Merge remote-tracking branch 'origin/master'

.gitignore

@@ -63,6 +63,8 @@ uwsgi
 #*.yaml
 
 .conf
+app/logs
+*.log
 *.jpg
-*.zip
+
-test
+feature/

app/core/config.py

@@ -27,8 +27,15 @@ MINIO_SECURE = False
 MINIO_ACCESS = "e8zc55mzDOh4IzRrZ9Oa"
 MINIO_SECRET = "uHfqJ7UkwA1PTDGfnA44Hp9ux5YkZTkzZLjeOYhE"
 
+<<<<<<< HEAD
+# OM_TRITON_IP = "10.1.1.150"
+# OM_TRITON_PORT = "7000"
+OM_TRITON_IP = "127.0.0.1"
+OM_TRITON_PORT = "8000"
+=======
 OM_TRITON_IP = "10.1.1.240"
 OM_TRITON_PORT = "10010"
+>>>>>>> 1f23781b16e59bfbcbbb4d252e6a61685267e6c7
 
 ATT_TRITON_IP = "10.1.1.240"
 ATT_TRITON_PORT = "10020"
@@ -39,6 +46,11 @@ ATT_TRITON_PORT = "10020"
 # FASHION_CATEGORIES_MAPPING = "app/service/outfit_matcher/config/fashion_category_mapping.json"
 
 # pycharm debug
+<<<<<<< HEAD
+LOGSPATH = "logs/errors.log"
+FASHION_CATEGORIES = "config/fashion_categories.json"
+FASHION_CATEGORIES_MAPPING = "config/fashion_category_mapping.json"
+=======
 LOGS_PATH = "logs/errors.log"
 FASHION_CATEGORIES = "service/outfit_matcher/config/fashion_categories.json"
 FASHION_CATEGORIES_MAPPING = "service/outfit_matcher/config/fashion_category_mapping.json"
@@ -47,3 +59,4 @@ FASHION_CATEGORIES_MAPPING = "service/outfit_matcher/config/fashion_category_map
 # LOGS_PATH = "app/logs/errors.log"
 # FASHION_CATEGORIES = "./config/fashion_categories.json"
 # FASHION_CATEGORIES_MAPPING = "./config/fashion_category_mapping.json"
+>>>>>>> 1f23781b16e59bfbcbbb4d252e6a61685267e6c7

app/service/outfit_matcher/outfit_evaluator.py

@@ -13,6 +13,103 @@ from app.service.outfit_matcher.foco import extract_main_colors
 from app.service.utils.decorator import RunTime
 
 
+class Backbone(object):
+    def __init__(self):
+        self.tritonclient = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
+        self.minio_client = Minio(
+            f"{MINIO_IP}:{MINIO_PORT}",
+            access_key=MINIO_ACCESS,
+            secret_key=MINIO_SECRET,
+            secure=MINIO_SECURE)
+    @RunTime
+    # TODO 用多线程读图片
+    def load_image(self, img_path):
+        try:
+            # 从 MinIO 中获取对象（图像文件）
+            image_data = self.minio_client.get_object(img_path.split("/", 1)[0], img_path.split("/", 1)[1])
+
+            # 读取图像数据并转换为 PIL 图像对象
+            pil_image = Image.open(io.BytesIO(image_data.data)).convert("RGB")
+
+            # 将 PIL 图像转换为 NumPy 数组
+            # image_array = np.array(pil_image)
+
+            return pil_image
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+    @staticmethod
+    def resize_image(img):
+        """
+        Args:
+            img: ndarray (height, width, channel)
+        """
+        image_transforms = transforms.Compose([
+            transforms.Resize(112),
+            transforms.CenterCrop(112),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                                 std=[0.229, 0.224, 0.225]),
+        ])
+        resized_img = image_transforms(img).numpy()
+        return resized_img
+
+    def preprocess(self, items):
+        images = []
+        for item in items:
+            image = self.load_image(item["image_path"])
+            image = self.resize_image(image)
+            images.append(image)
+        images = np.stack(images, axis=0)
+        return images
+
+    @RunTime
+    def get_result(self, items):
+        """Input items and output features for similiarity.
+        Args:
+            items: images of fashion items
+            Example:
+              [
+                {
+                    "item_name": "MSE_57987",
+                    "semantic_category": "BOTTOM/PANTS",
+                    "image_path": "D:\\PhD_Study\\MIXI\\mitu\\image\\2024 SS\\MSE_57987.jpg",
+                    "mapped_cate": "bottoms"
+                },
+                {
+                    "item_name": "MPO_SP7712",
+                    "semantic_category": "TOP/TANK",
+                    "image_path": "D:\\PhD_Study\\MIXI\\mitu\\image\\2024 SS\\MPO_SP7712.jpg",
+                    "mapped_cate": "tops"
+                },
+                {
+                    "item_name": "MWSS27195",
+                    "semantic_category": "OUTERWEAR/GILET",
+                    "image_path": "D:\\PhD_Study\\MIXI\\mitu\\image\\2024 SS\\MWSS27195.jpg",
+                    "mapped_cate": "outerwear"
+                }
+              ]
+        Returns:
+            scores: List of image features
+        """
+        image = self.preprocess(items)
+        client = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
+        # 输入集
+        inputs = [
+            httpclient.InferInput("input__0", image.shape, datatype="FP32"),
+        ]
+        inputs[0].set_data_from_numpy(image.astype(np.float32), binary_data=True)
+        # 输出集
+        outputs = [
+            httpclient.InferRequestedOutput("output__0", binary_data=True),
+        ]
+        results = client.infer(model_name="outfit_matcher_backbone", inputs=inputs, outputs=outputs)
+        # 推理
+        # 取结果
+        features = results.as_numpy("output__0")  # Shape (N, 64)
+        return features
+
+
 class OutfitMatcher(object):
     def __init__(self):
         self.tritonclient = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
@@ -22,6 +119,22 @@ class OutfitMatcher(object):
             secret_key=MINIO_SECRET,
             secure=MINIO_SECURE)
 
+    def load_image(self, img_path):
+        try:
+            # 从 MinIO 中获取对象（图像文件）
+            image_data = self.minio_client.get_object(img_path.split("/", 1)[0], img_path.split("/", 1)[1])
+
+            # 读取图像数据并转换为 PIL 图像对象
+            pil_image = Image.open(io.BytesIO(image_data.data)).convert("RGB")
+
+            # 将 PIL 图像转换为 NumPy 数组
+            # image_array = np.array(pil_image)
+
+            return pil_image
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
     @staticmethod
     def pad_array(input_value, value=0):
         """pad List of Array into same batch size
@@ -77,12 +190,11 @@ class OutfitMatcher(object):
     @RunTime
     def visualize(self, outfits, scores, topk=5, best=True, output_path=None):
         # 将outfits和scores按照scores的值进行排序
-        sorted_indices = np.argsort(-scores.flatten() if best else scores.flatten())[:topk]  # 使用负号进行降序排序
+        # sorted_indices = np.argsort(-scores if best else scores)[:topk]  # for HON
+        sorted_indices = np.argsort(scores if best else -scores)[:topk]  # type-aware
         outfits = [outfits[i] for i in sorted_indices]  # 最好或最差的五个
         scores = scores[sorted_indices]  # 这五个的分数
 
-        # 是否画出来
-        if output_path:
         # 设置子图的行列数
         num_rows = len(outfits)
         num_cols = max([len(x) for x in outfits]) + 1  # 一个是图片，一个是分数
@@ -96,11 +208,11 @@ class OutfitMatcher(object):
         # 遍历每套outfit并将其显示在对应的子图中
         for i, (outfit, score) in enumerate(zip(outfits, scores)):
             # 显示分数
-                axes[i, 0].text(0.1, 0.5, f"Score: {score[0]:.4f}", fontsize=12)
+            axes[i, 0].text(0.1, 0.5, f"Score: {score:.4f}", fontsize=12)
             axes[i, 0].axis("off")
             # 显示图片
             for j, item in enumerate(outfit):
-                    img = mpimg.imread(item['image_path'])  # 读取图片
+                img = self.load_image(item['image_path'])  # 读取图片
                 axes[i, j + 1].imshow(img)  # 在对应的子图中显示图片
                 axes[i, j + 1].axis('off')  # 关闭坐标轴
                 axes[i, j + 1].set_title(item["semantic_category"], fontsize=10)
@@ -120,8 +232,6 @@ class OutfitMatcher(object):
             plt.savefig(output_path)
         else:
             plt.show()
-        else:
-            return outfits, scores.numpy().flatten().tolist()
 
 
 class OutfitMatcherHon(OutfitMatcher):
@@ -216,60 +326,17 @@ class OutfitMaterTypeAware(OutfitMatcher):
         'outerwear', 'scarves', 'shoes', 'sunglasses', 'tops'
     ]
 
-    @RunTime
     def __init__(self):
         super().__init__()
 
-    @RunTime
+    def preprocess(self, outfits, features):
-    # TODO 用多线程读图片
-    def load_image(self, img_path):
-        try:
-            # 从 MinIO 中获取对象（图像文件）
-            image_data = self.minio_client.get_object(img_path.split("/", 1)[0], img_path.split("/", 1)[1])
-
-            # 读取图像数据并转换为 PIL 图像对象
-            pil_image = Image.open(io.BytesIO(image_data.data)).convert("RGB")
-
-            # 将 PIL 图像转换为 NumPy 数组
-            # image_array = np.array(pil_image)
-
-            return pil_image
-        except Exception as e:
-            print(f"An error occurred: {e}")
-            return None
-        # if 'http' in img_path:
-        #     file = requests.get(img_path)
-        #     image = cv2.imdecode(np.fromstring(file.content, np.uint8), 1)
-        #     image = Image.fromarray(image.astype('uint8'), 'RGB')
-        # else:
-        #     image = Image.open(img_path).convert('RGB')
-        # return np.array(image)
-
-    @staticmethod
-    def resize_image(img):
-        """
-        Args:
-            img: ndarray (height, width, channel)
-        """
-        image_transforms = transforms.Compose([
-            transforms.Resize(112),
-            transforms.CenterCrop(112),
-            transforms.ToTensor(),
-            transforms.Normalize(mean=[0.485, 0.456, 0.406],
-                                 std=[0.229, 0.224, 0.225]),
-        ])
-        resized_img = image_transforms(img).numpy()
-        return resized_img
-
-    def preprocess(self, outfits):
         outfit_images = []
         outfit_categories = []
         for outfit in outfits:
             images = []
             categories = []
             for item in outfit:
-                image = self.load_image(item["image_path"])
+                image = features[item["item_name"]]
-                image = self.resize_image(image)
                 images.append(image)
                 category = self.base_fashion_categories.index(item["mapped_cate"])
                 categories.append(category)
@@ -277,12 +344,10 @@ class OutfitMaterTypeAware(OutfitMatcher):
             outfit_images.append(images)  # List[(items, 3, 224, 224)]
             categories = np.array(categories)
             outfit_categories.append(categories)  # List[(items)]
-        outfit_images, mask = self.pad_array(outfit_images, value=0)
+        return outfit_images, outfit_categories
-        outfit_categories, _ = self.pad_array(outfit_categories, value=len(self.base_fashion_categories))
-        return outfit_images, outfit_categories, mask
 
     @RunTime
-    def get_result(self, outfits):
+    def get_result(self, outfits, features):
         """Input outfits structure and output scores.
         Args:
             outfits: outfits to be evaluated.
@@ -310,9 +375,32 @@ class OutfitMaterTypeAware(OutfitMatcher):
                   ],
                   ...
                 ]
+            features: dict(item_name = np.Array) image features of those items
         Returns:
             scores: List of float
         """
+<<<<<<< HEAD
+        outfit_images, outfit_categories = self.preprocess(outfits, features)
+        scores = []
+        for images, categories in zip(outfit_images, outfit_categories):
+            client = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
+            # 输入集
+            inputs = [
+                httpclient.InferInput("input__0", images.shape, datatype="FP32"),
+                httpclient.InferInput("input__1", categories.shape, datatype="INT16")
+            ]
+            inputs[0].set_data_from_numpy(images.astype(np.float32), binary_data=True)
+            inputs[1].set_data_from_numpy(categories.astype(np.int16), binary_data=True)
+            # 输出集
+            outputs = [
+                httpclient.InferRequestedOutput("output__0", binary_data=True),
+            ]
+            results = client.infer(model_name="outfit_matcher_type_aware", inputs=inputs, outputs=outputs)
+            scores.append(results.as_numpy("output__0"))  # Shape (N, 1)
+
+        scores = np.stack(scores, axis=0)
+        return scores.flatten()
+=======
         image, category, mask = self.preprocess(outfits)
         client = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
         # 输入集
@@ -336,3 +424,4 @@ class OutfitMaterTypeAware(OutfitMatcher):
         features = torch.from_numpy(results.as_numpy("output__1"))  # Shape (N, 64)
 
         return scores, features
+>>>>>>> 1f23781b16e59bfbcbbb4d252e6a61685267e6c7

app/service/outfit_matcher/service.py

@@ -1,19 +1,38 @@
 import json
 import os
 from pprint import pprint
+from tqdm import tqdm
+import numpy as np
 
 from app.service.outfit_matcher.dataset import FashionDataset
-from app.service.outfit_matcher.outfit_evaluator import OutfitMaterTypeAware
+from app.service.outfit_matcher.outfit_evaluator import OutfitMaterTypeAware, Backbone
 
 if __name__ == '__main__':
     with open("./test_param/recommendation_test.json", "r") as f:
         param = json.load(f)
     fashion_dataset = FashionDataset(param["database"])
+    backbone_service = Backbone()
     service = OutfitMaterTypeAware()
-    best_list = []
+
-    bad_list = []
+    # read feature from vector database
-    for item in param["query"]:
+    all_items = param["query"] + param["database"]
+    unextracted_item = []
+    prepared_feature = {}
+    for item in all_items:
+        if f'{item["item_name"]}.npy' not in os.listdir("feature"):
+            unextracted_item.append(item)
+    if len(unextracted_item) > 0:
+        extracted_features = backbone_service.get_result(unextracted_item)
+        for i, item in enumerate(unextracted_item):
+            np.save(f'feature/{item["item_name"]}.npy', extracted_features[i])
+    for item in all_items:
+        if item["item_name"] not in prepared_feature.keys():
+            prepared_feature[item["item_name"]] = np.load(f'feature/{item["item_name"]}.npy')
+    for item in tqdm(param["query"] * 10):
         outfits = fashion_dataset.generate_outfit(item, param["topk"], param["max_outfits"])
+<<<<<<< HEAD
+        scores = service.get_result(outfits, prepared_feature)
+=======
         scores, features = service.get_result(outfits)
         # save  features
 
@@ -22,16 +41,15 @@ if __name__ == '__main__':
         # 存入数据库
         # 关闭链接
 
+>>>>>>> 1f23781b16e59bfbcbbb4d252e6a61685267e6c7
         # print(scores)
         # print(len(scores))
-        best_outfits, best_scores = service.visualize(outfits, scores, param["topk"], best=True,
+        # service.visualize(outfits, scores, param["topk"], best=True,
-                                                      # output_path=os.path.join(r"E:\workspace\outfit_matcher\2024 SS Outfit", f"{item['item_name']}_best_{param['topk']}.png")
+        #                   output_path=os.path.join(r"D:\PhD_Study\MIXI\mitu\image\123",
-                                                      )
+        #                                            f"{item['item_name']}_best_{param['topk']}.png"))
-        bad_outfits, bad_scores = service.visualize(outfits, scores, param["topk"], best=False,
+        # service.visualize(outfits, scores, param["topk"], best=False,
-                                                    # output_path=os.path.join(r"E:\workspace\outfit_matcher\2024 SS Outfit", f"{item['item_name']}_worst_{param['topk']}.png")
+        #                   output_path=os.path.join(r"D:\PhD_Study\MIXI\mitu\image\123",
-                                                    )
+        #                                            f"{item['item_name']}_worst_{param['topk']}.png"))
-        best_list.append({"best_outfits": best_outfits, "best_scores": best_scores})
+        sorted_indices = np.argsort(scores)[:param["topk"]]  # type-aware
-        bad_list.append({"bad_outfits": bad_outfits, "bad_scores": bad_scores})
+        outfits = [outfits[i] for i in sorted_indices]  # 最好的五个
 
-    pprint(best_list)
-    pprint(bad_list)

app/service/outfit_matcher/test_param/recommendation_test.json

@@ -1,9 +1,114 @@
 {
+<<<<<<< HEAD
+  "topk": 5,
+  "max_outfits": 200,
+  "is_best": true,
+  "query": [
+    {
+      "item_name": "MSE_58107",
+      "semantic_category": "TOP/SHIRT",
+      "image_path": "test/2024 SS/MSE_58107.jpg"
+    },
+    {
+      "item_name": "MKTS27047",
+      "semantic_category": "ONE PIECE/DRESS",
+      "image_path": "test/2024 SS/MKTS27047.jpg"
+    },
+    {
+      "item_name": "MKTS27028",
+      "semantic_category": "OUTERWEAR/JACKET",
+      "image_path": "test/2024 SS/MKTS27028.jpg"
+    },
+    {
+      "item_name": "MSE_58057",
+      "semantic_category": "OUTERWEAR/BLAZER",
+      "image_path": "test/2024 SS/MSE_58057.jpg"
+    },
+    {
+      "item_name": "MSE_58495",
+      "semantic_category": "TOP/SHIRT",
+      "image_path": "test/2024 SS/MSE_58495.jpg"
+    }
+  ],
+  "database": [
+    {
+      "item_name": "MKTS27017",
+      "semantic_category": "OUTERWEAR/WINDBREAKER",
+      "image_path": "test/2024 SS/MKTS27017.jpg"
+    },
+    {
+      "item_name": "MKTS27047",
+      "semantic_category": "ONE PIECE/DRESS",
+      "image_path": "test/2024 SS/MKTS27047.jpg"
+    },
+    {
+      "item_name": "MKTS27000",
+      "semantic_category": "BOTTOM/PANTS",
+      "image_path": "test/2024 SS/MKTS27000.jpg"
+    },
+    {
+      "item_name": "MKTS27001",
+      "semantic_category": "BOTTOM/SHORTS",
+      "image_path": "test/2024 SS/MKTS27001.jpg"
+    },
+    {
+      "item_name": "MZOS27178",
+      "semantic_category": "KNIT/CARDIGAN",
+      "image_path": "test/2024 SS/MZOS27178.jpg"
+    },
+    {
+      "item_name": "MZOS27179",
+      "semantic_category": "KNIT/KNIT TOP",
+      "image_path": "test/2024 SS/MZOS27179.jpg"
+    },
+    {
+      "item_name": "MWSS27184",
+      "semantic_category": "TOP/TEE",
+      "image_path": "test/2024 SS/MWSS27184.jpg"
+    },
+    {
+      "item_name": "MWSS27191",
+      "semantic_category": "OUTERWEAR/TWIN SET",
+      "image_path": "test/2024 SS/MWSS27191.jpg"
+    },
+    {
+      "item_name": "MWSS27193",
+      "semantic_category": "OUTERWEAR/JACKET",
+      "image_path": "test/2024 SS/MWSS27193.jpg"
+    },
+    {
+      "item_name": "MWSS27195",
+      "semantic_category": "OUTERWEAR/GILET",
+      "image_path": "test/2024 SS/MWSS27195.jpg"
+    },
+    {
+      "item_name": "MWSS27200",
+      "semantic_category": "KNIT/VEST",
+      "image_path": "test/2024 SS/MWSS27200.jpg"
+    },
+    {
+      "item_name": "MWSS27209",
+      "semantic_category": "ONE PIECE/DRESS",
+      "image_path": "test/2024 SS/MWSS27209.jpg"
+    },
+    {
+      "item_name": "MWSS27210",
+      "semantic_category": "ONE PIECE/DRESS",
+      "image_path": "test/2024 SS/MWSS27210.jpg"
+    },
+    {
+      "item_name": "MWSS27211",
+      "semantic_category": "ONE PIECE/DRESS",
+      "image_path": "test/2024 SS/MWSS27211.jpg"
+    },
+    {
+=======
   "topk": 1,
   "max_outfits": 100,
   "is_best": true,
   "query": [
     {
+>>>>>>> 1f23781b16e59bfbcbbb4d252e6a61685267e6c7
       "item_name": "MWSS27212",
       "semantic_category": "TOP/BLOUSE",
       "image_path": "test/2024 SS/MWSS27212.jpg"