split into two models, one is resnet to extract feature, one is compatibility model

.gitignore

@@ -131,3 +131,5 @@ uwsgi
 app/logs
 *.log
 *.jpg
+
+feature/

app/core/config.py

@@ -27,8 +27,10 @@ MINIO_SECURE = False
 MINIO_ACCESS = "e8zc55mzDOh4IzRrZ9Oa"
 MINIO_SECRET = "uHfqJ7UkwA1PTDGfnA44Hp9ux5YkZTkzZLjeOYhE"
 
-OM_TRITON_IP = "10.1.1.150"
+# OM_TRITON_IP = "10.1.1.150"
-OM_TRITON_PORT = "7000"
+# OM_TRITON_PORT = "7000"
+OM_TRITON_IP = "127.0.0.1"
+OM_TRITON_PORT = "8000"
 
 ATT_TRITON_IP = "10.1.1.150"
 ATT_TRITON_PORT = "6000"
@@ -40,5 +42,5 @@ ATT_TRITON_PORT = "6000"
 
 # pycharm debug
 LOGSPATH = "logs/errors.log"
-FASHION_CATEGORIES = "service/outfit_matcher/config/fashion_categories.json"
+FASHION_CATEGORIES = "config/fashion_categories.json"
-FASHION_CATEGORIES_MAPPING = "service/outfit_matcher/config/fashion_category_mapping.json"
+FASHION_CATEGORIES_MAPPING = "config/fashion_category_mapping.json"

app/service/outfit_matcher/outfit_evaluator.py

@@ -13,6 +13,103 @@ from app.service.outfit_matcher.foco import extract_main_colors
 from app.service.utils.decorator import RunTime
 
 
+class Backbone(object):
+    def __init__(self):
+        self.tritonclient = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
+        self.minio_client = Minio(
+            f"{MINIO_IP}:{MINIO_PORT}",
+            access_key=MINIO_ACCESS,
+            secret_key=MINIO_SECRET,
+            secure=MINIO_SECURE)
+    @RunTime
+    # TODO 用多线程读图片
+    def load_image(self, img_path):
+        try:
+            # 从 MinIO 中获取对象（图像文件）
+            image_data = self.minio_client.get_object(img_path.split("/", 1)[0], img_path.split("/", 1)[1])
+
+            # 读取图像数据并转换为 PIL 图像对象
+            pil_image = Image.open(io.BytesIO(image_data.data)).convert("RGB")
+
+            # 将 PIL 图像转换为 NumPy 数组
+            # image_array = np.array(pil_image)
+
+            return pil_image
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+    @staticmethod
+    def resize_image(img):
+        """
+        Args:
+            img: ndarray (height, width, channel)
+        """
+        image_transforms = transforms.Compose([
+            transforms.Resize(112),
+            transforms.CenterCrop(112),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                                 std=[0.229, 0.224, 0.225]),
+        ])
+        resized_img = image_transforms(img).numpy()
+        return resized_img
+
+    def preprocess(self, items):
+        images = []
+        for item in items:
+            image = self.load_image(item["image_path"])
+            image = self.resize_image(image)
+            images.append(image)
+        images = np.stack(images, axis=0)
+        return images
+
+    @RunTime
+    def get_result(self, items):
+        """Input items and output features for similiarity.
+        Args:
+            items: images of fashion items
+            Example:
+              [
+                {
+                    "item_name": "MSE_57987",
+                    "semantic_category": "BOTTOM/PANTS",
+                    "image_path": "D:\\PhD_Study\\MIXI\\mitu\\image\\2024 SS\\MSE_57987.jpg",
+                    "mapped_cate": "bottoms"
+                },
+                {
+                    "item_name": "MPO_SP7712",
+                    "semantic_category": "TOP/TANK",
+                    "image_path": "D:\\PhD_Study\\MIXI\\mitu\\image\\2024 SS\\MPO_SP7712.jpg",
+                    "mapped_cate": "tops"
+                },
+                {
+                    "item_name": "MWSS27195",
+                    "semantic_category": "OUTERWEAR/GILET",
+                    "image_path": "D:\\PhD_Study\\MIXI\\mitu\\image\\2024 SS\\MWSS27195.jpg",
+                    "mapped_cate": "outerwear"
+                }
+              ]
+        Returns:
+            scores: List of image features
+        """
+        image = self.preprocess(items)
+        client = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
+        # 输入集
+        inputs = [
+            httpclient.InferInput("input__0", image.shape, datatype="FP32"),
+        ]
+        inputs[0].set_data_from_numpy(image.astype(np.float32), binary_data=True)
+        # 输出集
+        outputs = [
+            httpclient.InferRequestedOutput("output__0", binary_data=True),
+        ]
+        results = client.infer(model_name="outfit_matcher_backbone", inputs=inputs, outputs=outputs)
+        # 推理
+        # 取结果
+        features = results.as_numpy("output__0")  # Shape (N, 64)
+        return features
+
+
 class OutfitMatcher(object):
     def __init__(self):
         self.tritonclient = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
@@ -22,6 +119,22 @@ class OutfitMatcher(object):
             secret_key=MINIO_SECRET,
             secure=MINIO_SECURE)
 
+    def load_image(self, img_path):
+        try:
+            # 从 MinIO 中获取对象（图像文件）
+            image_data = self.minio_client.get_object(img_path.split("/", 1)[0], img_path.split("/", 1)[1])
+
+            # 读取图像数据并转换为 PIL 图像对象
+            pil_image = Image.open(io.BytesIO(image_data.data)).convert("RGB")
+
+            # 将 PIL 图像转换为 NumPy 数组
+            # image_array = np.array(pil_image)
+
+            return pil_image
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
     @staticmethod
     def pad_array(input_value, value=0):
         """pad List of Array into same batch size
@@ -77,51 +190,48 @@ class OutfitMatcher(object):
     @RunTime
     def visualize(self, outfits, scores, topk=5, best=True, output_path=None):
         # 将outfits和scores按照scores的值进行排序
-        sorted_indices = np.argsort(-scores.flatten() if best else scores.flatten())[:topk]  # 使用负号进行降序排序
+        # sorted_indices = np.argsort(-scores if best else scores)[:topk]  # for HON
+        sorted_indices = np.argsort(scores if best else -scores)[:topk]  # type-aware
         outfits = [outfits[i] for i in sorted_indices]  # 最好或最差的五个
         scores = scores[sorted_indices]  # 这五个的分数
 
-        # 是否画出来
+        # 设置子图的行列数
+        num_rows = len(outfits)
+        num_cols = max([len(x) for x in outfits]) + 1  # 一个是图片，一个是分数
+
+        # 创建一个新的图像，并指定子图的行列数
+        fig, axes = plt.subplots(num_rows, num_cols, figsize=(8, 15))
+
+        title = f"Best {topk} Outfits" if best else f"Worst {topk} Outfits"
+        fig.suptitle(title, fontsize=16)
+
+        # 遍历每套outfit并将其显示在对应的子图中
+        for i, (outfit, score) in enumerate(zip(outfits, scores)):
+            # 显示分数
+            axes[i, 0].text(0.1, 0.5, f"Score: {score:.4f}", fontsize=12)
+            axes[i, 0].axis("off")
+            # 显示图片
+            for j, item in enumerate(outfit):
+                img = self.load_image(item['image_path'])  # 读取图片
+                axes[i, j + 1].imshow(img)  # 在对应的子图中显示图片
+                axes[i, j + 1].axis('off')  # 关闭坐标轴
+                axes[i, j + 1].set_title(item["semantic_category"], fontsize=10)
+            for j in range(len(outfit), num_cols):
+                axes[i, j].axis("off")
+
+            # 在每一行的底部添加一条横线
+            axes[i, 0].axhline(y=0, color='black', linewidth=1)
+        # 隐藏最后一行的横线
+        axes[-1, 0].axhline(y=0, color='white', linewidth=1)
+
+        # 调整布局
+        plt.subplots_adjust(wspace=0.1, hspace=0.1)
+        plt.tight_layout()
+
         if output_path:
-            # 设置子图的行列数
+            plt.savefig(output_path)
-            num_rows = len(outfits)
-            num_cols = max([len(x) for x in outfits]) + 1  # 一个是图片，一个是分数
-
-            # 创建一个新的图像，并指定子图的行列数
-            fig, axes = plt.subplots(num_rows, num_cols, figsize=(8, 15))
-
-            title = f"Best {topk} Outfits" if best else f"Worst {topk} Outfits"
-            fig.suptitle(title, fontsize=16)
-
-            # 遍历每套outfit并将其显示在对应的子图中
-            for i, (outfit, score) in enumerate(zip(outfits, scores)):
-                # 显示分数
-                axes[i, 0].text(0.1, 0.5, f"Score: {score[0]:.4f}", fontsize=12)
-                axes[i, 0].axis("off")
-                # 显示图片
-                for j, item in enumerate(outfit):
-                    img = mpimg.imread(item['image_path'])  # 读取图片
-                    axes[i, j + 1].imshow(img)  # 在对应的子图中显示图片
-                    axes[i, j + 1].axis('off')  # 关闭坐标轴
-                    axes[i, j + 1].set_title(item["semantic_category"], fontsize=10)
-                for j in range(len(outfit), num_cols):
-                    axes[i, j].axis("off")
-
-                # 在每一行的底部添加一条横线
-                axes[i, 0].axhline(y=0, color='black', linewidth=1)
-            # 隐藏最后一行的横线
-            axes[-1, 0].axhline(y=0, color='white', linewidth=1)
-
-            # 调整布局
-            plt.subplots_adjust(wspace=0.1, hspace=0.1)
-            plt.tight_layout()
-
-            if output_path:
-                plt.savefig(output_path)
-            else:
-                plt.show()
         else:
-            return outfits, scores.numpy().flatten().tolist()
+            plt.show()
 
 
 class OutfitMatcherHon(OutfitMatcher):
@@ -216,60 +326,17 @@ class OutfitMaterTypeAware(OutfitMatcher):
         'outerwear', 'scarves', 'shoes', 'sunglasses', 'tops'
     ]
 
-    @RunTime
     def __init__(self):
         super().__init__()
 
-    @RunTime
+    def preprocess(self, outfits, features):
-    # TODO 用多线程读图片
-    def load_image(self, img_path):
-        try:
-            # 从 MinIO 中获取对象（图像文件）
-            image_data = self.minio_client.get_object(img_path.split("/", 1)[0], img_path.split("/", 1)[1])
-
-            # 读取图像数据并转换为 PIL 图像对象
-            pil_image = Image.open(io.BytesIO(image_data.data)).convert("RGB")
-
-            # 将 PIL 图像转换为 NumPy 数组
-            # image_array = np.array(pil_image)
-
-            return pil_image
-        except Exception as e:
-            print(f"An error occurred: {e}")
-            return None
-        # if 'http' in img_path:
-        #     file = requests.get(img_path)
-        #     image = cv2.imdecode(np.fromstring(file.content, np.uint8), 1)
-        #     image = Image.fromarray(image.astype('uint8'), 'RGB')
-        # else:
-        #     image = Image.open(img_path).convert('RGB')
-        # return np.array(image)
-
-    @staticmethod
-    def resize_image(img):
-        """
-        Args:
-            img: ndarray (height, width, channel)
-        """
-        image_transforms = transforms.Compose([
-            transforms.Resize(112),
-            transforms.CenterCrop(112),
-            transforms.ToTensor(),
-            transforms.Normalize(mean=[0.485, 0.456, 0.406],
-                                 std=[0.229, 0.224, 0.225]),
-        ])
-        resized_img = image_transforms(img).numpy()
-        return resized_img
-
-    def preprocess(self, outfits):
         outfit_images = []
         outfit_categories = []
         for outfit in outfits:
             images = []
             categories = []
             for item in outfit:
-                image = self.load_image(item["image_path"])
+                image = features[item["item_name"]]
-                image = self.resize_image(image)
                 images.append(image)
                 category = self.base_fashion_categories.index(item["mapped_cate"])
                 categories.append(category)
@@ -277,12 +344,10 @@ class OutfitMaterTypeAware(OutfitMatcher):
             outfit_images.append(images)  # List[(items, 3, 224, 224)]
             categories = np.array(categories)
             outfit_categories.append(categories)  # List[(items)]
-        outfit_images, mask = self.pad_array(outfit_images, value=0)
+        return outfit_images, outfit_categories
-        outfit_categories, _ = self.pad_array(outfit_categories, value=len(self.base_fashion_categories))
-        return outfit_images, outfit_categories, mask
 
     @RunTime
-    def get_result(self, outfits):
+    def get_result(self, outfits, features):
         """Input outfits structure and output scores.
         Args:
             outfits: outfits to be evaluated.
@@ -310,28 +375,27 @@ class OutfitMaterTypeAware(OutfitMatcher):
                   ],
                   ...
                 ]
+            features: dict(item_name = np.Array) image features of those items
         Returns:
             scores: List of float
         """
-        image, category, mask = self.preprocess(outfits)
+        outfit_images, outfit_categories = self.preprocess(outfits, features)
-        client = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
+        scores = []
-        # 输入集
+        for images, categories in zip(outfit_images, outfit_categories):
-        inputs = [
+            client = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
-            httpclient.InferInput("input__0", image.shape, datatype="FP32"),
+            # 输入集
-            httpclient.InferInput("input__1", category.shape, datatype="INT16"),
+            inputs = [
-            httpclient.InferInput("input__2", mask.shape, datatype="FP32"),
+                httpclient.InferInput("input__0", images.shape, datatype="FP32"),
-        ]
+                httpclient.InferInput("input__1", categories.shape, datatype="INT16")
-        inputs[0].set_data_from_numpy(image.astype(np.float32), binary_data=True)
+            ]
-        inputs[1].set_data_from_numpy(category.astype(np.int16), binary_data=True)
+            inputs[0].set_data_from_numpy(images.astype(np.float32), binary_data=True)
-        inputs[2].set_data_from_numpy(mask.astype(np.float32), binary_data=True)
+            inputs[1].set_data_from_numpy(categories.astype(np.int16), binary_data=True)
-        # 输出集
+            # 输出集
-        outputs = [
+            outputs = [
-            httpclient.InferRequestedOutput("output__0", binary_data=True),
+                httpclient.InferRequestedOutput("output__0", binary_data=True),
-            httpclient.InferRequestedOutput("output__1", binary_data=True),
+            ]
-        ]
+            results = client.infer(model_name="outfit_matcher_type_aware", inputs=inputs, outputs=outputs)
-        results = client.infer(model_name="outfit_matcher_type_aware", inputs=inputs, outputs=outputs)
+            scores.append(results.as_numpy("output__0"))  # Shape (N, 1)
-        # 推理
+
-        # 取结果
+        scores = np.stack(scores, axis=0)
-        scores = torch.from_numpy(results.as_numpy("output__0"))  # Shape (N, 1)
+        return scores.flatten()
-        features = torch.from_numpy(results.as_numpy("output__1"))  # Shape (N, -1, 64)
-        return scores, features

app/service/outfit_matcher/service.py

@@ -1,30 +1,44 @@
 import json
 import os
 from pprint import pprint
+from tqdm import tqdm
+import numpy as np
 
 from app.service.outfit_matcher.dataset import FashionDataset
-from app.service.outfit_matcher.outfit_evaluator import OutfitMaterTypeAware
+from app.service.outfit_matcher.outfit_evaluator import OutfitMaterTypeAware, Backbone
 
 if __name__ == '__main__':
     with open("./test_param/recommendation_test.json", "r") as f:
         param = json.load(f)
     fashion_dataset = FashionDataset(param["database"])
+    backbone_service = Backbone()
     service = OutfitMaterTypeAware()
-    best_list = []
+
-    bad_list = []
+    # read feature from vector database
-    for item in param["query"]:
+    all_items = param["query"] + param["database"]
+    unextracted_item = []
+    prepared_feature = {}
+    for item in all_items:
+        if f'{item["item_name"]}.npy' not in os.listdir("feature"):
+            unextracted_item.append(item)
+    if len(unextracted_item) > 0:
+        extracted_features = backbone_service.get_result(unextracted_item)
+        for i, item in enumerate(unextracted_item):
+            np.save(f'feature/{item["item_name"]}.npy', extracted_features[i])
+    for item in all_items:
+        if item["item_name"] not in prepared_feature.keys():
+            prepared_feature[item["item_name"]] = np.load(f'feature/{item["item_name"]}.npy')
+    for item in tqdm(param["query"] * 10):
         outfits = fashion_dataset.generate_outfit(item, param["topk"], param["max_outfits"])
-        scores = service.get_result(outfits)
+        scores = service.get_result(outfits, prepared_feature)
         # print(scores)
         # print(len(scores))
-        best_outfits, best_scores = service.visualize(outfits, scores, param["topk"], best=True,
+        # service.visualize(outfits, scores, param["topk"], best=True,
-                                                      # output_path=os.path.join(r"E:\workspace\outfit_matcher\2024 SS Outfit", f"{item['item_name']}_best_{param['topk']}.png")
+        #                   output_path=os.path.join(r"D:\PhD_Study\MIXI\mitu\image\123",
-                                                      )
+        #                                            f"{item['item_name']}_best_{param['topk']}.png"))
-        bad_outfits, bad_scores = service.visualize(outfits, scores, param["topk"], best=False,
+        # service.visualize(outfits, scores, param["topk"], best=False,
-                                                    # output_path=os.path.join(r"E:\workspace\outfit_matcher\2024 SS Outfit", f"{item['item_name']}_worst_{param['topk']}.png")
+        #                   output_path=os.path.join(r"D:\PhD_Study\MIXI\mitu\image\123",
-                                                    )
+        #                                            f"{item['item_name']}_worst_{param['topk']}.png"))
-        best_list.append({"best_outfits": best_outfits, "best_scores": best_scores})
+        sorted_indices = np.argsort(scores)[:param["topk"]]  # type-aware
-        bad_list.append({"bad_outfits": bad_outfits, "bad_scores": bad_scores})
+        outfits = [outfits[i] for i in sorted_indices]  # 最好的五个
 
-    pprint(best_list)
-    pprint(bad_list)

app/service/outfit_matcher/test_param/recommendation_test.json

@@ -1,8 +1,13 @@
 {
   "topk": 5,
-  "max_outfits": 100,
+  "max_outfits": 200,
   "is_best": true,
   "query": [
+    {
+      "item_name": "MSE_58107",
+      "semantic_category": "TOP/SHIRT",
+      "image_path": "test/2024 SS/MSE_58107.jpg"
+    },
     {
       "item_name": "MKTS27047",
       "semantic_category": "ONE PIECE/DRESS",
@@ -18,11 +23,6 @@
       "semantic_category": "OUTERWEAR/BLAZER",
       "image_path": "test/2024 SS/MSE_58057.jpg"
     },
-    {
-      "item_name": "MSE_58107",
-      "semantic_category": "TOP/SHIRT",
-      "image_path": "test/2024 SS/MSE_58107.jpg"
-    },
     {
       "item_name": "MSE_58495",
       "semantic_category": "TOP/SHIRT",