搭配服务修改

app/service/outfit_matcher/outfit_evaluator.py

@@ -13,6 +13,105 @@ from app.service.outfit_matcher.foco import extract_main_colors
 from app.service.utils.decorator import RunTime
 
 
+class Backbone(object):
+    def __init__(self):
+        self.tritonclient = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
+        self.minio_client = Minio(
+            f"{MINIO_IP}:{MINIO_PORT}",
+            access_key=MINIO_ACCESS,
+            secret_key=MINIO_SECRET,
+            secure=MINIO_SECURE)
+
+    @RunTime
+    # TODO 用多线程读图片
+    def load_image(self, img_path):
+        try:
+            # 从 MinIO 中获取对象（图像文件）
+            image_data = self.minio_client.get_object(img_path.split("/", 1)[0], img_path.split("/", 1)[1])
+
+            # 读取图像数据并转换为 PIL 图像对象
+            pil_image = Image.open(io.BytesIO(image_data.data)).convert("RGB")
+
+            # 将 PIL 图像转换为 NumPy 数组
+            # image_array = np.array(pil_image)
+
+            return pil_image
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
+    @staticmethod
+    def resize_image(img):
+        """
+        Args:
+            img: ndarray (height, width, channel)
+        """
+        image_transforms = transforms.Compose([
+            transforms.Resize(112),
+            transforms.CenterCrop(112),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                                 std=[0.229, 0.224, 0.225]),
+        ])
+        resized_img = image_transforms(img).numpy()
+        return resized_img
+
+    def preprocess(self, items):
+        images = []
+        for item in items:
+            image = self.load_image(item["image_path"])
+            image = self.resize_image(image)
+            images.append(image)
+        images = np.stack(images, axis=0)
+        return images
+
+    @RunTime
+    def get_result(self, items):
+        """Input items and output features for similiarity.
+        Args:
+            items: images of fashion items
+            Example:
+              [
+                {
+                    "item_name": "MSE_57987",
+                    "semantic_category": "BOTTOM/PANTS",
+                    "image_path": "D:\\PhD_Study\\MIXI\\mitu\\image\\2024 SS\\MSE_57987.jpg",
+                    "mapped_cate": "bottoms"
+                },
+                {
+                    "item_name": "MPO_SP7712",
+                    "semantic_category": "TOP/TANK",
+                    "image_path": "D:\\PhD_Study\\MIXI\\mitu\\image\\2024 SS\\MPO_SP7712.jpg",
+                    "mapped_cate": "tops"
+                },
+                {
+                    "item_name": "MWSS27195",
+                    "semantic_category": "OUTERWEAR/GILET",
+                    "image_path": "D:\\PhD_Study\\MIXI\\mitu\\image\\2024 SS\\MWSS27195.jpg",
+                    "mapped_cate": "outerwear"
+                }
+              ]
+        Returns:
+            scores: List of image features
+        """
+        image = self.preprocess(items)
+        client = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
+        # 输入集
+        inputs = [
+            httpclient.InferInput("input__0", image.shape, datatype="FP32"),
+        ]
+        inputs[0].set_data_from_numpy(image.astype(np.float32), binary_data=True)
+        # 输出集
+        outputs = [
+            httpclient.InferRequestedOutput("output__0", binary_data=True),
+        ]
+        results = client.infer(model_name="outfit_matcher_backbone", inputs=inputs, outputs=outputs)
+        # 推理
+        # 取结果
+        features = results.as_numpy("output__0")  # Shape (N, 64)
+        return features
+
+
 class OutfitMatcher(object):
     def __init__(self):
         self.tritonclient = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
@@ -22,6 +121,22 @@ class OutfitMatcher(object):
             secret_key=MINIO_SECRET,
             secure=MINIO_SECURE)
 
+    def load_image(self, img_path):
+        try:
+            # 从 MinIO 中获取对象（图像文件）
+            image_data = self.minio_client.get_object(img_path.split("/", 1)[0], img_path.split("/", 1)[1])
+
+            # 读取图像数据并转换为 PIL 图像对象
+            pil_image = Image.open(io.BytesIO(image_data.data)).convert("RGB")
+
+            # 将 PIL 图像转换为 NumPy 数组
+            # image_array = np.array(pil_image)
+
+            return pil_image
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return None
+
     @staticmethod
     def pad_array(input_value, value=0):
         """pad List of Array into same batch size
@@ -77,51 +192,48 @@ class OutfitMatcher(object):
     @RunTime
     def visualize(self, outfits, scores, topk=5, best=True, output_path=None):
         # 将outfits和scores按照scores的值进行排序
-        sorted_indices = np.argsort(-scores.flatten() if best else scores.flatten())[:topk]  # 使用负号进行降序排序
+        # sorted_indices = np.argsort(-scores if best else scores)[:topk]  # for HON
+        sorted_indices = np.argsort(scores if best else -scores)[:topk]  # type-aware
         outfits = [outfits[i] for i in sorted_indices]  # 最好或最差的五个
         scores = scores[sorted_indices]  # 这五个的分数
 
-        # 是否画出来
+        # 设置子图的行列数
+        num_rows = len(outfits)
+        num_cols = max([len(x) for x in outfits]) + 1  # 一个是图片，一个是分数
+
+        # 创建一个新的图像，并指定子图的行列数
+        fig, axes = plt.subplots(num_rows, num_cols, figsize=(8, 15))
+
+        title = f"Best {topk} Outfits" if best else f"Worst {topk} Outfits"
+        fig.suptitle(title, fontsize=16)
+
+        # 遍历每套outfit并将其显示在对应的子图中
+        for i, (outfit, score) in enumerate(zip(outfits, scores)):
+            # 显示分数
+            axes[i, 0].text(0.1, 0.5, f"Score: {score:.4f}", fontsize=12)
+            axes[i, 0].axis("off")
+            # 显示图片
+            for j, item in enumerate(outfit):
+                img = self.load_image(item['image_path'])  # 读取图片
+                axes[i, j + 1].imshow(img)  # 在对应的子图中显示图片
+                axes[i, j + 1].axis('off')  # 关闭坐标轴
+                axes[i, j + 1].set_title(item["semantic_category"], fontsize=10)
+            for j in range(len(outfit), num_cols):
+                axes[i, j].axis("off")
+
+            # 在每一行的底部添加一条横线
+            axes[i, 0].axhline(y=0, color='black', linewidth=1)
+        # 隐藏最后一行的横线
+        axes[-1, 0].axhline(y=0, color='white', linewidth=1)
+
+        # 调整布局
+        plt.subplots_adjust(wspace=0.1, hspace=0.1)
+        plt.tight_layout()
+
         if output_path:
-            # 设置子图的行列数
+            plt.savefig(output_path)
-            num_rows = len(outfits)
-            num_cols = max([len(x) for x in outfits]) + 1  # 一个是图片，一个是分数
-
-            # 创建一个新的图像，并指定子图的行列数
-            fig, axes = plt.subplots(num_rows, num_cols, figsize=(8, 15))
-
-            title = f"Best {topk} Outfits" if best else f"Worst {topk} Outfits"
-            fig.suptitle(title, fontsize=16)
-
-            # 遍历每套outfit并将其显示在对应的子图中
-            for i, (outfit, score) in enumerate(zip(outfits, scores)):
-                # 显示分数
-                axes[i, 0].text(0.1, 0.5, f"Score: {score[0]:.4f}", fontsize=12)
-                axes[i, 0].axis("off")
-                # 显示图片
-                for j, item in enumerate(outfit):
-                    img = mpimg.imread(item['image_path'])  # 读取图片
-                    axes[i, j + 1].imshow(img)  # 在对应的子图中显示图片
-                    axes[i, j + 1].axis('off')  # 关闭坐标轴
-                    axes[i, j + 1].set_title(item["semantic_category"], fontsize=10)
-                for j in range(len(outfit), num_cols):
-                    axes[i, j].axis("off")
-
-                # 在每一行的底部添加一条横线
-                axes[i, 0].axhline(y=0, color='black', linewidth=1)
-            # 隐藏最后一行的横线
-            axes[-1, 0].axhline(y=0, color='white', linewidth=1)
-
-            # 调整布局
-            plt.subplots_adjust(wspace=0.1, hspace=0.1)
-            plt.tight_layout()
-
-            if output_path:
-                plt.savefig(output_path)
-            else:
-                plt.show()
         else:
-            return outfits, scores.numpy().flatten().tolist()
+            plt.show()
 
 
 class OutfitMatcherHon(OutfitMatcher):
@@ -216,60 +328,17 @@ class OutfitMaterTypeAware(OutfitMatcher):
         'outerwear', 'scarves', 'shoes', 'sunglasses', 'tops'
     ]
 
-    @RunTime
     def __init__(self):
         super().__init__()
 
-    @RunTime
+    def preprocess(self, outfits, features):
-    # TODO 用多线程读图片
-    def load_image(self, img_path):
-        try:
-            # 从 MinIO 中获取对象（图像文件）
-            image_data = self.minio_client.get_object(img_path.split("/", 1)[0], img_path.split("/", 1)[1])
-
-            # 读取图像数据并转换为 PIL 图像对象
-            pil_image = Image.open(io.BytesIO(image_data.data)).convert("RGB")
-
-            # 将 PIL 图像转换为 NumPy 数组
-            # image_array = np.array(pil_image)
-
-            return pil_image
-        except Exception as e:
-            print(f"An error occurred: {e}")
-            return None
-        # if 'http' in img_path:
-        #     file = requests.get(img_path)
-        #     image = cv2.imdecode(np.fromstring(file.content, np.uint8), 1)
-        #     image = Image.fromarray(image.astype('uint8'), 'RGB')
-        # else:
-        #     image = Image.open(img_path).convert('RGB')
-        # return np.array(image)
-
-    @staticmethod
-    def resize_image(img):
-        """
-        Args:
-            img: ndarray (height, width, channel)
-        """
-        image_transforms = transforms.Compose([
-            transforms.Resize(112),
-            transforms.CenterCrop(112),
-            transforms.ToTensor(),
-            transforms.Normalize(mean=[0.485, 0.456, 0.406],
-                                 std=[0.229, 0.224, 0.225]),
-        ])
-        resized_img = image_transforms(img).numpy()
-        return resized_img
-
-    def preprocess(self, outfits):
         outfit_images = []
         outfit_categories = []
         for outfit in outfits:
             images = []
             categories = []
             for item in outfit:
-                image = self.load_image(item["image_path"])
+                image = features[item["item_name"]]
-                image = self.resize_image(image)
                 images.append(image)
                 category = self.base_fashion_categories.index(item["mapped_cate"])
                 categories.append(category)
@@ -277,12 +346,10 @@ class OutfitMaterTypeAware(OutfitMatcher):
             outfit_images.append(images)  # List[(items, 3, 224, 224)]
             categories = np.array(categories)
             outfit_categories.append(categories)  # List[(items)]
-        outfit_images, mask = self.pad_array(outfit_images, value=0)
+        return outfit_images, outfit_categories
-        outfit_categories, _ = self.pad_array(outfit_categories, value=len(self.base_fashion_categories))
-        return outfit_images, outfit_categories, mask
 
     @RunTime
-    def get_result(self, outfits):
+    def get_result(self, outfits, features):
         """Input outfits structure and output scores.
         Args:
             outfits: outfits to be evaluated.
@@ -310,29 +377,27 @@ class OutfitMaterTypeAware(OutfitMatcher):
                   ],
                   ...
                 ]
+            features: dict(item_name = np.Array) image features of those items
         Returns:
             scores: List of float
         """
-        image, category, mask = self.preprocess(outfits)
+        outfit_images, outfit_categories = self.preprocess(outfits, features)
-        client = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
+        scores = []
-        # 输入集
+        for images, categories in zip(outfit_images, outfit_categories):
-        inputs = [
+            client = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
-            httpclient.InferInput("input__0", image.shape, datatype="FP32"),
+            # 输入集
-            httpclient.InferInput("input__1", category.shape, datatype="INT16"),
+            inputs = [
-            httpclient.InferInput("input__2", mask.shape, datatype="FP32"),
+                httpclient.InferInput("input__0", images.shape, datatype="FP32"),
-        ]
+                httpclient.InferInput("input__1", categories.shape, datatype="INT16")
-        inputs[0].set_data_from_numpy(image.astype(np.float32), binary_data=True)
+            ]
-        inputs[1].set_data_from_numpy(category.astype(np.int16), binary_data=True)
+            inputs[0].set_data_from_numpy(images.astype(np.float32), binary_data=True)
-        inputs[2].set_data_from_numpy(mask.astype(np.float32), binary_data=True)
+            inputs[1].set_data_from_numpy(categories.astype(np.int16), binary_data=True)
-        # 输出集
+            # 输出集
-        outputs = [
+            outputs = [
-            httpclient.InferRequestedOutput("output__0", binary_data=True),
+                httpclient.InferRequestedOutput("output__0", binary_data=True),
-            httpclient.InferRequestedOutput("output__1", binary_data=True)
+            ]
-        ]
+            results = client.infer(model_name="outfit_matcher_type_aware", inputs=inputs, outputs=outputs)
-        results = client.infer(model_name="outfit_matcher_type_aware", inputs=inputs, outputs=outputs)
+            scores.append(results.as_numpy("output__0"))  # Shape (N, 1)
-        # 推理
-        # 取结果
-        scores = torch.from_numpy(results.as_numpy("output__0"))  # Shape (N, 1)
-        features = torch.from_numpy(results.as_numpy("output__1"))  # Shape (N, 64)
 
-        return scores, features
+        scores = np.stack(scores, axis=0)
+        return scores.flatten()