aidlab/sora_python
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
zhouchengrong
2024-03-28 10:38:57 +08:00
parent 1cf9ca55f8 8d34a4ed7f
commit e7f67b3c7b
6 changed files with 333 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
app/core/config.py
View File

@@ -27,8 +27,15 @@ MINIO_SECURE = False
MINIO_ACCESS = "e8zc55mzDOh4IzRrZ9Oa"
MINIO_SECRET = "uHfqJ7UkwA1PTDGfnA44Hp9ux5YkZTkzZLjeOYhE"
<<<<<<< HEAD
# OM_TRITON_IP = "10.1.1.150"
# OM_TRITON_PORT = "7000"
OM_TRITON_IP = "127.0.0.1"
OM_TRITON_PORT = "8000"
=======
OM_TRITON_IP = "10.1.1.240"
OM_TRITON_PORT = "10010"
>>>>>>> 1f23781b16e59bfbcbbb4d252e6a61685267e6c7
ATT_TRITON_IP = "10.1.1.240"
ATT_TRITON_PORT = "10020"
@@ -39,6 +46,11 @@ ATT_TRITON_PORT = "10020"
# FASHION_CATEGORIES_MAPPING = "app/service/outfit_matcher/config/fashion_category_mapping.json"
# pycharm debug
<<<<<<< HEAD
LOGSPATH = "logs/errors.log"
FASHION_CATEGORIES = "config/fashion_categories.json"
FASHION_CATEGORIES_MAPPING = "config/fashion_category_mapping.json"
=======
LOGS_PATH = "logs/errors.log"
FASHION_CATEGORIES = "service/outfit_matcher/config/fashion_categories.json"
FASHION_CATEGORIES_MAPPING = "service/outfit_matcher/config/fashion_category_mapping.json"
@@ -46,4 +58,5 @@ FASHION_CATEGORIES_MAPPING = "service/outfit_matcher/config/fashion_category_map
# LOGS_PATH = "app/logs/errors.log"
# FASHION_CATEGORIES = "./config/fashion_categories.json"
# FASHION_CATEGORIES_MAPPING = "./config/fashion_category_mapping.json"
# FASHION_CATEGORIES_MAPPING = "./config/fashion_category_mapping.json"
>>>>>>> 1f23781b16e59bfbcbbb4d252e6a61685267e6c7

267
app/service/outfit_matcher/outfit_evaluator.py
View File

@@ -13,6 +13,103 @@ from app.service.outfit_matcher.foco import extract_main_colors
from app.service.utils.decorator import RunTime
class Backbone(object):
def __init__(self):
self.tritonclient = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
self.minio_client = Minio(
f"{MINIO_IP}:{MINIO_PORT}",
access_key=MINIO_ACCESS,
secret_key=MINIO_SECRET,
secure=MINIO_SECURE)
@RunTime
# TODO 用多线程读图片
def load_image(self, img_path):
try:
# 从 MinIO 中获取对象（图像文件）
image_data = self.minio_client.get_object(img_path.split("/", 1)[0], img_path.split("/", 1)[1])
# 读取图像数据并转换为 PIL 图像对象
pil_image = Image.open(io.BytesIO(image_data.data)).convert("RGB")
# 将 PIL 图像转换为 NumPy 数组
# image_array = np.array(pil_image)
return pil_image
except Exception as e:
print(f"An error occurred: {e}")
return None
@staticmethod
def resize_image(img):
"""
Args:
img: ndarray (height, width, channel)
"""
image_transforms = transforms.Compose([
transforms.Resize(112),
transforms.CenterCrop(112),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
resized_img = image_transforms(img).numpy()
return resized_img
def preprocess(self, items):
images = []
for item in items:
image = self.load_image(item["image_path"])
image = self.resize_image(image)
images.append(image)
images = np.stack(images, axis=0)
return images
@RunTime
def get_result(self, items):
"""Input items and output features for similiarity.
Args:
items: images of fashion items
Example:
[
{
"item_name": "MSE_57987",
"semantic_category": "BOTTOM/PANTS",
"image_path": "D:\\PhD_Study\\MIXI\\mitu\\image\\2024 SS\\MSE_57987.jpg",
"mapped_cate": "bottoms"
},
{
"item_name": "MPO_SP7712",
"semantic_category": "TOP/TANK",
"image_path": "D:\\PhD_Study\\MIXI\\mitu\\image\\2024 SS\\MPO_SP7712.jpg",
"mapped_cate": "tops"
},
{
"item_name": "MWSS27195",
"semantic_category": "OUTERWEAR/GILET",
"image_path": "D:\\PhD_Study\\MIXI\\mitu\\image\\2024 SS\\MWSS27195.jpg",
"mapped_cate": "outerwear"
}
]
Returns:
scores: List of image features
"""
image = self.preprocess(items)
client = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
# 输入集
inputs = [
httpclient.InferInput("input__0", image.shape, datatype="FP32"),
]
inputs[0].set_data_from_numpy(image.astype(np.float32), binary_data=True)
# 输出集
outputs = [
httpclient.InferRequestedOutput("output__0", binary_data=True),
]
results = client.infer(model_name="outfit_matcher_backbone", inputs=inputs, outputs=outputs)
# 推理
# 取结果
features = results.as_numpy("output__0") # Shape (N, 64)
return features
class OutfitMatcher(object):
def __init__(self):
self.tritonclient = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
@@ -22,6 +119,22 @@ class OutfitMatcher(object):
secret_key=MINIO_SECRET,
secure=MINIO_SECURE)
def load_image(self, img_path):
try:
# 从 MinIO 中获取对象（图像文件）
image_data = self.minio_client.get_object(img_path.split("/", 1)[0], img_path.split("/", 1)[1])
# 读取图像数据并转换为 PIL 图像对象
pil_image = Image.open(io.BytesIO(image_data.data)).convert("RGB")
# 将 PIL 图像转换为 NumPy 数组
# image_array = np.array(pil_image)
return pil_image
except Exception as e:
print(f"An error occurred: {e}")
return None
@staticmethod
def pad_array(input_value, value=0):
"""pad List of Array into same batch size
@@ -77,51 +190,48 @@ class OutfitMatcher(object):
@RunTime
def visualize(self, outfits, scores, topk=5, best=True, output_path=None):
# 将outfits和scores按照scores的值进行排序
sorted_indices = np.argsort(-scores.flatten() if best else scores.flatten())[:topk] # 使用负号进行降序排序
# sorted_indices = np.argsort(-scores if best else scores)[:topk] # for HON
sorted_indices = np.argsort(scores if best else -scores)[:topk] # type-aware
outfits = [outfits[i] for i in sorted_indices] # 最好或最差的五个
scores = scores[sorted_indices] # 这五个的分数
# 是否画出来
# 设置子图的行列数
num_rows = len(outfits)
num_cols = max([len(x) for x in outfits]) + 1 # 一个是图片，一个是分数
# 创建一个新的图像，并指定子图的行列数
fig, axes = plt.subplots(num_rows, num_cols, figsize=(8, 15))
title = f"Best {topk} Outfits" if best else f"Worst {topk} Outfits"
fig.suptitle(title, fontsize=16)
# 遍历每套outfit并将其显示在对应的子图中
for i, (outfit, score) in enumerate(zip(outfits, scores)):
# 显示分数
axes[i, 0].text(0.1, 0.5, f"Score: {score:.4f}", fontsize=12)
axes[i, 0].axis("off")
# 显示图片
for j, item in enumerate(outfit):
img = self.load_image(item['image_path']) # 读取图片
axes[i, j + 1].imshow(img) # 在对应的子图中显示图片
axes[i, j + 1].axis('off') # 关闭坐标轴
axes[i, j + 1].set_title(item["semantic_category"], fontsize=10)
for j in range(len(outfit), num_cols):
axes[i, j].axis("off")
# 在每一行的底部添加一条横线
axes[i, 0].axhline(y=0, color='black', linewidth=1)
# 隐藏最后一行的横线
axes[-1, 0].axhline(y=0, color='white', linewidth=1)
# 调整布局
plt.subplots_adjust(wspace=0.1, hspace=0.1)
plt.tight_layout()
if output_path:
# 设置子图的行列数
num_rows = len(outfits)
num_cols = max([len(x) for x in outfits]) + 1 # 一个是图片，一个是分数
# 创建一个新的图像，并指定子图的行列数
fig, axes = plt.subplots(num_rows, num_cols, figsize=(8, 15))
title = f"Best {topk} Outfits" if best else f"Worst {topk} Outfits"
fig.suptitle(title, fontsize=16)
# 遍历每套outfit并将其显示在对应的子图中
for i, (outfit, score) in enumerate(zip(outfits, scores)):
# 显示分数
axes[i, 0].text(0.1, 0.5, f"Score: {score[0]:.4f}", fontsize=12)
axes[i, 0].axis("off")
# 显示图片
for j, item in enumerate(outfit):
img = mpimg.imread(item['image_path']) # 读取图片
axes[i, j + 1].imshow(img) # 在对应的子图中显示图片
axes[i, j + 1].axis('off') # 关闭坐标轴
axes[i, j + 1].set_title(item["semantic_category"], fontsize=10)
for j in range(len(outfit), num_cols):
axes[i, j].axis("off")
# 在每一行的底部添加一条横线
axes[i, 0].axhline(y=0, color='black', linewidth=1)
# 隐藏最后一行的横线
axes[-1, 0].axhline(y=0, color='white', linewidth=1)
# 调整布局
plt.subplots_adjust(wspace=0.1, hspace=0.1)
plt.tight_layout()
if output_path:
plt.savefig(output_path)
else:
plt.show()
plt.savefig(output_path)
else:
return outfits, scores.numpy().flatten().tolist()
plt.show()
class OutfitMatcherHon(OutfitMatcher):
@@ -216,60 +326,17 @@ class OutfitMaterTypeAware(OutfitMatcher):
'outerwear', 'scarves', 'shoes', 'sunglasses', 'tops'
]
@RunTime
def __init__(self):
super().__init__()
@RunTime
# TODO 用多线程读图片
def load_image(self, img_path):
try:
# 从 MinIO 中获取对象（图像文件）
image_data = self.minio_client.get_object(img_path.split("/", 1)[0], img_path.split("/", 1)[1])
# 读取图像数据并转换为 PIL 图像对象
pil_image = Image.open(io.BytesIO(image_data.data)).convert("RGB")
# 将 PIL 图像转换为 NumPy 数组
# image_array = np.array(pil_image)
return pil_image
except Exception as e:
print(f"An error occurred: {e}")
return None
# if 'http' in img_path:
# file = requests.get(img_path)
# image = cv2.imdecode(np.fromstring(file.content, np.uint8), 1)
# image = Image.fromarray(image.astype('uint8'), 'RGB')
# else:
# image = Image.open(img_path).convert('RGB')
# return np.array(image)
@staticmethod
def resize_image(img):
"""
Args:
img: ndarray (height, width, channel)
"""
image_transforms = transforms.Compose([
transforms.Resize(112),
transforms.CenterCrop(112),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
resized_img = image_transforms(img).numpy()
return resized_img
def preprocess(self, outfits):
def preprocess(self, outfits, features):
outfit_images = []
outfit_categories = []
for outfit in outfits:
images = []
categories = []
for item in outfit:
image = self.load_image(item["image_path"])
image = self.resize_image(image)
image = features[item["item_name"]]
images.append(image)
category = self.base_fashion_categories.index(item["mapped_cate"])
categories.append(category)
@@ -277,12 +344,10 @@ class OutfitMaterTypeAware(OutfitMatcher):
outfit_images.append(images) # List[(items, 3, 224, 224)]
categories = np.array(categories)
outfit_categories.append(categories) # List[(items)]
outfit_images, mask = self.pad_array(outfit_images, value=0)
outfit_categories, _ = self.pad_array(outfit_categories, value=len(self.base_fashion_categories))
return outfit_images, outfit_categories, mask
return outfit_images, outfit_categories
@RunTime
def get_result(self, outfits):
def get_result(self, outfits, features):
"""Input outfits structure and output scores.
Args:
outfits: outfits to be evaluated.
@@ -310,9 +375,32 @@ class OutfitMaterTypeAware(OutfitMatcher):
],
...
]
features: dict(item_name = np.Array) image features of those items
Returns:
scores: List of float
"""
<<<<<<< HEAD
outfit_images, outfit_categories = self.preprocess(outfits, features)
scores = []
for images, categories in zip(outfit_images, outfit_categories):
client = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
# 输入集
inputs = [
httpclient.InferInput("input__0", images.shape, datatype="FP32"),
httpclient.InferInput("input__1", categories.shape, datatype="INT16")
]
inputs[0].set_data_from_numpy(images.astype(np.float32), binary_data=True)
inputs[1].set_data_from_numpy(categories.astype(np.int16), binary_data=True)
# 输出集
outputs = [
httpclient.InferRequestedOutput("output__0", binary_data=True),
]
results = client.infer(model_name="outfit_matcher_type_aware", inputs=inputs, outputs=outputs)
scores.append(results.as_numpy("output__0")) # Shape (N, 1)
scores = np.stack(scores, axis=0)
return scores.flatten()
=======
image, category, mask = self.preprocess(outfits)
client = httpclient.InferenceServerClient(url=f"{OM_TRITON_IP}:{OM_TRITON_PORT}")
# 输入集
@@ -336,3 +424,4 @@ class OutfitMaterTypeAware(OutfitMatcher):
features = torch.from_numpy(results.as_numpy("output__1")) # Shape (N, 64)
return scores, features
>>>>>>> 1f23781b16e59bfbcbbb4d252e6a61685267e6c7

46
app/service/outfit_matcher/service.py
View File

@@ -1,19 +1,38 @@
import json
import os
from pprint import pprint
from tqdm import tqdm
import numpy as np
from app.service.outfit_matcher.dataset import FashionDataset
from app.service.outfit_matcher.outfit_evaluator import OutfitMaterTypeAware
from app.service.outfit_matcher.outfit_evaluator import OutfitMaterTypeAware, Backbone
if __name__ == '__main__':
with open("./test_param/recommendation_test.json", "r") as f:
param = json.load(f)
fashion_dataset = FashionDataset(param["database"])
backbone_service = Backbone()
service = OutfitMaterTypeAware()
best_list = []
bad_list = []
for item in param["query"]:
# read feature from vector database
all_items = param["query"] + param["database"]
unextracted_item = []
prepared_feature = {}
for item in all_items:
if f'{item["item_name"]}.npy' not in os.listdir("feature"):
unextracted_item.append(item)
if len(unextracted_item) > 0:
extracted_features = backbone_service.get_result(unextracted_item)
for i, item in enumerate(unextracted_item):
np.save(f'feature/{item["item_name"]}.npy', extracted_features[i])
for item in all_items:
if item["item_name"] not in prepared_feature.keys():
prepared_feature[item["item_name"]] = np.load(f'feature/{item["item_name"]}.npy')
for item in tqdm(param["query"] * 10):
outfits = fashion_dataset.generate_outfit(item, param["topk"], param["max_outfits"])
<<<<<<< HEAD
scores = service.get_result(outfits, prepared_feature)
=======
scores, features = service.get_result(outfits)
# save features
@@ -22,16 +41,15 @@ if __name__ == '__main__':
# 存入数据库
# 关闭链接
>>>>>>> 1f23781b16e59bfbcbbb4d252e6a61685267e6c7
# print(scores)
# print(len(scores))
best_outfits, best_scores = service.visualize(outfits, scores, param["topk"], best=True,
# output_path=os.path.join(r"E:\workspace\outfit_matcher\2024 SS Outfit", f"{item['item_name']}_best_{param['topk']}.png")
)
bad_outfits, bad_scores = service.visualize(outfits, scores, param["topk"], best=False,
# output_path=os.path.join(r"E:\workspace\outfit_matcher\2024 SS Outfit", f"{item['item_name']}_worst_{param['topk']}.png")
)
best_list.append({"best_outfits": best_outfits, "best_scores": best_scores})
bad_list.append({"bad_outfits": bad_outfits, "bad_scores": bad_scores})
# service.visualize(outfits, scores, param["topk"], best=True,
# output_path=os.path.join(r"D:\PhD_Study\MIXI\mitu\image\123",
# f"{item['item_name']}_best_{param['topk']}.png"))
# service.visualize(outfits, scores, param["topk"], best=False,
# output_path=os.path.join(r"D:\PhD_Study\MIXI\mitu\image\123",
# f"{item['item_name']}_worst_{param['topk']}.png"))
sorted_indices = np.argsort(scores)[:param["topk"]] # type-aware
outfits = [outfits[i] for i in sorted_indices] # 最好的五个
pprint(best_list)
pprint(bad_list)

105
app/service/outfit_matcher/test_param/recommendation_test.json
View File

@@ -1,9 +1,114 @@
{
<<<<<<< HEAD
"topk": 5,
"max_outfits": 200,
"is_best": true,
"query": [
{
"item_name": "MSE_58107",
"semantic_category": "TOP/SHIRT",
"image_path": "test/2024 SS/MSE_58107.jpg"
},
{
"item_name": "MKTS27047",
"semantic_category": "ONE PIECE/DRESS",
"image_path": "test/2024 SS/MKTS27047.jpg"
},
{
"item_name": "MKTS27028",
"semantic_category": "OUTERWEAR/JACKET",
"image_path": "test/2024 SS/MKTS27028.jpg"
},
{
"item_name": "MSE_58057",
"semantic_category": "OUTERWEAR/BLAZER",
"image_path": "test/2024 SS/MSE_58057.jpg"
},
{
"item_name": "MSE_58495",
"semantic_category": "TOP/SHIRT",
"image_path": "test/2024 SS/MSE_58495.jpg"
}
],
"database": [
{
"item_name": "MKTS27017",
"semantic_category": "OUTERWEAR/WINDBREAKER",
"image_path": "test/2024 SS/MKTS27017.jpg"
},
{
"item_name": "MKTS27047",
"semantic_category": "ONE PIECE/DRESS",
"image_path": "test/2024 SS/MKTS27047.jpg"
},
{
"item_name": "MKTS27000",
"semantic_category": "BOTTOM/PANTS",
"image_path": "test/2024 SS/MKTS27000.jpg"
},
{
"item_name": "MKTS27001",
"semantic_category": "BOTTOM/SHORTS",
"image_path": "test/2024 SS/MKTS27001.jpg"
},
{
"item_name": "MZOS27178",
"semantic_category": "KNIT/CARDIGAN",
"image_path": "test/2024 SS/MZOS27178.jpg"
},
{
"item_name": "MZOS27179",
"semantic_category": "KNIT/KNIT TOP",
"image_path": "test/2024 SS/MZOS27179.jpg"
},
{
"item_name": "MWSS27184",
"semantic_category": "TOP/TEE",
"image_path": "test/2024 SS/MWSS27184.jpg"
},
{
"item_name": "MWSS27191",
"semantic_category": "OUTERWEAR/TWIN SET",
"image_path": "test/2024 SS/MWSS27191.jpg"
},
{
"item_name": "MWSS27193",
"semantic_category": "OUTERWEAR/JACKET",
"image_path": "test/2024 SS/MWSS27193.jpg"
},
{
"item_name": "MWSS27195",
"semantic_category": "OUTERWEAR/GILET",
"image_path": "test/2024 SS/MWSS27195.jpg"
},
{
"item_name": "MWSS27200",
"semantic_category": "KNIT/VEST",
"image_path": "test/2024 SS/MWSS27200.jpg"
},
{
"item_name": "MWSS27209",
"semantic_category": "ONE PIECE/DRESS",
"image_path": "test/2024 SS/MWSS27209.jpg"
},
{
"item_name": "MWSS27210",
"semantic_category": "ONE PIECE/DRESS",
"image_path": "test/2024 SS/MWSS27210.jpg"
},
{
"item_name": "MWSS27211",
"semantic_category": "ONE PIECE/DRESS",
"image_path": "test/2024 SS/MWSS27211.jpg"
},
{
=======
"topk": 1,
"max_outfits": 100,
"is_best": true,
"query": [
{
>>>>>>> 1f23781b16e59bfbcbbb4d252e6a61685267e6c7
"item_name": "MWSS27212",
"semantic_category": "TOP/BLOUSE",
"image_path": "test/2024 SS/MWSS27212.jpg"