aidlab/AiDA_Python
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat design 提速测试

Browse Source 
fix
This commit is contained in:
zhouchengrong
2024-09-19 15:10:50 +08:00
parent 64dec8fb6a
commit fabe64785e
12 changed files with 129 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app/api/api_design.py
View File

@@ -7,6 +7,7 @@ from fastapi import APIRouter, HTTPException, UploadFile, File, Form
from app.schemas.design import DesignModel, DesignProgressModel, ModelProgressModel, DBGConfigModel from app.schemas.design import DesignModel, DesignProgressModel, ModelProgressModel, DBGConfigModel
from app.schemas.response_template import ResponseModel from app.schemas.response_template import ResponseModel
from app.service.design.model_process_service import model_transpose from app.service.design.model_process_service import model_transpose
from app.service.design.service import generate
from app.service.design.service_design_batch_generate import start_design_batch_generate from app.service.design.service_design_batch_generate import start_design_batch_generate
from app.service.design.utils.redis_utils import Redis from app.service.design.utils.redis_utils import Redis
from app.service.design_test.batch_design import design_generate from app.service.design_test.batch_design import design_generate
@@ -183,7 +184,7 @@ def design(request_data: DesignModel):
# logger.info(f"design request item is : @@@@@@:{json.dumps(request_data.dict())}") # logger.info(f"design request item is : @@@@@@:{json.dumps(request_data.dict())}")
# data = generate(request_data=request_data) # data = generate(request_data=request_data)
# logger.info(f"design response @@@@@@:{json.dumps(data)}") # logger.info(f"design response @@@@@@:{json.dumps(data)}")
#
logger.info(f"design request item is : @@@@@@:{json.dumps(request_data.dict())}") logger.info(f"design request item is : @@@@@@:{json.dumps(request_data.dict())}")
data = design_generate(request_data=request_data) data = design_generate(request_data=request_data)
logger.info(f"design response @@@@@@:{json.dumps(data)}") logger.info(f"design response @@@@@@:{json.dumps(data)}")

20
app/service/design/utils/upload_image.py
View File

@@ -17,7 +17,7 @@ from app.service.utils.oss_client import oss_upload_image
# @RunTime # @RunTime
def upload_png_mask(minio_client, front_image, object_name, mask=None): def upload_png_mask(front_image, object_name, mask=None):
try: try:
mask_url = None mask_url = None
if mask is not None: if mask is not None:
@@ -25,29 +25,15 @@ def upload_png_mask(minio_client, front_image, object_name, mask=None):
# 将掩模的3通道转换为4通道，白色部分不透明，黑色部分透明 # 将掩模的3通道转换为4通道，白色部分不透明，黑色部分透明
rgba_image = cv2.cvtColor(mask_inverted, cv2.COLOR_BGR2BGRA) rgba_image = cv2.cvtColor(mask_inverted, cv2.COLOR_BGR2BGRA)
rgba_image[rgba_image[:, :, 0] == 0] = [0, 0, 0, 0] rgba_image[rgba_image[:, :, 0] == 0] = [0, 0, 0, 0]
# image_bytes = io.BytesIO() req = oss_upload_image(bucket=AIDA_CLOTHING, object_name=f"mask/mask_{object_name}.png", image_bytes=cv2.imencode('.png', rgba_image)[1])
# image_bytes.write(cv2.imencode('.png', rgba_image)[1].tobytes())
# image_bytes.seek(0)
# mask_url = f"{AIDA_CLOTHING}/{minio_client.put_object('aida-clothing', f'mask/mask_{object_name}.png', image_bytes, len(image_bytes.getvalue()), content_type='image/png').object_name}"
# oss upload ####################
req = oss_upload_image(oss_client=minio_client, bucket=AIDA_CLOTHING, object_name=f"mask/mask_{object_name}.png", image_bytes=cv2.imencode('.png', rgba_image)[1])
mask_url = f"{AIDA_CLOTHING}/mask/mask_{object_name}.png" mask_url = f"{AIDA_CLOTHING}/mask/mask_{object_name}.png"
image_data = io.BytesIO() image_data = io.BytesIO()
front_image.save(image_data, format='PNG') front_image.save(image_data, format='PNG')
image_data.seek(0) image_data.seek(0)
image_bytes = image_data.read() image_bytes = image_data.read()
# image_url = f"{AIDA_CLOTHING}/{minio_client.put_object('aida-clothing', f'image/image_{object_name}.png', io.BytesIO(image_bytes), len(image_bytes), content_type='image/png').object_name}" req = oss_upload_image(bucket=AIDA_CLOTHING, object_name=f"image/image_{object_name}.png", image_bytes=image_bytes)
req = oss_upload_image(oss_client=minio_client, bucket=AIDA_CLOTHING, object_name=f"image/image_{object_name}.png", image_bytes=image_bytes)
image_url = f"{AIDA_CLOTHING}/image/image_{object_name}.png" image_url = f"{AIDA_CLOTHING}/image/image_{object_name}.png"
return front_image, image_url, mask_url return front_image, image_url, mask_url
except Exception as e: except Exception as e:
logging.warning(f"upload_png_mask runtime exception : {e}") logging.warning(f"upload_png_mask runtime exception : {e}")
# @RunTime
# def upload_png_mask(front_image, object_name, mask=None):
# mask_url = None
# if mask is not None:
# mask_url = f"{AIDA_CLOTHING}/mask/mask_{object_name}.png"
# image_url = f"{AIDA_CLOTHING}/image/image_{object_name}.png"
# return front_image, image_url, mask_url

8
app/service/design_test/batch_design.py
View File

@@ -13,7 +13,8 @@ from minio import Minio
from app.core.config import PRIORITY_DICT from app.core.config import PRIORITY_DICT
from app.service.design.utils.redis_utils import Redis from app.service.design.utils.redis_utils import Redis
from app.service.design_test.item import BodyItem, TopItem, BottomItem from app.service.design_test.item import BodyItem, TopItem, BottomItem
from app.service.utils.oss_client import oss_upload_image from app.service.utils.decorator import RunTime
from app.service.utils.new_oss_client import oss_upload_image
id_lock = threading.Lock() id_lock = threading.Lock()
@@ -298,10 +299,11 @@ def synthesis(data, size, basic_info):
logging.warning(f"synthesis runtime exception : {e}") logging.warning(f"synthesis runtime exception : {e}")
@RunTime
def design_generate(request_data): def design_generate(request_data):
objects_data = request_data.dict()['objects'] objects_data = request_data.dict()['objects']
process_id = request_data.dict()['process_id'] process_id = request_data.dict()['process_id']
object_response = [] object_response = {}
threads = [] threads = []
active_threads = 0 active_threads = 0
lock = threading.Lock() lock = threading.Lock()
@@ -362,7 +364,7 @@ def design_generate(request_data):
update_progress(process_id, total) update_progress(process_id, total)
with lock: with lock:
object_response.append(items_response) object_response[step] = items_response
active_threads -= 1 active_threads -= 1
for step, object in enumerate(objects_data): for step, object in enumerate(objects_data):

2
app/service/design_test/pipeline/color.py
View File

@@ -3,7 +3,7 @@ import logging
import cv2 import cv2
import numpy as np import numpy as np
from app.service.utils.oss_client import oss_get_image from app.service.utils.new_oss_client import oss_get_image
logger = logging.getLogger() logger = logging.getLogger()

2
app/service/design_test/pipeline/loading.py
View File

@@ -5,7 +5,7 @@ import cv2
import numpy as np import numpy as np
from PIL import Image from PIL import Image
from app.service.utils.oss_client import oss_get_image from app.service.utils.new_oss_client import oss_get_image
logger = logging.getLogger() logger = logging.getLogger()

2
app/service/design_test/pipeline/print_painting.py
View File

@@ -4,7 +4,7 @@ import cv2
import numpy as np import numpy as np
from PIL import Image from PIL import Image
from app.service.utils.oss_client import oss_get_image from app.service.utils.new_oss_client import oss_get_image
class PrintPainting: class PrintPainting:

2
app/service/design_test/pipeline/segmentation.py
View File

@@ -6,7 +6,7 @@ import numpy as np
from app.core.config import SEG_CACHE_PATH from app.core.config import SEG_CACHE_PATH
from app.service.design.utils.design_ensemble import get_seg_result from app.service.design.utils.design_ensemble import get_seg_result
from app.service.utils.oss_client import oss_get_image from app.service.utils.new_oss_client import oss_get_image
logger = logging.getLogger() logger = logging.getLogger()

4
app/service/design_test/pipeline/split.py
View File

@@ -8,9 +8,9 @@ from cv2 import cvtColor, COLOR_BGR2RGBA
from app.core.config import AIDA_CLOTHING from app.core.config import AIDA_CLOTHING
from app.service.design.utils.conversion_image import rgb_to_rgba from app.service.design.utils.conversion_image import rgb_to_rgba
from app.service.design.utils.upload_image import upload_png_mask from app.service.design_test.utils.upload_image import upload_png_mask
from app.service.utils.generate_uuid import generate_uuid from app.service.utils.generate_uuid import generate_uuid
from app.service.utils.oss_client import oss_upload_image from app.service.utils.new_oss_client import oss_upload_image
class Split(object): class Split(object):

24
app/service/design_test/utils/upload_image.py
View File

@@ -13,12 +13,11 @@ import logging
import cv2 import cv2
from app.core.config import * from app.core.config import *
from app.service.utils.decorator import RunTime from app.service.utils.new_oss_client import oss_upload_image
from app.service.utils.oss_client import oss_upload_image
# @RunTime # @RunTime
def upload_png_mask(front_image, object_name, mask=None): def upload_png_mask(minio_client, front_image, object_name, mask=None):
try: try:
mask_url = None mask_url = None
if mask is not None: if mask is not None:
@@ -26,30 +25,15 @@ def upload_png_mask(front_image, object_name, mask=None):
# 将掩模的3通道转换为4通道，白色部分不透明，黑色部分透明 # 将掩模的3通道转换为4通道，白色部分不透明，黑色部分透明
rgba_image = cv2.cvtColor(mask_inverted, cv2.COLOR_BGR2BGRA) rgba_image = cv2.cvtColor(mask_inverted, cv2.COLOR_BGR2BGRA)
rgba_image[rgba_image[:, :, 0] == 0] = [0, 0, 0, 0] rgba_image[rgba_image[:, :, 0] == 0] = [0, 0, 0, 0]
# image_bytes = io.BytesIO() req = oss_upload_image(oss_client=minio_client, bucket=AIDA_CLOTHING, object_name=f"mask/mask_{object_name}.png", image_bytes=cv2.imencode('.png', rgba_image)[1])
# image_bytes.write(cv2.imencode('.png', rgba_image)[1].tobytes())
# image_bytes.seek(0)
# mask_url = f"{AIDA_CLOTHING}/{minio_client.put_object('aida-clothing', f'mask/mask_{object_name}.png', image_bytes, len(image_bytes.getvalue()), content_type='image/png').object_name}"
# oss upload ####################
req = oss_upload_image(bucket=AIDA_CLOTHING, object_name=f"mask/mask_{object_name}.png", image_bytes=cv2.imencode('.png', rgba_image)[1])
mask_url = f"{AIDA_CLOTHING}/mask/mask_{object_name}.png" mask_url = f"{AIDA_CLOTHING}/mask/mask_{object_name}.png"
image_data = io.BytesIO() image_data = io.BytesIO()
front_image.save(image_data, format='PNG') front_image.save(image_data, format='PNG')
image_data.seek(0) image_data.seek(0)
image_bytes = image_data.read() image_bytes = image_data.read()
# image_url = f"{AIDA_CLOTHING}/{minio_client.put_object('aida-clothing', f'image/image_{object_name}.png', io.BytesIO(image_bytes), len(image_bytes), content_type='image/png').object_name}" req = oss_upload_image(oss_client=minio_client, bucket=AIDA_CLOTHING, object_name=f"image/image_{object_name}.png", image_bytes=image_bytes)
req = oss_upload_image(bucket=AIDA_CLOTHING, object_name=f"image/image_{object_name}.png", image_bytes=image_bytes)
image_url = f"{AIDA_CLOTHING}/image/image_{object_name}.png" image_url = f"{AIDA_CLOTHING}/image/image_{object_name}.png"
return front_image, image_url, mask_url return front_image, image_url, mask_url
except Exception as e: except Exception as e:
logging.warning(f"upload_png_mask runtime exception : {e}") logging.warning(f"upload_png_mask runtime exception : {e}")
# @RunTime
# def upload_png_mask(front_image, object_name, mask=None):
# mask_url = None
# if mask is not None:
# mask_url = f"{AIDA_CLOTHING}/mask/mask_{object_name}.png"
# image_url = f"{AIDA_CLOTHING}/image/image_{object_name}.png"
# return front_image, image_url, mask_url

91
app/service/utils/new_oss_client.py Normal file
View File

@@ -0,0 +1,91 @@
import io
import logging
from io import BytesIO
import cv2
import numpy as np
import urllib3
from PIL import Image
from minio import Minio
from app.core.config import *
minio_client = Minio(MINIO_URL, access_key=MINIO_ACCESS, secret_key=MINIO_SECRET, secure=MINIO_SECURE)
# 自定义 Retry 类
class CustomRetry(urllib3.Retry):
def increment(self, method=None, url=None, response=None, error=None, **kwargs):
# 调用父类的 increment 方法
new_retry = super(CustomRetry, self).increment(method, url, response, error, **kwargs)
# 打印重试信息
logger.info(f"重试连接: {method} {url}，错误: {error}，重试次数: {self.total - new_retry.total}")
return new_retry
logger = logging.getLogger()
timeout = urllib3.Timeout(connect=1, read=10.0) # 连接超时 5 秒，读取超时 10 秒
http_client = urllib3.PoolManager(
num_pools=10, # 设置连接池大小
maxsize=10,
timeout=timeout,
cert_reqs='CERT_REQUIRED', # 需要证书验证
retries=CustomRetry(
total=5,
backoff_factor=0.2,
status_forcelist=[500, 502, 503, 504],
),
)
# 获取图片
def oss_get_image(oss_client, bucket, object_name, data_type):
# cv2 默认全通道读取
image_object = None
try:
image_data = oss_client.get_object(bucket_name=bucket, object_name=object_name)
if data_type == "cv2":
image_bytes = image_data.read()
image_array = np.frombuffer(image_bytes, np.uint8) # 转成8位无符号整型
image_object = cv2.imdecode(image_array, cv2.IMREAD_UNCHANGED)
if image_object.dtype == np.uint16:
image_object = (image_object / 256).astype('uint8')
else:
data_bytes = BytesIO(image_data.read())
image_object = Image.open(data_bytes)
except Exception as e:
logger.warning(f"{OSS} | 获取图片出现异常 ######: {e}")
return image_object
def oss_upload_image(oss_client, bucket, object_name, image_bytes):
req = None
try:
req = oss_client.put_object(bucket_name=bucket, object_name=object_name, data=io.BytesIO(image_bytes), length=len(image_bytes), content_type='image/png')
except Exception as e:
logger.warning(f"{OSS} | 上传图片出现异常 ######: {e}")
return req
if __name__ == '__main__':
# url = "aida-results/result_0002186a-e631-11ee-86a6-b48351119060.png"
# url = "aida-collection-element/11523/Moodboard/f60af0d2-94c2-48f9-90ff-74b8e8a481b5.jpg"
# url = "aida-sys-image/images/female/outwear/0628000054.jpg"
# url = "aida-users/89/product_image/string-89.png"
# url = "test/845046c7-4f62-4f54-a4a9-c26d49c6969335b5b3a9-d335-4871-a46c-3cc3caf07da259629dfd1f1f555a2e2a9def7e719366.png"
# url = 'aida-users/89/relight_image/123-89.png'
# url = 'aida-users/89/relight_image/123-89.png'
# url = 'aida-users/89/relight_image/123-89.png'
# url = "aida-users/89/sketchboard/female/Dress/e6724ab7-8d3f-4677-abe0-c3e42ab7af85.jpeg"
# url = "aida-users/87/print/956614a2-7e75-4fbe-9ed0-c1831e37a2c9-4-87.png"
# url = "aida-users/89/single_logo/123-89.png"
url = "aida-users/31/sketchboard/female/dress/6edcbf92-7da9-4809-a0a8-a4b4f06dec1e0628000041.jpg"
# url = "aida-collection-element/12148/Sketchboard/95ea577b-305b-4a62-b30a-39c0dd3ddb3f.png"
read_type = "cv2"
if read_type == "cv2":
img = oss_get_image(oss_client=minio_client, bucket=url.split('/')[0], object_name=url[url.find('/') + 1:], data_type=read_type)
cv2.imshow("", img)
cv2.waitKey(0)
else:
img = oss_get_image(oss_client=minio_client, bucket=url.split('/')[0], object_name=url[url.find('/') + 1:], data_type=read_type)
img.show()

27
app/service/utils/oss_client.py
View File

@@ -2,6 +2,7 @@ import io
import logging import logging
from io import BytesIO from io import BytesIO
import boto3
import cv2 import cv2
import numpy as np import numpy as np
import urllib3 import urllib3
@@ -10,8 +11,6 @@ from minio import Minio
from app.core.config import * from app.core.config import *
minio_client = Minio(MINIO_URL, access_key=MINIO_ACCESS, secret_key=MINIO_SECRET, secure=MINIO_SECURE)
# 自定义 Retry 类 # 自定义 Retry 类
class CustomRetry(urllib3.Retry): class CustomRetry(urllib3.Retry):
@@ -39,11 +38,16 @@ http_client = urllib3.PoolManager(
# 获取图片 # 获取图片
def oss_get_image(oss_client, bucket, object_name, data_type): def oss_get_image(bucket, object_name, data_type):
# cv2 默认全通道读取 # cv2 默认全通道读取
image_object = None image_object = None
try: try:
image_data = oss_client.get_object(bucket_name=bucket, object_name=object_name) if OSS == "minio":
oss_client = Minio(MINIO_URL, access_key=MINIO_ACCESS, secret_key=MINIO_SECRET, secure=MINIO_SECURE, http_client=http_client)
image_data = oss_client.get_object(bucket_name=bucket, object_name=object_name)
else:
oss_client = boto3.client('s3', aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_AWS_SECRET_ACCESS_KEY, region_name=S3_REGION_NAME)
image_data = oss_client.get_object(Bucket=bucket, Key=object_name)['Body']
if data_type == "cv2": if data_type == "cv2":
image_bytes = image_data.read() image_bytes = image_data.read()
image_array = np.frombuffer(image_bytes, np.uint8) # 转成8位无符号整型 image_array = np.frombuffer(image_bytes, np.uint8) # 转成8位无符号整型
@@ -58,10 +62,15 @@ def oss_get_image(oss_client, bucket, object_name, data_type):
return image_object return image_object
def oss_upload_image(oss_client, bucket, object_name, image_bytes): def oss_upload_image(bucket, object_name, image_bytes):
req = None req = None
try: try:
req = oss_client.put_object(bucket_name=bucket, object_name=object_name, data=io.BytesIO(image_bytes), length=len(image_bytes), content_type='image/png') if OSS == "minio":
oss_client = Minio(MINIO_URL, access_key=MINIO_ACCESS, secret_key=MINIO_SECRET, secure=MINIO_SECURE)
req = oss_client.put_object(bucket_name=bucket, object_name=object_name, data=io.BytesIO(image_bytes), length=len(image_bytes), content_type='image/png')
else:
oss_client = boto3.client('s3', aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_AWS_SECRET_ACCESS_KEY, region_name=S3_REGION_NAME)
req = oss_client.put_object(Bucket=bucket, Key=object_name, Body=io.BytesIO(image_bytes), ContentType='image/png')
except Exception as e: except Exception as e:
logger.warning(f"{OSS} | 上传图片出现异常 ######: {e}") logger.warning(f"{OSS} | 上传图片出现异常 ######: {e}")
return req return req
@@ -79,13 +88,13 @@ if __name__ == '__main__':
# url = "aida-users/89/sketchboard/female/Dress/e6724ab7-8d3f-4677-abe0-c3e42ab7af85.jpeg" # url = "aida-users/89/sketchboard/female/Dress/e6724ab7-8d3f-4677-abe0-c3e42ab7af85.jpeg"
# url = "aida-users/87/print/956614a2-7e75-4fbe-9ed0-c1831e37a2c9-4-87.png" # url = "aida-users/87/print/956614a2-7e75-4fbe-9ed0-c1831e37a2c9-4-87.png"
# url = "aida-users/89/single_logo/123-89.png" # url = "aida-users/89/single_logo/123-89.png"
url = "aida-users/31/sketchboard/female/dress/6edcbf92-7da9-4809-a0a8-a4b4f06dec1e0628000041.jpg" url = "aida-clothing/mask/mask_f354afb5-6423-11ef-8b08-0826ae3ad6b3.png"
# url = "aida-collection-element/12148/Sketchboard/95ea577b-305b-4a62-b30a-39c0dd3ddb3f.png" # url = "aida-collection-element/12148/Sketchboard/95ea577b-305b-4a62-b30a-39c0dd3ddb3f.png"
read_type = "cv2" read_type = "cv2"
if read_type == "cv2": if read_type == "cv2":
img = oss_get_image(oss_client=minio_client, bucket=url.split('/')[0], object_name=url[url.find('/') + 1:], data_type=read_type) img = oss_get_image(bucket=url.split('/')[0], object_name=url[url.find('/') + 1:], data_type=read_type)
cv2.imshow("", img) cv2.imshow("", img)
cv2.waitKey(0) cv2.waitKey(0)
else: else:
img = oss_get_image(oss_client=minio_client, bucket=url.split('/')[0], object_name=url[url.find('/') + 1:], data_type=read_type) img = oss_get_image(bucket=url.split('/')[0], object_name=url[url.find('/') + 1:], data_type=read_type)
img.show() img.show()

BIN
requirements.txt
View File

Binary file not shown.