From 7ff1603583d653cde043fc49425b46546d39a596 Mon Sep 17 00:00:00 2001 From: zhouchengrong Date: Mon, 2 Dec 2024 23:38:06 +0800 Subject: [PATCH] =?UTF-8?q?feat=20=20=E4=BF=AE=E5=A4=8Dchatroboot=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/api_route.py | 3 +- app/core/config.py | 2 +- app/service/chat_robot/script/main.py | 2 +- app/service/chat_robot/script/prompt.py | 49 +++++++++++----- .../chat_robot/script/service/CallQWen.py | 57 ++++++++++++++++--- 5 files changed, 87 insertions(+), 26 deletions(-) diff --git a/app/api/api_route.py b/app/api/api_route.py index 7ee774d..0da3a66 100644 --- a/app/api/api_route.py +++ b/app/api/api_route.py @@ -1,6 +1,6 @@ from fastapi import APIRouter -from app.api import api_attribute_retrieve +from app.api import api_attribute_retrieve, api_query_image from app.api import api_brighten from app.api import api_chat_robot from app.api import api_design @@ -23,3 +23,4 @@ router.include_router(api_prompt_generation.router, tags=['prompt_generation'], router.include_router(api_design_pre_processing.router, tags=['design_pre_processing'], prefix="/api") router.include_router(api_image2sketch.router, tags=['api_image2sketch'], prefix="/api") router.include_router(api_brighten.router, tags=['api_brighten'], prefix="/api") +router.include_router(api_query_image.router, tags=['api_query_image'], prefix="/api") \ No newline at end of file diff --git a/app/core/config.py b/app/core/config.py index d369ff2..7629429 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -138,7 +138,7 @@ SEGMENTATION = { "output": "seg_output__0", } # ollama config -OLLAMA_URL = "http://10.1.1.243:11434/api/embeddings" +OLLAMA_URL = "http://10.1.1.240:11434/api/embeddings" # DESIGN config DESIGN_MODEL_URL = '10.1.1.240:10000' AIDA_CLOTHING = "aida-clothing" diff --git a/app/service/chat_robot/script/main.py b/app/service/chat_robot/script/main.py index cabe372..3342a5c 100644 --- a/app/service/chat_robot/script/main.py +++ b/app/service/chat_robot/script/main.py @@ -100,7 +100,7 @@ def chat(post_data): # session_key=f"buffer:{user_id}:{session_id}", # ) - final_outputs = CallQWen.call_with_messages(input_message) + final_outputs = CallQWen.call_with_messages(input_message, gender) # api_response = { # 'user_id': user_id, # 'session_id': session_id, diff --git a/app/service/chat_robot/script/prompt.py b/app/service/chat_robot/script/prompt.py index a88044d..ad6ac9e 100644 --- a/app/service/chat_robot/script/prompt.py +++ b/app/service/chat_robot/script/prompt.py @@ -1,16 +1,31 @@ +# FASHION_CHAT_BOT_PREFIX = """ +# You are a helpful assistant for fashion designers. You can chat with the users or answer their query as much as you can. +# The most crucial aspect is to accurately determine whether the user's inquiry requires a internet search or querying the database. +# Remember your answer should be very precise and the final output answer should not exceed 20 words. +# +# You may encounter the following types of questions: +# 1) If the query related to clothing retrieval, you are an agent designed to interact with a SQL database. +# Given an input question, create a syntactically correct mysql query to run, always fetching random data from tables. +# Unless the user specifies a specific number of examples they wish to obtain,always limit your query to at most 4 results. +# Never query for all the columns from a specific table, only ask for the relevant columns given the question. +# You MUST double check your query before executing it. If you get an error while executing a query, rewrite the query and try again. +# DO NOT make any DML statements (INSERT, UPDATE, DELETE, DROP etc.) to the database. +# If the question does not seem related to the database, just return "I don't know" as the answer. +# +# 2) If the query related to current events, you should use internet_search to seek help from the internet. +# +# 3) If the query is just casual conversation, engage in the conversation as a fashion designer assistant. +# +# Be careful to use the tools, since you are actually a chat bot. Tools can only be used when essential. +# """ + FASHION_CHAT_BOT_PREFIX = """ You are a helpful assistant for fashion designers. You can chat with the users or answer their query as much as you can. The most crucial aspect is to accurately determine whether the user's inquiry requires a internet search or querying the database. Remember your answer should be very precise and the final output answer should not exceed 20 words. You may encounter the following types of questions: -1) If the query related to clothing retrieval, you are an agent designed to interact with a SQL database. -Given an input question, create a syntactically correct mysql query to run, always fetching random data from tables. -Unless the user specifies a specific number of examples they wish to obtain,always limit your query to at most 4 results. -Never query for all the columns from a specific table, only ask for the relevant columns given the question. -You MUST double check your query before executing it. If you get an error while executing a query, rewrite the query and try again. -DO NOT make any DML statements (INSERT, UPDATE, DELETE, DROP etc.) to the database. -If the question does not seem related to the database, just return "I don't know" as the answer. +1) If you need to query information related to clothing retrieval, please use the get_image_from_vector_db tool. 2) If the query related to current events, you should use internet_search to seek help from the internet. @@ -37,15 +52,19 @@ ANSWER_FORMAT_SUFFIX = """ My final answer are limited to 20 words and be as much precise as possible. """ +# TOOLS_FUNCTIONS_SUFFIX = ( +# "If the input involves clothing queries," +# "I should look at the tables in the database to see what I can query. Then I should query the schema of the most relevant tables." +# "All SQL statements must use 'ORDER BY RAND()', for example:" +# "Example Input 1: 'SELECT img_name FROM skirt WHERE opening_type = 'Button' ORDER BY RAND() LIMIT 1'" +# "Example Input 2: 'SELECT img_name FROM top WHERE sleeve_length = 'Long' AND type = 'Blouse' ORDER BY RAND() LIMIT 2'" +# "If the input does not involve clothing queries, " +# "I should engage in conversation as an assistant or search from internet with internet_search tool." +# "If the database query returns no results, please respond directly with: 'Apologies, I couldn't find any images that match your description. Could you please give me more details about the clothing you're searching for?'" +# "Upon mentioning words related to 'tutorial' in the input, I should use tutorial_tool " +# ) TOOLS_FUNCTIONS_SUFFIX = ( - "If the input involves clothing queries," - "I should look at the tables in the database to see what I can query. Then I should query the schema of the most relevant tables." - "All SQL statements must use 'ORDER BY RAND()', for example:" - "Example Input 1: 'SELECT img_name FROM skirt WHERE opening_type = 'Button' ORDER BY RAND() LIMIT 1'" - "Example Input 2: 'SELECT img_name FROM top WHERE sleeve_length = 'Long' AND type = 'Blouse' ORDER BY RAND() LIMIT 2'" - "If the input does not involve clothing queries, " - "I should engage in conversation as an assistant or search from internet with internet_search tool." - "If the database query returns no results, please respond directly with: 'Apologies, I couldn't find any images that match your description. Could you please give me more details about the clothing you're searching for?'" + "If the input involves clothing queries,please use the get_image_from_vector_db tool." "Upon mentioning words related to 'tutorial' in the input, I should use tutorial_tool " ) diff --git a/app/service/chat_robot/script/service/CallQWen.py b/app/service/chat_robot/script/service/CallQWen.py index d2e2c06..33dcd04 100644 --- a/app/service/chat_robot/script/service/CallQWen.py +++ b/app/service/chat_robot/script/service/CallQWen.py @@ -8,6 +8,7 @@ from app.core.config import * from app.service.chat_robot.script.callbacks.qwen_callback_handler import QWenCallbackHandler from app.service.chat_robot.script.database import CustomDatabase from app.service.chat_robot.script.prompt import FASHION_CHAT_BOT_PREFIX, TOOLS_FUNCTIONS_SUFFIX, TUTORIAL_TOOL_RETURN +from app.service.search_image_with_text.service import query get_database_table_description = "Input is an empty string, output is a comma separated list of tables in the database." @@ -32,6 +33,12 @@ query_database_description = ( "order by rand() LIMIT 2'" ) +query_vector_db_description = ( + "Use this tool to find the clothing images that users need. " + "If the user's input includes clothing types such as blouse, skirt, dress, outerwear, pants, or trousers, please use this tool. " + "The input for the tool is the string provided by the user." +) + tutorial_description = ("Utilize this tool to retrieve specific statements related to user guidance tutorials." "Input is an empty string") @@ -105,15 +112,37 @@ tools = [ "function": { "name": "tutorial_tool", "description": tutorial_description, + # "parameters": { + # "type": "object", + # "properties": { + # "sql_string": { + # "type": "string", + # "description": "由模型生成的sql语句" + # } + # } + # }, + } + }, + { + "type": "function", + "function": { + "name": "get_image_from_vector_db", + "description": query_vector_db_description, "parameters": { - "type": "object", - "properties": { - "sql_string": { - "type": "string", - "description": "由模型生成的sql语句" + "parameters": { + "type": "object", + "properties": { + "gender": { + "type": "string", + "description": "性别" + }, + "content": { + "type": "string", + "description": "用户描述" + } } - } - }, + }, + } } } ] @@ -150,6 +179,10 @@ def query_database(sql_string): return CustomDatabase.run(db, sql_string) +def get_image_from_vector_db(gender, content): + return query(gender, content) + + @retry(exceptions=NewConnectionError, tries=3, delay=1) def get_response(messages): response = Generation.call( @@ -164,7 +197,8 @@ def get_response(messages): return response -def call_with_messages(message): +def call_with_messages(message, gender): + user_input = message print('\n') # messages = [ # { @@ -235,6 +269,12 @@ def call_with_messages(message): tool_info = {"name": "tutorial_tool", "role": "tool", 'content': tutorial_tool()} flag = False result_content = tool_info['content'] + elif assistant_output.tool_calls[0]['function']['name'] == 'get_image_from_vector_db': + tool_info = {"name": "get_image_from_vector_db", "role": "tool", + 'content': get_image_from_vector_db(gender, user_input)} + flag = False + result_content = tool_info['content'] + response_type = "image" print(f"工具输出信息：{tool_info['content']}\n") messages.append(tool_info) @@ -257,5 +297,6 @@ def call_with_messages(message): def tutorial_tool(): return TUTORIAL_TOOL_RETURN + if __name__ == '__main__': call_with_messages()