新增图片上下文存储
This commit is contained in:
@@ -10,18 +10,15 @@ from typing import AsyncGenerator
|
|||||||
from fastapi.responses import StreamingResponse
|
from fastapi.responses import StreamingResponse
|
||||||
from langchain_core.messages import SystemMessage, AIMessageChunk, ToolMessage, AIMessage, ToolMessageChunk
|
from langchain_core.messages import SystemMessage, AIMessageChunk, ToolMessage, AIMessage, ToolMessageChunk
|
||||||
|
|
||||||
from src.core.config import PROJECT_ROOT, settings, MONGO_URI
|
from src.core.config import PROJECT_ROOT, settings
|
||||||
from src.server.deep_agent.agents.main_agent import build_main_agent
|
from src.server.deep_agent.agents.main_agent import build_main_agent
|
||||||
from src.server.deep_agent.tools.conversation_title_tool import conversation_title
|
from src.server.deep_agent.tools.conversation_title_tool import conversation_title
|
||||||
from src.server.deep_agent.tools.generate_furniture_sketch import is_image_path_exist
|
|
||||||
from src.schemas.deep_agent_chat import DeepAgentChatRequest, HistoryResponse, HistoryItem
|
from src.schemas.deep_agent_chat import DeepAgentChatRequest, HistoryResponse, HistoryItem
|
||||||
from src.server.deep_agent.tools.extract_suggested_questions import generate_suggested_questions
|
from src.server.deep_agent.tools.extract_suggested_questions import generate_suggested_questions
|
||||||
from src.server.deep_agent.utils.mongodb_util import ThreadImageMinIOStore
|
from src.server.utils.new_oss_client import get_presigned_url
|
||||||
from src.server.utils.new_oss_client import is_minio_file_exist, oss_upload_image_file, oss_get_image, get_presigned_url
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/chat", tags=["Furniture Design Chat"])
|
router = APIRouter(prefix="/chat", tags=["Furniture Design Chat"])
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
image_store = ThreadImageMinIOStore(MONGO_URI, "agent_tool_generate_db")
|
|
||||||
minio_client = Minio(settings.MINIO_URL, access_key=settings.MINIO_ACCESS, secret_key=settings.MINIO_SECRET, secure=settings.MINIO_SECURE)
|
minio_client = Minio(settings.MINIO_URL, access_key=settings.MINIO_ACCESS, secret_key=settings.MINIO_SECRET, secure=settings.MINIO_SECURE)
|
||||||
|
|
||||||
|
|
||||||
@@ -108,6 +105,7 @@ async def chat_stream(request: DeepAgentChatRequest):
|
|||||||
workspace_dir = os.path.join(PROJECT_ROOT, f"agent_workspace/{target_thread_id}")
|
workspace_dir = os.path.join(PROJECT_ROOT, f"agent_workspace/{target_thread_id}")
|
||||||
logger.info(f"chat request data: {request} | target_thread_id : workspace_dir: {workspace_dir}")
|
logger.info(f"chat request data: {request} | target_thread_id : workspace_dir: {workspace_dir}")
|
||||||
main_agent = build_main_agent(request.use_report, workspace_dir, request.enable_thinking)
|
main_agent = build_main_agent(request.use_report, workspace_dir, request.enable_thinking)
|
||||||
|
|
||||||
# 2. 配置參數
|
# 2. 配置參數
|
||||||
temp = request.config_params.temperature if request.config_params else 0.7
|
temp = request.config_params.temperature if request.config_params else 0.7
|
||||||
|
|
||||||
@@ -142,11 +140,14 @@ async def chat_stream(request: DeepAgentChatRequest):
|
|||||||
"checkpoint_id": checkpoint_id
|
"checkpoint_id": checkpoint_id
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
last_checkpoint_id = await get_branch_checkpoint_id(main_agent, source_config)
|
||||||
older_state = await main_agent.aget_state(source_config)
|
older_state = await main_agent.aget_state(source_config)
|
||||||
combined_values = older_state.values.copy()
|
combined_values = older_state.values.copy()
|
||||||
if initial_messages:
|
if initial_messages:
|
||||||
combined_values["messages"] = list(combined_values.get("messages", [])) + initial_messages
|
combined_values["messages"] = list(combined_values.get("messages", [])) + initial_messages
|
||||||
await main_agent.aupdate_state(current_config, combined_values)
|
await main_agent.aupdate_state(current_config, combined_values)
|
||||||
|
else:
|
||||||
|
last_checkpoint_id = await get_checkpoint_id(main_agent, current_config)
|
||||||
|
|
||||||
async def event_generator() -> AsyncGenerator[str, None]:
|
async def event_generator() -> AsyncGenerator[str, None]:
|
||||||
is_first = True
|
is_first = True
|
||||||
@@ -171,15 +172,6 @@ async def chat_stream(request: DeepAgentChatRequest):
|
|||||||
content.append({"type": "image_url", "image_url": {"url": image_url}})
|
content.append({"type": "image_url", "image_url": {"url": image_url}})
|
||||||
files["quote_image"] = request.quote_image_path
|
files["quote_image"] = request.quote_image_path
|
||||||
|
|
||||||
# 用户最近生成图片
|
|
||||||
if image_store.get_image_path(target_thread_id):
|
|
||||||
current_image_path = image_store.get_image_path(target_thread_id).get("current_image_path", False)
|
|
||||||
if current_image_path:
|
|
||||||
bucket, object_name = current_image_path.split('/', 1)
|
|
||||||
image_url = get_presigned_url(oss_client=minio_client, bucket=bucket, object_name=object_name)
|
|
||||||
if image_url is not None:
|
|
||||||
content.append({"type": "image_url", "image_url": {"url": image_url}})
|
|
||||||
|
|
||||||
final_messages = {
|
final_messages = {
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
@@ -197,6 +189,17 @@ async def chat_stream(request: DeepAgentChatRequest):
|
|||||||
):
|
):
|
||||||
if is_first:
|
if is_first:
|
||||||
checkpoint_id = main_agent.get_state(current_config).config.get("configurable").get("checkpoint_id")
|
checkpoint_id = main_agent.get_state(current_config).config.get("configurable").get("checkpoint_id")
|
||||||
|
if not checkpoint_id:
|
||||||
|
print("123")
|
||||||
|
main_agent.store.put(
|
||||||
|
("image_history",),
|
||||||
|
"checkpoint_id",
|
||||||
|
{
|
||||||
|
"current_checkpoint_id": checkpoint_id,
|
||||||
|
"last_checkpoint_id": last_checkpoint_id,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
logger.info(f"*******************{checkpoint_id}**********************************")
|
||||||
yield f"data: {json.dumps({'thread_id': target_thread_id, 'is_branch': is_branching, 'status': 'start', "checkpoint_id": checkpoint_id}, ensure_ascii=False)}\n\n"
|
yield f"data: {json.dumps({'thread_id': target_thread_id, 'is_branch': is_branching, 'status': 'start', "checkpoint_id": checkpoint_id}, ensure_ascii=False)}\n\n"
|
||||||
is_first = False
|
is_first = False
|
||||||
_, mode, chunks = stream
|
_, mode, chunks = stream
|
||||||
@@ -257,7 +260,7 @@ async def chat_stream(request: DeepAgentChatRequest):
|
|||||||
# "tool_call_chunk": token.tool_call_chunks[0] if token.tool_call_chunks else None
|
# "tool_call_chunk": token.tool_call_chunks[0] if token.tool_call_chunks else None
|
||||||
})
|
})
|
||||||
else:
|
else:
|
||||||
print(f"[reasoning] {reasoning}*************************************************************************************")
|
logger.info(f"[reasoning] {reasoning}*************************************************************************************")
|
||||||
elif text:
|
elif text:
|
||||||
if len(text) == 1:
|
if len(text) == 1:
|
||||||
payload_out.update({
|
payload_out.update({
|
||||||
@@ -267,7 +270,7 @@ async def chat_stream(request: DeepAgentChatRequest):
|
|||||||
# "tool_call_chunk": token.tool_call_chunks[0] if token.tool_call_chunks else None
|
# "tool_call_chunk": token.tool_call_chunks[0] if token.tool_call_chunks else None
|
||||||
})
|
})
|
||||||
else:
|
else:
|
||||||
print(f"[text] {text}*************************************************************************************")
|
logger.info(f"[text] {text}*************************************************************************************")
|
||||||
else:
|
else:
|
||||||
payload_out.update({
|
payload_out.update({
|
||||||
"type": "tool_call",
|
"type": "tool_call",
|
||||||
@@ -395,7 +398,29 @@ async def get_chat_history(thread_id: str):
|
|||||||
))
|
))
|
||||||
|
|
||||||
return HistoryResponse(thread_id=thread_id, history=history_data)
|
return HistoryResponse(thread_id=thread_id, history=history_data)
|
||||||
# try:
|
|
||||||
|
|
||||||
# except Exception as e:
|
|
||||||
# raise HTTPException(status_code=404, detail=f"History not found: {str(e)}")
|
async def get_checkpoint_id(main_agent, current_config):
|
||||||
|
# 🔥 最优:边遍历边找,找到第一个就返回,不浪费内存
|
||||||
|
async for item in main_agent.aget_state_history(config=current_config):
|
||||||
|
if item.next == ("__start__",):
|
||||||
|
# 找到直接处理并返回
|
||||||
|
# if item.parent_config:
|
||||||
|
# return item.parent_config.get('configurable', {}).get('checkpoint_id')
|
||||||
|
return item.config.get('configurable', {}).get('checkpoint_id')
|
||||||
|
# 没找到
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def get_branch_checkpoint_id(main_agent, current_config):
|
||||||
|
# 🔥 最优:边遍历边找,找到第一个就返回,不浪费内存
|
||||||
|
async for item in main_agent.aget_state_history(config=current_config):
|
||||||
|
current_id = current_config.get('configurable', {}).get('checkpoint_id')
|
||||||
|
if item.next == ("__start__",) and item.config.get('configurable', {}).get('checkpoint_id') != current_id:
|
||||||
|
if item.parent_config:
|
||||||
|
if item.parent_config.get('configurable', {}).get('checkpoint_id') != current_id:
|
||||||
|
return item.config.get('configurable', {}).get('checkpoint_id')
|
||||||
|
else:
|
||||||
|
return item.config.get('configurable', {}).get('checkpoint_id')
|
||||||
|
# 没找到
|
||||||
|
return None
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ from deepagents.backends import FilesystemBackend
|
|||||||
from langchain.agents.middleware import SummarizationMiddleware
|
from langchain.agents.middleware import SummarizationMiddleware
|
||||||
from langgraph.checkpoint.mongodb import MongoDBSaver
|
from langgraph.checkpoint.mongodb import MongoDBSaver
|
||||||
from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer
|
from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer
|
||||||
|
from langgraph.store.memory import InMemoryStore
|
||||||
from pymongo import MongoClient
|
from pymongo import MongoClient
|
||||||
|
|
||||||
from src.core.config import MONGO_URI
|
from src.core.config import MONGO_URI
|
||||||
@@ -50,6 +51,7 @@ def build_main_agent(use_report, workspace_dir, enable_thinking):
|
|||||||
main_agent = create_deep_agent(
|
main_agent = create_deep_agent(
|
||||||
model=build_main_llm(enable_thinking=enable_thinking),
|
model=build_main_llm(enable_thinking=enable_thinking),
|
||||||
system_prompt=build_system_prompt(use_report=use_report),
|
system_prompt=build_system_prompt(use_report=use_report),
|
||||||
|
store=InMemoryStore(),
|
||||||
subagents=subagents,
|
subagents=subagents,
|
||||||
checkpointer=checkpointer,
|
checkpointer=checkpointer,
|
||||||
backend=FilesystemBackend(
|
backend=FilesystemBackend(
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
def build_system_prompt(use_report):
|
def build_system_prompt(use_report):
|
||||||
system_prompt = f"""
|
system_prompt = f"""
|
||||||
你是主调度 Agent(Supervisor),负责理解用户意图并选择合适的子Agent。
|
你是主调度 Agent(Supervisor),负责理解用户意图并选择合适的子Agent。
|
||||||
当前参数: use_report = {use_report}
|
当前参数:
|
||||||
|
use_report = {use_report}
|
||||||
系统中存在两个相关子Agent:
|
系统中存在两个相关子Agent:
|
||||||
1. user_profile_subagent
|
1. user_profile_subagent
|
||||||
负责收集和维护用户画像信息,包括但不限于:
|
负责收集和维护用户画像信息,包括但不限于:
|
||||||
@@ -16,30 +16,27 @@ def build_system_prompt(use_report):
|
|||||||
|
|
||||||
3. painter_subagent
|
3. painter_subagent
|
||||||
负责根据用户描述,构造适用于 生成家具sketch的prompt或编辑家具sketch的prompt
|
负责根据用户描述,构造适用于 生成家具sketch的prompt或编辑家具sketch的prompt
|
||||||
1.用prompt用工具生成图片.
|
1.利用prompt用工具生成图片.
|
||||||
2.用prompt和图片url用工具编辑图片.
|
2.利用prompt和图片路径用工具编辑图片.
|
||||||
|
|
||||||
========================
|
========================
|
||||||
执行规则
|
执行规则
|
||||||
========================
|
========================
|
||||||
【1】当 use_report = False 时:
|
【1】当用户请求报告 / 调研 / 分析 / 总结时:
|
||||||
- 严禁调用 research-subagent
|
|
||||||
- 如果用户明确请求报告、调研、总结、分析:
|
|
||||||
|
|
||||||
请礼貌回复:
|
|
||||||
"报告功能当前未开启,你可以打开 Trending report 后我来帮你生成报告。"
|
|
||||||
- 其他普通问题可以正常回答或调用其他子Agent
|
|
||||||
|
|
||||||
------------------------
|
|
||||||
|
|
||||||
【2】当用户请求报告 / 调研 / 分析 / 总结时:
|
|
||||||
先判断是否已经具备足够的用户画像信息。
|
先判断是否已经具备足够的用户画像信息。
|
||||||
如果用户需求信息不足(例如缺少风格、房间类型、预算、主题、范围等):
|
如果用户需求信息不足(例如缺少风格、房间类型、预算、主题、范围等):
|
||||||
→ 调用 user_profile_subagent 收集信息
|
→ 调用 user_profile_subagent 收集信息
|
||||||
不要直接生成报告。
|
不要直接生成报告。
|
||||||
如果用户画像信息已经完整:
|
如果用户画像信息已经完整:
|
||||||
→ 调用 research-subagent 生成报告。
|
→ 调用 research-subagent 生成报告。
|
||||||
。
|
------------------------
|
||||||
|
【2】当 use_report = False 时:
|
||||||
|
- 严禁调用 research-subagent
|
||||||
|
- 如果用户明确请求报告、调研、总结、分析:
|
||||||
|
|
||||||
|
请礼貌回复:
|
||||||
|
"报告功能当前未开启,你可以打开 use_report=True 后我来帮你生成报告。"
|
||||||
|
- 其他普通问题可以正常回答或调用其他子Agent。
|
||||||
------------------------
|
------------------------
|
||||||
【3】用户画像优先级规则
|
【3】用户画像优先级规则
|
||||||
只要用户输入包含以下情况:
|
只要用户输入包含以下情况:
|
||||||
@@ -55,42 +52,13 @@ def build_system_prompt(use_report):
|
|||||||
- user_profile_subagent 只负责 **信息收集**
|
- user_profile_subagent 只负责 **信息收集**
|
||||||
- research-subagent 只负责 **报告生成**
|
- research-subagent 只负责 **报告生成**
|
||||||
不要混用职责。
|
不要混用职责。
|
||||||
|
|
||||||
========================
|
========================
|
||||||
!禁止输出:
|
|
||||||
- 路径
|
|
||||||
- 图片url地址
|
|
||||||
- 工具参数
|
|
||||||
- 解释过程
|
|
||||||
========================
|
|
||||||
!输出规则:
|
|
||||||
- 当 painter_subagent 返回图片地址(image_url)时:
|
|
||||||
* 不要直接输出原始 URL 给用户。
|
|
||||||
* 请用 Markdown 格式回复,例如:
|
|
||||||
"已为你生成/编辑家具 sketch:"
|
|
||||||
<image-card alt="家具设计" src="image_url" ></image-card>
|
|
||||||
* 或者仅回复:"图片已生成,请查看。"(如果你想在前端单独显示图片)
|
|
||||||
- **禁止** 把工具返回的原始 image_url 直接暴露给用户。
|
|
||||||
- 你的输出必须**简短**。
|
|
||||||
========================
|
|
||||||
|
|
||||||
图片生成工具只会返回 MinIO 内部路径(如 "bucket/folder/xxx.png")。
|
|
||||||
重要规则:
|
|
||||||
- 当工具返回包含 "MinIO path:" 或类似 "test/a/v/xxx.png" 的内容时,你必须理解这是一张图片。
|
|
||||||
- 这张图片**不能直接用于多模态输入**,也不能直接发给用户查看。
|
|
||||||
- 任何时候如果你(或用户)需要“看”这张图片、描述这张图片、或把图片作为 vision 模型的输入,你**必须先调用 get_presigned_image_url 工具**,传入该 MinIO path,获取 presigned http URL。
|
|
||||||
- 之后用这个 presigned URL 进行多模态调用(例如把 URL 传给支持 image_url 的模型)。
|
|
||||||
- 在对话历史中,优先记住 MinIO path,并在需要时主动转换。
|
|
||||||
- 用户说“描述一下这张图”时,你应该先获取 presigned URL,再调用 vision 工具/模型。
|
|
||||||
|
|
||||||
永远不要假设 MinIO path 是可直接访问的 http 地址。
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
return system_prompt
|
return system_prompt
|
||||||
|
|
||||||
|
|
||||||
def build_painter_prompt():
|
def build_painter_prompt():
|
||||||
"""
|
prompt = """
|
||||||
你是 painter_subagent,专门负责「生成」或「编辑」 sketch 图像的工具调度助手。
|
你是 painter_subagent,专门负责「生成」或「编辑」 sketch 图像的工具调度助手。
|
||||||
你的唯一任务是:根据用户意图,严格选择正确的工具(generate_furniture 或 edit_furniture),并构造对应参数。
|
你的唯一任务是:根据用户意图,严格选择正确的工具(generate_furniture 或 edit_furniture),并构造对应参数。
|
||||||
--------------------------------
|
--------------------------------
|
||||||
@@ -128,8 +96,7 @@ def build_painter_prompt():
|
|||||||
调用 edit_furniture 时:
|
调用 edit_furniture 时:
|
||||||
- 只需要提供:
|
- 只需要提供:
|
||||||
{
|
{
|
||||||
"prompt": "<英文图像编辑描述>",
|
"prompt": "<英文图像编辑描述>"
|
||||||
"image_url": "<用户指定的图片url>"
|
|
||||||
}
|
}
|
||||||
- prompt 要求:
|
- prompt 要求:
|
||||||
- 清晰描述修改内容
|
- 清晰描述修改内容
|
||||||
@@ -144,33 +111,18 @@ def build_painter_prompt():
|
|||||||
- ❌ 忽略“修改类”意图
|
- ❌ 忽略“修改类”意图
|
||||||
- ❌ 因为信息少就拒绝调用工具
|
- ❌ 因为信息少就拒绝调用工具
|
||||||
--------------------------------
|
--------------------------------
|
||||||
|
【五、用户回复规则(必须遵守)】
|
||||||
|
你对用户的最终回复只能是以下格式之一:
|
||||||
|
- "图片已成功生成!"
|
||||||
|
- "已按你的要求完成修改,图片已更新!"
|
||||||
❗禁止输出:
|
❗禁止输出:
|
||||||
- 路径
|
- 路径
|
||||||
|
- URL
|
||||||
- 工具参数
|
- 工具参数
|
||||||
- 解释过程
|
- 解释过程
|
||||||
--------------------------------
|
--------------------------------
|
||||||
现在开始工作。
|
现在开始工作。
|
||||||
"""
|
"""
|
||||||
prompt = """
|
|
||||||
你是 painter_subagent,唯一任务是根据用户意图选择工具并构造参数。
|
|
||||||
|
|
||||||
【工具选择规则】
|
|
||||||
- 包含“修改 / 改成 / 调整 / 优化 / 换成 / 基于已有图片”等语义 → 必须用 edit_furniture。
|
|
||||||
- 明确“生成 / 创建 / 设计 / 画一个”等 → 用 generate_furniture。
|
|
||||||
- 不明确时默认视为编辑类,用 edit_furniture。
|
|
||||||
|
|
||||||
【图片规则】
|
|
||||||
- edit_furniture 自动使用上下文当前图片。
|
|
||||||
- 严禁自行编造 image_url。
|
|
||||||
|
|
||||||
【输出要求】
|
|
||||||
- 仅输出正确的工具调用。
|
|
||||||
- **禁止任何解释、过程、额外文字**。
|
|
||||||
- edit_furniture 只需提供 prompt(英文修改描述)和 image_url(如果用户指定)。
|
|
||||||
|
|
||||||
现在严格执行。
|
|
||||||
"""
|
|
||||||
|
|
||||||
return prompt
|
return prompt
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -2,48 +2,20 @@ import uuid
|
|||||||
import httpx
|
import httpx
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from langchain_core.messages import ToolMessage
|
|
||||||
from langchain_core.runnables import RunnableConfig
|
from langchain_core.runnables import RunnableConfig
|
||||||
from langchain_core.stores import BaseStore
|
|
||||||
from minio import Minio
|
from minio import Minio
|
||||||
from google import genai
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from langchain_core.tools import tool
|
from langchain_core.tools import tool
|
||||||
from google.oauth2 import service_account
|
|
||||||
from langgraph.prebuilt import ToolRuntime
|
from langgraph.prebuilt import ToolRuntime
|
||||||
from src.core.config import settings, MONGO_URI
|
from src.core.config import settings, MONGO_URI
|
||||||
from src.server.deep_agent.utils.mongodb_util import ThreadImageMinIOStore
|
from src.server.deep_agent.utils.mongodb_util import ThreadImageMinIOStore
|
||||||
from src.server.utils.new_oss_client import get_presigned_url, check_and_extract_minio_image
|
|
||||||
|
|
||||||
# from google.genai.types import GenerateContentConfig, Modality
|
|
||||||
# from langgraph.config import get_stream_writer
|
|
||||||
# from src.server.utils.new_oss_client import oss_upload_image, oss_get_image, is_minio_file_exist, oss_upload_image_file
|
|
||||||
|
|
||||||
# 初始化全局凭证和客户端
|
|
||||||
# creds = service_account.Credentials.from_service_account_file(
|
|
||||||
# settings.GOOGLE_GENAI_USE_VERTEXAI,
|
|
||||||
# scopes=["https://www.googleapis.com/auth/cloud-platform"],
|
|
||||||
# )
|
|
||||||
# client = genai.Client(
|
|
||||||
# credentials=creds,
|
|
||||||
# project=settings.GOOGLE_CLOUD_PROJECT,
|
|
||||||
# location=settings.GOOGLE_CLOUD_LOCATION,
|
|
||||||
# vertexai=True
|
|
||||||
# )
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
minio_client = Minio(settings.MINIO_URL, access_key=settings.MINIO_ACCESS, secret_key=settings.MINIO_SECRET, secure=settings.MINIO_SECURE)
|
minio_client = Minio(settings.MINIO_URL, access_key=settings.MINIO_ACCESS, secret_key=settings.MINIO_SECRET, secure=settings.MINIO_SECURE)
|
||||||
image_store = ThreadImageMinIOStore(MONGO_URI, "agent_tool_generate_db")
|
image_store = ThreadImageMinIOStore(MONGO_URI, "agent_tool_generate_db")
|
||||||
|
|
||||||
|
|
||||||
def is_image_path_exist(image_path):
|
|
||||||
try:
|
|
||||||
return Path(image_path).exists()
|
|
||||||
except:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
@tool
|
@tool
|
||||||
async def generate_furniture(prompt: str, runtime: ToolRuntime):
|
async def generate_furniture(prompt: str, runtime: ToolRuntime):
|
||||||
"""
|
"""
|
||||||
@@ -70,41 +42,16 @@ async def generate_furniture(prompt: str, runtime: ToolRuntime):
|
|||||||
- 生成的图片会自动携带到整个对话上下文中,支持后续使用 edit_furniture 等工具进行迭代修改。
|
- 生成的图片会自动携带到整个对话上下文中,支持后续使用 edit_furniture 等工具进行迭代修改。
|
||||||
- 如果需要生成多个方案,可以多次调用本工具或在 prompt 中明确要求生成不同变体。
|
- 如果需要生成多个方案,可以多次调用本工具或在 prompt 中明确要求生成不同变体。
|
||||||
"""
|
"""
|
||||||
logger.info(f"\n[系统日志] 正在调用 generate_furniture ...")
|
current_checkpoint_id = runtime.store.get(namespace=("image_history",), key="checkpoint_id", ).value.get("current_checkpoint_id")
|
||||||
# thread_id = runtime.config.get("configurable").get("thread_id")
|
|
||||||
|
logger.info(f"\n[系统日志] 正在调用 generate_furniture ...当前checkpoint_id={current_checkpoint_id}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 1. 生成图像 - local flux2-klein
|
image_url = await generate_or_edit_image(prompt=prompt)
|
||||||
object_name = f"furniture/sketches/{uuid.uuid4()}.png"
|
|
||||||
bucket_name = "fida-public-bucket" # 替换为你的 bucket 名称
|
|
||||||
request_data = {
|
|
||||||
"prompt": prompt,
|
|
||||||
"bucket_name": bucket_name,
|
|
||||||
"object_name": object_name,
|
|
||||||
"width": 1024,
|
|
||||||
"height": 1024
|
|
||||||
}
|
|
||||||
async with httpx.AsyncClient(timeout=120) as client:
|
|
||||||
resp = await client.post(
|
|
||||||
f"http://{settings.FLUX2_GEN_IMG_MODEL_URL}/predict",
|
|
||||||
json=request_data,
|
|
||||||
)
|
|
||||||
result = resp.json()
|
|
||||||
image_url = result.get("output_path", None)
|
|
||||||
# image_presigned_url = get_presigned_url(oss_client=minio_client, bucket=bucket_name, object_name=object_name)
|
|
||||||
if image_url:
|
if image_url:
|
||||||
# image_store.save_image_path(thread_id=thread_id, object_path=image_url, metadata={"prompt": prompt, "generated_at": str(datetime.now())})
|
image_store.save_image_path(thread_id=current_checkpoint_id, object_path=image_url, metadata={"prompt": prompt, "generated_at": str(datetime.now())})
|
||||||
# image_id = str(uuid.uuid4())
|
return image_url
|
||||||
# runtime.store.put(
|
|
||||||
# namespace=("images", thread_id), # e.g. ("images", thread_id)
|
|
||||||
# key=image_id,
|
|
||||||
# value={
|
|
||||||
# "url": image_url,
|
|
||||||
# "prompt": prompt,
|
|
||||||
# "timestamp": str(uuid.uuid4()), # 或用 datetime
|
|
||||||
# "version": "v1"
|
|
||||||
# }
|
|
||||||
# )
|
|
||||||
return f"Image has been generated: https://minio-api.aida.com.hk/{image_url}"
|
|
||||||
else:
|
else:
|
||||||
return "Image generation failed."
|
return "Image generation failed."
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -113,51 +60,75 @@ async def generate_furniture(prompt: str, runtime: ToolRuntime):
|
|||||||
|
|
||||||
|
|
||||||
@tool
|
@tool
|
||||||
async def edit_furniture(image_url: str, prompt: str, runtime: ToolRuntime, config: RunnableConfig):
|
async def edit_furniture(prompt: str, runtime: ToolRuntime, config: RunnableConfig):
|
||||||
"""
|
"""
|
||||||
使用先进的图像编辑模型(image editing model)对家具设计草图进行精准修改。
|
使用先进的图像编辑模型(image editing model)对家具设计草图进行精准修改。
|
||||||
|
|
||||||
功能说明:
|
功能说明:
|
||||||
- 输入一张家具图片(草图/效果图),根据用户提供的**详细英文提示词**,生成修改后的新家具图片。
|
- 根据用户提供的**详细英文提示词**,生成修改后的新家具图片。
|
||||||
- 修改后的图片会以 image_url 形式返回,自动加入对话上下文,后续 Agent 可以直接“看到”编辑结果并继续操作。
|
|
||||||
|
|
||||||
参数说明:
|
参数说明:
|
||||||
- image_url (str): 原始家具图片的 URL(支持公开 http/https 链接或 data:image/...;base64 格式)。
|
|
||||||
- prompt (str): **必须是详细的英文提示词**,描述想要的具体修改(风格、颜色、材质、形状、添加/删除元素、比例等)。
|
- prompt (str): **必须是详细的英文提示词**,描述想要的具体修改(风格、颜色、材质、形状、添加/删除元素、比例等)。
|
||||||
示例:"Change the sofa to a modern minimalist style with dark gray fabric and metal legs, add a matching coffee table, make the overall lighting warmer and more luxurious."
|
示例:"Change the sofa to a modern minimalist style with dark gray fabric and metal legs, add a matching coffee table, make the overall lighting warmer and more luxurious."
|
||||||
|
|
||||||
返回值:
|
|
||||||
返回新生成家具图片的 image_url,后续对话中 Agent 可直接引用该图片进行描述、进一步编辑或分析。
|
|
||||||
|
|
||||||
使用场景:
|
使用场景:
|
||||||
- 家具设计迭代
|
- 家具设计迭代
|
||||||
- 室内设计方案修改
|
- 室内设计方案修改
|
||||||
- 风格转换(现代/北欧/工业风等)
|
- 风格转换(现代/北欧/工业风等)
|
||||||
- 材质/颜色调整
|
- 材质/颜色调整
|
||||||
|
|
||||||
注意:如果需要多次迭代编辑,直接在下一次调用时传入上一次返回的新 image_url 即可。
|
|
||||||
"""
|
"""
|
||||||
logger.info(f"\n[系统日志] 正在调用 edit_furniture ...")
|
image_history = runtime.store.get(namespace=("image_history",), key="checkpoint_id", )
|
||||||
thread_id = runtime.config.get("configurable").get("thread_id")
|
last_checkpoint_id = image_history.value.get("last_checkpoint_id")
|
||||||
|
current_checkpoint_id = image_history.value.get("current_checkpoint_id")
|
||||||
|
|
||||||
|
logger.info(f"\n[系统日志] 正在调用 edit_furniture ...current_checkpoint_id={current_checkpoint_id} --- last_checkpoint_id={last_checkpoint_id}")
|
||||||
|
|
||||||
input_path = []
|
input_path = []
|
||||||
|
if image_store.get_image_path(last_checkpoint_id):
|
||||||
|
current_image_path = image_store.get_image_path(last_checkpoint_id).get("current_image_path", False)
|
||||||
|
else:
|
||||||
|
current_image_path = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
user_input_image_paths = runtime.state.get("files").get("input_image", [])
|
user_input_image_paths = runtime.state.get("files").get("input_image", [])
|
||||||
user_quote_image_path = runtime.state.get("files").get("quote_image", "")
|
user_quote_image_path = runtime.state.get("files").get("quote_image", "")
|
||||||
|
|
||||||
extract_result = check_and_extract_minio_image(url=image_url)
|
if current_image_path:
|
||||||
if extract_result['state']:
|
|
||||||
current_image_path = extract_result['data']
|
|
||||||
if len(user_input_image_paths) or current_image_path:
|
if len(user_input_image_paths) or current_image_path:
|
||||||
|
|
||||||
for path in user_input_image_paths:
|
for path in user_input_image_paths:
|
||||||
input_path.append(path)
|
input_path.append(path)
|
||||||
if user_quote_image_path:
|
if user_quote_image_path:
|
||||||
input_path.append(user_quote_image_path)
|
input_path.append(user_quote_image_path)
|
||||||
if not len(user_input_image_paths) and not user_quote_image_path:
|
if not len(user_input_image_paths) and not user_quote_image_path:
|
||||||
input_path = [current_image_path]
|
input_path = [current_image_path]
|
||||||
|
|
||||||
|
bucket_name = "fida-public-bucket"
|
||||||
object_name = f"furniture/sketches/{uuid.uuid4()}.png"
|
object_name = f"furniture/sketches/{uuid.uuid4()}.png"
|
||||||
bucket_name = "fida-public-bucket" # 替换为你的 bucket 名称
|
image_url = await generate_or_edit_image(input_path=input_path, prompt=prompt, bucket_name=bucket_name, object_name=object_name)
|
||||||
|
if image_url:
|
||||||
|
image_store.save_image_path(thread_id=current_checkpoint_id, object_path=image_url, metadata={"prompt": prompt, "generated_at": str(datetime.now())})
|
||||||
|
return image_url
|
||||||
|
else:
|
||||||
|
return "Image generation failed."
|
||||||
|
else:
|
||||||
|
return "The picture to be edited does not exist."
|
||||||
|
else:
|
||||||
|
return "No recent image found, please upload or cite it"
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"edit_furniture error :{e}")
|
||||||
|
return "edit_furniture error"
|
||||||
|
|
||||||
|
|
||||||
|
async def generate_or_edit_image(input_path=None, bucket_name="fida-public-bucket",
|
||||||
|
object_name=f"furniture/sketches/{uuid.uuid4()}.png",
|
||||||
|
prompt="Generate a modern minimalist dining chair made of light "
|
||||||
|
"oak wood and white leather, with slim metal legs, photographed "
|
||||||
|
"in a bright Scandinavian living room with natural sunlight, high detail, "
|
||||||
|
"8k resolution."):
|
||||||
|
if input_path is None:
|
||||||
|
input_path = []
|
||||||
request_data = {
|
request_data = {
|
||||||
"input_image_paths": input_path,
|
"input_image_paths": input_path,
|
||||||
"prompt": prompt,
|
"prompt": prompt,
|
||||||
@@ -173,81 +144,4 @@ async def edit_furniture(image_url: str, prompt: str, runtime: ToolRuntime, conf
|
|||||||
)
|
)
|
||||||
result = resp.json()
|
result = resp.json()
|
||||||
image_url = result.get("output_path", None)
|
image_url = result.get("output_path", None)
|
||||||
|
return image_url
|
||||||
if image_url:
|
|
||||||
# image_store.save_image_path(thread_id=thread_id, object_path=image_url, metadata={"prompt": prompt, "generated_at": str(datetime.now())})
|
|
||||||
return f"Image has been generated: https://minio-api.aida.com.hk/{image_url}"
|
|
||||||
else:
|
|
||||||
return "Image generation failed."
|
|
||||||
else:
|
|
||||||
return "The picture to be edited does not exist."
|
|
||||||
else:
|
|
||||||
return extract_result['message']
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"edit_furniture error :{e}")
|
|
||||||
return "edit_furniture error"
|
|
||||||
|
|
||||||
# def create_generate_furniture_tool(workspace_dir):
|
|
||||||
# @tool
|
|
||||||
# async def generate_furniture(prompt: str) -> str:
|
|
||||||
# """
|
|
||||||
# 使用 Gemini 图像生成模型根据详细的英文提示词生成家具设计草图。
|
|
||||||
# """
|
|
||||||
# print(f"\n[系统日志] 正在调用 Nano Banana (Gemini Image Gen) ...")
|
|
||||||
#
|
|
||||||
# try:
|
|
||||||
# response = client.models.generate_content(
|
|
||||||
# model="gemini-2.5-flash-image",
|
|
||||||
# contents=(f"Generate a professional furniture design sketch: {prompt}"),
|
|
||||||
# config=GenerateContentConfig(
|
|
||||||
# response_modalities=[Modality.TEXT, Modality.IMAGE],
|
|
||||||
# ),
|
|
||||||
# )
|
|
||||||
#
|
|
||||||
# image_bytes = None
|
|
||||||
# for part in response.candidates[0].content.parts:
|
|
||||||
# if part.inline_data:
|
|
||||||
# image_bytes = part.inline_data.data
|
|
||||||
# break
|
|
||||||
#
|
|
||||||
# if not image_bytes:
|
|
||||||
# return "未能生成图像数据。"
|
|
||||||
# # 1. 定义OSS存储路径和本地保存路径
|
|
||||||
# object_name = f"furniture/sketches/{uuid.uuid4()}.png"
|
|
||||||
# bucket = "fida-test" # 替换为你的 bucket 名称
|
|
||||||
# filename = os.path.join(workspace_dir, f"{bucket}/{object_name}")
|
|
||||||
#
|
|
||||||
# # 2. 创建本地目录(确保目录存在)
|
|
||||||
# local_dir = os.path.dirname(filename)
|
|
||||||
# if not os.path.exists(local_dir):
|
|
||||||
# os.makedirs(local_dir, exist_ok=True)
|
|
||||||
#
|
|
||||||
# # 3. 保存图片到本地文件(新增核心逻辑)
|
|
||||||
# try:
|
|
||||||
# with open(filename, "wb") as f:
|
|
||||||
# f.write(image_bytes)
|
|
||||||
# print(f"[系统日志] 图片已保存到本地:{filename}")
|
|
||||||
# except Exception as save_e:
|
|
||||||
# logger.warning(f"保存图片到本地失败:{save_e}")
|
|
||||||
# # 本地保存失败不中断上传流程,仅记录日志
|
|
||||||
#
|
|
||||||
# # 4. 上传图片到OSS(原有逻辑)
|
|
||||||
# upload_res = oss_upload_image(
|
|
||||||
# oss_client=minio_client,
|
|
||||||
# bucket=bucket,
|
|
||||||
# object_name=object_name,
|
|
||||||
# image_bytes=image_bytes
|
|
||||||
# )
|
|
||||||
#
|
|
||||||
# if upload_res:
|
|
||||||
# image_url = f"{bucket}/{object_name}"
|
|
||||||
# return image_url
|
|
||||||
# else:
|
|
||||||
# return f"图片生成成功(本地路径:{filename}),但上传至存储服务器失败。"
|
|
||||||
#
|
|
||||||
# except Exception as e:
|
|
||||||
# logger.warning(f"绘图流程异常:{e}")
|
|
||||||
# return "绘图流程异常"
|
|
||||||
#
|
|
||||||
# return generate_furniture
|
|
||||||
|
|||||||
Reference in New Issue
Block a user