diff --git a/backend/internal/service/provider.go b/backend/internal/service/provider.go index fe43e9ff9..088ea900b 100644 --- a/backend/internal/service/provider.go +++ b/backend/internal/service/provider.go @@ -35,11 +35,17 @@ type canvasGenerationInput struct { TextHistory []providerTextMessage `json:"textHistory"` Mask *providerMedia `json:"mask"` Metadata map[string]interface{} `json:"metadata"` + AgentRequests *agentToolRequests `json:"agentRequests"` ImageCapability *ImageCapabilityConfig `json:"-"` StreamText bool `json:"-"` // 分镜请求使用上游 SSE 保活；最终结构仍在流结束后统一校验。 VideoCapability *VideoCapabilityConfig `json:"-"` } +type agentToolRequests struct { + Responses map[string]interface{} `json:"responses"` + ChatCompletion map[string]interface{} `json:"chatCompletion"` +} + type providerTextMessage struct { Role string `json:"role"` Content string `json:"content"` @@ -247,6 +253,9 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string case "image": return runImageTask(ctx, input) case "text": + if input.AgentRequests != nil { + return runAgentToolTask(ctx, input) + } result, taskErr := runTextTask(ctx, input) if taskErr == nil && promptTemplateOperation != "" { taskErr = validatePromptTemplateResult(promptTemplateOperation, result) @@ -261,6 +270,78 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string } } +func runAgentToolTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) { + request := input.AgentRequests.ChatCompletion + path := "/chat/completions" + protocol := "chat-completion" + if input.Config.InterfaceType == string(model.ChannelInterfaceOpenAIResponse) { + request = input.AgentRequests.Responses + path = "/responses" + protocol = "responses" + } + if request == nil { + return nil, errors.New("画布 Agent 工具请求缺少协议参数") + } + body := cloneStringAnyMap(request) + body["model"] = input.Config.Model + var payload map[string]interface{} + if err := postJSON(ctx, input.Config, path, body, &payload); err != nil { + return nil, err + } + return parseAgentToolPayload(payload, protocol) +} + +func parseAgentToolPayload(payload map[string]interface{}, protocol string) (map[string]interface{}, error) { + if err := validateTextPayload(payload); err != nil { + return nil, err + } + result := map[string]interface{}{"mode": "text", "text": "", "toolCalls": []interface{}{}} + if protocol == "responses" { + result["text"] = firstNonEmptyString(stringField(payload, "output_text"), extractResponseText(payload)) + calls := make([]interface{}, 0) + for _, value := range interfaceSlice(payload["output"]) { + item, _ := value.(map[string]interface{}) + if stringField(item, "type") != "function_call" { + continue + } + calls = append(calls, map[string]interface{}{"id": firstNonEmptyString(stringField(item, "call_id"), stringField(item, "id")), "type": "function", "function": map[string]interface{}{"name": stringField(item, "name"), "arguments": stringField(item, "arguments")}}) + } + result["toolCalls"] = calls + return result, nil + } + choices := interfaceSlice(payload["choices"]) + if len(choices) == 0 { + return nil, errors.New("画布 Agent 接口没有返回 choices") + } + choice, _ := choices[0].(map[string]interface{}) + message, _ := choice["message"].(map[string]interface{}) + result["text"] = stringField(message, "content") + if reasoning := firstNonEmptyString(stringField(message, "reasoning_content"), stringField(message, "reasoning")); reasoning != "" { + result["reasoning"] = reasoning + } + calls := make([]interface{}, 0) + for _, value := range interfaceSlice(message["tool_calls"]) { + item, _ := value.(map[string]interface{}) + function, _ := item["function"].(map[string]interface{}) + calls = append(calls, map[string]interface{}{"id": stringField(item, "id"), "type": "function", "function": map[string]interface{}{"name": stringField(function, "name"), "arguments": stringField(function, "arguments")}}) + } + result["toolCalls"] = calls + return result, nil +} + +func interfaceSlice(value interface{}) []interface{} { + items, _ := value.([]interface{}) + return items +} + +func cloneStringAnyMap(value map[string]interface{}) map[string]interface{} { + cloned := make(map[string]interface{}, len(value)+1) + for key, item := range value { + cloned[key] = item + } + return cloned +} + type styleExecutionPlanDocument struct { SchemaVersion int `json:"schemaVersion"` ProfilePresetID string `json:"profilePresetId"` diff --git a/backend/internal/service/provider_test.go b/backend/internal/service/provider_test.go index 13190b4ab..b20741ce1 100644 --- a/backend/internal/service/provider_test.go +++ b/backend/internal/service/provider_test.go @@ -107,6 +107,59 @@ data: [DONE] } } +func TestParseAgentToolPayloadSupportsChatCompletions(t *testing.T) { + result, err := parseAgentToolPayload(map[string]interface{}{ + "choices": []interface{}{map[string]interface{}{ + "message": map[string]interface{}{ + "content": "准备读取画布", + "tool_calls": []interface{}{map[string]interface{}{ + "id": "call-1", + "function": map[string]interface{}{"name": "canvas_get_state", "arguments": `{}`}, + }}, + }, + }}, + }, "chat-completion") + if err != nil { + t.Fatalf("parseAgentToolPayload() error = %v", err) + } + if result["text"] != "准备读取画布" { + t.Fatalf("text = %v", result["text"]) + } + calls, _ := result["toolCalls"].([]interface{}) + if len(calls) != 1 { + t.Fatalf("toolCalls = %#v", result["toolCalls"]) + } + call, _ := calls[0].(map[string]interface{}) + function, _ := call["function"].(map[string]interface{}) + if call["id"] != "call-1" || function["name"] != "canvas_get_state" || function["arguments"] != `{}` { + t.Fatalf("tool call = %#v", call) + } +} + +func TestParseAgentToolPayloadSupportsResponses(t *testing.T) { + result, err := parseAgentToolPayload(map[string]interface{}{ + "output": []interface{}{ + map[string]interface{}{"type": "message", "content": []interface{}{map[string]interface{}{"type": "output_text", "text": "开始操作"}}}, + map[string]interface{}{"type": "function_call", "call_id": "call-2", "name": "canvas_apply_ops", "arguments": `{"ops":[]}`}, + }, + }, "responses") + if err != nil { + t.Fatalf("parseAgentToolPayload() error = %v", err) + } + if result["text"] != "开始操作" { + t.Fatalf("text = %v", result["text"]) + } + calls, _ := result["toolCalls"].([]interface{}) + if len(calls) != 1 { + t.Fatalf("toolCalls = %#v", result["toolCalls"]) + } + call, _ := calls[0].(map[string]interface{}) + function, _ := call["function"].(map[string]interface{}) + if call["id"] != "call-2" || function["name"] != "canvas_apply_ops" || function["arguments"] != `{"ops":[]}` { + t.Fatalf("tool call = %#v", call) + } +} + func TestPostStreamingTextSetsStreamHeaders(t *testing.T) { t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/web/src/components/canvas/canvas-assistant-panel.tsx b/web/src/components/canvas/canvas-assistant-panel.tsx index b930b1539..6c045a8f2 100644 --- a/web/src/components/canvas/canvas-assistant-panel.tsx +++ b/web/src/components/canvas/canvas-assistant-panel.tsx @@ -8,6 +8,7 @@ import { modelDisplayName, modelIcon, normalizeModelOptionValue, resolveModelCha import { canvasThemes } from "@/lib/canvas-theme"; import { nanoid } from "nanoid"; import { requestToolResponse, type ResponseFunctionTool, type ResponseInputMessage, type ResponseToolCall } from "@/services/api/image"; +import { logicalModelIDForConfig, runBackendToolGenerationTask } from "@/services/api/generation-task"; import { imageToDataUrl } from "@/services/image-storage"; import { isCanvasGenerationDurableAckError, persistCanvasCinematicSessionContinuationEffect } from "@/services/canvas-generation-consumer"; import { consumeGenerationTaskAgent } from "@/services/project-asset-sync"; @@ -376,6 +377,7 @@ export function CanvasAssistantPanel({ const hasMessages = messages.length > 0; const agentBusy = isRunning || safeSessions.some((session) => session.pendingBackendSession?.status === "pending"); const activeModel = effectiveConfig.textModel || effectiveConfig.model; + const activeModelName = activeModel ? modelDisplayName(effectiveConfig, activeModel) : ""; const selectedNodeKey = useMemo(() => Array.from(selectedNodeIds).sort().join(","), [selectedNodeIds]); const allSelectedReferences = useMemo(() => buildAssistantReferences(nodes, selectedNodeIds), [nodes, selectedNodeIds]); const selectedReferences = useMemo(() => allSelectedReferences.filter((item) => !removedReferenceIds.has(item.id)), [allSelectedReferences, removedReferenceIds]); @@ -606,17 +608,10 @@ export function CanvasAssistantPanel({ const messages = await buildToolAgentMessages(snapshotRef.current, history, userMessage); addOnlineLog(`Agent Tool Loop ${loop.step} 开始`, { toolChoice: "required" }); let streamed = ""; - const result = await requestToolResponse( - { ...requestConfig, systemPrompt: "" }, - messages, - ONLINE_AGENT_TOOLS, - "required", - (text) => { - streamed = text; - if (text.trim()) upsertMessage(sessionId, { id: assistantId, role: "assistant", text }); - }, - { promptCacheKey: canvasAgentPromptCacheKey(sessionId) }, - ); + const result = await requestOnlineAgentModel({ ...requestConfig, systemPrompt: "" }, messages, "required", userMessage.text, (text) => { + streamed = text; + if (text.trim()) upsertMessage(sessionId, { id: assistantId, role: "assistant", text }); + }, canvasAgentPromptCacheKey(sessionId)); addOnlineLog("模型工具回复", result); if (result.toolCalls.length) { const writableCalls = result.toolCalls.filter(isWritableToolCall); @@ -672,17 +667,10 @@ export function CanvasAssistantPanel({ } const requestConfig = { ...effectiveConfig, model: effectiveConfig.textModel || effectiveConfig.model }; let streamed = ""; - const next = await requestToolResponse( - { ...requestConfig, systemPrompt: "" }, - nextMessages, - ONLINE_AGENT_TOOLS, - "auto", - (text) => { - streamed = text; - if (text.trim()) upsertMessage(sessionId, { id: assistantId, role: "assistant", text }); - }, - { promptCacheKey: canvasAgentPromptCacheKey(sessionId) }, - ); + const next = await requestOnlineAgentModel({ ...requestConfig, systemPrompt: "" }, nextMessages, "auto", "继续处理画布工具结果", (text) => { + streamed = text; + if (text.trim()) upsertMessage(sessionId, { id: assistantId, role: "assistant", text }); + }, canvasAgentPromptCacheKey(sessionId)); addOnlineLog(`Agent Tool Loop ${step + 1} 回复`, next); if (next.toolCalls.length) { const writableCalls = next.toolCalls.filter(isWritableToolCall); @@ -1019,7 +1007,7 @@ export function CanvasAssistantPanel({ /> {view === "setup" ? ( - navigateToSettings({ continueCreation: true })} /> + navigateToSettings({ continueCreation: true })} /> ) : (
{view === "history" ? ( @@ -1036,7 +1024,7 @@ export function CanvasAssistantPanel({ setOnlineLogs([])} /> ) : messages.length ? ( @@ -1590,6 +1578,15 @@ function toolCallToResponseInput(call: ResponseToolCall): ResponseInputMessage { return { type: "function_call", call_id: call.id, name: call.function.name, arguments: call.function.arguments, ...(call.thoughtSignature ? { thoughtSignature: call.thoughtSignature } : {}) }; } +async function requestOnlineAgentModel(config: AiConfig, messages: ResponseInputMessage[], toolChoice: "auto" | "required", prompt: string, onDelta: (text: string) => void, promptCacheKey: string) { + if (logicalModelIDForConfig(config)) { + const result = await runBackendToolGenerationTask({ prompt, config, messages, tools: ONLINE_AGENT_TOOLS, toolChoice }); + if (result.content.trim()) onDelta(result.content); + return result; + } + return requestToolResponse(config, messages, ONLINE_AGENT_TOOLS, toolChoice, onDelta, { promptCacheKey }); +} + function summarizeToolCalls(calls: ResponseToolCall[]) { return calls.map((call) => toolCallLabel(call.function.name)).join("，") || "工具调用"; } diff --git a/web/src/services/api/generation-task.ts b/web/src/services/api/generation-task.ts index a463987fa..8f63919f5 100644 --- a/web/src/services/api/generation-task.ts +++ b/web/src/services/api/generation-task.ts @@ -10,6 +10,7 @@ import { modelOptionName, resolveModelChannel, resolveModelRequestConfig, type A import { useLocalDreaminaModelStore } from "@/stores/use-local-dreamina-model-store"; import type { ReferenceImage } from "@/types/image"; import type { ReferenceAudio, ReferenceVideo } from "@/types/media"; +import { buildBackendToolRequests, type ResponseFunctionTool, type ResponseInputMessage, type ToolChoice, type ToolResponseResult } from "@/services/api/image"; export type BackendGenerationMode = "text" | "image" | "video" | "audio"; @@ -19,6 +20,8 @@ export type BackendGenerationResult = { video?: { dataUrl: string; storageKey?: string; width?: number; height?: number; durationMs?: number; bytes?: number; mimeType?: string }; audio?: { dataUrl: string; storageKey?: string; durationMs?: number; bytes?: number; mimeType?: string; format?: string }; text?: string; + toolCalls?: Array<{ id: string; type: "function"; function: { name: string; arguments: string }; thoughtSignature?: string }>; + reasoning?: string; }; type BackendGenerationTaskOptions = { @@ -105,6 +108,40 @@ export async function runBackendGenerationTask( return createAndWaitGenerationTask({ projectId, mode, prompt, config, referenceImages, referenceVideos, referenceAudios, textHistory, signal, metadata, onTaskUpdate }, prepared, dependencies); } +export async function runBackendToolGenerationTask(options: { + prompt: string; + config: AiConfig; + messages: ResponseInputMessage[]; + tools: ResponseFunctionTool[]; + toolChoice: ToolChoice; + signal?: AbortSignal; +}): Promise { + throwIfAborted(options.signal); + const logicalModelId = logicalModelIDForConfig(options.config); + if (!logicalModelId) throw new Error("当前模型不是平台系统模型"); + const task = await createGenerationTask({ + type: "canvas_text", + operation: "text", + prompt: options.prompt, + model: options.config.model, + logicalModelId, + input: { + mode: "text", + prompt: options.prompt, + config: backendProviderConfig(options.config), + agentRequests: buildBackendToolRequests(options.messages, options.tools, options.toolChoice), + metadata: { source: "canvas-online-agent" }, + }, + }); + const completed = await waitForGenerationTask(task.id, { signal: options.signal, initialTask: task }); + const result = parseBackendGenerationResult(completed); + return { + content: result.text || "", + toolCalls: result.toolCalls || [], + ...(result.reasoning ? { reasoning: result.reasoning } : {}), + }; +} + export async function runBackendGenerationTaskBatch(options: BackendGenerationTaskOptions & { count: number }, dependencies: GenerationTaskDependencies = defaultDependencies) { const count = Math.max(1, Math.min(15, Math.floor(Number(options.count)) || 1)); throwIfAborted(options.signal); diff --git a/web/src/services/api/image.ts b/web/src/services/api/image.ts index fede3cce3..70c711cd4 100644 --- a/web/src/services/api/image.ts +++ b/web/src/services/api/image.ts @@ -46,7 +46,11 @@ export type ToolResponseResult = { reasoning?: string; }; -type ToolChoice = "auto" | "required" | { type: "function"; name: string }; +export type ToolChoice = "auto" | "required" | { type: "function"; name: string }; +export type BackendToolRequests = { + responses: Record; + chatCompletion: Record; +}; type ResponseMessageContent = AiTextMessage["content"] | string; type ResponseInputContent = { type: "input_text"; text: string } | { type: "input_image"; image_url: string } | { type: "input_file"; filename: string; file_data?: string; file_url?: string }; type ResponseInputItem = { role: "system" | "user" | "assistant"; content: string | ResponseInputContent[] } | { type: "function_call"; call_id: string; name: string; arguments: string } | { type: "function_call_output"; call_id: string; output: string }; @@ -407,6 +411,23 @@ function toChatCompletionToolChoice(toolChoice: ToolChoice) { return typeof toolChoice === "object" ? { type: "function", function: { name: toolChoice.name } } : toolChoice; } +export function buildBackendToolRequests(messages: ResponseInputMessage[], tools: ResponseFunctionTool[], toolChoice: ToolChoice): BackendToolRequests { + return { + responses: { + input: toResponseInput(messages), + tools: tools.map(toResponseTool), + tool_choice: toolChoice, + parallel_tool_calls: false, + }, + chatCompletion: { + messages: toChatCompletionMessages(messages), + tools, + tool_choice: toChatCompletionToolChoice(toolChoice), + parallel_tool_calls: false, + }, + }; +} + function isToolChoiceCompatibilityError(error: unknown) { const message = error instanceof Error ? error.message : ""; return /tool[_\s-]?choice|thinking\s+mode/i.test(message); diff --git a/web/test/generation-task-durability.node.test.ts b/web/test/generation-task-durability.node.test.ts index 009847d30..40b09a600 100644 --- a/web/test/generation-task-durability.node.test.ts +++ b/web/test/generation-task-durability.node.test.ts @@ -4,6 +4,31 @@ import { test } from "node:test"; import type { GenerationTask } from "../src/services/api/task-center"; import { applyGenerationConsumerEffect } from "../src/services/generation-consumer-dedupe"; import { createGenerationTaskMaterializer, type GenerationTaskEffectResult, type GenerationTaskEffectStore } from "../src/services/generation-task-materializer"; +import { buildBackendToolRequests } from "../src/services/api/image"; + +test("managed canvas agent task keeps tool definitions and tool results", () => { + const requests = buildBackendToolRequests( + [ + { role: "user", content: "读取画布" }, + { type: "function_call", call_id: "call-1", name: "canvas_get_state", arguments: "{}" }, + { role: "tool", tool_call_id: "call-1", content: "{\"nodes\":[]}" }, + ], + [{ type: "function", function: { name: "canvas_get_state", parameters: { type: "object" } } }], + "required", + ); + + assert.equal(requests.responses.tool_choice, "required"); + assert.deepEqual(requests.responses.input, [ + { role: "user", content: "读取画布" }, + { type: "function_call", call_id: "call-1", name: "canvas_get_state", arguments: "{}" }, + { type: "function_call_output", call_id: "call-1", output: "{\"nodes\":[]}" }, + ]); + assert.deepEqual(requests.chatCompletion.messages, [ + { role: "user", content: "读取画布" }, + { role: "assistant", content: null, tool_calls: [{ id: "call-1", type: "function", function: { name: "canvas_get_state", arguments: "{}" } }] }, + { role: "tool", tool_call_id: "call-1", content: "{\"nodes\":[]}" }, + ]); +}); test("node message and agent consumers receive stable effect keys across three replays", async () => { const completed = new Map();