diff --git a/web/src/components/canvas/canvas-assistant-online-tools.ts b/web/src/components/canvas/canvas-assistant-online-tools.ts index bc3e1cf33..36a511836 100644 --- a/web/src/components/canvas/canvas-assistant-online-tools.ts +++ b/web/src/components/canvas/canvas-assistant-online-tools.ts @@ -1,8 +1,9 @@ import { nanoid } from "nanoid"; -import { type ResponseInputMessage, type ResponseToolCall } from "@/services/api/image"; +import { type AiTextMessage, type ResponseInputMessage, type ResponseToolCall } from "@/services/api/image"; import { runBackendToolGenerationTask } from "@/services/api/generation-task"; import { resourceIdFromStorageKey } from "@/services/api/resources"; import { normalizeModelOptionValue, resolveModelRequestConfig, selectableModelsByCapability, type AiConfig } from "@/stores/use-config-store"; +import { modelCapabilityConfigFor } from "@/lib/model-capabilities"; import { NODE_DEFAULT_SIZE } from "@/constant/canvas"; import { CanvasNodeType, type CanvasAssistantMessage, type CanvasAssistantReference, type CanvasAssistantSession, type CanvasNodeData } from "@/types/canvas"; import { previewCanvasAgentOps, type CanvasAgentOp, type CanvasAgentOperationImpact, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops"; @@ -11,9 +12,35 @@ import { CANVAS_ONLINE_AGENT_TOOLS } from "@/lib/canvas/canvas-agent-tools"; import { buildOrderedCanvasResourceReferences, canvasResourceMentionToken } from "@/lib/canvas/canvas-resource-references"; import { buildCanvasWorkflowOps, looksLikeWorkflowRequest, type CanvasWorkflowInput } from "@/lib/canvas/canvas-agent-workflow"; import type { Skill } from "@/services/api/skills"; +import { AGENT_CAPABILITY_GUIDANCE } from "@/services/agent-capabilities"; +import { budgetCanvasAgentHistory } from "@/lib/canvas/canvas-agent-context-budget"; +import { CREATIVE_AGENT_SYSTEM_PROMPT } from "@/lib/creation/creative-agent-tools"; +import { creativeScenarioPrompt, type CreativeBrief, type CreativeScenarioId } from "@/lib/creation/creative-agent-contract"; +import type { CreativeDynamicPlan } from "@/lib/creation/creative-plan"; +import type { CreativeReference } from "@/lib/creation/creative-agent-state"; export type OnlineToolResult = { ok: true; message: string; data?: unknown; waitForUser?: boolean } | { ok: false; message: string }; +export type CreativeOnlineContext = { + scene: CreativeScenarioId; + brief: CreativeBrief; + plan?: CreativeDynamicPlan; + previousProposal?: unknown; + rejectedProposal?: unknown; + previousQuestions?: unknown; + answers?: unknown; + executionResults: unknown[]; + media?: unknown; + references: CreativeReference[]; + sourceContext?: unknown[]; +}; + +export type BuildToolAgentMessagesOptions = { + config?: AiConfig; + confirmTools?: boolean; + creative?: CreativeOnlineContext; +}; + export function objectDetail(value: unknown) { return value && typeof value === "object" ? (value as Record) : {}; } @@ -54,7 +81,7 @@ export function onlineToolToOps(name: string, input: Record, sn if (name === "canvas_create_text_nodes") { const items = requireRecordArray(input.items, "items"); const textBatch = items.map((item) => `${String(item.title || "")} ${String(item.text || "")}`).join(" "); - if (looksLikeWorkflowRequest(textBatch)) throw new Error("检测到流水线/工作流意图，请使用 canvas_create_workflow 创建真实类型节点和连线。" ); + if (looksLikeWorkflowRequest(textBatch)) throw new Error("检测到流水线/工作流意图，请使用 canvas_create_workflow 创建真实类型节点和连线。"); const x = numberOr(input.x, nextCanvasX(snapshot)); const y = numberOr(input.y, 0); const gap = numberOr(input.gap, 40); @@ -474,7 +501,7 @@ export function buildAssistantReferences(nodes: CanvasNodeData[], selectedNodeId .filter((item): item is CanvasAssistantReference => Boolean(item)); } -export async function buildToolAgentMessages(snapshot: CanvasAgentSnapshot, history: CanvasAssistantMessage[], userMessage: CanvasAssistantMessage, skills: Skill[] = []): Promise { +export async function buildToolAgentMessages(snapshot: CanvasAgentSnapshot, history: CanvasAssistantMessage[], userMessage: CanvasAssistantMessage, skills: Skill[] = [], options: BuildToolAgentMessagesOptions = {}): Promise { const refs = userMessage.references || []; const imageRefs = refs.filter((item) => item.type === CanvasNodeType.Image); const unavailableImageRefs = imageRefs.filter((item) => !resourceIdFromStorageKey(item.storageKey)); @@ -487,22 +514,54 @@ export async function buildToolAgentMessages(snapshot: CanvasAgentSnapshot, hist .slice(0, 40) .map((skill) => `- ${skill.skillName}（${skill.skillId}，v${skill.version || "1"}，${skill.fileCount || 1} 文件）：${skill.description}`) .join("\n"); - const systemContent = buildCanvasOnlineAgentSystemContent(skillCatalog); - return [ - { role: "system", content: systemContent }, - ...history - .filter((message) => message.role === "user" || message.role === "assistant" || message.role === "system") - .slice(-8) - .map((message): ResponseInputMessage => ({ role: message.role as "system" | "user" | "assistant", content: message.text })), - { - role: "user", - content: [ - ...refs.flatMap((item) => (item.text ? [{ type: "text" as const, text: `选中节点 ${item.title}：${item.text}` }] : [])), - { type: "text", text: `当前画布：${JSON.stringify(compactSnapshot(snapshot))}\n\n用户需求：${userMessage.text}` }, - ...imageRefs.map((item) => ({ type: "image_url" as const, image_url: { url: item.storageKey! } })), - ], - }, - ]; + const creative = options.creative; + const executionGuidance = + options.confirmTools === false + ? "当前普通画布工具的逐次确认已关闭。用户请求范围内的操作可以直接调用，不要先征求重复授权；这不代表可以擅自扩大删除范围或跳过结构化方案、具体费用的批准。" + : "当前普通画布工具由程序展示执行确认。用户目标和操作范围明确时直接提交工具，程序会处理确认，不要在工具确认之前再用文字问一遍。"; + const systemContent = [buildCanvasOnlineAgentSystemContent(skillCatalog), AGENT_CAPABILITY_GUIDANCE, executionGuidance, creative ? CREATIVE_AGENT_SYSTEM_PROMPT : "", creative ? creativeScenarioPrompt(creative.scene) : ""].filter(Boolean).join("\n\n"); + const historyMessages: AiTextMessage[] = history + .filter((message) => message.role === "user" || message.role === "assistant" || message.role === "system") + .map((message) => ({ role: message.role as "system" | "user" | "assistant", content: message.text })); + const creativeContextText = creative + ? `创作上下文：${JSON.stringify({ + brief: creative.brief, + plan: creative.plan, + previousProposal: creative.previousProposal, + rejectedProposal: creative.rejectedProposal, + previousQuestions: creative.previousQuestions, + answers: creative.answers, + executionResults: creative.executionResults, + media: creative.media, + references: creative.references, + availableModels: options.config ? (["image", "video"] as const).flatMap((mode) => selectableModelsByCapability(options.config!, mode).map((model) => ({ mode, model, capability: modelCapabilityConfigFor(options.config!, model) }))) : [], + })}\n已返回的执行状态不代表已观察画面。保留成功产物，若用户仅要求分析结果则只提供分析和下一步建议。新增或修改方案仍须确认，媒体生成仍须费用确认。` + : ""; + const currentMessage: AiTextMessage = { + role: "user", + content: [ + ...refs.flatMap((item) => (item.text ? [{ type: "text" as const, text: `选中节点 ${item.title}：${item.text}` }] : [])), + ...(creative?.sourceContext?.length + ? [ + { + type: "text" as const, + text: `首页接续记录（任务事实和资料目录，不代表新的执行批准）：${JSON.stringify(creative.sourceContext)}。沿用前序用户需求与技能引用；进入画布本身不要求重做作品，不把首页文本当成已获批准的画布操作。先检查当前真实节点，资源目录不等于已观察图片。`, + }, + ] + : []), + { type: "text", text: `当前画布：${JSON.stringify(compactSnapshot(snapshot))}\n${creativeContextText}\n用户需求：${userMessage.text}` }, + ...(creative + ? [ + { + type: "text" as const, + text: `本次引用的图片（仅素材目录，尚未观察图片）：${JSON.stringify(refs.filter((item) => item.dataUrl || item.storageKey).map((item) => ({ id: item.id, title: item.title })))}。需要观察画面时调用 canvas_inspect_image；不能仅凭素材名称断言画面内容。`, + }, + ] + : []), + ...imageRefs.map((item) => ({ type: "image_url" as const, image_url: { url: item.storageKey! } })), + ], + }; + return budgetCanvasAgentHistory({ role: "system", content: systemContent }, historyMessages, currentMessage); } export function compactSnapshot(snapshot: CanvasAgentSnapshot) { diff --git a/web/src/components/canvas/canvas-assistant-panel.tsx b/web/src/components/canvas/canvas-assistant-panel.tsx index 169c8b144..b42cace04 100644 --- a/web/src/components/canvas/canvas-assistant-panel.tsx +++ b/web/src/components/canvas/canvas-assistant-panel.tsx @@ -9,6 +9,7 @@ import { motion } from "motion/react"; import { resolveModelRequestConfig, useConfigStore, useEffectiveConfig, type AiConfig } from "@/stores/use-config-store"; import { nanoid } from "nanoid"; import { type ResponseInputMessage, type ResponseToolCall } from "@/services/api/image"; +import { runBackendToolGenerationTask } from "@/services/api/generation-task"; import { inspectAgentImage } from "@/services/agent-image-preview"; import { listAgentCapabilities, readAgentPluginDocumentation, readAgentPluginNode } from "@/services/agent-capabilities"; import { readAgentStoryboard } from "@/lib/canvas/canvas-agent-storyboard"; @@ -39,11 +40,14 @@ import { isWritableToolCall } from "@/lib/canvas/canvas-agent-protocol"; import { listAddedSkills, type Skill } from "@/services/api/skills"; import { buildSkillMentionReferences, skillRuntime } from "@/services/skill-runtime"; import { handleCinematicContinuationFailure, canvasCinematicContinuationEntryAdapters, type CinematicContinuationFailureDisposition, type CinematicContinuationLiveSessionState } from "./canvas-cinematic-continuation"; -import { CanvasCreativeInteraction, canvasCreativeDetail } from "./canvas-creative-interaction"; -import { creativePlan } from "@/lib/creation/creative-agent-state"; +import { CanvasCreativeInteraction, canvasCreativeDetail, creativeInteractionSeed, creativeProposalFailure, type CanvasCreativeDetail } from "./canvas-creative-interaction"; +import { creativePlan, type CreativeReference } from "@/lib/creation/creative-agent-state"; import { CreativePlanBar } from "@/components/creation/creative-agent-cards"; import { AgentTextModelPicker, AssistantHistory, AssistantReferenceChip, MessageReferences, assistantImageReferenceLabel, assistantMessageToChatMessage, assistantReferencesToMentionReferences, promptAlreadyHasMention } from "./canvas-assistant-panel-views"; -import { backendAgentProviderConfig, buildAssistantReferences, buildToolAgentMessages, capabilityBatchTitle, cinematicSessionMessageId, compactSnapshot, createSession, describeCanvasSnapshot, explainNoop, nodeToReference, objectDetail, onlineToolToOps, parseToolArguments, previewOnlineToolCalls, requestOnlineAgentModel, requireOps, requireString, snapshotSignature, summarizeToolCalls, toolCallToResponseInput, toolCallsFromDetail, toolResultText, upsertAssistantMessage, type OnlineToolResult } from "./canvas-assistant-online-tools"; +import { backendAgentProviderConfig, buildAssistantReferences, buildToolAgentMessages, capabilityBatchTitle, cinematicSessionMessageId, compactSnapshot, createSession, describeCanvasSnapshot, explainNoop, nodeToReference, objectDetail, onlineToolToOps, parseToolArguments, previewOnlineToolCalls, requestOnlineAgentModel, requireOps, requireString, snapshotSignature, summarizeToolCalls, toolCallToResponseInput, toolCallsFromDetail, toolResultText, upsertAssistantMessage, type CreativeOnlineContext, type OnlineToolResult } from "./canvas-assistant-online-tools"; +import { CREATIVE_AGENT_TOOLS } from "@/lib/creation/creative-agent-tools"; +import { resourceIdFromStorageKey } from "@/services/api/resources"; +import { recoverCreativeResponse } from "@/services/creative-agent-recovery"; export const CANVAS_AGENT_PANEL_MOTION_MS = 500; const PANEL_MOTION_SECONDS = CANVAS_AGENT_PANEL_MOTION_MS / 1000; @@ -80,6 +84,94 @@ type CanvasAssistantPanelProps = { resizing?: boolean; }; +function buildCreativeOnlineContext(history: CanvasAssistantMessage[], snapshot: CanvasAgentSnapshot, config: AiConfig): CreativeOnlineContext { + const seed = creativeInteractionSeed(history); + const previous = history.findLast((message) => canvasCreativeDetail(message)); + const previousState = previous ? canvasCreativeDetail(previous)!.state : undefined; + const previousDetail = previous ? canvasCreativeDetail(previous)! : undefined; + const validationError = previousDetail ? creativeProposalFailure(previousDetail, config) : undefined; + const rejectedProposal = validationError + ? { + error: validationError, + input: previousDetail!.input, + instruction: "此方案校验失败，未获得用户批准，未创建节点或生成媒体。修正错误并重新用 creative_respond 返回完整方案；不得省略 generationItems 来规避校验。", + } + : undefined; + const proposalMessage = history.findLast((message) => Boolean(canvasCreativeDetail(message)?.state.proposal)); + const proposalDetail = proposalMessage ? canvasCreativeDetail(proposalMessage)! : undefined; + const proposal = proposalDetail?.state.proposal; + const previousProposal = + proposal && proposalDetail?.state.canvasApplied + ? { + contextSummary: true, + instruction: "已落地方案的结构摘要，未包含原方案正文和生成提示词。需要修改具体内容时先读取对应画布节点，不得用摘要覆盖正文。", + id: proposal.id, + version: proposal.version, + title: proposal.title, + summary: proposal.summary, + deliverables: proposal.deliverables, + generationItems: proposal.generationItems, + nodes: proposal.workflow.nodes.map((node) => ({ ref: node.ref, kind: node.kind, title: node.title, referenceRefs: node.referenceRefs, referenceNodeIds: node.referenceNodeIds })), + edges: proposal.workflow.edges, + } + : proposal; + const executionByRun = new Map( + history.flatMap((message) => { + const detail = canvasCreativeDetail(message); + return detail && (detail.state.canvasApplied || detail.state.media.some((item) => item.taskId || item.error)) + ? [ + [ + detail.runId || message.id, + { + runId: detail.runId, + status: detail.status, + proposal: detail.state.proposal?.title, + canvasApplied: detail.state.canvasApplied, + approvedProposalVersion: detail.approvedProposalVersion, + media: detail.state.media.map((item) => ({ ref: item.ref, nodeId: item.nodeId, taskId: item.taskId, status: item.status, error: item.error })), + }, + ] as const, + ] + : []; + }), + ); + return { + scene: seed.scene, + brief: seed.brief, + plan: seed.dynamicPlan, + previousProposal, + rejectedProposal, + previousQuestions: previousState?.questions, + answers: previousState?.answers, + executionResults: [...executionByRun.values()], + media: previousState?.media, + references: creativeCanvasReferences(snapshot), + sourceContext: history.flatMap((message) => { + const detail = objectDetail(message.detail); + return detail.kind === "creation-handoff" ? [{ messageId: message.id, ...detail }] : []; + }), + }; +} + +function creativeCanvasReferences(snapshot: CanvasAgentSnapshot): CreativeReference[] { + return snapshot.nodes.flatMap((node): CreativeReference[] => + node.type === "image" && node.metadata?.storageKey && resourceIdFromStorageKey(node.metadata.storageKey) + ? [ + { + id: node.id, + title: node.title || "画布图片", + kind: "image", + storageKey: node.metadata.storageKey, + assetId: node.metadata.assetId, + mimeType: node.metadata.mimeType, + width: node.metadata.naturalWidth, + height: node.metadata.naturalHeight, + }, + ] + : [], + ); +} + export { handleCinematicContinuationFailure, runCanvasCinematicContinuationBoundary, canvasCinematicContinuationEntryAdapters } from "./canvas-cinematic-continuation"; export type { CinematicContinuationFailureDisposition } from "./canvas-cinematic-continuation"; export function CanvasAssistantPanel({ @@ -439,12 +531,18 @@ export function CanvasAssistantPanel({ const requestConfig = { ...effectiveConfig, model: effectiveConfig.textModel || effectiveConfig.model }; try { setIsRunning(true); - const messages = await buildToolAgentMessages(snapshotRef.current, history, userMessage, composerSkills); + const messages = await buildToolAgentMessages(snapshotRef.current, history, userMessage, composerSkills, { + config: effectiveConfig, + confirmTools, + creative: buildCreativeOnlineContext(history, snapshotRef.current, effectiveConfig), + }); let streamed = ""; - const result = await requestOnlineAgentModel({ ...requestConfig, systemPrompt: "" }, messages, "required", userMessage.text, (text) => { + const initial = await requestOnlineAgentModel({ ...requestConfig, systemPrompt: "" }, messages, "required", userMessage.text, (text) => { streamed = text; if (text.trim()) upsertMessage(sessionId, { id: assistantId, role: "assistant", text }); }); + const result = await repairCreativeReply(initial, messages, userMessage.text, history); + if (presentCreativeResponse(sessionId, assistantId, result)) return; if (result.toolCalls.length) { const writableCalls = result.toolCalls.filter(isWritableToolCall); if (confirmTools && writableCalls.length) { @@ -474,6 +572,44 @@ export function CanvasAssistantPanel({ } }; + const presentCreativeResponse = (sessionId: string, assistantId: string, result: { content: string; toolCalls: ResponseToolCall[] }) => { + const call = result.toolCalls.find((item) => item.function.name === "creative_respond"); + if (!call) return false; + const input = parseToolArguments(call.function.arguments); + const history = localSessionsRef.current.find((session) => session.id === sessionId)?.messages || []; + const state = creativeInteractionSeed(history.filter((message) => message.id !== assistantId)); + state.references = creativeCanvasReferences(snapshotRef.current); + const detail: CanvasCreativeDetail = { kind: "creative-interaction", input, state }; + upsertMessage(sessionId, { + id: assistantId, + role: "assistant", + text: typeof input.message === "string" ? input.message : result.content || "已整理当前需求。", + detail, + }); + return true; + }; + + const repairCreativeReply = async (reply: { content: string; toolCalls: ResponseToolCall[] }, protocol: ResponseInputMessage[], latestInput: string, history: CanvasAssistantMessage[]) => { + const state = creativeInteractionSeed(history); + state.references = creativeCanvasReferences(snapshotRef.current); + const config = { ...effectiveConfig, model: effectiveConfig.textModel || effectiveConfig.model, systemPrompt: "" }; + return recoverCreativeResponse( + reply, + protocol, + { config, state, latestInput }, + (messages) => + runBackendToolGenerationTask({ + prompt: "根据程序校验反馈修正创作交互", + config, + messages, + tools: CREATIVE_AGENT_TOOLS, + toolChoice: "required", + signal: generationConsumerControllerRef.current.signal, + }), + () => undefined, + ); + }; + const continueOnlineToolLoop = async (sessionId: string, assistantId: string, messages: ResponseInputMessage[], result: { content: string; toolCalls: ResponseToolCall[] }, step: number) => { const toolResults = await executeOnlineToolCalls(sessionId, result.toolCalls); appendMessage(sessionId, { @@ -487,6 +623,11 @@ export function CanvasAssistantPanel({ }; const continueOnlineToolLoopAfterResults = async (sessionId: string, assistantId: string, messages: ResponseInputMessage[], toolCalls: ResponseToolCall[], toolResults: OnlineExecutedToolCall[], step: number) => { + const waiting = toolResults.find((item) => item.result.ok && item.result.waitForUser); + if (waiting) { + upsertMessage(sessionId, { id: assistantId, role: "assistant", text: waiting.result.message }); + return; + } const nextMessages: ResponseInputMessage[] = [...messages, ...toolCalls.map(toolCallToResponseInput), ...toolResults.map((item) => ({ role: "tool" as const, tool_call_id: item.toolCallId, content: JSON.stringify(item.result) }))]; if (step >= ONLINE_AGENT_MAX_STEPS) { upsertMessage(sessionId, { id: assistantId, role: "assistant", text: toolResults.map((item) => toolResultText(item.result)).join("\n") || "工具已执行。" }); @@ -498,25 +639,28 @@ export function CanvasAssistantPanel({ streamed = text; if (text.trim()) upsertMessage(sessionId, { id: assistantId, role: "assistant", text }); }); - if (next.toolCalls.length) { - const writableCalls = next.toolCalls.filter(isWritableToolCall); + const history = localSessionsRef.current.find((session) => session.id === sessionId)?.messages || []; + const repaired = await repairCreativeReply(next, nextMessages, history.findLast((message) => message.role === "user")?.text || "", history); + if (presentCreativeResponse(sessionId, assistantId, repaired)) return; + if (repaired.toolCalls.length) { + const writableCalls = repaired.toolCalls.filter(isWritableToolCall); if (confirmTools && writableCalls.length) { - upsertMessage(sessionId, { id: assistantId, role: "assistant", text: next.content || streamed || "准备执行工具，等待确认。" }); + upsertMessage(sessionId, { id: assistantId, role: "assistant", text: repaired.content || streamed || "准备执行工具，等待确认。" }); const toolMessageId = nanoid(); - pendingToolContextRef.current.set(toolMessageId, { messages: nextMessages, toolCalls: next.toolCalls, assistantId, step: step + 1 }); + pendingToolContextRef.current.set(toolMessageId, { messages: nextMessages, toolCalls: repaired.toolCalls, assistantId, step: step + 1 }); appendMessage(sessionId, { id: toolMessageId, role: "tool", - title: `确认${capabilityBatchTitle(next.toolCalls)}`, - text: summarizeToolCalls(next.toolCalls), - detail: { status: "pending", step: step + 1, toolCalls: next.toolCalls, impact: previewOnlineToolCalls(next.toolCalls, snapshotRef.current, effectiveConfig) }, + title: `确认${capabilityBatchTitle(repaired.toolCalls)}`, + text: summarizeToolCalls(repaired.toolCalls), + detail: { status: "pending", step: step + 1, toolCalls: repaired.toolCalls, impact: previewOnlineToolCalls(repaired.toolCalls, snapshotRef.current, effectiveConfig) }, }); return; } - await continueOnlineToolLoop(sessionId, assistantId, nextMessages, next, step + 1); + await continueOnlineToolLoop(sessionId, assistantId, nextMessages, repaired, step + 1); return; } - upsertMessage(sessionId, { id: assistantId, role: "assistant", text: next.content || streamed || toolResults.map((item) => toolResultText(item.result)).join("\n") || "工具已执行。" }); + upsertMessage(sessionId, { id: assistantId, role: "assistant", text: repaired.content || streamed || toolResults.map((item) => toolResultText(item.result)).join("\n") || "工具已执行。" }); }; const executeOps = async (ops: CanvasAgentOp[], context?: { conversationId?: string; messageId?: string; source?: "online" | "local" }) => { diff --git a/web/src/lib/canvas/canvas-agent-tools.ts b/web/src/lib/canvas/canvas-agent-tools.ts index 5be25b7c6..1c5d1c9bf 100644 --- a/web/src/lib/canvas/canvas-agent-tools.ts +++ b/web/src/lib/canvas/canvas-agent-tools.ts @@ -1,5 +1,6 @@ import { type ResponseFunctionTool } from "@/services/api/image"; import { skillRuntime } from "@/services/skill-runtime"; +import { CREATIVE_AGENT_TOOLS } from "@/lib/creation/creative-agent-tools"; const JSON_RECORD_SCHEMA = { type: "object", additionalProperties: true }; const POSITION_SCHEMA = { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false }; @@ -73,6 +74,7 @@ function generationToolDefinition(name: string, description: string, mode?: "tex } export const CANVAS_ONLINE_AGENT_TOOLS: ResponseFunctionTool[] = [ + ...CREATIVE_AGENT_TOOLS, ...skillRuntime.agentTools("onlineAgent"), toolDefinition("canvas_get_state", "读取当前网页画布的节点、连线、选区和视口。", {}), toolDefinition("canvas_get_context", "读取语义化画布上下文、真实节点 id、连接关系、资源就绪状态和状态哈希。", {}), diff --git a/web/test/canvas-assistant-creative.test.ts b/web/test/canvas-assistant-creative.test.ts new file mode 100644 index 000000000..d7c1a774c --- /dev/null +++ b/web/test/canvas-assistant-creative.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, test } from "bun:test"; + +import { CANVAS_ONLINE_AGENT_TOOLS } from "@/lib/canvas/canvas-agent-tools"; +import { buildToolAgentMessages } from "@/components/canvas/canvas-assistant-online-tools"; +import type { CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops"; +import type { CanvasAssistantMessage } from "@/types/canvas"; + +const snapshot: CanvasAgentSnapshot = { + projectId: "canvas", + title: "测试画布", + nodes: [], + connections: [], + selectedNodeIds: [], + viewport: { x: 0, y: 0, k: 1 }, +}; + +describe("画布在线创作交互", () => { + test("在线工具清单包含 creative_respond", () => { + expect(CANVAS_ONLINE_AGENT_TOOLS.some((tool) => tool.function.name === "creative_respond")).toBe(true); + }); + + test("创作请求携带创作指导和结构化上下文", async () => { + const history: CanvasAssistantMessage[] = [{ id: "user-1", role: "user", text: "我想做一个短片" }]; + const messages = await buildToolAgentMessages(snapshot, history, { id: "user-2", role: "user", text: "先问我关键需求" }, [], { + creative: { + scene: "short-film", + brief: {}, + executionResults: [], + references: [{ id: "asset-1", title: "参考图", kind: "image" }], + }, + }); + const system = messages.find((message) => message.role === "system"); + const user = messages.findLast((message) => message.role === "user"); + expect(system?.content).toContain("多步骤创作使用 creative_respond"); + expect(user?.content).toEqual(expect.arrayContaining([expect.objectContaining({ type: "text", text: expect.stringContaining("创作上下文") })])); + }); +});