From 7043c8ac7a82d5d7a2488caf9fd644b913f984de Mon Sep 17 00:00:00 2001 From: ddCat <2695092045@qq.com> Date: Mon, 24 Aug 2026 12:09:15 +0800 Subject: [PATCH] =?UTF-8?q?feat(agent):=20=E7=94=BB=E5=B8=83=20Agent=20-?= =?UTF-8?q?=20=E9=87=8D=E6=9E=84=E5=B7=A5=E4=BD=9C=E6=B5=81=E4=B8=8A?= =?UTF-8?q?=E4=B8=8B=E6=96=87=E4=B8=8E=E6=8A=80=E8=83=BD=E4=BA=A4=E4=BA=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .claude/skills/superdesign/SKILL.md | 33 +++ README.md | 8 + canvas-agent/README.md | 32 ++- canvas-agent/bun.lock | 20 +- canvas-agent/package.json | 4 +- canvas-agent/src/agents.ts | 67 ++++- canvas-agent/src/canvas-context.ts | 191 +++++++++++++- canvas-agent/src/canvas-session.ts | 145 +++++++++- canvas-agent/src/config.ts | 5 +- canvas-agent/src/modules/canvas-agent-http.ts | 31 ++- canvas-agent/src/schemas.ts | 40 ++- canvas-agent/src/types.ts | 2 +- canvas-agent/test/canvas-agent-module.test.ts | 162 ++++++++++++ canvas-agent/test/canvas-context.test.ts | 86 +++++- canvas-agent/test/skills.test.ts | 57 ++++ plugins/yingce/skills/canvas-context/SKILL.md | 17 +- web/package.json | 2 +- .../canvas/canvas-agent-chat-ui.tsx | 98 ++++++- .../canvas/canvas-agent-panel-chrome.tsx | 66 +++-- .../canvas/canvas-assistant-panel.tsx | 247 ++++++++++++------ .../canvas/canvas-local-agent-panel.tsx | 186 +++++++++---- web/src/lib/canvas/canvas-agent-context.ts | 175 +++++++++++-- web/src/lib/canvas/canvas-agent-ops.ts | 230 ++++++++++++++++ web/src/lib/canvas/canvas-agent-workflow.ts | 220 ++++++++++++++++ web/src/lib/canvas/canvas-skill-mentions.ts | 42 ++- .../lib/canvas/local-runtime-connection.ts | 20 +- .../pages/canvas/canvas-project-top-bar.tsx | 4 +- web/src/pages/canvas/project.tsx | 3 +- .../stores/canvas/use-canvas-agent-store.ts | 4 +- web/src/styles/globals.css | 12 +- web/test/canvas-agent-context.test.ts | 116 ++++++++ web/test/canvas-agent-workflow.test.ts | 83 ++++++ web/test/canvas-skill-mentions.test.ts | 70 +++++ 33 files changed, 2230 insertions(+), 248 deletions(-) create mode 100644 .claude/skills/superdesign/SKILL.md create mode 100644 canvas-agent/test/skills.test.ts create mode 100644 web/src/lib/canvas/canvas-agent-workflow.ts create mode 100644 web/test/canvas-agent-context.test.ts create mode 100644 web/test/canvas-agent-workflow.test.ts create mode 100644 web/test/canvas-skill-mentions.test.ts diff --git a/.claude/skills/superdesign/SKILL.md b/.claude/skills/superdesign/SKILL.md new file mode 100644 index 000000000..92477850c --- /dev/null +++ b/.claude/skills/superdesign/SKILL.md @@ -0,0 +1,33 @@ +--- +name: superdesign +description: Superdesign is a design agent, where it specialised in frontend UI/UX design; Use this skill before you implement any UI that require some design thinking +metadata: + author: superdesign + version: "0.0.1" +--- + +SuperDesign helps you (1) find design inspirations/styles and (2) generate/iterate design drafts on an infinite canvas. + +--- + +# Core scenarios (what this skill handles) + +1. **Help me design X** (feature/page/flow) +2. **Set design system** +3. **Help me improve design of X** + +# How it works +Fetch fresh guidelines below: +``` +https://raw.githubusercontent.com/superdesigndev/superdesign-skill/main/skills/superdesign/SUPERDESIGN.md +``` + +Action accordingly based on instruction in the guidelines + +# Superdesign CLI + +``` +npm install -g @superdesign/cli@latest +superdesign login +superdesign --help +``` diff --git a/README.md b/README.md index e8e725113..07d842ce3 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,14 @@ | QAyong | QAyong
ID:QAyong
B站:QAyong
| [2110491559@qq.com](mailto:2110491559@qq.com) | AI 短剧合规,资产确权 | | _K37ix. | _K37ix. | [2773843782@qq.com](mailto:2773843782@qq.com) | Making things that think | +## 交流与反馈 + +Issue 反馈、技术讨论和产品升级建议都可以在 QQ 群中沟通。群内还会不定期组织 AI 学习与培训交流会。 + +

+ 影策 QQ 交流群 +

+ ## 当前能力 - **自由画布**:项目、节点、连线、框选、缩放、小地图、撤销重做、导入导出和公开只读分享。 diff --git a/canvas-agent/README.md b/canvas-agent/README.md index 70feff44a..781da0d9f 100644 --- a/canvas-agent/README.md +++ b/canvas-agent/README.md @@ -93,14 +93,40 @@ default_tools_approval_mode = "approve" - `canvas_get_state` - `canvas_get_context` - `canvas_find_nodes` +- `canvas_get_node` +- `canvas_get_connection` +- `canvas_get_generation_tasks` - `canvas_get_resources` - `canvas_validate_ops` - `canvas_get_selection` - `canvas_export_snapshot` - `canvas_apply_ops` +- `canvas_create_workflow` - `canvas_create_text_node` - `canvas_create_image_prompt_flow` +`canvas_create_workflow` 是创建流水线/节点图的高阶工具,不要把工作流退化成批量文本节点。它会根据节点语义自动选择真实节点类型、按实际尺寸布局、创建默认顺序连线,并复核连接与重叠结果: + +| kind | 画布节点类型 | 用途 | +| --- | --- | --- | +| `character_cards` | `image` | 角色拆分图片卡片 | +| `character_three_view` | `image` | 角色正面/侧面/背面三视图 | +| `storyboard_video` | `video` | 分镜剧情视频 | +| `script` | `script` | 剧本或分镜文字 | + +媒体节点优先提供 `prompt`/`content`;对三个影视语义节点,即使模型漏填提示词,工具也会从工作流标题和节点语义生成最小可用创作提示词。已有画布素材必须先通过 `canvas_find_nodes` 或 `canvas_get_resources` 获取真实 node id,再放入 `referenceNodeIds`。 + +```json +{ + "title": "搞笑修仙小说流水线", + "nodes": [ + { "ref": "cards", "kind": "character_cards", "title": "角色拆分图片卡片" }, + { "ref": "views", "kind": "character_three_view", "title": "角色三视图", "referenceRefs": ["cards"] }, + { "ref": "video", "kind": "storyboard_video", "title": "分镜剧情视频", "referenceRefs": ["views"], "runGeneration": false } + ] +} +``` + `canvas_apply_ops` 示例: ```json @@ -117,7 +143,9 @@ default_tools_approval_mode = "approve" } ``` -推荐的 Agent 工作流是:先调用 `canvas_get_context` 读取语义化上下文和 `stateHash`;不知道节点 id 时调用 `canvas_find_nodes`;涉及图片、视频或音频参考时调用 `canvas_get_resources`;复杂写操作先调用 `canvas_validate_ops`,通过后再调用 `canvas_apply_ops`。这样 Agent 不需要猜测节点 id,也不会把 loading/error/占位媒体误判成可用资源。 +画布写工具返回的结果包含 `ok`、`message` 和 `data`。`data.snapshot` 是本地 Runtime 写入后的最新快照,`data.verification` 会列出 `createdNodeIds`、`removedNodeIds`、缺失节点/连线、前后状态摘要和生成任务观察结果。生成任务的 `message` 会明确区分“已提交/生成中,尚未完成”和“已完成且资源就绪”;不要只根据节点已经创建就向用户报告生成完成。 + +推荐的 Agent 工作流是:先调用 `canvas_get_context` 读取语义化上下文和 `stateHash`;不知道节点 id 时调用 `canvas_find_nodes`,已经知道 id 后用 `canvas_get_node` 或 `canvas_get_connection` 做精确复核;需要观察生成中的节点时调用 `canvas_get_generation_tasks`;涉及图片、视频或音频参考时调用 `canvas_get_resources`;复杂写操作先调用 `canvas_validate_ops`,通过后再调用 `canvas_apply_ops`。这样 Agent 不需要猜测节点 id,也不会把 loading/error/占位媒体误判成可用资源。 ## 侧边栏 Codex @@ -127,6 +155,8 @@ default_tools_approval_mode = "approve" 侧边栏上传或粘贴的图片会先发到本机 Canvas Agent,再由 Canvas Agent 临时写入本机文件并作为 app-server `localImage` 输入传给 Codex;前端会提示附件体积,单次请求体限制为 30MB。 +侧边栏 Composer 中显式提及的网页技能不会被拼接进用户 Prompt。网页把技能 bundle 传给本机 Runtime,Runtime 为当前 turn 临时生成受限的 `SKILL.md`,并通过 Codex app-server 的原生 `skill` 输入项加载;技能不会被复制进文本输入,也不会添加 `$skill-name` 伪标记,turn 完成后删除临时文件。未被用户提及的技能不会进入该 turn。 + ## Claude Code Claude Code Adapter 代码暂时保留,但当前网页侧边栏只开放 Codex。后续开放 Claude 入口时,Canvas Agent 会调用本机 `claude -p --output-format stream-json` 并把流式 JSON 事件转发到侧边栏。 diff --git a/canvas-agent/bun.lock b/canvas-agent/bun.lock index dafae53cf..e23f0daf8 100644 --- a/canvas-agent/bun.lock +++ b/canvas-agent/bun.lock @@ -5,8 +5,8 @@ "": { "name": "@ddcat666/open-ai-canvas-agent", "dependencies": { - "@modelcontextprotocol/sdk": "^1.29.0", - "@openai/codex": "^0.147.0", + "@modelcontextprotocol/sdk": "^1.30.0", + "@openai/codex": "^0.149.1", "express": "^5.2.1", "json-canonicalize": "2.0.0", "zod": "^3.25.76", @@ -77,21 +77,21 @@ "@hono/node-server": ["@hono/node-server@1.19.14", "https://registry.npmmirror.com/@hono/node-server/-/node-server-1.19.14.tgz", { "peerDependencies": { "hono": "^4" } }, "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw=="], - "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "https://registry.npmmirror.com/@modelcontextprotocol/sdk/-/sdk-1.29.0.tgz", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="], + "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.30.0", "https://registry.npmmirror.com/@modelcontextprotocol/sdk/-/sdk-1.30.0.tgz", { "dependencies": { "@hono/node-server": "^1.19.9 || ^2.0.5", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "zod": "^3.25 || ^4.0", "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA=="], - "@openai/codex": ["@openai/codex@0.147.0", "", { "optionalDependencies": { "@openai/codex-darwin-arm64": "npm:@openai/codex@0.147.0-darwin-arm64", "@openai/codex-darwin-x64": "npm:@openai/codex@0.147.0-darwin-x64", "@openai/codex-linux-arm64": "npm:@openai/codex@0.147.0-linux-arm64", "@openai/codex-linux-x64": "npm:@openai/codex@0.147.0-linux-x64", "@openai/codex-win32-arm64": "npm:@openai/codex@0.147.0-win32-arm64", "@openai/codex-win32-x64": "npm:@openai/codex@0.147.0-win32-x64" }, "bin": { "codex": "bin/codex.js" } }, "sha512-EQLEXecAG2ptxI7UpBMo2TR/ga5596/c/OsYF/0LoUDh5JANZ7IoGqlzBEWbuEVQ76JePIbtTW/ihCkp1a7Z3w=="], + "@openai/codex": ["@openai/codex@0.149.1", "", { "optionalDependencies": { "@openai/codex-darwin-arm64": "npm:@openai/codex@0.149.1-darwin-arm64", "@openai/codex-darwin-x64": "npm:@openai/codex@0.149.1-darwin-x64", "@openai/codex-linux-arm64": "npm:@openai/codex@0.149.1-linux-arm64", "@openai/codex-linux-x64": "npm:@openai/codex@0.149.1-linux-x64", "@openai/codex-win32-arm64": "npm:@openai/codex@0.149.1-win32-arm64", "@openai/codex-win32-x64": "npm:@openai/codex@0.149.1-win32-x64" }, "bin": { "codex": "bin/codex.js" } }, "sha512-6q5pbcpFbJbqOpkubSDBwXmktQ55aD8eUzGzBF1zASob2DjwhBKDSNGtdZKalfrNJUdTDTPDMmzCXEXs5tMBYA=="], - "@openai/codex-darwin-arm64": ["@openai/codex@0.147.0-darwin-arm64", "", { "os": "darwin", "cpu": "arm64" }, "sha512-BEUVkiOW7kLcRyrMLfAr/h9wF8sRVJyZDy6OHtVn6QGDXiv3BvAZVTY1Pu9xF7KdIdkYXbp4uayN0aDQQaAUJw=="], + "@openai/codex-darwin-arm64": ["@openai/codex@0.149.1-darwin-arm64", "", { "os": "darwin", "cpu": "arm64" }, "sha512-6X84kTCbnTgPIJ2EdcPsrvwS0Wxsqpa+bCswGmRf4BjhcQ5nPMnBC6yCAaCMj+vrbXQHj+L6sa9FaR4QkmA1qw=="], - "@openai/codex-darwin-x64": ["@openai/codex@0.147.0-darwin-x64", "", { "os": "darwin", "cpu": "x64" }, "sha512-Tb8McE5SvJIH0Vs5R6sq7u+quiC931yan2KOOl6km1OdZ82+Wi7eF5XrSFPs5CF7xCgoIK4Vs+byMbT5hN+ZUw=="], + "@openai/codex-darwin-x64": ["@openai/codex@0.149.1-darwin-x64", "", { "os": "darwin", "cpu": "x64" }, "sha512-MfLBQLfcElJL9tvj6y45qVHHMGSXCPnQOixuD3/Zq0g1BW/eFizkrGLdn48cFpc+l8cK+gt5nYG5pQYwVs6g4A=="], - "@openai/codex-linux-arm64": ["@openai/codex@0.147.0-linux-arm64", "", { "os": "linux", "cpu": "arm64" }, "sha512-SLC1JXw2TYfr/c3HhrJubyyLelq7vTOLWVmiThFA+z0+WgzCPmaseJ/kzDD3Gge/TO7fCnnj7UcPmC0d2c8XAg=="], + "@openai/codex-linux-arm64": ["@openai/codex@0.149.1-linux-arm64", "", { "os": "linux", "cpu": "arm64" }, "sha512-OqxUfZ1TVvHd18zHPKK/8ZRlpk8Vy11mg5CMHaLxNWldTbwVImDKtSLWT+m8m4NM5Sz4PbjtZMrVT/RfpBW/mQ=="], - "@openai/codex-linux-x64": ["@openai/codex@0.147.0-linux-x64", "", { "os": "linux", "cpu": "x64" }, "sha512-0W9MBxPpWW0cSkNqrTDN2jR7rzzT7oNMhQY5446lT2Lw5cz5yhDTck4Va9rjkQEm+HlFzP/dmEMSZbXfJsINmw=="], + "@openai/codex-linux-x64": ["@openai/codex@0.149.1-linux-x64", "", { "os": "linux", "cpu": "x64" }, "sha512-Of5fGYgr7tAMsyj6vhXb4/RM/UoA3Zq8BLegUBDC09UNy1XTLGYP/2XD+UX8z3qh0NDwxYdCjFIWdDNijKZggQ=="], - "@openai/codex-win32-arm64": ["@openai/codex@0.147.0-win32-arm64", "", { "os": "win32", "cpu": "arm64" }, "sha512-e2ZstJ8zT8Rm1nvR7CUVO+Gr3cTChE41+VfOzGhynzDXEoW0wfbjUQbc2bWbh1arG94LMm4y3dqBtUIbSrfeGA=="], + "@openai/codex-win32-arm64": ["@openai/codex@0.149.1-win32-arm64", "", { "os": "win32", "cpu": "arm64" }, "sha512-5K0DmOKGK9Bos627p8sK8ATHjovPK0sDyT6h9Cb+4v+5CW5SGw1HLgjGxoLfJ8g3cg6mtg/pRCXXo2L/j71UVA=="], - "@openai/codex-win32-x64": ["@openai/codex@0.147.0-win32-x64", "", { "os": "win32", "cpu": "x64" }, "sha512-oT7Ss5fAPf2fiWE9QNURqZcQGAAawSVxmIUdgPzckq4KFZAM+pRz9JbM4Rr498CjtbNgTOjWvDJ+DXvIBSfOPA=="], + "@openai/codex-win32-x64": ["@openai/codex@0.149.1-win32-x64", "", { "os": "win32", "cpu": "x64" }, "sha512-G3QXGAg7nyyhqOeooAMUekBCeHd8a1QByhKcVAFyzNBaI06t6Ft7nsF+1SzFS0spuIdU4YyMi5YD26ukADBQUQ=="], "@types/body-parser": ["@types/body-parser@1.19.6", "https://registry.npmmirror.com/@types/body-parser/-/body-parser-1.19.6.tgz", { "dependencies": { "@types/connect": "*", "@types/node": "*" } }, "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g=="], diff --git a/canvas-agent/package.json b/canvas-agent/package.json index 8f1ddb090..537916f82 100644 --- a/canvas-agent/package.json +++ b/canvas-agent/package.json @@ -19,8 +19,8 @@ "prepack": "npm run build" }, "dependencies": { - "@modelcontextprotocol/sdk": "^1.29.0", - "@openai/codex": "^0.147.0", + "@modelcontextprotocol/sdk": "^1.30.0", + "@openai/codex": "^0.149.1", "express": "^5.2.1", "json-canonicalize": "2.0.0", "zod": "^3.25.76" diff --git a/canvas-agent/src/agents.ts b/canvas-agent/src/agents.ts index 41f19e5c8..e19e8f4b5 100644 --- a/canvas-agent/src/agents.ts +++ b/canvas-agent/src/agents.ts @@ -13,7 +13,9 @@ import type { AgentAttachment, AgentEmit } from "./types.js"; type Json = Record; type AgentEvent = Json & { type: string; usage?: unknown }; type PendingRequest = { resolve: (value: unknown) => void; reject: (error: Error) => void }; -type CodexRunOptions = { threadId?: string; cwd?: string; onThreadId?: (threadId: string) => void }; +export type AgentSkillReference = { skillId?: string; name: string; description?: string; instruction?: string }; +export type CodexSkillInput = { type: "skill"; name: string; path: string }; +type CodexRunOptions = { threadId?: string; cwd?: string; skills?: AgentSkillReference[]; onThreadId?: (threadId: string) => void }; type AgentHistoryMessage = { id: string; role: "user" | "assistant" | "tool" | "error"; title?: string; text: string; detail?: unknown; streamId?: string }; let codexQueue: Promise = Promise.resolve(); @@ -35,16 +37,20 @@ export async function runCodexTurn(prompt: string, emit: AgentEmit, attachments: async function runCodexTurnNow(prompt: string, emit: AgentEmit, attachments: AgentAttachment[], options: CodexRunOptions) { let files: string[] = []; + let skillDirectories: string[] = []; try { files = await writeAttachmentFiles(attachments); + const preparedSkills = await writeSkillFiles(options.skills || []); + skillDirectories = preparedSkills.directories; codexApp ||= await CodexAppClient.start(emit); const threadId = await ensureCodexThread(codexApp, options); if (threadId !== options.threadId) options.onThreadId?.(threadId); - await codexApp.startTurn(threadId, prompt, files); + await codexApp.startTurn(threadId, prompt, files, preparedSkills.inputs); } catch (error) { emit("agent_error", { message: errorMessage(error) }); } finally { await Promise.all(files.map((file) => fs.unlink(file).catch(() => undefined))); + await Promise.all(skillDirectories.map((directory) => fs.rm(directory, { recursive: true, force: true }).catch(() => undefined))); } } @@ -162,8 +168,8 @@ class CodexAppClient { return this.request("thread/archive", { threadId }); } - async startTurn(threadId: string, prompt: string, images: string[]) { - const result = await this.request("turn/start", { threadId, input: codexInput(prompt, images), approvalPolicy: "never" }); + async startTurn(threadId: string, prompt: string, images: string[], skills: CodexSkillInput[] = []) { + const result = await this.request("turn/start", { threadId, input: codexInput(prompt, images, skills), approvalPolicy: "never" }); const turnId = String(field(field(result, "turn"), "id") || ""); if (!turnId) throw new Error("Codex app-server 没有返回 turn id"); const completed = this.completedTurns.get(turnId); @@ -289,8 +295,16 @@ export function codexConfig(configDir = CONFIG_DIR) { }; } -function codexInput(prompt: string, images: string[]) { - return [{ type: "text", text: prompt, text_elements: [] }, ...images.map((file) => ({ type: "localImage", path: file }))]; +export function codexInput(prompt: string, images: string[], skills: CodexSkillInput[]) { + // Skill inputs are first-class app-server UserInput items. Do not add a + // `$skill` marker or copy SKILL.md into the text item: doing either turns + // native skill selection back into prompt expansion and makes the skill + // visible as ordinary user text in history. + return [ + { type: "text", text: prompt, text_elements: [] }, + ...images.map((file) => ({ type: "localImage", path: file })), + ...skills, + ]; } function normalizeCodexNotification(method: string, params: Json): AgentEvent | null { @@ -421,6 +435,9 @@ function toolName(name: string) { if (name === "canvas_get_state") return "读取画布"; if (name === "canvas_get_context") return "读取画布上下文"; if (name === "canvas_find_nodes") return "检索画布节点"; + if (name === "canvas_get_node") return "读取画布节点"; + if (name === "canvas_get_connection") return "读取画布连线"; + if (name === "canvas_get_generation_tasks") return "读取生成任务"; if (name === "canvas_get_resources") return "读取画布资源"; if (name === "canvas_validate_ops") return "校验画布操作"; if (name === "canvas_get_selection") return "读取选区"; @@ -448,6 +465,44 @@ async function writeAttachmentFile(item: AgentAttachment) { return file; } +export async function writeSkillFiles(skills: AgentSkillReference[]) { + const directories: string[] = []; + const inputs: CodexSkillInput[] = []; + const usedNames = new Set(); + try { + for (const skill of skills.slice(0, 8)) { + const instruction = String(skill.instruction || "").trim().slice(0, 24_000); + if (!instruction) continue; + const baseName = `canvas-${safeSkillSegment(skill.skillId || skill.name)}`; + const name = uniqueSkillName(baseName, usedNames); + usedNames.add(name); + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "infinite-canvas-skill-")); + directories.push(directory); + const file = path.join(directory, "SKILL.md"); + const description = String(skill.description || skill.name).trim().slice(0, 500).replace(/[\r\n]+/g, " "); + const body = [`---`, `name: ${name}`, `description: ${JSON.stringify(description)}`, `---`, ``, `# ${skill.name}`, ``, instruction, ``].join("\n"); + await fs.writeFile(file, body, "utf8"); + inputs.push({ type: "skill", name, path: file }); + } + return { directories, inputs }; + } catch (error) { + await Promise.all(directories.map((directory) => fs.rm(directory, { recursive: true, force: true }).catch(() => undefined))); + throw error; + } +} + +function uniqueSkillName(baseName: string, usedNames: Set) { + if (!usedNames.has(baseName)) return baseName; + let suffix = 2; + while (usedNames.has(`${baseName}-${suffix}`)) suffix += 1; + return `${baseName}-${suffix}`; +} + +function safeSkillSegment(value: string) { + const normalized = value.toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48); + return normalized || "skill"; +} + function imageExt(type = "") { if (type.includes("png")) return "png"; if (type.includes("webp")) return "webp"; diff --git a/canvas-agent/src/canvas-context.ts b/canvas-agent/src/canvas-context.ts index db4ccb262..2ccfcdc0b 100644 --- a/canvas-agent/src/canvas-context.ts +++ b/canvas-agent/src/canvas-context.ts @@ -19,8 +19,24 @@ export type CanvasResourceSummary = { isReady: boolean; }; +export type CanvasGenerationTaskSummary = { + taskId: string; + nodeId: string; + nodeTitle: string; + mode?: string; + nodeStatus: string; + status: string; + progress?: number; + stage?: string; + provider?: string; + errorCode?: string; + officialStatus?: string; + resourceReady: boolean; +}; + export type CanvasContext = { schemaVersion: 1; + revision: number; stateHash: string; canvas: { projectId?: string; @@ -67,6 +83,7 @@ export function buildCanvasContext(state: CanvasSnapshot | null): CanvasContext return { schemaVersion: 1, + revision: state.revision ?? 0, stateHash: hashState(state), canvas: { projectId: state.projectId, @@ -106,6 +123,66 @@ export function findCanvasNodes(state: CanvasSnapshot | null, input: { query?: s return { query: input.query || "", total: nodes.length, truncated: nodes.length > limit, nodes: nodes.slice(0, limit).map(compactContextNode) }; } +export function getCanvasNode(state: CanvasSnapshot | null, input: { id: string }) { + if (!state) throw new Error("当前没有已连接画布"); + const node = (state.nodes || []).find((candidate) => candidate.id === input.id); + if (!node) return { found: false, id: input.id, node: null, connections: [] }; + const nodeById = new Map((state.nodes || []).map((candidate) => [candidate.id, candidate])); + const connections = (state.connections || []) + .filter((connection) => connection.fromNodeId === node.id || connection.toNodeId === node.id) + .map((connection) => connectionSummary(connection, nodeById)); + return { + found: true, + id: node.id, + node: compactContextNode(node), + resource: resourceSummary(node), + connections, + }; +} + +export function getCanvasConnection(state: CanvasSnapshot | null, input: { id: string }) { + if (!state) throw new Error("当前没有已连接画布"); + const connection = (state.connections || []).find((candidate) => candidate.id === input.id); + if (!connection) return { found: false, id: input.id, connection: null }; + const nodeById = new Map((state.nodes || []).map((candidate) => [candidate.id, candidate])); + return { + found: true, + id: connection.id, + connection: connectionSummary(connection, nodeById), + fromNode: nodeById.get(connection.fromNodeId) ? compactContextNode(nodeById.get(connection.fromNodeId)!) : null, + toNode: nodeById.get(connection.toNodeId) ? compactContextNode(nodeById.get(connection.toNodeId)!) : null, + }; +} + +export function getCanvasGenerationTasks(state: CanvasSnapshot | null, input: { status?: string; nodeIds?: string[]; limit?: number }) { + if (!state) throw new Error("当前没有已连接画布"); + const nodeIds = input.nodeIds?.length ? new Set(input.nodeIds) : null; + const tasks = (state.nodes || []).flatMap((node) => { + const metadata = node.metadata || {}; + const taskId = stringValue(metadata.taskId); + if (!taskId || (nodeIds && !nodeIds.has(node.id))) return []; + const status = stringValue(metadata.taskStatus) || stringValue(metadata.status) || "unknown"; + if (input.status && status !== input.status) return []; + const resource = resourceSummary(node); + return [{ + taskId, + nodeId: node.id, + nodeTitle: node.title || "未命名节点", + mode: stringValue(metadata.generationMode), + nodeStatus: stringValue(metadata.status) || "idle", + status, + progress: numberValue(metadata.taskProgress ?? metadata.progress), + stage: stringValue(metadata.taskStage), + provider: stringValue(metadata.taskProvider), + errorCode: stringValue(metadata.taskErrorCode || metadata.generationErrorCode), + officialStatus: stringValue(metadata.taskOfficialStatus), + resourceReady: Boolean(resource?.isReady), + }]; + }); + const limit = Math.min(Math.max(input.limit || 100, 1), 200); + return { total: tasks.length, truncated: tasks.length > limit, tasks: tasks.slice(0, limit) }; +} + export function getCanvasResources(state: CanvasSnapshot | null, input: { nodeIds?: string[]; status?: string; limit?: number }) { if (!state) throw new Error("当前没有已连接画布"); const nodeIds = input.nodeIds?.length ? new Set(input.nodeIds) : null; @@ -123,11 +200,23 @@ export function validateCanvasOps(state: CanvasSnapshot | null, ops: unknown[]) if (!state) throw new Error("当前没有已连接画布"); const nodes = state.nodes || []; const nodeIds = new Set(nodes.map((node) => node.id)); + const connectionKeys = new Set((state.connections || []).map(connectionKey)); + const liveConnections = new Map(); + (state.connections || []).forEach((connection) => { + liveConnections.set(connection.id, { key: connectionKey(connection), fromNodeId: connection.fromNodeId, toNodeId: connection.toNodeId }); + }); const issues: Array<{ index: number; severity: "error" | "warning"; message: string }> = []; const addedIds = new Set(); + const liveNodeIds = new Set(nodeIds); + const removeConnection = (id: string) => { + const connection = liveConnections.get(id); + if (!connection) return; + liveConnections.delete(id); + if (![...liveConnections.values()].some((item) => item.key === connection.key)) connectionKeys.delete(connection.key); + }; const requireNode = (index: number, id: unknown, label: string) => { if (typeof id !== "string" || !id) issues.push({ index, severity: "error", message: `${label} 缺少节点 id` }); - else if (!nodeIds.has(id) && !addedIds.has(id)) issues.push({ index, severity: "error", message: `${label}「${id}」不在当前画布状态中;先重新读取 canvas_get_context 或 canvas_find_nodes` }); + else if (!liveNodeIds.has(id)) issues.push({ index, severity: "error", message: `${label}「${id}」不在当前画布状态中;先重新读取 canvas_get_context 或 canvas_find_nodes` }); }; ops.forEach((raw, index) => { if (!raw || typeof raw !== "object" || Array.isArray(raw)) { @@ -137,32 +226,84 @@ export function validateCanvasOps(state: CanvasSnapshot | null, ops: unknown[]) const op = raw as Record; switch (op.type) { case "add_node": + if (op.nodeType !== undefined && !["image", "text", "script", "config", "video", "audio", "frame"].includes(String(op.nodeType))) issues.push({ index, severity: "error", message: `新增节点类型「${String(op.nodeType)}」不受支持` }); + validateFiniteNumber(issues, index, op.x, "x 坐标"); + validateFiniteNumber(issues, index, op.y, "y 坐标"); + validatePosition(issues, index, op.position, "position"); + validatePositiveNumber(issues, index, op.width, "节点宽度"); + validatePositiveNumber(issues, index, op.height, "节点高度"); if (typeof op.id === "string") { if (nodeIds.has(op.id) || addedIds.has(op.id)) issues.push({ index, severity: "error", message: `新增节点 id「${op.id}」重复` }); addedIds.add(op.id); + liveNodeIds.add(op.id); } break; - case "update_node": + case "update_node": { + requireNode(index, op.id, "更新目标"); + const targetNode = nodes.find((node) => node.id === op.id); + if (op.patch && typeof op.patch === "object" && !Array.isArray(op.patch)) { + const patch = op.patch as Record; + if ("id" in patch || "type" in patch) issues.push({ index, severity: "error", message: "不能通过 update_node 修改节点 id 或类型" }); + validateFiniteNumber(issues, index, patch.x, "节点 x 坐标"); + validateFiniteNumber(issues, index, patch.y, "节点 y 坐标"); + validatePosition(issues, index, patch.position, "节点 position"); + validatePositiveNumber(issues, index, patch.width, "节点宽度"); + validatePositiveNumber(issues, index, patch.height, "节点高度"); + } + const metadata = op.metadata && typeof op.metadata === "object" && !Array.isArray(op.metadata) ? op.metadata as Record : null; + const patchMetadata = op.patch && typeof op.patch === "object" && !Array.isArray(op.patch) && (op.patch as Record).metadata && typeof (op.patch as Record).metadata === "object" && !Array.isArray((op.patch as Record).metadata) ? (op.patch as Record).metadata as Record : null; + if (isMediaNodeType(targetNode?.type) && ((metadata && "status" in metadata) || (patchMetadata && "status" in patchMetadata))) issues.push({ index, severity: "error", message: "不能直接修改媒体节点 status;生成状态必须由任务结果回写" }); + break; + } case "run_generation": - requireNode(index, op.nodeId ?? op.id, op.type === "run_generation" ? "生成目标" : "更新目标"); + requireNode(index, op.nodeId, "生成目标"); + if (typeof op.mode === "string") { + const node = nodes.find((item) => item.id === op.nodeId); + if (node && op.mode !== node.type && !(op.mode === "text" && node.type === "script")) issues.push({ index, severity: "error", message: `生成模式「${op.mode}」与节点类型「${node.type}」不匹配` }); + } break; - case "delete_node": - if (Array.isArray(op.ids)) op.ids.forEach((id) => requireNode(index, id, "删除目标")); - else requireNode(index, op.id, "删除目标"); + case "delete_node": { + const ids = Array.isArray(op.ids) ? op.ids : (typeof op.id === "string" ? [op.id] : op.nodeType ? nodes.filter((node) => node.type === op.nodeType).map((node) => node.id) : []); + if (!ids.length) issues.push({ index, severity: "error", message: "删除节点必须提供 id、ids 或 nodeType" }); + const unique = new Set(ids); + if (unique.size !== ids.length) issues.push({ index, severity: "error", message: "删除节点 id 不能重复" }); + ids.forEach((id) => requireNode(index, id, "删除目标")); + ids.forEach((id) => liveNodeIds.delete(id)); + for (const [connectionId, connection] of liveConnections) if (ids.includes(connection.fromNodeId) || ids.includes(connection.toNodeId)) removeConnection(connectionId); break; - case "connect_nodes": + } + case "connect_nodes": { requireNode(index, op.fromNodeId, "连接起点"); requireNode(index, op.toNodeId, "连接终点"); if (op.fromNodeId === op.toNodeId) issues.push({ index, severity: "error", message: "不能连接节点自身" }); + if (typeof op.id === "string" && liveConnections.has(op.id)) issues.push({ index, severity: "error", message: `连线 id「${op.id}」重复` }); + const key = connectionKey(op as CanvasConnection); + if (connectionKeys.has(key)) issues.push({ index, severity: "error", message: "相同端点和 handle 的连线已存在" }); + connectionKeys.add(key); + liveConnections.set(typeof op.id === "string" ? op.id : `__anonymous_${index}`, { key, fromNodeId: String(op.fromNodeId), toNodeId: String(op.toNodeId) }); break; + } case "select_nodes": if (Array.isArray(op.ids)) op.ids.forEach((id) => requireNode(index, id, "选区节点")); break; case "delete_connections": + if (op.all && (op.id || (Array.isArray(op.ids) && op.ids.length))) issues.push({ index, severity: "error", message: "delete_connections 不能同时使用 all 和 id/ids" }); if (!op.all && !op.id && !(Array.isArray(op.ids) && op.ids.length)) issues.push({ index, severity: "error", message: "删除连线必须提供 id、ids 或 all=true" }); + const connectionTargets = Array.isArray(op.ids) ? op.ids : (typeof op.id === "string" ? [op.id] : []); + if (new Set(connectionTargets).size !== connectionTargets.length) issues.push({ index, severity: "error", message: "删除连线 id 不能重复" }); + connectionTargets.filter((id) => !liveConnections.has(id)).forEach((id) => issues.push({ index, severity: "error", message: `连线「${id}」不存在,请先读取 canvas_get_context` })); + if (op.all) [...liveConnections.keys()].forEach(removeConnection); + else connectionTargets.forEach(removeConnection); break; case "set_viewport": if (!op.viewport || typeof op.viewport !== "object") issues.push({ index, severity: "error", message: "视口参数无效" }); + else { + const viewport = op.viewport as Record; + validateFiniteNumber(issues, index, viewport.x, "视口 x"); + validateFiniteNumber(issues, index, viewport.y, "视口 y"); + validateFiniteNumber(issues, index, viewport.k, "视口缩放"); + if (typeof viewport.k === "number" && (viewport.k < 0.05 || viewport.k > 8)) issues.push({ index, severity: "error", message: "视口缩放 k 必须在 0.05 到 8 之间" }); + } break; default: issues.push({ index, severity: "error", message: `不支持的操作类型「${String(op.type)}」` }); @@ -172,6 +313,33 @@ export function validateCanvasOps(state: CanvasSnapshot | null, ops: unknown[]) return { ok: errors.length === 0, issues, operationCount: ops.length, currentStateHash: hashState(state) }; } +function validateFiniteNumber(issues: Array<{ index: number; severity: "error" | "warning"; message: string }>, index: number, value: unknown, label: string) { + if (value !== undefined && (typeof value !== "number" || !Number.isFinite(value))) issues.push({ index, severity: "error", message: `${label}必须是有限数字` }); +} + +function validatePositiveNumber(issues: Array<{ index: number; severity: "error" | "warning"; message: string }>, index: number, value: unknown, label: string) { + if (value !== undefined && (typeof value !== "number" || !Number.isFinite(value) || value <= 0)) issues.push({ index, severity: "error", message: `${label}必须是正数` }); +} + +function validatePosition(issues: Array<{ index: number; severity: "error" | "warning"; message: string }>, index: number, value: unknown, label: string) { + if (value === undefined) return; + if (!value || typeof value !== "object" || Array.isArray(value)) { + issues.push({ index, severity: "error", message: `${label}必须是包含 x/y 的对象` }); + return; + } + const position = value as Record; + validateFiniteNumber(issues, index, position.x, `${label}.x`); + validateFiniteNumber(issues, index, position.y, `${label}.y`); +} + +function connectionKey(connection: Pick) { + return [connection.fromNodeId, connection.toNodeId, connection.fromHandleId || "", connection.toHandleId || ""].join("\0"); +} + +function isMediaNodeType(type: CanvasNode["type"] | undefined) { + return type === "image" || type === "video" || type === "audio"; +} + export function hashState(state: CanvasSnapshot) { return crypto.createHash("sha256").update(stableStringify({ projectId: state.projectId, @@ -197,12 +365,19 @@ function compactContextNode(node: CanvasNode) { status: String(metadata.status || "idle"), content: preview(metadata.content, 240), prompt: preview(metadata.prompt || metadata.composerContent, 300), - generation: metadata.generationMode || metadata.workflowKind ? { + generation: metadata.generationMode || metadata.workflowKind || metadata.taskId ? { mode: metadata.generationMode, model: metadata.model, workflowKind: metadata.workflowKind, workflowTitle: metadata.workflowTitle, + taskId: metadata.taskId, + status: metadata.taskStatus || metadata.status, + progress: numberValue(metadata.taskProgress ?? metadata.progress), + stage: metadata.taskStage, + provider: metadata.taskProvider, + errorCode: metadata.taskErrorCode || metadata.generationErrorCode, } : undefined, + error: metadata.status === "error" ? preview(metadata.errorDetails || metadata.generationErrorCode, 360) : undefined, asset: metadata.assetId || metadata.characterAssetId ? { assetId: metadata.assetId || metadata.characterAssetId, versionId: metadata.characterVersionId, diff --git a/canvas-agent/src/canvas-session.ts b/canvas-agent/src/canvas-session.ts index 76c94e035..389c805ab 100644 --- a/canvas-agent/src/canvas-session.ts +++ b/canvas-agent/src/canvas-session.ts @@ -2,7 +2,7 @@ import crypto from "node:crypto"; import type { ServerResponse } from "node:http"; import { CANVAS_GENERATION_CONTINUATION_TIMEOUT_MS } from "./canvas-tool-timeouts.js"; -import { buildCanvasContext, findCanvasNodes, getCanvasResources, validateCanvasOps } from "./canvas-context.js"; +import { buildCanvasContext, findCanvasNodes, getCanvasConnection, getCanvasGenerationTasks, getCanvasNode, getCanvasResources, hashState, validateCanvasOps } from "./canvas-context.js"; import { type ToolName } from "./schemas.js"; import { compactCanvasState, compactNode, isToolName, nextCanvasX, parseToolInput } from "./tools.js"; import type { CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js"; @@ -52,7 +52,20 @@ export class CanvasSession { } updateState(body: unknown, clientId?: string) { - this.canvasState = { ...((body && typeof body === "object" && !Array.isArray(body) ? body : {}) as Record), clientId } as CanvasSnapshot; + const candidate = { ...((body && typeof body === "object" && !Array.isArray(body) ? body : {}) as Record), clientId } as CanvasSnapshot; + const incomingRevision = typeof candidate.revision === "number" && Number.isInteger(candidate.revision) && candidate.revision >= 0 ? candidate.revision : undefined; + const actualHash = hashState(candidate); + const suppliedHash = typeof (body as Record | null)?.stateHash === "string" ? String((body as Record).stateHash) : undefined; + if (suppliedHash && suppliedHash !== actualHash) return { accepted: false, revision: this.canvasState?.revision ?? 0, stateHash: this.canvasState ? hashState(this.canvasState) : actualHash, reason: "state_hash_mismatch" as const }; + const previousState = this.canvasState; + const currentRevision = previousState?.revision ?? 0; + if (previousState) { + if (incomingRevision !== undefined && incomingRevision < currentRevision) return { accepted: false, revision: currentRevision, stateHash: hashState(previousState), reason: "stale_revision" as const }; + if (incomingRevision === currentRevision && actualHash !== hashState(previousState)) return { accepted: false, revision: currentRevision, stateHash: hashState(previousState), reason: "revision_conflict" as const }; + } + const revision = incomingRevision ?? (this.canvasState ? currentRevision + 1 : 0); + this.canvasState = { ...candidate, revision }; + return { accepted: true, idempotent: Boolean(previousState && incomingRevision === currentRevision && actualHash === hashState(previousState)), revision, stateHash: actualHash }; } resolveResult(body: { requestId?: string; error?: string; result?: unknown }) { @@ -89,17 +102,24 @@ export class CanvasSession { if (!input.projectId) throw new Error("当前画布没有关联短剧项目"); return await this.requestCanvasTool(tool, input); } - const readTool = ["canvas_get_state", "canvas_get_context", "canvas_find_nodes", "canvas_get_resources", "canvas_validate_ops", "canvas_get_selection", "canvas_export_snapshot"].includes(tool); + const readTool = ["canvas_get_state", "canvas_get_context", "canvas_find_nodes", "canvas_get_node", "canvas_get_connection", "canvas_get_generation_tasks", "canvas_get_resources", "canvas_validate_ops", "canvas_get_selection", "canvas_export_snapshot"].includes(tool); if (readTool && (!this.clients.size || !this.canvasState)) throw new Error("当前没有已连接画布"); if (tool === "canvas_get_state" || tool === "canvas_export_snapshot") return compactCanvasState(this.canvasState); if (tool === "canvas_get_context") return buildCanvasContext(this.canvasState); if (tool === "canvas_find_nodes") return findCanvasNodes(this.canvasState, input as Parameters[1]); + if (tool === "canvas_get_node") return getCanvasNode(this.canvasState, input as Parameters[1]); + if (tool === "canvas_get_connection") return getCanvasConnection(this.canvasState, input as Parameters[1]); + if (tool === "canvas_get_generation_tasks") return getCanvasGenerationTasks(this.canvasState, input as Parameters[1]); if (tool === "canvas_get_resources") return getCanvasResources(this.canvasState, input as Parameters[1]); if (tool === "canvas_validate_ops") return validateCanvasOps(this.canvasState, (input as { ops: unknown[] }).ops); if (tool === "canvas_get_selection") { const ids = new Set(this.canvasState?.selectedNodeIds || []); return { nodes: (this.canvasState?.nodes || []).filter((node) => ids.has(node.id)).map(compactNode) }; } + if (tool === "canvas_create_workflow") { + input = { ops: workflowOps(input as Record, this.canvasState) }; + tool = "canvas_apply_ops"; + } if (tool === "canvas_create_node") { const data = input as { nodeType: CanvasNodeType; title?: string; x?: number; y?: number; width?: number; height?: number; metadata?: Record }; input = { ops: [{ type: "add_node", nodeType: data.nodeType, title: data.title, position: { x: data.x ?? nextCanvasX(this.canvasState), y: data.y ?? 0 }, width: data.width, height: data.height, metadata: data.metadata }] }; @@ -112,6 +132,8 @@ export class CanvasSession { } if (tool === "canvas_create_text_nodes") { const data = input as { items: Array<{ text: string; title?: string; x?: number; y?: number; width?: number; height?: number }>; x?: number; y?: number; gap?: number; direction?: "row" | "column" }; + const batchText = data.items.map((item) => `${item.title || ""} ${item.text || ""}`).join(" "); + if (/流水线|工作流|工作流图|管线|节点图|连线|pipeline|workflow/i.test(batchText)) throw new Error("检测到工作流意图,请使用 canvas_create_workflow 创建真实类型节点和连线"); const x = Number(data.x ?? nextCanvasX(this.canvasState)); const y = Number(data.y ?? 0); const gap = Number(data.gap ?? 40); @@ -181,8 +203,13 @@ export class CanvasSession { } if (tool !== "canvas_apply_ops") throw new Error(`未知工具:${tool}`); if (!this.clients.size) throw new Error("当前没有已连接画布"); + const currentContext = buildCanvasContext(this.canvasState); + const expectedRevision = typeof input.expectedRevision === "number" ? input.expectedRevision : undefined; const expectedStateHash = typeof input.expectedStateHash === "string" ? input.expectedStateHash : ""; - if (expectedStateHash && expectedStateHash !== buildCanvasContext(this.canvasState).stateHash) { + if (expectedRevision !== undefined && expectedRevision !== currentContext.revision) { + throw new Error(`画布 revision 已从 ${expectedRevision} 变为 ${currentContext.revision},请重新读取 canvas_get_context 后再执行写操作`); + } + if (expectedStateHash && expectedStateHash !== currentContext.stateHash) { throw new Error("画布状态已变化,请重新读取 canvas_get_context 后再执行写操作"); } const validation = validateCanvasOps(this.canvasState, (input as { ops: unknown[] }).ops); @@ -230,6 +257,116 @@ function sendEvent(res: ServerResponse, type: string, payload: unknown) { res.write(`event: ${type}\ndata: ${JSON.stringify(payload)}\n\n`); } +function workflowOps(input: Record, state: CanvasSnapshot | null) { + const nodes = Array.isArray(input.nodes) ? input.nodes as Array> : []; + if (!nodes.length) throw new Error("工作流至少需要一个节点"); + const refs = new Set(); + for (const node of nodes) { + const ref = String(node.ref || "").trim(); + const title = String(node.title || "").trim(); + const kind = String(node.kind || "text"); + const prompt = String(node.prompt || node.content || workflowPrompt(kind, title, input)).trim(); + if (!ref || !title) throw new Error("工作流节点必须包含 ref 和 title"); + if (refs.has(ref)) throw new Error(`工作流节点 ref「${ref}」重复`); + if (!["text", "script"].includes(workflowNodeType(kind)) && !prompt) throw new Error(`媒体工作流节点「${title}」缺少 prompt/content,不能创建空资源节点`); + refs.add(ref); + for (const nodeId of Array.isArray(node.referenceNodeIds) ? node.referenceNodeIds : []) { + if (!existingNodeId(state, String(nodeId))) throw new Error(`节点「${title}」引用的现有节点「${String(nodeId)}」不存在`); + } + } + const direction = input.direction === "vertical" ? "vertical" : "horizontal"; + const gap = Math.max(48, Number(input.gap || 120)); + const existing = state?.nodes || []; + const maxX = existing.reduce((max, node) => Math.max(max, node.position.x + node.width), 0); + const maxY = existing.reduce((max, node) => Math.max(max, node.position.y + node.height), 0); + const start = input.start && typeof input.start === "object" ? input.start as { x: number; y: number } : { x: existing.length ? maxX + 160 : 80, y: existing.length ? Math.max(80, maxY - 520) : 80 }; + const ids = new Map(nodes.map((node) => [String(node.ref), `agent-workflow-${slug(String(node.ref))}-${crypto.randomUUID().slice(0, 8)}`])); + const ops: Array> = []; + let cursor = { x: Number(start.x), y: Number(start.y) }; + for (const node of nodes) { + const kind = String(node.kind || "text"); + const type = workflowNodeType(kind); + const size = workflowNodeSize(type, kind, node.width, node.height); + const prompt = String(node.prompt || node.content || workflowPrompt(kind, String(node.title), input)); + const position = { ...cursor }; + const internalReferenceIds = Array.isArray(node.referenceRefs) ? node.referenceRefs.map((ref) => ids.get(String(ref))).filter(Boolean) : []; + const externalReferenceIds = Array.isArray(node.referenceNodeIds) ? node.referenceNodeIds.map(String) : []; + ops.push({ type: "add_node", id: ids.get(String(node.ref)), nodeType: type, title: String(node.title), position, width: size.width, height: size.height, metadata: { content: type === "text" ? String(node.content || prompt) : "", composerContent: prompt || undefined, prompt: prompt || undefined, workflowKind: workflowKind(kind), workflowTitle: input.title, workflowDescription: node.description || input.description, generationMode: type === "image" ? "image" : type === "video" ? "video" : type === "audio" ? "audio" : undefined, status: type === "text" || type === "script" ? "success" : "idle", referenceNodeIds: [...internalReferenceIds, ...externalReferenceIds].length ? [...internalReferenceIds, ...externalReferenceIds] : undefined } }); + cursor = direction === "vertical" ? { x: Number(start.x), y: cursor.y + size.height + gap } : { x: cursor.x + size.width + gap, y: Number(start.y) }; + } + const edges = Array.isArray(input.edges) && input.edges.length ? input.edges as Array> : nodes.slice(0, -1).map((node, index) => ({ from: node.ref, to: nodes[index + 1].ref })); + const keys = new Set(); + for (const edge of edges) { + const from = String(edge.from || ""); + const to = String(edge.to || ""); + if (!ids.has(from) || !ids.has(to)) throw new Error(`工作流连线引用不存在的节点:${from} → ${to}`); + const key = `${from}\0${to}`; + if (keys.has(key)) continue; + keys.add(key); + ops.push({ type: "connect_nodes", fromNodeId: ids.get(from), toNodeId: ids.get(to) }); + } + for (const node of nodes) for (const ref of Array.isArray(node.referenceRefs) ? node.referenceRefs : []) { + const from = String(ref); + const to = String(node.ref); + if (!ids.has(from)) throw new Error(`节点「${to}」引用了不存在的节点「${from}」`); + const key = `${from}\0${to}`; + if (keys.has(key)) continue; + keys.add(key); + ops.push({ type: "connect_nodes", fromNodeId: ids.get(from), toNodeId: ids.get(to) }); + } + for (const node of nodes) for (const ref of Array.isArray(node.referenceNodeIds) ? node.referenceNodeIds : []) { + const from = String(ref); + const to = String(node.ref); + const key = `${from}\0${to}`; + if (keys.has(key)) continue; + keys.add(key); + ops.push({ type: "connect_nodes", fromNodeId: from, toNodeId: ids.get(to) }); + } + ops.push({ type: "select_nodes", ids: nodes.map((node) => ids.get(String(node.ref))) }); + if (input.autoRun === true || nodes.some((node) => node.runGeneration === true)) for (const node of nodes) { + const type = workflowNodeType(String(node.kind || "text")); + if (!["image", "video", "audio"].includes(type) || (input.autoRun !== true && node.runGeneration !== true)) continue; + ops.push({ type: "run_generation", nodeId: ids.get(String(node.ref)), mode: type, prompt: node.prompt || node.content || workflowPrompt(String(node.kind || "text"), String(node.title), input) }); + } + return ops; +} + +function existingNodeId(state: CanvasSnapshot | null, id: string) { + return Boolean(state?.nodes?.some((node) => node.id === id)); +} + +function workflowNodeType(kind: string) { + if (kind === "script") return "script"; + if (["image", "character_cards", "character_three_view"].includes(kind)) return "image"; + if (["video", "storyboard_video"].includes(kind)) return "video"; + if (kind === "audio") return "audio"; + return "text"; +} + +function workflowNodeSize(type: string, kind: string, width: unknown, height: unknown) { + const defaults = type === "image" ? { width: 560, height: 380 } : type === "video" ? { width: 640, height: 360 } : type === "script" ? { width: 920, height: 360 } : type === "audio" ? { width: 340, height: 160 } : { width: 420, height: 240 }; + return { width: typeof width === "number" && width > 0 ? width : defaults.width, height: typeof height === "number" && height > 0 ? height : defaults.height }; +} + +function workflowPrompt(kind: string, title: string, input: Record) { + const workflowTitle = String(input.title || input.description || "当前创作项目").trim(); + if (kind === "character_cards") return `请基于「${workflowTitle}」拆分主要角色,并为每个角色生成可用于后续创作的角色图片卡片:外观、服饰、身份、性格和视觉辨识点。`; + if (kind === "character_three_view") return `请基于上游角色卡片生成「${title}」:同一角色的正面、侧面、背面三视图,保持服饰、发型、道具和比例一致。`; + if (kind === "storyboard_video") return `请基于上游角色三视图,为「${workflowTitle}」制作分镜剧情视频方案:包含镜头顺序、景别、动作、节奏和画面连续性。`; + return ""; +} + +function workflowKind(kind: string) { + if (["character_cards", "character_three_view"].includes(kind)) return "character"; + if (kind === "storyboard_video") return "storyboard"; + if (kind === "script") return "script"; + return "free"; +} + +function slug(value: string) { + return value.toLowerCase().replace(/[^a-z0-9\u4e00-\u9fff]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40) || "node"; +} + function textNodeOp(input: { id?: string; text?: string; title?: string; width?: number; height?: number }, x: number, y: number) { return { type: "add_node", id: input.id, nodeType: "text", title: input.title, position: { x, y }, width: input.width, height: input.height, metadata: { content: input.text || "", status: "success", fontSize: 14 } }; } diff --git a/canvas-agent/src/config.ts b/canvas-agent/src/config.ts index da3db00e6..785a2c488 100644 --- a/canvas-agent/src/config.ts +++ b/canvas-agent/src/config.ts @@ -15,7 +15,7 @@ export const AGENT_PROMPT = `你是影策的画布执行 Agent,不是只会生 【上下文协议】 - 涉及“这个/当前/已有/选中的”对象时,先 canvas_get_context;用户明确指向选中对象时再补 canvas_get_selection。 -- 不要从记忆或用户描述猜节点 id。需要找节点时用 canvas_find_nodes;需要判断媒体能否作为参考时用 canvas_get_resources。 +- 不要从记忆或用户描述猜节点 id。需要找节点时用 canvas_find_nodes;已经知道真实 id 时用 canvas_get_node 或 canvas_get_connection 做精确复核;需要观察生成进度时用 canvas_get_generation_tasks;需要判断媒体能否作为参考时用 canvas_get_resources。 - canvas_get_context 返回 stateHash、语义化节点、连接关系和资源就绪状态。资源 ready=false、status=loading/error 或只有占位 metadata 时,必须明确说明,不要把它当成可用素材。 【执行协议】 @@ -23,7 +23,8 @@ export const AGENT_PROMPT = `你是影策的画布执行 Agent,不是只会生 - 写操作只使用当前上下文中真实存在的 id;新增节点要避免重叠,优先沿现有内容的右侧或下方网格布局。 - 操作完成后检查工具返回的真实结果;如果没有改变、部分失败或生成仍在进行,必须如实报告,不要说“已完成”。 - 删除、覆盖、批量移动、触发生成属于高影响操作,先给出简短计划并等待网页侧确认;不要用模拟鼠标点击绕过确认。 -- 优先使用语义化工具(canvas_create_text_node、canvas_generate_*、canvas_update_node_text 等),只有确实需要批量事务时才使用 canvas_apply_ops。 +- 流水线、工作流、管线、节点图或用户明确要求连线时,必须使用 canvas_create_workflow:将业务阶段拆成真实的文本/脚本/图片/视频/音频节点;character_cards 表示角色拆分图片卡片,character_three_view 表示角色三视图,storyboard_video 表示分镜剧情视频。媒体节点必须有真实 prompt/content;涉及已有素材时先 canvas_find_nodes/canvas_get_resources,再使用返回的真实 node id 填入 referenceNodeIds。工具会按实际尺寸布局并建立 edges/referenceRefs/referenceNodeIds 连线,禁止把工作流退化成批量空文本节点。 +- 优先使用语义化工具(canvas_create_workflow、canvas_create_text_node、canvas_generate_*、canvas_update_node_text 等),只有确实需要批量事务时才使用 canvas_apply_ops。 【资源与生成】 - 生成前先检查已有提示词、参考节点、资产引用和就绪状态;有合适资源就复用真实 node id,不要重复上传或创建孤立副本。 diff --git a/canvas-agent/src/modules/canvas-agent-http.ts b/canvas-agent/src/modules/canvas-agent-http.ts index 6bab6ff22..a20f203ae 100644 --- a/canvas-agent/src/modules/canvas-agent-http.ts +++ b/canvas-agent/src/modules/canvas-agent-http.ts @@ -40,8 +40,16 @@ export function createCanvasAgentHttpModule( ); }, { queryKeys: ["clientId"], lastEventId: true }), canvasRoute("POST", "/canvas/state", (req, res) => { - session.updateState(jsonBody(req), queryValue(req, "clientId") || undefined); - res.json({ ok: true }); + const result = session.updateState(jsonBody(req), queryValue(req, "clientId") || undefined); + if (!result) { + res.json({ ok: true }); + return; + } + if (result && !result.accepted) { + res.status(409).json({ ok: false, ...result }); + return; + } + res.json({ ok: true, ...result }); }, { queryKeys: ["clientId"] }), canvasRoute("POST", "/canvas/result", (req, res) => { session.resolveResult(jsonBody(req) as { requestId?: string; error?: string; result?: unknown }); @@ -112,6 +120,7 @@ export function createCanvasAgentHttpModule( const attachments = Array.isArray(body.attachments) ? body.attachments as AgentAttachment[] : []; + const skills = parseAgentSkills(body.skills); const workspace = ensureCanvasWorkspace(config, String(body.canvasId || "")); let threadId = String(body.threadId || workspace.activeThreadId || ""); void (async () => { @@ -128,6 +137,7 @@ export function createCanvasAgentHttpModule( emit, attachments, { + skills, threadId, cwd: workspace.workspacePath, onThreadId: (nextThreadId) => updateCanvasWorkspace( @@ -204,6 +214,23 @@ function jsonRecord(req: Request) { return value as Record; } +export function parseAgentSkills(value: unknown) { + if (!Array.isArray(value)) return []; + return value.slice(0, 8).flatMap((item) => { + if (!item || typeof item !== "object" || Array.isArray(item)) return []; + const input = item as Record; + const name = typeof input.name === "string" ? input.name.trim().slice(0, 120) : ""; + const instruction = typeof input.instruction === "string" ? input.instruction.trim().slice(0, 24_000) : ""; + if (!name || !instruction) return []; + return [{ + ...(typeof input.skillId === "string" ? { skillId: input.skillId.trim().slice(0, 120) } : {}), + name, + ...(typeof input.description === "string" ? { description: input.description.trim().slice(0, 500) } : {}), + instruction, + }]; + }); +} + function queryValue(req: Request, key: string) { const value = req.query[key]; return Array.isArray(value) ? String(value[0] ?? "") : String(value ?? ""); diff --git a/canvas-agent/src/schemas.ts b/canvas-agent/src/schemas.ts index c239fd296..b3100b85c 100644 --- a/canvas-agent/src/schemas.ts +++ b/canvas-agent/src/schemas.ts @@ -13,11 +13,15 @@ export const toolNames = [ "canvas_get_state", "canvas_get_context", "canvas_find_nodes", + "canvas_get_node", + "canvas_get_connection", + "canvas_get_generation_tasks", "canvas_get_resources", "canvas_validate_ops", "canvas_get_selection", "canvas_export_snapshot", "canvas_apply_ops", + "canvas_create_workflow", "canvas_create_node", "canvas_create_text_node", "canvas_create_text_nodes", @@ -52,7 +56,7 @@ export type ToolName = (typeof toolNames)[number]; export const canvasOpSchema = z.discriminatedUnion("type", [ z.object({ type: z.literal("add_node"), nodeType: nodeTypeSchema.optional(), id: z.string().optional(), title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), position: positionSchema.optional(), metadata: recordSchema.optional() }).passthrough(), z.object({ type: z.literal("update_node"), id: z.string(), patch: recordSchema.optional(), metadata: recordSchema.optional() }).passthrough(), - z.object({ type: z.literal("delete_node"), id: z.string().optional(), ids: z.array(z.string()).optional() }).passthrough(), + z.object({ type: z.literal("delete_node"), id: z.string().optional(), ids: z.array(z.string()).optional(), nodeType: nodeTypeSchema.optional() }).passthrough(), z.object({ type: z.literal("delete_connections"), id: z.string().optional(), ids: z.array(z.string()).optional(), all: z.boolean().optional() }).passthrough(), z.object({ type: z.literal("connect_nodes"), id: z.string().optional(), fromNodeId: z.string(), toNodeId: z.string(), fromHandleId: z.string().optional(), toHandleId: z.string().optional() }).passthrough(), z.object({ type: z.literal("set_viewport"), viewport: viewportSchema }).passthrough(), @@ -69,6 +73,30 @@ const textNodeSchema = z.object({ height: z.number().optional(), }); +const workflowNodeSchema = z.object({ + ref: z.string().min(1), + kind: z.enum(["text", "script", "image", "video", "audio", "character_cards", "character_three_view", "storyboard_video"]), + title: z.string().min(1), + content: z.string().optional(), + prompt: z.string().optional(), + description: z.string().optional(), + referenceRefs: z.array(z.string()).optional(), + referenceNodeIds: z.array(z.string()).optional(), + runGeneration: z.boolean().optional(), + width: z.number().positive().optional(), + height: z.number().positive().optional(), +}); +const workflowSchema = z.object({ + title: z.string().optional(), + description: z.string().optional(), + nodes: z.array(workflowNodeSchema).min(1), + edges: z.array(z.object({ from: z.string().min(1), to: z.string().min(1) })).optional(), + direction: z.enum(["horizontal", "vertical"]).optional(), + start: positionSchema.optional(), + gap: z.number().min(48).optional(), + autoRun: z.boolean().optional(), +}); + const generationOptionsSchema = z.object({ model: z.string().optional(), size: z.string().optional(), @@ -96,11 +124,15 @@ export const toolInputSchemas = { canvas_get_state: z.object({}).passthrough(), canvas_get_context: z.object({}).passthrough(), canvas_find_nodes: z.object({ query: z.string().optional(), ids: z.array(z.string()).optional(), types: z.array(z.string()).optional(), statuses: z.array(z.string()).optional(), resourceOnly: z.boolean().optional(), limit: z.number().int().min(1).max(200).optional() }), + canvas_get_node: z.object({ id: z.string().min(1) }), + canvas_get_connection: z.object({ id: z.string().min(1) }), + canvas_get_generation_tasks: z.object({ status: z.string().optional(), nodeIds: z.array(z.string()).optional(), limit: z.number().int().min(1).max(200).optional() }), canvas_get_resources: z.object({ nodeIds: z.array(z.string()).optional(), status: z.string().optional(), limit: z.number().int().min(1).max(300).optional() }), canvas_validate_ops: z.object({ ops: z.array(canvasOpSchema) }), canvas_get_selection: z.object({}).passthrough(), canvas_export_snapshot: z.object({}).passthrough(), - canvas_apply_ops: z.object({ ops: z.array(canvasOpSchema), expectedStateHash: z.string().min(1).optional() }), + canvas_apply_ops: z.object({ ops: z.array(canvasOpSchema), expectedRevision: z.number().int().nonnegative().optional(), expectedStateHash: z.string().min(1).optional() }), + canvas_create_workflow: workflowSchema, canvas_create_node: z.object({ nodeType: nodeTypeSchema, title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), metadata: recordSchema.optional() }), canvas_create_text_node: z.object({ text: z.string().optional(), x: z.number().optional(), y: z.number().optional(), title: z.string().optional(), width: z.number().optional(), height: z.number().optional() }), canvas_create_text_nodes: z.object({ items: z.array(textNodeSchema).min(1), x: z.number().optional(), y: z.number().optional(), gap: z.number().optional(), direction: z.enum(["row", "column"]).optional() }), @@ -135,11 +167,15 @@ export const toolDescriptions: Record = { canvas_get_state: "读取当前网页画布的节点、连线、选区和视口。", canvas_get_context: "读取面向 Agent 的语义化画布上下文:节点用途、连接关系、选区、资源引用、生成状态和状态哈希。优先于直接猜测节点 metadata。", canvas_find_nodes: "按标题、内容、提示词、类型、状态、资产或工作流检索真实节点,返回可用于后续写操作的节点 id。", + canvas_get_node: "按真实节点 id 精确读取单个节点的语义、完整安全 metadata、资源状态和关联连线;找不到时明确返回未找到。", + canvas_get_connection: "按真实连线 id 精确读取连线端点、端点节点摘要和 handle 信息;找不到时明确返回未找到。", + canvas_get_generation_tasks: "读取当前画布节点已经绑定的生成任务观察状态,包括 taskId、节点、任务状态、进度、阶段和错误;这是画布快照观察,不会主动轮询上游。", canvas_get_resources: "读取当前画布引用的媒体资源清单,包括资源/资产引用、类型、尺寸、大小、时长和就绪状态;不会返回媒体 URL。", canvas_validate_ops: "在真正写入前校验批量操作中的节点 id、连接关系和参数,避免对不存在节点或错误资源执行操作。", canvas_get_selection: "读取当前网页画布选中的节点。", canvas_export_snapshot: "导出当前画布快照,用于理解布局。", canvas_apply_ops: "批量操作当前网页画布。ops 支持 add_node、update_node、delete_node、delete_connections、connect_nodes、set_viewport、select_nodes、run_generation。", + canvas_create_workflow: "创建语义化工作流/流水线。character_cards 是角色拆分图片卡片,character_three_view 是角色三视图,storyboard_video 是分镜剧情视频。媒体节点必须有 prompt/content;已有素材使用先读取到的真实 node id 填入 referenceNodeIds。自动分配 id、根据真实节点尺寸布局、建立 edges/referenceRefs/referenceNodeIds 连线并返回可验证结果;不要用批量文本节点代替工作流。", canvas_create_node: "创建任意类型节点:text、script、image、video、audio、frame。适合创建脚本、媒体占位、背板或自定义 metadata 节点。", canvas_create_text_node: "在当前画布创建单个文本节点。", canvas_create_text_nodes: "批量创建文本节点,适合生成标题、段落、脚本、说明等内容块。", diff --git a/canvas-agent/src/types.ts b/canvas-agent/src/types.ts index 28490ab14..df11db099 100644 --- a/canvas-agent/src/types.ts +++ b/canvas-agent/src/types.ts @@ -3,6 +3,6 @@ export type Viewport = { x: number; y: number; k: number }; export type CanvasNodeType = "image" | "text" | "script" | "config" | "video" | "audio" | "frame"; export type CanvasNode = { id: string; type: CanvasNodeType; title?: string; position: Position; width: number; height: number; parentId?: string; metadata?: Record }; export type CanvasConnection = { id: string; fromNodeId: string; toNodeId: string; fromHandleId?: string; toHandleId?: string }; -export type CanvasSnapshot = { projectId?: string; domainProjectId?: string; title?: string; nodes?: CanvasNode[]; connections?: CanvasConnection[]; selectedNodeIds?: string[]; viewport?: Viewport; clientId?: string }; +export type CanvasSnapshot = { projectId?: string; domainProjectId?: string; title?: string; nodes?: CanvasNode[]; connections?: CanvasConnection[]; selectedNodeIds?: string[]; viewport?: Viewport; clientId?: string; revision?: number }; export type AgentEmit = (type: string, payload: unknown) => void; export type AgentAttachment = { name?: string; type?: string; dataUrl?: string }; diff --git a/canvas-agent/test/canvas-agent-module.test.ts b/canvas-agent/test/canvas-agent-module.test.ts index d8138e56d..c0bb9ec88 100644 --- a/canvas-agent/test/canvas-agent-module.test.ts +++ b/canvas-agent/test/canvas-agent-module.test.ts @@ -3,10 +3,12 @@ import { EventEmitter } from "node:events"; import http, { type Server } from "node:http"; import { test } from "node:test"; +import { buildCanvasContext } from "../src/canvas-context.js"; import { CanvasSession } from "../src/canvas-session.js"; import { createLocalRuntimeApp } from "../src/local-runtime.js"; import { LocalRuntimeSessionManager } from "../src/local-runtime-session.js"; import { createCanvasAgentHttpModule } from "../src/modules/canvas-agent-http.js"; +import { toolDescriptions, toolInputSchemas, toolNames } from "../src/schemas.js"; import type { LocalRuntimeConfig } from "../src/config.js"; const authority = "127.0.0.1:41743"; @@ -14,6 +16,38 @@ const endpoint = `http://${authority}`; const origin = "http://127.0.0.1:3001"; const token = "legacy-canvas-token-fixture"; +test("MCP manifest exposes the semantic canvas read tools with schemas and descriptions", () => { + const expected = [ + "canvas_get_context", + "canvas_find_nodes", + "canvas_get_node", + "canvas_get_connection", + "canvas_get_generation_tasks", + "canvas_get_resources", + "canvas_validate_ops", + ]; + for (const name of expected) { + assert.ok(toolNames.includes(name as typeof toolNames[number]), `${name} is missing from toolNames`); + assert.ok(toolDescriptions[name as keyof typeof toolDescriptions], `${name} is missing a description`); + assert.ok(toolInputSchemas[name as keyof typeof toolInputSchemas], `${name} is missing an input schema`); + } + assert.deepEqual( + toolNames.filter((name) => name.startsWith("canvas_")).slice(0, 10), + [ + "canvas_get_state", + "canvas_get_context", + "canvas_find_nodes", + "canvas_get_node", + "canvas_get_connection", + "canvas_get_generation_tasks", + "canvas_get_resources", + "canvas_validate_ops", + "canvas_get_selection", + "canvas_export_snapshot", + ], + ); +}); + test("Canvas module declares only Canvas scopes and constructs without CLI side effects", () => { const calls: string[] = []; const module = createCanvasAgentHttpModule(fixtureConfig(), sessionFixture(calls)); @@ -101,6 +135,23 @@ test("CanvasSession dispose closes streams and a replaced stream cannot clear th second.response.emit("close"); }); +test("CanvasSession exposes precise node and connection reads", async () => { + const session = new CanvasSession(); + const response = eventResponse(); + session.openEvents(new URL("http://127.0.0.1/events?clientId=precise-read"), response.response as never); + session.updateState({ + nodes: [ + { id: "node-a", type: "text", title: "A", position: { x: 0, y: 0 }, width: 320, height: 240 }, + { id: "node-b", type: "image", title: "B", position: { x: 400, y: 0 }, width: 320, height: 320, metadata: { status: "success", storageKey: "resource:b" } }, + ], + connections: [{ id: "connection-1", fromNodeId: "node-a", toNodeId: "node-b" }], + }, "precise-read"); + assert.equal((await session.callTool("canvas_get_node", { id: "node-b" }) as { found: boolean }).found, true); + assert.equal((await session.callTool("canvas_get_connection", { id: "connection-1" }) as { found: boolean }).found, true); + assert.equal((await session.callTool("canvas_get_node", { id: "missing" }) as { found: boolean }).found, false); + session.dispose(); +}); + test("CanvasSession closes only streams owned by a revoked Runtime session", async () => { const session = new CanvasSession(); const closeRuntimeSession = (session as CanvasSession & { @@ -159,6 +210,64 @@ test("Canvas generation tool continuation survives a browser stream reconnect un } }); +test("CanvasSession expands a workflow into semantic nodes, non-overlapping layout, real edges, and selective generation", async () => { + const session = new CanvasSession(); + const events = eventResponse(); + session.openEvents(new URL("http://127.0.0.1/events?clientId=agent-workflow"), events.response as never); + session.updateState({ + nodes: [{ id: "existing-character", type: "image", title: "角色原画", position: { x: 0, y: 0 }, width: 560, height: 380, metadata: { status: "success", storageKey: "resource:character" } }], + connections: [], + }, "agent-workflow"); + + try { + const pending = session.callTool("canvas_create_workflow", { + title: "搞笑修仙小说流水线", + nodes: [ + { ref: "cards", kind: "character_cards", title: "角色拆分图片卡片", referenceNodeIds: ["existing-character"] }, + { ref: "views", kind: "character_three_view", title: "角色三视图", prompt: "基于角色卡片生成正面、侧面、背面三视图", referenceRefs: ["cards"] }, + { ref: "storyboard", kind: "storyboard_video", title: "分镜剧情视频", prompt: "基于三视图制作分镜剧情视频", referenceRefs: ["views"], runGeneration: true }, + ], + }); + const call = latestToolCall(events.writes()); + const ops = (call.input as { ops: Array> }).ops; + const added = ops.filter((op) => op.type === "add_node"); + const edges = ops.filter((op) => op.type === "connect_nodes"); + const runs = ops.filter((op) => op.type === "run_generation"); + + assert.deepEqual(added.map((op) => op.nodeType), ["image", "image", "video"]); + assert.match(String((added[0]?.metadata as Record)?.prompt), /拆分主要角色/); + assert.equal(edges.length, 3, "two workflow edges plus one existing reference edge"); + assert.equal(runs.length, 1, "runGeneration only affects the explicitly requested node"); + assert.equal(runs[0]?.nodeId, added[2]?.id); + assert.ok(Number((added[1]?.position as { x: number }).x) > Number((added[0]?.position as { x: number }).x) + Number(added[0]?.width)); + assert.ok(Number((added[2]?.position as { x: number }).x) > Number((added[1]?.position as { x: number }).x) + Number(added[1]?.width)); + assert.ok(edges.some((op) => op.fromNodeId === "existing-character" && op.toNodeId === added[0]?.id)); + + session.resolveResult({ requestId: call.requestId, result: { accepted: true } }); + assert.deepEqual(await pending, { accepted: true }); + } finally { + session.dispose(); + } +}); + +test("CanvasSession rejects media workflow nodes without real creative content", async () => { + const session = new CanvasSession(); + const events = eventResponse(); + session.openEvents(new URL("http://127.0.0.1/events?clientId=agent-workflow-invalid"), events.response as never); + session.updateState({ nodes: [] }, "agent-workflow-invalid"); + try { + await assert.rejects( + session.callTool("canvas_create_workflow", { + nodes: [{ ref: "empty-image", kind: "image", title: "空图片节点" }], + }), + /缺少 prompt\/content/, + ); + assert.equal(events.writes().some((value) => value.includes("event: tool_call")), false); + } finally { + session.dispose(); + } +}); + test("Canvas Dreamina image generation preserves the shared product model and auto quality before run_generation", async () => { const session = new CanvasSession(); const events = eventResponse(); @@ -372,3 +481,56 @@ function listening(server: Server) { function close(server: Server) { return new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); } + +test("CanvasSession rejects stale state revisions and accepts idempotent retries", () => { + const session = new CanvasSession(); + const first = session.updateState({ projectId: "canvas-1", nodes: [], connections: [], viewport: { x: 0, y: 0, k: 1 } }, "fixture"); + assert.equal(first.accepted, true); + assert.equal(first.revision, 0); + + const idempotent = session.updateState({ projectId: "canvas-1", nodes: [], connections: [], viewport: { x: 0, y: 0, k: 1 }, revision: 0 }, "fixture"); + assert.equal(idempotent.accepted, true); + assert.equal(idempotent.idempotent, true); + + const conflict = session.updateState({ projectId: "canvas-1", nodes: [{ id: "n-1", type: "text", position: { x: 0, y: 0 }, width: 100, height: 100 }], connections: [], viewport: { x: 0, y: 0, k: 1 }, revision: 0 }, "fixture"); + assert.equal(conflict.accepted, false); + assert.equal(conflict.reason, "revision_conflict"); + + const next = session.updateState({ projectId: "canvas-1", nodes: [{ id: "n-1", type: "text", position: { x: 0, y: 0 }, width: 100, height: 100 }], connections: [], viewport: { x: 0, y: 0, k: 1 }, revision: 1 }, "fixture"); + assert.equal(next.accepted, true); + assert.equal(next.revision, 1); + + const stale = session.updateState({ projectId: "canvas-1", nodes: [], connections: [], viewport: { x: 0, y: 0, k: 1 }, revision: 0 }, "fixture"); + assert.equal(stale.accepted, false); + assert.equal(stale.reason, "stale_revision"); + session.dispose(); +}); + + +test("canvas_apply_ops enforces expected revision and state hash before dispatch", async () => { + const session = new CanvasSession(); + const events = eventResponse(); + session.openEvents(new URL("http://127.0.0.1/events?clientId=guarded-write"), events.response as never); + session.updateState({ nodes: [], connections: [], viewport: { x: 0, y: 0, k: 1 } }, "guarded-write"); + const context = buildCanvasContext({ nodes: [], connections: [], viewport: { x: 0, y: 0, k: 1 }, revision: 0 }); + try { + const accepted = session.callTool("canvas_apply_ops", { ops: [], expectedRevision: 0, expectedStateHash: context.stateHash }); + const call = latestToolCall(events.writes()); + session.resolveResult({ requestId: call.requestId, result: { accepted: true } }); + assert.deepEqual(await accepted, { accepted: true }); + + const writeCount = events.writes().length; + await assert.rejects( + session.callTool("canvas_apply_ops", { ops: [], expectedRevision: 1 }), + /revision.*重新读取 canvas_get_context/ + ); + assert.equal(events.writes().length, writeCount); + await assert.rejects( + session.callTool("canvas_apply_ops", { ops: [], expectedRevision: 0, expectedStateHash: "bad-hash" }), + /画布状态已变化.*重新读取 canvas_get_context/ + ); + assert.equal(events.writes().length, writeCount); + } finally { + session.dispose(); + } +}); diff --git a/canvas-agent/test/canvas-context.test.ts b/canvas-agent/test/canvas-context.test.ts index cf72ea557..800a0c859 100644 --- a/canvas-agent/test/canvas-context.test.ts +++ b/canvas-agent/test/canvas-context.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { buildCanvasContext, findCanvasNodes, getCanvasResources, validateCanvasOps } from "../src/canvas-context.js"; +import { buildCanvasContext, findCanvasNodes, getCanvasConnection, getCanvasGenerationTasks, getCanvasNode, getCanvasResources, validateCanvasOps } from "../src/canvas-context.js"; import type { CanvasSnapshot } from "../src/types.js"; const state: CanvasSnapshot = { @@ -12,7 +12,7 @@ const state: CanvasSnapshot = { nodes: [ { id: "prompt-1", type: "text", title: "提示词", position: { x: 0, y: 0 }, width: 340, height: 240, metadata: { content: "夜雨中的城市", status: "success" } }, { id: "ref-1", type: "image", title: "角色参考", position: { x: 420, y: 0 }, width: 320, height: 320, metadata: { status: "success", storageKey: "resource:abc", mimeType: "image/png", assetId: "asset-1", assetCategory: "character", naturalWidth: 1024, naturalHeight: 1024 } }, - { id: "loading-1", type: "video", title: "待生成视频", position: { x: 840, y: 0 }, width: 480, height: 270, metadata: { status: "loading" } }, + { id: "loading-1", type: "video", title: "待生成视频", position: { x: 840, y: 0 }, width: 480, height: 270, metadata: { status: "loading", taskId: "task-1", taskStatus: "running", taskProgress: 42, taskStage: "rendering", taskProvider: "backend" } }, ], connections: [{ id: "c-1", fromNodeId: "prompt-1", toNodeId: "ref-1" }], }; @@ -25,6 +25,9 @@ test("builds semantic canvas context without media URLs", () => { assert.equal(context.resources[0]?.isReady, true); assert.equal(context.resources[1]?.isReady, false); assert.equal((context.nodes[1] as { resource?: { ready?: boolean } }).resource?.ready, true); + assert.equal((context.nodes[2] as { generation?: { taskId?: string; progress?: number } }).generation?.taskId, "task-1"); + assert.equal((context.nodes[2] as { generation?: { taskId?: string; progress?: number; stage?: string } }).generation?.progress, 42); + assert.equal((context.nodes[2] as { generation?: { taskId?: string; progress?: number; stage?: string } }).generation?.stage, "rendering"); assert.equal("url" in context.nodes[1], false); assert.match(context.stateHash, /^[a-f0-9]{16}$/); }); @@ -34,6 +37,43 @@ test("finds real nodes and resources by semantic query", () => { assert.equal(getCanvasResources(state, { status: "loading" }).resources[0]?.nodeId, "loading-1"); }); +test("reads one node and its related connections without media URLs", () => { + const result = getCanvasNode(state, { id: "ref-1" }); + assert.equal(result.found, true); + assert.equal(result.node?.id, "ref-1"); + assert.equal(result.resource?.isReady, true); + assert.equal(result.connections[0]?.id, "c-1"); + assert.equal("url" in (result.node || {}), false); + assert.equal(getCanvasNode(state, { id: "missing" }).found, false); +}); + +test("reads one connection with endpoint summaries", () => { + const result = getCanvasConnection(state, { id: "c-1" }); + assert.equal(result.found, true); + assert.equal(result.connection?.fromTitle, "提示词"); + assert.equal(result.toNode?.id, "ref-1"); + assert.equal(getCanvasConnection(state, { id: "missing" }).found, false); +}); + +test("projects observable generation task state from canvas nodes", () => { + const result = getCanvasGenerationTasks(state, { status: "running" }); + assert.equal(result.total, 1); + assert.deepEqual(result.tasks[0], { + taskId: "task-1", + nodeId: "loading-1", + nodeTitle: "待生成视频", + mode: undefined, + nodeStatus: "loading", + status: "running", + progress: 42, + stage: "rendering", + provider: "backend", + errorCode: undefined, + officialStatus: undefined, + resourceReady: false, + }); +}); + test("rejects stale or unsafe operations before dispatch", () => { const result = validateCanvasOps(state, [ { type: "update_node", id: "missing" }, @@ -42,3 +82,45 @@ test("rejects stale or unsafe operations before dispatch", () => { assert.equal(result.ok, false); assert.equal(result.issues.length, 2); }); + +test("validates a batch transaction against its evolving node and connection state", () => { + const result = validateCanvasOps(state, [ + { type: "add_node", id: "new-1", nodeType: "image", position: { x: 10, y: 20 }, width: 320, height: 240 }, + { type: "connect_nodes", fromNodeId: "new-1", toNodeId: "ref-1", id: "new-connection" }, + { type: "delete_node", id: "new-1" }, + { type: "update_node", id: "new-1", patch: { title: "should fail" } }, + ]); + assert.equal(result.ok, false); + assert.match(result.issues.at(-1)?.message || "", /不在当前画布状态中/); +}); + +test("rejects duplicate, missing, and unsafe connection or viewport operations", () => { + const result = validateCanvasOps(state, [ + { type: "connect_nodes", fromNodeId: "prompt-1", toNodeId: "ref-1" }, + { type: "delete_connections", ids: ["missing-connection"] }, + { type: "set_viewport", viewport: { x: 0, y: 0, k: 99 } }, + { type: "add_node", nodeType: "image", width: -1, height: 0, position: { x: Number.NaN, y: 0 } }, + ]); + assert.equal(result.ok, false); + assert.ok(result.issues.some((issue) => issue.message.includes("相同端点"))); + assert.ok(result.issues.some((issue) => issue.message.includes("不存在"))); + assert.ok(result.issues.some((issue) => issue.message.includes("0.05 到 8"))); + assert.ok(result.issues.some((issue) => issue.message.includes("正数"))); +}); + + +test("evolving connection state allows delete then recreate in one atomic batch", () => { + const result = validateCanvasOps(state, [ + { type: "delete_connections", id: "c-1" }, + { type: "connect_nodes", id: "c-2", fromNodeId: "prompt-1", toNodeId: "ref-1" }, + ]); + assert.equal(result.ok, true); +}); + +test("rejects direct media status forgery as a write error", () => { + const result = validateCanvasOps(state, [ + { type: "update_node", id: "ref-1", patch: { metadata: { status: "success" } } }, + ]); + assert.equal(result.ok, false); + assert.equal(result.issues[0]?.severity, "error"); +}); diff --git a/canvas-agent/test/skills.test.ts b/canvas-agent/test/skills.test.ts new file mode 100644 index 000000000..119efced6 --- /dev/null +++ b/canvas-agent/test/skills.test.ts @@ -0,0 +1,57 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import test from "node:test"; + +import { codexInput, writeSkillFiles } from "../src/agents.js"; +import { parseAgentSkills } from "../src/modules/canvas-agent-http.js"; + +test("codexInput keeps skills as native UserInput items", () => { + const input = codexInput("请按技能执行", ["/tmp/reference.png"], [{ + type: "skill", + name: "canvas-shot-list", + path: "/tmp/skill/SKILL.md", + }]); + + assert.deepEqual(input, [ + { type: "text", text: "请按技能执行", text_elements: [] }, + { type: "localImage", path: "/tmp/reference.png" }, + { type: "skill", name: "canvas-shot-list", path: "/tmp/skill/SKILL.md" }, + ]); + assert.equal(JSON.stringify(input).includes("$canvas-shot-list"), false); +}); + +test("writeSkillFiles creates Codex-compatible SKILL.md inputs and unique names", async () => { + const prepared = await writeSkillFiles([ + { skillId: "same", name: "第一个技能", description: "first", instruction: "do first" }, + { skillId: "same", name: "第二个技能", description: "second", instruction: "do second" }, + { skillId: "empty", name: "空技能", instruction: " " }, + ]); + + try { + assert.equal(prepared.inputs.length, 2); + assert.deepEqual(prepared.inputs.map((item) => item.type), ["skill", "skill"]); + assert.deepEqual(prepared.inputs.map((item) => item.name), ["canvas-same", "canvas-same-2"]); + for (const item of prepared.inputs) { + assert.equal(item.path.endsWith("/SKILL.md"), true); + const body = await fs.readFile(item.path, "utf8"); + assert.match(body, /^---\nname: canvas-same(?:-2)?\ndescription: /); + assert.match(body, /\n---\n\n#/); + } + } finally { + await Promise.all(prepared.directories.map((directory) => fs.rm(directory, { recursive: true, force: true }))); + } +}); + +test("parseAgentSkills validates and bounds browser skill bundles", () => { + const input = [ + { skillId: "safe", name: " safe ", description: " desc ", instruction: " instruction " }, + { name: "missing instruction" }, + null, + { name: "too long", instruction: "x".repeat(25_000) }, + ]; + const parsed = parseAgentSkills(input); + + assert.equal(parsed.length, 2); + assert.deepEqual(parsed[0], { skillId: "safe", name: "safe", description: "desc", instruction: "instruction" }); + assert.equal(parsed[1].instruction.length, 24_000); +}); diff --git a/plugins/yingce/skills/canvas-context/SKILL.md b/plugins/yingce/skills/canvas-context/SKILL.md index a9041b1a6..afda05357 100644 --- a/plugins/yingce/skills/canvas-context/SKILL.md +++ b/plugins/yingce/skills/canvas-context/SKILL.md @@ -10,7 +10,20 @@ description: 理解影策当前画布的语义结构、选区、连接关系和 1. `canvas_get_context`:读取语义化节点、真实 id、连接关系、选区、资源清单和 `stateHash`。 2. 用户说“这个/选中内容”时,补 `canvas_get_selection`。 3. 不知道节点 id 时用 `canvas_find_nodes`,不要猜 id。 -4. 涉及图片、视频、音频参考时用 `canvas_get_resources`;只有 `ready=true` 且有持久化引用的资源才可作为可用素材。 -5. 看到 `loading`、`error`、缺少 `storageKey/resourceId` 的节点时,向用户说明它是未就绪或占位状态。 +4. 已经知道真实节点或连线 id 时,分别用 `canvas_get_node` / `canvas_get_connection` 精确读取,不要为确认一个对象反复传输整张画布。 +5. 需要知道画布内生成任务的绑定节点、`taskId`、状态、进度或阶段时,用 `canvas_get_generation_tasks`;它只观察画布快照,不等于主动轮询上游。 +6. 涉及图片、视频、音频参考时用 `canvas_get_resources`;只有 `ready=true` 且有持久化引用的资源才可作为可用素材。 +7. 看到 `loading`、`error`、缺少 `storageKey/resourceId` 的节点时,向用户说明它是未就绪或占位状态。 上下文只作为事实来源,不要把 `storageKey`、内部 id 或资源状态编造成媒体 URL。工具结果返回后,以结果为准继续下一步。 + +## 工作流与流水线 + +用户提出“流水线、工作流、节点图、管线、连线”时,优先使用 `canvas_create_workflow`,不要用 `canvas_create_text_nodes` 伪造流程。将阶段拆成真实语义节点: + +- `character_cards`:角色拆分图片卡片,实际类型为 `image` +- `character_three_view`:角色三视图,实际类型为 `image` +- `storyboard_video`:分镜剧情视频,实际类型为 `video` +- `script`:剧本或分镜文字,实际类型为 `script` + +媒体节点要有 `prompt` 或 `content`。已知画布素材时,先检索并将真实 node id 放入 `referenceNodeIds`;工作流内部依赖放入 `referenceRefs`。工具会自动布局、默认建立顺序连线并返回连接与重叠复核结果。只有结果中的 `verification` 确认连接实际存在时,才能向用户报告“已连线”。 diff --git a/web/package.json b/web/package.json index b19618375..ccbc178a1 100644 --- a/web/package.json +++ b/web/package.json @@ -7,7 +7,7 @@ "dev": "vite --host 0.0.0.0 --port 3000", "build": "tsc --noEmit && vite build", "typecheck": "tsc --noEmit", - "test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/agent-prompt-cache.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-image-batch-retry.test.ts test/canvas-leafer-viewport.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts && bun test test/local-dreamina-cross-runtime.test.ts", + "test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-image-batch-retry.test.ts test/canvas-leafer-viewport.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts && bun test test/local-dreamina-cross-runtime.test.ts", "start": "vite preview --host 0.0.0.0 --port 3000", "format": "prettier --write .", "format:check": "prettier --check ." diff --git a/web/src/components/canvas/canvas-agent-chat-ui.tsx b/web/src/components/canvas/canvas-agent-chat-ui.tsx index 1333b3a35..cc975a7f5 100644 --- a/web/src/components/canvas/canvas-agent-chat-ui.tsx +++ b/web/src/components/canvas/canvas-agent-chat-ui.tsx @@ -1,5 +1,6 @@ import { useEffect, useRef, useState, type ClipboardEvent as ReactClipboardEvent, type KeyboardEvent as ReactKeyboardEvent, type ReactNode } from "react"; import { Button, Tooltip } from "antd"; +import { motion, useReducedMotion } from "motion/react"; import { ArrowUp, CheckCircle2, CircleAlert, ImagePlus, LoaderCircle, Sparkles, UserRound, Wrench, X, XCircle } from "lucide-react"; import { canvasThemes } from "@/lib/canvas-theme"; @@ -22,12 +23,36 @@ export type CanvasAgentChatMessage = { attachments?: CanvasAgentChatAttachment[]; }; +export type CanvasAgentQuickAction = { label: string; prompt: string }; + +/** + * Turn the short numbered choices the Agent already emits into real UI actions. + * This deliberately stays conservative: only assistant messages with 1–4 + * numbered lines are eligible, and code blocks are ignored. + */ +export function extractCanvasAgentQuickActions(text: string): CanvasAgentQuickAction[] { + if (!text.trim() || text.includes("```")) return []; + const actions: CanvasAgentQuickAction[] = []; + const seen = new Set(); + for (const line of text.split(/\r?\n/u)) { + const match = /^\s*(?:[-*]\s*)?(\d{1,2})[.)、]\s*(.+?)\s*$/u.exec(line); + if (!match) continue; + const label = match[2].replace(/^[*_\s]+|[*_\s]+$/gu, "").trim(); + if (!label || label.length > 96 || seen.has(label)) continue; + seen.add(label); + actions.push({ label, prompt: label }); + if (actions.length >= 4) break; + } + return actions; +} + const WORKING_TEXT = "正在推演..."; -export function AgentChatMessage({ item, theme, user, isStreaming = false, onRejectTool, onApproveTool }: { item: CanvasAgentChatMessage; theme: (typeof canvasThemes)[keyof typeof canvasThemes]; user: LocalUser | null; isStreaming?: boolean; onRejectTool?: (id: string) => void; onApproveTool?: (id: string) => void }) { +export function AgentChatMessage({ item, theme, user, isStreaming = false, onRejectTool, onApproveTool, onQuickAction }: { item: CanvasAgentChatMessage; theme: (typeof canvasThemes)[keyof typeof canvasThemes]; user: LocalUser | null; isStreaming?: boolean; onRejectTool?: (id: string) => void; onApproveTool?: (id: string) => void; onQuickAction?: (prompt: string) => void }) { const isUser = item.role === "user"; const isSystem = item.role === "system"; const color = item.role === "error" ? "#dc2626" : item.role === "tool" ? "#2563eb" : theme.node.text; + const quickActions = item.role === "assistant" && !isStreaming ? extractCanvasAgentQuickActions(item.text) : []; if (isSystem) { return (
@@ -52,6 +77,22 @@ export function AgentChatMessage({ item, theme, user, isStreaming = false, onRej {!isUser ? : null}
{item.role === "assistant" ? {item.text} :
{item.text}
} + {quickActions.length && onQuickAction ? ( +
+ {quickActions.map((action) => ( + onQuickAction(action.prompt)} + > + {action.label} + + ))} +
+ ) : null} {item.attachments?.length ? : null} {item.meta ?
{item.meta}
: null}
@@ -209,6 +250,7 @@ export function AgentChatComposer({ const [slashIndex, setSlashIndex] = useState(0); const availableSlashSkills = slashSkills ?? []; const canSubmit = !disabled && !sending && Boolean(prompt.trim() || attachments.length); + const reducedMotion = useReducedMotion(); const activeSlashIndex = Math.min(Math.max(slashIndex, 0), Math.max(availableSlashSkills.length - 1, 0)); // 在输入值末尾检测「/关键词」打开技能候选;选择后替换为 @[skill:xxx] 引用 token(保持在 prompt 文本里)。 @@ -270,15 +312,22 @@ export function AgentChatComposer({ }; return ( -
event.stopPropagation()}> -
+
event.stopPropagation()}> +
{attachments.length ? (
{attachments.map((item) => (
{item.name} {onRemoveAttachment ? ( - ) : null} @@ -306,8 +355,8 @@ export function AgentChatComposer({ {slash && availableSlashSkills.length ? (
event.preventDefault()} > {availableSlashSkills.map((skill, index) => ( @@ -336,13 +385,38 @@ export function AgentChatComposer({ event.target.value = ""; }} /> -
-
@@ -351,11 +425,11 @@ export function AgentChatComposer({ export function AgentPanelTabs({ value, items, theme, right, onChange }: { value: T; items: { value: T; label: string; icon?: ReactNode; count?: number }[]; theme: (typeof canvasThemes)[keyof typeof canvasThemes]; right?: ReactNode; onChange: (value: T) => void }) { return ( -
-
-