feat(platform): 开源发布 - 初始化 Open AI Canvas

canvas-agent/src/agents.ts

@@ -0,0 +1,491 @@
+import { spawn, type ChildProcess, type StdioOptions } from "node:child_process";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { createRequire } from "node:module";
+import { fileURLToPath } from "node:url";
+
+import { AGENT_PROMPT, VERSION } from "./config.js";
+import type { AgentAttachment, AgentEmit } from "./types.js";
+
+type Json = Record<string, unknown>;
+type AgentEvent = Json & { type: string; usage?: unknown };
+type PendingRequest = { resolve: (value: unknown) => void; reject: (error: Error) => void };
+type CodexRunOptions = { threadId?: string; cwd?: string };
+type AgentHistoryMessage = { id: string; role: "user" | "assistant" | "tool" | "error"; title?: string; text: string; detail?: unknown; streamId?: string };
+
+let codexQueue: Promise<unknown> = Promise.resolve();
+let codexApp: CodexAppClient | null = null;
+let codexThreadId = "";
+const canvasAgentMcp = canvasAgentMcpCommand();
+const require = createRequire(import.meta.url);
+
+export function withAgentPrompt(prompt: string) {
+    return prompt.trim() ? `${AGENT_PROMPT}\n\n用户请求：${prompt}` : "";
+}
+
+export async function runCodexTurn(prompt: string, emit: AgentEmit, attachments: AgentAttachment[] = [], options: CodexRunOptions = {}) {
+    if (!prompt.trim()) return;
+    codexQueue = codexQueue.catch(() => undefined).then(() => runCodexTurnNow(prompt, emit, attachments, options));
+    await codexQueue;
+}
+
+async function runCodexTurnNow(prompt: string, emit: AgentEmit, attachments: AgentAttachment[], options: CodexRunOptions) {
+    let files: string[] = [];
+    try {
+        files = await writeAttachmentFiles(attachments);
+        codexApp ||= await CodexAppClient.start(emit);
+        const threadId = await ensureCodexThread(codexApp, options);
+        await codexApp.startTurn(threadId, prompt, files);
+    } catch (error) {
+        emit("agent_error", { message: errorMessage(error) });
+    } finally {
+        await Promise.all(files.map((file) => fs.unlink(file).catch(() => undefined)));
+    }
+}
+
+export async function startCodexThread(emit: AgentEmit, cwd?: string) {
+    codexApp ||= await CodexAppClient.start(emit);
+    const thread = await codexApp.startThread(cwd);
+    codexThreadId = String(field(thread, "id") || "");
+    return thread;
+}
+
+export async function resumeCodexThread(emit: AgentEmit, threadId: string, cwd?: string) {
+    codexApp ||= await CodexAppClient.start(emit);
+    await loadCodexThread(emit, threadId, cwd, false);
+    const thread = await codexApp.resumeThread(threadId, cwd);
+    assertThreadWorkspace(thread, cwd);
+    codexThreadId = String(field(thread, "id") || threadId);
+    return { thread, messages: threadMessages(thread) };
+}
+
+export async function listCodexThreads(emit: AgentEmit, options: { cwd: string; searchTerm?: string; limit?: number }) {
+    codexApp ||= await CodexAppClient.start(emit);
+    const result = await codexApp.listThreads({
+        limit: options.limit || 40,
+        sortKey: "updated_at",
+        sortDirection: "desc",
+        sourceKinds: ["cli", "vscode", "appServer", "exec"],
+        cwd: options.cwd,
+        ...(options.searchTerm ? { searchTerm: options.searchTerm } : {}),
+    });
+    const data = Array.isArray(field(result, "data")) ? (field(result, "data") as unknown[]).map(summarizeCodexThread).filter((thread) => threadInWorkspace(thread, options.cwd)) : [];
+    return { data, nextCursor: field(result, "nextCursor") || null, backwardsCursor: field(result, "backwardsCursor") || null };
+}
+
+export async function readCodexThread(emit: AgentEmit, threadId: string, cwd?: string) {
+    const thread = await loadCodexThread(emit, threadId, cwd, true);
+    return { thread: summarizeCodexThread(thread), messages: threadMessages(thread) };
+}
+
+export async function verifyCodexThreadWorkspace(emit: AgentEmit, threadId: string, cwd: string) {
+    await loadCodexThread(emit, threadId, cwd, false);
+}
+
+export async function archiveCodexThread(emit: AgentEmit, threadId: string, cwd?: string) {
+    codexApp ||= await CodexAppClient.start(emit);
+    await loadCodexThread(emit, threadId, cwd, false);
+    await codexApp.archiveThread(threadId);
+}
+
+export function runClaudeTurn(prompt: string, emit: AgentEmit) {
+    if (!prompt.trim()) return;
+    const child = spawnAgent("claude", ["-p", "--output-format", "stream-json", "--verbose", "--include-partial-messages", "--allowedTools", "mcp__infinite-canvas__*", prompt], ["ignore", "pipe", "pipe"], emit);
+    if (!child) return;
+    pipeJsonLines(child, emit, "claude");
+}
+
+async function ensureCodexThread(app: CodexAppClient, options: CodexRunOptions) {
+    if (options.threadId) {
+        const result = await app.readThread(options.threadId, false);
+        assertThreadWorkspace(field(result, "thread") || {}, options.cwd);
+        const thread = await app.resumeThread(options.threadId, options.cwd);
+        assertThreadWorkspace(thread, options.cwd);
+        codexThreadId = String(field(thread, "id") || options.threadId);
+        return codexThreadId;
+    }
+    if (!codexThreadId) {
+        const thread = await app.startThread(options.cwd);
+        codexThreadId = String(field(thread, "id") || "");
+    }
+    return codexThreadId;
+}
+
+class CodexAppClient {
+    private nextId = 1;
+    private buffer = "";
+    private textByItem = new Map<string, string>();
+    private deltaCount = 0;
+    private lastUsage: unknown = null;
+    private pending = new Map<number, PendingRequest>();
+    private activeTurns = new Map<string, PendingRequest>();
+    private completedTurns = new Map<string, Error | null>();
+
+    private constructor(private child: ChildProcess, private emit: AgentEmit) {}
+
+    static async start(emit: AgentEmit) {
+        const child = spawn(process.execPath, [codexBin(), "app-server", "--stdio"], { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
+        const client = new CodexAppClient(child, emit);
+        child.stdout?.on("data", (chunk) => client.read(chunk.toString()));
+        child.stderr?.on("data", (chunk) => emit("agent_log", { text: chunk.toString() }));
+        child.on("error", (error) => emit("agent_error", { message: error.message }));
+        child.on("exit", (code) => {
+            client.failAll(`Codex app-server exited: ${code ?? 0}`);
+            codexApp = null;
+            codexThreadId = "";
+            emit("agent_log", { text: `Codex app-server exited: ${code ?? 0}` });
+        });
+        await client.request("initialize", { clientInfo: { name: "canvas-agent", title: "Infinite Canvas Agent", version: VERSION }, capabilities: { experimentalApi: true, requestAttestation: false } });
+        client.notify("initialized");
+        return client;
+    }
+
+    async startThread(cwd?: string) {
+        const result = await this.request("thread/start", { approvalPolicy: "never", sandbox: "workspace-write", config: codexConfig(), ...(cwd ? { cwd } : {}), threadSource: "user" });
+        const thread = field(result, "thread") as Json | undefined;
+        const id = String(field(thread, "id") || "");
+        if (!id) throw new Error("Codex app-server 没有返回 thread id");
+        return thread || {};
+    }
+
+    async resumeThread(threadId: string, cwd?: string) {
+        const result = await this.request("thread/resume", { threadId, approvalPolicy: "never", sandbox: "workspace-write", config: codexConfig(), ...(cwd ? { cwd } : {}) });
+        const thread = field(result, "thread") as Json | undefined;
+        const id = String(field(thread, "id") || "");
+        if (!id) throw new Error("Codex app-server 没有返回 thread id");
+        return thread || {};
+    }
+
+    listThreads(params: Json) {
+        return this.request("thread/list", params);
+    }
+
+    readThread(threadId: string, includeTurns = true) {
+        return this.request("thread/read", { threadId, includeTurns });
+    }
+
+    archiveThread(threadId: string) {
+        return this.request("thread/archive", { threadId });
+    }
+
+    async startTurn(threadId: string, prompt: string, images: string[]) {
+        const result = await this.request("turn/start", { threadId, input: codexInput(prompt, images), approvalPolicy: "never" });
+        const turnId = String(field(field(result, "turn"), "id") || "");
+        if (!turnId) throw new Error("Codex app-server 没有返回 turn id");
+        const completed = this.completedTurns.get(turnId);
+        if (this.completedTurns.has(turnId)) {
+            this.completedTurns.delete(turnId);
+            if (completed) throw completed;
+            return;
+        }
+        await new Promise((resolve, reject) => this.activeTurns.set(turnId, { resolve, reject }));
+    }
+
+    private request(method: string, params: unknown) {
+        const id = this.nextId++;
+        this.write({ id, method, params });
+        return new Promise((resolve, reject) => this.pending.set(id, { resolve, reject }));
+    }
+
+    private notify(method: string, params?: unknown) {
+        this.write(params === undefined ? { method } : { method, params });
+    }
+
+    private write(value: unknown) {
+        this.child.stdin?.write(`${JSON.stringify(value)}\n`);
+    }
+
+    private read(chunk: string) {
+        this.buffer += chunk;
+        const lines = this.buffer.split(/\r?\n/);
+        this.buffer = lines.pop() || "";
+        lines.filter(Boolean).forEach((line) => {
+            try {
+                this.handle(JSON.parse(line) as Json);
+            } catch {
+                this.emit("agent_log", { text: line });
+            }
+        });
+    }
+
+    private handle(message: Json) {
+        const id = Number(message.id);
+        if (message.error && this.pending.has(id)) return this.reject(id, String(field(message.error, "message") || "Codex request failed"));
+        if (this.pending.has(id)) return this.resolve(id, message.result);
+        if (typeof message.method === "string" && "id" in message) return this.answerServerRequest(message);
+        if (typeof message.method === "string") this.handleNotification(message.method, (message.params || {}) as Json);
+    }
+
+    private handleNotification(method: string, params: Json) {
+        if (method === "item/agentMessage/delta") return this.emitDelta(params);
+        if (method === "thread/tokenUsage/updated") this.lastUsage = normalizeUsage(params);
+        const event = normalizeCodexNotification(method, params);
+        if (!event) return;
+        if (event.type === "turn.completed") event.usage = this.lastUsage;
+        this.emit("agent_event", { agent: "codex", ...event });
+        if (event.type === "turn.completed") {
+            const turnId = String(field(params, "turnId") || field(field(params, "turn"), "id") || "");
+            const pending = this.activeTurns.get(turnId);
+            const error = field(field(params, "turn"), "error");
+            if (pending) {
+                this.activeTurns.delete(turnId);
+                error ? pending.reject(new Error(String(field(error, "message") || "Codex turn failed"))) : pending.resolve(event);
+            } else if (turnId) {
+                this.completedTurns.set(turnId, error ? new Error(String(field(error, "message") || "Codex turn failed")) : null);
+            }
+            this.emit("agent_event", { agent: "codex", type: "stream.summary", delta_count: this.deltaCount });
+            this.deltaCount = 0;
+            this.emit("agent_done", { agent: "codex", usage: event.usage });
+        }
+    }
+
+    private emitDelta(params: Json) {
+        const id = String(field(params, "itemId") || "");
+        const text = `${this.textByItem.get(id) || ""}${String(field(params, "delta") || "")}`;
+        this.deltaCount += 1;
+        this.textByItem.set(id, text);
+        this.emit("agent_event", { agent: "codex", type: "item.updated", item: { id, type: "agent_message", text } });
+    }
+
+    private answerServerRequest(message: Json) {
+        const method = String(message.method);
+        const result = method === "mcpServer/elicitation/request" ? { action: "accept", content: {}, _meta: null } : { decision: "decline" };
+        this.write({ id: message.id, result });
+        this.emit("agent_event", { agent: "codex", type: "server.request", method, params: message.params, result });
+    }
+
+    private resolve(id: number, result: unknown) {
+        const pending = this.pending.get(id);
+        if (pending) (this.pending.delete(id), pending.resolve(result));
+    }
+
+    private reject(id: number, message: string) {
+        const pending = this.pending.get(id);
+        if (pending) (this.pending.delete(id), pending.reject(new Error(message)));
+    }
+
+    failAll(message: string) {
+        [...this.pending.values(), ...this.activeTurns.values()].forEach((item) => item.reject(new Error(message)));
+        this.pending.clear();
+        this.activeTurns.clear();
+    }
+}
+
+function canvasAgentMcpCommand() {
+    const current = process.argv.find((arg) => /index\.(t|j)s$/.test(arg)) || "";
+    const entry = path.resolve(current || fileURLToPath(new URL("./index.js", import.meta.url)));
+    const tsx = path.join(path.dirname(entry), "..", "node_modules", "tsx", "dist", "cli.mjs");
+    return entry.endsWith(".ts") ? { command: process.execPath, args: [tsx, entry, "mcp"] } : { command: process.execPath, args: [entry, "mcp"] };
+}
+
+function codexConfig() {
+    return { mcp_servers: { "infinite-canvas": { command: canvasAgentMcp.command, args: canvasAgentMcp.args, default_tools_approval_mode: "approve", startup_timeout_sec: 20, tool_timeout_sec: 90 } } };
+}
+
+function codexInput(prompt: string, images: string[]) {
+    return [{ type: "text", text: prompt, text_elements: [] }, ...images.map((file) => ({ type: "localImage", path: file }))];
+}
+
+function normalizeCodexNotification(method: string, params: Json): AgentEvent | null {
+    if (method === "thread/started") return { type: "thread.started", thread_id: field(field(params, "thread"), "id") };
+    if (method === "turn/started") return { type: "turn.started" };
+    if (method === "turn/completed") return { type: "turn.completed", usage: null };
+    if (method === "item/started") return { type: "item.started", item: normalizeItem(field(params, "item")) };
+    if (method === "item/completed") return { type: "item.completed", item: normalizeItem(field(params, "item")) };
+    if (method === "error") return { type: "error", message: field(params, "message") };
+    return null;
+}
+
+async function loadCodexThread(emit: AgentEmit, threadId: string, cwd: string | undefined, includeTurns: boolean) {
+    codexApp ||= await CodexAppClient.start(emit);
+    const result = await codexApp.readThread(threadId, includeTurns);
+    const thread = field(result, "thread") || {};
+    assertThreadWorkspace(thread, cwd);
+    return thread;
+}
+
+function assertThreadWorkspace(thread: unknown, cwd?: string) {
+    if (!cwd || threadInWorkspace(thread, cwd)) return;
+    throw new Error("该 Codex 会话不属于当前画布工作空间");
+}
+
+function threadInWorkspace(thread: unknown, cwd: string) {
+    const threadCwd = String(field(thread, "cwd") || "");
+    return Boolean(threadCwd && path.resolve(threadCwd) === path.resolve(cwd));
+}
+
+function normalizeItem(item: unknown) {
+    const value = item && typeof item === "object" ? { ...(item as Json) } : {};
+    if (value.type === "agentMessage") value.type = "agent_message";
+    if (value.type === "mcpToolCall") value.type = "mcp_tool_call";
+    if (value.type === "agent_message" && typeof value.id === "string") value.text = String(value.text || "");
+    if ("arguments" in value) value.arguments = parseMaybeJson(value.arguments);
+    return value;
+}
+
+function normalizeUsage(params: Json) {
+    const total = field(field(params, "tokenUsage"), "total") as Json | undefined;
+    return {
+        input_tokens: field(total, "inputTokens"),
+        cached_input_tokens: field(total, "cachedInputTokens"),
+        output_tokens: field(total, "outputTokens"),
+        reasoning_output_tokens: field(total, "reasoningOutputTokens"),
+    };
+}
+
+function parseMaybeJson(value: unknown) {
+    if (typeof value !== "string") return value;
+    try {
+        return JSON.parse(value);
+    } catch {
+        return value;
+    }
+}
+
+function field(value: unknown, key: string) {
+    return value && typeof value === "object" ? (value as Json)[key] : undefined;
+}
+
+export function summarizeCodexThread(thread: unknown) {
+    return {
+        id: String(field(thread, "id") || ""),
+        sessionId: String(field(thread, "sessionId") || ""),
+        preview: displayUserText(String(field(thread, "preview") || "")),
+        name: stringOrNull(field(thread, "name")),
+        cwd: String(field(thread, "cwd") || ""),
+        status: String(field(thread, "status") || ""),
+        source: field(thread, "source"),
+        threadSource: field(thread, "threadSource"),
+        createdAt: Number(field(thread, "createdAt") || 0),
+        updatedAt: Number(field(thread, "updatedAt") || 0),
+    };
+}
+
+function threadMessages(thread: unknown): AgentHistoryMessage[] {
+    const turns = arrayValue(field(thread, "turns"));
+    const messages: AgentHistoryMessage[] = [];
+    turns.forEach((turn, turnIndex) => {
+        arrayValue(field(turn, "items")).forEach((item, itemIndex) => {
+            const type = String(field(item, "type") || "");
+            const id = String(field(item, "id") || `${turnIndex}-${itemIndex}`);
+            if (type === "userMessage") {
+                const text = displayUserText(userInputText(field(item, "content")));
+                if (text) messages.push({ id, role: "user", text });
+            }
+            if (type === "agentMessage") {
+                const text = String(field(item, "text") || "").trim();
+                if (text) messages.push({ id, role: "assistant", title: "Codex", text, streamId: id });
+            }
+            if (type === "mcpToolCall") {
+                const tool = String(field(item, "tool") || "工具调用");
+                const error = field(field(item, "error"), "message");
+                messages.push({ id, role: error ? "error" : "tool", title: toolName(tool), text: error ? String(error) : `${toolName(tool)} ${String(field(item, "status") || "完成")}`, detail: item });
+            }
+            if (type === "commandExecution") {
+                const command = String(field(item, "command") || "").trim();
+                if (command) messages.push({ id, role: "tool", title: "命令", text: command, detail: { cwd: field(item, "cwd"), status: field(item, "status"), exitCode: field(item, "exitCode") } });
+            }
+            if (type === "fileChange") messages.push({ id, role: "tool", title: "文件变更", text: "Codex 修改了文件", detail: item });
+        });
+    });
+    return messages.filter((item) => item.text).slice(-120);
+}
+
+function userInputText(content: unknown) {
+    return arrayValue(content)
+        .map((item) => {
+            const type = String(field(item, "type") || "");
+            if (type === "text") return String(field(item, "text") || "");
+            if (type === "image" || type === "localImage") return "图片附件";
+            if (type === "mention") return `@${String(field(item, "name") || "文件")}`;
+            return "";
+        })
+        .filter(Boolean)
+        .join("\n");
+}
+
+function displayUserText(text: string) {
+    const value = text.trim();
+    const marker = "用户请求：";
+    const index = value.lastIndexOf(marker);
+    return (index >= 0 ? value.slice(index + marker.length) : value).trim();
+}
+
+function arrayValue(value: unknown) {
+    return Array.isArray(value) ? value : [];
+}
+
+function stringOrNull(value: unknown) {
+    return typeof value === "string" && value.trim() ? value : null;
+}
+
+function toolName(name: string) {
+    if (name === "canvas_apply_ops") return "画布操作";
+    if (name === "canvas_get_state") return "读取画布";
+    if (name === "canvas_get_selection") return "读取选区";
+    if (name === "canvas_export_snapshot") return "导出快照";
+    if (name === "canvas_create_text_node") return "创建文本";
+    if (name === "canvas_create_image_prompt_flow") return "创建生图流程";
+    if (name === "canvas_create_generation_flow") return "创建生成流程";
+    if (name === "canvas_generate_text") return "生成文本";
+    if (name === "canvas_generate_image") return "生成图片";
+    if (name === "canvas_generate_video") return "生成视频";
+    if (name === "canvas_generate_audio") return "生成音频";
+    if (name === "canvas_run_generation") return "触发生成";
+    return name;
+}
+
+async function writeAttachmentFiles(attachments: AgentAttachment[]) {
+    return await Promise.all(attachments.filter((item) => item.dataUrl?.startsWith("data:image/")).map(writeAttachmentFile));
+}
+
+async function writeAttachmentFile(item: AgentAttachment) {
+    const [, meta = "", data = ""] = item.dataUrl?.match(/^data:([^;]+);base64,(.+)$/) || [];
+    if (!data) throw new Error(`图片附件无效：${item.name || "未命名图片"}`);
+    const file = path.join(os.tmpdir(), `infinite-canvas-${Date.now()}-${Math.random().toString(16).slice(2)}.${imageExt(meta || item.type)}`);
+    await fs.writeFile(file, Buffer.from(data, "base64"));
+    return file;
+}
+
+function imageExt(type = "") {
+    if (type.includes("png")) return "png";
+    if (type.includes("webp")) return "webp";
+    return "jpg";
+}
+
+function codexBin() {
+    return path.join(path.dirname(require.resolve("@openai/codex/package.json")), "bin", "codex.js");
+}
+
+function pipeJsonLines(child: ReturnType<typeof spawn>, emit: AgentEmit, agent: string) {
+    let out = "";
+    child.stdout?.on("data", (chunk) => {
+        out += chunk.toString();
+        const lines = out.split(/\r?\n/);
+        out = lines.pop() || "";
+        lines.filter(Boolean).forEach((line) => {
+            try {
+                emit("agent_event", { agent, ...JSON.parse(line) });
+            } catch {
+                emit("agent_event", { agent, type: "raw", text: line });
+            }
+        });
+    });
+    child.stderr?.on("data", (chunk) => emit("agent_log", { text: chunk.toString() }));
+    child.on("error", (error) => emit("agent_error", { message: error.message }));
+    child.on("close", (code) => emit("agent_done", { agent, code }));
+}
+
+function spawnAgent(name: string, args: string[], stdio: StdioOptions, emit: AgentEmit) {
+    try {
+        return spawn(name, args, { stdio, shell: process.platform === "win32", windowsHide: true });
+    } catch (error) {
+        emit("agent_error", { message: errorMessage(error) });
+        return null;
+    }
+}
+
+function errorMessage(error: unknown) {
+    return error instanceof Error ? error.message : String(error);
+}

canvas-agent/src/canvas-session.ts

@@ -0,0 +1,248 @@
+import crypto from "node:crypto";
+import type { ServerResponse } from "node:http";
+
+import { type ToolName } from "./schemas.js";
+import { compactCanvasState, compactNode, isToolName, nextCanvasX, parseToolInput } from "./tools.js";
+import type { CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js";
+
+type PendingRequest = { resolve: (value: unknown) => void; reject: (error: Error) => void };
+
+export class CanvasSession {
+    private clients = new Map<string, ServerResponse>();
+    private pending = new Map<string, PendingRequest>();
+    private canvasState: CanvasSnapshot | null = null;
+
+    health() {
+        return { ok: true, hasCanvas: Boolean(this.canvasState), clients: this.clients.size };
+    }
+
+    openEvents(url: URL, res: ServerResponse) {
+        const clientId = url.searchParams.get("clientId") || crypto.randomUUID();
+        res.writeHead(200, { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", Connection: "keep-alive" });
+        this.clients.set(clientId, res);
+        sendEvent(res, "hello", { ok: true, clientId });
+        const timer = setInterval(() => sendEvent(res, "ping", { time: Date.now() }), 15000);
+        res.on("close", () => {
+            clearInterval(timer);
+            this.clients.delete(clientId);
+            if (this.canvasState?.clientId === clientId) this.canvasState = null;
+        });
+    }
+
+    updateState(body: unknown, clientId?: string) {
+        this.canvasState = { ...((body && typeof body === "object" && !Array.isArray(body) ? body : {}) as Record<string, unknown>), clientId } as CanvasSnapshot;
+    }
+
+    resolveResult(body: { requestId?: string; error?: string; result?: unknown }) {
+        const item = body.requestId ? this.pending.get(body.requestId) : null;
+        if (!item || !body.requestId) return;
+        this.pending.delete(body.requestId);
+        body.error ? item.reject(new Error(body.error)) : item.resolve(body.result);
+    }
+
+    emitAll(type: string, payload: unknown) {
+        this.clients.forEach((client) => sendEvent(client, type, payload));
+    }
+
+    async callTool(name: unknown, rawInput: unknown) {
+        if (!isToolName(name)) throw new Error(`未知工具：${String(name)}`);
+        let tool: ToolName = name;
+        let input = parseToolInput(tool, rawInput) as Record<string, unknown>;
+        const readTool = ["canvas_get_state", "canvas_get_selection", "canvas_export_snapshot"].includes(tool);
+        if (readTool && (!this.clients.size || !this.canvasState)) throw new Error("当前没有已连接画布");
+        if (tool === "canvas_get_state" || tool === "canvas_export_snapshot") return compactCanvasState(this.canvasState);
+        if (tool === "canvas_get_selection") {
+            const ids = new Set(this.canvasState?.selectedNodeIds || []);
+            return { nodes: (this.canvasState?.nodes || []).filter((node) => ids.has(node.id)).map(compactNode) };
+        }
+        if (tool === "canvas_create_node") {
+            const data = input as { nodeType: CanvasNodeType; title?: string; x?: number; y?: number; width?: number; height?: number; metadata?: Record<string, unknown> };
+            input = { ops: [{ type: "add_node", nodeType: data.nodeType, title: data.title, position: { x: data.x ?? nextCanvasX(this.canvasState), y: data.y ?? 0 }, width: data.width, height: data.height, metadata: data.metadata }] };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_create_text_node") {
+            const text = input as { text?: string; x?: number; y?: number; title?: string; width?: number; height?: number };
+            input = { ops: [textNodeOp(text, text.x ?? nextCanvasX(this.canvasState), text.y ?? 0)] };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_create_text_nodes") {
+            const data = input as { items: Array<{ text: string; title?: string; x?: number; y?: number; width?: number; height?: number }>; x?: number; y?: number; gap?: number; direction?: "row" | "column" };
+            const x = Number(data.x ?? nextCanvasX(this.canvasState));
+            const y = Number(data.y ?? 0);
+            const gap = Number(data.gap ?? 40);
+            input = {
+                ops: data.items.map((item, index) => textNodeOp(item, item.x ?? (data.direction === "row" ? x + index * (340 + gap) : x), item.y ?? (data.direction === "row" ? y : y + index * (240 + gap)))),
+            };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_create_image_prompt_flow") {
+            input = { ops: generationFlowOps({ ...(input as Record<string, unknown>), mode: "image" }, this.canvasState) };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_create_config_node") {
+            const data = input as Record<string, unknown>;
+            const x = Number(data.x ?? nextCanvasX(this.canvasState));
+            const y = Number(data.y ?? 0);
+            const configId = `config-${crypto.randomUUID()}`;
+            const mode = generationMode(data.mode);
+            const prompt = String(data.prompt || "");
+            input = { ops: [configNodeOp(configId, data, x, y), ...(data.autoRun ? [runGenerationOp(configId, mode, prompt)] : [])] };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_create_generation_flow") {
+            input = { ops: generationFlowOps(input as Record<string, unknown>, this.canvasState) };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_generate_text" || tool === "canvas_generate_image" || tool === "canvas_generate_video" || tool === "canvas_generate_audio") {
+            input = { ops: generationFlowOps({ ...(input as Record<string, unknown>), mode: tool.replace("canvas_generate_", ""), autoRun: true }, this.canvasState) };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_update_node") {
+            const data = input as { id: string; patch?: Record<string, unknown>; metadata?: Record<string, unknown> };
+            input = { ops: [{ type: "update_node", id: data.id, patch: data.patch, metadata: data.metadata }] };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_update_node_text") {
+            const data = input as { id: string; text: string; title?: string };
+            input = { ops: [{ type: "update_node", id: data.id, patch: { ...(data.title ? { title: data.title } : {}) }, metadata: { content: data.text, status: "success" } }] };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_move_nodes") {
+            const data = input as { items: Array<{ id: string; x?: number; y?: number; dx?: number; dy?: number }> };
+            input = {
+                ops: data.items.map((item) => {
+                    const current = findNode(this.canvasState, item.id);
+                    return { type: "update_node", id: item.id, patch: { position: { x: item.x ?? ((current?.position.x || 0) + (item.dx || 0)), y: item.y ?? ((current?.position.y || 0) + (item.dy || 0)) } } };
+                }),
+            };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_resize_node") {
+            const data = input as { id: string; width: number; height: number; freeResize?: boolean };
+            input = { ops: [{ type: "update_node", id: data.id, patch: { width: data.width, height: data.height }, metadata: data.freeResize === undefined ? undefined : { freeResize: data.freeResize } }] };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_delete_nodes") {
+            input = { ops: [{ type: "delete_node", ids: (input as { ids: string[] }).ids }] };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_connect_nodes") {
+            const data = input as { connections: Array<{ fromNodeId: string; toNodeId: string; fromHandleId?: string; toHandleId?: string }> };
+            input = { ops: data.connections.map((connection) => ({ type: "connect_nodes", ...connection })) };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_select_nodes") {
+            input = { ops: [{ type: "select_nodes", ids: (input as { ids: string[] }).ids }] };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_set_viewport") {
+            input = { ops: [{ type: "set_viewport", viewport: (input as { viewport: unknown }).viewport }] };
+            tool = "canvas_apply_ops";
+        }
+        if (tool === "canvas_run_generation") {
+            const data = input as { nodeId: string; mode?: string; prompt?: string };
+            input = { ops: [runGenerationOp(data.nodeId, generationMode(data.mode), data.prompt)] };
+            tool = "canvas_apply_ops";
+        }
+        if (tool !== "canvas_apply_ops") throw new Error(`未知工具：${tool}`);
+        if (!this.clients.size) throw new Error("当前没有已连接画布");
+        return await this.requestCanvasTool(tool, input);
+    }
+
+    private async requestCanvasTool(name: ToolName, input: Record<string, unknown>) {
+        const requestId = crypto.randomUUID();
+        const client = this.clients.get(this.canvasState?.clientId || "") || this.clients.values().next().value;
+        if (!client) throw new Error("当前没有已连接画布");
+        sendEvent(client, "tool_call", { requestId, name, input });
+        return await new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.pending.delete(requestId);
+                reject(new Error("画布操作超时"));
+            }, 30000);
+            this.pending.set(requestId, { resolve: (value) => (clearTimeout(timer), resolve(value)), reject: (error) => (clearTimeout(timer), reject(error)) });
+        });
+    }
+}
+
+function sendEvent(res: ServerResponse, type: string, payload: unknown) {
+    res.write(`event: ${type}\ndata: ${JSON.stringify(payload)}\n\n`);
+}
+
+function textNodeOp(input: { id?: string; text?: string; title?: string; width?: number; height?: number }, x: number, y: number) {
+    return { type: "add_node", id: input.id, nodeType: "text", title: input.title, position: { x, y }, width: input.width, height: input.height, metadata: { content: input.text || "", status: "success", fontSize: 14 } };
+}
+
+function configNodeOp(id: string, input: Record<string, unknown>, x: number, y: number) {
+    const mode = generationMode(input.mode);
+    const prompt = String(input.prompt || "");
+    return {
+        type: "add_node",
+        id,
+        nodeType: "config",
+        title: String(input.title || generationTitle(mode)),
+        position: { x, y },
+        width: typeof input.width === "number" ? input.width : undefined,
+        height: typeof input.height === "number" ? input.height : undefined,
+        metadata: cleanRecord({
+            generationMode: mode,
+            composerContent: prompt,
+            prompt,
+            status: "idle",
+            model: input.model,
+            size: input.size,
+            quality: input.quality,
+            count: input.count,
+            seconds: input.seconds,
+            vquality: input.vquality,
+            generateAudio: input.generateAudio,
+            watermark: input.watermark,
+            audioVoice: input.audioVoice,
+            audioFormat: input.audioFormat,
+            audioSpeed: input.audioSpeed,
+            audioInstructions: input.audioInstructions,
+        }),
+    };
+}
+
+function generationFlowOps(input: Record<string, unknown>, state: CanvasSnapshot | null) {
+    const mode = generationMode(input.mode);
+    const prompt = String(input.prompt || "");
+    const x = Number(input.x ?? nextCanvasX(state));
+    const y = Number(input.y ?? 0);
+    const textId = `text-${crypto.randomUUID()}`;
+    const configId = `config-${crypto.randomUUID()}`;
+    const referenceNodeIds = Array.isArray(input.referenceNodeIds) ? input.referenceNodeIds.filter((id): id is string => typeof id === "string") : [];
+    const tokens = [`@[node:${textId}]`, ...referenceNodeIds.map((id) => `@[node:${id}]`)];
+    const configInput = { ...input, prompt: tokens.join("\n") };
+    return [
+        textNodeOp({ id: textId, text: prompt, title: String(input.title || "提示词") }, x, y),
+        configNodeOp(configId, configInput, x + 420, y),
+        { type: "connect_nodes", fromNodeId: textId, toNodeId: configId },
+        ...referenceNodeIds.map((fromNodeId) => ({ type: "connect_nodes", fromNodeId, toNodeId: configId })),
+        { type: "select_nodes", ids: [configId] },
+        ...(input.autoRun ? [runGenerationOp(configId, mode, tokens.join("\n"))] : []),
+    ];
+}
+
+function runGenerationOp(nodeId: string, mode: "text" | "image" | "video" | "audio", prompt?: string) {
+    return { type: "run_generation", nodeId, mode, prompt };
+}
+
+function generationMode(value: unknown): "text" | "image" | "video" | "audio" {
+    return value === "text" || value === "video" || value === "audio" ? value : "image";
+}
+
+function generationTitle(mode: "text" | "image" | "video" | "audio") {
+    if (mode === "text") return "文本生成";
+    if (mode === "video") return "视频生成";
+    if (mode === "audio") return "音频生成";
+    return "图片生成";
+}
+
+function findNode(state: CanvasSnapshot | null, id: string): CanvasNode | undefined {
+    return (state?.nodes || []).find((node) => node.id === id);
+}
+
+function cleanRecord(value: Record<string, unknown>) {
+    return Object.fromEntries(Object.entries(value).filter(([, item]) => item !== undefined && item !== ""));
+}

canvas-agent/src/config.ts

@@ -0,0 +1,73 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+export const DEFAULT_PORT = 17371;
+export const CONFIG_DIR = path.join(os.homedir(), ".infinite-canvas");
+export const CONFIG_FILE = path.join(CONFIG_DIR, "canvas-agent.json");
+export const VERSION = readPackageVersion();
+export const AGENT_PROMPT = "你正在帮助用户操作 Infinite Canvas 网页画布。需要改动画布时优先使用已配置的 infinite-canvas MCP 工具：先 canvas_get_state 读取当前画布，再根据任务使用 canvas_create_text_node、canvas_generate_text、canvas_generate_image、canvas_generate_video、canvas_generate_audio、canvas_create_generation_flow、canvas_create_config_node、canvas_run_generation、canvas_update_node、canvas_connect_nodes 等通用工具；复杂批量改动再用 canvas_apply_ops，删除连线可用 delete_connections。需要生成内容时直接调用对应生成工具，不要绑定特定业务场景。不要模拟鼠标点击，不要要求用户手动复制 JSON。";
+
+export type CanvasWorkspaceConfig = { workspacePath: string; activeThreadId?: string; pinnedThreadIds?: string[] };
+export type CanvasAgentConfig = { url: string; token: string; origins?: string[]; canvases?: Record<string, CanvasWorkspaceConfig> };
+
+export function loadConfig(create = false): CanvasAgentConfig {
+    try {
+        return JSON.parse(fs.readFileSync(CONFIG_FILE, "utf8")) as CanvasAgentConfig;
+    } catch {
+        const config = { url: `http://127.0.0.1:${Number(process.env.PORT) || DEFAULT_PORT}`, token: crypto.randomBytes(18).toString("hex") };
+        if (create) saveConfig(config);
+        return config;
+    }
+}
+
+export function saveConfig(config: CanvasAgentConfig) {
+    fs.mkdirSync(CONFIG_DIR, { recursive: true });
+    fs.writeFileSync(CONFIG_FILE, JSON.stringify(config, null, 2));
+}
+
+export function ensureCanvasWorkspace(config: CanvasAgentConfig, canvasId: string) {
+    const id = safeSegment(canvasId || "default");
+    config.canvases ||= {};
+    const current = config.canvases[id];
+    if (current?.workspacePath) {
+        fs.mkdirSync(resolveWorkspacePath(current.workspacePath), { recursive: true });
+        return { canvasId: id, ...current, workspacePath: resolveWorkspacePath(current.workspacePath) };
+    }
+    const workspacePath = path.join(CONFIG_DIR, "codex-workspaces", id);
+    config.canvases[id] = { workspacePath };
+    fs.mkdirSync(workspacePath, { recursive: true });
+    saveConfig(config);
+    return { canvasId: id, workspacePath };
+}
+
+export function updateCanvasWorkspace(config: CanvasAgentConfig, canvasId: string, patch: Partial<CanvasWorkspaceConfig>) {
+    const current = ensureCanvasWorkspace(config, canvasId);
+    const workspacePath = patch.workspacePath ? resolveWorkspacePath(patch.workspacePath) : current.workspacePath;
+    const next = { ...current, ...patch, workspacePath };
+    config.canvases ||= {};
+    config.canvases[current.canvasId] = { workspacePath: next.workspacePath, activeThreadId: next.activeThreadId, pinnedThreadIds: next.pinnedThreadIds };
+    fs.mkdirSync(workspacePath, { recursive: true });
+    saveConfig(config);
+    return { canvasId: current.canvasId, ...config.canvases[current.canvasId] };
+}
+
+function resolveWorkspacePath(value: string) {
+    if (value === "~") return os.homedir();
+    if (value.startsWith("~/")) return path.join(os.homedir(), value.slice(2));
+    return path.resolve(value);
+}
+
+function safeSegment(value: string) {
+    return value.replace(/[^a-zA-Z0-9._-]/g, "-").slice(0, 120) || "default";
+}
+
+function readPackageVersion() {
+    try {
+        const pkg = JSON.parse(fs.readFileSync(new URL("../package.json", import.meta.url), "utf8")) as { version?: string };
+        return pkg.version || "0.0.0";
+    } catch {
+        return "0.0.0";
+    }
+}

canvas-agent/src/http-server.ts

@@ -0,0 +1,137 @@
+import express, { type NextFunction, type Request, type Response } from "express";
+
+import { DEFAULT_PORT, ensureCanvasWorkspace, loadConfig, saveConfig, updateCanvasWorkspace, type CanvasAgentConfig } from "./config.js";
+import { CanvasSession } from "./canvas-session.js";
+import { archiveCodexThread, listCodexThreads, readCodexThread, resumeCodexThread, runClaudeTurn, runCodexTurn, startCodexThread, summarizeCodexThread, verifyCodexThreadWorkspace, withAgentPrompt } from "./agents.js";
+import type { AgentAttachment } from "./types.js";
+
+export function startHttpServer() {
+    const config = loadConfig(true);
+    const port = Number(process.env.PORT) || Number(new URL(config.url).port) || DEFAULT_PORT;
+    config.url = `http://127.0.0.1:${port}`;
+    saveConfig(config);
+
+    const session = new CanvasSession();
+    const emit = (type: string, payload: unknown) => session.emitAll(type, payload);
+    const app = express();
+    app.disable("x-powered-by");
+    app.use(express.json({ limit: "30mb" }));
+    app.use((req, res, next) => {
+        const url = requestUrl(req, config);
+        if (!setCors(req, res, url, config)) return void res.status(403).json({ ok: false, error: "origin not allowed" });
+        if (req.method === "OPTIONS") return void res.json({});
+        next();
+    });
+    app.get("/health", (_req, res) => res.json(session.health()));
+    app.get("/config", (_req, res) => res.json({ ok: true, url: config.url, hasToken: true }));
+    app.use((req, res, next) => {
+        if (validToken(req, requestUrl(req, config), config.token)) return next();
+        res.status(401).json({ ok: false, error: "invalid token" });
+    });
+    app.get("/events", (req, res) => session.openEvents(requestUrl(req, config), res));
+    app.post("/canvas/state", (req, res) => {
+        session.updateState(req.body, String(req.query.clientId || "") || undefined);
+        res.json({ ok: true });
+    });
+    app.post("/canvas/result", (req, res) => {
+        session.resolveResult(req.body);
+        res.json({ ok: true });
+    });
+    app.post("/api/tools", route(async (req, res) => res.json({ ok: true, result: await session.callTool(req.body?.name, req.body?.input || {}) })));
+    app.get("/agent/codex/workspace", (req, res) => {
+        const workspace = ensureCanvasWorkspace(config, String(req.query.canvasId || ""));
+        res.json({ ok: true, workspace });
+    });
+    app.get("/agent/codex/threads", route(async (req, res) => {
+        const workspace = ensureCanvasWorkspace(config, String(req.query.canvasId || ""));
+        const result = await listCodexThreads(emit, { cwd: workspace.workspacePath, searchTerm: String(req.query.searchTerm || "") });
+        res.json({ ok: true, workspace, ...result });
+    }));
+    app.post("/agent/codex/threads/new", route(async (req, res) => {
+        const workspace = ensureCanvasWorkspace(config, String(req.body?.canvasId || ""));
+        const thread = await startCodexThread(emit, workspace.workspacePath);
+        const activeThreadId = String((thread as Record<string, unknown>).id || "");
+        updateCanvasWorkspace(config, workspace.canvasId, { activeThreadId });
+        res.json({ ok: true, workspace: { ...workspace, activeThreadId }, thread: summarizeCodexThread(thread), messages: [] });
+    }));
+    app.get("/agent/codex/threads/:threadId", route(async (req, res) => {
+        const workspace = ensureCanvasWorkspace(config, String(req.query.canvasId || ""));
+        const threadId = routeParam(req.params.threadId);
+        res.json({ ok: true, workspace, ...(await readCodexThread(emit, threadId, workspace.workspacePath)) });
+    }));
+    app.post("/agent/codex/threads/:threadId/resume", route(async (req, res) => {
+        const workspace = ensureCanvasWorkspace(config, String(req.body?.canvasId || ""));
+        const threadId = routeParam(req.params.threadId);
+        const result = await resumeCodexThread(emit, threadId, workspace.workspacePath);
+        updateCanvasWorkspace(config, workspace.canvasId, { activeThreadId: threadId });
+        res.json({ ok: true, workspace: { ...workspace, activeThreadId: threadId }, ...result });
+    }));
+    app.post("/agent/codex/threads/:threadId/delete", route(async (req, res) => {
+        const workspace = ensureCanvasWorkspace(config, String(req.body?.canvasId || ""));
+        const threadId = routeParam(req.params.threadId);
+        await archiveCodexThread(emit, threadId, workspace.workspacePath);
+        if (workspace.activeThreadId === threadId) updateCanvasWorkspace(config, workspace.canvasId, { activeThreadId: undefined });
+        res.json({ ok: true });
+    }));
+    app.post("/agent/codex/turn", route(async (req, res) => {
+        const attachments = Array.isArray(req.body?.attachments) ? (req.body.attachments as AgentAttachment[]) : [];
+        const workspace = ensureCanvasWorkspace(config, String(req.body?.canvasId || ""));
+        let threadId = String(req.body?.threadId || workspace.activeThreadId || "");
+        if (!threadId) {
+            const thread = await startCodexThread(emit, workspace.workspacePath);
+            threadId = String((thread as Record<string, unknown>).id || "");
+            updateCanvasWorkspace(config, workspace.canvasId, { activeThreadId: threadId });
+        } else if (threadId !== workspace.activeThreadId) {
+            await verifyCodexThreadWorkspace(emit, threadId, workspace.workspacePath);
+            updateCanvasWorkspace(config, workspace.canvasId, { activeThreadId: threadId });
+        }
+        void runCodexTurn(withAgentPrompt(String(req.body?.prompt || "")), emit, attachments, { threadId, cwd: workspace.workspacePath });
+        res.json({ ok: true, threadId });
+    }));
+    app.post("/agent/claude/turn", (req, res) => {
+        runClaudeTurn(withAgentPrompt(String(req.body?.prompt || "")), emit);
+        res.json({ ok: true });
+    });
+    app.use((_req, res) => res.status(404).json({ ok: false, error: "not found" }));
+    app.use((error: Error, _req: Request, res: Response, _next: NextFunction) => res.status(500).json({ ok: false, error: error.message }));
+
+    app.listen(port, "127.0.0.1", () => {
+        console.log("Infinite Canvas Agent");
+        console.log(`Local URL: ${config.url}`);
+        console.log(`Connect token: ${config.token}`);
+        console.log("Codex MCP: codex mcp add infinite-canvas -- npx -y @ddcat666/open-ai-canvas-agent mcp");
+    });
+}
+
+function route(handler: (req: Request, res: Response) => Promise<unknown>) {
+    return (req: Request, res: Response, next: NextFunction) => void handler(req, res).catch(next);
+}
+
+function routeParam(value: string | string[]) {
+    return Array.isArray(value) ? value[0] || "" : value;
+}
+
+function requestUrl(req: Request, config: CanvasAgentConfig) {
+    return new URL(req.originalUrl || req.url || "/", config.url);
+}
+
+function setCors(req: Request, res: Response, url: URL, config: CanvasAgentConfig) {
+    const origin = req.headers.origin;
+    res.setHeader("Access-Control-Allow-Origin", origin || "*");
+    res.setHeader("Access-Control-Allow-Headers", "content-type,x-canvas-agent-token");
+    res.setHeader("Access-Control-Allow-Methods", "GET,POST,OPTIONS");
+    res.setHeader("Access-Control-Allow-Private-Network", "true");
+    if (!origin || req.method === "OPTIONS" || url.pathname === "/health" || url.pathname === "/config") return true;
+    config.origins ||= [];
+    if (validToken(req, url, config.token) && !config.origins.includes(origin)) {
+        config.origins.push(origin);
+        saveConfig(config);
+    }
+    res.setHeader("Vary", "Origin");
+    return config.origins.includes(origin);
+}
+
+function validToken(req: Request, url: URL, token: string) {
+    const header = req.headers["x-canvas-agent-token"];
+    return url.searchParams.get("token") === token || header === token || (Array.isArray(header) && header.includes(token));
+}

canvas-agent/src/index.ts

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+import { startHttpServer } from "./http-server.js";
+import { startMcpServer } from "./mcp-server.js";
+
+if (process.argv[2] === "mcp") await startMcpServer();
+else startHttpServer();

canvas-agent/src/mcp-server.ts

@@ -0,0 +1,29 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+
+import { AGENT_PROMPT, loadConfig, type CanvasAgentConfig, VERSION } from "./config.js";
+import { toolDescriptions, toolInputSchemas, toolNames, type ToolName } from "./schemas.js";
+
+type CanvasAgentToolResponse = { ok?: boolean; result?: unknown; error?: string };
+
+export async function startMcpServer() {
+    const config = loadConfig(true);
+    const server = new McpServer({ name: "canvas-agent", version: VERSION }, { instructions: AGENT_PROMPT });
+    toolNames.forEach((name) => registerCanvasTool(server, config, name));
+    await server.connect(new StdioServerTransport());
+}
+
+function registerCanvasTool(server: McpServer, config: CanvasAgentConfig, name: ToolName) {
+    const schema = toolInputSchemas[name];
+    server.registerTool(name, { description: toolDescriptions[name], inputSchema: schema.shape }, async (input: unknown) => {
+        const result = await postCanvasAgentTool(config, name, schema.parse(input));
+        return { content: [{ type: "text" as const, text: JSON.stringify(result, null, 2) }] };
+    });
+}
+
+async function postCanvasAgentTool(config: CanvasAgentConfig, name: ToolName, input: unknown) {
+    const res = await fetch(`${config.url}/api/tools`, { method: "POST", headers: { "content-type": "application/json", "x-canvas-agent-token": config.token }, body: JSON.stringify({ name, input }) });
+    const body = (await res.json()) as CanvasAgentToolResponse;
+    if (!body.ok) throw new Error(body.error || "tool call failed");
+    return body.result;
+}

canvas-agent/src/schemas.ts

@@ -0,0 +1,129 @@
+import { z } from "zod";
+
+const recordSchema = z.record(z.unknown());
+const positionSchema = z.object({ x: z.number(), y: z.number() });
+const viewportSchema = z.object({ x: z.number(), y: z.number(), k: z.number() });
+const nodeTypeSchema = z.enum(["image", "text", "script", "config", "video", "audio", "frame"]);
+const generationModeSchema = z.enum(["text", "image", "video", "audio"]);
+
+export const toolNames = [
+    "canvas_get_state",
+    "canvas_get_selection",
+    "canvas_export_snapshot",
+    "canvas_apply_ops",
+    "canvas_create_node",
+    "canvas_create_text_node",
+    "canvas_create_text_nodes",
+    "canvas_create_config_node",
+    "canvas_create_image_prompt_flow",
+    "canvas_create_generation_flow",
+    "canvas_generate_text",
+    "canvas_generate_image",
+    "canvas_generate_video",
+    "canvas_generate_audio",
+    "canvas_update_node",
+    "canvas_update_node_text",
+    "canvas_move_nodes",
+    "canvas_resize_node",
+    "canvas_delete_nodes",
+    "canvas_connect_nodes",
+    "canvas_select_nodes",
+    "canvas_set_viewport",
+    "canvas_run_generation",
+] as const;
+export type ToolName = (typeof toolNames)[number];
+
+export const canvasOpSchema = z.discriminatedUnion("type", [
+    z.object({ type: z.literal("add_node"), nodeType: nodeTypeSchema.optional(), id: z.string().optional(), title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), position: positionSchema.optional(), metadata: recordSchema.optional() }).passthrough(),
+    z.object({ type: z.literal("update_node"), id: z.string(), patch: recordSchema.optional(), metadata: recordSchema.optional() }).passthrough(),
+    z.object({ type: z.literal("delete_node"), id: z.string().optional(), ids: z.array(z.string()).optional() }).passthrough(),
+    z.object({ type: z.literal("delete_connections"), id: z.string().optional(), ids: z.array(z.string()).optional(), all: z.boolean().optional() }).passthrough(),
+    z.object({ type: z.literal("connect_nodes"), id: z.string().optional(), fromNodeId: z.string(), toNodeId: z.string(), fromHandleId: z.string().optional(), toHandleId: z.string().optional() }).passthrough(),
+    z.object({ type: z.literal("set_viewport"), viewport: viewportSchema }).passthrough(),
+    z.object({ type: z.literal("select_nodes"), ids: z.array(z.string()) }).passthrough(),
+    z.object({ type: z.literal("run_generation"), nodeId: z.string(), mode: generationModeSchema.optional(), prompt: z.string().optional() }).passthrough(),
+]);
+
+const textNodeSchema = z.object({
+    text: z.string(),
+    title: z.string().optional(),
+    x: z.number().optional(),
+    y: z.number().optional(),
+    width: z.number().optional(),
+    height: z.number().optional(),
+});
+
+const generationOptionsSchema = z.object({
+    model: z.string().optional(),
+    size: z.string().optional(),
+    quality: z.string().optional(),
+    count: z.number().optional(),
+    seconds: z.string().optional(),
+    vquality: z.string().optional(),
+    generateAudio: z.string().optional(),
+    watermark: z.string().optional(),
+    audioVoice: z.string().optional(),
+    audioFormat: z.string().optional(),
+    audioSpeed: z.string().optional(),
+    audioInstructions: z.string().optional(),
+});
+
+const generationFlowSchema = z.object({
+    prompt: z.string(),
+    title: z.string().optional(),
+    x: z.number().optional(),
+    y: z.number().optional(),
+    referenceNodeIds: z.array(z.string()).optional(),
+});
+
+export const toolInputSchemas = {
+    canvas_get_state: z.object({}).passthrough(),
+    canvas_get_selection: z.object({}).passthrough(),
+    canvas_export_snapshot: z.object({}).passthrough(),
+    canvas_apply_ops: z.object({ ops: z.array(canvasOpSchema) }),
+    canvas_create_node: z.object({ nodeType: nodeTypeSchema, title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), metadata: recordSchema.optional() }),
+    canvas_create_text_node: z.object({ text: z.string().optional(), x: z.number().optional(), y: z.number().optional(), title: z.string().optional(), width: z.number().optional(), height: z.number().optional() }),
+    canvas_create_text_nodes: z.object({ items: z.array(textNodeSchema).min(1), x: z.number().optional(), y: z.number().optional(), gap: z.number().optional(), direction: z.enum(["row", "column"]).optional() }),
+    canvas_create_config_node: z.object({ prompt: z.string().optional(), mode: generationModeSchema.optional(), title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), autoRun: z.boolean().optional() }).merge(generationOptionsSchema),
+    canvas_create_image_prompt_flow: z.object({ prompt: z.string(), x: z.number().optional(), y: z.number().optional(), autoRun: z.boolean().optional() }).merge(generationOptionsSchema),
+    canvas_create_generation_flow: generationFlowSchema.extend({ mode: generationModeSchema.optional(), autoRun: z.boolean().optional() }).merge(generationOptionsSchema),
+    canvas_generate_text: generationFlowSchema.merge(generationOptionsSchema),
+    canvas_generate_image: generationFlowSchema.merge(generationOptionsSchema),
+    canvas_generate_video: generationFlowSchema.merge(generationOptionsSchema),
+    canvas_generate_audio: generationFlowSchema.merge(generationOptionsSchema),
+    canvas_update_node: z.object({ id: z.string(), patch: recordSchema.optional(), metadata: recordSchema.optional() }),
+    canvas_update_node_text: z.object({ id: z.string(), text: z.string(), title: z.string().optional() }),
+    canvas_move_nodes: z.object({ items: z.array(z.object({ id: z.string(), x: z.number().optional(), y: z.number().optional(), dx: z.number().optional(), dy: z.number().optional() })).min(1) }),
+    canvas_resize_node: z.object({ id: z.string(), width: z.number(), height: z.number(), freeResize: z.boolean().optional() }),
+    canvas_delete_nodes: z.object({ ids: z.array(z.string()).min(1) }),
+    canvas_connect_nodes: z.object({ connections: z.array(z.object({ fromNodeId: z.string(), toNodeId: z.string(), fromHandleId: z.string().optional(), toHandleId: z.string().optional() })).min(1) }),
+    canvas_select_nodes: z.object({ ids: z.array(z.string()) }),
+    canvas_set_viewport: z.object({ viewport: viewportSchema }),
+    canvas_run_generation: z.object({ nodeId: z.string(), mode: generationModeSchema.optional(), prompt: z.string().optional() }),
+} satisfies Record<ToolName, z.AnyZodObject>;
+
+export const toolDescriptions: Record<ToolName, string> = {
+    canvas_get_state: "读取当前网页画布的节点、连线、选区和视口。",
+    canvas_get_selection: "读取当前网页画布选中的节点。",
+    canvas_export_snapshot: "导出当前画布快照，用于理解布局。",
+    canvas_apply_ops: "批量操作当前网页画布。ops 支持 add_node、update_node、delete_node、delete_connections、connect_nodes、set_viewport、select_nodes、run_generation。",
+    canvas_create_node: "创建任意类型节点：text、script、image、config、video、audio、frame。适合创建脚本、媒体占位、配置节点、背板或自定义 metadata 节点。",
+    canvas_create_text_node: "在当前画布创建单个文本节点。",
+    canvas_create_text_nodes: "批量创建文本节点，适合生成标题、段落、脚本、说明等内容块。",
+    canvas_create_config_node: "创建生成配置节点，可指定 text/image/video/audio 模式和生成参数，可选择立即触发生成。",
+    canvas_create_image_prompt_flow: "创建提示词文本节点和图片生成配置节点，并自动连线，可选择立即触发生图。",
+    canvas_create_generation_flow: "创建通用生成流程：提示词文本节点、生成配置节点、参考节点连线，可用于文案、生图、视频或音频。",
+    canvas_generate_text: "创建通用文本生成流程并立即触发生成。",
+    canvas_generate_image: "创建通用图片生成流程并立即触发生成。",
+    canvas_generate_video: "创建通用视频生成流程并立即触发生成。",
+    canvas_generate_audio: "创建通用音频生成流程并立即触发生成。",
+    canvas_update_node: "更新节点基础字段或 metadata。",
+    canvas_update_node_text: "更新文本节点内容和标题。",
+    canvas_move_nodes: "移动一个或多个节点，支持绝对坐标或 dx/dy 偏移。",
+    canvas_resize_node: "调整节点尺寸。",
+    canvas_delete_nodes: "删除指定节点及相关连线。",
+    canvas_connect_nodes: "批量连接节点。",
+    canvas_select_nodes: "设置当前选中节点。",
+    canvas_set_viewport: "调整画布视口。",
+    canvas_run_generation: "触发指定节点生成，通常用于配置节点或文本/图片/视频/音频节点。",
+};

canvas-agent/src/tools.ts

@@ -0,0 +1,26 @@
+import { toolInputSchemas, toolNames, type ToolName } from "./schemas.js";
+import type { CanvasNode, CanvasSnapshot } from "./types.js";
+
+export function isToolName(name: unknown): name is ToolName {
+    return typeof name === "string" && toolNames.includes(name as ToolName);
+}
+
+export function parseToolInput(name: ToolName, input: unknown) {
+    return toolInputSchemas[name].parse(input ?? {});
+}
+
+export function compactCanvasState(state: CanvasSnapshot | null) {
+    if (!state) throw new Error("当前没有已连接画布");
+    return { ...state, nodes: (state.nodes || []).map(compactNode) };
+}
+
+export function compactNode(node: CanvasNode) {
+    const metadata = { ...(node.metadata || {}) };
+    if (typeof metadata.content === "string" && metadata.content.length > 240) metadata.content = `${metadata.content.slice(0, 120)}...`;
+    return { id: node.id, type: node.type, title: node.title, position: node.position, width: node.width, height: node.height, metadata };
+}
+
+export function nextCanvasX(state: CanvasSnapshot | null) {
+    const nodes = state?.nodes || [];
+    return nodes.length ? Math.max(...nodes.map((node) => node.position.x + node.width)) + 80 : 0;
+}

canvas-agent/src/types.ts

@@ -0,0 +1,8 @@
+export type Position = { x: number; y: number };
+export type Viewport = { x: number; y: number; k: number };
+export type CanvasNodeType = "image" | "text" | "script" | "config" | "video" | "audio" | "frame";
+export type CanvasNode = { id: string; type: CanvasNodeType; title?: string; position: Position; width: number; height: number; parentId?: string; metadata?: Record<string, unknown> };
+export type CanvasConnection = { id: string; fromNodeId: string; toNodeId: string; fromHandleId?: string; toHandleId?: string };
+export type CanvasSnapshot = { projectId?: string; title?: string; nodes?: CanvasNode[]; connections?: CanvasConnection[]; selectedNodeIds?: string[]; viewport?: Viewport; clientId?: string };
+export type AgentEmit = (type: string, payload: unknown) => void;
+export type AgentAttachment = { name?: string; type?: string; dataUrl?: string };