feat(agent): 画布 Agent - 重构工作流上下文与技能交互
This commit is contained in:
@@ -13,7 +13,9 @@ import type { AgentAttachment, AgentEmit } from "./types.js";
|
||||
type Json = Record<string, unknown>;
|
||||
type AgentEvent = Json & { type: string; usage?: unknown };
|
||||
type PendingRequest = { resolve: (value: unknown) => void; reject: (error: Error) => void };
|
||||
type CodexRunOptions = { threadId?: string; cwd?: string; onThreadId?: (threadId: string) => void };
|
||||
export type AgentSkillReference = { skillId?: string; name: string; description?: string; instruction?: string };
|
||||
export type CodexSkillInput = { type: "skill"; name: string; path: string };
|
||||
type CodexRunOptions = { threadId?: string; cwd?: string; skills?: AgentSkillReference[]; onThreadId?: (threadId: string) => void };
|
||||
type AgentHistoryMessage = { id: string; role: "user" | "assistant" | "tool" | "error"; title?: string; text: string; detail?: unknown; streamId?: string };
|
||||
|
||||
let codexQueue: Promise<unknown> = Promise.resolve();
|
||||
@@ -35,16 +37,20 @@ export async function runCodexTurn(prompt: string, emit: AgentEmit, attachments:
|
||||
|
||||
async function runCodexTurnNow(prompt: string, emit: AgentEmit, attachments: AgentAttachment[], options: CodexRunOptions) {
|
||||
let files: string[] = [];
|
||||
let skillDirectories: string[] = [];
|
||||
try {
|
||||
files = await writeAttachmentFiles(attachments);
|
||||
const preparedSkills = await writeSkillFiles(options.skills || []);
|
||||
skillDirectories = preparedSkills.directories;
|
||||
codexApp ||= await CodexAppClient.start(emit);
|
||||
const threadId = await ensureCodexThread(codexApp, options);
|
||||
if (threadId !== options.threadId) options.onThreadId?.(threadId);
|
||||
await codexApp.startTurn(threadId, prompt, files);
|
||||
await codexApp.startTurn(threadId, prompt, files, preparedSkills.inputs);
|
||||
} catch (error) {
|
||||
emit("agent_error", { message: errorMessage(error) });
|
||||
} finally {
|
||||
await Promise.all(files.map((file) => fs.unlink(file).catch(() => undefined)));
|
||||
await Promise.all(skillDirectories.map((directory) => fs.rm(directory, { recursive: true, force: true }).catch(() => undefined)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,8 +168,8 @@ class CodexAppClient {
|
||||
return this.request("thread/archive", { threadId });
|
||||
}
|
||||
|
||||
async startTurn(threadId: string, prompt: string, images: string[]) {
|
||||
const result = await this.request("turn/start", { threadId, input: codexInput(prompt, images), approvalPolicy: "never" });
|
||||
async startTurn(threadId: string, prompt: string, images: string[], skills: CodexSkillInput[] = []) {
|
||||
const result = await this.request("turn/start", { threadId, input: codexInput(prompt, images, skills), approvalPolicy: "never" });
|
||||
const turnId = String(field(field(result, "turn"), "id") || "");
|
||||
if (!turnId) throw new Error("Codex app-server 没有返回 turn id");
|
||||
const completed = this.completedTurns.get(turnId);
|
||||
@@ -289,8 +295,16 @@ export function codexConfig(configDir = CONFIG_DIR) {
|
||||
};
|
||||
}
|
||||
|
||||
function codexInput(prompt: string, images: string[]) {
|
||||
return [{ type: "text", text: prompt, text_elements: [] }, ...images.map((file) => ({ type: "localImage", path: file }))];
|
||||
export function codexInput(prompt: string, images: string[], skills: CodexSkillInput[]) {
|
||||
// Skill inputs are first-class app-server UserInput items. Do not add a
|
||||
// `$skill` marker or copy SKILL.md into the text item: doing either turns
|
||||
// native skill selection back into prompt expansion and makes the skill
|
||||
// visible as ordinary user text in history.
|
||||
return [
|
||||
{ type: "text", text: prompt, text_elements: [] },
|
||||
...images.map((file) => ({ type: "localImage", path: file })),
|
||||
...skills,
|
||||
];
|
||||
}
|
||||
|
||||
function normalizeCodexNotification(method: string, params: Json): AgentEvent | null {
|
||||
@@ -421,6 +435,9 @@ function toolName(name: string) {
|
||||
if (name === "canvas_get_state") return "读取画布";
|
||||
if (name === "canvas_get_context") return "读取画布上下文";
|
||||
if (name === "canvas_find_nodes") return "检索画布节点";
|
||||
if (name === "canvas_get_node") return "读取画布节点";
|
||||
if (name === "canvas_get_connection") return "读取画布连线";
|
||||
if (name === "canvas_get_generation_tasks") return "读取生成任务";
|
||||
if (name === "canvas_get_resources") return "读取画布资源";
|
||||
if (name === "canvas_validate_ops") return "校验画布操作";
|
||||
if (name === "canvas_get_selection") return "读取选区";
|
||||
@@ -448,6 +465,44 @@ async function writeAttachmentFile(item: AgentAttachment) {
|
||||
return file;
|
||||
}
|
||||
|
||||
export async function writeSkillFiles(skills: AgentSkillReference[]) {
|
||||
const directories: string[] = [];
|
||||
const inputs: CodexSkillInput[] = [];
|
||||
const usedNames = new Set<string>();
|
||||
try {
|
||||
for (const skill of skills.slice(0, 8)) {
|
||||
const instruction = String(skill.instruction || "").trim().slice(0, 24_000);
|
||||
if (!instruction) continue;
|
||||
const baseName = `canvas-${safeSkillSegment(skill.skillId || skill.name)}`;
|
||||
const name = uniqueSkillName(baseName, usedNames);
|
||||
usedNames.add(name);
|
||||
const directory = await fs.mkdtemp(path.join(os.tmpdir(), "infinite-canvas-skill-"));
|
||||
directories.push(directory);
|
||||
const file = path.join(directory, "SKILL.md");
|
||||
const description = String(skill.description || skill.name).trim().slice(0, 500).replace(/[\r\n]+/g, " ");
|
||||
const body = [`---`, `name: ${name}`, `description: ${JSON.stringify(description)}`, `---`, ``, `# ${skill.name}`, ``, instruction, ``].join("\n");
|
||||
await fs.writeFile(file, body, "utf8");
|
||||
inputs.push({ type: "skill", name, path: file });
|
||||
}
|
||||
return { directories, inputs };
|
||||
} catch (error) {
|
||||
await Promise.all(directories.map((directory) => fs.rm(directory, { recursive: true, force: true }).catch(() => undefined)));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function uniqueSkillName(baseName: string, usedNames: Set<string>) {
|
||||
if (!usedNames.has(baseName)) return baseName;
|
||||
let suffix = 2;
|
||||
while (usedNames.has(`${baseName}-${suffix}`)) suffix += 1;
|
||||
return `${baseName}-${suffix}`;
|
||||
}
|
||||
|
||||
function safeSkillSegment(value: string) {
|
||||
const normalized = value.toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48);
|
||||
return normalized || "skill";
|
||||
}
|
||||
|
||||
function imageExt(type = "") {
|
||||
if (type.includes("png")) return "png";
|
||||
if (type.includes("webp")) return "webp";
|
||||
|
||||
@@ -19,8 +19,24 @@ export type CanvasResourceSummary = {
|
||||
isReady: boolean;
|
||||
};
|
||||
|
||||
export type CanvasGenerationTaskSummary = {
|
||||
taskId: string;
|
||||
nodeId: string;
|
||||
nodeTitle: string;
|
||||
mode?: string;
|
||||
nodeStatus: string;
|
||||
status: string;
|
||||
progress?: number;
|
||||
stage?: string;
|
||||
provider?: string;
|
||||
errorCode?: string;
|
||||
officialStatus?: string;
|
||||
resourceReady: boolean;
|
||||
};
|
||||
|
||||
export type CanvasContext = {
|
||||
schemaVersion: 1;
|
||||
revision: number;
|
||||
stateHash: string;
|
||||
canvas: {
|
||||
projectId?: string;
|
||||
@@ -67,6 +83,7 @@ export function buildCanvasContext(state: CanvasSnapshot | null): CanvasContext
|
||||
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
revision: state.revision ?? 0,
|
||||
stateHash: hashState(state),
|
||||
canvas: {
|
||||
projectId: state.projectId,
|
||||
@@ -106,6 +123,66 @@ export function findCanvasNodes(state: CanvasSnapshot | null, input: { query?: s
|
||||
return { query: input.query || "", total: nodes.length, truncated: nodes.length > limit, nodes: nodes.slice(0, limit).map(compactContextNode) };
|
||||
}
|
||||
|
||||
export function getCanvasNode(state: CanvasSnapshot | null, input: { id: string }) {
|
||||
if (!state) throw new Error("当前没有已连接画布");
|
||||
const node = (state.nodes || []).find((candidate) => candidate.id === input.id);
|
||||
if (!node) return { found: false, id: input.id, node: null, connections: [] };
|
||||
const nodeById = new Map((state.nodes || []).map((candidate) => [candidate.id, candidate]));
|
||||
const connections = (state.connections || [])
|
||||
.filter((connection) => connection.fromNodeId === node.id || connection.toNodeId === node.id)
|
||||
.map((connection) => connectionSummary(connection, nodeById));
|
||||
return {
|
||||
found: true,
|
||||
id: node.id,
|
||||
node: compactContextNode(node),
|
||||
resource: resourceSummary(node),
|
||||
connections,
|
||||
};
|
||||
}
|
||||
|
||||
export function getCanvasConnection(state: CanvasSnapshot | null, input: { id: string }) {
|
||||
if (!state) throw new Error("当前没有已连接画布");
|
||||
const connection = (state.connections || []).find((candidate) => candidate.id === input.id);
|
||||
if (!connection) return { found: false, id: input.id, connection: null };
|
||||
const nodeById = new Map((state.nodes || []).map((candidate) => [candidate.id, candidate]));
|
||||
return {
|
||||
found: true,
|
||||
id: connection.id,
|
||||
connection: connectionSummary(connection, nodeById),
|
||||
fromNode: nodeById.get(connection.fromNodeId) ? compactContextNode(nodeById.get(connection.fromNodeId)!) : null,
|
||||
toNode: nodeById.get(connection.toNodeId) ? compactContextNode(nodeById.get(connection.toNodeId)!) : null,
|
||||
};
|
||||
}
|
||||
|
||||
export function getCanvasGenerationTasks(state: CanvasSnapshot | null, input: { status?: string; nodeIds?: string[]; limit?: number }) {
|
||||
if (!state) throw new Error("当前没有已连接画布");
|
||||
const nodeIds = input.nodeIds?.length ? new Set(input.nodeIds) : null;
|
||||
const tasks = (state.nodes || []).flatMap<CanvasGenerationTaskSummary>((node) => {
|
||||
const metadata = node.metadata || {};
|
||||
const taskId = stringValue(metadata.taskId);
|
||||
if (!taskId || (nodeIds && !nodeIds.has(node.id))) return [];
|
||||
const status = stringValue(metadata.taskStatus) || stringValue(metadata.status) || "unknown";
|
||||
if (input.status && status !== input.status) return [];
|
||||
const resource = resourceSummary(node);
|
||||
return [{
|
||||
taskId,
|
||||
nodeId: node.id,
|
||||
nodeTitle: node.title || "未命名节点",
|
||||
mode: stringValue(metadata.generationMode),
|
||||
nodeStatus: stringValue(metadata.status) || "idle",
|
||||
status,
|
||||
progress: numberValue(metadata.taskProgress ?? metadata.progress),
|
||||
stage: stringValue(metadata.taskStage),
|
||||
provider: stringValue(metadata.taskProvider),
|
||||
errorCode: stringValue(metadata.taskErrorCode || metadata.generationErrorCode),
|
||||
officialStatus: stringValue(metadata.taskOfficialStatus),
|
||||
resourceReady: Boolean(resource?.isReady),
|
||||
}];
|
||||
});
|
||||
const limit = Math.min(Math.max(input.limit || 100, 1), 200);
|
||||
return { total: tasks.length, truncated: tasks.length > limit, tasks: tasks.slice(0, limit) };
|
||||
}
|
||||
|
||||
export function getCanvasResources(state: CanvasSnapshot | null, input: { nodeIds?: string[]; status?: string; limit?: number }) {
|
||||
if (!state) throw new Error("当前没有已连接画布");
|
||||
const nodeIds = input.nodeIds?.length ? new Set(input.nodeIds) : null;
|
||||
@@ -123,11 +200,23 @@ export function validateCanvasOps(state: CanvasSnapshot | null, ops: unknown[])
|
||||
if (!state) throw new Error("当前没有已连接画布");
|
||||
const nodes = state.nodes || [];
|
||||
const nodeIds = new Set(nodes.map((node) => node.id));
|
||||
const connectionKeys = new Set((state.connections || []).map(connectionKey));
|
||||
const liveConnections = new Map<string, { key: string; fromNodeId: string; toNodeId: string }>();
|
||||
(state.connections || []).forEach((connection) => {
|
||||
liveConnections.set(connection.id, { key: connectionKey(connection), fromNodeId: connection.fromNodeId, toNodeId: connection.toNodeId });
|
||||
});
|
||||
const issues: Array<{ index: number; severity: "error" | "warning"; message: string }> = [];
|
||||
const addedIds = new Set<string>();
|
||||
const liveNodeIds = new Set(nodeIds);
|
||||
const removeConnection = (id: string) => {
|
||||
const connection = liveConnections.get(id);
|
||||
if (!connection) return;
|
||||
liveConnections.delete(id);
|
||||
if (![...liveConnections.values()].some((item) => item.key === connection.key)) connectionKeys.delete(connection.key);
|
||||
};
|
||||
const requireNode = (index: number, id: unknown, label: string) => {
|
||||
if (typeof id !== "string" || !id) issues.push({ index, severity: "error", message: `${label} 缺少节点 id` });
|
||||
else if (!nodeIds.has(id) && !addedIds.has(id)) issues.push({ index, severity: "error", message: `${label}「${id}」不在当前画布状态中;先重新读取 canvas_get_context 或 canvas_find_nodes` });
|
||||
else if (!liveNodeIds.has(id)) issues.push({ index, severity: "error", message: `${label}「${id}」不在当前画布状态中;先重新读取 canvas_get_context 或 canvas_find_nodes` });
|
||||
};
|
||||
ops.forEach((raw, index) => {
|
||||
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
||||
@@ -137,32 +226,84 @@ export function validateCanvasOps(state: CanvasSnapshot | null, ops: unknown[])
|
||||
const op = raw as Record<string, unknown>;
|
||||
switch (op.type) {
|
||||
case "add_node":
|
||||
if (op.nodeType !== undefined && !["image", "text", "script", "config", "video", "audio", "frame"].includes(String(op.nodeType))) issues.push({ index, severity: "error", message: `新增节点类型「${String(op.nodeType)}」不受支持` });
|
||||
validateFiniteNumber(issues, index, op.x, "x 坐标");
|
||||
validateFiniteNumber(issues, index, op.y, "y 坐标");
|
||||
validatePosition(issues, index, op.position, "position");
|
||||
validatePositiveNumber(issues, index, op.width, "节点宽度");
|
||||
validatePositiveNumber(issues, index, op.height, "节点高度");
|
||||
if (typeof op.id === "string") {
|
||||
if (nodeIds.has(op.id) || addedIds.has(op.id)) issues.push({ index, severity: "error", message: `新增节点 id「${op.id}」重复` });
|
||||
addedIds.add(op.id);
|
||||
liveNodeIds.add(op.id);
|
||||
}
|
||||
break;
|
||||
case "update_node":
|
||||
case "update_node": {
|
||||
requireNode(index, op.id, "更新目标");
|
||||
const targetNode = nodes.find((node) => node.id === op.id);
|
||||
if (op.patch && typeof op.patch === "object" && !Array.isArray(op.patch)) {
|
||||
const patch = op.patch as Record<string, unknown>;
|
||||
if ("id" in patch || "type" in patch) issues.push({ index, severity: "error", message: "不能通过 update_node 修改节点 id 或类型" });
|
||||
validateFiniteNumber(issues, index, patch.x, "节点 x 坐标");
|
||||
validateFiniteNumber(issues, index, patch.y, "节点 y 坐标");
|
||||
validatePosition(issues, index, patch.position, "节点 position");
|
||||
validatePositiveNumber(issues, index, patch.width, "节点宽度");
|
||||
validatePositiveNumber(issues, index, patch.height, "节点高度");
|
||||
}
|
||||
const metadata = op.metadata && typeof op.metadata === "object" && !Array.isArray(op.metadata) ? op.metadata as Record<string, unknown> : null;
|
||||
const patchMetadata = op.patch && typeof op.patch === "object" && !Array.isArray(op.patch) && (op.patch as Record<string, unknown>).metadata && typeof (op.patch as Record<string, unknown>).metadata === "object" && !Array.isArray((op.patch as Record<string, unknown>).metadata) ? (op.patch as Record<string, unknown>).metadata as Record<string, unknown> : null;
|
||||
if (isMediaNodeType(targetNode?.type) && ((metadata && "status" in metadata) || (patchMetadata && "status" in patchMetadata))) issues.push({ index, severity: "error", message: "不能直接修改媒体节点 status;生成状态必须由任务结果回写" });
|
||||
break;
|
||||
}
|
||||
case "run_generation":
|
||||
requireNode(index, op.nodeId ?? op.id, op.type === "run_generation" ? "生成目标" : "更新目标");
|
||||
requireNode(index, op.nodeId, "生成目标");
|
||||
if (typeof op.mode === "string") {
|
||||
const node = nodes.find((item) => item.id === op.nodeId);
|
||||
if (node && op.mode !== node.type && !(op.mode === "text" && node.type === "script")) issues.push({ index, severity: "error", message: `生成模式「${op.mode}」与节点类型「${node.type}」不匹配` });
|
||||
}
|
||||
break;
|
||||
case "delete_node":
|
||||
if (Array.isArray(op.ids)) op.ids.forEach((id) => requireNode(index, id, "删除目标"));
|
||||
else requireNode(index, op.id, "删除目标");
|
||||
case "delete_node": {
|
||||
const ids = Array.isArray(op.ids) ? op.ids : (typeof op.id === "string" ? [op.id] : op.nodeType ? nodes.filter((node) => node.type === op.nodeType).map((node) => node.id) : []);
|
||||
if (!ids.length) issues.push({ index, severity: "error", message: "删除节点必须提供 id、ids 或 nodeType" });
|
||||
const unique = new Set(ids);
|
||||
if (unique.size !== ids.length) issues.push({ index, severity: "error", message: "删除节点 id 不能重复" });
|
||||
ids.forEach((id) => requireNode(index, id, "删除目标"));
|
||||
ids.forEach((id) => liveNodeIds.delete(id));
|
||||
for (const [connectionId, connection] of liveConnections) if (ids.includes(connection.fromNodeId) || ids.includes(connection.toNodeId)) removeConnection(connectionId);
|
||||
break;
|
||||
case "connect_nodes":
|
||||
}
|
||||
case "connect_nodes": {
|
||||
requireNode(index, op.fromNodeId, "连接起点");
|
||||
requireNode(index, op.toNodeId, "连接终点");
|
||||
if (op.fromNodeId === op.toNodeId) issues.push({ index, severity: "error", message: "不能连接节点自身" });
|
||||
if (typeof op.id === "string" && liveConnections.has(op.id)) issues.push({ index, severity: "error", message: `连线 id「${op.id}」重复` });
|
||||
const key = connectionKey(op as CanvasConnection);
|
||||
if (connectionKeys.has(key)) issues.push({ index, severity: "error", message: "相同端点和 handle 的连线已存在" });
|
||||
connectionKeys.add(key);
|
||||
liveConnections.set(typeof op.id === "string" ? op.id : `__anonymous_${index}`, { key, fromNodeId: String(op.fromNodeId), toNodeId: String(op.toNodeId) });
|
||||
break;
|
||||
}
|
||||
case "select_nodes":
|
||||
if (Array.isArray(op.ids)) op.ids.forEach((id) => requireNode(index, id, "选区节点"));
|
||||
break;
|
||||
case "delete_connections":
|
||||
if (op.all && (op.id || (Array.isArray(op.ids) && op.ids.length))) issues.push({ index, severity: "error", message: "delete_connections 不能同时使用 all 和 id/ids" });
|
||||
if (!op.all && !op.id && !(Array.isArray(op.ids) && op.ids.length)) issues.push({ index, severity: "error", message: "删除连线必须提供 id、ids 或 all=true" });
|
||||
const connectionTargets = Array.isArray(op.ids) ? op.ids : (typeof op.id === "string" ? [op.id] : []);
|
||||
if (new Set(connectionTargets).size !== connectionTargets.length) issues.push({ index, severity: "error", message: "删除连线 id 不能重复" });
|
||||
connectionTargets.filter((id) => !liveConnections.has(id)).forEach((id) => issues.push({ index, severity: "error", message: `连线「${id}」不存在,请先读取 canvas_get_context` }));
|
||||
if (op.all) [...liveConnections.keys()].forEach(removeConnection);
|
||||
else connectionTargets.forEach(removeConnection);
|
||||
break;
|
||||
case "set_viewport":
|
||||
if (!op.viewport || typeof op.viewport !== "object") issues.push({ index, severity: "error", message: "视口参数无效" });
|
||||
else {
|
||||
const viewport = op.viewport as Record<string, unknown>;
|
||||
validateFiniteNumber(issues, index, viewport.x, "视口 x");
|
||||
validateFiniteNumber(issues, index, viewport.y, "视口 y");
|
||||
validateFiniteNumber(issues, index, viewport.k, "视口缩放");
|
||||
if (typeof viewport.k === "number" && (viewport.k < 0.05 || viewport.k > 8)) issues.push({ index, severity: "error", message: "视口缩放 k 必须在 0.05 到 8 之间" });
|
||||
}
|
||||
break;
|
||||
default:
|
||||
issues.push({ index, severity: "error", message: `不支持的操作类型「${String(op.type)}」` });
|
||||
@@ -172,6 +313,33 @@ export function validateCanvasOps(state: CanvasSnapshot | null, ops: unknown[])
|
||||
return { ok: errors.length === 0, issues, operationCount: ops.length, currentStateHash: hashState(state) };
|
||||
}
|
||||
|
||||
function validateFiniteNumber(issues: Array<{ index: number; severity: "error" | "warning"; message: string }>, index: number, value: unknown, label: string) {
|
||||
if (value !== undefined && (typeof value !== "number" || !Number.isFinite(value))) issues.push({ index, severity: "error", message: `${label}必须是有限数字` });
|
||||
}
|
||||
|
||||
function validatePositiveNumber(issues: Array<{ index: number; severity: "error" | "warning"; message: string }>, index: number, value: unknown, label: string) {
|
||||
if (value !== undefined && (typeof value !== "number" || !Number.isFinite(value) || value <= 0)) issues.push({ index, severity: "error", message: `${label}必须是正数` });
|
||||
}
|
||||
|
||||
function validatePosition(issues: Array<{ index: number; severity: "error" | "warning"; message: string }>, index: number, value: unknown, label: string) {
|
||||
if (value === undefined) return;
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
issues.push({ index, severity: "error", message: `${label}必须是包含 x/y 的对象` });
|
||||
return;
|
||||
}
|
||||
const position = value as Record<string, unknown>;
|
||||
validateFiniteNumber(issues, index, position.x, `${label}.x`);
|
||||
validateFiniteNumber(issues, index, position.y, `${label}.y`);
|
||||
}
|
||||
|
||||
function connectionKey(connection: Pick<CanvasConnection, "fromNodeId" | "toNodeId" | "fromHandleId" | "toHandleId">) {
|
||||
return [connection.fromNodeId, connection.toNodeId, connection.fromHandleId || "", connection.toHandleId || ""].join("\0");
|
||||
}
|
||||
|
||||
function isMediaNodeType(type: CanvasNode["type"] | undefined) {
|
||||
return type === "image" || type === "video" || type === "audio";
|
||||
}
|
||||
|
||||
export function hashState(state: CanvasSnapshot) {
|
||||
return crypto.createHash("sha256").update(stableStringify({
|
||||
projectId: state.projectId,
|
||||
@@ -197,12 +365,19 @@ function compactContextNode(node: CanvasNode) {
|
||||
status: String(metadata.status || "idle"),
|
||||
content: preview(metadata.content, 240),
|
||||
prompt: preview(metadata.prompt || metadata.composerContent, 300),
|
||||
generation: metadata.generationMode || metadata.workflowKind ? {
|
||||
generation: metadata.generationMode || metadata.workflowKind || metadata.taskId ? {
|
||||
mode: metadata.generationMode,
|
||||
model: metadata.model,
|
||||
workflowKind: metadata.workflowKind,
|
||||
workflowTitle: metadata.workflowTitle,
|
||||
taskId: metadata.taskId,
|
||||
status: metadata.taskStatus || metadata.status,
|
||||
progress: numberValue(metadata.taskProgress ?? metadata.progress),
|
||||
stage: metadata.taskStage,
|
||||
provider: metadata.taskProvider,
|
||||
errorCode: metadata.taskErrorCode || metadata.generationErrorCode,
|
||||
} : undefined,
|
||||
error: metadata.status === "error" ? preview(metadata.errorDetails || metadata.generationErrorCode, 360) : undefined,
|
||||
asset: metadata.assetId || metadata.characterAssetId ? {
|
||||
assetId: metadata.assetId || metadata.characterAssetId,
|
||||
versionId: metadata.characterVersionId,
|
||||
|
||||
@@ -2,7 +2,7 @@ import crypto from "node:crypto";
|
||||
import type { ServerResponse } from "node:http";
|
||||
|
||||
import { CANVAS_GENERATION_CONTINUATION_TIMEOUT_MS } from "./canvas-tool-timeouts.js";
|
||||
import { buildCanvasContext, findCanvasNodes, getCanvasResources, validateCanvasOps } from "./canvas-context.js";
|
||||
import { buildCanvasContext, findCanvasNodes, getCanvasConnection, getCanvasGenerationTasks, getCanvasNode, getCanvasResources, hashState, validateCanvasOps } from "./canvas-context.js";
|
||||
import { type ToolName } from "./schemas.js";
|
||||
import { compactCanvasState, compactNode, isToolName, nextCanvasX, parseToolInput } from "./tools.js";
|
||||
import type { CanvasNode, CanvasNodeType, CanvasSnapshot } from "./types.js";
|
||||
@@ -52,7 +52,20 @@ export class CanvasSession {
|
||||
}
|
||||
|
||||
updateState(body: unknown, clientId?: string) {
|
||||
this.canvasState = { ...((body && typeof body === "object" && !Array.isArray(body) ? body : {}) as Record<string, unknown>), clientId } as CanvasSnapshot;
|
||||
const candidate = { ...((body && typeof body === "object" && !Array.isArray(body) ? body : {}) as Record<string, unknown>), clientId } as CanvasSnapshot;
|
||||
const incomingRevision = typeof candidate.revision === "number" && Number.isInteger(candidate.revision) && candidate.revision >= 0 ? candidate.revision : undefined;
|
||||
const actualHash = hashState(candidate);
|
||||
const suppliedHash = typeof (body as Record<string, unknown> | null)?.stateHash === "string" ? String((body as Record<string, unknown>).stateHash) : undefined;
|
||||
if (suppliedHash && suppliedHash !== actualHash) return { accepted: false, revision: this.canvasState?.revision ?? 0, stateHash: this.canvasState ? hashState(this.canvasState) : actualHash, reason: "state_hash_mismatch" as const };
|
||||
const previousState = this.canvasState;
|
||||
const currentRevision = previousState?.revision ?? 0;
|
||||
if (previousState) {
|
||||
if (incomingRevision !== undefined && incomingRevision < currentRevision) return { accepted: false, revision: currentRevision, stateHash: hashState(previousState), reason: "stale_revision" as const };
|
||||
if (incomingRevision === currentRevision && actualHash !== hashState(previousState)) return { accepted: false, revision: currentRevision, stateHash: hashState(previousState), reason: "revision_conflict" as const };
|
||||
}
|
||||
const revision = incomingRevision ?? (this.canvasState ? currentRevision + 1 : 0);
|
||||
this.canvasState = { ...candidate, revision };
|
||||
return { accepted: true, idempotent: Boolean(previousState && incomingRevision === currentRevision && actualHash === hashState(previousState)), revision, stateHash: actualHash };
|
||||
}
|
||||
|
||||
resolveResult(body: { requestId?: string; error?: string; result?: unknown }) {
|
||||
@@ -89,17 +102,24 @@ export class CanvasSession {
|
||||
if (!input.projectId) throw new Error("当前画布没有关联短剧项目");
|
||||
return await this.requestCanvasTool(tool, input);
|
||||
}
|
||||
const readTool = ["canvas_get_state", "canvas_get_context", "canvas_find_nodes", "canvas_get_resources", "canvas_validate_ops", "canvas_get_selection", "canvas_export_snapshot"].includes(tool);
|
||||
const readTool = ["canvas_get_state", "canvas_get_context", "canvas_find_nodes", "canvas_get_node", "canvas_get_connection", "canvas_get_generation_tasks", "canvas_get_resources", "canvas_validate_ops", "canvas_get_selection", "canvas_export_snapshot"].includes(tool);
|
||||
if (readTool && (!this.clients.size || !this.canvasState)) throw new Error("当前没有已连接画布");
|
||||
if (tool === "canvas_get_state" || tool === "canvas_export_snapshot") return compactCanvasState(this.canvasState);
|
||||
if (tool === "canvas_get_context") return buildCanvasContext(this.canvasState);
|
||||
if (tool === "canvas_find_nodes") return findCanvasNodes(this.canvasState, input as Parameters<typeof findCanvasNodes>[1]);
|
||||
if (tool === "canvas_get_node") return getCanvasNode(this.canvasState, input as Parameters<typeof getCanvasNode>[1]);
|
||||
if (tool === "canvas_get_connection") return getCanvasConnection(this.canvasState, input as Parameters<typeof getCanvasConnection>[1]);
|
||||
if (tool === "canvas_get_generation_tasks") return getCanvasGenerationTasks(this.canvasState, input as Parameters<typeof getCanvasGenerationTasks>[1]);
|
||||
if (tool === "canvas_get_resources") return getCanvasResources(this.canvasState, input as Parameters<typeof getCanvasResources>[1]);
|
||||
if (tool === "canvas_validate_ops") return validateCanvasOps(this.canvasState, (input as { ops: unknown[] }).ops);
|
||||
if (tool === "canvas_get_selection") {
|
||||
const ids = new Set(this.canvasState?.selectedNodeIds || []);
|
||||
return { nodes: (this.canvasState?.nodes || []).filter((node) => ids.has(node.id)).map(compactNode) };
|
||||
}
|
||||
if (tool === "canvas_create_workflow") {
|
||||
input = { ops: workflowOps(input as Record<string, unknown>, this.canvasState) };
|
||||
tool = "canvas_apply_ops";
|
||||
}
|
||||
if (tool === "canvas_create_node") {
|
||||
const data = input as { nodeType: CanvasNodeType; title?: string; x?: number; y?: number; width?: number; height?: number; metadata?: Record<string, unknown> };
|
||||
input = { ops: [{ type: "add_node", nodeType: data.nodeType, title: data.title, position: { x: data.x ?? nextCanvasX(this.canvasState), y: data.y ?? 0 }, width: data.width, height: data.height, metadata: data.metadata }] };
|
||||
@@ -112,6 +132,8 @@ export class CanvasSession {
|
||||
}
|
||||
if (tool === "canvas_create_text_nodes") {
|
||||
const data = input as { items: Array<{ text: string; title?: string; x?: number; y?: number; width?: number; height?: number }>; x?: number; y?: number; gap?: number; direction?: "row" | "column" };
|
||||
const batchText = data.items.map((item) => `${item.title || ""} ${item.text || ""}`).join(" ");
|
||||
if (/流水线|工作流|工作流图|管线|节点图|连线|pipeline|workflow/i.test(batchText)) throw new Error("检测到工作流意图,请使用 canvas_create_workflow 创建真实类型节点和连线");
|
||||
const x = Number(data.x ?? nextCanvasX(this.canvasState));
|
||||
const y = Number(data.y ?? 0);
|
||||
const gap = Number(data.gap ?? 40);
|
||||
@@ -181,8 +203,13 @@ export class CanvasSession {
|
||||
}
|
||||
if (tool !== "canvas_apply_ops") throw new Error(`未知工具:${tool}`);
|
||||
if (!this.clients.size) throw new Error("当前没有已连接画布");
|
||||
const currentContext = buildCanvasContext(this.canvasState);
|
||||
const expectedRevision = typeof input.expectedRevision === "number" ? input.expectedRevision : undefined;
|
||||
const expectedStateHash = typeof input.expectedStateHash === "string" ? input.expectedStateHash : "";
|
||||
if (expectedStateHash && expectedStateHash !== buildCanvasContext(this.canvasState).stateHash) {
|
||||
if (expectedRevision !== undefined && expectedRevision !== currentContext.revision) {
|
||||
throw new Error(`画布 revision 已从 ${expectedRevision} 变为 ${currentContext.revision},请重新读取 canvas_get_context 后再执行写操作`);
|
||||
}
|
||||
if (expectedStateHash && expectedStateHash !== currentContext.stateHash) {
|
||||
throw new Error("画布状态已变化,请重新读取 canvas_get_context 后再执行写操作");
|
||||
}
|
||||
const validation = validateCanvasOps(this.canvasState, (input as { ops: unknown[] }).ops);
|
||||
@@ -230,6 +257,116 @@ function sendEvent(res: ServerResponse, type: string, payload: unknown) {
|
||||
res.write(`event: ${type}\ndata: ${JSON.stringify(payload)}\n\n`);
|
||||
}
|
||||
|
||||
function workflowOps(input: Record<string, unknown>, state: CanvasSnapshot | null) {
|
||||
const nodes = Array.isArray(input.nodes) ? input.nodes as Array<Record<string, unknown>> : [];
|
||||
if (!nodes.length) throw new Error("工作流至少需要一个节点");
|
||||
const refs = new Set<string>();
|
||||
for (const node of nodes) {
|
||||
const ref = String(node.ref || "").trim();
|
||||
const title = String(node.title || "").trim();
|
||||
const kind = String(node.kind || "text");
|
||||
const prompt = String(node.prompt || node.content || workflowPrompt(kind, title, input)).trim();
|
||||
if (!ref || !title) throw new Error("工作流节点必须包含 ref 和 title");
|
||||
if (refs.has(ref)) throw new Error(`工作流节点 ref「${ref}」重复`);
|
||||
if (!["text", "script"].includes(workflowNodeType(kind)) && !prompt) throw new Error(`媒体工作流节点「${title}」缺少 prompt/content,不能创建空资源节点`);
|
||||
refs.add(ref);
|
||||
for (const nodeId of Array.isArray(node.referenceNodeIds) ? node.referenceNodeIds : []) {
|
||||
if (!existingNodeId(state, String(nodeId))) throw new Error(`节点「${title}」引用的现有节点「${String(nodeId)}」不存在`);
|
||||
}
|
||||
}
|
||||
const direction = input.direction === "vertical" ? "vertical" : "horizontal";
|
||||
const gap = Math.max(48, Number(input.gap || 120));
|
||||
const existing = state?.nodes || [];
|
||||
const maxX = existing.reduce((max, node) => Math.max(max, node.position.x + node.width), 0);
|
||||
const maxY = existing.reduce((max, node) => Math.max(max, node.position.y + node.height), 0);
|
||||
const start = input.start && typeof input.start === "object" ? input.start as { x: number; y: number } : { x: existing.length ? maxX + 160 : 80, y: existing.length ? Math.max(80, maxY - 520) : 80 };
|
||||
const ids = new Map(nodes.map((node) => [String(node.ref), `agent-workflow-${slug(String(node.ref))}-${crypto.randomUUID().slice(0, 8)}`]));
|
||||
const ops: Array<Record<string, unknown>> = [];
|
||||
let cursor = { x: Number(start.x), y: Number(start.y) };
|
||||
for (const node of nodes) {
|
||||
const kind = String(node.kind || "text");
|
||||
const type = workflowNodeType(kind);
|
||||
const size = workflowNodeSize(type, kind, node.width, node.height);
|
||||
const prompt = String(node.prompt || node.content || workflowPrompt(kind, String(node.title), input));
|
||||
const position = { ...cursor };
|
||||
const internalReferenceIds = Array.isArray(node.referenceRefs) ? node.referenceRefs.map((ref) => ids.get(String(ref))).filter(Boolean) : [];
|
||||
const externalReferenceIds = Array.isArray(node.referenceNodeIds) ? node.referenceNodeIds.map(String) : [];
|
||||
ops.push({ type: "add_node", id: ids.get(String(node.ref)), nodeType: type, title: String(node.title), position, width: size.width, height: size.height, metadata: { content: type === "text" ? String(node.content || prompt) : "", composerContent: prompt || undefined, prompt: prompt || undefined, workflowKind: workflowKind(kind), workflowTitle: input.title, workflowDescription: node.description || input.description, generationMode: type === "image" ? "image" : type === "video" ? "video" : type === "audio" ? "audio" : undefined, status: type === "text" || type === "script" ? "success" : "idle", referenceNodeIds: [...internalReferenceIds, ...externalReferenceIds].length ? [...internalReferenceIds, ...externalReferenceIds] : undefined } });
|
||||
cursor = direction === "vertical" ? { x: Number(start.x), y: cursor.y + size.height + gap } : { x: cursor.x + size.width + gap, y: Number(start.y) };
|
||||
}
|
||||
const edges = Array.isArray(input.edges) && input.edges.length ? input.edges as Array<Record<string, unknown>> : nodes.slice(0, -1).map((node, index) => ({ from: node.ref, to: nodes[index + 1].ref }));
|
||||
const keys = new Set<string>();
|
||||
for (const edge of edges) {
|
||||
const from = String(edge.from || "");
|
||||
const to = String(edge.to || "");
|
||||
if (!ids.has(from) || !ids.has(to)) throw new Error(`工作流连线引用不存在的节点:${from} → ${to}`);
|
||||
const key = `${from}\0${to}`;
|
||||
if (keys.has(key)) continue;
|
||||
keys.add(key);
|
||||
ops.push({ type: "connect_nodes", fromNodeId: ids.get(from), toNodeId: ids.get(to) });
|
||||
}
|
||||
for (const node of nodes) for (const ref of Array.isArray(node.referenceRefs) ? node.referenceRefs : []) {
|
||||
const from = String(ref);
|
||||
const to = String(node.ref);
|
||||
if (!ids.has(from)) throw new Error(`节点「${to}」引用了不存在的节点「${from}」`);
|
||||
const key = `${from}\0${to}`;
|
||||
if (keys.has(key)) continue;
|
||||
keys.add(key);
|
||||
ops.push({ type: "connect_nodes", fromNodeId: ids.get(from), toNodeId: ids.get(to) });
|
||||
}
|
||||
for (const node of nodes) for (const ref of Array.isArray(node.referenceNodeIds) ? node.referenceNodeIds : []) {
|
||||
const from = String(ref);
|
||||
const to = String(node.ref);
|
||||
const key = `${from}\0${to}`;
|
||||
if (keys.has(key)) continue;
|
||||
keys.add(key);
|
||||
ops.push({ type: "connect_nodes", fromNodeId: from, toNodeId: ids.get(to) });
|
||||
}
|
||||
ops.push({ type: "select_nodes", ids: nodes.map((node) => ids.get(String(node.ref))) });
|
||||
if (input.autoRun === true || nodes.some((node) => node.runGeneration === true)) for (const node of nodes) {
|
||||
const type = workflowNodeType(String(node.kind || "text"));
|
||||
if (!["image", "video", "audio"].includes(type) || (input.autoRun !== true && node.runGeneration !== true)) continue;
|
||||
ops.push({ type: "run_generation", nodeId: ids.get(String(node.ref)), mode: type, prompt: node.prompt || node.content || workflowPrompt(String(node.kind || "text"), String(node.title), input) });
|
||||
}
|
||||
return ops;
|
||||
}
|
||||
|
||||
function existingNodeId(state: CanvasSnapshot | null, id: string) {
|
||||
return Boolean(state?.nodes?.some((node) => node.id === id));
|
||||
}
|
||||
|
||||
function workflowNodeType(kind: string) {
|
||||
if (kind === "script") return "script";
|
||||
if (["image", "character_cards", "character_three_view"].includes(kind)) return "image";
|
||||
if (["video", "storyboard_video"].includes(kind)) return "video";
|
||||
if (kind === "audio") return "audio";
|
||||
return "text";
|
||||
}
|
||||
|
||||
function workflowNodeSize(type: string, kind: string, width: unknown, height: unknown) {
|
||||
const defaults = type === "image" ? { width: 560, height: 380 } : type === "video" ? { width: 640, height: 360 } : type === "script" ? { width: 920, height: 360 } : type === "audio" ? { width: 340, height: 160 } : { width: 420, height: 240 };
|
||||
return { width: typeof width === "number" && width > 0 ? width : defaults.width, height: typeof height === "number" && height > 0 ? height : defaults.height };
|
||||
}
|
||||
|
||||
function workflowPrompt(kind: string, title: string, input: Record<string, unknown>) {
|
||||
const workflowTitle = String(input.title || input.description || "当前创作项目").trim();
|
||||
if (kind === "character_cards") return `请基于「${workflowTitle}」拆分主要角色,并为每个角色生成可用于后续创作的角色图片卡片:外观、服饰、身份、性格和视觉辨识点。`;
|
||||
if (kind === "character_three_view") return `请基于上游角色卡片生成「${title}」:同一角色的正面、侧面、背面三视图,保持服饰、发型、道具和比例一致。`;
|
||||
if (kind === "storyboard_video") return `请基于上游角色三视图,为「${workflowTitle}」制作分镜剧情视频方案:包含镜头顺序、景别、动作、节奏和画面连续性。`;
|
||||
return "";
|
||||
}
|
||||
|
||||
function workflowKind(kind: string) {
|
||||
if (["character_cards", "character_three_view"].includes(kind)) return "character";
|
||||
if (kind === "storyboard_video") return "storyboard";
|
||||
if (kind === "script") return "script";
|
||||
return "free";
|
||||
}
|
||||
|
||||
function slug(value: string) {
|
||||
return value.toLowerCase().replace(/[^a-z0-9\u4e00-\u9fff]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40) || "node";
|
||||
}
|
||||
|
||||
function textNodeOp(input: { id?: string; text?: string; title?: string; width?: number; height?: number }, x: number, y: number) {
|
||||
return { type: "add_node", id: input.id, nodeType: "text", title: input.title, position: { x, y }, width: input.width, height: input.height, metadata: { content: input.text || "", status: "success", fontSize: 14 } };
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ export const AGENT_PROMPT = `你是影策的画布执行 Agent,不是只会生
|
||||
|
||||
【上下文协议】
|
||||
- 涉及“这个/当前/已有/选中的”对象时,先 canvas_get_context;用户明确指向选中对象时再补 canvas_get_selection。
|
||||
- 不要从记忆或用户描述猜节点 id。需要找节点时用 canvas_find_nodes;需要判断媒体能否作为参考时用 canvas_get_resources。
|
||||
- 不要从记忆或用户描述猜节点 id。需要找节点时用 canvas_find_nodes;已经知道真实 id 时用 canvas_get_node 或 canvas_get_connection 做精确复核;需要观察生成进度时用 canvas_get_generation_tasks;需要判断媒体能否作为参考时用 canvas_get_resources。
|
||||
- canvas_get_context 返回 stateHash、语义化节点、连接关系和资源就绪状态。资源 ready=false、status=loading/error 或只有占位 metadata 时,必须明确说明,不要把它当成可用素材。
|
||||
|
||||
【执行协议】
|
||||
@@ -23,7 +23,8 @@ export const AGENT_PROMPT = `你是影策的画布执行 Agent,不是只会生
|
||||
- 写操作只使用当前上下文中真实存在的 id;新增节点要避免重叠,优先沿现有内容的右侧或下方网格布局。
|
||||
- 操作完成后检查工具返回的真实结果;如果没有改变、部分失败或生成仍在进行,必须如实报告,不要说“已完成”。
|
||||
- 删除、覆盖、批量移动、触发生成属于高影响操作,先给出简短计划并等待网页侧确认;不要用模拟鼠标点击绕过确认。
|
||||
- 优先使用语义化工具(canvas_create_text_node、canvas_generate_*、canvas_update_node_text 等),只有确实需要批量事务时才使用 canvas_apply_ops。
|
||||
- 流水线、工作流、管线、节点图或用户明确要求连线时,必须使用 canvas_create_workflow:将业务阶段拆成真实的文本/脚本/图片/视频/音频节点;character_cards 表示角色拆分图片卡片,character_three_view 表示角色三视图,storyboard_video 表示分镜剧情视频。媒体节点必须有真实 prompt/content;涉及已有素材时先 canvas_find_nodes/canvas_get_resources,再使用返回的真实 node id 填入 referenceNodeIds。工具会按实际尺寸布局并建立 edges/referenceRefs/referenceNodeIds 连线,禁止把工作流退化成批量空文本节点。
|
||||
- 优先使用语义化工具(canvas_create_workflow、canvas_create_text_node、canvas_generate_*、canvas_update_node_text 等),只有确实需要批量事务时才使用 canvas_apply_ops。
|
||||
|
||||
【资源与生成】
|
||||
- 生成前先检查已有提示词、参考节点、资产引用和就绪状态;有合适资源就复用真实 node id,不要重复上传或创建孤立副本。
|
||||
|
||||
@@ -40,8 +40,16 @@ export function createCanvasAgentHttpModule(
|
||||
);
|
||||
}, { queryKeys: ["clientId"], lastEventId: true }),
|
||||
canvasRoute("POST", "/canvas/state", (req, res) => {
|
||||
session.updateState(jsonBody(req), queryValue(req, "clientId") || undefined);
|
||||
res.json({ ok: true });
|
||||
const result = session.updateState(jsonBody(req), queryValue(req, "clientId") || undefined);
|
||||
if (!result) {
|
||||
res.json({ ok: true });
|
||||
return;
|
||||
}
|
||||
if (result && !result.accepted) {
|
||||
res.status(409).json({ ok: false, ...result });
|
||||
return;
|
||||
}
|
||||
res.json({ ok: true, ...result });
|
||||
}, { queryKeys: ["clientId"] }),
|
||||
canvasRoute("POST", "/canvas/result", (req, res) => {
|
||||
session.resolveResult(jsonBody(req) as { requestId?: string; error?: string; result?: unknown });
|
||||
@@ -112,6 +120,7 @@ export function createCanvasAgentHttpModule(
|
||||
const attachments = Array.isArray(body.attachments)
|
||||
? body.attachments as AgentAttachment[]
|
||||
: [];
|
||||
const skills = parseAgentSkills(body.skills);
|
||||
const workspace = ensureCanvasWorkspace(config, String(body.canvasId || ""));
|
||||
let threadId = String(body.threadId || workspace.activeThreadId || "");
|
||||
void (async () => {
|
||||
@@ -128,6 +137,7 @@ export function createCanvasAgentHttpModule(
|
||||
emit,
|
||||
attachments,
|
||||
{
|
||||
skills,
|
||||
threadId,
|
||||
cwd: workspace.workspacePath,
|
||||
onThreadId: (nextThreadId) => updateCanvasWorkspace(
|
||||
@@ -204,6 +214,23 @@ function jsonRecord(req: Request) {
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function parseAgentSkills(value: unknown) {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.slice(0, 8).flatMap((item) => {
|
||||
if (!item || typeof item !== "object" || Array.isArray(item)) return [];
|
||||
const input = item as Record<string, unknown>;
|
||||
const name = typeof input.name === "string" ? input.name.trim().slice(0, 120) : "";
|
||||
const instruction = typeof input.instruction === "string" ? input.instruction.trim().slice(0, 24_000) : "";
|
||||
if (!name || !instruction) return [];
|
||||
return [{
|
||||
...(typeof input.skillId === "string" ? { skillId: input.skillId.trim().slice(0, 120) } : {}),
|
||||
name,
|
||||
...(typeof input.description === "string" ? { description: input.description.trim().slice(0, 500) } : {}),
|
||||
instruction,
|
||||
}];
|
||||
});
|
||||
}
|
||||
|
||||
function queryValue(req: Request, key: string) {
|
||||
const value = req.query[key];
|
||||
return Array.isArray(value) ? String(value[0] ?? "") : String(value ?? "");
|
||||
|
||||
@@ -13,11 +13,15 @@ export const toolNames = [
|
||||
"canvas_get_state",
|
||||
"canvas_get_context",
|
||||
"canvas_find_nodes",
|
||||
"canvas_get_node",
|
||||
"canvas_get_connection",
|
||||
"canvas_get_generation_tasks",
|
||||
"canvas_get_resources",
|
||||
"canvas_validate_ops",
|
||||
"canvas_get_selection",
|
||||
"canvas_export_snapshot",
|
||||
"canvas_apply_ops",
|
||||
"canvas_create_workflow",
|
||||
"canvas_create_node",
|
||||
"canvas_create_text_node",
|
||||
"canvas_create_text_nodes",
|
||||
@@ -52,7 +56,7 @@ export type ToolName = (typeof toolNames)[number];
|
||||
export const canvasOpSchema = z.discriminatedUnion("type", [
|
||||
z.object({ type: z.literal("add_node"), nodeType: nodeTypeSchema.optional(), id: z.string().optional(), title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), position: positionSchema.optional(), metadata: recordSchema.optional() }).passthrough(),
|
||||
z.object({ type: z.literal("update_node"), id: z.string(), patch: recordSchema.optional(), metadata: recordSchema.optional() }).passthrough(),
|
||||
z.object({ type: z.literal("delete_node"), id: z.string().optional(), ids: z.array(z.string()).optional() }).passthrough(),
|
||||
z.object({ type: z.literal("delete_node"), id: z.string().optional(), ids: z.array(z.string()).optional(), nodeType: nodeTypeSchema.optional() }).passthrough(),
|
||||
z.object({ type: z.literal("delete_connections"), id: z.string().optional(), ids: z.array(z.string()).optional(), all: z.boolean().optional() }).passthrough(),
|
||||
z.object({ type: z.literal("connect_nodes"), id: z.string().optional(), fromNodeId: z.string(), toNodeId: z.string(), fromHandleId: z.string().optional(), toHandleId: z.string().optional() }).passthrough(),
|
||||
z.object({ type: z.literal("set_viewport"), viewport: viewportSchema }).passthrough(),
|
||||
@@ -69,6 +73,30 @@ const textNodeSchema = z.object({
|
||||
height: z.number().optional(),
|
||||
});
|
||||
|
||||
const workflowNodeSchema = z.object({
|
||||
ref: z.string().min(1),
|
||||
kind: z.enum(["text", "script", "image", "video", "audio", "character_cards", "character_three_view", "storyboard_video"]),
|
||||
title: z.string().min(1),
|
||||
content: z.string().optional(),
|
||||
prompt: z.string().optional(),
|
||||
description: z.string().optional(),
|
||||
referenceRefs: z.array(z.string()).optional(),
|
||||
referenceNodeIds: z.array(z.string()).optional(),
|
||||
runGeneration: z.boolean().optional(),
|
||||
width: z.number().positive().optional(),
|
||||
height: z.number().positive().optional(),
|
||||
});
|
||||
const workflowSchema = z.object({
|
||||
title: z.string().optional(),
|
||||
description: z.string().optional(),
|
||||
nodes: z.array(workflowNodeSchema).min(1),
|
||||
edges: z.array(z.object({ from: z.string().min(1), to: z.string().min(1) })).optional(),
|
||||
direction: z.enum(["horizontal", "vertical"]).optional(),
|
||||
start: positionSchema.optional(),
|
||||
gap: z.number().min(48).optional(),
|
||||
autoRun: z.boolean().optional(),
|
||||
});
|
||||
|
||||
const generationOptionsSchema = z.object({
|
||||
model: z.string().optional(),
|
||||
size: z.string().optional(),
|
||||
@@ -96,11 +124,15 @@ export const toolInputSchemas = {
|
||||
canvas_get_state: z.object({}).passthrough(),
|
||||
canvas_get_context: z.object({}).passthrough(),
|
||||
canvas_find_nodes: z.object({ query: z.string().optional(), ids: z.array(z.string()).optional(), types: z.array(z.string()).optional(), statuses: z.array(z.string()).optional(), resourceOnly: z.boolean().optional(), limit: z.number().int().min(1).max(200).optional() }),
|
||||
canvas_get_node: z.object({ id: z.string().min(1) }),
|
||||
canvas_get_connection: z.object({ id: z.string().min(1) }),
|
||||
canvas_get_generation_tasks: z.object({ status: z.string().optional(), nodeIds: z.array(z.string()).optional(), limit: z.number().int().min(1).max(200).optional() }),
|
||||
canvas_get_resources: z.object({ nodeIds: z.array(z.string()).optional(), status: z.string().optional(), limit: z.number().int().min(1).max(300).optional() }),
|
||||
canvas_validate_ops: z.object({ ops: z.array(canvasOpSchema) }),
|
||||
canvas_get_selection: z.object({}).passthrough(),
|
||||
canvas_export_snapshot: z.object({}).passthrough(),
|
||||
canvas_apply_ops: z.object({ ops: z.array(canvasOpSchema), expectedStateHash: z.string().min(1).optional() }),
|
||||
canvas_apply_ops: z.object({ ops: z.array(canvasOpSchema), expectedRevision: z.number().int().nonnegative().optional(), expectedStateHash: z.string().min(1).optional() }),
|
||||
canvas_create_workflow: workflowSchema,
|
||||
canvas_create_node: z.object({ nodeType: nodeTypeSchema, title: z.string().optional(), x: z.number().optional(), y: z.number().optional(), width: z.number().optional(), height: z.number().optional(), metadata: recordSchema.optional() }),
|
||||
canvas_create_text_node: z.object({ text: z.string().optional(), x: z.number().optional(), y: z.number().optional(), title: z.string().optional(), width: z.number().optional(), height: z.number().optional() }),
|
||||
canvas_create_text_nodes: z.object({ items: z.array(textNodeSchema).min(1), x: z.number().optional(), y: z.number().optional(), gap: z.number().optional(), direction: z.enum(["row", "column"]).optional() }),
|
||||
@@ -135,11 +167,15 @@ export const toolDescriptions: Record<ToolName, string> = {
|
||||
canvas_get_state: "读取当前网页画布的节点、连线、选区和视口。",
|
||||
canvas_get_context: "读取面向 Agent 的语义化画布上下文:节点用途、连接关系、选区、资源引用、生成状态和状态哈希。优先于直接猜测节点 metadata。",
|
||||
canvas_find_nodes: "按标题、内容、提示词、类型、状态、资产或工作流检索真实节点,返回可用于后续写操作的节点 id。",
|
||||
canvas_get_node: "按真实节点 id 精确读取单个节点的语义、完整安全 metadata、资源状态和关联连线;找不到时明确返回未找到。",
|
||||
canvas_get_connection: "按真实连线 id 精确读取连线端点、端点节点摘要和 handle 信息;找不到时明确返回未找到。",
|
||||
canvas_get_generation_tasks: "读取当前画布节点已经绑定的生成任务观察状态,包括 taskId、节点、任务状态、进度、阶段和错误;这是画布快照观察,不会主动轮询上游。",
|
||||
canvas_get_resources: "读取当前画布引用的媒体资源清单,包括资源/资产引用、类型、尺寸、大小、时长和就绪状态;不会返回媒体 URL。",
|
||||
canvas_validate_ops: "在真正写入前校验批量操作中的节点 id、连接关系和参数,避免对不存在节点或错误资源执行操作。",
|
||||
canvas_get_selection: "读取当前网页画布选中的节点。",
|
||||
canvas_export_snapshot: "导出当前画布快照,用于理解布局。",
|
||||
canvas_apply_ops: "批量操作当前网页画布。ops 支持 add_node、update_node、delete_node、delete_connections、connect_nodes、set_viewport、select_nodes、run_generation。",
|
||||
canvas_create_workflow: "创建语义化工作流/流水线。character_cards 是角色拆分图片卡片,character_three_view 是角色三视图,storyboard_video 是分镜剧情视频。媒体节点必须有 prompt/content;已有素材使用先读取到的真实 node id 填入 referenceNodeIds。自动分配 id、根据真实节点尺寸布局、建立 edges/referenceRefs/referenceNodeIds 连线并返回可验证结果;不要用批量文本节点代替工作流。",
|
||||
canvas_create_node: "创建任意类型节点:text、script、image、video、audio、frame。适合创建脚本、媒体占位、背板或自定义 metadata 节点。",
|
||||
canvas_create_text_node: "在当前画布创建单个文本节点。",
|
||||
canvas_create_text_nodes: "批量创建文本节点,适合生成标题、段落、脚本、说明等内容块。",
|
||||
|
||||
@@ -3,6 +3,6 @@ export type Viewport = { x: number; y: number; k: number };
|
||||
export type CanvasNodeType = "image" | "text" | "script" | "config" | "video" | "audio" | "frame";
|
||||
export type CanvasNode = { id: string; type: CanvasNodeType; title?: string; position: Position; width: number; height: number; parentId?: string; metadata?: Record<string, unknown> };
|
||||
export type CanvasConnection = { id: string; fromNodeId: string; toNodeId: string; fromHandleId?: string; toHandleId?: string };
|
||||
export type CanvasSnapshot = { projectId?: string; domainProjectId?: string; title?: string; nodes?: CanvasNode[]; connections?: CanvasConnection[]; selectedNodeIds?: string[]; viewport?: Viewport; clientId?: string };
|
||||
export type CanvasSnapshot = { projectId?: string; domainProjectId?: string; title?: string; nodes?: CanvasNode[]; connections?: CanvasConnection[]; selectedNodeIds?: string[]; viewport?: Viewport; clientId?: string; revision?: number };
|
||||
export type AgentEmit = (type: string, payload: unknown) => void;
|
||||
export type AgentAttachment = { name?: string; type?: string; dataUrl?: string };
|
||||
|
||||
Reference in New Issue
Block a user