198 lines
13 KiB
TypeScript
198 lines
13 KiB
TypeScript
import { type ResponseFunctionTool } from "@/services/api/image";
|
||
import { skillRuntime } from "@/services/skill-runtime";
|
||
import { CREATIVE_AGENT_TOOLS } from "@/lib/creation/creative-agent-tools";
|
||
|
||
const JSON_RECORD_SCHEMA = { type: "object", additionalProperties: true };
|
||
const POSITION_SCHEMA = { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false };
|
||
const VIEWPORT_SCHEMA = { type: "object", properties: { x: { type: "number" }, y: { type: "number" }, k: { type: "number" } }, required: ["x", "y", "k"], additionalProperties: false };
|
||
const NODE_TYPE_SCHEMA = { type: "string", enum: ["image", "text", "skill", "video", "audio"] };
|
||
const WORKFLOW_NODE_KIND_SCHEMA = { type: "string", enum: ["text", "script", "image", "video", "audio", "character_cards", "character_three_view", "storyboard_video"] };
|
||
const GENERATION_MODE_SCHEMA = { type: "string", enum: ["text", "image", "video", "audio"] };
|
||
const GENERATION_OPTION_PROPERTIES = {
|
||
model: { type: "string" },
|
||
size: { type: "string" },
|
||
quality: { type: "string" },
|
||
transparentBackground: { type: "string", enum: ["true", "false"] },
|
||
count: { type: "number" },
|
||
seconds: { type: "string" },
|
||
vquality: { type: "string" },
|
||
generateAudio: { type: "string" },
|
||
watermark: { type: "string" },
|
||
audioVoice: { type: "string" },
|
||
audioFormat: { type: "string" },
|
||
audioSpeed: { type: "string" },
|
||
audioInstructions: { type: "string" },
|
||
};
|
||
const CANVAS_OP_SCHEMA = {
|
||
type: "object",
|
||
properties: {
|
||
type: { type: "string", enum: ["add_node", "update_node", "delete_node", "delete_connections", "connect_nodes", "set_viewport", "select_nodes", "run_generation"] },
|
||
id: { type: "string" },
|
||
ids: { type: "array", items: { type: "string" } },
|
||
nodeType: NODE_TYPE_SCHEMA,
|
||
title: { type: "string" },
|
||
x: { type: "number" },
|
||
y: { type: "number" },
|
||
width: { type: "number" },
|
||
height: { type: "number" },
|
||
position: POSITION_SCHEMA,
|
||
metadata: JSON_RECORD_SCHEMA,
|
||
patch: JSON_RECORD_SCHEMA,
|
||
all: { type: "boolean" },
|
||
fromNodeId: { type: "string" },
|
||
toNodeId: { type: "string" },
|
||
viewport: VIEWPORT_SCHEMA,
|
||
nodeId: { type: "string" },
|
||
mode: GENERATION_MODE_SCHEMA,
|
||
prompt: { type: "string" },
|
||
retry: { type: "boolean" },
|
||
},
|
||
required: ["type"],
|
||
additionalProperties: false,
|
||
};
|
||
|
||
function toolDefinition(name: string, description: string, properties: Record<string, unknown>, required: string[] = [], strict = false): ResponseFunctionTool {
|
||
return { type: "function", function: { name, description, parameters: { type: "object", properties, required, additionalProperties: false }, strict } };
|
||
}
|
||
|
||
function generationToolDefinition(name: string, description: string, mode?: "text" | "image" | "video" | "audio") {
|
||
return toolDefinition(
|
||
name,
|
||
description,
|
||
{
|
||
prompt: { type: "string" },
|
||
title: { type: "string" },
|
||
x: { type: "number" },
|
||
y: { type: "number" },
|
||
referenceNodeIds: { type: "array", items: { type: "string" } },
|
||
...(mode ? {} : { mode: GENERATION_MODE_SCHEMA }),
|
||
autoRun: { type: "boolean" },
|
||
...GENERATION_OPTION_PROPERTIES,
|
||
},
|
||
["prompt"],
|
||
);
|
||
}
|
||
|
||
export const CANVAS_ONLINE_AGENT_TOOLS: ResponseFunctionTool[] = [
|
||
...CREATIVE_AGENT_TOOLS,
|
||
...skillRuntime.agentTools("onlineAgent"),
|
||
toolDefinition("canvas_get_state", "读取当前网页画布的节点、连线、选区和视口。", {}),
|
||
toolDefinition("canvas_get_context", "读取语义化画布上下文、真实节点 id、连接关系、资源就绪状态和状态哈希。", {}),
|
||
toolDefinition("canvas_find_nodes", "按标题、内容、提示词、类型、状态或资产检索真实节点。", { query: { type: "string" }, ids: { type: "array", items: { type: "string" } }, types: { type: "array", items: { type: "string" } }, statuses: { type: "array", items: { type: "string" } }, resourceOnly: { type: "boolean" }, limit: { type: "number" } }),
|
||
toolDefinition("canvas_get_node", "按真实节点 id 精确读取单个节点、资源状态和关联连线。", { id: { type: "string" } }, ["id"]),
|
||
toolDefinition("canvas_get_connection", "按真实连线 id 精确读取端点节点和 handle 信息。", { id: { type: "string" } }, ["id"]),
|
||
toolDefinition("canvas_get_generation_tasks", "读取当前画布绑定的生成任务观察状态，不主动轮询上游。", { status: { type: "string" }, nodeIds: { type: "array", items: { type: "string" } }, limit: { type: "number" } }),
|
||
toolDefinition("canvas_get_resources", "读取画布媒体资源引用、类型、尺寸、大小、时长和就绪状态，不返回媒体 URL。", { nodeIds: { type: "array", items: { type: "string" } }, status: { type: "string" }, limit: { type: "number" } }),
|
||
toolDefinition("canvas_validate_ops", "在写入前校验节点 id、连接关系和批量操作参数。", { ops: { type: "array", items: CANVAS_OP_SCHEMA } }, ["ops"]),
|
||
toolDefinition("canvas_get_selection", "读取当前网页画布选中的节点。", {}),
|
||
toolDefinition("canvas_export_snapshot", "导出当前画布快照，用于理解布局。", {}),
|
||
toolDefinition(
|
||
"canvas_apply_ops",
|
||
"批量操作当前网页画布。复杂写操作应先 canvas_validate_ops；可传 canvas_get_context 返回的 expectedStateHash 防止基于过期状态写入。",
|
||
{ ops: { type: "array", items: CANVAS_OP_SCHEMA }, expectedRevision: { type: "number" }, expectedStateHash: { type: "string" } },
|
||
["ops"],
|
||
false,
|
||
),
|
||
toolDefinition(
|
||
"canvas_create_workflow",
|
||
"创建语义化工作流/流水线：节点使用真实的文本、脚本、图片、视频或音频类型；character_cards=角色拆分图片卡片，character_three_view=角色三视图，storyboard_video=分镜剧情视频。工具会自动生成唯一 id、按节点实际尺寸布局、创建 edges/referenceRefs/referenceNodeIds 连线、选择新节点并复核重叠。媒体节点必须提供有意义的 prompt 或 content；已有素材先 canvas_find_nodes/canvas_get_resources，再把真实 node id 放入 referenceNodeIds。不要用 canvas_create_text_nodes 代替工作流。",
|
||
{
|
||
title: { type: "string" },
|
||
description: { type: "string" },
|
||
nodes: {
|
||
type: "array",
|
||
minItems: 1,
|
||
items: {
|
||
type: "object",
|
||
properties: {
|
||
ref: { type: "string" },
|
||
kind: WORKFLOW_NODE_KIND_SCHEMA,
|
||
title: { type: "string" },
|
||
content: { type: "string" },
|
||
prompt: { type: "string" },
|
||
description: { type: "string" },
|
||
referenceRefs: { type: "array", items: { type: "string" } },
|
||
referenceNodeIds: { type: "array", items: { type: "string" } },
|
||
runGeneration: { type: "boolean" },
|
||
width: { type: "number" },
|
||
height: { type: "number" },
|
||
},
|
||
required: ["ref", "kind", "title"],
|
||
additionalProperties: false,
|
||
},
|
||
},
|
||
edges: { type: "array", items: { type: "object", properties: { from: { type: "string" }, to: { type: "string" } }, required: ["from", "to"], additionalProperties: false } },
|
||
direction: { type: "string", enum: ["horizontal", "vertical"] },
|
||
start: POSITION_SCHEMA,
|
||
gap: { type: "number" },
|
||
autoRun: { type: "boolean" },
|
||
},
|
||
["nodes"],
|
||
),
|
||
toolDefinition(
|
||
"canvas_create_node",
|
||
"创建任意类型节点：text、image、video、audio。适合创建文本、媒体占位或自定义 metadata 节点。",
|
||
{ nodeType: NODE_TYPE_SCHEMA, title: { type: "string" }, x: { type: "number" }, y: { type: "number" }, width: { type: "number" }, height: { type: "number" }, metadata: JSON_RECORD_SCHEMA },
|
||
["nodeType"],
|
||
),
|
||
toolDefinition("canvas_create_text_node", "在当前画布创建单个文本节点。", { text: { type: "string" }, x: { type: "number" }, y: { type: "number" }, title: { type: "string" }, width: { type: "number" }, height: { type: "number" } }),
|
||
toolDefinition(
|
||
"canvas_create_text_nodes",
|
||
"批量创建文本节点，适合生成标题、段落、脚本、说明等内容块。",
|
||
{
|
||
items: {
|
||
type: "array",
|
||
minItems: 1,
|
||
items: {
|
||
type: "object",
|
||
properties: { text: { type: "string" }, title: { type: "string" }, x: { type: "number" }, y: { type: "number" }, width: { type: "number" }, height: { type: "number" } },
|
||
required: ["text"],
|
||
additionalProperties: false,
|
||
},
|
||
},
|
||
x: { type: "number" },
|
||
y: { type: "number" },
|
||
gap: { type: "number" },
|
||
direction: { type: "string", enum: ["row", "column"] },
|
||
},
|
||
["items"],
|
||
),
|
||
toolDefinition("canvas_create_cinematic_session", "把自然语言创作指令提交给后端影视 Agent 会话，后端拆解为剧本、场景、分镜、镜头和成片节点，并返回可写回画布的操作。", { prompt: { type: "string" } }, ["prompt"]),
|
||
toolDefinition(
|
||
"canvas_create_image_prompt_flow",
|
||
"创建提示词文本节点和图片目标节点并自动连线，可选择立即触发生图。",
|
||
{ prompt: { type: "string" }, x: { type: "number" }, y: { type: "number" }, autoRun: { type: "boolean" }, ...GENERATION_OPTION_PROPERTIES },
|
||
["prompt"],
|
||
),
|
||
generationToolDefinition("canvas_create_generation_flow", "创建通用生成流程：提示词文本节点、对应类型的生成目标节点和参考节点连线，可用于文案、生图、视频或音频。"),
|
||
generationToolDefinition("canvas_generate_text", "创建通用文本生成流程并立即触发生成。", "text"),
|
||
generationToolDefinition("canvas_generate_image", "创建通用图片生成流程并立即触发生成。", "image"),
|
||
generationToolDefinition("canvas_generate_video", "创建通用视频生成流程并立即触发生成。", "video"),
|
||
generationToolDefinition("canvas_generate_audio", "创建通用音频生成流程并立即触发生成。", "audio"),
|
||
toolDefinition("canvas_update_node", "更新节点基础字段或 metadata。", { id: { type: "string" }, patch: JSON_RECORD_SCHEMA, metadata: JSON_RECORD_SCHEMA }, ["id"]),
|
||
toolDefinition("canvas_update_node_text", "更新文本节点内容和标题。", { id: { type: "string" }, text: { type: "string" }, title: { type: "string" } }, ["id", "text"]),
|
||
toolDefinition(
|
||
"canvas_move_nodes",
|
||
"移动一个或多个节点，支持绝对坐标或 dx/dy 偏移。",
|
||
{
|
||
items: {
|
||
type: "array",
|
||
minItems: 1,
|
||
items: { type: "object", properties: { id: { type: "string" }, x: { type: "number" }, y: { type: "number" }, dx: { type: "number" }, dy: { type: "number" } }, required: ["id"], additionalProperties: false },
|
||
},
|
||
},
|
||
["items"],
|
||
),
|
||
toolDefinition("canvas_resize_node", "调整节点尺寸。", { id: { type: "string" }, width: { type: "number" }, height: { type: "number" }, freeResize: { type: "boolean" } }, ["id", "width", "height"]),
|
||
toolDefinition("canvas_delete_nodes", "删除指定节点及相关连线。", { ids: { type: "array", items: { type: "string" }, minItems: 1 } }, ["ids"]),
|
||
toolDefinition(
|
||
"canvas_connect_nodes",
|
||
"批量连接节点。",
|
||
{ connections: { type: "array", minItems: 1, items: { type: "object", properties: { fromNodeId: { type: "string" }, toNodeId: { type: "string" } }, required: ["fromNodeId", "toNodeId"], additionalProperties: false } } },
|
||
["connections"],
|
||
),
|
||
toolDefinition("canvas_select_nodes", "设置当前选中节点。", { ids: { type: "array", items: { type: "string" } } }, ["ids"]),
|
||
toolDefinition("canvas_set_viewport", "调整画布视口。", { viewport: VIEWPORT_SCHEMA }, ["viewport"]),
|
||
toolDefinition("canvas_run_generation", "触发指定节点生成；对已有生成任务明确重试时传 retry=true。", { nodeId: { type: "string" }, mode: GENERATION_MODE_SCHEMA, prompt: { type: "string" }, retry: { type: "boolean" } }, ["nodeId"]),
|
||
];
|