tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(agent): 创作能力 - 接通专业节点发现与插件执行基础

Browse Source
This commit is contained in:
王志伟
2026-09-08 20:56:57 +08:00
parent a594c8266f
commit 757d1dd2c7
29 changed files with 617 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
web/src/services/agent-capabilities.ts Normal file
View File

@@ -0,0 +1,52 @@
import { getNodeDefinition, listNodeDefinitions } from "@/lib/canvas/node-registry";
import { getRegisteredPlugin, listRegisteredPlugins } from "@/lib/plugins/plugin-registry";
import { isPluginEffectivelyEnabled } from "@/stores/use-plugin-store";
import type { CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops";
// Query live registries on every discovery; installation is not execution availability.
export function listAgentCapabilities(query = "") {
const nodes = listNodeDefinitions().filter((node) => !node.plugin || isPluginEffectivelyEnabled(node.plugin.pluginId)).map((node) => ({
id: node.type, name: node.label, source: node.plugin?.pluginId || "builtin",
scope: "canvas", actions: node.type === "script" ? ["create", "canvas_read_storyboard", "canvas_edit_storyboard"] : ["create", "read", "update"],
defaultMetadata: node.defaultMetadata, defaultSize: node.defaultSize,
inputKind: node.acceptsInputKind, schema: node.plugin?.schema,
execution: node.plugin ? "仅开放节点创建和数据更新；插件专用执行需提供动作适配器，不能用媒体生成替代。" : "通过已有画布工具操作；专业节点不等同于普通媒体生成。",
}));
const plugins = listRegisteredPlugins().filter((plugin) => isPluginEffectivelyEnabled(plugin.manifest.id)).map(({ manifest, agentActions, readAgentNode }) => ({
id: manifest.id, name: manifest.name, source: manifest.id, version: manifest.version,
description: manifest.description, contributions: manifest.contributes,
actions: (agentActions || []).map(({ id, description, inputSchema }) => ({ id, description, inputSchema })),
readTool: readAgentNode ? "canvas_read_plugin_node" : undefined,
execution: agentActions?.length ? "使用 canvas_plugin_action 调用 actions 中的真实动作。" : "贡献声明是发现信息，不是任意命令执行入口。只调用已挂载工具。",
}));
const terms = query.toLocaleLowerCase().trim().split(/\s+/).filter(Boolean);
return [...nodes, ...plugins, { id: "project-style", name: "项目画风", source: "builtin", actions: ["canvas_list_styles", "canvas_apply_style"], scope: "canvas", description: "读取系统或个人风格，通过同一保存入口应用画风并同步关联项目。" }].filter((item) => terms.every((term) => JSON.stringify(item).toLocaleLowerCase().includes(term)));
}
export function readAgentPluginNode(snapshot: CanvasAgentSnapshot, nodeId: string) {
const node = snapshot.nodes.find((item) => item.id === nodeId);
const owner = node && getNodeDefinition(node.type)?.plugin?.pluginId;
const plugin = owner && isPluginEffectivelyEnabled(owner) ? getRegisteredPlugin(owner) : undefined;
if (!node || !plugin?.readAgentNode || !plugin.manifest.permissions.includes("canvas.read")) throw new Error("当前插件未启用或未提供节点读取能力");
return { nodeId, pluginId: owner, version: plugin.manifest.version, data: plugin.readAgentNode(structuredClone(node), structuredClone(snapshot)) };
}
export function readAgentPluginDocumentation(pluginId: string) {
const plugin = getRegisteredPlugin(pluginId);
if (!plugin || !isPluginEffectivelyEnabled(pluginId)) throw new Error("插件不可用，请重新发现能力");
return { id: pluginId, version: plugin.manifest.version, documentation: plugin.manifest.documentation || plugin.manifest.description };
}
export function buildAgentPluginOperations(pluginId: string, actionId: string, input: Record<string, unknown>, snapshot: CanvasAgentSnapshot) {
const plugin = getRegisteredPlugin(pluginId);
if (!plugin || !isPluginEffectivelyEnabled(pluginId)) throw new Error("插件已停用或不可用");
const action = plugin.agentActions?.find((item) => item.id === actionId);
if (!action) throw new Error("插件未提供此 Agent 动作，请先读取能力目录");
if (!plugin.manifest.permissions.includes("canvas.write")) throw new Error("插件未声明画布写入权限");
const operations = action.buildOperations(structuredClone(input), structuredClone(snapshot));
if (!Array.isArray(operations) || !operations.length || operations.length > 100) throw new Error("插件必须返回 1 至 100 个画布操作");
if (operations.some((op) => op.type === "run_generation") && !plugin.manifest.permissions.includes("generation.run")) throw new Error("插件未声明生成权限");
return operations;
}
export const AGENT_CAPABILITY_GUIDANCE = "处理专业任务前先调用 canvas_list_capabilities 查找系统现有节点和已启用插件，并用 canvas_list_skills 检索相关专业技能、读取 SKILL.md 后按需使用；用户不必手动选择技能。不要用普通文本模拟分镜表或插件结果。发现信息不代表已经执行，未挂载的专用动作须如实说明。已授权范围内自主推进，不重复征求技能读取或普通操作许可。";

5
web/src/services/api/generation-task.ts
View File

@@ -143,6 +143,8 @@ type BackendToolGenerationOptions = {
toolChoice: ToolChoice;
signal?: AbortSignal;
onDelta?: (text: string) => void;
onTaskCreated?: (task: GenerationTask) => void;
metadata?: { source: string; nodeId?: string; runId?: string; stage?: string };
};
// 报价和执行复用完全相同的任务协议，准备阶段不提交模型任务。
@@ -174,7 +176,7 @@ export function prepareBackendToolGenerationTask(options: BackendToolGenerationO
config: backendProviderConfig(options.config, "text"),
agentRequests: buildBackendToolRequests(options.messages, options.tools, options.toolChoice, options.config),
referenceImages: [...imageKeys].map((storageKey) => ({ storageKey })),
metadata: { source: "canvas-online-agent" },
metadata: options.metadata || { source: "canvas-online-agent" },
},
};
if (new Blob([JSON.stringify(task)]).size > 15 * 1024 * 1024) {
@@ -185,6 +187,7 @@ export function prepareBackendToolGenerationTask(options: BackendToolGenerationO
export async function runBackendToolGenerationTask(options: BackendToolGenerationOptions): Promise<ToolResponseResult> {
const task = await createGenerationTask(prepareBackendToolGenerationTask(options));
options.onTaskCreated?.(task);
const completed = await waitForGenerationTask(task.id, { signal: options.signal, initialTask: task, onTextDelta: options.onDelta });
const result = parseBackendGenerationResult(completed);
return {

52
web/src/services/art-critique-execution.ts Normal file
View File

@@ -0,0 +1,52 @@
import { inspectAgentImage } from "./agent-image-preview";
import { runBackendToolGenerationTask } from "./api/generation-task";
import { runArtCritiquePipeline, type ArtCritiquePipelineOptions } from "@/lib/art-critique/pipeline";
import { artCritiqueSourceFingerprint, isArtCritiqueImageInput } from "@/lib/art-critique/contracts";
import type { CanvasNodeData } from "@/types/canvas";
import type { AiConfig } from "@/stores/use-config-store";
import { getActiveUserScope } from "@/lib/user-scope";
export const ART_CRITIQUE_MAX_MODEL_CALLS = 9;
// One execution boundary for manual UI and future approved Agent runs.
// The executor may be replaced by an approval-aware task submitter; parsing stays in the pipeline.
export async function executeArtCritique(input: {
nodeId: string;
runId: string;
source: CanvasNodeData;
config: AiConfig;
signal: AbortSignal;
onTaskCreated: (stage: string, taskId: string) => void;
onStage?: ArtCritiquePipelineOptions["onStage"];
onDraftReport?: ArtCritiquePipelineOptions["onDraftReport"];
}, dependencies = { inspectImage: inspectAgentImage, runTask: runBackendToolGenerationTask, runPipeline: runArtCritiquePipeline }) {
if (!input.nodeId || !input.runId || !isArtCritiqueImageInput(input.source)) throw new Error("分析需要真实目标节点、执行标识和已有图片");
const scope = getActiveUserScope();
const assertLive = () => {
if (input.signal.aborted || getActiveUserScope() !== scope) throw new DOMException("分析已停止或账号已切换", "AbortError");
};
assertLive();
const content = input.source.metadata?.content || input.source.metadata?.previewContent || "";
const image = await dependencies.inspectImage({ storageKey: input.source.metadata?.storageKey, ...(content.startsWith("data:") ? { dataUrl: content } : { url: content }), name: input.source.title }, "original");
assertLive();
const calledStages = new Set<string>();
return dependencies.runPipeline(input.config, { dataUrl: image.storageKey, title: input.source.title, sourceFingerprint: artCritiqueSourceFingerprint(input.source) }, {
signal: input.signal,
onStage: (stage) => { assertLive(); input.onStage?.(stage); },
onDraftReport: (report) => { assertLive(); input.onDraftReport?.(report); },
requestStage: async ({ config, messages, tool, toolName, signal }) => {
assertLive();
if (calledStages.has(toolName) || calledStages.size >= ART_CRITIQUE_MAX_MODEL_CALLS) throw new Error("分析阶段重复或超过本次调用数量上限");
calledStages.add(toolName);
const result = await dependencies.runTask({
config, messages, tools: [tool], toolChoice: { type: "function", name: toolName },
prompt: `图片审美分析：${toolName}`, signal,
metadata: { source: "art-critique", nodeId: input.nodeId, runId: input.runId, stage: toolName },
// A task can be accepted while the user stops observing; retain that receipt.
onTaskCreated: (task) => { if (getActiveUserScope() === scope) input.onTaskCreated(toolName, task.id); },
});
assertLive();
return result;
},
});
}

12
web/src/services/skill-runtime.ts
View File

@@ -130,7 +130,7 @@ function functionTool(name: string, description: string, properties: Record<stri
}
const PROGRESSIVE_SKILL_TOOLS: ResponseFunctionTool[] = [
functionTool("canvas_list_skills", "列出当前用户已加入、可按需加载的画布技能；只返回元数据，不返回完整指令。", {}),
functionTool("canvas_list_skills", "自动发现当前用户已加入的技能；可按 query 检索名称、描述或标签，offset/limit 分页；只返回元数据。", { query: { type: "string" }, offset: { type: "number" }, limit: { type: "number" } }),
functionTool("canvas_get_skill", "按 skillId 或名称读取技能元数据和入口 SKILL.md；不会加载 references、scripts、assets 等其他文件。", { skillId: { type: "string" }, name: { type: "string" } }),
functionTool("canvas_list_skill_files", "列出一个技能包的文件路径、类型和大小，不读取文件正文。", { skillId: { type: "string" }, name: { type: "string" } }),
functionTool("canvas_read_skill_file", "按路径读取技能包中的一个文本文件。先从入口引用或文件清单确定路径，禁止猜测路径。", { skillId: { type: "string" }, name: { type: "string" }, path: { type: "string" } }, ["path"]),
@@ -343,9 +343,13 @@ function nativePackage(skill: Skill, bundle: SkillPackageBundle): NativeSkillPac
function createProgressiveToolsAdapter(dependencies: SkillRuntimeDependencies): SkillToolAdapter {
const handlers: Record<string, (args: Record<string, unknown>, skills: Skill[]) => Promise<SkillRuntimeToolResult>> = {
canvas_list_skills: async (_args, skills) => {
const data = skills.filter((skill) => skill.is_added).map((skill) => ({ skillId: skill.skill_id, name: skill.skill_name, description: skill.description, tag: skill.tag, version: skill.version, fileCount: skill.file_count, sourceType: skill.source_type }));
return { ok: true, message: data.length ? "已列出当前可用技能。" : "当前没有已加入技能。", data };
canvas_list_skills: async (args, skills) => {
const terms = typeof args.query === "string" ? args.query.trim().toLocaleLowerCase().split(/\s+/).filter(Boolean) : [];
const matches = skills.filter((skill) => skill.is_added && terms.every((term) => `${skill.skill_name} ${skill.description} ${skill.tag}`.toLocaleLowerCase().includes(term)));
const offset = typeof args.offset === "number" && Number.isFinite(args.offset) ? Math.max(0, Math.floor(args.offset)) : 0;
const limit = typeof args.limit === "number" && Number.isFinite(args.limit) ? Math.max(1, Math.min(100, Math.floor(args.limit))) : 40;
const items = matches.slice(offset, offset + limit).map((skill) => ({ skillId: skill.skill_id, name: skill.skill_name, description: skill.description, tag: skill.tag, version: skill.version, fileCount: skill.file_count, sourceType: skill.source_type }));
return { ok: true, message: items.length ? "已发现相关技能，请按需读取入口；无匹配时可缩短关键词或查询完整目录。" : "没有匹配技能，可调整关键词。", data: { items, total: matches.length, nextOffset: offset + items.length < matches.length ? offset + items.length : null } };
},
canvas_get_skill: async (args, skills) => {
const skill = requireAddedSkill(skills, args);