feat(canvas): 创作助手 - 合并创作对话、能力发现与画布接续
This commit is contained in:
ddCat
52
web/src/services/agent-capabilities.ts
Normal file
52
web/src/services/agent-capabilities.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
import { getNodeDefinition, listNodeDefinitions } from "@/lib/canvas/node-registry";
|
||||
import { getRegisteredPlugin, listRegisteredPlugins } from "@/lib/plugins/plugin-registry";
|
||||
import { isPluginEffectivelyEnabled } from "@/stores/use-plugin-store";
|
||||
import type { CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops";
|
||||
|
||||
// Query live registries on every discovery; installation is not execution availability.
|
||||
export function listAgentCapabilities(query = "") {
|
||||
const nodes = listNodeDefinitions().filter((node) => !node.plugin || isPluginEffectivelyEnabled(node.plugin.pluginId)).map((node) => ({
|
||||
id: node.type, name: node.label, source: node.plugin?.pluginId || "builtin",
|
||||
scope: "canvas", actions: node.type === "script" ? ["create", "canvas_read_storyboard", "canvas_edit_storyboard"] : ["create", "read", "update"],
|
||||
defaultMetadata: node.defaultMetadata, defaultSize: node.defaultSize,
|
||||
inputKind: node.acceptsInputKind, schema: node.plugin?.schema,
|
||||
execution: node.plugin ? "仅开放节点创建和数据更新；插件专用执行需提供动作适配器，不能用媒体生成替代。" : "通过已有画布工具操作；专业节点不等同于普通媒体生成。",
|
||||
}));
|
||||
const plugins = listRegisteredPlugins().filter((plugin) => isPluginEffectivelyEnabled(plugin.manifest.id)).map(({ manifest, agentActions, readAgentNode }) => ({
|
||||
id: manifest.id, name: manifest.name, source: manifest.id, version: manifest.version,
|
||||
description: manifest.description, contributions: manifest.contributes,
|
||||
actions: (agentActions || []).map(({ id, description, inputSchema }) => ({ id, description, inputSchema })),
|
||||
readTool: readAgentNode ? "canvas_read_plugin_node" : undefined,
|
||||
execution: agentActions?.length ? "使用 canvas_plugin_action 调用 actions 中的真实动作。" : "贡献声明是发现信息，不是任意命令执行入口。只调用已挂载工具。",
|
||||
}));
|
||||
const terms = query.toLocaleLowerCase().trim().split(/\s+/).filter(Boolean);
|
||||
return [...nodes, ...plugins, { id: "project-style", name: "项目画风", source: "builtin", actions: ["canvas_list_styles", "canvas_apply_style"], scope: "canvas", description: "读取系统或个人风格，通过同一保存入口应用画风并同步关联项目。" }].filter((item) => terms.every((term) => JSON.stringify(item).toLocaleLowerCase().includes(term)));
|
||||
}
|
||||
|
||||
export function readAgentPluginNode(snapshot: CanvasAgentSnapshot, nodeId: string) {
|
||||
const node = snapshot.nodes.find((item) => item.id === nodeId);
|
||||
const owner = node && getNodeDefinition(node.type)?.plugin?.pluginId;
|
||||
const plugin = owner && isPluginEffectivelyEnabled(owner) ? getRegisteredPlugin(owner) : undefined;
|
||||
if (!node || !plugin?.readAgentNode || !plugin.manifest.permissions.includes("canvas.read")) throw new Error("当前插件未启用或未提供节点读取能力");
|
||||
return { nodeId, pluginId: owner, version: plugin.manifest.version, data: plugin.readAgentNode(structuredClone(node), structuredClone(snapshot)) };
|
||||
}
|
||||
|
||||
export function readAgentPluginDocumentation(pluginId: string) {
|
||||
const plugin = getRegisteredPlugin(pluginId);
|
||||
if (!plugin || !isPluginEffectivelyEnabled(pluginId)) throw new Error("插件不可用，请重新发现能力");
|
||||
return { id: pluginId, version: plugin.manifest.version, documentation: plugin.manifest.documentation || plugin.manifest.description };
|
||||
}
|
||||
|
||||
export function buildAgentPluginOperations(pluginId: string, actionId: string, input: Record<string, unknown>, snapshot: CanvasAgentSnapshot) {
|
||||
const plugin = getRegisteredPlugin(pluginId);
|
||||
if (!plugin || !isPluginEffectivelyEnabled(pluginId)) throw new Error("插件已停用或不可用");
|
||||
const action = plugin.agentActions?.find((item) => item.id === actionId);
|
||||
if (!action) throw new Error("插件未提供此 Agent 动作，请先读取能力目录");
|
||||
if (!plugin.manifest.permissions.includes("canvas.write")) throw new Error("插件未声明画布写入权限");
|
||||
const operations = action.buildOperations(structuredClone(input), structuredClone(snapshot));
|
||||
if (!Array.isArray(operations) || !operations.length || operations.length > 100) throw new Error("插件必须返回 1 至 100 个画布操作");
|
||||
if (operations.some((op) => op.type === "run_generation") && !plugin.manifest.permissions.includes("generation.run")) throw new Error("插件未声明生成权限");
|
||||
return operations;
|
||||
}
|
||||
|
||||
export const AGENT_CAPABILITY_GUIDANCE = "处理专业任务前先调用 canvas_list_capabilities 查找系统现有节点和已启用插件，并用 canvas_list_skills 检索相关专业技能、读取 SKILL.md 后按需使用；用户不必手动选择技能。不要用普通文本模拟分镜表或插件结果。发现信息不代表已经执行，未挂载的专用动作须如实说明。已授权范围内自主推进，不重复征求技能读取或普通操作许可。";
|
||||
89
web/src/services/agent-image-preview.ts
Normal file
89
web/src/services/agent-image-preview.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import { imageToDataUrl } from "@/services/image-storage";
|
||||
import { resourceIdFromStorageKey, resourceStorageKey, uploadResourceFile } from "@/services/api/resources";
|
||||
import { getActiveUserScope } from "@/lib/user-scope";
|
||||
|
||||
// 预算只作用于视觉分析副本，不修改原素材。
|
||||
export async function agentImagePreviews(images: Parameters<typeof imageToDataUrl>[0][]): Promise<string[]> {
|
||||
if (!images.length) return [];
|
||||
const limit = Math.min(256 * 1024, Math.floor(2 * 1024 * 1024 / images.length));
|
||||
if (limit < 16 * 1024) throw new Error("本次参考图片过多，请分批选择图片后继续。");
|
||||
const previews: string[] = [];
|
||||
// 顺序解码，避免多张原图同时占用内存。
|
||||
for (const source of images) {
|
||||
const url = await imageToDataUrl(source);
|
||||
if (!url) throw new Error("参考图片无法读取，请重新选择图片。");
|
||||
const image = new Image();
|
||||
image.src = url;
|
||||
try { await image.decode(); }
|
||||
catch { throw new Error("参考图片无法解码，请重新选择图片。"); }
|
||||
const canvas = document.createElement("canvas");
|
||||
try {
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("无法准备看图预览，请刷新后重试。");
|
||||
let edge = Math.min(1600, Math.max(image.naturalWidth, image.naturalHeight));
|
||||
let preview = "";
|
||||
while (edge > 0) {
|
||||
const scale = Math.min(1, edge / Math.max(image.naturalWidth, image.naturalHeight));
|
||||
canvas.width = Math.max(1, Math.round(image.naturalWidth * scale));
|
||||
canvas.height = Math.max(1, Math.round(image.naturalHeight * scale));
|
||||
context.fillStyle = "#ffffff";
|
||||
context.fillRect(0, 0, canvas.width, canvas.height);
|
||||
context.drawImage(image, 0, 0, canvas.width, canvas.height);
|
||||
preview = canvas.toDataURL("image/jpeg", 0.82);
|
||||
if (preview.length <= limit) break;
|
||||
if (edge <= 128) break;
|
||||
edge = Math.floor(edge * 0.75);
|
||||
}
|
||||
if (!preview || preview.length > limit) throw new Error("参考图片体积过大，请减少本次选择的图片后继续。");
|
||||
previews.push(preview);
|
||||
} finally {
|
||||
canvas.width = 0;
|
||||
canvas.height = 0;
|
||||
image.src = "";
|
||||
}
|
||||
}
|
||||
return previews;
|
||||
}
|
||||
|
||||
type AgentImageCrop = { x: number; y: number; width: number; height: number };
|
||||
|
||||
export async function inspectAgentImage(source: Parameters<typeof imageToDataUrl>[0], detail: "preview" | "original" | "crop", crop?: AgentImageCrop) {
|
||||
const scope = getActiveUserScope();
|
||||
const dataUrl = await encodeAgentImage(source, detail, crop);
|
||||
if (scope !== getActiveUserScope()) throw new DOMException("账号已切换", "AbortError");
|
||||
if (detail === "original" && resourceIdFromStorageKey(source.storageKey)) return { storageKey: source.storageKey!, encodedBytes: dataUrl.length };
|
||||
const response = await fetch(dataUrl);
|
||||
if (!response.ok) throw new Error("读取看图素材失败");
|
||||
const blob = await response.blob();
|
||||
const digest = await crypto.subtle.digest("SHA-256", await blob.arrayBuffer());
|
||||
const hash = Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||
if (scope !== getActiveUserScope()) throw new DOMException("账号已切换", "AbortError");
|
||||
const resource = await uploadResourceFile(blob, "image", { fileName: `agent-inspect-${hash.slice(0, 12)}.${blob.type === "image/png" ? "png" : "jpg"}`, idempotencyKey: `agent-inspect:${hash}` });
|
||||
if (scope !== getActiveUserScope()) throw new DOMException("账号已切换", "AbortError");
|
||||
return { storageKey: resourceStorageKey(resource.id), encodedBytes: dataUrl.length };
|
||||
}
|
||||
|
||||
async function encodeAgentImage(source: Parameters<typeof imageToDataUrl>[0], detail: "preview" | "original" | "crop", crop?: AgentImageCrop) {
|
||||
if (detail === "preview") return (await agentImagePreviews([source]))[0];
|
||||
const url = await imageToDataUrl(source);
|
||||
if (!url) throw new Error("参考图片无法读取，请重新选择图片。");
|
||||
if (detail === "original") {
|
||||
if (url.length > 6 * 1024 * 1024) throw new Error("原图超过本次看图预算，请改用 crop 分区检查，不要降低细节精度。");
|
||||
return url;
|
||||
}
|
||||
if (!crop || ![crop.x, crop.y, crop.width, crop.height].every((value) => typeof value === "number" && Number.isFinite(value)) || crop.x < 0 || crop.y < 0 || crop.width <= 0 || crop.height <= 0 || crop.x + crop.width > 1 || crop.y + crop.height > 1) throw new Error("裁剪范围必须是 0 到 1 的归一化坐标，且完整位于原图内。");
|
||||
const image = new Image();
|
||||
image.src = url;
|
||||
await image.decode();
|
||||
const canvas = document.createElement("canvas");
|
||||
try {
|
||||
canvas.width = Math.max(1, Math.round(image.naturalWidth * crop.width));
|
||||
canvas.height = Math.max(1, Math.round(image.naturalHeight * crop.height));
|
||||
const context = canvas.getContext("2d");
|
||||
if (!context) throw new Error("无法读取图片局部，请刷新后重试。");
|
||||
context.drawImage(image, image.naturalWidth * crop.x, image.naturalHeight * crop.y, image.naturalWidth * crop.width, image.naturalHeight * crop.height, 0, 0, canvas.width, canvas.height);
|
||||
const result = canvas.toDataURL("image/png");
|
||||
if (result.length > 6 * 1024 * 1024) throw new Error("该区域超过本次看图预算，请缩小裁剪范围后继续检查。");
|
||||
return result;
|
||||
} finally { canvas.width = 0; canvas.height = 0; image.src = ""; }
|
||||
}
|
||||
43
web/src/services/api/creation-runs.ts
Normal file
43
web/src/services/api/creation-runs.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import { http } from "./request";
|
||||
import type { CanvasAgentOp } from "@/lib/canvas/canvas-agent-ops";
|
||||
import type { CanvasProject } from "@/stores/canvas/use-canvas-store";
|
||||
import type { CreateTaskInput, GenerationTask } from "./task-center";
|
||||
|
||||
export type CreationStatus = "idle" | "running" | "waiting_answer" | "waiting_proposal" | "waiting_canvas" | "waiting_payment" | "waiting_task" | "paused" | "completed" | "cancelled";
|
||||
export type CreationGuard = { executionEpoch: number; owner: string };
|
||||
export type CreationRun = {
|
||||
id: string; userId: string; canvasId?: string; revision: number;
|
||||
executionEpoch: number; executionOwner: string; leaseExpiresAt?: string;
|
||||
status: CreationStatus; state: Record<string, unknown>;
|
||||
approvedProposalVersion?: number; approvedProposalHash?: string;
|
||||
createdAt: string; updatedAt: string;
|
||||
};
|
||||
export type CreationSubmission = {
|
||||
id: string; runId: string; itemKey: string; requestHash: string;
|
||||
quote: { model: string; billingMode: string; quantity: number; amountMicrocredits: number; estimated: boolean; expiresAt: string; quoteHash: string; options?: Record<string, unknown> };
|
||||
approvedAt?: string; revokedAt?: string; taskId?: string;
|
||||
};
|
||||
export type CreationRunDetail = { run: CreationRun; submissions: CreationSubmission[] };
|
||||
|
||||
const path = (id: string) => `/creation-runs/${encodeURIComponent(id)}`;
|
||||
export const creationRuns = {
|
||||
create: (input: { clientKey: string; canvasId?: string; state: Record<string, unknown> }, signal?: AbortSignal) => http.post<CreationRunDetail>("/creation-runs", input, { signal }),
|
||||
get: (id: string, signal?: AbortSignal) => http.get<CreationRunDetail>(path(id), { signal }),
|
||||
list: (signal?: AbortSignal) => http.get<{ runs: CreationRun[] }>("/creation-runs", { signal }),
|
||||
save: (id: string, input: CreationGuard & { revision: number; state: Record<string, unknown>; status: CreationStatus }, signal?: AbortSignal) => http.patch<CreationRun>(path(id), input, { signal }),
|
||||
claim: (id: string, input: { expectedEpoch: number; owner: string }, signal?: AbortSignal) => http.post<CreationRun>(`${path(id)}/claim`, input, { signal }),
|
||||
heartbeat: (id: string, guard: CreationGuard, signal?: AbortSignal) => http.post<{ leaseExpiresAt: string }>(`${path(id)}/heartbeat`, guard, { signal }),
|
||||
release: (id: string, guard: CreationGuard) => http.post<{ released: boolean }>(`${path(id)}/release`, guard),
|
||||
approveProposal: (id: string, input: CreationGuard & { revision: number; proposalVersion: number; proposal: unknown; ops: CanvasAgentOp[] }, signal?: AbortSignal) => http.post<CreationRun>(`${path(id)}/proposal-approve`, input, { signal }),
|
||||
invalidateProposal: (id: string, input: CreationGuard & { revision: number }, signal?: AbortSignal) => http.post<CreationRun>(`${path(id)}/proposal-invalidate`, input, { signal }),
|
||||
canvas: (id: string, guard: CreationGuard, signal?: AbortSignal) => http.post<{ run: CreationRun; canvasId: string }>(`${path(id)}/canvas`, guard, { signal }),
|
||||
prepare: (id: string, input: CreationGuard & { itemKey: string; proposalVersion?: number; request: CreateTaskInput }, signal?: AbortSignal) => http.post<CreationSubmission>(`${path(id)}/submissions/prepare`, input, { signal }),
|
||||
approve: (id: string, input: CreationGuard & { submissionIds: string[] }, signal?: AbortSignal) => http.post<{ submissions: CreationSubmission[] }>(`${path(id)}/submissions/approve`, input, { signal }),
|
||||
refreshQuote: (id: string, input: CreationGuard & { submissionId: string }, signal?: AbortSignal) => http.post<CreationSubmission>(`${path(id)}/submissions/refresh`, input, { signal }),
|
||||
execute: (id: string, input: CreationGuard & { submissionId: string }, signal?: AbortSignal) => http.post<GenerationTask>(`${path(id)}/execute`, input, { signal }).then((task) => {
|
||||
if (typeof window !== "undefined") window.dispatchEvent(new CustomEvent("wallet:updated"));
|
||||
return task;
|
||||
}),
|
||||
canvasSnapshot: (id: string, signal?: AbortSignal) => http.get<{ document: CanvasProject; snapshotHash: string }>(`${path(id)}/canvas-snapshot`, { signal }),
|
||||
commitCanvas: (id: string, input: CreationGuard & { expectedSnapshotHash: string; document: CanvasProject }, signal?: AbortSignal) => http.post<{ snapshotHash: string }>(`${path(id)}/canvas-commit`, input, { signal }),
|
||||
};;
|
||||
@@ -1,7 +1,7 @@
|
||||
import { getMediaBlob } from "@/services/file-storage";
|
||||
import { getImageBlob } from "@/services/image-storage";
|
||||
import { resourceIdFromStorageKey, resourceStorageKey, uploadResourceFile } from "@/services/api/resources";
|
||||
import { createGenerationTask, waitForGenerationTask, type GenerationTask } from "@/services/api/task-center";
|
||||
import { createGenerationTask, waitForGenerationTask, type GenerationTask, type CreateTaskInput } from "@/services/api/task-center";
|
||||
import { LOCAL_DREAMINA_WAIT_STOPPED_CODE, LocalDreaminaGenerationClientError, runLocalDreaminaGenerationTask, type LocalDreaminaGenerationInput, type LocalDreaminaGenerationTask } from "@/services/local-dreamina-generation";
|
||||
import { isLocalDreaminaBackgroundTask, localDreaminaTaskId, projectLocalDreaminaTask, stripLocalDreaminaTaskPrefix } from "@/services/local-dreamina-task-projection";
|
||||
import { modelCapabilityConfigFor } from "@/lib/model-capabilities";
|
||||
@@ -13,6 +13,7 @@ import { useLocalDreaminaModelStore } from "@/stores/use-local-dreamina-model-st
|
||||
import type { ReferenceImage } from "@/types/image";
|
||||
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
|
||||
import { buildBackendToolRequests, type ResponseFunctionTool, type ResponseInputMessage, type ToolChoice, type ToolResponseResult } from "@/services/api/image";
|
||||
import { assertAgentExchangeBudget } from "@/lib/canvas/canvas-agent-context-budget";
|
||||
|
||||
export { logicalModelIDForConfig };
|
||||
|
||||
@@ -134,7 +135,7 @@ export async function submitBackendGenerationTask(
|
||||
return createBackendGenerationTask(options, prepared, dependencies);
|
||||
}
|
||||
|
||||
export async function runBackendToolGenerationTask(options: {
|
||||
type BackendToolGenerationOptions = {
|
||||
prompt: string;
|
||||
config: AiConfig;
|
||||
messages: ResponseInputMessage[];
|
||||
@@ -142,12 +143,28 @@ export async function runBackendToolGenerationTask(options: {
|
||||
toolChoice: ToolChoice;
|
||||
signal?: AbortSignal;
|
||||
onDelta?: (text: string) => void;
|
||||
}): Promise<ToolResponseResult> {
|
||||
onTaskCreated?: (task: GenerationTask) => void;
|
||||
metadata?: { source: string; nodeId?: string; runId?: string; stage?: string };
|
||||
};
|
||||
|
||||
// 报价和执行复用完全相同的任务协议，准备阶段不提交模型任务。
|
||||
export function prepareBackendToolGenerationTask(options: BackendToolGenerationOptions): CreateTaskInput {
|
||||
throwIfAborted(options.signal);
|
||||
assertAgentExchangeBudget(options.messages, options.tools, options.config.systemPrompt || "");
|
||||
const imageKeys = new Set<string>();
|
||||
for (const message of options.messages) {
|
||||
if ("type" in message || message.role === "tool" || !Array.isArray(message.content)) continue;
|
||||
for (const part of message.content) {
|
||||
if (part.type !== "image_url") continue;
|
||||
const key = part.image_url.url;
|
||||
if (!resourceIdFromStorageKey(key)) throw new Error("看图素材尚未保存为资源，请让助手重新读取图片后继续。");
|
||||
imageKeys.add(key);
|
||||
}
|
||||
}
|
||||
const logicalModelId = logicalModelIDForConfig(options.config);
|
||||
const requestConfig = resolveModelRequestConfig(options.config, options.config.model);
|
||||
if (!logicalModelId && !requestConfig.channelId && !requestConfig.interfaceType) throw new Error("当前模型未选择可用请求协议");
|
||||
const task = await createGenerationTask({
|
||||
const task: CreateTaskInput = {
|
||||
type: "canvas_text",
|
||||
operation: "text",
|
||||
prompt: options.prompt,
|
||||
@@ -156,11 +173,21 @@ export async function runBackendToolGenerationTask(options: {
|
||||
input: {
|
||||
mode: "text",
|
||||
prompt: options.prompt,
|
||||
config: backendProviderConfig(options.config),
|
||||
config: backendProviderConfig(options.config, "text"),
|
||||
agentRequests: buildBackendToolRequests(options.messages, options.tools, options.toolChoice, options.config),
|
||||
metadata: { source: "canvas-online-agent" },
|
||||
referenceImages: [...imageKeys].map((storageKey) => ({ storageKey })),
|
||||
metadata: options.metadata || { source: "canvas-online-agent" },
|
||||
},
|
||||
});
|
||||
};
|
||||
if (new Blob([JSON.stringify(task)]).size > 15 * 1024 * 1024) {
|
||||
throw new Error("本次图片或对话内容过多，请减少参考图片，或新建会话后继续。已有作品会保留。");
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
export async function runBackendToolGenerationTask(options: BackendToolGenerationOptions): Promise<ToolResponseResult> {
|
||||
const task = await createGenerationTask(prepareBackendToolGenerationTask(options));
|
||||
options.onTaskCreated?.(task);
|
||||
const completed = await waitForGenerationTask(task.id, { signal: options.signal, initialTask: task, onTextDelta: options.onDelta });
|
||||
const result = parseBackendGenerationResult(completed);
|
||||
return {
|
||||
@@ -425,11 +452,27 @@ async function createAndWaitGenerationTask(options: BackendGenerationTaskOptions
|
||||
}
|
||||
|
||||
async function createBackendGenerationTask(options: BackendGenerationTaskOptions, prepared: PreparedGenerationReferences, dependencies: GenerationTaskDependencies) {
|
||||
const { projectId, mode, prompt, config, metadata, onTaskUpdate } = options;
|
||||
const task = await dependencies.createTask(backendGenerationTaskInput(options, prepared));
|
||||
options.onTaskUpdate?.(task);
|
||||
return task;
|
||||
}
|
||||
|
||||
export async function prepareBackendGenerationTask(options: BackendGenerationTaskOptions): Promise<CreateTaskInput> {
|
||||
throwIfAborted(options.signal);
|
||||
if (usesLocalDreamina(options.config)) throw new Error("智能创作的在线授权暂不支持本机即梦，请使用现有本机入口");
|
||||
assertClientPromptLimit(options.mode, options.prompt, options.config, options.metadata);
|
||||
assertBackendRuntimeConfigured(options.config, options.mode);
|
||||
const prepared = await prepareGenerationReferences(options);
|
||||
throwIfAborted(options.signal);
|
||||
return backendGenerationTaskInput(options, prepared);
|
||||
}
|
||||
|
||||
function backendGenerationTaskInput(options: BackendGenerationTaskOptions, prepared: PreparedGenerationReferences): CreateTaskInput {
|
||||
const { projectId, mode, prompt, config, metadata } = options;
|
||||
const videoOperation = generationOperation(options);
|
||||
const workflow = resolveGenerationWorkflowExecution(config, mode);
|
||||
const logicalModelId = workflow ? "" : logicalModelIDForConfig(config);
|
||||
const task = await dependencies.createTask({
|
||||
return {
|
||||
...(projectId ? { projectId } : {}),
|
||||
type: `canvas_${mode}`,
|
||||
operation: mode === "video" ? videoOperation : mode,
|
||||
@@ -451,9 +494,7 @@ async function createBackendGenerationTask(options: BackendGenerationTaskOptions
|
||||
mask: prepared.mask,
|
||||
metadata: generationMetadata(config, metadata),
|
||||
},
|
||||
});
|
||||
onTaskUpdate?.(task);
|
||||
return task;
|
||||
};
|
||||
}
|
||||
|
||||
function generationMetadata(config: AiConfig, metadata?: Record<string, unknown>) {
|
||||
|
||||
@@ -33,10 +33,12 @@ export type ToolResponseResult = {
|
||||
export type ToolChoice = "auto" | "required" | { type: "function"; name: string };
|
||||
|
||||
export type BackendToolRequests = {
|
||||
responses: Record<string, unknown>;
|
||||
chatCompletion: Record<string, unknown>;
|
||||
claude?: Record<string, unknown>;
|
||||
gemini?: Record<string, unknown>;
|
||||
canonical: {
|
||||
messages: ResponseInputMessage[];
|
||||
tools: ResponseFunctionTool[];
|
||||
toolChoice: ToolChoice;
|
||||
systemPrompt: string;
|
||||
};
|
||||
};
|
||||
|
||||
export type ResponseMessageContent = AiTextMessage["content"] | string;
|
||||
|
||||
@@ -102,28 +102,14 @@ export function toChatCompletionToolChoice(toolChoice: ToolChoice) {
|
||||
}
|
||||
|
||||
export function buildBackendToolRequests(messages: ResponseInputMessage[], tools: ResponseFunctionTool[], toolChoice: ToolChoice, config?: AiConfig): BackendToolRequests {
|
||||
const requests: BackendToolRequests = {
|
||||
responses: {
|
||||
input: toResponseInput(messages),
|
||||
tools: tools.map(toResponseTool),
|
||||
tool_choice: toolChoice,
|
||||
parallel_tool_calls: false,
|
||||
},
|
||||
chatCompletion: {
|
||||
messages: toChatCompletionMessages(messages),
|
||||
return {
|
||||
canonical: {
|
||||
messages,
|
||||
tools,
|
||||
tool_choice: toChatCompletionToolChoice(toolChoice),
|
||||
parallel_tool_calls: false,
|
||||
toolChoice,
|
||||
systemPrompt: config?.systemPrompt || "",
|
||||
},
|
||||
};
|
||||
if (config) {
|
||||
requests.claude = {
|
||||
...toClaudeBody(config, messages, tools),
|
||||
tool_choice: typeof toolChoice === "object" ? { type: "tool", name: toolChoice.name } : { type: toolChoice === "required" ? "any" : "auto" },
|
||||
};
|
||||
requests.gemini = toGeminiBody(config, messages, toGeminiToolOptions(tools, toolChoice));
|
||||
}
|
||||
return requests;
|
||||
}
|
||||
|
||||
export function toGeminiBody(config: AiConfig, messages: ResponseInputMessage[], extra?: Record<string, unknown>) {
|
||||
|
||||
169
web/src/services/approved-tool-execution.ts
Normal file
169
web/src/services/approved-tool-execution.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
import { nanoid } from "nanoid";
|
||||
import { creationRuns, type CreationRun, type CreationSubmission } from "./api/creation-runs";
|
||||
import { prepareBackendToolGenerationTask, parseBackendGenerationResult, runBackendToolGenerationTask } from "./api/generation-task";
|
||||
import { waitForGenerationTask } from "./api/task-center";
|
||||
import type { CreativeQuote } from "@/lib/creation/creative-agent-contract";
|
||||
import { getActiveUserScope } from "@/lib/user-scope";
|
||||
import { logicalModelIDForConfig, resolveModelRequestConfig } from "@/stores/use-config-store";
|
||||
|
||||
type Request = Parameters<typeof runBackendToolGenerationTask>[0];
|
||||
type Pending = { submission: CreationSubmission; externalBilling: boolean; resolve: () => void; reject: (error: Error) => void };
|
||||
export type ApprovedToolExecutionState = { quote?: CreativeQuote; busy: boolean; error?: string };
|
||||
|
||||
// Only this boundary approves and executes quoted tasks. Pipeline code never grants approval.
|
||||
export class ApprovedToolExecution {
|
||||
private run?: CreationRun;
|
||||
private owner = `plugin:${nanoid()}`;
|
||||
private scope = getActiveUserScope();
|
||||
private pending = new Map<string, Pending>();
|
||||
private heartbeat?: ReturnType<typeof setInterval>;
|
||||
private disposed = false;
|
||||
private busy = false;
|
||||
private error?: string;
|
||||
private stopReason?: string;
|
||||
private ready?: Promise<void>;
|
||||
private controller = new AbortController();
|
||||
|
||||
constructor(private options: {
|
||||
clientKey: string; runId?: string; identity: Record<string, string>;
|
||||
signal: AbortSignal; onRun: (id: string) => void;
|
||||
onState: (state: ApprovedToolExecutionState) => void;
|
||||
}, private api = creationRuns, private waitTask = waitForGenerationTask) {
|
||||
options.signal.addEventListener("abort", this.abort, { once: true });
|
||||
}
|
||||
|
||||
private assertLive() {
|
||||
if (this.disposed || this.options.signal.aborted || this.scope !== getActiveUserScope() || this.controller.signal.aborted) throw new DOMException(this.stopReason || "分析已停止，请重新打开后继续", "AbortError");
|
||||
}
|
||||
private guard() { return { owner: this.owner, executionEpoch: this.run!.executionEpoch }; }
|
||||
private initialize() {
|
||||
return this.ready ||= (async () => {
|
||||
this.assertLive();
|
||||
const detail = this.options.runId ? await this.api.get(this.options.runId, this.controller.signal) : await this.api.create({ clientKey: this.options.clientKey, state: this.options.identity }, this.controller.signal);
|
||||
this.assertLive();
|
||||
if (Object.entries(this.options.identity).some(([key, value]) => detail.run.state[key] !== value)) throw new Error("原分析的图片、节点或模型已变化，请开始新的分析");
|
||||
this.run = await this.api.claim(detail.run.id, { expectedEpoch: detail.run.executionEpoch, owner: this.owner }, this.controller.signal);
|
||||
this.assertLive();
|
||||
this.options.onRun(this.run.id);
|
||||
this.heartbeat = setInterval(() => {
|
||||
if (this.disposed) return;
|
||||
void (async () => { this.assertLive(); await this.api.heartbeat(this.run!.id, this.guard(), this.controller.signal); })().catch(() => this.abort(new Error("分析连接已中断，请重新打开后继续")));
|
||||
}, 15_000);
|
||||
})();
|
||||
}
|
||||
|
||||
execute = async (request: Request) => {
|
||||
// A control/quote failure must also unblock sibling stages waiting for approval.
|
||||
// Model task failures below keep the pipeline's existing partial-result handling.
|
||||
const task = await this.submit(request).catch((error: unknown) => {
|
||||
this.abort(error);
|
||||
throw new DOMException(error instanceof Error ? error.message : "分析提交失败，请重新打开后继续", "AbortError");
|
||||
});
|
||||
if (this.scope === getActiveUserScope()) request.onTaskCreated?.(task);
|
||||
this.assertLive();
|
||||
const completed = await this.waitTask(task.id, { initialTask: task, signal: this.controller.signal, onTextDelta: request.onDelta });
|
||||
this.assertLive();
|
||||
const result = parseBackendGenerationResult(completed);
|
||||
return { content: result.text || "", toolCalls: result.toolCalls || [], ...(result.reasoning ? { reasoning: result.reasoning } : {}) };
|
||||
};
|
||||
|
||||
private async submit(request: Request) {
|
||||
await this.initialize();
|
||||
this.assertLive();
|
||||
const stage = request.metadata?.stage;
|
||||
if (!stage) throw new Error("分析阶段缺少稳定标识");
|
||||
let submission = await this.api.prepare(this.run!.id, { ...this.guard(), itemKey: `stage:${stage}`, request: prepareBackendToolGenerationTask(request) }, this.controller.signal);
|
||||
this.assertLive();
|
||||
if (submission.revokedAt) {
|
||||
const detail = await this.api.get(this.run!.id, this.controller.signal);
|
||||
submission = detail.submissions.find((item) => item.requestHash === submission.requestHash && !item.revokedAt) || submission;
|
||||
if (submission.revokedAt) throw new Error("原报价已失效，无法恢复此阶段");
|
||||
}
|
||||
if (!submission.taskId && Date.parse(submission.quote.expiresAt) <= Date.now()) submission = await this.api.refreshQuote(this.run!.id, { ...this.guard(), submissionId: submission.id }, this.controller.signal);
|
||||
this.assertLive();
|
||||
if (!submission.taskId && !submission.approvedAt) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const externalBilling = !logicalModelIDForConfig(request.config) && !resolveModelRequestConfig(request.config, request.config.model).channelId;
|
||||
this.pending.set(stage, { submission, externalBilling, resolve, reject });
|
||||
this.emit();
|
||||
});
|
||||
submission = this.pending.get(stage)!.submission;
|
||||
this.pending.delete(stage);
|
||||
this.emit();
|
||||
}
|
||||
this.assertLive();
|
||||
return this.api.execute(this.run!.id, { ...this.guard(), submissionId: submission.id }, this.controller.signal);
|
||||
}
|
||||
|
||||
async approve(submissionIds: string[]) {
|
||||
if (this.busy || this.disposed || this.controller.signal.aborted) return;
|
||||
const ids = new Set(submissionIds);
|
||||
const batch = [...this.pending.values()].filter((item) => ids.has(item.submission.id) && !item.submission.approvedAt);
|
||||
if (batch.length !== ids.size) { this.error = "报价已变化，请核对当前费用卡后确认"; this.emit(); return; }
|
||||
if (!batch.length) return;
|
||||
this.busy = true; this.error = undefined; this.emit();
|
||||
try {
|
||||
this.assertLive();
|
||||
const result = await this.api.approve(this.run!.id, { ...this.guard(), submissionIds: batch.map((item) => item.submission.id) }, this.controller.signal);
|
||||
this.assertLive();
|
||||
if (batch.some((item) => !result.submissions.some((candidate) => candidate.id === item.submission.id && candidate.approvedAt))) {
|
||||
const error = new Error("费用确认回执不完整，请继续上次分析以读取实际状态");
|
||||
this.abort(error);
|
||||
throw error;
|
||||
}
|
||||
for (const item of batch) {
|
||||
const approved = result.submissions.find((candidate) => candidate.id === item.submission.id && candidate.approvedAt);
|
||||
if (!approved) throw new Error("费用确认结果不完整，请重新核对");
|
||||
item.submission = approved;
|
||||
}
|
||||
batch.forEach((item) => item.resolve());
|
||||
} catch (error) { this.error = error instanceof Error ? error.message : "费用确认失败"; }
|
||||
finally { this.busy = false; this.emit(); }
|
||||
}
|
||||
|
||||
async refresh() {
|
||||
if (this.busy || this.disposed || this.controller.signal.aborted) return;
|
||||
this.busy = true; this.error = undefined; this.emit();
|
||||
try {
|
||||
this.assertLive();
|
||||
for (const item of [...this.pending.values()]) {
|
||||
if (item.submission.approvedAt) continue;
|
||||
item.submission = await this.api.refreshQuote(this.run!.id, { ...this.guard(), submissionId: item.submission.id }, this.controller.signal);
|
||||
this.assertLive();
|
||||
}
|
||||
} catch (error) { this.error = error instanceof Error ? error.message : "报价刷新失败"; }
|
||||
finally { this.busy = false; this.emit(); }
|
||||
}
|
||||
|
||||
private emit() {
|
||||
if (this.disposed || this.scope !== getActiveUserScope()) return;
|
||||
const items = [...this.pending.entries()].filter(([, item]) => !item.submission.approvedAt).map(([stage, item]) => ({ ...item.submission, stage }));
|
||||
const externalBilling = [...this.pending.values()].some((item) => !item.submission.approvedAt && item.externalBilling);
|
||||
const estimated = items.some((item) => item.quote.estimated);
|
||||
this.options.onState({ busy: this.busy, error: this.error, quote: items.length ? {
|
||||
id: items.map((item) => item.id).join(":"), title: "确认本批分析费用",
|
||||
items: items.map((item) => ({ id: item.id, label: stageLabel(item.stage), model: item.quote.model, quantity: item.quote.quantity, specification: `${item.quote.estimated ? "预计" : ""} ${item.quote.amountMicrocredits / 1_000_000} 积分` })),
|
||||
amountLabel: `${externalBilling ? "平台 " : ""}${estimated ? "预计 " : ""}${items.reduce((sum, item) => sum + item.quote.amountMicrocredits, 0) / 1_000_000} 积分${externalBilling ? " + 外部渠道费用" : ""}`,
|
||||
externalBilling,
|
||||
basis: "仅批准本批列出的模型调用；按用量计费时按实际结算，估算不是费用上限。后续阶段根据分析结果另行报价。",
|
||||
expiresAt: items.map((item) => item.quote.expiresAt).sort()[0],
|
||||
} : undefined });
|
||||
}
|
||||
|
||||
private abort = (reason?: unknown) => {
|
||||
if (!this.controller.signal.aborted && reason instanceof Error) this.stopReason = reason.message;
|
||||
this.controller.abort();
|
||||
if (this.heartbeat) clearInterval(this.heartbeat);
|
||||
for (const item of this.pending.values()) item.reject(new DOMException(this.stopReason || "分析已停止", "AbortError"));
|
||||
};
|
||||
dispose() {
|
||||
this.disposed = true; this.abort();
|
||||
if (this.heartbeat) clearInterval(this.heartbeat);
|
||||
this.options.signal.removeEventListener("abort", this.abort);
|
||||
if (this.run && this.scope === getActiveUserScope()) void this.api.release(this.run.id, this.guard()).catch(() => { /* Lease expiry releases an offline owner. */ });
|
||||
}
|
||||
}
|
||||
|
||||
function stageLabel(key: string) {
|
||||
return ({ analyze_art_scene: "理解画面", review_art_composition: "分析构图", review_art_color: "分析色彩", review_art_lighting: "分析光线", review_art_structure: "分析结构", aggregate_art_critique: "整理问题", ground_art_critique_issues: "定位问题", verify_art_critique: "复核结论", generate_art_edit_prompts: "编写修改建议" } as Record<string, string>)[key.replace(/^stage:/, "")] || "分析阶段";
|
||||
}
|
||||
53
web/src/services/art-critique-execution.ts
Normal file
53
web/src/services/art-critique-execution.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
import { inspectAgentImage } from "./agent-image-preview";
|
||||
import { runBackendToolGenerationTask } from "./api/generation-task";
|
||||
import { runArtCritiquePipeline, type ArtCritiquePipelineOptions } from "@/lib/art-critique/pipeline";
|
||||
import { artCritiqueSourceFingerprint, isArtCritiqueImageInput } from "@/lib/art-critique/contracts";
|
||||
import type { CanvasNodeData } from "@/types/canvas";
|
||||
import type { AiConfig } from "@/stores/use-config-store";
|
||||
import { getActiveUserScope } from "@/lib/user-scope";
|
||||
|
||||
export const ART_CRITIQUE_MAX_MODEL_CALLS = 9;
|
||||
|
||||
// One execution boundary for manual UI and future approved Agent runs.
|
||||
// The executor may be replaced by an approval-aware task submitter; parsing stays in the pipeline.
|
||||
export async function executeArtCritique(input: {
|
||||
nodeId: string;
|
||||
runId: string;
|
||||
source: CanvasNodeData;
|
||||
config: AiConfig;
|
||||
signal: AbortSignal;
|
||||
onTaskCreated: (stage: string, taskId: string) => void;
|
||||
onStage?: ArtCritiquePipelineOptions["onStage"];
|
||||
onDraftReport?: ArtCritiquePipelineOptions["onDraftReport"];
|
||||
submitStage?: typeof runBackendToolGenerationTask;
|
||||
}, dependencies = { inspectImage: inspectAgentImage, runTask: runBackendToolGenerationTask, runPipeline: runArtCritiquePipeline }) {
|
||||
if (!input.nodeId || !input.runId || !isArtCritiqueImageInput(input.source)) throw new Error("分析需要真实目标节点、执行标识和已有图片");
|
||||
const scope = getActiveUserScope();
|
||||
const assertLive = () => {
|
||||
if (input.signal.aborted || getActiveUserScope() !== scope) throw new DOMException("分析已停止或账号已切换", "AbortError");
|
||||
};
|
||||
assertLive();
|
||||
const content = input.source.metadata?.content || input.source.metadata?.previewContent || "";
|
||||
const image = await dependencies.inspectImage({ storageKey: input.source.metadata?.storageKey, ...(content.startsWith("data:") ? { dataUrl: content } : { url: content }), name: input.source.title }, "original");
|
||||
assertLive();
|
||||
const calledStages = new Set<string>();
|
||||
return dependencies.runPipeline(input.config, { dataUrl: image.storageKey, title: input.source.title, sourceFingerprint: artCritiqueSourceFingerprint(input.source) }, {
|
||||
signal: input.signal,
|
||||
onStage: (stage) => { assertLive(); input.onStage?.(stage); },
|
||||
onDraftReport: (report) => { assertLive(); input.onDraftReport?.(report); },
|
||||
requestStage: async ({ config, messages, tool, toolName, signal }) => {
|
||||
assertLive();
|
||||
if (calledStages.has(toolName) || calledStages.size >= ART_CRITIQUE_MAX_MODEL_CALLS) throw new Error("分析阶段重复或超过本次调用数量上限");
|
||||
calledStages.add(toolName);
|
||||
const result = await (input.submitStage || dependencies.runTask)({
|
||||
config, messages, tools: [tool], toolChoice: { type: "function", name: toolName },
|
||||
prompt: `图片审美分析：${toolName}`, signal,
|
||||
metadata: { source: "art-critique", nodeId: input.nodeId, runId: input.runId, stage: toolName },
|
||||
// A task can be accepted while the user stops observing; retain that receipt.
|
||||
onTaskCreated: (task) => { if (getActiveUserScope() === scope) input.onTaskCreated(toolName, task.id); },
|
||||
});
|
||||
assertLive();
|
||||
return result;
|
||||
},
|
||||
});
|
||||
}
|
||||
67
web/src/services/creation-canvas-conversation.ts
Normal file
67
web/src/services/creation-canvas-conversation.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
import type { CanvasAssistantMessage, CanvasAssistantSession } from "@/types/canvas";
|
||||
import { getActiveUserScope } from "@/lib/user-scope";
|
||||
import { flushCanvasStorePersistence, useCanvasStore } from "@/stores/canvas/use-canvas-store";
|
||||
import { createCanvasProjectWithRemoteSync, hasRemoteUserDataSyncSession, loadCanvasProjectForEditing, saveRemoteUserDataNow } from "@/services/user-data-sync";
|
||||
|
||||
type SourceMessage = {
|
||||
id: string; role: "user" | "assistant"; content: string; createdAt: string;
|
||||
mode?: string; status?: string; taskIds?: string[]; error?: string; model?: string;
|
||||
settings?: unknown; attachments?: Array<{ name: string; storageKey?: string }>;
|
||||
references?: Array<{ id: string; label: string; kind: string }>;
|
||||
};
|
||||
type SourceConversation = { id: string; title: string; updatedAt: string; canvasId?: string; messages: SourceMessage[] };
|
||||
|
||||
// Same source IDs merge into the same canvas conversation, without replaying tasks
|
||||
// or replacing messages produced by the canvas Agent after the handoff.
|
||||
export async function continueCreationConversationOnCanvas(source: SourceConversation) {
|
||||
const scope = getActiveUserScope();
|
||||
const assertScope = () => { if (scope !== getActiveUserScope()) throw new DOMException("账号已切换，请重新打开创作", "AbortError"); };
|
||||
if (!source.id || !source.messages.length) throw new Error("请先开始一段创作对话");
|
||||
if (source.messages.some((item) => item.status === "pending" || item.status === "streaming")) throw new Error("当前创作仍在生成，请完成或停止后再转入画布。");
|
||||
if (!useCanvasStore.getState().hydrated) throw new Error("画布资料正在加载，请稍后继续。");
|
||||
const sessionId = `creation:${source.id}`;
|
||||
const messages: CanvasAssistantMessage[] = source.messages.map((item) => ({
|
||||
id: `creation:${source.id}:${item.id}`, role: item.role,
|
||||
text: item.content || (item.status === "done" ? "首页作品已生成，可在画布中继续创作。" : item.status === "error" ? "首页生成未完成。" : item.status === "cancelled" ? "首页生成已停止。" : "来自首页的创作记录"),
|
||||
meta: "来自首页",
|
||||
detail: { kind: "creation-handoff", sourceConversationId: source.id, sourceMessageId: item.id, createdAt: item.createdAt,
|
||||
status: item.status, mode: item.mode, model: item.model, taskIds: item.taskIds, error: item.error, settings: item.settings,
|
||||
references: item.references?.map(({ id, label, kind }) => ({ id, label, kind })),
|
||||
attachments: item.attachments?.map(({ name, storageKey }) => ({ name, storageKey })),
|
||||
},
|
||||
}));
|
||||
const local = useCanvasStore.getState().projects.find((item) => item.chatSessions?.some((session) => session.id === sessionId));
|
||||
const existingId = source.canvasId || local?.id;
|
||||
let id: string;
|
||||
let syncError: unknown;
|
||||
if (existingId) {
|
||||
const project = await loadCanvasProjectForEditing(existingId);
|
||||
assertScope();
|
||||
if (!project) throw new Error("关联画布已不存在，请先恢复画布后继续。");
|
||||
const sessions = project.chatSessions || [];
|
||||
const previous = sessions.find((item) => item.id === sessionId);
|
||||
const incoming = new Map(messages.map((item) => [item.id, item]));
|
||||
const merged = (previous?.messages || []).map((item) => {
|
||||
const replacement = incoming.get(item.id);
|
||||
incoming.delete(item.id);
|
||||
return replacement || item;
|
||||
});
|
||||
const session: CanvasAssistantSession = { ...previous, id: sessionId, title: source.title, createdAt: previous?.createdAt || source.messages[0].createdAt, updatedAt: new Date().toISOString(), messages: [...merged, ...incoming.values()] };
|
||||
useCanvasStore.getState().updateProject(project.id, { chatSessions: [...sessions.filter((item) => item.id !== sessionId), session], activeChatId: sessionId });
|
||||
id = project.id;
|
||||
await flushCanvasStorePersistence();
|
||||
assertScope();
|
||||
try {
|
||||
if (!hasRemoteUserDataSyncSession()) throw new Error("尚未建立云端同步会话");
|
||||
await saveRemoteUserDataNow();
|
||||
} catch (cause) { syncError = cause; }
|
||||
} else {
|
||||
const session: CanvasAssistantSession = { id: sessionId, title: source.title, createdAt: source.messages[0].createdAt, updatedAt: source.updatedAt, messages };
|
||||
const created = await createCanvasProjectWithRemoteSync(source.title || "创作画布", undefined, { chatSessions: [session], activeChatId: sessionId });
|
||||
id = created.id;
|
||||
syncError = created.syncError;
|
||||
await flushCanvasStorePersistence();
|
||||
}
|
||||
assertScope();
|
||||
return { id, sessionId, syncError };
|
||||
}
|
||||
558
web/src/services/creative-agent-controller.ts
Normal file
558
web/src/services/creative-agent-controller.ts
Normal file
@@ -0,0 +1,558 @@
|
||||
import { nanoid } from "nanoid";
|
||||
import { applyCanvasAgentOps, type CanvasAgentOp, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops";
|
||||
import { buildCanvasAgentContext } from "@/lib/canvas/canvas-agent-context";
|
||||
import { applyCreativeAnswers, creativeScenarioPrompt, normalizeCreativeQuestions, CREATIVE_SCENARIOS, type CreativeAnswers, type CreativeQuote } from "@/lib/creation/creative-agent-contract";
|
||||
import { assertCreativeBriefSpecifications, creativeNodeId, creativeProposalOps, initialCreativeState, mergeCreativeBrief, normalizeCreativeProposal, readCreativeState, type CreativeAgentState, type CreativeMediaState } from "@/lib/creation/creative-agent-state";
|
||||
import { CREATIVE_AGENT_SYSTEM_PROMPT, CREATIVE_AGENT_TOOLS, parseCreativeToolArguments } from "@/lib/creation/creative-agent-tools";
|
||||
import { modelCapabilityConfigFor } from "@/lib/model-capabilities";
|
||||
import { getActiveUserScope } from "@/lib/user-scope";
|
||||
import { logicalModelIDForConfig, modelDisplayName, resolveModelRequestConfig, selectableModelsByCapability, type AiConfig } from "@/stores/use-config-store";
|
||||
import { creationRuns, type CreationGuard, type CreationRun, type CreationRunDetail, type CreationStatus, type CreationSubmission } from "./api/creation-runs";
|
||||
import { parseBackendGenerationResult, prepareBackendGenerationTask, prepareBackendToolGenerationTask } from "./api/generation-task";
|
||||
import { queryGenerationTask, waitForGenerationTask, type GenerationTask } from "./api/task-center";
|
||||
import { resourceFileUrl, resourceIdFromStorageKey } from "./api/resources";
|
||||
import type { ResponseInputMessage, ResponseToolCall } from "./api/image";
|
||||
import { listAddedSkills } from "./api/skills";
|
||||
import { skillRuntime } from "./skill-runtime";
|
||||
import { ensureCanvasNodeAsset } from "./project-asset-sync";
|
||||
import { withRemoteUserDataSyncExclusive } from "./user-data-sync";
|
||||
import { useCanvasStore } from "@/stores/canvas/use-canvas-store";
|
||||
import { creativeVideoSpecificationError } from "@/lib/creation/creative-agent-state";
|
||||
import { updateCreativePlan } from "@/lib/creation/creative-plan";
|
||||
|
||||
export type CreativeCanvasAdapter = { canvasId: string; read: () => CanvasAgentSnapshot; apply: (ops: CanvasAgentOp[]) => Promise<CanvasAgentSnapshot> };
|
||||
export type CreativeControllerView = { run?: CreationRun; state: CreativeAgentState; busy: boolean; hasControl: boolean; error?: string; quote?: CreativeQuote };
|
||||
type ControllerOptions = { clientKey?: string; config: () => AiConfig; canvas: () => CreativeCanvasAdapter | undefined; onChange: (view: CreativeControllerView) => void; onOpenCanvas: (canvasId: string, runId: string) => void; api?: typeof creationRuns; waitTask?: typeof waitForGenerationTask; queryTask?: typeof queryGenerationTask; ensureAsset?: typeof ensureCanvasNodeAsset };
|
||||
|
||||
// 单一浏览器执行器；运行事实和费用批准以服务端记录为准，页面只提供就绪的画布适配器。
|
||||
export class CreativeAgentController {
|
||||
private run?: CreationRun;
|
||||
private state = initialCreativeState();
|
||||
private submissions: CreationSubmission[] = [];
|
||||
private owner = nanoid();
|
||||
private scope = getActiveUserScope();
|
||||
private abort = new AbortController();
|
||||
private busy = false;
|
||||
private hasControl = false;
|
||||
private disposed = false;
|
||||
private heartbeat?: ReturnType<typeof setInterval>;
|
||||
private error?: string;
|
||||
private api: typeof creationRuns;
|
||||
private waitTask: typeof waitForGenerationTask;
|
||||
private queryTask: typeof queryGenerationTask;
|
||||
private getAsset() { return this.options.ensureAsset || ensureCanvasNodeAsset; }
|
||||
constructor(private options: ControllerOptions) { this.api = options.api || creationRuns; this.waitTask = options.waitTask || waitForGenerationTask; this.queryTask = options.queryTask || queryGenerationTask; }
|
||||
|
||||
private emit() { if (!this.disposed) this.options.onChange({ run: this.run, state: this.state, busy: this.busy, hasControl: this.hasControl, error: this.error, quote: this.quote() }); }
|
||||
private assertLive() { if (this.disposed || this.abort.signal.aborted || this.scope !== getActiveUserScope()) throw new DOMException("执行已停止或账号已切换", "AbortError"); }
|
||||
private guard(): CreationGuard { this.assertLive(); if (!this.run || !this.hasControl) throw new Error("请先接管当前任务，再继续操作"); return { owner: this.owner, executionEpoch: this.run.executionEpoch }; }
|
||||
private accept(detail: CreationRunDetail) { this.run = detail.run; this.state = readCreativeState(detail.run.state); this.submissions = detail.submissions || []; }
|
||||
private async save(status: CreationStatus = this.run?.status || "idle") {
|
||||
const guard = this.guard();
|
||||
this.run = await this.api.save(this.run!.id, { ...guard, revision: this.run!.revision, status, state: this.state as unknown as Record<string, unknown> }, this.abort.signal);
|
||||
this.assertLive(); this.emit();
|
||||
}
|
||||
private async action(operation: () => Promise<void>, takeOver = false) {
|
||||
if (this.busy) throw new Error("上一项操作仍在处理，请稍候");
|
||||
if (this.disposed || this.scope !== getActiveUserScope()) throw new DOMException("页面已关闭", "AbortError");
|
||||
this.busy = true; this.error = undefined; this.emit();
|
||||
try { if (this.run) await this.ensureControl(takeOver); await operation(); }
|
||||
catch (error) {
|
||||
if (!this.disposed && !(error instanceof DOMException && error.name === "AbortError")) { this.error = error instanceof Error ? error.message : "操作未完成"; this.emit(); }
|
||||
throw error;
|
||||
} finally { this.busy = false; this.emit(); }
|
||||
}
|
||||
|
||||
async load(runId?: string, seed?: Partial<CreativeAgentState>, readOnly = false) {
|
||||
await this.action(async () => {
|
||||
if (runId) this.accept(await this.api.get(runId, this.abort.signal));
|
||||
else this.accept(await this.api.create({ clientKey: this.options.clientKey || nanoid(), canvasId: this.options.canvas()?.canvasId, state: { ...initialCreativeState(), ...seed } }, this.abort.signal));
|
||||
this.assertLive();
|
||||
if (!readOnly) {
|
||||
this.startHeartbeat();
|
||||
await this.ensureControl(true);
|
||||
}
|
||||
this.emit();
|
||||
});
|
||||
}
|
||||
private async claim() {
|
||||
this.run = await this.api.claim(this.run!.id, { expectedEpoch: this.run!.executionEpoch, owner: this.owner }, this.abort.signal);
|
||||
this.assertLive(); this.hasControl = true;
|
||||
this.startHeartbeat();
|
||||
}
|
||||
private startHeartbeat() {
|
||||
if (this.heartbeat) return;
|
||||
this.heartbeat = setInterval(() => {
|
||||
if (this.disposed || !this.run) return;
|
||||
if (!this.hasControl) {
|
||||
if (!this.busy) void this.ensureControl().then(() => { this.error = undefined; this.emit(); }).catch((error) => {
|
||||
if (this.disposed) return;
|
||||
this.error = error instanceof Error ? error.message : "连接恢复失败，请重试";
|
||||
this.emit();
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (this.abort.signal.aborted) return;
|
||||
void this.api.heartbeat(this.run.id, this.guard(), this.abort.signal).then((result) => {
|
||||
if (!this.disposed && this.run) this.run = { ...this.run, leaseExpiresAt: result.leaseExpiresAt };
|
||||
}).catch(() => { if (this.disposed) return; this.hasControl = false; this.abort.abort(); this.error = "连接暂时中断，正在恢复。已有作品会保留。"; this.emit(); });
|
||||
}, 15_000);
|
||||
}
|
||||
private restoring?: Promise<void>;
|
||||
private async ensureControl(takeOver = false): Promise<void> {
|
||||
if (this.hasControl && !this.abort.signal.aborted && Date.parse(this.run?.leaseExpiresAt || "") > Date.now() + 3000) return;
|
||||
if (this.restoring) {
|
||||
if (!takeOver) return this.restoring;
|
||||
try { await this.restoring; } catch { /* Explicit reconnect retries after the passive recovery attempt. */ }
|
||||
return this.ensureControl(true);
|
||||
}
|
||||
this.restoring = (async () => {
|
||||
if (this.disposed || this.scope !== getActiveUserScope()) throw new DOMException("页面已关闭", "AbortError");
|
||||
if (this.abort.signal.aborted) this.abort = new AbortController();
|
||||
const detail = await this.api.get(this.run!.id, this.abort.signal);
|
||||
this.assertLive();
|
||||
const run = detail.run;
|
||||
if (!takeOver && run.executionOwner && run.executionOwner !== this.owner && Date.parse(run.leaseExpiresAt || "") > Date.now()) throw new Error("当前创作连接已切换，请点击重试连接继续。");
|
||||
// Only replace the lease here; never overwrite local edits or unknown submission results.
|
||||
this.run = { ...this.run!, executionOwner: run.executionOwner, executionEpoch: run.executionEpoch, leaseExpiresAt: run.leaseExpiresAt };
|
||||
await this.claim();
|
||||
})();
|
||||
try { await this.restoring; } finally { this.restoring = undefined; }
|
||||
}
|
||||
async takeControl() {
|
||||
if (this.disposed) return;
|
||||
await this.action(async () => { this.accept(await this.api.get(this.run!.id, this.abort.signal)); }, true);
|
||||
}
|
||||
async refresh() { await this.action(async () => { this.accept(await this.api.get(this.run!.id, this.abort.signal)); }); }
|
||||
dispose() {
|
||||
this.disposed = true; this.abort.abort(); if (this.heartbeat) clearInterval(this.heartbeat);
|
||||
if (this.hasControl && this.run) void this.api.release(this.run.id, { executionEpoch: this.run.executionEpoch, owner: this.owner }).catch(() => undefined);
|
||||
this.hasControl = false;
|
||||
}
|
||||
async pause() {
|
||||
if (!this.hasControl || !this.run) return;
|
||||
const guard = this.guard();
|
||||
this.abort.abort();
|
||||
this.state = { ...this.state, error: "已暂停后续操作。已提交的生成任务仍可在任务中心继续运行。" };
|
||||
try {
|
||||
this.run = await this.api.save(this.run.id, { ...guard, revision: this.run.revision, status: "paused", state: this.state as unknown as Record<string, unknown> });
|
||||
} finally {
|
||||
try { await this.api.release(this.run.id, guard); }
|
||||
finally { this.hasControl = false; this.startHeartbeat(); this.emit(); }
|
||||
}
|
||||
}
|
||||
async configure(patch: Pick<CreativeAgentState, "scene" | "references" | "selectedSkillIds" | "textModel">) { await this.action(async () => { this.state = { ...this.state, ...patch }; await this.save(); }); }
|
||||
|
||||
async requestModification() {
|
||||
await this.action(async () => {
|
||||
if (this.state.media.some((item) => ["queued", "running"].includes(item.status))) throw new Error("已有生成还在处理，请继续处理以同步结果，完成后再修改方案。");
|
||||
this.run = await this.api.invalidateProposal(this.run!.id, { ...this.guard(), revision: this.run!.revision }, this.abort.signal);
|
||||
this.state = { ...this.state, modificationRequested: true, pendingPayment: undefined, pendingEdits: undefined, pendingRedo: undefined };
|
||||
await this.save("idle");
|
||||
});
|
||||
}
|
||||
|
||||
async send(text: string) {
|
||||
if (!text.trim()) return;
|
||||
await this.action(async () => {
|
||||
this.guard();
|
||||
if (this.state.media.some((item) => ["queued", "running"].includes(item.status))) throw new Error("请先暂停或等当前生成完成，再调整需求；已提交任务不会被文字修改取消");
|
||||
if (this.run?.approvedProposalVersion) this.run = await this.api.invalidateProposal(this.run.id, { ...this.guard(), revision: this.run.revision }, this.abort.signal);
|
||||
this.state = { ...this.state, questions: undefined, answers: undefined, pendingPayment: undefined, planning: undefined, pendingEdits: undefined, pendingRedo: undefined, error: undefined, messages: [...this.state.messages, { id: nanoid(), role: "user", text: text.trim() }] };
|
||||
await this.save("running");
|
||||
await this.preparePlanning(text.trim());
|
||||
});
|
||||
}
|
||||
async answer(answers: CreativeAnswers, continuePlanning = true) {
|
||||
await this.action(async () => {
|
||||
const question = this.state.questions;
|
||||
if (!question) throw new Error("当前没有待回答问题");
|
||||
const brief = applyCreativeAnswers(question, question, answers, this.state.brief, this.assets());
|
||||
const summary = question.questions.map((item) => `${item.title}：${JSON.stringify(brief[item.field]?.value ?? "由助手建议")}`).join("\n");
|
||||
this.state = { ...this.state, brief, questions: { ...question, status: "submitted" }, answers, messages: [...this.state.messages, { id: nanoid(), role: "user", text: summary, question: { ...question, status: "submitted" }, answers }] };
|
||||
await this.save(continuePlanning ? "running" : "idle");
|
||||
if (continuePlanning) await this.preparePlanning(summary);
|
||||
});
|
||||
}
|
||||
// 原在线 Agent 工具循环已完成模型请求；此入口仅校验并持久化交互，不再调用第二个模型。
|
||||
async presentInteraction(data: Record<string, unknown>, revalidate = false) {
|
||||
await this.action(async () => {
|
||||
this.guard();
|
||||
if (revalidate && (this.state.proposal || this.state.canvasApplied || this.run?.approvedProposalVersion || this.state.pendingPayment?.length || this.state.media.some((item) => item.taskId))) throw new Error("此方案已有批准或执行记录，不能重复呈现，请修改方案后重新确认");
|
||||
if (this.state.externalInteractionPresented && !revalidate) return;
|
||||
this.state = { ...this.state, externalInteractionPresented: true };
|
||||
await this.consumePlanning({ resultJson: JSON.stringify({ toolCalls: [{ id: "interaction", type: "function", function: { name: "creative_respond", arguments: JSON.stringify(data) } }] }) } as GenerationTask);
|
||||
});
|
||||
}
|
||||
private assets() { return this.state.references.map((item) => ({ id: item.id, label: item.title })); }
|
||||
private requestConfig(model?: string): AiConfig {
|
||||
const config = this.options.config(), selected = model || this.state.textModel || config.textModel || config.model;
|
||||
const next: AiConfig = { ...config, model: selected, count: "1", taskWorkflowProvider: "model" };
|
||||
const requestConfig = resolveModelRequestConfig(next, selected);
|
||||
if (!logicalModelIDForConfig(next) && !requestConfig.channelId) throw new Error("智能创作需要系统受管模型，请选择系统模型；自定义渠道和本机模型仍可使用原直接生成入口");
|
||||
return next;
|
||||
}
|
||||
private async preparePlanning(prompt: string) {
|
||||
if (this.state.modelCalls >= 32) throw new Error("此任务已达到 32 次分析调用上限，请新建任务并沿用当前方案");
|
||||
const config = this.requestConfig();
|
||||
const skills = await listAddedSkills();
|
||||
const prepared = await skillRuntime.prepare({ profile: "creation", prompt, skills: skills.skills || [], selectedSkillIds: this.state.selectedSkillIds });
|
||||
this.assertLive();
|
||||
const catalogue = (["image", "video"] as const).flatMap((mode) => selectableModelsByCapability(config, mode).slice(0, 20).map((model) => ({ mode, model, name: modelDisplayName(config, model), capability: modelCapabilityConfigFor(config, model) })));
|
||||
const canvas = this.options.canvas();
|
||||
const imageReferences = this.state.references.filter((reference) => reference.kind === "image" && reference.storageKey);
|
||||
const protocol: ResponseInputMessage[] = [
|
||||
{ role: "system", content: `${CREATIVE_AGENT_SYSTEM_PROMPT}\n${creativeScenarioPrompt(this.state.scene)}\n用户系统提示：${config.systemPrompt || ""}` },
|
||||
...this.state.messages.slice(-20).map((message) => ({ role: message.role, content: message.text })),
|
||||
{ role: "user", content: [{ type: "text", text: JSON.stringify({ brief: this.state.brief, proposal: this.state.proposal, availableModels: catalogue, references: this.state.references, canvas: canvas ? buildCanvasAgentContext(canvas.read()) : { available: false, instruction: "首页不能操作画布" }, currentRequest: prepared.prompt }) }, ...imageReferences.map((reference) => ({ type: "image_url" as const, image_url: { url: reference.storageKey! } }))] },
|
||||
];
|
||||
const itemKey = `planning:${nanoid()}`;
|
||||
this.state = { ...this.state, planning: { itemKey, protocol, model: config.model, prompt }, pendingPayment: undefined };
|
||||
await this.save("running");
|
||||
await this.ensurePlanningSubmission();
|
||||
}
|
||||
private async ensurePlanningSubmission() {
|
||||
const planning = this.state.planning;
|
||||
if (!planning) throw new Error("缺少待恢复的规划内容");
|
||||
let submission = this.submissions.find((item) => item.itemKey === planning.itemKey);
|
||||
if (!submission) {
|
||||
const config = this.requestConfig(planning.model);
|
||||
const request = prepareBackendToolGenerationTask({ prompt: planning.prompt || "继续创作", config: { ...config, systemPrompt: "" }, messages: planning.protocol, tools: CREATIVE_AGENT_TOOLS, toolChoice: "auto", signal: this.abort.signal });
|
||||
request.input = { ...request.input, referenceImages: this.state.references.filter((reference) => reference.kind === "image" && reference.storageKey).map((reference) => ({ id: reference.id, name: reference.title, storageKey: reference.storageKey })) };
|
||||
submission = await this.api.prepare(this.run!.id, { ...this.guard(), itemKey: planning.itemKey, request }, this.abort.signal);
|
||||
}
|
||||
this.assertLive(); this.upsertSubmission(submission);
|
||||
this.state = { ...this.state, planning: { ...planning, submissionId: submission.id }, pendingPayment: [submission.id] };
|
||||
await this.save("waiting_payment");
|
||||
}
|
||||
private upsertSubmission(submission: CreationSubmission) { this.submissions = [...this.submissions.filter((item) => item.id !== submission.id), submission]; }
|
||||
private quote(): CreativeQuote | undefined {
|
||||
// pendingPayment also retains submitted IDs for recovery; only unsubmitted items need a fee card.
|
||||
const pending = this.state.pendingPayment?.map((id) => this.submissions.find((item) => item.id === id)).filter((item): item is CreationSubmission => Boolean(item && !item.taskId && !item.revokedAt));
|
||||
if (!pending?.length) return undefined;
|
||||
const amount = pending.reduce((sum, item) => sum + item.quote.amountMicrocredits, 0);
|
||||
const estimated = pending.some((item) => item.quote.estimated);
|
||||
return { id: pending.map((item) => item.id).join(":"), title: this.state.planning && !this.state.planning.consumed ? "确认本次理解与规划费用" : "确认本批生成费用", items: pending.map((item) => { const media = this.state.media.find((media) => media.submissionId === item.id); const node = this.state.proposal?.workflow.nodes.find((node) => node.ref === media?.ref); return { id: item.id, label: node?.title || "理解需求并生成问答或方案", model: item.quote.model, quantity: 1, specification: [...Object.entries(item.quote.options || {}).filter(([, value]) => value !== undefined && value !== "").map(([key, value]) => `${({ size: "比例/尺寸", videoSeconds: "时长（秒）", vquality: "清晰度", quality: "质量", count: "数量" } as Record<string, string>)[key] || key}：${String(value)}`), `${item.quote.billingMode === "token" ? "按用量" : item.quote.billingMode === "per_second" ? "按秒" : "按次"}计费，本项${item.quote.estimated ? "预计" : ""} ${(item.quote.amountMicrocredits / 1_000_000).toLocaleString("zh-CN", { maximumFractionDigits: 6 })} 积分`].join(" · ") }; }), amountLabel: `${estimated ? "预计 " : ""}${(amount / 1_000_000).toLocaleString("zh-CN", { maximumFractionDigits: 6 })} 积分`, basis: estimated ? "按实际用量结算，此金额为估算，不是费用上限。仅批准列出的调用。" : "按当前报价，仅批准本批列出的生成项。", expiresAt: pending.map((item) => item.quote.expiresAt).sort()[0], approvedQuantity: pending.filter((item) => item.approvedAt).length };
|
||||
}
|
||||
|
||||
async approvePayment() {
|
||||
const expiresAt = this.quote()?.expiresAt;
|
||||
if (expiresAt && Date.parse(expiresAt) <= Date.now()) {
|
||||
await this.refreshQuotes();
|
||||
return;
|
||||
}
|
||||
await this.action(async () => {
|
||||
const ids = this.state.pendingPayment;
|
||||
if (!ids?.length) throw new Error("当前没有待批准费用");
|
||||
const unapproved = ids.filter((id) => { const item = this.submissions.find((item) => item.id === id); return item && !item.taskId && !item.approvedAt && !item.revokedAt; });
|
||||
if (this.run!.status === "paused") await this.save("waiting_payment");
|
||||
if (unapproved.length) {
|
||||
const result = await this.api.approve(this.run!.id, { ...this.guard(), submissionIds: unapproved }, this.abort.signal);
|
||||
result.submissions.forEach((item) => this.upsertSubmission(item));
|
||||
}
|
||||
await this.executeApproved(ids);
|
||||
});
|
||||
}
|
||||
async refreshQuotes() {
|
||||
await this.action(async () => {
|
||||
if (!this.state.pendingPayment?.length) throw new Error("当前没有待更新报价");
|
||||
for (const id of [...this.state.pendingPayment]) {
|
||||
if (this.submissions.find((item) => item.id === id)?.taskId) continue;
|
||||
const next = await this.api.refreshQuote(this.run!.id, { ...this.guard(), submissionId: id }, this.abort.signal);
|
||||
this.guard(); this.upsertSubmission(next);
|
||||
this.state = { ...this.state,
|
||||
pendingPayment: this.state.pendingPayment!.map((item) => item === id ? next.id : item),
|
||||
planning: this.state.planning?.submissionId === id ? { ...this.state.planning, submissionId: next.id, itemKey: next.itemKey } : this.state.planning,
|
||||
media: this.state.media.map((item) => item.submissionId === id ? { ...item, submissionId: next.id } : item),
|
||||
};
|
||||
await this.save("waiting_payment");
|
||||
}
|
||||
});
|
||||
}
|
||||
private async executeApproved(ids: string[]) {
|
||||
const planning = this.state.planning;
|
||||
if (planning?.submissionId && ids.includes(planning.submissionId) && !planning.consumed) {
|
||||
await this.save("waiting_task");
|
||||
const task = await this.api.execute(this.run!.id, { ...this.guard(), submissionId: planning.submissionId }, this.abort.signal);
|
||||
this.upsertSubmission({ ...this.submissions.find((item) => item.id === planning.submissionId)!, taskId: task.id });
|
||||
const completed = await this.waitTask(task.id, { initialTask: task, signal: this.abort.signal });
|
||||
this.assertLive();
|
||||
await this.consumePlanning(completed);
|
||||
return;
|
||||
}
|
||||
if (!this.options.canvas()) throw new Error("媒体生成只能在对应画布页面继续");
|
||||
await this.save("running");
|
||||
const errors: string[] = [];
|
||||
for (const id of ids) {
|
||||
this.guard();
|
||||
const media = this.state.media.find((item) => item.submissionId === id);
|
||||
if (!media || media.status === "ready") continue;
|
||||
try {
|
||||
const task = await this.api.execute(this.run!.id, { ...this.guard(), submissionId: id }, this.abort.signal);
|
||||
this.upsertSubmission({ ...this.submissions.find((item) => item.id === id)!, taskId: task.id });
|
||||
this.setMedia(media.ref, { taskId: task.id, status: task.status === "succeeded" ? "running" : "queued", error: undefined });
|
||||
await this.save("waiting_task");
|
||||
} catch (error) { this.assertLive(); errors.push(error instanceof Error ? error.message : "提交未确认"); }
|
||||
}
|
||||
// 等待已提交项，不会因观察失败创建新的任务。
|
||||
for (const id of ids) {
|
||||
const media = this.state.media.find((item) => item.submissionId === id);
|
||||
if (media?.taskId && media.status !== "ready") {
|
||||
try { await this.observeMedia(media); } catch (error) { this.assertLive(); errors.push(error instanceof Error ? error.message : "生成未完成"); }
|
||||
}
|
||||
}
|
||||
if (errors.length) { await this.save("paused"); throw new Error([...new Set(errors)].join("；")); }
|
||||
this.state = { ...this.state, pendingPayment: undefined };
|
||||
await this.prepareMediaBatch();
|
||||
}
|
||||
private async consumePlanning(task: GenerationTask) {
|
||||
const output = parseBackendGenerationResult(task);
|
||||
const calls = output.toolCalls || [];
|
||||
const first = calls.find((call) => call.function.name === "creative_respond");
|
||||
const results = calls.map((call) => ({ role: "tool" as const, tool_call_id: call.id, content: JSON.stringify({ status: call === first ? "presented_to_user" : "cancelled", reason: call === first ? "内容待程序校验和用户确认" : "交互形成等待屏障，后续调用必须重新规划" }) }));
|
||||
const protocol = [...(this.state.planning?.protocol || []), ...calls.map(toolInput), ...results];
|
||||
this.state = { ...this.state, modelCalls: this.state.modelCalls + 1, pendingPayment: undefined, planning: this.state.planning ? { ...this.state.planning, consumed: true, protocol } : undefined };
|
||||
if (!first) {
|
||||
if (!output.text?.trim()) throw new Error("助手没有返回可展示的内容");
|
||||
this.state = { ...this.state, messages: [...this.state.messages, { id: nanoid(), role: "assistant", text: output.text }] };
|
||||
await this.save("idle"); return;
|
||||
}
|
||||
const data = parseCreativeToolArguments(first.function.arguments);
|
||||
const scene = typeof data.scenario === "string" && Object.hasOwn(CREATIVE_SCENARIOS, data.scenario) ? data.scenario as CreativeAgentState["scene"] : this.state.scene;
|
||||
const brief = mergeCreativeBrief(this.state.brief, data.brief, scene, this.state.messages.findLast((message) => message.role === "user")?.text || "");
|
||||
const questions = normalizeCreativeQuestions(data.questions, scene, brief, { interactionId: nanoid(), revision: 1 }, this.assets());
|
||||
const message = typeof data.message === "string" ? data.message : output.text || "已整理你的要求。";
|
||||
this.state = { ...this.state, scene, brief, dynamicPlan: updateCreativePlan(data.plan, this.state.dynamicPlan, nanoid()), messages: [...this.state.messages, { id: nanoid(), role: "assistant", text: message }], questions: undefined, answers: undefined };
|
||||
if (questions.questions.length) {
|
||||
this.state = { ...this.state, questions };
|
||||
await this.save("waiting_answer"); return;
|
||||
}
|
||||
if (data.proposal) {
|
||||
const proposal = normalizeCreativeProposal(data.proposal, nanoid(), Math.max(this.state.proposal?.version || 0, this.run?.approvedProposalVersion || 0) + 1, this.options.config(), this.state.references);
|
||||
assertCreativeBriefSpecifications(proposal, brief);
|
||||
this.state = { ...this.state, proposal, operations: undefined, canvasApplied: false, media: proposal.generationItems.map((item) => ({ ref: item.ref, nodeId: creativeNodeId(this.run!.id, proposal.version, item.ref), attempt: 1, status: "pending" })) };
|
||||
await this.save("waiting_proposal"); return;
|
||||
}
|
||||
if (Array.isArray(data.edits) && data.edits.length) {
|
||||
const canvas = this.options.canvas();
|
||||
if (!canvas) throw new Error("请进入画布后再调整节点");
|
||||
const current = canvas.read();
|
||||
const ops: CanvasAgentOp[] = data.edits.map((raw) => {
|
||||
const edit = raw as Record<string, unknown>;
|
||||
const id = typeof edit.nodeId === "string" ? edit.nodeId : "";
|
||||
if (!current.nodes.some((node) => node.id === id)) throw new Error("助手建议修改的节点不存在");
|
||||
return { type: "update_node", id, patch: { ...(typeof edit.title === "string" ? { title: edit.title } : {}) }, metadata: { ...(typeof edit.prompt === "string" ? { prompt: edit.prompt, composerContent: edit.prompt } : {}), ...(typeof edit.content === "string" && current.nodes.find((node) => node.id === id)?.type === "text" ? { content: edit.content } : {}) } };
|
||||
});
|
||||
this.state = { ...this.state, pendingEdits: ops };
|
||||
await this.save("waiting_proposal"); return;
|
||||
}
|
||||
await this.save("idle");
|
||||
}
|
||||
async approveProposal() {
|
||||
await this.action(async () => {
|
||||
const proposal = this.state.proposal;
|
||||
if (!proposal || this.run?.status !== "waiting_proposal") throw new Error("当前方案已过期或无需确认");
|
||||
assertCreativeBriefSpecifications(proposal, this.state.brief);
|
||||
const snapshot = this.options.canvas()?.read() || { projectId: "", title: "", nodes: [], connections: [], selectedNodeIds: [], viewport: { x: 0, y: 0, k: 1 } };
|
||||
const ops = creativeProposalOps(this.run!.id, proposal, snapshot, this.options.config());
|
||||
proposal.extra = { ...(proposal.extra as Record<string, unknown> || {}), nodeIds: Object.fromEntries(proposal.workflow.nodes.map((node) => [node.ref, creativeNodeId(this.run!.id, proposal.version, node.ref)])) };
|
||||
this.state = { ...this.state, proposal, operations: ops, messages: [...this.state.messages, { id: nanoid(), role: "assistant", text: `方案 v${proposal.version}：${proposal.title}`, proposal }] };
|
||||
await this.save("waiting_proposal");
|
||||
this.run = await this.api.approveProposal(this.run!.id, { ...this.guard(), revision: this.run!.revision, proposalVersion: proposal.version, proposal, ops }, this.abort.signal);
|
||||
const linked = await this.api.canvas(this.run.id, this.guard(), this.abort.signal);
|
||||
this.run = linked.run;
|
||||
await this.save("waiting_canvas");
|
||||
if (this.options.canvas()?.canvasId === linked.canvasId) await this.applyProposal();
|
||||
else { await this.api.release(this.run.id, this.guard()); this.hasControl = false; this.options.onOpenCanvas(linked.canvasId, this.run.id); }
|
||||
});
|
||||
}
|
||||
async approveEdits() {
|
||||
await this.action(async () => {
|
||||
const ops = this.state.pendingEdits;
|
||||
if (!ops?.length) throw new Error("当前没有待确认节点调整");
|
||||
const version = Math.max(this.state.proposal?.version || 0, this.run?.approvedProposalVersion || 0) + 1;
|
||||
this.run = await this.api.approveProposal(this.run!.id, { ...this.guard(), revision: this.run!.revision, proposalVersion: version, proposal: { title: "局部节点调整", ops }, ops }, this.abort.signal);
|
||||
await this.commitOps(ops);
|
||||
this.state = { ...this.state, pendingEdits: undefined, messages: [...this.state.messages, { id: nanoid(), role: "assistant", text: "已更新指定节点字段。" }] };
|
||||
await this.save("idle");
|
||||
});
|
||||
}
|
||||
private async applyProposal() {
|
||||
if (this.state.canvasApplied) return;
|
||||
if (!this.state.operations || !this.state.proposal) throw new Error("缺少已批准方案的操作记录，请重新核对");
|
||||
await this.commitOps(this.state.operations);
|
||||
this.state = { ...this.state, canvasApplied: true };
|
||||
await this.save("running");
|
||||
await this.prepareMediaBatch();
|
||||
}
|
||||
private async commitOps(ops: CanvasAgentOp[]) {
|
||||
return withRemoteUserDataSyncExclusive(() => this.commitOpsExclusive(ops));
|
||||
}
|
||||
private async commitOpsExclusive(ops: CanvasAgentOp[]) {
|
||||
const adapter = this.options.canvas();
|
||||
if (!adapter || adapter.canvasId !== this.run?.canvasId) throw new Error("对应画布尚未加载，不能执行画布操作");
|
||||
this.guard();
|
||||
const remote = await this.api.canvasSnapshot(this.run.id, this.abort.signal);
|
||||
const remoteSnapshot = { ...adapter.read(), projectId: remote.document.id, nodes: remote.document.nodes, connections: remote.document.connections };
|
||||
const missing = reconcileOps(ops, remoteSnapshot);
|
||||
if (missing.length) {
|
||||
const next = applyCanvasAgentOps(remoteSnapshot, missing);
|
||||
await this.api.commitCanvas(this.run.id, { ...this.guard(), expectedSnapshotHash: remote.snapshotHash, document: { ...remote.document, nodes: next.nodes, connections: next.connections } }, this.abort.signal);
|
||||
}
|
||||
this.guard();
|
||||
const localOps = reconcileOps(ops, adapter.read());
|
||||
if (localOps.length) {
|
||||
const applied = await adapter.apply(localOps);
|
||||
// 同一保存队列释放前更新持久化源，避免排队的自动保存仍提交操作前快照。
|
||||
if (useCanvasStore.getState().projects.some((project) => project.id === adapter.canvasId)) {
|
||||
useCanvasStore.getState().updateProject(adapter.canvasId, { nodes: applied.nodes, connections: applied.connections });
|
||||
}
|
||||
}
|
||||
this.assertLive();
|
||||
}
|
||||
private setMedia(ref: string, patch: Partial<CreativeMediaState>) { this.state = { ...this.state, media: this.state.media.map((item) => item.ref === ref ? { ...item, ...patch } : item) }; this.emit(); }
|
||||
private async prepareMediaBatch() {
|
||||
const proposal = this.state.proposal;
|
||||
const adapter = this.options.canvas();
|
||||
if (!proposal || !adapter || !this.state.canvasApplied) throw new Error("请先在画布中完成方案接续");
|
||||
const existingRefs = Object.keys((proposal.extra as { existingAssets?: Record<string, unknown> } | undefined)?.existingAssets || {});
|
||||
const ready = new Set([...existingRefs, ...this.state.media.filter((item) => item.status === "ready").map((item) => item.ref)]);
|
||||
if (this.state.media.every((item) => item.status === "ready")) {
|
||||
this.state = { ...this.state, pendingPayment: undefined, messages: [...this.state.messages, { id: nanoid(), role: "assistant", text: "本次产物已生成并保存，可以在会话和画布查看。" }] };
|
||||
await this.save("completed"); return;
|
||||
}
|
||||
const candidates = this.state.media.filter((media) => media.status === "pending" && proposal.generationItems.find((item) => item.ref === media.ref)?.referenceRefs?.every((ref) => ready.has(ref)) !== false);
|
||||
const ids: string[] = [];
|
||||
for (const media of candidates) {
|
||||
const item = proposal.generationItems.find((item) => item.ref === media.ref)!;
|
||||
const snapshot = adapter.read(), node = snapshot.nodes.find((node) => node.id === media.nodeId);
|
||||
if (!node) throw new Error(`节点 ${media.ref} 已被删除，请核对后继续，不会重新创建`);
|
||||
const nodeIds = (proposal.extra as { nodeIds?: Record<string, string> } | undefined)?.nodeIds;
|
||||
const referenceNodes = [...(item.referenceRefs || []).map((ref) => nodeIds?.[ref] || this.state.media.find((media) => media.ref === ref)?.nodeId || creativeNodeId(this.run!.id, proposal.version, ref)), ...(proposal.workflow.nodes.find((node) => node.ref === item.ref)?.referenceNodeIds || [])].map((id) => snapshot.nodes.find((node) => node.id === id));
|
||||
const referenceImages = referenceNodes.map((reference) => {
|
||||
const storageKey = typeof reference?.metadata?.storageKey === "string" ? reference.metadata.storageKey : "";
|
||||
const resourceId = resourceIdFromStorageKey(storageKey);
|
||||
if (!reference || !resourceId || reference.metadata?.status !== "success") throw new Error("引用素材尚未就绪，请选择实际可用的资源");
|
||||
return { id: reference.id, name: reference.title || "参考图", type: reference.metadata?.mimeType || "image/png", storageKey, dataUrl: resourceFileUrl(resourceId) };
|
||||
});
|
||||
const config = { ...this.requestConfig(item.model), count: "1", ...(item.size ? { size: item.size } : {}), ...(item.seconds !== undefined ? { videoSeconds: String(item.seconds) } : {}), ...(item.quality ? item.mode === "video" ? { vquality: item.quality } : { quality: item.quality } : {}) };
|
||||
const request = await prepareBackendGenerationTask({ projectId: this.run!.canvasId, mode: item.mode, prompt: node.metadata?.prompt || "", config, referenceImages, signal: this.abort.signal, metadata: { source: "creative-agent", creationRunId: this.run!.id, nodeId: node.id, conversationId: this.run!.id } });
|
||||
request.input = { ...request.input, nodeId: node.id };
|
||||
const submission = await this.api.prepare(this.run!.id, { ...this.guard(), itemKey: `media:v${proposal.version}:${media.ref}:${media.attempt}`, proposalVersion: proposal.version, request }, this.abort.signal);
|
||||
this.upsertSubmission(submission); this.setMedia(media.ref, { submissionId: submission.id }); ids.push(submission.id);
|
||||
this.state = { ...this.state, pendingPayment: [...ids] }; await this.save("waiting_payment");
|
||||
}
|
||||
if (!ids.length) { await this.save("paused"); throw new Error("当前没有可执行项，请先处理失败任务或核对素材依赖"); }
|
||||
}
|
||||
private async observeMedia(media: CreativeMediaState) {
|
||||
let task: GenerationTask;
|
||||
let observedStatus: GenerationTask["status"] | undefined;
|
||||
try {
|
||||
task = await this.waitTask(media.taskId!, { signal: this.abort.signal, onTaskUpdate: (task) => { observedStatus = task.status; if (!this.disposed) this.setMedia(media.ref, { status: task.status === "queued" ? "queued" : "running" }); } });
|
||||
} catch (error) {
|
||||
if (this.abort.signal.aborted) throw error;
|
||||
this.setMedia(media.ref, { status: "failed", failureKind: observedStatus === "failed" || observedStatus === "cancelled" ? "generation" : "observation", error: error instanceof Error ? error.message : "生成失败" }); await this.save("paused"); throw error;
|
||||
}
|
||||
this.guard();
|
||||
try {
|
||||
const result = parseBackendGenerationResult(task), output = result.video || result.images?.[0];
|
||||
if (!output?.storageKey || !resourceIdFromStorageKey(output.storageKey)) throw new Error("任务已完成，但资源尚未就绪；请恢复回写，不要重新生成");
|
||||
const adapter = this.options.canvas();
|
||||
const node = adapter?.read().nodes.find((node) => node.id === media.nodeId);
|
||||
if (!node) throw new Error("目标节点已删除，产物仍保留在任务中心和素材库");
|
||||
const metadata = { content: resourceFileUrl(resourceIdFromStorageKey(output.storageKey)!), storageKey: output.storageKey, status: "success" as const, naturalWidth: output.width, naturalHeight: output.height, bytes: output.bytes, mimeType: output.mimeType, ...(result.video ? { durationMs: result.video.durationMs } : {}), generationTaskId: task.id };
|
||||
const asset = await this.getAsset()({ canvasId: this.run!.canvasId!, node: { ...node, metadata: { ...node.metadata, ...metadata } }, source: "canvas-generation", taskId: task.id, signal: this.abort.signal });
|
||||
this.guard();
|
||||
await this.commitOps([{ type: "update_node", id: media.nodeId, metadata: { ...metadata, assetId: asset.assetId } }]);
|
||||
const specificationError = result.video && creativeVideoSpecificationError(this.state.proposal?.generationItems.find((item) => item.ref === media.ref), result.video);
|
||||
if (specificationError) {
|
||||
this.setMedia(media.ref, { status: "failed", storageKey: output.storageKey, error: specificationError });
|
||||
throw new Error(specificationError);
|
||||
}
|
||||
this.setMedia(media.ref, { status: "ready", storageKey: output.storageKey, error: undefined }); await this.save("waiting_task");
|
||||
} catch (error) { if (this.abort.signal.aborted) throw error; this.setMedia(media.ref, { status: this.state.media.find((item) => item.ref === media.ref)?.status === "failed" ? "failed" : "write_failed", error: error instanceof Error ? error.message : "资源回写失败" }); await this.save("paused"); throw error; }
|
||||
}
|
||||
async resume() {
|
||||
await this.action(async () => {
|
||||
this.guard(); this.accept(await this.api.get(this.run!.id, this.abort.signal));
|
||||
if (this.state.modificationRequested) throw new Error("请先说明想修改的内容，调整后的方案需要重新确认。");
|
||||
this.state = { ...this.state, error: undefined };
|
||||
await this.save(this.state.questions?.status === "pending" ? "waiting_answer" : "running");
|
||||
if (this.state.pendingRedo) { await this.continueRedo(); return; }
|
||||
if (!this.state.planning && this.state.messages.at(-1)?.role === "user") {
|
||||
if (this.state.externalInteractionPresented) { await this.save("idle"); return; }
|
||||
await this.preparePlanning(this.state.messages.at(-1)!.text); return;
|
||||
}
|
||||
if (this.state.planning && !this.state.planning.consumed && !this.state.planning.submissionId) await this.ensurePlanningSubmission();
|
||||
const planning = this.state.planning;
|
||||
if (planning?.submissionId && !planning.consumed) {
|
||||
const submission = this.submissions.find((item) => item.id === planning.submissionId);
|
||||
if (submission?.taskId) { const task = await this.waitTask(submission.taskId, { signal: this.abort.signal }); await this.consumePlanning(task); return; }
|
||||
if (submission?.approvedAt) { await this.executeApproved([submission.id]); return; }
|
||||
this.state = { ...this.state, pendingPayment: [planning.submissionId] }; await this.save("waiting_payment"); return;
|
||||
}
|
||||
if (!this.state.canvasApplied && this.state.proposal && this.run!.approvedProposalHash && this.run!.approvedProposalVersion === this.state.proposal.version) {
|
||||
const linked = await this.api.canvas(this.run!.id, this.guard(), this.abort.signal); this.run = linked.run;
|
||||
if (this.options.canvas()?.canvasId !== linked.canvasId) { await this.save("waiting_canvas"); await this.api.release(this.run.id, this.guard()); this.hasControl = false; this.options.onOpenCanvas(linked.canvasId, this.run.id); return; }
|
||||
await this.applyProposal(); return;
|
||||
}
|
||||
if (this.state.questions?.status === "pending") return;
|
||||
if (this.state.pendingEdits) { await this.save("waiting_proposal"); return; }
|
||||
if (this.state.proposal && !this.state.canvasApplied) { await this.save("waiting_proposal"); return; }
|
||||
const errors: string[] = [];
|
||||
for (const media of this.state.media) {
|
||||
const submission = this.submissions.find((item) => item.id === media.submissionId);
|
||||
if (submission?.taskId && media.status !== "ready") { this.setMedia(media.ref, { taskId: submission.taskId }); try { await this.observeMedia({ ...media, taskId: submission.taskId }); } catch (error) { this.assertLive(); errors.push(error instanceof Error ? error.message : "生成未完成"); } }
|
||||
}
|
||||
if (errors.length) { await this.save("paused"); throw new Error([...new Set(errors)].join("；")); }
|
||||
const approved = this.state.pendingPayment?.filter((id) => this.submissions.some((item) => item.id === id && item.approvedAt));
|
||||
if (approved?.length) await this.executeApproved(approved);
|
||||
else if (this.state.pendingPayment?.length) await this.save("waiting_payment");
|
||||
else if (this.state.proposal && this.state.canvasApplied) await this.prepareMediaBatch();
|
||||
});
|
||||
}
|
||||
async redo(ref: string) {
|
||||
await this.action(async () => {
|
||||
if (this.state.pendingRedo) throw new Error("已有待恢复的重做操作，请先核对状态并继续");
|
||||
const media = this.state.media.find((item) => item.ref === ref);
|
||||
if (!media) throw new Error("生成项不存在");
|
||||
if (media.taskId) { const task = await this.queryTask(media.taskId, { signal: this.abort.signal }); if (["running", "queued"].includes(task.status) || task.billing?.status === "uncertain") throw new Error("原任务仍在执行或账务待核对，不能重新扣费"); }
|
||||
const canvas = this.options.canvas();
|
||||
const proposal = this.state.proposal;
|
||||
if (!canvas || !proposal) throw new Error("请先打开对应画布");
|
||||
const snapshot = canvas.read();
|
||||
const node = snapshot.nodes.find((node) => node.id === media.nodeId);
|
||||
if (!node) throw new Error("目标节点已删除，不能自动重建");
|
||||
const version = Math.max(proposal.version, this.run!.approvedProposalVersion || 0) + 1;
|
||||
const nodeIds = (proposal.extra as { nodeIds?: Record<string, string> } | undefined)?.nodeIds || Object.fromEntries(proposal.workflow.nodes.map((node) => [node.ref, creativeNodeId(this.run!.id, proposal.version, node.ref)]));
|
||||
const operations: CanvasAgentOp[] = Object.values(nodeIds).map((id) => { const current = snapshot.nodes.find((node) => node.id === id); if (!current) throw new Error("原方案节点已被删除，请核对方案后继续"); return { type: "update_node", id, metadata: { ...current.metadata } }; });
|
||||
const nextProposal = { ...proposal, version, extra: { ...(proposal.extra as Record<string, unknown> || {}), nodeIds }, generationItems: proposal.generationItems.map((item) => item.ref === ref ? { ...item, model: String(node.metadata?.model || item.model), size: node.metadata?.size || item.size, seconds: node.metadata?.seconds ? Number(node.metadata.seconds) : item.seconds, quality: (item.mode === "video" ? node.metadata?.vquality : node.metadata?.quality) || item.quality } : item) };
|
||||
assertCreativeBriefSpecifications(nextProposal, this.state.brief);
|
||||
this.state = { ...this.state, proposal: nextProposal, operations, pendingPayment: undefined, planning: undefined, pendingRedo: { ref, attempt: media.attempt + 1, proposalVersion: version } };
|
||||
await this.save("running");
|
||||
await this.continueRedo();
|
||||
});
|
||||
}
|
||||
private async continueRedo() {
|
||||
const intent = this.state.pendingRedo, proposal = this.state.proposal, operations = this.state.operations;
|
||||
if (!intent || !proposal || proposal.version !== intent.proposalVersion || !operations || !this.state.media.some((media) => media.ref === intent.ref)) throw new Error("重做记录不完整，请重新核对方案");
|
||||
if (this.run!.approvedProposalVersion !== intent.proposalVersion || !this.run!.approvedProposalHash) {
|
||||
this.run = await this.api.approveProposal(this.run!.id, { ...this.guard(), revision: this.run!.revision, proposalVersion: intent.proposalVersion, proposal, ops: operations }, this.abort.signal);
|
||||
}
|
||||
this.guard();
|
||||
this.setMedia(intent.ref, { status: "pending", attempt: intent.attempt, submissionId: undefined, taskId: undefined, error: undefined });
|
||||
this.state = { ...this.state, pendingRedo: undefined };
|
||||
await this.save("running");
|
||||
await this.prepareMediaBatch();
|
||||
}
|
||||
}
|
||||
|
||||
function reconcileOps(ops: CanvasAgentOp[], snapshot: CanvasAgentSnapshot): CanvasAgentOp[] {
|
||||
return ops.filter((op) => {
|
||||
if (op.type === "add_node") return !snapshot.nodes.some((node) => node.id === op.id);
|
||||
if (op.type === "connect_nodes") return !snapshot.connections.some((edge) => edge.id === op.id || edge.fromNodeId === op.fromNodeId && edge.toNodeId === op.toNodeId);
|
||||
if (op.type === "run_generation") throw new Error("画布写入不能直接提交收费生成");
|
||||
return true;
|
||||
});
|
||||
}
|
||||
function toolInput(call: ResponseToolCall): ResponseInputMessage { return { type: "function_call", call_id: call.id, name: call.function.name, arguments: call.function.arguments, ...(call.thoughtSignature ? { thoughtSignature: call.thoughtSignature } : {}) }; }
|
||||
68
web/src/services/creative-agent-recovery.ts
Normal file
68
web/src/services/creative-agent-recovery.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
import { nanoid } from "nanoid";
|
||||
import type { AiConfig } from "@/stores/use-config-store";
|
||||
import type { ResponseInputMessage, ResponseToolCall } from "./api/image";
|
||||
import { CREATIVE_SCENARIOS, normalizeCreativeQuestions, type CreativeScenarioId } from "@/lib/creation/creative-agent-contract";
|
||||
import { assertCreativeBriefSpecifications, mergeCreativeBrief, normalizeCreativeProposal, type CreativeAgentState } from "@/lib/creation/creative-agent-state";
|
||||
import { parseCreativeToolArguments } from "@/lib/creation/creative-agent-tools";
|
||||
|
||||
type Reply = { content: string; toolCalls: ResponseToolCall[] };
|
||||
type Context = { config: AiConfig; state: CreativeAgentState; latestInput: string };
|
||||
|
||||
function validate(reply: Reply, context: Context) {
|
||||
const call = reply.toolCalls.find((item) => item.function.name === "creative_respond");
|
||||
if (!call) return;
|
||||
const input = parseCreativeToolArguments(call.function.arguments);
|
||||
const scene = typeof input.scenario === "string" && Object.hasOwn(CREATIVE_SCENARIOS, input.scenario) ? input.scenario as CreativeScenarioId : context.state.scene;
|
||||
const brief = mergeCreativeBrief(context.state.brief, input.brief, scene, context.latestInput);
|
||||
const questions = normalizeCreativeQuestions(input.questions, scene, brief, { interactionId: "validation", revision: 1 }, context.state.references.map((asset) => ({ id: asset.id, label: asset.title })));
|
||||
if (questions.questions.length) return;
|
||||
if (Array.isArray(input.questions) && input.questions.length && !input.proposal) throw new Error("问题均为已知或无效字段。请使用已知答案继续规划，不要重复提问；若缺少其他关键信息，请提出具体的新问题。");
|
||||
if (input.proposal) {
|
||||
const proposal = normalizeCreativeProposal(input.proposal, "validation", 1, context.config, context.state.references);
|
||||
assertCreativeBriefSpecifications(proposal, brief);
|
||||
}
|
||||
}
|
||||
|
||||
function recoveryQuestion(context: Context, error: string): Reply {
|
||||
const assetIssue = /引用|素材|图片节点/.test(error);
|
||||
const modelIssue = /模型|能力|尺寸|比例|时长/.test(error);
|
||||
const field = `recovery.choice-${nanoid(8)}`;
|
||||
const message = assetIssue ? "这份方案的素材依赖仍未能可靠绑定。已有作品保留，请选择下一步，我会按你的选择继续规划。"
|
||||
: modelIssue ? "当前可用生成能力还不能可靠满足这份方案。请决定接下来如何安排，已确认的目标会保留。"
|
||||
: "我尝试修正了方案，但它仍未达到可执行条件。已有内容保留，你可以先完善创意，或补充调整要求后继续。";
|
||||
const choices = assetIssue
|
||||
? [{ id: "choose", label: "先让我选择参考素材", description: "助手下一轮列出当前真实可用素材供选择。" }, { id: "text", label: "先完善文字方案", description: "先讨论内容，不创建或生成媒体。" }]
|
||||
: [{ id: "text", label: "先完善文字方案", description: "保留创意和已确认需求，暂不进入媒体制作。" }, { id: "alternatives", label: "说明可行替代方案", description: "列出差异，由我决定是否修改目标。" }];
|
||||
const input = { scenario: context.state.scene, message, brief: [], questions: [{ field, type: "single", title: "接下来希望怎样推进？", required: true, allowCustom: true, options: choices }] };
|
||||
return { content: message, toolCalls: [{ id: `recovery-${nanoid()}`, type: "function", function: { name: "creative_respond", arguments: JSON.stringify(input) } }] };
|
||||
}
|
||||
|
||||
// Correct rejected arguments before presentation. This loop never executes canvas or media tools.
|
||||
export async function recoverCreativeResponse(initial: Reply, messages: ResponseInputMessage[], context: Context,
|
||||
request: (messages: ResponseInputMessage[]) => Promise<Reply>, onProgress: (attempt: number, error: string) => void): Promise<Reply> {
|
||||
let reply = initial;
|
||||
let protocol = [...messages];
|
||||
for (let attempt = 0; attempt <= 2; attempt++) {
|
||||
let failure: string;
|
||||
try { validate(reply, context); return reply; }
|
||||
catch (error) { failure = error instanceof Error ? error.message : "交互格式无效"; }
|
||||
onProgress(attempt + 1, failure);
|
||||
if (attempt === 2) return recoveryQuestion(context, failure);
|
||||
protocol = [...protocol,
|
||||
...reply.toolCalls.map((call): ResponseInputMessage => ({ type: "function_call", call_id: call.id, name: call.function.name, arguments: call.function.arguments, ...(call.thoughtSignature ? { thoughtSignature: call.thoughtSignature } : {}) })),
|
||||
...reply.toolCalls.map((call): ResponseInputMessage => ({ role: "tool", tool_call_id: call.id, content: JSON.stringify({ status: call.function.name === "creative_respond" ? "validation_failed" : "cancelled", error: failure, executed: false }) })),
|
||||
{ role: "user", content: "这是程序校验反馈，不是用户修改需求。未创建节点或消费媒体报价。请自行修复参数，保留已知要求与真实引用；不让用户填写技术字段。缺少真实素材、事实或能力时，用creative_respond.questions主动提问，选项来自当前上下文且允许自由补充。不再仅说‘已修复’或‘请确认’，必须返回可校验的完整方案或可回答问题。仍无法组成媒体方案时可先提供纯文字创意并提问下一步，不虚构媒体配置。" },
|
||||
];
|
||||
// Only proposal/question tools are supplied to the repair model; there are no writable tools.
|
||||
try {
|
||||
reply = await request(protocol);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") throw error;
|
||||
onProgress(3, error instanceof Error ? error.message : "修正请求失败");
|
||||
return recoveryQuestion(context, failure);
|
||||
}
|
||||
reply = { ...reply, toolCalls: reply.toolCalls.filter((call) => call.function.name === "creative_respond") };
|
||||
if (!reply.toolCalls.length && !reply.content.trim()) return recoveryQuestion(context, failure);
|
||||
}
|
||||
return reply;
|
||||
}
|
||||
@@ -130,7 +130,7 @@ function functionTool(name: string, description: string, properties: Record<stri
|
||||
}
|
||||
|
||||
const PROGRESSIVE_SKILL_TOOLS: ResponseFunctionTool[] = [
|
||||
functionTool("canvas_list_skills", "列出当前用户已加入、可按需加载的画布技能；只返回元数据，不返回完整指令。", {}),
|
||||
functionTool("canvas_list_skills", "自动发现当前用户已加入的技能；可按 query 检索名称、描述或标签，offset/limit 分页；只返回元数据。", { query: { type: "string" }, offset: { type: "number" }, limit: { type: "number" } }),
|
||||
functionTool("canvas_get_skill", "按 skillId 或名称读取技能元数据和入口 SKILL.md；不会加载 references、scripts、assets 等其他文件。", { skillId: { type: "string" }, name: { type: "string" } }),
|
||||
functionTool("canvas_list_skill_files", "列出一个技能包的文件路径、类型和大小，不读取文件正文。", { skillId: { type: "string" }, name: { type: "string" } }),
|
||||
functionTool("canvas_read_skill_file", "按路径读取技能包中的一个文本文件。先从入口引用或文件清单确定路径，禁止猜测路径。", { skillId: { type: "string" }, name: { type: "string" }, path: { type: "string" } }, ["path"]),
|
||||
@@ -343,9 +343,37 @@ function nativePackage(skill: Skill, bundle: SkillPackageBundle): NativeSkillPac
|
||||
|
||||
function createProgressiveToolsAdapter(dependencies: SkillRuntimeDependencies): SkillToolAdapter {
|
||||
const handlers: Record<string, (args: Record<string, unknown>, skills: Skill[]) => Promise<SkillRuntimeToolResult>> = {
|
||||
canvas_list_skills: async (_args, skills) => {
|
||||
const data = skills.filter((skill) => skill.isAdded).map((skill) => ({ skillId: skill.skillId, name: skill.skillName, description: skill.description, tag: skill.tag, version: skill.version, fileCount: skill.fileCount, sourceType: skill.sourceType }));
|
||||
return { ok: true, message: data.length ? "已列出当前可用技能。" : "当前没有已加入技能。", data };
|
||||
canvas_list_skills: async (args, skills) => {
|
||||
const terms = typeof args.query === "string"
|
||||
? args.query.trim().toLocaleLowerCase().split(/\s+/).filter(Boolean)
|
||||
: [];
|
||||
const matches = skills.filter((skill) => skill.isAdded && terms.every((term) =>
|
||||
`${skill.skillName} ${skill.description} ${skill.tag}`.toLocaleLowerCase().includes(term),
|
||||
));
|
||||
const offset = typeof args.offset === "number" && Number.isFinite(args.offset)
|
||||
? Math.max(0, Math.floor(args.offset))
|
||||
: 0;
|
||||
const limit = typeof args.limit === "number" && Number.isFinite(args.limit)
|
||||
? Math.max(1, Math.min(100, Math.floor(args.limit)))
|
||||
: 40;
|
||||
const items = matches.slice(offset, offset + limit).map((skill) => ({
|
||||
skillId: skill.skillId,
|
||||
name: skill.skillName,
|
||||
description: skill.description,
|
||||
tag: skill.tag,
|
||||
version: skill.version,
|
||||
fileCount: skill.fileCount,
|
||||
sourceType: skill.sourceType,
|
||||
}));
|
||||
return {
|
||||
ok: true,
|
||||
message: items.length ? "已发现相关技能，请按需读取入口；无匹配时可缩短关键词或查询完整目录。" : "没有匹配技能，可调整关键词。",
|
||||
data: {
|
||||
items,
|
||||
total: matches.length,
|
||||
nextOffset: offset + items.length < matches.length ? offset + items.length : null,
|
||||
},
|
||||
};
|
||||
},
|
||||
canvas_get_skill: async (args, skills) => {
|
||||
const skill = requireAddedSkill(skills, args);
|
||||
|
||||
35
web/src/services/storyboard-submission.ts
Normal file
35
web/src/services/storyboard-submission.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
import { nanoid } from "nanoid";
|
||||
import { creationRuns, type CreationSubmission } from "./api/creation-runs";
|
||||
import type { CreateTaskInput } from "./api/task-center";
|
||||
|
||||
// Both the node composer and Agent submit the same quoted professional task.
|
||||
export async function submitStoryboardTask(request: CreateTaskInput, options: {
|
||||
signal: AbortSignal;
|
||||
assertCurrent: () => void;
|
||||
confirm: (submission: CreationSubmission) => Promise<boolean>;
|
||||
}, api = creationRuns) {
|
||||
const { signal, assertCurrent } = options;
|
||||
const detail = await api.create({ clientKey: `storyboard:${nanoid()}`, canvasId: request.projectId, state: { kind: "storyboard" } }, signal);
|
||||
const run = await api.claim(detail.run.id, { expectedEpoch: detail.run.executionEpoch, owner: `storyboard:${nanoid()}` }, signal);
|
||||
const guard = { executionEpoch: run.executionEpoch, owner: run.executionOwner };
|
||||
let connectionError: unknown;
|
||||
const heartbeat = setInterval(() => {
|
||||
void api.heartbeat(run.id, guard, signal).catch((error) => { connectionError = error; });
|
||||
}, 15_000);
|
||||
try {
|
||||
const submission = await api.prepare(run.id, { ...guard, itemKey: "storyboard", request }, signal);
|
||||
if (!await options.confirm(submission)) return undefined;
|
||||
signal.throwIfAborted();
|
||||
if (connectionError) throw connectionError;
|
||||
assertCurrent();
|
||||
if (Date.parse(submission.quote.expiresAt) <= Date.now()) throw new Error("分镜报价已过期，请重新发起拆镜并确认最新费用");
|
||||
const approved = await api.approve(run.id, { ...guard, submissionIds: [submission.id] }, signal);
|
||||
if (!approved.submissions.some((item) => item.id === submission.id && item.approvedAt)) throw new Error("费用确认回执不完整，尚未提交拆镜任务");
|
||||
signal.throwIfAborted();
|
||||
assertCurrent();
|
||||
return await api.execute(run.id, { ...guard, submissionId: submission.id }, signal);
|
||||
} finally {
|
||||
clearInterval(heartbeat);
|
||||
void api.release(run.id, guard).catch(() => { /* Offline leases expire on the server. */ });
|
||||
}
|
||||
}
|
||||
@@ -224,7 +224,7 @@ export function localSavedRemotePendingMessage(localAction: string, error: unkno
|
||||
return formatLocalSavedRemotePending(localAction, error);
|
||||
}
|
||||
|
||||
export async function createCanvasProjectWithRemoteSync(title: string, projectId?: string, initialContent?: Partial<Pick<CanvasProject, "nodes" | "connections">>) {
|
||||
export async function createCanvasProjectWithRemoteSync(title: string, projectId?: string, initialContent?: Partial<Pick<CanvasProject, "nodes" | "connections" | "chatSessions" | "activeChatId">>) {
|
||||
const id = useCanvasStore.getState().createProject(title, projectId);
|
||||
if (initialContent) useCanvasStore.getState().updateProject(id, initialContent);
|
||||
if (!activeRemoteUserId) return { id, syncError: new Error("尚未建立云端同步会话") };
|
||||
|
||||
Reference in New Issue
Block a user