From abddb6c7509f559a08b8f95002a4da205090919c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=BF=97=E4=BC=9F?= Date: Tue, 8 Sep 2026 22:24:17 +0800 Subject: [PATCH] =?UTF-8?q?fix(agent):=20=E5=88=9B=E4=BD=9C=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=20-=20=E4=BF=AE=E5=A4=8D=E4=BC=9A=E8=AF=9D=E6=81=A2?= =?UTF-8?q?=E5=A4=8D=E5=B9=B6=E6=A0=A1=E9=AA=8C=E7=94=BB=E9=A3=8E=E4=B8=8E?= =?UTF-8?q?=E7=9C=9F=E5=AE=9E=E5=88=86=E9=95=9C=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/design/agent-capabilities.mdx | 4 +- .../canvas/canvas-assistant-panel.tsx | 10 ++-- .../canvas/canvas-creative-interaction.tsx | 19 +++++-- web/src/lib/canvas/canvas-agent-context.ts | 2 + web/src/lib/canvas/canvas-agent-storyboard.ts | 12 ++++- web/src/lib/canvas/canvas-agent-workflow.ts | 1 + .../lib/canvas/canvas-storyboard-context.ts | 24 +++++++++ web/src/lib/creation/creative-agent-state.ts | 3 +- web/src/lib/creation/creative-agent-tools.ts | 3 +- web/src/services/creative-agent-controller.ts | 40 ++++++++++----- web/test/agent-capabilities.test.ts | 7 +++ web/test/agent-storyboard.test.ts | 31 +++++++++++- web/test/creative-agent-controller.test.ts | 50 +++++++++++++++++++ 13 files changed, 179 insertions(+), 27 deletions(-) diff --git a/docs/design/agent-capabilities.mdx b/docs/design/agent-capabilities.mdx index e61c0ea97..72c778f08 100644 --- a/docs/design/agent-capabilities.mdx +++ b/docs/design/agent-capabilities.mdx @@ -17,7 +17,9 @@ description: 在线画布助手的节点发现、技能渐进读取和插件动 创作方案和通用工作流支持 `styleboard`、`story_input`、`script`，复用手动短剧入口的数据结构。画风节点初始为空，显示系统已有的选择入口；故事节点使用专用编辑卡片；分镜节点使用真实表格。 -`script.shots` 可填写逐镜 `durationSeconds`、`videoMotionPrompt`、`dialogue`，最多 100 行，写入稳定行 ID。方案连到分镜节点时使用 `storyboard:context` 输入。仅创建这些节点不产生媒体生成项；图像、视频仍沿用既有方案和费用确认。 +`script.shots` 必须逐镜填写正数 `durationSeconds` 和非空 `videoMotionPrompt`，`dialogue` 按需填写，最多 100 行，写入稳定行 ID。仅有正文或提示词的分镜方案会返回校验错误供模型修正；通用节点创建工具也校验真实分镜行，不能以空表代替交付。方案连到分镜节点时使用 `storyboard:context` 输入。仅创建这些节点不产生媒体生成项；图像、视频仍沿用既有方案和费用确认。 + +`canvas_get_context.storyboardReadiness` 与 `canvas_read_storyboard.readiness` 复用原分镜生成前置校验，返回画风状态、阻塞原因、真实行数量、未填完整的行 ID 和实际总时长。助手先检查这些状态；缺画风时读取真实目录，结合已确认风格与用户授权应用，只有关键取舍尚不明确时才追问。已有空表通过逐行编辑补齐，保留原行 ID 和素材关联；有足够剧本信息时直接编写镜头，不要求用户重复手动生成分镜。提示词引导不替代生成入口的业务校验，写入镜头也不代表媒体已生成。 `canvas_list_styles` 按 `source=system|user` 查询系统预设或当前用户保存的风格。`canvas_apply_style` 只接受该目录返回的真实 ID，复用页面的异步画风保存入口：关联项目时先保存项目设置，再应用画布节点；不关联项目时走原画布保存机制。异步返回时校验画布、项目及用户上下文，避免写到新页面。该动作是原画风选择逻辑，不冒充普通节点撤销能够回滚服务端项目设置；需要改回时重新应用原风格。 diff --git a/web/src/components/canvas/canvas-assistant-panel.tsx b/web/src/components/canvas/canvas-assistant-panel.tsx index 99348ddcc..1e7620e73 100644 --- a/web/src/components/canvas/canvas-assistant-panel.tsx +++ b/web/src/components/canvas/canvas-assistant-panel.tsx @@ -15,7 +15,7 @@ import { runBackendToolGenerationTask } from "@/services/api/generation-task"; import { inspectAgentImage } from "@/services/agent-image-preview"; import { AGENT_CAPABILITY_GUIDANCE, buildAgentPluginOperations, listAgentCapabilities, readAgentPluginDocumentation, readAgentPluginNode } from "@/services/agent-capabilities"; import { getNodeDefinition } from "@/lib/canvas/node-registry"; -import { buildAgentStoryboardOperations, readAgentStoryboard } from "@/lib/canvas/canvas-agent-storyboard"; +import { buildAgentStoryboardOperations, readAgentStoryboard, validateAgentStoryboardCreation } from "@/lib/canvas/canvas-agent-storyboard"; import { canvasStylePresets, userStylePreset, type CanvasStylePreset } from "./canvas-style-picker-modal"; import { listStyleProfiles } from "@/services/api/style-profiles"; import { getActiveUserScope } from "@/lib/user-scope"; @@ -879,8 +879,8 @@ export function CanvasAssistantPanel({ } if (name === "canvas_read_plugin_node") return { ok: true, message: "已读取插件节点状态。", data: readAgentPluginNode(current, requireString(args.nodeId, "nodeId")) }; if (name === "canvas_read_storyboard") { - const { node, rows } = readAgentStoryboard(current, requireString(args.nodeId, "nodeId")); - return { ok: true, message: "已读取分镜表。", data: { nodeId: node.id, title: node.title, rows } }; + const { node, rows, readiness } = readAgentStoryboard(current, requireString(args.nodeId, "nodeId")); + return { ok: true, message: "已读取分镜表及画风前置条件。", data: { nodeId: node.id, title: node.title, rows, readiness } }; } if (name === "canvas_list_styles" || name === "canvas_apply_style") { const originScope = getActiveUserScope(); @@ -1564,13 +1564,13 @@ function parseToolArguments(value: string) { export function onlineToolToOps(name: string, input: Record, snapshot: CanvasAgentSnapshot, config: AiConfig): CanvasAgentOp[] { if (name === "canvas_edit_storyboard") return buildAgentStoryboardOperations(snapshot, input); if (name === "canvas_plugin_action") return buildAgentPluginOperations(requireString(input.pluginId, "pluginId"), requireString(input.actionId, "actionId"), recordOptional(input.input) || {}, snapshot); - if (name === "canvas_apply_ops") return requireOps(input.ops); + if (name === "canvas_apply_ops") return validateAgentStoryboardCreation(requireOps(input.ops)); if (name === "canvas_create_workflow") return buildCanvasWorkflowOps(input as unknown as CanvasWorkflowInput, snapshot, config); if (name === "canvas_create_node") { const nodeType = requireNodeType(input.nodeType); const x = numberOr(input.x, nextCanvasX(snapshot)); const y = numberOr(input.y, 0); - return [{ type: "add_node", nodeType, title: stringOptional(input.title), position: { x, y }, width: numberOptional(input.width), height: numberOptional(input.height), metadata: recordOptional(input.metadata) as CanvasNodeData["metadata"] }]; + return validateAgentStoryboardCreation([{ type: "add_node", nodeType, title: stringOptional(input.title), position: { x, y }, width: numberOptional(input.width), height: numberOptional(input.height), metadata: recordOptional(input.metadata) as CanvasNodeData["metadata"] }]); } if (name === "canvas_create_text_node") return [textNodeOp(input, numberOr(input.x, nextCanvasX(snapshot)), numberOr(input.y, 0))]; if (name === "canvas_create_text_nodes") { diff --git a/web/src/components/canvas/canvas-creative-interaction.tsx b/web/src/components/canvas/canvas-creative-interaction.tsx index 80c36d0e9..312c98289 100644 --- a/web/src/components/canvas/canvas-creative-interaction.tsx +++ b/web/src/components/canvas/canvas-creative-interaction.tsx @@ -57,12 +57,14 @@ export function CanvasCreativeInteraction(props: Props) { const quoteHistory = useRef(detail.quotes || []); const answersEdited = useRef(false); const presentationAttempted = useRef(false); + const hadControl = useRef(false); const scope = getActiveUserScope(); const invoke = (promise?: Promise) => { if (promise) { setError(""); void promise.catch((cause) => { if (!(cause instanceof DOMException && cause.name === "AbortError")) setError(cause instanceof Error ? cause.message : "操作未完成"); }); } }; useEffect(() => { let live = true; presentationAttempted.current = false; + hadControl.current = false; const original = canvasCreativeDetail(latest.current.message)!; setView({ state: original.state, busy: false, hasControl: false }); const instance = new CreativeAgentController({ @@ -72,6 +74,9 @@ export function CanvasCreativeInteraction(props: Props) { onOpenCanvas: () => setError("当前会话仅操作所在画布，请在对应画布打开历史会话。"), onChange: (next) => { if (!live) return; + // Recovery clears the controller error and the rejected action's local UI error together. + if (next.hasControl && !hadControl.current && !next.error) setError(""); + hadControl.current = next.hasControl; setView(next); latest.current.onBusy(props.message.id, next.busy); if (!next.run || !latest.current.active) return; if (next.quote) quoteHistory.current = [...quoteHistory.current.filter((quote) => quote.id !== next.quote!.id), next.quote]; @@ -126,7 +131,9 @@ export function CanvasCreativeInteraction(props: Props) { props.onContinue("保留已确认的要求，在已授权范围内继续当前任务，自行处理常规选择和技术问题；确实阻塞时再询问。"); }; const rawError = proposalFailure || error || view.error; - const feedback = rawError ? /恢复|连接|网络|接管|执行权|代次/.test(rawError) ? "连接暂时中断，正在恢复。已有作品会保留。" + const failedGenerations = view.state.media.filter((item) => item.status === "failed" && item.failureKind !== "observation"); + const onlyFailedRemaining = failedGenerations.length > 0 && !view.state.media.some((item) => ["queued", "running", "write_failed"].includes(item.status) || item.failureKind === "observation" && item.status === "failed"); + const feedback = onlyFailedRemaining ? "部分作品生成失败。请在失败作品下方重新生成，先核对费用；已成功的作品会保留。" : rawError ? /恢复|连接|网络|接管|执行权|代次/.test(rawError) ? "状态读取未完成，可以继续处理重试。已有作品会保留。" : /报价|费用|价格|过期/.test(rawError) ? "生成费用需要重新确认，继续后会显示最新价格。" : /素材|参考|引用/.test(rawError) ? "参考素材还需要核对，助手会继续处理，必要时请你选择。" : "这一步还没有完成。已有内容已保留，可以继续处理或修改要求。" : undefined; @@ -147,23 +154,27 @@ export function CanvasCreativeInteraction(props: Props) { {view.state.pendingEdits &&

将调整 {view.state.pendingEdits.length} 个节点的指定字段。

} {props.active && view.quote && invoke(controller.current?.approvePayment())} onRefresh={() => invoke(controller.current?.refreshQuotes())} />} {(detail.quotes || []).filter((quote) => !props.active || quote.id !== view.quote?.id).map((quote) =>
{quote.approvedQuantity ? "已确认生成" : "历史报价"} · {quote.amountLabel} · {quote.items.length} 项

{quote.basis}

    {quote.items.map((item) =>
  • {item.label} · {item.model} · {item.specification}
    • )}
)} - {view.state.media.filter((item) => item.storageKey || item.error).map((item) => { + {view.state.media.filter((item) => item.taskId || item.storageKey || item.error).map((item) => { const resource = resourceIdFromStorageKey(item.storageKey), url = resource ? resourceFileUrl(resource) : ""; const video = view.state.proposal?.generationItems.find((entry) => entry.ref === item.ref)?.mode === "video"; return
{view.state.proposal?.workflow.nodes.find((node) => node.ref === item.ref)?.title || item.ref} + {!item.error && !url && item.taskId &&

{view.busy ? item.status === "running" ? "正在生成，完成后自动显示。" : "已提交，正在等待生成结果。" : "任务已提交。点击下方“继续处理”读取最新结果，无需重复确认费用。"}

} {url && (video ?
; })} {!props.active && rawError &&
这一步未完成，后续对话中可继续调整

{feedback}

} {props.active &&
-

{view.busy ? "正在处理，请稍候…" : !view.hasControl ? "正在恢复当前创作，请稍候…" : view.state.modificationRequested ? "在下方告诉我想改哪里，调整后再确认。" : feedback || (view.quote ? "确认费用后，就开始本批制作。" : awaitingProposal ? "看看方案是否符合你的想法，确认后继续。" : question?.status === "pending" ? "选择一个答案，也可以在下方直接补充。" : status === "completed" ? "本阶段已完成。可以查看作品，或告诉我想改哪里。" : "准备好了，可以继续。")}

+

{view.busy ? "正在处理，请稍候…" : !view.hasControl ? rawError ? "暂时无法恢复当前创作，可以重试连接。已有作品会保留。" : "正在恢复当前创作，请稍候…" : view.state.modificationRequested ? "在下方告诉我想改哪里，调整后再确认。" : feedback || (view.quote ? "确认费用后，就开始本批制作。" : awaitingProposal ? "看看方案是否符合你的想法，确认后继续。" : question?.status === "pending" ? "选择一个答案，也可以在下方直接补充。" : status === "completed" ? "本阶段已完成。可以查看作品，或告诉我想改哪里。" : "准备好了，可以继续。")}

+ {!view.busy && !view.hasControl && } {view.busy && ["running", "waiting_task", "waiting_canvas"].includes(status || "") ? : !view.busy && view.hasControl && !view.state.modificationRequested && !view.quote && !awaitingProposal && !view.state.pendingEdits && question?.status !== "pending" ? <> - + {!onlyFailedRemaining && } : null}
diff --git a/web/src/lib/canvas/canvas-agent-context.ts b/web/src/lib/canvas/canvas-agent-context.ts index c3cc17dc6..5d912c0a5 100644 --- a/web/src/lib/canvas/canvas-agent-context.ts +++ b/web/src/lib/canvas/canvas-agent-context.ts @@ -2,6 +2,7 @@ import { hashCanvasAgentSnapshot, type CanvasAgentOp, type CanvasAgentSnapshot } import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas"; import { getNodeDefinition } from "./node-registry"; import { isPluginEffectivelyEnabled } from "@/stores/use-plugin-store"; +import { inspectStoryboardReadiness } from "./canvas-storyboard-context"; export type CanvasAgentResource = { nodeId: string; @@ -47,6 +48,7 @@ export function buildCanvasAgentContext(snapshot: CanvasAgentSnapshot, options: nodes: nodes.map(compactNode), connections: snapshot.connections.map((connection) => ({ id: connection.id, fromNodeId: connection.fromNodeId, fromTitle: nodeById.get(connection.fromNodeId)?.title || "未知节点", toNodeId: connection.toNodeId, toTitle: nodeById.get(connection.toNodeId)?.title || "未知节点", fromHandleId: connection.fromHandleId, toHandleId: connection.toHandleId })), resources, + storyboardReadiness: inspectStoryboardReadiness(snapshot.nodes), warnings: [ ...(nodes.some((node) => node.metadata?.status === "error") ? ["画布中存在生成失败节点；重试前先检查错误信息。"] : []), ...(resources.some((resource) => !resource.ready) ? ["存在未就绪或缺少持久化引用的媒体节点，不要把占位节点当作可用参考素材。"] : []), diff --git a/web/src/lib/canvas/canvas-agent-storyboard.ts b/web/src/lib/canvas/canvas-agent-storyboard.ts index 68a0e20e4..cbee247ed 100644 --- a/web/src/lib/canvas/canvas-agent-storyboard.ts +++ b/web/src/lib/canvas/canvas-agent-storyboard.ts @@ -1,13 +1,23 @@ import type { CanvasAgentOp, CanvasAgentSnapshot } from "./canvas-agent-ops"; import { createStoryboardRow } from "./canvas-project-domain"; import type { StoryboardRow } from "@/types/canvas"; +import { inspectStoryboardReadiness } from "./canvas-storyboard-context"; const textFields = ["plotDescription", "dialogue", "videoMotionPrompt", "imageGenerationPrompt", "camera", "motion", "shotSize", "emotion", "lightingAndAtmosphere", "audioEffects", "narrativeIntent", "viewerPOV", "performanceBlocking", "timeBeats", "continuityOut", "negativePrompt"] as const; +export function validateAgentStoryboardCreation(ops: CanvasAgentOp[]) { + for (const op of ops) { + if (op.type !== "add_node" || op.nodeType !== "script") continue; + const rows = op.metadata?.storyboard?.rows; + if (!Array.isArray(rows) || !rows.length || rows.length > 100 || rows.some((row) => !Number.isFinite(row.durationSeconds) || row.durationSeconds <= 0 || !row.videoMotionPrompt?.trim())) throw new Error("分镜节点必须包含真实镜头行。请使用 canvas_create_workflow 的 script.shots 填写每镜时长和视频提示词，不能仅填正文；已有分镜使用 canvas_edit_storyboard"); + } + return ops; +} + export function readAgentStoryboard(snapshot: CanvasAgentSnapshot, nodeId: string) { const node = snapshot.nodes.find((item) => item.id === nodeId && item.type === "script"); if (!node) throw new Error("未找到当前画布的分镜脚本节点，请先查找真实节点"); - return { node, rows: node.metadata?.storyboard?.rows || [] }; + return { node, rows: node.metadata?.storyboard?.rows || [], readiness: inspectStoryboardReadiness(snapshot.nodes) }; } export function buildAgentStoryboardOperations(snapshot: CanvasAgentSnapshot, input: Record): CanvasAgentOp[] { diff --git a/web/src/lib/canvas/canvas-agent-workflow.ts b/web/src/lib/canvas/canvas-agent-workflow.ts index 3d5f48310..390f1d58f 100644 --- a/web/src/lib/canvas/canvas-agent-workflow.ts +++ b/web/src/lib/canvas/canvas-agent-workflow.ts @@ -112,6 +112,7 @@ export function buildCanvasWorkflowOps(input: CanvasWorkflowInput, snapshot: Can const source = template.nodes[node.kind === "styleboard" ? 0 : node.kind === "story_input" ? 1 : 2]; Object.assign(metadata, source.metadata, { content: node.kind === "styleboard" ? "" : node.content || "", composerContent: prompt }); if (node.kind === "script") { + if (!node.shots?.length) throw new Error("创建分镜工作流必须提供非空 shots，请把镜头逐行填写 durationSeconds、videoMotionPrompt、dialogue，不能仅写正文"); if (node.shots && (!Array.isArray(node.shots) || node.shots.length > 100)) throw new Error("分镜最多 100 行"); metadata.storyboard = { ...source.metadata!.storyboard!, rows: (node.shots || []).map((shot, rowIndex) => { if (!Number.isFinite(shot.durationSeconds) || shot.durationSeconds <= 0 || typeof shot.videoMotionPrompt !== "string" || !shot.videoMotionPrompt.trim()) throw new Error("每条分镜需要有效时长和视频提示词"); diff --git a/web/src/lib/canvas/canvas-storyboard-context.ts b/web/src/lib/canvas/canvas-storyboard-context.ts index 066ce2354..8b12e5f3e 100644 --- a/web/src/lib/canvas/canvas-storyboard-context.ts +++ b/web/src/lib/canvas/canvas-storyboard-context.ts @@ -44,3 +44,27 @@ export function resolveStoryboardGenerationContext(nodes: CanvasNodeData[]): Sto })), }; } + +// Expose the same generation prerequisites to the Agent before it starts a workflow. +export function inspectStoryboardReadiness(nodes: CanvasNodeData[]) { + let generationContext: StoryboardGenerationContext | undefined; + let blockingReason: string | undefined; + try { generationContext = resolveStoryboardGenerationContext(nodes); } + catch (error) { blockingReason = error instanceof Error ? error.message : "分镜上下文未就绪"; } + const styleNode = nodes.find((node) => node.metadata?.workflowKind === "styleboard"); + const styleReady = Boolean(styleNode?.metadata?.stylePresetId?.trim() && String(styleNode.metadata.content || styleNode.metadata.prompt || "").trim()); + const storyboards = nodes.filter((node) => node.type === "script").map((node) => { + const rows = node.metadata?.storyboard?.rows || []; + const incompleteRowIds = rows.filter((row) => !Number.isFinite(row.durationSeconds) || row.durationSeconds <= 0 || !row.videoMotionPrompt?.trim()).map((row) => row.id); + return { nodeId: node.id, title: node.title, rowCount: rows.length, populated: rows.length > 0 && incompleteRowIds.length === 0, incompleteRowIds, totalDurationSeconds: rows.reduce((total, row) => total + (Number.isFinite(row.durationSeconds) && row.durationSeconds > 0 ? row.durationSeconds : 0), 0) }; + }); + return { + canGenerateStoryboard: Boolean(generationContext), blockingReason, + style: { ready: styleReady, nodeId: styleNode?.id, presetId: styleNode?.metadata?.stylePresetId, title: styleNode?.title }, + storyboards, + nextActions: [ + ...(!styleReady ? ["先 canvas_list_styles 读取真实画风，结合已确认风格调用 canvas_apply_style；该工具会复用或创建画风节点。仅在缺少关键风格取舍时询问用户。"] : []), + ...(storyboards.some((item) => !item.populated) ? ["先 canvas_read_storyboard 读取真实行 ID，再 canvas_edit_storyboard 更新空行并按需新增镜头；正文或标题不代表已填充分镜表。"] : []), + ], + }; +} diff --git a/web/src/lib/creation/creative-agent-state.ts b/web/src/lib/creation/creative-agent-state.ts index e1fb78e5c..f716924ef 100644 --- a/web/src/lib/creation/creative-agent-state.ts +++ b/web/src/lib/creation/creative-agent-state.ts @@ -10,7 +10,7 @@ import { CREATIVE_SCENARIOS, type CreativeAnswers, type CreativeBrief, type Crea export type CreativeReference = { id: string; title: string; kind: "image" | "text"; assetId?: string; storageKey?: string; text?: string; mimeType?: string; width?: number; height?: number }; export type CreativeMessage = { id: string; role: "user" | "assistant"; text: string; question?: CreativeQuestionRequest; answers?: CreativeAnswers; proposal?: CreativeProposal }; -export type CreativeMediaState = { ref: string; nodeId: string; attempt: number; submissionId?: string; taskId?: string; status: "pending" | "queued" | "running" | "ready" | "failed" | "write_failed"; storageKey?: string; error?: string }; +export type CreativeMediaState = { ref: string; nodeId: string; attempt: number; submissionId?: string; taskId?: string; status: "pending" | "queued" | "running" | "ready" | "failed" | "write_failed"; storageKey?: string; error?: string; failureKind?: "generation" | "observation" }; export type CreativeAgentState = { schemaVersion: 1; scene: CreativeScenarioId; brief: CreativeBrief; messages: CreativeMessage[]; references: CreativeReference[]; selectedSkillIds: string[]; textModel?: string; @@ -70,6 +70,7 @@ export function normalizeCreativeProposal(raw: unknown, id: string, version: num if (!assetId && (["text", "story_input"].includes(kind) ? !content : ["image", "video"].includes(kind) && !prompt)) throw new Error("文本节点需要正文，媒体节点需要生成提示词"); const shots = Array.isArray(node.shots) ? node.shots.map((value) => { const shot = record(value); const durationSeconds = Number(shot.durationSeconds); if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) throw new Error("分镜时长必须大于零"); return { durationSeconds, videoMotionPrompt: required(shot.videoMotionPrompt, "分镜视频提示词"), dialogue: str(shot.dialogue) }; }) : undefined; if (shots && shots.length > 100) throw new Error("分镜最多 100 行"); + if (kind === "script" && !shots?.length) throw new Error("分镜方案必须提供非空 script.shots，每镜填写 durationSeconds 和 videoMotionPrompt；请把已设计镜头转成真实行，不能只填 content 或 prompt"); return { ref: required(node.ref, "节点引用"), kind: kind as "text" | "image" | "video" | "styleboard" | "story_input" | "script", title: required(node.title, "节点标题"), content, prompt, shots, runGeneration: false, referenceRefs: Array.isArray(node.referenceRefs) ? node.referenceRefs.map(String) : [], referenceNodeIds: Array.isArray(node.referenceNodeIds) ? node.referenceNodeIds.map(String) : [] }; }); const refs = new Set(nodes.map((node) => node.ref)); diff --git a/web/src/lib/creation/creative-agent-tools.ts b/web/src/lib/creation/creative-agent-tools.ts index 5ef8a8ddd..202c5b4bf 100644 --- a/web/src/lib/creation/creative-agent-tools.ts +++ b/web/src/lib/creation/creative-agent-tools.ts @@ -31,7 +31,8 @@ export const CREATIVE_AGENT_SYSTEM_PROMPT = `你是影策的通用创作助手 多步骤创作使用 creative_respond。先结合全部需求摘要、会话、用户素材、专业角色指导、选中技能与真实画布，提取已知信息；只有缺失或冲突且确实影响执行的信息才补问，通常1至3题，确需一起决定时最多6题，不凑题。允许自定义安全的需求字段，优先复用已有字段，不通过改名重复询问。用户直接输入的自然语言回答与卡片回答同等有效，可以同时更新多个字段。不得将猜测写成用户确认。用户允许自主决定时可以直接给方案。 多步骤任务可携带 plan，按本次实际工作命名步骤，例如“确定结尾情绪”“设计商品构图”“制作首张主图”；不套固定步骤数。每步 ref 在同一工作持续时保持稳定，新工作使用新ref。phase只用于程序关联真实状态：questions=需求选择，proposal=内容方案，canvas=节点落地，media=产物生成（mediaRefs关联方案生成项ref）。可以调整未完成步骤并在reason说明原因；不编造完成状态。普通咨询不创建计划。已完成工作不可抹掉，新增任务不能复用已完成步骤ref。 问题与方案不能在同一次响应同时推进。已经确认的题材/时长/数量/比例/风格不再问。覆盖已确认字段必须在 evidence 引用最新用户消息中明确修改该字段的原文；不能将建议或旧消息作为修改授权。素材题的 option.id 只能使用上下文提供的真实素材ID。 -方案应是可阅读的完整 Markdown，包含内容、已有素材、待生成产物、规格和真实依赖。workflow 支持 text/image/video/styleboard/story_input/script；搭建短剧工作流时使用真实的 styleboard 项目画风选择节点、story_input 故事梗概和 script 分镜表，已有节点优先沿用。script.shots 可填写逐镜 durationSeconds、videoMotionPrompt、dialogue，不用普通文本模拟表格。styleboard 是待选择入口，不把任意文字当作已应用画风；用户已有画风时不重复创建。只有 image/video 填写 generationItems；创建分镜表和画风入口不代表生成媒体。媒体必须有完整提示词，文本必须有正文。每个待生成媒体节点有且仅有一项 generationItems，使用可用模型的完整 model ID。用户已有图片优先用 image 节点的 assetId 指向真实 references.id，此类节点不写 generationItems、不重复生成，可被其他节点用 referenceRefs 引用。 +方案应是可阅读的完整 Markdown，包含内容、已有素材、待生成产物、规格和真实依赖。workflow 支持 text/image/video/styleboard/story_input/script；搭建短剧工作流时使用真实的 styleboard 项目画风选择节点、story_input 故事梗概和 script 分镜表，已有节点优先沿用。script.shots 必须逐镜填写 durationSeconds、videoMotionPrompt、dialogue，不用普通文本模拟表格。styleboard 是待选择入口，不把任意文字当作已应用画风；用户已有画风时不重复创建。只有 image/video 填写 generationItems；创建分镜表和画风入口不代表生成媒体。媒体必须有完整提示词，文本必须有正文。每个待生成媒体节点有且仅有一项 generationItems，使用可用模型的完整 model ID。用户已有图片优先用 image 节点的 assetId 指向真实 references.id，此类节点不写 generationItems、不重复生成，可被其他节点用 referenceRefs 引用。 +分镜任务先检查 canvas_get_context.storyboardReadiness 或 canvas_read_storyboard.readiness。缺少画风时先 canvas_list_styles，再按用户已确认的风格或自主决策授权调用 canvas_apply_style；只有确实需要用户取舍时才提问，不重复询问已有风格。应用后重新读取上下文确认 canGenerateStoryboard，不能只在正文声称画风已设置。方案批准后创建 script 时必须提供非空 shots，不能只写 content/prompt 后把默认空行当成完成。已有分镜用 canvas_read_storyboard 取得真实行 ID，再 canvas_edit_storyboard 更新空行、按需追加镜头，保留已有行 ID 与素材；已有剧本足够时由你直接编写镜头，不要求用户再次点击生成分镜。0–2秒对应 durationSeconds=2，2–4秒为2秒，4–7秒为3秒，7–9秒为2秒，9–10秒为1秒；写入后核对总时长与约定一致。不用改标题代替修改时长，不声称有分镜行就已经生成图片视频。 generationItems 是方案的声明式生成配置，用于校验和之后的报价，不会立即调用媒体模型。因此“确认后再生成”仍必须填写它，不能省略。每项 ref 必须对应节点 ref，mode 必须等于节点 kind，model 必须来自 availableModels 中相同 mode 的条目；图片模型不能用于视频。上下文若有 rejectedProposal，先根据其中真实校验错误修正，保留已知需求并返回完整方案。结构化方案和费用的批准以程序记录为准；若仍缺按钮确认，直接显示对应卡片，不再先问一遍是否同意、再让用户点卡片。 引用已有画布图片时，将真实 references.id 放到 workflow.nodes[].referenceNodeIds；referenceRefs 优先用于本次方案内其他图片节点的 ref。不要为引用现有图片再生成一张副本，不要在两个字段重复放同一张图。程序会核实真实资源并整理引用，未知ID不能通过。 单段视频可以在正文描述多个叙事节奏，不把它们拆成未合成的几段视频。用户要求15秒时只选实际支持15秒的模型；不存在则解释缺少能力并询问可接受替代，绝不能偷偷缩短。图像引用数量、视频输入角色、尺寸均须符合上下文能力。边表达真实依赖，不为了好看造线。 diff --git a/web/src/services/creative-agent-controller.ts b/web/src/services/creative-agent-controller.ts index 68c79cf64..824fcca03 100644 --- a/web/src/services/creative-agent-controller.ts +++ b/web/src/services/creative-agent-controller.ts @@ -52,11 +52,11 @@ export class CreativeAgentController { this.run = await this.api.save(this.run!.id, { ...guard, revision: this.run!.revision, status, state: this.state as unknown as Record }, this.abort.signal); this.assertLive(); this.emit(); } - private async action(operation: () => Promise) { + private async action(operation: () => Promise, takeOver = false) { if (this.busy) throw new Error("上一项操作仍在处理，请稍候"); if (this.disposed || this.scope !== getActiveUserScope()) throw new DOMException("页面已关闭", "AbortError"); this.busy = true; this.error = undefined; this.emit(); - try { if (this.run) await this.ensureControl(); await operation(); } + try { if (this.run) await this.ensureControl(takeOver); await operation(); } catch (error) { if (!this.disposed && !(error instanceof DOMException && error.name === "AbortError")) { this.error = error instanceof Error ? error.message : "操作未完成"; this.emit(); } throw error; @@ -70,7 +70,7 @@ export class CreativeAgentController { this.assertLive(); if (!readOnly) { this.startHeartbeat(); - await this.ensureControl(); + await this.ensureControl(true); } this.emit(); }); @@ -85,7 +85,11 @@ export class CreativeAgentController { this.heartbeat = setInterval(() => { if (this.disposed || !this.run) return; if (!this.hasControl) { - if (!this.busy) void this.ensureControl().then(() => { this.error = undefined; this.emit(); }).catch(() => undefined); + if (!this.busy) void this.ensureControl().then(() => { this.error = undefined; this.emit(); }).catch((error) => { + if (this.disposed) return; + this.error = error instanceof Error ? error.message : "连接恢复失败，请重试"; + this.emit(); + }); return; } if (this.abort.signal.aborted) return; @@ -95,16 +99,20 @@ export class CreativeAgentController { }, 15_000); } private restoring?: Promise; - private async ensureControl() { + private async ensureControl(takeOver = false): Promise { if (this.hasControl && !this.abort.signal.aborted && Date.parse(this.run?.leaseExpiresAt || "") > Date.now() + 3000) return; - if (this.restoring) return this.restoring; + if (this.restoring) { + if (!takeOver) return this.restoring; + try { await this.restoring; } catch { /* Explicit reconnect retries after the passive recovery attempt. */ } + return this.ensureControl(true); + } this.restoring = (async () => { if (this.disposed || this.scope !== getActiveUserScope()) throw new DOMException("页面已关闭", "AbortError"); if (this.abort.signal.aborted) this.abort = new AbortController(); const detail = await this.api.get(this.run!.id, this.abort.signal); this.assertLive(); const run = detail.run; - if (run.executionOwner && run.executionOwner !== this.owner && Date.parse(run.leaseExpiresAt || "") > Date.now()) throw new Error("正在恢复当前创作，请稍候再继续。"); + if (!takeOver && run.executionOwner && run.executionOwner !== this.owner && Date.parse(run.leaseExpiresAt || "") > Date.now()) throw new Error("当前创作连接已切换，请点击重试连接继续。"); // Only replace the lease here; never overwrite local edits or unknown submission results. this.run = { ...this.run!, executionOwner: run.executionOwner, executionEpoch: run.executionEpoch, leaseExpiresAt: run.leaseExpiresAt }; await this.claim(); @@ -113,7 +121,7 @@ export class CreativeAgentController { } async takeControl() { if (this.disposed) return; - await this.action(async () => { this.accept(await this.api.get(this.run!.id, this.abort.signal)); }); + await this.action(async () => { this.accept(await this.api.get(this.run!.id, this.abort.signal)); }, true); } async refresh() { await this.action(async () => { this.accept(await this.api.get(this.run!.id, this.abort.signal)); }); } dispose() { @@ -219,7 +227,8 @@ export class CreativeAgentController { } private upsertSubmission(submission: CreationSubmission) { this.submissions = [...this.submissions.filter((item) => item.id !== submission.id), submission]; } private quote(): CreativeQuote | undefined { - const pending = this.state.pendingPayment?.map((id) => this.submissions.find((item) => item.id === id)).filter((item): item is CreationSubmission => Boolean(item)); + // pendingPayment also retains submitted IDs for recovery; only unsubmitted items need a fee card. + const pending = this.state.pendingPayment?.map((id) => this.submissions.find((item) => item.id === id)).filter((item): item is CreationSubmission => Boolean(item && !item.taskId && !item.revokedAt)); if (!pending?.length) return undefined; const amount = pending.reduce((sum, item) => sum + item.quote.amountMicrocredits, 0); const estimated = pending.some((item) => item.quote.estimated); @@ -235,8 +244,12 @@ export class CreativeAgentController { await this.action(async () => { const ids = this.state.pendingPayment; if (!ids?.length) throw new Error("当前没有待批准费用"); - const result = await this.api.approve(this.run!.id, { ...this.guard(), submissionIds: ids }, this.abort.signal); - result.submissions.forEach((item) => this.upsertSubmission(item)); + const unapproved = ids.filter((id) => { const item = this.submissions.find((item) => item.id === id); return item && !item.taskId && !item.approvedAt && !item.revokedAt; }); + if (this.run!.status === "paused") await this.save("waiting_payment"); + if (unapproved.length) { + const result = await this.api.approve(this.run!.id, { ...this.guard(), submissionIds: unapproved }, this.abort.signal); + result.submissions.forEach((item) => this.upsertSubmission(item)); + } await this.executeApproved(ids); }); } @@ -433,11 +446,12 @@ export class CreativeAgentController { } private async observeMedia(media: CreativeMediaState) { let task: GenerationTask; + let observedStatus: GenerationTask["status"] | undefined; try { - task = await this.waitTask(media.taskId!, { signal: this.abort.signal, onTaskUpdate: (task) => { if (!this.disposed) this.setMedia(media.ref, { status: task.status === "queued" ? "queued" : "running" }); } }); + task = await this.waitTask(media.taskId!, { signal: this.abort.signal, onTaskUpdate: (task) => { observedStatus = task.status; if (!this.disposed) this.setMedia(media.ref, { status: task.status === "queued" ? "queued" : "running" }); } }); } catch (error) { if (this.abort.signal.aborted) throw error; - this.setMedia(media.ref, { status: "failed", error: error instanceof Error ? error.message : "生成失败" }); await this.save("paused"); throw error; + this.setMedia(media.ref, { status: "failed", failureKind: observedStatus === "failed" || observedStatus === "cancelled" ? "generation" : "observation", error: error instanceof Error ? error.message : "生成失败" }); await this.save("paused"); throw error; } this.guard(); try { diff --git a/web/test/agent-capabilities.test.ts b/web/test/agent-capabilities.test.ts index 68ec18fca..8f4729e0f 100644 --- a/web/test/agent-capabilities.test.ts +++ b/web/test/agent-capabilities.test.ts @@ -5,11 +5,18 @@ import { listAgentCapabilities, buildAgentPluginOperations } from "../src/servic import { applyCanvasAgentOps, type CanvasAgentSnapshot } from "../src/lib/canvas/canvas-agent-ops"; import { normalizeCreativeProposal, creativeProposalOps } from "../src/lib/creation/creative-agent-state"; import { defaultConfig } from "../src/stores/use-config-store"; +import { buildCanvasWorkflowOps } from "../src/lib/canvas/canvas-agent-workflow"; const snapshot: CanvasAgentSnapshot = { projectId: "test", title: "test", nodes: [], connections: [], selectedNodeIds: [], viewport: { x: 0, y: 0, k: 1 } }; const pluginId = "agent-extension-test"; afterEach(() => { unregisterPlugin(pluginId); usePluginStore.setState({ runtimeStatuses: {} }); }); +test("方案与直接工作流均拒绝只有正文的分镜", () => { + const nodes = [{ ref: "shots", kind: "script" as const, title: "10秒分镜", content: "0–2秒近景，2–10秒特写" }]; + expect(() => normalizeCreativeProposal({ title: "短剧", summary: "分镜", markdown: "方案", workflow: { nodes, edges: [] }, generationItems: [] }, "p", 1, defaultConfig)).toThrow("script.shots"); + expect(() => buildCanvasWorkflowOps({ nodes, edges: [] }, snapshot, defaultConfig)).toThrow("shots"); +}); + test("专业方案创建可编辑分镜与上下文连线，不伪造画风已选择或收费任务", () => { const proposal = normalizeCreativeProposal({ title: "短剧", summary: "搭建", markdown: "完整方案", workflow: { nodes: [ { ref: "style", kind: "styleboard", title: "项目画风" }, diff --git a/web/test/agent-storyboard.test.ts b/web/test/agent-storyboard.test.ts index 4b3cf9662..273cb6721 100644 --- a/web/test/agent-storyboard.test.ts +++ b/web/test/agent-storyboard.test.ts @@ -1,5 +1,7 @@ import { expect, test } from "bun:test"; -import { buildAgentStoryboardOperations } from "../src/lib/canvas/canvas-agent-storyboard"; +import { buildAgentStoryboardOperations, validateAgentStoryboardCreation } from "../src/lib/canvas/canvas-agent-storyboard"; +import { inspectStoryboardReadiness } from "../src/lib/canvas/canvas-storyboard-context"; +import { buildCanvasAgentContext } from "../src/lib/canvas/canvas-agent-context"; import { createShortDramaPipeline } from "../src/lib/canvas/canvas-short-drama"; import { createStoryboardRow } from "../src/lib/canvas/canvas-project-domain"; import { applyCanvasAgentOps, type CanvasAgentSnapshot } from "../src/lib/canvas/canvas-agent-ops"; @@ -13,6 +15,33 @@ function fixture() { return { snapshot, script }; } +test("Agent 上下文提前指出未选择画风及分镜空行", () => { + const { snapshot, script } = fixture(); + const readiness = buildCanvasAgentContext(snapshot).storyboardReadiness; + expect(readiness.canGenerateStoryboard).toBe(false); + expect(readiness.style.ready).toBe(false); + expect(readiness.blockingReason).toContain("画风"); + expect(readiness.storyboards.find((item) => item.nodeId === script.id)?.incompleteRowIds).toEqual(["row-1", "row-2"]); + expect(readiness.nextActions.join(" ")).toContain("canvas_apply_style"); +}); + +test("已应用画风与真实分镜写入后，返回实际总时长", () => { + const { snapshot, script } = fixture(); + const style = snapshot.nodes.find((node) => node.metadata?.workflowKind === "styleboard")!; + style.metadata = { ...style.metadata, stylePresetId: "preset", content: "电影写实" }; + script.metadata!.storyboard!.rows = [2, 2, 3, 2, 1].map((durationSeconds, i) => createStoryboardRow(i + 1, { durationSeconds, videoMotionPrompt: `镜头${i + 1}` })); + const readiness = inspectStoryboardReadiness(snapshot.nodes); + expect(readiness.canGenerateStoryboard).toBe(true); + expect(readiness.storyboards[0]).toMatchObject({ rowCount: 5, populated: true, totalDurationSeconds: 10 }); + expect(readiness.nextActions).toEqual([]); +}); + +test("通用节点创建不能用正文代替真实分镜行", () => { + expect(() => validateAgentStoryboardCreation([{ type: "add_node", nodeType: "script", metadata: { content: "镜头1：0–2秒近景；镜头2：2–4秒特写" } }])).toThrow("真实镜头行"); + const { script } = fixture(); + expect(() => validateAgentStoryboardCreation([{ type: "add_node", nodeType: "script", metadata: script.metadata }])).toThrow(); +}); + test("按行修改保留其他镜头及已有生成素材关联", () => { const { snapshot, script } = fixture(); const next = applyCanvasAgentOps(snapshot, buildAgentStoryboardOperations(snapshot, { nodeId: script.id, action: "update", rowId: "row-1", patch: { dialogue: "新对白", durationSeconds: 8 } })); diff --git a/web/test/creative-agent-controller.test.ts b/web/test/creative-agent-controller.test.ts index 2e0b1c895..ca6ddad5d 100644 --- a/web/test/creative-agent-controller.test.ts +++ b/web/test/creative-agent-controller.test.ts @@ -30,6 +30,56 @@ const proposal = { id: "p", version: 1, title: "方案", summary: "摘要", mark const submission = (id: string, itemKey: string, taskId?: string): CreationSubmission => ({ id, runId: "run", itemKey, requestHash: "hash", taskId, quote: { model: "model", billingMode: "fixed_request", quantity: 1, amountMicrocredits: 1, estimated: false, expiresAt: "2099-01-01", quoteHash: "quote" } }); describe("创作控制器恢复", () => { + for (const terminal of [true, false]) test(terminal ? "任务已失败时标记生成失败，不能引导反复读取" : "仅查询断线时保留结果恢复语义，不当作需要重做", async () => { + const h = harness({ ...initialCreativeState(), proposal, canvasApplied: true, media: [{ ref: "a", nodeId: "a", attempt: 1, submissionId: "a", taskId: "a", status: "queued" }] }, "waiting_task", [submission("a", "a", "a")], async (_id, options) => { + if (terminal) options?.onTaskUpdate?.({ id: "a", status: "failed" } as GenerationTask); + throw new Error("网络异常。"); + }); + try { await h.controller.load("run"); await expect(h.controller.resume()).rejects.toThrow(); expect(h.view().state.media[0].failureKind).toBe(terminal ? "generation" : "observation"); expect(h.counters().executions).toBe(0); } + finally { h.controller.dispose(); } + }); + test("刷新页面可接续仍有效的旧连接，不等待旧页面租约过期", async () => { + const h = harness(initialCreativeState()); + const get = h.api.get, claim = h.api.claim; + let foreign = true, claims = 0; + h.api.get = async (...args) => { const detail = await get(...args); return foreign ? { ...detail, run: { ...detail.run, executionOwner: "previous-page", leaseExpiresAt: "2099-01-01" } } : detail; }; + h.api.claim = async (...args) => { claims++; foreign = false; return claim(...args); }; + try { await h.controller.load("run"); expect(h.view().hasControl).toBe(true); expect(claims).toBe(1); expect(h.counters().executions).toBe(0); } + finally { h.controller.dispose(); } + }); + test("普通恢复不争抢其他连接，用户重试连接可以显式接续", async () => { + const h = harness(initialCreativeState()); + const get = h.api.get, claim = h.api.claim; + let foreign = false, claims = 0; + h.api.get = async (...args) => { const detail = await get(...args); return foreign ? { ...detail, run: { ...detail.run, executionOwner: "other-page", leaseExpiresAt: "2099-01-01" } } : detail; }; + h.api.claim = async (...args) => { claims++; foreign = false; return claim(...args); }; + try { + await h.controller.load("run"); foreign = true; + await expect(h.controller.refresh()).rejects.toThrow("重试连接"); expect(claims).toBe(1); + await h.controller.takeControl(); expect(claims).toBe(2); expect(h.view().hasControl).toBe(true); expect(h.view().error).toBeUndefined(); + } finally { h.controller.dispose(); } + }); + test("已提交任务即使保留待处理批次也不能再次展示费用确认", async () => { + const h = harness({ ...initialCreativeState(), proposal, canvasApplied: true, pendingPayment: ["a"], media: [{ ref: "a", nodeId: "a", attempt: 1, submissionId: "a", taskId: "task-a", status: "queued" }] }, "waiting_task", [{ ...submission("a", "a", "task-a"), approvedAt: "2026-01-01" }]); + try { await h.controller.load("run"); expect(h.view().quote).toBeUndefined(); } + finally { h.controller.dispose(); } + }); + test("部分提交后费用卡只保留未提交项", async () => { + const h = harness({ ...initialCreativeState(), pendingPayment: ["a", "b"] }, "paused", [submission("a", "a", "task-a"), submission("b", "b")]); + try { await h.controller.load("run"); expect(h.view().quote?.items.map((item) => item.id)).toEqual(["b"]); } + finally { h.controller.dispose(); } + }); + test("迟到的确认点击复用已提交任务，不重新批准或刷新过期费用", async () => { + const old = { ...submission("a", "a", "task-a"), approvedAt: "2000-01-01" }; + old.quote.expiresAt = "2000-01-01"; + const completed = { id: "task-a", status: "succeeded", resultJson: JSON.stringify({ images: [{ storageKey: "resource:output" }] }) } as GenerationTask; + const h = harness({ ...initialCreativeState(), proposal, canvasApplied: true, pendingPayment: ["a"], media: [{ ref: "a", nodeId: "a", attempt: 1, submissionId: "a", taskId: "task-a", status: "queued" }] }, "waiting_task", [old], async () => completed); + h.api.approve = async () => { throw new Error("不应重新批准已提交任务"); }; + h.api.refreshQuote = async () => { throw new Error("不应刷新已提交任务报价"); }; + h.api.execute = async (_id, input) => { expect(input.submissionId).toBe("a"); return completed; }; + try { await h.controller.load("run"); await h.controller.approvePayment(); expect(h.view().state.media[0].status).toBe("ready"); expect(h.view().quote).toBeUndefined(); expect(h.counters().prepares).toBe(0); } + finally { h.controller.dispose(); } + }); test("更新过期报价替换分析关联并等待新批准，不执行模型", async () => { const old = submission("old", "planning:key"); const fresh = submission("fresh", "requote:old");