tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(agent): 创作执行 - 修复会话恢复并校验画风与真实分镜行

Browse Source
This commit is contained in:
王志伟
2026-09-08 22:24:17 +08:00
parent 2856bc7f58
commit 45de256deb
13 changed files with 179 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/design/agent-capabilities.mdx
View File

@@ -17,7 +17,9 @@ description: 在线画布助手的节点发现、技能渐进读取和插件动
创作方案和通用工作流支持 `styleboard`、`story_input`、`script`，复用手动短剧入口的数据结构。画风节点初始为空，显示系统已有的选择入口；故事节点使用专用编辑卡片；分镜节点使用真实表格。
`script.shots` 可填写逐镜 `durationSeconds`、`videoMotionPrompt`、`dialogue`，最多 100 行，写入稳定行 ID。方案连到分镜节点时使用 `storyboard:context` 输入。仅创建这些节点不产生媒体生成项；图像、视频仍沿用既有方案和费用确认。
`script.shots` 必须逐镜填写正数 `durationSeconds` 和非空 `videoMotionPrompt`，`dialogue` 按需填写，最多 100 行，写入稳定行 ID。仅有正文或提示词的分镜方案会返回校验错误供模型修正；通用节点创建工具也校验真实分镜行，不能以空表代替交付。方案连到分镜节点时使用 `storyboard:context` 输入。仅创建这些节点不产生媒体生成项；图像、视频仍沿用既有方案和费用确认。
`canvas_get_context.storyboardReadiness` 与 `canvas_read_storyboard.readiness` 复用原分镜生成前置校验，返回画风状态、阻塞原因、真实行数量、未填完整的行 ID 和实际总时长。助手先检查这些状态；缺画风时读取真实目录，结合已确认风格与用户授权应用，只有关键取舍尚不明确时才追问。已有空表通过逐行编辑补齐，保留原行 ID 和素材关联；有足够剧本信息时直接编写镜头，不要求用户重复手动生成分镜。提示词引导不替代生成入口的业务校验，写入镜头也不代表媒体已生成。
`canvas_list_styles` 按 `source=system|user` 查询系统预设或当前用户保存的风格。`canvas_apply_style` 只接受该目录返回的真实 ID，复用页面的异步画风保存入口：关联项目时先保存项目设置，再应用画布节点；不关联项目时走原画布保存机制。异步返回时校验画布、项目及用户上下文，避免写到新页面。该动作是原画风选择逻辑，不冒充普通节点撤销能够回滚服务端项目设置；需要改回时重新应用原风格。

10
web/src/components/canvas/canvas-assistant-panel.tsx
View File

@@ -15,7 +15,7 @@ import { runBackendToolGenerationTask } from "@/services/api/generation-task";
import { inspectAgentImage } from "@/services/agent-image-preview";
import { AGENT_CAPABILITY_GUIDANCE, buildAgentPluginOperations, listAgentCapabilities, readAgentPluginDocumentation, readAgentPluginNode } from "@/services/agent-capabilities";
import { getNodeDefinition } from "@/lib/canvas/node-registry";
import { buildAgentStoryboardOperations, readAgentStoryboard } from "@/lib/canvas/canvas-agent-storyboard";
import { buildAgentStoryboardOperations, readAgentStoryboard, validateAgentStoryboardCreation } from "@/lib/canvas/canvas-agent-storyboard";
import { canvasStylePresets, userStylePreset, type CanvasStylePreset } from "./canvas-style-picker-modal";
import { listStyleProfiles } from "@/services/api/style-profiles";
import { getActiveUserScope } from "@/lib/user-scope";
@@ -879,8 +879,8 @@ export function CanvasAssistantPanel({
}
if (name === "canvas_read_plugin_node") return { ok: true, message: "已读取插件节点状态。", data: readAgentPluginNode(current, requireString(args.nodeId, "nodeId")) };
if (name === "canvas_read_storyboard") {
const { node, rows } = readAgentStoryboard(current, requireString(args.nodeId, "nodeId"));
return { ok: true, message: "已读取分镜表。", data: { nodeId: node.id, title: node.title, rows } };
const { node, rows, readiness } = readAgentStoryboard(current, requireString(args.nodeId, "nodeId"));
return { ok: true, message: "已读取分镜表及画风前置条件。", data: { nodeId: node.id, title: node.title, rows, readiness } };
}
if (name === "canvas_list_styles" || name === "canvas_apply_style") {
const originScope = getActiveUserScope();
@@ -1564,13 +1564,13 @@ function parseToolArguments(value: string) {
export function onlineToolToOps(name: string, input: Record<string, unknown>, snapshot: CanvasAgentSnapshot, config: AiConfig): CanvasAgentOp[] {
if (name === "canvas_edit_storyboard") return buildAgentStoryboardOperations(snapshot, input);
if (name === "canvas_plugin_action") return buildAgentPluginOperations(requireString(input.pluginId, "pluginId"), requireString(input.actionId, "actionId"), recordOptional(input.input) || {}, snapshot);
if (name === "canvas_apply_ops") return requireOps(input.ops);
if (name === "canvas_apply_ops") return validateAgentStoryboardCreation(requireOps(input.ops));
if (name === "canvas_create_workflow") return buildCanvasWorkflowOps(input as unknown as CanvasWorkflowInput, snapshot, config);
if (name === "canvas_create_node") {
const nodeType = requireNodeType(input.nodeType);
const x = numberOr(input.x, nextCanvasX(snapshot));
const y = numberOr(input.y, 0);
return [{ type: "add_node", nodeType, title: stringOptional(input.title), position: { x, y }, width: numberOptional(input.width), height: numberOptional(input.height), metadata: recordOptional(input.metadata) as CanvasNodeData["metadata"] }];
return validateAgentStoryboardCreation([{ type: "add_node", nodeType, title: stringOptional(input.title), position: { x, y }, width: numberOptional(input.width), height: numberOptional(input.height), metadata: recordOptional(input.metadata) as CanvasNodeData["metadata"] }]);
}
if (name === "canvas_create_text_node") return [textNodeOp(input, numberOr(input.x, nextCanvasX(snapshot)), numberOr(input.y, 0))];
if (name === "canvas_create_text_nodes") {

19
web/src/components/canvas/canvas-creative-interaction.tsx
View File

@@ -57,12 +57,14 @@ export function CanvasCreativeInteraction(props: Props) {
const quoteHistory = useRef<CreativeQuote[]>(detail.quotes || []);
const answersEdited = useRef(false);
const presentationAttempted = useRef(false);
const hadControl = useRef(false);
const scope = getActiveUserScope();
const invoke = (promise?: Promise<unknown>) => { if (promise) { setError(""); void promise.catch((cause) => { if (!(cause instanceof DOMException && cause.name === "AbortError")) setError(cause instanceof Error ? cause.message : "操作未完成"); }); } };
useEffect(() => {
let live = true;
presentationAttempted.current = false;
hadControl.current = false;
const original = canvasCreativeDetail(latest.current.message)!;
setView({ state: original.state, busy: false, hasControl: false });
const instance = new CreativeAgentController({
@@ -72,6 +74,9 @@ export function CanvasCreativeInteraction(props: Props) {
onOpenCanvas: () => setError("当前会话仅操作所在画布，请在对应画布打开历史会话。"),
onChange: (next) => {
if (!live) return;
// Recovery clears the controller error and the rejected action's local UI error together.
if (next.hasControl && !hadControl.current && !next.error) setError("");
hadControl.current = next.hasControl;
setView(next); latest.current.onBusy(props.message.id, next.busy);
if (!next.run || !latest.current.active) return;
if (next.quote) quoteHistory.current = [...quoteHistory.current.filter((quote) => quote.id !== next.quote!.id), next.quote];
@@ -126,7 +131,9 @@ export function CanvasCreativeInteraction(props: Props) {
props.onContinue("保留已确认的要求，在已授权范围内继续当前任务，自行处理常规选择和技术问题；确实阻塞时再询问。");
};
const rawError = proposalFailure || error || view.error;
const feedback = rawError ? /恢复|连接|网络|接管|执行权|代次/.test(rawError) ? "连接暂时中断，正在恢复。已有作品会保留。"
const failedGenerations = view.state.media.filter((item) => item.status === "failed" && item.failureKind !== "observation");
const onlyFailedRemaining = failedGenerations.length > 0 && !view.state.media.some((item) => ["queued", "running", "write_failed"].includes(item.status) || item.failureKind === "observation" && item.status === "failed");
const feedback = onlyFailedRemaining ? "部分作品生成失败。请在失败作品下方重新生成，先核对费用；已成功的作品会保留。" : rawError ? /恢复|连接|网络|接管|执行权|代次/.test(rawError) ? "状态读取未完成，可以继续处理重试。已有作品会保留。"
: /报价|费用|价格|过期/.test(rawError) ? "生成费用需要重新确认，继续后会显示最新价格。"
: /素材|参考|引用/.test(rawError) ? "参考素材还需要核对，助手会继续处理，必要时请你选择。"
: "这一步还没有完成。已有内容已保留，可以继续处理或修改要求。" : undefined;
@@ -147,23 +154,27 @@ export function CanvasCreativeInteraction(props: Props) {
{view.state.pendingEdits && <div className="creative-agent-card"><p>将调整 {view.state.pendingEdits.length} 个节点的指定字段。</p><Button disabled={blocked} onClick={() => invoke(controller.current?.approveEdits())}>确认节点调整</Button></div>}
{props.active && view.quote && <CreativeQuoteCard quote={view.quote} busy={view.busy} disabled={blocked} onApprove={() => invoke(controller.current?.approvePayment())} onRefresh={() => invoke(controller.current?.refreshQuotes())} />}
{(detail.quotes || []).filter((quote) => !props.active || quote.id !== view.quote?.id).map((quote) => <details key={quote.id} className="creative-agent-receipt"><summary>{quote.approvedQuantity ? "已确认生成" : "历史报价"} · {quote.amountLabel} · {quote.items.length} 项</summary><p>{quote.basis}</p><ul>{quote.items.map((item) => <li key={item.id}>{item.label} · {item.model} · {item.specification}</li>)}</ul></details>)}
{view.state.media.filter((item) => item.storageKey || item.error).map((item) => {
{view.state.media.filter((item) => item.taskId || item.storageKey || item.error).map((item) => {
const resource = resourceIdFromStorageKey(item.storageKey), url = resource ? resourceFileUrl(resource) : "";
const video = view.state.proposal?.generationItems.find((entry) => entry.ref === item.ref)?.mode === "video";
return <div className="creative-agent-card" key={item.ref}>
<strong>{view.state.proposal?.workflow.nodes.find((node) => node.ref === item.ref)?.title || item.ref}</strong>
{!item.error && !url && item.taskId && <p className="creative-agent-muted">{view.busy ? item.status === "running" ? "正在生成，完成后自动显示。" : "已提交，正在等待生成结果。" : "任务已提交。点击下方“继续处理”读取最新结果，无需重复确认费用。"}</p>}
{url && (video ? <video className="w-full max-h-80 object-contain" src={url} controls preload="metadata" /> : <img className="w-full max-h-80 object-contain" src={url} alt={item.ref} />)}
{item.error && <p className="creative-agent-muted">这项作品还需要处理，已有结果会保留。</p>}
{item.error && <details className="creative-agent-receipt"><summary>查看失败原因</summary><p>{item.error}</p></details>}
{props.active && item.status === "failed" && item.failureKind !== "observation" && <Button type="primary" disabled={blocked} onClick={() => invoke(controller.current?.redo(item.ref))}>重新生成此项 · 先确认费用</Button>}
<div className="creative-agent-actions">{url && <a href={url} target="_blank" rel="noreferrer">查看作品</a>}{props.active && <Dropdown trigger={["click"]} disabled={blocked} menu={{ items: [{ key: "adjust", label: "调整作品" }, { key: "redo", label: "重新生成（费用另行确认）" }], onClick: ({ key }) => key === "redo" ? invoke(controller.current?.redo(item.ref)) : modify(`请调整《${view.state.proposal?.workflow.nodes.find((entry) => entry.ref === item.ref)?.title || item.ref}》，保留其他作品：`) }}><Button type="text" disabled={blocked}>更多</Button></Dropdown>}</div>
</div>;
})}
{!props.active && rawError && <details className="creative-agent-receipt"><summary>这一步未完成，后续对话中可继续调整</summary><p>{feedback}</p></details>}
{props.active && <div className="creative-agent-next" role="status">
<p>{view.busy ? "正在处理，请稍候…" : !view.hasControl ? "正在恢复当前创作，请稍候…" : view.state.modificationRequested ? "在下方告诉我想改哪里，调整后再确认。" : feedback || (view.quote ? "确认费用后，就开始本批制作。" : awaitingProposal ? "看看方案是否符合你的想法，确认后继续。" : question?.status === "pending" ? "选择一个答案，也可以在下方直接补充。" : status === "completed" ? "本阶段已完成。可以查看作品，或告诉我想改哪里。" : "准备好了，可以继续。")}</p>
<p>{view.busy ? "正在处理，请稍候…" : !view.hasControl ? rawError ? "暂时无法恢复当前创作，可以重试连接。已有作品会保留。" : "正在恢复当前创作，请稍候…" : view.state.modificationRequested ? "在下方告诉我想改哪里，调整后再确认。" : feedback || (view.quote ? "确认费用后，就开始本批制作。" : awaitingProposal ? "看看方案是否符合你的想法，确认后继续。" : question?.status === "pending" ? "选择一个答案，也可以在下方直接补充。" : status === "completed" ? "本阶段已完成。可以查看作品，或告诉我想改哪里。" : "准备好了，可以继续。")}</p>
<div className="creative-agent-actions">
{!view.busy && !view.hasControl && <Button onClick={() => invoke(view.run ? controller.current?.takeControl() : controller.current?.load(detail.runId, detail.state))}>重试连接</Button>}
{view.busy && ["running", "waiting_task", "waiting_canvas"].includes(status || "") ? <Button onClick={() => invoke(controller.current?.pause())}>停止后续制作</Button>
: !view.busy && view.hasControl && !view.state.modificationRequested && !view.quote && !awaitingProposal && !view.state.pendingEdits && question?.status !== "pending" ? <>
<Button type="primary" onClick={() => status === "completed" && !feedback ? props.onReview() : invoke(continueWork())}>{status === "completed" && !feedback ? "继续创作" : "继续处理"}</Button>
{!onlyFailedRemaining && <Button type="primary" onClick={() => status === "completed" && !feedback ? props.onReview() : invoke(continueWork())}>{status === "completed" && !feedback ? "继续创作" : "继续处理"}</Button>}
<Button onClick={() => modify("我想调整当前创作要求：")}>修改</Button>
</> : null}
</div>

2
web/src/lib/canvas/canvas-agent-context.ts
View File

@@ -2,6 +2,7 @@ import { hashCanvasAgentSnapshot, type CanvasAgentOp, type CanvasAgentSnapshot }
import { CanvasNodeType, type CanvasNodeData } from "@/types/canvas";
import { getNodeDefinition } from "./node-registry";
import { isPluginEffectivelyEnabled } from "@/stores/use-plugin-store";
import { inspectStoryboardReadiness } from "./canvas-storyboard-context";
export type CanvasAgentResource = {
nodeId: string;
@@ -47,6 +48,7 @@ export function buildCanvasAgentContext(snapshot: CanvasAgentSnapshot, options:
nodes: nodes.map(compactNode),
connections: snapshot.connections.map((connection) => ({ id: connection.id, fromNodeId: connection.fromNodeId, fromTitle: nodeById.get(connection.fromNodeId)?.title || "未知节点", toNodeId: connection.toNodeId, toTitle: nodeById.get(connection.toNodeId)?.title || "未知节点", fromHandleId: connection.fromHandleId, toHandleId: connection.toHandleId })),
resources,
storyboardReadiness: inspectStoryboardReadiness(snapshot.nodes),
warnings: [
...(nodes.some((node) => node.metadata?.status === "error") ? ["画布中存在生成失败节点；重试前先检查错误信息。"] : []),
...(resources.some((resource) => !resource.ready) ? ["存在未就绪或缺少持久化引用的媒体节点，不要把占位节点当作可用参考素材。"] : []),

12
web/src/lib/canvas/canvas-agent-storyboard.ts
View File

@@ -1,13 +1,23 @@
import type { CanvasAgentOp, CanvasAgentSnapshot } from "./canvas-agent-ops";
import { createStoryboardRow } from "./canvas-project-domain";
import type { StoryboardRow } from "@/types/canvas";
import { inspectStoryboardReadiness } from "./canvas-storyboard-context";
const textFields = ["plotDescription", "dialogue", "videoMotionPrompt", "imageGenerationPrompt", "camera", "motion", "shotSize", "emotion", "lightingAndAtmosphere", "audioEffects", "narrativeIntent", "viewerPOV", "performanceBlocking", "timeBeats", "continuityOut", "negativePrompt"] as const;
export function validateAgentStoryboardCreation(ops: CanvasAgentOp[]) {
for (const op of ops) {
if (op.type !== "add_node" || op.nodeType !== "script") continue;
const rows = op.metadata?.storyboard?.rows;
if (!Array.isArray(rows) || !rows.length || rows.length > 100 || rows.some((row) => !Number.isFinite(row.durationSeconds) || row.durationSeconds <= 0 || !row.videoMotionPrompt?.trim())) throw new Error("分镜节点必须包含真实镜头行。请使用 canvas_create_workflow 的 script.shots 填写每镜时长和视频提示词，不能仅填正文；已有分镜使用 canvas_edit_storyboard");
}
return ops;
}
export function readAgentStoryboard(snapshot: CanvasAgentSnapshot, nodeId: string) {
const node = snapshot.nodes.find((item) => item.id === nodeId && item.type === "script");
if (!node) throw new Error("未找到当前画布的分镜脚本节点，请先查找真实节点");
return { node, rows: node.metadata?.storyboard?.rows || [] };
return { node, rows: node.metadata?.storyboard?.rows || [], readiness: inspectStoryboardReadiness(snapshot.nodes) };
}
export function buildAgentStoryboardOperations(snapshot: CanvasAgentSnapshot, input: Record<string, unknown>): CanvasAgentOp[] {

1
web/src/lib/canvas/canvas-agent-workflow.ts
View File

@@ -112,6 +112,7 @@ export function buildCanvasWorkflowOps(input: CanvasWorkflowInput, snapshot: Can
const source = template.nodes[node.kind === "styleboard" ? 0 : node.kind === "story_input" ? 1 : 2];
Object.assign(metadata, source.metadata, { content: node.kind === "styleboard" ? "" : node.content || "", composerContent: prompt });
if (node.kind === "script") {
if (!node.shots?.length) throw new Error("创建分镜工作流必须提供非空 shots，请把镜头逐行填写 durationSeconds、videoMotionPrompt、dialogue，不能仅写正文");
if (node.shots && (!Array.isArray(node.shots) || node.shots.length > 100)) throw new Error("分镜最多 100 行");
metadata.storyboard = { ...source.metadata!.storyboard!, rows: (node.shots || []).map((shot, rowIndex) => {
if (!Number.isFinite(shot.durationSeconds) || shot.durationSeconds <= 0 || typeof shot.videoMotionPrompt !== "string" || !shot.videoMotionPrompt.trim()) throw new Error("每条分镜需要有效时长和视频提示词");

24
web/src/lib/canvas/canvas-storyboard-context.ts
View File

@@ -44,3 +44,27 @@ export function resolveStoryboardGenerationContext(nodes: CanvasNodeData[]): Sto
})),
};
}
// Expose the same generation prerequisites to the Agent before it starts a workflow.
export function inspectStoryboardReadiness(nodes: CanvasNodeData[]) {
let generationContext: StoryboardGenerationContext | undefined;
let blockingReason: string | undefined;
try { generationContext = resolveStoryboardGenerationContext(nodes); }
catch (error) { blockingReason = error instanceof Error ? error.message : "分镜上下文未就绪"; }
const styleNode = nodes.find((node) => node.metadata?.workflowKind === "styleboard");
const styleReady = Boolean(styleNode?.metadata?.stylePresetId?.trim() && String(styleNode.metadata.content || styleNode.metadata.prompt || "").trim());
const storyboards = nodes.filter((node) => node.type === "script").map((node) => {
const rows = node.metadata?.storyboard?.rows || [];
const incompleteRowIds = rows.filter((row) => !Number.isFinite(row.durationSeconds) || row.durationSeconds <= 0 || !row.videoMotionPrompt?.trim()).map((row) => row.id);
return { nodeId: node.id, title: node.title, rowCount: rows.length, populated: rows.length > 0 && incompleteRowIds.length === 0, incompleteRowIds, totalDurationSeconds: rows.reduce((total, row) => total + (Number.isFinite(row.durationSeconds) && row.durationSeconds > 0 ? row.durationSeconds : 0), 0) };
});
return {
canGenerateStoryboard: Boolean(generationContext), blockingReason,
style: { ready: styleReady, nodeId: styleNode?.id, presetId: styleNode?.metadata?.stylePresetId, title: styleNode?.title },
storyboards,
nextActions: [
...(!styleReady ? ["先 canvas_list_styles 读取真实画风，结合已确认风格调用 canvas_apply_style；该工具会复用或创建画风节点。仅在缺少关键风格取舍时询问用户。"] : []),
...(storyboards.some((item) => !item.populated) ? ["先 canvas_read_storyboard 读取真实行 ID，再 canvas_edit_storyboard 更新空行并按需新增镜头；正文或标题不代表已填充分镜表。"] : []),
],
};
}

3
web/src/lib/creation/creative-agent-state.ts
View File

@@ -10,7 +10,7 @@ import { CREATIVE_SCENARIOS, type CreativeAnswers, type CreativeBrief, type Crea
export type CreativeReference = { id: string; title: string; kind: "image" | "text"; assetId?: string; storageKey?: string; text?: string; mimeType?: string; width?: number; height?: number };
export type CreativeMessage = { id: string; role: "user" | "assistant"; text: string; question?: CreativeQuestionRequest; answers?: CreativeAnswers; proposal?: CreativeProposal };
export type CreativeMediaState = { ref: string; nodeId: string; attempt: number; submissionId?: string; taskId?: string; status: "pending" | "queued" | "running" | "ready" | "failed" | "write_failed"; storageKey?: string; error?: string };
export type CreativeMediaState = { ref: string; nodeId: string; attempt: number; submissionId?: string; taskId?: string; status: "pending" | "queued" | "running" | "ready" | "failed" | "write_failed"; storageKey?: string; error?: string; failureKind?: "generation" | "observation" };
export type CreativeAgentState = {
schemaVersion: 1; scene: CreativeScenarioId; brief: CreativeBrief; messages: CreativeMessage[];
references: CreativeReference[]; selectedSkillIds: string[]; textModel?: string;
@@ -70,6 +70,7 @@ export function normalizeCreativeProposal(raw: unknown, id: string, version: num
if (!assetId && (["text", "story_input"].includes(kind) ? !content : ["image", "video"].includes(kind) && !prompt)) throw new Error("文本节点需要正文，媒体节点需要生成提示词");
const shots = Array.isArray(node.shots) ? node.shots.map((value) => { const shot = record(value); const durationSeconds = Number(shot.durationSeconds); if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) throw new Error("分镜时长必须大于零"); return { durationSeconds, videoMotionPrompt: required(shot.videoMotionPrompt, "分镜视频提示词"), dialogue: str(shot.dialogue) }; }) : undefined;
if (shots && shots.length > 100) throw new Error("分镜最多 100 行");
if (kind === "script" && !shots?.length) throw new Error("分镜方案必须提供非空 script.shots，每镜填写 durationSeconds 和 videoMotionPrompt；请把已设计镜头转成真实行，不能只填 content 或 prompt");
return { ref: required(node.ref, "节点引用"), kind: kind as "text" | "image" | "video" | "styleboard" | "story_input" | "script", title: required(node.title, "节点标题"), content, prompt, shots, runGeneration: false, referenceRefs: Array.isArray(node.referenceRefs) ? node.referenceRefs.map(String) : [], referenceNodeIds: Array.isArray(node.referenceNodeIds) ? node.referenceNodeIds.map(String) : [] };
});
const refs = new Set(nodes.map((node) => node.ref));

3
web/src/lib/creation/creative-agent-tools.ts
View File

@@ -31,7 +31,8 @@ export const CREATIVE_AGENT_SYSTEM_PROMPT = `你是影策的通用创作助手
多步骤创作使用 creative_respond。先结合全部需求摘要、会话、用户素材、专业角色指导、选中技能与真实画布，提取已知信息；只有缺失或冲突且确实影响执行的信息才补问，通常1至3题，确需一起决定时最多6题，不凑题。允许自定义安全的需求字段，优先复用已有字段，不通过改名重复询问。用户直接输入的自然语言回答与卡片回答同等有效，可以同时更新多个字段。不得将猜测写成用户确认。用户允许自主决定时可以直接给方案。
多步骤任务可携带 plan，按本次实际工作命名步骤，例如“确定结尾情绪”“设计商品构图”“制作首张主图”；不套固定步骤数。每步 ref 在同一工作持续时保持稳定，新工作使用新ref。phase只用于程序关联真实状态：questions=需求选择，proposal=内容方案，canvas=节点落地，media=产物生成（mediaRefs关联方案生成项ref）。可以调整未完成步骤并在reason说明原因；不编造完成状态。普通咨询不创建计划。已完成工作不可抹掉，新增任务不能复用已完成步骤ref。
问题与方案不能在同一次响应同时推进。已经确认的题材/时长/数量/比例/风格不再问。覆盖已确认字段必须在 evidence 引用最新用户消息中明确修改该字段的原文；不能将建议或旧消息作为修改授权。素材题的 option.id 只能使用上下文提供的真实素材ID。
方案应是可阅读的完整 Markdown，包含内容、已有素材、待生成产物、规格和真实依赖。workflow 支持 text/image/video/styleboard/story_input/script；搭建短剧工作流时使用真实的 styleboard 项目画风选择节点、story_input 故事梗概和 script 分镜表，已有节点优先沿用。script.shots 可填写逐镜 durationSeconds、videoMotionPrompt、dialogue，不用普通文本模拟表格。styleboard 是待选择入口，不把任意文字当作已应用画风；用户已有画风时不重复创建。只有 image/video 填写 generationItems；创建分镜表和画风入口不代表生成媒体。媒体必须有完整提示词，文本必须有正文。每个待生成媒体节点有且仅有一项 generationItems，使用可用模型的完整 model ID。用户已有图片优先用 image 节点的 assetId 指向真实 references.id，此类节点不写 generationItems、不重复生成，可被其他节点用 referenceRefs 引用。
方案应是可阅读的完整 Markdown，包含内容、已有素材、待生成产物、规格和真实依赖。workflow 支持 text/image/video/styleboard/story_input/script；搭建短剧工作流时使用真实的 styleboard 项目画风选择节点、story_input 故事梗概和 script 分镜表，已有节点优先沿用。script.shots 必须逐镜填写 durationSeconds、videoMotionPrompt、dialogue，不用普通文本模拟表格。styleboard 是待选择入口，不把任意文字当作已应用画风；用户已有画风时不重复创建。只有 image/video 填写 generationItems；创建分镜表和画风入口不代表生成媒体。媒体必须有完整提示词，文本必须有正文。每个待生成媒体节点有且仅有一项 generationItems，使用可用模型的完整 model ID。用户已有图片优先用 image 节点的 assetId 指向真实 references.id，此类节点不写 generationItems、不重复生成，可被其他节点用 referenceRefs 引用。
分镜任务先检查 canvas_get_context.storyboardReadiness 或 canvas_read_storyboard.readiness。缺少画风时先 canvas_list_styles，再按用户已确认的风格或自主决策授权调用 canvas_apply_style；只有确实需要用户取舍时才提问，不重复询问已有风格。应用后重新读取上下文确认 canGenerateStoryboard，不能只在正文声称画风已设置。方案批准后创建 script 时必须提供非空 shots，不能只写 content/prompt 后把默认空行当成完成。已有分镜用 canvas_read_storyboard 取得真实行 ID，再 canvas_edit_storyboard 更新空行、按需追加镜头，保留已有行 ID 与素材；已有剧本足够时由你直接编写镜头，不要求用户再次点击生成分镜。0–2秒对应 durationSeconds=2，2–4秒为2秒，4–7秒为3秒，7–9秒为2秒，9–10秒为1秒；写入后核对总时长与约定一致。不用改标题代替修改时长，不声称有分镜行就已经生成图片视频。
generationItems 是方案的声明式生成配置，用于校验和之后的报价，不会立即调用媒体模型。因此“确认后再生成”仍必须填写它，不能省略。每项 ref 必须对应节点 ref，mode 必须等于节点 kind，model 必须来自 availableModels 中相同 mode 的条目；图片模型不能用于视频。上下文若有 rejectedProposal，先根据其中真实校验错误修正，保留已知需求并返回完整方案。结构化方案和费用的批准以程序记录为准；若仍缺按钮确认，直接显示对应卡片，不再先问一遍是否同意、再让用户点卡片。
引用已有画布图片时，将真实 references.id 放到 workflow.nodes[].referenceNodeIds；referenceRefs 优先用于本次方案内其他图片节点的 ref。不要为引用现有图片再生成一张副本，不要在两个字段重复放同一张图。程序会核实真实资源并整理引用，未知ID不能通过。
单段视频可以在正文描述多个叙事节奏，不把它们拆成未合成的几段视频。用户要求15秒时只选实际支持15秒的模型；不存在则解释缺少能力并询问可接受替代，绝不能偷偷缩短。图像引用数量、视频输入角色、尺寸均须符合上下文能力。边表达真实依赖，不为了好看造线。

40
web/src/services/creative-agent-controller.ts
View File

@@ -52,11 +52,11 @@ export class CreativeAgentController {
this.run = await this.api.save(this.run!.id, { ...guard, revision: this.run!.revision, status, state: this.state as unknown as Record<string, unknown> }, this.abort.signal);
this.assertLive(); this.emit();
}
private async action(operation: () => Promise<void>) {
private async action(operation: () => Promise<void>, takeOver = false) {
if (this.busy) throw new Error("上一项操作仍在处理，请稍候");
if (this.disposed || this.scope !== getActiveUserScope()) throw new DOMException("页面已关闭", "AbortError");
this.busy = true; this.error = undefined; this.emit();
try { if (this.run) await this.ensureControl(); await operation(); }
try { if (this.run) await this.ensureControl(takeOver); await operation(); }
catch (error) {
if (!this.disposed && !(error instanceof DOMException && error.name === "AbortError")) { this.error = error instanceof Error ? error.message : "操作未完成"; this.emit(); }
throw error;
@@ -70,7 +70,7 @@ export class CreativeAgentController {
this.assertLive();
if (!readOnly) {
this.startHeartbeat();
await this.ensureControl();
await this.ensureControl(true);
}
this.emit();
});
@@ -85,7 +85,11 @@ export class CreativeAgentController {
this.heartbeat = setInterval(() => {
if (this.disposed || !this.run) return;
if (!this.hasControl) {
if (!this.busy) void this.ensureControl().then(() => { this.error = undefined; this.emit(); }).catch(() => undefined);
if (!this.busy) void this.ensureControl().then(() => { this.error = undefined; this.emit(); }).catch((error) => {
if (this.disposed) return;
this.error = error instanceof Error ? error.message : "连接恢复失败，请重试";
this.emit();
});
return;
}
if (this.abort.signal.aborted) return;
@@ -95,16 +99,20 @@ export class CreativeAgentController {
}, 15_000);
}
private restoring?: Promise<void>;
private async ensureControl() {
private async ensureControl(takeOver = false): Promise<void> {
if (this.hasControl && !this.abort.signal.aborted && Date.parse(this.run?.leaseExpiresAt || "") > Date.now() + 3000) return;
if (this.restoring) return this.restoring;
if (this.restoring) {
if (!takeOver) return this.restoring;
try { await this.restoring; } catch { /* Explicit reconnect retries after the passive recovery attempt. */ }
return this.ensureControl(true);
}
this.restoring = (async () => {
if (this.disposed || this.scope !== getActiveUserScope()) throw new DOMException("页面已关闭", "AbortError");
if (this.abort.signal.aborted) this.abort = new AbortController();
const detail = await this.api.get(this.run!.id, this.abort.signal);
this.assertLive();
const run = detail.run;
if (run.executionOwner && run.executionOwner !== this.owner && Date.parse(run.leaseExpiresAt || "") > Date.now()) throw new Error("正在恢复当前创作，请稍候再继续。");
if (!takeOver && run.executionOwner && run.executionOwner !== this.owner && Date.parse(run.leaseExpiresAt || "") > Date.now()) throw new Error("当前创作连接已切换，请点击重试连接继续。");
// Only replace the lease here; never overwrite local edits or unknown submission results.
this.run = { ...this.run!, executionOwner: run.executionOwner, executionEpoch: run.executionEpoch, leaseExpiresAt: run.leaseExpiresAt };
await this.claim();
@@ -113,7 +121,7 @@ export class CreativeAgentController {
}
async takeControl() {
if (this.disposed) return;
await this.action(async () => { this.accept(await this.api.get(this.run!.id, this.abort.signal)); });
await this.action(async () => { this.accept(await this.api.get(this.run!.id, this.abort.signal)); }, true);
}
async refresh() { await this.action(async () => { this.accept(await this.api.get(this.run!.id, this.abort.signal)); }); }
dispose() {
@@ -219,7 +227,8 @@ export class CreativeAgentController {
}
private upsertSubmission(submission: CreationSubmission) { this.submissions = [...this.submissions.filter((item) => item.id !== submission.id), submission]; }
private quote(): CreativeQuote | undefined {
const pending = this.state.pendingPayment?.map((id) => this.submissions.find((item) => item.id === id)).filter((item): item is CreationSubmission => Boolean(item));
// pendingPayment also retains submitted IDs for recovery; only unsubmitted items need a fee card.
const pending = this.state.pendingPayment?.map((id) => this.submissions.find((item) => item.id === id)).filter((item): item is CreationSubmission => Boolean(item && !item.taskId && !item.revokedAt));
if (!pending?.length) return undefined;
const amount = pending.reduce((sum, item) => sum + item.quote.amountMicrocredits, 0);
const estimated = pending.some((item) => item.quote.estimated);
@@ -235,8 +244,12 @@ export class CreativeAgentController {
await this.action(async () => {
const ids = this.state.pendingPayment;
if (!ids?.length) throw new Error("当前没有待批准费用");
const result = await this.api.approve(this.run!.id, { ...this.guard(), submissionIds: ids }, this.abort.signal);
result.submissions.forEach((item) => this.upsertSubmission(item));
const unapproved = ids.filter((id) => { const item = this.submissions.find((item) => item.id === id); return item && !item.taskId && !item.approvedAt && !item.revokedAt; });
if (this.run!.status === "paused") await this.save("waiting_payment");
if (unapproved.length) {
const result = await this.api.approve(this.run!.id, { ...this.guard(), submissionIds: unapproved }, this.abort.signal);
result.submissions.forEach((item) => this.upsertSubmission(item));
}
await this.executeApproved(ids);
});
}
@@ -433,11 +446,12 @@ export class CreativeAgentController {
}
private async observeMedia(media: CreativeMediaState) {
let task: GenerationTask;
let observedStatus: GenerationTask["status"] | undefined;
try {
task = await this.waitTask(media.taskId!, { signal: this.abort.signal, onTaskUpdate: (task) => { if (!this.disposed) this.setMedia(media.ref, { status: task.status === "queued" ? "queued" : "running" }); } });
task = await this.waitTask(media.taskId!, { signal: this.abort.signal, onTaskUpdate: (task) => { observedStatus = task.status; if (!this.disposed) this.setMedia(media.ref, { status: task.status === "queued" ? "queued" : "running" }); } });
} catch (error) {
if (this.abort.signal.aborted) throw error;
this.setMedia(media.ref, { status: "failed", error: error instanceof Error ? error.message : "生成失败" }); await this.save("paused"); throw error;
this.setMedia(media.ref, { status: "failed", failureKind: observedStatus === "failed" || observedStatus === "cancelled" ? "generation" : "observation", error: error instanceof Error ? error.message : "生成失败" }); await this.save("paused"); throw error;
}
this.guard();
try {

7
web/test/agent-capabilities.test.ts
View File

@@ -5,11 +5,18 @@ import { listAgentCapabilities, buildAgentPluginOperations } from "../src/servic
import { applyCanvasAgentOps, type CanvasAgentSnapshot } from "../src/lib/canvas/canvas-agent-ops";
import { normalizeCreativeProposal, creativeProposalOps } from "../src/lib/creation/creative-agent-state";
import { defaultConfig } from "../src/stores/use-config-store";
import { buildCanvasWorkflowOps } from "../src/lib/canvas/canvas-agent-workflow";
const snapshot: CanvasAgentSnapshot = { projectId: "test", title: "test", nodes: [], connections: [], selectedNodeIds: [], viewport: { x: 0, y: 0, k: 1 } };
const pluginId = "agent-extension-test";
afterEach(() => { unregisterPlugin(pluginId); usePluginStore.setState({ runtimeStatuses: {} }); });
test("方案与直接工作流均拒绝只有正文的分镜", () => {
const nodes = [{ ref: "shots", kind: "script" as const, title: "10秒分镜", content: "0–2秒近景，2–10秒特写" }];
expect(() => normalizeCreativeProposal({ title: "短剧", summary: "分镜", markdown: "方案", workflow: { nodes, edges: [] }, generationItems: [] }, "p", 1, defaultConfig)).toThrow("script.shots");
expect(() => buildCanvasWorkflowOps({ nodes, edges: [] }, snapshot, defaultConfig)).toThrow("shots");
});
test("专业方案创建可编辑分镜与上下文连线，不伪造画风已选择或收费任务", () => {
const proposal = normalizeCreativeProposal({ title: "短剧", summary: "搭建", markdown: "完整方案", workflow: { nodes: [
{ ref: "style", kind: "styleboard", title: "项目画风" },

31
web/test/agent-storyboard.test.ts
View File

@@ -1,5 +1,7 @@
import { expect, test } from "bun:test";
import { buildAgentStoryboardOperations } from "../src/lib/canvas/canvas-agent-storyboard";
import { buildAgentStoryboardOperations, validateAgentStoryboardCreation } from "../src/lib/canvas/canvas-agent-storyboard";
import { inspectStoryboardReadiness } from "../src/lib/canvas/canvas-storyboard-context";
import { buildCanvasAgentContext } from "../src/lib/canvas/canvas-agent-context";
import { createShortDramaPipeline } from "../src/lib/canvas/canvas-short-drama";
import { createStoryboardRow } from "../src/lib/canvas/canvas-project-domain";
import { applyCanvasAgentOps, type CanvasAgentSnapshot } from "../src/lib/canvas/canvas-agent-ops";
@@ -13,6 +15,33 @@ function fixture() {
return { snapshot, script };
}
test("Agent 上下文提前指出未选择画风及分镜空行", () => {
const { snapshot, script } = fixture();
const readiness = buildCanvasAgentContext(snapshot).storyboardReadiness;
expect(readiness.canGenerateStoryboard).toBe(false);
expect(readiness.style.ready).toBe(false);
expect(readiness.blockingReason).toContain("画风");
expect(readiness.storyboards.find((item) => item.nodeId === script.id)?.incompleteRowIds).toEqual(["row-1", "row-2"]);
expect(readiness.nextActions.join(" ")).toContain("canvas_apply_style");
});
test("已应用画风与真实分镜写入后，返回实际总时长", () => {
const { snapshot, script } = fixture();
const style = snapshot.nodes.find((node) => node.metadata?.workflowKind === "styleboard")!;
style.metadata = { ...style.metadata, stylePresetId: "preset", content: "电影写实" };
script.metadata!.storyboard!.rows = [2, 2, 3, 2, 1].map((durationSeconds, i) => createStoryboardRow(i + 1, { durationSeconds, videoMotionPrompt: `镜头${i + 1}` }));
const readiness = inspectStoryboardReadiness(snapshot.nodes);
expect(readiness.canGenerateStoryboard).toBe(true);
expect(readiness.storyboards[0]).toMatchObject({ rowCount: 5, populated: true, totalDurationSeconds: 10 });
expect(readiness.nextActions).toEqual([]);
});
test("通用节点创建不能用正文代替真实分镜行", () => {
expect(() => validateAgentStoryboardCreation([{ type: "add_node", nodeType: "script", metadata: { content: "镜头1：0–2秒近景；镜头2：2–4秒特写" } }])).toThrow("真实镜头行");
const { script } = fixture();
expect(() => validateAgentStoryboardCreation([{ type: "add_node", nodeType: "script", metadata: script.metadata }])).toThrow();
});
test("按行修改保留其他镜头及已有生成素材关联", () => {
const { snapshot, script } = fixture();
const next = applyCanvasAgentOps(snapshot, buildAgentStoryboardOperations(snapshot, { nodeId: script.id, action: "update", rowId: "row-1", patch: { dialogue: "新对白", durationSeconds: 8 } }));

50
web/test/creative-agent-controller.test.ts
View File

@@ -30,6 +30,56 @@ const proposal = { id: "p", version: 1, title: "方案", summary: "摘要", mark
const submission = (id: string, itemKey: string, taskId?: string): CreationSubmission => ({ id, runId: "run", itemKey, requestHash: "hash", taskId, quote: { model: "model", billingMode: "fixed_request", quantity: 1, amountMicrocredits: 1, estimated: false, expiresAt: "2099-01-01", quoteHash: "quote" } });
describe("创作控制器恢复", () => {
for (const terminal of [true, false]) test(terminal ? "任务已失败时标记生成失败，不能引导反复读取" : "仅查询断线时保留结果恢复语义，不当作需要重做", async () => {
const h = harness({ ...initialCreativeState(), proposal, canvasApplied: true, media: [{ ref: "a", nodeId: "a", attempt: 1, submissionId: "a", taskId: "a", status: "queued" }] }, "waiting_task", [submission("a", "a", "a")], async (_id, options) => {
if (terminal) options?.onTaskUpdate?.({ id: "a", status: "failed" } as GenerationTask);
throw new Error("网络异常。");
});
try { await h.controller.load("run"); await expect(h.controller.resume()).rejects.toThrow(); expect(h.view().state.media[0].failureKind).toBe(terminal ? "generation" : "observation"); expect(h.counters().executions).toBe(0); }
finally { h.controller.dispose(); }
});
test("刷新页面可接续仍有效的旧连接，不等待旧页面租约过期", async () => {
const h = harness(initialCreativeState());
const get = h.api.get, claim = h.api.claim;
let foreign = true, claims = 0;
h.api.get = async (...args) => { const detail = await get(...args); return foreign ? { ...detail, run: { ...detail.run, executionOwner: "previous-page", leaseExpiresAt: "2099-01-01" } } : detail; };
h.api.claim = async (...args) => { claims++; foreign = false; return claim(...args); };
try { await h.controller.load("run"); expect(h.view().hasControl).toBe(true); expect(claims).toBe(1); expect(h.counters().executions).toBe(0); }
finally { h.controller.dispose(); }
});
test("普通恢复不争抢其他连接，用户重试连接可以显式接续", async () => {
const h = harness(initialCreativeState());
const get = h.api.get, claim = h.api.claim;
let foreign = false, claims = 0;
h.api.get = async (...args) => { const detail = await get(...args); return foreign ? { ...detail, run: { ...detail.run, executionOwner: "other-page", leaseExpiresAt: "2099-01-01" } } : detail; };
h.api.claim = async (...args) => { claims++; foreign = false; return claim(...args); };
try {
await h.controller.load("run"); foreign = true;
await expect(h.controller.refresh()).rejects.toThrow("重试连接"); expect(claims).toBe(1);
await h.controller.takeControl(); expect(claims).toBe(2); expect(h.view().hasControl).toBe(true); expect(h.view().error).toBeUndefined();
} finally { h.controller.dispose(); }
});
test("已提交任务即使保留待处理批次也不能再次展示费用确认", async () => {
const h = harness({ ...initialCreativeState(), proposal, canvasApplied: true, pendingPayment: ["a"], media: [{ ref: "a", nodeId: "a", attempt: 1, submissionId: "a", taskId: "task-a", status: "queued" }] }, "waiting_task", [{ ...submission("a", "a", "task-a"), approvedAt: "2026-01-01" }]);
try { await h.controller.load("run"); expect(h.view().quote).toBeUndefined(); }
finally { h.controller.dispose(); }
});
test("部分提交后费用卡只保留未提交项", async () => {
const h = harness({ ...initialCreativeState(), pendingPayment: ["a", "b"] }, "paused", [submission("a", "a", "task-a"), submission("b", "b")]);
try { await h.controller.load("run"); expect(h.view().quote?.items.map((item) => item.id)).toEqual(["b"]); }
finally { h.controller.dispose(); }
});
test("迟到的确认点击复用已提交任务，不重新批准或刷新过期费用", async () => {
const old = { ...submission("a", "a", "task-a"), approvedAt: "2000-01-01" };
old.quote.expiresAt = "2000-01-01";
const completed = { id: "task-a", status: "succeeded", resultJson: JSON.stringify({ images: [{ storageKey: "resource:output" }] }) } as GenerationTask;
const h = harness({ ...initialCreativeState(), proposal, canvasApplied: true, pendingPayment: ["a"], media: [{ ref: "a", nodeId: "a", attempt: 1, submissionId: "a", taskId: "task-a", status: "queued" }] }, "waiting_task", [old], async () => completed);
h.api.approve = async () => { throw new Error("不应重新批准已提交任务"); };
h.api.refreshQuote = async () => { throw new Error("不应刷新已提交任务报价"); };
h.api.execute = async (_id, input) => { expect(input.submissionId).toBe("a"); return completed; };
try { await h.controller.load("run"); await h.controller.approvePayment(); expect(h.view().state.media[0].status).toBe("ready"); expect(h.view().quote).toBeUndefined(); expect(h.counters().prepares).toBe(0); }
finally { h.controller.dispose(); }
});
test("更新过期报价替换分析关联并等待新批准，不执行模型", async () => {
const old = submission("old", "planning:key");
const fresh = submission("fresh", "requote:old");