tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(*): 稳定期治理 - 收敛核心架构、错误语义并修复跨项目数据竞态

Browse Source
This commit is contained in:
ddCat
2026-09-09 22:02:10 +08:00
parent bacd502f5c
commit b5f3413ded
298 changed files with 12681 additions and 26428 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
web/src/lib/asset-category.ts
View File

@@ -29,6 +29,15 @@ export function normalizeAssetCategory(value: unknown, fallback: AssetCategory =
return LEGACY_ASSET_CATEGORY_MAP[normalized] || fallback;
}
export function parseAssetCategory(value: unknown): AssetCategory {
if (typeof value !== "string") throw new Error("素材 category 必须是字符串");
const normalized = value.trim().toLowerCase();
if ((ASSET_CATEGORIES as readonly string[]).includes(normalized)) return normalized as AssetCategory;
const mapped = LEGACY_ASSET_CATEGORY_MAP[normalized];
if (!mapped) throw new Error("素材 category 无效");
return mapped;
}
export function defaultAssetCategoryForKind(kind: string): AssetCategory {
if (kind === "entity") return "character";
if (kind === "image" || kind === "video" || kind === "audio" || kind === "model") return "material";

214
web/src/lib/asset-record.ts Normal file
View File

@@ -0,0 +1,214 @@
import type { Asset, AssetKind, AssetStatus, AudioAsset, EntityAsset, ImageAsset, ModelAsset, TextAsset, VideoAsset } from "@/stores/use-asset-store";
import { parseAssetCategory } from "@/lib/asset-category";
const ASSET_KINDS: readonly AssetKind[] = ["text", "image", "video", "audio", "model", "entity"];
const ASSET_STATUSES: readonly AssetStatus[] = ["draft", "review", "confirmed", "archived"];
/**
* 素材持久化合同的运行时边界。它只接受能被后续业务安全消费的完整记录，绝不通过空 URL、通配 MIME 或零尺寸伪造默认值。
* 展示层可以隔离历史坏记录，但新增、更新和写回前必须经过这里的严格校验。
*/
export function parseAssetRecord(value: unknown): Asset {
if (!isRecord(value)) throw new Error("素材记录不是对象");
const id = requireTrimmedString(value, "id");
const kind = requireTrimmedString(value, "kind");
if (!ASSET_KINDS.includes(kind as AssetKind)) throw new Error(`不支持的素材类型 ${kind}`);
const title = requireString(value, "title");
const coverUrl = requireString(value, "coverUrl");
const tags = requireStringArray(value.tags, "tags");
const createdAt = requireString(value, "createdAt");
const updatedAt = requireString(value, "updatedAt");
const folderId = optionalTrimmedString(value, "folderId");
const status = optionalAssetStatus(value.status);
const primaryVersionId = optionalTrimmedString(value, "primaryVersionId");
const source = optionalString(value, "source");
const note = optionalString(value, "note");
const metadata = optionalRecord(value.metadata, "metadata");
const category = value.category === undefined ? undefined : parseAssetCategory(value.category);
const data = requireRecord(value.data, "data");
const base = {
id,
title,
coverUrl,
tags,
createdAt,
updatedAt,
...(folderId ? { folderId } : {}),
...(category ? { category } : {}),
...(status ? { status } : {}),
...(primaryVersionId ? { primaryVersionId } : {}),
...(source !== undefined ? { source } : {}),
...(note !== undefined ? { note } : {}),
...(metadata ? { metadata } : {}),
};
if (kind === "text") {
return { ...base, kind, data: { content: requireString(data, "content") } } satisfies TextAsset;
}
if (kind === "image") {
const parsed = {
dataUrl: requireString(data, "dataUrl"),
storageKey: optionalString(data, "storageKey"),
width: requirePositiveNumber(data, "width"),
height: requirePositiveNumber(data, "height"),
bytes: requireNonNegativeNumber(data, "bytes"),
mimeType: requireMediaMimeType(data, "image"),
};
requireMediaLocator(parsed.dataUrl, parsed.storageKey, "图片", "dataUrl");
return { ...base, kind, data: parsed } satisfies ImageAsset;
}
if (kind === "video") {
const parsed = {
url: requireString(data, "url"),
storageKey: optionalString(data, "storageKey"),
width: requirePositiveNumber(data, "width"),
height: requirePositiveNumber(data, "height"),
durationMs: optionalNonNegativeNumber(data, "durationMs"),
hasAudio: optionalBoolean(data, "hasAudio"),
bytes: requireNonNegativeNumber(data, "bytes"),
mimeType: requireMediaMimeType(data, "video"),
};
requireMediaLocator(parsed.url, parsed.storageKey, "视频", "url");
return { ...base, kind, data: parsed } satisfies VideoAsset;
}
if (kind === "audio") {
const parsed = {
url: requireString(data, "url"),
storageKey: optionalString(data, "storageKey"),
durationMs: optionalNonNegativeNumber(data, "durationMs"),
bytes: requireNonNegativeNumber(data, "bytes"),
mimeType: requireMediaMimeType(data, "audio"),
};
requireMediaLocator(parsed.url, parsed.storageKey, "音频", "url");
return { ...base, kind, data: parsed } satisfies AudioAsset;
}
if (kind === "model") {
const parsed = {
url: requireString(data, "url"),
storageKey: optionalString(data, "storageKey"),
bytes: requireNonNegativeNumber(data, "bytes"),
mimeType: requireConcreteMimeType(data),
fileName: requireTrimmedString(data, "fileName"),
};
requireMediaLocator(parsed.url, parsed.storageKey, "模型", "url");
return { ...base, kind, data: parsed } satisfies ModelAsset;
}
if (kind === "entity") {
return { ...base, kind, data: { definition: requireRecord(data.definition, "definition") } } satisfies EntityAsset;
}
throw new Error(`不支持的素材类型 ${kind}`);
}
export function parseAssetRecordList(values: unknown): Asset[] {
if (!Array.isArray(values)) throw new Error("素材列表无效");
return values.map((item, index) => {
try {
return parseAssetRecord(item);
} catch (error) {
const id = isRecord(item) && typeof item.id === "string" && item.id.trim() ? item.id : `#${index}`;
throw new Error(`素材 ${id} 合同无效：${error instanceof Error ? error.message : String(error)}`);
}
});
}
function requireMediaLocator(primary: string, storageKey: string | undefined, label: string, primaryKey: string) {
if (!primary.trim() && !storageKey?.trim()) {
throw new Error(`${label}素材缺少 ${primaryKey} 或 storageKey`);
}
}
function requireString(record: Record<string, unknown>, key: string): string {
const value = record[key];
if (typeof value !== "string") throw new Error(`素材字段 ${key} 必须是字符串`);
return value;
}
function requireTrimmedString(record: Record<string, unknown>, key: string): string {
const value = requireString(record, key).trim();
if (!value) throw new Error(`素材字段 ${key} 不能为空`);
return value;
}
function optionalString(record: Record<string, unknown>, key: string): string | undefined {
if (!Object.prototype.hasOwnProperty.call(record, key) || record[key] === undefined) return undefined;
if (typeof record[key] !== "string") throw new Error(`素材字段 ${key} 必须是字符串`);
return record[key] as string;
}
function optionalTrimmedString(record: Record<string, unknown>, key: string): string | undefined {
const value = optionalString(record, key)?.trim();
return value || undefined;
}
function requireStringArray(value: unknown, key: string): string[] {
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
throw new Error(`素材字段 ${key} 必须是字符串数组`);
}
return value;
}
function requireRecord(value: unknown, key: string): Record<string, unknown> {
if (!isRecord(value)) throw new Error(`素材字段 ${key} 必须是对象`);
return value;
}
function optionalRecord(value: unknown, key: string): Record<string, unknown> | undefined {
if (value === undefined) return undefined;
return requireRecord(value, key);
}
function requireNonNegativeNumber(record: Record<string, unknown>, key: string): number {
const value = record[key];
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
throw new Error(`素材字段 ${key} 必须是非负数字`);
}
return value;
}
function requirePositiveNumber(record: Record<string, unknown>, key: string): number {
const value = requireNonNegativeNumber(record, key);
if (value === 0) throw new Error(`素材字段 ${key} 必须大于 0`);
return value;
}
function requireConcreteMimeType(record: Record<string, unknown>): string {
const mimeType = requireTrimmedString(record, "mimeType").toLowerCase();
if (!/^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+$/.test(mimeType) || mimeType.endsWith("/*")) {
throw new Error("素材字段 mimeType 必须是具体 MIME 类型");
}
return mimeType;
}
function requireMediaMimeType(record: Record<string, unknown>, kind: "image" | "video" | "audio"): string {
const mimeType = requireConcreteMimeType(record);
// application/octet-stream 是诚实的“类型未知”；除此之外必须与素材种类一致，
// 防止图片记录携带 audio/* 等跨类型元数据后在下载、转码阶段才暴露错误。
if (mimeType !== "application/octet-stream" && !mimeType.startsWith(`${kind}/`)) {
throw new Error(`素材字段 mimeType 与 ${kind} 类型不匹配`);
}
return mimeType;
}
function optionalNonNegativeNumber(record: Record<string, unknown>, key: string): number | undefined {
if (!Object.prototype.hasOwnProperty.call(record, key) || record[key] === undefined) return undefined;
return requireNonNegativeNumber(record, key);
}
function optionalBoolean(record: Record<string, unknown>, key: string): boolean | undefined {
if (!Object.prototype.hasOwnProperty.call(record, key) || record[key] === undefined) return undefined;
if (typeof record[key] !== "boolean") throw new Error(`素材字段 ${key} 必须是布尔值`);
return record[key] as boolean;
}
function optionalAssetStatus(value: unknown): AssetStatus | undefined {
if (value === undefined) return undefined;
if (typeof value !== "string" || !ASSET_STATUSES.includes(value as AssetStatus)) {
throw new Error("素材 status 无效");
}
return value as AssetStatus;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}

13
web/src/lib/asset-storage-revision.test.ts
View File

@@ -2,10 +2,10 @@ import assert from "node:assert/strict";
import test from "node:test";
// @ts-expect-error -- Node 原生 TypeScript 测试运行器需要保留扩展名。
import { normalizeAssetRecord } from "./asset-storage-revision.ts";
import { parseAssetRecord } from "./asset-record.ts";
test("缺失图片尺寸的历史素材会补齐可渲染数据", () => {
const asset = normalizeAssetRecord({
test("缺失图片数据的历史素材会被拒绝，而不是补齐可渲染字段", () => {
assert.throws(() => parseAssetRecord({
id: "legacy-image",
kind: "image",
title: "历史图片",
@@ -14,10 +14,5 @@ test("缺失图片尺寸的历史素材会补齐可渲染数据", () => {
createdAt: "2026-01-01T00:00:00.000Z",
updatedAt: "2026-01-01T00:00:00.000Z",
data: undefined,
} as never);
assert.equal(asset.kind, "image");
assert.equal(asset.data.width, 0);
assert.equal(asset.data.height, 0);
assert.equal(asset.data.dataUrl, "https://example.com/image.png");
}), /data/);
});

116
web/src/lib/asset-storage-revision.ts
View File

@@ -1,5 +1,5 @@
import type { Asset } from "@/stores/use-asset-store";
import { defaultAssetCategoryForKind, normalizeAssetCategory } from "@/lib/asset-category";
import { parseAssetRecord, parseAssetRecordList } from "@/lib/asset-record";
export type AssetStorageDocument = {
state: { assets: Asset[] };
@@ -8,51 +8,10 @@ export type AssetStorageDocument = {
tombstones: { assets: Record<string, number> };
};
export function normalizeAssetRecord(asset: Asset): Asset {
const category = normalizeAssetCategory(asset.category, defaultAssetCategoryForKind(asset.kind));
const folderId = typeof asset.folderId === "string" ? asset.folderId.trim() : "";
const normalizedData = normalizeAssetData(asset);
if (Array.isArray(asset.tags) && asset.tags.every((tag) => typeof tag === "string") && asset.category === category && (asset.folderId || "") === folderId && normalizedData === asset.data) return asset;
return {
...asset,
tags: Array.isArray(asset.tags) ? asset.tags.filter((tag): tag is string => typeof tag === "string") : [],
folderId: folderId || undefined,
category,
data: normalizedData,
} as Asset;
}
function normalizeAssetData(asset: Asset): Asset["data"] {
const raw = asset.data && typeof asset.data === "object" ? asset.data as Record<string, unknown> : {};
const numberValue = (value: unknown) => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0;
if (asset.kind === "text") return typeof raw.content === "string" ? asset.data : { content: "" };
if (asset.kind === "image") {
if (typeof raw.dataUrl === "string" && typeof raw.width === "number" && typeof raw.height === "number") return asset.data;
return { dataUrl: typeof raw.dataUrl === "string" ? raw.dataUrl : asset.coverUrl || "", storageKey: typeof raw.storageKey === "string" ? raw.storageKey : undefined, width: numberValue(raw.width), height: numberValue(raw.height), bytes: numberValue(raw.bytes), mimeType: typeof raw.mimeType === "string" ? raw.mimeType : "image/*" };
}
if (asset.kind === "video") {
if (typeof raw.url === "string" && typeof raw.width === "number" && typeof raw.height === "number") return asset.data;
return { url: typeof raw.url === "string" ? raw.url : asset.coverUrl || "", storageKey: typeof raw.storageKey === "string" ? raw.storageKey : undefined, width: numberValue(raw.width), height: numberValue(raw.height), durationMs: numberValue(raw.durationMs), hasAudio: raw.hasAudio === true, bytes: numberValue(raw.bytes), mimeType: typeof raw.mimeType === "string" ? raw.mimeType : "video/*" };
}
if (asset.kind === "audio") {
if (typeof raw.url === "string") return asset.data;
return { url: "", storageKey: typeof raw.storageKey === "string" ? raw.storageKey : undefined, durationMs: numberValue(raw.durationMs), bytes: numberValue(raw.bytes), mimeType: typeof raw.mimeType === "string" ? raw.mimeType : "audio/*" };
}
if (asset.kind === "model") {
if (typeof raw.url === "string" && typeof raw.fileName === "string") return asset.data;
return { url: "", storageKey: typeof raw.storageKey === "string" ? raw.storageKey : undefined, bytes: numberValue(raw.bytes), mimeType: typeof raw.mimeType === "string" ? raw.mimeType : "model/*", fileName: typeof raw.fileName === "string" ? raw.fileName : asset.title || "未命名模型" };
}
return asset.data;
}
function normalizeAssets(assets: Asset[]) {
return assets.map(normalizeAssetRecord);
}
export function parseAssetStorageDocument(value: string | null, fallback: Asset[] = []): AssetStorageDocument {
if (!value) {
return {
state: { assets: normalizeAssets(fallback) },
state: { assets: parseAssetRecordList(fallback) },
version: 0,
storageRevision: 0,
tombstones: { assets: {} },
@@ -69,13 +28,77 @@ export function parseAssetStorageDocument(value: string | null, fallback: Asset[
const tombstones =
rawTombstones && typeof rawTombstones === "object" && !Array.isArray(rawTombstones) ? Object.fromEntries(Object.entries(rawTombstones).filter((entry): entry is [string, number] => typeof entry[1] === "number" && Number.isFinite(entry[1]))) : {};
return {
state: { assets: normalizeAssets(parsed.state.assets as Asset[]) },
state: { assets: parseAssetRecordList(parsed.state.assets) },
version: typeof parsed.version === "number" ? parsed.version : 0,
storageRevision: typeof parsed.storageRevision === "number" && Number.isFinite(parsed.storageRevision) ? parsed.storageRevision : 0,
tombstones: { assets: tombstones },
};
}
export type InvalidAssetRecord = {
index: number;
id?: string;
error: string;
};
export type AssetStorageRecovery = {
document: AssetStorageDocument;
invalid: InvalidAssetRecord[];
};
/**
* 仅用于读取历史本地缓存：保留仍符合合同的素材，并把坏记录隔离到诊断结果。
*
* 这里不能把坏记录改成空 URL、通配 MIME 或其他“看起来可用”的默认值；
* 读路径允许恢复可用部分，但写路径仍由 parseAssetRecord 严格阻断损坏状态。
*/
export function parseAssetStorageDocumentRecovering(value: string | null, fallback: Asset[] = []): AssetStorageRecovery {
if (!value) {
return {
document: parseAssetStorageDocument(value, fallback),
invalid: [],
};
}
const parsed = JSON.parse(value) as {
state?: { assets?: unknown };
version?: unknown;
storageRevision?: unknown;
tombstones?: { assets?: unknown };
};
if (!Array.isArray(parsed.state?.assets)) throw new Error("素材持久状态无效");
const invalid: InvalidAssetRecord[] = [];
const assets: Asset[] = [];
parsed.state.assets.forEach((item, index) => {
try {
assets.push(parseAssetRecord(item));
} catch (error) {
const id = isRecord(item) && typeof item.id === "string" && item.id.trim() ? item.id : undefined;
invalid.push({
index,
...(id ? { id } : {}),
error: error instanceof Error ? error.message : String(error),
});
}
});
const rawTombstones = parsed.tombstones?.assets;
const tombstones =
rawTombstones && typeof rawTombstones === "object" && !Array.isArray(rawTombstones) ? Object.fromEntries(Object.entries(rawTombstones).filter((entry): entry is [string, number] => typeof entry[1] === "number" && Number.isFinite(entry[1]))) : {};
return {
document: {
state: { assets },
version: typeof parsed.version === "number" ? parsed.version : 0,
storageRevision: typeof parsed.storageRevision === "number" && Number.isFinite(parsed.storageRevision) ? parsed.storageRevision : 0,
tombstones: { assets: tombstones },
},
invalid,
};
}
export function serializeAssetStorageDocument(document: AssetStorageDocument) {
return JSON.stringify(document);
}
@@ -124,6 +147,11 @@ function mergeRecord(base: Record<string, unknown>, local: Record<string, unknow
return merged;
}
/**
* 将本地内存快照合并到最新 durable 文档：base 是读取时看到的版本，local 是当前用户修改，durable 是另一写入者已落盘的版本。
* 删除通过 tombstone 记录 revision，避免旧标签页把已删除素材重新写回来；字段冲突按三方变更而不是最后写入时间粗暴覆盖。
*/
export function rebaseAssetSnapshot(input: { document: AssetStorageDocument; baseAssets: Asset[]; localAssets: Asset[]; baseRevision: number }) {
const nextRevision = input.document.storageRevision + 1;
const tombstones = { assets: { ...input.document.tombstones.assets } };

32
web/src/lib/canvas/canvas-agent-protocol.ts Normal file
View File

@@ -0,0 +1,32 @@
import { SKILL_RUNTIME_AGENT_GUIDANCE, skillRuntime } from "@/services/skill-runtime";
import type { ResponseToolCall } from "@/services/api/image";
export const CANVAS_ONLINE_AGENT_PROMPT =
`你是当前创作工作台内置的在线画布助手。首轮必须先调用 canvas_get_context；涉及已有节点时用 canvas_find_nodes 获取真实 id，涉及媒体参考时用 canvas_get_resources。流水线、工作流、管线、节点图或用户要求连线时，必须使用 canvas_create_workflow：把需求拆成有语义的节点类型、真实内容/提示词、边和布局，禁止把业务阶段退化成几个空文本卡片；工具会自动分配 id、布局并建立连线。复杂写操作先 canvas_validate_ops，再执行 canvas_apply_ops。任何写入后都必须检查工具返回的真实节点类型、connectionCount、overlapWarnings 和 verification；没有真实连线时绝不能说已连线，没有生成资源时绝不能说已完成。不要输出 JSON ops、不要猜 id、不要把未就绪资源当作可用素材、不要编造执行结果。需要用户选择时，给出可点击的短选项，不要只让用户输入 1、2、3。${SKILL_RUNTIME_AGENT_GUIDANCE}`;
export const CANVAS_READ_TOOL_NAMES = [
"canvas_get_state",
"canvas_get_context",
"canvas_find_nodes",
"canvas_get_node",
"canvas_get_connection",
"canvas_get_generation_tasks",
"canvas_get_resources",
"canvas_validate_ops",
"canvas_get_selection",
"canvas_export_snapshot",
] as const;
const CANVAS_READ_TOOL_NAME_SET = new Set<string>(CANVAS_READ_TOOL_NAMES);
export function isCanvasReadTool(name: string) {
return CANVAS_READ_TOOL_NAME_SET.has(name) || skillRuntime.agentToolNames("onlineAgent").has(name);
}
export function isWritableToolCall(call: ResponseToolCall) {
return !isCanvasReadTool(call.function.name);
}
export function buildCanvasOnlineAgentSystemContent(skillCatalog: string) {
return [CANVAS_ONLINE_AGENT_PROMPT, skillCatalog ? `当前可按需加载的技能（仅元数据）：\n${skillCatalog}` : ""].filter(Boolean).join("\n\n");
}

195
web/src/lib/canvas/canvas-agent-tools.ts Normal file
View File

@@ -0,0 +1,195 @@
import { type ResponseFunctionTool } from "@/services/api/image";
import { skillRuntime } from "@/services/skill-runtime";
const JSON_RECORD_SCHEMA = { type: "object", additionalProperties: true };
const POSITION_SCHEMA = { type: "object", properties: { x: { type: "number" }, y: { type: "number" } }, required: ["x", "y"], additionalProperties: false };
const VIEWPORT_SCHEMA = { type: "object", properties: { x: { type: "number" }, y: { type: "number" }, k: { type: "number" } }, required: ["x", "y", "k"], additionalProperties: false };
const NODE_TYPE_SCHEMA = { type: "string", enum: ["image", "text", "skill", "video", "audio"] };
const WORKFLOW_NODE_KIND_SCHEMA = { type: "string", enum: ["text", "script", "image", "video", "audio", "character_cards", "character_three_view", "storyboard_video"] };
const GENERATION_MODE_SCHEMA = { type: "string", enum: ["text", "image", "video", "audio"] };
const GENERATION_OPTION_PROPERTIES = {
model: { type: "string" },
size: { type: "string" },
quality: { type: "string" },
transparentBackground: { type: "string", enum: ["true", "false"] },
count: { type: "number" },
seconds: { type: "string" },
vquality: { type: "string" },
generateAudio: { type: "string" },
watermark: { type: "string" },
audioVoice: { type: "string" },
audioFormat: { type: "string" },
audioSpeed: { type: "string" },
audioInstructions: { type: "string" },
};
const CANVAS_OP_SCHEMA = {
type: "object",
properties: {
type: { type: "string", enum: ["add_node", "update_node", "delete_node", "delete_connections", "connect_nodes", "set_viewport", "select_nodes", "run_generation"] },
id: { type: "string" },
ids: { type: "array", items: { type: "string" } },
nodeType: NODE_TYPE_SCHEMA,
title: { type: "string" },
x: { type: "number" },
y: { type: "number" },
width: { type: "number" },
height: { type: "number" },
position: POSITION_SCHEMA,
metadata: JSON_RECORD_SCHEMA,
patch: JSON_RECORD_SCHEMA,
all: { type: "boolean" },
fromNodeId: { type: "string" },
toNodeId: { type: "string" },
viewport: VIEWPORT_SCHEMA,
nodeId: { type: "string" },
mode: GENERATION_MODE_SCHEMA,
prompt: { type: "string" },
retry: { type: "boolean" },
},
required: ["type"],
additionalProperties: false,
};
function toolDefinition(name: string, description: string, properties: Record<string, unknown>, required: string[] = [], strict = false): ResponseFunctionTool {
return { type: "function", function: { name, description, parameters: { type: "object", properties, required, additionalProperties: false }, strict } };
}
function generationToolDefinition(name: string, description: string, mode?: "text" | "image" | "video" | "audio") {
return toolDefinition(
name,
description,
{
prompt: { type: "string" },
title: { type: "string" },
x: { type: "number" },
y: { type: "number" },
referenceNodeIds: { type: "array", items: { type: "string" } },
...(mode ? {} : { mode: GENERATION_MODE_SCHEMA }),
autoRun: { type: "boolean" },
...GENERATION_OPTION_PROPERTIES,
},
["prompt"],
);
}
export const CANVAS_ONLINE_AGENT_TOOLS: ResponseFunctionTool[] = [
...skillRuntime.agentTools("onlineAgent"),
toolDefinition("canvas_get_state", "读取当前网页画布的节点、连线、选区和视口。", {}),
toolDefinition("canvas_get_context", "读取语义化画布上下文、真实节点 id、连接关系、资源就绪状态和状态哈希。", {}),
toolDefinition("canvas_find_nodes", "按标题、内容、提示词、类型、状态或资产检索真实节点。", { query: { type: "string" }, ids: { type: "array", items: { type: "string" } }, types: { type: "array", items: { type: "string" } }, statuses: { type: "array", items: { type: "string" } }, resourceOnly: { type: "boolean" }, limit: { type: "number" } }),
toolDefinition("canvas_get_node", "按真实节点 id 精确读取单个节点、资源状态和关联连线。", { id: { type: "string" } }, ["id"]),
toolDefinition("canvas_get_connection", "按真实连线 id 精确读取端点节点和 handle 信息。", { id: { type: "string" } }, ["id"]),
toolDefinition("canvas_get_generation_tasks", "读取当前画布绑定的生成任务观察状态，不主动轮询上游。", { status: { type: "string" }, nodeIds: { type: "array", items: { type: "string" } }, limit: { type: "number" } }),
toolDefinition("canvas_get_resources", "读取画布媒体资源引用、类型、尺寸、大小、时长和就绪状态，不返回媒体 URL。", { nodeIds: { type: "array", items: { type: "string" } }, status: { type: "string" }, limit: { type: "number" } }),
toolDefinition("canvas_validate_ops", "在写入前校验节点 id、连接关系和批量操作参数。", { ops: { type: "array", items: CANVAS_OP_SCHEMA } }, ["ops"]),
toolDefinition("canvas_get_selection", "读取当前网页画布选中的节点。", {}),
toolDefinition("canvas_export_snapshot", "导出当前画布快照，用于理解布局。", {}),
toolDefinition(
"canvas_apply_ops",
"批量操作当前网页画布。复杂写操作应先 canvas_validate_ops；可传 canvas_get_context 返回的 expectedStateHash 防止基于过期状态写入。",
{ ops: { type: "array", items: CANVAS_OP_SCHEMA }, expectedRevision: { type: "number" }, expectedStateHash: { type: "string" } },
["ops"],
false,
),
toolDefinition(
"canvas_create_workflow",
"创建语义化工作流/流水线：节点使用真实的文本、脚本、图片、视频或音频类型；character_cards=角色拆分图片卡片，character_three_view=角色三视图，storyboard_video=分镜剧情视频。工具会自动生成唯一 id、按节点实际尺寸布局、创建 edges/referenceRefs/referenceNodeIds 连线、选择新节点并复核重叠。媒体节点必须提供有意义的 prompt 或 content；已有素材先 canvas_find_nodes/canvas_get_resources，再把真实 node id 放入 referenceNodeIds。不要用 canvas_create_text_nodes 代替工作流。",
{
title: { type: "string" },
description: { type: "string" },
nodes: {
type: "array",
minItems: 1,
items: {
type: "object",
properties: {
ref: { type: "string" },
kind: WORKFLOW_NODE_KIND_SCHEMA,
title: { type: "string" },
content: { type: "string" },
prompt: { type: "string" },
description: { type: "string" },
referenceRefs: { type: "array", items: { type: "string" } },
referenceNodeIds: { type: "array", items: { type: "string" } },
runGeneration: { type: "boolean" },
width: { type: "number" },
height: { type: "number" },
},
required: ["ref", "kind", "title"],
additionalProperties: false,
},
},
edges: { type: "array", items: { type: "object", properties: { from: { type: "string" }, to: { type: "string" } }, required: ["from", "to"], additionalProperties: false } },
direction: { type: "string", enum: ["horizontal", "vertical"] },
start: POSITION_SCHEMA,
gap: { type: "number" },
autoRun: { type: "boolean" },
},
["nodes"],
),
toolDefinition(
"canvas_create_node",
"创建任意类型节点：text、image、video、audio。适合创建文本、媒体占位或自定义 metadata 节点。",
{ nodeType: NODE_TYPE_SCHEMA, title: { type: "string" }, x: { type: "number" }, y: { type: "number" }, width: { type: "number" }, height: { type: "number" }, metadata: JSON_RECORD_SCHEMA },
["nodeType"],
),
toolDefinition("canvas_create_text_node", "在当前画布创建单个文本节点。", { text: { type: "string" }, x: { type: "number" }, y: { type: "number" }, title: { type: "string" }, width: { type: "number" }, height: { type: "number" } }),
toolDefinition(
"canvas_create_text_nodes",
"批量创建文本节点，适合生成标题、段落、脚本、说明等内容块。",
{
items: {
type: "array",
minItems: 1,
items: {
type: "object",
properties: { text: { type: "string" }, title: { type: "string" }, x: { type: "number" }, y: { type: "number" }, width: { type: "number" }, height: { type: "number" } },
required: ["text"],
additionalProperties: false,
},
},
x: { type: "number" },
y: { type: "number" },
gap: { type: "number" },
direction: { type: "string", enum: ["row", "column"] },
},
["items"],
),
toolDefinition("canvas_create_cinematic_session", "把自然语言创作指令提交给后端影视 Agent 会话，后端拆解为剧本、场景、分镜、镜头和成片节点，并返回可写回画布的操作。", { prompt: { type: "string" } }, ["prompt"]),
toolDefinition(
"canvas_create_image_prompt_flow",
"创建提示词文本节点和图片目标节点并自动连线，可选择立即触发生图。",
{ prompt: { type: "string" }, x: { type: "number" }, y: { type: "number" }, autoRun: { type: "boolean" }, ...GENERATION_OPTION_PROPERTIES },
["prompt"],
),
generationToolDefinition("canvas_create_generation_flow", "创建通用生成流程：提示词文本节点、对应类型的生成目标节点和参考节点连线，可用于文案、生图、视频或音频。"),
generationToolDefinition("canvas_generate_text", "创建通用文本生成流程并立即触发生成。", "text"),
generationToolDefinition("canvas_generate_image", "创建通用图片生成流程并立即触发生成。", "image"),
generationToolDefinition("canvas_generate_video", "创建通用视频生成流程并立即触发生成。", "video"),
generationToolDefinition("canvas_generate_audio", "创建通用音频生成流程并立即触发生成。", "audio"),
toolDefinition("canvas_update_node", "更新节点基础字段或 metadata。", { id: { type: "string" }, patch: JSON_RECORD_SCHEMA, metadata: JSON_RECORD_SCHEMA }, ["id"]),
toolDefinition("canvas_update_node_text", "更新文本节点内容和标题。", { id: { type: "string" }, text: { type: "string" }, title: { type: "string" } }, ["id", "text"]),
toolDefinition(
"canvas_move_nodes",
"移动一个或多个节点，支持绝对坐标或 dx/dy 偏移。",
{
items: {
type: "array",
minItems: 1,
items: { type: "object", properties: { id: { type: "string" }, x: { type: "number" }, y: { type: "number" }, dx: { type: "number" }, dy: { type: "number" } }, required: ["id"], additionalProperties: false },
},
},
["items"],
),
toolDefinition("canvas_resize_node", "调整节点尺寸。", { id: { type: "string" }, width: { type: "number" }, height: { type: "number" }, freeResize: { type: "boolean" } }, ["id", "width", "height"]),
toolDefinition("canvas_delete_nodes", "删除指定节点及相关连线。", { ids: { type: "array", items: { type: "string" }, minItems: 1 } }, ["ids"]),
toolDefinition(
"canvas_connect_nodes",
"批量连接节点。",
{ connections: { type: "array", minItems: 1, items: { type: "object", properties: { fromNodeId: { type: "string" }, toNodeId: { type: "string" } }, required: ["fromNodeId", "toNodeId"], additionalProperties: false } } },
["connections"],
),
toolDefinition("canvas_select_nodes", "设置当前选中节点。", { ids: { type: "array", items: { type: "string" } } }, ["ids"]),
toolDefinition("canvas_set_viewport", "调整画布视口。", { viewport: VIEWPORT_SCHEMA }, ["viewport"]),
toolDefinition("canvas_run_generation", "触发指定节点生成；对已有生成任务明确重试时传 retry=true。", { nodeId: { type: "string" }, mode: GENERATION_MODE_SCHEMA, prompt: { type: "string" }, retry: { type: "boolean" } }, ["nodeId"]),
];

7
web/src/lib/canvas/canvas-agent-workflow.ts
View File

@@ -1,6 +1,7 @@
import { nanoid } from "nanoid";
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { workflowStarterPrompt } from "@/lib/prompts";
import { CanvasNodeType, type CanvasNodeData, type CanvasNodeMetadata } from "@/types/canvas";
import type { AiConfig } from "@/stores/use-config-store";
import type { CanvasAgentOp, CanvasAgentSnapshot } from "./canvas-agent-ops";
@@ -143,9 +144,9 @@ export function buildCanvasWorkflowOps(input: CanvasWorkflowInput, snapshot: Can
function workflowPrompt(kind: CanvasWorkflowNodeKind, title: string, input: CanvasWorkflowInput) {
const workflowTitle = (input.title || input.description || "当前创作项目").trim();
if (kind === "character_cards") return `请基于「${workflowTitle}」拆分主要角色，并为每个角色生成可用于后续创作的角色图片卡片：外观、服饰、身份、性格和视觉辨识点。`;
if (kind === "character_three_view") return `请基于上游角色卡片生成「${title}」：同一角色的正面、侧面、背面三视图，保持服饰、发型、道具和比例一致。`;
if (kind === "storyboard_video") return `请基于上游角色三视图，为「${workflowTitle}」制作分镜剧情视频方案：包含镜头顺序、景别、动作、节奏和画面连续性。`;
if (kind === "character_cards" || kind === "character_three_view" || kind === "storyboard_video") {
return workflowStarterPrompt(kind, title, workflowTitle);
}
return "";
}

21
web/src/lib/canvas/canvas-node-asset.ts
View File

@@ -12,8 +12,15 @@ type CanvasNodeAssetOptions = {
};
export function canvasNodeToAsset(node: CanvasNodeData, options: CanvasNodeAssetOptions): NewAsset | null {
const content = node.metadata?.content?.trim();
if (!content) return null;
const content = node.metadata?.content?.trim() || "";
const storageKey = node.metadata?.storageKey?.trim() || undefined;
const isMedia = node.type === CanvasNodeType.Image || node.type === CanvasNodeType.Video || node.type === CanvasNodeType.Audio;
// 文本素材必须直接携带正文；媒体则允许只有持久 storageKey。后者常见于页面重载后：
// blob/object URL 已失效，但资源定位符仍可从后端重新换取可访问地址，不能因此丢掉素材记录。
if (node.type === CanvasNodeType.Text && !content) return null;
if (isMedia && !content && !storageKey) return null;
if (!isMedia && node.type !== CanvasNodeType.Text) return null;
const title = node.metadata?.prompt?.slice(0, 24) || node.title || canvasAssetFallbackTitle(node.type);
const metadata = {
source: options.source,
@@ -21,7 +28,7 @@ export function canvasNodeToAsset(node: CanvasNodeData, options: CanvasNodeAsset
nodeId: node.id,
taskId: options.taskId || node.metadata?.taskId,
prompt: node.metadata?.prompt,
resourceKey: node.metadata?.storageKey,
resourceKey: storageKey,
};
const base = {
title,
@@ -37,13 +44,13 @@ export function canvasNodeToAsset(node: CanvasNodeData, options: CanvasNodeAsset
return { ...base, kind: "text", data: { content } };
}
if (node.type === CanvasNodeType.Image) {
const dataUrl = node.metadata?.storageKey ? "" : content;
const dataUrl = storageKey ? "" : content;
return {
...base,
kind: "image",
data: {
dataUrl,
storageKey: node.metadata?.storageKey,
storageKey,
width: node.metadata?.naturalWidth || node.width,
height: node.metadata?.naturalHeight || node.height,
bytes: node.metadata?.bytes || getDataUrlByteSize(dataUrl),
@@ -57,7 +64,7 @@ export function canvasNodeToAsset(node: CanvasNodeData, options: CanvasNodeAsset
kind: "video",
data: {
url: content,
storageKey: node.metadata?.storageKey,
storageKey,
width: node.metadata?.naturalWidth || node.width,
height: node.metadata?.naturalHeight || node.height,
durationMs: node.metadata?.durationMs,
@@ -73,7 +80,7 @@ export function canvasNodeToAsset(node: CanvasNodeData, options: CanvasNodeAsset
kind: "audio",
data: {
url: content,
storageKey: node.metadata?.storageKey,
storageKey,
durationMs: node.metadata?.durationMs,
bytes: node.metadata?.bytes || 0,
mimeType: node.metadata?.mimeType || "audio/mpeg",

3
web/src/lib/canvas/canvas-project-domain.ts
View File

@@ -1,4 +1,5 @@
import { NODE_DEFAULT_SIZE, getNodeSpec } from "@/constant/canvas";
import { PromptTemplateOperation } from "@/lib/prompts";
import { STORYBOARD_HEADER_HEIGHT, STORYBOARD_ROW_HEIGHT, storyboardTableHeight } from "@/lib/canvas/canvas-storyboard-layout";
import { normalizeStoryboardAssetBindings } from "@/lib/canvas/canvas-storyboard-assets";
import { bindingForConnectedNode, storyboardComposerContent, storyboardRowReferenceNodeIds } from "@/lib/canvas/canvas-storyboard-materializer";
@@ -93,7 +94,7 @@ export function createStoryboardRow(shotNumber: number, patch: Partial<Storyboar
export function storyboardPromptTemplateMetadata(row: StoryboardRow, kind: "image" | "video"): Pick<CanvasNodeMetadata, "promptTemplateOperation" | "promptTemplateVariables"> {
const variables = kind === "image" ? row.imagePromptTemplateVariables : row.videoPromptTemplateVariables;
return variables
? { promptTemplateOperation: kind === "image" ? "storyboard_first_frame" : "storyboard_video", promptTemplateVariables: variables }
? { promptTemplateOperation: kind === "image" ? PromptTemplateOperation.StoryboardFirstFrame : PromptTemplateOperation.StoryboardVideo, promptTemplateVariables: variables }
: { promptTemplateOperation: undefined, promptTemplateVariables: undefined };
}

7
web/src/lib/canvas/canvas-resource-references.ts
View File

@@ -39,7 +39,7 @@ export function canvasNodeMentionToken(nodeId: string) {
export function canvasResourceMentionToken(reference: CanvasResourceReference) {
if (reference.mentionToken) return reference.mentionToken;
if (reference.kind === "skill" && reference.skill?.skill_id) return canvasSkillMentionToken(reference.skill.skill_id);
if (reference.kind === "skill" && reference.skill?.skillId) return canvasSkillMentionToken(reference.skill.skillId);
if (reference.assetId) return `@[asset:${reference.assetId}]`;
return `@${reference.label}`;
}
@@ -239,7 +239,10 @@ export function getMentionResourceNodes(nodeId: string, nodes: CanvasNodeData[],
if (configInputs.length) return configInputs;
const ownInputs = getContextResourceNodes(nodeId, nodes, connections);
if (ownInputs.length) return ownInputs;
return [];
// 没有入边时，资源节点可以把自身当作 @图片1 / @视频1，用于图生图、视频再编辑。
// 有入边时仍只暴露上游，避免自身把槽位序号挤掉。
const self = nodes.find((node) => node.id === nodeId);
return self && isResourceNode(self) ? [self] : [];
}
export function getGenerationResourceNodes(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]) {

9
web/src/lib/canvas/director/director-onboarding.ts
View File

@@ -2,7 +2,7 @@ import { localForageStorageForScope } from "@/lib/localforage-storage";
import type { DirectorMode } from "@/lib/canvas/director/director-modes";
/**
* 导演台首次上手引导（P2）。
* 导演台首次上手引导。
*
* 为什么存在：导演台把模板、四个一级模式、对象操作、保存/导出堆在同一个工作台里，
* 第一次进来的人没有任何线索该先做什么。引导给出一条确定的最短路径。
@@ -44,7 +44,7 @@ export type DirectorOnboardingStep = {
mode?: DirectorMode;
};
/** 顺序与 Issue #305 的首次任务一致：添加演员 → 移动 → 调姿 → 轨迹 → CAM → 应用。 */
/** 引导步骤按首次使用的最短操作路径排列：添加演员 → 移动 → 调姿 → 轨迹 → CAM → 应用。 */
export const DIRECTOR_ONBOARDING_STEPS: DirectorOnboardingStep[] = [
{ id: "actor", title: "添加并选中演员", detail: "在左侧「快速添加」点演员；模板已有演员时，直接在场景树或视口选中它。", mode: "layout" },
{ id: "move", title: "把演员移动到表演位置", detail: "保持摆场模式，在底部 dock 选移动工具（W），拖动轴手柄；右栏可输入精确坐标。", mode: "layout" },
@@ -207,9 +207,8 @@ export async function advanceDirectorOnboarding(scope: string, progress: Directo
}
/**
* 独立的重启 API。不依赖调用方持有当前内存进度 —— 未来工作台设置里的
* 「重新开始引导」按钮不必先加载一次旧进度才能重置，直接落盘初始状态即可，
* 对已 dismissed/completed 的账号同样生效。
* 独立的重启 API。不依赖调用方持有当前内存进度；设置页的「重新开始引导」
* 按钮可以直接落盘初始状态，对已 dismissed/completed 的账号同样生效。
*/
export async function resetDirectorOnboardingProgress(scope: string, storage?: DirectorOnboardingStorage): Promise<DirectorOnboardingProgress> {
const normalizedScope = requireScope(scope);

28
web/src/lib/canvas/skill-drafting.ts
View File

@@ -1,28 +1,15 @@
import { buildGenerationConfig } from "@/lib/canvas/canvas-project-generation";
import { PromptTemplateOperation, promptTemplateTaskPlaceholder } from "@/lib/prompts";
import { runBackendGenerationTask } from "@/services/api/generation-task";
import type { AiConfig } from "@/stores/use-config-store";
export type SkillDraft = {
skill_name: string;
skillName: string;
description: string;
instruction: string;
tag?: string;
};
const SKILL_DRAFT_PROMPT = `你是一位技能编写助手。根据用户的想法，为一个「可复用的创作技能」生成一份草稿。
只输出一个 JSON 对象，不要输出任何其他文字、解释或 Markdown 代码块。JSON 字段：
- "skill_name": 技能名称，简短（不超过 20 个字）
- "tag": 分类，从以下值中选一个字符串：drama（短剧影视）、ecommerce（电商营销）、creative（创意设计）、social（社媒内容）、others（其他）
- "description": 技能简介（不超过 120 字），说明适用场景、输入条件和最终产出
- "instruction": 技能指令（Markdown 格式，至少 300 字），写给后续在画布中使用该技能的 AI 模型阅读，必须包含：角色设定、输入与约束、分步执行流程、检查清单、输出格式
用户的想法：
{{{IDEA}}}`;
export function buildSkillDraftPrompt(idea: string): string {
return SKILL_DRAFT_PROMPT.replace("{{{IDEA}}}", idea.trim());
}
const MAX_NAME_LENGTH = 80;
const MAX_DESCRIPTION_LENGTH = 500;
@@ -36,7 +23,7 @@ export function parseSkillDraft(text: string): Partial<SkillDraft> {
const raw = JSON.parse(trimmed.slice(start, end + 1)) as Record<string, unknown>;
if (raw && typeof raw === "object") {
return {
skill_name: String(raw.skill_name ?? "").trim().slice(0, MAX_NAME_LENGTH),
skillName: String(raw.skillName ?? raw.skill_name ?? "").trim().slice(0, MAX_NAME_LENGTH),
description: String(raw.description ?? "").trim().slice(0, MAX_DESCRIPTION_LENGTH),
instruction: String(raw.instruction ?? raw.instructions ?? "").trim(),
tag: typeof raw.tag === "string" && raw.tag.trim() ? raw.tag.trim() : undefined,
@@ -49,7 +36,7 @@ export function parseSkillDraft(text: string): Partial<SkillDraft> {
// 回退：首行作为名称，正文作为简介与指令，用户随后可自由编辑
const firstLine = trimmed.split(/\r?\n/)[0]?.trim().slice(0, 30) || "未命名技能";
return {
skill_name: firstLine,
skillName: firstLine,
description: trimmed.slice(0, MAX_DESCRIPTION_LENGTH),
instruction: trimmed,
};
@@ -59,8 +46,13 @@ export async function generateSkillDraft(idea: string, config: AiConfig, signal?
const generationConfig = buildGenerationConfig(config, undefined, "text");
const result = await runBackendGenerationTask({
mode: "text",
prompt: buildSkillDraftPrompt(idea),
prompt: promptTemplateTaskPlaceholder("技能草稿"),
config: generationConfig,
metadata: {
source: "skill-draft",
promptTemplateOperation: PromptTemplateOperation.SkillDraft,
promptTemplateVariables: { 用户想法: idea.trim() },
},
signal,
});
return parseSkillDraft(result.text || "");

86
web/src/lib/creation-text-replay.ts
View File

@@ -1,86 +0,0 @@
import { appendTaskTextDelta, completeTextReplayTask, createGenerationTask } from "@/services/api/task-center";
import type { AiConfig } from "@/stores/use-config-store";
/**
* text-replay 前端发布器：为前端直连模型生成的文本，在后端维护一个
* text_replay 任务作持久化存档 + 回放。
*
* - start()：创建后端 text_replay 任务（不排队、不计费），拿到 taskId；
* - publish(fullText)：按阈值节流地把新增片段通过 appendTaskTextDelta 上报
* （生成中途刷新页面也能读到已生成部分）；
* - finish(finalText)：把最终正文写入任务并置为 succeeded，queryTaskTextReplay
* 即可读到 finalText。
*
* 全部失败静默降级：文本直连生成是主流程，后端持久化只是增强，失败不阻塞生成。
*/
export function createTextReplayPublisher(config: AiConfig, prompt: string) {
let taskId: string | null = null;
let lastLen = 0;
let buffered = "";
let flushTimer: ReturnType<typeof setTimeout> | null = null;
let closed = false;
// 增量累积到该阈值才落一次库，避免逐 token 高频写。
const FLUSH_THRESHOLD = 2048;
const flush = (force = false) => {
if (flushTimer) {
clearTimeout(flushTimer);
flushTimer = null;
}
if (closed || !taskId) return;
if (buffered.length >= FLUSH_THRESHOLD || force) {
const chunk = buffered;
buffered = "";
if (chunk.trim()) void appendTaskTextDelta(taskId, chunk).catch(() => {});
} else if (buffered) {
flushTimer = setTimeout(() => flush(true), 1000);
}
};
const start = async () => {
try {
const task = await createGenerationTask({
type: "text",
prompt: prompt.trim() || "文本创作",
model: config.model,
input: {
mode: "text",
replay: true,
prompt: prompt.trim() || "文本创作",
config: {
model: config.model,
baseUrl: config.baseUrl,
apiKey: config.apiKey,
apiFormat: config.apiFormat,
},
},
});
taskId = task?.id || null;
} catch {
taskId = null; // 降级：无持久化，不影响主流程
}
};
const publish = (fullText: string) => {
if (!taskId) return;
if (fullText.length <= lastLen) return;
buffered += fullText.slice(lastLen);
lastLen = fullText.length;
flush();
};
const finish = (finalText: string) => {
closed = true;
if (flushTimer) {
clearTimeout(flushTimer);
flushTimer = null;
}
if (!taskId) return;
void appendTaskTextDelta(taskId, buffered).catch(() => {});
if (finalText && finalText.trim()) {
void completeTextReplayTask(taskId, finalText).catch(() => {});
}
};
return { start, publish, finish };
}

30
web/src/lib/media-metadata.ts
View File

@@ -1,7 +1,5 @@
// 媒体时长探测（纯前端工具，M3.4）。
// 上传视频/音频前用 <video>/<audio> 元数据读取真实时长（毫秒），
// 随上传 meta 传给后端存 Resource.DurationMs → 资产 summary 透出 → 时间线片段真实时长。
// 解码能力仅用于读取 duration，不需要能播放编码。
// 媒体时长探测工具：上传前通过 <video>/<audio> 元数据读取真实时长（毫秒）。
// 结果随上传 meta 传给后端，供资源摘要和时间线片段使用；这里不负责验证完整播放能力。
const PROBE_TIMEOUT_MS = 8000;
@@ -9,31 +7,37 @@ function isProbeable(file: File): boolean {
return /^video\//.test(file.type) || /^audio\//.test(file.type);
}
/**
* 返回可确认的媒体时长；非音视频、浏览器不支持解码、元数据错误或超时都返回 undefined。
* 这是上传附加信息而不是写入前置条件，调用方必须允许缺少时长，但不能编造默认时长。
*/
export async function probeMediaDurationMs(file: File): Promise<number | undefined> {
if (!isProbeable(file)) return undefined;
const url = URL.createObjectURL(file);
const el: HTMLVideoElement | HTMLAudioElement = /^video\//.test(file.type)
? document.createElement("video")
: document.createElement("audio");
let timeoutId: number | undefined;
el.preload = "metadata";
el.muted = true; // video 自动加载元数据在某些浏览器需要非 autoplay；muted 仅避免噪音
// 某些浏览器只有在静音时才允许视频元素主动加载元数据；这里只禁止声音，不触发自动播放。
el.muted = true;
try {
const timeout = new Promise<never>((_, reject) => {
window.setTimeout(() => reject(new Error("probe timeout")), PROBE_TIMEOUT_MS);
});
const meta = new Promise<number>((resolve) => {
const durationMs = await new Promise<number>((resolve, reject) => {
timeoutId = window.setTimeout(() => reject(new Error("媒体元数据读取超时")), PROBE_TIMEOUT_MS);
el.onloadedmetadata = () => {
const d = el.duration;
resolve(Number.isFinite(d) && d > 0 ? Math.round(d * 1000) : 0);
const durationSeconds = el.duration;
resolve(Number.isFinite(durationSeconds) && durationSeconds > 0 ? Math.round(durationSeconds * 1000) : 0);
};
el.onerror = () => resolve(0);
el.src = url;
});
el.src = url;
const durationMs = await Promise.race([meta, timeout]);
return durationMs > 0 ? durationMs : undefined;
} catch {
return undefined;
} finally {
if (timeoutId !== undefined) window.clearTimeout(timeoutId);
el.onloadedmetadata = null;
el.onerror = null;
el.removeAttribute("src");
el.load();
URL.revokeObjectURL(url);

10
web/src/lib/plugins/builtin/editor/editor-ai-assistant.tsx
View File

@@ -1,10 +1,6 @@
// AI 编辑助手（editor-shell 预设插件贡献 ai-assistant 插槽，M3.8 注册；入口在编辑器顶部工具按钮浮层）。
// M6.3：对话面板 —— 时间线摘要注入 system prompt，LLM 输出受 ai-command-schema 约束；
// ≤3 条命令直执行，>3 条先预览确认（ADR-0007：AI 编辑交互=预设插件且命令受约束）。
// 命令逐条 dispatch；宿主 fail-closed：无效命令被 registry 拒绝并写入 saveError（顶部提示条），
// 面板汇报为中性“已提交”，执行结果以时间线实际变化为准。
// 模型显式选择：不再盲信 config.model 默认值（历史默认可能是图像模型导致文本请求失败），
// 面板顶部提供文本能力模型下拉，请求时同时覆写 config.model/textModel，避免默认模型错误。
// 时间线摘要作为上下文发送给文本模型，模型输出必须通过 ai-command-schema 校验。
// 小批命令直接进入统一 dispatch，大批命令先由用户确认；宿主拒绝无效命令并显示 saveError。
// 请求显式携带用户选择的文本模型，不能沿用可能指向图片模型的全局默认值。
import { useRef, useState, useEffect } from "react";
import { Bot, Loader2, RotateCcw, Send, ShieldAlert } from "lucide-react";
import { requestImageQuestion } from "@/services/api/image";

3
web/src/lib/plugins/builtin/editor/editor-asset-ingest.tsx
View File

@@ -1,4 +1,3 @@
// 资产导入（editor-shell 预设插件贡献 asset-ingest 插槽，M3.4）。
// 展示项目资产库，按来源区分「本地上传」与「项目素材」；点击资产 → makeClipFromAsset
// → dispatch addClip 加入时间线（添加到匹配 kind 的轨道末尾）。资产是
// "仅时间线作用域"直连媒体（nodeId=asset:<id>）。导入链路：uploadResourceFile →
@@ -111,7 +110,7 @@ function AssetThumb({ asset }: { asset: ProjectAsset }) {
);
}
/** 尝试用 <video> 抓取视频首帧作为封面 dataURL；失败返回 null（保持图标占位）。 */
/** 尝试用 <video> 抓取视频首帧作为封面 dataURL；失败时保留媒体类型图标。 */
function useVideoPoster(url: string | null): string | null {
const [poster, setPoster] = useState<string | null>(null);
useEffect(() => {

7
web/src/lib/plugins/builtin/editor/editor-export.tsx
View File

@@ -1,8 +1,5 @@
// 导出（editor-shell 预设插件贡献 export-renderer 插槽，M3.7）。
// M4.2：默认提交后端渲染任务（POST /timeline/renders，ffmpeg 服务端合成，
// 产物落资源存储，ResultJSON 返回 resourceId 供内嵌预览/下载）；
// ffmpeg.wasm 本地导出保留为离线/降级路径。
// §3.1 引用契约：缺失 nodeId 源（节点已删除）按计划跳过，成片不因悬空引用失败。
// 默认由后端 ffmpeg 渲染并把产物写入资源存储；浏览器 ffmpeg.wasm 只承担离线降级。
// 构建渲染计划时跳过已失去媒体来源的片段，避免悬空 nodeId 阻断其余有效片段导出。
import { useMemo, useState } from "react";
import { Download, Loader2, PackageOpen, Server } from "lucide-react";

4
web/src/lib/plugins/builtin/editor/editor-inspector.tsx
View File

@@ -1,7 +1,5 @@
// 片段检查器（editor-shell 预设插件贡献 inspector 插槽，M3.3）。
// 读取选中片段（store.selectedClipId），属性编辑经 setClipProperty 命令入队
// （可撤销、可回放、进黄金文件语义）；无选中时显示项目概览。
// 视觉参照 Concat inspector：分组小标题 + 紧凑控件 + sunken 滑块，token 随主题联动。
// 并进入统一撤销历史；检查器不直接修改 project，避免绕过命令校验和持久化。
import { useEffect, useState } from "react";

7
web/src/lib/plugins/builtin/editor/editor-preview-monitor.tsx
View File

@@ -1,7 +1,6 @@
// 预览监视器（editor-shell 预设插件贡献 preview-renderer 插槽，M3.2）。
// 浏览器内近似预览：真实媒体帧（storageKey → resolveMediaUrl）+ 播放头 + 时间码。
// 近似渲染与导出（M3.7 的 buildTimelineRenderPlan）共享同一条"片段→媒体"解析路径，
// 但这里只做时序呈现，不承诺像素级预览——像素级交给导出任务（M4）。
// 预览与 buildTimelineRenderPlan 共享“片段→媒体”解析规则，但浏览器预览只保证时序一致；
// 最终画面、编码和滤镜结果以后端导出任务为准。
//
// 播放模型：时间线 transport（store.transportMs）与监视器共享。
// - 本地 playbackRef 以 rAF 逐帧推进（60fps 时间码），节流回写 store（~80ms），
@@ -461,7 +460,7 @@ export function EditorPreviewMonitor() {
<div className="max-w-xs text-xs leading-relaxed text-white/70">
{project && project.clips.length > 0
? activeClip
? "当前时间点片段无媒体源（占位/字幕/音频）"
? "当前时间点没有可显示画面（字幕或音频片段）"
: "播放头位于空白处"
: "时间线暂无片段，导入素材后在此预览"}
</div>

12
web/src/lib/plugins/builtin/editor/editor-shell.tsx
View File

@@ -13,9 +13,9 @@ import { EditorAiAssistant } from "./editor-ai-assistant";
const manifest: PluginManifestV2 = {
apiVersion: "yingce.plugin/v2",
id: "editor-shell",
name: "剪辑工作台（壳）",
name: "剪辑工作台",
version: "0.1.0",
description: "编辑器预设插件垂直切片：注册时间线面板插槽，验证 v2 插件插槽注册→渲染链路。",
description: "注册时间线、预览、检查器、素材、字幕、转写、导出和 AI 编辑八个工作台插槽。",
author: "影策团队",
surfaces: ["fullscreen"],
permissions: ["timeline.read", "timeline.command", "export.run"],
@@ -42,55 +42,47 @@ export const editorShellPlugin: RegisteredPlugin = {
registerPlugin(editorShellPlugin);
// M2.4：插槽渲染函数 = 真实时间线面板（命令状态机 + 手势）。
registerEditorSlot({
pluginId: manifest.id,
slot: "timeline-panel",
render: () => <EditorTimelinePanel />,
});
// M3.2：插槽渲染函数 = 预览监视器（浏览器内近似预览 + 播放头）。
registerEditorSlot({
pluginId: manifest.id,
slot: "preview-renderer",
render: () => <EditorPreviewMonitor />,
});
// M3.3：插槽渲染函数 = 片段检查器（属性编辑入队可撤销）。
registerEditorSlot({
pluginId: manifest.id,
slot: "inspector",
render: () => <EditorInspector />,
});
// M3.4：插槽渲染函数 = 素材库（点击资产 → addClip 入轨）。
registerEditorSlot({
pluginId: manifest.id,
slot: "asset-ingest",
render: () => <EditorAssetIngest />,
});
// M3.5：插槽渲染函数 = 字幕工具（SRT 导入导出 + 从节点重建字幕快照）。
registerEditorSlot({
pluginId: manifest.id,
slot: "subtitle-tool",
render: () => <EditorSubtitleTools />,
});
// M3.6：转写（mock ASR → 字幕轨道，M4 接后端任务客户端）。
registerEditorSlot({
pluginId: manifest.id,
slot: "transcription-provider",
render: () => <EditorTranscription />,
});
// M3.7：导出（渲染计划 + ffmpeg.wasm 降级，M4 接后端任务）。
registerEditorSlot({
pluginId: manifest.id,
slot: "export-renderer",
render: () => <EditorExport />,
});
// M3.8：AI 助手占位（M6 补实现 ai-command-schema + timeline-summary）。
registerEditorSlot({
pluginId: manifest.id,
slot: "ai-assistant",

10
web/src/lib/plugins/builtin/editor/editor-subtitle-tools.tsx
View File

@@ -1,11 +1,7 @@
// 字幕工具（editor-shell 预设插件贡献 subtitle-tool 插槽，M3.5）。
// SRT 导入导出（复用 srt-parser）+ 「从节点重建字幕片段」命令（§3.1 快照契约：
// SRT 导入导出（复用 srt-parser）+ 「从节点重建字幕片段」命令（快照契约：
// 节点 subtitleEntries 权威 → 重建时间线字幕快照，替换过期快照，单向显式同步）。
// AI 字幕高亮与关键词标注已在 M6 落地，入口在画布「字幕」弹窗
//（canvas-subtitle-dialog.tsx，走 ai.text 权限门禁）：对权威节点字幕条目生成
// 高亮/关键词，并随字幕条目渲染。此处（时间线字幕工具）字幕片段只是快照，
// 编辑器预览尚无字幕文本渲染层，高亮视觉无处挂载，故不在此重复接 AI；
// 待 M4 数据源接入、预览可渲染字幕文本后，再将高亮打通到时间线片段。
// 时间线字幕片段是节点字幕的只读快照；AI 高亮只写权威节点字幕。
// 当前预览不渲染字幕高亮，因此本工具不重复发起 AI 请求，避免产生两份互相漂移的结果。
import { useMemo, useState } from "react";

9
web/src/lib/plugins/builtin/editor/editor-timeline-panel.tsx
View File

@@ -1,8 +1,7 @@
// 时间线面板（editor-shell 预设插件贡献，M2.4 手势 echo 最小件）。
// 渲染链路：store.project（当前命令状态）→ 轨道/片段绝对定位 → 拖拽手势
// （moveClip / trimClip）经 previewGesture 逐帧预览、commitGesture 一次性入历史。
// 手势数学与渲染同源（同一 pxPerMs），保证拖拽所见即所得。
// 撤销/重做与保存状态移入宿主顶栏（Concat 主菜单区），本面板保留缩放与片段统计。
// 撤销、重做和保存由编辑器宿主统一处理，本面板只负责时间线交互和可视状态。
import { useEffect, useRef, useState } from "react";
import { Eye, EyeOff, Film, Magnet, Maximize, MousePointer2, Music2, Plus, Scissors, Slice, Subtitles, Trash2, Volume2, VolumeX, X, ZoomIn, ZoomOut } from "lucide-react";
@@ -42,7 +41,7 @@ const TOOL_FG_ACTIVE = "bg-[var(--director-dock-active-surface)] text-[var(--dir
type GestureMode = "move" | "trim-start" | "trim-end" | null;
// 与时间线相关的场景：playheadMs 是标尺交互的临时坐标，随缩放同步换算。
// 工具模式只影响指针手势解释；时间线数据修改仍必须通过命令分发。
type TimelineTool = "select" | "razor";
type GestureState = {
@@ -300,7 +299,7 @@ export function EditorTimelinePanel() {
{project.clips.length === 0 ? (
<div className="flex h-full flex-col items-center justify-center gap-2 text-[var(--director-dock-fg)]">
<Scissors className="size-8" />
<p className="text-sm">时间线暂无片段，从素材库拖入或后续接入画布节点</p>
<p className="text-sm">从素材库加入素材后即可开始剪辑</p>
</div>
) : (
<div className="relative min-w-full" style={{ width: contentWidth }}>
@@ -399,7 +398,7 @@ function TimelineRuler({
};
return (
<div className="sticky top-0 z-10 flex h-6 shrink-0 w-full items-end border-b border-[var(--director-sequencer-border)] bg-[var(--director-sequencer-surface-raised)]">
{/* 左列占位与 TrackRow 轨道标签列（w-48）同宽，sticky 跟随横向滚动，保证 0ms 刻度与片段区起点对齐 */}
{/* 左侧空白列与轨道标签列同宽，保证 0ms 刻度和片段绘制区域使用同一坐标原点。 */}
<div className="sticky left-0 z-10 w-48 shrink-0 self-stretch border-r border-[var(--director-sequencer-border)] bg-[var(--director-sequencer-surface-raised)]">
{/* 当前播放头时码，跟随拖动实时刷新 */}
<span className="absolute bottom-1 right-2 text-[10px] tabular-nums text-[var(--director-danger)]">{formatTimelineTime(playheadMs)}</span>

6
web/src/lib/plugins/builtin/editor/editor-transcription.tsx
View File

@@ -1,7 +1,5 @@
// 字幕转写（editor-shell 预设插件贡献 transcription-provider 插槽，M3.6）。
// M4 后端转写任务（whisper.cpp 本地执行）：创建 POST /timeline/transcriptions
// → waitForGenerationTask 轮询进度 → 结果 segments 按 ADR-0004 协议以
// SrtEntry[] 落字幕轨道（rebuildSubtitleClips 重建快照），UI 契约与 M3.6 mock 一致。
// 转写由后端 whisper.cpp 任务执行；前端只提交资源 ID、观察任务状态，
// 并将成功结果转换为 SrtEntry[]，通过 rebuildSubtitleClips 原子替换字幕轨道快照。
import { useEffect, useMemo, useState } from "react";
import { AudioLines, Loader2 } from "lucide-react";

1
web/src/lib/plugins/builtin/editor/index.ts
View File

@@ -1,4 +1,3 @@
// 编辑器预设插件集合入口（M3.8）。
// editor-shell 单个预设插件贡献全部 8 种 editorSlots；此处聚合导出，
// builtin/index.ts 的静态注册经 editor-shell 模块副作用完成。

12
web/src/lib/plugins/plugin-permission-check.ts
View File

@@ -1,12 +1,12 @@
// 编辑器插件执行期权限校验（M5，fail-closed）。
// v2 插件贡献 editorSlots / 代发 timeline 命令时，其 manifest.permissions 必须覆盖
// 对应能力域；缺失即拒绝渲染/执行。v1 插件不贡献 editorSlots，天然不受此域约束。
// 本模块只做纯判定，接入点在插槽渲染（SlotStack）与宿主命令代发通道。
// 编辑器插件执行期权限校验，默认拒绝未知插件和缺失权限。
// v2 插件贡献 editorSlots 或代发 timeline 命令时，manifest.permissions 必须覆盖对应能力域；
// v1 插件不贡献 editorSlots，因此不经过这条插槽权限路径。本模块只做纯判定，
// 接入点分别位于插槽渲染（SlotStack）和宿主命令代发通道。
import { getRegisteredPlugin } from "@/lib/plugins/plugin-registry";
import type { EditorPluginPermission, EditorSlotKind } from "@/lib/plugins/plugin-types";
/** 每个编辑器插槽所需的最小权限域（M5.2 表驱动：新插槽必须补表，否则 fail-closed 拒渲染）。 */
/** 每个编辑器插槽所需的最小权限域；新增插槽必须补表，否则会因找不到权限而拒绝渲染。 */
export const EDITOR_SLOT_REQUIRED_PERMISSION: Record<EditorSlotKind, EditorPluginPermission> = {
"timeline-panel": "timeline.command",
"preview-renderer": "timeline.read",
@@ -51,7 +51,7 @@ export function pluginMayRenderEditorSlot(pluginId: string, slot: EditorSlotKind
return { allowed: true };
}
/** 宿主命令代发通道的权限断言：插件未声明 permission 时抛错（M6 AI 命令、第三方代发复用）。 */
/** 宿主命令代发通道的权限断言：未注册插件或未声明目标权限时抛错，调用方必须阻止命令执行。 */
export function assertEditorPermission(pluginId: string, permission: EditorPluginPermission): void {
const permissions = pluginPermissions(pluginId);
if (permissions === null) {

127
web/src/lib/prompts/catalog.ts Normal file
View File

@@ -0,0 +1,127 @@
import type { CanvasGenerationMode } from "@/types/canvas";
import type { SrtEntry } from "@/types/timeline";
export type CanvasPromptPreset = {
id: string;
name: string;
description: string;
prompt: string;
modes: CanvasGenerationMode[];
source: "builtin" | "skill";
};
export const CANVAS_BUILTIN_PRESETS: CanvasPromptPreset[] = [
{
id: "character-sheet",
name: "角色设定图",
description: "正面、侧面、背面与表情参考，锁定角色一致性",
prompt: "生成角色设定图：保持同一角色身份、五官、发型、服装和体态一致，包含正面、侧面、背面和关键表情参考，背景简洁，便于后续镜头复用。",
modes: ["image"],
source: "builtin",
},
{
id: "multi-angle",
name: "多机位视角",
description: "围绕同一主体生成连续、可衔接的机位变化",
prompt: "围绕同一主体设计多机位画面，保持人物、服装、场景和光线一致，分别给出远景、全景、中景、近景、特写、侧面、背面和俯拍视角，镜头之间具有连续性。",
modes: ["image", "video"],
source: "builtin",
},
{
id: "next-shot",
name: "画面推演",
description: "推演当前画面的前后动作与镜头衔接",
prompt: "基于当前画面推演下一个连续镜头：保持角色和场景一致，明确主体接下来的动作、视线、环境变化、镜头运动和自然衔接方式，不要跳变构图或身份。",
modes: ["image", "video"],
source: "builtin",
},
{
id: "story-beats",
name: "连续镜头",
description: "将短剧情拆成可生成的连续镜头节拍",
prompt: "把这段内容拆成连续镜头节拍。每个镜头写清主体动作、景别、构图、机位、运镜、光线、情绪和与前后镜头的衔接，并保持角色、场景和道具一致。",
modes: ["text", "image", "video"],
source: "builtin",
},
{
id: "cinematic-light",
name: "电影光影优化",
description: "保留内容，优化真实光线、层次和融合感",
prompt: "保留主体身份、动作和原始构图，优化为真实电影摄影光线：明确主光方向、环境反射、阴影层次、肤色和背景融合，降低塑料感与过度锐化，不改变画面内容。",
modes: ["image", "video"],
source: "builtin",
},
{
id: "video-prompt",
name: "视频提示词优化",
description: "整理为模型更容易执行的时序化镜头指令",
prompt: "将当前要求改写为结构化视频提示词，按时间顺序描述开场画面、主体动作、镜头运动、环境变化、声音和结束画面；消除冲突指令，保留所有关键约束。",
modes: ["text", "video"],
source: "builtin",
},
];
export const IMAGE_PROMPT_REVERSE = `请根据参考图片反推一段适合用于 AI 生图的提示词。
要求：
1. 只输出提示词正文，不要解释。
2. 覆盖主体、构图、风格、光线、色彩、材质、镜头和氛围。
3. 尽量写成可直接用于生图模型的完整提示词。`;
export function workflowStarterPrompt(kind: "character_cards" | "character_three_view" | "storyboard_video", title: string, workflowTitle: string) {
if (kind === "character_cards") return `请基于「${workflowTitle}」拆分主要角色，并为每个角色生成可用于后续创作的角色图片卡片：外观、服饰、身份、性格和视觉辨识点。`;
if (kind === "character_three_view") return `请基于上游角色卡片生成「${title}」：同一角色的正面、侧面、背面三视图，保持服饰、发型、道具和比例一致。`;
return `请基于上游角色三视图，为「${workflowTitle}」制作分镜剧情视频方案：包含镜头顺序、景别、动作、节奏和画面连续性。`;
}
export function buildTextRewritePrompt(sourceText: string, instruction: string) {
return `请根据要求修改以下文本。\n\n原文：\n${sourceText}\n\n修改要求：\n${instruction}`;
}
export function buildVideoOperationPrompt(operation: string, prompt: string, operationLabel: string) {
if (operation === "compare_versions") return `请对以下视频结果版本做对比分析，输出推荐版本、差异点和修改建议：\n${prompt}`;
return `视频编辑任务：${operationLabel}\n创作要求：${prompt}`;
}
export function buildSubtitleHighlightSystemPrompt() {
return `你是一个视频字幕关键词高亮助手。请输出严格 JSON。
任务目标：
- 为每条字幕判断是否需要高亮
- 每条字幕最多高亮 1 段
- 只返回最值得高亮的一个关键词或短语
- start 和 end 使用 JavaScript 字符串下标
- end 为 exclusive
- 没有明确重点时返回 shouldHighlight=false
高亮标准：
- 优先高亮结论词、数字、身份词、首次/突破词、强反差词
- 不要高亮虚词、口头语、整句长短语
边界约束：
- 不负责颜色、字号、位置、动效
- 只返回 highlights 数组
- highlightText 必须严格等于原字幕文本切片结果`;
}
export function buildSubtitleHighlightUserMessage(entries: SrtEntry[]) {
const lines = entries.map((entry) => {
return `entryIndex=${entry.index} | startMs=${entry.startMs} | endMs=${entry.endMs} | text=${entry.text}`;
});
return `请根据以下字幕和上下文，为每条字幕判断是否需要高亮，并返回 JSON：
{
"highlights": [
{
"entryIndex": 1,
"shouldHighlight": true,
"highlightText": "示例",
"start": 0,
"end": 2
}
]
}
字幕列表：
${lines.join("\n")}`;
}

22
web/src/lib/prompts/index.ts Normal file
View File

@@ -0,0 +1,22 @@
export {
PromptTemplateOperation,
promptTemplateTaskPlaceholder,
type PromptTemplateOperationId,
} from "./operations";
export {
CANVAS_BUILTIN_PRESETS,
IMAGE_PROMPT_REVERSE,
buildSubtitleHighlightSystemPrompt,
buildSubtitleHighlightUserMessage,
buildTextRewritePrompt,
buildVideoOperationPrompt,
workflowStarterPrompt,
type CanvasPromptPreset,
} from "./catalog";
export {
parseGeneratedStory,
shortDramaOutlineVariables,
storyTitleFromAnswer,
type GeneratedShortDramaChapter,
type GeneratedShortDramaOutline,
} from "./short-drama-outline";

17
web/src/lib/prompts/operations.ts Normal file
View File

@@ -0,0 +1,17 @@
export const PromptTemplateOperation = {
ChapterAssetsExtract: "chapter_assets_extract",
CharacterExtract: "character_extract",
CharacterTurnaround: "character_turnaround",
StoryboardPlan: "storyboard_plan",
StoryboardRepair: "storyboard_repair",
StoryboardFirstFrame: "storyboard_first_frame",
StoryboardVideo: "storyboard_video",
ShortDramaOutline: "short_drama_outline",
SkillDraft: "skill_draft",
} as const;
export type PromptTemplateOperationId = (typeof PromptTemplateOperation)[keyof typeof PromptTemplateOperation];
export function promptTemplateTaskPlaceholder(label: string) {
return `使用当前启用的${label}模板。`;
}

57
web/src/lib/prompts/short-drama-outline.ts Normal file
View File

@@ -0,0 +1,57 @@
export type GeneratedShortDramaChapter = {
title: string;
content: string;
};
export type GeneratedShortDramaOutline = {
title: string;
synopsis: string;
chapters: GeneratedShortDramaChapter[];
};
export function parseGeneratedStory(answer: string): GeneratedShortDramaOutline {
const cleaned = answer
.replace(/```json/gi, "")
.replace(/```/g, "")
.trim();
const match = cleaned.match(/\{[\s\S]*\}/);
const payload = match ? JSON.parse(match[0]) as Record<string, unknown> : {};
const title = String(payload.title || "").trim();
const synopsis = String(payload.synopsis || "").trim();
const chapters = Array.isArray(payload.chapters)
? payload.chapters
.map((chapter: unknown) => {
const item = typeof chapter === "object" && chapter ? chapter as Record<string, unknown> : {};
return { title: String(item.title || "").trim(), content: String(item.content || "").trim() };
})
.filter((chapter: GeneratedShortDramaChapter) => chapter.title && chapter.content)
: [];
return { title: title || storyTitleFromAnswer(answer), synopsis, chapters };
}
export function storyTitleFromAnswer(answer: string) {
const line = answer.split(/\r?\n/).find((item) => item.trim());
return line ? line.trim().slice(0, 24) : "AI 生成短剧";
}
export function shortDramaOutlineVariables(input: {
story: string;
chapterCount: string;
structure: string;
wordCount: string;
perspective: string;
tone: string;
characterScale: string;
chapterLength: string;
}) {
return {
用户故事: input.story,
章节数量: input.chapterCount,
叙事结构: input.structure,
每章字数: input.wordCount,
叙事视角: input.perspective,
整体基调: input.tone,
角色规模: input.characterScale,
章节篇幅: input.chapterLength,
};
}

23
web/src/lib/timeline/ai-command-schema.ts
View File

@@ -1,7 +1,6 @@
// AI 编辑命令契约（ADR-0007 / Runbook M6.1）：宿主校验 + LLM 提示词约束的双契约。
// 本文件不复制 handler 约束——命令合法性硬守门在 M2.1 命令注册表（fail-closed），
// 这里只做：(1) LLM 输出 JSON 的结构解析与批级预检（dry-run 走同一注册表，零重复），
// (2) 生成喂给 LLM 的 op 契约（op 集合与注册表黄金同步，见测试）。
// AI 编辑命令的边界层：同一份注册表同时约束模型输出和宿主执行，避免提示词与真实能力列表分叉。
// 本文件只负责 JSON 结构解析、批量预检和提示词契约生成；命令的参数合法性仍由编辑命令注册表统一判定。
// 预检使用与正式执行相同的纯函数注册表，保证“模型看见的能力”和“宿主实际允许的能力”一致。
import { getEditorCommandRegistry, type EditCommand } from "@/lib/timeline/editor-commands";
import type { TimelineProject } from "@/types/timeline";
@@ -79,8 +78,8 @@ function assertPlanShape(raw: unknown): asserts raw is AiCommandPlan {
}
/**
* 解析 LLM 文本输出为 AiCommandPlan；结构非法时抛 AiCommandPlanError（供 UI 展示并回填模型）。
* commands 为空数组表示「只读问答 / 无法用现有命令完成」，面板直接展示 reasoning 作为回答。
* 解析模型文本为 AiCommandPlan；结构非法时抛 AiCommandPlanError，调用方可将明确原因展示给用户或回传模型修正。
* commands 为空数组表示只读问答或当前能力无法完成，调用方不应把它当成“执行成功的空修改”。
*/
export function parseAiCommandPlan(text: string): AiCommandPlan {
let raw: unknown;
@@ -96,8 +95,8 @@ export function parseAiCommandPlan(text: string): AiCommandPlan {
/**
* 批级预检：把整批命令按序 dry-run 到当前时间线（immutable 纯函数，无副作用）。
* 任何一条触发注册表 fail-closed（未知 op / 非法 payload / 引用不存在）即整批拒绝，
* 返回首个出错命令的下标与消息——ADR-0007「任一非法→整批拒绝，错误回填 LLM 修正」。
* 任何一条触发注册表的拒绝规则（未知 op、非法 payload 或引用不存在）即整批拒绝，
* 返回首个失败命令的位置和原因；正式执行不得绕过这一步或只提交部分命令。
*/
export function validateAiCommandBatch(project: TimelineProject, plan: AiCommandPlan): AiCommandBatchVerdict {
const registry = getEditorCommandRegistry();
@@ -114,13 +113,13 @@ export function validateAiCommandBatch(project: TimelineProject, plan: AiCommand
}
// ---------------------------------------------------------------------------
// LLM 提示词契约（ADR-0007：schema 同时用于宿主校验与模型输出约束）
// 模型提示词契约：schema 同时用于宿主校验与模型输出约束，具体交互决策见 ADR-0007。
// ---------------------------------------------------------------------------
/** AI 命令 schema 版本（Runbook M6.1）：提示词契约/解析规则变更时递增，供 golden 与缓存失效对齐。 */
/** AI 命令 schema 版本；提示词契约或解析规则变化时递增，用于 golden 测试和缓存失效对齐。 */
export const AI_COMMAND_SCHEMA_VERSION = 1;
/** 单次 AI 编辑最多允许的命令条数（ADR-0007：防止模型一次输出失控编辑）。 */
/** 单次 AI 编辑最多允许的命令条数，防止一次模型输出造成不可控的大范围变更。 */
export const AI_EDITING_MAX_COMMANDS = 8;
/** 12 个黄金 op 的 LLM 可读 payload 契约；op 集合与注册表黄金同步（见测试）。 */
@@ -203,7 +202,7 @@ export const AI_EDITING_OUTPUT_FORMAT = `输出要求：
const AI_EDITING_OUTPUT_EXAMPLE = `示例：
{ "reasoning": "把第一个视频片段移到第 5 秒", "commands": [ { "op": "moveClip", "payload": { "id": "clip-a", "startMs": 5000 } } ] }`;
/** 组装 AI 编辑系统提示词：时间线摘要 + op 契约 + 输出约束（M6.3 面板喂给模型）。 */
/** 组装 AI 编辑系统提示词：时间线摘要、可用 op 契约和输出约束必须来自同一份注册表。 */
export function buildAiEditingSystemPrompt(timelineSummary: string): string {
const catalogLines = AI_EDITING_OP_CATALOG.map(
(entry) => `- ${entry.op}：${entry.desc}。payload：${entry.payload}`,

2
web/src/lib/timeline/asset-ingest.ts
View File

@@ -1,4 +1,4 @@
// 资产 → 时间线片段 的纯转换（M3.4 asset-ingest）。
// 资产到时间线片段的纯转换：只做类型映射和结构组装，不读取网络或修改编辑器状态。
// 资产是"仅时间线作用域"的直连媒体：nodeId 用 `asset:<id>` 前缀（不指向画布节点），
// 预览/导出从 directMedia 解析媒体，不回画布查节点。

2
web/src/lib/timeline/editor-commands.ts
View File

@@ -17,7 +17,7 @@ export type EditorCommandRegistry = {
};
// ---------------------------------------------------------------------------
// 黄金命令集合 payload 类型（首版一期能力，对应 Runbook M2.1）
// 内置命令的 payload 类型；注册表、AI 命令契约和黄金测试必须共享同一组 op。
// ---------------------------------------------------------------------------
export type AddClipPayload = { clip: TimelineClip };

36
web/src/lib/timeline/mock-transcription.ts
View File

@@ -1,36 +0,0 @@
// Mock 转写（M3.6）：M4 后端转写任务未就绪前，用确定性伪转写模拟 ASR 结果。
// 同一 title + durationMs 永远生成相同的 SrtEntry[]，便于测试与回归；
// M4 接真实任务客户端后本模块移除，UI 契约（SrtEntry[] 落字幕轨道）不变。
import type { SrtEntry } from "@/types/timeline";
const MOCK_CHUNK_MS = 1800;
const MOCK_GAP_MS = 200;
/** 从标题确定性生成一组模拟字幕条目（基于字符串 hash 稳定切分）。 */
export function mockTranscriptionEntries(title: string, durationMs: number): SrtEntry[] {
const text = title.trim() || "未命名素材";
const chunks = splitStable(text, 6);
const usable = Math.max(1, Math.floor(durationMs / (MOCK_CHUNK_MS + MOCK_GAP_MS)));
const count = Math.min(chunks.length, usable);
const step = durationMs / count;
const entries: SrtEntry[] = [];
for (let i = 0; i < count; i += 1) {
const startMs = Math.round(i * step);
const endMs = Math.min(durationMs, Math.round(startMs + MOCK_CHUNK_MS));
if (endMs <= startMs) break;
entries.push({ index: i, startMs, endMs, text: chunks[i % chunks.length] });
}
return entries;
}
/** 稳定切分：按可打印字符逐段取满 maxLen，确定性无随机。 */
function splitStable(text: string, maxLen: number): string[] {
const chars = Array.from(text);
const chunks: string[] = [];
for (let i = 0; i < chars.length; i += maxLen) {
chunks.push(chars.slice(i, i + maxLen).join(""));
}
return chunks;
}

53
web/src/lib/timeline/subtitle-highlight-ai.ts
View File

@@ -1,53 +0,0 @@
import type { SrtEntry } from "@/types/timeline";
export interface SubtitleHighlightLLMResult {
entryIndex: number;
shouldHighlight: boolean;
highlightText: string;
start: number;
end: number;
}
export function buildSubtitleHighlightSystemPrompt(): string {
return `你是一个视频字幕关键词高亮助手。请输出严格 JSON。
任务目标：
- 为每条字幕判断是否需要高亮
- 每条字幕最多高亮 1 段
- 只返回最值得高亮的一个关键词或短语
- start 和 end 使用 JavaScript 字符串下标
- end 为 exclusive
- 没有明确重点时返回 shouldHighlight=false
高亮标准：
- 优先高亮结论词、数字、身份词、首次/突破词、强反差词
- 不要高亮虚词、口头语、整句长短语
边界约束：
- 不负责颜色、字号、位置、动效
- 只返回 highlights 数组
- highlightText 必须严格等于原字幕文本切片结果`;
}
export function buildSubtitleHighlightUserMessage(entries: SrtEntry[]): string {
const lines = entries.map((entry) => {
return `entryIndex=${entry.index} | startMs=${entry.startMs} | endMs=${entry.endMs} | text=${entry.text}`;
});
return `请根据以下字幕和上下文，为每条字幕判断是否需要高亮，并返回 JSON：
{
"highlights": [
{
"entryIndex": 1,
"shouldHighlight": true,
"highlightText": "示例",
"start": 0,
"end": 2
}
]
}
字幕列表：
${lines.join("\n")}`;
}

2
web/src/lib/timeline/subtitle-highlight-runner.ts
View File

@@ -1,7 +1,7 @@
import { runBackendCanvasGenerationTask } from "@/lib/canvas/canvas-project-generation";
import type { AiConfig } from "@/stores/use-config-store";
import type { SrtEntry, SubtitleHighlight } from "@/types/timeline";
import { buildSubtitleHighlightSystemPrompt, buildSubtitleHighlightUserMessage } from "./subtitle-highlight-ai";
import { buildSubtitleHighlightSystemPrompt, buildSubtitleHighlightUserMessage } from "@/lib/prompts";
import { parseSubtitleHighlightResponse } from "./subtitle-highlight-service";
export interface SubtitleHighlightProgress {

8
web/src/lib/timeline/subtitle-highlight-service.ts
View File

@@ -1,6 +1,12 @@
import type { SrtEntry, SubtitleHighlight } from "@/types/timeline";
import { filterValidSubtitleHighlights } from "./subtitle-highlights";
import type { SubtitleHighlightLLMResult } from "./subtitle-highlight-ai";
interface SubtitleHighlightLLMResult {
entryIndex: number;
shouldHighlight: boolean;
highlightText: string;
start: number;
end: number;
}
function isSubtitleHighlightLLMResult(value: unknown): value is SubtitleHighlightLLMResult {
if (!value || typeof value !== "object") {

2
web/src/lib/timeline/timeline-summary.ts
View File

@@ -1,4 +1,4 @@
// 时间线确定性摘要（Runbook M6.2）：把 TimelineProject v2 压缩成喂给 LLM 的中文上下文。
// 时间线确定性摘要：把 TimelineProject 压缩成稳定、可控长度的中文上下文，供 AI 编辑和诊断使用。
// 同一输入永远产出同一输出（无时间戳/无随机），供命令引用 id 与 AI 面板上下文注入。
import type { TimelineClip, TimelineProject, TimelineTrack } from "@/types/timeline";

37
web/src/lib/user-session.ts
View File

@@ -1,5 +1,5 @@
import { getFeatureAvailability, type AuthSessionPayload } from "@/services/api/auth";
import { getModelCatalog, listLogicalModels, type CapabilitySpec, type ModelCatalogResponse, type OptionConstraint, type PublicChannelCatalog, type PublicLogicalModel } from "@/services/api/logical-models";
import { getModelCatalog, type CapabilitySpec, type ModelCatalogResponse, type OptionConstraint, type PublicChannelCatalog, type PublicLogicalModel } from "@/services/api/logical-models";
import { localForageStorage } from "@/lib/localforage-storage";
import { appQueryClient } from "@/lib/query-client";
import { scopedLocalStorage, setActiveUserScope } from "@/lib/user-scope";
@@ -61,17 +61,10 @@ export async function applyUserSession(payload: AuthSessionPayload) {
if (!persistedCreationPreferences) useCreationPreferencesStore.setState({ preferences: {} });
if (!persistedConfig) {
// 只有首次配置缺失时才生成能力推荐；已有配置中的空数组代表用户明确清空。
// 使用统一模型目录接口
const catalog = await getModelCatalog();
let channels: ModelChannel[] = [];
if (catalog.source === "frontend" && catalog.models) {
channels = managedModelChannels(catalog.models);
} else if (catalog.source === "system" && catalog.channels) {
channels = systemChannelModelChannels(catalog.channels);
}
const initialSystemConfig = {
...defaultConfig,
channels,
channels: modelCatalogChannels(catalog),
imageModels: undefined,
videoModels: undefined,
textModels: undefined,
@@ -79,13 +72,8 @@ export async function applyUserSession(payload: AuthSessionPayload) {
};
useConfigStore.getState().replaceConfig(normalizeConfigSnapshot({ config: initialSystemConfig }).config);
} else {
// 已有配置时也需要合并最新的系统渠道
const catalog = await getModelCatalog();
if (catalog.source === "frontend" && catalog.models) {
useConfigStore.getState().mergeSystemChannels(managedModelChannels(catalog.models));
} else if (catalog.source === "system" && catalog.channels) {
useConfigStore.getState().mergeSystemChannels(systemChannelModelChannels(catalog.channels));
}
useConfigStore.getState().mergeSystemChannels(modelCatalogChannels(catalog));
}
installRemoteUserDataAutoSync();
if (payload.user?.id) {
@@ -97,16 +85,21 @@ export async function applyUserSession(payload: AuthSessionPayload) {
}
export async function refreshSystemChannels() {
// 使用统一模型目录接口，根据 frontendModelsEnabled 自动返回前台模型或系统渠道模型
const catalog = await getModelCatalog();
useConfigStore.getState().mergeSystemChannels(modelCatalogChannels(catalog));
}
if (catalog.source === "frontend" && catalog.models) {
// 前台模型模式
useConfigStore.getState().mergeSystemChannels(managedModelChannels(catalog.models));
} else if (catalog.source === "system" && catalog.channels) {
// 系统渠道模式
useConfigStore.getState().mergeSystemChannels(systemChannelModelChannels(catalog.channels));
// 模型目录来源决定数据形状；这里统一做运行时收口，避免畸形响应被当成“空目录”写入用户配置。
function modelCatalogChannels(catalog: ModelCatalogResponse): ModelChannel[] {
if (catalog.source === "frontend") {
if (!Array.isArray(catalog.models)) throw new Error("模型目录响应缺少前台模型列表");
return managedModelChannels(catalog.models);
}
if (catalog.source === "system") {
if (!Array.isArray(catalog.channels)) throw new Error("模型目录响应缺少系统渠道列表");
return systemChannelModelChannels(catalog.channels);
}
throw new Error("模型目录响应来源无效");
}
function managedModelChannels(models: PublicLogicalModel[]) {