fix(canvas): 分镜视频生成 - 正确附加角色场景与声音资产及镜头台词

2026-08-30 17:54:23 +08:00
parent 37eb2b7e48
commit f2fe163bc7
9 changed files with 266 additions and 27 deletions
--- a/web/src/pages/projects/detail/workflow-production-workbench.tsx
+++ b/web/src/pages/projects/detail/workflow-production-workbench.tsx
@@ -118,7 +118,13 @@ export default function WorkflowProductionWorkbench(props: Props) {
    const [selectedSkillIds, setSelectedSkillIds] = useState<string[]>([]);
    const { skills: availableSkills, loading: skillsLoading } = useSkillRuntimeCatalog();
    const shotAssetReferenceContext = useMemo(() => buildShotAssetReferenceContext(detail, selectedShot?.id || ""), [detail, selectedShot?.id]);
-    const referenceByVersionId = useMemo(() => new Map((detail.shotReferences || []).filter((reference) => reference.shotId === selectedShot?.id && reference.role === "reference" && reference.status === "linked").map((reference) => [reference.assetVersionId, reference])), [detail.shotReferences, selectedShot?.id]);
+    const referenceByVersionId = useMemo(() => {
+        const references = (detail.shotReferences || []).filter((reference) => reference.shotId === selectedShot?.id && reference.role === "reference" && reference.status === "linked");
+        return new Map(references.flatMap((reference) => [
+            [reference.assetVersionId, reference] as const,
+            ...(reference.asset?.primaryVersionId ? [[reference.asset.primaryVersionId, reference] as const] : []),
+        ]));
+    }, [detail.shotReferences, selectedShot?.id]);
    const currentDurationSeconds = Number(watchedDuration || Math.max(0.5, (selectedShot?.durationMs || 3000) / 1000));
    const generationSeconds = String(Math.max(1, Math.round(currentDurationSeconds)));
    const selectedVideoProfile = generationCapability === "video" && selectedModel ? modelCapabilityConfigFor(effectiveConfig, selectedModel).video : undefined;
@@ -324,7 +330,7 @@ export default function WorkflowProductionWorkbench(props: Props) {
            const basePrompt = mode === "video"
                ? [values.videoPrompt || values.plotDescription, values.action, values.dialogue && `台词：${values.dialogue}`, values.continuityNotes].filter(Boolean).join("\n")
                : [values.imagePrompt || values.plotDescription, values.action, "黑白分镜草图，清晰动作节拍，电影构图"].filter(Boolean).join("\n");
-            const resolvedPrompt = resolveShotAssetMentionPrompt(basePrompt, shotAssetReferenceContext);
+            const resolvedPrompt = resolveShotAssetMentionPrompt(basePrompt, shotAssetReferenceContext, { dialogue: values.dialogue });
            const skillExecution = await skillRuntime.prepare({
                profile: "shortDrama",
                prompt: resolvedPrompt,
@@ -637,7 +643,7 @@ function BoundAssets({ detail, shotId, changing, onUnlink }: { detail: ProjectDe
            <Image.PreviewGroup>
                <div className="workflow-bound-assets-content">
                    {references.length ? references.map((reference) => {
-                        const asset = assetByVersionId.get(reference.assetVersionId);
+                        const asset = reference.asset || assetByVersionId.get(reference.assetVersionId);
                        const title = asset?.title || "历史资产版本";
                        const previewUrl = asset ? assetPreviewUrl(asset) : "";
                        return <div key={reference.id} className="workflow-bound-asset-chip">
--- a/web/src/pages/projects/detail/workflow-shot-references.ts
+++ b/web/src/pages/projects/detail/workflow-shot-references.ts
@@ -1,6 +1,6 @@
 import type { CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
 import { resourceFileUrl, resourceIdFromStorageKey, resourceStorageKey } from "@/services/api/resources";
-import type { ProjectAsset, ProjectDetail } from "@/services/api/projects";
+import type { CharacterRepresentation, ProjectAsset, ProjectDetail, ShotAssetReference } from "@/services/api/projects";
 import type { AssetCategory } from "@/stores/use-asset-store";
 import type { ReferenceImage } from "@/types/image";
 import type { ReferenceAudio } from "@/types/media";
@@ -20,6 +20,7 @@ export type ShotPromptAssetReference = {
    imageIndex: number;
    audioIndex?: number;
    voiceDescription?: string;
+    dialogue?: string;
 };

 const assetCategories = new Set<AssetCategory>(["character", "environment", "wardrobe", "prop", "weapon", "style", "other"]);
@@ -29,12 +30,12 @@ export function buildShotAssetReferenceContext(detail: ProjectDetail, shotId: st
    const seenAssetIds = new Set<string>();
    const entries = (detail.shotReferences || []).flatMap((reference) => {
        if (reference.shotId !== shotId || reference.status !== "linked") return [];
-        const asset = assetByVersionId.get(reference.assetVersionId);
+        const asset = reference.asset || assetByVersionId.get(reference.assetVersionId);
        if (!asset || seenAssetIds.has(asset.id)) return [];
-        const image = projectAssetReferenceImage(asset);
+        const image = projectAssetReferenceImage(asset, reference);
        if (!image) return [];
        seenAssetIds.add(asset.id);
-        return [{ asset, image }];
+        return [{ asset, image, reference }];
    });

    const referenceAudios: ReferenceAudio[] = [];
@@ -82,11 +83,11 @@ export function buildShotAssetReferenceContext(detail: ProjectDetail, shotId: st
        referenceImages: entries.map(({ image }) => image),
        referenceAudios,
        assetReferences,
-        resolvedCharacterVersions: entries.flatMap(({ asset }) => asset.character?.versionId ? [{ assetId: asset.id, versionId: asset.character.versionId }] : []),
+        resolvedCharacterVersions: entries.flatMap(({ asset, reference }) => asset.character ? [{ assetId: asset.id, versionId: reference.referencedVersion?.id || reference.assetVersionId || asset.character.versionId }] : []),
    };
 }

-export function resolveShotAssetMentionPrompt(prompt: string, context: ShotAssetReferenceContext) {
+export function resolveShotAssetMentionPrompt(prompt: string, context: ShotAssetReferenceContext, options: { dialogue?: string } = {}) {
    const imageLabelByAssetId = new Map(context.mentionReferences.map((reference, index) => [reference.assetId, `图片${index + 1}`]));
    const unresolved = new Set<string>();
    const resolved = prompt.replace(/@\[asset:([^\]]+)\]/g, (token, assetId: string) => {
@@ -98,8 +99,10 @@ export function resolveShotAssetMentionPrompt(prompt: string, context: ShotAsset
        return label;
    });
    if (unresolved.size) throw new Error(`提示词中的 ${Array.from(unresolved).join("、")} 未绑定到当前镜头，请重新选择资产或删除引用`);
-    const assetBlock = compileShotAssetReferencePrompt(context.assetReferences);
-    return [resolved.trim(), assetBlock].filter(Boolean).join("\n\n");
+    const dialogueContext = withShotDialogue(context.assetReferences, options.dialogue);
+    const assetBlock = compileShotAssetReferencePrompt(dialogueContext.references);
+    const dialogueBlock = dialogueContext.unassignedDialogue ? `【镜头台词】\n\n${dialogueContext.unassignedDialogue}` : "";
+    return [resolved.trim(), assetBlock, dialogueBlock].filter(Boolean).join("\n\n");
 }

 export function compileShotAssetReferencePrompt(references: ShotPromptAssetReference[]) {
@@ -109,18 +112,17 @@ export function compileShotAssetReferencePrompt(references: ShotPromptAssetRefer
        const media = [`${visualLabel}：图片${reference.imageIndex}`, reference.audioIndex ? `声音参考：音频${reference.audioIndex}` : ""].filter(Boolean).join("；");
        return [
            `- ${reference.title}：${media}`,
-            reference.voiceDescription ? `  声音说明：${reference.voiceDescription}` : "",
+            reference.voiceDescription ? `  声音画像：${reference.voiceDescription}` : "",
+            reference.dialogue ? `  镜头台词：${reference.dialogue}` : "",
        ].filter(Boolean);
    });
    return ["【资产参考】", "", ...lines].join("\n");
 }

-function projectAssetReferenceImage(asset: ProjectAsset): ReferenceImage | undefined {
-    const representation = asset.character
-        ? asset.character.representations.find((item) => item.role === "turnaround_sheet")
-            || asset.character.representations.find((item) => item.role === "primary")
-            || asset.character.representations.find((item) => item.role === "front")
-        : undefined;
+function projectAssetReferenceImage(asset: ProjectAsset, reference?: ShotAssetReference): ReferenceImage | undefined {
+    const boundRepresentations = reference?.referencedVersion?.representations || [];
+    const representation = preferredVisualRepresentation(boundRepresentations)
+        || (asset.character ? preferredVisualRepresentation(asset.character.representations) : undefined);
    if (representation) {
        return {
            id: asset.id,
@@ -143,6 +145,40 @@ function projectAssetReferenceImage(asset: ProjectAsset): ReferenceImage | undef
    };
 }

+function preferredVisualRepresentation(representations: CharacterRepresentation[]) {
+    return representations.find((item) => item.role === "turnaround_sheet")
+        || representations.find((item) => item.role === "primary")
+        || representations.find((item) => item.role === "front")
+        || representations.find((item) => item.mediaType.startsWith("image"));
+}
+
+function withShotDialogue(references: ShotPromptAssetReference[], dialogue?: string) {
+    const value = stringValue(dialogue);
+    if (!value) return { references, unassignedDialogue: "" };
+    const characters = references.filter((reference) => reference.category === "character");
+    if (characters.length === 1) {
+        return {
+            references: references.map((reference) => reference.assetId === characters[0]?.assetId ? { ...reference, dialogue: value } : reference),
+            unassignedDialogue: "",
+        };
+    }
+    const resolved = references.map((reference) => {
+        if (reference.category !== "character") return reference;
+        const matched = dialogueForCharacter(value, reference.title);
+        return matched ? { ...reference, dialogue: matched } : reference;
+    });
+    return {
+        references: resolved,
+        unassignedDialogue: resolved.some((reference) => reference.dialogue) ? "" : value,
+    };
+}
+
+function dialogueForCharacter(dialogue: string, characterName: string) {
+    const escaped = characterName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const pattern = new RegExp(`(?:^|\\n)\\s*${escaped}\\s*[：:]\\s*([^\\n]+)`, "g");
+    return Array.from(dialogue.matchAll(pattern), (match) => match[1]?.trim()).filter(Boolean).join("；");
+}
+
 function projectAssetCategory(value: string): AssetCategory {
    return assetCategories.has(value as AssetCategory) ? value as AssetCategory : "other";
 }
--- a/web/src/services/api/projects.ts
+++ b/web/src/services/api/projects.ts
@@ -202,6 +202,13 @@ export type ShotAssetReference = {
    role: "reference" | "start_frame" | "end_frame" | "keyframe" | "storyboard" | "output" | string;
    status: string;
    createdAt: string;
+    asset?: ProjectAsset;
+    referencedVersion?: {
+        id: string;
+        assetId: string;
+        version: number;
+        representations: CharacterRepresentation[];
+    };
 };

 export type WorkflowStep = {
--- a/web/test/workflow-shot-generation.test.ts
+++ b/web/test/workflow-shot-generation.test.ts
@@ -6,7 +6,7 @@ import type { GenerationTask } from "../src/services/api/task-center";
 import type { ProjectDetail } from "../src/services/api/projects";
 import { buildShotAssetReferenceContext, resolveShotAssetMentionPrompt } from "../src/pages/projects/detail/workflow-shot-references";

-test("shot asset prompt keeps character names and maps character image, voice and scene assets", () => {
+test("shot generation submits historical character image, current voice and asset prompt", async () => {
    const detail = {
        assets: [
            {
@@ -14,9 +14,9 @@ test("shot asset prompt keeps character names and maps character image, voice an
                title: "张天昊",
                category: "character",
                mediaType: "image",
-                primaryVersionId: "character-version-1",
+                primaryVersionId: "character-version-2",
                character: {
-                    versionId: "character-version-1",
+                    versionId: "character-version-2",
                    definition: { voiceLanguage: "普通话", voiceAge: "青年男性", voiceTimbre: "略带疲惫和震惊" },
                    representations: [{ id: "representation-1", resourceId: "character-image-1", mediaType: "image/png", role: "primary" }],
                    voice: { profile: { sampleResourceId: "character-audio-1", language: "普通话", timbre: "沉稳" }, instructions: "内心独白语气" },
@@ -32,22 +32,77 @@ test("shot asset prompt keeps character names and maps character image, voice an
            },
        ],
        shotReferences: [
-            { shotId: "shot-1", assetVersionId: "character-version-1", status: "linked" },
+            {
+                shotId: "shot-1",
+                assetVersionId: "character-version-1",
+                status: "linked",
+                asset: {
+                    id: "character-1",
+                    title: "张天昊",
+                    category: "character",
+                    mediaType: "entity",
+                    primaryVersionId: "character-version-2",
+                    character: {
+                        versionId: "character-version-2",
+                        definition: { voiceLanguage: "普通话", voiceAge: "青年男性", voiceTimbre: "略带疲惫和震惊" },
+                        representations: [{ id: "representation-2", resourceId: "character-image-2", mediaType: "image/png", role: "primary" }],
+                        voice: { profile: { sampleResourceId: "character-audio-1", language: "普通话", timbre: "沉稳" }, instructions: "内心独白语气" },
+                    },
+                },
+                referencedVersion: {
+                    id: "character-version-1",
+                    assetId: "character-1",
+                    version: 1,
+                    representations: [{ id: "representation-1", resourceId: "character-image-1", mediaType: "image/png", role: "primary" }],
+                },
+            },
            { shotId: "shot-1", assetVersionId: "scene-version-1", status: "linked" },
        ],
    } as ProjectDetail;

    const context = buildShotAssetReferenceContext(detail, "shot-1");
-    const prompt = resolveShotAssetMentionPrompt("张天昊在 @[asset:scene-1] 睁开眼睛", context);
+    const prompt = resolveShotAssetMentionPrompt("张天昊在 @[asset:scene-1] 睁开眼睛", context, { dialogue: "我穿越了？瓦西国张家废柴少主？" });

    expect(context.referenceImages).toHaveLength(2);
+    expect(context.referenceImages[0]?.storageKey).toBe("resource:character-image-1");
    expect(context.referenceAudios).toHaveLength(1);
    expect(context.referenceAudios[0]?.storageKey).toBe("resource:character-audio-1");
    expect(context.resolvedCharacterVersions).toEqual([{ assetId: "character-1", versionId: "character-version-1" }]);
    expect(prompt).toContain("张天昊在 图片2 睁开眼睛");
    expect(prompt).toContain("- 张天昊：人物参考：图片1；声音参考：音频1");
-    expect(prompt).toContain("声音说明：普通话；青年男性；略带疲惫和震惊；内心独白语气");
+    expect(prompt).toContain("声音画像：普通话；青年男性；略带疲惫和震惊；内心独白语气");
+    expect(prompt).toContain("镜头台词：我穿越了？瓦西国张家废柴少主？");
    expect(prompt).toContain("- 坑底场景：场景参考：图片2");
+
+    let createdInput: Parameters<GenerationTaskDependencies["createTask"]>[0] | undefined;
+    const task = {
+        id: "shot-task-1",
+        type: "canvas_video",
+        status: "queued",
+        prompt,
+        attempts: 0,
+        createdAt: "2026-08-30T00:00:00.000Z",
+        updatedAt: "2026-08-30T00:00:00.000Z",
+    } satisfies GenerationTask;
+    await submitBackendGenerationTask({
+        projectId: "project-1",
+        mode: "video",
+        prompt,
+        config: { ...defaultConfig, model: "MiniMax-H3", videoModel: "MiniMax-H3" },
+        referenceImages: context.referenceImages,
+        referenceAudios: context.referenceAudios,
+        metadata: { shotId: "shot-1", videoEditOperation: "reference_to_video" },
+    }, {
+        createTask: async (input) => { createdInput = input; return task; },
+        waitTask: async () => { throw new Error("should not wait"); },
+        runLocal: async () => ({ mode: "video" }),
+        createId: () => "id-1",
+        now: () => "2026-08-30T00:00:00.000Z",
+    });
+
+    expect(createdInput?.prompt).toContain("【资产参考】");
+    expect(createdInput?.input.referenceImages.map((reference) => reference.storageKey)).toEqual(["resource:character-image-1", "resource:scene-image-1"]);
+    expect(createdInput?.input.referenceAudios.map((reference) => reference.storageKey)).toEqual(["resource:character-audio-1"]);
 });

 test("background generation submission returns after task creation without waiting", async () => {
--- a/web/test/workflow-shot-references.test.ts
+++ b/web/test/workflow-shot-references.test.ts
@@ -18,7 +18,7 @@ describe("workflow shot asset references", () => {
        const [scene, character] = context.mentionReferences;
        const prompt = `让 ${canvasResourceMentionToken(character)} 从 ${canvasResourceMentionToken(scene)} 走近镜头`;

-        expect(resolveShotAssetMentionPrompt(prompt, context)).toBe("让 图片2 从 图片1 走近镜头");
+        expect(resolveShotAssetMentionPrompt(prompt, context)).toContain("让 图片2 从 图片1 走近镜头");
    });

    test("rejects stale mentions after an asset is unbound", () => {