fix(canvas): 分镜视频生成 - 正确附加角色场景与声音资产及镜头台词

2026-08-30 17:54:23 +08:00
parent 37eb2b7e48
commit f2fe163bc7
9 changed files with 266 additions and 27 deletions
--- a/web/src/pages/projects/detail/workflow-production-workbench.tsx
+++ b/web/src/pages/projects/detail/workflow-production-workbench.tsx
@@ -118,7 +118,13 @@ export default function WorkflowProductionWorkbench(props: Props) {
    const [selectedSkillIds, setSelectedSkillIds] = useState<string[]>([]);
    const { skills: availableSkills, loading: skillsLoading } = useSkillRuntimeCatalog();
    const shotAssetReferenceContext = useMemo(() => buildShotAssetReferenceContext(detail, selectedShot?.id || ""), [detail, selectedShot?.id]);
-    const referenceByVersionId = useMemo(() => new Map((detail.shotReferences || []).filter((reference) => reference.shotId === selectedShot?.id && reference.role === "reference" && reference.status === "linked").map((reference) => [reference.assetVersionId, reference])), [detail.shotReferences, selectedShot?.id]);
+    const referenceByVersionId = useMemo(() => {
+        const references = (detail.shotReferences || []).filter((reference) => reference.shotId === selectedShot?.id && reference.role === "reference" && reference.status === "linked");
+        return new Map(references.flatMap((reference) => [
+            [reference.assetVersionId, reference] as const,
+            ...(reference.asset?.primaryVersionId ? [[reference.asset.primaryVersionId, reference] as const] : []),
+        ]));
+    }, [detail.shotReferences, selectedShot?.id]);
    const currentDurationSeconds = Number(watchedDuration || Math.max(0.5, (selectedShot?.durationMs || 3000) / 1000));
    const generationSeconds = String(Math.max(1, Math.round(currentDurationSeconds)));
    const selectedVideoProfile = generationCapability === "video" && selectedModel ? modelCapabilityConfigFor(effectiveConfig, selectedModel).video : undefined;
@@ -324,7 +330,7 @@ export default function WorkflowProductionWorkbench(props: Props) {
            const basePrompt = mode === "video"
                ? [values.videoPrompt || values.plotDescription, values.action, values.dialogue && `台词：${values.dialogue}`, values.continuityNotes].filter(Boolean).join("\n")
                : [values.imagePrompt || values.plotDescription, values.action, "黑白分镜草图，清晰动作节拍，电影构图"].filter(Boolean).join("\n");
-            const resolvedPrompt = resolveShotAssetMentionPrompt(basePrompt, shotAssetReferenceContext);
+            const resolvedPrompt = resolveShotAssetMentionPrompt(basePrompt, shotAssetReferenceContext, { dialogue: values.dialogue });
            const skillExecution = await skillRuntime.prepare({
                profile: "shortDrama",
                prompt: resolvedPrompt,
@@ -637,7 +643,7 @@ function BoundAssets({ detail, shotId, changing, onUnlink }: { detail: ProjectDe
            <Image.PreviewGroup>
                <div className="workflow-bound-assets-content">
                    {references.length ? references.map((reference) => {
-                        const asset = assetByVersionId.get(reference.assetVersionId);
+                        const asset = reference.asset || assetByVersionId.get(reference.assetVersionId);
                        const title = asset?.title || "历史资产版本";
                        const previewUrl = asset ? assetPreviewUrl(asset) : "";
                        return <div key={reference.id} className="workflow-bound-asset-chip">
--- a/web/src/pages/projects/detail/workflow-shot-references.ts
+++ b/web/src/pages/projects/detail/workflow-shot-references.ts
@@ -1,6 +1,6 @@
 import type { CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
 import { resourceFileUrl, resourceIdFromStorageKey, resourceStorageKey } from "@/services/api/resources";
-import type { ProjectAsset, ProjectDetail } from "@/services/api/projects";
+import type { CharacterRepresentation, ProjectAsset, ProjectDetail, ShotAssetReference } from "@/services/api/projects";
 import type { AssetCategory } from "@/stores/use-asset-store";
 import type { ReferenceImage } from "@/types/image";
 import type { ReferenceAudio } from "@/types/media";
@@ -20,6 +20,7 @@ export type ShotPromptAssetReference = {
    imageIndex: number;
    audioIndex?: number;
    voiceDescription?: string;
+    dialogue?: string;
 };

 const assetCategories = new Set<AssetCategory>(["character", "environment", "wardrobe", "prop", "weapon", "style", "other"]);
@@ -29,12 +30,12 @@ export function buildShotAssetReferenceContext(detail: ProjectDetail, shotId: st
    const seenAssetIds = new Set<string>();
    const entries = (detail.shotReferences || []).flatMap((reference) => {
        if (reference.shotId !== shotId || reference.status !== "linked") return [];
-        const asset = assetByVersionId.get(reference.assetVersionId);
+        const asset = reference.asset || assetByVersionId.get(reference.assetVersionId);
        if (!asset || seenAssetIds.has(asset.id)) return [];
-        const image = projectAssetReferenceImage(asset);
+        const image = projectAssetReferenceImage(asset, reference);
        if (!image) return [];
        seenAssetIds.add(asset.id);
-        return [{ asset, image }];
+        return [{ asset, image, reference }];
    });

    const referenceAudios: ReferenceAudio[] = [];
@@ -82,11 +83,11 @@ export function buildShotAssetReferenceContext(detail: ProjectDetail, shotId: st
        referenceImages: entries.map(({ image }) => image),
        referenceAudios,
        assetReferences,
-        resolvedCharacterVersions: entries.flatMap(({ asset }) => asset.character?.versionId ? [{ assetId: asset.id, versionId: asset.character.versionId }] : []),
+        resolvedCharacterVersions: entries.flatMap(({ asset, reference }) => asset.character ? [{ assetId: asset.id, versionId: reference.referencedVersion?.id || reference.assetVersionId || asset.character.versionId }] : []),
    };
 }

-export function resolveShotAssetMentionPrompt(prompt: string, context: ShotAssetReferenceContext) {
+export function resolveShotAssetMentionPrompt(prompt: string, context: ShotAssetReferenceContext, options: { dialogue?: string } = {}) {
    const imageLabelByAssetId = new Map(context.mentionReferences.map((reference, index) => [reference.assetId, `图片${index + 1}`]));
    const unresolved = new Set<string>();
    const resolved = prompt.replace(/@\[asset:([^\]]+)\]/g, (token, assetId: string) => {
@@ -98,8 +99,10 @@ export function resolveShotAssetMentionPrompt(prompt: string, context: ShotAsset
        return label;
    });
    if (unresolved.size) throw new Error(`提示词中的 ${Array.from(unresolved).join("、")} 未绑定到当前镜头，请重新选择资产或删除引用`);
-    const assetBlock = compileShotAssetReferencePrompt(context.assetReferences);
-    return [resolved.trim(), assetBlock].filter(Boolean).join("\n\n");
+    const dialogueContext = withShotDialogue(context.assetReferences, options.dialogue);
+    const assetBlock = compileShotAssetReferencePrompt(dialogueContext.references);
+    const dialogueBlock = dialogueContext.unassignedDialogue ? `【镜头台词】\n\n${dialogueContext.unassignedDialogue}` : "";
+    return [resolved.trim(), assetBlock, dialogueBlock].filter(Boolean).join("\n\n");
 }

 export function compileShotAssetReferencePrompt(references: ShotPromptAssetReference[]) {
@@ -109,18 +112,17 @@ export function compileShotAssetReferencePrompt(references: ShotPromptAssetRefer
        const media = [`${visualLabel}：图片${reference.imageIndex}`, reference.audioIndex ? `声音参考：音频${reference.audioIndex}` : ""].filter(Boolean).join("；");
        return [
            `- ${reference.title}：${media}`,
-            reference.voiceDescription ? `  声音说明：${reference.voiceDescription}` : "",
+            reference.voiceDescription ? `  声音画像：${reference.voiceDescription}` : "",
+            reference.dialogue ? `  镜头台词：${reference.dialogue}` : "",
        ].filter(Boolean);
    });
    return ["【资产参考】", "", ...lines].join("\n");
 }

-function projectAssetReferenceImage(asset: ProjectAsset): ReferenceImage | undefined {
-    const representation = asset.character
-        ? asset.character.representations.find((item) => item.role === "turnaround_sheet")
-            || asset.character.representations.find((item) => item.role === "primary")
-            || asset.character.representations.find((item) => item.role === "front")
-        : undefined;
+function projectAssetReferenceImage(asset: ProjectAsset, reference?: ShotAssetReference): ReferenceImage | undefined {
+    const boundRepresentations = reference?.referencedVersion?.representations || [];
+    const representation = preferredVisualRepresentation(boundRepresentations)
+        || (asset.character ? preferredVisualRepresentation(asset.character.representations) : undefined);
    if (representation) {
        return {
            id: asset.id,
@@ -143,6 +145,40 @@ function projectAssetReferenceImage(asset: ProjectAsset): ReferenceImage | undef
    };
 }

+function preferredVisualRepresentation(representations: CharacterRepresentation[]) {
+    return representations.find((item) => item.role === "turnaround_sheet")
+        || representations.find((item) => item.role === "primary")
+        || representations.find((item) => item.role === "front")
+        || representations.find((item) => item.mediaType.startsWith("image"));
+}
+
+function withShotDialogue(references: ShotPromptAssetReference[], dialogue?: string) {
+    const value = stringValue(dialogue);
+    if (!value) return { references, unassignedDialogue: "" };
+    const characters = references.filter((reference) => reference.category === "character");
+    if (characters.length === 1) {
+        return {
+            references: references.map((reference) => reference.assetId === characters[0]?.assetId ? { ...reference, dialogue: value } : reference),
+            unassignedDialogue: "",
+        };
+    }
+    const resolved = references.map((reference) => {
+        if (reference.category !== "character") return reference;
+        const matched = dialogueForCharacter(value, reference.title);
+        return matched ? { ...reference, dialogue: matched } : reference;
+    });
+    return {
+        references: resolved,
+        unassignedDialogue: resolved.some((reference) => reference.dialogue) ? "" : value,
+    };
+}
+
+function dialogueForCharacter(dialogue: string, characterName: string) {
+    const escaped = characterName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const pattern = new RegExp(`(?:^|\\n)\\s*${escaped}\\s*[：:]\\s*([^\\n]+)`, "g");
+    return Array.from(dialogue.matchAll(pattern), (match) => match[1]?.trim()).filter(Boolean).join("；");
+}
+
 function projectAssetCategory(value: string): AssetCategory {
    return assetCategories.has(value as AssetCategory) ? value as AssetCategory : "other";
 }
--- a/web/src/services/api/projects.ts
+++ b/web/src/services/api/projects.ts
@@ -202,6 +202,13 @@ export type ShotAssetReference = {
    role: "reference" | "start_frame" | "end_frame" | "keyframe" | "storyboard" | "output" | string;
    status: string;
    createdAt: string;
+    asset?: ProjectAsset;
+    referencedVersion?: {
+        id: string;
+        assetId: string;
+        version: number;
+        representations: CharacterRepresentation[];
+    };
 };

 export type WorkflowStep = {