fix(canvas): 分镜生成 - 强化模型选择提示并补齐资产引用与角色音频
This commit is contained in:
@@ -1,5 +1,6 @@
|
|||||||
import type { StoryboardRow } from "@/types/canvas";
|
import type { StoryboardRow } from "@/types/canvas";
|
||||||
import type { ProjectDetail, ShotRevisionInput } from "@/services/api/projects";
|
import type { ProjectDetail, ShotRevisionInput } from "@/services/api/projects";
|
||||||
|
import { ensureShotAssetMentionPrompt } from "./workflow-shot-references";
|
||||||
|
|
||||||
export type ChapterStoryboardAsset = {
|
export type ChapterStoryboardAsset = {
|
||||||
id: string;
|
id: string;
|
||||||
@@ -74,10 +75,20 @@ export function chapterStoryboardAssets(detail: ProjectDetail): ChapterStoryboar
|
|||||||
|
|
||||||
export function storyboardRowsToProjectShots(rows: StoryboardRow[], detail?: ProjectDetail): ProjectShotReplacementInput[] {
|
export function storyboardRowsToProjectShots(rows: StoryboardRow[], detail?: ProjectDetail): ProjectShotReplacementInput[] {
|
||||||
const versionByAssetId = new Map((detail?.assets || []).flatMap((asset) => asset.primaryVersionId ? [[asset.id, asset.primaryVersionId] as const] : []));
|
const versionByAssetId = new Map((detail?.assets || []).flatMap((asset) => asset.primaryVersionId ? [[asset.id, asset.primaryVersionId] as const] : []));
|
||||||
|
const mentionReferenceByAssetId = new Map((detail?.assets || []).map((asset) => [asset.id, {
|
||||||
|
assetId: asset.id,
|
||||||
|
kind: asset.category === "character" ? "character" as const : "image" as const,
|
||||||
|
label: asset.title,
|
||||||
|
title: asset.title,
|
||||||
|
}]));
|
||||||
return rows.map((row, index) => {
|
return rows.map((row, index) => {
|
||||||
const shotNumber = index + 1;
|
const shotNumber = index + 1;
|
||||||
const description = row.plotDescription.trim() || `镜头 ${shotNumber}`;
|
const description = row.plotDescription.trim() || `镜头 ${shotNumber}`;
|
||||||
const durationMs = Math.max(1000, Math.round((Number(row.durationSeconds) || 1) * 1000));
|
const durationMs = Math.max(1000, Math.round((Number(row.durationSeconds) || 1) * 1000));
|
||||||
|
const mentionReferences = storyboardRowAssetIds(row).flatMap((assetId) => {
|
||||||
|
const reference = mentionReferenceByAssetId.get(assetId);
|
||||||
|
return reference ? [reference] : [];
|
||||||
|
});
|
||||||
return {
|
return {
|
||||||
title: `SC.${String(shotNumber).padStart(2, "0")}`,
|
title: `SC.${String(shotNumber).padStart(2, "0")}`,
|
||||||
description,
|
description,
|
||||||
@@ -92,7 +103,7 @@ export function storyboardRowsToProjectShots(rows: StoryboardRow[], detail?: Pro
|
|||||||
cameraMovement: row.motion.trim(),
|
cameraMovement: row.motion.trim(),
|
||||||
durationMs,
|
durationMs,
|
||||||
imagePrompt: row.imageGenerationPrompt.trim(),
|
imagePrompt: row.imageGenerationPrompt.trim(),
|
||||||
videoPrompt: row.videoMotionPrompt.trim(),
|
videoPrompt: ensureShotAssetMentionPrompt(row.videoMotionPrompt, mentionReferences),
|
||||||
negativePrompt: row.negativePrompt.trim(),
|
negativePrompt: row.negativePrompt.trim(),
|
||||||
continuityNotes: row.continuityOut.trim(),
|
continuityNotes: row.continuityOut.trim(),
|
||||||
actionBeats: row.timeBeats.trim() ? [{ description: row.timeBeats.trim() }] : [],
|
actionBeats: row.timeBeats.trim() ? [{ description: row.timeBeats.trim() }] : [],
|
||||||
@@ -114,16 +125,19 @@ export function chapterStoryboardReplaceImpact(detail: ProjectDetail, unitId: st
|
|||||||
}
|
}
|
||||||
|
|
||||||
function storyboardRowAssetVersionIds(row: StoryboardRow, versionByAssetId: Map<string, string>) {
|
function storyboardRowAssetVersionIds(row: StoryboardRow, versionByAssetId: Map<string, string>) {
|
||||||
const assetIds = [
|
return storyboardRowAssetIds(row).flatMap((assetId) => {
|
||||||
...row.assetBindings.map((binding) => binding.nodeId),
|
|
||||||
...row.characters.flatMap((character) => character.characterAssetId ? [character.characterAssetId] : []),
|
|
||||||
];
|
|
||||||
return Array.from(new Set(assetIds)).flatMap((assetId) => {
|
|
||||||
const versionId = versionByAssetId.get(assetId);
|
const versionId = versionByAssetId.get(assetId);
|
||||||
return versionId ? [versionId] : [];
|
return versionId ? [versionId] : [];
|
||||||
}).slice(0, 6);
|
}).slice(0, 6);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function storyboardRowAssetIds(row: StoryboardRow) {
|
||||||
|
return Array.from(new Set([
|
||||||
|
...row.assetBindings.map((binding) => binding.nodeId),
|
||||||
|
...row.characters.flatMap((character) => character.characterAssetId ? [character.characterAssetId] : []),
|
||||||
|
])).slice(0, 6);
|
||||||
|
}
|
||||||
|
|
||||||
function storyboardAssetType(mediaType: string, category: string): ChapterStoryboardAsset["type"] | null {
|
function storyboardAssetType(mediaType: string, category: string): ChapterStoryboardAsset["type"] | null {
|
||||||
if (category === "character") return "character";
|
if (category === "character") return "character";
|
||||||
if (mediaType === "image" || mediaType === "video" || mediaType === "audio") return mediaType;
|
if (mediaType === "image" || mediaType === "video" || mediaType === "audio") return mediaType;
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ import {
|
|||||||
formatDuration,
|
formatDuration,
|
||||||
type ShortDramaWorkflowStage,
|
type ShortDramaWorkflowStage,
|
||||||
} from "./workflow-shared";
|
} from "./workflow-shared";
|
||||||
import { buildShotAssetReferenceContext, resolveShotAssetMentionPrompt } from "./workflow-shot-references";
|
import { buildShotAssetReferenceContext, ensureShotAssetMentionPrompt, resolveShotAssetMentionPrompt } from "./workflow-shot-references";
|
||||||
|
|
||||||
type ShotEditorValues = Omit<ShotRevisionInput, "durationMs"> & {
|
type ShotEditorValues = Omit<ShotRevisionInput, "durationMs"> & {
|
||||||
title: string;
|
title: string;
|
||||||
@@ -127,8 +127,7 @@ export default function WorkflowProductionWorkbench(props: Props) {
|
|||||||
}, [detail.shotReferences, selectedShot?.id]);
|
}, [detail.shotReferences, selectedShot?.id]);
|
||||||
const currentDurationSeconds = Number(watchedDuration || Math.max(0.5, (selectedShot?.durationMs || 3000) / 1000));
|
const currentDurationSeconds = Number(watchedDuration || Math.max(0.5, (selectedShot?.durationMs || 3000) / 1000));
|
||||||
const generationSeconds = String(Math.max(1, Math.round(currentDurationSeconds)));
|
const generationSeconds = String(Math.max(1, Math.round(currentDurationSeconds)));
|
||||||
const selectedVideoProfile = generationCapability === "video" && selectedModel ? modelCapabilityConfigFor(effectiveConfig, selectedModel).video : undefined;
|
const generationReferenceAudios = generationCapability === "video" ? shotAssetReferenceContext.referenceAudios : [];
|
||||||
const generationReferenceAudios = generationCapability === "video" && (selectedVideoProfile?.references.maxAudios || 0) > 0 ? shotAssetReferenceContext.referenceAudios : [];
|
|
||||||
const videoEditOperation = generationCapability === "video" && shotAssetReferenceContext.referenceImages.length ? "reference_to_video" : undefined;
|
const videoEditOperation = generationCapability === "video" && shotAssetReferenceContext.referenceImages.length ? "reference_to_video" : undefined;
|
||||||
const modelRequirements = useMemo<ModelRequirements>(() => ({
|
const modelRequirements = useMemo<ModelRequirements>(() => ({
|
||||||
capability: generationCapability,
|
capability: generationCapability,
|
||||||
@@ -218,6 +217,7 @@ export default function WorkflowProductionWorkbench(props: Props) {
|
|||||||
const normalizedDurationSeconds = generationCapability === "video" && currentModel
|
const normalizedDurationSeconds = generationCapability === "video" && currentModel
|
||||||
? Number(normalizeVideoValue(modelCapabilityConfigFor(effectiveConfig, currentModel).video!, { seconds: String(shotDurationSeconds) }).seconds)
|
? Number(normalizeVideoValue(modelCapabilityConfigFor(effectiveConfig, currentModel).video!, { seconds: String(shotDurationSeconds) }).seconds)
|
||||||
: shotDurationSeconds;
|
: shotDurationSeconds;
|
||||||
|
const videoPrompt = ensureShotAssetMentionPrompt(revision?.videoPrompt || "", shotAssetReferenceContext.mentionReferences);
|
||||||
form.setFieldsValue({
|
form.setFieldsValue({
|
||||||
title: selectedShot?.title || "",
|
title: selectedShot?.title || "",
|
||||||
plotDescription: revision?.plotDescription || selectedShot?.description || "",
|
plotDescription: revision?.plotDescription || selectedShot?.description || "",
|
||||||
@@ -228,13 +228,13 @@ export default function WorkflowProductionWorkbench(props: Props) {
|
|||||||
cameraMovement: revision?.cameraMovement || "",
|
cameraMovement: revision?.cameraMovement || "",
|
||||||
durationSeconds: normalizedDurationSeconds,
|
durationSeconds: normalizedDurationSeconds,
|
||||||
imagePrompt: revision?.imagePrompt || "",
|
imagePrompt: revision?.imagePrompt || "",
|
||||||
videoPrompt: revision?.videoPrompt || "",
|
videoPrompt,
|
||||||
negativePrompt: revision?.negativePrompt || "",
|
negativePrompt: revision?.negativePrompt || "",
|
||||||
continuityNotes: revision?.continuityNotes || "",
|
continuityNotes: revision?.continuityNotes || "",
|
||||||
});
|
});
|
||||||
setPreviewArtifactId("");
|
setPreviewArtifactId("");
|
||||||
setEditorDirty(!revision);
|
setEditorDirty(!revision || videoPrompt !== revision.videoPrompt);
|
||||||
}, [effectiveConfig, form, generationCapability, initialModel, revision?.id, selectedShot?.id]);
|
}, [effectiveConfig, form, generationCapability, initialModel, revision?.id, selectedShot?.id, shotAssetReferenceContext.mentionReferences]);
|
||||||
|
|
||||||
const changeGenerationModel = (nextModel: string) => {
|
const changeGenerationModel = (nextModel: string) => {
|
||||||
selectedModelRef.current = nextModel;
|
selectedModelRef.current = nextModel;
|
||||||
@@ -359,7 +359,6 @@ export default function WorkflowProductionWorkbench(props: Props) {
|
|||||||
artifactMetadata: { model: routedModel, aspectRatio, resolution, durationSeconds: values.durationSeconds, ...skillExecution.metadata },
|
artifactMetadata: { model: routedModel, aspectRatio, resolution, durationSeconds: values.durationSeconds, ...skillExecution.metadata },
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
if (shotAssetReferenceContext.referenceAudios.length && !generationReferenceAudios.length) message.warning("当前模型不支持参考音频,已保留角色声音文字说明,但未附加声音样本");
|
|
||||||
if (activeShotIdRef.current === submittingShot.id) setEditorDirty(false);
|
if (activeShotIdRef.current === submittingShot.id) setEditorDirty(false);
|
||||||
await onRefresh();
|
await onRefresh();
|
||||||
message.success(`${productionStageCopy[activeStage as "storyboard" | "previz" | "video"].label}任务已提交`);
|
message.success(`${productionStageCopy[activeStage as "storyboard" | "previz" | "video"].label}任务已提交`);
|
||||||
|
|||||||
@@ -105,6 +105,20 @@ export function resolveShotAssetMentionPrompt(prompt: string, context: ShotAsset
|
|||||||
return [resolved.trim(), assetBlock, dialogueBlock].filter(Boolean).join("\n\n");
|
return [resolved.trim(), assetBlock, dialogueBlock].filter(Boolean).join("\n\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function ensureShotAssetMentionPrompt(prompt: string, references: Array<Pick<CanvasResourceReference, "assetId" | "kind" | "label" | "title">>) {
|
||||||
|
const value = prompt.trim();
|
||||||
|
if (!value) return "";
|
||||||
|
const mentionedAssetIds = new Set(Array.from(value.matchAll(/@\[asset:([^\]]+)\]/g), (match) => match[1]));
|
||||||
|
const missing = references.filter((reference): reference is typeof reference & { assetId: string } => Boolean(reference.assetId && !mentionedAssetIds.has(reference.assetId)));
|
||||||
|
if (!missing.length) return value;
|
||||||
|
const block = (title: string, items: typeof missing) => items.length
|
||||||
|
? [title, ...items.map((reference) => `${reference.title || reference.label}:@[asset:${reference.assetId}]`)].join("\n")
|
||||||
|
: "";
|
||||||
|
const characters = missing.filter((reference) => reference.kind === "character");
|
||||||
|
const otherAssets = missing.filter((reference) => reference.kind !== "character");
|
||||||
|
return [value, block("【角色参考】", characters), block("【场景与道具参考】", otherAssets)].filter(Boolean).join("\n\n");
|
||||||
|
}
|
||||||
|
|
||||||
export function compileShotAssetReferencePrompt(references: ShotPromptAssetReference[]) {
|
export function compileShotAssetReferencePrompt(references: ShotPromptAssetReference[]) {
|
||||||
if (!references.length) return "";
|
if (!references.length) return "";
|
||||||
const lines = references.flatMap((reference) => {
|
const lines = references.flatMap((reference) => {
|
||||||
|
|||||||
@@ -366,11 +366,29 @@
|
|||||||
|
|
||||||
.workflow-model-picker.canvas-composer-model-picker {
|
.workflow-model-picker.canvas-composer-model-picker {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
height: 40px;
|
height: 44px;
|
||||||
justify-content: space-between;
|
justify-content: space-between;
|
||||||
border-color: var(--workspace-border);
|
border: 1px solid var(--workspace-border-strong);
|
||||||
border-radius: var(--r-md);
|
border-radius: var(--r-md);
|
||||||
background: var(--workspace-surface-strong);
|
background: var(--workspace-surface-strong);
|
||||||
|
padding-inline: 11px;
|
||||||
|
transition: border-color var(--motion-instant) ease-out, background-color var(--motion-instant) ease-out, box-shadow var(--motion-instant) ease-out;
|
||||||
|
}
|
||||||
|
|
||||||
|
.workflow-model-picker.canvas-composer-model-picker:hover {
|
||||||
|
border-color: color-mix(in srgb, var(--workspace-accent) 48%, var(--workspace-border-strong));
|
||||||
|
}
|
||||||
|
|
||||||
|
.workflow-model-picker.canvas-composer-model-picker:focus-visible,
|
||||||
|
.workflow-model-picker.canvas-composer-model-picker[aria-expanded="true"] {
|
||||||
|
border-color: color-mix(in srgb, var(--workspace-accent) 72%, var(--workspace-border-strong));
|
||||||
|
box-shadow: 0 0 0 2px color-mix(in srgb, var(--workspace-accent) 18%, transparent) !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
.workflow-model-picker .canvas-model-picker-chevron {
|
||||||
|
width: 17px;
|
||||||
|
height: 17px;
|
||||||
|
color: color-mix(in srgb, var(--foreground) 72%, transparent);
|
||||||
}
|
}
|
||||||
|
|
||||||
.workflow-settings-section + .workflow-settings-section {
|
.workflow-settings-section + .workflow-settings-section {
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ describe("章节分镜写入分镜制作", () => {
|
|||||||
cameraAngle: "平视,35mm",
|
cameraAngle: "平视,35mm",
|
||||||
cameraMovement: "缓慢推进",
|
cameraMovement: "缓慢推进",
|
||||||
imagePrompt: "雨夜旧屋中的林默",
|
imagePrompt: "雨夜旧屋中的林默",
|
||||||
videoPrompt: "林默推门,镜头缓慢推进",
|
videoPrompt: "林默推门,镜头缓慢推进\n\n【角色参考】\n林默:@[asset:character-1]\n\n【场景与道具参考】\n旧信封:@[asset:prop-1]",
|
||||||
continuityNotes: "林默停在门内,右手仍扶着门把",
|
continuityNotes: "林默停在门内,右手仍扶着门把",
|
||||||
actionBeats: [{ description: "0-2 秒推门;2-5 秒环顾" }],
|
actionBeats: [{ description: "0-2 秒推门;2-5 秒环顾" }],
|
||||||
},
|
},
|
||||||
|
|||||||
12
web/test/workflow-model-picker-style.test.ts
Normal file
12
web/test/workflow-model-picker-style.test.ts
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
import { expect, test } from "bun:test";
|
||||||
|
|
||||||
|
test("分镜生成模型选择器具有明确的下拉控件状态", async () => {
|
||||||
|
const css = await Bun.file(new URL("../src/pages/projects/detail/workflow.css", import.meta.url)).text();
|
||||||
|
const rule = css.match(/\.workflow-model-picker\.canvas-composer-model-picker\s*\{([\s\S]*?)\}/)?.[1] || "";
|
||||||
|
|
||||||
|
expect(rule).toContain("border: 1px solid var(--workspace-border-strong)");
|
||||||
|
expect(rule).toContain("height: 44px");
|
||||||
|
expect(css).toContain('.workflow-model-picker.canvas-composer-model-picker[aria-expanded="true"]');
|
||||||
|
expect(css).toContain(".workflow-model-picker .canvas-model-picker-chevron");
|
||||||
|
expect(css).toContain("box-shadow: 0 0 0 2px");
|
||||||
|
});
|
||||||
@@ -6,6 +6,13 @@ import type { GenerationTask } from "../src/services/api/task-center";
|
|||||||
import type { ProjectDetail } from "../src/services/api/projects";
|
import type { ProjectDetail } from "../src/services/api/projects";
|
||||||
import { buildShotAssetReferenceContext, resolveShotAssetMentionPrompt } from "../src/pages/projects/detail/workflow-shot-references";
|
import { buildShotAssetReferenceContext, resolveShotAssetMentionPrompt } from "../src/pages/projects/detail/workflow-shot-references";
|
||||||
|
|
||||||
|
test("production workbench does not silently drop bound voice samples before backend validation", async () => {
|
||||||
|
const source = await Bun.file(new URL("../src/pages/projects/detail/workflow-production-workbench.tsx", import.meta.url)).text();
|
||||||
|
|
||||||
|
expect(source).toContain('const generationReferenceAudios = generationCapability === "video" ? shotAssetReferenceContext.referenceAudios : [];');
|
||||||
|
expect(source).not.toContain("selectedVideoProfile?.references.maxAudios");
|
||||||
|
});
|
||||||
|
|
||||||
test("shot generation submits historical character image, current voice and asset prompt", async () => {
|
test("shot generation submits historical character image, current voice and asset prompt", async () => {
|
||||||
const detail = {
|
const detail = {
|
||||||
assets: [
|
assets: [
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { describe, expect, test } from "bun:test";
|
|||||||
|
|
||||||
import { canvasResourceMentionToken } from "@/lib/canvas/canvas-resource-references";
|
import { canvasResourceMentionToken } from "@/lib/canvas/canvas-resource-references";
|
||||||
import type { ProjectAsset, ProjectDetail } from "@/services/api/projects";
|
import type { ProjectAsset, ProjectDetail } from "@/services/api/projects";
|
||||||
import { buildShotAssetReferenceContext, resolveShotAssetMentionPrompt } from "@/pages/projects/detail/workflow-shot-references";
|
import { buildShotAssetReferenceContext, ensureShotAssetMentionPrompt, resolveShotAssetMentionPrompt } from "@/pages/projects/detail/workflow-shot-references";
|
||||||
|
|
||||||
describe("workflow shot asset references", () => {
|
describe("workflow shot asset references", () => {
|
||||||
test("builds ordered image references for bound media and characters", () => {
|
test("builds ordered image references for bound media and characters", () => {
|
||||||
@@ -26,6 +26,15 @@ describe("workflow shot asset references", () => {
|
|||||||
|
|
||||||
expect(() => resolveShotAssetMentionPrompt("跟随 @[asset:missing]", context)).toThrow("未绑定到当前镜头");
|
expect(() => resolveShotAssetMentionPrompt("跟随 @[asset:missing]", context)).toThrow("未绑定到当前镜头");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("adds bound assets as editable mention tokens without duplicating existing mentions", () => {
|
||||||
|
const context = buildShotAssetReferenceContext(detail(), "shot-1");
|
||||||
|
const prompt = ensureShotAssetMentionPrompt("让 @[asset:asset-character] 走近镜头", context.mentionReferences);
|
||||||
|
|
||||||
|
expect(prompt).toContain("@[asset:asset-character]");
|
||||||
|
expect(prompt.match(/@\[asset:asset-character\]/g)).toHaveLength(1);
|
||||||
|
expect(prompt).toContain("【场景与道具参考】\n雨夜街道:@[asset:asset-scene]");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
function detail(): ProjectDetail {
|
function detail(): ProjectDetail {
|
||||||
|
|||||||
Reference in New Issue
Block a user