From f2fe163bc793f421a8b26855dff01d4d3d97b017 Mon Sep 17 00:00:00 2001 From: ddCat <2695092045@qq.com> Date: Sun, 30 Aug 2026 17:54:23 +0800 Subject: [PATCH] =?UTF-8?q?fix(canvas):=20=E5=88=86=E9=95=9C=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E7=94=9F=E6=88=90=20-=20=E6=AD=A3=E7=A1=AE=E9=99=84?= =?UTF-8?q?=E5=8A=A0=E8=A7=92=E8=89=B2=E5=9C=BA=E6=99=AF=E4=B8=8E=E5=A3=B0?= =?UTF-8?q?=E9=9F=B3=E8=B5=84=E4=BA=A7=E5=8F=8A=E9=95=9C=E5=A4=B4=E5=8F=B0?= =?UTF-8?q?=E8=AF=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../repository/project_workbench_read.go | 21 ++++++ .../service/project_workbench_read.go | 69 ++++++++++++++++++- .../service/project_workbench_read_test.go | 47 +++++++++++++ docs/content/docs/progress/pending-test.mdx | 2 + .../detail/workflow-production-workbench.tsx | 12 +++- .../detail/workflow-shot-references.ts | 66 ++++++++++++++---- web/src/services/api/projects.ts | 7 ++ web/test/workflow-shot-generation.test.ts | 67 ++++++++++++++++-- web/test/workflow-shot-references.test.ts | 2 +- 9 files changed, 266 insertions(+), 27 deletions(-) diff --git a/backend/internal/repository/project_workbench_read.go b/backend/internal/repository/project_workbench_read.go index f8b1475b1..340f5c6fc 100644 --- a/backend/internal/repository/project_workbench_read.go +++ b/backend/internal/repository/project_workbench_read.go @@ -109,6 +109,27 @@ func (r *Repository) ProjectUnitShotAssetReferences(projectID string, unitID str return references, err } +func (r *Repository) ProjectAssetVersionsByIDs(projectID string, versionIDs []string) ([]model.AssetVersion, error) { + if len(versionIDs) == 0 { + return []model.AssetVersion{}, nil + } + var versions []model.AssetVersion + err := r.db.Table("asset_versions").Select("asset_versions.*"). + Joins("JOIN project_asset_links ON project_asset_links.asset_id = asset_versions.asset_id"). + Where("project_asset_links.project_id = ? AND asset_versions.id IN ?", projectID, versionIDs). + Find(&versions).Error + return versions, err +} + +func (r *Repository) AssetRepresentationsByVersionIDs(versionIDs []string) ([]model.AssetRepresentation, error) { + if len(versionIDs) == 0 { + return []model.AssetRepresentation{}, nil + } + var representations []model.AssetRepresentation + err := r.db.Where("asset_version_id IN ?", versionIDs).Order("asset_version_id asc, role asc, created_at asc").Find(&representations).Error + return representations, err +} + func (r *Repository) ProjectUnitAssetCandidates(projectID string, unitID string) ([]model.ProjectAssetCandidate, error) { var candidates []model.ProjectAssetCandidate err := r.db.Where("project_id = ? AND (unit_id = ? OR unit_id = '')", projectID, unitID).Order("created_at asc").Find(&candidates).Error diff --git a/backend/internal/service/project_workbench_read.go b/backend/internal/service/project_workbench_read.go index 360f0b803..c32539a81 100644 --- a/backend/internal/service/project_workbench_read.go +++ b/backend/internal/service/project_workbench_read.go @@ -49,12 +49,28 @@ type ProjectUnitWorkspace struct { Shots []model.Shot `json:"shots"` ShotRevisions []model.ShotRevision `json:"shotRevisions"` ShotArtifacts []model.ShotArtifact `json:"shotArtifacts"` - ShotReferences []model.ShotAssetReference `json:"shotReferences"` + ShotReferences []ProjectShotAssetReference `json:"shotReferences"` AssetCandidates []model.ProjectAssetCandidate `json:"assetCandidates"` Assets []ProjectAssetSummary `json:"assets"` Tasks []TaskSummary `json:"tasks"` } +// ProjectShotAssetReference keeps the immutable visual version selected by the +// shot while also exposing the asset's current card configuration (notably its +// voice binding) for generation-time prompt assembly. +type ProjectShotAssetReference struct { + model.ShotAssetReference + Asset ProjectAssetSummary `json:"asset"` + ReferencedVersion ProjectShotAssetReferenceVersion `json:"referencedVersion"` +} + +type ProjectShotAssetReferenceVersion struct { + ID string `json:"id"` + AssetID string `json:"assetId"` + Version int `json:"version"` + Representations []CharacterRepresentationSummary `json:"representations"` +} + type ProjectCanvasPage struct { Canvases []model.CanvasProject `json:"canvases"` CanvasUnitLinks []model.CanvasUnitLink `json:"canvasUnitLinks"` @@ -167,6 +183,7 @@ func (s *Service) ProjectUnitWorkspace(userID string, projectID string, unitID s } result := ProjectUnitWorkspace{Unit: *unit} var projectTasks []TaskSummary + var shotReferences []model.ShotAssetReference var group errgroup.Group group.Go(func() error { var err error @@ -185,7 +202,7 @@ func (s *Service) ProjectUnitWorkspace(userID string, projectID string, unitID s }) group.Go(func() error { var err error - result.ShotReferences, err = s.repo.ProjectUnitShotAssetReferences(projectID, unitID) + shotReferences, err = s.repo.ProjectUnitShotAssetReferences(projectID, unitID) return err }) group.Go(func() error { @@ -251,6 +268,54 @@ func (s *Service) ProjectUnitWorkspace(userID string, projectID string, unitID s if err := group.Wait(); err != nil { return ProjectUnitWorkspace{}, err } + assetByID := make(map[string]ProjectAssetSummary, len(result.Assets)) + for _, asset := range result.Assets { + assetByID[asset.ID] = asset + } + versionIDs := make([]string, 0, len(shotReferences)) + for _, reference := range shotReferences { + versionIDs = append(versionIDs, reference.AssetVersionID) + } + versions, err := s.repo.ProjectAssetVersionsByIDs(projectID, versionIDs) + if err != nil { + return ProjectUnitWorkspace{}, err + } + storedRepresentations, err := s.repo.AssetRepresentationsByVersionIDs(versionIDs) + if err != nil { + return ProjectUnitWorkspace{}, err + } + versionByID := make(map[string]model.AssetVersion, len(versions)) + for _, version := range versions { + versionByID[version.ID] = version + } + representationsByVersionID := make(map[string][]CharacterRepresentationSummary, len(versionIDs)) + for _, representation := range storedRepresentations { + representationsByVersionID[representation.AssetVersionID] = append(representationsByVersionID[representation.AssetVersionID], CharacterRepresentationSummary{ + ID: representation.ID, ResourceID: representation.ResourceID, MediaType: representation.MediaType, Role: representation.Role, + }) + } + result.ShotReferences = make([]ProjectShotAssetReference, 0, len(shotReferences)) + for _, reference := range shotReferences { + version, exists := versionByID[reference.AssetVersionID] + if !exists { + return ProjectUnitWorkspace{}, BadAuthRequest("镜头引用的资产版本不可用") + } + asset, exists := assetByID[version.AssetID] + if !exists { + return ProjectUnitWorkspace{}, BadAuthRequest("镜头引用的项目资产不可用") + } + representations := representationsByVersionID[version.ID] + if representations == nil { + representations = []CharacterRepresentationSummary{} + } + result.ShotReferences = append(result.ShotReferences, ProjectShotAssetReference{ + ShotAssetReference: reference, + Asset: asset, + ReferencedVersion: ProjectShotAssetReferenceVersion{ + ID: version.ID, AssetID: version.AssetID, Version: version.Version, Representations: representations, + }, + }) + } shotIDs := make(map[string]struct{}, len(result.Shots)) for _, shot := range result.Shots { shotIDs[shot.ID] = struct{}{} diff --git a/backend/internal/service/project_workbench_read_test.go b/backend/internal/service/project_workbench_read_test.go index d49e24a4d..c1a1668ad 100644 --- a/backend/internal/service/project_workbench_read_test.go +++ b/backend/internal/service/project_workbench_read_test.go @@ -21,6 +21,7 @@ func newProjectWorkbenchReadTestService(t *testing.T) (*Service, *gorm.DB) { if err := db.AutoMigrate( &model.Project{}, &model.ProjectUnit{}, &model.CanvasProject{}, &model.CanvasUnitLink{}, &model.Asset{}, &model.AssetVersion{}, &model.AssetRepresentation{}, &model.ProjectAssetLink{}, &model.ProjectAssetCandidate{}, + &model.CharacterVoiceBinding{}, &model.VoiceProfile{}, &model.Shot{}, &model.ShotRevision{}, &model.ShotArtifact{}, &model.ShotAssetReference{}, &model.WorkflowInstance{}, &model.WorkflowStepInstance{}, &model.Task{}, ); err != nil { @@ -29,6 +30,52 @@ func newProjectWorkbenchReadTestService(t *testing.T) (*Service, *gorm.DB) { return &Service{repo: repository.New(db)}, db } +func TestProjectUnitWorkspaceResolvesHistoricalVisualAndCurrentCharacterVoice(t *testing.T) { + service, db := newProjectWorkbenchReadTestService(t) + project := seedWorkbenchProject(t, db) + unit := model.ProjectUnit{ID: "unit-1", ProjectID: project.ID, Title: "第一章", Status: model.ProjectUnitStatusReady} + asset := model.Asset{ + ID: "character-1", UserID: "user-1", Title: "张天昊", Kind: "entity", Category: model.AssetCategoryCharacter, + Status: model.AssetVersionStatusConfirmed, PrimaryVersionID: "character-v2", PayloadJSON: `{}`, + } + seed := []any{ + &unit, + &asset, + &model.ProjectAssetLink{ID: "link-1", ProjectID: project.ID, AssetID: asset.ID}, + &model.AssetVersion{ID: "character-v1", AssetID: asset.ID, Version: 1, Status: model.AssetVersionStatusConfirmed, DefinitionJSON: `{}`}, + &model.AssetVersion{ID: "character-v2", AssetID: asset.ID, Version: 2, Status: model.AssetVersionStatusConfirmed, DefinitionJSON: `{"voiceLanguage":"普通话","voiceAge":"青年男性","voiceTimbre":"略带疲惫和震惊"}`}, + &model.AssetRepresentation{ID: "representation-v1", TaskID: "visual-v1", AssetVersionID: "character-v1", ResourceID: "character-image-v1", MediaType: "image/png", Role: "primary"}, + &model.AssetRepresentation{ID: "representation-v2", TaskID: "visual-v2", AssetVersionID: "character-v2", ResourceID: "character-image-v2", MediaType: "image/png", Role: "primary"}, + &model.VoiceProfile{ID: "voice-1", UserID: "user-1", Name: "张天昊声音", Provider: "custom", VoiceKey: "voice-1", Language: "普通话", Timbre: "青年男声", SampleResourceID: "voice-sample-1", Status: "active", CompatibleModelsJSON: `[]`}, + &model.CharacterVoiceBinding{ID: "binding-1", AssetVersionID: "character-v2", VoiceProfileID: "voice-1", Instructions: "内心独白语气"}, + &model.Shot{ID: "shot-1", ProjectID: project.ID, UnitID: unit.ID, Position: 0}, + &model.ShotAssetReference{ID: "reference-1", ShotID: "shot-1", AssetVersionID: "character-v1", Role: "reference", Status: "linked"}, + } + for _, item := range seed { + if err := db.Create(item).Error; err != nil { + t.Fatal(err) + } + } + + workspace, err := service.ProjectUnitWorkspace("user-1", project.ID, unit.ID) + if err != nil { + t.Fatal(err) + } + if len(workspace.ShotReferences) != 1 { + t.Fatalf("shot references = %d, want 1", len(workspace.ShotReferences)) + } + reference := workspace.ShotReferences[0] + if reference.ReferencedVersion.ID != "character-v1" || len(reference.ReferencedVersion.Representations) != 1 || reference.ReferencedVersion.Representations[0].ResourceID != "character-image-v1" { + t.Fatalf("historical visual snapshot not preserved: %+v", reference.ReferencedVersion) + } + if reference.Asset.Character == nil || reference.Asset.Character.VersionID != "character-v2" || reference.Asset.Character.Voice == nil { + t.Fatalf("current character card not resolved: %+v", reference.Asset.Character) + } + if reference.Asset.Character.Voice.Profile.SampleResourceID != "voice-sample-1" || reference.Asset.Character.Voice.Instructions != "内心独白语气" { + t.Fatalf("current character voice not resolved: %+v", reference.Asset.Character.Voice) + } +} + func seedWorkbenchProject(t *testing.T, db *gorm.DB) model.Project { t.Helper() project := model.Project{ID: "project-1", UserID: "user-1", Name: "长篇短剧", Status: model.ProjectStatusActive} diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index b50c10579..88f392d6d 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -531,10 +531,12 @@ description: 影策已实现功能的手工验证清单与未确认边界 - [x] 镜头保存使用真实版本写接口；资产点击绑定具体版本；生成统一进入后端任务生命周期，成功后登记对应镜头产物；下载使用带登录凭据的资源接口。 - [x] “生成设置”新增技能库多选，分镜图、动作预演和镜头视频统一通过 Skill Runtime 解析，任务与镜头产物记录技能版本和实际引用文件。 - [x] 镜头资产支持再次点击或从已绑定标签取消引用；解绑使用归属校验的独立写接口，并使该镜头旧产物失效。动态提示词复用画布 `@` 引用编辑器，已绑定图片和角色会按实际参考图顺序解析后提交；后端全量测试、前端专项测试和生产构建通过。 +- [x] 镜头生成资产解析改为同时保留绑定版本视觉快照与当前角色卡声音配置；角色主图、场景图和声音样本按 `图片N` / `音频N` 写入 `【资产参考】`，并自动补充声音画像与镜头台词。历史角色版本 + 新版声音绑定的后端/前端专项回归通过。 - [x] 镜头编辑器底部在“保存脚本”左侧新增“删除镜头”；二次确认后原子清理该镜头的脚本版本、资产引用、生成产物、候选资产和生产任务关联，压紧剩余镜头顺序并自动切换到相邻镜头。 - [ ] 使用包含角色、场景、道具和画风的真实项目，确认左侧分类、绑定状态、参考素材标签和镜头切换保持一致。 - [ ] 分别删除首个、中间、末尾和唯一镜头，确认二次提示、相邻镜头切换、空态、镜头编号、总镜数以及刷新后的数据一致；有未保存修改时确认提示明确且取消不会丢失内容。 - [ ] 在同一镜头绑定角色和场景后输入 `@`，确认菜单、行内缩略图和生成请求中的“图片 N”顺序一致；分别从左侧资产行和已绑定标签取消引用，确认刷新后状态一致，旧 token 会阻止生成并提示重新选择或删除引用。 +- [ ] 使用“分镜绑定角色旧版本、角色卡随后绑定声音”的真实项目生成镜头视频，确认请求包含旧版角色主图、场景图、当前声音样本、声音画像和镜头台词，且媒体角色为参考资产而非首帧。 - [ ] 配置一个图片模型和一个视频模型，分别生成分镜预览、动作预演与视频，确认任务中心、镜头历史、最新预览和阶段状态同步更新。 - [ ] 在浅色/深色主题及 1280px、1050px、760px 三个断点检查项目导航、表单滚动、右侧预览和底部镜头轨道，确认没有遮挡或不可达控件。 - [ ] 验证下载视频、复制生成参数、将任务物化产物加入项目素材库、章节切换与新增分镜；确认失败任务展示真实错误且不会伪造产物成功。 diff --git a/web/src/pages/projects/detail/workflow-production-workbench.tsx b/web/src/pages/projects/detail/workflow-production-workbench.tsx index 9202d09e2..392952886 100644 --- a/web/src/pages/projects/detail/workflow-production-workbench.tsx +++ b/web/src/pages/projects/detail/workflow-production-workbench.tsx @@ -118,7 +118,13 @@ export default function WorkflowProductionWorkbench(props: Props) { const [selectedSkillIds, setSelectedSkillIds] = useState([]); const { skills: availableSkills, loading: skillsLoading } = useSkillRuntimeCatalog(); const shotAssetReferenceContext = useMemo(() => buildShotAssetReferenceContext(detail, selectedShot?.id || ""), [detail, selectedShot?.id]); - const referenceByVersionId = useMemo(() => new Map((detail.shotReferences || []).filter((reference) => reference.shotId === selectedShot?.id && reference.role === "reference" && reference.status === "linked").map((reference) => [reference.assetVersionId, reference])), [detail.shotReferences, selectedShot?.id]); + const referenceByVersionId = useMemo(() => { + const references = (detail.shotReferences || []).filter((reference) => reference.shotId === selectedShot?.id && reference.role === "reference" && reference.status === "linked"); + return new Map(references.flatMap((reference) => [ + [reference.assetVersionId, reference] as const, + ...(reference.asset?.primaryVersionId ? [[reference.asset.primaryVersionId, reference] as const] : []), + ])); + }, [detail.shotReferences, selectedShot?.id]); const currentDurationSeconds = Number(watchedDuration || Math.max(0.5, (selectedShot?.durationMs || 3000) / 1000)); const generationSeconds = String(Math.max(1, Math.round(currentDurationSeconds))); const selectedVideoProfile = generationCapability === "video" && selectedModel ? modelCapabilityConfigFor(effectiveConfig, selectedModel).video : undefined; @@ -324,7 +330,7 @@ export default function WorkflowProductionWorkbench(props: Props) { const basePrompt = mode === "video" ? [values.videoPrompt || values.plotDescription, values.action, values.dialogue && `台词：${values.dialogue}`, values.continuityNotes].filter(Boolean).join("\n") : [values.imagePrompt || values.plotDescription, values.action, "黑白分镜草图，清晰动作节拍，电影构图"].filter(Boolean).join("\n"); - const resolvedPrompt = resolveShotAssetMentionPrompt(basePrompt, shotAssetReferenceContext); + const resolvedPrompt = resolveShotAssetMentionPrompt(basePrompt, shotAssetReferenceContext, { dialogue: values.dialogue }); const skillExecution = await skillRuntime.prepare({ profile: "shortDrama", prompt: resolvedPrompt, @@ -637,7 +643,7 @@ function BoundAssets({ detail, shotId, changing, onUnlink }: { detail: ProjectDe
{references.length ? references.map((reference) => { - const asset = assetByVersionId.get(reference.assetVersionId); + const asset = reference.asset || assetByVersionId.get(reference.assetVersionId); const title = asset?.title || "历史资产版本"; const previewUrl = asset ? assetPreviewUrl(asset) : ""; return
diff --git a/web/src/pages/projects/detail/workflow-shot-references.ts b/web/src/pages/projects/detail/workflow-shot-references.ts index c59fc140f..262b036bf 100644 --- a/web/src/pages/projects/detail/workflow-shot-references.ts +++ b/web/src/pages/projects/detail/workflow-shot-references.ts @@ -1,6 +1,6 @@ import type { CanvasResourceReference } from "@/lib/canvas/canvas-resource-references"; import { resourceFileUrl, resourceIdFromStorageKey, resourceStorageKey } from "@/services/api/resources"; -import type { ProjectAsset, ProjectDetail } from "@/services/api/projects"; +import type { CharacterRepresentation, ProjectAsset, ProjectDetail, ShotAssetReference } from "@/services/api/projects"; import type { AssetCategory } from "@/stores/use-asset-store"; import type { ReferenceImage } from "@/types/image"; import type { ReferenceAudio } from "@/types/media"; @@ -20,6 +20,7 @@ export type ShotPromptAssetReference = { imageIndex: number; audioIndex?: number; voiceDescription?: string; + dialogue?: string; }; const assetCategories = new Set(["character", "environment", "wardrobe", "prop", "weapon", "style", "other"]); @@ -29,12 +30,12 @@ export function buildShotAssetReferenceContext(detail: ProjectDetail, shotId: st const seenAssetIds = new Set(); const entries = (detail.shotReferences || []).flatMap((reference) => { if (reference.shotId !== shotId || reference.status !== "linked") return []; - const asset = assetByVersionId.get(reference.assetVersionId); + const asset = reference.asset || assetByVersionId.get(reference.assetVersionId); if (!asset || seenAssetIds.has(asset.id)) return []; - const image = projectAssetReferenceImage(asset); + const image = projectAssetReferenceImage(asset, reference); if (!image) return []; seenAssetIds.add(asset.id); - return [{ asset, image }]; + return [{ asset, image, reference }]; }); const referenceAudios: ReferenceAudio[] = []; @@ -82,11 +83,11 @@ export function buildShotAssetReferenceContext(detail: ProjectDetail, shotId: st referenceImages: entries.map(({ image }) => image), referenceAudios, assetReferences, - resolvedCharacterVersions: entries.flatMap(({ asset }) => asset.character?.versionId ? [{ assetId: asset.id, versionId: asset.character.versionId }] : []), + resolvedCharacterVersions: entries.flatMap(({ asset, reference }) => asset.character ? [{ assetId: asset.id, versionId: reference.referencedVersion?.id || reference.assetVersionId || asset.character.versionId }] : []), }; } -export function resolveShotAssetMentionPrompt(prompt: string, context: ShotAssetReferenceContext) { +export function resolveShotAssetMentionPrompt(prompt: string, context: ShotAssetReferenceContext, options: { dialogue?: string } = {}) { const imageLabelByAssetId = new Map(context.mentionReferences.map((reference, index) => [reference.assetId, `图片${index + 1}`])); const unresolved = new Set(); const resolved = prompt.replace(/@\[asset:([^\]]+)\]/g, (token, assetId: string) => { @@ -98,8 +99,10 @@ export function resolveShotAssetMentionPrompt(prompt: string, context: ShotAsset return label; }); if (unresolved.size) throw new Error(`提示词中的 ${Array.from(unresolved).join("、")} 未绑定到当前镜头，请重新选择资产或删除引用`); - const assetBlock = compileShotAssetReferencePrompt(context.assetReferences); - return [resolved.trim(), assetBlock].filter(Boolean).join("\n\n"); + const dialogueContext = withShotDialogue(context.assetReferences, options.dialogue); + const assetBlock = compileShotAssetReferencePrompt(dialogueContext.references); + const dialogueBlock = dialogueContext.unassignedDialogue ? `【镜头台词】\n\n${dialogueContext.unassignedDialogue}` : ""; + return [resolved.trim(), assetBlock, dialogueBlock].filter(Boolean).join("\n\n"); } export function compileShotAssetReferencePrompt(references: ShotPromptAssetReference[]) { @@ -109,18 +112,17 @@ export function compileShotAssetReferencePrompt(references: ShotPromptAssetRefer const media = [`${visualLabel}：图片${reference.imageIndex}`, reference.audioIndex ? `声音参考：音频${reference.audioIndex}` : ""].filter(Boolean).join("；"); return [ `- ${reference.title}：${media}`, - reference.voiceDescription ? ` 声音说明：${reference.voiceDescription}` : "", + reference.voiceDescription ? ` 声音画像：${reference.voiceDescription}` : "", + reference.dialogue ? ` 镜头台词：${reference.dialogue}` : "", ].filter(Boolean); }); return ["【资产参考】", "", ...lines].join("\n"); } -function projectAssetReferenceImage(asset: ProjectAsset): ReferenceImage | undefined { - const representation = asset.character - ? asset.character.representations.find((item) => item.role === "turnaround_sheet") - || asset.character.representations.find((item) => item.role === "primary") - || asset.character.representations.find((item) => item.role === "front") - : undefined; +function projectAssetReferenceImage(asset: ProjectAsset, reference?: ShotAssetReference): ReferenceImage | undefined { + const boundRepresentations = reference?.referencedVersion?.representations || []; + const representation = preferredVisualRepresentation(boundRepresentations) + || (asset.character ? preferredVisualRepresentation(asset.character.representations) : undefined); if (representation) { return { id: asset.id, @@ -143,6 +145,40 @@ function projectAssetReferenceImage(asset: ProjectAsset): ReferenceImage | undef }; } +function preferredVisualRepresentation(representations: CharacterRepresentation[]) { + return representations.find((item) => item.role === "turnaround_sheet") + || representations.find((item) => item.role === "primary") + || representations.find((item) => item.role === "front") + || representations.find((item) => item.mediaType.startsWith("image")); +} + +function withShotDialogue(references: ShotPromptAssetReference[], dialogue?: string) { + const value = stringValue(dialogue); + if (!value) return { references, unassignedDialogue: "" }; + const characters = references.filter((reference) => reference.category === "character"); + if (characters.length === 1) { + return { + references: references.map((reference) => reference.assetId === characters[0]?.assetId ? { ...reference, dialogue: value } : reference), + unassignedDialogue: "", + }; + } + const resolved = references.map((reference) => { + if (reference.category !== "character") return reference; + const matched = dialogueForCharacter(value, reference.title); + return matched ? { ...reference, dialogue: matched } : reference; + }); + return { + references: resolved, + unassignedDialogue: resolved.some((reference) => reference.dialogue) ? "" : value, + }; +} + +function dialogueForCharacter(dialogue: string, characterName: string) { + const escaped = characterName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const pattern = new RegExp(`(?:^|\\n)\\s*${escaped}\\s*[：:]\\s*([^\\n]+)`, "g"); + return Array.from(dialogue.matchAll(pattern), (match) => match[1]?.trim()).filter(Boolean).join("；"); +} + function projectAssetCategory(value: string): AssetCategory { return assetCategories.has(value as AssetCategory) ? value as AssetCategory : "other"; } diff --git a/web/src/services/api/projects.ts b/web/src/services/api/projects.ts index 2567f2f23..32e24fc97 100644 --- a/web/src/services/api/projects.ts +++ b/web/src/services/api/projects.ts @@ -202,6 +202,13 @@ export type ShotAssetReference = { role: "reference" | "start_frame" | "end_frame" | "keyframe" | "storyboard" | "output" | string; status: string; createdAt: string; + asset?: ProjectAsset; + referencedVersion?: { + id: string; + assetId: string; + version: number; + representations: CharacterRepresentation[]; + }; }; export type WorkflowStep = { diff --git a/web/test/workflow-shot-generation.test.ts b/web/test/workflow-shot-generation.test.ts index c520de257..48cfb7504 100644 --- a/web/test/workflow-shot-generation.test.ts +++ b/web/test/workflow-shot-generation.test.ts @@ -6,7 +6,7 @@ import type { GenerationTask } from "../src/services/api/task-center"; import type { ProjectDetail } from "../src/services/api/projects"; import { buildShotAssetReferenceContext, resolveShotAssetMentionPrompt } from "../src/pages/projects/detail/workflow-shot-references"; -test("shot asset prompt keeps character names and maps character image, voice and scene assets", () => { +test("shot generation submits historical character image, current voice and asset prompt", async () => { const detail = { assets: [ { @@ -14,9 +14,9 @@ test("shot asset prompt keeps character names and maps character image, voice an title: "张天昊", category: "character", mediaType: "image", - primaryVersionId: "character-version-1", + primaryVersionId: "character-version-2", character: { - versionId: "character-version-1", + versionId: "character-version-2", definition: { voiceLanguage: "普通话", voiceAge: "青年男性", voiceTimbre: "略带疲惫和震惊" }, representations: [{ id: "representation-1", resourceId: "character-image-1", mediaType: "image/png", role: "primary" }], voice: { profile: { sampleResourceId: "character-audio-1", language: "普通话", timbre: "沉稳" }, instructions: "内心独白语气" }, @@ -32,22 +32,77 @@ test("shot asset prompt keeps character names and maps character image, voice an }, ], shotReferences: [ - { shotId: "shot-1", assetVersionId: "character-version-1", status: "linked" }, + { + shotId: "shot-1", + assetVersionId: "character-version-1", + status: "linked", + asset: { + id: "character-1", + title: "张天昊", + category: "character", + mediaType: "entity", + primaryVersionId: "character-version-2", + character: { + versionId: "character-version-2", + definition: { voiceLanguage: "普通话", voiceAge: "青年男性", voiceTimbre: "略带疲惫和震惊" }, + representations: [{ id: "representation-2", resourceId: "character-image-2", mediaType: "image/png", role: "primary" }], + voice: { profile: { sampleResourceId: "character-audio-1", language: "普通话", timbre: "沉稳" }, instructions: "内心独白语气" }, + }, + }, + referencedVersion: { + id: "character-version-1", + assetId: "character-1", + version: 1, + representations: [{ id: "representation-1", resourceId: "character-image-1", mediaType: "image/png", role: "primary" }], + }, + }, { shotId: "shot-1", assetVersionId: "scene-version-1", status: "linked" }, ], } as ProjectDetail; const context = buildShotAssetReferenceContext(detail, "shot-1"); - const prompt = resolveShotAssetMentionPrompt("张天昊在 @[asset:scene-1] 睁开眼睛", context); + const prompt = resolveShotAssetMentionPrompt("张天昊在 @[asset:scene-1] 睁开眼睛", context, { dialogue: "我穿越了？瓦西国张家废柴少主？" }); expect(context.referenceImages).toHaveLength(2); + expect(context.referenceImages[0]?.storageKey).toBe("resource:character-image-1"); expect(context.referenceAudios).toHaveLength(1); expect(context.referenceAudios[0]?.storageKey).toBe("resource:character-audio-1"); expect(context.resolvedCharacterVersions).toEqual([{ assetId: "character-1", versionId: "character-version-1" }]); expect(prompt).toContain("张天昊在 图片2 睁开眼睛"); expect(prompt).toContain("- 张天昊：人物参考：图片1；声音参考：音频1"); - expect(prompt).toContain("声音说明：普通话；青年男性；略带疲惫和震惊；内心独白语气"); + expect(prompt).toContain("声音画像：普通话；青年男性；略带疲惫和震惊；内心独白语气"); + expect(prompt).toContain("镜头台词：我穿越了？瓦西国张家废柴少主？"); expect(prompt).toContain("- 坑底场景：场景参考：图片2"); + + let createdInput: Parameters[0] | undefined; + const task = { + id: "shot-task-1", + type: "canvas_video", + status: "queued", + prompt, + attempts: 0, + createdAt: "2026-08-30T00:00:00.000Z", + updatedAt: "2026-08-30T00:00:00.000Z", + } satisfies GenerationTask; + await submitBackendGenerationTask({ + projectId: "project-1", + mode: "video", + prompt, + config: { ...defaultConfig, model: "MiniMax-H3", videoModel: "MiniMax-H3" }, + referenceImages: context.referenceImages, + referenceAudios: context.referenceAudios, + metadata: { shotId: "shot-1", videoEditOperation: "reference_to_video" }, + }, { + createTask: async (input) => { createdInput = input; return task; }, + waitTask: async () => { throw new Error("should not wait"); }, + runLocal: async () => ({ mode: "video" }), + createId: () => "id-1", + now: () => "2026-08-30T00:00:00.000Z", + }); + + expect(createdInput?.prompt).toContain("【资产参考】"); + expect(createdInput?.input.referenceImages.map((reference) => reference.storageKey)).toEqual(["resource:character-image-1", "resource:scene-image-1"]); + expect(createdInput?.input.referenceAudios.map((reference) => reference.storageKey)).toEqual(["resource:character-audio-1"]); }); test("background generation submission returns after task creation without waiting", async () => { diff --git a/web/test/workflow-shot-references.test.ts b/web/test/workflow-shot-references.test.ts index d65efcfb2..7be7b373b 100644 --- a/web/test/workflow-shot-references.test.ts +++ b/web/test/workflow-shot-references.test.ts @@ -18,7 +18,7 @@ describe("workflow shot asset references", () => { const [scene, character] = context.mentionReferences; const prompt = `让 ${canvasResourceMentionToken(character)} 从 ${canvasResourceMentionToken(scene)} 走近镜头`; - expect(resolveShotAssetMentionPrompt(prompt, context)).toBe("让 图片2 从 图片1 走近镜头"); + expect(resolveShotAssetMentionPrompt(prompt, context)).toContain("让 图片2 从 图片1 走近镜头"); }); test("rejects stale mentions after an asset is unbound", () => {