From 625a97560ba68ef986a90d0403c8680639320bfc Mon Sep 17 00:00:00 2001 From: ddCat <2695092045@qq.com> Date: Tue, 25 Aug 2026 11:24:47 +0800 Subject: [PATCH] =?UTF-8?q?feat(canvas):=20=E5=88=86=E9=95=9C=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=20-=20=E5=85=B3=E8=81=94=E7=94=BB=E5=B8=83=E8=B5=84?= =?UTF-8?q?=E4=BA=A7=E5=B9=B6=E8=87=AA=E5=8A=A8=E7=89=A9=E5=8C=96=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/internal/service/canvas_share.go | 2 +- .../service/prompt_template_defaults.go | 18 +- backend/internal/service/storyboard.go | 250 +++++++++--------- .../service/storyboard_prompt_test.go | 49 +++- .../internal/service/storyboard_workflow.go | 24 +- docs/content/docs/progress/pending-test.mdx | 8 + web/package.json | 2 +- .../components/canvas/canvas-script-node.tsx | 92 +++++-- .../canvas/storyboard-assets-cell.tsx | 107 ++++++++ web/src/constant/canvas.ts | 2 +- web/src/lib/canvas/canvas-node-copy.ts | 2 +- web/src/lib/canvas/canvas-project-domain.ts | 27 +- web/src/lib/canvas/canvas-short-drama.ts | 2 +- .../lib/canvas/canvas-storyboard-assets.ts | 68 +++++ .../canvas/canvas-storyboard-materializer.ts | 76 ++++++ .../lib/canvas/project-chapter-storyboard.ts | 4 +- web/src/pages/canvas/project.tsx | 2 + web/src/pages/canvas/use-canvas-storyboard.ts | 93 +++---- web/src/types/canvas.ts | 14 +- web/test/canvas-batch-connection.test.ts | 2 +- web/test/canvas-storyboard-assets.test.ts | 89 +++++++ 21 files changed, 684 insertions(+), 249 deletions(-) create mode 100644 web/src/components/canvas/storyboard-assets-cell.tsx create mode 100644 web/src/lib/canvas/canvas-storyboard-assets.ts create mode 100644 web/src/lib/canvas/canvas-storyboard-materializer.ts create mode 100644 web/test/canvas-storyboard-assets.test.ts diff --git a/backend/internal/service/canvas_share.go b/backend/internal/service/canvas_share.go index d42d7ae6d..7413309f5 100644 --- a/backend/internal/service/canvas_share.go +++ b/backend/internal/service/canvas_share.go @@ -43,7 +43,7 @@ var publicCanvasMetadataKeys = map[string]bool{ "batchRootId": true, "batchChildIds": true, "batchUsesReferenceImages": true, "primaryImageId": true, "imageBatchExpanded": true, "mimeType": true, "bytes": true, "durationMs": true, "assetTags": true, "workflowKind": true, "workflowTitle": true, "workflowDescription": true, "shotIndex": true, - "sceneId": true, "characterIds": true, "referenceSetId": true, "referenceAssetNodeIds": true, + "sceneId": true, "characterIds": true, "referenceSetId": true, "referenceAssetNodeIds": true, "assetBindings": true, "characterName": true, "characterPrompt": true, "characterAliases": true, "characterView": true, "characterViewNodeIds": true, "videoEditOperation": true, "videoCameraMoveId": true, "videoCameraMovePrompt": true, "videoStartFrameNodeId": true, "videoEndFrameNodeId": true, "versionOfNodeId": true, diff --git a/backend/internal/service/prompt_template_defaults.go b/backend/internal/service/prompt_template_defaults.go index 8a500a684..1858f97c7 100644 --- a/backend/internal/service/prompt_template_defaults.go +++ b/backend/internal/service/prompt_template_defaults.go @@ -139,7 +139,7 @@ const storyboardPlanJSONSchema = `{ "items": { "type": "object", "additionalProperties": false, - "required": ["title", "description", "durationSeconds", "dialogue", "characterIds", "narrativeIntent", "viewerPOV", "performanceBlocking", "shotSize", "emotion", "lightingAndAtmosphere", "audioEffects", "visualPrompt", "videoPrompt", "camera", "motion", "timeBeats", "mustHave", "optionalDetails", "continuityOut", "negativePrompt", "assetTags"], + "required": ["title", "description", "durationSeconds", "dialogue", "characterIds", "narrativeIntent", "viewerPOV", "performanceBlocking", "shotSize", "emotion", "lightingAndAtmosphere", "audioEffects", "visualPrompt", "videoPrompt", "camera", "motion", "timeBeats", "mustHave", "optionalDetails", "continuityOut", "negativePrompt", "assetRefs"], "properties": { "title": {"type": "string"}, "description": {"type": "string"}, @@ -162,7 +162,20 @@ const storyboardPlanJSONSchema = `{ "optionalDetails": {"type": "array", "items": {"type": "string"}}, "continuityOut": {"type": "string"}, "negativePrompt": {"type": "string"}, - "assetTags": {"type": "array", "items": {"type": "string"}} + "assetRefs": { + "type": "array", + "maxItems": 6, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["nodeId", "role", "priority"], + "properties": { + "nodeId": {"type": "string"}, + "role": {"type": "string", "enum": ["character", "environment", "wardrobe", "prop", "weapon", "style", "motion", "audio"]}, + "priority": {"type": "integer", "minimum": 0, "maximum": 100} + } + } + } } } } @@ -231,6 +244,7 @@ func storyboardExecutionContract(durationRule string, countRule string) string { - 单镜头最多 2 名主要角色、1 个主运镜、1 条主要动作链、3 个 timeBeats 和 3 个 mustHave；超限必须拆镜或在固定镜头数内重新分配。 - dialogue 只写本镜头实际念出的台词或简短旁白，字数上限按 1 秒最多约 5 个中文字符计算（至少 24 字）；超长台词/旁白必须拆镜或精简，不得用 dialogue 承载长段叙述。 - characterIds 只能引用当前角色版本中的 assetId；没有角色时返回空数组。 +- assetRefs 只能引用当前画布资产中的 nodeId；不要根据相似名称编造 ID。每镜最多 6 个，priority 越大表示越重要。 - styleGuide 最多 120 个中文字符；visualPrompt 只描述首帧，videoPrompt 只描述运动和结尾状态。 - 画幅比例由视频节点参数控制，提示词不得写入具体比例，也不要讨论画幅配置。 - 只返回完整 JSON，不要 Markdown 或解释。 diff --git a/backend/internal/service/storyboard.go b/backend/internal/service/storyboard.go index c3e49c2b4..a0245639c 100644 --- a/backend/internal/service/storyboard.go +++ b/backend/internal/service/storyboard.go @@ -39,12 +39,19 @@ type storyboardAsset struct { ID string `json:"id"` Title string `json:"title"` Type string `json:"type"` + Category string `json:"category,omitempty"` Tags []string `json:"tags"` Prompt string `json:"prompt"` CharacterAssetID string `json:"characterAssetId,omitempty"` CharacterVersionID string `json:"characterVersionId,omitempty"` } +type storyboardAssetRef struct { + NodeID string `json:"nodeId"` + Role string `json:"role"` + Priority int `json:"priority"` +} + type agentStoryboardPlan struct { Title string `json:"title"` Logline string `json:"logline"` @@ -56,28 +63,28 @@ type agentStoryboardPlan struct { } type agentStoryboardShot struct { - Title string `json:"title"` - Description string `json:"description"` - Duration int `json:"durationSeconds"` - Dialogue string `json:"dialogue"` - ShotSize string `json:"shotSize"` - Emotion string `json:"emotion"` - Lighting string `json:"lightingAndAtmosphere"` - AudioEffects string `json:"audioEffects"` - VisualPrompt string `json:"visualPrompt"` - VideoPrompt string `json:"videoPrompt"` - Camera string `json:"camera"` - Motion string `json:"motion"` - TimeBeats string `json:"timeBeats"` - Negative string `json:"negativePrompt"` - AssetTags []string `json:"assetTags"` - CharacterIDs []string `json:"characterIds"` - Intent string `json:"narrativeIntent"` - ViewerPOV string `json:"viewerPOV"` - Performance string `json:"performanceBlocking"` - MustHave []string `json:"mustHave"` - Optional []string `json:"optionalDetails"` - ContinuityOut string `json:"continuityOut"` + Title string `json:"title"` + Description string `json:"description"` + Duration int `json:"durationSeconds"` + Dialogue string `json:"dialogue"` + ShotSize string `json:"shotSize"` + Emotion string `json:"emotion"` + Lighting string `json:"lightingAndAtmosphere"` + AudioEffects string `json:"audioEffects"` + VisualPrompt string `json:"visualPrompt"` + VideoPrompt string `json:"videoPrompt"` + Camera string `json:"camera"` + Motion string `json:"motion"` + TimeBeats string `json:"timeBeats"` + Negative string `json:"negativePrompt"` + AssetRefs []storyboardAssetRef `json:"assetRefs"` + CharacterIDs []string `json:"characterIds"` + Intent string `json:"narrativeIntent"` + ViewerPOV string `json:"viewerPOV"` + Performance string `json:"performanceBlocking"` + MustHave []string `json:"mustHave"` + Optional []string `json:"optionalDetails"` + ContinuityOut string `json:"continuityOut"` } func parseAgentStoryboardPlan(raw string) (agentStoryboardPlan, error) { @@ -106,7 +113,9 @@ func parseAgentStoryboardPlan(raw string) (agentStoryboardPlan, error) { plan.Shots[i].Title = fmt.Sprintf("镜头 %d", i+1) } plan.Shots[i].CharacterIDs = nonNilStrings(plan.Shots[i].CharacterIDs) - plan.Shots[i].AssetTags = nonNilStrings(plan.Shots[i].AssetTags) + if plan.Shots[i].AssetRefs == nil { + plan.Shots[i].AssetRefs = []storyboardAssetRef{} + } plan.Shots[i].Optional = nonNilStrings(plan.Shots[i].Optional) plan.Shots[i].Intent = defaultString(strings.TrimSpace(plan.Shots[i].Intent), strings.TrimSpace(plan.Shots[i].Description)) plan.Shots[i].ViewerPOV = defaultString(strings.TrimSpace(plan.Shots[i].ViewerPOV), "客观观察当前主要角色与事件") @@ -145,7 +154,7 @@ func validateStoryboardJSONFields(jsonText string) error { if err := json.Unmarshal(root["shots"], &shots); err != nil { return errors.New("分镜 JSON 的 shots 必须是数组") } - required := []string{"title", "description", "durationSeconds", "dialogue", "characterIds", "narrativeIntent", "viewerPOV", "performanceBlocking", "shotSize", "emotion", "lightingAndAtmosphere", "audioEffects", "visualPrompt", "videoPrompt", "camera", "motion", "timeBeats", "mustHave", "optionalDetails", "continuityOut", "negativePrompt", "assetTags"} + required := []string{"title", "description", "durationSeconds", "dialogue", "characterIds", "narrativeIntent", "viewerPOV", "performanceBlocking", "shotSize", "emotion", "lightingAndAtmosphere", "audioEffects", "visualPrompt", "videoPrompt", "camera", "motion", "timeBeats", "mustHave", "optionalDetails", "continuityOut", "negativePrompt", "assetRefs"} for index, shot := range shots { for _, field := range required { if _, ok := shot[field]; !ok { @@ -195,7 +204,7 @@ func validateStoryboardShotDuration(plan agentStoryboardPlan, target int) error return nil } -func validateStoryboardPlan(plan agentStoryboardPlan, shotDuration int, shotCount int, characters []storyboardCharacterCard) error { +func validateStoryboardPlan(plan agentStoryboardPlan, shotDuration int, shotCount int, characters []storyboardCharacterCard, assets []storyboardAsset) error { if utf8.RuneCountInString(strings.TrimSpace(plan.StyleGuide)) > 120 { return errors.New("styleGuide 最多 120 个中文字符") } @@ -208,6 +217,43 @@ func validateStoryboardPlan(plan agentStoryboardPlan, shotDuration int, shotCoun if err := validateStoryboardCharacterIDs(plan, characters); err != nil { return err } + if err := validateStoryboardAssetRefs(plan, assets); err != nil { + return err + } + return nil +} + +func validateStoryboardAssetRefs(plan agentStoryboardPlan, assets []storyboardAsset) error { + allowed := make(map[string]storyboardAsset, len(assets)) + for _, asset := range assets { + allowed[asset.ID] = asset + } + validRoles := map[string]bool{ + "character": true, "environment": true, "wardrobe": true, "prop": true, + "weapon": true, "style": true, "motion": true, "audio": true, + } + for shotIndex, shot := range plan.Shots { + if len(shot.AssetRefs) > 6 { + return fmt.Errorf("镜头 %d 最多关联 6 个画布资产", shotIndex+1) + } + seen := make(map[string]bool, len(shot.AssetRefs)) + for _, ref := range shot.AssetRefs { + asset, ok := allowed[ref.NodeID] + if !ok { + return fmt.Errorf("镜头 %d 引用了不在当前画布资产目录中的 nodeId：%s", shotIndex+1, ref.NodeID) + } + if seen[ref.NodeID] { + return fmt.Errorf("镜头 %d 重复引用画布资产：%s", shotIndex+1, ref.NodeID) + } + seen[ref.NodeID] = true + if !validRoles[ref.Role] { + return fmt.Errorf("镜头 %d 的资产 %s 使用了不支持的角色类型：%s", shotIndex+1, asset.Title, ref.Role) + } + if ref.Priority < 0 || ref.Priority > 100 { + return fmt.Errorf("镜头 %d 的资产 %s 优先级必须在 0 到 100 之间", shotIndex+1, asset.Title) + } + } + } return nil } @@ -395,8 +441,10 @@ func extractStoryboardAssets(snapshot map[string]any) []storyboardAsset { metadata = map[string]interface{}{} } nodeType := stringValue(node["type"]) - isCharacterCard := stringValue(metadata["workflowKind"]) == "character" && stringValue(metadata["characterAssetId"]) != "" && stringValue(metadata["characterVersionId"]) != "" - if nodeType != "image" && !isCharacterCard { + workflowKind := stringValue(metadata["workflowKind"]) + isCharacterCard := workflowKind == "character" && stringValue(metadata["characterAssetId"]) != "" && stringValue(metadata["characterVersionId"]) != "" + isMedia := nodeType == "image" || nodeType == "video" || nodeType == "audio" + if (!isMedia && !isCharacterCard) || workflowKind == "shot" || workflowKind == "action_board" || workflowKind == "final" { continue } id := stringValue(node["id"]) @@ -411,122 +459,82 @@ func extractStoryboardAssets(snapshot map[string]any) []storyboardAsset { } assets = append(assets, storyboardAsset{ ID: id, - Title: defaultString(stringValue(node["title"]), "未命名图片"), + Title: defaultString(stringValue(node["title"]), "未命名资产"), Type: defaultString(nodeType, "reference"), + Category: stringValue(metadata["assetCategory"]), Tags: tags, Prompt: prompt, CharacterAssetID: stringValue(metadata["characterAssetId"]), CharacterVersionID: stringValue(metadata["characterVersionId"]), }) - if len(assets) >= 30 { + if len(assets) >= 60 { break } } return assets } -func matchStoryboardAssets(assets []storyboardAsset, shotTags []string) []storyboardAsset { - wanted := map[string]bool{} - for _, tag := range shotTags { - for _, token := range storyboardTagTokens(tag) { - wanted[token] = true - } - } - if len(wanted) == 0 { - return nil - } - matched := make([]storyboardAsset, 0) +func resolveStoryboardAssets(assets []storyboardAsset, refs []storyboardAssetRef) []storyboardAsset { + byID := make(map[string]storyboardAsset, len(assets)) for _, asset := range assets { - tokens := map[string]bool{} - for _, token := range storyboardTagTokens(asset.Title) { - tokens[token] = true + byID[asset.ID] = asset + } + resolved := make([]storyboardAsset, 0, len(refs)) + for _, ref := range refs { + if asset, ok := byID[ref.NodeID]; ok { + resolved = append(resolved, asset) } - for _, tag := range asset.Tags { - for _, token := range storyboardTagTokens(tag) { - tokens[token] = true + } + return resolved +} + +func normalizeStoryboardAssets(input []storyboardAsset) []storyboardAsset { + allowedTypes := map[string]bool{"image": true, "video": true, "audio": true, "character": true} + seenIDs := make(map[string]bool, len(input)) + assets := make([]storyboardAsset, 0, min(len(input), 60)) + for _, asset := range input { + id := strings.TrimSpace(asset.ID) + assetType := strings.ToLower(strings.TrimSpace(asset.Type)) + if id == "" || utf8.RuneCountInString(id) > 160 || seenIDs[id] || !allowedTypes[assetType] { + continue + } + seenIDs[id] = true + tags := make([]string, 0, min(len(asset.Tags), 12)) + seenTags := make(map[string]bool, len(asset.Tags)) + for _, value := range asset.Tags { + tag := clipStoryboardAssetText(value, 64) + if tag == "" || seenTags[tag] { + continue + } + seenTags[tag] = true + tags = append(tags, tag) + if len(tags) == 12 { + break } } - if storyboardTokensMatch(wanted, tokens) { - matched = append(matched, asset) - } - if len(matched) >= 6 { + assets = append(assets, storyboardAsset{ + ID: id, + Title: defaultString(clipStoryboardAssetText(asset.Title, 120), "未命名资产"), + Type: assetType, + Category: clipStoryboardAssetText(asset.Category, 40), + Tags: tags, + Prompt: clipStoryboardAssetText(asset.Prompt, 600), + CharacterAssetID: clipStoryboardAssetText(asset.CharacterAssetID, 160), + CharacterVersionID: clipStoryboardAssetText(asset.CharacterVersionID, 160), + }) + if len(assets) == 60 { break } } - return matched + return assets } -func matchStoryboardAssetsForShot(assets []storyboardAsset, shot agentStoryboardShot) []storyboardAsset { - matched := make([]storyboardAsset, 0, 6) - seen := make(map[string]bool, 6) - wantedCharacters := make(map[string]bool, len(shot.CharacterIDs)) - for _, assetID := range shot.CharacterIDs { - wantedCharacters[assetID] = true +func clipStoryboardAssetText(value string, limit int) string { + runes := []rune(strings.TrimSpace(value)) + if len(runes) <= limit { + return string(runes) } - for _, asset := range assets { - if len(matched) >= 6 { - break - } - if !seen[asset.ID] && wantedCharacters[asset.CharacterAssetID] { - matched = append(matched, asset) - seen[asset.ID] = true - } - } - for _, asset := range matchStoryboardAssets(assets, shot.AssetTags) { - if len(matched) >= 6 { - break - } - if !seen[asset.ID] { - matched = append(matched, asset) - seen[asset.ID] = true - } - } - return matched -} - -func storyboardTokensMatch(wanted map[string]bool, tokens map[string]bool) bool { - for want := range wanted { - if tokens[want] { - return true - } - for token := range tokens { - if meaningfulStoryboardTagToken(want) && meaningfulStoryboardTagToken(token) && (strings.Contains(token, want) || strings.Contains(want, token)) { - return true - } - } - } - return false -} - -func storyboardTagTokens(value string) []string { - normalized := strings.ToLower(strings.ReplaceAll(strings.Join(strings.Fields(strings.ReplaceAll(value, "：", ":")), ""), "，", ",")) - if normalized == "" { - return nil - } - tokens := []string{normalized} - if index := strings.Index(normalized, ":"); index >= 0 { - tokens = append(tokens, normalized[index+1:]) - } - unique := make([]string, 0, len(tokens)) - seen := map[string]bool{} - for _, token := range tokens { - if meaningfulStoryboardTagToken(token) && !seen[token] { - seen[token] = true - unique = append(unique, token) - } - } - return unique -} - -func meaningfulStoryboardTagToken(value string) bool { - if len([]rune(value)) < 2 { - return false - } - switch value { - case "角色", "环境", "场景", "道具", "武器", "风格": - return false - } - return true + return string(runes[:limit]) } func listContent(title string, items []string) string { diff --git a/backend/internal/service/storyboard_prompt_test.go b/backend/internal/service/storyboard_prompt_test.go index c14a61a1c..22e18bf83 100644 --- a/backend/internal/service/storyboard_prompt_test.go +++ b/backend/internal/service/storyboard_prompt_test.go @@ -3,6 +3,7 @@ package service import ( "strings" "testing" + "unicode/utf8" ) func TestStoryboardCinematicQualityContractIncludesRequestedCountAndDuration(t *testing.T) { @@ -82,7 +83,7 @@ func TestValidateStoryboardPlanTreatsComplexityAsAdvisory(t *testing.T) { }}} characters := []storyboardCharacterCard{{AssetID: "a"}, {AssetID: "b"}, {AssetID: "c"}} - if err := validateStoryboardPlan(plan, 0, 0, characters); err != nil { + if err := validateStoryboardPlan(plan, 0, 0, characters, nil); err != nil { t.Fatalf("complexity should be advisory for an otherwise valid plan: %v", err) } } @@ -112,13 +113,51 @@ func TestNormalizeAutomaticStoryboardDurationsPreservesLongDialogue(t *testing.T } } -func TestMatchStoryboardAssetsForShotPrioritizesCurrentCharacterCards(t *testing.T) { +func TestResolveStoryboardAssetsUsesStableCanvasNodeIDs(t *testing.T) { assets := []storyboardAsset{ {ID: "environment", Title: "天宫", Tags: []string{"场景:天宫"}}, {ID: "character-node", Title: "青莲剑仙", CharacterAssetID: "character-1", CharacterVersionID: "version-3"}, } - matched := matchStoryboardAssetsForShot(assets, agentStoryboardShot{CharacterIDs: []string{"character-1"}, AssetTags: []string{"场景:天宫"}}) - if len(matched) != 2 || matched[0].ID != "character-node" { - t.Fatalf("expected current character card before optional tagged assets: %#v", matched) + resolved := resolveStoryboardAssets(assets, []storyboardAssetRef{{NodeID: "character-node", Role: "character", Priority: 100}, {NodeID: "environment", Role: "environment", Priority: 80}}) + if len(resolved) != 2 || resolved[0].ID != "character-node" || resolved[1].ID != "environment" { + t.Fatalf("expected stable asset order from model references: %#v", resolved) + } +} + +func TestValidateStoryboardAssetRefsRejectsInventedNodeID(t *testing.T) { + plan := agentStoryboardPlan{Shots: []agentStoryboardShot{{AssetRefs: []storyboardAssetRef{{NodeID: "invented", Role: "prop", Priority: 50}}}}} + err := validateStoryboardAssetRefs(plan, []storyboardAsset{{ID: "real", Title: "真实道具"}}) + if err == nil || !strings.Contains(err.Error(), "不在当前画布资产目录") { + t.Fatalf("expected invented node id to fail, got %v", err) + } +} + +func TestExtractStoryboardAssetsIncludesMediaAndExcludesShotOutputs(t *testing.T) { + snapshot := map[string]any{"nodes": []interface{}{ + map[string]interface{}{"id": "image", "type": "image", "title": "角色参考", "metadata": map[string]interface{}{"content": "resource:image", "assetCategory": "character"}}, + map[string]interface{}{"id": "video", "type": "video", "title": "动作参考", "metadata": map[string]interface{}{"content": "resource:video"}}, + map[string]interface{}{"id": "audio", "type": "audio", "title": "环境声", "metadata": map[string]interface{}{"content": "resource:audio"}}, + map[string]interface{}{"id": "output", "type": "video", "title": "旧镜头", "metadata": map[string]interface{}{"content": "resource:output", "workflowKind": "shot"}}, + }} + assets := extractStoryboardAssets(snapshot) + if len(assets) != 3 || assets[0].ID != "image" || assets[1].ID != "video" || assets[2].ID != "audio" { + t.Fatalf("unexpected assets: %#v", assets) + } +} + +func TestNormalizeStoryboardAssetsBoundsAndDeduplicatesInput(t *testing.T) { + assets := normalizeStoryboardAssets([]storyboardAsset{ + {ID: " asset-1 ", Title: " 场景图 ", Type: "IMAGE", Tags: []string{"雨夜", "雨夜", strings.Repeat("长", 80)}, Prompt: strings.Repeat("镜", 700)}, + {ID: "asset-1", Title: "重复", Type: "image"}, + {ID: "text-1", Title: "文本", Type: "text"}, + }) + if len(assets) != 1 { + t.Fatalf("expected one normalized asset, got %#v", assets) + } + if assets[0].ID != "asset-1" || assets[0].Type != "image" || len(assets[0].Tags) != 2 { + t.Fatalf("unexpected normalized asset: %#v", assets[0]) + } + if utf8.RuneCountInString(assets[0].Prompt) != 600 || utf8.RuneCountInString(assets[0].Tags[1]) != 64 { + t.Fatalf("expected bounded prompt and tag lengths: %#v", assets[0]) } } diff --git a/backend/internal/service/storyboard_workflow.go b/backend/internal/service/storyboard_workflow.go index 8c624e087..db939c177 100644 --- a/backend/internal/service/storyboard_workflow.go +++ b/backend/internal/service/storyboard_workflow.go @@ -27,7 +27,7 @@ func (s *Service) processStoryboardRowsTask(ctx context.Context, task model.Task if err != nil { return nil, nil, err } - plan, assets, err := s.generateStoryboardPlan(ctx, task, input, input.ShotDuration, input.ShotCount) + plan, _, err := s.generateStoryboardPlan(ctx, task, input, input.ShotDuration, input.ShotCount) if err != nil { return nil, nil, err } @@ -44,11 +44,6 @@ func (s *Service) processStoryboardRowsTask(ctx context.Context, task model.Task if promptErr != nil { return nil, nil, promptErr } - matchedAssets := matchStoryboardAssets(assets, shot.AssetTags) - referenceNodeIDs := make([]string, 0, len(matchedAssets)) - for _, asset := range matchedAssets { - referenceNodeIDs = append(referenceNodeIDs, asset.ID) - } rows = append(rows, map[string]any{ "shotNumber": index + 1, "durationSeconds": shot.Duration, "plotDescription": shot.Description, "dialogue": shot.Dialogue, "characters": storyboardRowCharacters(shot, input.Characters), "shotSize": shot.ShotSize, "emotion": shot.Emotion, @@ -58,7 +53,7 @@ func (s *Service) processStoryboardRowsTask(ctx context.Context, task model.Task "camera": shot.Camera, "motion": shot.Motion, "timeBeats": shot.TimeBeats, "negativePrompt": shot.Negative, "narrativeIntent": shot.Intent, "viewerPOV": shot.ViewerPOV, "performanceBlocking": shot.Performance, "mustHave": shot.MustHave, "optionalDetails": shot.Optional, "continuityOut": shot.ContinuityOut, - "referenceNodeIds": referenceNodeIDs, "assetTags": shot.AssetTags, + "assetBindings": shot.AssetRefs, }) } return map[string]interface{}{"title": plan.Title, "rows": rows}, nil, nil @@ -85,7 +80,7 @@ func (s *Service) repairStoryboardPlan(ctx context.Context, task model.Task, inp plan, parseErr := parseAgentStoryboardPlan(repairedText) if parseErr == nil { normalizeAutomaticStoryboardDurations(&plan, shotDuration) - parseErr = validateStoryboardPlan(plan, shotDuration, shotCount, input.Characters) + parseErr = validateStoryboardPlan(plan, shotDuration, shotCount, input.Characters, input.CanvasAssets) } if parseErr == nil { return plan, nil @@ -119,10 +114,11 @@ func (s *Service) generateStoryboardPlan(ctx context.Context, task model.Task, i if err := validateStoryboardContext(input.ProjectStyle, input.Characters); err != nil { return agentStoryboardPlan{}, nil, err } - assets := input.CanvasAssets + assets := normalizeStoryboardAssets(input.CanvasAssets) if len(assets) == 0 { - assets = extractStoryboardAssets(input.CanvasSnapshot) + assets = normalizeStoryboardAssets(extractStoryboardAssets(input.CanvasSnapshot)) } + input.CanvasAssets = assets config, err := s.resolveProviderConfig(input.Config) if err != nil { return agentStoryboardPlan{}, nil, err @@ -140,7 +136,7 @@ func (s *Service) generateStoryboardPlan(ctx context.Context, task model.Task, i plan, err := parseAgentStoryboardPlan(text) if err == nil { normalizeAutomaticStoryboardDurations(&plan, shotDuration) - err = validateStoryboardPlan(plan, shotDuration, shotCount, input.Characters) + err = validateStoryboardPlan(plan, shotDuration, shotCount, input.Characters, assets) } if err != nil { plan, err = s.repairStoryboardPlan(ctx, task, input, config, text, err, shotDuration, shotCount) @@ -178,7 +174,7 @@ func (s *Service) buildAgentStoryboardResult(task model.Task, plan agentStoryboa return nil, nil, err } shotID := fmt.Sprintf("%s-shot-%d", prefix, index+1) - matchedAssets := matchStoryboardAssetsForShot(assets, shot) + matchedAssets := resolveStoryboardAssets(assets, shot.AssetRefs) assetIDs := make([]string, 0, len(matchedAssets)) for _, asset := range matchedAssets { assetIDs = append(assetIDs, asset.ID) @@ -193,7 +189,7 @@ func (s *Service) buildAgentStoryboardResult(task model.Task, plan agentStoryboa "prompt": videoPrompt, "composerContent": shotComposerContent(videoPrompt, matchedAssets), "videoEditOperation": "text_to_video", - "assetTags": shot.AssetTags, + "assetBindings": shot.AssetRefs, "referenceAssetNodeIds": assetIDs, "status": "idle", }), @@ -203,7 +199,7 @@ func (s *Service) buildAgentStoryboardResult(task model.Task, plan agentStoryboa for _, asset := range matchedAssets { ops = append(ops, connectOp(asset.ID, shotID)) } - resultShots = append(resultShots, map[string]any{"title": shot.Title, "description": shot.Description, "assetTags": shot.AssetTags, "referenceAssetNodeIds": assetIDs}) + resultShots = append(resultShots, map[string]any{"title": shot.Title, "description": shot.Description, "assetBindings": shot.AssetRefs, "referenceAssetNodeIds": assetIDs}) } ops = append(ops, map[string]any{"type": "select_nodes", "ids": shotIDs(prefix, len(plan.Shots))}) result := map[string]any{ diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index cbe04e404..81832d397 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -1,5 +1,13 @@ # 待测试清单 +## 分镜脚本资产关联 + +1. 在同一画布放入角色、场景图片、参考视频和音频后生成分镜，确认各镜头只关联模型返回的真实画布资产，不出现失效或编造 ID。 +2. 分镜卡片按“序号、时长、视频提示词、台词/旁白、关联资产”五列显示；图片、视频和音频缩略图可点击并打开对应预览。 +3. 从单个镜头及批量入口创建视频节点，确认脚本行输出和每个关联资产都自动连到视频节点，重复同步不会产生重复连线。 +4. 打开视频节点提示词，确认关联资产以标准 `@资产` 缩略图展示，生成请求能解析图片、视频和音频引用。 +5. 修改镜头资产关联后再次同步，确认已移除的托管资产连线被清理，而用户手工创建的普通连线仍保留。 + ## 零价免费模型 1. 管理员设置启用系统渠道模型的按次价格为 `0` 后，普通用户刷新设置即可看到并成功创建任务。 diff --git a/web/package.json b/web/package.json index d5739c34a..2ae5e24c3 100644 --- a/web/package.json +++ b/web/package.json @@ -7,7 +7,7 @@ "dev": "vite --host 0.0.0.0 --port 3000", "build": "tsc --noEmit && vite build", "typecheck": "tsc --noEmit", - "test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-protocols.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-image-batch-retry.test.ts test/canvas-leafer-viewport.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts && bun test test/local-dreamina-cross-runtime.test.ts", + "test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-protocols.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-image-batch-retry.test.ts test/canvas-leafer-viewport.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts && bun test test/local-dreamina-cross-runtime.test.ts", "start": "vite preview --host 0.0.0.0 --port 3000", "format": "prettier --write .", "format:check": "prettier --check ." diff --git a/web/src/components/canvas/canvas-script-node.tsx b/web/src/components/canvas/canvas-script-node.tsx index 6b0b5d9cf..424ca7fda 100644 --- a/web/src/components/canvas/canvas-script-node.tsx +++ b/web/src/components/canvas/canvas-script-node.tsx @@ -5,6 +5,7 @@ import type { ColumnsType } from "antd/es/table"; import { ChevronDown, ChevronUp, Clapperboard, Copy, Expand, Film, Grid3X3, Image as ImageIcon, ListTree, Merge, MoreHorizontal, Plus, RefreshCw, Send, Square, Trash2, Video } from "lucide-react"; import { CanvasResourceMentionTextarea } from "@/components/canvas/canvas-resource-mention-textarea"; +import { StoryboardAssetsCell } from "@/components/canvas/storyboard-assets-cell"; import { ModelPicker } from "@/components/model-picker"; import { buildGenerationConfig } from "@/lib/canvas/canvas-project-generation"; import type { CanvasResourceReference } from "@/lib/canvas/canvas-resource-references"; @@ -33,13 +34,16 @@ import type { TaskStatus } from "@/services/api/task-center"; const STORYBOARD_PROMPT_MIN_HEIGHT = 40; const STORYBOARD_PROMPT_MAX_HEIGHT = 116; -const SCRIPT_GRID_TEMPLATE = "72px minmax(220px, 1fr) minmax(300px, 1.4fr) minmax(220px, 1fr) 58px"; +const SCRIPT_GRID_TEMPLATE = "64px 76px minmax(300px, 1.55fr) minmax(220px, 1fr) 180px"; const EMPTY_STORYBOARD_ROWS: StoryboardRow[] = []; -const DEFAULT_STORYBOARD_COLUMNS: StoryboardColumn[] = ["shotNumber", "plotDescription", "videoMotionPrompt", "dialogue"]; +const DEFAULT_STORYBOARD_COLUMNS: StoryboardColumn[] = ["shotNumber", "durationSeconds", "videoMotionPrompt", "dialogue", "assets"]; const LEGACY_STORYBOARD_COLUMNS: StoryboardColumn[] = ["shotNumber", "durationSeconds", "plotDescription", "dialogue"]; +const PREVIOUS_DEFAULT_STORYBOARD_COLUMNS: StoryboardColumn[] = ["shotNumber", "plotDescription", "videoMotionPrompt", "dialogue"]; function resolveStoryboardVisibleColumns(columns?: StoryboardColumn[]) { - if (!columns?.length || (columns.length === LEGACY_STORYBOARD_COLUMNS.length && LEGACY_STORYBOARD_COLUMNS.every((column) => columns.includes(column)))) { + const isLegacyDefault = columns?.length === LEGACY_STORYBOARD_COLUMNS.length && LEGACY_STORYBOARD_COLUMNS.every((column) => columns.includes(column)); + const isPreviousDefault = columns?.length === PREVIOUS_DEFAULT_STORYBOARD_COLUMNS.length && PREVIOUS_DEFAULT_STORYBOARD_COLUMNS.every((column) => columns.includes(column)); + if (!columns?.length || isLegacyDefault || isPreviousDefault) { return DEFAULT_STORYBOARD_COLUMNS; } return columns; @@ -62,12 +66,14 @@ const columnOptions: Array<{ label: string; value: StoryboardColumn }> = [ { label: "时间节拍", value: "timeBeats" }, { label: "图片提示词", value: "imageGenerationPrompt" }, { label: "视频提示词", value: "videoMotionPrompt" }, + { label: "关联资产", value: "assets" }, { label: "连续性出口", value: "continuityOut" }, { label: "负面要求", value: "negativePrompt" }, ]; export function CanvasScriptNodeContent({ node, + nodes, batch, pipeline, scale, @@ -97,6 +103,7 @@ export function CanvasScriptNodeContent({ workspaceMode = "professional", }: { node: CanvasNodeData; + nodes: CanvasNodeData[]; batch?: CanvasGenerationBatch; pipeline: CanvasStoryboardPipelineProgress; scale: number; @@ -285,10 +292,10 @@ export function CanvasScriptNodeContent({ 序号 - 画面 + 时长 视频提示词 台词/旁白 - 操作 + 关联资产
(
-
- {row.shotNumber} +
+
+ {row.shotNumber} + , danger: true, disabled: rows.length <= 1, onClick: () => onRemoveRow(row.id) }] }} + > + + +
{batchItemByRowId.get(row.id) ? ( - + {generationBatchItemLabel(batchItemByRowId.get(row.id)!)} ) : null}
- onUpdateRow(row.id, { plotDescription: value })} borderColor={theme.node.stroke} /> + onUpdateRow(row.id, { durationSeconds })} /> onUpdateRow(row.id, { videoMotionPrompt: value })} borderColor={theme.node.stroke} /> onUpdateRow(row.id, { dialogue: value })} borderColor={theme.node.stroke} /> -
- +
+
)) @@ -588,6 +598,7 @@ function batchItemTone(item?: CanvasGenerationBatchItem): CanvasNodeStatus | und export function CanvasScriptEditor({ node, + nodes, open, onClose, onUpdateRows, @@ -597,6 +608,7 @@ export function CanvasScriptEditor({ onVideoInputModeChange, }: { node: CanvasNodeData | null; + nodes: CanvasNodeData[]; open: boolean; onClose: () => void; onUpdateRows: (rows: StoryboardRow[]) => void; @@ -610,18 +622,19 @@ export function CanvasScriptEditor({ const rows = node?.metadata?.storyboard?.rows || EMPTY_STORYBOARD_ROWS; const visibleColumns = resolveStoryboardVisibleColumns(node?.metadata?.storyboard?.visibleColumns); const videoInputMode = node?.metadata?.storyboardVideoInputMode || "direct"; + const nodeById = useMemo(() => new Map(nodes.map((item) => [item.id, item])), [nodes]); const filteredRows = useMemo(() => { const keyword = query.trim().toLowerCase(); return keyword ? rows.filter((row) => - [row.plotDescription, row.dialogue, row.camera, row.motion, row.timeBeats, row.imageGenerationPrompt, row.videoMotionPrompt, row.negativePrompt].some((value) => + [row.plotDescription, row.dialogue, row.camera, row.motion, row.timeBeats, row.imageGenerationPrompt, row.videoMotionPrompt, row.negativePrompt, ...(row.assetBindings || []).map((binding) => nodeById.get(binding.nodeId)?.title || "")].some((value) => String(value || "") .toLowerCase() .includes(keyword), ), ) : rows; - }, [query, rows]); + }, [nodeById, query, rows]); useEffect(() => { setSelectedIds((current) => { const next = current.filter((id) => rows.some((row) => row.id === id)); @@ -651,13 +664,15 @@ export function CanvasScriptEditor({ title: option.label, dataIndex: option.value, key: option.value, - width: option.value === "shotNumber" ? 72 : option.value === "durationSeconds" ? 100 : option.value === "plotDescription" || option.value === "dialogue" || option.value === "timeBeats" || option.value.endsWith("Prompt") ? 260 : 170, + width: option.value === "shotNumber" ? 72 : option.value === "durationSeconds" ? 100 : option.value === "assets" ? 220 : option.value === "plotDescription" || option.value === "dialogue" || option.value === "timeBeats" || option.value.endsWith("Prompt") ? 260 : 170, fixed: option.value === "shotNumber" ? ("left" as const) : undefined, render: (_: unknown, row: StoryboardRow) => option.value === "shotNumber" ? ( {row.shotNumber} ) : option.value === "durationSeconds" ? ( updateRow(row.id, { durationSeconds: Number(value) || 1 })} /> + ) : option.value === "assets" ? ( + ) : option.value === "shotSize" ? ( onChange(Math.max(1, Math.min(60, Number(event.target.value) || 1)))} + onMouseDown={(event) => event.stopPropagation()} + onPointerDown={(event) => event.stopPropagation()} + /> + 秒 + + ); +} + function HeaderCell({ children, borderColor, align = "left" }: { children: ReactNode; borderColor: string; align?: "left" | "center" }) { return ( @@ -802,7 +836,7 @@ function editorRow(shotNumber: number): StoryboardRow { optionalDetails: [], continuityOut: "", negativePrompt: "", - referenceNodeIds: [], + assetBindings: [], status: "idle", }; } diff --git a/web/src/components/canvas/storyboard-assets-cell.tsx b/web/src/components/canvas/storyboard-assets-cell.tsx new file mode 100644 index 000000000..c252f1134 --- /dev/null +++ b/web/src/components/canvas/storyboard-assets-cell.tsx @@ -0,0 +1,107 @@ +import { useEffect, useMemo, useState } from "react"; +import { Modal, Tooltip } from "antd"; +import { Image as ImageIcon, Music2, Play, UserRound } from "lucide-react"; + +import { isStoryboardPreviewAsset } from "@/lib/canvas/canvas-storyboard-materializer"; +import { resolveMediaUrl } from "@/services/file-storage"; +import { CanvasNodeType, type CanvasNodeData, type StoryboardAssetBinding } from "@/types/canvas"; + +const ROLE_LABELS: Record = { + character: "角色", + environment: "场景", + wardrobe: "服装", + prop: "道具", + weapon: "武器", + style: "风格", + motion: "动态", + audio: "音频", +}; + +export function StoryboardAssetsCell({ bindings, nodes, limit = 4 }: { bindings: StoryboardAssetBinding[]; nodes: CanvasNodeData[]; limit?: number }) { + const [previewNode, setPreviewNode] = useState(null); + const nodeById = useMemo(() => new Map(nodes.map((node) => [node.id, node])), [nodes]); + const assets = bindings.map((binding) => ({ binding, node: nodeById.get(binding.nodeId) })).filter((item) => !item.node || isStoryboardPreviewAsset(item.node)); + const visible = assets.slice(0, limit); + const hiddenCount = Math.max(0, assets.length - visible.length); + + if (!assets.length) return 未关联; + return ( + <> +
+ {visible.map(({ binding, node }) => ( + + + + ))} + {hiddenCount ? +{hiddenCount} : null} +
+ setPreviewNode(null)} /> + + ); +} + +function AssetThumbnail({ node }: { node: CanvasNodeData }) { + const source = useNodeMediaSource(node); + if (node.type === CanvasNodeType.Audio) return ; + if (node.metadata?.workflowKind === "character" && !source) return ; + if (node.type === CanvasNodeType.Video) { + return source ? ( + <> +