diff --git a/backend/internal/protocol/builtin.go b/backend/internal/protocol/builtin.go index 9ac43bd0b..88c5b773a 100644 --- a/backend/internal/protocol/builtin.go +++ b/backend/internal/protocol/builtin.go @@ -340,7 +340,7 @@ func newAPIChannel1Adapter() Adapter { return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) { media := make([]map[string]string, 0, len(r.Images)+len(r.Videos)+len(r.Audios)) for _, image := range r.Images { - media = append(media, map[string]string{"type": "reference_image", "url": mediaValue(image)}) + media = append(media, map[string]string{"type": mediaRole(image, "reference_image"), "url": mediaValue(image)}) } for _, video := range r.Videos { media = append(media, map[string]string{"type": "reference_video", "url": mediaValue(video)}) @@ -365,11 +365,26 @@ func xAIVideosAdapter() Adapter { info.Parameters = videoParams() return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) { body := map[string]any{"model": r.Model, "prompt": r.Prompt, "duration": defaultInt(r.Duration, 6), "aspect_ratio": defaultValue(r.AspectRatio, "16:9"), "resolution": defaultValue(r.Resolution, "720p")} - if len(r.Images) == 1 { - body["image"] = map[string]any{"url": mediaValue(r.Images[0])} - } else if len(r.Images) > 1 { + frameImages, referenceImages, unspecifiedImages := splitVideoImages(r.Images) + if len(frameImages) > 0 && len(referenceImages) > 0 { + return RequestSpec{}, fmt.Errorf("xAI 视频协议不能同时混用首尾帧和角色参考图") + } + if len(frameImages) > 1 { + return RequestSpec{}, fmt.Errorf("xAI 视频协议最多支持 1 张起始图,不支持尾帧输入") + } + if len(frameImages) == 1 { + body["image"] = map[string]any{"url": mediaValue(frameImages[0])} + } else if len(referenceImages) > 0 { + refs := make([]any, 0, len(referenceImages)) + for _, image := range referenceImages { + refs = append(refs, map[string]any{"url": mediaValue(image)}) + } + body["reference_images"] = refs + } else if len(unspecifiedImages) == 1 { + body["image"] = map[string]any{"url": mediaValue(unspecifiedImages[0])} + } else if len(unspecifiedImages) > 1 { refs := make([]any, 0, len(r.Images)) - for _, image := range r.Images { + for _, image := range unspecifiedImages { refs = append(refs, map[string]any{"url": mediaValue(image)}) } body["reference_images"] = refs @@ -390,7 +405,7 @@ func arkVideosAdapter() Adapter { } content := []any{map[string]any{"type": "text", "text": r.Prompt}} for _, image := range r.Images { - content = append(content, map[string]any{"type": "image_url", "image_url": map[string]any{"url": mediaValue(image)}, "role": "reference_image"}) + content = append(content, map[string]any{"type": "image_url", "image_url": map[string]any{"url": mediaValue(image)}, "role": mediaRole(image, "reference_image")}) } for _, video := range r.Videos { content = append(content, map[string]any{"type": "video_url", "video_url": map[string]any{"url": mediaValue(video)}, "role": "reference_video"}) @@ -428,6 +443,9 @@ func geminiVeoAdapter() Adapter { info := metadata("gemini-veo", "Gemini Veo", "Google", CapabilityVideo, "POST /v1beta/models/{model}:predictLongRunning", "GET /v1beta/{operation_name}", "application/json") info.Parameters = videoParams() create := func(r GenerationRequest) (RequestSpec, error) { + if hasVideoImageRole(r.Images, "reference_image") { + return RequestSpec{}, fmt.Errorf("Gemini Veo 当前不支持角色或风格参考图生视频,请改用支持 reference_to_video 的模型") + } instance := map[string]any{"prompt": r.Prompt} if len(r.Images) > 0 { image := r.Images[0] @@ -450,6 +468,9 @@ func novitaVideosAdapter() Adapter { info := metadata("novita-video", "Novita 视频", "Novita", CapabilityVideo, "POST /v3/video/create", "GET /v3/async/task-result?task_id={id}", "application/json") info.Parameters = videoParams() return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) { + if hasVideoImageRole(r.Images, "reference_image") { + return RequestSpec{}, fmt.Errorf("Novita 视频当前不支持角色或风格参考图生视频,请改用支持 reference_to_video 的模型") + } body := map[string]any{"model_name": r.Model, "prompt": r.Prompt, "duration": defaultInt(r.Duration, 5), "aspect_ratio": defaultValue(r.AspectRatio, "16:9")} if len(r.Images) > 0 { body["image_url"] = mediaValue(r.Images[0]) @@ -464,7 +485,13 @@ func miniMaxVideosAdapter() Adapter { return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) { content := []any{map[string]any{"type": "text", "text": r.Prompt}} for _, image := range r.Images { - content = append(content, map[string]any{"type": "image_url", "image_url": map[string]any{"url": mediaValue(image)}, "role": "reference_image"}) + content = append(content, map[string]any{"type": "image_url", "image_url": map[string]any{"url": mediaValue(image)}, "role": mediaRole(image, "reference_image")}) + } + for _, video := range r.Videos { + content = append(content, map[string]any{"type": "video_url", "video_url": map[string]any{"url": mediaValue(video)}, "role": "reference_video"}) + } + for _, audio := range r.Audios { + content = append(content, map[string]any{"type": "audio_url", "audio_url": map[string]any{"url": mediaValue(audio)}, "role": "reference_audio"}) } body := map[string]any{"model": r.Model, "prompt": r.Prompt, "duration": defaultInt(r.Duration, 6), "content": content} mergeExtra(body, r.Extra, "model", "prompt", "duration", "content", "resolution", "aspect_ratio") @@ -541,6 +568,10 @@ func buildAgnesCreate(r GenerationRequest) (RequestSpec, error) { } func buildAgnesV25Body(r GenerationRequest) (map[string]any, error) { + frameImages, referenceImages, _ := splitVideoImages(r.Images) + if len(frameImages) > 0 && (len(referenceImages) > 0 || len(r.Videos)+len(r.Audios) > 0) { + return nil, fmt.Errorf("Agnes 2.5 不能同时混用首尾帧和角色、视频或音频参考素材") + } mode := firstString(r.Extra, "mode") if mode == "" { switch strings.TrimSpace(r.Operation) { @@ -550,8 +581,10 @@ func buildAgnesV25Body(r GenerationRequest) (map[string]any, error) { mode = "keyframe" default: switch { - case len(r.Videos) > 0 || len(r.Audios) > 0: + case len(r.Videos) > 0 || len(r.Audios) > 0 || len(referenceImages) > 0: mode = "reference" + case len(frameImages) > 0: + mode = "keyframe" case len(r.Images) > 2: mode = "reference" case len(r.Images) > 0: @@ -610,9 +643,17 @@ func buildAgnesV25Body(r GenerationRequest) (map[string]any, error) { if len(r.Images) == 0 || len(r.Images) > 2 || len(r.Videos)+len(r.Audios) > 0 { return nil, fmt.Errorf("Agnes 2.5 keyframe 模式需要 1 到 2 张图片,且不接受音视频参考") } - body["first_frame"] = mediaValue(r.Images[0]) - if len(r.Images) > 1 { - body["last_frame"] = mediaValue(r.Images[1]) + _, referenceImages, _ := splitVideoImages(r.Images) + if len(referenceImages) > 0 { + return nil, fmt.Errorf("Agnes 2.5 keyframe 模式不能混用角色或风格参考图") + } + frameImages := orderedFrameImages(r.Images) + if len(frameImages) == 0 { + return nil, fmt.Errorf("Agnes 2.5 keyframe 模式缺少首帧图片") + } + body["first_frame"] = mediaValue(frameImages[0]) + if len(frameImages) > 1 { + body["last_frame"] = mediaValue(frameImages[1]) } case "reference": if len(r.Images)+len(r.Videos)+len(r.Audios) == 0 { @@ -681,6 +722,10 @@ func buildAgnesV20Body(r GenerationRequest) (map[string]any, error) { if len(r.Videos)+len(r.Audios) > 0 { return nil, fmt.Errorf("Agnes Video V2.0 不支持参考视频或音频") } + _, referenceImages, _ := splitVideoImages(r.Images) + if r.Operation == "reference_to_video" || len(referenceImages) > 0 { + return nil, fmt.Errorf("Agnes Video V2.0 不支持角色或风格参考图模式,请改用 Agnes Video 2.5") + } body := map[string]any{"model": r.Model, "prompt": r.Prompt} mergeExtra(body, r.Extra, "mode", "width", "height", "num_frames", "frame_rate", "num_inference_steps", "seed", "negative_prompt") if r.Duration > 0 { @@ -703,16 +748,17 @@ func buildAgnesV20Body(r GenerationRequest) (map[string]any, error) { body["num_frames"] = frames } } - if len(r.Images) == 1 { - body["image"] = mediaValue(r.Images[0]) - } else if len(r.Images) > 1 { + orderedImages := orderedFrameImages(r.Images) + if len(orderedImages) == 1 { + body["image"] = mediaValue(orderedImages[0]) + } else if len(orderedImages) > 1 { extraBody := map[string]any{} if configured := object(r.Extra["extra_body"]); configured != nil { for key, value := range configured { extraBody[key] = value } } - extraBody["image"] = mediaValues(r.Images) + extraBody["image"] = mediaValues(orderedImages) extraBody["mode"] = "keyframes" body["extra_body"] = extraBody } else if configured := object(r.Extra["extra_body"]); configured != nil { @@ -1120,6 +1166,47 @@ func compactMap(body map[string]any) map[string]any { return result } func mediaValue(media MediaReference) string { return defaultValue(media.URL, media.DataURL) } +func mediaRole(media MediaReference, fallback string) string { + return defaultValue(strings.TrimSpace(media.Role), fallback) +} + +func splitVideoImages(images []MediaReference) (frames []MediaReference, references []MediaReference, unspecified []MediaReference) { + firstFrames := make([]MediaReference, 0, 1) + lastFrames := make([]MediaReference, 0, 1) + for _, image := range images { + switch strings.TrimSpace(image.Role) { + case "first_frame": + firstFrames = append(firstFrames, image) + case "last_frame": + lastFrames = append(lastFrames, image) + case "reference_image": + references = append(references, image) + default: + unspecified = append(unspecified, image) + } + } + frames = append(frames, firstFrames...) + frames = append(frames, lastFrames...) + return frames, references, unspecified +} + +func orderedFrameImages(images []MediaReference) []MediaReference { + frames, _, unspecified := splitVideoImages(images) + if len(frames) == 0 { + return unspecified + } + return append(frames, unspecified...) +} + +func hasVideoImageRole(images []MediaReference, role string) bool { + for _, image := range images { + if strings.TrimSpace(image.Role) == role { + return true + } + } + return false +} + func mediaValues(values []MediaReference) []string { result := make([]string, 0, len(values)) for _, value := range values { diff --git a/backend/internal/protocol/builtin_test.go b/backend/internal/protocol/builtin_test.go index 9d9e62955..49045fc29 100644 --- a/backend/internal/protocol/builtin_test.go +++ b/backend/internal/protocol/builtin_test.go @@ -196,6 +196,62 @@ func TestArkVideoAdapterMapsFullModalReferences(t *testing.T) { } } +func TestVideoAdaptersTranslateImageRolesWithoutGuessingFromCount(t *testing.T) { + tests := []struct { + name string + id string + role string + }{ + {name: "newapi channel 1", id: "newapi-channel-1", role: "reference_image"}, + {name: "volcengine ark", id: "volcengine-ark-video", role: "first_frame"}, + {name: "minimax", id: "minimax-video", role: "last_frame"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + adapter, _ := Builtins().Get(test.id) + spec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{ + Model: "video-model", Prompt: "test", Duration: 5, + Images: []MediaReference{{ID: "image-1", URL: "https://cdn.example/image.png", Role: test.role}}, + }}) + if err != nil { + t.Fatal(err) + } + body, _ := json.Marshal(spec.Body) + if !strings.Contains(string(body), `"`+test.role+`"`) { + t.Fatalf("body does not contain role %q: %s", test.role, body) + } + }) + } +} + +func TestXAIVideoUsesReferenceImagesEvenForOneReferenceAsset(t *testing.T) { + adapter, _ := Builtins().Get("xai-video") + spec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{ + Model: "grok-video", Prompt: "keep the character", Images: []MediaReference{{URL: "https://cdn.example/character.png", Role: "reference_image"}}, + }}) + if err != nil { + t.Fatal(err) + } + body := spec.Body.(map[string]any) + if body["image"] != nil || body["reference_images"] == nil { + t.Fatalf("xAI reference body = %#v", body) + } +} + +func TestStartFrameOnlyAdaptersRejectReferenceAssetSemantics(t *testing.T) { + for _, id := range []string{"gemini-veo", "novita-video"} { + t.Run(id, func(t *testing.T) { + adapter, _ := Builtins().Get(id) + _, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{ + Model: id, Prompt: "keep the character", Images: []MediaReference{{URL: "https://cdn.example/character.png", Role: "reference_image"}}, + }}) + if err == nil || !strings.Contains(err.Error(), "reference_to_video") { + t.Fatalf("BuildCreate() error = %v", err) + } + }) + } +} + func TestAsyncVideoResponseNormalizesStatusAndResult(t *testing.T) { adapter, _ := Builtins().Get("minimax-video") created, err := adapter.ParseCreate(context.Background(), []byte(`{"task":{"id":"mm-1","status":"processing"}}`)) diff --git a/backend/internal/protocol/types.go b/backend/internal/protocol/types.go index a8b8a6bc7..e97f7b64c 100644 --- a/backend/internal/protocol/types.go +++ b/backend/internal/protocol/types.go @@ -37,9 +37,11 @@ const ( ) type MediaReference struct { + ID string `json:"id,omitempty"` URL string `json:"url,omitempty"` DataURL string `json:"dataUrl,omitempty"` Kind string `json:"kind,omitempty"` + Role string `json:"role,omitempty"` Ephemeral bool `json:"ephemeral,omitempty"` } diff --git a/backend/internal/service/provider.go b/backend/internal/service/provider.go index c27d5afa4..eb8fd9ca9 100644 --- a/backend/internal/service/provider.go +++ b/backend/internal/service/provider.go @@ -2341,7 +2341,7 @@ func protocolRequestFromInput(input canvasGenerationInput) protocol.GenerationRe request := protocol.GenerationRequest{ Model: input.Config.Model, Prompt: input.Prompt, - Images: protocolMediaReferences(input.ReferenceImages, "image"), + Images: protocolVideoImageReferences(input), Videos: protocolMediaReferences(input.ReferenceVideos, "video"), Audios: protocolMediaReferences(input.ReferenceAudios, "audio"), AspectRatio: input.Config.Size, @@ -2370,10 +2370,31 @@ func protocolRequestFromInput(input canvasGenerationInput) protocol.GenerationRe return request } +func protocolVideoImageReferences(input canvasGenerationInput) []protocol.MediaReference { + result := make([]protocol.MediaReference, 0, len(input.ReferenceImages)) + fallbackRole := "" + if metadataString(input.Metadata, "videoStartFrameNodeId") != "" || metadataString(input.Metadata, "videoEndFrameNodeId") != "" { + fallbackRole = "reference_image" + } + for _, value := range input.ReferenceImages { + item := protocol.MediaReference{ + ID: strings.TrimSpace(value.ID), + URL: strings.TrimSpace(value.URL), + DataURL: strings.TrimSpace(value.DataURL), + Kind: "image", + Role: videoImageRoleOrDefault(input, value, fallbackRole), + } + if item.URL != "" || item.DataURL != "" { + result = append(result, item) + } + } + return result +} + func protocolMediaReferences(values []providerMedia, kind string) []protocol.MediaReference { result := make([]protocol.MediaReference, 0, len(values)) for _, value := range values { - item := protocol.MediaReference{URL: strings.TrimSpace(value.URL), DataURL: strings.TrimSpace(value.DataURL), Kind: kind} + item := protocol.MediaReference{ID: strings.TrimSpace(value.ID), URL: strings.TrimSpace(value.URL), DataURL: strings.TrimSpace(value.DataURL), Kind: kind} if item.URL != "" || item.DataURL != "" { result = append(result, item) } @@ -2868,16 +2889,9 @@ func runMiniMaxVideoTask(ctx context.Context, input canvasGenerationInput) (map[ if err != nil { return nil, fmt.Errorf("MiniMax 参考图无效:%w", err) } - role := "reference_image" - if frameMode { - switch { - case startFrameID != "" && image.ID == startFrameID: - role = "first_frame" - case endFrameID != "" && image.ID == endFrameID: - role = "last_frame" - default: - return nil, errors.New("MiniMax 首尾帧模式不能混合未标记的参考图") - } + role := videoImageRole(input, image) + if frameMode && role == "reference_image" { + return nil, errors.New("MiniMax 首尾帧模式不能混合未标记的参考图") } content = append(content, miniMaxVideoContent{Type: "image_url", ImageURL: &miniMaxMediaURL{URL: url}, Role: role}) } @@ -3014,6 +3028,9 @@ func runGeminiVeoVideoTask(ctx context.Context, input canvasGenerationInput) (ma if len(input.ReferenceImages) > 1 || len(input.ReferenceVideos) > 0 || len(input.ReferenceAudios) > 0 { return nil, errors.New("Gemini Veo 当前只支持 1 张起始图,不支持参考视频或音频") } + if len(input.ReferenceImages) > 0 && metadataString(input.Metadata, "videoEditOperation") == "reference_to_video" { + return nil, errors.New("Gemini Veo 当前不支持角色或风格参考图生视频,请改用支持 reference_to_video 的模型") + } id := resumedProviderRequestID(ctx) if id == "" { instance := geminiVeoInstance{Prompt: strings.TrimSpace(input.Prompt)} @@ -3074,6 +3091,9 @@ func runNovitaVideoTask(ctx context.Context, input canvasGenerationInput) (map[s if len(input.ReferenceImages) > 1 || len(input.ReferenceVideos) > 0 || len(input.ReferenceAudios) > 0 { return nil, errors.New("Novita 视频当前只支持 1 张起始图,不支持参考视频或参考音频") } + if len(input.ReferenceImages) > 0 && metadataString(input.Metadata, "videoEditOperation") == "reference_to_video" { + return nil, errors.New("Novita 视频当前不支持角色或风格参考图生视频,请改用支持 reference_to_video 的模型") + } id := resumedProviderRequestID(ctx) if id == "" { body := map[string]interface{}{ @@ -3636,7 +3656,7 @@ func newAPIChannel1VideoBody(input canvasGenerationInput) (map[string]interface{ if err != nil { return nil, err } - media = append(media, map[string]string{"type": seedanceImageRole(input, image), "url": url}) + media = append(media, map[string]string{"type": videoImageRole(input, image), "url": url}) } } for _, video := range input.ReferenceVideos { @@ -3798,6 +3818,9 @@ func xaiVideoRequestBody(input canvasGenerationInput) (xaiVideoRequest, error) { return body, nil } startFrameID := metadataString(input.Metadata, "videoStartFrameNodeId") + if metadataString(input.Metadata, "videoEditOperation") == "reference_to_video" { + startFrameID = "" + } if startFrameID == "" { // 未设置首帧:所有参考图作为 R2V 参考,不受单张起始图限制。 for index := range input.ReferenceImages { @@ -4874,7 +4897,7 @@ func seedanceContent(input canvasGenerationInput) ([]map[string]interface{}, err if err != nil { return nil, err } - content = append(content, map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": url}, "role": seedanceImageRole(input, image)}) + content = append(content, map[string]interface{}{"type": "image_url", "image_url": map[string]interface{}{"url": url}, "role": videoImageRole(input, image)}) } for _, video := range input.ReferenceVideos { url, err := mediaReferenceURL(video) @@ -4943,7 +4966,9 @@ func seedanceVideosRequestBody(input canvasGenerationInput) (seedanceVideosReque if err != nil { return seedanceVideosRequest{}, err } - if len(frameImageURLs) > 0 { + if metadataString(input.Metadata, "videoEditOperation") == "reference_to_video" { + body.ReferenceImageURLs = imageURLs + } else if len(frameImageURLs) > 0 { body.ImageURLs = frameImageURLs } else if len(imageURLs) > 0 { body.ImageURL = imageURLs[0] @@ -4993,17 +5018,27 @@ func seedanceVideosPromptText(input canvasGenerationInput) string { return strings.TrimSpace(input.Prompt) } -func seedanceImageRole(input canvasGenerationInput, image providerMedia) string { +func videoImageRole(input canvasGenerationInput, image providerMedia) string { + return videoImageRoleOrDefault(input, image, "reference_image") +} + +func videoImageRoleOrDefault(input canvasGenerationInput, image providerMedia, fallback string) string { + if metadataString(input.Metadata, "videoEditOperation") == "reference_to_video" { + return "reference_image" + } if id := metadataString(input.Metadata, "videoStartFrameNodeId"); id != "" && image.ID == id { return "first_frame" } if id := metadataString(input.Metadata, "videoEndFrameNodeId"); id != "" && image.ID == id { return "last_frame" } - return "reference_image" + return fallback } func videoFrameImageURLs(input canvasGenerationInput, imageURLs []string) ([]string, error) { + if metadataString(input.Metadata, "videoEditOperation") == "reference_to_video" { + return nil, nil + } startFrameID := metadataString(input.Metadata, "videoStartFrameNodeId") endFrameID := metadataString(input.Metadata, "videoEndFrameNodeId") if startFrameID == "" && endFrameID == "" { diff --git a/backend/internal/service/provider_test.go b/backend/internal/service/provider_test.go index 39e078b80..d425dcf6c 100644 --- a/backend/internal/service/provider_test.go +++ b/backend/internal/service/provider_test.go @@ -1372,6 +1372,31 @@ func TestSeedanceVideosBodyUsesOrderedFrameImageURLsWhenConfigured(t *testing.T) } } +func TestSeedanceVideosBodyKeepsProjectAssetsAsReferenceImages(t *testing.T) { + body, err := seedanceVideosRequestBody(canvasGenerationInput{ + Prompt: "keep the character consistent", + Config: providerConfig{Model: "seedance-2.0"}, + ReferenceImages: []providerMedia{ + {ID: "character-1", DataURL: testReferenceImageDataURL}, + {ID: "character-2", DataURL: "data:image/png;base64,d29ybGQ="}, + }, + Metadata: map[string]interface{}{ + "videoEditOperation": "reference_to_video", + "videoStartFrameNodeId": "character-1", + }, + }) + if err != nil { + t.Fatalf("seedanceVideosRequestBody() error = %v", err) + } + want := []string{testReferenceImageDataURL, "data:image/png;base64,d29ybGQ="} + if !reflect.DeepEqual(body.ReferenceImageURLs, want) { + t.Fatalf("reference_image_urls = %#v, want %#v", body.ReferenceImageURLs, want) + } + if body.ImageURL != "" || body.ImageURLs != nil { + t.Fatalf("reference operation leaked frame fields: %#v", body) + } +} + func TestRunVideoTaskUsesNewAPIForAnyVideoModel(t *testing.T) { t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true") paths := make([]string, 0, 3) @@ -1731,6 +1756,22 @@ func TestXAIVideoBodyWithStartFrameKeepsOfficialImageShape(t *testing.T) { } } +func TestXAIVideoReferenceOperationIgnoresStaleStartFrameMetadata(t *testing.T) { + body, err := xaiVideoRequestBody(canvasGenerationInput{ + Config: providerConfig{Model: "grok-imagine-video-1.5", InterfaceType: "xai-video"}, + ReferenceImages: []providerMedia{ + {ID: "character", DataURL: testReferenceImageDataURL}, + }, + Metadata: map[string]interface{}{"videoEditOperation": "reference_to_video", "videoStartFrameNodeId": "character"}, + }) + if err != nil { + t.Fatalf("xaiVideoRequestBody() error = %v", err) + } + if body.Image != nil || len(body.ReferenceImages) != 1 { + t.Fatalf("xAI reference operation body = %#v", body) + } +} + func TestXAIVideoBodyWithStartFrameRejectsMultipleImages(t *testing.T) { _, err := xaiVideoRequestBody(canvasGenerationInput{ Config: providerConfig{Model: "grok-imagine-video-1.5", InterfaceType: "xai-video"}, @@ -1919,6 +1960,38 @@ func TestNewAPIChannel1VideoBodyMapsFramesAndReferences(t *testing.T) { } } +func TestProtocolRequestPreservesVideoImageIDsAndRoles(t *testing.T) { + request := protocolRequestFromInput(canvasGenerationInput{ + Mode: "video", + ReferenceImages: []providerMedia{ + {ID: "start", URL: "https://example.com/start.png"}, + {ID: "character", URL: "https://example.com/character.png"}, + }, + Metadata: map[string]interface{}{ + "videoEditOperation": "image_to_video", + "videoStartFrameNodeId": "start", + }, + }) + if len(request.Images) != 2 { + t.Fatalf("images = %#v", request.Images) + } + if request.Images[0].ID != "start" || request.Images[0].Role != "first_frame" { + t.Fatalf("start image = %#v", request.Images[0]) + } + if request.Images[1].ID != "character" || request.Images[1].Role != "reference_image" { + t.Fatalf("unmarked image role = %#v", request.Images[1]) + } + + request = protocolRequestFromInput(canvasGenerationInput{ + Mode: "video", + ReferenceImages: []providerMedia{{ID: "character", URL: "https://example.com/character.png"}}, + Metadata: map[string]interface{}{"videoEditOperation": "reference_to_video", "videoStartFrameNodeId": "character"}, + }) + if request.Images[0].Role != "reference_image" { + t.Fatalf("reference operation image = %#v", request.Images[0]) + } +} + func TestNewAPIChannel1VideoBodyRejectsInlineMedia(t *testing.T) { _, err := newAPIChannel1VideoBody(canvasGenerationInput{ Prompt: "make it move", diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index c52c1787b..b50c10579 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -42,6 +42,7 @@ description: 影策已实现功能的手工验证清单与未确认边界 - [x] 镜头轨道按当前制作阶段显示待生成、生成中、已生成和生成失败状态,状态标签固定在卡片右下角;横向镜头只在轨道内部滚动,不再撑宽脚本区或挤出产物预览。制作概览六个流程节点改为独立可点击卡片;前端类型检查、生产构建与相关专项测试通过。 - [x] 分镜生成设置接入统一模型选择、能力和计价链路,支持前台模型、系统模型与用户自定义模型;切换模型会归一画幅、分辨率/画质和时长,MiniMax-H3 的旧 `720` 会切换为 `768P`。项目设置可保存默认生图/视频模型;后端全量测试、MiniMax 回归测试和前端生产构建通过。 - [x] 单镜生成改为任务提交后立即返回,不同镜头可并行进入排队/生成状态;角色卡主图、绑定声音样本和场景资产会按 `图片N` / `音频N` 编译进提示词。MiniMax 项目资产使用 `reference_image` / `reference_audio`,仅显式首尾帧节点使用帧角色;产物固定绑定任务提交时的镜头版本。前后端全量测试与生产构建通过。 +- [x] 视频图片语义统一为首帧、尾帧和普通参考图三类,并贯穿前端直连、后端任务与声明式协议;Agnes、Seedance `/videos`、火山方舟、NewAPI 媒体任务、xAI 和 MiniMax 不再按图片数量误猜项目角色资产。Gemini Veo、Novita 等仅支持起始图的协议会明确拒绝 `reference_to_video`,不再静默降级成首帧。 - [ ] 在真实项目中初始化一个已有章节,确认不会误用旧 v1 工作流,刷新和切换章节后阶段、镜头和当前版本保持一致。 - [ ] 在同一章节准备待生成、生成中、已生成和生成失败镜头,确认灰、蓝、绿、红标签始终显示在卡片右下角;增加到几十个镜头后确认只有底部轨道横向滚动,中间脚本和右侧产物预览仍完整可见。 - [ ] 在浅色、深色、常规桌面和窄屏下检查制作概览六张流程卡片的边界、间距、完成态、悬停和键盘焦点,确认跳转目标正确。 @@ -51,6 +52,7 @@ description: 影策已实现功能的手工验证清单与未确认边界 - [ ] 在项目设置分别选择系统、用户自定义的默认生图/视频模型,刷新后进入分镜图、动作预演和镜头视频,确认项目默认优先、已下架模型回退全局默认;切换 Seedance、MiniMax-H3 等能力不同的模型时,画幅、时长、分辨率和本次预计积分立即更新,并与实际账单一致。 - [ ] 连续选择两个镜头并分别提交视频生成,确认两个镜头独立显示“正在提交/生成中”,第二个任务不会被第一个全局锁阻止;刷新页面后两个任务状态都能恢复,完成产物分别绑定各自提交时的镜头版本。 - [ ] 给镜头绑定带主图和声音样本的角色卡及场景资产,使用 MiniMax-H3 生成并检查请求日志:文本包含 `【资产参考】`、角色声音说明和台词,角色/场景图片为 `reference_image`、声音为 `reference_audio`,请求不再出现误判的 `first_frame`,画幅和 `768P` / `2K` 与界面选择一致。 +- [ ] 使用同一镜头资产依次切换 Agnes 2.5、Seedance `/videos`、火山方舟、NewAPI 媒体任务和 xAI,确认单张角色图仍走 reference 字段;再切换 Gemini Veo、Novita,确认提交前给出“不支持 reference_to_video”的明确提示,而不是把角色图当起始帧生成。 - [ ] 当前“资产拆分 AI、多宫格生成提交、批量视频任务、后端交付包”仍是后续阶段,界面不得显示为已完成或可用。 ## 角色声音素材绑定 diff --git a/web/src/services/api/video-provider-agnes.ts b/web/src/services/api/video-provider-agnes.ts index eae4db660..6f0aacfef 100644 --- a/web/src/services/api/video-provider-agnes.ts +++ b/web/src/services/api/video-provider-agnes.ts @@ -5,6 +5,7 @@ import type { ReferenceAudio, ReferenceVideo } from "@/types/media"; import type { ApiEnvelope, RequestOptions, ResolvedAiConfig, VideoGenerationTask, VideoGenerationTaskState } from "./video-contracts"; import type { VideoProviderDeps } from "./video-provider-deps"; +import { resolveVideoImageReferences, type VideoImageRole } from "./video-reference-roles"; type AgnesTaskResponse = { id?: string; @@ -44,9 +45,10 @@ export async function createAgnesVideoTask( Promise.all(videoReferences.map((item) => resolveAgnesMediaUrl(item.url, item.storageKey))), Promise.all(audioReferences.map((item) => resolveAgnesMediaUrl(item.url, item.storageKey))), ]); + const imageRoles = resolveVideoImageReferences(references, options, { videoCount: videoUrls.length, audioCount: audioUrls.length }).map(({ role }) => role); const payload = modelName === "agnes-video-v2.0" - ? buildAgnesV20Payload(config, modelName, normalizedPrompt, imageUrls, videoUrls, audioUrls) - : buildAgnesV25Payload(config, modelName, normalizedPrompt, imageUrls, videoUrls, audioUrls); + ? buildAgnesV20Payload(config, modelName, normalizedPrompt, imageUrls, imageRoles, videoUrls, audioUrls, options) + : buildAgnesV25Payload(config, modelName, normalizedPrompt, imageUrls, imageRoles, videoUrls, audioUrls); try { const raw = await deps.transport.post>(deps.transport.apiUrl("/videos"), payload, options); @@ -92,7 +94,7 @@ export function agnesPollUrl(config: ResolvedAiConfig, videoId: string, modelNam return base.toString(); } -function buildAgnesV25Payload(config: ResolvedAiConfig, model: string, prompt: string, images: string[], videos: string[], audios: string[]) { +function buildAgnesV25Payload(config: ResolvedAiConfig, model: string, prompt: string, images: string[], imageRoles: VideoImageRole[], videos: string[], audios: string[]) { if (!AGNES_V25_MODELS.has(model)) throw new Error(`Agnes 视频协议不支持模型 ${model}`); const seconds = agnesSeconds(config.videoSeconds); const size = agnesResolution(config.vquality); @@ -101,7 +103,10 @@ function buildAgnesV25Payload(config: ResolvedAiConfig, model: string, prompt: s if (images.length > 5) throw new Error("Agnes Video 2.5 Flash 最多支持 5 张参考图"); if (videos.length) throw new Error("Agnes Video 2.5 Flash 不支持参考视频"); } - const mode = videos.length || audios.length || images.length > 2 ? "reference" : images.length ? "keyframe" : "text"; + const hasReferenceImages = imageRoles.includes("reference_image"); + const hasFrameImages = imageRoles.includes("first_frame") || imageRoles.includes("last_frame"); + if (hasFrameImages && (hasReferenceImages || videos.length || audios.length)) throw new Error("Agnes Video 2.5 不能同时混用首尾帧和角色、视频或音频参考素材"); + const mode = videos.length || audios.length || hasReferenceImages ? "reference" : images.length ? "keyframe" : "text"; const payload: Record = { model, prompt, @@ -112,8 +117,10 @@ function buildAgnesV25Payload(config: ResolvedAiConfig, model: string, prompt: s n: 1, }; if (mode === "keyframe") { - payload.first_frame = images[0]; - if (images[1]) payload.last_frame = images[1]; + const firstFrame = imageForRole(images, imageRoles, "first_frame") || images[0]; + const lastFrame = imageForRole(images, imageRoles, "last_frame"); + payload.first_frame = firstFrame; + if (lastFrame) payload.last_frame = lastFrame; } else if (mode === "reference") { if (images.length) payload.images = images; if (audios.length) payload.audios = audios; @@ -122,8 +129,9 @@ function buildAgnesV25Payload(config: ResolvedAiConfig, model: string, prompt: s return payload; } -function buildAgnesV20Payload(config: ResolvedAiConfig, model: string, prompt: string, images: string[], videos: string[], audios: string[]) { +function buildAgnesV20Payload(config: ResolvedAiConfig, model: string, prompt: string, images: string[], imageRoles: VideoImageRole[], videos: string[], audios: string[], options?: RequestOptions) { if (videos.length || audios.length) throw new Error("Agnes Video V2.0 不支持参考视频或音频"); + if (options?.videoEditOperation === "reference_to_video" || imageRoles.includes("reference_image")) throw new Error("Agnes Video V2.0 不支持角色或风格参考图模式,请改用 Agnes Video 2.5"); const seconds = Math.max(1, Math.floor(Number(config.videoSeconds) || 5)); const frameRate = 24; const targetFrames = seconds * frameRate; @@ -133,11 +141,17 @@ function buildAgnesV20Payload(config: ResolvedAiConfig, model: string, prompt: s frame_rate: frameRate, num_frames: Math.min(441, Math.floor((targetFrames - 1 + 4) / 8) * 8 + 1), }; - if (images.length === 1) payload.image = images[0]; - if (images.length > 1) payload.extra_body = { image: images, mode: "keyframes" }; + const orderedImages = [imageForRole(images, imageRoles, "first_frame"), imageForRole(images, imageRoles, "last_frame")].filter((value): value is string => Boolean(value)); + if (orderedImages.length === 1) payload.image = orderedImages[0]; + if (orderedImages.length > 1) payload.extra_body = { image: orderedImages, mode: "keyframes" }; return payload; } +function imageForRole(images: string[], roles: VideoImageRole[], role: VideoImageRole) { + const index = roles.indexOf(role); + return index >= 0 ? images[index] : undefined; +} + function agnesSeconds(value: string) { const seconds = Math.floor(Number(value) || 5); if (seconds < 4 || seconds > 12) throw new Error("Agnes Video 2.5 时长必须在 4–12 秒之间"); diff --git a/web/src/services/api/video-provider-gemini.ts b/web/src/services/api/video-provider-gemini.ts index cfe0f5885..854dae35c 100644 --- a/web/src/services/api/video-provider-gemini.ts +++ b/web/src/services/api/video-provider-gemini.ts @@ -16,6 +16,7 @@ type GeminiVeoOperation = { export async function createGeminiVeoTask(deps: VideoProviderDeps, config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], options?: RequestOptions): Promise { if (references.length > 1 || videoReferences.length || audioReferences.length) throw new Error("Gemini Veo 当前只支持 1 张起始图,不支持参考视频或音频"); + if (references.length && options?.videoEditOperation === "reference_to_video") throw new Error("Gemini Veo 当前不支持角色或风格参考图生视频,请改用支持 reference_to_video 的模型"); const instance: Record = { prompt: prompt.trim() }; if (references[0]) { const dataUrl = await imageToDataUrl(references[0]); diff --git a/web/src/services/api/video-provider-minimax.ts b/web/src/services/api/video-provider-minimax.ts index 06f3b25b8..df9785b27 100644 --- a/web/src/services/api/video-provider-minimax.ts +++ b/web/src/services/api/video-provider-minimax.ts @@ -9,22 +9,22 @@ import type { ReferenceAudio, ReferenceVideo } from "@/types/media"; import type { MiniMaxVideoCreateResponse, MiniMaxVideoTask, RequestOptions, ResolvedAiConfig, VideoGenerationTask, VideoGenerationTaskState } from "./video-contracts"; import type { VideoProviderDeps } from "./video-provider-deps"; +import { resolveVideoImageReferences } from "./video-reference-roles"; export async function createMiniMaxVideoTask(deps: VideoProviderDeps, config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], options?: RequestOptions): Promise { const imageUrls = await Promise.all(references.slice(0, 9).map((image) => resolveMiniMaxImageUrl(image))); const videoUrls = await Promise.all(videoReferences.slice(0, 3).map((video) => resolveMiniMaxMediaUrl(video, "参考视频"))); const audioUrls = await Promise.all(audioReferences.slice(0, 3).map((audio) => resolveMiniMaxMediaUrl(audio, "参考音频"))); const content: Array> = [{ type: "text", text: prompt.trim() }]; - const explicitFrameMode = options?.videoEditOperation !== "reference_to_video" && Boolean(options?.videoStartFrameNodeId || options?.videoEndFrameNodeId); + const imagePlan = resolveVideoImageReferences(references.slice(0, 9), options, { videoCount: videoUrls.length, audioCount: audioUrls.length }); + const frameMode = imagePlan.some(({ role }) => role === "first_frame" || role === "last_frame"); imageUrls.forEach((url, index) => { - const reference = references[index]; - const role = miniMaxImageRole(reference?.id, options); - if (explicitFrameMode && role === "reference_image") throw new Error("MiniMax 首尾帧模式不能混合未标记的参考图"); + const role = imagePlan[index]?.role || "reference_image"; + if (frameMode && role === "reference_image") throw new Error("MiniMax 首尾帧模式不能混合未标记的参考图"); content.push({ type: "image_url", image_url: { url }, role }); }); videoUrls.forEach((url) => content.push({ type: "video_url", video_url: { url }, role: "reference_video" })); audioUrls.forEach((url) => content.push({ type: "audio_url", audio_url: { url }, role: "reference_audio" })); - const frameMode = explicitFrameMode; const payload = { model: modelOptionName(model), content, @@ -43,13 +43,6 @@ export async function createMiniMaxVideoTask(deps: VideoProviderDeps, config: Re } } -function miniMaxImageRole(imageId: string | undefined, options?: RequestOptions) { - if (options?.videoEditOperation === "reference_to_video") return "reference_image"; - if (imageId && imageId === options?.videoStartFrameNodeId) return "first_frame"; - if (imageId && imageId === options?.videoEndFrameNodeId) return "last_frame"; - return "reference_image"; -} - export async function pollMiniMaxVideoTask(deps: VideoProviderDeps, config: ResolvedAiConfig, task: VideoGenerationTask, options?: RequestOptions): Promise { try { const response = await deps.transport.get<{ task?: MiniMaxVideoTask }>(miniMaxVideoUrl(config, `/query/video_generation/${encodeURIComponent(task.id)}`), options); diff --git a/web/src/services/api/video-provider-novita.ts b/web/src/services/api/video-provider-novita.ts index 54d91675d..5d7c9972a 100644 --- a/web/src/services/api/video-provider-novita.ts +++ b/web/src/services/api/video-provider-novita.ts @@ -12,6 +12,7 @@ type NovitaVideoResult = { task?: { status?: string; reason?: string }; videos?: export async function createNovitaVideoTask(deps: VideoProviderDeps, config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], options?: RequestOptions): Promise { if (references.length > 1 || videoReferences.length || audioReferences.length) throw new Error("Novita 视频当前只支持 1 张起始图,不支持参考视频或音频"); + if (references.length && options?.videoEditOperation === "reference_to_video") throw new Error("Novita 视频当前不支持角色或风格参考图生视频,请改用支持 reference_to_video 的模型"); const payload: Record = { model: modelOptionName(model), prompt: prompt.trim(), diff --git a/web/src/services/api/video-provider-openai.ts b/web/src/services/api/video-provider-openai.ts index eb013c1f6..f03f3f4e5 100644 --- a/web/src/services/api/video-provider-openai.ts +++ b/web/src/services/api/video-provider-openai.ts @@ -13,13 +13,21 @@ export async function createOpenAIVideoTask(deps: VideoProviderDeps, config: Res if (config.interfaceType === "xai-video" || modelName.toLowerCase().includes("grok")) { const images = await Promise.all(references.slice(0, 7).map((image) => imageToDataUrl(image))); const seconds = normalizeVideoSeconds(config.videoSeconds); + const referenceMode = options?.videoEditOperation === "reference_to_video"; + const explicitFrameMode = !referenceMode && Boolean(options?.videoStartFrameNodeId || options?.videoEndFrameNodeId); + if (config.interfaceType === "xai-video" && explicitFrameMode && (images.length > 1 || options?.videoEndFrameNodeId)) throw new Error("xAI 视频协议最多支持 1 张起始图,不支持尾帧或混合角色参考图"); + const imagePayload = config.interfaceType === "xai-video" + ? referenceMode && images.length + ? { reference_images: images.map((url) => ({ url })) } + : images.length ? { image: { url: images[0] } } : {} + : images.length ? { image: images[0], images } : {}; const payload = { model: modelName, prompt, duration: Number.parseInt(seconds, 10) || 6, seconds, ...(normalizeVideoSize(config.size) ? { size: normalizeVideoSize(config.size) } : {}), - ...(images.length ? { image: images[0], images } : {}), + ...imagePayload, }; try { const createPath = config.interfaceType === "xai-video" ? "/videos/generations" : "/videos"; diff --git a/web/src/services/api/video-provider-seedance.ts b/web/src/services/api/video-provider-seedance.ts index a82e69d09..12e84b9e0 100644 --- a/web/src/services/api/video-provider-seedance.ts +++ b/web/src/services/api/video-provider-seedance.ts @@ -10,6 +10,7 @@ import type { ReferenceAudio, ReferenceVideo } from "@/types/media"; import { isPublicMediaUrl } from "./video-validation"; import type { ApiEnvelope, RequestOptions, ResolvedAiConfig, SeedanceTask, VideoGenerationTask, VideoGenerationTaskState } from "./video-contracts"; import type { VideoProviderDeps } from "./video-provider-deps"; +import { hasExplicitVideoFrames, resolveVideoImageReferences } from "./video-reference-roles"; export function isSeedanceConfig(config: ResolvedAiConfig) { return config.interfaceType === "volcengine-ark-video" || isSeedanceVideoConfig(config); @@ -20,8 +21,8 @@ export async function createSeedanceTask(deps: VideoProviderDeps, config: Resolv assertSeedanceAudioReferences(audioReferences); const isVolcengineArk = config.interfaceType === "volcengine-ark-video"; const payload = isVolcengineArk || isArkPlanBaseUrl(config.baseUrl) - ? await buildSeedanceAgentPlanPayload(config, model, prompt, references, videoReferences, audioReferences, deps) - : await buildSeedanceVideosPayload(config, model, prompt, references, videoReferences, audioReferences, deps); + ? await buildSeedanceAgentPlanPayload(config, model, prompt, references, videoReferences, audioReferences, deps, options) + : await buildSeedanceVideosPayload(config, model, prompt, references, videoReferences, audioReferences, deps, options); try { const raw = await deps.transport.post>(seedanceApiUrl(config), payload, options); @@ -80,13 +81,13 @@ function seedanceApiUrl(config: ResolvedAiConfig, taskId?: string) { return buildApiUrl(config.baseUrl, `/videos${taskId ? `/${encodeURIComponent(taskId)}` : ""}`); } -async function buildSeedanceAgentPlanPayload(config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps) { +async function buildSeedanceAgentPlanPayload(config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps, options?: RequestOptions) { if (audioReferences.length && !references.length && !videoReferences.length) { throw new Error(config.interfaceType === "volcengine-ark-video" ? "火山方舟全模态参考不支持纯音频或文本+音频,请同时添加参考图片或参考视频" : "Seedance 参考音频不能单独使用,请同时添加参考图或参考视频"); } const content = config.interfaceType === "volcengine-ark-video" - ? await buildVolcengineArkContent(prompt, references, videoReferences, audioReferences) - : await buildSeedanceContent(config, prompt, references, videoReferences, audioReferences, deps); + ? await buildVolcengineArkContent(prompt, references, videoReferences, audioReferences, options) + : await buildSeedanceContent(config, prompt, references, videoReferences, audioReferences, deps, options); if (!content.length) throw new Error("请输入视频提示词,或连接参考图片/视频/音频"); const profile = modelCapabilityConfigFor(config, model).video!; return { @@ -100,11 +101,12 @@ async function buildSeedanceAgentPlanPayload(config: ResolvedAiConfig, model: st }; } -async function buildVolcengineArkContent(prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) { +async function buildVolcengineArkContent(prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], options?: RequestOptions) { const content: Array> = []; + const imagePlan = resolveVideoImageReferences(references.slice(0, SEEDANCE_REFERENCE_LIMITS.images), options, { videoCount: videoReferences.length, audioCount: audioReferences.length }); if (prompt.trim()) content.push({ type: "text", text: prompt.trim() }); - for (const image of references.slice(0, SEEDANCE_REFERENCE_LIMITS.images)) { - content.push({ type: "image_url", image_url: { url: await resolveVolcengineArkReferenceUrl(image.url || image.dataUrl, image.storageKey) }, role: "reference_image" }); + for (const { image, role } of imagePlan) { + content.push({ type: "image_url", image_url: { url: await resolveVolcengineArkReferenceUrl(image.url || image.dataUrl, image.storageKey) }, role }); } for (const video of videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos)) { content.push({ type: "video_url", video_url: { url: await resolveVolcengineArkReferenceUrl(video.url, video.storageKey) }, role: "reference_video" }); @@ -121,35 +123,45 @@ async function resolveVolcengineArkReferenceUrl(value: string | undefined, stora throw new Error("火山方舟视频参考素材需要公网 URL 或 asset:// 素材 ID;请先将本地素材保存到对象存储"); } -async function buildSeedanceVideosPayload(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps) { +async function buildSeedanceVideosPayload(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps, options?: RequestOptions) { if ((videoReferences.length || audioReferences.length) && !references.length) { throw new Error("Seedance 参考视频或参考音频需要同时连接至少 1 张主参考图"); } const imageUrls = await Promise.all(references.slice(0, SEEDANCE_REFERENCE_LIMITS.images).map(resolveSeedanceVideosImageUrl)); + const imagePlan = resolveVideoImageReferences(references.slice(0, SEEDANCE_REFERENCE_LIMITS.images), options, { videoCount: videoReferences.length, audioCount: audioReferences.length }); const videoUrls = await Promise.all(videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos).map((media) => resolveSeedanceVideosMediaUrl(media, deps))); const audioUrls = await Promise.all(audioReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.audios).map((media) => resolveSeedanceVideosMediaUrl(media, deps))); const ratio = normalizeSeedanceRatio(config.size); const duration = normalizeSeedanceDuration(config.videoSeconds); const profile = modelCapabilityConfigFor(config, model).video!; + const imagePayload: Record = {}; + if (options?.videoEditOperation === "reference_to_video") { + if (imageUrls.length) imagePayload.reference_image_urls = imageUrls; + } else if (hasExplicitVideoFrames(options)) { + imagePayload.image_urls = orderedImageUrls(imageUrls, imagePlan.map(({ role }) => role)); + } else if (imageUrls.length) { + imagePayload.image_url = imageUrls[0]; + if (imageUrls.length > 1) imagePayload.reference_image_urls = imageUrls.slice(1); + } return { model: modelOptionName(model), prompt: prompt.trim(), aspect_ratio: ratio === "adaptive" ? "16:9" : ratio, duration, ...(profile.generateAudio.supported ? { generate_audio: boolConfig(config.videoGenerateAudio, profile.generateAudio.default) } : {}), - ...(imageUrls[0] ? { image_url: imageUrls[0] } : {}), - ...(imageUrls.length > 1 ? { reference_image_urls: imageUrls.slice(1) } : {}), + ...imagePayload, ...(videoUrls.length ? { reference_videos: videoUrls } : {}), ...(audioUrls.length ? { reference_audios: audioUrls } : {}), }; } -async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps) { +async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps, options?: RequestOptions) { const content: Array> = []; + const imagePlan = resolveVideoImageReferences(references.slice(0, SEEDANCE_REFERENCE_LIMITS.images), options, { videoCount: videoReferences.length, audioCount: audioReferences.length }); const text = buildSeedancePromptText(prompt, references, videoReferences, audioReferences); if (text) content.push({ type: "text", text }); - for (const image of references.slice(0, SEEDANCE_REFERENCE_LIMITS.images)) { - content.push({ type: "image_url", image_url: { url: await resolveSeedanceImageUrl(image) }, role: "reference_image" }); + for (const { image, role } of imagePlan) { + content.push({ type: "image_url", image_url: { url: await resolveSeedanceImageUrl(image) }, role }); } for (const video of videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos)) { content.push({ type: "video_url", video_url: { url: await resolveSeedanceMediaUrl(video, deps, "参考视频") }, role: "reference_video" }); @@ -160,6 +172,10 @@ async function buildSeedanceContent(config: AiConfig, prompt: string, references return content; } +function orderedImageUrls(imageUrls: string[], roles: Array<"first_frame" | "last_frame" | "reference_image">) { + return ["first_frame", "last_frame", "reference_image"].flatMap((role) => imageUrls.filter((_, index) => roles[index] === role)); +} + async function resolveSeedanceImageUrl(image: ReferenceImage) { const directUrl = image.url || image.dataUrl; if (isPublicMediaUrl(directUrl) || directUrl.startsWith("asset://")) return directUrl; diff --git a/web/src/services/api/video-reference-roles.ts b/web/src/services/api/video-reference-roles.ts new file mode 100644 index 000000000..2284107c8 --- /dev/null +++ b/web/src/services/api/video-reference-roles.ts @@ -0,0 +1,45 @@ +import type { ReferenceImage } from "@/types/image"; + +import type { RequestOptions } from "./video-contracts"; + +export type VideoImageRole = "first_frame" | "last_frame" | "reference_image"; + +export type ResolvedVideoImageReference = { + image: ReferenceImage; + role: VideoImageRole; +}; + +type VideoReferenceContext = { + videoCount?: number; + audioCount?: number; +}; + +export function resolveVideoImageReferences(images: ReferenceImage[], options?: RequestOptions, context: VideoReferenceContext = {}): ResolvedVideoImageReference[] { + const operation = options?.videoEditOperation?.trim(); + if (operation === "reference_to_video") { + return images.map((image) => ({ image, role: "reference_image" })); + } + + const startFrameId = options?.videoStartFrameNodeId?.trim(); + const endFrameId = options?.videoEndFrameNodeId?.trim(); + if (startFrameId || endFrameId) { + return images.map((image) => ({ + image, + role: image.id === startFrameId ? "first_frame" : image.id === endFrameId ? "last_frame" : "reference_image", + })); + } + + if (context.videoCount || context.audioCount) { + return images.map((image) => ({ image, role: "reference_image" })); + } + + // 传统创作页没有帧节点 ID:保留单图首帧、双图首尾帧的历史语义。 + return images.map((image, index) => ({ + image, + role: index === 0 ? "first_frame" : index === 1 && images.length === 2 ? "last_frame" : "reference_image", + })); +} + +export function hasExplicitVideoFrames(options?: RequestOptions) { + return options?.videoEditOperation !== "reference_to_video" && Boolean(options?.videoStartFrameNodeId || options?.videoEndFrameNodeId); +} diff --git a/web/test/agnes-video.test.ts b/web/test/agnes-video.test.ts index 01cb73896..16a531a10 100644 --- a/web/test/agnes-video.test.ts +++ b/web/test/agnes-video.test.ts @@ -124,6 +124,27 @@ describe("Agnes Video 2.5 request contract", () => { }); }); + test("treats a single storyboard character image as a reference asset", async () => { + const config = resolveModelRequestConfig(configForAgnes(), "agnes::agnes-video-2.5"); + const { calls, deps } = providerDeps(config); + await createAgnesVideoTask( + deps, + config, + "agnes::agnes-video-2.5", + "保持角色一致", + [{ id: "character-1", name: "character.png", type: "image/png", dataUrl: "", url: "https://cdn.example.com/character.png" }], + [], + [], + { videoEditOperation: "reference_to_video" }, + ); + + expect(calls[0]?.body).toMatchObject({ + mode: "reference", + images: ["https://cdn.example.com/character.png"], + }); + expect(calls[0]?.body as Record).not.toHaveProperty("first_frame"); + }); + test("polls the Agnes host root and reads metadata.url", async () => { const config = resolveModelRequestConfig(configForAgnes(), "agnes::agnes-video-2.5"); const { calls, deps } = providerDeps(config, { polled: { video_id: "video-1", status: "completed", metadata: { url: "https://cdn.example.com/result.mp4" } } }); diff --git a/web/test/video-api-contracts.test.ts b/web/test/video-api-contracts.test.ts index d85469f0e..f97b03436 100644 --- a/web/test/video-api-contracts.test.ts +++ b/web/test/video-api-contracts.test.ts @@ -100,6 +100,63 @@ describe("Volcengine Ark full-modal references", () => { const deps = { transport: {}, response: videoResponseTools } as unknown as VideoProviderDeps; await expect(createSeedanceTask(deps, config as never, model, "跟随节奏", [], [], [{ id: "audio-1", name: "audio.mp3", type: "audio/mpeg", url: "https://cdn.example.com/audio.mp3" }])).rejects.toThrow("不支持纯音频或文本+音频"); }); + + test("显式首帧与项目角色参考图使用不同角色", async () => { + let requestBody: unknown; + const deps = { + transport: { post: async (_url: string, body: unknown) => (requestBody = body, { id: "task-1" }) }, + response: videoResponseTools, + } as unknown as VideoProviderDeps; + const images = [ + { id: "character", name: "character.png", type: "image/png", dataUrl: "", url: "https://cdn.example.com/character.png" }, + { id: "start", name: "start.png", type: "image/png", dataUrl: "", url: "https://cdn.example.com/start.png" }, + ]; + + await createSeedanceTask(deps, config as never, model, "开始运动", images, [], [], { + videoEditOperation: "image_to_video", + videoStartFrameNodeId: "start", + }); + + expect((requestBody as { content: unknown[] }).content).toContainEqual({ + type: "image_url", + image_url: { url: "https://cdn.example.com/start.png" }, + role: "first_frame", + }); + expect((requestBody as { content: unknown[] }).content).toContainEqual({ + type: "image_url", + image_url: { url: "https://cdn.example.com/character.png" }, + role: "reference_image", + }); + }); +}); + +describe("Seedance /videos image roles", () => { + test("reference_to_video does not promote the first character image to image_url", async () => { + let requestBody: Record = {}; + const deps = { + transport: { post: async (_url: string, body: unknown) => (requestBody = body as Record, { id: "task-1" }) }, + response: videoResponseTools, + } as unknown as VideoProviderDeps; + const config = { ...defaultConfig, baseUrl: "https://video.example.com/v1", model: "seedance-2.0", videoModel: "seedance-2.0" }; + + await createSeedanceTask( + deps, + config as never, + "seedance-2.0", + "保持角色一致", + [ + { id: "character-1", name: "1.png", type: "image/png", dataUrl: "", url: "https://cdn.example.com/1.png" }, + { id: "character-2", name: "2.png", type: "image/png", dataUrl: "", url: "https://cdn.example.com/2.png" }, + ], + [], + [], + { videoEditOperation: "reference_to_video" }, + ); + + expect(requestBody.reference_image_urls).toEqual(["https://cdn.example.com/1.png", "https://cdn.example.com/2.png"]); + expect(requestBody).not.toHaveProperty("image_url"); + expect(requestBody).not.toHaveProperty("image_urls"); + }); }); describe("MiniMax explicit reference roles", () => {