From 393b7f220faa2ceb46b4543daf8dad2177f14e0c Mon Sep 17 00:00:00 2001 From: Devin Kw Zhu <34890117+Mr-Devin@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:14:00 +0800 Subject: [PATCH] reference_images (#208) --- backend/internal/service/provider.go | 19 ++++- .../service/provider_request_types.go | 13 ++-- backend/internal/service/provider_test.go | 76 ++++++++++++++----- 3 files changed, 82 insertions(+), 26 deletions(-) diff --git a/backend/internal/service/provider.go b/backend/internal/service/provider.go index cfa8d1dea..7dc9a794d 100644 --- a/backend/internal/service/provider.go +++ b/backend/internal/service/provider.go @@ -2058,6 +2058,7 @@ func grokVideoBody(input canvasGenerationInput) (map[string]interface{}, error) } // xAI 生成接口与 legacy /videos 使用不同字段，保持独立可避免兼容字段触发上游 422。 +// 设置首帧时按 image-to-video 只传 image；未设置首帧时按 reference-to-video 把所有参考图放入 reference_images。 func xaiVideoRequestBody(input canvasGenerationInput) (xaiVideoRequest, error) { body := xaiVideoRequest{ Model: input.Config.Model, @@ -2069,8 +2070,24 @@ func xaiVideoRequestBody(input canvasGenerationInput) (xaiVideoRequest, error) { if !shouldSendNewAPIVideoImages(input) || len(input.ReferenceImages) == 0 { return body, nil } + startFrameID := metadataString(input.Metadata, "videoStartFrameNodeId") + if startFrameID == "" { + // 未设置首帧：所有参考图作为 R2V 参考，不受单张起始图限制。 + for index := range input.ReferenceImages { + imageURL, err := openAIImageInputURL(input.ReferenceImages[index]) + if err != nil { + return xaiVideoRequest{}, err + } + body.ReferenceImages = append(body.ReferenceImages, xaiVideoImage{URL: imageURL}) + } + return body, nil + } + // 设置了首帧：xAI 不允许 image 与 reference_images 同时出现，只把首帧作为起始图。 if len(input.ReferenceImages) > 1 { - return xaiVideoRequest{}, fmt.Errorf("xAI 图生视频只支持 1 张起始图，当前连接了 %d 张", len(input.ReferenceImages)) + return xaiVideoRequest{}, fmt.Errorf("xAI 设置首帧后只支持 1 张起始图，当前连接了 %d 张", len(input.ReferenceImages)) + } + if input.ReferenceImages[0].ID != startFrameID { + return xaiVideoRequest{}, errors.New("已配置的首帧参考图未包含在视频请求中") } imageURL, err := openAIImageInputURL(input.ReferenceImages[0]) if err != nil { diff --git a/backend/internal/service/provider_request_types.go b/backend/internal/service/provider_request_types.go index c6be13f0c..9ce618fae 100644 --- a/backend/internal/service/provider_request_types.go +++ b/backend/internal/service/provider_request_types.go @@ -40,12 +40,13 @@ type seedanceVideosRequest struct { } type xaiVideoRequest struct { - Model string `json:"model"` - Prompt string `json:"prompt"` - Duration int `json:"duration"` - AspectRatio string `json:"aspect_ratio"` - Resolution string `json:"resolution"` - Image *xaiVideoImage `json:"image,omitempty"` + Model string `json:"model"` + Prompt string `json:"prompt"` + Duration int `json:"duration"` + AspectRatio string `json:"aspect_ratio"` + Resolution string `json:"resolution"` + Image *xaiVideoImage `json:"image,omitempty"` + ReferenceImages []xaiVideoImage `json:"reference_images,omitempty"` } type xaiVideoImage struct { diff --git a/backend/internal/service/provider_test.go b/backend/internal/service/provider_test.go index 78ee34f3e..89f740be2 100644 --- a/backend/internal/service/provider_test.go +++ b/backend/internal/service/provider_test.go @@ -691,7 +691,7 @@ func TestRunVideoTaskUsesXAIVideoGenerationEndpoint(t *testing.T) { } } -func TestXAIVideoBodyUsesOfficialImageShapeAndNormalizesSettings(t *testing.T) { +func TestXAIVideoBodyWithoutStartFramePutsAllImagesIntoReferenceImages(t *testing.T) { body, err := xaiVideoRequestBody(canvasGenerationInput{ Prompt: "make it move", Config: providerConfig{ @@ -701,31 +701,69 @@ func TestXAIVideoBodyUsesOfficialImageShapeAndNormalizesSettings(t *testing.T) { Size: "1024x1792", VQuality: "1080", }, - ReferenceImages: []providerMedia{{ID: "image-1", DataURL: testReferenceImageDataURL}}, - Metadata: map[string]interface{}{"videoEditOperation": "image_to_video"}, - }) - if err != nil { - t.Fatalf("grokVideoBody() error = %v", err) - } - if body.Duration != 20 || body.AspectRatio != "9:16" || body.Resolution != "1080p" { - t.Fatalf("xAI settings = %#v", body) - } - if body.Image == nil || body.Image.URL != testReferenceImageDataURL { - t.Fatalf("image = %#v", body.Image) - } -} - -func TestXAIVideoBodyRejectsMultipleStartImages(t *testing.T) { - _, err := xaiVideoRequestBody(canvasGenerationInput{ - Config: providerConfig{Model: "grok-imagine-video-1.5", InterfaceType: "xai-video"}, ReferenceImages: []providerMedia{ {ID: "image-1", DataURL: testReferenceImageDataURL}, {ID: "image-2", DataURL: testReferenceImageDataURL}, }, Metadata: map[string]interface{}{"videoEditOperation": "image_to_video"}, }) + if err != nil { + t.Fatalf("xaiVideoRequestBody() error = %v", err) + } + if body.Duration != 20 || body.AspectRatio != "9:16" || body.Resolution != "1080p" { + t.Fatalf("xAI settings = %#v", body) + } + if body.Image != nil { + t.Fatalf("image should be nil without start frame, got %#v", body.Image) + } + if len(body.ReferenceImages) != 2 || body.ReferenceImages[0].URL != testReferenceImageDataURL || body.ReferenceImages[1].URL != testReferenceImageDataURL { + t.Fatalf("reference_images = %#v", body.ReferenceImages) + } +} + +func TestXAIVideoBodyWithStartFrameKeepsOfficialImageShape(t *testing.T) { + body, err := xaiVideoRequestBody(canvasGenerationInput{ + Config: providerConfig{Model: "grok-imagine-video-1.5", InterfaceType: "xai-video"}, + ReferenceImages: []providerMedia{ + {ID: "image-1", DataURL: testReferenceImageDataURL}, + }, + Metadata: map[string]interface{}{"videoEditOperation": "image_to_video", "videoStartFrameNodeId": "image-1"}, + }) + if err != nil { + t.Fatalf("xaiVideoRequestBody() error = %v", err) + } + if body.Image == nil || body.Image.URL != testReferenceImageDataURL { + t.Fatalf("image = %#v", body.Image) + } + if len(body.ReferenceImages) != 0 { + t.Fatalf("reference_images = %#v", body.ReferenceImages) + } +} + +func TestXAIVideoBodyWithStartFrameRejectsMultipleImages(t *testing.T) { + _, err := xaiVideoRequestBody(canvasGenerationInput{ + Config: providerConfig{Model: "grok-imagine-video-1.5", InterfaceType: "xai-video"}, + ReferenceImages: []providerMedia{ + {ID: "image-1", DataURL: testReferenceImageDataURL}, + {ID: "image-2", DataURL: testReferenceImageDataURL}, + }, + Metadata: map[string]interface{}{"videoEditOperation": "image_to_video", "videoStartFrameNodeId": "image-1"}, + }) if err == nil || !strings.Contains(err.Error(), "只支持 1 张起始图") { - t.Fatalf("grokVideoBody() error = %v", err) + t.Fatalf("xaiVideoRequestBody() error = %v", err) + } +} + +func TestXAIVideoBodyWithMissingStartFrameImageErrors(t *testing.T) { + _, err := xaiVideoRequestBody(canvasGenerationInput{ + Config: providerConfig{Model: "grok-imagine-video-1.5", InterfaceType: "xai-video"}, + ReferenceImages: []providerMedia{ + {ID: "image-2", DataURL: testReferenceImageDataURL}, + }, + Metadata: map[string]interface{}{"videoEditOperation": "image_to_video", "videoStartFrameNodeId": "image-1"}, + }) + if err == nil || !strings.Contains(err.Error(), "首帧参考图未包含") { + t.Fatalf("xaiVideoRequestBody() error = %v", err) } }