tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reference_images (#208)

Browse Source
This commit is contained in:
Devin Kw Zhu
2026-08-12 12:14:00 +08:00
committed by GitHub
parent feb5631237
commit 393b7f220f
3 changed files with 82 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
backend/internal/service/provider.go
View File

@@ -2058,6 +2058,7 @@ func grokVideoBody(input canvasGenerationInput) (map[string]interface{}, error)
}
// xAI 生成接口与 legacy /videos 使用不同字段，保持独立可避免兼容字段触发上游 422。
// 设置首帧时按 image-to-video 只传 image；未设置首帧时按 reference-to-video 把所有参考图放入 reference_images。
func xaiVideoRequestBody(input canvasGenerationInput) (xaiVideoRequest, error) {
body := xaiVideoRequest{
Model: input.Config.Model,
@@ -2069,8 +2070,24 @@ func xaiVideoRequestBody(input canvasGenerationInput) (xaiVideoRequest, error) {
if !shouldSendNewAPIVideoImages(input) || len(input.ReferenceImages) == 0 {
return body, nil
}
startFrameID := metadataString(input.Metadata, "videoStartFrameNodeId")
if startFrameID == "" {
// 未设置首帧：所有参考图作为 R2V 参考，不受单张起始图限制。
for index := range input.ReferenceImages {
imageURL, err := openAIImageInputURL(input.ReferenceImages[index])
if err != nil {
return xaiVideoRequest{}, err
}
body.ReferenceImages = append(body.ReferenceImages, xaiVideoImage{URL: imageURL})
}
return body, nil
}
// 设置了首帧：xAI 不允许 image 与 reference_images 同时出现，只把首帧作为起始图。
if len(input.ReferenceImages) > 1 {
return xaiVideoRequest{}, fmt.Errorf("xAI 图生视频只支持 1 张起始图，当前连接了 %d 张", len(input.ReferenceImages))
return xaiVideoRequest{}, fmt.Errorf("xAI 设置首帧后只支持 1 张起始图，当前连接了 %d 张", len(input.ReferenceImages))
}
if input.ReferenceImages[0].ID != startFrameID {
return xaiVideoRequest{}, errors.New("已配置的首帧参考图未包含在视频请求中")
}
imageURL, err := openAIImageInputURL(input.ReferenceImages[0])
if err != nil {

13
backend/internal/service/provider_request_types.go
View File

@@ -40,12 +40,13 @@ type seedanceVideosRequest struct {
}
type xaiVideoRequest struct {
Model string `json:"model"`
Prompt string `json:"prompt"`
Duration int `json:"duration"`
AspectRatio string `json:"aspect_ratio"`
Resolution string `json:"resolution"`
Image *xaiVideoImage `json:"image,omitempty"`
Model string `json:"model"`
Prompt string `json:"prompt"`
Duration int `json:"duration"`
AspectRatio string `json:"aspect_ratio"`
Resolution string `json:"resolution"`
Image *xaiVideoImage `json:"image,omitempty"`
ReferenceImages []xaiVideoImage `json:"reference_images,omitempty"`
}
type xaiVideoImage struct {

76
backend/internal/service/provider_test.go
View File

@@ -691,7 +691,7 @@ func TestRunVideoTaskUsesXAIVideoGenerationEndpoint(t *testing.T) {
}
}
func TestXAIVideoBodyUsesOfficialImageShapeAndNormalizesSettings(t *testing.T) {
func TestXAIVideoBodyWithoutStartFramePutsAllImagesIntoReferenceImages(t *testing.T) {
body, err := xaiVideoRequestBody(canvasGenerationInput{
Prompt: "make it move",
Config: providerConfig{
@@ -701,31 +701,69 @@ func TestXAIVideoBodyUsesOfficialImageShapeAndNormalizesSettings(t *testing.T) {
Size: "1024x1792",
VQuality: "1080",
},
ReferenceImages: []providerMedia{{ID: "image-1", DataURL: testReferenceImageDataURL}},
Metadata: map[string]interface{}{"videoEditOperation": "image_to_video"},
})
if err != nil {
t.Fatalf("grokVideoBody() error = %v", err)
}
if body.Duration != 20 || body.AspectRatio != "9:16" || body.Resolution != "1080p" {
t.Fatalf("xAI settings = %#v", body)
}
if body.Image == nil || body.Image.URL != testReferenceImageDataURL {
t.Fatalf("image = %#v", body.Image)
}
}
func TestXAIVideoBodyRejectsMultipleStartImages(t *testing.T) {
_, err := xaiVideoRequestBody(canvasGenerationInput{
Config: providerConfig{Model: "grok-imagine-video-1.5", InterfaceType: "xai-video"},
ReferenceImages: []providerMedia{
{ID: "image-1", DataURL: testReferenceImageDataURL},
{ID: "image-2", DataURL: testReferenceImageDataURL},
},
Metadata: map[string]interface{}{"videoEditOperation": "image_to_video"},
})
if err != nil {
t.Fatalf("xaiVideoRequestBody() error = %v", err)
}
if body.Duration != 20 || body.AspectRatio != "9:16" || body.Resolution != "1080p" {
t.Fatalf("xAI settings = %#v", body)
}
if body.Image != nil {
t.Fatalf("image should be nil without start frame, got %#v", body.Image)
}
if len(body.ReferenceImages) != 2 || body.ReferenceImages[0].URL != testReferenceImageDataURL || body.ReferenceImages[1].URL != testReferenceImageDataURL {
t.Fatalf("reference_images = %#v", body.ReferenceImages)
}
}
func TestXAIVideoBodyWithStartFrameKeepsOfficialImageShape(t *testing.T) {
body, err := xaiVideoRequestBody(canvasGenerationInput{
Config: providerConfig{Model: "grok-imagine-video-1.5", InterfaceType: "xai-video"},
ReferenceImages: []providerMedia{
{ID: "image-1", DataURL: testReferenceImageDataURL},
},
Metadata: map[string]interface{}{"videoEditOperation": "image_to_video", "videoStartFrameNodeId": "image-1"},
})
if err != nil {
t.Fatalf("xaiVideoRequestBody() error = %v", err)
}
if body.Image == nil || body.Image.URL != testReferenceImageDataURL {
t.Fatalf("image = %#v", body.Image)
}
if len(body.ReferenceImages) != 0 {
t.Fatalf("reference_images = %#v", body.ReferenceImages)
}
}
func TestXAIVideoBodyWithStartFrameRejectsMultipleImages(t *testing.T) {
_, err := xaiVideoRequestBody(canvasGenerationInput{
Config: providerConfig{Model: "grok-imagine-video-1.5", InterfaceType: "xai-video"},
ReferenceImages: []providerMedia{
{ID: "image-1", DataURL: testReferenceImageDataURL},
{ID: "image-2", DataURL: testReferenceImageDataURL},
},
Metadata: map[string]interface{}{"videoEditOperation": "image_to_video", "videoStartFrameNodeId": "image-1"},
})
if err == nil || !strings.Contains(err.Error(), "只支持 1 张起始图") {
t.Fatalf("grokVideoBody() error = %v", err)
t.Fatalf("xaiVideoRequestBody() error = %v", err)
}
}
func TestXAIVideoBodyWithMissingStartFrameImageErrors(t *testing.T) {
_, err := xaiVideoRequestBody(canvasGenerationInput{
Config: providerConfig{Model: "grok-imagine-video-1.5", InterfaceType: "xai-video"},
ReferenceImages: []providerMedia{
{ID: "image-2", DataURL: testReferenceImageDataURL},
},
Metadata: map[string]interface{}{"videoEditOperation": "image_to_video", "videoStartFrameNodeId": "image-1"},
})
if err == nil || !strings.Contains(err.Error(), "首帧参考图未包含") {
t.Fatalf("xaiVideoRequestBody() error = %v", err)
}
}