feat(video): support Ark full-modal references (#300)

backend/internal/service/model_capability.go

@@ -206,6 +206,7 @@ func DefaultModelCapabilityConfigForModel(protocol string, modelName string) *Mo
 		video.Duration = VideoDurationConfig{Selection: "enum", Values: []int{4, 6, 8}, Default: 6}
 		video.Resolutions = []string{"720p", "1080p"}
 	case model.ChannelInterfaceVolcengineArkVideo:
+		video.Operations = append(video.Operations, "reference_to_video", "audio_to_video")
 		video.References.MaxVideos, video.References.MaxAudios = 3, 3
 		video.References.MaxVideoBytes, video.References.MaxAudioBytes = 200*1024*1024, 15*1024*1024
 		video.References.MaxVideoDuration, video.References.MaxAudioDuration = 15, 15
@@ -601,6 +602,9 @@ func validateVideoTask(profile *VideoCapabilityConfig, input canvasGenerationInp
 	if len(input.ReferenceImages) > profile.References.MaxImages || len(input.ReferenceVideos) > profile.References.MaxVideos || len(input.ReferenceAudios) > profile.References.MaxAudios {
 		return BadAuthRequest("参考素材数量超过当前模型限制")
 	}
+	if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) && len(input.ReferenceAudios) > 0 && len(input.ReferenceImages) == 0 && len(input.ReferenceVideos) == 0 {
+		return BadAuthRequest("火山方舟全模态参考不支持纯音频或文本+音频，请同时添加参考图片或参考视频")
+	}
 	if len(input.ReferenceImages) < profile.References.MinImages {
 		return BadAuthRequest(fmt.Sprintf("当前视频模型至少需要 %d 张参考图", profile.References.MinImages))
 	}

backend/internal/service/model_capability_test.go

@@ -93,6 +93,40 @@ func TestDefaultMiniMaxVideoCapabilitySupportsReferenceGeneration(t *testing.T)
 	}
 }
 
+func TestDefaultVolcengineArkVideoCapabilitySupportsFullModalReference(t *testing.T) {
+	profile := DefaultModelCapabilityConfigForModel("volcengine-ark-video", "doubao-seedance-2-0-260128")
+	if profile == nil || profile.Video == nil {
+		t.Fatal("Volcengine Ark video profile = nil")
+	}
+	for _, operation := range []string{"reference_to_video", "audio_to_video"} {
+		if !containsCapabilityString(profile.Video.Operations, operation) {
+			t.Fatalf("operations = %v, want %s", profile.Video.Operations, operation)
+		}
+	}
+	if profile.Video.References.MaxImages != 9 || profile.Video.References.MaxVideos != 3 || profile.Video.References.MaxAudios != 3 {
+		t.Fatalf("reference limits = %#v", profile.Video.References)
+	}
+}
+
+func TestValidateVolcengineArkFullModalReferenceRejectsTextAndAudioOnly(t *testing.T) {
+	profile := DefaultModelCapabilityConfigForModel("volcengine-ark-video", "doubao-seedance-2-0-260128").Video
+	input := canvasGenerationInput{
+		Prompt:          "follow the soundtrack",
+		Config:          providerConfig{InterfaceType: "volcengine-ark-video", VideoSeconds: "6", Size: "16:9", VQuality: "720p"},
+		ReferenceAudios: []providerMedia{{URL: "https://example.com/music.mp3"}},
+		Metadata:        map[string]interface{}{"videoEditOperation": "audio_to_video"},
+	}
+	err := validateVideoTask(profile, input)
+	if err == nil || !strings.Contains(err.Error(), "文本+音频") {
+		t.Fatalf("validateVideoTask() error = %v", err)
+	}
+
+	input.ReferenceImages = []providerMedia{{URL: "https://example.com/subject.png"}}
+	if err := validateVideoTask(profile, input); err != nil {
+		t.Fatalf("validateVideoTask(full modal) error = %v", err)
+	}
+}
+
 func TestCapabilitySpecFromModelCapabilityConfigRestoresLegacyWildcardImageSizes(t *testing.T) {
 	config := &ModelCapabilityConfig{
 		Version: 1,

backend/internal/service/provider_test.go

@@ -1446,12 +1446,19 @@ func TestVolcengineArkVideoProtocolUsesContentTaskAndDownloadsResult(t *testing.
 				t.Fatalf("decode request: %v", err)
 			}
 			content, _ := body["content"].([]interface{})
-			if len(content) != 2 {
+			if len(content) != 4 {
 				t.Errorf("body = %#v", body)
 				return
 			}
-			imageContent, _ := content[1].(map[string]interface{})
-			if body["model"] != "doubao-seedance-test" || imageContent["role"] != "reference_image" {
+			wantTypes := []string{"text", "image_url", "video_url", "audio_url"}
+			wantRoles := []string{"", "reference_image", "reference_video", "reference_audio"}
+			for index, item := range content {
+				entry, _ := item.(map[string]interface{})
+				if entry["type"] != wantTypes[index] || (wantRoles[index] != "" && entry["role"] != wantRoles[index]) {
+					t.Errorf("content[%d] = %#v", index, entry)
+				}
+			}
+			if body["model"] != "doubao-seedance-test" {
 				t.Errorf("body = %#v", body)
 			}
 			_, _ = w.Write([]byte(`{"id":"ark-task-1","status":"running"}`))
@@ -1470,6 +1477,8 @@ func TestVolcengineArkVideoProtocolUsesContentTaskAndDownloadsResult(t *testing.
 		Prompt:          "make it move",
 		Config:          providerConfig{BaseURL: server.URL + "/api/v3", APIKey: "test-key", Model: "doubao-seedance-test", InterfaceType: "volcengine-ark-video"},
 		ReferenceImages: []providerMedia{{ID: "start", URL: server.URL + "/reference.png"}},
+		ReferenceVideos: []providerMedia{{ID: "motion", URL: server.URL + "/reference.mp4"}},
+		ReferenceAudios: []providerMedia{{ID: "music", URL: server.URL + "/reference.mp3"}},
 		Metadata:        map[string]interface{}{"videoStartFrameNodeId": "start"},
 	})
 	if err != nil {