feat(video): support Ark full-modal references (#300)

backend/internal/protocol/builtin.go

@@ -367,10 +367,22 @@ func arkVideosAdapter() Adapter {
 	info := metadata("volcengine-ark-video", "火山方舟视频", "Volcengine Ark", CapabilityVideo, "POST /api/v3/contents/generations/tasks", "GET /api/v3/contents/generations/tasks/{task_id}", "application/json")
 	info.Parameters = videoParams()
 	return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) {
+		if len(r.Images) > 9 || len(r.Videos) > 3 || len(r.Audios) > 3 {
+			return RequestSpec{}, fmt.Errorf("火山方舟全模态参考最多支持 9 张图片、3 个视频和 3 个音频")
+		}
+		if len(r.Audios) > 0 && len(r.Images) == 0 && len(r.Videos) == 0 {
+			return RequestSpec{}, fmt.Errorf("火山方舟全模态参考不支持纯音频或文本+音频，请同时添加参考图片或参考视频")
+		}
 		content := []any{map[string]any{"type": "text", "text": r.Prompt}}
 		for _, image := range r.Images {
 			content = append(content, map[string]any{"type": "image_url", "image_url": map[string]any{"url": mediaValue(image)}, "role": "reference_image"})
 		}
+		for _, video := range r.Videos {
+			content = append(content, map[string]any{"type": "video_url", "video_url": map[string]any{"url": mediaValue(video)}, "role": "reference_video"})
+		}
+		for _, audio := range r.Audios {
+			content = append(content, map[string]any{"type": "audio_url", "audio_url": map[string]any{"url": mediaValue(audio)}, "role": "reference_audio"})
+		}
 		body := map[string]any{"model": r.Model, "content": content, "ratio": defaultValue(r.AspectRatio, "16:9"), "resolution": defaultValue(r.Resolution, "720p"), "duration": defaultInt(r.Duration, 5)}
 		if r.GenerateAudio {
 			body["generate_audio"] = true

backend/internal/protocol/builtin_test.go

@@ -150,6 +150,41 @@ func TestImageAndVideoAdaptersMapProviderShapes(t *testing.T) {
 	}
 }
 
+func TestArkVideoAdapterMapsFullModalReferences(t *testing.T) {
+	adapter, ok := Builtins().Get("volcengine-ark-video")
+	if !ok {
+		t.Fatal("adapter missing")
+	}
+	spec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{
+		Model:  "doubao-seedance-2-0-260128",
+		Prompt: "follow all references",
+		Images: []MediaReference{{URL: "https://example.com/subject.png"}},
+		Videos: []MediaReference{{URL: "https://example.com/motion.mp4"}},
+		Audios: []MediaReference{{URL: "https://example.com/music.mp3"}},
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	body := spec.Body.(map[string]any)
+	content := body["content"].([]any)
+	want := []string{"text", "image_url", "video_url", "audio_url"}
+	if len(content) != len(want) {
+		t.Fatalf("content = %#v", content)
+	}
+	for index, item := range content {
+		if item.(map[string]any)["type"] != want[index] {
+			t.Fatalf("content[%d] = %#v, want type %q", index, item, want[index])
+		}
+	}
+
+	_, err = adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{
+		Model: "doubao-seedance-2-0-260128", Prompt: "follow the soundtrack", Audios: []MediaReference{{URL: "https://example.com/music.mp3"}},
+	}})
+	if err == nil || !strings.Contains(err.Error(), "文本+音频") {
+		t.Fatalf("audio-only combination error = %v", err)
+	}
+}
+
 func TestAsyncVideoResponseNormalizesStatusAndResult(t *testing.T) {
 	adapter, _ := Builtins().Get("minimax-video")
 	created, err := adapter.ParseCreate(context.Background(), []byte(`{"task":{"id":"mm-1","status":"processing"}}`))

backend/internal/protocol/docs/volcengine-ark-video.md

@@ -1,6 +1,6 @@
 # 火山方舟视频任务
 
-本插件使用 Ark Contents Generations Tasks 异步接口。提示词和参考图组成 `content[]`；比例、分辨率、时长、音频和水印位于顶层任务参数。
+本插件使用 Ark Contents Generations Tasks 异步接口。提示词、参考图、参考视频和参考音频组成 `content[]`；比例、分辨率、时长、生成音频和水印位于顶层任务参数。
 
 ## 接口与鉴权
 
@@ -21,7 +21,13 @@ Content-Type: application/json
 
 {{PARAMETERS}}
 
-每张图片构造为 `{"type":"image_url","image_url":{"url":"..."},"role":"reference_image"}`。当前不发送参考视频与参考音频；`generate_audio` 和 `watermark` 只在 true 时发送。
+全模态参考的素材映射如下：
+
+- 图片：`{"type":"image_url","image_url":{"url":"..."},"role":"reference_image"}`
+- 视频：`{"type":"video_url","video_url":{"url":"..."},"role":"reference_video"}`
+- 音频：`{"type":"audio_url","audio_url":{"url":"..."},"role":"reference_audio"}`
+
+单次最多发送 9 张图片、3 个视频和 3 个音频。参考素材可以组合使用，但上游不支持纯音频或“文本 + 音频”；使用音频时必须同时提供至少一张参考图或一个参考视频。`generate_audio` 和 `watermark` 只在 true 时发送。
 
 ## 创建任务示例
 
@@ -32,8 +38,10 @@ curl "{channel_base_url}/api/v3/contents/generations/tasks" \
   -d '{
     "model":"YOUR_ARK_VIDEO_MODEL",
     "content":[
-      {"type":"text","text":"保持人物一致，镜头缓慢环绕"},
-      {"type":"image_url","image_url":{"url":"https://example.com/subject.png"},"role":"reference_image"}
+      {"type":"text","text":"沿用图片 1 的主体、视频 1 的运镜和音频 1 的音乐风格"},
+      {"type":"image_url","image_url":{"url":"https://example.com/subject.png"},"role":"reference_image"},
+      {"type":"video_url","video_url":{"url":"https://example.com/motion.mp4"},"role":"reference_video"},
+      {"type":"audio_url","audio_url":{"url":"https://example.com/music.mp3"},"role":"reference_audio"}
     ],
     "ratio":"16:9",
     "resolution":"720p",
@@ -53,5 +61,6 @@ curl "{channel_base_url}/api/v3/contents/generations/tasks" \
 
 - [火山方舟视频生成 API](https://www.volcengine.com/docs/82379)
 - [火山方舟内容生成任务文档](https://www.volcengine.com/docs/82379/1520757)
+- [Doubao Seedance 2.0 全模态参考教程](https://console.volcengine.com/ark/region:cn-beijing/docs/82379/2291680?lang=zh#50e1b4ea)
 
 {{CONTRACT}}