tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(video): support Ark full-modal references (#300)

Browse Source
This commit is contained in:
MasterChiefCN
2026-08-24 16:55:45 +08:00
committed by GitHub
parent b039d558f0
commit 8ea2e738a2
18 changed files with 195 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
backend/internal/protocol/builtin.go
View File

@@ -367,10 +367,22 @@ func arkVideosAdapter() Adapter {
info := metadata("volcengine-ark-video", "火山方舟视频", "Volcengine Ark", CapabilityVideo, "POST /api/v3/contents/generations/tasks", "GET /api/v3/contents/generations/tasks/{task_id}", "application/json")
info.Parameters = videoParams()
return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) {
if len(r.Images) > 9 || len(r.Videos) > 3 || len(r.Audios) > 3 {
return RequestSpec{}, fmt.Errorf("火山方舟全模态参考最多支持 9 张图片、3 个视频和 3 个音频")
}
if len(r.Audios) > 0 && len(r.Images) == 0 && len(r.Videos) == 0 {
return RequestSpec{}, fmt.Errorf("火山方舟全模态参考不支持纯音频或文本+音频，请同时添加参考图片或参考视频")
}
content := []any{map[string]any{"type": "text", "text": r.Prompt}}
for _, image := range r.Images {
content = append(content, map[string]any{"type": "image_url", "image_url": map[string]any{"url": mediaValue(image)}, "role": "reference_image"})
}
for _, video := range r.Videos {
content = append(content, map[string]any{"type": "video_url", "video_url": map[string]any{"url": mediaValue(video)}, "role": "reference_video"})
}
for _, audio := range r.Audios {
content = append(content, map[string]any{"type": "audio_url", "audio_url": map[string]any{"url": mediaValue(audio)}, "role": "reference_audio"})
}
body := map[string]any{"model": r.Model, "content": content, "ratio": defaultValue(r.AspectRatio, "16:9"), "resolution": defaultValue(r.Resolution, "720p"), "duration": defaultInt(r.Duration, 5)}
if r.GenerateAudio {
body["generate_audio"] = true

35
backend/internal/protocol/builtin_test.go
View File

@@ -150,6 +150,41 @@ func TestImageAndVideoAdaptersMapProviderShapes(t *testing.T) {
}
}
func TestArkVideoAdapterMapsFullModalReferences(t *testing.T) {
adapter, ok := Builtins().Get("volcengine-ark-video")
if !ok {
t.Fatal("adapter missing")
}
spec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{
Model: "doubao-seedance-2-0-260128",
Prompt: "follow all references",
Images: []MediaReference{{URL: "https://example.com/subject.png"}},
Videos: []MediaReference{{URL: "https://example.com/motion.mp4"}},
Audios: []MediaReference{{URL: "https://example.com/music.mp3"}},
}})
if err != nil {
t.Fatal(err)
}
body := spec.Body.(map[string]any)
content := body["content"].([]any)
want := []string{"text", "image_url", "video_url", "audio_url"}
if len(content) != len(want) {
t.Fatalf("content = %#v", content)
}
for index, item := range content {
if item.(map[string]any)["type"] != want[index] {
t.Fatalf("content[%d] = %#v, want type %q", index, item, want[index])
}
}
_, err = adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{
Model: "doubao-seedance-2-0-260128", Prompt: "follow the soundtrack", Audios: []MediaReference{{URL: "https://example.com/music.mp3"}},
}})
if err == nil || !strings.Contains(err.Error(), "文本+音频") {
t.Fatalf("audio-only combination error = %v", err)
}
}
func TestAsyncVideoResponseNormalizesStatusAndResult(t *testing.T) {
adapter, _ := Builtins().Get("minimax-video")
created, err := adapter.ParseCreate(context.Background(), []byte(`{"task":{"id":"mm-1","status":"processing"}}`))

17
backend/internal/protocol/docs/volcengine-ark-video.md
View File

@@ -1,6 +1,6 @@
# 火山方舟视频任务
本插件使用 Ark Contents Generations Tasks 异步接口。提示词和参考图组成 `content[]`；比例、分辨率、时长、音频和水印位于顶层任务参数。
本插件使用 Ark Contents Generations Tasks 异步接口。提示词、参考图、参考视频和参考音频组成 `content[]`；比例、分辨率、时长、生成音频和水印位于顶层任务参数。
## 接口与鉴权
@@ -21,7 +21,13 @@ Content-Type: application/json
{{PARAMETERS}}
每张图片构造为 `{"type":"image_url","image_url":{"url":"..."},"role":"reference_image"}`。当前不发送参考视频与参考音频；`generate_audio` 和 `watermark` 只在 true 时发送。
全模态参考的素材映射如下：
- 图片：`{"type":"image_url","image_url":{"url":"..."},"role":"reference_image"}`
- 视频：`{"type":"video_url","video_url":{"url":"..."},"role":"reference_video"}`
- 音频：`{"type":"audio_url","audio_url":{"url":"..."},"role":"reference_audio"}`
单次最多发送 9 张图片、3 个视频和 3 个音频。参考素材可以组合使用，但上游不支持纯音频或“文本 + 音频”；使用音频时必须同时提供至少一张参考图或一个参考视频。`generate_audio` 和 `watermark` 只在 true 时发送。
## 创建任务示例
@@ -32,8 +38,10 @@ curl "{channel_base_url}/api/v3/contents/generations/tasks" \
-d '{
"model":"YOUR_ARK_VIDEO_MODEL",
"content":[
{"type":"text","text":"保持人物一致，镜头缓慢环绕"},
{"type":"image_url","image_url":{"url":"https://example.com/subject.png"},"role":"reference_image"}
{"type":"text","text":"沿用图片 1 的主体、视频 1 的运镜和音频 1 的音乐风格"},
{"type":"image_url","image_url":{"url":"https://example.com/subject.png"},"role":"reference_image"},
{"type":"video_url","video_url":{"url":"https://example.com/motion.mp4"},"role":"reference_video"},
{"type":"audio_url","audio_url":{"url":"https://example.com/music.mp3"},"role":"reference_audio"}
],
"ratio":"16:9",
"resolution":"720p",
@@ -53,5 +61,6 @@ curl "{channel_base_url}/api/v3/contents/generations/tasks" \
- [火山方舟视频生成 API](https://www.volcengine.com/docs/82379)
- [火山方舟内容生成任务文档](https://www.volcengine.com/docs/82379/1520757)
- [Doubao Seedance 2.0 全模态参考教程](https://console.volcengine.com/ark/region:cn-beijing/docs/82379/2291680?lang=zh#50e1b4ea)
{{CONTRACT}}

4
backend/internal/service/model_capability.go
View File

@@ -206,6 +206,7 @@ func DefaultModelCapabilityConfigForModel(protocol string, modelName string) *Mo
video.Duration = VideoDurationConfig{Selection: "enum", Values: []int{4, 6, 8}, Default: 6}
video.Resolutions = []string{"720p", "1080p"}
case model.ChannelInterfaceVolcengineArkVideo:
video.Operations = append(video.Operations, "reference_to_video", "audio_to_video")
video.References.MaxVideos, video.References.MaxAudios = 3, 3
video.References.MaxVideoBytes, video.References.MaxAudioBytes = 200*1024*1024, 15*1024*1024
video.References.MaxVideoDuration, video.References.MaxAudioDuration = 15, 15
@@ -601,6 +602,9 @@ func validateVideoTask(profile *VideoCapabilityConfig, input canvasGenerationInp
if len(input.ReferenceImages) > profile.References.MaxImages || len(input.ReferenceVideos) > profile.References.MaxVideos || len(input.ReferenceAudios) > profile.References.MaxAudios {
return BadAuthRequest("参考素材数量超过当前模型限制")
}
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) && len(input.ReferenceAudios) > 0 && len(input.ReferenceImages) == 0 && len(input.ReferenceVideos) == 0 {
return BadAuthRequest("火山方舟全模态参考不支持纯音频或文本+音频，请同时添加参考图片或参考视频")
}
if len(input.ReferenceImages) < profile.References.MinImages {
return BadAuthRequest(fmt.Sprintf("当前视频模型至少需要 %d 张参考图", profile.References.MinImages))
}

34
backend/internal/service/model_capability_test.go
View File

@@ -93,6 +93,40 @@ func TestDefaultMiniMaxVideoCapabilitySupportsReferenceGeneration(t *testing.T)
}
}
func TestDefaultVolcengineArkVideoCapabilitySupportsFullModalReference(t *testing.T) {
profile := DefaultModelCapabilityConfigForModel("volcengine-ark-video", "doubao-seedance-2-0-260128")
if profile == nil || profile.Video == nil {
t.Fatal("Volcengine Ark video profile = nil")
}
for _, operation := range []string{"reference_to_video", "audio_to_video"} {
if !containsCapabilityString(profile.Video.Operations, operation) {
t.Fatalf("operations = %v, want %s", profile.Video.Operations, operation)
}
}
if profile.Video.References.MaxImages != 9 || profile.Video.References.MaxVideos != 3 || profile.Video.References.MaxAudios != 3 {
t.Fatalf("reference limits = %#v", profile.Video.References)
}
}
func TestValidateVolcengineArkFullModalReferenceRejectsTextAndAudioOnly(t *testing.T) {
profile := DefaultModelCapabilityConfigForModel("volcengine-ark-video", "doubao-seedance-2-0-260128").Video
input := canvasGenerationInput{
Prompt: "follow the soundtrack",
Config: providerConfig{InterfaceType: "volcengine-ark-video", VideoSeconds: "6", Size: "16:9", VQuality: "720p"},
ReferenceAudios: []providerMedia{{URL: "https://example.com/music.mp3"}},
Metadata: map[string]interface{}{"videoEditOperation": "audio_to_video"},
}
err := validateVideoTask(profile, input)
if err == nil || !strings.Contains(err.Error(), "文本+音频") {
t.Fatalf("validateVideoTask() error = %v", err)
}
input.ReferenceImages = []providerMedia{{URL: "https://example.com/subject.png"}}
if err := validateVideoTask(profile, input); err != nil {
t.Fatalf("validateVideoTask(full modal) error = %v", err)
}
}
func TestCapabilitySpecFromModelCapabilityConfigRestoresLegacyWildcardImageSizes(t *testing.T) {
config := &ModelCapabilityConfig{
Version: 1,

15
backend/internal/service/provider_test.go
View File

@@ -1446,12 +1446,19 @@ func TestVolcengineArkVideoProtocolUsesContentTaskAndDownloadsResult(t *testing.
t.Fatalf("decode request: %v", err)
}
content, _ := body["content"].([]interface{})
if len(content) != 2 {
if len(content) != 4 {
t.Errorf("body = %#v", body)
return
}
imageContent, _ := content[1].(map[string]interface{})
if body["model"] != "doubao-seedance-test" || imageContent["role"] != "reference_image" {
wantTypes := []string{"text", "image_url", "video_url", "audio_url"}
wantRoles := []string{"", "reference_image", "reference_video", "reference_audio"}
for index, item := range content {
entry, _ := item.(map[string]interface{})
if entry["type"] != wantTypes[index] || (wantRoles[index] != "" && entry["role"] != wantRoles[index]) {
t.Errorf("content[%d] = %#v", index, entry)
}
}
if body["model"] != "doubao-seedance-test" {
t.Errorf("body = %#v", body)
}
_, _ = w.Write([]byte(`{"id":"ark-task-1","status":"running"}`))
@@ -1470,6 +1477,8 @@ func TestVolcengineArkVideoProtocolUsesContentTaskAndDownloadsResult(t *testing.
Prompt: "make it move",
Config: providerConfig{BaseURL: server.URL + "/api/v3", APIKey: "test-key", Model: "doubao-seedance-test", InterfaceType: "volcengine-ark-video"},
ReferenceImages: []providerMedia{{ID: "start", URL: server.URL + "/reference.png"}},
ReferenceVideos: []providerMedia{{ID: "motion", URL: server.URL + "/reference.mp4"}},
ReferenceAudios: []providerMedia{{ID: "music", URL: server.URL + "/reference.mp3"}},
Metadata: map[string]interface{}{"videoStartFrameNodeId": "start"},
})
if err != nil {

1
web/src/components/canvas/canvas-config-node-panel.tsx
View File

@@ -30,6 +30,7 @@ type CanvasConfigNodePanelProps = {
const videoOperationOptions: Array<{ label: string; value: CanvasVideoEditOperation }> = [
{ label: "文生视频", value: "text_to_video" },
{ label: "图生视频", value: "image_to_video" },
{ label: "全模态参考", value: "reference_to_video" },
{ label: "视频续写", value: "extend" },
{ label: "局部修改", value: "inpaint" },
{ label: "元素替换", value: "replace_element" },

1
web/src/components/canvas/canvas-video-segment-dialog.tsx
View File

@@ -413,6 +413,7 @@ function videoOperationLabel(operation: CanvasVideoEditOperation) {
const labels: Record<CanvasVideoEditOperation, string> = {
text_to_video: "文生视频",
image_to_video: "图生视频",
reference_to_video: "全模态参考",
extend: "视频续写",
inpaint: "局部修改",
replace_element: "元素替换",

1
web/src/components/model-capability-editor.tsx
View File

@@ -9,6 +9,7 @@ const ratioOptions = ["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"];
const operationOptions = [
{ label: "文生视频", value: "text_to_video" },
{ label: "图生视频", value: "image_to_video" },
{ label: "全模态参考", value: "reference_to_video" },
{ label: "视频续写", value: "extend" },
{ label: "局部修改", value: "inpaint" },
{ label: "元素替换", value: "replace_element" },

1
web/src/components/model-picker.tsx
View File

@@ -294,6 +294,7 @@ function logicalCapabilitySummary(spec: NonNullable<NonNullable<AiConfig["channe
const operationLabels: Record<string, string> = {
text_to_video: "文生视频",
image_to_video: "图生视频",
reference_to_video: "全模态参考",
audio_to_video: "音频生视频",
extend: "视频续写",
inpaint: "局部修改",

1
web/src/lib/generation-task-display.ts
View File

@@ -43,6 +43,7 @@ export const operationOptions = [
{ label: "Agent 会话：拆解影视工作流", value: "agent_session" },
{ label: "文生视频", value: "text_to_video" },
{ label: "图生视频", value: "image_to_video" },
{ label: "全模态参考", value: "reference_to_video" },
{ label: "视频续写", value: "extend" },
{ label: "视频局部修改", value: "inpaint" },
{ label: "元素替换", value: "replace_element" },

5
web/src/lib/model-capabilities.ts
View File

@@ -292,7 +292,10 @@ export function defaultModelCapabilityConfig(protocol?: ModelProtocol, model = "
video.generateAudio = { supported: true, default: true };
}
if (protocol === "volcengine-ark-video" || protocol === "newapi-channel-1") video.resolutions = ["480p", "720p", "1080p"];
if (protocol === "volcengine-ark-video") video.watermark = { supported: true, default: false };
if (protocol === "volcengine-ark-video") {
video.watermark = { supported: true, default: false };
video.operations.push("reference_to_video", "audio_to_video");
}
if (protocol === "novita-video") {
video.references.maxImages = 1;
video.references.maxImageBytes = 10 * 1024 * 1024;

2
web/src/lib/model-selection.ts
View File

@@ -336,7 +336,7 @@ function videoOperationLabel(operation: string) {
if (operation === "text_to_video") return "文生视频";
if (operation === "image_to_video") return "图生视频";
if (operation === "audio_to_video") return "音频生视频";
if (operation === "reference_to_video") return "参考素材生视频";
if (operation === "reference_to_video") return "全模态参考";
if (operation === "extend") return "视频续写";
return "当前生成模式";
}

1
web/src/pages/admin/logical-models/model-routing-capabilities.tsx
View File

@@ -637,6 +637,7 @@ export function operationLabel(value: string) {
{
text_to_video: "文生视频",
image_to_video: "图生视频",
reference_to_video: "全模态参考",
extend: "视频续写",
inpaint: "局部修改",
replace_element: "元素替换",

4
web/src/services/api/video-provider-seedance.ts
View File

@@ -81,8 +81,8 @@ function seedanceApiUrl(config: ResolvedAiConfig, taskId?: string) {
}
async function buildSeedanceAgentPlanPayload(config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps) {
if (config.interfaceType !== "volcengine-ark-video" && audioReferences.length && !references.length && !videoReferences.length) {
throw new Error("Seedance 参考音频不能单独使用，请同时添加参考图或参考视频");
if (audioReferences.length && !references.length && !videoReferences.length) {
throw new Error(config.interfaceType === "volcengine-ark-video" ? "火山方舟全模态参考不支持纯音频或文本+音频，请同时添加参考图片或参考视频" : "Seedance 参考音频不能单独使用，请同时添加参考图或参考视频");
}
const content = config.interfaceType === "volcengine-ark-video"
? await buildVolcengineArkContent(prompt, references, videoReferences, audioReferences)

2
web/src/types/canvas.ts
View File

@@ -50,7 +50,7 @@ export type CanvasGenerationBatchStatus = "queued" | "running" | "partial_failed
export type CanvasGenerationBatchItemStatus = "waiting" | "submitting" | "queued" | "running" | "succeeded" | "failed" | "cancelled";
export type CanvasImageGenerationType = "generation" | "edit";
export type CanvasWorkflowKind = "free" | "script" | "story_input" | "character" | "scene" | "storyboard" | "shot" | "final" | "styleboard" | "reference_set" | "reference_video" | "action_board";
export type CanvasVideoEditOperation = "text_to_video" | "image_to_video" | "extend" | "inpaint" | "replace_element" | "camera_motion" | "style_transfer" | "audio_to_video" | "compare_versions" | "concat";
export type CanvasVideoEditOperation = "text_to_video" | "image_to_video" | "reference_to_video" | "extend" | "inpaint" | "replace_element" | "camera_motion" | "style_transfer" | "audio_to_video" | "compare_versions" | "concat";
export type CanvasSkillCategory = "writing" | "storyboard" | "image" | "video" | "utility";
export type CanvasSkillOutputMode = "text" | "json" | "image_prompt" | "workflow";
export type StoryboardColumn =

68
web/test/video-api-contracts.test.ts
View File

@@ -1,6 +1,9 @@
import { describe, expect, test } from "bun:test";
import { unwrapEnvelope, videoTaskId } from "../src/services/api/video-response";
import { defaultConfig } from "../src/stores/use-config-store";
import { createSeedanceTask } from "../src/services/api/video-provider-seedance";
import type { VideoProviderDeps } from "../src/services/api/video-provider-deps";
import { unwrapEnvelope, videoResponseTools, videoTaskId } from "../src/services/api/video-response";
import { isPublicMediaUrl, normalizeVideoResolution, normalizeVideoSeconds, normalizeVideoSize } from "../src/services/api/video-validation";
describe("video API response contracts", () => {
@@ -34,3 +37,66 @@ describe("video request normalization", () => {
expect(isPublicMediaUrl("/resources/video.mp4")).toBe(false);
});
});
describe("Volcengine Ark full-modal references", () => {
const model = "seedance-2-0-250824";
const config = {
...defaultConfig,
baseUrl: "https://ark.cn-beijing.volces.com/api/v3",
apiKey: "test-key",
interfaceType: "volcengine-ark-video",
channels: [
{
id: "ark",
name: "Ark",
baseUrl: "https://ark.cn-beijing.volces.com/api/v3",
apiKey: "test-key",
secretKey: "",
headers: [],
apiFormat: "openai",
interfaceType: "volcengine-ark-video",
models: [model],
scope: "user",
enabled: true,
},
],
};
test("映射图片、视频和音频参考素材", async () => {
let requestUrl = "";
let requestBody: unknown;
const deps = {
transport: {
post: async (url: string, body: unknown) => {
requestUrl = url;
requestBody = body;
return { id: "task-1", status: "queued" };
},
},
response: videoResponseTools,
} as unknown as VideoProviderDeps;
await createSeedanceTask(
deps,
config as never,
model,
"保持主体一致",
[{ id: "image-1", name: "image.png", type: "image/png", dataUrl: "", url: "https://cdn.example.com/image.png" }],
[{ id: "video-1", name: "video.mp4", type: "video/mp4", url: "https://cdn.example.com/video.mp4" }],
[{ id: "audio-1", name: "audio.mp3", type: "audio/mpeg", url: "https://cdn.example.com/audio.mp3" }],
);
expect(requestUrl).toBe("https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks");
expect((requestBody as { content: unknown[] }).content).toEqual([
{ type: "text", text: "保持主体一致" },
{ type: "image_url", image_url: { url: "https://cdn.example.com/image.png" }, role: "reference_image" },
{ type: "video_url", video_url: { url: "https://cdn.example.com/video.mp4" }, role: "reference_video" },
{ type: "audio_url", audio_url: { url: "https://cdn.example.com/audio.mp3" }, role: "reference_audio" },
]);
});
test("拒绝纯音频和文本加音频", async () => {
const deps = { transport: {}, response: videoResponseTools } as unknown as VideoProviderDeps;
await expect(createSeedanceTask(deps, config as never, model, "跟随节奏", [], [], [{ id: "audio-1", name: "audio.mp3", type: "audio/mpeg", url: "https://cdn.example.com/audio.mp3" }])).rejects.toThrow("不支持纯音频或文本+音频");
});
});

4
web/test/video-generation-options.test.ts
View File

@@ -22,4 +22,8 @@ describe("video generation resolution options", () => {
expect(defaultModelCapabilityConfig("volcengine-jimeng-video").video?.resolutions).toEqual(["720p"]);
expect(defaultModelCapabilityConfig("gemini-veo").video?.resolutions).toEqual(["720p", "1080p"]);
});
test("火山方舟默认开启全模态参考模式", () => {
expect(defaultModelCapabilityConfig("volcengine-ark-video").video?.operations).toEqual(expect.arrayContaining(["reference_to_video", "audio_to_video"]));
});
});