feat(video): support Ark full-modal references (#300)
This commit is contained in:
1 parent
b039d558f0
commit
8ea2e738a2
18 files changed
+195
-13
No files matched your search
@@ -367,10 +367,22 @@ func arkVideosAdapter() Adapter {
|
|||||||
info := metadata("volcengine-ark-video", "火山方舟视频", "Volcengine Ark", CapabilityVideo, "POST /api/v3/contents/generations/tasks", "GET /api/v3/contents/generations/tasks/{task_id}", "application/json")
|
info := metadata("volcengine-ark-video", "火山方舟视频", "Volcengine Ark", CapabilityVideo, "POST /api/v3/contents/generations/tasks", "GET /api/v3/contents/generations/tasks/{task_id}", "application/json")
|
||||||
info.Parameters = videoParams()
|
info.Parameters = videoParams()
|
||||||
return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) {
|
return videoAdapter(info, func(r GenerationRequest) (RequestSpec, error) {
|
||||||
|
if len(r.Images) > 9 || len(r.Videos) > 3 || len(r.Audios) > 3 {
|
||||||
|
return RequestSpec{}, fmt.Errorf("火山方舟全模态参考最多支持 9 张图片、3 个视频和 3 个音频")
|
||||||
|
}
|
||||||
|
if len(r.Audios) > 0 && len(r.Images) == 0 && len(r.Videos) == 0 {
|
||||||
|
return RequestSpec{}, fmt.Errorf("火山方舟全模态参考不支持纯音频或文本+音频,请同时添加参考图片或参考视频")
|
||||||
|
}
|
||||||
content := []any{map[string]any{"type": "text", "text": r.Prompt}}
|
content := []any{map[string]any{"type": "text", "text": r.Prompt}}
|
||||||
for _, image := range r.Images {
|
for _, image := range r.Images {
|
||||||
content = append(content, map[string]any{"type": "image_url", "image_url": map[string]any{"url": mediaValue(image)}, "role": "reference_image"})
|
content = append(content, map[string]any{"type": "image_url", "image_url": map[string]any{"url": mediaValue(image)}, "role": "reference_image"})
|
||||||
}
|
}
|
||||||
|
for _, video := range r.Videos {
|
||||||
|
content = append(content, map[string]any{"type": "video_url", "video_url": map[string]any{"url": mediaValue(video)}, "role": "reference_video"})
|
||||||
|
}
|
||||||
|
for _, audio := range r.Audios {
|
||||||
|
content = append(content, map[string]any{"type": "audio_url", "audio_url": map[string]any{"url": mediaValue(audio)}, "role": "reference_audio"})
|
||||||
|
}
|
||||||
body := map[string]any{"model": r.Model, "content": content, "ratio": defaultValue(r.AspectRatio, "16:9"), "resolution": defaultValue(r.Resolution, "720p"), "duration": defaultInt(r.Duration, 5)}
|
body := map[string]any{"model": r.Model, "content": content, "ratio": defaultValue(r.AspectRatio, "16:9"), "resolution": defaultValue(r.Resolution, "720p"), "duration": defaultInt(r.Duration, 5)}
|
||||||
if r.GenerateAudio {
|
if r.GenerateAudio {
|
||||||
body["generate_audio"] = true
|
body["generate_audio"] = true
|
||||||
|
|||||||
@@ -150,6 +150,41 @@ func TestImageAndVideoAdaptersMapProviderShapes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestArkVideoAdapterMapsFullModalReferences(t *testing.T) {
|
||||||
|
adapter, ok := Builtins().Get("volcengine-ark-video")
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("adapter missing")
|
||||||
|
}
|
||||||
|
spec, err := adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{
|
||||||
|
Model: "doubao-seedance-2-0-260128",
|
||||||
|
Prompt: "follow all references",
|
||||||
|
Images: []MediaReference{{URL: "https://example.com/subject.png"}},
|
||||||
|
Videos: []MediaReference{{URL: "https://example.com/motion.mp4"}},
|
||||||
|
Audios: []MediaReference{{URL: "https://example.com/music.mp3"}},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body := spec.Body.(map[string]any)
|
||||||
|
content := body["content"].([]any)
|
||||||
|
want := []string{"text", "image_url", "video_url", "audio_url"}
|
||||||
|
if len(content) != len(want) {
|
||||||
|
t.Fatalf("content = %#v", content)
|
||||||
|
}
|
||||||
|
for index, item := range content {
|
||||||
|
if item.(map[string]any)["type"] != want[index] {
|
||||||
|
t.Fatalf("content[%d] = %#v, want type %q", index, item, want[index])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = adapter.BuildCreate(context.Background(), RequestContext{Request: GenerationRequest{
|
||||||
|
Model: "doubao-seedance-2-0-260128", Prompt: "follow the soundtrack", Audios: []MediaReference{{URL: "https://example.com/music.mp3"}},
|
||||||
|
}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "文本+音频") {
|
||||||
|
t.Fatalf("audio-only combination error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAsyncVideoResponseNormalizesStatusAndResult(t *testing.T) {
|
func TestAsyncVideoResponseNormalizesStatusAndResult(t *testing.T) {
|
||||||
adapter, _ := Builtins().Get("minimax-video")
|
adapter, _ := Builtins().Get("minimax-video")
|
||||||
created, err := adapter.ParseCreate(context.Background(), []byte(`{"task":{"id":"mm-1","status":"processing"}}`))
|
created, err := adapter.ParseCreate(context.Background(), []byte(`{"task":{"id":"mm-1","status":"processing"}}`))
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# 火山方舟视频任务
|
# 火山方舟视频任务
|
||||||
|
|
||||||
本插件使用 Ark Contents Generations Tasks 异步接口。提示词和参考图组成 `content[]`;比例、分辨率、时长、音频和水印位于顶层任务参数。
|
本插件使用 Ark Contents Generations Tasks 异步接口。提示词、参考图、参考视频和参考音频组成 `content[]`;比例、分辨率、时长、生成音频和水印位于顶层任务参数。
|
||||||
|
|
||||||
## 接口与鉴权
|
## 接口与鉴权
|
||||||
|
|
||||||
@@ -21,7 +21,13 @@ Content-Type: application/json
|
|||||||
|
|
||||||
{{PARAMETERS}}
|
{{PARAMETERS}}
|
||||||
|
|
||||||
每张图片构造为 `{"type":"image_url","image_url":{"url":"..."},"role":"reference_image"}`。当前不发送参考视频与参考音频;`generate_audio` 和 `watermark` 只在 true 时发送。
|
全模态参考的素材映射如下:
|
||||||
|
|
||||||
|
- 图片:`{"type":"image_url","image_url":{"url":"..."},"role":"reference_image"}`
|
||||||
|
- 视频:`{"type":"video_url","video_url":{"url":"..."},"role":"reference_video"}`
|
||||||
|
- 音频:`{"type":"audio_url","audio_url":{"url":"..."},"role":"reference_audio"}`
|
||||||
|
|
||||||
|
单次最多发送 9 张图片、3 个视频和 3 个音频。参考素材可以组合使用,但上游不支持纯音频或“文本 + 音频”;使用音频时必须同时提供至少一张参考图或一个参考视频。`generate_audio` 和 `watermark` 只在 true 时发送。
|
||||||
|
|
||||||
## 创建任务示例
|
## 创建任务示例
|
||||||
|
|
||||||
@@ -32,8 +38,10 @@ curl "{channel_base_url}/api/v3/contents/generations/tasks" \
|
|||||||
-d '{
|
-d '{
|
||||||
"model":"YOUR_ARK_VIDEO_MODEL",
|
"model":"YOUR_ARK_VIDEO_MODEL",
|
||||||
"content":[
|
"content":[
|
||||||
{"type":"text","text":"保持人物一致,镜头缓慢环绕"},
|
{"type":"text","text":"沿用图片 1 的主体、视频 1 的运镜和音频 1 的音乐风格"},
|
||||||
{"type":"image_url","image_url":{"url":"https://example.com/subject.png"},"role":"reference_image"}
|
{"type":"image_url","image_url":{"url":"https://example.com/subject.png"},"role":"reference_image"},
|
||||||
|
{"type":"video_url","video_url":{"url":"https://example.com/motion.mp4"},"role":"reference_video"},
|
||||||
|
{"type":"audio_url","audio_url":{"url":"https://example.com/music.mp3"},"role":"reference_audio"}
|
||||||
],
|
],
|
||||||
"ratio":"16:9",
|
"ratio":"16:9",
|
||||||
"resolution":"720p",
|
"resolution":"720p",
|
||||||
@@ -53,5 +61,6 @@ curl "{channel_base_url}/api/v3/contents/generations/tasks" \
|
|||||||
|
|
||||||
- [火山方舟视频生成 API](https://www.volcengine.com/docs/82379)
|
- [火山方舟视频生成 API](https://www.volcengine.com/docs/82379)
|
||||||
- [火山方舟内容生成任务文档](https://www.volcengine.com/docs/82379/1520757)
|
- [火山方舟内容生成任务文档](https://www.volcengine.com/docs/82379/1520757)
|
||||||
|
- [Doubao Seedance 2.0 全模态参考教程](https://console.volcengine.com/ark/region:cn-beijing/docs/82379/2291680?lang=zh#50e1b4ea)
|
||||||
|
|
||||||
{{CONTRACT}}
|
{{CONTRACT}}
|
||||||
@@ -206,6 +206,7 @@ func DefaultModelCapabilityConfigForModel(protocol string, modelName string) *Mo
|
|||||||
video.Duration = VideoDurationConfig{Selection: "enum", Values: []int{4, 6, 8}, Default: 6}
|
video.Duration = VideoDurationConfig{Selection: "enum", Values: []int{4, 6, 8}, Default: 6}
|
||||||
video.Resolutions = []string{"720p", "1080p"}
|
video.Resolutions = []string{"720p", "1080p"}
|
||||||
case model.ChannelInterfaceVolcengineArkVideo:
|
case model.ChannelInterfaceVolcengineArkVideo:
|
||||||
|
video.Operations = append(video.Operations, "reference_to_video", "audio_to_video")
|
||||||
video.References.MaxVideos, video.References.MaxAudios = 3, 3
|
video.References.MaxVideos, video.References.MaxAudios = 3, 3
|
||||||
video.References.MaxVideoBytes, video.References.MaxAudioBytes = 200*1024*1024, 15*1024*1024
|
video.References.MaxVideoBytes, video.References.MaxAudioBytes = 200*1024*1024, 15*1024*1024
|
||||||
video.References.MaxVideoDuration, video.References.MaxAudioDuration = 15, 15
|
video.References.MaxVideoDuration, video.References.MaxAudioDuration = 15, 15
|
||||||
@@ -601,6 +602,9 @@ func validateVideoTask(profile *VideoCapabilityConfig, input canvasGenerationInp
|
|||||||
if len(input.ReferenceImages) > profile.References.MaxImages || len(input.ReferenceVideos) > profile.References.MaxVideos || len(input.ReferenceAudios) > profile.References.MaxAudios {
|
if len(input.ReferenceImages) > profile.References.MaxImages || len(input.ReferenceVideos) > profile.References.MaxVideos || len(input.ReferenceAudios) > profile.References.MaxAudios {
|
||||||
return BadAuthRequest("参考素材数量超过当前模型限制")
|
return BadAuthRequest("参考素材数量超过当前模型限制")
|
||||||
}
|
}
|
||||||
|
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkVideo) && len(input.ReferenceAudios) > 0 && len(input.ReferenceImages) == 0 && len(input.ReferenceVideos) == 0 {
|
||||||
|
return BadAuthRequest("火山方舟全模态参考不支持纯音频或文本+音频,请同时添加参考图片或参考视频")
|
||||||
|
}
|
||||||
if len(input.ReferenceImages) < profile.References.MinImages {
|
if len(input.ReferenceImages) < profile.References.MinImages {
|
||||||
return BadAuthRequest(fmt.Sprintf("当前视频模型至少需要 %d 张参考图", profile.References.MinImages))
|
return BadAuthRequest(fmt.Sprintf("当前视频模型至少需要 %d 张参考图", profile.References.MinImages))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -93,6 +93,40 @@ func TestDefaultMiniMaxVideoCapabilitySupportsReferenceGeneration(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestDefaultVolcengineArkVideoCapabilitySupportsFullModalReference(t *testing.T) {
|
||||||
|
profile := DefaultModelCapabilityConfigForModel("volcengine-ark-video", "doubao-seedance-2-0-260128")
|
||||||
|
if profile == nil || profile.Video == nil {
|
||||||
|
t.Fatal("Volcengine Ark video profile = nil")
|
||||||
|
}
|
||||||
|
for _, operation := range []string{"reference_to_video", "audio_to_video"} {
|
||||||
|
if !containsCapabilityString(profile.Video.Operations, operation) {
|
||||||
|
t.Fatalf("operations = %v, want %s", profile.Video.Operations, operation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile.Video.References.MaxImages != 9 || profile.Video.References.MaxVideos != 3 || profile.Video.References.MaxAudios != 3 {
|
||||||
|
t.Fatalf("reference limits = %#v", profile.Video.References)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateVolcengineArkFullModalReferenceRejectsTextAndAudioOnly(t *testing.T) {
|
||||||
|
profile := DefaultModelCapabilityConfigForModel("volcengine-ark-video", "doubao-seedance-2-0-260128").Video
|
||||||
|
input := canvasGenerationInput{
|
||||||
|
Prompt: "follow the soundtrack",
|
||||||
|
Config: providerConfig{InterfaceType: "volcengine-ark-video", VideoSeconds: "6", Size: "16:9", VQuality: "720p"},
|
||||||
|
ReferenceAudios: []providerMedia{{URL: "https://example.com/music.mp3"}},
|
||||||
|
Metadata: map[string]interface{}{"videoEditOperation": "audio_to_video"},
|
||||||
|
}
|
||||||
|
err := validateVideoTask(profile, input)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "文本+音频") {
|
||||||
|
t.Fatalf("validateVideoTask() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
input.ReferenceImages = []providerMedia{{URL: "https://example.com/subject.png"}}
|
||||||
|
if err := validateVideoTask(profile, input); err != nil {
|
||||||
|
t.Fatalf("validateVideoTask(full modal) error = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCapabilitySpecFromModelCapabilityConfigRestoresLegacyWildcardImageSizes(t *testing.T) {
|
func TestCapabilitySpecFromModelCapabilityConfigRestoresLegacyWildcardImageSizes(t *testing.T) {
|
||||||
config := &ModelCapabilityConfig{
|
config := &ModelCapabilityConfig{
|
||||||
Version: 1,
|
Version: 1,
|
||||||
|
|||||||
@@ -1446,12 +1446,19 @@ func TestVolcengineArkVideoProtocolUsesContentTaskAndDownloadsResult(t *testing.
|
|||||||
t.Fatalf("decode request: %v", err)
|
t.Fatalf("decode request: %v", err)
|
||||||
}
|
}
|
||||||
content, _ := body["content"].([]interface{})
|
content, _ := body["content"].([]interface{})
|
||||||
if len(content) != 2 {
|
if len(content) != 4 {
|
||||||
t.Errorf("body = %#v", body)
|
t.Errorf("body = %#v", body)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
imageContent, _ := content[1].(map[string]interface{})
|
wantTypes := []string{"text", "image_url", "video_url", "audio_url"}
|
||||||
if body["model"] != "doubao-seedance-test" || imageContent["role"] != "reference_image" {
|
wantRoles := []string{"", "reference_image", "reference_video", "reference_audio"}
|
||||||
|
for index, item := range content {
|
||||||
|
entry, _ := item.(map[string]interface{})
|
||||||
|
if entry["type"] != wantTypes[index] || (wantRoles[index] != "" && entry["role"] != wantRoles[index]) {
|
||||||
|
t.Errorf("content[%d] = %#v", index, entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if body["model"] != "doubao-seedance-test" {
|
||||||
t.Errorf("body = %#v", body)
|
t.Errorf("body = %#v", body)
|
||||||
}
|
}
|
||||||
_, _ = w.Write([]byte(`{"id":"ark-task-1","status":"running"}`))
|
_, _ = w.Write([]byte(`{"id":"ark-task-1","status":"running"}`))
|
||||||
@@ -1470,6 +1477,8 @@ func TestVolcengineArkVideoProtocolUsesContentTaskAndDownloadsResult(t *testing.
|
|||||||
Prompt: "make it move",
|
Prompt: "make it move",
|
||||||
Config: providerConfig{BaseURL: server.URL + "/api/v3", APIKey: "test-key", Model: "doubao-seedance-test", InterfaceType: "volcengine-ark-video"},
|
Config: providerConfig{BaseURL: server.URL + "/api/v3", APIKey: "test-key", Model: "doubao-seedance-test", InterfaceType: "volcengine-ark-video"},
|
||||||
ReferenceImages: []providerMedia{{ID: "start", URL: server.URL + "/reference.png"}},
|
ReferenceImages: []providerMedia{{ID: "start", URL: server.URL + "/reference.png"}},
|
||||||
|
ReferenceVideos: []providerMedia{{ID: "motion", URL: server.URL + "/reference.mp4"}},
|
||||||
|
ReferenceAudios: []providerMedia{{ID: "music", URL: server.URL + "/reference.mp3"}},
|
||||||
Metadata: map[string]interface{}{"videoStartFrameNodeId": "start"},
|
Metadata: map[string]interface{}{"videoStartFrameNodeId": "start"},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ type CanvasConfigNodePanelProps = {
|
|||||||
const videoOperationOptions: Array<{ label: string; value: CanvasVideoEditOperation }> = [
|
const videoOperationOptions: Array<{ label: string; value: CanvasVideoEditOperation }> = [
|
||||||
{ label: "文生视频", value: "text_to_video" },
|
{ label: "文生视频", value: "text_to_video" },
|
||||||
{ label: "图生视频", value: "image_to_video" },
|
{ label: "图生视频", value: "image_to_video" },
|
||||||
|
{ label: "全模态参考", value: "reference_to_video" },
|
||||||
{ label: "视频续写", value: "extend" },
|
{ label: "视频续写", value: "extend" },
|
||||||
{ label: "局部修改", value: "inpaint" },
|
{ label: "局部修改", value: "inpaint" },
|
||||||
{ label: "元素替换", value: "replace_element" },
|
{ label: "元素替换", value: "replace_element" },
|
||||||
|
|||||||
@@ -413,6 +413,7 @@ function videoOperationLabel(operation: CanvasVideoEditOperation) {
|
|||||||
const labels: Record<CanvasVideoEditOperation, string> = {
|
const labels: Record<CanvasVideoEditOperation, string> = {
|
||||||
text_to_video: "文生视频",
|
text_to_video: "文生视频",
|
||||||
image_to_video: "图生视频",
|
image_to_video: "图生视频",
|
||||||
|
reference_to_video: "全模态参考",
|
||||||
extend: "视频续写",
|
extend: "视频续写",
|
||||||
inpaint: "局部修改",
|
inpaint: "局部修改",
|
||||||
replace_element: "元素替换",
|
replace_element: "元素替换",
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ const ratioOptions = ["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"];
|
|||||||
const operationOptions = [
|
const operationOptions = [
|
||||||
{ label: "文生视频", value: "text_to_video" },
|
{ label: "文生视频", value: "text_to_video" },
|
||||||
{ label: "图生视频", value: "image_to_video" },
|
{ label: "图生视频", value: "image_to_video" },
|
||||||
|
{ label: "全模态参考", value: "reference_to_video" },
|
||||||
{ label: "视频续写", value: "extend" },
|
{ label: "视频续写", value: "extend" },
|
||||||
{ label: "局部修改", value: "inpaint" },
|
{ label: "局部修改", value: "inpaint" },
|
||||||
{ label: "元素替换", value: "replace_element" },
|
{ label: "元素替换", value: "replace_element" },
|
||||||
|
|||||||
@@ -294,6 +294,7 @@ function logicalCapabilitySummary(spec: NonNullable<NonNullable<AiConfig["channe
|
|||||||
const operationLabels: Record<string, string> = {
|
const operationLabels: Record<string, string> = {
|
||||||
text_to_video: "文生视频",
|
text_to_video: "文生视频",
|
||||||
image_to_video: "图生视频",
|
image_to_video: "图生视频",
|
||||||
|
reference_to_video: "全模态参考",
|
||||||
audio_to_video: "音频生视频",
|
audio_to_video: "音频生视频",
|
||||||
extend: "视频续写",
|
extend: "视频续写",
|
||||||
inpaint: "局部修改",
|
inpaint: "局部修改",
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ export const operationOptions = [
|
|||||||
{ label: "Agent 会话:拆解影视工作流", value: "agent_session" },
|
{ label: "Agent 会话:拆解影视工作流", value: "agent_session" },
|
||||||
{ label: "文生视频", value: "text_to_video" },
|
{ label: "文生视频", value: "text_to_video" },
|
||||||
{ label: "图生视频", value: "image_to_video" },
|
{ label: "图生视频", value: "image_to_video" },
|
||||||
|
{ label: "全模态参考", value: "reference_to_video" },
|
||||||
{ label: "视频续写", value: "extend" },
|
{ label: "视频续写", value: "extend" },
|
||||||
{ label: "视频局部修改", value: "inpaint" },
|
{ label: "视频局部修改", value: "inpaint" },
|
||||||
{ label: "元素替换", value: "replace_element" },
|
{ label: "元素替换", value: "replace_element" },
|
||||||
|
|||||||
@@ -292,7 +292,10 @@ export function defaultModelCapabilityConfig(protocol?: ModelProtocol, model = "
|
|||||||
video.generateAudio = { supported: true, default: true };
|
video.generateAudio = { supported: true, default: true };
|
||||||
}
|
}
|
||||||
if (protocol === "volcengine-ark-video" || protocol === "newapi-channel-1") video.resolutions = ["480p", "720p", "1080p"];
|
if (protocol === "volcengine-ark-video" || protocol === "newapi-channel-1") video.resolutions = ["480p", "720p", "1080p"];
|
||||||
if (protocol === "volcengine-ark-video") video.watermark = { supported: true, default: false };
|
if (protocol === "volcengine-ark-video") {
|
||||||
|
video.watermark = { supported: true, default: false };
|
||||||
|
video.operations.push("reference_to_video", "audio_to_video");
|
||||||
|
}
|
||||||
if (protocol === "novita-video") {
|
if (protocol === "novita-video") {
|
||||||
video.references.maxImages = 1;
|
video.references.maxImages = 1;
|
||||||
video.references.maxImageBytes = 10 * 1024 * 1024;
|
video.references.maxImageBytes = 10 * 1024 * 1024;
|
||||||
|
|||||||
@@ -336,7 +336,7 @@ function videoOperationLabel(operation: string) {
|
|||||||
if (operation === "text_to_video") return "文生视频";
|
if (operation === "text_to_video") return "文生视频";
|
||||||
if (operation === "image_to_video") return "图生视频";
|
if (operation === "image_to_video") return "图生视频";
|
||||||
if (operation === "audio_to_video") return "音频生视频";
|
if (operation === "audio_to_video") return "音频生视频";
|
||||||
if (operation === "reference_to_video") return "参考素材生视频";
|
if (operation === "reference_to_video") return "全模态参考";
|
||||||
if (operation === "extend") return "视频续写";
|
if (operation === "extend") return "视频续写";
|
||||||
return "当前生成模式";
|
return "当前生成模式";
|
||||||
}
|
}
|
||||||
@@ -637,6 +637,7 @@ export function operationLabel(value: string) {
|
|||||||
{
|
{
|
||||||
text_to_video: "文生视频",
|
text_to_video: "文生视频",
|
||||||
image_to_video: "图生视频",
|
image_to_video: "图生视频",
|
||||||
|
reference_to_video: "全模态参考",
|
||||||
extend: "视频续写",
|
extend: "视频续写",
|
||||||
inpaint: "局部修改",
|
inpaint: "局部修改",
|
||||||
replace_element: "元素替换",
|
replace_element: "元素替换",
|
||||||
|
|||||||
@@ -81,8 +81,8 @@ function seedanceApiUrl(config: ResolvedAiConfig, taskId?: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function buildSeedanceAgentPlanPayload(config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps) {
|
async function buildSeedanceAgentPlanPayload(config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps) {
|
||||||
if (config.interfaceType !== "volcengine-ark-video" && audioReferences.length && !references.length && !videoReferences.length) {
|
if (audioReferences.length && !references.length && !videoReferences.length) {
|
||||||
throw new Error("Seedance 参考音频不能单独使用,请同时添加参考图或参考视频");
|
throw new Error(config.interfaceType === "volcengine-ark-video" ? "火山方舟全模态参考不支持纯音频或文本+音频,请同时添加参考图片或参考视频" : "Seedance 参考音频不能单独使用,请同时添加参考图或参考视频");
|
||||||
}
|
}
|
||||||
const content = config.interfaceType === "volcengine-ark-video"
|
const content = config.interfaceType === "volcengine-ark-video"
|
||||||
? await buildVolcengineArkContent(prompt, references, videoReferences, audioReferences)
|
? await buildVolcengineArkContent(prompt, references, videoReferences, audioReferences)
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ export type CanvasGenerationBatchStatus = "queued" | "running" | "partial_failed
|
|||||||
export type CanvasGenerationBatchItemStatus = "waiting" | "submitting" | "queued" | "running" | "succeeded" | "failed" | "cancelled";
|
export type CanvasGenerationBatchItemStatus = "waiting" | "submitting" | "queued" | "running" | "succeeded" | "failed" | "cancelled";
|
||||||
export type CanvasImageGenerationType = "generation" | "edit";
|
export type CanvasImageGenerationType = "generation" | "edit";
|
||||||
export type CanvasWorkflowKind = "free" | "script" | "story_input" | "character" | "scene" | "storyboard" | "shot" | "final" | "styleboard" | "reference_set" | "reference_video" | "action_board";
|
export type CanvasWorkflowKind = "free" | "script" | "story_input" | "character" | "scene" | "storyboard" | "shot" | "final" | "styleboard" | "reference_set" | "reference_video" | "action_board";
|
||||||
export type CanvasVideoEditOperation = "text_to_video" | "image_to_video" | "extend" | "inpaint" | "replace_element" | "camera_motion" | "style_transfer" | "audio_to_video" | "compare_versions" | "concat";
|
export type CanvasVideoEditOperation = "text_to_video" | "image_to_video" | "reference_to_video" | "extend" | "inpaint" | "replace_element" | "camera_motion" | "style_transfer" | "audio_to_video" | "compare_versions" | "concat";
|
||||||
export type CanvasSkillCategory = "writing" | "storyboard" | "image" | "video" | "utility";
|
export type CanvasSkillCategory = "writing" | "storyboard" | "image" | "video" | "utility";
|
||||||
export type CanvasSkillOutputMode = "text" | "json" | "image_prompt" | "workflow";
|
export type CanvasSkillOutputMode = "text" | "json" | "image_prompt" | "workflow";
|
||||||
export type StoryboardColumn =
|
export type StoryboardColumn =
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
import { describe, expect, test } from "bun:test";
|
import { describe, expect, test } from "bun:test";
|
||||||
|
|
||||||
import { unwrapEnvelope, videoTaskId } from "../src/services/api/video-response";
|
import { defaultConfig } from "../src/stores/use-config-store";
|
||||||
|
import { createSeedanceTask } from "../src/services/api/video-provider-seedance";
|
||||||
|
import type { VideoProviderDeps } from "../src/services/api/video-provider-deps";
|
||||||
|
import { unwrapEnvelope, videoResponseTools, videoTaskId } from "../src/services/api/video-response";
|
||||||
import { isPublicMediaUrl, normalizeVideoResolution, normalizeVideoSeconds, normalizeVideoSize } from "../src/services/api/video-validation";
|
import { isPublicMediaUrl, normalizeVideoResolution, normalizeVideoSeconds, normalizeVideoSize } from "../src/services/api/video-validation";
|
||||||
|
|
||||||
describe("video API response contracts", () => {
|
describe("video API response contracts", () => {
|
||||||
@@ -34,3 +37,66 @@ describe("video request normalization", () => {
|
|||||||
expect(isPublicMediaUrl("/resources/video.mp4")).toBe(false);
|
expect(isPublicMediaUrl("/resources/video.mp4")).toBe(false);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("Volcengine Ark full-modal references", () => {
|
||||||
|
const model = "seedance-2-0-250824";
|
||||||
|
const config = {
|
||||||
|
...defaultConfig,
|
||||||
|
baseUrl: "https://ark.cn-beijing.volces.com/api/v3",
|
||||||
|
apiKey: "test-key",
|
||||||
|
interfaceType: "volcengine-ark-video",
|
||||||
|
channels: [
|
||||||
|
{
|
||||||
|
id: "ark",
|
||||||
|
name: "Ark",
|
||||||
|
baseUrl: "https://ark.cn-beijing.volces.com/api/v3",
|
||||||
|
apiKey: "test-key",
|
||||||
|
secretKey: "",
|
||||||
|
headers: [],
|
||||||
|
apiFormat: "openai",
|
||||||
|
interfaceType: "volcengine-ark-video",
|
||||||
|
models: [model],
|
||||||
|
scope: "user",
|
||||||
|
enabled: true,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
test("映射图片、视频和音频参考素材", async () => {
|
||||||
|
let requestUrl = "";
|
||||||
|
let requestBody: unknown;
|
||||||
|
const deps = {
|
||||||
|
transport: {
|
||||||
|
post: async (url: string, body: unknown) => {
|
||||||
|
requestUrl = url;
|
||||||
|
requestBody = body;
|
||||||
|
return { id: "task-1", status: "queued" };
|
||||||
|
},
|
||||||
|
},
|
||||||
|
response: videoResponseTools,
|
||||||
|
} as unknown as VideoProviderDeps;
|
||||||
|
|
||||||
|
await createSeedanceTask(
|
||||||
|
deps,
|
||||||
|
config as never,
|
||||||
|
model,
|
||||||
|
"保持主体一致",
|
||||||
|
[{ id: "image-1", name: "image.png", type: "image/png", dataUrl: "", url: "https://cdn.example.com/image.png" }],
|
||||||
|
[{ id: "video-1", name: "video.mp4", type: "video/mp4", url: "https://cdn.example.com/video.mp4" }],
|
||||||
|
[{ id: "audio-1", name: "audio.mp3", type: "audio/mpeg", url: "https://cdn.example.com/audio.mp3" }],
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(requestUrl).toBe("https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks");
|
||||||
|
expect((requestBody as { content: unknown[] }).content).toEqual([
|
||||||
|
{ type: "text", text: "保持主体一致" },
|
||||||
|
{ type: "image_url", image_url: { url: "https://cdn.example.com/image.png" }, role: "reference_image" },
|
||||||
|
{ type: "video_url", video_url: { url: "https://cdn.example.com/video.mp4" }, role: "reference_video" },
|
||||||
|
{ type: "audio_url", audio_url: { url: "https://cdn.example.com/audio.mp3" }, role: "reference_audio" },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("拒绝纯音频和文本加音频", async () => {
|
||||||
|
const deps = { transport: {}, response: videoResponseTools } as unknown as VideoProviderDeps;
|
||||||
|
await expect(createSeedanceTask(deps, config as never, model, "跟随节奏", [], [], [{ id: "audio-1", name: "audio.mp3", type: "audio/mpeg", url: "https://cdn.example.com/audio.mp3" }])).rejects.toThrow("不支持纯音频或文本+音频");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -22,4 +22,8 @@ describe("video generation resolution options", () => {
|
|||||||
expect(defaultModelCapabilityConfig("volcengine-jimeng-video").video?.resolutions).toEqual(["720p"]);
|
expect(defaultModelCapabilityConfig("volcengine-jimeng-video").video?.resolutions).toEqual(["720p"]);
|
||||||
expect(defaultModelCapabilityConfig("gemini-veo").video?.resolutions).toEqual(["720p", "1080p"]);
|
expect(defaultModelCapabilityConfig("gemini-veo").video?.resolutions).toEqual(["720p", "1080p"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("火山方舟默认开启全模态参考模式", () => {
|
||||||
|
expect(defaultModelCapabilityConfig("volcengine-ark-video").video?.operations).toEqual(expect.arrayContaining(["reference_to_video", "audio_to_video"]));
|
||||||
|
});
|
||||||
});
|
});
|
||||||
Reference in new issue
Block a user