feat(video): support Ark full-modal references (#300)
This commit is contained in:
MasterChiefCN
@@ -30,6 +30,7 @@ type CanvasConfigNodePanelProps = {
|
||||
const videoOperationOptions: Array<{ label: string; value: CanvasVideoEditOperation }> = [
|
||||
{ label: "文生视频", value: "text_to_video" },
|
||||
{ label: "图生视频", value: "image_to_video" },
|
||||
{ label: "全模态参考", value: "reference_to_video" },
|
||||
{ label: "视频续写", value: "extend" },
|
||||
{ label: "局部修改", value: "inpaint" },
|
||||
{ label: "元素替换", value: "replace_element" },
|
||||
|
||||
@@ -413,6 +413,7 @@ function videoOperationLabel(operation: CanvasVideoEditOperation) {
|
||||
const labels: Record<CanvasVideoEditOperation, string> = {
|
||||
text_to_video: "文生视频",
|
||||
image_to_video: "图生视频",
|
||||
reference_to_video: "全模态参考",
|
||||
extend: "视频续写",
|
||||
inpaint: "局部修改",
|
||||
replace_element: "元素替换",
|
||||
|
||||
@@ -9,6 +9,7 @@ const ratioOptions = ["16:9", "9:16", "1:1", "4:3", "3:4", "21:9"];
|
||||
const operationOptions = [
|
||||
{ label: "文生视频", value: "text_to_video" },
|
||||
{ label: "图生视频", value: "image_to_video" },
|
||||
{ label: "全模态参考", value: "reference_to_video" },
|
||||
{ label: "视频续写", value: "extend" },
|
||||
{ label: "局部修改", value: "inpaint" },
|
||||
{ label: "元素替换", value: "replace_element" },
|
||||
|
||||
@@ -294,6 +294,7 @@ function logicalCapabilitySummary(spec: NonNullable<NonNullable<AiConfig["channe
|
||||
const operationLabels: Record<string, string> = {
|
||||
text_to_video: "文生视频",
|
||||
image_to_video: "图生视频",
|
||||
reference_to_video: "全模态参考",
|
||||
audio_to_video: "音频生视频",
|
||||
extend: "视频续写",
|
||||
inpaint: "局部修改",
|
||||
|
||||
@@ -43,6 +43,7 @@ export const operationOptions = [
|
||||
{ label: "Agent 会话：拆解影视工作流", value: "agent_session" },
|
||||
{ label: "文生视频", value: "text_to_video" },
|
||||
{ label: "图生视频", value: "image_to_video" },
|
||||
{ label: "全模态参考", value: "reference_to_video" },
|
||||
{ label: "视频续写", value: "extend" },
|
||||
{ label: "视频局部修改", value: "inpaint" },
|
||||
{ label: "元素替换", value: "replace_element" },
|
||||
|
||||
@@ -292,7 +292,10 @@ export function defaultModelCapabilityConfig(protocol?: ModelProtocol, model = "
|
||||
video.generateAudio = { supported: true, default: true };
|
||||
}
|
||||
if (protocol === "volcengine-ark-video" || protocol === "newapi-channel-1") video.resolutions = ["480p", "720p", "1080p"];
|
||||
if (protocol === "volcengine-ark-video") video.watermark = { supported: true, default: false };
|
||||
if (protocol === "volcengine-ark-video") {
|
||||
video.watermark = { supported: true, default: false };
|
||||
video.operations.push("reference_to_video", "audio_to_video");
|
||||
}
|
||||
if (protocol === "novita-video") {
|
||||
video.references.maxImages = 1;
|
||||
video.references.maxImageBytes = 10 * 1024 * 1024;
|
||||
|
||||
@@ -336,7 +336,7 @@ function videoOperationLabel(operation: string) {
|
||||
if (operation === "text_to_video") return "文生视频";
|
||||
if (operation === "image_to_video") return "图生视频";
|
||||
if (operation === "audio_to_video") return "音频生视频";
|
||||
if (operation === "reference_to_video") return "参考素材生视频";
|
||||
if (operation === "reference_to_video") return "全模态参考";
|
||||
if (operation === "extend") return "视频续写";
|
||||
return "当前生成模式";
|
||||
}
|
||||
|
||||
@@ -637,6 +637,7 @@ export function operationLabel(value: string) {
|
||||
{
|
||||
text_to_video: "文生视频",
|
||||
image_to_video: "图生视频",
|
||||
reference_to_video: "全模态参考",
|
||||
extend: "视频续写",
|
||||
inpaint: "局部修改",
|
||||
replace_element: "元素替换",
|
||||
|
||||
@@ -81,8 +81,8 @@ function seedanceApiUrl(config: ResolvedAiConfig, taskId?: string) {
|
||||
}
|
||||
|
||||
async function buildSeedanceAgentPlanPayload(config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps) {
|
||||
if (config.interfaceType !== "volcengine-ark-video" && audioReferences.length && !references.length && !videoReferences.length) {
|
||||
throw new Error("Seedance 参考音频不能单独使用，请同时添加参考图或参考视频");
|
||||
if (audioReferences.length && !references.length && !videoReferences.length) {
|
||||
throw new Error(config.interfaceType === "volcengine-ark-video" ? "火山方舟全模态参考不支持纯音频或文本+音频，请同时添加参考图片或参考视频" : "Seedance 参考音频不能单独使用，请同时添加参考图或参考视频");
|
||||
}
|
||||
const content = config.interfaceType === "volcengine-ark-video"
|
||||
? await buildVolcengineArkContent(prompt, references, videoReferences, audioReferences)
|
||||
|
||||
@@ -50,7 +50,7 @@ export type CanvasGenerationBatchStatus = "queued" | "running" | "partial_failed
|
||||
export type CanvasGenerationBatchItemStatus = "waiting" | "submitting" | "queued" | "running" | "succeeded" | "failed" | "cancelled";
|
||||
export type CanvasImageGenerationType = "generation" | "edit";
|
||||
export type CanvasWorkflowKind = "free" | "script" | "story_input" | "character" | "scene" | "storyboard" | "shot" | "final" | "styleboard" | "reference_set" | "reference_video" | "action_board";
|
||||
export type CanvasVideoEditOperation = "text_to_video" | "image_to_video" | "extend" | "inpaint" | "replace_element" | "camera_motion" | "style_transfer" | "audio_to_video" | "compare_versions" | "concat";
|
||||
export type CanvasVideoEditOperation = "text_to_video" | "image_to_video" | "reference_to_video" | "extend" | "inpaint" | "replace_element" | "camera_motion" | "style_transfer" | "audio_to_video" | "compare_versions" | "concat";
|
||||
export type CanvasSkillCategory = "writing" | "storyboard" | "image" | "video" | "utility";
|
||||
export type CanvasSkillOutputMode = "text" | "json" | "image_prompt" | "workflow";
|
||||
export type StoryboardColumn =
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
|
||||
import { unwrapEnvelope, videoTaskId } from "../src/services/api/video-response";
|
||||
import { defaultConfig } from "../src/stores/use-config-store";
|
||||
import { createSeedanceTask } from "../src/services/api/video-provider-seedance";
|
||||
import type { VideoProviderDeps } from "../src/services/api/video-provider-deps";
|
||||
import { unwrapEnvelope, videoResponseTools, videoTaskId } from "../src/services/api/video-response";
|
||||
import { isPublicMediaUrl, normalizeVideoResolution, normalizeVideoSeconds, normalizeVideoSize } from "../src/services/api/video-validation";
|
||||
|
||||
describe("video API response contracts", () => {
|
||||
@@ -34,3 +37,66 @@ describe("video request normalization", () => {
|
||||
expect(isPublicMediaUrl("/resources/video.mp4")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Volcengine Ark full-modal references", () => {
|
||||
const model = "seedance-2-0-250824";
|
||||
const config = {
|
||||
...defaultConfig,
|
||||
baseUrl: "https://ark.cn-beijing.volces.com/api/v3",
|
||||
apiKey: "test-key",
|
||||
interfaceType: "volcengine-ark-video",
|
||||
channels: [
|
||||
{
|
||||
id: "ark",
|
||||
name: "Ark",
|
||||
baseUrl: "https://ark.cn-beijing.volces.com/api/v3",
|
||||
apiKey: "test-key",
|
||||
secretKey: "",
|
||||
headers: [],
|
||||
apiFormat: "openai",
|
||||
interfaceType: "volcengine-ark-video",
|
||||
models: [model],
|
||||
scope: "user",
|
||||
enabled: true,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
test("映射图片、视频和音频参考素材", async () => {
|
||||
let requestUrl = "";
|
||||
let requestBody: unknown;
|
||||
const deps = {
|
||||
transport: {
|
||||
post: async (url: string, body: unknown) => {
|
||||
requestUrl = url;
|
||||
requestBody = body;
|
||||
return { id: "task-1", status: "queued" };
|
||||
},
|
||||
},
|
||||
response: videoResponseTools,
|
||||
} as unknown as VideoProviderDeps;
|
||||
|
||||
await createSeedanceTask(
|
||||
deps,
|
||||
config as never,
|
||||
model,
|
||||
"保持主体一致",
|
||||
[{ id: "image-1", name: "image.png", type: "image/png", dataUrl: "", url: "https://cdn.example.com/image.png" }],
|
||||
[{ id: "video-1", name: "video.mp4", type: "video/mp4", url: "https://cdn.example.com/video.mp4" }],
|
||||
[{ id: "audio-1", name: "audio.mp3", type: "audio/mpeg", url: "https://cdn.example.com/audio.mp3" }],
|
||||
);
|
||||
|
||||
expect(requestUrl).toBe("https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks");
|
||||
expect((requestBody as { content: unknown[] }).content).toEqual([
|
||||
{ type: "text", text: "保持主体一致" },
|
||||
{ type: "image_url", image_url: { url: "https://cdn.example.com/image.png" }, role: "reference_image" },
|
||||
{ type: "video_url", video_url: { url: "https://cdn.example.com/video.mp4" }, role: "reference_video" },
|
||||
{ type: "audio_url", audio_url: { url: "https://cdn.example.com/audio.mp3" }, role: "reference_audio" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("拒绝纯音频和文本加音频", async () => {
|
||||
const deps = { transport: {}, response: videoResponseTools } as unknown as VideoProviderDeps;
|
||||
await expect(createSeedanceTask(deps, config as never, model, "跟随节奏", [], [], [{ id: "audio-1", name: "audio.mp3", type: "audio/mpeg", url: "https://cdn.example.com/audio.mp3" }])).rejects.toThrow("不支持纯音频或文本+音频");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -22,4 +22,8 @@ describe("video generation resolution options", () => {
|
||||
expect(defaultModelCapabilityConfig("volcengine-jimeng-video").video?.resolutions).toEqual(["720p"]);
|
||||
expect(defaultModelCapabilityConfig("gemini-veo").video?.resolutions).toEqual(["720p", "1080p"]);
|
||||
});
|
||||
|
||||
test("火山方舟默认开启全模态参考模式", () => {
|
||||
expect(defaultModelCapabilityConfig("volcengine-ark-video").video?.operations).toEqual(expect.arrayContaining(["reference_to_video", "audio_to_video"]));
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user