tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(canvas): 视频参考协议 - 统一首尾帧与角色资产语义

Browse Source
This commit is contained in:
ddCat
2026-08-30 16:52:09 +08:00
parent b5f8b66622
commit 37eb2b7e48
15 changed files with 479 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
web/src/services/api/video-provider-agnes.ts
View File

@@ -5,6 +5,7 @@ import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
import type { ApiEnvelope, RequestOptions, ResolvedAiConfig, VideoGenerationTask, VideoGenerationTaskState } from "./video-contracts";
import type { VideoProviderDeps } from "./video-provider-deps";
import { resolveVideoImageReferences, type VideoImageRole } from "./video-reference-roles";
type AgnesTaskResponse = {
id?: string;
@@ -44,9 +45,10 @@ export async function createAgnesVideoTask(
Promise.all(videoReferences.map((item) => resolveAgnesMediaUrl(item.url, item.storageKey))),
Promise.all(audioReferences.map((item) => resolveAgnesMediaUrl(item.url, item.storageKey))),
]);
const imageRoles = resolveVideoImageReferences(references, options, { videoCount: videoUrls.length, audioCount: audioUrls.length }).map(({ role }) => role);
const payload = modelName === "agnes-video-v2.0"
? buildAgnesV20Payload(config, modelName, normalizedPrompt, imageUrls, videoUrls, audioUrls)
: buildAgnesV25Payload(config, modelName, normalizedPrompt, imageUrls, videoUrls, audioUrls);
? buildAgnesV20Payload(config, modelName, normalizedPrompt, imageUrls, imageRoles, videoUrls, audioUrls, options)
: buildAgnesV25Payload(config, modelName, normalizedPrompt, imageUrls, imageRoles, videoUrls, audioUrls);
try {
const raw = await deps.transport.post<ApiEnvelope<AgnesTaskResponse>>(deps.transport.apiUrl("/videos"), payload, options);
@@ -92,7 +94,7 @@ export function agnesPollUrl(config: ResolvedAiConfig, videoId: string, modelNam
return base.toString();
}
function buildAgnesV25Payload(config: ResolvedAiConfig, model: string, prompt: string, images: string[], videos: string[], audios: string[]) {
function buildAgnesV25Payload(config: ResolvedAiConfig, model: string, prompt: string, images: string[], imageRoles: VideoImageRole[], videos: string[], audios: string[]) {
if (!AGNES_V25_MODELS.has(model)) throw new Error(`Agnes 视频协议不支持模型 ${model}`);
const seconds = agnesSeconds(config.videoSeconds);
const size = agnesResolution(config.vquality);
@@ -101,7 +103,10 @@ function buildAgnesV25Payload(config: ResolvedAiConfig, model: string, prompt: s
if (images.length > 5) throw new Error("Agnes Video 2.5 Flash 最多支持 5 张参考图");
if (videos.length) throw new Error("Agnes Video 2.5 Flash 不支持参考视频");
}
const mode = videos.length || audios.length || images.length > 2 ? "reference" : images.length ? "keyframe" : "text";
const hasReferenceImages = imageRoles.includes("reference_image");
const hasFrameImages = imageRoles.includes("first_frame") || imageRoles.includes("last_frame");
if (hasFrameImages && (hasReferenceImages || videos.length || audios.length)) throw new Error("Agnes Video 2.5 不能同时混用首尾帧和角色、视频或音频参考素材");
const mode = videos.length || audios.length || hasReferenceImages ? "reference" : images.length ? "keyframe" : "text";
const payload: Record<string, unknown> = {
model,
prompt,
@@ -112,8 +117,10 @@ function buildAgnesV25Payload(config: ResolvedAiConfig, model: string, prompt: s
n: 1,
};
if (mode === "keyframe") {
payload.first_frame = images[0];
if (images[1]) payload.last_frame = images[1];
const firstFrame = imageForRole(images, imageRoles, "first_frame") || images[0];
const lastFrame = imageForRole(images, imageRoles, "last_frame");
payload.first_frame = firstFrame;
if (lastFrame) payload.last_frame = lastFrame;
} else if (mode === "reference") {
if (images.length) payload.images = images;
if (audios.length) payload.audios = audios;
@@ -122,8 +129,9 @@ function buildAgnesV25Payload(config: ResolvedAiConfig, model: string, prompt: s
return payload;
}
function buildAgnesV20Payload(config: ResolvedAiConfig, model: string, prompt: string, images: string[], videos: string[], audios: string[]) {
function buildAgnesV20Payload(config: ResolvedAiConfig, model: string, prompt: string, images: string[], imageRoles: VideoImageRole[], videos: string[], audios: string[], options?: RequestOptions) {
if (videos.length || audios.length) throw new Error("Agnes Video V2.0 不支持参考视频或音频");
if (options?.videoEditOperation === "reference_to_video" || imageRoles.includes("reference_image")) throw new Error("Agnes Video V2.0 不支持角色或风格参考图模式，请改用 Agnes Video 2.5");
const seconds = Math.max(1, Math.floor(Number(config.videoSeconds) || 5));
const frameRate = 24;
const targetFrames = seconds * frameRate;
@@ -133,11 +141,17 @@ function buildAgnesV20Payload(config: ResolvedAiConfig, model: string, prompt: s
frame_rate: frameRate,
num_frames: Math.min(441, Math.floor((targetFrames - 1 + 4) / 8) * 8 + 1),
};
if (images.length === 1) payload.image = images[0];
if (images.length > 1) payload.extra_body = { image: images, mode: "keyframes" };
const orderedImages = [imageForRole(images, imageRoles, "first_frame"), imageForRole(images, imageRoles, "last_frame")].filter((value): value is string => Boolean(value));
if (orderedImages.length === 1) payload.image = orderedImages[0];
if (orderedImages.length > 1) payload.extra_body = { image: orderedImages, mode: "keyframes" };
return payload;
}
function imageForRole(images: string[], roles: VideoImageRole[], role: VideoImageRole) {
const index = roles.indexOf(role);
return index >= 0 ? images[index] : undefined;
}
function agnesSeconds(value: string) {
const seconds = Math.floor(Number(value) || 5);
if (seconds < 4 || seconds > 12) throw new Error("Agnes Video 2.5 时长必须在 4–12 秒之间");

1
web/src/services/api/video-provider-gemini.ts
View File

@@ -16,6 +16,7 @@ type GeminiVeoOperation = {
export async function createGeminiVeoTask(deps: VideoProviderDeps, config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], options?: RequestOptions): Promise<VideoGenerationTask> {
if (references.length > 1 || videoReferences.length || audioReferences.length) throw new Error("Gemini Veo 当前只支持 1 张起始图，不支持参考视频或音频");
if (references.length && options?.videoEditOperation === "reference_to_video") throw new Error("Gemini Veo 当前不支持角色或风格参考图生视频，请改用支持 reference_to_video 的模型");
const instance: Record<string, unknown> = { prompt: prompt.trim() };
if (references[0]) {
const dataUrl = await imageToDataUrl(references[0]);

17
web/src/services/api/video-provider-minimax.ts
View File

@@ -9,22 +9,22 @@ import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
import type { MiniMaxVideoCreateResponse, MiniMaxVideoTask, RequestOptions, ResolvedAiConfig, VideoGenerationTask, VideoGenerationTaskState } from "./video-contracts";
import type { VideoProviderDeps } from "./video-provider-deps";
import { resolveVideoImageReferences } from "./video-reference-roles";
export async function createMiniMaxVideoTask(deps: VideoProviderDeps, config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], options?: RequestOptions): Promise<VideoGenerationTask> {
const imageUrls = await Promise.all(references.slice(0, 9).map((image) => resolveMiniMaxImageUrl(image)));
const videoUrls = await Promise.all(videoReferences.slice(0, 3).map((video) => resolveMiniMaxMediaUrl(video, "参考视频")));
const audioUrls = await Promise.all(audioReferences.slice(0, 3).map((audio) => resolveMiniMaxMediaUrl(audio, "参考音频")));
const content: Array<Record<string, unknown>> = [{ type: "text", text: prompt.trim() }];
const explicitFrameMode = options?.videoEditOperation !== "reference_to_video" && Boolean(options?.videoStartFrameNodeId || options?.videoEndFrameNodeId);
const imagePlan = resolveVideoImageReferences(references.slice(0, 9), options, { videoCount: videoUrls.length, audioCount: audioUrls.length });
const frameMode = imagePlan.some(({ role }) => role === "first_frame" || role === "last_frame");
imageUrls.forEach((url, index) => {
const reference = references[index];
const role = miniMaxImageRole(reference?.id, options);
if (explicitFrameMode && role === "reference_image") throw new Error("MiniMax 首尾帧模式不能混合未标记的参考图");
const role = imagePlan[index]?.role || "reference_image";
if (frameMode && role === "reference_image") throw new Error("MiniMax 首尾帧模式不能混合未标记的参考图");
content.push({ type: "image_url", image_url: { url }, role });
});
videoUrls.forEach((url) => content.push({ type: "video_url", video_url: { url }, role: "reference_video" }));
audioUrls.forEach((url) => content.push({ type: "audio_url", audio_url: { url }, role: "reference_audio" }));
const frameMode = explicitFrameMode;
const payload = {
model: modelOptionName(model),
content,
@@ -43,13 +43,6 @@ export async function createMiniMaxVideoTask(deps: VideoProviderDeps, config: Re
}
}
function miniMaxImageRole(imageId: string | undefined, options?: RequestOptions) {
if (options?.videoEditOperation === "reference_to_video") return "reference_image";
if (imageId && imageId === options?.videoStartFrameNodeId) return "first_frame";
if (imageId && imageId === options?.videoEndFrameNodeId) return "last_frame";
return "reference_image";
}
export async function pollMiniMaxVideoTask(deps: VideoProviderDeps, config: ResolvedAiConfig, task: VideoGenerationTask, options?: RequestOptions): Promise<VideoGenerationTaskState> {
try {
const response = await deps.transport.get<{ task?: MiniMaxVideoTask }>(miniMaxVideoUrl(config, `/query/video_generation/${encodeURIComponent(task.id)}`), options);

1
web/src/services/api/video-provider-novita.ts
View File

@@ -12,6 +12,7 @@ type NovitaVideoResult = { task?: { status?: string; reason?: string }; videos?:
export async function createNovitaVideoTask(deps: VideoProviderDeps, config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], options?: RequestOptions): Promise<VideoGenerationTask> {
if (references.length > 1 || videoReferences.length || audioReferences.length) throw new Error("Novita 视频当前只支持 1 张起始图，不支持参考视频或音频");
if (references.length && options?.videoEditOperation === "reference_to_video") throw new Error("Novita 视频当前不支持角色或风格参考图生视频，请改用支持 reference_to_video 的模型");
const payload: Record<string, unknown> = {
model: modelOptionName(model),
prompt: prompt.trim(),

10
web/src/services/api/video-provider-openai.ts
View File

@@ -13,13 +13,21 @@ export async function createOpenAIVideoTask(deps: VideoProviderDeps, config: Res
if (config.interfaceType === "xai-video" || modelName.toLowerCase().includes("grok")) {
const images = await Promise.all(references.slice(0, 7).map((image) => imageToDataUrl(image)));
const seconds = normalizeVideoSeconds(config.videoSeconds);
const referenceMode = options?.videoEditOperation === "reference_to_video";
const explicitFrameMode = !referenceMode && Boolean(options?.videoStartFrameNodeId || options?.videoEndFrameNodeId);
if (config.interfaceType === "xai-video" && explicitFrameMode && (images.length > 1 || options?.videoEndFrameNodeId)) throw new Error("xAI 视频协议最多支持 1 张起始图，不支持尾帧或混合角色参考图");
const imagePayload = config.interfaceType === "xai-video"
? referenceMode && images.length
? { reference_images: images.map((url) => ({ url })) }
: images.length ? { image: { url: images[0] } } : {}
: images.length ? { image: images[0], images } : {};
const payload = {
model: modelName,
prompt,
duration: Number.parseInt(seconds, 10) || 6,
seconds,
...(normalizeVideoSize(config.size) ? { size: normalizeVideoSize(config.size) } : {}),
...(images.length ? { image: images[0], images } : {}),
...imagePayload,
};
try {
const createPath = config.interfaceType === "xai-video" ? "/videos/generations" : "/videos";

44
web/src/services/api/video-provider-seedance.ts
View File

@@ -10,6 +10,7 @@ import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
import { isPublicMediaUrl } from "./video-validation";
import type { ApiEnvelope, RequestOptions, ResolvedAiConfig, SeedanceTask, VideoGenerationTask, VideoGenerationTaskState } from "./video-contracts";
import type { VideoProviderDeps } from "./video-provider-deps";
import { hasExplicitVideoFrames, resolveVideoImageReferences } from "./video-reference-roles";
export function isSeedanceConfig(config: ResolvedAiConfig) {
return config.interfaceType === "volcengine-ark-video" || isSeedanceVideoConfig(config);
@@ -20,8 +21,8 @@ export async function createSeedanceTask(deps: VideoProviderDeps, config: Resolv
assertSeedanceAudioReferences(audioReferences);
const isVolcengineArk = config.interfaceType === "volcengine-ark-video";
const payload = isVolcengineArk || isArkPlanBaseUrl(config.baseUrl)
? await buildSeedanceAgentPlanPayload(config, model, prompt, references, videoReferences, audioReferences, deps)
: await buildSeedanceVideosPayload(config, model, prompt, references, videoReferences, audioReferences, deps);
? await buildSeedanceAgentPlanPayload(config, model, prompt, references, videoReferences, audioReferences, deps, options)
: await buildSeedanceVideosPayload(config, model, prompt, references, videoReferences, audioReferences, deps, options);
try {
const raw = await deps.transport.post<ApiEnvelope<SeedanceTask>>(seedanceApiUrl(config), payload, options);
@@ -80,13 +81,13 @@ function seedanceApiUrl(config: ResolvedAiConfig, taskId?: string) {
return buildApiUrl(config.baseUrl, `/videos${taskId ? `/${encodeURIComponent(taskId)}` : ""}`);
}
async function buildSeedanceAgentPlanPayload(config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps) {
async function buildSeedanceAgentPlanPayload(config: ResolvedAiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps, options?: RequestOptions) {
if (audioReferences.length && !references.length && !videoReferences.length) {
throw new Error(config.interfaceType === "volcengine-ark-video" ? "火山方舟全模态参考不支持纯音频或文本+音频，请同时添加参考图片或参考视频" : "Seedance 参考音频不能单独使用，请同时添加参考图或参考视频");
}
const content = config.interfaceType === "volcengine-ark-video"
? await buildVolcengineArkContent(prompt, references, videoReferences, audioReferences)
: await buildSeedanceContent(config, prompt, references, videoReferences, audioReferences, deps);
? await buildVolcengineArkContent(prompt, references, videoReferences, audioReferences, options)
: await buildSeedanceContent(config, prompt, references, videoReferences, audioReferences, deps, options);
if (!content.length) throw new Error("请输入视频提示词，或连接参考图片/视频/音频");
const profile = modelCapabilityConfigFor(config, model).video!;
return {
@@ -100,11 +101,12 @@ async function buildSeedanceAgentPlanPayload(config: ResolvedAiConfig, model: st
};
}
async function buildVolcengineArkContent(prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) {
async function buildVolcengineArkContent(prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], options?: RequestOptions) {
const content: Array<Record<string, unknown>> = [];
const imagePlan = resolveVideoImageReferences(references.slice(0, SEEDANCE_REFERENCE_LIMITS.images), options, { videoCount: videoReferences.length, audioCount: audioReferences.length });
if (prompt.trim()) content.push({ type: "text", text: prompt.trim() });
for (const image of references.slice(0, SEEDANCE_REFERENCE_LIMITS.images)) {
content.push({ type: "image_url", image_url: { url: await resolveVolcengineArkReferenceUrl(image.url || image.dataUrl, image.storageKey) }, role: "reference_image" });
for (const { image, role } of imagePlan) {
content.push({ type: "image_url", image_url: { url: await resolveVolcengineArkReferenceUrl(image.url || image.dataUrl, image.storageKey) }, role });
}
for (const video of videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos)) {
content.push({ type: "video_url", video_url: { url: await resolveVolcengineArkReferenceUrl(video.url, video.storageKey) }, role: "reference_video" });
@@ -121,35 +123,45 @@ async function resolveVolcengineArkReferenceUrl(value: string | undefined, stora
throw new Error("火山方舟视频参考素材需要公网 URL 或 asset:// 素材 ID；请先将本地素材保存到对象存储");
}
async function buildSeedanceVideosPayload(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps) {
async function buildSeedanceVideosPayload(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps, options?: RequestOptions) {
if ((videoReferences.length || audioReferences.length) && !references.length) {
throw new Error("Seedance 参考视频或参考音频需要同时连接至少 1 张主参考图");
}
const imageUrls = await Promise.all(references.slice(0, SEEDANCE_REFERENCE_LIMITS.images).map(resolveSeedanceVideosImageUrl));
const imagePlan = resolveVideoImageReferences(references.slice(0, SEEDANCE_REFERENCE_LIMITS.images), options, { videoCount: videoReferences.length, audioCount: audioReferences.length });
const videoUrls = await Promise.all(videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos).map((media) => resolveSeedanceVideosMediaUrl(media, deps)));
const audioUrls = await Promise.all(audioReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.audios).map((media) => resolveSeedanceVideosMediaUrl(media, deps)));
const ratio = normalizeSeedanceRatio(config.size);
const duration = normalizeSeedanceDuration(config.videoSeconds);
const profile = modelCapabilityConfigFor(config, model).video!;
const imagePayload: Record<string, unknown> = {};
if (options?.videoEditOperation === "reference_to_video") {
if (imageUrls.length) imagePayload.reference_image_urls = imageUrls;
} else if (hasExplicitVideoFrames(options)) {
imagePayload.image_urls = orderedImageUrls(imageUrls, imagePlan.map(({ role }) => role));
} else if (imageUrls.length) {
imagePayload.image_url = imageUrls[0];
if (imageUrls.length > 1) imagePayload.reference_image_urls = imageUrls.slice(1);
}
return {
model: modelOptionName(model),
prompt: prompt.trim(),
aspect_ratio: ratio === "adaptive" ? "16:9" : ratio,
duration,
...(profile.generateAudio.supported ? { generate_audio: boolConfig(config.videoGenerateAudio, profile.generateAudio.default) } : {}),
...(imageUrls[0] ? { image_url: imageUrls[0] } : {}),
...(imageUrls.length > 1 ? { reference_image_urls: imageUrls.slice(1) } : {}),
...imagePayload,
...(videoUrls.length ? { reference_videos: videoUrls } : {}),
...(audioUrls.length ? { reference_audios: audioUrls } : {}),
};
}
async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps) {
async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], deps: VideoProviderDeps, options?: RequestOptions) {
const content: Array<Record<string, unknown>> = [];
const imagePlan = resolveVideoImageReferences(references.slice(0, SEEDANCE_REFERENCE_LIMITS.images), options, { videoCount: videoReferences.length, audioCount: audioReferences.length });
const text = buildSeedancePromptText(prompt, references, videoReferences, audioReferences);
if (text) content.push({ type: "text", text });
for (const image of references.slice(0, SEEDANCE_REFERENCE_LIMITS.images)) {
content.push({ type: "image_url", image_url: { url: await resolveSeedanceImageUrl(image) }, role: "reference_image" });
for (const { image, role } of imagePlan) {
content.push({ type: "image_url", image_url: { url: await resolveSeedanceImageUrl(image) }, role });
}
for (const video of videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos)) {
content.push({ type: "video_url", video_url: { url: await resolveSeedanceMediaUrl(video, deps, "参考视频") }, role: "reference_video" });
@@ -160,6 +172,10 @@ async function buildSeedanceContent(config: AiConfig, prompt: string, references
return content;
}
function orderedImageUrls(imageUrls: string[], roles: Array<"first_frame" | "last_frame" | "reference_image">) {
return ["first_frame", "last_frame", "reference_image"].flatMap((role) => imageUrls.filter((_, index) => roles[index] === role));
}
async function resolveSeedanceImageUrl(image: ReferenceImage) {
const directUrl = image.url || image.dataUrl;
if (isPublicMediaUrl(directUrl) || directUrl.startsWith("asset://")) return directUrl;

45
web/src/services/api/video-reference-roles.ts Normal file
View File

@@ -0,0 +1,45 @@
import type { ReferenceImage } from "@/types/image";
import type { RequestOptions } from "./video-contracts";
export type VideoImageRole = "first_frame" | "last_frame" | "reference_image";
export type ResolvedVideoImageReference = {
image: ReferenceImage;
role: VideoImageRole;
};
type VideoReferenceContext = {
videoCount?: number;
audioCount?: number;
};
export function resolveVideoImageReferences(images: ReferenceImage[], options?: RequestOptions, context: VideoReferenceContext = {}): ResolvedVideoImageReference[] {
const operation = options?.videoEditOperation?.trim();
if (operation === "reference_to_video") {
return images.map((image) => ({ image, role: "reference_image" }));
}
const startFrameId = options?.videoStartFrameNodeId?.trim();
const endFrameId = options?.videoEndFrameNodeId?.trim();
if (startFrameId || endFrameId) {
return images.map((image) => ({
image,
role: image.id === startFrameId ? "first_frame" : image.id === endFrameId ? "last_frame" : "reference_image",
}));
}
if (context.videoCount || context.audioCount) {
return images.map((image) => ({ image, role: "reference_image" }));
}
// 传统创作页没有帧节点 ID：保留单图首帧、双图首尾帧的历史语义。
return images.map((image, index) => ({
image,
role: index === 0 ? "first_frame" : index === 1 && images.length === 2 ? "last_frame" : "reference_image",
}));
}
export function hasExplicitVideoFrames(options?: RequestOptions) {
return options?.videoEditOperation !== "reference_to_video" && Boolean(options?.videoStartFrameNodeId || options?.videoEndFrameNodeId);
}