feat(platform): 开源发布 - 初始化 Open AI Canvas

This commit is contained in:
ddCat
2026-07-22 14:18:14 +08:00
commit aea35896eb
365 changed files with 66410 additions and 0 deletions

View File

@@ -0,0 +1,412 @@
import axios from "axios";
import { dataUrlToFile } from "@/lib/image-utils";
import { getMediaBlob, uploadMediaFile, type UploadedFile } from "@/services/file-storage";
import { imageToDataUrl } from "@/services/image-storage";
import { boolConfig, buildSeedancePromptText, isArkPlanBaseUrl, isSeedanceVideoConfig, normalizeSeedanceDuration, normalizeSeedanceRatio, normalizeSeedanceResolution, seedanceVideoReferenceError, SEEDANCE_REFERENCE_LIMITS } from "@/lib/seedance-video";
import { buildApiUrl, isSystemProxyBaseUrl, modelOptionName, resolveModelRequestConfig, type AiConfig } from "@/stores/use-config-store";
import type { ReferenceImage } from "@/types/image";
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
type VideoResponse = { id: string; status?: string; error?: { message?: string } };
type ApiVideoResponse = VideoResponse | { code?: number; data?: VideoResponse | null; msg?: string };
type SeedanceTask = {
id: string;
task_id?: string;
status?: "queued" | "running" | "succeeded" | "completed" | "failed" | "cancelled" | "expired";
error?: { code?: string; message?: string } | null;
error_code?: string | null;
video_url?: string | null;
content?: { video_url?: string; last_frame_url?: string } | null;
};
type ApiEnvelope<T> = T | { code?: number; data?: T | null; msg?: string };
type RequestOptions = { signal?: AbortSignal };
export type VideoGenerationResult = { blob?: Blob; url?: string; mimeType?: string };
export type VideoGenerationTask = { id: string; provider: "openai" | "seedance"; model: string };
export type VideoGenerationTaskState = { status: "pending" } | { status: "completed"; result: VideoGenerationResult } | { status: "failed"; error: string };
function aiApiUrl(config: AiConfig, path: string) {
return buildApiUrl(config.baseUrl, path);
}
function aiHeaders(config: AiConfig, contentType?: string) {
return {
Authorization: `Bearer ${config.apiKey}`,
...(contentType ? { "Content-Type": contentType } : {}),
...(isSystemProxyBaseUrl(config.baseUrl) ? { "X-Canvas-Scene": "video", "X-Idempotency-Key": crypto.randomUUID() } : {}),
};
}
export async function requestVideoGeneration(config: AiConfig, prompt: string, references: ReferenceImage[] = [], videoReferences: ReferenceVideo[] = [], audioReferences: ReferenceAudio[] = [], options?: RequestOptions): Promise<VideoGenerationResult> {
const task = await createVideoGenerationTask(config, prompt, references, videoReferences, audioReferences, options);
const delayMs = task.provider === "seedance" ? 5000 : 2500;
for (let attempt = 0; attempt < 120; attempt += 1) {
if (options?.signal?.aborted) throw new DOMException("Aborted", "AbortError");
const state = await pollVideoGenerationTask(config, task, options);
if (state.status === "completed") return state.result;
if (state.status === "failed") throw new Error(state.error);
if (attempt === 119) throw new Error(`${task.provider === "seedance" ? "Seedance " : ""}视频生成超时,请稍后重试`);
await delay(delayMs, options?.signal);
}
throw new Error("视频生成超时,请稍后重试");
}
export async function createVideoGenerationTask(config: AiConfig, prompt: string, references: ReferenceImage[] = [], videoReferences: ReferenceVideo[] = [], audioReferences: ReferenceAudio[] = [], options?: RequestOptions): Promise<VideoGenerationTask> {
const selectedModel = (config.model || config.videoModel).trim();
const requestConfig = resolveModelRequestConfig(config, selectedModel);
assertVideoConfig(requestConfig, requestConfig.model);
if (isSeedanceVideoConfig(requestConfig)) {
return createSeedanceTask(requestConfig, selectedModel, prompt, references, videoReferences, audioReferences, options);
}
if (videoReferences.length || audioReferences.length) {
throw new Error("当前视频接口不支持参考视频或参考音频,请切换到 Seedance 2.0 / 火山 Agent Plan 模型,或移除参考素材");
}
return createOpenAIVideoTask(requestConfig, selectedModel, prompt, references, options);
}
export async function pollVideoGenerationTask(config: AiConfig, task: VideoGenerationTask, options?: RequestOptions): Promise<VideoGenerationTaskState> {
const requestConfig = resolveModelRequestConfig(config, task.model);
assertVideoConfig(requestConfig, requestConfig.model);
return task.provider === "seedance" ? pollSeedanceTask(requestConfig, task, options) : pollOpenAIVideoTask(requestConfig, task, options);
}
export async function storeGeneratedVideo(result: VideoGenerationResult): Promise<UploadedFile> {
if (result.blob) return uploadMediaFile(result.blob, "video");
if (result.url) return { url: result.url, storageKey: "", bytes: 0, mimeType: result.mimeType || "video/mp4" };
throw new Error("视频接口没有返回可播放的视频");
}
async function createOpenAIVideoTask(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], options?: RequestOptions): Promise<VideoGenerationTask> {
const modelName = modelOptionName(model);
if (modelName.toLowerCase().includes("grok")) {
const images = await Promise.all(references.slice(0, 7).map((image) => imageToDataUrl(image)));
const seconds = normalizeVideoSeconds(config.videoSeconds);
const payload = {
model: modelName,
prompt,
duration: Number.parseInt(seconds, 10) || 6,
seconds,
...(normalizeVideoSize(config.size) ? { size: normalizeVideoSize(config.size) } : {}),
...(images.length ? { image: images[0], images } : {}),
};
try {
const created = unwrapVideoResponse((await axios.post<ApiVideoResponse>(aiApiUrl(config, "/videos"), payload, { headers: aiHeaders(config, "application/json"), signal: options?.signal })).data);
if (!created.id) throw new Error("视频接口没有返回任务 ID");
return { id: created.id, provider: "openai", model };
} catch (error) {
throw new Error(readAxiosError(error, "视频任务创建失败"));
}
}
const body = new FormData();
body.append("model", modelName);
body.append("prompt", prompt);
body.append("seconds", normalizeVideoSeconds(config.videoSeconds));
if (normalizeVideoSize(config.size)) body.append("size", normalizeVideoSize(config.size)!);
body.append("resolution_name", normalizeVideoResolution(config.vquality));
body.append("preset", "normal");
const files = await Promise.all(references.slice(0, 7).map(async (image) => dataUrlToFile({ ...image, dataUrl: await imageToDataUrl(image) })));
files.forEach((file) => body.append("input_reference[]", file));
try {
const created = unwrapVideoResponse((await axios.post<ApiVideoResponse>(aiApiUrl(config, "/videos"), body, { headers: aiHeaders(config), signal: options?.signal })).data);
if (!created.id) throw new Error("视频接口没有返回任务 ID");
return { id: created.id, provider: "openai", model };
} catch (error) {
throw new Error(readAxiosError(error, "视频任务创建失败"));
}
}
async function pollOpenAIVideoTask(config: AiConfig, task: VideoGenerationTask, options?: RequestOptions): Promise<VideoGenerationTaskState> {
try {
const video = unwrapVideoResponse((await axios.get<ApiVideoResponse>(aiApiUrl(config, `/videos/${task.id}`), { headers: aiHeaders(config), signal: options?.signal })).data);
if (video.status === "completed") {
const content = await axios.get<Blob>(aiApiUrl(config, `/videos/${task.id}/content`), { headers: aiHeaders(config), responseType: "blob", signal: options?.signal });
await assertVideoBlob(content.data);
return { status: "completed", result: { blob: content.data } };
}
if (video.status === "failed" || video.status === "cancelled") return { status: "failed", error: video.error?.message || "视频生成失败" };
return { status: "pending" };
} catch (error) {
throw new Error(readAxiosError(error, "视频任务查询失败"));
}
}
async function createSeedanceTask(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[], options?: RequestOptions): Promise<VideoGenerationTask> {
assertSeedanceVideoReferences(videoReferences);
assertSeedanceAudioReferences(audioReferences);
const payload = isArkPlanBaseUrl(config.baseUrl) ? await buildSeedanceAgentPlanPayload(config, model, prompt, references, videoReferences, audioReferences) : await buildSeedanceVideosPayload(config, model, prompt, references, videoReferences, audioReferences);
try {
const created = unwrapSeedanceTask((await axios.post<ApiEnvelope<SeedanceTask>>(seedanceApiUrl(config), payload, { headers: aiHeaders(config, "application/json"), signal: options?.signal })).data);
const id = created.id || created.task_id;
if (!id) throw new Error("Seedance 接口没有返回任务 ID");
return { id, provider: "seedance", model };
} catch (error) {
throw new Error(readAxiosError(error, "Seedance 任务创建失败"));
}
}
async function pollSeedanceTask(config: AiConfig, task: VideoGenerationTask, options?: RequestOptions): Promise<VideoGenerationTaskState> {
try {
const state = unwrapSeedanceTask((await axios.get<ApiEnvelope<SeedanceTask>>(seedanceApiUrl(config, task.id), { headers: aiHeaders(config), signal: options?.signal })).data);
if (state.status === "succeeded" || state.status === "completed") {
const url = state.video_url || state.content?.video_url;
if (url) return { status: "completed", result: await videoResultFromUrl(url, options) };
if (isArkPlanBaseUrl(config.baseUrl)) return { status: "failed", error: "Seedance 任务成功但没有返回视频 URL" };
const content = await axios.get<Blob>(aiApiUrl(config, `/videos/${task.id}/content`), { headers: aiHeaders(config), responseType: "blob", signal: options?.signal });
await assertVideoBlob(content.data);
return { status: "completed", result: { blob: content.data } };
}
if (state.status === "failed" || state.status === "cancelled" || state.status === "expired") return { status: "failed", error: seedanceErrorMessage(state) || `Seedance 视频生成${state.status === "expired" ? "超时" : "失败"}` };
return { status: "pending" };
} catch (error) {
throw new Error(readAxiosError(error, "Seedance 任务查询失败"));
}
}
function assertSeedanceVideoReferences(videoReferences: ReferenceVideo[]) {
const error = seedanceVideoReferenceError(videoReferences);
if (error) throw new Error(error);
let total = 0;
for (const video of videoReferences) {
if (!video.durationMs) continue;
if (video.durationMs < 2000 || video.durationMs > 15000) throw new Error("Seedance 参考视频单个时长需要在 2-15 秒之间");
total += video.durationMs;
}
if (total > 15000) throw new Error("Seedance 参考视频总时长不能超过 15 秒");
}
function assertSeedanceAudioReferences(audioReferences: ReferenceAudio[]) {
let total = 0;
for (const audio of audioReferences) {
if (!audio.durationMs) continue;
if (audio.durationMs < 2000 || audio.durationMs > 15000) throw new Error("Seedance 参考音频单个时长需要在 2-15 秒之间");
total += audio.durationMs;
}
if (total > 15000) throw new Error("Seedance 参考音频总时长不能超过 15 秒");
}
function seedanceApiUrl(config: AiConfig, taskId?: string) {
if (isArkPlanBaseUrl(config.baseUrl)) return buildApiUrl(config.baseUrl, `/contents/generations/tasks${taskId ? `/${encodeURIComponent(taskId)}` : ""}`);
return buildApiUrl(config.baseUrl, `/videos${taskId ? `/${encodeURIComponent(taskId)}` : ""}`);
}
async function buildSeedanceAgentPlanPayload(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) {
if (audioReferences.length && !references.length && !videoReferences.length) {
throw new Error("Seedance 参考音频不能单独使用,请同时添加参考图或参考视频");
}
const content = await buildSeedanceContent(config, prompt, references, videoReferences, audioReferences);
if (!content.length) throw new Error("请输入视频提示词,或连接参考图片/视频/音频");
return {
model: modelOptionName(model),
content,
ratio: normalizeSeedanceRatio(config.size),
resolution: normalizeSeedanceResolution(config.vquality, modelOptionName(model)),
duration: normalizeSeedanceDuration(config.videoSeconds),
generate_audio: boolConfig(config.videoGenerateAudio, true),
watermark: boolConfig(config.videoWatermark, false),
};
}
async function buildSeedanceVideosPayload(config: AiConfig, model: string, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) {
if ((videoReferences.length || audioReferences.length) && !references.length) {
throw new Error("Seedance 参考视频或参考音频需要同时连接至少 1 张主参考图");
}
const imageUrls = await Promise.all(references.slice(0, SEEDANCE_REFERENCE_LIMITS.images).map(resolveSeedanceVideosImageUrl));
const videoUrls = await Promise.all(videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos).map(resolveSeedanceVideosMediaUrl));
const audioUrls = await Promise.all(audioReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.audios).map(resolveSeedanceVideosMediaUrl));
const ratio = normalizeSeedanceRatio(config.size);
const duration = normalizeSeedanceDuration(config.videoSeconds);
return {
model: modelOptionName(model),
prompt: buildSeedanceVideosPromptText(prompt, imageUrls.length, videoUrls.length, audioUrls.length),
aspect_ratio: ratio === "adaptive" ? "16:9" : ratio,
duration: duration < 4 ? 5 : duration,
...(imageUrls[0] ? { image_url: imageUrls[0] } : {}),
...(imageUrls.length > 1 ? { reference_image_urls: imageUrls.slice(1) } : {}),
...(videoUrls.length ? { reference_videos: videoUrls } : {}),
...(audioUrls.length ? { reference_audios: audioUrls } : {}),
};
}
async function buildSeedanceContent(config: AiConfig, prompt: string, references: ReferenceImage[], videoReferences: ReferenceVideo[], audioReferences: ReferenceAudio[]) {
const content: Array<Record<string, unknown>> = [];
const text = buildSeedancePromptText(prompt, references, videoReferences, audioReferences);
if (text) content.push({ type: "text", text });
for (const image of references.slice(0, SEEDANCE_REFERENCE_LIMITS.images)) {
content.push({ type: "image_url", image_url: { url: await resolveSeedanceImageUrl(config, image) }, role: "reference_image" });
}
for (const video of videoReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.videos)) {
content.push({ type: "video_url", video_url: { url: await resolveSeedanceVideoUrl(video) }, role: "reference_video" });
}
for (const audio of audioReferences.slice(0, SEEDANCE_REFERENCE_LIMITS.audios)) {
content.push({ type: "audio_url", audio_url: { url: await resolveSeedanceAudioUrl(audio) }, role: "reference_audio" });
}
return content;
}
function buildSeedanceVideosPromptText(prompt: string, _imageCount: number, _videoCount: number, _audioCount: number) {
return prompt.trim();
}
async function resolveSeedanceImageUrl(config: AiConfig, image: ReferenceImage) {
const directUrl = image.url || image.dataUrl;
if (isPublicMediaUrl(directUrl) || directUrl.startsWith("asset://")) return directUrl;
const dataUrl = await imageToDataUrl(image);
if (!dataUrl) throw new Error("参考图读取失败,请换一张图片或重新上传");
return dataUrl;
}
async function resolveSeedanceVideosImageUrl(image: ReferenceImage) {
const directUrl = image.url || image.dataUrl;
if (isPublicMediaUrl(directUrl) || directUrl.startsWith("data:")) return directUrl;
const dataUrl = await imageToDataUrl(image);
if (!dataUrl) throw new Error("参考图读取失败,请换一张图片或重新上传");
return dataUrl;
}
async function resolveSeedanceVideoUrl(video: ReferenceVideo) {
if (isPublicMediaUrl(video.url) || video.url.startsWith("asset://")) return video.url;
let blob: Blob | null = null;
if (video.storageKey) blob = await getMediaBlob(video.storageKey);
if (!blob && video.url?.startsWith("blob:")) blob = await (await fetch(video.url)).blob();
if (!blob) throw new Error("参考视频必须是公网 URL、素材 ID,或本地已保存的视频");
return blobToDataUrl(blob);
}
async function resolveSeedanceAudioUrl(audio: ReferenceAudio) {
if (isPublicMediaUrl(audio.url) || audio.url.startsWith("asset://")) return audio.url;
let blob: Blob | null = null;
if (audio.storageKey) blob = await getMediaBlob(audio.storageKey);
if (!blob && audio.url?.startsWith("blob:")) blob = await (await fetch(audio.url)).blob();
if (!blob) throw new Error("参考音频必须是公网 URL、素材 ID,或本地已保存的音频");
return blobToDataUrl(blob);
}
async function resolveSeedanceVideosMediaUrl(media: ReferenceVideo | ReferenceAudio) {
if (isPublicMediaUrl(media.url) || media.url?.startsWith("data:")) return media.url;
let blob: Blob | null = null;
if (media.storageKey) blob = await getMediaBlob(media.storageKey);
if (!blob && media.url?.startsWith("blob:")) blob = await (await fetch(media.url)).blob();
if (!blob) throw new Error("Seedance /videos 参考素材必须是公网 URL、data URL,或本地已保存素材");
return blobToDataUrl(blob);
}
async function videoResultFromUrl(url: string, options?: RequestOptions): Promise<VideoGenerationResult> {
try {
const response = await axios.get<Blob>(url, { responseType: "blob", signal: options?.signal });
await assertVideoBlob(response.data);
return { blob: response.data };
} catch (error) {
if (axios.isCancel(error) || options?.signal?.aborted) throw error;
return { url, mimeType: "video/mp4" };
}
}
function seedanceErrorMessage(state: SeedanceTask) {
if (state.error?.message && state.error.code) return `${state.error.code}:${state.error.message}`;
return state.error?.message || state.error_code || "";
}
function assertVideoConfig(config: AiConfig, model: string) {
if (!model) throw new Error("请先配置视频模型");
if (!config.baseUrl.trim()) throw new Error("请先配置 Base URL");
if (!config.apiKey.trim()) throw new Error("请先配置 API Key");
if (config.apiFormat === "gemini") throw new Error("Gemini 调用格式暂不支持视频生成,请使用 OpenAI 格式渠道");
}
function normalizeVideoSeconds(value: string) {
const seconds = Math.floor(Number(value) || 6);
return String(Math.max(1, Math.min(20, seconds)));
}
function normalizeVideoSize(value: string) {
if (value === "auto") return null;
const size = value || "1280x720";
if (/^\d+x\d+$/.test(size)) return size;
return ["9:16", "2:3", "3:4"].includes(size) ? "720x1280" : "1280x720";
}
function normalizeVideoResolution(value: string) {
if (value === "low") return "480p";
if (value === "auto" || value === "high" || value === "medium") return "720p";
const resolution = value.replace(/p$/i, "") || "720";
return `${resolution}p`;
}
function unwrapVideoResponse(payload: ApiVideoResponse) {
return unwrapEnvelope(payload, "接口没有返回视频任务");
}
function unwrapSeedanceTask(payload: ApiEnvelope<SeedanceTask>) {
return unwrapEnvelope(payload, "Seedance 接口没有返回任务");
}
function unwrapEnvelope<T>(payload: ApiEnvelope<T>, emptyMessage: string): T {
if (!payload) throw new Error(emptyMessage);
if (typeof payload === "object" && "code" in payload && typeof payload.code === "number") {
if (payload.code !== 0) throw new Error(payload.msg || "请求失败");
if (!payload.data) throw new Error(emptyMessage);
return payload.data;
}
return payload as T;
}
function readAxiosError(error: unknown, fallback: string) {
if (axios.isCancel(error)) return "请求已取消";
if (axios.isAxiosError<{ error?: { message?: string }; msg?: string; code?: number }>(error)) {
const responseData = error.response?.data;
return responseData?.msg || responseData?.error?.message || statusMessage(error.response?.status, fallback);
}
if (error instanceof DOMException && error.name === "AbortError") return "请求已取消";
return error instanceof Error ? error.message : fallback;
}
function statusMessage(status: number | undefined, fallback: string) {
if (status === 401 || status === 403) return "鉴权失败,请检查 API Key、套餐权限或模型权限";
if (status === 429) return "请求被限流或额度不足,请稍后重试";
return status ? `${fallback}(${status})` : fallback;
}
async function assertVideoBlob(blob: Blob) {
if (!blob.type.includes("json")) return;
let payload: { code?: number; msg?: string; error?: { message?: string } };
try {
payload = JSON.parse(await blob.text()) as { code?: number; msg?: string; error?: { message?: string } };
} catch {
return;
}
if (typeof payload.code === "number" && payload.code !== 0) throw new Error(payload.msg || "视频下载失败");
if (payload.error?.message) throw new Error(payload.error.message);
}
function isPublicMediaUrl(value: string) {
return /^https?:\/\//i.test(value || "");
}
function delay(ms: number, signal?: AbortSignal) {
return new Promise<void>((resolve, reject) => {
if (signal?.aborted) {
reject(new DOMException("Aborted", "AbortError"));
return;
}
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
reject(new DOMException("Aborted", "AbortError"));
},
{ once: true },
);
});
}
function blobToDataUrl(blob: Blob) {
return new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(String(reader.result || ""));
reader.onerror = () => reject(new Error("读取本地素材失败"));
reader.readAsDataURL(blob);
});
}