From d0f4d38c6a963e0979164428710e9ca4793372a7 Mon Sep 17 00:00:00 2001 From: ddCat <2695092045@qq.com> Date: Wed, 5 Aug 2026 19:38:04 +0800 Subject: [PATCH] =?UTF-8?q?fix(media):=20=E9=9F=B3=E9=A2=91=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=20-=20=E6=94=AF=E6=8C=81=E5=BC=82=E6=AD=A5=20Provider?= =?UTF-8?q?=20=E5=8D=8F=E8=AE=AE=E4=B8=8E=E7=BB=93=E6=9E=9C=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=20(#146)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #146 --- backend/internal/handler/security.go | 2 +- backend/internal/model/models.go | 1 + backend/internal/service/admin.go | 2 +- backend/internal/service/channel_models.go | 2 +- backend/internal/service/provider.go | 209 ++++++++++++++++++++- web/src/lib/model-protocols.ts | 24 ++- web/src/services/api/audio.ts | 126 +++++++++++-- 7 files changed, 346 insertions(+), 20 deletions(-) diff --git a/backend/internal/handler/security.go b/backend/internal/handler/security.go index 056d75f83..f86b805b9 100644 --- a/backend/internal/handler/security.go +++ b/backend/internal/handler/security.go @@ -189,7 +189,7 @@ func interfaceAllowsProxyPath(interfaceType model.ChannelInterfaceType, requestP return requestPath == "/images/generations" case model.ChannelInterfaceOpenAIAudio: return requestPath == "/audio/speech" - case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo: + case model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo: return false default: return true diff --git a/backend/internal/model/models.go b/backend/internal/model/models.go index 4abce0eb3..787825af5 100644 --- a/backend/internal/model/models.go +++ b/backend/internal/model/models.go @@ -65,6 +65,7 @@ const ( ChannelInterfaceVolcengineArkImage ChannelInterfaceType = "volcengine-ark-image" ChannelInterfaceVolcengineJiMengImage ChannelInterfaceType = "volcengine-jimeng-image" ChannelInterfaceOpenAIAudio ChannelInterfaceType = "openai-audio" + ChannelInterfaceAsyncAudio ChannelInterfaceType = "async-audio" ChannelInterfaceNewAPIVideo ChannelInterfaceType = "newapi" ChannelInterfaceNewAPIChannel1 ChannelInterfaceType = "newapi-channel-1" ChannelInterfaceNewAPIChannel2 ChannelInterfaceType = "newapi-channel-2" diff --git a/backend/internal/service/admin.go b/backend/internal/service/admin.go index a1b87d355..66cfbc271 100644 --- a/backend/internal/service/admin.go +++ b/backend/internal/service/admin.go @@ -733,7 +733,7 @@ func mergeChannelRequest(req ChannelRequest, channel model.ModelChannel) Channel func validChannelInterfaceType(value model.ChannelInterfaceType) bool { switch value { - case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo: + case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo: return true default: return false diff --git a/backend/internal/service/channel_models.go b/backend/internal/service/channel_models.go index 7d0269ca8..60e9b0c1c 100644 --- a/backend/internal/service/channel_models.go +++ b/backend/internal/service/channel_models.go @@ -408,7 +408,7 @@ func capabilityForProtocol(protocol model.ChannelInterfaceType) string { switch protocol { case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage: return "image" - case model.ChannelInterfaceOpenAIAudio: + case model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio: return "audio" case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo: return "video" diff --git a/backend/internal/service/provider.go b/backend/internal/service/provider.go index 3c3e56775..168f63025 100644 --- a/backend/internal/service/provider.go +++ b/backend/internal/service/provider.go @@ -735,13 +735,220 @@ func runAudioTask(ctx context.Context, input canvasGenerationInput) (map[string] if input.Config.AudioInstructions != "" { body["instructions"] = input.Config.AudioInstructions } + if input.Config.InterfaceType == string(model.ChannelInterfaceAsyncAudio) { + return runAsyncAudioTask(ctx, input, body, format) + } data, mimeType, err := postBinary(ctx, input.Config, "/audio/speech", body) if err != nil { return nil, err } + mimeType, err = validateGeneratedAudio(mimeType, data, format) + if err != nil { + return nil, err + } return map[string]interface{}{"mode": "audio", "audio": map[string]interface{}{"dataUrl": dataURL(mimeType, data), "mimeType": mimeType, "format": format}}, nil } +func runAsyncAudioTask(ctx context.Context, input canvasGenerationInput, body map[string]interface{}, format string) (map[string]interface{}, error) { + id := resumedProviderRequestID(ctx) + var state map[string]interface{} + if id == "" { + if err := postJSON(ctx, input.Config, "/audio/tasks", body, &state); err != nil { + return nil, err + } + state = asyncAudioPayload(state) + id = firstNonEmptyString(stringField(state, "id"), stringField(state, "task_id"), stringField(state, "request_id")) + if id == "" { + return nil, errors.New("异步音频接口没有返回任务 ID") + } + if asyncAudioSucceeded(state) { + return asyncAudioResult(ctx, input.Config, id, state, format) + } + } + for deadline := providerPollingDeadline(ctx); time.Now().Before(deadline); { + state = map[string]interface{}{} + pollCtx := withProviderRequestKind(ctx, "poll") + if err := getJSON(pollCtx, input.Config, "/audio/tasks/"+url.PathEscape(id), &state); err != nil { + return nil, err + } + state = asyncAudioPayload(state) + if asyncAudioSucceeded(state) { + return asyncAudioResult(ctx, input.Config, id, state, format) + } + status := strings.ToLower(strings.TrimSpace(stringField(state, "status"))) + if status == "failed" || status == "cancelled" || status == "canceled" || status == "expired" || status == "error" { + return nil, fmt.Errorf("异步音频生成失败（任务 %s）：%s", id, asyncAudioErrorMessage(state)) + } + if err := sleepContext(ctx, 2500*time.Millisecond); err != nil { + return nil, err + } + } + return nil, fmt.Errorf("异步音频生成超时（任务 %s）", id) +} + +func asyncAudioPayload(payload map[string]interface{}) map[string]interface{} { + for _, key := range []string{"data", "result", "output"} { + if nested, ok := payload[key].(map[string]interface{}); ok { + for parentKey, parentValue := range payload { + if parentKey == "data" || parentKey == "result" || parentKey == "output" { + continue + } + if _, exists := nested[parentKey]; !exists { + nested[parentKey] = parentValue + } + } + return nested + } + } + return payload +} + +func asyncAudioSucceeded(state map[string]interface{}) bool { + status := strings.ToLower(strings.TrimSpace(stringField(state, "status"))) + done, _ := state["done"].(bool) + return done || status == "completed" || status == "succeeded" || status == "success" || status == "done" || (status == "" && asyncAudioResultURL(state) != "") +} + +func asyncAudioResult(ctx context.Context, config providerConfig, id string, state map[string]interface{}, format string) (map[string]interface{}, error) { + resultURL := asyncAudioResultURL(state) + var data []byte + var mimeType string + var err error + if strings.HasPrefix(resultURL, "data:") { + mimeType, data, err = decodeProviderDataURL(resultURL) + if err == nil { + limit, limitErr := providerGeneratedFileLimit(ctx) + if limitErr != nil { + err = limitErr + } else if int64(len(data)) > limit { + err = fmt.Errorf("异步音频结果超过 %s 限制", formatStorageLimit(limit)) + } + } + } else if isPublicMediaURL(resultURL) { + data, mimeType, err = getExternalBinary(withProviderRequestKind(ctx, "download"), resultURL) + } else { + data, mimeType, err = getBinary(withProviderRequestKind(ctx, "download"), config, "/audio/tasks/"+url.PathEscape(id)+"/content") + } + if err != nil { + return nil, fmt.Errorf("异步音频结果下载失败（任务 %s）：%w", id, err) + } + mimeType, err = validateGeneratedAudio(mimeType, data, format) + if err != nil { + return nil, fmt.Errorf("异步音频结果无效（任务 %s）：%w", id, err) + } + return map[string]interface{}{"mode": "audio", "audio": map[string]interface{}{"dataUrl": dataURL(mimeType, data), "mimeType": mimeType, "format": format}}, nil +} + +func asyncAudioResultURL(state map[string]interface{}) string { + for _, key := range []string{"audio_url", "audioUrl", "result_url", "resultUrl", "output_url", "outputUrl", "url", "data"} { + if value := strings.TrimSpace(stringField(state, key)); strings.HasPrefix(value, "data:") || isPublicMediaURL(value) { + return value + } + } + for _, key := range []string{"audio", "data", "result", "output"} { + if nested, ok := state[key].(map[string]interface{}); ok { + if value := asyncAudioResultURL(nested); value != "" { + return value + } + } + } + return "" +} + +func asyncAudioErrorMessage(state map[string]interface{}) string { + _, message := providerFailureDetails(state) + return defaultString(message, firstNonEmptyString(stringField(state, "message"), "上游返回失败状态")) +} + +func decodeProviderDataURL(value string) (string, []byte, error) { + header, encoded, ok := strings.Cut(value, ",") + if !ok || !strings.HasPrefix(header, "data:") || !strings.HasSuffix(strings.ToLower(header), ";base64") { + return "", nil, errors.New("音频 data URL 格式无效") + } + mimeType := strings.TrimSuffix(strings.TrimPrefix(header, "data:"), ";base64") + data, err := base64.StdEncoding.DecodeString(encoded) + return mimeType, data, err +} + +func providerGeneratedFileLimit(ctx context.Context) (int64, error) { + metadata, ok := ctx.Value(providerAnalyticsKey{}).(providerAnalyticsContext) + if !ok || metadata.Service == nil { + return maxProviderResponseBytes, nil + } + policy, err := metadata.Service.RuntimePolicy() + if err != nil { + return 0, fmt.Errorf("读取生成资源限制失败：%w", err) + } + return megabytes(policy.Resource.GeneratedFileMB), nil +} + +func validateGeneratedAudio(declared string, data []byte, format string) (string, error) { + if len(data) == 0 { + return "", errors.New("音频内容为空") + } + detected := strings.ToLower(strings.TrimSpace(strings.Split(http.DetectContentType(data), ";")[0])) + if strings.Contains(detected, "json") || strings.HasPrefix(detected, "text/") || strings.HasPrefix(detected, "image/") || strings.HasPrefix(detected, "video/") { + return "", fmt.Errorf("上游返回了非音频内容：%s", detected) + } + mimeType := strings.ToLower(strings.TrimSpace(strings.Split(declared, ";")[0])) + resolved := "" + if strings.HasPrefix(mimeType, "audio/") { + resolved = mimeType + } else if strings.HasPrefix(detected, "audio/") { + resolved = detected + } else if fallback := audioFormatMimeType(format); fallback != "" && (mimeType == "" || mimeType == "application/octet-stream") { + resolved = fallback + } + if resolved == "" { + return "", fmt.Errorf("上游响应类型不是音频：%s", defaultString(mimeType, detected)) + } + if !audioSignatureMatches(resolved, data) { + return "", fmt.Errorf("音频内容与格式不匹配：%s", resolved) + } + return resolved, nil +} + +func audioSignatureMatches(mimeType string, data []byte) bool { + if strings.Contains(mimeType, "pcm") || mimeType == "audio/l16" { + return len(data) > 0 + } + if strings.Contains(mimeType, "mpeg") || strings.Contains(mimeType, "mp3") { + return bytes.HasPrefix(data, []byte("ID3")) || (len(data) >= 2 && data[0] == 0xff && data[1]&0xe0 == 0xe0) + } + if strings.Contains(mimeType, "wav") || strings.Contains(mimeType, "wave") { + return len(data) >= 12 && bytes.Equal(data[:4], []byte("RIFF")) && bytes.Equal(data[8:12], []byte("WAVE")) + } + if strings.Contains(mimeType, "opus") || strings.Contains(mimeType, "ogg") { + return bytes.HasPrefix(data, []byte("OggS")) + } + if strings.Contains(mimeType, "flac") { + return bytes.HasPrefix(data, []byte("fLaC")) + } + if strings.Contains(mimeType, "aac") { + return bytes.HasPrefix(data, []byte("ADIF")) || (len(data) >= 2 && data[0] == 0xff && data[1]&0xf0 == 0xf0) + } + return false +} + +func audioFormatMimeType(format string) string { + switch strings.ToLower(strings.TrimSpace(format)) { + case "wav": + return "audio/wav" + case "opus": + return "audio/opus" + case "aac": + return "audio/aac" + case "flac": + return "audio/flac" + case "pcm": + return "audio/pcm" + case "mp3": + return "audio/mpeg" + default: + return "" + } +} + func runVideoTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) { if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineJiMengVideo) { return runVolcengineJiMengVideoTask(ctx, input) @@ -1413,7 +1620,7 @@ func validateGenerationInterface(mode string, interfaceType string) error { "text": {"chat-completion": true, "openai-response": true}, "image": {"openai-image": true, "volcengine-ark-image": true, "volcengine-jimeng-image": true}, "video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true, "xai-video": true, "volcengine-ark-video": true, "volcengine-jimeng-video": true, "gemini-veo": true}, - "audio": {"openai-audio": true}, + "audio": {"openai-audio": true, "async-audio": true}, } if allowed[mode] != nil && !allowed[mode][interfaceType] { return fmt.Errorf("接口类型 %s 不支持%s生成", interfaceType, mode) diff --git a/web/src/lib/model-protocols.ts b/web/src/lib/model-protocols.ts index bf94034b2..e2ccdb403 100644 --- a/web/src/lib/model-protocols.ts +++ b/web/src/lib/model-protocols.ts @@ -5,6 +5,7 @@ export type ModelProtocol = | "volcengine-ark-image" | "volcengine-jimeng-image" | "openai-audio" + | "async-audio" | "newapi" | "newapi-channel-1" | "newapi-channel-2" @@ -32,11 +33,28 @@ export const MODEL_PROTOCOLS: ModelProtocolDefinition[] = [ { value: "volcengine-ark-image", label: "火山方舟图片", capability: "image", create: "POST /api/v3/images/generations", contentType: "application/json", media: "文生图与 image 参考图，不支持蒙版" }, { value: "volcengine-jimeng-image", label: "即梦官方图片", capability: "image", create: "POST CVSync2AsyncSubmitTask", poll: "POST CVSync2AsyncGetResult", contentType: "application/json + AK/SK 签名", media: "0-14 张参考图，模型标识填写 req_key" }, { value: "openai-audio", label: "OpenAI Audio", capability: "audio", create: "POST /v1/audio/speech", contentType: "application/json", media: "文本转语音" }, + { value: "async-audio", label: "异步音频任务", capability: "audio", create: "POST /v1/audio/tasks", poll: "GET /v1/audio/tasks/{task_id}", contentType: "application/json", media: "语音、音效与音乐生成" }, { value: "newapi", label: "OpenAI / NewAPI Videos", capability: "video", create: "POST /v1/videos", poll: "GET /v1/videos/{task_id}", contentType: "multipart/form-data", media: "input_reference[] 参考图" }, { value: "newapi-channel-1", label: "NewAPI 媒体任务", capability: "video", create: "POST /v1/videos", poll: "GET /v1/videos/{task_id}", contentType: "application/json", media: "图片、视频、音频公网 URL" }, - { value: "newapi-channel-2", label: "NewAPI Video Generations", capability: "video", create: "POST /v1/video/generations", poll: "GET /v1/video/generations/{task_id}", contentType: "application/json", media: "image_urls（首帧、尾帧、其他参考图） / video_urls / audio_urls" }, + { + value: "newapi-channel-2", + label: "NewAPI Video Generations", + capability: "video", + create: "POST /v1/video/generations", + poll: "GET /v1/video/generations/{task_id}", + contentType: "application/json", + media: "image_urls（首帧、尾帧、其他参考图） / video_urls / audio_urls", + }, { value: "xai-video", label: "xAI 官方视频", capability: "video", create: "POST /v1/videos/generations", poll: "GET /v1/videos/{request_id}", contentType: "application/json", media: "单张起始图" }, - { value: "volcengine-ark-video", label: "火山方舟视频", capability: "video", create: "POST /api/v3/contents/generations/tasks", poll: "GET /api/v3/contents/generations/tasks/{task_id}", contentType: "application/json", media: "图片、视频、音频参考素材" }, + { + value: "volcengine-ark-video", + label: "火山方舟视频", + capability: "video", + create: "POST /api/v3/contents/generations/tasks", + poll: "GET /api/v3/contents/generations/tasks/{task_id}", + contentType: "application/json", + media: "图片、视频、音频参考素材", + }, { value: "volcengine-jimeng-video", label: "即梦官方视频", capability: "video", create: "POST CVSync2AsyncSubmitTask", poll: "POST CVSync2AsyncGetResult", contentType: "application/json + AK/SK 签名", media: "文本或一张首帧图，模型标识填写 req_key" }, { value: "gemini-veo", label: "Gemini Veo", capability: "video", create: "POST /v1beta/models/{model}:predictLongRunning", poll: "GET /v1beta/{operation_name}", contentType: "application/json", media: "文本与单张起始图" }, ]; @@ -73,7 +91,7 @@ export function modelProtocolSummary(value?: string) { } export function normalizeModelProtocol(value: unknown): ModelProtocol | undefined { - return typeof value === "string" && MODEL_PROTOCOLS.some((item) => item.value === value) ? value as ModelProtocol : undefined; + return typeof value === "string" && MODEL_PROTOCOLS.some((item) => item.value === value) ? (value as ModelProtocol) : undefined; } function protocolOptions(capability: ProtocolCapability) { diff --git a/web/src/services/api/audio.ts b/web/src/services/api/audio.ts index 317c6259e..b8bb890fa 100644 --- a/web/src/services/api/audio.ts +++ b/web/src/services/api/audio.ts @@ -26,21 +26,21 @@ export async function requestAudioGeneration(config: AiConfig, prompt: string, o assertAudioConfig(requestConfig, model); const format = normalizeAudioFormatValue(config.audioFormat); const instructions = config.audioInstructions.trim(); + const payload = { + model, + input: prompt, + voice: normalizeAudioVoiceValue(config.audioVoice), + response_format: format, + speed: Number(normalizeAudioSpeedValue(config.audioSpeed)), + ...(instructions ? { instructions } : {}), + }; try { + if (requestConfig.interfaceType === "async-audio") { + return await requestAsyncAudioGeneration(requestConfig, payload, format, options); + } const request = channelRequest(requestConfig, aiApiUrl(requestConfig, "/audio/speech"), aiHeaders(requestConfig)); - const response = await axios.post( - request.url, - { - model, - input: prompt, - voice: normalizeAudioVoiceValue(config.audioVoice), - response_format: format, - speed: Number(normalizeAudioSpeedValue(config.audioSpeed)), - ...(instructions ? { instructions } : {}), - }, - { headers: request.headers, withCredentials: request.credentials === "include", responseType: "blob", signal: options?.signal }, - ); + const response = await axios.post(request.url, payload, { headers: request.headers, withCredentials: request.credentials === "include", responseType: "blob", signal: options?.signal }); await assertAudioBlob(response.data); return response.data.type.startsWith("audio/") ? response.data : new Blob([response.data], { type: audioMimeType(format) }); } catch (error) { @@ -48,6 +48,104 @@ export async function requestAudioGeneration(config: AiConfig, prompt: string, o } } +async function requestAsyncAudioGeneration(config: AiConfig, payload: Record, format: string, options?: RequestOptions) { + const createRequest = channelRequest(config, aiApiUrl(config, "/audio/tasks"), aiHeaders(config)); + const created = await axios.post>(createRequest.url, payload, { headers: createRequest.headers, withCredentials: createRequest.credentials === "include", signal: options?.signal }); + let state = asyncAudioPayload(created.data); + const taskId = asyncAudioTaskId(state); + if (!taskId) throw new Error("异步音频接口没有返回任务 ID"); + for (let attempt = 0; attempt < 120; attempt += 1) { + if (asyncAudioSucceeded(state)) return downloadAsyncAudio(config, taskId, state, format, options); + const status = String(state.status || "").toLowerCase(); + if (["failed", "cancelled", "canceled", "expired", "error"].includes(status)) { + throw new Error(asyncAudioError(state)); + } + await waitForAudioPoll(options?.signal); + const pollRequest = channelRequest(config, aiApiUrl(config, `/audio/tasks/${encodeURIComponent(taskId)}`), aiHeaders(config)); + const polled = await axios.get>(pollRequest.url, { headers: pollRequest.headers, withCredentials: pollRequest.credentials === "include", signal: options?.signal }); + state = asyncAudioPayload(polled.data); + } + throw new Error(`异步音频生成超时（任务 ${taskId}）`); +} + +function asyncAudioPayload(payload: Record): Record { + for (const key of ["data", "result", "output"]) { + const nested = payload[key]; + if (nested && typeof nested === "object" && !Array.isArray(nested)) return { ...payload, ...(nested as Record) }; + } + return payload; +} + +function asyncAudioTaskId(payload: Record) { + for (const value of [payload.id, payload.task_id, payload.request_id]) { + if (typeof value === "string" && value.trim()) return value.trim(); + } + return undefined; +} + +function asyncAudioSucceeded(payload: Record) { + const status = String(payload.status || "").toLowerCase(); + return payload.done === true || ["completed", "succeeded", "success", "done"].includes(status) || (!status && Boolean(asyncAudioResultUrl(payload))); +} + +function asyncAudioResultUrl(payload: Record): string { + for (const key of ["audio_url", "audioUrl", "result_url", "resultUrl", "output_url", "outputUrl", "url", "data"]) { + const value = payload[key]; + if (typeof value === "string" && (/^https?:\/\//i.test(value) || value.startsWith("data:audio/"))) return value; + } + for (const key of ["audio", "data", "result", "output"]) { + const value = payload[key]; + if (value && typeof value === "object" && !Array.isArray(value)) { + const nested = asyncAudioResultUrl(value as Record); + if (nested) return nested; + } + } + return ""; +} + +async function downloadAsyncAudio(config: AiConfig, taskId: string, state: Record, format: string, options?: RequestOptions) { + const resultUrl = asyncAudioResultUrl(state); + let blob: Blob; + if (resultUrl.startsWith("data:audio/")) { + blob = await (await fetch(resultUrl, { signal: options?.signal })).blob(); + } else if (/^https?:\/\//i.test(resultUrl)) { + blob = (await axios.get(resultUrl, { responseType: "blob", signal: options?.signal })).data; + } else { + const contentRequest = channelRequest(config, aiApiUrl(config, `/audio/tasks/${encodeURIComponent(taskId)}/content`), aiHeaders(config)); + blob = (await axios.get(contentRequest.url, { headers: contentRequest.headers, withCredentials: contentRequest.credentials === "include", responseType: "blob", signal: options?.signal })).data; + } + await assertAudioBlob(blob); + return blob.type.startsWith("audio/") ? blob : new Blob([blob], { type: audioMimeType(format) }); +} + +function asyncAudioError(payload: Record) { + const error = payload.error; + if (typeof error === "string" && error.trim()) return error; + if (error && typeof error === "object" && !Array.isArray(error)) { + const message = (error as Record).message; + if (typeof message === "string" && message.trim()) return message; + } + return typeof payload.message === "string" && payload.message.trim() ? payload.message : "异步音频生成失败"; +} + +function waitForAudioPoll(signal?: AbortSignal) { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new DOMException("Aborted", "AbortError")); + return; + } + const onAbort = () => { + window.clearTimeout(timer); + reject(new DOMException("Aborted", "AbortError")); + }; + const timer = window.setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, 2500); + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + export async function storeGeneratedAudio(blob: Blob, format = "mp3"): Promise { const audio = blob.type.startsWith("audio/") ? blob : new Blob([blob], { type: audioMimeType(format) }); return uploadMediaFile(audio, "audio"); @@ -61,7 +159,9 @@ function assertAudioConfig(config: AiConfig, model: string) { } async function assertAudioBlob(blob: Blob) { - if (!blob.type.includes("json")) return; + const mimeType = blob.type.toLowerCase(); + if (mimeType.startsWith("image/") || mimeType.startsWith("video/") || mimeType.startsWith("text/")) throw new Error(`上游返回了非音频内容：${mimeType}`); + if (!mimeType.includes("json")) return; let payload: { code?: number; msg?: string; error?: { message?: string } }; try { payload = JSON.parse(await blob.text()) as { code?: number; msg?: string; error?: { message?: string } };