fix(media): 音频任务 - 支持异步 Provider 协议与结果校验 (#146)

Closes #146
This commit is contained in:
ddCat committed 2026-08-05 19:38:04 +08:00
1 parent cd85a45936
commit 6ae5692849
7 files changed
+346 -20

No files matched your search

+1 -1
View File
@@ -189,7 +189,7 @@ func interfaceAllowsProxyPath(interfaceType model.ChannelInterfaceType, requestP
return requestPath == "/images/generations"
case model.ChannelInterfaceOpenAIAudio:
return requestPath == "/audio/speech"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo:
case model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo:
return false
default:
return true
+1
View File
@@ -65,6 +65,7 @@ const (
ChannelInterfaceVolcengineArkImage ChannelInterfaceType = "volcengine-ark-image"
ChannelInterfaceVolcengineJiMengImage ChannelInterfaceType = "volcengine-jimeng-image"
ChannelInterfaceOpenAIAudio ChannelInterfaceType = "openai-audio"
ChannelInterfaceAsyncAudio ChannelInterfaceType = "async-audio"
ChannelInterfaceNewAPIVideo ChannelInterfaceType = "newapi"
ChannelInterfaceNewAPIChannel1 ChannelInterfaceType = "newapi-channel-1"
ChannelInterfaceNewAPIChannel2 ChannelInterfaceType = "newapi-channel-2"
+1 -1
View File
@@ -733,7 +733,7 @@ func mergeChannelRequest(req ChannelRequest, channel model.ModelChannel) Channel
func validChannelInterfaceType(value model.ChannelInterfaceType) bool {
switch value {
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo:
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo:
return true
default:
return false
+1 -1
View File
@@ -408,7 +408,7 @@ func capabilityForProtocol(protocol model.ChannelInterfaceType) string {
switch protocol {
case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage:
return "image"
case model.ChannelInterfaceOpenAIAudio:
case model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio:
return "audio"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo:
return "video"
+208 -1
View File
@@ -735,13 +735,220 @@ func runAudioTask(ctx context.Context, input canvasGenerationInput) (map[string]
if input.Config.AudioInstructions != "" {
body["instructions"] = input.Config.AudioInstructions
}
if input.Config.InterfaceType == string(model.ChannelInterfaceAsyncAudio) {
return runAsyncAudioTask(ctx, input, body, format)
}
data, mimeType, err := postBinary(ctx, input.Config, "/audio/speech", body)
if err != nil {
return nil, err
}
mimeType, err = validateGeneratedAudio(mimeType, data, format)
if err != nil {
return nil, err
}
return map[string]interface{}{"mode": "audio", "audio": map[string]interface{}{"dataUrl": dataURL(mimeType, data), "mimeType": mimeType, "format": format}}, nil
}
func runAsyncAudioTask(ctx context.Context, input canvasGenerationInput, body map[string]interface{}, format string) (map[string]interface{}, error) {
id := resumedProviderRequestID(ctx)
var state map[string]interface{}
if id == "" {
if err := postJSON(ctx, input.Config, "/audio/tasks", body, &state); err != nil {
return nil, err
}
state = asyncAudioPayload(state)
id = firstNonEmptyString(stringField(state, "id"), stringField(state, "task_id"), stringField(state, "request_id"))
if id == "" {
return nil, errors.New("异步音频接口没有返回任务 ID")
}
if asyncAudioSucceeded(state) {
return asyncAudioResult(ctx, input.Config, id, state, format)
}
}
for deadline := providerPollingDeadline(ctx); time.Now().Before(deadline); {
state = map[string]interface{}{}
pollCtx := withProviderRequestKind(ctx, "poll")
if err := getJSON(pollCtx, input.Config, "/audio/tasks/"+url.PathEscape(id), &state); err != nil {
return nil, err
}
state = asyncAudioPayload(state)
if asyncAudioSucceeded(state) {
return asyncAudioResult(ctx, input.Config, id, state, format)
}
status := strings.ToLower(strings.TrimSpace(stringField(state, "status")))
if status == "failed" || status == "cancelled" || status == "canceled" || status == "expired" || status == "error" {
return nil, fmt.Errorf("异步音频生成失败(任务 %s):%s", id, asyncAudioErrorMessage(state))
}
if err := sleepContext(ctx, 2500*time.Millisecond); err != nil {
return nil, err
}
}
return nil, fmt.Errorf("异步音频生成超时(任务 %s)", id)
}
func asyncAudioPayload(payload map[string]interface{}) map[string]interface{} {
for _, key := range []string{"data", "result", "output"} {
if nested, ok := payload[key].(map[string]interface{}); ok {
for parentKey, parentValue := range payload {
if parentKey == "data" || parentKey == "result" || parentKey == "output" {
continue
}
if _, exists := nested[parentKey]; !exists {
nested[parentKey] = parentValue
}
}
return nested
}
}
return payload
}
func asyncAudioSucceeded(state map[string]interface{}) bool {
status := strings.ToLower(strings.TrimSpace(stringField(state, "status")))
done, _ := state["done"].(bool)
return done || status == "completed" || status == "succeeded" || status == "success" || status == "done" || (status == "" && asyncAudioResultURL(state) != "")
}
func asyncAudioResult(ctx context.Context, config providerConfig, id string, state map[string]interface{}, format string) (map[string]interface{}, error) {
resultURL := asyncAudioResultURL(state)
var data []byte
var mimeType string
var err error
if strings.HasPrefix(resultURL, "data:") {
mimeType, data, err = decodeProviderDataURL(resultURL)
if err == nil {
limit, limitErr := providerGeneratedFileLimit(ctx)
if limitErr != nil {
err = limitErr
} else if int64(len(data)) > limit {
err = fmt.Errorf("异步音频结果超过 %s 限制", formatStorageLimit(limit))
}
}
} else if isPublicMediaURL(resultURL) {
data, mimeType, err = getExternalBinary(withProviderRequestKind(ctx, "download"), resultURL)
} else {
data, mimeType, err = getBinary(withProviderRequestKind(ctx, "download"), config, "/audio/tasks/"+url.PathEscape(id)+"/content")
}
if err != nil {
return nil, fmt.Errorf("异步音频结果下载失败(任务 %s):%w", id, err)
}
mimeType, err = validateGeneratedAudio(mimeType, data, format)
if err != nil {
return nil, fmt.Errorf("异步音频结果无效(任务 %s):%w", id, err)
}
return map[string]interface{}{"mode": "audio", "audio": map[string]interface{}{"dataUrl": dataURL(mimeType, data), "mimeType": mimeType, "format": format}}, nil
}
func asyncAudioResultURL(state map[string]interface{}) string {
for _, key := range []string{"audio_url", "audioUrl", "result_url", "resultUrl", "output_url", "outputUrl", "url", "data"} {
if value := strings.TrimSpace(stringField(state, key)); strings.HasPrefix(value, "data:") || isPublicMediaURL(value) {
return value
}
}
for _, key := range []string{"audio", "data", "result", "output"} {
if nested, ok := state[key].(map[string]interface{}); ok {
if value := asyncAudioResultURL(nested); value != "" {
return value
}
}
}
return ""
}
func asyncAudioErrorMessage(state map[string]interface{}) string {
_, message := providerFailureDetails(state)
return defaultString(message, firstNonEmptyString(stringField(state, "message"), "上游返回失败状态"))
}
func decodeProviderDataURL(value string) (string, []byte, error) {
header, encoded, ok := strings.Cut(value, ",")
if !ok || !strings.HasPrefix(header, "data:") || !strings.HasSuffix(strings.ToLower(header), ";base64") {
return "", nil, errors.New("音频 data URL 格式无效")
}
mimeType := strings.TrimSuffix(strings.TrimPrefix(header, "data:"), ";base64")
data, err := base64.StdEncoding.DecodeString(encoded)
return mimeType, data, err
}
func providerGeneratedFileLimit(ctx context.Context) (int64, error) {
metadata, ok := ctx.Value(providerAnalyticsKey{}).(providerAnalyticsContext)
if !ok || metadata.Service == nil {
return maxProviderResponseBytes, nil
}
policy, err := metadata.Service.RuntimePolicy()
if err != nil {
return 0, fmt.Errorf("读取生成资源限制失败:%w", err)
}
return megabytes(policy.Resource.GeneratedFileMB), nil
}
func validateGeneratedAudio(declared string, data []byte, format string) (string, error) {
if len(data) == 0 {
return "", errors.New("音频内容为空")
}
detected := strings.ToLower(strings.TrimSpace(strings.Split(http.DetectContentType(data), ";")[0]))
if strings.Contains(detected, "json") || strings.HasPrefix(detected, "text/") || strings.HasPrefix(detected, "image/") || strings.HasPrefix(detected, "video/") {
return "", fmt.Errorf("上游返回了非音频内容:%s", detected)
}
mimeType := strings.ToLower(strings.TrimSpace(strings.Split(declared, ";")[0]))
resolved := ""
if strings.HasPrefix(mimeType, "audio/") {
resolved = mimeType
} else if strings.HasPrefix(detected, "audio/") {
resolved = detected
} else if fallback := audioFormatMimeType(format); fallback != "" && (mimeType == "" || mimeType == "application/octet-stream") {
resolved = fallback
}
if resolved == "" {
return "", fmt.Errorf("上游响应类型不是音频:%s", defaultString(mimeType, detected))
}
if !audioSignatureMatches(resolved, data) {
return "", fmt.Errorf("音频内容与格式不匹配:%s", resolved)
}
return resolved, nil
}
func audioSignatureMatches(mimeType string, data []byte) bool {
if strings.Contains(mimeType, "pcm") || mimeType == "audio/l16" {
return len(data) > 0
}
if strings.Contains(mimeType, "mpeg") || strings.Contains(mimeType, "mp3") {
return bytes.HasPrefix(data, []byte("ID3")) || (len(data) >= 2 && data[0] == 0xff && data[1]&0xe0 == 0xe0)
}
if strings.Contains(mimeType, "wav") || strings.Contains(mimeType, "wave") {
return len(data) >= 12 && bytes.Equal(data[:4], []byte("RIFF")) && bytes.Equal(data[8:12], []byte("WAVE"))
}
if strings.Contains(mimeType, "opus") || strings.Contains(mimeType, "ogg") {
return bytes.HasPrefix(data, []byte("OggS"))
}
if strings.Contains(mimeType, "flac") {
return bytes.HasPrefix(data, []byte("fLaC"))
}
if strings.Contains(mimeType, "aac") {
return bytes.HasPrefix(data, []byte("ADIF")) || (len(data) >= 2 && data[0] == 0xff && data[1]&0xf0 == 0xf0)
}
return false
}
func audioFormatMimeType(format string) string {
switch strings.ToLower(strings.TrimSpace(format)) {
case "wav":
return "audio/wav"
case "opus":
return "audio/opus"
case "aac":
return "audio/aac"
case "flac":
return "audio/flac"
case "pcm":
return "audio/pcm"
case "mp3":
return "audio/mpeg"
default:
return ""
}
}
func runVideoTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) {
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineJiMengVideo) {
return runVolcengineJiMengVideoTask(ctx, input)
@@ -1413,7 +1620,7 @@ func validateGenerationInterface(mode string, interfaceType string) error {
"text": {"chat-completion": true, "openai-response": true},
"image": {"openai-image": true, "volcengine-ark-image": true, "volcengine-jimeng-image": true},
"video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true, "xai-video": true, "volcengine-ark-video": true, "volcengine-jimeng-video": true, "gemini-veo": true},
"audio": {"openai-audio": true},
"audio": {"openai-audio": true, "async-audio": true},
}
if allowed[mode] != nil && !allowed[mode][interfaceType] {
return fmt.Errorf("接口类型 %s 不支持%s生成", interfaceType, mode)