fix(*): 模型请求协议 - 支持模型独立选协议并修正视频多参考请求

This commit is contained in:
ddCat committed 2026-07-28 23:05:15 +08:00
1 parent 7093b4e161
commit e3c501620d
20 files changed
+691 -190

No files matched your search

+25 -20
View File
@@ -663,7 +663,9 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User,
switch channel.InterfaceType {
case model.ChannelInterfaceOpenAIImage:
capability = "image"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo:
case model.ChannelInterfaceOpenAIAudio:
capability = "audio"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceGeminiVeo:
capability = "video"
}
order, err := svc.ReserveProxyBilling(user.ID, channel.ID, strings.TrimPrefix(modelName, "models/"), capability, c.GetHeader("X-Canvas-Scene"), c.GetHeader("X-Idempotency-Key"), proxyRequestVideoSeconds(c.GetHeader("Content-Type"), body))
@@ -752,7 +754,9 @@ func apiCallLog(user *model.User, channel *model.ModelChannel, billingOrderID st
switch channel.InterfaceType {
case model.ChannelInterfaceOpenAIImage:
capability = "image"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo:
case model.ChannelInterfaceOpenAIAudio:
capability = "audio"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceGeminiVeo:
capability = "video"
}
requestKind := "create"
@@ -763,24 +767,25 @@ func apiCallLog(user *model.User, channel *model.ModelChannel, billingOrderID st
}
}
return model.ApiCallLog{
UserID: user.ID,
ChannelID: channel.ID,
BillingOrderID: billingOrderID,
Source: "system-channel",
Capability: capability,
RequestKind: requestKind,
Billable: method == http.MethodPost,
APIFormat: channel.APIFormat,
Method: method,
Path: path,
Model: readPayloadModel(body),
Status: status,
StatusCode: statusCode,
DurationMs: duration.Milliseconds(),
Error: errorText,
ConcurrencyLimit: concurrencyLimit,
UpstreamURL: target,
RequestBody: service.SanitizeAPICallPayload(body, contentType),
UserID: user.ID,
ChannelID: channel.ID,
BillingOrderID: billingOrderID,
Source: "system-channel",
Capability: capability,
RequestKind: requestKind,
Billable: method == http.MethodPost,
APIFormat: channel.APIFormat,
Method: method,
Path: path,
Model: readPayloadModel(body),
Status: status,
StatusCode: statusCode,
DurationMs: duration.Milliseconds(),
Error: errorText,
ConcurrencyLimit: concurrencyLimit,
UpstreamURL: target,
RequestContentType: contentType,
RequestBody: service.SanitizeAPICallPayload(body, contentType),
}
}
+18 -7
View File
@@ -23,10 +23,13 @@ import (
)
var (
runtimeService *service.Service
geminiGeneratePath = regexp.MustCompile(`^/models/([^/:]+):(generateContent|streamGenerateContent)$`)
customGeminiRelayPath = regexp.MustCompile(`(?:^|/)models/[^/:]+:(generateContent|streamGenerateContent)$`)
openAIPostEndpoints = map[string]bool{
runtimeService *service.Service
geminiGeneratePath = regexp.MustCompile(`^/models/([^/:]+):(generateContent|streamGenerateContent)$`)
customGeminiRelayPath = regexp.MustCompile(`(?:^|/)models/[^/:]+:(generateContent|streamGenerateContent|predictLongRunning)$`)
customVideoTaskPath = regexp.MustCompile(`(?:^|/)video/generations/[^/]+$`)
customXAIVideoTaskPath = regexp.MustCompile(`(?:^|/)videos/[^/]+$`)
customGeminiOperationPath = regexp.MustCompile(`(?:^|/)(?:models/[^/]+/)?operations/[^/]+$`)
openAIPostEndpoints = map[string]bool{
"/responses": true, "/chat/completions": true, "/images/generations": true, "/images/edits": true,
"/audio/speech": true,
}
@@ -57,7 +60,13 @@ func authorizeCustomRelay(method string, target *url.URL, apiFormat string, cont
return errors.New("自定义渠道调用格式无效")
}
if method == http.MethodGet {
if len(query) != 0 || (requestPath != "/models" && !strings.HasSuffix(requestPath, "/models")) {
allowed := requestPath == "/models" || strings.HasSuffix(requestPath, "/models")
if apiFormat == "openai" {
allowed = allowed || customVideoTaskPath.MatchString(requestPath) || customXAIVideoTaskPath.MatchString(requestPath)
} else {
allowed = allowed || customGeminiOperationPath.MatchString(requestPath)
}
if len(query) != 0 || !allowed {
return errors.New("自定义渠道不允许访问该上游接口")
}
return nil
@@ -70,7 +79,7 @@ func authorizeCustomRelay(method string, target *url.URL, apiFormat string, cont
return errors.New("自定义渠道不允许使用该请求方法")
}
if apiFormat == "openai" {
if len(query) != 0 || (!strings.HasSuffix(requestPath, "/responses") && !strings.HasSuffix(requestPath, "/chat/completions")) {
if len(query) != 0 || (!strings.HasSuffix(requestPath, "/responses") && !strings.HasSuffix(requestPath, "/chat/completions") && !strings.HasSuffix(requestPath, "/video/generations") && !strings.HasSuffix(requestPath, "/videos/generations")) {
return errors.New("自定义渠道不允许访问该上游接口")
}
return nil
@@ -169,7 +178,9 @@ func interfaceAllowsProxyPath(interfaceType model.ChannelInterfaceType, requestP
return requestPath == "/responses"
case model.ChannelInterfaceOpenAIImage:
return requestPath == "/images/generations" || requestPath == "/images/edits"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo:
case model.ChannelInterfaceOpenAIAudio:
return requestPath == "/audio/speech"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceGeminiVeo:
return false
default:
return true
@@ -28,7 +28,13 @@ func TestAuthorizeCustomRelayAllowsModelsAndAgentEndpoints(t *testing.T) {
{method: http.MethodGet, target: "https://api.example.com/v1/models", apiFormat: "openai"},
{method: http.MethodPost, target: "https://api.example.com/v1/responses", apiFormat: "openai", contentType: "application/json"},
{method: http.MethodPost, target: "https://api.example.com/v1/chat/completions", apiFormat: "openai", contentType: "application/json; charset=utf-8"},
{method: http.MethodPost, target: "https://api.example.com/v1/video/generations", apiFormat: "openai", contentType: "application/json"},
{method: http.MethodGet, target: "https://api.example.com/v1/video/generations/task-1", apiFormat: "openai"},
{method: http.MethodPost, target: "https://api.x.ai/v1/videos/generations", apiFormat: "openai", contentType: "application/json"},
{method: http.MethodGet, target: "https://api.x.ai/v1/videos/request-1", apiFormat: "openai"},
{method: http.MethodPost, target: "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse", apiFormat: "gemini", contentType: "application/json"},
{method: http.MethodPost, target: "https://generativelanguage.googleapis.com/v1beta/models/veo-3.0-generate-preview:predictLongRunning", apiFormat: "gemini", contentType: "application/json"},
{method: http.MethodGet, target: "https://generativelanguage.googleapis.com/v1beta/operations/operation-1", apiFormat: "gemini"},
}
for _, test := range tests {
target, err := url.Parse(test.target)
+17 -13
View File
@@ -62,10 +62,12 @@ const (
ChannelInterfaceChatCompletion ChannelInterfaceType = "chat-completion"
ChannelInterfaceOpenAIResponse ChannelInterfaceType = "openai-response"
ChannelInterfaceOpenAIImage ChannelInterfaceType = "openai-image"
ChannelInterfaceOpenAIAudio ChannelInterfaceType = "openai-audio"
ChannelInterfaceNewAPIVideo ChannelInterfaceType = "newapi"
ChannelInterfaceNewAPIChannel1 ChannelInterfaceType = "newapi-channel-1"
ChannelInterfaceNewAPIChannel2 ChannelInterfaceType = "newapi-channel-2"
ChannelInterfaceXAIVideo ChannelInterfaceType = "xai-video"
ChannelInterfaceGeminiVeo ChannelInterfaceType = "gemini-veo"
ApiCallStatusSucceeded ApiCallStatus = "succeeded"
ApiCallStatusFailed ApiCallStatus = "failed"
@@ -210,19 +212,20 @@ type ModelChannel struct {
}
type ChannelModel struct {
ID string `json:"id" gorm:"primaryKey;size:36"`
ChannelID string `json:"channelId" gorm:"size:36;index;uniqueIndex:idx_channel_model_key_active,priority:1,where:deleted_at IS NULL"`
ModelKey string `json:"modelKey" gorm:"size:120;uniqueIndex:idx_channel_model_key_active,priority:2,where:deleted_at IS NULL"`
DisplayName string `json:"displayName" gorm:"size:160"`
Capability string `json:"capability" gorm:"size:32;index"`
BillingMode string `json:"billingMode" gorm:"size:32"`
UnitPriceMicrocredits int64 `json:"unitPriceMicrocredits"`
PriceConfigured bool `json:"priceConfigured" gorm:"index"`
Enabled bool `json:"enabled" gorm:"index"`
PriceVersion int64 `json:"priceVersion"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
DeletedAt gorm.DeletedAt `json:"-" gorm:"index"`
ID string `json:"id" gorm:"primaryKey;size:36"`
ChannelID string `json:"channelId" gorm:"size:36;index;uniqueIndex:idx_channel_model_key_active,priority:1,where:deleted_at IS NULL"`
ModelKey string `json:"modelKey" gorm:"size:120;uniqueIndex:idx_channel_model_key_active,priority:2,where:deleted_at IS NULL"`
DisplayName string `json:"displayName" gorm:"size:160"`
Capability string `json:"capability" gorm:"size:32;index"`
Protocol ChannelInterfaceType `json:"protocol" gorm:"size:32;index"`
BillingMode string `json:"billingMode" gorm:"size:32"`
UnitPriceMicrocredits int64 `json:"unitPriceMicrocredits"`
PriceConfigured bool `json:"priceConfigured" gorm:"index"`
Enabled bool `json:"enabled" gorm:"index"`
PriceVersion int64 `json:"priceVersion"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
DeletedAt gorm.DeletedAt `json:"-" gorm:"index"`
}
type ApiCallLog struct {
@@ -264,6 +267,7 @@ type ApiCallLog struct {
Error string `json:"error"`
ConcurrencyLimit int `json:"concurrencyLimit"`
UpstreamURL string `json:"upstreamUrl"`
RequestContentType string `json:"requestContentType,omitempty" gorm:"size:160"`
RequestBody string `json:"requestBody,omitempty" gorm:"type:text"`
ResponseBody string `json:"responseBody,omitempty" gorm:"type:text"`
StartedAt time.Time `json:"startedAt"`
+17 -9
View File
@@ -104,11 +104,12 @@ type PublicModelChannel struct {
}
type PublicChannelModelPrice struct {
Model string `json:"model"`
DisplayName string `json:"displayName"`
Capability string `json:"capability"`
BillingMode string `json:"billingMode"`
UnitPriceMicrocredits int64 `json:"unitPriceMicrocredits"`
Model string `json:"model"`
DisplayName string `json:"displayName"`
Capability string `json:"capability"`
Protocol model.ChannelInterfaceType `json:"protocol"`
BillingMode string `json:"billingMode"`
UnitPriceMicrocredits int64 `json:"unitPriceMicrocredits"`
}
func (s *Service) RequireAdmin(user *model.User) error {
@@ -511,7 +512,7 @@ func (s *Service) LogAPICall(log model.ApiCallLog) error {
if err != nil {
return err
}
incomingBytes := int64(len(log.Path) + len(log.Model) + len(log.ProviderRequestID) + len(log.ErrorCode) + len(log.Error) + len(log.UpstreamURL) + len(log.RequestBody) + len(log.ResponseBody))
incomingBytes := int64(len(log.Path) + len(log.Model) + len(log.ProviderRequestID) + len(log.ErrorCode) + len(log.Error) + len(log.UpstreamURL) + len(log.RequestContentType) + len(log.RequestBody) + len(log.ResponseBody))
if err := validateAPICallLogQuotaWithPolicy(usage, incomingBytes, policy.Resource); err != nil {
return err
}
@@ -600,8 +601,11 @@ func channelFromRequest(req ChannelRequest, channel model.ModelChannel) (model.M
if req.APIKey != "" {
channel.APIKey = req.APIKey
}
// 系统渠道均由后端按已声明的接口类型分发,调用格式固定为 Bearer/OpenAI 兼容鉴权。
// Gemini Veo 使用 x-goog-api-key,其余系统协议使用 Bearer 鉴权。
channel.APIFormat = "openai"
if interfaceType == model.ChannelInterfaceGeminiVeo {
channel.APIFormat = "gemini"
}
channel.InterfaceType = interfaceType
if req.UseGlobalConcurrency != nil && *req.UseGlobalConcurrency {
channel.ConcurrencyLimit = 0
@@ -641,7 +645,7 @@ func mergeChannelRequest(req ChannelRequest, channel model.ModelChannel) Channel
func validChannelInterfaceType(value model.ChannelInterfaceType) bool {
switch value {
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo:
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceGeminiVeo:
return true
default:
return false
@@ -673,7 +677,11 @@ func publicChannel(channel model.ModelChannel, admin bool, channelModels []model
}
models = append(models, item.ModelKey)
if item.Enabled && item.PriceConfigured {
modelCosts = append(modelCosts, PublicChannelModelPrice{Model: item.ModelKey, DisplayName: item.DisplayName, Capability: item.Capability, BillingMode: item.BillingMode, UnitPriceMicrocredits: item.UnitPriceMicrocredits})
protocol := item.Protocol
if protocol == "" {
protocol = channel.InterfaceType
}
modelCosts = append(modelCosts, PublicChannelModelPrice{Model: item.ModelKey, DisplayName: item.DisplayName, Capability: item.Capability, Protocol: protocol, BillingMode: item.BillingMode, UnitPriceMicrocredits: item.UnitPriceMicrocredits})
}
}
if len(models) == 0 {
@@ -55,6 +55,21 @@ func TestChannelFromRequestStoresXAIVideoInterfaceType(t *testing.T) {
}
}
func TestChannelFromRequestUsesGeminiAuthForVeo(t *testing.T) {
channel, err := channelFromRequest(ChannelRequest{
Name: "Gemini Veo",
BaseURL: "https://generativelanguage.googleapis.com",
APIKey: "key",
InterfaceType: "gemini-veo",
}, model.ModelChannel{})
if err != nil {
t.Fatalf("channelFromRequest() error = %v", err)
}
if channel.InterfaceType != model.ChannelInterfaceGeminiVeo || channel.APIFormat != "gemini" {
t.Fatalf("channel = %#v", channel)
}
}
func TestMergeChannelRequestSupportsEnabledOnlyPatch(t *testing.T) {
enabled := false
req := mergeChannelRequest(ChannelRequest{Enabled: &enabled}, model.ModelChannel{
+25 -5
View File
@@ -16,6 +16,7 @@ type ChannelModelRequest struct {
ModelKey string `json:"modelKey"`
DisplayName string `json:"displayName"`
Capability string `json:"capability"`
Protocol string `json:"protocol"`
BillingMode string `json:"billingMode"`
UnitPriceMicrocredits int64 `json:"unitPriceMicrocredits"`
PriceConfigured bool `json:"priceConfigured"`
@@ -85,7 +86,7 @@ func (s *Service) FetchAdminChannelModels(ctx context.Context, actor *model.User
continue
}
// 自动发现不能绕过定价边界;新模型由管理员定价后再手动启用。
missing = append(missing, model.ChannelModel{ID: newID(), ChannelID: channelID, ModelKey: name, DisplayName: name, Capability: capabilityForChannel(*channel), BillingMode: "fixed_request", Enabled: false, PriceVersion: 1})
missing = append(missing, model.ChannelModel{ID: newID(), ChannelID: channelID, ModelKey: name, DisplayName: name, Capability: capabilityForChannel(*channel), Protocol: channel.InterfaceType, BillingMode: "fixed_request", Enabled: false, PriceVersion: 1})
}
added, err := s.repo.CreateMissingChannelModels(missing)
if err != nil {
@@ -113,6 +114,16 @@ func (s *Service) SaveAdminChannelModel(actor *model.User, channelID string, id
if capability == "" {
return nil, BadAuthRequest("请选择模型能力")
}
protocol := model.ChannelInterfaceType(strings.TrimSpace(req.Protocol))
if protocol == "" {
protocol = channel.InterfaceType
}
if !validChannelInterfaceType(protocol) {
return nil, BadAuthRequest("请选择有效的模型请求协议")
}
if expected := capabilityForProtocol(protocol); expected != "" && expected != capability {
return nil, BadAuthRequest("模型能力与请求协议不匹配")
}
billingMode := strings.TrimSpace(req.BillingMode)
if billingMode == "" {
billingMode = "fixed_request"
@@ -140,6 +151,7 @@ func (s *Service) SaveAdminChannelModel(actor *model.User, channelID string, id
item.DisplayName = modelKey
}
item.Capability = capability
item.Protocol = protocol
item.BillingMode = billingMode
item.UnitPriceMicrocredits = req.UnitPriceMicrocredits
item.PriceConfigured = req.PriceConfigured
@@ -216,7 +228,7 @@ func (s *Service) syncInitialChannelModels(channel *model.ModelChannel, names []
}
continue
}
item := model.ChannelModel{ID: newID(), ChannelID: channel.ID, ModelKey: name, DisplayName: name, Capability: capabilityForChannel(*channel), BillingMode: "fixed_request", Enabled: true, PriceVersion: 1}
item := model.ChannelModel{ID: newID(), ChannelID: channel.ID, ModelKey: name, DisplayName: name, Capability: capabilityForChannel(*channel), Protocol: channel.InterfaceType, BillingMode: "fixed_request", Enabled: true, PriceVersion: 1}
if err := s.repo.SaveChannelModel(&item); err != nil {
return err
}
@@ -266,12 +278,20 @@ func (s *Service) syncChannelModelNames(channel *model.ModelChannel) error {
}
func capabilityForChannel(channel model.ModelChannel) string {
switch channel.InterfaceType {
return capabilityForProtocol(channel.InterfaceType)
}
func capabilityForProtocol(protocol model.ChannelInterfaceType) string {
switch protocol {
case model.ChannelInterfaceOpenAIImage:
return "image"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo:
case model.ChannelInterfaceOpenAIAudio:
return "audio"
case model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceGeminiVeo:
return "video"
default:
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse:
return "text"
default:
return ""
}
}
+202 -35
View File
@@ -12,6 +12,7 @@ import (
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"strconv"
"strings"
"time"
@@ -157,7 +158,7 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string
return nil, err
}
input.Config = config
if input.Config.APIFormat == "gemini" {
if input.Config.APIFormat == "gemini" && input.Config.InterfaceType != string(model.ChannelInterfaceGeminiVeo) {
return nil, errors.New("后端任务队列暂不支持 Gemini 调用格式,请使用 OpenAI 兼容渠道")
}
if strings.TrimSpace(input.Config.BaseURL) == "" || strings.TrimSpace(input.Config.APIKey) == "" || strings.TrimSpace(input.Config.Model) == "" {
@@ -167,7 +168,8 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string
return nil, err
}
if resumedProviderRequestID(ctx) == "" {
if err := s.hydrateGenerationMedia(userID, &input, input.Config.InterfaceType == "newapi-channel-1"); err != nil {
requirePublicURL := input.Config.InterfaceType == "newapi-channel-1" || input.Config.InterfaceType == "newapi-channel-2"
if err := s.hydrateGenerationMedia(userID, &input, requirePublicURL); err != nil {
return nil, err
}
}
@@ -203,7 +205,7 @@ func (s *Service) hydrateGenerationMedia(userID string, input *canvasGenerationI
func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, requirePublicURL bool) error {
if !strings.HasPrefix(media.StorageKey, "resource:") {
if requirePublicURL && strings.HasPrefix(strings.TrimSpace(media.DataURL), "data:") {
return errors.New("NewAPI 渠道 1 的参考素材不能使用内嵌数据,请先上传到 OSS 或提供公网素材地址")
return errors.New("当前 JSON 视频协议的参考素材不能使用内嵌数据,请先上传到 OSS 或提供公网素材地址")
}
return nil
}
@@ -217,18 +219,18 @@ func (s *Service) hydrateProviderMedia(userID string, media *providerMedia, requ
return errors.New("任务参考资源尚未上传完成")
}
if resource.Provider == "local" {
return errors.New("NewAPI 渠道 1 的参考素材需要公网可访问地址,请启用 OSS 后重新上传该素材")
return errors.New("当前 JSON 视频协议的参考素材需要公网可访问地址,请启用 OSS 后重新上传该素材")
}
setting, err := s.ossSettingForResource(userID, resource)
if err != nil {
return err
}
if setting.Provider != "aliyun" {
return errors.New("NewAPI 渠道 1 的参考素材暂时只支持阿里云 OSS 签名地址")
return errors.New("当前 JSON 视频协议的私有参考素材暂时只支持阿里云 OSS 签名地址")
}
signedURL, err := signedOSSObjectURL(setting, resource.ObjectKey, time.Now().Add(providerResourceURLTTL))
if err != nil {
return fmt.Errorf("生成 NewAPI 渠道 1 参考素材地址失败:%w", err)
return fmt.Errorf("生成 JSON 视频协议参考素材地址失败:%w", err)
}
media.URL = signedURL
media.DataURL = ""
@@ -299,6 +301,15 @@ func (s *Service) resolveProviderConfig(config providerConfig) (providerConfig,
config.ChannelID = channel.ID
config.APIFormat = channel.APIFormat
config.InterfaceType = string(channel.InterfaceType)
if channelModel, modelErr := s.repo.ChannelModelByKey(channel.ID, modelName); modelErr == nil && channelModel.Protocol != "" {
config.InterfaceType = string(channelModel.Protocol)
}
// 模型协议是实际请求契约;混合渠道中鉴权格式也必须随模型协议切换。
if config.InterfaceType == string(model.ChannelInterfaceGeminiVeo) {
config.APIFormat = "gemini"
} else if config.InterfaceType != "" {
config.APIFormat = "openai"
}
config.BaseURL = channel.BaseURL
config.APIKey = channel.APIKey
config.Model = modelName
@@ -585,6 +596,9 @@ func runAudioTask(ctx context.Context, input canvasGenerationInput) (map[string]
}
func runVideoTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) {
if input.Config.InterfaceType == "gemini-veo" {
return runGeminiVeoVideoTask(ctx, input)
}
if input.Config.InterfaceType == "newapi-channel-2" {
return runNewAPIChannel2VideoTask(ctx, input)
}
@@ -684,6 +698,126 @@ func runVideoTask(ctx context.Context, input canvasGenerationInput) (map[string]
return nil, errors.New("视频生成超时")
}
func runGeminiVeoVideoTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) {
if len(input.ReferenceImages) > 1 || len(input.ReferenceVideos) > 0 || len(input.ReferenceAudios) > 0 {
return nil, errors.New("Gemini Veo 当前只支持 1 张起始图,不支持参考视频或音频")
}
id := resumedProviderRequestID(ctx)
if id == "" {
instance := map[string]interface{}{"prompt": strings.TrimSpace(input.Prompt)}
if len(input.ReferenceImages) == 1 {
raw, mimeType, err := mediaBytes(input.ReferenceImages[0])
if err != nil {
return nil, fmt.Errorf("读取 Gemini Veo 起始图失败:%w", err)
}
instance["image"] = map[string]interface{}{"bytesBase64Encoded": base64.StdEncoding.EncodeToString(raw), "mimeType": mimeType}
}
body := map[string]interface{}{
"instances": []interface{}{instance},
"parameters": map[string]interface{}{
"aspectRatio": normalizeNewAPIChannel2Ratio(input.Config.Size, strings.ToLower(input.Config.Model)),
"durationSeconds": normalizeSeedanceVideosDuration(input.Config.VideoSeconds),
"resolution": normalizeNewAPIChannel2Resolution(input.Config.VQuality, strings.ToLower(input.Config.Model)),
"sampleCount": 1,
},
}
var created map[string]interface{}
if err := postGeminiJSON(ctx, input.Config, "/models/"+url.PathEscape(input.Config.Model)+":predictLongRunning", body, &created); err != nil {
return nil, err
}
id = strings.TrimSpace(stringField(created, "name"))
}
if id == "" {
return nil, errors.New("Gemini Veo 没有返回 operation name")
}
for deadline := providerPollingDeadline(ctx); time.Now().Before(deadline); {
var operation map[string]interface{}
if err := getGeminiJSON(ctx, input.Config, "/"+strings.TrimLeft(id, "/"), &operation); err != nil {
return nil, err
}
if errorValue, ok := operation["error"].(map[string]interface{}); ok && stringField(errorValue, "message") != "" {
return nil, fmt.Errorf("Gemini Veo 视频生成失败(任务 %s):%s", id, stringField(errorValue, "message"))
}
done, _ := operation["done"].(bool)
if done {
videoURL := findProviderMediaURL(operation["response"])
if videoURL == "" {
return nil, fmt.Errorf("Gemini Veo 任务 %s 已完成但没有返回视频地址", id)
}
data, mimeType, err := getGeminiBinary(withProviderRequestKind(ctx, "download"), input.Config, videoURL)
if err != nil {
return nil, fmt.Errorf("Gemini Veo 视频下载失败(任务 %s):%w", id, err)
}
mimeType = normalizedMediaMimeType(mimeType, data)
return map[string]interface{}{"mode": "video", "video": map[string]interface{}{"dataUrl": dataURL(mimeType, data), "mimeType": mimeType}}, nil
}
if err := sleepContext(ctx, 5*time.Second); err != nil {
return nil, err
}
}
return nil, fmt.Errorf("Gemini Veo 视频生成超时(任务 %s)", id)
}
func postGeminiJSON(ctx context.Context, config providerConfig, path string, body interface{}, target interface{}) error {
data, _ := json.Marshal(body)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, geminiVeoURL(config.BaseURL, path), bytes.NewReader(data))
if err != nil {
return err
}
req.Header.Set("x-goog-api-key", config.APIKey)
req.Header.Set("Content-Type", "application/json")
return doJSON(req, target)
}
func getGeminiJSON(ctx context.Context, config providerConfig, path string, target interface{}) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, geminiVeoURL(config.BaseURL, path), nil)
if err != nil {
return err
}
req.Header.Set("x-goog-api-key", config.APIKey)
return doJSON(req, target)
}
func getGeminiBinary(ctx context.Context, config providerConfig, rawURL string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return nil, "", err
}
req.Header.Set("x-goog-api-key", config.APIKey)
return doBinary(req)
}
func geminiVeoURL(baseURL string, path string) string {
base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
if !strings.HasSuffix(strings.ToLower(base), "/v1beta") {
base += "/v1beta"
}
return base + "/" + strings.TrimLeft(path, "/")
}
func findProviderMediaURL(value interface{}) string {
switch typed := value.(type) {
case map[string]interface{}:
for _, key := range []string{"uri", "url", "videoUri", "video_url"} {
if candidate := strings.TrimSpace(stringField(typed, key)); isPublicMediaURL(candidate) {
return candidate
}
}
for _, child := range typed {
if candidate := findProviderMediaURL(child); candidate != "" {
return candidate
}
}
case []interface{}:
for _, child := range typed {
if candidate := findProviderMediaURL(child); candidate != "" {
return candidate
}
}
}
return ""
}
func newAPIVideoResultURL(state map[string]interface{}) string {
return nestedNewAPIVideoResultURL(state, false, 0)
}
@@ -736,7 +870,7 @@ func runNewAPIChannel2VideoTask(ctx context.Context, input canvasGenerationInput
}
}
if id == "" {
return nil, errors.New("NewAPI 渠道 2 没有返回任务 ID")
return nil, errors.New("NewAPI Video Generations 没有返回任务 ID")
}
for deadline := time.Now().Add(newAPIChannel2VideoPollTimeout); time.Now().Before(deadline); {
@@ -752,32 +886,32 @@ func runNewAPIChannel2VideoTask(ctx context.Context, input canvasGenerationInput
case "SUCCESS":
videoURL := strings.TrimSpace(stringField(state, "result_url"))
if videoURL == "" {
return nil, fmt.Errorf("NewAPI 渠道 2 任务 %s 已成功但没有返回视频地址", id)
return nil, fmt.Errorf("NewAPI Video Generations 任务 %s 已成功但没有返回视频地址", id)
}
data, mimeType, err := getExternalBinary(withProviderRequestKind(ctx, "download"), videoURL)
if err != nil {
return nil, fmt.Errorf("NewAPI 渠道 2 视频结果下载失败(任务 %s):%w", id, err)
return nil, fmt.Errorf("NewAPI Video Generations 视频结果下载失败(任务 %s):%w", id, err)
}
mimeType = normalizedMediaMimeType(mimeType, data)
return map[string]interface{}{"mode": "video", "video": map[string]interface{}{"dataUrl": dataURL(mimeType, data), "mimeType": mimeType}}, nil
case "FAILURE":
reason := strings.TrimSpace(stringField(state, "fail_reason"))
return nil, fmt.Errorf("NewAPI 渠道 2 视频生成失败(任务 %s):%s", id, defaultString(reason, "上游返回失败"))
return nil, fmt.Errorf("NewAPI Video Generations 视频生成失败(任务 %s):%s", id, defaultString(reason, "上游返回失败"))
case "SUBMITTED", "QUEUED", "IN_PROGRESS", "NOT_START", "":
// 按上游协议继续轮询;空状态也可能出现在任务刚写入队列时。
default:
return nil, fmt.Errorf("NewAPI 渠道 2 任务 %s 返回未知状态:%s", id, status)
return nil, fmt.Errorf("NewAPI Video Generations 任务 %s 返回未知状态:%s", id, status)
}
if err := sleepContext(ctx, newAPIChannel2VideoPollInterval); err != nil {
return nil, err
}
}
return nil, fmt.Errorf("NewAPI 渠道 2 视频生成超时(任务 %s)", id)
return nil, fmt.Errorf("NewAPI Video Generations 视频生成超时(任务 %s)", id)
}
func newAPIChannel2VideoBody(input canvasGenerationInput) (map[string]interface{}, error) {
if len(input.ReferenceVideos) > 0 || len(input.ReferenceAudios) > 0 {
return nil, errors.New("NewAPI 渠道 2 只支持参考图片,不支持参考视频或音频")
if len(input.ReferenceImages) > 9 || len(input.ReferenceVideos) > 3 || len(input.ReferenceAudios) > 3 {
return nil, errors.New("NewAPI Video Generations 最多支持 9 张参考图、3 个参考视频和 3 个参考音频")
}
modelName := strings.ToLower(strings.TrimSpace(input.Config.Model))
requiresSingleImage := modelName == "grok-video-1.5" || modelName == "grok-video-1.5-1080p"
@@ -785,19 +919,16 @@ func newAPIChannel2VideoBody(input canvasGenerationInput) (map[string]interface{
// 单图模型以实际参考图为准,兼容旧画布中未随连接关系更新的 text_to_video 元数据。
if shouldSendNewAPIVideoImages(input) || requiresSingleImage {
for _, image := range input.ReferenceImages {
url, err := openAIImageInputURL(image)
url, err := videoGenerationsMediaURL(image)
if err != nil {
return nil, err
}
images = append(images, url)
}
}
if len(images) > 7 {
return nil, errors.New("NewAPI 渠道 2 最多支持 7 张参考图")
}
if requiresSingleImage {
if len(images) != 1 {
return nil, fmt.Errorf("NewAPI 渠道 2 的 %s 必须且只能提供 1 张参考图(当前 %d 张)", input.Config.Model, len(images))
return nil, fmt.Errorf("NewAPI Video Generations 的 %s 必须且只能提供 1 张参考图(当前 %d 张)", input.Config.Model, len(images))
}
}
@@ -813,18 +944,49 @@ func newAPIChannel2VideoBody(input canvasGenerationInput) (map[string]interface{
ratio := normalizeNewAPIChannel2Ratio(input.Config.Size, modelName)
resolution := normalizeNewAPIChannel2Resolution(input.Config.VQuality, modelName)
body := map[string]interface{}{
"model": input.Config.Model,
"prompt": strings.TrimSpace(input.Prompt),
"seconds": seconds,
"aspect_ratio": ratio,
"resolution": resolution,
"model": input.Config.Model,
"prompt": strings.TrimSpace(input.Prompt),
"seconds": seconds,
"aspect_ratio": ratio,
"resolution": resolution,
"generate_audio": parseBool(input.Config.VideoGenerateAudio, true),
}
if len(images) > 0 {
body["image_urls"] = images
}
videoURLs := make([]string, 0, len(input.ReferenceVideos))
for _, video := range input.ReferenceVideos {
url, err := videoGenerationsMediaURL(video)
if err != nil {
return nil, err
}
videoURLs = append(videoURLs, url)
}
if len(videoURLs) > 0 {
body["video_urls"] = videoURLs
}
audioURLs := make([]string, 0, len(input.ReferenceAudios))
for _, audio := range input.ReferenceAudios {
url, err := videoGenerationsMediaURL(audio)
if err != nil {
return nil, err
}
audioURLs = append(audioURLs, url)
}
if len(audioURLs) > 0 {
body["audio_urls"] = audioURLs
}
return body, nil
}
func videoGenerationsMediaURL(media providerMedia) (string, error) {
value := strings.TrimSpace(firstNonEmpty(media.URL, media.DataURL))
if isPublicMediaURL(value) || strings.HasPrefix(value, "data:") {
return value, nil
}
return "", errors.New("NewAPI Video Generations 的参考素材需要公网 URL;私有素材请先保存到 OSS")
}
func normalizeNewAPIChannel2Ratio(value string, modelName string) string {
ratio := strings.TrimSpace(value)
if strings.Contains(ratio, "x") {
@@ -886,10 +1048,10 @@ func runNewAPIChannel1VideoTask(ctx context.Context, input canvasGenerationInput
}
status := strings.ToUpper(strings.TrimSpace(stringField(created, "status")))
if strings.HasPrefix(status, "FAILED") {
return nil, fmt.Errorf("NewAPI 渠道 1 视频生成失败(任务 %s):%s", id, strings.TrimSpace(strings.TrimPrefix(status, "FAILED:")))
return nil, fmt.Errorf("NewAPI 媒体任务视频生成失败(任务 %s):%s", id, strings.TrimSpace(strings.TrimPrefix(status, "FAILED:")))
}
if id == "" {
return nil, errors.New("NewAPI 渠道 1 没有返回任务 ID")
return nil, errors.New("NewAPI 媒体任务没有返回任务 ID")
}
for deadline := providerPollingDeadline(ctx); time.Now().Before(deadline); {
var state map[string]interface{}
@@ -904,27 +1066,27 @@ func runNewAPIChannel1VideoTask(ctx context.Context, input canvasGenerationInput
case status == "SUCCEEDED":
videoURL := stringField(state, "object")
if videoURL == "" {
return nil, fmt.Errorf("NewAPI 渠道 1 任务 %s 已完成但没有返回视频 URL", id)
return nil, fmt.Errorf("NewAPI 媒体任务 %s 已完成但没有返回视频 URL", id)
}
data, mimeType, err := getExternalBinary(withProviderRequestKind(ctx, "download"), videoURL)
if err != nil {
return nil, fmt.Errorf("NewAPI 渠道 1 视频结果下载失败(任务 %s):%w", id, err)
return nil, fmt.Errorf("NewAPI 媒体任务视频结果下载失败(任务 %s):%w", id, err)
}
return map[string]interface{}{"mode": "video", "video": map[string]interface{}{"dataUrl": dataURL(mimeType, data), "mimeType": mimeType}}, nil
case strings.HasPrefix(status, "FAILED"):
message := strings.TrimSpace(strings.TrimPrefix(status, "FAILED:"))
return nil, fmt.Errorf("NewAPI 渠道 1 视频生成失败(任务 %s):%s", id, defaultString(message, "上游返回失败"))
return nil, fmt.Errorf("NewAPI 媒体任务视频生成失败(任务 %s):%s", id, defaultString(message, "上游返回失败"))
}
if err := sleepContext(ctx, newAPIChannel1VideoPollInterval); err != nil {
return nil, err
}
}
return nil, fmt.Errorf("NewAPI 渠道 1 视频生成超时(任务 %s)", id)
return nil, fmt.Errorf("NewAPI 媒体任务视频生成超时(任务 %s)", id)
}
func newAPIChannel1VideoBody(input canvasGenerationInput) (map[string]interface{}, error) {
if len(input.ReferenceImages) > 9 || len(input.ReferenceVideos) > 3 || len(input.ReferenceAudios) > 3 {
return nil, errors.New("NewAPI 渠道 1 最多支持 9 张参考图、3 个参考视频和 3 个参考音频")
return nil, errors.New("NewAPI 媒体任务最多支持 9 张参考图、3 个参考视频和 3 个参考音频")
}
media := make([]map[string]string, 0, len(input.ReferenceImages)+len(input.ReferenceVideos)+len(input.ReferenceAudios))
if shouldSendNewAPIVideoImages(input) {
@@ -970,7 +1132,7 @@ func newAPIChannel1VideoBody(input canvasGenerationInput) (map[string]interface{
func newAPIChannel1MediaURL(media providerMedia) (string, error) {
value := strings.TrimSpace(media.URL)
if !isPublicMediaURL(value) {
return "", errors.New("NewAPI 渠道 1 的参考素材必须使用公网 HTTP(S) URL,请启用 OSS 或提供公网素材地址")
return "", errors.New("NewAPI 媒体任务的参考素材必须使用公网 HTTP(S) URL,请启用 OSS 或提供公网素材地址")
}
if _, err := ValidateOutboundURL(value); err != nil {
return "", err
@@ -1012,7 +1174,8 @@ func validateGenerationInterface(mode string, interfaceType string) error {
allowed := map[string]map[string]bool{
"text": {"chat-completion": true, "openai-response": true},
"image": {"openai-image": true},
"video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true, "xai-video": true},
"video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true, "xai-video": true, "gemini-veo": true},
"audio": {"openai-audio": true},
}
if allowed[mode] != nil && !allowed[mode][interfaceType] {
return fmt.Errorf("接口类型 %s 不支持%s生成", interfaceType, mode)
@@ -1389,14 +1552,18 @@ func recordProviderRequest(req *http.Request, startedAt time.Time, statusCode in
if metadata.RequestKind != "" {
requestKind = metadata.RequestKind
}
apiFormat := "openai"
if req.Header.Get("x-goog-api-key") != "" {
apiFormat = "gemini"
}
log := model.ApiCallLog{
UserID: metadata.UserID, ChannelID: metadata.ChannelID, TaskID: metadata.TaskID, BillingOrderID: metadata.BillingOrderID,
Source: "backend-task", Capability: metadata.Capability, Operation: metadata.Operation,
RequestKind: requestKind, Billable: req.Method == http.MethodPost,
APIFormat: "openai", Method: req.Method, Path: req.URL.Path, Model: metadata.Model,
APIFormat: apiFormat, Method: req.Method, Path: req.URL.Path, Model: metadata.Model,
Status: status, StatusCode: statusCode, DurationMs: time.Since(startedAt).Milliseconds(),
Error: errorText, ConcurrencyLimit: metadata.ConcurrencyLimit, UpstreamURL: req.URL.Scheme + "://" + req.URL.Host + req.URL.Path,
ProviderRequestID: metadata.ProviderRequestID, RequestBody: requestPayloadForLog(req), ResponseBody: SanitizeAPICallPayload(responseBody, ""),
ProviderRequestID: metadata.ProviderRequestID, RequestContentType: req.Header.Get("Content-Type"), RequestBody: requestPayloadForLog(req), ResponseBody: SanitizeAPICallPayload(responseBody, ""),
}
channelSlotFailure := false
if code, message := ChannelSlotFailureDetails(requestErr); code != "" {
+50 -1
View File
@@ -695,6 +695,11 @@ func TestRunNewAPIChannel2VideoTaskDownloadsTemporaryResult(t *testing.T) {
if !ok || len(images) != 2 || images[0] != testReferenceImageDataURL {
t.Errorf("image_urls = %#v", body["image_urls"])
}
videos, _ := body["video_urls"].([]interface{})
audios, _ := body["audio_urls"].([]interface{})
if len(videos) != 1 || len(audios) != 1 || body["generate_audio"] != true {
t.Errorf("multi-reference body = %#v", body)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"task_id":"grok-task","status":"queued"}`))
case "GET /v1/video/generations/grok-task":
@@ -716,7 +721,9 @@ func TestRunNewAPIChannel2VideoTaskDownloadsTemporaryResult(t *testing.T) {
{ID: "image-1", DataURL: testReferenceImageDataURL},
{ID: "image-2", DataURL: testReferenceImageDataURL},
},
Metadata: map[string]interface{}{"videoEditOperation": "image_to_video"},
ReferenceVideos: []providerMedia{{ID: "video-1", URL: server.URL + "/reference.mp4"}},
ReferenceAudios: []providerMedia{{ID: "audio-1", URL: server.URL + "/reference.mp3"}},
Metadata: map[string]interface{}{"videoEditOperation": "image_to_video"},
})
if err != nil {
t.Fatalf("runVideoTask() error = %v", err)
@@ -731,6 +738,48 @@ func TestRunNewAPIChannel2VideoTaskDownloadsTemporaryResult(t *testing.T) {
}
}
func TestRunGeminiVeoVideoTaskUsesLongRunningOperation(t *testing.T) {
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
paths := make([]string, 0, 3)
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
paths = append(paths, r.Method+" "+r.URL.Path)
if r.Header.Get("x-goog-api-key") != "test-key" {
t.Errorf("x-goog-api-key = %q", r.Header.Get("x-goog-api-key"))
}
switch r.Method + " " + r.URL.Path {
case "POST /v1beta/models/veo-test:predictLongRunning":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"name":"operations/op-1"}`))
case "GET /v1beta/operations/op-1":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"done":true,"response":{"generatedSamples":[{"video":{"uri":"` + server.URL + `/video.mp4"}}]}}`))
case "GET /video.mp4":
w.Header().Set("Content-Type", "video/mp4")
_, _ = w.Write([]byte("video"))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
result, err := runVideoTask(context.Background(), canvasGenerationInput{
Prompt: "make it move",
Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", APIFormat: "gemini", Model: "veo-test", InterfaceType: "gemini-veo", VideoSeconds: "6", Size: "16:9", VQuality: "720"},
})
if err != nil {
t.Fatalf("runVideoTask() error = %v", err)
}
video := result["video"].(map[string]interface{})
if video["dataUrl"] != "data:video/mp4;base64,dmlkZW8=" {
t.Fatalf("video = %#v", video)
}
want := "POST /v1beta/models/veo-test:predictLongRunning,GET /v1beta/operations/op-1,GET /video.mp4"
if got := strings.Join(paths, ","); got != want {
t.Fatalf("paths = %q, want %q", got, want)
}
}
func TestNewAPIChannel2SingleImageModelsRequireOneReference(t *testing.T) {
_, err := newAPIChannel2VideoBody(canvasGenerationInput{Config: providerConfig{Model: "grok-video-1.5", VideoSeconds: "6"}})
if err == nil {