feat(platform): 创作能力整合 - 新增 Gemini Images 图片请求协议
This commit is contained in:
ddCat
@@ -805,7 +805,7 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User,
|
||||
}
|
||||
service.ApplyOutboundHeaders(upstreamReq, channelHeaders)
|
||||
service.ApplyDefaultOutboundHeaders(upstreamReq)
|
||||
if protocol == model.ChannelInterfaceGeminiVeo {
|
||||
if protocol == model.ChannelInterfaceGeminiVeo || protocol == model.ChannelInterfaceGeminiImage {
|
||||
upstreamReq.Header.Set("x-goog-api-key", channel.APIKey)
|
||||
} else {
|
||||
upstreamReq.Header.Set("Authorization", "Bearer "+channel.APIKey)
|
||||
@@ -867,7 +867,7 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User,
|
||||
func apiCallLog(user *model.User, channel *model.ModelChannel, billingOrderID string, capability string, protocol model.ChannelInterfaceType, method string, path string, target string, body []byte, contentType string, status model.ApiCallStatus, statusCode int, duration time.Duration, errorText string, concurrencyLimit int) model.ApiCallLog {
|
||||
requestKind := "create"
|
||||
apiFormat := "openai"
|
||||
if protocol == model.ChannelInterfaceGeminiVeo {
|
||||
if protocol == model.ChannelInterfaceGeminiVeo || protocol == model.ChannelInterfaceGeminiImage {
|
||||
apiFormat = "gemini"
|
||||
}
|
||||
if method == http.MethodGet {
|
||||
|
||||
@@ -160,7 +160,7 @@ func authorizeSystemProxy(channel *model.ModelChannel, protocol model.ChannelInt
|
||||
if method == http.MethodGet && requestPath == "/models" {
|
||||
return nil
|
||||
}
|
||||
if protocol == model.ChannelInterfaceGeminiVeo {
|
||||
if protocol == model.ChannelInterfaceGeminiVeo || protocol == model.ChannelInterfaceGeminiImage {
|
||||
matches := geminiGeneratePath.FindStringSubmatch(requestPath)
|
||||
if method != http.MethodPost || len(matches) != 3 {
|
||||
return errors.New("系统渠道不允许访问该上游接口")
|
||||
@@ -196,7 +196,7 @@ func interfaceAllowsProxyPath(interfaceType model.ChannelInterfaceType, requestP
|
||||
return requestPath == "/images/generations"
|
||||
case model.ChannelInterfaceOpenAIAudio:
|
||||
return requestPath == "/audio/speech"
|
||||
case model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo, model.ChannelInterfaceNovitaVideo:
|
||||
case model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo, model.ChannelInterfaceGeminiImage, model.ChannelInterfaceNovitaVideo:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
|
||||
@@ -64,6 +64,7 @@ const (
|
||||
ChannelInterfaceGrokImage ChannelInterfaceType = "grok-image"
|
||||
ChannelInterfaceVolcengineArkImage ChannelInterfaceType = "volcengine-ark-image"
|
||||
ChannelInterfaceVolcengineJiMengImage ChannelInterfaceType = "volcengine-jimeng-image"
|
||||
ChannelInterfaceGeminiImage ChannelInterfaceType = "gemini-image"
|
||||
ChannelInterfaceOpenAIAudio ChannelInterfaceType = "openai-audio"
|
||||
ChannelInterfaceAsyncAudio ChannelInterfaceType = "async-audio"
|
||||
ChannelInterfaceNewAPIVideo ChannelInterfaceType = "newapi"
|
||||
|
||||
@@ -816,7 +816,7 @@ func mergeChannelRequest(req ChannelRequest, channel model.ModelChannel) Channel
|
||||
|
||||
func validChannelInterfaceType(value model.ChannelInterfaceType) bool {
|
||||
switch value {
|
||||
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo:
|
||||
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceGeminiImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
|
||||
@@ -510,7 +510,7 @@ func (s *Service) syncChannelModelNames(channel *model.ModelChannel) error {
|
||||
|
||||
func capabilityForProtocol(protocol model.ChannelInterfaceType) string {
|
||||
switch protocol {
|
||||
case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage:
|
||||
case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceGeminiImage:
|
||||
return "image"
|
||||
case model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio:
|
||||
return "audio"
|
||||
|
||||
@@ -143,6 +143,14 @@ func DefaultImageCapabilityConfig(protocol string, modelName string) *ImageCapab
|
||||
image.TransparentBackground.Supported = false
|
||||
image.ResponseFormat.Supported = false
|
||||
image.OutputFormat.Supported = false
|
||||
case model.ChannelInterfaceGeminiImage:
|
||||
image.References.MaskSupported = false
|
||||
// Gemini Images uses imageConfig.aspectRatio, not the OpenAI-style pixel size field.
|
||||
image.Size = ImageSizeConfig{Parameter: "aspect_ratio", Values: []string{"auto", "1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9", "21:9"}, Default: "1:1", AllowCustom: false}
|
||||
image.TransparentBackground.Supported = false
|
||||
image.ResponseFormat.Supported = false
|
||||
image.OutputFormat.Supported = false
|
||||
image.MaxOutputs = 4
|
||||
}
|
||||
if model.ChannelInterfaceType(protocol) != model.ChannelInterfaceGrokImage && strings.HasPrefix(strings.ToLower(strings.TrimSpace(modelName)), "grok-imagine-image") {
|
||||
image.References.MaxImages = 0
|
||||
|
||||
@@ -210,7 +210,7 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if input.Config.APIFormat == "gemini" && input.Config.InterfaceType != string(model.ChannelInterfaceGeminiVeo) {
|
||||
if input.Config.APIFormat == "gemini" && input.Config.InterfaceType != string(model.ChannelInterfaceGeminiVeo) && input.Config.InterfaceType != string(model.ChannelInterfaceGeminiImage) {
|
||||
return nil, errors.New("后端任务队列暂不支持 Gemini 调用格式，请使用 OpenAI 兼容渠道")
|
||||
}
|
||||
if strings.TrimSpace(input.Config.BaseURL) == "" || strings.TrimSpace(input.Config.APIKey) == "" || strings.TrimSpace(input.Config.Model) == "" {
|
||||
@@ -621,7 +621,7 @@ func (s *Service) resolveProviderConfig(config providerConfig) (providerConfig,
|
||||
}
|
||||
config.InterfaceType = string(channelModel.Protocol)
|
||||
// 模型协议是实际请求契约；混合渠道中鉴权格式也必须随模型协议切换。
|
||||
if config.InterfaceType == string(model.ChannelInterfaceGeminiVeo) {
|
||||
if config.InterfaceType == string(model.ChannelInterfaceGeminiVeo) || config.InterfaceType == string(model.ChannelInterfaceGeminiImage) {
|
||||
config.APIFormat = "gemini"
|
||||
} else if config.InterfaceType != "" {
|
||||
config.APIFormat = "openai"
|
||||
@@ -674,6 +674,9 @@ func runImageTask(ctx context.Context, input canvasGenerationInput) (map[string]
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkImage) {
|
||||
return runVolcengineArkImageTask(ctx, input)
|
||||
}
|
||||
if input.Config.InterfaceType == string(model.ChannelInterfaceGeminiImage) {
|
||||
return runGeminiImageTask(ctx, input)
|
||||
}
|
||||
var payload imageResponse
|
||||
if input.Mask != nil {
|
||||
// 蒙版编辑是强校验写路径：协议能力不明确时必须失败，不能静默退化为整图重绘。
|
||||
@@ -753,6 +756,143 @@ func runImageTask(ctx context.Context, input canvasGenerationInput) (map[string]
|
||||
return map[string]interface{}{"mode": "image", "images": images}, nil
|
||||
}
|
||||
|
||||
func runGeminiImageTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) {
|
||||
if input.Mask != nil {
|
||||
return nil, errors.New("Gemini Images 不支持蒙版编辑，请移除蒙版后重试")
|
||||
}
|
||||
if len(input.ReferenceVideos) > 0 || len(input.ReferenceAudios) > 0 {
|
||||
return nil, errors.New("Gemini Images 不支持参考视频或音频")
|
||||
}
|
||||
|
||||
parts := make([]geminiImageContentPart, 0, 1+len(input.ReferenceImages))
|
||||
if prompt := strings.TrimSpace(input.Prompt); prompt != "" {
|
||||
parts = append(parts, geminiImageContentPart{Text: prompt})
|
||||
}
|
||||
for _, image := range input.ReferenceImages {
|
||||
raw, mimeType, err := geminiImageBytes(image)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取 Gemini Images 参考图失败：%w", err)
|
||||
}
|
||||
parts = append(parts, geminiImageContentPart{InlineData: &geminiImageInlineData{MIMEType: mimeType, Data: base64.StdEncoding.EncodeToString(raw)}})
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return nil, errors.New("Gemini Images 请求缺少提示词或参考图")
|
||||
}
|
||||
|
||||
body := geminiImageRequest{
|
||||
Contents: []geminiImageContent{{Role: "user", Parts: parts}},
|
||||
GenerationConfig: geminiImageGenerationConfig{
|
||||
ResponseModalities: []string{"TEXT", "IMAGE"},
|
||||
ImageConfig: geminiImageConfigFor(input.Config),
|
||||
},
|
||||
}
|
||||
if systemPrompt := strings.TrimSpace(input.Config.SystemPrompt); systemPrompt != "" {
|
||||
body.SystemInstruction = &geminiImageContent{Parts: []geminiImageContentPart{{Text: systemPrompt}}}
|
||||
}
|
||||
var payload map[string]interface{}
|
||||
path := "/models/" + url.PathEscape(input.Config.Model) + ":generateContent"
|
||||
if err := postGeminiJSON(ctx, input.Config, path, body, &payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
images, err := geminiImageDataURLs(payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]interface{}{"mode": "image", "images": images}, nil
|
||||
}
|
||||
|
||||
func geminiImageConfigFor(config providerConfig) *geminiImageConfig {
|
||||
imageConfig := &geminiImageConfig{}
|
||||
size := strings.TrimSpace(config.Size)
|
||||
if size != "" && size != "auto" && strings.Count(size, ":") == 1 {
|
||||
imageConfig.AspectRatio = size
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(config.Quality)) {
|
||||
case "low", "1k":
|
||||
imageConfig.ImageSize = "1K"
|
||||
case "medium", "2k":
|
||||
imageConfig.ImageSize = "2K"
|
||||
case "high", "4k":
|
||||
imageConfig.ImageSize = "4K"
|
||||
}
|
||||
if imageConfig.AspectRatio == "" && imageConfig.ImageSize == "" {
|
||||
return nil
|
||||
}
|
||||
return imageConfig
|
||||
}
|
||||
|
||||
func geminiImageBytes(media providerMedia) ([]byte, string, error) {
|
||||
raw, mimeType, err := mediaBytes(media)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, "", errors.New("参考图片数据为空")
|
||||
}
|
||||
detected := strings.TrimSpace(strings.Split(http.DetectContentType(raw), ";")[0])
|
||||
if !strings.HasPrefix(strings.ToLower(mimeType), "image/") {
|
||||
if !strings.HasPrefix(strings.ToLower(detected), "image/") {
|
||||
return nil, "", fmt.Errorf("参考图片 MIME 类型无效：%s", defaultString(mimeType, detected))
|
||||
}
|
||||
mimeType = detected
|
||||
} else if !strings.HasPrefix(strings.ToLower(detected), "image/") {
|
||||
// 以实际字节签名为准，避免错误的 data URL MIME 把非图片内容伪装成图片。
|
||||
return nil, "", fmt.Errorf("参考图片内容不是有效图片：%s", defaultString(detected, mimeType))
|
||||
}
|
||||
return raw, mimeType, nil
|
||||
}
|
||||
|
||||
func geminiImageDataURLs(payload map[string]interface{}) ([]map[string]string, error) {
|
||||
if errorValue, ok := payload["error"].(map[string]interface{}); ok {
|
||||
if message := stringField(errorValue, "message"); message != "" {
|
||||
return nil, errors.New(message)
|
||||
}
|
||||
}
|
||||
candidates, _ := payload["candidates"].([]interface{})
|
||||
images := make([]map[string]string, 0)
|
||||
for _, candidateValue := range candidates {
|
||||
candidate, _ := candidateValue.(map[string]interface{})
|
||||
content, _ := candidate["content"].(map[string]interface{})
|
||||
parts, _ := content["parts"].([]interface{})
|
||||
for _, partValue := range parts {
|
||||
part, _ := partValue.(map[string]interface{})
|
||||
inlineData, _ := part["inlineData"].(map[string]interface{})
|
||||
if inlineData == nil {
|
||||
inlineData, _ = part["inline_data"].(map[string]interface{})
|
||||
}
|
||||
if inlineData != nil {
|
||||
data := strings.TrimSpace(stringField(inlineData, "data"))
|
||||
mimeType := firstNonEmptyString(stringField(inlineData, "mimeType"), stringField(inlineData, "mime_type"))
|
||||
if data == "" {
|
||||
continue
|
||||
}
|
||||
if mimeType == "" {
|
||||
mimeType = "image/png"
|
||||
}
|
||||
if !strings.HasPrefix(strings.ToLower(mimeType), "image/") {
|
||||
return nil, fmt.Errorf("Gemini Images 返回了非图片 MIME 类型：%s", mimeType)
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Gemini Images 返回的图片数据无效：%w", err)
|
||||
}
|
||||
images = append(images, map[string]string{"dataUrl": dataURL(mimeType, decoded)})
|
||||
continue
|
||||
}
|
||||
fileData, _ := part["fileData"].(map[string]interface{})
|
||||
if fileData != nil {
|
||||
if fileURL := firstNonEmptyString(stringField(fileData, "fileUri"), stringField(fileData, "file_uri")); fileURL != "" {
|
||||
images = append(images, map[string]string{"dataUrl": fileURL})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(images) == 0 {
|
||||
return nil, errors.New("Gemini Images 接口没有返回图片")
|
||||
}
|
||||
return images, nil
|
||||
}
|
||||
|
||||
func runGrokImageTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) {
|
||||
body, path, err := grokImageRequestBody(input)
|
||||
if err != nil {
|
||||
@@ -2257,7 +2397,7 @@ func validateGenerationInterface(mode string, interfaceType string) error {
|
||||
}
|
||||
allowed := map[string]map[string]bool{
|
||||
"text": {"chat-completion": true, "openai-response": true},
|
||||
"image": {"openai-image": true, "grok-image": true, "volcengine-ark-image": true, "volcengine-jimeng-image": true},
|
||||
"image": {"openai-image": true, "grok-image": true, "volcengine-ark-image": true, "volcengine-jimeng-image": true, "gemini-image": true},
|
||||
"video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true, "xai-video": true, "volcengine-ark-video": true, "volcengine-jimeng-video": true, "gemini-veo": true, "novita-video": true},
|
||||
"audio": {"openai-audio": true, "async-audio": true},
|
||||
}
|
||||
|
||||
@@ -90,6 +90,37 @@ type geminiVeoParameters struct {
|
||||
SampleCount int `json:"sampleCount"`
|
||||
}
|
||||
|
||||
type geminiImageRequest struct {
|
||||
Contents []geminiImageContent `json:"contents"`
|
||||
SystemInstruction *geminiImageContent `json:"systemInstruction,omitempty"`
|
||||
GenerationConfig geminiImageGenerationConfig `json:"generationConfig"`
|
||||
}
|
||||
|
||||
type geminiImageContent struct {
|
||||
Role string `json:"role,omitempty"`
|
||||
Parts []geminiImageContentPart `json:"parts"`
|
||||
}
|
||||
|
||||
type geminiImageContentPart struct {
|
||||
Text string `json:"text,omitempty"`
|
||||
InlineData *geminiImageInlineData `json:"inlineData,omitempty"`
|
||||
}
|
||||
|
||||
type geminiImageInlineData struct {
|
||||
MIMEType string `json:"mimeType"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
type geminiImageGenerationConfig struct {
|
||||
ResponseModalities []string `json:"responseModalities"`
|
||||
ImageConfig *geminiImageConfig `json:"imageConfig,omitempty"`
|
||||
}
|
||||
|
||||
type geminiImageConfig struct {
|
||||
AspectRatio string `json:"aspectRatio,omitempty"`
|
||||
ImageSize string `json:"imageSize,omitempty"`
|
||||
}
|
||||
|
||||
type seedanceAgentPlanRequest struct {
|
||||
Model string `json:"model"`
|
||||
Content []map[string]interface{} `json:"content"`
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
)
|
||||
|
||||
const testReferenceImageDataURL = "data:image/png;base64,aGVsbG8="
|
||||
const testGeminiReferenceImageDataURL = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
|
||||
|
||||
func TestWriteMediaPartSanitizesFilenameAndSetsMimeType(t *testing.T) {
|
||||
var body bytes.Buffer
|
||||
@@ -236,6 +237,86 @@ func TestVolcengineArkImageRejectsMaskBeforeRequest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunGeminiImageTaskUsesInlineDataAndImageConfig(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1beta/models/gemini-test:generateContent" {
|
||||
t.Errorf("path = %q, want /v1beta/models/gemini-test:generateContent", r.URL.Path)
|
||||
}
|
||||
if got := r.Header.Get("x-goog-api-key"); got != "test-key" {
|
||||
t.Errorf("x-goog-api-key = %q, want test-key", got)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
t.Fatalf("decode request body: %v", err)
|
||||
}
|
||||
contents, ok := body["contents"].([]interface{})
|
||||
if !ok || len(contents) != 1 {
|
||||
t.Fatalf("contents = %#v", body["contents"])
|
||||
}
|
||||
content, _ := contents[0].(map[string]interface{})
|
||||
parts, ok := content["parts"].([]interface{})
|
||||
if !ok || len(parts) != 2 {
|
||||
t.Fatalf("parts = %#v", content["parts"])
|
||||
}
|
||||
textPart, _ := parts[0].(map[string]interface{})
|
||||
if textPart["text"] != "edit this image" {
|
||||
t.Errorf("text part = %#v", textPart)
|
||||
}
|
||||
imagePart, _ := parts[1].(map[string]interface{})
|
||||
inlineData, _ := imagePart["inlineData"].(map[string]interface{})
|
||||
if inlineData["mimeType"] != "image/png" || inlineData["data"] != "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" {
|
||||
t.Errorf("inlineData = %#v", inlineData)
|
||||
}
|
||||
generationConfig, _ := body["generationConfig"].(map[string]interface{})
|
||||
modalities, _ := generationConfig["responseModalities"].([]interface{})
|
||||
if !reflect.DeepEqual(modalities, []interface{}{"TEXT", "IMAGE"}) {
|
||||
t.Errorf("responseModalities = %#v", modalities)
|
||||
}
|
||||
imageConfig, _ := generationConfig["imageConfig"].(map[string]interface{})
|
||||
if imageConfig["aspectRatio"] != "16:9" || imageConfig["imageSize"] != "4K" {
|
||||
t.Errorf("imageConfig = %#v", imageConfig)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"candidates":[{"content":{"parts":[{"inlineData":{"mimeType":"image/png","data":"aGVsbG8="}}]}}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
result, err := runImageTask(context.Background(), canvasGenerationInput{
|
||||
Prompt: "edit this image",
|
||||
Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", APIFormat: "gemini", Model: "gemini-test", InterfaceType: "gemini-image", Size: "16:9", Quality: "high"},
|
||||
ReferenceImages: []providerMedia{{DataURL: testGeminiReferenceImageDataURL}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runImageTask() error = %v", err)
|
||||
}
|
||||
images, _ := result["images"].([]map[string]string)
|
||||
if len(images) != 1 || images[0]["dataUrl"] != testReferenceImageDataURL {
|
||||
t.Fatalf("images = %#v", result["images"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunGeminiImageTaskRejectsInvalidReferenceBeforeRequest(t *testing.T) {
|
||||
called := false
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
called = true
|
||||
http.Error(w, "unexpected upstream request", http.StatusInternalServerError)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
_, err := runImageTask(context.Background(), canvasGenerationInput{
|
||||
Prompt: "edit this image",
|
||||
Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", APIFormat: "gemini", Model: "gemini-test", InterfaceType: "gemini-image"},
|
||||
ReferenceImages: []providerMedia{{DataURL: "data:text/plain;base64,aGVsbG8="}},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "读取 Gemini Images 参考图失败") || !strings.Contains(err.Error(), "MIME 类型无效") {
|
||||
t.Fatalf("runImageTask() error = %v", err)
|
||||
}
|
||||
if called {
|
||||
t.Fatal("invalid reference image must be rejected before upstream request")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunImageTaskOmitsAutomaticQualityAndSize(t *testing.T) {
|
||||
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"dev": "vite --host 0.0.0.0 --port 3000",
|
||||
"build": "tsc --noEmit && vite build",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "bun test test/channel-model-catalog.test.ts test/agent-prompt-cache.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-generation-layout.test.ts test/canvas-image-batch-retry.test.ts test/canvas-leafer-viewport.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-references.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts && bun test test/local-dreamina-cross-runtime.test.ts",
|
||||
"test": "bun test test/channel-model-catalog.test.ts test/agent-prompt-cache.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-generation-layout.test.ts test/canvas-image-batch-retry.test.ts test/canvas-leafer-viewport.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-references.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts && bun test test/local-dreamina-cross-runtime.test.ts",
|
||||
"start": "vite preview --host 0.0.0.0 --port 3000",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check ."
|
||||
|
||||
@@ -42,7 +42,11 @@ export function CapabilityCardPicker({ value, onChange, density = "comfortable"
|
||||
onClick={() => onChange?.(item.value)}
|
||||
>
|
||||
<span className={cn("grid place-items-center rounded-md", density === "compact" ? "size-6" : "size-8", selected ? "bg-foreground text-background" : "bg-muted text-foreground/65")}>{item.icon}</span>
|
||||
{selected ? <span className={cn("absolute grid place-items-center rounded-full bg-foreground text-background", density === "compact" ? "right-1.5 top-1.5 size-4" : "right-2.5 top-2.5 size-5")}><Check className={density === "compact" ? "size-2.5" : "size-3"} /></span> : null}
|
||||
{selected ? (
|
||||
<span className={cn("absolute grid place-items-center rounded-full bg-foreground text-background", density === "compact" ? "right-1.5 top-1.5 size-4" : "right-2.5 top-2.5 size-5")}>
|
||||
<Check className={density === "compact" ? "size-2.5" : "size-3"} />
|
||||
</span>
|
||||
) : null}
|
||||
<span className={cn("block font-semibold", density === "compact" ? "mt-1 text-xs" : "mt-2 text-sm")}>{item.label}</span>
|
||||
<span className={cn("block text-foreground/48", density === "compact" ? "text-[var(--fs-micro)]" : "text-xs")}>{item.description}</span>
|
||||
{density === "comfortable" ? <BrandIconRow models={item.brands} className="mt-auto pt-2" /> : null}
|
||||
@@ -79,14 +83,20 @@ export function ProtocolCardPicker({ capability, value, onChange, density = "com
|
||||
<div className={cn("mt-0.5 truncate font-mono text-foreground/48", density === "compact" ? "text-[var(--fs-micro)]" : "text-[var(--fs-tiny)]")}>{protocol.create}</div>
|
||||
</div>
|
||||
</div>
|
||||
{selected ? <span className={cn("absolute grid place-items-center rounded-full bg-foreground text-background", density === "compact" ? "right-1.5 top-1.5 size-4" : "right-2.5 top-2.5 size-5")}><Check className={density === "compact" ? "size-2.5" : "size-3"} /></span> : null}
|
||||
{density === "comfortable" ? <>
|
||||
<div className="mt-2 line-clamp-2 text-xs leading-5 text-foreground/58">{protocol.media}</div>
|
||||
<div className="mt-auto flex items-center justify-between gap-2 pt-2 text-[var(--fs-tiny)] text-foreground/42">
|
||||
<span className="truncate">{protocol.contentType}</span>
|
||||
{protocol.poll ? <span className="shrink-0">异步轮询</span> : <span className="shrink-0">同步响应</span>}
|
||||
</div>
|
||||
</> : null}
|
||||
{selected ? (
|
||||
<span className={cn("absolute grid place-items-center rounded-full bg-foreground text-background", density === "compact" ? "right-1.5 top-1.5 size-4" : "right-2.5 top-2.5 size-5")}>
|
||||
<Check className={density === "compact" ? "size-2.5" : "size-3"} />
|
||||
</span>
|
||||
) : null}
|
||||
{density === "comfortable" ? (
|
||||
<>
|
||||
<div className="mt-2 line-clamp-2 text-xs leading-5 text-foreground/58">{protocol.media}</div>
|
||||
<div className="mt-auto flex items-center justify-between gap-2 pt-2 text-[var(--fs-tiny)] text-foreground/42">
|
||||
<span className="truncate">{protocol.contentType}</span>
|
||||
{protocol.poll ? <span className="shrink-0">异步轮询</span> : <span className="shrink-0">同步响应</span>}
|
||||
</div>
|
||||
</>
|
||||
) : null}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
@@ -103,16 +113,24 @@ export function ModelCapabilityProtocolModal({ value, onChange }: { value: { cap
|
||||
}, [open, value]);
|
||||
|
||||
const updateCapability = (capability: ModelCapabilityChoice) => {
|
||||
const protocol = draft.protocol && MODEL_PROTOCOLS.some((item) => item.value === draft.protocol && item.capability === capability)
|
||||
? draft.protocol
|
||||
: MODEL_PROTOCOLS.find((item) => item.capability === capability)?.value || "chat-completion";
|
||||
const protocol = draft.protocol && MODEL_PROTOCOLS.some((item) => item.value === draft.protocol && item.capability === capability) ? draft.protocol : MODEL_PROTOCOLS.find((item) => item.capability === capability)?.value || "chat-completion";
|
||||
setDraft({ capability, protocol });
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<Button size="small" className="max-w-full justify-start" icon={<Settings2 className="size-3.5" />} onClick={() => { setDraft(value); setOpen(true); }}>
|
||||
<span className="max-w-[min(56vw,360px)] truncate">{capabilityLabel(value.capability)} · {modelProtocolLabel(value.protocol)}</span>
|
||||
<Button
|
||||
size="small"
|
||||
className="max-w-full justify-start"
|
||||
icon={<Settings2 className="size-3.5" />}
|
||||
onClick={() => {
|
||||
setDraft(value);
|
||||
setOpen(true);
|
||||
}}
|
||||
>
|
||||
<span className="max-w-[min(56vw,360px)] truncate">
|
||||
{capabilityLabel(value.capability)} · {modelProtocolLabel(value.protocol)}
|
||||
</span>
|
||||
</Button>
|
||||
<Modal
|
||||
title="配置模型能力与请求协议"
|
||||
@@ -123,7 +141,10 @@ export function ModelCapabilityProtocolModal({ value, onChange }: { value: { cap
|
||||
onCancel={() => setOpen(false)}
|
||||
okText="应用配置"
|
||||
cancelText="取消"
|
||||
onOk={() => { onChange(draft); setOpen(false); }}
|
||||
onOk={() => {
|
||||
onChange(draft);
|
||||
setOpen(false);
|
||||
}}
|
||||
>
|
||||
<div className="space-y-4">
|
||||
<section>
|
||||
@@ -143,17 +164,36 @@ export function ModelCapabilityProtocolModal({ value, onChange }: { value: { cap
|
||||
function ProtocolBrandMark({ protocol, compact = false }: { protocol: ModelProtocolDefinition; compact?: boolean }) {
|
||||
const iconSize = compact ? "size-6" : "size-8";
|
||||
if (protocol.value === "chat-completion") return <BrandIconRow models={["openai", "deepseek", "glm"]} compact={compact} />;
|
||||
if (protocol.value.startsWith("volcengine-jimeng-")) return <span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}><Sparkles className={compact ? "size-3" : "size-4"} /></span>;
|
||||
if (protocol.value.startsWith("volcengine-")) return <span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}><Flame className={compact ? "size-3" : "size-4"} /></span>;
|
||||
if (protocol.value.startsWith("newapi-channel-") || protocol.value === "novita-video") return <span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}><Network className={compact ? "size-3" : "size-4"} /></span>;
|
||||
const brand = protocol.value === "gemini-veo" ? "gemini" : protocol.value === "grok-image" || protocol.value === "xai-video" ? "grok" : "openai";
|
||||
if (protocol.value.startsWith("volcengine-jimeng-"))
|
||||
return (
|
||||
<span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}>
|
||||
<Sparkles className={compact ? "size-3" : "size-4"} />
|
||||
</span>
|
||||
);
|
||||
if (protocol.value.startsWith("volcengine-"))
|
||||
return (
|
||||
<span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}>
|
||||
<Flame className={compact ? "size-3" : "size-4"} />
|
||||
</span>
|
||||
);
|
||||
if (protocol.value.startsWith("newapi-channel-") || protocol.value === "novita-video")
|
||||
return (
|
||||
<span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}>
|
||||
<Network className={compact ? "size-3" : "size-4"} />
|
||||
</span>
|
||||
);
|
||||
const brand = protocol.value === "gemini-veo" || protocol.value === "gemini-image" ? "gemini" : protocol.value === "grok-image" || protocol.value === "xai-video" ? "grok" : "openai";
|
||||
return <BrandIconRow models={[brand]} compact={compact} />;
|
||||
}
|
||||
|
||||
function BrandIconRow({ models, compact = false, className }: { models: string[]; compact?: boolean; className?: string }) {
|
||||
return (
|
||||
<span className={cn("flex items-center -space-x-1", compact && "shrink-0", className)} aria-hidden="true">
|
||||
{models.map((model) => <span key={model} className={cn("grid shrink-0 place-items-center rounded-md border border-border/70 bg-background", compact ? "size-6" : "size-6")} title={modelBrandLabel(model)}><ModelIcon model={model} /></span>)}
|
||||
{models.map((model) => (
|
||||
<span key={model} className={cn("grid shrink-0 place-items-center rounded-md border border-border/70 bg-background", compact ? "size-6" : "size-6")} title={modelBrandLabel(model)}>
|
||||
<ModelIcon model={model} />
|
||||
</span>
|
||||
))}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
40
web/src/lib/gemini-image.ts
Normal file
40
web/src/lib/gemini-image.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
export type GeminiImageGenerationConfig = {
|
||||
responseModalities: ["TEXT", "IMAGE"];
|
||||
imageConfig?: { aspectRatio?: string; imageSize?: "1K" | "2K" | "4K" };
|
||||
};
|
||||
|
||||
export type GeminiInlineImage = {
|
||||
mimeType: string;
|
||||
data: string;
|
||||
};
|
||||
|
||||
const GEMINI_IMAGE_SIZES: Record<string, GeminiInlineImageGenerationSize> = {
|
||||
low: "1K",
|
||||
medium: "2K",
|
||||
high: "4K",
|
||||
"1k": "1K",
|
||||
"2k": "2K",
|
||||
"4k": "4K",
|
||||
};
|
||||
|
||||
type GeminiInlineImageGenerationSize = "1K" | "2K" | "4K";
|
||||
|
||||
export function parseGeminiImageDataUrl(dataUrl: string): GeminiInlineImage {
|
||||
const match = dataUrl.trim().match(/^data:([^;,\s]+)(?:;[^,]*)?;base64,([A-Za-z0-9+/=_-]+)$/i);
|
||||
if (!match) throw new Error("参考图片读取失败：未得到有效的 image data URL");
|
||||
const mimeType = match[1].toLowerCase();
|
||||
const data = match[2].replace(/-/g, "+").replace(/_/g, "/");
|
||||
if (!mimeType.startsWith("image/")) throw new Error(`参考图片 MIME 类型无效：${mimeType}`);
|
||||
if (!data) throw new Error("参考图片读取失败：图片数据为空");
|
||||
return { mimeType, data };
|
||||
}
|
||||
|
||||
export function buildGeminiImageGenerationConfig(size?: string, quality?: string): GeminiImageGenerationConfig {
|
||||
const imageConfig: NonNullable<GeminiImageGenerationConfig["imageConfig"]> = {};
|
||||
const normalizedSize = size?.trim();
|
||||
const normalizedQuality = quality?.trim().toLowerCase();
|
||||
if (normalizedSize && normalizedSize !== "auto" && /^\d+:\d+$/.test(normalizedSize)) imageConfig.aspectRatio = normalizedSize;
|
||||
const imageSize = GEMINI_IMAGE_SIZES[normalizedQuality || ""];
|
||||
if (imageSize) imageConfig.imageSize = imageSize;
|
||||
return { responseModalities: ["TEXT", "IMAGE"], ...(Object.keys(imageConfig).length ? { imageConfig } : {}) };
|
||||
}
|
||||
@@ -77,10 +77,45 @@ export type VideoCapabilityConfig = {
|
||||
// Keep explicit pixel presets for each resolution tier so the settings panel can
|
||||
// switch between 1K, 2K and 4K without silently converting the requested ratio.
|
||||
const defaultImageSizes = [
|
||||
"auto", "1:1", "3:2", "2:3", "4:3", "3:4", "16:9", "21:9", "9:16",
|
||||
"1024x1024", "1360x1024", "1024x1360", "1536x1024", "1024x1536", "1024x1280", "1280x1024", "2048x878", "1824x1024", "1024x1824",
|
||||
"2048x2048", "2304x1728", "1728x2304", "2496x1664", "1664x2496", "1792x2240", "2240x1792", "3136x1344", "2752x1536", "1536x2752",
|
||||
"2880x2880", "3264x2448", "2448x3264", "3504x2336", "2336x3504", "2560x3200", "3200x2560", "3808x1632", "3840x2160", "2160x3840",
|
||||
"auto",
|
||||
"1:1",
|
||||
"3:2",
|
||||
"2:3",
|
||||
"4:3",
|
||||
"3:4",
|
||||
"16:9",
|
||||
"21:9",
|
||||
"9:16",
|
||||
"1024x1024",
|
||||
"1360x1024",
|
||||
"1024x1360",
|
||||
"1536x1024",
|
||||
"1024x1536",
|
||||
"1024x1280",
|
||||
"1280x1024",
|
||||
"2048x878",
|
||||
"1824x1024",
|
||||
"1024x1824",
|
||||
"2048x2048",
|
||||
"2304x1728",
|
||||
"1728x2304",
|
||||
"2496x1664",
|
||||
"1664x2496",
|
||||
"1792x2240",
|
||||
"2240x1792",
|
||||
"3136x1344",
|
||||
"2752x1536",
|
||||
"1536x2752",
|
||||
"2880x2880",
|
||||
"3264x2448",
|
||||
"2448x3264",
|
||||
"3504x2336",
|
||||
"2336x3504",
|
||||
"2560x3200",
|
||||
"3200x2560",
|
||||
"3808x1632",
|
||||
"3840x2160",
|
||||
"2160x3840",
|
||||
];
|
||||
|
||||
export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = ""): ImageCapabilityConfig {
|
||||
@@ -123,6 +158,20 @@ export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = "
|
||||
image.responseFormat.supported = false;
|
||||
image.outputFormat.supported = false;
|
||||
}
|
||||
if (protocol === "gemini-image") {
|
||||
image.references.maskSupported = false;
|
||||
// Gemini Images uses imageConfig.aspectRatio, not the OpenAI-style pixel size field.
|
||||
image.size = {
|
||||
parameter: "aspect_ratio",
|
||||
values: ["auto", "1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9", "21:9"],
|
||||
default: "1:1",
|
||||
allowCustom: false,
|
||||
};
|
||||
image.transparentBackground = { supported: false, default: false };
|
||||
image.responseFormat = { supported: false };
|
||||
image.outputFormat = { supported: false };
|
||||
image.maxOutputs = 4;
|
||||
}
|
||||
if (protocol !== "grok-image" && model.trim().toLowerCase().startsWith("grok-imagine-image")) {
|
||||
image.references.maxImages = 0;
|
||||
image.references.maskSupported = false;
|
||||
@@ -207,20 +256,14 @@ export function modelCapabilityConfigFor(config: { channels: Array<{ id: string;
|
||||
const cost = channel?.modelCosts?.find((item) => item.model === modelName);
|
||||
const fallback = defaultModelCapabilityConfig(cost?.protocol, modelName);
|
||||
if (!cost?.capabilityConfig) return fallback;
|
||||
const text = cost.capabilityConfig.text
|
||||
? { ...fallback.text!, ...cost.capabilityConfig.text, references: { ...fallback.text!.references, ...cost.capabilityConfig.text.references } }
|
||||
: fallback.text;
|
||||
const video = cost.capabilityConfig.video
|
||||
? { ...fallback.video!, ...cost.capabilityConfig.video, references: { ...fallback.video!.references, ...cost.capabilityConfig.video.references } }
|
||||
: fallback.video;
|
||||
const text = cost.capabilityConfig.text ? { ...fallback.text!, ...cost.capabilityConfig.text, references: { ...fallback.text!.references, ...cost.capabilityConfig.text.references } } : fallback.text;
|
||||
const video = cost.capabilityConfig.video ? { ...fallback.video!, ...cost.capabilityConfig.video, references: { ...fallback.video!.references, ...cost.capabilityConfig.video.references } } : fallback.video;
|
||||
return { ...fallback, ...cost.capabilityConfig, text, image: cost.capabilityConfig.image || fallback.image, video };
|
||||
}
|
||||
|
||||
export function normalizeImageValue(profile: ImageCapabilityConfig, value: { size?: string; quality?: string; count?: string; transparentBackground?: string }) {
|
||||
const size = normalizeImageSizeSetting(profile, value.size);
|
||||
const quality = profile.quality.supported
|
||||
? (value.quality && profile.quality.values.includes(value.quality) ? value.quality : profile.quality.default || "auto")
|
||||
: profile.quality.default || "auto";
|
||||
const quality = profile.quality.supported ? (value.quality && profile.quality.values.includes(value.quality) ? value.quality : profile.quality.default || "auto") : profile.quality.default || "auto";
|
||||
const count = String(Math.max(1, Math.min(profile.maxOutputs, Math.floor(Math.abs(Number(value.count)) || 1))));
|
||||
const transparentBackground = profile.transparentBackground.supported && value.transparentBackground === "true" ? "true" : "false";
|
||||
return { size, quality, count, transparentBackground };
|
||||
@@ -242,16 +285,16 @@ export function imageSizeRequest(profile: ImageCapabilityConfig, value?: string)
|
||||
}
|
||||
|
||||
export function normalizeVideoValue(profile: VideoCapabilityConfig, value: { seconds?: string; ratio?: string; resolution?: string }) {
|
||||
const duration = profile.duration.selection === "enum"
|
||||
? (profile.duration.values || []).includes(Number(value.seconds)) ? Number(value.seconds) : profile.duration.default
|
||||
: normalizeRangeDuration(profile, Number(value.seconds));
|
||||
const duration = profile.duration.selection === "enum" ? ((profile.duration.values || []).includes(Number(value.seconds)) ? Number(value.seconds) : profile.duration.default) : normalizeRangeDuration(profile, Number(value.seconds));
|
||||
const ratio = profile.ratios.includes(value.ratio || "") ? value.ratio! : profile.defaultRatio;
|
||||
const resolution = profile.resolutions.includes(value.resolution || "") ? value.resolution! : profile.defaultResolution;
|
||||
return { seconds: String(duration), ratio, resolution };
|
||||
}
|
||||
|
||||
export function videoResolutionRequest(profile: VideoCapabilityConfig, value: string | undefined) {
|
||||
const requested = String(value || "").trim().toLowerCase();
|
||||
const requested = String(value || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (!requested || requested === "auto" || requested === "default" || requested === "medium" || requested === "high") return undefined;
|
||||
const candidates = [requested];
|
||||
if (/^\d+$/.test(requested)) candidates.push(`${requested}p`);
|
||||
|
||||
@@ -5,6 +5,7 @@ export type ModelProtocol =
|
||||
| "grok-image"
|
||||
| "volcengine-ark-image"
|
||||
| "volcengine-jimeng-image"
|
||||
| "gemini-image"
|
||||
| "openai-audio"
|
||||
| "async-audio"
|
||||
| "newapi"
|
||||
@@ -35,6 +36,7 @@ export const MODEL_PROTOCOLS: ModelProtocolDefinition[] = [
|
||||
{ value: "grok-image", label: "Grok Images", capability: "image", create: "POST /v1/images/generations / edits", contentType: "application/json", media: "文生图与单张 URL 参考图，不支持蒙版" },
|
||||
{ value: "volcengine-ark-image", label: "火山方舟图片", capability: "image", create: "POST /api/v3/images/generations", contentType: "application/json", media: "文生图与 image 参考图，不支持蒙版" },
|
||||
{ value: "volcengine-jimeng-image", label: "即梦官方图片", capability: "image", create: "POST CVSync2AsyncSubmitTask", poll: "POST CVSync2AsyncGetResult", contentType: "application/json + AK/SK 签名", media: "0-14 张参考图，模型标识填写 req_key" },
|
||||
{ value: "gemini-image", label: "Gemini Images", capability: "image", create: "POST /v1beta/models/{model}:generateContent", contentType: "application/json", media: "文生图、图片编辑与多张内嵌参考图" },
|
||||
{ value: "openai-audio", label: "OpenAI Audio", capability: "audio", create: "POST /v1/audio/speech", contentType: "application/json", media: "文本转语音" },
|
||||
{ value: "async-audio", label: "异步音频任务", capability: "audio", create: "POST /v1/audio/tasks", poll: "GET /v1/audio/tasks/{task_id}", contentType: "application/json", media: "语音、音效与音乐生成" },
|
||||
{ value: "newapi", label: "OpenAI / NewAPI Videos", capability: "video", create: "POST /v1/videos", poll: "GET /v1/videos/{task_id}", contentType: "multipart/form-data", media: "input_reference[] 参考图" },
|
||||
@@ -96,6 +98,7 @@ export function protocolForModelCatalog(endpointTypes: string[] = []): ModelProt
|
||||
const normalized = new Set(endpointTypes.map((value) => value.trim().toLowerCase()));
|
||||
if (normalized.has("openai-chat") || normalized.has("chat-completion") || normalized.has("chat")) return "chat-completion";
|
||||
if (normalized.has("openai-response") || normalized.has("responses")) return "openai-response";
|
||||
if (normalized.has("gemini-image") || normalized.has("gemini-images")) return "gemini-image";
|
||||
if (normalized.has("openai-image") || normalized.has("image")) return "openai-image";
|
||||
if (normalized.has("openai-video") || normalized.has("video")) return "newapi-channel-2";
|
||||
if (normalized.has("openai-audio") || normalized.has("audio")) return "openai-audio";
|
||||
|
||||
@@ -12,16 +12,14 @@ import { imageToDataUrl } from "@/services/image-storage";
|
||||
import type { ReferenceImage } from "@/types/image";
|
||||
import { withOpenAIPromptCacheKey } from "@/lib/openai-prompt-cache";
|
||||
import { imageSizeRequest, modelCapabilityConfigFor, normalizeImageValue, type ImageCapabilityConfig } from "@/lib/model-capabilities";
|
||||
import { buildGeminiImageGenerationConfig, parseGeminiImageDataUrl, type GeminiImageGenerationConfig } from "@/lib/gemini-image";
|
||||
|
||||
export type AiTextMessage = {
|
||||
role: "system" | "user" | "assistant";
|
||||
content: string | AiTextContentPart[];
|
||||
};
|
||||
|
||||
export type AiTextContentPart =
|
||||
| { type: "text"; text: string }
|
||||
| { type: "image_url"; image_url: { url: string } }
|
||||
| { type: "file_url"; file_url: { url: string; name: string; mimeType: string } };
|
||||
export type AiTextContentPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } } | { type: "file_url"; file_url: { url: string; name: string; mimeType: string } };
|
||||
|
||||
export type ResponseToolCall = {
|
||||
id: string;
|
||||
@@ -358,9 +356,7 @@ function toResponseContent(content: ResponseMessageContent): string | ResponseIn
|
||||
return content.map((item) => {
|
||||
if (item.type === "text") return { type: "input_text" as const, text: item.text };
|
||||
if (item.type === "image_url") return { type: "input_image" as const, image_url: item.image_url.url };
|
||||
return item.file_url.url.startsWith("data:")
|
||||
? { type: "input_file" as const, filename: item.file_url.name, file_data: item.file_url.url }
|
||||
: { type: "input_file" as const, filename: item.file_url.name, file_url: item.file_url.url };
|
||||
return item.file_url.url.startsWith("data:") ? { type: "input_file" as const, filename: item.file_url.name, file_data: item.file_url.url } : { type: "input_file" as const, filename: item.file_url.name, file_url: item.file_url.url };
|
||||
});
|
||||
}
|
||||
|
||||
@@ -402,9 +398,7 @@ function toChatCompletionContent(content: ResponseMessageContent) {
|
||||
if (!Array.isArray(content)) return content;
|
||||
return content.map((item) => {
|
||||
if (item.type !== "file_url") return item;
|
||||
const file = item.file_url.url.startsWith("data:")
|
||||
? { filename: item.file_url.name, file_data: item.file_url.url }
|
||||
: { filename: item.file_url.name, file_url: item.file_url.url };
|
||||
const file = item.file_url.url.startsWith("data:") ? { filename: item.file_url.name, file_data: item.file_url.url } : { filename: item.file_url.name, file_url: item.file_url.url };
|
||||
return { type: "file", file };
|
||||
});
|
||||
}
|
||||
@@ -710,7 +704,7 @@ function toGeminiFilePart(url: string, mimeType: string): GeminiPart {
|
||||
|
||||
function geminiTextContent(content: ResponseMessageContent) {
|
||||
if (!Array.isArray(content)) return String(content || "");
|
||||
return content.map((item) => item.type === "text" ? item.text : item.type === "image_url" ? item.image_url.url : `${item.file_url.name}: ${item.file_url.url}`).join("\n");
|
||||
return content.map((item) => (item.type === "text" ? item.text : item.type === "image_url" ? item.image_url.url : `${item.file_url.name}: ${item.file_url.url}`)).join("\n");
|
||||
}
|
||||
|
||||
function jsonObject(value: string): Record<string, unknown> {
|
||||
@@ -809,8 +803,14 @@ function consumeGeminiStreamBlock(block: string, state: GeminiStreamState, onDel
|
||||
function parseGeminiToolResponse(payload: GeminiPayload): ToolResponseResult {
|
||||
validateGeminiPayload(payload);
|
||||
const parts = payload.candidates?.flatMap((candidate) => candidate.content?.parts || []) || [];
|
||||
const content = parts.filter((part) => !part.thought).map((part) => part.text || "").join("");
|
||||
const reasoning = parts.filter((part) => part.thought).map((part) => part.text || "").join("");
|
||||
const content = parts
|
||||
.filter((part) => !part.thought)
|
||||
.map((part) => part.text || "")
|
||||
.join("");
|
||||
const reasoning = parts
|
||||
.filter((part) => part.thought)
|
||||
.map((part) => part.text || "")
|
||||
.join("");
|
||||
const toolCalls = parts
|
||||
.map((part) => part.functionCall)
|
||||
.filter((call): call is NonNullable<GeminiPart["functionCall"]> => Boolean(call?.name))
|
||||
@@ -827,21 +827,27 @@ function parseGeminiToolResponse(payload: GeminiPayload): ToolResponseResult {
|
||||
return { content, toolCalls, ...(reasoning ? { reasoning } : {}) };
|
||||
}
|
||||
|
||||
async function requestGeminiImages(config: AiConfig, prompt: string, references: ReferenceImage[], count: number, options?: RequestOptions) {
|
||||
const requests = Array.from({ length: count }, () => requestGeminiImagesOnce(config, prompt, references, options));
|
||||
async function requestGeminiImages(config: AiConfig, prompt: string, references: ReferenceImage[], count: number, generationConfig: GeminiImageGenerationConfig, options?: RequestOptions) {
|
||||
// 参考图先完整读取并校验一次，再复用已解析的 inlineData；不能让每个输出重新读取 storageKey，
|
||||
// 否则资源缓存瞬时未命中时会出现“偶发缺参考图”或把 storageKey 当 URL 请求的问题。
|
||||
const referenceParts = await Promise.all(
|
||||
references.map(async (image) => {
|
||||
const dataUrl = await imageToDataUrl(image);
|
||||
const inlineData = parseGeminiImageDataUrl(dataUrl);
|
||||
return { inlineData };
|
||||
}),
|
||||
);
|
||||
const requests = Array.from({ length: count }, () => requestGeminiImagesOnce(config, prompt, referenceParts, generationConfig, options));
|
||||
return (await Promise.all(requests)).flat();
|
||||
}
|
||||
|
||||
async function requestGeminiImagesOnce(config: AiConfig, prompt: string, references: ReferenceImage[], options?: RequestOptions) {
|
||||
const parts: GeminiPart[] = [{ text: prompt }];
|
||||
for (const image of references) {
|
||||
parts.push(toGeminiFilePart(await imageToDataUrl(image), image.type || "image/png"));
|
||||
}
|
||||
async function requestGeminiImagesOnce(config: AiConfig, prompt: string, referenceParts: GeminiPart[], generationConfig: GeminiImageGenerationConfig, options?: RequestOptions) {
|
||||
const parts: GeminiPart[] = [{ text: prompt }, ...referenceParts];
|
||||
const request = channelRequest(config, geminiApiUrl(config, "generateContent"), geminiHeaders(config));
|
||||
const response = await axios.post<GeminiPayload>(
|
||||
request.url,
|
||||
{
|
||||
...toGeminiBody(config, [{ role: "user", content: prompt }], { generationConfig: { responseModalities: ["TEXT", "IMAGE"] } }),
|
||||
...toGeminiBody(config, [{ role: "user", content: prompt }], { generationConfig }),
|
||||
contents: [{ role: "user", parts }],
|
||||
},
|
||||
{ headers: request.headers, withCredentials: request.credentials === "include", signal: options?.signal },
|
||||
@@ -872,9 +878,9 @@ export async function requestGeneration(config: AiConfig, prompt: string, option
|
||||
validateImageCapability(imageProfile, []);
|
||||
const normalizedImage = normalizeImageValue(imageProfile, config);
|
||||
const n = Number(normalizedImage.count);
|
||||
if (requestConfig.apiFormat === "gemini") {
|
||||
if (requestConfig.interfaceType === "gemini-image") {
|
||||
try {
|
||||
return await requestGeminiImages(requestConfig, prompt, [], n, options);
|
||||
return await requestGeminiImages(requestConfig, prompt, [], n, buildGeminiImageGenerationConfig(normalizedImage.size, normalizedImage.quality), options);
|
||||
} catch (error) {
|
||||
throw new Error(readAxiosError(error, "请求失败"));
|
||||
}
|
||||
@@ -959,10 +965,10 @@ export async function requestEdit(config: AiConfig, prompt: string, references:
|
||||
const normalizedImage = normalizeImageValue(imageProfile, config);
|
||||
const n = Number(normalizedImage.count);
|
||||
const requestPrompt = buildImageReferencePromptText(prompt, references);
|
||||
if (requestConfig.apiFormat === "gemini") {
|
||||
if (requestConfig.interfaceType === "gemini-image") {
|
||||
if (mask) throw new Error("Gemini 调用格式暂不支持蒙版编辑");
|
||||
try {
|
||||
return await requestGeminiImages(requestConfig, requestPrompt, references, n, options);
|
||||
return await requestGeminiImages(requestConfig, requestPrompt, references, n, buildGeminiImageGenerationConfig(normalizedImage.size, normalizedImage.quality), options);
|
||||
} catch (error) {
|
||||
throw new Error(readAxiosError(error, "请求失败"));
|
||||
}
|
||||
|
||||
@@ -516,7 +516,7 @@ export function resolveModelRequestConfig(config: AiConfig, value: string) {
|
||||
apiKey: channel.apiKey,
|
||||
secretKey: channel.secretKey,
|
||||
headers: channel.headers,
|
||||
apiFormat: interfaceType ? (interfaceType === "gemini-veo" ? ("gemini" as const) : ("openai" as const)) : channel.apiFormat,
|
||||
apiFormat: interfaceType ? (interfaceType === "gemini-veo" || interfaceType === "gemini-image" ? ("gemini" as const) : ("openai" as const)) : channel.apiFormat,
|
||||
interfaceType,
|
||||
channelId: channel.scope === "system" ? channel.id : "",
|
||||
});
|
||||
@@ -564,7 +564,7 @@ export function defaultBaseUrlForApiFormat(apiFormat: ApiCallFormat) {
|
||||
}
|
||||
|
||||
export function defaultBaseUrlForChannelInterface(interfaceType?: ChannelInterfaceType) {
|
||||
if (interfaceType === "gemini-veo") return GEMINI_BASE_URL;
|
||||
if (interfaceType === "gemini-veo" || interfaceType === "gemini-image") return GEMINI_BASE_URL;
|
||||
if (interfaceType === "novita-video") return "https://api.novita.ai/v3";
|
||||
if (interfaceType === "volcengine-ark-image" || interfaceType === "volcengine-ark-video") return "https://ark.cn-beijing.volces.com/api/v3";
|
||||
if (interfaceType === "volcengine-jimeng-image" || interfaceType === "volcengine-jimeng-video") return "https://visual.volcengineapi.com";
|
||||
|
||||
22
web/test/gemini-image.test.ts
Normal file
22
web/test/gemini-image.test.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
|
||||
import { buildGeminiImageGenerationConfig, parseGeminiImageDataUrl } from "../src/lib/gemini-image";
|
||||
|
||||
describe("Gemini Images request helpers", () => {
|
||||
test("拆分参考图 data URL 时使用真实 MIME 和裸 base64", () => {
|
||||
expect(parseGeminiImageDataUrl("data:image/jpeg;base64,aGVsbG8=")).toEqual({ mimeType: "image/jpeg", data: "aGVsbG8=" });
|
||||
});
|
||||
|
||||
test("拒绝空图和非图片 data URL，避免把错误参考图发给 Gemini", () => {
|
||||
expect(() => parseGeminiImageDataUrl("")).toThrow("未得到有效的 image data URL");
|
||||
expect(() => parseGeminiImageDataUrl("data:text/plain;base64,aGVsbG8=")).toThrow("MIME 类型无效");
|
||||
});
|
||||
|
||||
test("把统一图片选项映射到 Gemini imageConfig", () => {
|
||||
expect(buildGeminiImageGenerationConfig("16:9", "high")).toEqual({
|
||||
responseModalities: ["TEXT", "IMAGE"],
|
||||
imageConfig: { aspectRatio: "16:9", imageSize: "4K" },
|
||||
});
|
||||
expect(buildGeminiImageGenerationConfig("auto", "auto")).toEqual({ responseModalities: ["TEXT", "IMAGE"] });
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user