diff --git a/backend/internal/handler/auth.go b/backend/internal/handler/auth.go index 233f13d8a..18c73f804 100644 --- a/backend/internal/handler/auth.go +++ b/backend/internal/handler/auth.go @@ -805,7 +805,7 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User, } service.ApplyOutboundHeaders(upstreamReq, channelHeaders) service.ApplyDefaultOutboundHeaders(upstreamReq) - if protocol == model.ChannelInterfaceGeminiVeo { + if protocol == model.ChannelInterfaceGeminiVeo || protocol == model.ChannelInterfaceGeminiImage { upstreamReq.Header.Set("x-goog-api-key", channel.APIKey) } else { upstreamReq.Header.Set("Authorization", "Bearer "+channel.APIKey) @@ -867,7 +867,7 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User, func apiCallLog(user *model.User, channel *model.ModelChannel, billingOrderID string, capability string, protocol model.ChannelInterfaceType, method string, path string, target string, body []byte, contentType string, status model.ApiCallStatus, statusCode int, duration time.Duration, errorText string, concurrencyLimit int) model.ApiCallLog { requestKind := "create" apiFormat := "openai" - if protocol == model.ChannelInterfaceGeminiVeo { + if protocol == model.ChannelInterfaceGeminiVeo || protocol == model.ChannelInterfaceGeminiImage { apiFormat = "gemini" } if method == http.MethodGet { diff --git a/backend/internal/handler/security.go b/backend/internal/handler/security.go index 472951c0d..b1febb58b 100644 --- a/backend/internal/handler/security.go +++ b/backend/internal/handler/security.go @@ -160,7 +160,7 @@ func authorizeSystemProxy(channel *model.ModelChannel, protocol model.ChannelInt if method == http.MethodGet && requestPath == "/models" { return nil } - if protocol == model.ChannelInterfaceGeminiVeo { + if protocol == model.ChannelInterfaceGeminiVeo || protocol == model.ChannelInterfaceGeminiImage { matches := geminiGeneratePath.FindStringSubmatch(requestPath) if method != http.MethodPost || len(matches) != 3 { return errors.New("系统渠道不允许访问该上游接口") @@ -196,7 +196,7 @@ func interfaceAllowsProxyPath(interfaceType model.ChannelInterfaceType, requestP return requestPath == "/images/generations" case model.ChannelInterfaceOpenAIAudio: return requestPath == "/audio/speech" - case model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo, model.ChannelInterfaceNovitaVideo: + case model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo, model.ChannelInterfaceGeminiImage, model.ChannelInterfaceNovitaVideo: return false default: return true diff --git a/backend/internal/model/models.go b/backend/internal/model/models.go index 1bbe08a81..4b4f72008 100644 --- a/backend/internal/model/models.go +++ b/backend/internal/model/models.go @@ -64,6 +64,7 @@ const ( ChannelInterfaceGrokImage ChannelInterfaceType = "grok-image" ChannelInterfaceVolcengineArkImage ChannelInterfaceType = "volcengine-ark-image" ChannelInterfaceVolcengineJiMengImage ChannelInterfaceType = "volcengine-jimeng-image" + ChannelInterfaceGeminiImage ChannelInterfaceType = "gemini-image" ChannelInterfaceOpenAIAudio ChannelInterfaceType = "openai-audio" ChannelInterfaceAsyncAudio ChannelInterfaceType = "async-audio" ChannelInterfaceNewAPIVideo ChannelInterfaceType = "newapi" diff --git a/backend/internal/service/admin.go b/backend/internal/service/admin.go index 6915f6998..545fff8b6 100644 --- a/backend/internal/service/admin.go +++ b/backend/internal/service/admin.go @@ -816,7 +816,7 @@ func mergeChannelRequest(req ChannelRequest, channel model.ModelChannel) Channel func validChannelInterfaceType(value model.ChannelInterfaceType) bool { switch value { - case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo: + case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceGeminiImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo: return true default: return false diff --git a/backend/internal/service/channel_models.go b/backend/internal/service/channel_models.go index b3fa8aa0e..14da78c7e 100644 --- a/backend/internal/service/channel_models.go +++ b/backend/internal/service/channel_models.go @@ -510,7 +510,7 @@ func (s *Service) syncChannelModelNames(channel *model.ModelChannel) error { func capabilityForProtocol(protocol model.ChannelInterfaceType) string { switch protocol { - case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage: + case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceGeminiImage: return "image" case model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio: return "audio" diff --git a/backend/internal/service/model_capability.go b/backend/internal/service/model_capability.go index 74aa40039..223cc70e4 100644 --- a/backend/internal/service/model_capability.go +++ b/backend/internal/service/model_capability.go @@ -143,6 +143,14 @@ func DefaultImageCapabilityConfig(protocol string, modelName string) *ImageCapab image.TransparentBackground.Supported = false image.ResponseFormat.Supported = false image.OutputFormat.Supported = false + case model.ChannelInterfaceGeminiImage: + image.References.MaskSupported = false + // Gemini Images uses imageConfig.aspectRatio, not the OpenAI-style pixel size field. + image.Size = ImageSizeConfig{Parameter: "aspect_ratio", Values: []string{"auto", "1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9", "21:9"}, Default: "1:1", AllowCustom: false} + image.TransparentBackground.Supported = false + image.ResponseFormat.Supported = false + image.OutputFormat.Supported = false + image.MaxOutputs = 4 } if model.ChannelInterfaceType(protocol) != model.ChannelInterfaceGrokImage && strings.HasPrefix(strings.ToLower(strings.TrimSpace(modelName)), "grok-imagine-image") { image.References.MaxImages = 0 diff --git a/backend/internal/service/provider.go b/backend/internal/service/provider.go index 8b010bcb5..23ab60c82 100644 --- a/backend/internal/service/provider.go +++ b/backend/internal/service/provider.go @@ -210,7 +210,7 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string return nil, err } } - if input.Config.APIFormat == "gemini" && input.Config.InterfaceType != string(model.ChannelInterfaceGeminiVeo) { + if input.Config.APIFormat == "gemini" && input.Config.InterfaceType != string(model.ChannelInterfaceGeminiVeo) && input.Config.InterfaceType != string(model.ChannelInterfaceGeminiImage) { return nil, errors.New("后端任务队列暂不支持 Gemini 调用格式，请使用 OpenAI 兼容渠道") } if strings.TrimSpace(input.Config.BaseURL) == "" || strings.TrimSpace(input.Config.APIKey) == "" || strings.TrimSpace(input.Config.Model) == "" { @@ -621,7 +621,7 @@ func (s *Service) resolveProviderConfig(config providerConfig) (providerConfig, } config.InterfaceType = string(channelModel.Protocol) // 模型协议是实际请求契约；混合渠道中鉴权格式也必须随模型协议切换。 - if config.InterfaceType == string(model.ChannelInterfaceGeminiVeo) { + if config.InterfaceType == string(model.ChannelInterfaceGeminiVeo) || config.InterfaceType == string(model.ChannelInterfaceGeminiImage) { config.APIFormat = "gemini" } else if config.InterfaceType != "" { config.APIFormat = "openai" @@ -674,6 +674,9 @@ func runImageTask(ctx context.Context, input canvasGenerationInput) (map[string] if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkImage) { return runVolcengineArkImageTask(ctx, input) } + if input.Config.InterfaceType == string(model.ChannelInterfaceGeminiImage) { + return runGeminiImageTask(ctx, input) + } var payload imageResponse if input.Mask != nil { // 蒙版编辑是强校验写路径：协议能力不明确时必须失败，不能静默退化为整图重绘。 @@ -753,6 +756,143 @@ func runImageTask(ctx context.Context, input canvasGenerationInput) (map[string] return map[string]interface{}{"mode": "image", "images": images}, nil } +func runGeminiImageTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) { + if input.Mask != nil { + return nil, errors.New("Gemini Images 不支持蒙版编辑，请移除蒙版后重试") + } + if len(input.ReferenceVideos) > 0 || len(input.ReferenceAudios) > 0 { + return nil, errors.New("Gemini Images 不支持参考视频或音频") + } + + parts := make([]geminiImageContentPart, 0, 1+len(input.ReferenceImages)) + if prompt := strings.TrimSpace(input.Prompt); prompt != "" { + parts = append(parts, geminiImageContentPart{Text: prompt}) + } + for _, image := range input.ReferenceImages { + raw, mimeType, err := geminiImageBytes(image) + if err != nil { + return nil, fmt.Errorf("读取 Gemini Images 参考图失败：%w", err) + } + parts = append(parts, geminiImageContentPart{InlineData: &geminiImageInlineData{MIMEType: mimeType, Data: base64.StdEncoding.EncodeToString(raw)}}) + } + if len(parts) == 0 { + return nil, errors.New("Gemini Images 请求缺少提示词或参考图") + } + + body := geminiImageRequest{ + Contents: []geminiImageContent{{Role: "user", Parts: parts}}, + GenerationConfig: geminiImageGenerationConfig{ + ResponseModalities: []string{"TEXT", "IMAGE"}, + ImageConfig: geminiImageConfigFor(input.Config), + }, + } + if systemPrompt := strings.TrimSpace(input.Config.SystemPrompt); systemPrompt != "" { + body.SystemInstruction = &geminiImageContent{Parts: []geminiImageContentPart{{Text: systemPrompt}}} + } + var payload map[string]interface{} + path := "/models/" + url.PathEscape(input.Config.Model) + ":generateContent" + if err := postGeminiJSON(ctx, input.Config, path, body, &payload); err != nil { + return nil, err + } + images, err := geminiImageDataURLs(payload) + if err != nil { + return nil, err + } + return map[string]interface{}{"mode": "image", "images": images}, nil +} + +func geminiImageConfigFor(config providerConfig) *geminiImageConfig { + imageConfig := &geminiImageConfig{} + size := strings.TrimSpace(config.Size) + if size != "" && size != "auto" && strings.Count(size, ":") == 1 { + imageConfig.AspectRatio = size + } + switch strings.ToLower(strings.TrimSpace(config.Quality)) { + case "low", "1k": + imageConfig.ImageSize = "1K" + case "medium", "2k": + imageConfig.ImageSize = "2K" + case "high", "4k": + imageConfig.ImageSize = "4K" + } + if imageConfig.AspectRatio == "" && imageConfig.ImageSize == "" { + return nil + } + return imageConfig +} + +func geminiImageBytes(media providerMedia) ([]byte, string, error) { + raw, mimeType, err := mediaBytes(media) + if err != nil { + return nil, "", err + } + if len(raw) == 0 { + return nil, "", errors.New("参考图片数据为空") + } + detected := strings.TrimSpace(strings.Split(http.DetectContentType(raw), ";")[0]) + if !strings.HasPrefix(strings.ToLower(mimeType), "image/") { + if !strings.HasPrefix(strings.ToLower(detected), "image/") { + return nil, "", fmt.Errorf("参考图片 MIME 类型无效：%s", defaultString(mimeType, detected)) + } + mimeType = detected + } else if !strings.HasPrefix(strings.ToLower(detected), "image/") { + // 以实际字节签名为准，避免错误的 data URL MIME 把非图片内容伪装成图片。 + return nil, "", fmt.Errorf("参考图片内容不是有效图片：%s", defaultString(detected, mimeType)) + } + return raw, mimeType, nil +} + +func geminiImageDataURLs(payload map[string]interface{}) ([]map[string]string, error) { + if errorValue, ok := payload["error"].(map[string]interface{}); ok { + if message := stringField(errorValue, "message"); message != "" { + return nil, errors.New(message) + } + } + candidates, _ := payload["candidates"].([]interface{}) + images := make([]map[string]string, 0) + for _, candidateValue := range candidates { + candidate, _ := candidateValue.(map[string]interface{}) + content, _ := candidate["content"].(map[string]interface{}) + parts, _ := content["parts"].([]interface{}) + for _, partValue := range parts { + part, _ := partValue.(map[string]interface{}) + inlineData, _ := part["inlineData"].(map[string]interface{}) + if inlineData == nil { + inlineData, _ = part["inline_data"].(map[string]interface{}) + } + if inlineData != nil { + data := strings.TrimSpace(stringField(inlineData, "data")) + mimeType := firstNonEmptyString(stringField(inlineData, "mimeType"), stringField(inlineData, "mime_type")) + if data == "" { + continue + } + if mimeType == "" { + mimeType = "image/png" + } + if !strings.HasPrefix(strings.ToLower(mimeType), "image/") { + return nil, fmt.Errorf("Gemini Images 返回了非图片 MIME 类型：%s", mimeType) + } + decoded, err := base64.StdEncoding.DecodeString(data) + if err != nil { + return nil, fmt.Errorf("Gemini Images 返回的图片数据无效：%w", err) + } + images = append(images, map[string]string{"dataUrl": dataURL(mimeType, decoded)}) + continue + } + fileData, _ := part["fileData"].(map[string]interface{}) + if fileData != nil { + if fileURL := firstNonEmptyString(stringField(fileData, "fileUri"), stringField(fileData, "file_uri")); fileURL != "" { + images = append(images, map[string]string{"dataUrl": fileURL}) + } + } + } + } + if len(images) == 0 { + return nil, errors.New("Gemini Images 接口没有返回图片") + } + return images, nil +} + func runGrokImageTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) { body, path, err := grokImageRequestBody(input) if err != nil { @@ -2257,7 +2397,7 @@ func validateGenerationInterface(mode string, interfaceType string) error { } allowed := map[string]map[string]bool{ "text": {"chat-completion": true, "openai-response": true}, - "image": {"openai-image": true, "grok-image": true, "volcengine-ark-image": true, "volcengine-jimeng-image": true}, + "image": {"openai-image": true, "grok-image": true, "volcengine-ark-image": true, "volcengine-jimeng-image": true, "gemini-image": true}, "video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true, "xai-video": true, "volcengine-ark-video": true, "volcengine-jimeng-video": true, "gemini-veo": true, "novita-video": true}, "audio": {"openai-audio": true, "async-audio": true}, } diff --git a/backend/internal/service/provider_request_types.go b/backend/internal/service/provider_request_types.go index a646018a0..f3a2f11db 100644 --- a/backend/internal/service/provider_request_types.go +++ b/backend/internal/service/provider_request_types.go @@ -90,6 +90,37 @@ type geminiVeoParameters struct { SampleCount int `json:"sampleCount"` } +type geminiImageRequest struct { + Contents []geminiImageContent `json:"contents"` + SystemInstruction *geminiImageContent `json:"systemInstruction,omitempty"` + GenerationConfig geminiImageGenerationConfig `json:"generationConfig"` +} + +type geminiImageContent struct { + Role string `json:"role,omitempty"` + Parts []geminiImageContentPart `json:"parts"` +} + +type geminiImageContentPart struct { + Text string `json:"text,omitempty"` + InlineData *geminiImageInlineData `json:"inlineData,omitempty"` +} + +type geminiImageInlineData struct { + MIMEType string `json:"mimeType"` + Data string `json:"data"` +} + +type geminiImageGenerationConfig struct { + ResponseModalities []string `json:"responseModalities"` + ImageConfig *geminiImageConfig `json:"imageConfig,omitempty"` +} + +type geminiImageConfig struct { + AspectRatio string `json:"aspectRatio,omitempty"` + ImageSize string `json:"imageSize,omitempty"` +} + type seedanceAgentPlanRequest struct { Model string `json:"model"` Content []map[string]interface{} `json:"content"` diff --git a/backend/internal/service/provider_test.go b/backend/internal/service/provider_test.go index 5dd6ee521..8f4d3993f 100644 --- a/backend/internal/service/provider_test.go +++ b/backend/internal/service/provider_test.go @@ -19,6 +19,7 @@ import ( ) const testReferenceImageDataURL = "data:image/png;base64,aGVsbG8=" +const testGeminiReferenceImageDataURL = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" func TestWriteMediaPartSanitizesFilenameAndSetsMimeType(t *testing.T) { var body bytes.Buffer @@ -236,6 +237,86 @@ func TestVolcengineArkImageRejectsMaskBeforeRequest(t *testing.T) { } } +func TestRunGeminiImageTaskUsesInlineDataAndImageConfig(t *testing.T) { + t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1beta/models/gemini-test:generateContent" { + t.Errorf("path = %q, want /v1beta/models/gemini-test:generateContent", r.URL.Path) + } + if got := r.Header.Get("x-goog-api-key"); got != "test-key" { + t.Errorf("x-goog-api-key = %q, want test-key", got) + } + var body map[string]interface{} + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("decode request body: %v", err) + } + contents, ok := body["contents"].([]interface{}) + if !ok || len(contents) != 1 { + t.Fatalf("contents = %#v", body["contents"]) + } + content, _ := contents[0].(map[string]interface{}) + parts, ok := content["parts"].([]interface{}) + if !ok || len(parts) != 2 { + t.Fatalf("parts = %#v", content["parts"]) + } + textPart, _ := parts[0].(map[string]interface{}) + if textPart["text"] != "edit this image" { + t.Errorf("text part = %#v", textPart) + } + imagePart, _ := parts[1].(map[string]interface{}) + inlineData, _ := imagePart["inlineData"].(map[string]interface{}) + if inlineData["mimeType"] != "image/png" || inlineData["data"] != "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" { + t.Errorf("inlineData = %#v", inlineData) + } + generationConfig, _ := body["generationConfig"].(map[string]interface{}) + modalities, _ := generationConfig["responseModalities"].([]interface{}) + if !reflect.DeepEqual(modalities, []interface{}{"TEXT", "IMAGE"}) { + t.Errorf("responseModalities = %#v", modalities) + } + imageConfig, _ := generationConfig["imageConfig"].(map[string]interface{}) + if imageConfig["aspectRatio"] != "16:9" || imageConfig["imageSize"] != "4K" { + t.Errorf("imageConfig = %#v", imageConfig) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"candidates":[{"content":{"parts":[{"inlineData":{"mimeType":"image/png","data":"aGVsbG8="}}]}}]}`)) + })) + defer server.Close() + + result, err := runImageTask(context.Background(), canvasGenerationInput{ + Prompt: "edit this image", + Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", APIFormat: "gemini", Model: "gemini-test", InterfaceType: "gemini-image", Size: "16:9", Quality: "high"}, + ReferenceImages: []providerMedia{{DataURL: testGeminiReferenceImageDataURL}}, + }) + if err != nil { + t.Fatalf("runImageTask() error = %v", err) + } + images, _ := result["images"].([]map[string]string) + if len(images) != 1 || images[0]["dataUrl"] != testReferenceImageDataURL { + t.Fatalf("images = %#v", result["images"]) + } +} + +func TestRunGeminiImageTaskRejectsInvalidReferenceBeforeRequest(t *testing.T) { + called := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + called = true + http.Error(w, "unexpected upstream request", http.StatusInternalServerError) + })) + defer server.Close() + + _, err := runImageTask(context.Background(), canvasGenerationInput{ + Prompt: "edit this image", + Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", APIFormat: "gemini", Model: "gemini-test", InterfaceType: "gemini-image"}, + ReferenceImages: []providerMedia{{DataURL: "data:text/plain;base64,aGVsbG8="}}, + }) + if err == nil || !strings.Contains(err.Error(), "读取 Gemini Images 参考图失败") || !strings.Contains(err.Error(), "MIME 类型无效") { + t.Fatalf("runImageTask() error = %v", err) + } + if called { + t.Fatal("invalid reference image must be rejected before upstream request") + } +} + func TestRunImageTaskOmitsAutomaticQualityAndSize(t *testing.T) { t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/web/package.json b/web/package.json index 093cafeff..4f938780b 100644 --- a/web/package.json +++ b/web/package.json @@ -7,7 +7,7 @@ "dev": "vite --host 0.0.0.0 --port 3000", "build": "tsc --noEmit && vite build", "typecheck": "tsc --noEmit", - "test": "bun test test/channel-model-catalog.test.ts test/agent-prompt-cache.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-generation-layout.test.ts test/canvas-image-batch-retry.test.ts test/canvas-leafer-viewport.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-references.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts && bun test test/local-dreamina-cross-runtime.test.ts", + "test": "bun test test/channel-model-catalog.test.ts test/agent-prompt-cache.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-generation-layout.test.ts test/canvas-image-batch-retry.test.ts test/canvas-leafer-viewport.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-references.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts && bun test test/local-dreamina-cross-runtime.test.ts", "start": "vite preview --host 0.0.0.0 --port 3000", "format": "prettier --write .", "format:check": "prettier --check ." diff --git a/web/src/components/model-protocol-picker.tsx b/web/src/components/model-protocol-picker.tsx index fb3b052d0..22b63bdd1 100644 --- a/web/src/components/model-protocol-picker.tsx +++ b/web/src/components/model-protocol-picker.tsx @@ -42,7 +42,11 @@ export function CapabilityCardPicker({ value, onChange, density = "comfortable" onClick={() => onChange?.(item.value)} > {item.icon} - {selected ? : null} + {selected ? ( + + + + ) : null} {item.label} {item.description} {density === "comfortable" ? : null} @@ -79,14 +83,20 @@ export function ProtocolCardPicker({ capability, value, onChange, density = "com
{protocol.create}
- {selected ? : null} - {density === "comfortable" ? <> -
{protocol.media}
-
- {protocol.contentType} - {protocol.poll ? 异步轮询 : 同步响应} -
- : null} + {selected ? ( + + + + ) : null} + {density === "comfortable" ? ( + <> +
{protocol.media}
+
+ {protocol.contentType} + {protocol.poll ? 异步轮询 : 同步响应} +
+ + ) : null} ); })} @@ -103,16 +113,24 @@ export function ModelCapabilityProtocolModal({ value, onChange }: { value: { cap }, [open, value]); const updateCapability = (capability: ModelCapabilityChoice) => { - const protocol = draft.protocol && MODEL_PROTOCOLS.some((item) => item.value === draft.protocol && item.capability === capability) - ? draft.protocol - : MODEL_PROTOCOLS.find((item) => item.capability === capability)?.value || "chat-completion"; + const protocol = draft.protocol && MODEL_PROTOCOLS.some((item) => item.value === draft.protocol && item.capability === capability) ? draft.protocol : MODEL_PROTOCOLS.find((item) => item.capability === capability)?.value || "chat-completion"; setDraft({ capability, protocol }); }; return ( <> - setOpen(false)} okText="应用配置" cancelText="取消" - onOk={() => { onChange(draft); setOpen(false); }} + onOk={() => { + onChange(draft); + setOpen(false); + }} >
@@ -143,17 +164,36 @@ export function ModelCapabilityProtocolModal({ value, onChange }: { value: { cap function ProtocolBrandMark({ protocol, compact = false }: { protocol: ModelProtocolDefinition; compact?: boolean }) { const iconSize = compact ? "size-6" : "size-8"; if (protocol.value === "chat-completion") return ; - if (protocol.value.startsWith("volcengine-jimeng-")) return ; - if (protocol.value.startsWith("volcengine-")) return ; - if (protocol.value.startsWith("newapi-channel-") || protocol.value === "novita-video") return ; - const brand = protocol.value === "gemini-veo" ? "gemini" : protocol.value === "grok-image" || protocol.value === "xai-video" ? "grok" : "openai"; + if (protocol.value.startsWith("volcengine-jimeng-")) + return ( + + + + ); + if (protocol.value.startsWith("volcengine-")) + return ( + + + + ); + if (protocol.value.startsWith("newapi-channel-") || protocol.value === "novita-video") + return ( + + + + ); + const brand = protocol.value === "gemini-veo" || protocol.value === "gemini-image" ? "gemini" : protocol.value === "grok-image" || protocol.value === "xai-video" ? "grok" : "openai"; return ; } function BrandIconRow({ models, compact = false, className }: { models: string[]; compact?: boolean; className?: string }) { return ( ); } diff --git a/web/src/lib/gemini-image.ts b/web/src/lib/gemini-image.ts new file mode 100644 index 000000000..56391b539 --- /dev/null +++ b/web/src/lib/gemini-image.ts @@ -0,0 +1,40 @@ +export type GeminiImageGenerationConfig = { + responseModalities: ["TEXT", "IMAGE"]; + imageConfig?: { aspectRatio?: string; imageSize?: "1K" | "2K" | "4K" }; +}; + +export type GeminiInlineImage = { + mimeType: string; + data: string; +}; + +const GEMINI_IMAGE_SIZES: Record = { + low: "1K", + medium: "2K", + high: "4K", + "1k": "1K", + "2k": "2K", + "4k": "4K", +}; + +type GeminiInlineImageGenerationSize = "1K" | "2K" | "4K"; + +export function parseGeminiImageDataUrl(dataUrl: string): GeminiInlineImage { + const match = dataUrl.trim().match(/^data:([^;,\s]+)(?:;[^,]*)?;base64,([A-Za-z0-9+/=_-]+)$/i); + if (!match) throw new Error("参考图片读取失败：未得到有效的 image data URL"); + const mimeType = match[1].toLowerCase(); + const data = match[2].replace(/-/g, "+").replace(/_/g, "/"); + if (!mimeType.startsWith("image/")) throw new Error(`参考图片 MIME 类型无效：${mimeType}`); + if (!data) throw new Error("参考图片读取失败：图片数据为空"); + return { mimeType, data }; +} + +export function buildGeminiImageGenerationConfig(size?: string, quality?: string): GeminiImageGenerationConfig { + const imageConfig: NonNullable = {}; + const normalizedSize = size?.trim(); + const normalizedQuality = quality?.trim().toLowerCase(); + if (normalizedSize && normalizedSize !== "auto" && /^\d+:\d+$/.test(normalizedSize)) imageConfig.aspectRatio = normalizedSize; + const imageSize = GEMINI_IMAGE_SIZES[normalizedQuality || ""]; + if (imageSize) imageConfig.imageSize = imageSize; + return { responseModalities: ["TEXT", "IMAGE"], ...(Object.keys(imageConfig).length ? { imageConfig } : {}) }; +} diff --git a/web/src/lib/model-capabilities.ts b/web/src/lib/model-capabilities.ts index a20bd9f0c..c9dd6b8fb 100644 --- a/web/src/lib/model-capabilities.ts +++ b/web/src/lib/model-capabilities.ts @@ -77,10 +77,45 @@ export type VideoCapabilityConfig = { // Keep explicit pixel presets for each resolution tier so the settings panel can // switch between 1K, 2K and 4K without silently converting the requested ratio. const defaultImageSizes = [ - "auto", "1:1", "3:2", "2:3", "4:3", "3:4", "16:9", "21:9", "9:16", - "1024x1024", "1360x1024", "1024x1360", "1536x1024", "1024x1536", "1024x1280", "1280x1024", "2048x878", "1824x1024", "1024x1824", - "2048x2048", "2304x1728", "1728x2304", "2496x1664", "1664x2496", "1792x2240", "2240x1792", "3136x1344", "2752x1536", "1536x2752", - "2880x2880", "3264x2448", "2448x3264", "3504x2336", "2336x3504", "2560x3200", "3200x2560", "3808x1632", "3840x2160", "2160x3840", + "auto", + "1:1", + "3:2", + "2:3", + "4:3", + "3:4", + "16:9", + "21:9", + "9:16", + "1024x1024", + "1360x1024", + "1024x1360", + "1536x1024", + "1024x1536", + "1024x1280", + "1280x1024", + "2048x878", + "1824x1024", + "1024x1824", + "2048x2048", + "2304x1728", + "1728x2304", + "2496x1664", + "1664x2496", + "1792x2240", + "2240x1792", + "3136x1344", + "2752x1536", + "1536x2752", + "2880x2880", + "3264x2448", + "2448x3264", + "3504x2336", + "2336x3504", + "2560x3200", + "3200x2560", + "3808x1632", + "3840x2160", + "2160x3840", ]; export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = ""): ImageCapabilityConfig { @@ -123,6 +158,20 @@ export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = " image.responseFormat.supported = false; image.outputFormat.supported = false; } + if (protocol === "gemini-image") { + image.references.maskSupported = false; + // Gemini Images uses imageConfig.aspectRatio, not the OpenAI-style pixel size field. + image.size = { + parameter: "aspect_ratio", + values: ["auto", "1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9", "21:9"], + default: "1:1", + allowCustom: false, + }; + image.transparentBackground = { supported: false, default: false }; + image.responseFormat = { supported: false }; + image.outputFormat = { supported: false }; + image.maxOutputs = 4; + } if (protocol !== "grok-image" && model.trim().toLowerCase().startsWith("grok-imagine-image")) { image.references.maxImages = 0; image.references.maskSupported = false; @@ -207,20 +256,14 @@ export function modelCapabilityConfigFor(config: { channels: Array<{ id: string; const cost = channel?.modelCosts?.find((item) => item.model === modelName); const fallback = defaultModelCapabilityConfig(cost?.protocol, modelName); if (!cost?.capabilityConfig) return fallback; - const text = cost.capabilityConfig.text - ? { ...fallback.text!, ...cost.capabilityConfig.text, references: { ...fallback.text!.references, ...cost.capabilityConfig.text.references } } - : fallback.text; - const video = cost.capabilityConfig.video - ? { ...fallback.video!, ...cost.capabilityConfig.video, references: { ...fallback.video!.references, ...cost.capabilityConfig.video.references } } - : fallback.video; + const text = cost.capabilityConfig.text ? { ...fallback.text!, ...cost.capabilityConfig.text, references: { ...fallback.text!.references, ...cost.capabilityConfig.text.references } } : fallback.text; + const video = cost.capabilityConfig.video ? { ...fallback.video!, ...cost.capabilityConfig.video, references: { ...fallback.video!.references, ...cost.capabilityConfig.video.references } } : fallback.video; return { ...fallback, ...cost.capabilityConfig, text, image: cost.capabilityConfig.image || fallback.image, video }; } export function normalizeImageValue(profile: ImageCapabilityConfig, value: { size?: string; quality?: string; count?: string; transparentBackground?: string }) { const size = normalizeImageSizeSetting(profile, value.size); - const quality = profile.quality.supported - ? (value.quality && profile.quality.values.includes(value.quality) ? value.quality : profile.quality.default || "auto") - : profile.quality.default || "auto"; + const quality = profile.quality.supported ? (value.quality && profile.quality.values.includes(value.quality) ? value.quality : profile.quality.default || "auto") : profile.quality.default || "auto"; const count = String(Math.max(1, Math.min(profile.maxOutputs, Math.floor(Math.abs(Number(value.count)) || 1)))); const transparentBackground = profile.transparentBackground.supported && value.transparentBackground === "true" ? "true" : "false"; return { size, quality, count, transparentBackground }; @@ -242,16 +285,16 @@ export function imageSizeRequest(profile: ImageCapabilityConfig, value?: string) } export function normalizeVideoValue(profile: VideoCapabilityConfig, value: { seconds?: string; ratio?: string; resolution?: string }) { - const duration = profile.duration.selection === "enum" - ? (profile.duration.values || []).includes(Number(value.seconds)) ? Number(value.seconds) : profile.duration.default - : normalizeRangeDuration(profile, Number(value.seconds)); + const duration = profile.duration.selection === "enum" ? ((profile.duration.values || []).includes(Number(value.seconds)) ? Number(value.seconds) : profile.duration.default) : normalizeRangeDuration(profile, Number(value.seconds)); const ratio = profile.ratios.includes(value.ratio || "") ? value.ratio! : profile.defaultRatio; const resolution = profile.resolutions.includes(value.resolution || "") ? value.resolution! : profile.defaultResolution; return { seconds: String(duration), ratio, resolution }; } export function videoResolutionRequest(profile: VideoCapabilityConfig, value: string | undefined) { - const requested = String(value || "").trim().toLowerCase(); + const requested = String(value || "") + .trim() + .toLowerCase(); if (!requested || requested === "auto" || requested === "default" || requested === "medium" || requested === "high") return undefined; const candidates = [requested]; if (/^\d+$/.test(requested)) candidates.push(`${requested}p`); diff --git a/web/src/lib/model-protocols.ts b/web/src/lib/model-protocols.ts index 867082a97..d6a9cc707 100644 --- a/web/src/lib/model-protocols.ts +++ b/web/src/lib/model-protocols.ts @@ -5,6 +5,7 @@ export type ModelProtocol = | "grok-image" | "volcengine-ark-image" | "volcengine-jimeng-image" + | "gemini-image" | "openai-audio" | "async-audio" | "newapi" @@ -35,6 +36,7 @@ export const MODEL_PROTOCOLS: ModelProtocolDefinition[] = [ { value: "grok-image", label: "Grok Images", capability: "image", create: "POST /v1/images/generations / edits", contentType: "application/json", media: "文生图与单张 URL 参考图，不支持蒙版" }, { value: "volcengine-ark-image", label: "火山方舟图片", capability: "image", create: "POST /api/v3/images/generations", contentType: "application/json", media: "文生图与 image 参考图，不支持蒙版" }, { value: "volcengine-jimeng-image", label: "即梦官方图片", capability: "image", create: "POST CVSync2AsyncSubmitTask", poll: "POST CVSync2AsyncGetResult", contentType: "application/json + AK/SK 签名", media: "0-14 张参考图，模型标识填写 req_key" }, + { value: "gemini-image", label: "Gemini Images", capability: "image", create: "POST /v1beta/models/{model}:generateContent", contentType: "application/json", media: "文生图、图片编辑与多张内嵌参考图" }, { value: "openai-audio", label: "OpenAI Audio", capability: "audio", create: "POST /v1/audio/speech", contentType: "application/json", media: "文本转语音" }, { value: "async-audio", label: "异步音频任务", capability: "audio", create: "POST /v1/audio/tasks", poll: "GET /v1/audio/tasks/{task_id}", contentType: "application/json", media: "语音、音效与音乐生成" }, { value: "newapi", label: "OpenAI / NewAPI Videos", capability: "video", create: "POST /v1/videos", poll: "GET /v1/videos/{task_id}", contentType: "multipart/form-data", media: "input_reference[] 参考图" }, @@ -96,6 +98,7 @@ export function protocolForModelCatalog(endpointTypes: string[] = []): ModelProt const normalized = new Set(endpointTypes.map((value) => value.trim().toLowerCase())); if (normalized.has("openai-chat") || normalized.has("chat-completion") || normalized.has("chat")) return "chat-completion"; if (normalized.has("openai-response") || normalized.has("responses")) return "openai-response"; + if (normalized.has("gemini-image") || normalized.has("gemini-images")) return "gemini-image"; if (normalized.has("openai-image") || normalized.has("image")) return "openai-image"; if (normalized.has("openai-video") || normalized.has("video")) return "newapi-channel-2"; if (normalized.has("openai-audio") || normalized.has("audio")) return "openai-audio"; diff --git a/web/src/services/api/image.ts b/web/src/services/api/image.ts index 2870f2414..4f83e98c0 100644 --- a/web/src/services/api/image.ts +++ b/web/src/services/api/image.ts @@ -12,16 +12,14 @@ import { imageToDataUrl } from "@/services/image-storage"; import type { ReferenceImage } from "@/types/image"; import { withOpenAIPromptCacheKey } from "@/lib/openai-prompt-cache"; import { imageSizeRequest, modelCapabilityConfigFor, normalizeImageValue, type ImageCapabilityConfig } from "@/lib/model-capabilities"; +import { buildGeminiImageGenerationConfig, parseGeminiImageDataUrl, type GeminiImageGenerationConfig } from "@/lib/gemini-image"; export type AiTextMessage = { role: "system" | "user" | "assistant"; content: string | AiTextContentPart[]; }; -export type AiTextContentPart = - | { type: "text"; text: string } - | { type: "image_url"; image_url: { url: string } } - | { type: "file_url"; file_url: { url: string; name: string; mimeType: string } }; +export type AiTextContentPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } } | { type: "file_url"; file_url: { url: string; name: string; mimeType: string } }; export type ResponseToolCall = { id: string; @@ -358,9 +356,7 @@ function toResponseContent(content: ResponseMessageContent): string | ResponseIn return content.map((item) => { if (item.type === "text") return { type: "input_text" as const, text: item.text }; if (item.type === "image_url") return { type: "input_image" as const, image_url: item.image_url.url }; - return item.file_url.url.startsWith("data:") - ? { type: "input_file" as const, filename: item.file_url.name, file_data: item.file_url.url } - : { type: "input_file" as const, filename: item.file_url.name, file_url: item.file_url.url }; + return item.file_url.url.startsWith("data:") ? { type: "input_file" as const, filename: item.file_url.name, file_data: item.file_url.url } : { type: "input_file" as const, filename: item.file_url.name, file_url: item.file_url.url }; }); } @@ -402,9 +398,7 @@ function toChatCompletionContent(content: ResponseMessageContent) { if (!Array.isArray(content)) return content; return content.map((item) => { if (item.type !== "file_url") return item; - const file = item.file_url.url.startsWith("data:") - ? { filename: item.file_url.name, file_data: item.file_url.url } - : { filename: item.file_url.name, file_url: item.file_url.url }; + const file = item.file_url.url.startsWith("data:") ? { filename: item.file_url.name, file_data: item.file_url.url } : { filename: item.file_url.name, file_url: item.file_url.url }; return { type: "file", file }; }); } @@ -710,7 +704,7 @@ function toGeminiFilePart(url: string, mimeType: string): GeminiPart { function geminiTextContent(content: ResponseMessageContent) { if (!Array.isArray(content)) return String(content || ""); - return content.map((item) => item.type === "text" ? item.text : item.type === "image_url" ? item.image_url.url : `${item.file_url.name}: ${item.file_url.url}`).join("\n"); + return content.map((item) => (item.type === "text" ? item.text : item.type === "image_url" ? item.image_url.url : `${item.file_url.name}: ${item.file_url.url}`)).join("\n"); } function jsonObject(value: string): Record { @@ -809,8 +803,14 @@ function consumeGeminiStreamBlock(block: string, state: GeminiStreamState, onDel function parseGeminiToolResponse(payload: GeminiPayload): ToolResponseResult { validateGeminiPayload(payload); const parts = payload.candidates?.flatMap((candidate) => candidate.content?.parts || []) || []; - const content = parts.filter((part) => !part.thought).map((part) => part.text || "").join(""); - const reasoning = parts.filter((part) => part.thought).map((part) => part.text || "").join(""); + const content = parts + .filter((part) => !part.thought) + .map((part) => part.text || "") + .join(""); + const reasoning = parts + .filter((part) => part.thought) + .map((part) => part.text || "") + .join(""); const toolCalls = parts .map((part) => part.functionCall) .filter((call): call is NonNullable => Boolean(call?.name)) @@ -827,21 +827,27 @@ function parseGeminiToolResponse(payload: GeminiPayload): ToolResponseResult { return { content, toolCalls, ...(reasoning ? { reasoning } : {}) }; } -async function requestGeminiImages(config: AiConfig, prompt: string, references: ReferenceImage[], count: number, options?: RequestOptions) { - const requests = Array.from({ length: count }, () => requestGeminiImagesOnce(config, prompt, references, options)); +async function requestGeminiImages(config: AiConfig, prompt: string, references: ReferenceImage[], count: number, generationConfig: GeminiImageGenerationConfig, options?: RequestOptions) { + // 参考图先完整读取并校验一次，再复用已解析的 inlineData；不能让每个输出重新读取 storageKey， + // 否则资源缓存瞬时未命中时会出现“偶发缺参考图”或把 storageKey 当 URL 请求的问题。 + const referenceParts = await Promise.all( + references.map(async (image) => { + const dataUrl = await imageToDataUrl(image); + const inlineData = parseGeminiImageDataUrl(dataUrl); + return { inlineData }; + }), + ); + const requests = Array.from({ length: count }, () => requestGeminiImagesOnce(config, prompt, referenceParts, generationConfig, options)); return (await Promise.all(requests)).flat(); } -async function requestGeminiImagesOnce(config: AiConfig, prompt: string, references: ReferenceImage[], options?: RequestOptions) { - const parts: GeminiPart[] = [{ text: prompt }]; - for (const image of references) { - parts.push(toGeminiFilePart(await imageToDataUrl(image), image.type || "image/png")); - } +async function requestGeminiImagesOnce(config: AiConfig, prompt: string, referenceParts: GeminiPart[], generationConfig: GeminiImageGenerationConfig, options?: RequestOptions) { + const parts: GeminiPart[] = [{ text: prompt }, ...referenceParts]; const request = channelRequest(config, geminiApiUrl(config, "generateContent"), geminiHeaders(config)); const response = await axios.post( request.url, { - ...toGeminiBody(config, [{ role: "user", content: prompt }], { generationConfig: { responseModalities: ["TEXT", "IMAGE"] } }), + ...toGeminiBody(config, [{ role: "user", content: prompt }], { generationConfig }), contents: [{ role: "user", parts }], }, { headers: request.headers, withCredentials: request.credentials === "include", signal: options?.signal }, @@ -872,9 +878,9 @@ export async function requestGeneration(config: AiConfig, prompt: string, option validateImageCapability(imageProfile, []); const normalizedImage = normalizeImageValue(imageProfile, config); const n = Number(normalizedImage.count); - if (requestConfig.apiFormat === "gemini") { + if (requestConfig.interfaceType === "gemini-image") { try { - return await requestGeminiImages(requestConfig, prompt, [], n, options); + return await requestGeminiImages(requestConfig, prompt, [], n, buildGeminiImageGenerationConfig(normalizedImage.size, normalizedImage.quality), options); } catch (error) { throw new Error(readAxiosError(error, "请求失败")); } @@ -959,10 +965,10 @@ export async function requestEdit(config: AiConfig, prompt: string, references: const normalizedImage = normalizeImageValue(imageProfile, config); const n = Number(normalizedImage.count); const requestPrompt = buildImageReferencePromptText(prompt, references); - if (requestConfig.apiFormat === "gemini") { + if (requestConfig.interfaceType === "gemini-image") { if (mask) throw new Error("Gemini 调用格式暂不支持蒙版编辑"); try { - return await requestGeminiImages(requestConfig, requestPrompt, references, n, options); + return await requestGeminiImages(requestConfig, requestPrompt, references, n, buildGeminiImageGenerationConfig(normalizedImage.size, normalizedImage.quality), options); } catch (error) { throw new Error(readAxiosError(error, "请求失败")); } diff --git a/web/src/stores/use-config-store.ts b/web/src/stores/use-config-store.ts index 02a236155..be76725ee 100644 --- a/web/src/stores/use-config-store.ts +++ b/web/src/stores/use-config-store.ts @@ -516,7 +516,7 @@ export function resolveModelRequestConfig(config: AiConfig, value: string) { apiKey: channel.apiKey, secretKey: channel.secretKey, headers: channel.headers, - apiFormat: interfaceType ? (interfaceType === "gemini-veo" ? ("gemini" as const) : ("openai" as const)) : channel.apiFormat, + apiFormat: interfaceType ? (interfaceType === "gemini-veo" || interfaceType === "gemini-image" ? ("gemini" as const) : ("openai" as const)) : channel.apiFormat, interfaceType, channelId: channel.scope === "system" ? channel.id : "", }); @@ -564,7 +564,7 @@ export function defaultBaseUrlForApiFormat(apiFormat: ApiCallFormat) { } export function defaultBaseUrlForChannelInterface(interfaceType?: ChannelInterfaceType) { - if (interfaceType === "gemini-veo") return GEMINI_BASE_URL; + if (interfaceType === "gemini-veo" || interfaceType === "gemini-image") return GEMINI_BASE_URL; if (interfaceType === "novita-video") return "https://api.novita.ai/v3"; if (interfaceType === "volcengine-ark-image" || interfaceType === "volcengine-ark-video") return "https://ark.cn-beijing.volces.com/api/v3"; if (interfaceType === "volcengine-jimeng-image" || interfaceType === "volcengine-jimeng-video") return "https://visual.volcengineapi.com"; diff --git a/web/test/gemini-image.test.ts b/web/test/gemini-image.test.ts new file mode 100644 index 000000000..712ead416 --- /dev/null +++ b/web/test/gemini-image.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, test } from "bun:test"; + +import { buildGeminiImageGenerationConfig, parseGeminiImageDataUrl } from "../src/lib/gemini-image"; + +describe("Gemini Images request helpers", () => { + test("拆分参考图 data URL 时使用真实 MIME 和裸 base64", () => { + expect(parseGeminiImageDataUrl("data:image/jpeg;base64,aGVsbG8=")).toEqual({ mimeType: "image/jpeg", data: "aGVsbG8=" }); + }); + + test("拒绝空图和非图片 data URL，避免把错误参考图发给 Gemini", () => { + expect(() => parseGeminiImageDataUrl("")).toThrow("未得到有效的 image data URL"); + expect(() => parseGeminiImageDataUrl("data:text/plain;base64,aGVsbG8=")).toThrow("MIME 类型无效"); + }); + + test("把统一图片选项映射到 Gemini imageConfig", () => { + expect(buildGeminiImageGenerationConfig("16:9", "high")).toEqual({ + responseModalities: ["TEXT", "IMAGE"], + imageConfig: { aspectRatio: "16:9", imageSize: "4K" }, + }); + expect(buildGeminiImageGenerationConfig("auto", "auto")).toEqual({ responseModalities: ["TEXT", "IMAGE"] }); + }); +});