tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(platform): 创作能力整合 - 新增 Gemini Images 图片请求协议

Browse Source
This commit is contained in:
ddCat
2026-08-18 21:42:22 +08:00
parent a23b4dda93
commit 778551127c
17 changed files with 489 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/internal/handler/auth.go
View File

@@ -805,7 +805,7 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User,
}
service.ApplyOutboundHeaders(upstreamReq, channelHeaders)
service.ApplyDefaultOutboundHeaders(upstreamReq)
if protocol == model.ChannelInterfaceGeminiVeo {
if protocol == model.ChannelInterfaceGeminiVeo || protocol == model.ChannelInterfaceGeminiImage {
upstreamReq.Header.Set("x-goog-api-key", channel.APIKey)
} else {
upstreamReq.Header.Set("Authorization", "Bearer "+channel.APIKey)
@@ -867,7 +867,7 @@ func proxySystemRequest(c *gin.Context, svc *service.Service, user *model.User,
func apiCallLog(user *model.User, channel *model.ModelChannel, billingOrderID string, capability string, protocol model.ChannelInterfaceType, method string, path string, target string, body []byte, contentType string, status model.ApiCallStatus, statusCode int, duration time.Duration, errorText string, concurrencyLimit int) model.ApiCallLog {
requestKind := "create"
apiFormat := "openai"
if protocol == model.ChannelInterfaceGeminiVeo {
if protocol == model.ChannelInterfaceGeminiVeo || protocol == model.ChannelInterfaceGeminiImage {
apiFormat = "gemini"
}
if method == http.MethodGet {

4
backend/internal/handler/security.go
View File

@@ -160,7 +160,7 @@ func authorizeSystemProxy(channel *model.ModelChannel, protocol model.ChannelInt
if method == http.MethodGet && requestPath == "/models" {
return nil
}
if protocol == model.ChannelInterfaceGeminiVeo {
if protocol == model.ChannelInterfaceGeminiVeo || protocol == model.ChannelInterfaceGeminiImage {
matches := geminiGeneratePath.FindStringSubmatch(requestPath)
if method != http.MethodPost || len(matches) != 3 {
return errors.New("系统渠道不允许访问该上游接口")
@@ -196,7 +196,7 @@ func interfaceAllowsProxyPath(interfaceType model.ChannelInterfaceType, requestP
return requestPath == "/images/generations"
case model.ChannelInterfaceOpenAIAudio:
return requestPath == "/audio/speech"
case model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo, model.ChannelInterfaceNovitaVideo:
case model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo, model.ChannelInterfaceGeminiImage, model.ChannelInterfaceNovitaVideo:
return false
default:
return true

1
backend/internal/model/models.go
View File

@@ -64,6 +64,7 @@ const (
ChannelInterfaceGrokImage ChannelInterfaceType = "grok-image"
ChannelInterfaceVolcengineArkImage ChannelInterfaceType = "volcengine-ark-image"
ChannelInterfaceVolcengineJiMengImage ChannelInterfaceType = "volcengine-jimeng-image"
ChannelInterfaceGeminiImage ChannelInterfaceType = "gemini-image"
ChannelInterfaceOpenAIAudio ChannelInterfaceType = "openai-audio"
ChannelInterfaceAsyncAudio ChannelInterfaceType = "async-audio"
ChannelInterfaceNewAPIVideo ChannelInterfaceType = "newapi"

2
backend/internal/service/admin.go
View File

@@ -816,7 +816,7 @@ func mergeChannelRequest(req ChannelRequest, channel model.ModelChannel) Channel
func validChannelInterfaceType(value model.ChannelInterfaceType) bool {
switch value {
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo:
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceGeminiImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo:
return true
default:
return false

2
backend/internal/service/channel_models.go
View File

@@ -510,7 +510,7 @@ func (s *Service) syncChannelModelNames(channel *model.ModelChannel) error {
func capabilityForProtocol(protocol model.ChannelInterfaceType) string {
switch protocol {
case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage:
case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceGeminiImage:
return "image"
case model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio:
return "audio"

8
backend/internal/service/model_capability.go
View File

@@ -143,6 +143,14 @@ func DefaultImageCapabilityConfig(protocol string, modelName string) *ImageCapab
image.TransparentBackground.Supported = false
image.ResponseFormat.Supported = false
image.OutputFormat.Supported = false
case model.ChannelInterfaceGeminiImage:
image.References.MaskSupported = false
// Gemini Images uses imageConfig.aspectRatio, not the OpenAI-style pixel size field.
image.Size = ImageSizeConfig{Parameter: "aspect_ratio", Values: []string{"auto", "1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9", "21:9"}, Default: "1:1", AllowCustom: false}
image.TransparentBackground.Supported = false
image.ResponseFormat.Supported = false
image.OutputFormat.Supported = false
image.MaxOutputs = 4
}
if model.ChannelInterfaceType(protocol) != model.ChannelInterfaceGrokImage && strings.HasPrefix(strings.ToLower(strings.TrimSpace(modelName)), "grok-imagine-image") {
image.References.MaxImages = 0

146
backend/internal/service/provider.go
View File

@@ -210,7 +210,7 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string
return nil, err
}
}
if input.Config.APIFormat == "gemini" && input.Config.InterfaceType != string(model.ChannelInterfaceGeminiVeo) {
if input.Config.APIFormat == "gemini" && input.Config.InterfaceType != string(model.ChannelInterfaceGeminiVeo) && input.Config.InterfaceType != string(model.ChannelInterfaceGeminiImage) {
return nil, errors.New("后端任务队列暂不支持 Gemini 调用格式，请使用 OpenAI 兼容渠道")
}
if strings.TrimSpace(input.Config.BaseURL) == "" || strings.TrimSpace(input.Config.APIKey) == "" || strings.TrimSpace(input.Config.Model) == "" {
@@ -621,7 +621,7 @@ func (s *Service) resolveProviderConfig(config providerConfig) (providerConfig,
}
config.InterfaceType = string(channelModel.Protocol)
// 模型协议是实际请求契约；混合渠道中鉴权格式也必须随模型协议切换。
if config.InterfaceType == string(model.ChannelInterfaceGeminiVeo) {
if config.InterfaceType == string(model.ChannelInterfaceGeminiVeo) || config.InterfaceType == string(model.ChannelInterfaceGeminiImage) {
config.APIFormat = "gemini"
} else if config.InterfaceType != "" {
config.APIFormat = "openai"
@@ -674,6 +674,9 @@ func runImageTask(ctx context.Context, input canvasGenerationInput) (map[string]
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineArkImage) {
return runVolcengineArkImageTask(ctx, input)
}
if input.Config.InterfaceType == string(model.ChannelInterfaceGeminiImage) {
return runGeminiImageTask(ctx, input)
}
var payload imageResponse
if input.Mask != nil {
// 蒙版编辑是强校验写路径：协议能力不明确时必须失败，不能静默退化为整图重绘。
@@ -753,6 +756,143 @@ func runImageTask(ctx context.Context, input canvasGenerationInput) (map[string]
return map[string]interface{}{"mode": "image", "images": images}, nil
}
func runGeminiImageTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) {
if input.Mask != nil {
return nil, errors.New("Gemini Images 不支持蒙版编辑，请移除蒙版后重试")
}
if len(input.ReferenceVideos) > 0 || len(input.ReferenceAudios) > 0 {
return nil, errors.New("Gemini Images 不支持参考视频或音频")
}
parts := make([]geminiImageContentPart, 0, 1+len(input.ReferenceImages))
if prompt := strings.TrimSpace(input.Prompt); prompt != "" {
parts = append(parts, geminiImageContentPart{Text: prompt})
}
for _, image := range input.ReferenceImages {
raw, mimeType, err := geminiImageBytes(image)
if err != nil {
return nil, fmt.Errorf("读取 Gemini Images 参考图失败：%w", err)
}
parts = append(parts, geminiImageContentPart{InlineData: &geminiImageInlineData{MIMEType: mimeType, Data: base64.StdEncoding.EncodeToString(raw)}})
}
if len(parts) == 0 {
return nil, errors.New("Gemini Images 请求缺少提示词或参考图")
}
body := geminiImageRequest{
Contents: []geminiImageContent{{Role: "user", Parts: parts}},
GenerationConfig: geminiImageGenerationConfig{
ResponseModalities: []string{"TEXT", "IMAGE"},
ImageConfig: geminiImageConfigFor(input.Config),
},
}
if systemPrompt := strings.TrimSpace(input.Config.SystemPrompt); systemPrompt != "" {
body.SystemInstruction = &geminiImageContent{Parts: []geminiImageContentPart{{Text: systemPrompt}}}
}
var payload map[string]interface{}
path := "/models/" + url.PathEscape(input.Config.Model) + ":generateContent"
if err := postGeminiJSON(ctx, input.Config, path, body, &payload); err != nil {
return nil, err
}
images, err := geminiImageDataURLs(payload)
if err != nil {
return nil, err
}
return map[string]interface{}{"mode": "image", "images": images}, nil
}
func geminiImageConfigFor(config providerConfig) *geminiImageConfig {
imageConfig := &geminiImageConfig{}
size := strings.TrimSpace(config.Size)
if size != "" && size != "auto" && strings.Count(size, ":") == 1 {
imageConfig.AspectRatio = size
}
switch strings.ToLower(strings.TrimSpace(config.Quality)) {
case "low", "1k":
imageConfig.ImageSize = "1K"
case "medium", "2k":
imageConfig.ImageSize = "2K"
case "high", "4k":
imageConfig.ImageSize = "4K"
}
if imageConfig.AspectRatio == "" && imageConfig.ImageSize == "" {
return nil
}
return imageConfig
}
func geminiImageBytes(media providerMedia) ([]byte, string, error) {
raw, mimeType, err := mediaBytes(media)
if err != nil {
return nil, "", err
}
if len(raw) == 0 {
return nil, "", errors.New("参考图片数据为空")
}
detected := strings.TrimSpace(strings.Split(http.DetectContentType(raw), ";")[0])
if !strings.HasPrefix(strings.ToLower(mimeType), "image/") {
if !strings.HasPrefix(strings.ToLower(detected), "image/") {
return nil, "", fmt.Errorf("参考图片 MIME 类型无效：%s", defaultString(mimeType, detected))
}
mimeType = detected
} else if !strings.HasPrefix(strings.ToLower(detected), "image/") {
// 以实际字节签名为准，避免错误的 data URL MIME 把非图片内容伪装成图片。
return nil, "", fmt.Errorf("参考图片内容不是有效图片：%s", defaultString(detected, mimeType))
}
return raw, mimeType, nil
}
func geminiImageDataURLs(payload map[string]interface{}) ([]map[string]string, error) {
if errorValue, ok := payload["error"].(map[string]interface{}); ok {
if message := stringField(errorValue, "message"); message != "" {
return nil, errors.New(message)
}
}
candidates, _ := payload["candidates"].([]interface{})
images := make([]map[string]string, 0)
for _, candidateValue := range candidates {
candidate, _ := candidateValue.(map[string]interface{})
content, _ := candidate["content"].(map[string]interface{})
parts, _ := content["parts"].([]interface{})
for _, partValue := range parts {
part, _ := partValue.(map[string]interface{})
inlineData, _ := part["inlineData"].(map[string]interface{})
if inlineData == nil {
inlineData, _ = part["inline_data"].(map[string]interface{})
}
if inlineData != nil {
data := strings.TrimSpace(stringField(inlineData, "data"))
mimeType := firstNonEmptyString(stringField(inlineData, "mimeType"), stringField(inlineData, "mime_type"))
if data == "" {
continue
}
if mimeType == "" {
mimeType = "image/png"
}
if !strings.HasPrefix(strings.ToLower(mimeType), "image/") {
return nil, fmt.Errorf("Gemini Images 返回了非图片 MIME 类型：%s", mimeType)
}
decoded, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return nil, fmt.Errorf("Gemini Images 返回的图片数据无效：%w", err)
}
images = append(images, map[string]string{"dataUrl": dataURL(mimeType, decoded)})
continue
}
fileData, _ := part["fileData"].(map[string]interface{})
if fileData != nil {
if fileURL := firstNonEmptyString(stringField(fileData, "fileUri"), stringField(fileData, "file_uri")); fileURL != "" {
images = append(images, map[string]string{"dataUrl": fileURL})
}
}
}
}
if len(images) == 0 {
return nil, errors.New("Gemini Images 接口没有返回图片")
}
return images, nil
}
func runGrokImageTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) {
body, path, err := grokImageRequestBody(input)
if err != nil {
@@ -2257,7 +2397,7 @@ func validateGenerationInterface(mode string, interfaceType string) error {
}
allowed := map[string]map[string]bool{
"text": {"chat-completion": true, "openai-response": true},
"image": {"openai-image": true, "grok-image": true, "volcengine-ark-image": true, "volcengine-jimeng-image": true},
"image": {"openai-image": true, "grok-image": true, "volcengine-ark-image": true, "volcengine-jimeng-image": true, "gemini-image": true},
"video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true, "xai-video": true, "volcengine-ark-video": true, "volcengine-jimeng-video": true, "gemini-veo": true, "novita-video": true},
"audio": {"openai-audio": true, "async-audio": true},
}

31
backend/internal/service/provider_request_types.go
View File

@@ -90,6 +90,37 @@ type geminiVeoParameters struct {
SampleCount int `json:"sampleCount"`
}
type geminiImageRequest struct {
Contents []geminiImageContent `json:"contents"`
SystemInstruction *geminiImageContent `json:"systemInstruction,omitempty"`
GenerationConfig geminiImageGenerationConfig `json:"generationConfig"`
}
type geminiImageContent struct {
Role string `json:"role,omitempty"`
Parts []geminiImageContentPart `json:"parts"`
}
type geminiImageContentPart struct {
Text string `json:"text,omitempty"`
InlineData *geminiImageInlineData `json:"inlineData,omitempty"`
}
type geminiImageInlineData struct {
MIMEType string `json:"mimeType"`
Data string `json:"data"`
}
type geminiImageGenerationConfig struct {
ResponseModalities []string `json:"responseModalities"`
ImageConfig *geminiImageConfig `json:"imageConfig,omitempty"`
}
type geminiImageConfig struct {
AspectRatio string `json:"aspectRatio,omitempty"`
ImageSize string `json:"imageSize,omitempty"`
}
type seedanceAgentPlanRequest struct {
Model string `json:"model"`
Content []map[string]interface{} `json:"content"`

81
backend/internal/service/provider_test.go
View File

@@ -19,6 +19,7 @@ import (
)
const testReferenceImageDataURL = "data:image/png;base64,aGVsbG8="
const testGeminiReferenceImageDataURL = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
func TestWriteMediaPartSanitizesFilenameAndSetsMimeType(t *testing.T) {
var body bytes.Buffer
@@ -236,6 +237,86 @@ func TestVolcengineArkImageRejectsMaskBeforeRequest(t *testing.T) {
}
}
func TestRunGeminiImageTaskUsesInlineDataAndImageConfig(t *testing.T) {
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1beta/models/gemini-test:generateContent" {
t.Errorf("path = %q, want /v1beta/models/gemini-test:generateContent", r.URL.Path)
}
if got := r.Header.Get("x-goog-api-key"); got != "test-key" {
t.Errorf("x-goog-api-key = %q, want test-key", got)
}
var body map[string]interface{}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode request body: %v", err)
}
contents, ok := body["contents"].([]interface{})
if !ok || len(contents) != 1 {
t.Fatalf("contents = %#v", body["contents"])
}
content, _ := contents[0].(map[string]interface{})
parts, ok := content["parts"].([]interface{})
if !ok || len(parts) != 2 {
t.Fatalf("parts = %#v", content["parts"])
}
textPart, _ := parts[0].(map[string]interface{})
if textPart["text"] != "edit this image" {
t.Errorf("text part = %#v", textPart)
}
imagePart, _ := parts[1].(map[string]interface{})
inlineData, _ := imagePart["inlineData"].(map[string]interface{})
if inlineData["mimeType"] != "image/png" || inlineData["data"] != "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=" {
t.Errorf("inlineData = %#v", inlineData)
}
generationConfig, _ := body["generationConfig"].(map[string]interface{})
modalities, _ := generationConfig["responseModalities"].([]interface{})
if !reflect.DeepEqual(modalities, []interface{}{"TEXT", "IMAGE"}) {
t.Errorf("responseModalities = %#v", modalities)
}
imageConfig, _ := generationConfig["imageConfig"].(map[string]interface{})
if imageConfig["aspectRatio"] != "16:9" || imageConfig["imageSize"] != "4K" {
t.Errorf("imageConfig = %#v", imageConfig)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"candidates":[{"content":{"parts":[{"inlineData":{"mimeType":"image/png","data":"aGVsbG8="}}]}}]}`))
}))
defer server.Close()
result, err := runImageTask(context.Background(), canvasGenerationInput{
Prompt: "edit this image",
Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", APIFormat: "gemini", Model: "gemini-test", InterfaceType: "gemini-image", Size: "16:9", Quality: "high"},
ReferenceImages: []providerMedia{{DataURL: testGeminiReferenceImageDataURL}},
})
if err != nil {
t.Fatalf("runImageTask() error = %v", err)
}
images, _ := result["images"].([]map[string]string)
if len(images) != 1 || images[0]["dataUrl"] != testReferenceImageDataURL {
t.Fatalf("images = %#v", result["images"])
}
}
func TestRunGeminiImageTaskRejectsInvalidReferenceBeforeRequest(t *testing.T) {
called := false
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
called = true
http.Error(w, "unexpected upstream request", http.StatusInternalServerError)
}))
defer server.Close()
_, err := runImageTask(context.Background(), canvasGenerationInput{
Prompt: "edit this image",
Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", APIFormat: "gemini", Model: "gemini-test", InterfaceType: "gemini-image"},
ReferenceImages: []providerMedia{{DataURL: "data:text/plain;base64,aGVsbG8="}},
})
if err == nil || !strings.Contains(err.Error(), "读取 Gemini Images 参考图失败") || !strings.Contains(err.Error(), "MIME 类型无效") {
t.Fatalf("runImageTask() error = %v", err)
}
if called {
t.Fatal("invalid reference image must be rejected before upstream request")
}
}
func TestRunImageTaskOmitsAutomaticQualityAndSize(t *testing.T) {
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

2
web/package.json
View File

@@ -7,7 +7,7 @@
"dev": "vite --host 0.0.0.0 --port 3000",
"build": "tsc --noEmit && vite build",
"typecheck": "tsc --noEmit",
"test": "bun test test/channel-model-catalog.test.ts test/agent-prompt-cache.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-generation-layout.test.ts test/canvas-image-batch-retry.test.ts test/canvas-leafer-viewport.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-references.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts && bun test test/local-dreamina-cross-runtime.test.ts",
"test": "bun test test/channel-model-catalog.test.ts test/agent-prompt-cache.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-generation-layout.test.ts test/canvas-image-batch-retry.test.ts test/canvas-leafer-viewport.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-references.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts && bun test test/local-dreamina-cross-runtime.test.ts",
"start": "vite preview --host 0.0.0.0 --port 3000",
"format": "prettier --write .",
"format:check": "prettier --check ."

80
web/src/components/model-protocol-picker.tsx
View File

@@ -42,7 +42,11 @@ export function CapabilityCardPicker({ value, onChange, density = "comfortable"
onClick={() => onChange?.(item.value)}
>
<span className={cn("grid place-items-center rounded-md", density === "compact" ? "size-6" : "size-8", selected ? "bg-foreground text-background" : "bg-muted text-foreground/65")}>{item.icon}</span>
{selected ? <span className={cn("absolute grid place-items-center rounded-full bg-foreground text-background", density === "compact" ? "right-1.5 top-1.5 size-4" : "right-2.5 top-2.5 size-5")}><Check className={density === "compact" ? "size-2.5" : "size-3"} /></span> : null}
{selected ? (
<span className={cn("absolute grid place-items-center rounded-full bg-foreground text-background", density === "compact" ? "right-1.5 top-1.5 size-4" : "right-2.5 top-2.5 size-5")}>
<Check className={density === "compact" ? "size-2.5" : "size-3"} />
</span>
) : null}
<span className={cn("block font-semibold", density === "compact" ? "mt-1 text-xs" : "mt-2 text-sm")}>{item.label}</span>
<span className={cn("block text-foreground/48", density === "compact" ? "text-[var(--fs-micro)]" : "text-xs")}>{item.description}</span>
{density === "comfortable" ? <BrandIconRow models={item.brands} className="mt-auto pt-2" /> : null}
@@ -79,14 +83,20 @@ export function ProtocolCardPicker({ capability, value, onChange, density = "com
<div className={cn("mt-0.5 truncate font-mono text-foreground/48", density === "compact" ? "text-[var(--fs-micro)]" : "text-[var(--fs-tiny)]")}>{protocol.create}</div>
</div>
</div>
{selected ? <span className={cn("absolute grid place-items-center rounded-full bg-foreground text-background", density === "compact" ? "right-1.5 top-1.5 size-4" : "right-2.5 top-2.5 size-5")}><Check className={density === "compact" ? "size-2.5" : "size-3"} /></span> : null}
{density === "comfortable" ? <>
<div className="mt-2 line-clamp-2 text-xs leading-5 text-foreground/58">{protocol.media}</div>
<div className="mt-auto flex items-center justify-between gap-2 pt-2 text-[var(--fs-tiny)] text-foreground/42">
<span className="truncate">{protocol.contentType}</span>
{protocol.poll ? <span className="shrink-0">异步轮询</span> : <span className="shrink-0">同步响应</span>}
</div>
</> : null}
{selected ? (
<span className={cn("absolute grid place-items-center rounded-full bg-foreground text-background", density === "compact" ? "right-1.5 top-1.5 size-4" : "right-2.5 top-2.5 size-5")}>
<Check className={density === "compact" ? "size-2.5" : "size-3"} />
</span>
) : null}
{density === "comfortable" ? (
<>
<div className="mt-2 line-clamp-2 text-xs leading-5 text-foreground/58">{protocol.media}</div>
<div className="mt-auto flex items-center justify-between gap-2 pt-2 text-[var(--fs-tiny)] text-foreground/42">
<span className="truncate">{protocol.contentType}</span>
{protocol.poll ? <span className="shrink-0">异步轮询</span> : <span className="shrink-0">同步响应</span>}
</div>
</>
) : null}
</button>
);
})}
@@ -103,16 +113,24 @@ export function ModelCapabilityProtocolModal({ value, onChange }: { value: { cap
}, [open, value]);
const updateCapability = (capability: ModelCapabilityChoice) => {
const protocol = draft.protocol && MODEL_PROTOCOLS.some((item) => item.value === draft.protocol && item.capability === capability)
? draft.protocol
: MODEL_PROTOCOLS.find((item) => item.capability === capability)?.value || "chat-completion";
const protocol = draft.protocol && MODEL_PROTOCOLS.some((item) => item.value === draft.protocol && item.capability === capability) ? draft.protocol : MODEL_PROTOCOLS.find((item) => item.capability === capability)?.value || "chat-completion";
setDraft({ capability, protocol });
};
return (
<>
<Button size="small" className="max-w-full justify-start" icon={<Settings2 className="size-3.5" />} onClick={() => { setDraft(value); setOpen(true); }}>
<span className="max-w-[min(56vw,360px)] truncate">{capabilityLabel(value.capability)} · {modelProtocolLabel(value.protocol)}</span>
<Button
size="small"
className="max-w-full justify-start"
icon={<Settings2 className="size-3.5" />}
onClick={() => {
setDraft(value);
setOpen(true);
}}
>
<span className="max-w-[min(56vw,360px)] truncate">
{capabilityLabel(value.capability)} · {modelProtocolLabel(value.protocol)}
</span>
</Button>
<Modal
title="配置模型能力与请求协议"
@@ -123,7 +141,10 @@ export function ModelCapabilityProtocolModal({ value, onChange }: { value: { cap
onCancel={() => setOpen(false)}
okText="应用配置"
cancelText="取消"
onOk={() => { onChange(draft); setOpen(false); }}
onOk={() => {
onChange(draft);
setOpen(false);
}}
>
<div className="space-y-4">
<section>
@@ -143,17 +164,36 @@ export function ModelCapabilityProtocolModal({ value, onChange }: { value: { cap
function ProtocolBrandMark({ protocol, compact = false }: { protocol: ModelProtocolDefinition; compact?: boolean }) {
const iconSize = compact ? "size-6" : "size-8";
if (protocol.value === "chat-completion") return <BrandIconRow models={["openai", "deepseek", "glm"]} compact={compact} />;
if (protocol.value.startsWith("volcengine-jimeng-")) return <span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}><Sparkles className={compact ? "size-3" : "size-4"} /></span>;
if (protocol.value.startsWith("volcengine-")) return <span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}><Flame className={compact ? "size-3" : "size-4"} /></span>;
if (protocol.value.startsWith("newapi-channel-") || protocol.value === "novita-video") return <span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}><Network className={compact ? "size-3" : "size-4"} /></span>;
const brand = protocol.value === "gemini-veo" ? "gemini" : protocol.value === "grok-image" || protocol.value === "xai-video" ? "grok" : "openai";
if (protocol.value.startsWith("volcengine-jimeng-"))
return (
<span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}>
<Sparkles className={compact ? "size-3" : "size-4"} />
</span>
);
if (protocol.value.startsWith("volcengine-"))
return (
<span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}>
<Flame className={compact ? "size-3" : "size-4"} />
</span>
);
if (protocol.value.startsWith("newapi-channel-") || protocol.value === "novita-video")
return (
<span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}>
<Network className={compact ? "size-3" : "size-4"} />
</span>
);
const brand = protocol.value === "gemini-veo" || protocol.value === "gemini-image" ? "gemini" : protocol.value === "grok-image" || protocol.value === "xai-video" ? "grok" : "openai";
return <BrandIconRow models={[brand]} compact={compact} />;
}
function BrandIconRow({ models, compact = false, className }: { models: string[]; compact?: boolean; className?: string }) {
return (
<span className={cn("flex items-center -space-x-1", compact && "shrink-0", className)} aria-hidden="true">
{models.map((model) => <span key={model} className={cn("grid shrink-0 place-items-center rounded-md border border-border/70 bg-background", compact ? "size-6" : "size-6")} title={modelBrandLabel(model)}><ModelIcon model={model} /></span>)}
{models.map((model) => (
<span key={model} className={cn("grid shrink-0 place-items-center rounded-md border border-border/70 bg-background", compact ? "size-6" : "size-6")} title={modelBrandLabel(model)}>
<ModelIcon model={model} />
</span>
))}
</span>
);
}

40
web/src/lib/gemini-image.ts Normal file
View File

@@ -0,0 +1,40 @@
export type GeminiImageGenerationConfig = {
responseModalities: ["TEXT", "IMAGE"];
imageConfig?: { aspectRatio?: string; imageSize?: "1K" | "2K" | "4K" };
};
export type GeminiInlineImage = {
mimeType: string;
data: string;
};
const GEMINI_IMAGE_SIZES: Record<string, GeminiInlineImageGenerationSize> = {
low: "1K",
medium: "2K",
high: "4K",
"1k": "1K",
"2k": "2K",
"4k": "4K",
};
type GeminiInlineImageGenerationSize = "1K" | "2K" | "4K";
export function parseGeminiImageDataUrl(dataUrl: string): GeminiInlineImage {
const match = dataUrl.trim().match(/^data:([^;,\s]+)(?:;[^,]*)?;base64,([A-Za-z0-9+/=_-]+)$/i);
if (!match) throw new Error("参考图片读取失败：未得到有效的 image data URL");
const mimeType = match[1].toLowerCase();
const data = match[2].replace(/-/g, "+").replace(/_/g, "/");
if (!mimeType.startsWith("image/")) throw new Error(`参考图片 MIME 类型无效：${mimeType}`);
if (!data) throw new Error("参考图片读取失败：图片数据为空");
return { mimeType, data };
}
export function buildGeminiImageGenerationConfig(size?: string, quality?: string): GeminiImageGenerationConfig {
const imageConfig: NonNullable<GeminiImageGenerationConfig["imageConfig"]> = {};
const normalizedSize = size?.trim();
const normalizedQuality = quality?.trim().toLowerCase();
if (normalizedSize && normalizedSize !== "auto" && /^\d+:\d+$/.test(normalizedSize)) imageConfig.aspectRatio = normalizedSize;
const imageSize = GEMINI_IMAGE_SIZES[normalizedQuality || ""];
if (imageSize) imageConfig.imageSize = imageSize;
return { responseModalities: ["TEXT", "IMAGE"], ...(Object.keys(imageConfig).length ? { imageConfig } : {}) };
}

77
web/src/lib/model-capabilities.ts
View File

@@ -77,10 +77,45 @@ export type VideoCapabilityConfig = {
// Keep explicit pixel presets for each resolution tier so the settings panel can
// switch between 1K, 2K and 4K without silently converting the requested ratio.
const defaultImageSizes = [
"auto", "1:1", "3:2", "2:3", "4:3", "3:4", "16:9", "21:9", "9:16",
"1024x1024", "1360x1024", "1024x1360", "1536x1024", "1024x1536", "1024x1280", "1280x1024", "2048x878", "1824x1024", "1024x1824",
"2048x2048", "2304x1728", "1728x2304", "2496x1664", "1664x2496", "1792x2240", "2240x1792", "3136x1344", "2752x1536", "1536x2752",
"2880x2880", "3264x2448", "2448x3264", "3504x2336", "2336x3504", "2560x3200", "3200x2560", "3808x1632", "3840x2160", "2160x3840",
"auto",
"1:1",
"3:2",
"2:3",
"4:3",
"3:4",
"16:9",
"21:9",
"9:16",
"1024x1024",
"1360x1024",
"1024x1360",
"1536x1024",
"1024x1536",
"1024x1280",
"1280x1024",
"2048x878",
"1824x1024",
"1024x1824",
"2048x2048",
"2304x1728",
"1728x2304",
"2496x1664",
"1664x2496",
"1792x2240",
"2240x1792",
"3136x1344",
"2752x1536",
"1536x2752",
"2880x2880",
"3264x2448",
"2448x3264",
"3504x2336",
"2336x3504",
"2560x3200",
"3200x2560",
"3808x1632",
"3840x2160",
"2160x3840",
];
export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = ""): ImageCapabilityConfig {
@@ -123,6 +158,20 @@ export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = "
image.responseFormat.supported = false;
image.outputFormat.supported = false;
}
if (protocol === "gemini-image") {
image.references.maskSupported = false;
// Gemini Images uses imageConfig.aspectRatio, not the OpenAI-style pixel size field.
image.size = {
parameter: "aspect_ratio",
values: ["auto", "1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9", "21:9"],
default: "1:1",
allowCustom: false,
};
image.transparentBackground = { supported: false, default: false };
image.responseFormat = { supported: false };
image.outputFormat = { supported: false };
image.maxOutputs = 4;
}
if (protocol !== "grok-image" && model.trim().toLowerCase().startsWith("grok-imagine-image")) {
image.references.maxImages = 0;
image.references.maskSupported = false;
@@ -207,20 +256,14 @@ export function modelCapabilityConfigFor(config: { channels: Array<{ id: string;
const cost = channel?.modelCosts?.find((item) => item.model === modelName);
const fallback = defaultModelCapabilityConfig(cost?.protocol, modelName);
if (!cost?.capabilityConfig) return fallback;
const text = cost.capabilityConfig.text
? { ...fallback.text!, ...cost.capabilityConfig.text, references: { ...fallback.text!.references, ...cost.capabilityConfig.text.references } }
: fallback.text;
const video = cost.capabilityConfig.video
? { ...fallback.video!, ...cost.capabilityConfig.video, references: { ...fallback.video!.references, ...cost.capabilityConfig.video.references } }
: fallback.video;
const text = cost.capabilityConfig.text ? { ...fallback.text!, ...cost.capabilityConfig.text, references: { ...fallback.text!.references, ...cost.capabilityConfig.text.references } } : fallback.text;
const video = cost.capabilityConfig.video ? { ...fallback.video!, ...cost.capabilityConfig.video, references: { ...fallback.video!.references, ...cost.capabilityConfig.video.references } } : fallback.video;
return { ...fallback, ...cost.capabilityConfig, text, image: cost.capabilityConfig.image || fallback.image, video };
}
export function normalizeImageValue(profile: ImageCapabilityConfig, value: { size?: string; quality?: string; count?: string; transparentBackground?: string }) {
const size = normalizeImageSizeSetting(profile, value.size);
const quality = profile.quality.supported
? (value.quality && profile.quality.values.includes(value.quality) ? value.quality : profile.quality.default || "auto")
: profile.quality.default || "auto";
const quality = profile.quality.supported ? (value.quality && profile.quality.values.includes(value.quality) ? value.quality : profile.quality.default || "auto") : profile.quality.default || "auto";
const count = String(Math.max(1, Math.min(profile.maxOutputs, Math.floor(Math.abs(Number(value.count)) || 1))));
const transparentBackground = profile.transparentBackground.supported && value.transparentBackground === "true" ? "true" : "false";
return { size, quality, count, transparentBackground };
@@ -242,16 +285,16 @@ export function imageSizeRequest(profile: ImageCapabilityConfig, value?: string)
}
export function normalizeVideoValue(profile: VideoCapabilityConfig, value: { seconds?: string; ratio?: string; resolution?: string }) {
const duration = profile.duration.selection === "enum"
? (profile.duration.values || []).includes(Number(value.seconds)) ? Number(value.seconds) : profile.duration.default
: normalizeRangeDuration(profile, Number(value.seconds));
const duration = profile.duration.selection === "enum" ? ((profile.duration.values || []).includes(Number(value.seconds)) ? Number(value.seconds) : profile.duration.default) : normalizeRangeDuration(profile, Number(value.seconds));
const ratio = profile.ratios.includes(value.ratio || "") ? value.ratio! : profile.defaultRatio;
const resolution = profile.resolutions.includes(value.resolution || "") ? value.resolution! : profile.defaultResolution;
return { seconds: String(duration), ratio, resolution };
}
export function videoResolutionRequest(profile: VideoCapabilityConfig, value: string | undefined) {
const requested = String(value || "").trim().toLowerCase();
const requested = String(value || "")
.trim()
.toLowerCase();
if (!requested || requested === "auto" || requested === "default" || requested === "medium" || requested === "high") return undefined;
const candidates = [requested];
if (/^\d+$/.test(requested)) candidates.push(`${requested}p`);

3
web/src/lib/model-protocols.ts
View File

@@ -5,6 +5,7 @@ export type ModelProtocol =
| "grok-image"
| "volcengine-ark-image"
| "volcengine-jimeng-image"
| "gemini-image"
| "openai-audio"
| "async-audio"
| "newapi"
@@ -35,6 +36,7 @@ export const MODEL_PROTOCOLS: ModelProtocolDefinition[] = [
{ value: "grok-image", label: "Grok Images", capability: "image", create: "POST /v1/images/generations / edits", contentType: "application/json", media: "文生图与单张 URL 参考图，不支持蒙版" },
{ value: "volcengine-ark-image", label: "火山方舟图片", capability: "image", create: "POST /api/v3/images/generations", contentType: "application/json", media: "文生图与 image 参考图，不支持蒙版" },
{ value: "volcengine-jimeng-image", label: "即梦官方图片", capability: "image", create: "POST CVSync2AsyncSubmitTask", poll: "POST CVSync2AsyncGetResult", contentType: "application/json + AK/SK 签名", media: "0-14 张参考图，模型标识填写 req_key" },
{ value: "gemini-image", label: "Gemini Images", capability: "image", create: "POST /v1beta/models/{model}:generateContent", contentType: "application/json", media: "文生图、图片编辑与多张内嵌参考图" },
{ value: "openai-audio", label: "OpenAI Audio", capability: "audio", create: "POST /v1/audio/speech", contentType: "application/json", media: "文本转语音" },
{ value: "async-audio", label: "异步音频任务", capability: "audio", create: "POST /v1/audio/tasks", poll: "GET /v1/audio/tasks/{task_id}", contentType: "application/json", media: "语音、音效与音乐生成" },
{ value: "newapi", label: "OpenAI / NewAPI Videos", capability: "video", create: "POST /v1/videos", poll: "GET /v1/videos/{task_id}", contentType: "multipart/form-data", media: "input_reference[] 参考图" },
@@ -96,6 +98,7 @@ export function protocolForModelCatalog(endpointTypes: string[] = []): ModelProt
const normalized = new Set(endpointTypes.map((value) => value.trim().toLowerCase()));
if (normalized.has("openai-chat") || normalized.has("chat-completion") || normalized.has("chat")) return "chat-completion";
if (normalized.has("openai-response") || normalized.has("responses")) return "openai-response";
if (normalized.has("gemini-image") || normalized.has("gemini-images")) return "gemini-image";
if (normalized.has("openai-image") || normalized.has("image")) return "openai-image";
if (normalized.has("openai-video") || normalized.has("video")) return "newapi-channel-2";
if (normalized.has("openai-audio") || normalized.has("audio")) return "openai-audio";

56
web/src/services/api/image.ts
View File

@@ -12,16 +12,14 @@ import { imageToDataUrl } from "@/services/image-storage";
import type { ReferenceImage } from "@/types/image";
import { withOpenAIPromptCacheKey } from "@/lib/openai-prompt-cache";
import { imageSizeRequest, modelCapabilityConfigFor, normalizeImageValue, type ImageCapabilityConfig } from "@/lib/model-capabilities";
import { buildGeminiImageGenerationConfig, parseGeminiImageDataUrl, type GeminiImageGenerationConfig } from "@/lib/gemini-image";
export type AiTextMessage = {
role: "system" | "user" | "assistant";
content: string | AiTextContentPart[];
};
export type AiTextContentPart =
| { type: "text"; text: string }
| { type: "image_url"; image_url: { url: string } }
| { type: "file_url"; file_url: { url: string; name: string; mimeType: string } };
export type AiTextContentPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } } | { type: "file_url"; file_url: { url: string; name: string; mimeType: string } };
export type ResponseToolCall = {
id: string;
@@ -358,9 +356,7 @@ function toResponseContent(content: ResponseMessageContent): string | ResponseIn
return content.map((item) => {
if (item.type === "text") return { type: "input_text" as const, text: item.text };
if (item.type === "image_url") return { type: "input_image" as const, image_url: item.image_url.url };
return item.file_url.url.startsWith("data:")
? { type: "input_file" as const, filename: item.file_url.name, file_data: item.file_url.url }
: { type: "input_file" as const, filename: item.file_url.name, file_url: item.file_url.url };
return item.file_url.url.startsWith("data:") ? { type: "input_file" as const, filename: item.file_url.name, file_data: item.file_url.url } : { type: "input_file" as const, filename: item.file_url.name, file_url: item.file_url.url };
});
}
@@ -402,9 +398,7 @@ function toChatCompletionContent(content: ResponseMessageContent) {
if (!Array.isArray(content)) return content;
return content.map((item) => {
if (item.type !== "file_url") return item;
const file = item.file_url.url.startsWith("data:")
? { filename: item.file_url.name, file_data: item.file_url.url }
: { filename: item.file_url.name, file_url: item.file_url.url };
const file = item.file_url.url.startsWith("data:") ? { filename: item.file_url.name, file_data: item.file_url.url } : { filename: item.file_url.name, file_url: item.file_url.url };
return { type: "file", file };
});
}
@@ -710,7 +704,7 @@ function toGeminiFilePart(url: string, mimeType: string): GeminiPart {
function geminiTextContent(content: ResponseMessageContent) {
if (!Array.isArray(content)) return String(content || "");
return content.map((item) => item.type === "text" ? item.text : item.type === "image_url" ? item.image_url.url : `${item.file_url.name}: ${item.file_url.url}`).join("\n");
return content.map((item) => (item.type === "text" ? item.text : item.type === "image_url" ? item.image_url.url : `${item.file_url.name}: ${item.file_url.url}`)).join("\n");
}
function jsonObject(value: string): Record<string, unknown> {
@@ -809,8 +803,14 @@ function consumeGeminiStreamBlock(block: string, state: GeminiStreamState, onDel
function parseGeminiToolResponse(payload: GeminiPayload): ToolResponseResult {
validateGeminiPayload(payload);
const parts = payload.candidates?.flatMap((candidate) => candidate.content?.parts || []) || [];
const content = parts.filter((part) => !part.thought).map((part) => part.text || "").join("");
const reasoning = parts.filter((part) => part.thought).map((part) => part.text || "").join("");
const content = parts
.filter((part) => !part.thought)
.map((part) => part.text || "")
.join("");
const reasoning = parts
.filter((part) => part.thought)
.map((part) => part.text || "")
.join("");
const toolCalls = parts
.map((part) => part.functionCall)
.filter((call): call is NonNullable<GeminiPart["functionCall"]> => Boolean(call?.name))
@@ -827,21 +827,27 @@ function parseGeminiToolResponse(payload: GeminiPayload): ToolResponseResult {
return { content, toolCalls, ...(reasoning ? { reasoning } : {}) };
}
async function requestGeminiImages(config: AiConfig, prompt: string, references: ReferenceImage[], count: number, options?: RequestOptions) {
const requests = Array.from({ length: count }, () => requestGeminiImagesOnce(config, prompt, references, options));
async function requestGeminiImages(config: AiConfig, prompt: string, references: ReferenceImage[], count: number, generationConfig: GeminiImageGenerationConfig, options?: RequestOptions) {
// 参考图先完整读取并校验一次，再复用已解析的 inlineData；不能让每个输出重新读取 storageKey，
// 否则资源缓存瞬时未命中时会出现“偶发缺参考图”或把 storageKey 当 URL 请求的问题。
const referenceParts = await Promise.all(
references.map(async (image) => {
const dataUrl = await imageToDataUrl(image);
const inlineData = parseGeminiImageDataUrl(dataUrl);
return { inlineData };
}),
);
const requests = Array.from({ length: count }, () => requestGeminiImagesOnce(config, prompt, referenceParts, generationConfig, options));
return (await Promise.all(requests)).flat();
}
async function requestGeminiImagesOnce(config: AiConfig, prompt: string, references: ReferenceImage[], options?: RequestOptions) {
const parts: GeminiPart[] = [{ text: prompt }];
for (const image of references) {
parts.push(toGeminiFilePart(await imageToDataUrl(image), image.type || "image/png"));
}
async function requestGeminiImagesOnce(config: AiConfig, prompt: string, referenceParts: GeminiPart[], generationConfig: GeminiImageGenerationConfig, options?: RequestOptions) {
const parts: GeminiPart[] = [{ text: prompt }, ...referenceParts];
const request = channelRequest(config, geminiApiUrl(config, "generateContent"), geminiHeaders(config));
const response = await axios.post<GeminiPayload>(
request.url,
{
...toGeminiBody(config, [{ role: "user", content: prompt }], { generationConfig: { responseModalities: ["TEXT", "IMAGE"] } }),
...toGeminiBody(config, [{ role: "user", content: prompt }], { generationConfig }),
contents: [{ role: "user", parts }],
},
{ headers: request.headers, withCredentials: request.credentials === "include", signal: options?.signal },
@@ -872,9 +878,9 @@ export async function requestGeneration(config: AiConfig, prompt: string, option
validateImageCapability(imageProfile, []);
const normalizedImage = normalizeImageValue(imageProfile, config);
const n = Number(normalizedImage.count);
if (requestConfig.apiFormat === "gemini") {
if (requestConfig.interfaceType === "gemini-image") {
try {
return await requestGeminiImages(requestConfig, prompt, [], n, options);
return await requestGeminiImages(requestConfig, prompt, [], n, buildGeminiImageGenerationConfig(normalizedImage.size, normalizedImage.quality), options);
} catch (error) {
throw new Error(readAxiosError(error, "请求失败"));
}
@@ -959,10 +965,10 @@ export async function requestEdit(config: AiConfig, prompt: string, references:
const normalizedImage = normalizeImageValue(imageProfile, config);
const n = Number(normalizedImage.count);
const requestPrompt = buildImageReferencePromptText(prompt, references);
if (requestConfig.apiFormat === "gemini") {
if (requestConfig.interfaceType === "gemini-image") {
if (mask) throw new Error("Gemini 调用格式暂不支持蒙版编辑");
try {
return await requestGeminiImages(requestConfig, requestPrompt, references, n, options);
return await requestGeminiImages(requestConfig, requestPrompt, references, n, buildGeminiImageGenerationConfig(normalizedImage.size, normalizedImage.quality), options);
} catch (error) {
throw new Error(readAxiosError(error, "请求失败"));
}

4
web/src/stores/use-config-store.ts
View File

@@ -516,7 +516,7 @@ export function resolveModelRequestConfig(config: AiConfig, value: string) {
apiKey: channel.apiKey,
secretKey: channel.secretKey,
headers: channel.headers,
apiFormat: interfaceType ? (interfaceType === "gemini-veo" ? ("gemini" as const) : ("openai" as const)) : channel.apiFormat,
apiFormat: interfaceType ? (interfaceType === "gemini-veo" || interfaceType === "gemini-image" ? ("gemini" as const) : ("openai" as const)) : channel.apiFormat,
interfaceType,
channelId: channel.scope === "system" ? channel.id : "",
});
@@ -564,7 +564,7 @@ export function defaultBaseUrlForApiFormat(apiFormat: ApiCallFormat) {
}
export function defaultBaseUrlForChannelInterface(interfaceType?: ChannelInterfaceType) {
if (interfaceType === "gemini-veo") return GEMINI_BASE_URL;
if (interfaceType === "gemini-veo" || interfaceType === "gemini-image") return GEMINI_BASE_URL;
if (interfaceType === "novita-video") return "https://api.novita.ai/v3";
if (interfaceType === "volcengine-ark-image" || interfaceType === "volcengine-ark-video") return "https://ark.cn-beijing.volces.com/api/v3";
if (interfaceType === "volcengine-jimeng-image" || interfaceType === "volcengine-jimeng-video") return "https://visual.volcengineapi.com";

22
web/test/gemini-image.test.ts Normal file
View File

@@ -0,0 +1,22 @@
import { describe, expect, test } from "bun:test";
import { buildGeminiImageGenerationConfig, parseGeminiImageDataUrl } from "../src/lib/gemini-image";
describe("Gemini Images request helpers", () => {
test("拆分参考图 data URL 时使用真实 MIME 和裸 base64", () => {
expect(parseGeminiImageDataUrl("data:image/jpeg;base64,aGVsbG8=")).toEqual({ mimeType: "image/jpeg", data: "aGVsbG8=" });
});
test("拒绝空图和非图片 data URL，避免把错误参考图发给 Gemini", () => {
expect(() => parseGeminiImageDataUrl("")).toThrow("未得到有效的 image data URL");
expect(() => parseGeminiImageDataUrl("data:text/plain;base64,aGVsbG8=")).toThrow("MIME 类型无效");
});
test("把统一图片选项映射到 Gemini imageConfig", () => {
expect(buildGeminiImageGenerationConfig("16:9", "high")).toEqual({
responseModalities: ["TEXT", "IMAGE"],
imageConfig: { aspectRatio: "16:9", imageSize: "4K" },
});
expect(buildGeminiImageGenerationConfig("auto", "auto")).toEqual({ responseModalities: ["TEXT", "IMAGE"] });
});
});