diff --git a/backend/internal/app/model_capability.go b/backend/internal/app/model_capability.go index 240ab5699..1a05e0ebb 100644 --- a/backend/internal/app/model_capability.go +++ b/backend/internal/app/model_capability.go @@ -207,9 +207,13 @@ func legacyImageSizeValues() []string { } func DefaultModelCapabilityConfigForModel(protocol string, modelName string) *ModelCapabilityConfig { - // 文本模型是否支持视觉输入不能从协议或模型名可靠推断，默认关闭，由管理员按真实上游能力开启。 streaming := true - text := &TextCapabilityConfig{Streaming: &streaming, References: TextReferenceConfig{PromptMaxChars: 32000}} + textReferences := TextReferenceConfig{PromptMaxChars: 32000} + if isKnownMultimodalTextModel(protocol, modelName) { + textReferences.MaxImages = 16 + textReferences.MaxImageBytes = 30 * 1024 * 1024 + } + text := &TextCapabilityConfig{Streaming: &streaming, References: textReferences} video := &VideoCapabilityConfig{ References: VideoReferenceConfig{PromptMaxChars: DefaultVideoPromptMaxChars, MinImages: 0, MaxImages: 9, MaxImageBytes: 30 * 1024 * 1024, MaxVideos: 0, MaxVideoBytes: 0, MaxVideoDuration: 0, MaxAudios: 0, MaxAudioBytes: 0, MaxAudioDuration: 0}, Duration: VideoDurationConfig{Selection: "range", Min: 1, Max: 15, Step: 1, Default: 6}, @@ -275,6 +279,11 @@ func DefaultModelCapabilityConfigForModel(protocol string, modelName string) *Mo return &ModelCapabilityConfig{Version: 1, Text: text, Image: DefaultImageCapabilityConfig(protocol, modelName), Video: video} } +func isKnownMultimodalTextModel(protocol string, modelName string) bool { + model := strings.ToLower(strings.TrimSpace(modelName)) + return strings.Contains(model, "gemini") || strings.Contains(model, "gpt") || strings.Contains(model, "doubao") || strings.Contains(model, "豆包") || strings.EqualFold(strings.TrimSpace(protocol), "gemini") +} + func DecodeModelCapabilityConfig(raw string) (*ModelCapabilityConfig, error) { if strings.TrimSpace(raw) == "" { return nil, nil @@ -323,6 +332,14 @@ func NormalizeModelCapabilityConfigForModel(capability string, protocol string, return nil, BadAuthRequest("请配置文本模型能力参数") } text := *input.Text + if isKnownMultimodalTextModel(protocol, modelName) && text.References.MaxImages == 0 { + // 旧配置是在文本模型默认不接收图片时生成的；Gemini 文本模型 + // 通过兼容协议仍然会接收图片 parts，因此自动修复旧的 0 上限。 + text.References.MaxImages = 16 + if text.References.MaxImageBytes == 0 { + text.References.MaxImageBytes = 30 * 1024 * 1024 + } + } if text.Streaming == nil { streaming := true text.Streaming = &streaming diff --git a/backend/internal/app/model_capability_test.go b/backend/internal/app/model_capability_test.go index 2028dc406..8e15472b8 100644 --- a/backend/internal/app/model_capability_test.go +++ b/backend/internal/app/model_capability_test.go @@ -6,6 +6,23 @@ import ( "testing" ) +func TestDefaultModelCapabilityConfigRecognizesKnownTextVisionFamilies(t *testing.T) { + gemini := DefaultModelCapabilityConfigForModel("chat-completion", "gemini-3.8-flash-high") + gpt := DefaultModelCapabilityConfigForModel("chat-completion", "gpt-4o") + doubao := DefaultModelCapabilityConfigForModel("chat-completion", "doubao-seed-1.6-flash") + plain := DefaultModelCapabilityConfigForModel("chat-completion", "plain-text-model") + + if gemini.Text == nil || gemini.Text.References.MaxImages != 16 || gemini.Text.References.MaxImageBytes != 30*1024*1024 { + t.Fatalf("Gemini text vision defaults = %#v", gemini.Text) + } + if gpt.Text == nil || gpt.Text.References.MaxImages != 16 || doubao.Text == nil || doubao.Text.References.MaxImages != 16 { + t.Fatalf("known multimodal text vision defaults: gpt=%#v doubao=%#v", gpt.Text, doubao.Text) + } + if plain.Text == nil || plain.Text.References.MaxImages != 0 { + t.Fatalf("plain text vision defaults = %#v", plain.Text) + } +} + func TestValidateImageTaskRejectsOversizedGrokPromptByUTF8Bytes(t *testing.T) { prompt := strings.Repeat("中", 4001) input := canvasGenerationInput{ diff --git a/web/src/lib/model-capabilities.ts b/web/src/lib/model-capabilities.ts index 3f2e93493..658a3d685 100644 --- a/web/src/lib/model-capabilities.ts +++ b/web/src/lib/model-capabilities.ts @@ -105,6 +105,14 @@ export const STANDARD_IMAGE_SIZE_VALUES = [ "1024x1536", ] as const; +const DEFAULT_TEXT_REFERENCE_MAX_IMAGES = 16; + +function isKnownMultimodalTextModel(protocol?: ModelProtocol, model = "") { + // These model families currently expose vision through their text/chat + // endpoint, including when they are wrapped by an OpenAI-compatible gateway. + return /gemini|gpt|doubao|豆包/i.test(model) || protocol === "gemini"; +} + export function normalizeCapabilityString(value: string) { const normalized = value.trim(); return normalized.startsWith("string:") ? normalized.slice("string:".length) : normalized; @@ -286,8 +294,13 @@ export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = " export function defaultModelCapabilityConfig(protocol?: ModelProtocol, model = ""): ModelCapabilityConfig { const text: TextCapabilityConfig = { streaming: true, - // 文本模型的视觉能力必须由管理员明确开启，不能根据模型名猜测。 - references: { promptMaxChars: 32000, maxImages: 0, maxImageBytes: 0, maxVideos: 0, maxVideoBytes: 0 }, + references: { + promptMaxChars: 32000, + maxImages: isKnownMultimodalTextModel(protocol, model) ? DEFAULT_TEXT_REFERENCE_MAX_IMAGES : 0, + maxImageBytes: isKnownMultimodalTextModel(protocol, model) ? 30 * 1024 * 1024 : 0, + maxVideos: 0, + maxVideoBytes: 0, + }, }; const video: VideoCapabilityConfig = { references: { @@ -398,7 +411,10 @@ export function modelCapabilityConfigFor(config: { channels: Array<{ id: string; const fallback = defaultModelCapabilityConfig(cost?.protocol, modelName); if (!cost?.capabilityConfig) return fallback; const capabilityConfig = normalizeModelCapabilityConfig(cost.capabilityConfig); - const text = capabilityConfig.text ? { ...fallback.text!, ...capabilityConfig.text, references: { ...fallback.text!.references, ...capabilityConfig.text.references } } : fallback.text; + const mergedText = capabilityConfig.text ? { ...fallback.text!, ...capabilityConfig.text, references: { ...fallback.text!.references, ...capabilityConfig.text.references } } : fallback.text; + const text = mergedText && isKnownMultimodalTextModel(cost?.protocol, modelName) && mergedText.references.maxImages === 0 + ? { ...mergedText, references: { ...mergedText.references, maxImages: DEFAULT_TEXT_REFERENCE_MAX_IMAGES, maxImageBytes: mergedText.references.maxImageBytes || 30 * 1024 * 1024 } } + : mergedText; const video = capabilityConfig.video ? { ...fallback.video!, ...capabilityConfig.video, references: { ...fallback.video!.references, ...capabilityConfig.video.references } } : fallback.video; const configuredImage = capabilityConfig.image; const image = configuredImage diff --git a/web/test/model-capabilities.test.ts b/web/test/model-capabilities.test.ts index d320615ca..ba4bfe768 100644 --- a/web/test/model-capabilities.test.ts +++ b/web/test/model-capabilities.test.ts @@ -31,3 +31,16 @@ test("raising the video default leaves text and image limits untouched", () => { assert.equal(profile.text!.references.promptMaxChars, 32000); assert.equal(profile.image!.references.promptMaxChars, 32000); }); + +test("known multimodal text model families default to image input", () => { + const gemini = defaultModelCapabilityConfig("chat-completion", "gemini-3.8-flash-high").text!; + const gpt = defaultModelCapabilityConfig("chat-completion", "gpt-4o").text!; + const doubao = defaultModelCapabilityConfig("chat-completion", "doubao-seed-1.6-flash").text!; + const plain = defaultModelCapabilityConfig("chat-completion", "plain-text-model").text!; + + assert.equal(gemini.references.maxImages, 16); + assert.equal(gemini.references.maxImageBytes, 30 * 1024 * 1024); + assert.equal(gpt.references.maxImages, 16); + assert.equal(doubao.references.maxImages, 16); + assert.equal(plain.references.maxImages, 0); +});