fix: recognize common multimodal text models
This commit is contained in:
1 parent
376770aad6
commit
43f30b9a84
4 files changed
+68
-5
No files matched your search
@@ -207,9 +207,13 @@ func legacyImageSizeValues() []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func DefaultModelCapabilityConfigForModel(protocol string, modelName string) *ModelCapabilityConfig {
|
func DefaultModelCapabilityConfigForModel(protocol string, modelName string) *ModelCapabilityConfig {
|
||||||
// 文本模型是否支持视觉输入不能从协议或模型名可靠推断,默认关闭,由管理员按真实上游能力开启。
|
|
||||||
streaming := true
|
streaming := true
|
||||||
text := &TextCapabilityConfig{Streaming: &streaming, References: TextReferenceConfig{PromptMaxChars: 32000}}
|
textReferences := TextReferenceConfig{PromptMaxChars: 32000}
|
||||||
|
if isKnownMultimodalTextModel(protocol, modelName) {
|
||||||
|
textReferences.MaxImages = 16
|
||||||
|
textReferences.MaxImageBytes = 30 * 1024 * 1024
|
||||||
|
}
|
||||||
|
text := &TextCapabilityConfig{Streaming: &streaming, References: textReferences}
|
||||||
video := &VideoCapabilityConfig{
|
video := &VideoCapabilityConfig{
|
||||||
References: VideoReferenceConfig{PromptMaxChars: DefaultVideoPromptMaxChars, MinImages: 0, MaxImages: 9, MaxImageBytes: 30 * 1024 * 1024, MaxVideos: 0, MaxVideoBytes: 0, MaxVideoDuration: 0, MaxAudios: 0, MaxAudioBytes: 0, MaxAudioDuration: 0},
|
References: VideoReferenceConfig{PromptMaxChars: DefaultVideoPromptMaxChars, MinImages: 0, MaxImages: 9, MaxImageBytes: 30 * 1024 * 1024, MaxVideos: 0, MaxVideoBytes: 0, MaxVideoDuration: 0, MaxAudios: 0, MaxAudioBytes: 0, MaxAudioDuration: 0},
|
||||||
Duration: VideoDurationConfig{Selection: "range", Min: 1, Max: 15, Step: 1, Default: 6},
|
Duration: VideoDurationConfig{Selection: "range", Min: 1, Max: 15, Step: 1, Default: 6},
|
||||||
@@ -275,6 +279,11 @@ func DefaultModelCapabilityConfigForModel(protocol string, modelName string) *Mo
|
|||||||
return &ModelCapabilityConfig{Version: 1, Text: text, Image: DefaultImageCapabilityConfig(protocol, modelName), Video: video}
|
return &ModelCapabilityConfig{Version: 1, Text: text, Image: DefaultImageCapabilityConfig(protocol, modelName), Video: video}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isKnownMultimodalTextModel(protocol string, modelName string) bool {
|
||||||
|
model := strings.ToLower(strings.TrimSpace(modelName))
|
||||||
|
return strings.Contains(model, "gemini") || strings.Contains(model, "gpt") || strings.Contains(model, "doubao") || strings.Contains(model, "豆包") || strings.EqualFold(strings.TrimSpace(protocol), "gemini")
|
||||||
|
}
|
||||||
|
|
||||||
func DecodeModelCapabilityConfig(raw string) (*ModelCapabilityConfig, error) {
|
func DecodeModelCapabilityConfig(raw string) (*ModelCapabilityConfig, error) {
|
||||||
if strings.TrimSpace(raw) == "" {
|
if strings.TrimSpace(raw) == "" {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
@@ -323,6 +332,14 @@ func NormalizeModelCapabilityConfigForModel(capability string, protocol string,
|
|||||||
return nil, BadAuthRequest("请配置文本模型能力参数")
|
return nil, BadAuthRequest("请配置文本模型能力参数")
|
||||||
}
|
}
|
||||||
text := *input.Text
|
text := *input.Text
|
||||||
|
if isKnownMultimodalTextModel(protocol, modelName) && text.References.MaxImages == 0 {
|
||||||
|
// 旧配置是在文本模型默认不接收图片时生成的;Gemini 文本模型
|
||||||
|
// 通过兼容协议仍然会接收图片 parts,因此自动修复旧的 0 上限。
|
||||||
|
text.References.MaxImages = 16
|
||||||
|
if text.References.MaxImageBytes == 0 {
|
||||||
|
text.References.MaxImageBytes = 30 * 1024 * 1024
|
||||||
|
}
|
||||||
|
}
|
||||||
if text.Streaming == nil {
|
if text.Streaming == nil {
|
||||||
streaming := true
|
streaming := true
|
||||||
text.Streaming = &streaming
|
text.Streaming = &streaming
|
||||||
|
|||||||
@@ -6,6 +6,23 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestDefaultModelCapabilityConfigRecognizesKnownTextVisionFamilies(t *testing.T) {
|
||||||
|
gemini := DefaultModelCapabilityConfigForModel("chat-completion", "gemini-3.8-flash-high")
|
||||||
|
gpt := DefaultModelCapabilityConfigForModel("chat-completion", "gpt-4o")
|
||||||
|
doubao := DefaultModelCapabilityConfigForModel("chat-completion", "doubao-seed-1.6-flash")
|
||||||
|
plain := DefaultModelCapabilityConfigForModel("chat-completion", "plain-text-model")
|
||||||
|
|
||||||
|
if gemini.Text == nil || gemini.Text.References.MaxImages != 16 || gemini.Text.References.MaxImageBytes != 30*1024*1024 {
|
||||||
|
t.Fatalf("Gemini text vision defaults = %#v", gemini.Text)
|
||||||
|
}
|
||||||
|
if gpt.Text == nil || gpt.Text.References.MaxImages != 16 || doubao.Text == nil || doubao.Text.References.MaxImages != 16 {
|
||||||
|
t.Fatalf("known multimodal text vision defaults: gpt=%#v doubao=%#v", gpt.Text, doubao.Text)
|
||||||
|
}
|
||||||
|
if plain.Text == nil || plain.Text.References.MaxImages != 0 {
|
||||||
|
t.Fatalf("plain text vision defaults = %#v", plain.Text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateImageTaskRejectsOversizedGrokPromptByUTF8Bytes(t *testing.T) {
|
func TestValidateImageTaskRejectsOversizedGrokPromptByUTF8Bytes(t *testing.T) {
|
||||||
prompt := strings.Repeat("中", 4001)
|
prompt := strings.Repeat("中", 4001)
|
||||||
input := canvasGenerationInput{
|
input := canvasGenerationInput{
|
||||||
|
|||||||
@@ -105,6 +105,14 @@ export const STANDARD_IMAGE_SIZE_VALUES = [
|
|||||||
"1024x1536",
|
"1024x1536",
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
|
const DEFAULT_TEXT_REFERENCE_MAX_IMAGES = 16;
|
||||||
|
|
||||||
|
function isKnownMultimodalTextModel(protocol?: ModelProtocol, model = "") {
|
||||||
|
// These model families currently expose vision through their text/chat
|
||||||
|
// endpoint, including when they are wrapped by an OpenAI-compatible gateway.
|
||||||
|
return /gemini|gpt|doubao|豆包/i.test(model) || protocol === "gemini";
|
||||||
|
}
|
||||||
|
|
||||||
export function normalizeCapabilityString(value: string) {
|
export function normalizeCapabilityString(value: string) {
|
||||||
const normalized = value.trim();
|
const normalized = value.trim();
|
||||||
return normalized.startsWith("string:") ? normalized.slice("string:".length) : normalized;
|
return normalized.startsWith("string:") ? normalized.slice("string:".length) : normalized;
|
||||||
@@ -286,8 +294,13 @@ export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = "
|
|||||||
export function defaultModelCapabilityConfig(protocol?: ModelProtocol, model = ""): ModelCapabilityConfig {
|
export function defaultModelCapabilityConfig(protocol?: ModelProtocol, model = ""): ModelCapabilityConfig {
|
||||||
const text: TextCapabilityConfig = {
|
const text: TextCapabilityConfig = {
|
||||||
streaming: true,
|
streaming: true,
|
||||||
// 文本模型的视觉能力必须由管理员明确开启,不能根据模型名猜测。
|
references: {
|
||||||
references: { promptMaxChars: 32000, maxImages: 0, maxImageBytes: 0, maxVideos: 0, maxVideoBytes: 0 },
|
promptMaxChars: 32000,
|
||||||
|
maxImages: isKnownMultimodalTextModel(protocol, model) ? DEFAULT_TEXT_REFERENCE_MAX_IMAGES : 0,
|
||||||
|
maxImageBytes: isKnownMultimodalTextModel(protocol, model) ? 30 * 1024 * 1024 : 0,
|
||||||
|
maxVideos: 0,
|
||||||
|
maxVideoBytes: 0,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
const video: VideoCapabilityConfig = {
|
const video: VideoCapabilityConfig = {
|
||||||
references: {
|
references: {
|
||||||
@@ -398,7 +411,10 @@ export function modelCapabilityConfigFor(config: { channels: Array<{ id: string;
|
|||||||
const fallback = defaultModelCapabilityConfig(cost?.protocol, modelName);
|
const fallback = defaultModelCapabilityConfig(cost?.protocol, modelName);
|
||||||
if (!cost?.capabilityConfig) return fallback;
|
if (!cost?.capabilityConfig) return fallback;
|
||||||
const capabilityConfig = normalizeModelCapabilityConfig(cost.capabilityConfig);
|
const capabilityConfig = normalizeModelCapabilityConfig(cost.capabilityConfig);
|
||||||
const text = capabilityConfig.text ? { ...fallback.text!, ...capabilityConfig.text, references: { ...fallback.text!.references, ...capabilityConfig.text.references } } : fallback.text;
|
const mergedText = capabilityConfig.text ? { ...fallback.text!, ...capabilityConfig.text, references: { ...fallback.text!.references, ...capabilityConfig.text.references } } : fallback.text;
|
||||||
|
const text = mergedText && isKnownMultimodalTextModel(cost?.protocol, modelName) && mergedText.references.maxImages === 0
|
||||||
|
? { ...mergedText, references: { ...mergedText.references, maxImages: DEFAULT_TEXT_REFERENCE_MAX_IMAGES, maxImageBytes: mergedText.references.maxImageBytes || 30 * 1024 * 1024 } }
|
||||||
|
: mergedText;
|
||||||
const video = capabilityConfig.video ? { ...fallback.video!, ...capabilityConfig.video, references: { ...fallback.video!.references, ...capabilityConfig.video.references } } : fallback.video;
|
const video = capabilityConfig.video ? { ...fallback.video!, ...capabilityConfig.video, references: { ...fallback.video!.references, ...capabilityConfig.video.references } } : fallback.video;
|
||||||
const configuredImage = capabilityConfig.image;
|
const configuredImage = capabilityConfig.image;
|
||||||
const image = configuredImage
|
const image = configuredImage
|
||||||
|
|||||||
@@ -31,3 +31,16 @@ test("raising the video default leaves text and image limits untouched", () => {
|
|||||||
assert.equal(profile.text!.references.promptMaxChars, 32000);
|
assert.equal(profile.text!.references.promptMaxChars, 32000);
|
||||||
assert.equal(profile.image!.references.promptMaxChars, 32000);
|
assert.equal(profile.image!.references.promptMaxChars, 32000);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("known multimodal text model families default to image input", () => {
|
||||||
|
const gemini = defaultModelCapabilityConfig("chat-completion", "gemini-3.8-flash-high").text!;
|
||||||
|
const gpt = defaultModelCapabilityConfig("chat-completion", "gpt-4o").text!;
|
||||||
|
const doubao = defaultModelCapabilityConfig("chat-completion", "doubao-seed-1.6-flash").text!;
|
||||||
|
const plain = defaultModelCapabilityConfig("chat-completion", "plain-text-model").text!;
|
||||||
|
|
||||||
|
assert.equal(gemini.references.maxImages, 16);
|
||||||
|
assert.equal(gemini.references.maxImageBytes, 30 * 1024 * 1024);
|
||||||
|
assert.equal(gpt.references.maxImages, 16);
|
||||||
|
assert.equal(doubao.references.maxImages, 16);
|
||||||
|
assert.equal(plain.references.maxImages, 0);
|
||||||
|
});
|
||||||
Reference in new issue
Block a user