feat(*): 图片模型渠道 - 新增 Grok JSON 生图与单图编辑协议

This commit is contained in:
ddCat
2026-08-07 14:58:26 +08:00
parent 8ef218eb74
commit 6b5f176baa
15 changed files with 207 additions and 10 deletions

View File

@@ -82,7 +82,7 @@ func authorizeCustomRelay(method string, target *url.URL, apiFormat string, cont
}
if apiFormat == "openai" {
multipartAllowed := mediaType == "multipart/form-data" && (strings.HasSuffix(requestPath, "/images/edits") || strings.HasSuffix(requestPath, "/videos"))
jsonAllowed := mediaType == "application/json" && (strings.HasSuffix(requestPath, "/responses") || strings.HasSuffix(requestPath, "/chat/completions") || strings.HasSuffix(requestPath, "/images/generations") || strings.HasSuffix(requestPath, "/audio/speech") || strings.HasSuffix(requestPath, "/video/generations") || strings.HasSuffix(requestPath, "/videos/generations") || strings.HasSuffix(requestPath, "/videos") || strings.HasSuffix(requestPath, "/contents/generations/tasks"))
jsonAllowed := mediaType == "application/json" && (strings.HasSuffix(requestPath, "/responses") || strings.HasSuffix(requestPath, "/chat/completions") || strings.HasSuffix(requestPath, "/images/generations") || strings.HasSuffix(requestPath, "/images/edits") || strings.HasSuffix(requestPath, "/audio/speech") || strings.HasSuffix(requestPath, "/video/generations") || strings.HasSuffix(requestPath, "/videos/generations") || strings.HasSuffix(requestPath, "/videos") || strings.HasSuffix(requestPath, "/contents/generations/tasks"))
if len(query) != 0 || (!multipartAllowed && !jsonAllowed) {
return errors.New("自定义渠道不允许访问该上游接口")
}
@@ -183,7 +183,7 @@ func interfaceAllowsProxyPath(interfaceType model.ChannelInterfaceType, requestP
return requestPath == "/chat/completions"
case model.ChannelInterfaceOpenAIResponse:
return requestPath == "/responses"
case model.ChannelInterfaceOpenAIImage:
case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage:
return requestPath == "/images/generations" || requestPath == "/images/edits"
case model.ChannelInterfaceVolcengineArkImage:
return requestPath == "/images/generations"

View File

@@ -18,6 +18,14 @@ func TestAuthorizeSystemProxyAllowsConfiguredGenerationModel(t *testing.T) {
}
}
func TestAuthorizeSystemProxyAllowsGrokImageJSONEdits(t *testing.T) {
channel := &model.ModelChannel{APIFormat: "openai", ModelsJSON: `["grok-imagine-image"]`}
body := []byte(`{"model":"grok-imagine-image","prompt":"edit"}`)
if err := authorizeSystemProxy(channel, model.ChannelInterfaceGrokImage, http.MethodPost, "/images/edits", "application/json", body); err != nil {
t.Fatalf("authorizeSystemProxy() error = %v", err)
}
}
func TestAuthorizeCustomRelayAllowsModelsAndAgentEndpoints(t *testing.T) {
tests := []struct {
method string
@@ -30,6 +38,7 @@ func TestAuthorizeCustomRelayAllowsModelsAndAgentEndpoints(t *testing.T) {
{method: http.MethodPost, target: "https://api.example.com/v1/chat/completions", apiFormat: "openai", contentType: "application/json; charset=utf-8"},
{method: http.MethodPost, target: "https://api.example.com/v1/audio/speech", apiFormat: "openai", contentType: "application/json"},
{method: http.MethodPost, target: "https://api.example.com/v1/images/edits", apiFormat: "openai", contentType: "multipart/form-data; boundary=test"},
{method: http.MethodPost, target: "https://api.example.com/v1/images/edits", apiFormat: "openai", contentType: "application/json"},
{method: http.MethodPost, target: "https://api.example.com/v1/videos", apiFormat: "openai", contentType: "application/json"},
{method: http.MethodPost, target: "https://api.example.com/v1/videos", apiFormat: "openai", contentType: "multipart/form-data; boundary=test"},
{method: http.MethodGet, target: "https://api.example.com/v1/videos/task-1/content", apiFormat: "openai"},

View File

@@ -61,6 +61,7 @@ const (
ChannelInterfaceChatCompletion ChannelInterfaceType = "chat-completion"
ChannelInterfaceOpenAIResponse ChannelInterfaceType = "openai-response"
ChannelInterfaceOpenAIImage ChannelInterfaceType = "openai-image"
ChannelInterfaceGrokImage ChannelInterfaceType = "grok-image"
ChannelInterfaceVolcengineArkImage ChannelInterfaceType = "volcengine-ark-image"
ChannelInterfaceVolcengineJiMengImage ChannelInterfaceType = "volcengine-jimeng-image"
ChannelInterfaceOpenAIAudio ChannelInterfaceType = "openai-audio"

View File

@@ -734,7 +734,7 @@ func mergeChannelRequest(req ChannelRequest, channel model.ModelChannel) Channel
func validChannelInterfaceType(value model.ChannelInterfaceType) bool {
switch value {
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo:
case model.ChannelInterfaceChatCompletion, model.ChannelInterfaceOpenAIResponse, model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage, model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio, model.ChannelInterfaceNewAPIVideo, model.ChannelInterfaceNewAPIChannel1, model.ChannelInterfaceNewAPIChannel2, model.ChannelInterfaceXAIVideo, model.ChannelInterfaceVolcengineArkVideo, model.ChannelInterfaceVolcengineJiMengVideo, model.ChannelInterfaceGeminiVeo:
return true
default:
return false

View File

@@ -454,7 +454,7 @@ func (s *Service) syncChannelModelNames(channel *model.ModelChannel) error {
func capabilityForProtocol(protocol model.ChannelInterfaceType) string {
switch protocol {
case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage:
case model.ChannelInterfaceOpenAIImage, model.ChannelInterfaceGrokImage, model.ChannelInterfaceVolcengineArkImage, model.ChannelInterfaceVolcengineJiMengImage:
return "image"
case model.ChannelInterfaceOpenAIAudio, model.ChannelInterfaceAsyncAudio:
return "audio"

View File

@@ -106,6 +106,15 @@ func DefaultImageCapabilityConfig(protocol string, modelName string) *ImageCapab
MaxOutputs: 15,
}
switch model.ChannelInterfaceType(protocol) {
case model.ChannelInterfaceGrokImage:
image.References.MaxImages = 1
image.References.MaskSupported = false
image.Size = ImageSizeConfig{Parameter: "none", Values: []string{}, Default: "auto", AllowCustom: false}
image.Quality = ImageQualityConfig{Supported: false, Values: []string{}, Default: "auto"}
image.TransparentBackground = VideoBooleanConfig{Supported: false, Default: false}
image.ResponseFormat = ParameterSupport{Supported: true}
image.OutputFormat = ParameterSupport{Supported: false}
image.MaxOutputs = 1
case model.ChannelInterfaceVolcengineArkImage:
image.References.MaskSupported = false
image.Quality.Supported = false
@@ -120,7 +129,7 @@ func DefaultImageCapabilityConfig(protocol string, modelName string) *ImageCapab
image.ResponseFormat.Supported = false
image.OutputFormat.Supported = false
}
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(modelName)), "grok-imagine-image") {
if model.ChannelInterfaceType(protocol) != model.ChannelInterfaceGrokImage && strings.HasPrefix(strings.ToLower(strings.TrimSpace(modelName)), "grok-imagine-image") {
image.References.MaxImages = 0
image.References.MaskSupported = false
image.Size = ImageSizeConfig{Parameter: "none", Values: []string{}, Default: "auto", AllowCustom: false}

View File

@@ -635,6 +635,9 @@ func systemChannelIDFromBaseURL(baseURL string) string {
}
func runImageTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) {
if input.Config.InterfaceType == string(model.ChannelInterfaceGrokImage) {
return runGrokImageTask(ctx, input)
}
if input.Config.InterfaceType == string(model.ChannelInterfaceVolcengineJiMengImage) {
return runVolcengineJiMengImageTask(ctx, input)
}
@@ -720,6 +723,48 @@ func runImageTask(ctx context.Context, input canvasGenerationInput) (map[string]
return map[string]interface{}{"mode": "image", "images": images}, nil
}
func runGrokImageTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) {
body, path, err := grokImageRequestBody(input)
if err != nil {
return nil, err
}
var payload imageResponse
if err := postJSON(ctx, input.Config, path, body, &payload); err != nil {
return nil, err
}
images, err := imageDataURLs(payload)
if err != nil {
return nil, err
}
return map[string]interface{}{"mode": "image", "images": images}, nil
}
func grokImageRequestBody(input canvasGenerationInput) (grokImageRequest, string, error) {
if input.Mask != nil {
return grokImageRequest{}, "", errors.New("Grok 图片协议不支持蒙版编辑,请移除蒙版后重试")
}
body := grokImageRequest{Model: input.Config.Model, Prompt: withSystemPrompt(input.Config, input.Prompt), N: 1, ResponseFormat: "url"}
if len(input.ReferenceImages) == 0 {
return body, "/images/generations", nil
}
if len(input.ReferenceImages) != 1 {
return grokImageRequest{}, "", fmt.Errorf("Grok 图片编辑只支持 1 张参考图,当前连接了 %d 张", len(input.ReferenceImages))
}
imageURL, err := grokImageInputURL(input.ReferenceImages[0])
if err != nil {
return grokImageRequest{}, "", err
}
body.Image = &grokImageInput{URL: imageURL}
return body, "/images/edits", nil
}
func grokImageInputURL(media providerMedia) (string, error) {
if isPublicMediaURL(strings.TrimSpace(media.URL)) {
return strings.TrimSpace(media.URL), nil
}
return openAIImageInputURL(media)
}
const volcengineArkImageMaxPixels = 4624220
func runVolcengineArkImageTask(ctx context.Context, input canvasGenerationInput) (map[string]interface{}, error) {
@@ -1909,7 +1954,7 @@ func validateGenerationInterface(mode string, interfaceType string) error {
}
allowed := map[string]map[string]bool{
"text": {"chat-completion": true, "openai-response": true},
"image": {"openai-image": true, "volcengine-ark-image": true, "volcengine-jimeng-image": true},
"image": {"openai-image": true, "grok-image": true, "volcengine-ark-image": true, "volcengine-jimeng-image": true},
"video": {"newapi": true, "newapi-channel-1": true, "newapi-channel-2": true, "xai-video": true, "volcengine-ark-video": true, "volcengine-jimeng-video": true, "gemini-veo": true},
"audio": {"openai-audio": true, "async-audio": true},
}

View File

@@ -52,6 +52,18 @@ type xaiVideoImage struct {
URL string `json:"url"`
}
type grokImageRequest struct {
Model string `json:"model"`
Prompt string `json:"prompt"`
Image *grokImageInput `json:"image,omitempty"`
N int `json:"n"`
ResponseFormat string `json:"response_format"`
}
type grokImageInput struct {
URL string `json:"url"`
}
type geminiVeoRequest struct {
Instances []geminiVeoInstance `json:"instances"`
Parameters geminiVeoParameters `json:"parameters"`

View File

@@ -104,6 +104,66 @@ func TestVolcengineArkImageRejectsMaskBeforeRequest(t *testing.T) {
}
}
func TestRunGrokImageTaskUsesJSONEditContract(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/images/edits" {
t.Errorf("path = %q, want /v1/images/edits", r.URL.Path)
}
if contentType := r.Header.Get("Content-Type"); !strings.HasPrefix(contentType, "application/json") {
t.Errorf("Content-Type = %q, want application/json", contentType)
}
var body grokImageRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Fatalf("decode request body: %v", err)
}
if body.Model != "grok-imagine-image-quality" || body.N != 1 || body.ResponseFormat != "url" {
t.Fatalf("request body = %#v", body)
}
if body.Image == nil || body.Image.URL != testReferenceImageDataURL {
t.Fatalf("image = %#v", body.Image)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[{"url":"https://example.com/result.png"}]}`))
}))
defer server.Close()
result, err := runImageTask(context.Background(), canvasGenerationInput{
Mode: "image",
Prompt: "edit the reference",
Config: providerConfig{BaseURL: server.URL, APIKey: "key", Model: "grok-imagine-image-quality", InterfaceType: "grok-image"},
ReferenceImages: []providerMedia{{DataURL: testReferenceImageDataURL}},
})
if err != nil {
t.Fatalf("runImageTask() error = %v", err)
}
images, _ := result["images"].([]map[string]string)
if len(images) != 1 || images[0]["dataUrl"] != "https://example.com/result.png" {
t.Fatalf("images = %#v", result["images"])
}
}
func TestGrokImageRequestBodyRejectsMaskAndMultipleReferences(t *testing.T) {
if _, _, err := grokImageRequestBody(canvasGenerationInput{Config: providerConfig{InterfaceType: "grok-image"}, Mask: &providerMedia{DataURL: testReferenceImageDataURL}}); err == nil || !strings.Contains(err.Error(), "不支持蒙版") {
t.Fatalf("mask error = %v", err)
}
if _, _, err := grokImageRequestBody(canvasGenerationInput{Config: providerConfig{InterfaceType: "grok-image"}, ReferenceImages: []providerMedia{{DataURL: testReferenceImageDataURL}, {DataURL: testReferenceImageDataURL}}}); err == nil || !strings.Contains(err.Error(), "只支持 1 张") {
t.Fatalf("multiple reference error = %v", err)
}
}
func TestGrokImageRequestBodyPrefersPublicURL(t *testing.T) {
body, path, err := grokImageRequestBody(canvasGenerationInput{
Config: providerConfig{Model: "grok-imagine-image", InterfaceType: "grok-image"},
ReferenceImages: []providerMedia{{URL: "https://example.com/reference.png", DataURL: testReferenceImageDataURL}},
})
if err != nil {
t.Fatalf("grokImageRequestBody() error = %v", err)
}
if path != "/images/edits" || body.Image == nil || body.Image.URL != "https://example.com/reference.png" {
t.Fatalf("path = %q, image = %#v", path, body.Image)
}
}
func TestNormalizePixelSizeConvertsCanvasAspectRatios(t *testing.T) {
tests := map[string]string{
"1:1": "1024x1024",
@@ -974,6 +1034,9 @@ func TestValidateGenerationInterfaceRejectsMismatchedType(t *testing.T) {
if err := validateGenerationInterface("video", "xai-video"); err != nil {
t.Fatalf("validateGenerationInterface() error = %v", err)
}
if err := validateGenerationInterface("image", "grok-image"); err != nil {
t.Fatalf("validateGenerationInterface() error = %v", err)
}
}
func TestProcessTaskValidatesInterfaceBeforeHydratingMedia(t *testing.T) {

View File

@@ -146,7 +146,7 @@ function ProtocolBrandMark({ protocol, compact = false }: { protocol: ModelProto
if (protocol.value.startsWith("volcengine-jimeng-")) return <span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}><Sparkles className={compact ? "size-3" : "size-4"} /></span>;
if (protocol.value.startsWith("volcengine-")) return <span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}><Flame className={compact ? "size-3" : "size-4"} /></span>;
if (protocol.value.startsWith("newapi-channel-")) return <span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}><Network className={compact ? "size-3" : "size-4"} /></span>;
const brand = protocol.value === "gemini-veo" ? "gemini" : protocol.value === "xai-video" ? "grok" : "openai";
const brand = protocol.value === "gemini-veo" ? "gemini" : protocol.value === "grok-image" || protocol.value === "xai-video" ? "grok" : "openai";
return <BrandIconRow models={[brand]} compact={compact} />;
}

View File

@@ -74,7 +74,16 @@ export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = "
outputFormat: { supported: true },
maxOutputs: 15,
};
if (protocol === "volcengine-ark-image") {
if (protocol === "grok-image") {
image.references.maxImages = 1;
image.references.maskSupported = false;
image.size = { parameter: "none", values: [], default: "auto", allowCustom: false };
image.quality = { supported: false, values: [], default: "auto" };
image.transparentBackground = { supported: false, default: false };
image.responseFormat = { supported: true };
image.outputFormat = { supported: false };
image.maxOutputs = 1;
} else if (protocol === "volcengine-ark-image") {
image.references.maskSupported = false;
image.quality.supported = false;
image.transparentBackground.supported = false;
@@ -89,7 +98,7 @@ export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = "
image.responseFormat.supported = false;
image.outputFormat.supported = false;
}
if (model.trim().toLowerCase().startsWith("grok-imagine-image")) {
if (protocol !== "grok-image" && model.trim().toLowerCase().startsWith("grok-imagine-image")) {
image.references.maxImages = 0;
image.references.maskSupported = false;
image.size = { parameter: "none", values: [], default: "auto", allowCustom: false };

View File

@@ -2,6 +2,7 @@ export type ModelProtocol =
| "chat-completion"
| "openai-response"
| "openai-image"
| "grok-image"
| "volcengine-ark-image"
| "volcengine-jimeng-image"
| "openai-audio"
@@ -30,6 +31,7 @@ export const MODEL_PROTOCOLS: ModelProtocolDefinition[] = [
{ value: "chat-completion", label: "OpenAI Chat Completions", capability: "text", create: "POST /v1/chat/completions", contentType: "application/json", media: "文本与多模态消息" },
{ value: "openai-response", label: "OpenAI Responses", capability: "text", create: "POST /v1/responses", contentType: "application/json", media: "文本与多模态输入" },
{ value: "openai-image", label: "OpenAI Images", capability: "image", create: "POST /v1/images/generations", contentType: "application/json / multipart", media: "生成、编辑与参考图" },
{ value: "grok-image", label: "Grok Images", capability: "image", create: "POST /v1/images/generations / edits", contentType: "application/json", media: "文生图与单张 URL 参考图,不支持蒙版" },
{ value: "volcengine-ark-image", label: "火山方舟图片", capability: "image", create: "POST /api/v3/images/generations", contentType: "application/json", media: "文生图与 image 参考图,不支持蒙版" },
{ value: "volcengine-jimeng-image", label: "即梦官方图片", capability: "image", create: "POST CVSync2AsyncSubmitTask", poll: "POST CVSync2AsyncGetResult", contentType: "application/json + AK/SK 签名", media: "0-14 张参考图,模型标识填写 req_key" },
{ value: "openai-audio", label: "OpenAI Audio", capability: "audio", create: "POST /v1/audio/speech", contentType: "application/json", media: "文本转语音" },

View File

@@ -181,6 +181,7 @@ function defaultProtocolForModel(channel: ModelChannel, model: string): ModelPro
if (channel.interfaceType) return channel.interfaceType;
if (channel.apiFormat === "gemini" && modelMatchesCapability(model, "video")) return "gemini-veo";
if (modelMatchesCapability(model, "video")) return "newapi";
if (modelOptionName(model).trim().toLowerCase().startsWith("grok-imagine-image")) return "grok-image";
if (modelMatchesCapability(model, "image")) return "openai-image";
return "chat-completion";
}

View File

@@ -812,6 +812,24 @@ export async function requestGeneration(config: AiConfig, prompt: string, option
throw new Error(readAxiosError(error, "请求失败"));
}
}
if (requestConfig.interfaceType === "grok-image") {
try {
const responseData = await postChannelJSON<ImageApiResponse>(
requestConfig,
aiApiUrl(requestConfig, "/images/generations"),
{
model: requestConfig.model,
prompt: withSystemPrompt(requestConfig, prompt),
n,
response_format: "url",
},
options,
);
return parseImagePayload(responseData);
} catch (error) {
throw new Error(readAxiosError(error, "Grok 图片生成失败"));
}
}
const quality = imageProfile.quality.supported && normalizedImage.quality !== "auto" ? normalizeQuality(normalizedImage.quality) || normalizedImage.quality : undefined;
const requestSize = resolveImageRequestSize(imageProfile, quality, normalizedImage.size);
const isVolcengineArk = requestConfig.interfaceType === "volcengine-ark-image";
@@ -853,6 +871,12 @@ async function postChannelJSON<T>(config: ReturnType<typeof resolveModelRequestC
).data;
}
async function grokImageInputURL(image: ReferenceImage) {
const candidate = image.url?.trim() || "";
if (/^https?:\/\//i.test(candidate)) return candidate;
return imageToDataUrl(image);
}
export async function requestEdit(config: AiConfig, prompt: string, references: ReferenceImage[], mask?: ReferenceImage, options?: RequestOptions) {
const selectedModel = config.model || config.imageModel;
const requestConfig = resolveModelRequestConfig(config, selectedModel);
@@ -869,6 +893,28 @@ export async function requestEdit(config: AiConfig, prompt: string, references:
throw new Error(readAxiosError(error, "请求失败"));
}
}
if (requestConfig.interfaceType === "grok-image") {
if (mask) throw new Error("Grok 图片协议不支持蒙版编辑,请移除蒙版后重试");
if (references.length !== 1) throw new Error("Grok 图片编辑必须提供且仅支持 1 张参考图");
try {
const imageUrl = await grokImageInputURL(references[0]);
const response = await postChannelJSON<ImageApiResponse>(
requestConfig,
aiApiUrl(requestConfig, "/images/edits"),
{
model: requestConfig.model,
prompt: withSystemPrompt(requestConfig, requestPrompt),
image: { url: imageUrl },
n,
response_format: "url",
},
options,
);
return parseImagePayload(response);
} catch (error) {
throw new Error(readAxiosError(error, "Grok 图片编辑失败"));
}
}
if (requestConfig.interfaceType === "volcengine-ark-image") {
if (mask) throw new Error("火山方舟图片协议不支持蒙版编辑,请移除蒙版后重试");
const quality = imageProfile.quality.supported && normalizedImage.quality !== "auto" ? normalizeQuality(normalizedImage.quality) || normalizedImage.quality : undefined;

View File

@@ -491,7 +491,7 @@ export function defaultBaseUrlForChannelInterface(interfaceType?: ChannelInterfa
if (interfaceType === "gemini-veo") return GEMINI_BASE_URL;
if (interfaceType === "volcengine-ark-image" || interfaceType === "volcengine-ark-video") return "https://ark.cn-beijing.volces.com/api/v3";
if (interfaceType === "volcengine-jimeng-image" || interfaceType === "volcengine-jimeng-video") return "https://visual.volcengineapi.com";
if (interfaceType === "newapi" || interfaceType === "newapi-channel-1" || interfaceType === "newapi-channel-2" || interfaceType === "xai-video") return "";
if (interfaceType === "grok-image" || interfaceType === "newapi" || interfaceType === "newapi-channel-1" || interfaceType === "newapi-channel-2" || interfaceType === "xai-video") return "";
return OPENAI_BASE_URL;
}