feat(platform): 创作能力整合 - 新增 Gemini Images 图片请求协议
This commit is contained in:
ddCat
@@ -42,7 +42,11 @@ export function CapabilityCardPicker({ value, onChange, density = "comfortable"
|
||||
onClick={() => onChange?.(item.value)}
|
||||
>
|
||||
<span className={cn("grid place-items-center rounded-md", density === "compact" ? "size-6" : "size-8", selected ? "bg-foreground text-background" : "bg-muted text-foreground/65")}>{item.icon}</span>
|
||||
{selected ? <span className={cn("absolute grid place-items-center rounded-full bg-foreground text-background", density === "compact" ? "right-1.5 top-1.5 size-4" : "right-2.5 top-2.5 size-5")}><Check className={density === "compact" ? "size-2.5" : "size-3"} /></span> : null}
|
||||
{selected ? (
|
||||
<span className={cn("absolute grid place-items-center rounded-full bg-foreground text-background", density === "compact" ? "right-1.5 top-1.5 size-4" : "right-2.5 top-2.5 size-5")}>
|
||||
<Check className={density === "compact" ? "size-2.5" : "size-3"} />
|
||||
</span>
|
||||
) : null}
|
||||
<span className={cn("block font-semibold", density === "compact" ? "mt-1 text-xs" : "mt-2 text-sm")}>{item.label}</span>
|
||||
<span className={cn("block text-foreground/48", density === "compact" ? "text-[var(--fs-micro)]" : "text-xs")}>{item.description}</span>
|
||||
{density === "comfortable" ? <BrandIconRow models={item.brands} className="mt-auto pt-2" /> : null}
|
||||
@@ -79,14 +83,20 @@ export function ProtocolCardPicker({ capability, value, onChange, density = "com
|
||||
<div className={cn("mt-0.5 truncate font-mono text-foreground/48", density === "compact" ? "text-[var(--fs-micro)]" : "text-[var(--fs-tiny)]")}>{protocol.create}</div>
|
||||
</div>
|
||||
</div>
|
||||
{selected ? <span className={cn("absolute grid place-items-center rounded-full bg-foreground text-background", density === "compact" ? "right-1.5 top-1.5 size-4" : "right-2.5 top-2.5 size-5")}><Check className={density === "compact" ? "size-2.5" : "size-3"} /></span> : null}
|
||||
{density === "comfortable" ? <>
|
||||
<div className="mt-2 line-clamp-2 text-xs leading-5 text-foreground/58">{protocol.media}</div>
|
||||
<div className="mt-auto flex items-center justify-between gap-2 pt-2 text-[var(--fs-tiny)] text-foreground/42">
|
||||
<span className="truncate">{protocol.contentType}</span>
|
||||
{protocol.poll ? <span className="shrink-0">异步轮询</span> : <span className="shrink-0">同步响应</span>}
|
||||
</div>
|
||||
</> : null}
|
||||
{selected ? (
|
||||
<span className={cn("absolute grid place-items-center rounded-full bg-foreground text-background", density === "compact" ? "right-1.5 top-1.5 size-4" : "right-2.5 top-2.5 size-5")}>
|
||||
<Check className={density === "compact" ? "size-2.5" : "size-3"} />
|
||||
</span>
|
||||
) : null}
|
||||
{density === "comfortable" ? (
|
||||
<>
|
||||
<div className="mt-2 line-clamp-2 text-xs leading-5 text-foreground/58">{protocol.media}</div>
|
||||
<div className="mt-auto flex items-center justify-between gap-2 pt-2 text-[var(--fs-tiny)] text-foreground/42">
|
||||
<span className="truncate">{protocol.contentType}</span>
|
||||
{protocol.poll ? <span className="shrink-0">异步轮询</span> : <span className="shrink-0">同步响应</span>}
|
||||
</div>
|
||||
</>
|
||||
) : null}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
@@ -103,16 +113,24 @@ export function ModelCapabilityProtocolModal({ value, onChange }: { value: { cap
|
||||
}, [open, value]);
|
||||
|
||||
const updateCapability = (capability: ModelCapabilityChoice) => {
|
||||
const protocol = draft.protocol && MODEL_PROTOCOLS.some((item) => item.value === draft.protocol && item.capability === capability)
|
||||
? draft.protocol
|
||||
: MODEL_PROTOCOLS.find((item) => item.capability === capability)?.value || "chat-completion";
|
||||
const protocol = draft.protocol && MODEL_PROTOCOLS.some((item) => item.value === draft.protocol && item.capability === capability) ? draft.protocol : MODEL_PROTOCOLS.find((item) => item.capability === capability)?.value || "chat-completion";
|
||||
setDraft({ capability, protocol });
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<Button size="small" className="max-w-full justify-start" icon={<Settings2 className="size-3.5" />} onClick={() => { setDraft(value); setOpen(true); }}>
|
||||
<span className="max-w-[min(56vw,360px)] truncate">{capabilityLabel(value.capability)} · {modelProtocolLabel(value.protocol)}</span>
|
||||
<Button
|
||||
size="small"
|
||||
className="max-w-full justify-start"
|
||||
icon={<Settings2 className="size-3.5" />}
|
||||
onClick={() => {
|
||||
setDraft(value);
|
||||
setOpen(true);
|
||||
}}
|
||||
>
|
||||
<span className="max-w-[min(56vw,360px)] truncate">
|
||||
{capabilityLabel(value.capability)} · {modelProtocolLabel(value.protocol)}
|
||||
</span>
|
||||
</Button>
|
||||
<Modal
|
||||
title="配置模型能力与请求协议"
|
||||
@@ -123,7 +141,10 @@ export function ModelCapabilityProtocolModal({ value, onChange }: { value: { cap
|
||||
onCancel={() => setOpen(false)}
|
||||
okText="应用配置"
|
||||
cancelText="取消"
|
||||
onOk={() => { onChange(draft); setOpen(false); }}
|
||||
onOk={() => {
|
||||
onChange(draft);
|
||||
setOpen(false);
|
||||
}}
|
||||
>
|
||||
<div className="space-y-4">
|
||||
<section>
|
||||
@@ -143,17 +164,36 @@ export function ModelCapabilityProtocolModal({ value, onChange }: { value: { cap
|
||||
function ProtocolBrandMark({ protocol, compact = false }: { protocol: ModelProtocolDefinition; compact?: boolean }) {
|
||||
const iconSize = compact ? "size-6" : "size-8";
|
||||
if (protocol.value === "chat-completion") return <BrandIconRow models={["openai", "deepseek", "glm"]} compact={compact} />;
|
||||
if (protocol.value.startsWith("volcengine-jimeng-")) return <span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}><Sparkles className={compact ? "size-3" : "size-4"} /></span>;
|
||||
if (protocol.value.startsWith("volcengine-")) return <span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}><Flame className={compact ? "size-3" : "size-4"} /></span>;
|
||||
if (protocol.value.startsWith("newapi-channel-") || protocol.value === "novita-video") return <span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}><Network className={compact ? "size-3" : "size-4"} /></span>;
|
||||
const brand = protocol.value === "gemini-veo" ? "gemini" : protocol.value === "grok-image" || protocol.value === "xai-video" ? "grok" : "openai";
|
||||
if (protocol.value.startsWith("volcengine-jimeng-"))
|
||||
return (
|
||||
<span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}>
|
||||
<Sparkles className={compact ? "size-3" : "size-4"} />
|
||||
</span>
|
||||
);
|
||||
if (protocol.value.startsWith("volcengine-"))
|
||||
return (
|
||||
<span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}>
|
||||
<Flame className={compact ? "size-3" : "size-4"} />
|
||||
</span>
|
||||
);
|
||||
if (protocol.value.startsWith("newapi-channel-") || protocol.value === "novita-video")
|
||||
return (
|
||||
<span className={cn("grid shrink-0 place-items-center rounded-md bg-muted text-foreground/65", iconSize)}>
|
||||
<Network className={compact ? "size-3" : "size-4"} />
|
||||
</span>
|
||||
);
|
||||
const brand = protocol.value === "gemini-veo" || protocol.value === "gemini-image" ? "gemini" : protocol.value === "grok-image" || protocol.value === "xai-video" ? "grok" : "openai";
|
||||
return <BrandIconRow models={[brand]} compact={compact} />;
|
||||
}
|
||||
|
||||
function BrandIconRow({ models, compact = false, className }: { models: string[]; compact?: boolean; className?: string }) {
|
||||
return (
|
||||
<span className={cn("flex items-center -space-x-1", compact && "shrink-0", className)} aria-hidden="true">
|
||||
{models.map((model) => <span key={model} className={cn("grid shrink-0 place-items-center rounded-md border border-border/70 bg-background", compact ? "size-6" : "size-6")} title={modelBrandLabel(model)}><ModelIcon model={model} /></span>)}
|
||||
{models.map((model) => (
|
||||
<span key={model} className={cn("grid shrink-0 place-items-center rounded-md border border-border/70 bg-background", compact ? "size-6" : "size-6")} title={modelBrandLabel(model)}>
|
||||
<ModelIcon model={model} />
|
||||
</span>
|
||||
))}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
40
web/src/lib/gemini-image.ts
Normal file
40
web/src/lib/gemini-image.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
export type GeminiImageGenerationConfig = {
|
||||
responseModalities: ["TEXT", "IMAGE"];
|
||||
imageConfig?: { aspectRatio?: string; imageSize?: "1K" | "2K" | "4K" };
|
||||
};
|
||||
|
||||
export type GeminiInlineImage = {
|
||||
mimeType: string;
|
||||
data: string;
|
||||
};
|
||||
|
||||
const GEMINI_IMAGE_SIZES: Record<string, GeminiInlineImageGenerationSize> = {
|
||||
low: "1K",
|
||||
medium: "2K",
|
||||
high: "4K",
|
||||
"1k": "1K",
|
||||
"2k": "2K",
|
||||
"4k": "4K",
|
||||
};
|
||||
|
||||
type GeminiInlineImageGenerationSize = "1K" | "2K" | "4K";
|
||||
|
||||
export function parseGeminiImageDataUrl(dataUrl: string): GeminiInlineImage {
|
||||
const match = dataUrl.trim().match(/^data:([^;,\s]+)(?:;[^,]*)?;base64,([A-Za-z0-9+/=_-]+)$/i);
|
||||
if (!match) throw new Error("参考图片读取失败：未得到有效的 image data URL");
|
||||
const mimeType = match[1].toLowerCase();
|
||||
const data = match[2].replace(/-/g, "+").replace(/_/g, "/");
|
||||
if (!mimeType.startsWith("image/")) throw new Error(`参考图片 MIME 类型无效：${mimeType}`);
|
||||
if (!data) throw new Error("参考图片读取失败：图片数据为空");
|
||||
return { mimeType, data };
|
||||
}
|
||||
|
||||
export function buildGeminiImageGenerationConfig(size?: string, quality?: string): GeminiImageGenerationConfig {
|
||||
const imageConfig: NonNullable<GeminiImageGenerationConfig["imageConfig"]> = {};
|
||||
const normalizedSize = size?.trim();
|
||||
const normalizedQuality = quality?.trim().toLowerCase();
|
||||
if (normalizedSize && normalizedSize !== "auto" && /^\d+:\d+$/.test(normalizedSize)) imageConfig.aspectRatio = normalizedSize;
|
||||
const imageSize = GEMINI_IMAGE_SIZES[normalizedQuality || ""];
|
||||
if (imageSize) imageConfig.imageSize = imageSize;
|
||||
return { responseModalities: ["TEXT", "IMAGE"], ...(Object.keys(imageConfig).length ? { imageConfig } : {}) };
|
||||
}
|
||||
@@ -77,10 +77,45 @@ export type VideoCapabilityConfig = {
|
||||
// Keep explicit pixel presets for each resolution tier so the settings panel can
|
||||
// switch between 1K, 2K and 4K without silently converting the requested ratio.
|
||||
const defaultImageSizes = [
|
||||
"auto", "1:1", "3:2", "2:3", "4:3", "3:4", "16:9", "21:9", "9:16",
|
||||
"1024x1024", "1360x1024", "1024x1360", "1536x1024", "1024x1536", "1024x1280", "1280x1024", "2048x878", "1824x1024", "1024x1824",
|
||||
"2048x2048", "2304x1728", "1728x2304", "2496x1664", "1664x2496", "1792x2240", "2240x1792", "3136x1344", "2752x1536", "1536x2752",
|
||||
"2880x2880", "3264x2448", "2448x3264", "3504x2336", "2336x3504", "2560x3200", "3200x2560", "3808x1632", "3840x2160", "2160x3840",
|
||||
"auto",
|
||||
"1:1",
|
||||
"3:2",
|
||||
"2:3",
|
||||
"4:3",
|
||||
"3:4",
|
||||
"16:9",
|
||||
"21:9",
|
||||
"9:16",
|
||||
"1024x1024",
|
||||
"1360x1024",
|
||||
"1024x1360",
|
||||
"1536x1024",
|
||||
"1024x1536",
|
||||
"1024x1280",
|
||||
"1280x1024",
|
||||
"2048x878",
|
||||
"1824x1024",
|
||||
"1024x1824",
|
||||
"2048x2048",
|
||||
"2304x1728",
|
||||
"1728x2304",
|
||||
"2496x1664",
|
||||
"1664x2496",
|
||||
"1792x2240",
|
||||
"2240x1792",
|
||||
"3136x1344",
|
||||
"2752x1536",
|
||||
"1536x2752",
|
||||
"2880x2880",
|
||||
"3264x2448",
|
||||
"2448x3264",
|
||||
"3504x2336",
|
||||
"2336x3504",
|
||||
"2560x3200",
|
||||
"3200x2560",
|
||||
"3808x1632",
|
||||
"3840x2160",
|
||||
"2160x3840",
|
||||
];
|
||||
|
||||
export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = ""): ImageCapabilityConfig {
|
||||
@@ -123,6 +158,20 @@ export function defaultImageCapabilityConfig(protocol?: ModelProtocol, model = "
|
||||
image.responseFormat.supported = false;
|
||||
image.outputFormat.supported = false;
|
||||
}
|
||||
if (protocol === "gemini-image") {
|
||||
image.references.maskSupported = false;
|
||||
// Gemini Images uses imageConfig.aspectRatio, not the OpenAI-style pixel size field.
|
||||
image.size = {
|
||||
parameter: "aspect_ratio",
|
||||
values: ["auto", "1:1", "2:3", "3:2", "3:4", "4:3", "9:16", "16:9", "21:9"],
|
||||
default: "1:1",
|
||||
allowCustom: false,
|
||||
};
|
||||
image.transparentBackground = { supported: false, default: false };
|
||||
image.responseFormat = { supported: false };
|
||||
image.outputFormat = { supported: false };
|
||||
image.maxOutputs = 4;
|
||||
}
|
||||
if (protocol !== "grok-image" && model.trim().toLowerCase().startsWith("grok-imagine-image")) {
|
||||
image.references.maxImages = 0;
|
||||
image.references.maskSupported = false;
|
||||
@@ -207,20 +256,14 @@ export function modelCapabilityConfigFor(config: { channels: Array<{ id: string;
|
||||
const cost = channel?.modelCosts?.find((item) => item.model === modelName);
|
||||
const fallback = defaultModelCapabilityConfig(cost?.protocol, modelName);
|
||||
if (!cost?.capabilityConfig) return fallback;
|
||||
const text = cost.capabilityConfig.text
|
||||
? { ...fallback.text!, ...cost.capabilityConfig.text, references: { ...fallback.text!.references, ...cost.capabilityConfig.text.references } }
|
||||
: fallback.text;
|
||||
const video = cost.capabilityConfig.video
|
||||
? { ...fallback.video!, ...cost.capabilityConfig.video, references: { ...fallback.video!.references, ...cost.capabilityConfig.video.references } }
|
||||
: fallback.video;
|
||||
const text = cost.capabilityConfig.text ? { ...fallback.text!, ...cost.capabilityConfig.text, references: { ...fallback.text!.references, ...cost.capabilityConfig.text.references } } : fallback.text;
|
||||
const video = cost.capabilityConfig.video ? { ...fallback.video!, ...cost.capabilityConfig.video, references: { ...fallback.video!.references, ...cost.capabilityConfig.video.references } } : fallback.video;
|
||||
return { ...fallback, ...cost.capabilityConfig, text, image: cost.capabilityConfig.image || fallback.image, video };
|
||||
}
|
||||
|
||||
export function normalizeImageValue(profile: ImageCapabilityConfig, value: { size?: string; quality?: string; count?: string; transparentBackground?: string }) {
|
||||
const size = normalizeImageSizeSetting(profile, value.size);
|
||||
const quality = profile.quality.supported
|
||||
? (value.quality && profile.quality.values.includes(value.quality) ? value.quality : profile.quality.default || "auto")
|
||||
: profile.quality.default || "auto";
|
||||
const quality = profile.quality.supported ? (value.quality && profile.quality.values.includes(value.quality) ? value.quality : profile.quality.default || "auto") : profile.quality.default || "auto";
|
||||
const count = String(Math.max(1, Math.min(profile.maxOutputs, Math.floor(Math.abs(Number(value.count)) || 1))));
|
||||
const transparentBackground = profile.transparentBackground.supported && value.transparentBackground === "true" ? "true" : "false";
|
||||
return { size, quality, count, transparentBackground };
|
||||
@@ -242,16 +285,16 @@ export function imageSizeRequest(profile: ImageCapabilityConfig, value?: string)
|
||||
}
|
||||
|
||||
export function normalizeVideoValue(profile: VideoCapabilityConfig, value: { seconds?: string; ratio?: string; resolution?: string }) {
|
||||
const duration = profile.duration.selection === "enum"
|
||||
? (profile.duration.values || []).includes(Number(value.seconds)) ? Number(value.seconds) : profile.duration.default
|
||||
: normalizeRangeDuration(profile, Number(value.seconds));
|
||||
const duration = profile.duration.selection === "enum" ? ((profile.duration.values || []).includes(Number(value.seconds)) ? Number(value.seconds) : profile.duration.default) : normalizeRangeDuration(profile, Number(value.seconds));
|
||||
const ratio = profile.ratios.includes(value.ratio || "") ? value.ratio! : profile.defaultRatio;
|
||||
const resolution = profile.resolutions.includes(value.resolution || "") ? value.resolution! : profile.defaultResolution;
|
||||
return { seconds: String(duration), ratio, resolution };
|
||||
}
|
||||
|
||||
export function videoResolutionRequest(profile: VideoCapabilityConfig, value: string | undefined) {
|
||||
const requested = String(value || "").trim().toLowerCase();
|
||||
const requested = String(value || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (!requested || requested === "auto" || requested === "default" || requested === "medium" || requested === "high") return undefined;
|
||||
const candidates = [requested];
|
||||
if (/^\d+$/.test(requested)) candidates.push(`${requested}p`);
|
||||
|
||||
@@ -5,6 +5,7 @@ export type ModelProtocol =
|
||||
| "grok-image"
|
||||
| "volcengine-ark-image"
|
||||
| "volcengine-jimeng-image"
|
||||
| "gemini-image"
|
||||
| "openai-audio"
|
||||
| "async-audio"
|
||||
| "newapi"
|
||||
@@ -35,6 +36,7 @@ export const MODEL_PROTOCOLS: ModelProtocolDefinition[] = [
|
||||
{ value: "grok-image", label: "Grok Images", capability: "image", create: "POST /v1/images/generations / edits", contentType: "application/json", media: "文生图与单张 URL 参考图，不支持蒙版" },
|
||||
{ value: "volcengine-ark-image", label: "火山方舟图片", capability: "image", create: "POST /api/v3/images/generations", contentType: "application/json", media: "文生图与 image 参考图，不支持蒙版" },
|
||||
{ value: "volcengine-jimeng-image", label: "即梦官方图片", capability: "image", create: "POST CVSync2AsyncSubmitTask", poll: "POST CVSync2AsyncGetResult", contentType: "application/json + AK/SK 签名", media: "0-14 张参考图，模型标识填写 req_key" },
|
||||
{ value: "gemini-image", label: "Gemini Images", capability: "image", create: "POST /v1beta/models/{model}:generateContent", contentType: "application/json", media: "文生图、图片编辑与多张内嵌参考图" },
|
||||
{ value: "openai-audio", label: "OpenAI Audio", capability: "audio", create: "POST /v1/audio/speech", contentType: "application/json", media: "文本转语音" },
|
||||
{ value: "async-audio", label: "异步音频任务", capability: "audio", create: "POST /v1/audio/tasks", poll: "GET /v1/audio/tasks/{task_id}", contentType: "application/json", media: "语音、音效与音乐生成" },
|
||||
{ value: "newapi", label: "OpenAI / NewAPI Videos", capability: "video", create: "POST /v1/videos", poll: "GET /v1/videos/{task_id}", contentType: "multipart/form-data", media: "input_reference[] 参考图" },
|
||||
@@ -96,6 +98,7 @@ export function protocolForModelCatalog(endpointTypes: string[] = []): ModelProt
|
||||
const normalized = new Set(endpointTypes.map((value) => value.trim().toLowerCase()));
|
||||
if (normalized.has("openai-chat") || normalized.has("chat-completion") || normalized.has("chat")) return "chat-completion";
|
||||
if (normalized.has("openai-response") || normalized.has("responses")) return "openai-response";
|
||||
if (normalized.has("gemini-image") || normalized.has("gemini-images")) return "gemini-image";
|
||||
if (normalized.has("openai-image") || normalized.has("image")) return "openai-image";
|
||||
if (normalized.has("openai-video") || normalized.has("video")) return "newapi-channel-2";
|
||||
if (normalized.has("openai-audio") || normalized.has("audio")) return "openai-audio";
|
||||
|
||||
@@ -12,16 +12,14 @@ import { imageToDataUrl } from "@/services/image-storage";
|
||||
import type { ReferenceImage } from "@/types/image";
|
||||
import { withOpenAIPromptCacheKey } from "@/lib/openai-prompt-cache";
|
||||
import { imageSizeRequest, modelCapabilityConfigFor, normalizeImageValue, type ImageCapabilityConfig } from "@/lib/model-capabilities";
|
||||
import { buildGeminiImageGenerationConfig, parseGeminiImageDataUrl, type GeminiImageGenerationConfig } from "@/lib/gemini-image";
|
||||
|
||||
export type AiTextMessage = {
|
||||
role: "system" | "user" | "assistant";
|
||||
content: string | AiTextContentPart[];
|
||||
};
|
||||
|
||||
export type AiTextContentPart =
|
||||
| { type: "text"; text: string }
|
||||
| { type: "image_url"; image_url: { url: string } }
|
||||
| { type: "file_url"; file_url: { url: string; name: string; mimeType: string } };
|
||||
export type AiTextContentPart = { type: "text"; text: string } | { type: "image_url"; image_url: { url: string } } | { type: "file_url"; file_url: { url: string; name: string; mimeType: string } };
|
||||
|
||||
export type ResponseToolCall = {
|
||||
id: string;
|
||||
@@ -358,9 +356,7 @@ function toResponseContent(content: ResponseMessageContent): string | ResponseIn
|
||||
return content.map((item) => {
|
||||
if (item.type === "text") return { type: "input_text" as const, text: item.text };
|
||||
if (item.type === "image_url") return { type: "input_image" as const, image_url: item.image_url.url };
|
||||
return item.file_url.url.startsWith("data:")
|
||||
? { type: "input_file" as const, filename: item.file_url.name, file_data: item.file_url.url }
|
||||
: { type: "input_file" as const, filename: item.file_url.name, file_url: item.file_url.url };
|
||||
return item.file_url.url.startsWith("data:") ? { type: "input_file" as const, filename: item.file_url.name, file_data: item.file_url.url } : { type: "input_file" as const, filename: item.file_url.name, file_url: item.file_url.url };
|
||||
});
|
||||
}
|
||||
|
||||
@@ -402,9 +398,7 @@ function toChatCompletionContent(content: ResponseMessageContent) {
|
||||
if (!Array.isArray(content)) return content;
|
||||
return content.map((item) => {
|
||||
if (item.type !== "file_url") return item;
|
||||
const file = item.file_url.url.startsWith("data:")
|
||||
? { filename: item.file_url.name, file_data: item.file_url.url }
|
||||
: { filename: item.file_url.name, file_url: item.file_url.url };
|
||||
const file = item.file_url.url.startsWith("data:") ? { filename: item.file_url.name, file_data: item.file_url.url } : { filename: item.file_url.name, file_url: item.file_url.url };
|
||||
return { type: "file", file };
|
||||
});
|
||||
}
|
||||
@@ -710,7 +704,7 @@ function toGeminiFilePart(url: string, mimeType: string): GeminiPart {
|
||||
|
||||
function geminiTextContent(content: ResponseMessageContent) {
|
||||
if (!Array.isArray(content)) return String(content || "");
|
||||
return content.map((item) => item.type === "text" ? item.text : item.type === "image_url" ? item.image_url.url : `${item.file_url.name}: ${item.file_url.url}`).join("\n");
|
||||
return content.map((item) => (item.type === "text" ? item.text : item.type === "image_url" ? item.image_url.url : `${item.file_url.name}: ${item.file_url.url}`)).join("\n");
|
||||
}
|
||||
|
||||
function jsonObject(value: string): Record<string, unknown> {
|
||||
@@ -809,8 +803,14 @@ function consumeGeminiStreamBlock(block: string, state: GeminiStreamState, onDel
|
||||
function parseGeminiToolResponse(payload: GeminiPayload): ToolResponseResult {
|
||||
validateGeminiPayload(payload);
|
||||
const parts = payload.candidates?.flatMap((candidate) => candidate.content?.parts || []) || [];
|
||||
const content = parts.filter((part) => !part.thought).map((part) => part.text || "").join("");
|
||||
const reasoning = parts.filter((part) => part.thought).map((part) => part.text || "").join("");
|
||||
const content = parts
|
||||
.filter((part) => !part.thought)
|
||||
.map((part) => part.text || "")
|
||||
.join("");
|
||||
const reasoning = parts
|
||||
.filter((part) => part.thought)
|
||||
.map((part) => part.text || "")
|
||||
.join("");
|
||||
const toolCalls = parts
|
||||
.map((part) => part.functionCall)
|
||||
.filter((call): call is NonNullable<GeminiPart["functionCall"]> => Boolean(call?.name))
|
||||
@@ -827,21 +827,27 @@ function parseGeminiToolResponse(payload: GeminiPayload): ToolResponseResult {
|
||||
return { content, toolCalls, ...(reasoning ? { reasoning } : {}) };
|
||||
}
|
||||
|
||||
async function requestGeminiImages(config: AiConfig, prompt: string, references: ReferenceImage[], count: number, options?: RequestOptions) {
|
||||
const requests = Array.from({ length: count }, () => requestGeminiImagesOnce(config, prompt, references, options));
|
||||
async function requestGeminiImages(config: AiConfig, prompt: string, references: ReferenceImage[], count: number, generationConfig: GeminiImageGenerationConfig, options?: RequestOptions) {
|
||||
// 参考图先完整读取并校验一次，再复用已解析的 inlineData；不能让每个输出重新读取 storageKey，
|
||||
// 否则资源缓存瞬时未命中时会出现“偶发缺参考图”或把 storageKey 当 URL 请求的问题。
|
||||
const referenceParts = await Promise.all(
|
||||
references.map(async (image) => {
|
||||
const dataUrl = await imageToDataUrl(image);
|
||||
const inlineData = parseGeminiImageDataUrl(dataUrl);
|
||||
return { inlineData };
|
||||
}),
|
||||
);
|
||||
const requests = Array.from({ length: count }, () => requestGeminiImagesOnce(config, prompt, referenceParts, generationConfig, options));
|
||||
return (await Promise.all(requests)).flat();
|
||||
}
|
||||
|
||||
async function requestGeminiImagesOnce(config: AiConfig, prompt: string, references: ReferenceImage[], options?: RequestOptions) {
|
||||
const parts: GeminiPart[] = [{ text: prompt }];
|
||||
for (const image of references) {
|
||||
parts.push(toGeminiFilePart(await imageToDataUrl(image), image.type || "image/png"));
|
||||
}
|
||||
async function requestGeminiImagesOnce(config: AiConfig, prompt: string, referenceParts: GeminiPart[], generationConfig: GeminiImageGenerationConfig, options?: RequestOptions) {
|
||||
const parts: GeminiPart[] = [{ text: prompt }, ...referenceParts];
|
||||
const request = channelRequest(config, geminiApiUrl(config, "generateContent"), geminiHeaders(config));
|
||||
const response = await axios.post<GeminiPayload>(
|
||||
request.url,
|
||||
{
|
||||
...toGeminiBody(config, [{ role: "user", content: prompt }], { generationConfig: { responseModalities: ["TEXT", "IMAGE"] } }),
|
||||
...toGeminiBody(config, [{ role: "user", content: prompt }], { generationConfig }),
|
||||
contents: [{ role: "user", parts }],
|
||||
},
|
||||
{ headers: request.headers, withCredentials: request.credentials === "include", signal: options?.signal },
|
||||
@@ -872,9 +878,9 @@ export async function requestGeneration(config: AiConfig, prompt: string, option
|
||||
validateImageCapability(imageProfile, []);
|
||||
const normalizedImage = normalizeImageValue(imageProfile, config);
|
||||
const n = Number(normalizedImage.count);
|
||||
if (requestConfig.apiFormat === "gemini") {
|
||||
if (requestConfig.interfaceType === "gemini-image") {
|
||||
try {
|
||||
return await requestGeminiImages(requestConfig, prompt, [], n, options);
|
||||
return await requestGeminiImages(requestConfig, prompt, [], n, buildGeminiImageGenerationConfig(normalizedImage.size, normalizedImage.quality), options);
|
||||
} catch (error) {
|
||||
throw new Error(readAxiosError(error, "请求失败"));
|
||||
}
|
||||
@@ -959,10 +965,10 @@ export async function requestEdit(config: AiConfig, prompt: string, references:
|
||||
const normalizedImage = normalizeImageValue(imageProfile, config);
|
||||
const n = Number(normalizedImage.count);
|
||||
const requestPrompt = buildImageReferencePromptText(prompt, references);
|
||||
if (requestConfig.apiFormat === "gemini") {
|
||||
if (requestConfig.interfaceType === "gemini-image") {
|
||||
if (mask) throw new Error("Gemini 调用格式暂不支持蒙版编辑");
|
||||
try {
|
||||
return await requestGeminiImages(requestConfig, requestPrompt, references, n, options);
|
||||
return await requestGeminiImages(requestConfig, requestPrompt, references, n, buildGeminiImageGenerationConfig(normalizedImage.size, normalizedImage.quality), options);
|
||||
} catch (error) {
|
||||
throw new Error(readAxiosError(error, "请求失败"));
|
||||
}
|
||||
|
||||
@@ -516,7 +516,7 @@ export function resolveModelRequestConfig(config: AiConfig, value: string) {
|
||||
apiKey: channel.apiKey,
|
||||
secretKey: channel.secretKey,
|
||||
headers: channel.headers,
|
||||
apiFormat: interfaceType ? (interfaceType === "gemini-veo" ? ("gemini" as const) : ("openai" as const)) : channel.apiFormat,
|
||||
apiFormat: interfaceType ? (interfaceType === "gemini-veo" || interfaceType === "gemini-image" ? ("gemini" as const) : ("openai" as const)) : channel.apiFormat,
|
||||
interfaceType,
|
||||
channelId: channel.scope === "system" ? channel.id : "",
|
||||
});
|
||||
@@ -564,7 +564,7 @@ export function defaultBaseUrlForApiFormat(apiFormat: ApiCallFormat) {
|
||||
}
|
||||
|
||||
export function defaultBaseUrlForChannelInterface(interfaceType?: ChannelInterfaceType) {
|
||||
if (interfaceType === "gemini-veo") return GEMINI_BASE_URL;
|
||||
if (interfaceType === "gemini-veo" || interfaceType === "gemini-image") return GEMINI_BASE_URL;
|
||||
if (interfaceType === "novita-video") return "https://api.novita.ai/v3";
|
||||
if (interfaceType === "volcengine-ark-image" || interfaceType === "volcengine-ark-video") return "https://ark.cn-beijing.volces.com/api/v3";
|
||||
if (interfaceType === "volcengine-jimeng-image" || interfaceType === "volcengine-jimeng-video") return "https://visual.volcengineapi.com";
|
||||
|
||||
Reference in New Issue
Block a user