fix(canvas): 分镜与视频模型 - 按端点协议适配并修复分镜生成
* fix(backend): 分镜生成 - 自动修复携带原始剧情并支持有界重试 * fix(canvas): 视频模型 - 按端点协议拉取模型并适配分镜视频节点 * fix(canvas): 视频模型 - 保留已有协议并刷新分镜节点模型 * fix(canvas): 引用图预览 - 修正提示面板代码格式 --------- Co-authored-by: sikongyue <3155157671@qq.com> Co-authored-by: ddCat <2695092045@qq.com>
This commit is contained in:
爱笑的毛毛虫
@@ -388,14 +388,22 @@ function ConnectedReferenceShelf({ references, theme, onInsert }: { references:
|
||||
style={{ background: theme.toolbar.itemHover, color: theme.node.text, outlineColor: theme.node.activeStroke, boxShadow: `0 4px 14px ${theme.spatial.shadow}` }}
|
||||
title={canPreview ? `预览 ${reference.title}` : `插入 @${reference.label}`}
|
||||
aria-label={canPreview ? `预览 ${reference.title}` : `插入 @${reference.label}`}
|
||||
onClick={() => canPreview ? setImagePreview(reference) : onInsert(reference)}
|
||||
onClick={() => (canPreview ? setImagePreview(reference) : onInsert(reference))}
|
||||
>
|
||||
<span className="block size-full overflow-hidden rounded-md">
|
||||
<ReferenceThumbnail reference={reference} />
|
||||
</span>
|
||||
<span className="absolute left-0.5 top-0.5 grid size-3.5 place-items-center rounded-full bg-black/65 text-[var(--fs-micro)] font-semibold text-white backdrop-blur-sm">{index + 1}</span>
|
||||
{canPreview ? <span className="absolute bottom-0.5 left-0.5 grid size-3.5 place-items-center rounded-full bg-black/65 text-white backdrop-blur-sm"><Maximize2 className="size-2" /></span> : null}
|
||||
{!canPreview ? <span className="absolute bottom-0.5 right-0.5 grid size-3.5 place-items-center rounded-full bg-black/65 text-white backdrop-blur-sm"><AtSign className="size-2" /></span> : null}
|
||||
{canPreview ? (
|
||||
<span className="absolute bottom-0.5 left-0.5 grid size-3.5 place-items-center rounded-full bg-black/65 text-white backdrop-blur-sm">
|
||||
<Maximize2 className="size-2" />
|
||||
</span>
|
||||
) : null}
|
||||
{!canPreview ? (
|
||||
<span className="absolute bottom-0.5 right-0.5 grid size-3.5 place-items-center rounded-full bg-black/65 text-white backdrop-blur-sm">
|
||||
<AtSign className="size-2" />
|
||||
</span>
|
||||
) : null}
|
||||
</button>
|
||||
{canPreview ? (
|
||||
<button
|
||||
|
||||
@@ -86,6 +86,19 @@ export function modelProtocolCapability(value?: string) {
|
||||
return modelProtocolDefinition(value)?.capability;
|
||||
}
|
||||
|
||||
export function protocolForModelCatalog(endpointTypes: string[] = []): ModelProtocol | undefined {
|
||||
const normalized = new Set(endpointTypes.map((value) => value.trim().toLowerCase()));
|
||||
if (normalized.has("openai-chat") || normalized.has("chat-completion") || normalized.has("chat")) return "chat-completion";
|
||||
if (normalized.has("openai-response") || normalized.has("responses")) return "openai-response";
|
||||
if (normalized.has("openai-image") || normalized.has("image")) return "openai-image";
|
||||
if (normalized.has("openai-video") || normalized.has("video")) return "newapi-channel-2";
|
||||
if (normalized.has("openai-audio") || normalized.has("audio")) return "openai-audio";
|
||||
if (normalized.has("grok-image")) return "grok-image";
|
||||
if (normalized.has("gemini-veo") || normalized.has("gemini-video") || normalized.has("veo")) return "gemini-veo";
|
||||
if (normalized.has("volcengine-ark-video") || normalized.has("volcengine-ark")) return "volcengine-ark-video";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function modelProtocolSummary(value?: string) {
|
||||
const protocol = modelProtocolDefinition(value);
|
||||
if (!protocol) return "保存模型时选择协议后，将在这里显示实际请求方式。";
|
||||
|
||||
@@ -276,6 +276,7 @@ export function useCanvasStoryboard({
|
||||
if (!scriptNode || !rows.length) return;
|
||||
const videoSpec = NODE_DEFAULT_SIZE[CanvasNodeType.Video];
|
||||
const startLeft = scriptNode.position.x + scriptNode.width + 120;
|
||||
const videoModel = buildGenerationConfig(effectiveConfig, undefined, "video").model;
|
||||
const nextNodes = [...nodesRef.current];
|
||||
const nextConnections = [...connectionsRef.current];
|
||||
const videoNodeByRowId = new Map<string, string>();
|
||||
@@ -286,14 +287,14 @@ export function useCanvasStoryboard({
|
||||
if (existingIndex >= 0) {
|
||||
const existing = nextNodes[existingIndex];
|
||||
const existingMetadata = existing.metadata?.content ? existing.metadata : resetGenerationTaskMetadata(existing.metadata);
|
||||
nextNodes[existingIndex] = { ...existing, metadata: { ...existingMetadata, prompt, composerContent: prompt, ...storyboardPromptTemplateMetadata(row, "video"), seconds: String(row.durationSeconds), shotIndex: row.shotNumber, workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 视频`, generationMode: "video", videoEditOperation: existing.metadata?.videoEditOperation || "text_to_video" } };
|
||||
nextNodes[existingIndex] = { ...existing, metadata: { ...existingMetadata, prompt, composerContent: prompt, model: videoModel, ...storyboardPromptTemplateMetadata(row, "video"), seconds: String(row.durationSeconds), shotIndex: row.shotNumber, workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 视频`, generationMode: "video", videoEditOperation: existing.metadata?.videoEditOperation || "text_to_video" } };
|
||||
storyboardRowReferenceNodeIds(scriptNode, row, nextNodes, nextConnections, false).forEach((referenceId) => {
|
||||
if (!nextConnections.some((connection) => connection.fromNodeId === referenceId && connection.toNodeId === existing.id)) nextConnections.push({ id: nanoid(), fromNodeId: referenceId, toNodeId: existing.id });
|
||||
});
|
||||
videoNodeByRowId.set(row.id, existing.id);
|
||||
return;
|
||||
}
|
||||
const videoNode = createCanvasNode(CanvasNodeType.Video, { x: startLeft + videoSpec.width / 2, y: scriptNode.position.y + index * (videoSpec.height + 36) + videoSpec.height / 2 }, { prompt, composerContent: prompt, ...storyboardPromptTemplateMetadata(row, "video"), workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 视频`, shotIndex: row.shotNumber, generationMode: "video", videoEditOperation: "text_to_video", status: NODE_STATUS_IDLE, seconds: String(row.durationSeconds) });
|
||||
const videoNode = createCanvasNode(CanvasNodeType.Video, { x: startLeft + videoSpec.width / 2, y: scriptNode.position.y + index * (videoSpec.height + 36) + videoSpec.height / 2 }, { prompt, composerContent: prompt, model: videoModel, ...storyboardPromptTemplateMetadata(row, "video"), workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 视频`, shotIndex: row.shotNumber, generationMode: "video", videoEditOperation: "text_to_video", status: NODE_STATUS_IDLE, seconds: String(row.durationSeconds) });
|
||||
videoNode.title = `镜头 ${row.shotNumber} · 视频`;
|
||||
nextNodes.push(videoNode);
|
||||
nextConnections.push({ id: nanoid(), fromNodeId: scriptNode.id, toNodeId: videoNode.id, fromHandleId: `row:${row.id}` });
|
||||
@@ -320,7 +321,7 @@ export function useCanvasStoryboard({
|
||||
setNodes(nextNodes);
|
||||
setConnections(nextConnections);
|
||||
if (!silent) message.success(createdCount ? `已创建 ${createdCount} 个视频节点` : "已同步现有视频节点的提示词");
|
||||
}, [connectionsRef, message, nodesRef, setConnections, setNodes]);
|
||||
}, [connectionsRef, effectiveConfig, message, nodesRef, setConnections, setNodes]);
|
||||
|
||||
const createAndGenerateScriptVideos = useCallback(async (nodeId: string, rowIds?: string[]) => {
|
||||
const videoModel = effectiveConfig.videoModel || effectiveConfig.model;
|
||||
@@ -358,7 +359,7 @@ export function useCanvasStoryboard({
|
||||
const targetById = new Map(targets.map((target) => [target.videoNode.id, target]));
|
||||
const nextNodes = nodesRef.current.map((node) => {
|
||||
const target = targetById.get(node.id);
|
||||
return target ? { ...node, metadata: { ...node.metadata, prompt: target.prompt, composerContent: target.prompt, ...storyboardPromptTemplateMetadata(target.row, "video"), generationMode: "video" as const, videoEditOperation: "text_to_video" as const, videoStartFrameNodeId: undefined } } : node;
|
||||
return target ? { ...node, metadata: { ...node.metadata, prompt: target.prompt, composerContent: target.prompt, model: videoModel, ...storyboardPromptTemplateMetadata(target.row, "video"), generationMode: "video" as const, videoEditOperation: "text_to_video" as const, videoStartFrameNodeId: undefined } } : node;
|
||||
});
|
||||
const dedicatedFirstFrameConnections = new Set(targets
|
||||
.filter((target) => Boolean(target.row.imageNodeId))
|
||||
@@ -453,8 +454,8 @@ export function useCanvasStoryboard({
|
||||
const existing = row.videoNodeId ? nextNodes.find((node) => node.id === row.videoNodeId && node.type === CanvasNodeType.Video) : undefined;
|
||||
const existingMetadata = existing?.metadata?.content ? existing.metadata : resetGenerationTaskMetadata(existing?.metadata);
|
||||
const videoNode = existing
|
||||
? { ...existing, metadata: { ...existingMetadata, prompt, composerContent: prompt, ...storyboardPromptTemplateMetadata(row, "video"), workflowKind: "shot" as const, workflowTitle: `镜头 ${row.shotNumber} 视频`, shotIndex: row.shotNumber, generationMode: "video" as const, videoEditOperation: "image_to_video" as const, videoStartFrameNodeId: row.imageNodeId, seconds: String(row.durationSeconds) } }
|
||||
: createCanvasNode(CanvasNodeType.Video, { x: startX, y: currentScriptNode.position.y + index * (videoSpec.height + 36) + videoSpec.height / 2 }, { prompt, ...storyboardPromptTemplateMetadata(row, "video"), workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 视频`, shotIndex: row.shotNumber, generationMode: "video", videoEditOperation: "image_to_video", videoStartFrameNodeId: row.imageNodeId, status: NODE_STATUS_IDLE, seconds: String(row.durationSeconds) });
|
||||
? { ...existing, metadata: { ...existingMetadata, prompt, composerContent: prompt, model: videoModel, ...storyboardPromptTemplateMetadata(row, "video"), workflowKind: "shot" as const, workflowTitle: `镜头 ${row.shotNumber} 视频`, shotIndex: row.shotNumber, generationMode: "video" as const, videoEditOperation: "image_to_video" as const, videoStartFrameNodeId: row.imageNodeId, seconds: String(row.durationSeconds) } }
|
||||
: createCanvasNode(CanvasNodeType.Video, { x: startX, y: currentScriptNode.position.y + index * (videoSpec.height + 36) + videoSpec.height / 2 }, { prompt, model: videoModel, ...storyboardPromptTemplateMetadata(row, "video"), workflowKind: "shot", workflowTitle: `镜头 ${row.shotNumber} 视频`, shotIndex: row.shotNumber, generationMode: "video", videoEditOperation: "image_to_video", videoStartFrameNodeId: row.imageNodeId, status: NODE_STATUS_IDLE, seconds: String(row.durationSeconds) });
|
||||
if (!existing) {
|
||||
videoNode.title = `镜头 ${row.shotNumber} · 视频`;
|
||||
nextNodes.push(videoNode);
|
||||
|
||||
@@ -8,7 +8,9 @@ import { WorkspaceState } from "@/components/layout/workspace-state";
|
||||
import { WorkspaceSignalIcon } from "@/components/ui/aceternity/workspace-signal-icon";
|
||||
import { ChannelHeadersEditor, validateChannelHeaders } from "@/components/channel-headers-editor";
|
||||
import { refreshSystemChannels } from "@/lib/user-session";
|
||||
import { fetchChannelModels } from "@/services/api/image";
|
||||
import { fetchChannelModels, type ChannelModelCatalogItem } from "@/services/api/image";
|
||||
import { defaultModelCapabilityConfig } from "@/lib/model-capabilities";
|
||||
import { modelProtocolCapability, protocolForModelCatalog } from "@/lib/model-protocols";
|
||||
import { audioFormatOptions, audioVoiceOptions, normalizeAudioSpeedValue } from "@/lib/audio-generation";
|
||||
import {
|
||||
createModelChannel,
|
||||
@@ -166,8 +168,8 @@ export default function SettingsPage() {
|
||||
}
|
||||
setChannelLoading(channel.id, true);
|
||||
try {
|
||||
const models = await fetchChannelModels(channel, true);
|
||||
if (!models.length) {
|
||||
const result = await fetchChannelModels(channel, true);
|
||||
if (!result.models.length) {
|
||||
message.warning(`${channel.name || "当前渠道"}未返回模型，已保留现有手工模型`);
|
||||
return;
|
||||
}
|
||||
@@ -179,7 +181,7 @@ export default function SettingsPage() {
|
||||
return;
|
||||
}
|
||||
updateChannels(
|
||||
latestConfig.channels.map((item) => (item.id === channel.id ? { ...item, models } : item)),
|
||||
latestConfig.channels.map((item) => (item.id === channel.id ? { ...item, models: result.models, modelCosts: mergeFetchedModelCosts(item, result.catalog) } : item)),
|
||||
latestConfig,
|
||||
);
|
||||
message.success(`${latestChannel.name || "当前渠道"}模型列表已更新`);
|
||||
@@ -203,27 +205,30 @@ export default function SettingsPage() {
|
||||
const results = await Promise.all(
|
||||
runnable.map(async (channel) => {
|
||||
try {
|
||||
const models = await fetchChannelModels(channel, true);
|
||||
return { channel, models, error: "" };
|
||||
const result = await fetchChannelModels(channel, true);
|
||||
return { channel, result, error: "" };
|
||||
} catch (error) {
|
||||
return { channel, models: [] as string[], error: error instanceof Error ? error.message : "读取失败" };
|
||||
return { channel, result: { models: [], catalog: [] }, error: error instanceof Error ? error.message : "读取失败" };
|
||||
}
|
||||
}),
|
||||
);
|
||||
const latestConfig = useConfigStore.getState().config;
|
||||
const successful = results.filter((item) => {
|
||||
const latestChannel = latestConfig.channels.find((channel) => channel.id === item.channel.id);
|
||||
return Boolean(item.models.length && latestChannel && channelConnectionSignature(latestChannel) === channelConnectionSignature(item.channel));
|
||||
return Boolean(item.result.models.length && latestChannel && channelConnectionSignature(latestChannel) === channelConnectionSignature(item.channel));
|
||||
});
|
||||
const stale = results.filter((item) => {
|
||||
const latestChannel = latestConfig.channels.find((channel) => channel.id === item.channel.id);
|
||||
return Boolean(item.models.length && (!latestChannel || channelConnectionSignature(latestChannel) !== channelConnectionSignature(item.channel)));
|
||||
return Boolean(item.result.models.length && (!latestChannel || channelConnectionSignature(latestChannel) !== channelConnectionSignature(item.channel)));
|
||||
});
|
||||
const failed = results.filter((item) => !item.models.length);
|
||||
const failed = results.filter((item) => !item.result.models.length);
|
||||
if (successful.length) {
|
||||
const modelMap = new Map(successful.map((item) => [item.channel.id, item.models] as const));
|
||||
const resultMap = new Map(successful.map((item) => [item.channel.id, item.result] as const));
|
||||
updateChannels(
|
||||
latestConfig.channels.map((channel) => (modelMap.has(channel.id) ? { ...channel, models: modelMap.get(channel.id) || channel.models } : channel)),
|
||||
latestConfig.channels.map((channel) => {
|
||||
const fetched = resultMap.get(channel.id);
|
||||
return fetched ? { ...channel, models: fetched.models, modelCosts: mergeFetchedModelCosts(channel, fetched.catalog) } : channel;
|
||||
}),
|
||||
latestConfig,
|
||||
);
|
||||
message.success(`已更新 ${successful.length} 个渠道的模型`);
|
||||
@@ -590,6 +595,45 @@ function normalizeImageCount(value: string) {
|
||||
return String(Math.max(1, Math.min(15, Math.floor(Math.abs(Number(value)) || Number(defaultConfig.canvasImageCount)))));
|
||||
}
|
||||
|
||||
type ChannelModelCost = NonNullable<ModelChannel["modelCosts"]>[number];
|
||||
|
||||
// 拉取模型目录时按上游 supported_endpoint_types 推导协议和能力；
|
||||
// 保留已有定价，但目录端点类型与旧配置冲突时自动纠正协议，避免模型名猜测和 multipart 错配。
|
||||
function mergeFetchedModelCosts(channel: ModelChannel, catalog: ChannelModelCatalogItem[]): ChannelModelCost[] {
|
||||
const existingByModel = new Map((channel.modelCosts || []).map((cost) => [cost.model, cost]));
|
||||
const next: ChannelModelCost[] = [];
|
||||
for (const item of catalog) {
|
||||
const existing = existingByModel.get(item.id);
|
||||
const inferredProtocol = protocolForModelCatalog(item.supportedEndpointTypes);
|
||||
if (existing) {
|
||||
if (inferredProtocol && existing.protocol !== inferredProtocol) {
|
||||
const inferredCapability = modelProtocolCapability(inferredProtocol);
|
||||
next.push({
|
||||
...existing,
|
||||
protocol: inferredProtocol,
|
||||
capability: inferredCapability || existing.capability,
|
||||
capabilityConfig: inferredCapability === "image" || inferredCapability === "video" ? defaultModelCapabilityConfig(inferredProtocol, item.id) : undefined,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
next.push(existing);
|
||||
continue;
|
||||
}
|
||||
const catalogProtocol = inferredProtocol || channel.interfaceType;
|
||||
const catalogCapability = catalogProtocol ? modelProtocolCapability(catalogProtocol) : undefined;
|
||||
if (!catalogProtocol || !catalogCapability) continue;
|
||||
next.push({
|
||||
model: item.id,
|
||||
capability: catalogCapability,
|
||||
protocol: catalogProtocol,
|
||||
billingMode: "fixed_request",
|
||||
unitPriceMicrocredits: 0,
|
||||
capabilityConfig: catalogCapability === "image" || catalogCapability === "video" ? defaultModelCapabilityConfig(catalogProtocol, item.id) : undefined,
|
||||
});
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function uniqueModels(models: string[]) {
|
||||
return Array.from(new Set(models.map((model) => model.trim()).filter(Boolean)));
|
||||
}
|
||||
|
||||
@@ -1095,13 +1095,18 @@ export async function fetchImageModels(config: Pick<AiConfig, "baseUrl" | "apiKe
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchChannelModels(channel: ModelChannel, viaBackend = false) {
|
||||
export type ChannelModelCatalogItem = { id: string; supportedEndpointTypes?: string[] };
|
||||
|
||||
export type ChannelModelFetchResult = { models: string[]; catalog: ChannelModelCatalogItem[] };
|
||||
|
||||
export async function fetchChannelModels(channel: ModelChannel, viaBackend = false): Promise<ChannelModelFetchResult> {
|
||||
if (!viaBackend) {
|
||||
return fetchImageModels({ baseUrl: channel.baseUrl, apiKey: channel.apiKey, apiFormat: channel.apiFormat });
|
||||
const models = await fetchImageModels({ baseUrl: channel.baseUrl, apiKey: channel.apiKey, apiFormat: channel.apiFormat });
|
||||
return { models, catalog: models.map((id) => ({ id })) };
|
||||
}
|
||||
try {
|
||||
// 登录态由同源后端代取模型目录，避免每个 OpenAI 兼容服务分别维护浏览器 CORS 白名单。
|
||||
const response = await axios.post<{ code?: number; data?: { models?: string[] }; msg?: string }>(
|
||||
const response = await axios.post<{ code?: number; data?: { models?: Array<string | ChannelModelCatalogItem> }; msg?: string }>(
|
||||
resolveBackendApiUrl("/api/ai/models"),
|
||||
{
|
||||
baseUrl: channel.baseUrl,
|
||||
@@ -1114,7 +1119,16 @@ export async function fetchChannelModels(channel: ModelChannel, viaBackend = fal
|
||||
if (typeof response.data.code === "number" && response.data.code !== 0) {
|
||||
throw new Error(response.data.msg || "读取模型失败");
|
||||
}
|
||||
return Array.from(new Set((response.data.data?.models || []).map((model) => model.trim()).filter(Boolean))).sort((a, b) => a.localeCompare(b));
|
||||
const catalog = new Map<string, ChannelModelCatalogItem>();
|
||||
for (const item of response.data.data?.models || []) {
|
||||
const entry = typeof item === "string" ? { id: item.trim() } : { id: String(item.id || "").trim(), supportedEndpointTypes: Array.isArray(item.supportedEndpointTypes) ? item.supportedEndpointTypes : undefined };
|
||||
if (!entry.id) continue;
|
||||
const existing = catalog.get(entry.id);
|
||||
catalog.set(entry.id, existing || entry);
|
||||
}
|
||||
const models = Array.from(catalog.keys()).sort((a, b) => a.localeCompare(b));
|
||||
const sortedCatalog = Array.from(catalog.values()).sort((a, b) => a.id.localeCompare(b.id));
|
||||
return { models, catalog: sortedCatalog };
|
||||
} catch (error) {
|
||||
throw new Error(readAxiosError(error, "读取模型失败"));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user