feat(runtime): 即梦本机运行时 - 接入安全生成闭环与桌面本机渠道

Refs #225

Co-authored-by: zkhyww <127009345+zkhyww@users.noreply.github.com>
This commit is contained in:
ddCatandzkhyww committed 2026-08-15 12:45:29 +08:00
1 parent 8f2232feee
commit 57e1820f41
179 files changed
+49746 -1897

No files matched your search

@@ -3,15 +3,13 @@ import { App } from "antd";
import { buildNodeGenerationContext, hydrateNodeGenerationContext } from "@/components/canvas/canvas-node-generation";
import type { CanvasNodeGenerationMode } from "@/components/canvas/canvas-node-prompt-panel";
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { audioMetadata, imageMetadata, videoMetadata } from "@/lib/canvas/canvas-generation-task-sync";
import { fitNodeSize, nodeSizeFromRatio, VIDEO_NODE_MAX_SIZE } from "@/lib/canvas/canvas-node-size";
import { buildEmotionImageArtifacts, compositeEmotionImage, emotionGenerationSize } from "@/lib/canvas/canvas-emotion";
import { buildEmotionImageArtifacts, emotionGenerationSize } from "@/lib/canvas/canvas-emotion";
import {
buildAudioGenerationMetadata,
buildGenerationConfig,
buildImageGenerationMetadata,
buildVideoGenerationMetadata,
createGenerationRetryContext,
findRetrySourceNode,
generationReferenceUrls,
isGenerationCanceled,
@@ -19,6 +17,7 @@ import {
resolveMetadataReferences,
resolveStoredReferenceImages,
runBackendCanvasGenerationTask,
runCanvasGenerationTaskToConsumer,
sourceNodeReferenceImages,
supportsVideoReferenceAudio,
} from "@/lib/canvas/canvas-project-generation";
@@ -27,11 +26,9 @@ import { buildPortraitTexturePrompt } from "@/lib/canvas/canvas-portrait-texture
import { resolveCanvasStyleExecution } from "@/lib/canvas/canvas-style-execution";
import { generationFailureMetadata, unchangedModeratedPrompt } from "@/lib/generation-error";
import { navigateToSettings } from "@/lib/settings-navigation";
import { storeGeneratedAudio } from "@/services/api/audio";
import { storeGeneratedVideo } from "@/services/api/video";
import type { Skill } from "@/services/api/skills";
import type { GenerationTask } from "@/services/api/task-center";
import { resolveImageUrl, uploadImage } from "@/services/image-storage";
import { resolveImageUrl } from "@/services/image-storage";
import { resolveModelRequestConfig, useConfigStore, useEffectiveConfig } from "@/stores/use-config-store";
import type { Asset } from "@/stores/use-asset-store";
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "@/types/canvas";
@@ -48,13 +45,27 @@ type UseCanvasGenerationRetryOptions = {
startGenerationRequest: (targetNodeId: string, originNodeId: string, runningId?: string, controller?: AbortController) => AbortController;
finishGenerationRequest: (targetNodeId: string, controller: AbortController) => void;
bindGenerationTask: (targetNodeId: string, task: GenerationTask) => void;
applyGenerationTaskResult: (targetNodeId: string, task: GenerationTask) => Promise<void>;
};
const NODE_STATUS_LOADING = "loading" as const;
const NODE_STATUS_SUCCESS = "success" as const;
const NODE_STATUS_ERROR = "error" as const;
export function useCanvasGenerationRetry({ projectId, domainProjectId, addedSkills, assets, nodesRef, connectionsRef, setNodes, setRunningNodeId, startGenerationRequest, finishGenerationRequest, bindGenerationTask }: UseCanvasGenerationRetryOptions) {
export function useCanvasGenerationRetry({
projectId,
domainProjectId,
addedSkills,
assets,
nodesRef,
connectionsRef,
setNodes,
setRunningNodeId,
startGenerationRequest,
finishGenerationRequest,
bindGenerationTask,
applyGenerationTaskResult,
}: UseCanvasGenerationRetryOptions) {
const { message } = App.useApp();
const effectiveConfig = useEffectiveConfig();
const isAiConfigReady = useConfigStore((state) => state.isAiConfigReady);
@@ -72,7 +83,14 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, addedSkil
const hasSavedImageMetadata = Boolean(savedImageMetadata?.generationType);
let generationConfig =
hasSavedImageMetadata && savedImageMetadata
? { ...effectiveConfig, model: savedImageMetadata.model || effectiveConfig.imageModel || effectiveConfig.model, quality: savedImageMetadata.quality || effectiveConfig.quality, size: savedImageMetadata.size || effectiveConfig.size, transparentBackground: (savedImageMetadata.transparentBackground || effectiveConfig.transparentBackground) === "true" ? "true" : "false", count: "1" }
? {
...effectiveConfig,
model: savedImageMetadata.model || effectiveConfig.imageModel || effectiveConfig.model,
quality: savedImageMetadata.quality || effectiveConfig.quality,
size: savedImageMetadata.size || effectiveConfig.size,
transparentBackground: (savedImageMetadata.transparentBackground || effectiveConfig.transparentBackground) === "true" ? "true" : "false",
count: "1",
}
: { ...buildGenerationConfig(effectiveConfig, sourceNode, retryMode), count: "1" };
if (!isAiConfigReady(generationConfig, generationConfig.model)) {
navigateToSettings({ continueCreation: true });
@@ -80,9 +98,7 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, addedSkil
}
const retryPromptSource = sourceNode.metadata?.composerContent || sourceNode.metadata?.prompt || node.metadata?.prompt || "";
const retryContextPrompt = retryMode === "image" && sourceNode.metadata?.portraitTexture
? buildPortraitTexturePrompt(retryPromptSource, sourceNode.metadata.portraitTexture)
: retryPromptSource;
const retryContextPrompt = retryMode === "image" && sourceNode.metadata?.portraitTexture ? buildPortraitTexturePrompt(retryPromptSource, sourceNode.metadata.portraitTexture) : retryPromptSource;
if (unchangedModeratedPrompt(node.metadata, retryPromptSource)) {
message.warning("该提示词未通过内容审核,请先修改提示词再重新生成");
return;
@@ -137,7 +153,15 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, addedSkil
return;
}
const useReferenceImages = isEmotionRetry ? false : context?.characterReferences.length ? true : generationType ? generationType === "edit" : Boolean(context?.referenceImages.length);
const retryReferenceImages = isEmotionRetry ? [] : hasSavedImageMetadata && savedImageMetadata && !context?.characterReferences.length ? await resolveMetadataReferences(savedImageMetadata) : useReferenceImages ? (context?.referenceImages.length ? context.referenceImages : sourceNodeReferenceImages(batchRoot || sourceNode)) : [];
const retryReferenceImages = isEmotionRetry
? []
: hasSavedImageMetadata && savedImageMetadata && !context?.characterReferences.length
? await resolveMetadataReferences(savedImageMetadata)
: useReferenceImages
? context?.referenceImages.length
? context.referenceImages
: sourceNodeReferenceImages(batchRoot || sourceNode)
: [];
if (useReferenceImages && !retryReferenceImages) {
markMissingReferences(node.id, setNodes);
message.error("参考图片已丢失,无法继续重试");
@@ -151,34 +175,108 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, addedSkil
return;
}
const videoReferenceImages = context?.referenceImages.length ? context.referenceImages : storedVideoImages;
const videoContext = node.type === CanvasNodeType.Video ? { prompt, referenceImages: videoReferenceImages, referenceVideos: context?.referenceVideos || [], referenceAudios: context?.referenceAudios || [], textCount: context?.textCount || 0, imageCount: videoReferenceImages.length, videoCount: context?.referenceVideos.length || 0, audioCount: context?.referenceAudios.length || 0 } : undefined;
const videoContext =
node.type === CanvasNodeType.Video
? {
prompt,
referenceImages: videoReferenceImages,
referenceVideos: context?.referenceVideos || [],
referenceAudios: context?.referenceAudios || [],
textCount: context?.textCount || 0,
imageCount: videoReferenceImages.length,
videoCount: context?.referenceVideos.length || 0,
audioCount: context?.referenceAudios.length || 0,
}
: undefined;
setRunningNodeId(node.id);
setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, status: NODE_STATUS_LOADING, errorDetails: undefined, generationErrorCode: undefined, failedPromptFingerprint: undefined } } : item)));
const controller = startGenerationRequest(node.id, sourceNode.id, node.id);
const retryContext = node.metadata?.taskId ? await createGenerationRetryContext(node.metadata.taskId, node.metadata.attemptGroupId) : {};
const runAndConsumeRetry = async (input: Parameters<typeof runBackendCanvasGenerationTask>[0]) => {
await runCanvasGenerationTaskToConsumer(
{ ...input, ...retryContext },
{
bindTask: (task) => bindGenerationTask(node.id, task),
consumeTask: (task) => applyGenerationTaskResult(node.id, task),
},
);
};
try {
if (node.type === CanvasNodeType.Text) {
if (!context) return;
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "text", prompt, config: generationConfig, referenceImages: context.referenceImages, referenceVideos: context.referenceVideos, signal: controller.signal, metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context.resolvedCharacterVersions }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
if (!result.text) throw new Error("后端任务没有返回文本");
setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, type: CanvasNodeType.Text, metadata: { ...item.metadata, content: result.text, richText: undefined, prompt, status: NODE_STATUS_SUCCESS, errorDetails: undefined, generationErrorCode: undefined, failedPromptFingerprint: undefined } } : item)));
await runAndConsumeRetry({
projectId,
nodeId: node.id,
mode: "text",
prompt,
config: generationConfig,
referenceImages: context.referenceImages,
referenceVideos: context.referenceVideos,
signal: controller.signal,
metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context.resolvedCharacterVersions },
});
return;
}
if (node.type === CanvasNodeType.Video) {
const videoGenerationMetadata = buildVideoGenerationMetadata(node, videoContext);
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "video", prompt: mediaPrompt, config: generationConfig, referenceImages: videoContext?.referenceImages || [], referenceVideos: videoContext?.referenceVideos || [], referenceAudios: videoContext?.referenceAudios || [], signal: controller.signal, metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context?.resolvedCharacterVersions || [], resolvedCharacterVoices: context?.resolvedCharacterVoices || [], promptTemplateOperation: node.metadata?.promptTemplateOperation, promptTemplateVariables: node.metadata?.promptTemplateVariables, ...videoGenerationMetadata, ...styleMetadata }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
if (!result.video?.dataUrl) throw new Error("后端任务没有返回视频");
const video = await storeGeneratedVideo({ url: result.video.dataUrl, mimeType: result.video.mimeType || "video/mp4" });
const videoSize = fitNodeSize(video.width || node.width, video.height || node.height, VIDEO_NODE_MAX_SIZE.width, VIDEO_NODE_MAX_SIZE.height);
setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, width: videoSize.width, height: videoSize.height, position: { x: item.position.x + item.width / 2 - videoSize.width / 2, y: item.position.y + item.height / 2 - videoSize.height / 2 }, metadata: { ...item.metadata, ...videoMetadata(video), prompt: mediaPrompt, model: generationConfig.model, size: generationConfig.size, seconds: generationConfig.videoSeconds, vquality: generationConfig.vquality, generateAudio: generationConfig.videoGenerateAudio, watermark: generationConfig.videoWatermark, ...videoGenerationMetadata, ...styleMetadata, references: videoContext ? generationReferenceUrls(videoContext) : item.metadata?.references } } : item)));
setNodes((current) =>
current.map((item) =>
item.id === node.id
? {
...item,
metadata: {
...item.metadata,
prompt: mediaPrompt,
model: generationConfig.model,
size: generationConfig.size,
seconds: generationConfig.videoSeconds,
vquality: generationConfig.vquality,
generateAudio: generationConfig.videoGenerateAudio,
watermark: generationConfig.videoWatermark,
...videoGenerationMetadata,
...styleMetadata,
references: videoContext ? generationReferenceUrls(videoContext) : item.metadata?.references,
},
}
: item,
),
);
await runAndConsumeRetry({
projectId,
nodeId: node.id,
mode: "video",
prompt: mediaPrompt,
config: generationConfig,
referenceImages: videoContext?.referenceImages || [],
referenceVideos: videoContext?.referenceVideos || [],
referenceAudios: videoContext?.referenceAudios || [],
signal: controller.signal,
metadata: {
retry: true,
sourceNodeId: sourceNode.id,
resolvedCharacterVersions: context?.resolvedCharacterVersions || [],
resolvedCharacterVoices: context?.resolvedCharacterVoices || [],
promptTemplateOperation: node.metadata?.promptTemplateOperation,
promptTemplateVariables: node.metadata?.promptTemplateVariables,
...videoGenerationMetadata,
...styleMetadata,
},
});
return;
}
if (node.type === CanvasNodeType.Audio) {
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "audio", prompt, config: generationConfig, signal: controller.signal, metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context?.resolvedCharacterVersions || [], resolvedCharacterVoiceKey: context?.resolvedCharacterVoices[0]?.voiceKey }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
if (!result.audio?.dataUrl) throw new Error("后端任务没有返回音频");
const audio = await storeGeneratedAudio(await (await fetch(result.audio.dataUrl)).blob(), generationConfig.audioFormat);
setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, ...audioMetadata(audio), prompt, ...buildAudioGenerationMetadata(generationConfig) } } : item)));
setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, prompt, ...buildAudioGenerationMetadata(generationConfig) } } : item)));
await runAndConsumeRetry({
projectId,
nodeId: node.id,
mode: "audio",
prompt,
config: generationConfig,
signal: controller.signal,
metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context?.resolvedCharacterVersions || [], resolvedCharacterVoiceKey: context?.resolvedCharacterVoices[0]?.voiceKey },
});
return;
}
@@ -195,30 +293,69 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, addedSkil
const editReference = { id: `${emotionSource.id}-${emotionEdit.presetId}-edit-region`, name: "emotion-edit-region.png", type: "image/png", dataUrl: artifacts.sourceDataUrl };
const characterReference = { id: `${emotionSource.id}-${emotionEdit.presetId}-character`, name: `${emotionEdit.characterName}-face.jpg`, type: "image/jpeg", dataUrl: artifacts.characterDataUrl };
const nextEmotionEdit = { ...emotionEdit, editRegion: artifacts.editRegion, sourceWidth: artifacts.imageWidth, sourceHeight: artifacts.imageHeight, providerSize: emotionConfig.size };
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "image", prompt: mediaPrompt, config: emotionConfig, referenceImages: [editReference, characterReference], mask: { id: `${emotionSource.id}-emotion-mask`, name: "emotion-mask.png", type: "image/png", dataUrl: artifacts.maskDataUrl }, signal: controller.signal, metadata: { retry: true, sourceNodeId: emotionSource.id, edit: "emotion", emotion: nextEmotionEdit, resolvedCharacterVersions: context?.resolvedCharacterVersions || [], ...styleMetadata }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
const image = result.images?.[0];
if (!image?.dataUrl) throw new Error("后端任务没有返回图片");
const composited = await compositeEmotionImage(sourceDataUrl, image.dataUrl, artifacts.editRegion, emotionEdit.faceBox);
const uploadedImage = await uploadImage(composited);
const imageSize = fitNodeSize(uploadedImage.width, uploadedImage.height, node.width, node.height);
const generationMetadata = { ...buildImageGenerationMetadata("edit", emotionConfig, 1, [sourceReference]), size: `${artifacts.imageWidth}x${artifacts.imageHeight}` };
setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, type: CanvasNodeType.Image, width: imageSize.width, height: imageSize.height, metadata: { ...item.metadata, ...imageMetadata(uploadedImage), prompt: mediaPrompt, ...generationMetadata, ...styleMetadata, emotionEdit: nextEmotionEdit } } : item)));
setNodes((current) =>
current.map((item) =>
item.id === node.id
? {
...item,
metadata: { ...item.metadata, prompt: mediaPrompt, ...generationMetadata, ...styleMetadata, emotionEdit: nextEmotionEdit },
}
: item,
),
);
await runAndConsumeRetry({
projectId,
nodeId: node.id,
mode: "image",
prompt: mediaPrompt,
config: emotionConfig,
referenceImages: [editReference, characterReference],
mask: { id: `${emotionSource.id}-emotion-mask`, name: "emotion-mask.png", type: "image/png", dataUrl: artifacts.maskDataUrl },
signal: controller.signal,
metadata: { retry: true, sourceNodeId: emotionSource.id, edit: "emotion", emotion: nextEmotionEdit, resolvedCharacterVersions: context?.resolvedCharacterVersions || [], ...styleMetadata },
});
return;
}
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "image", prompt: mediaPrompt, config: generationConfig, referenceImages: useReferenceImages ? retryImages : [], signal: controller.signal, metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context?.resolvedCharacterVersions || [], promptTemplateOperation: node.metadata?.promptTemplateOperation, promptTemplateVariables: node.metadata?.promptTemplateVariables, ...styleMetadata }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
const image = result.images?.[0];
if (!image?.dataUrl) throw new Error("后端任务没有返回图片");
const uploadedImage = await uploadImage(image.dataUrl);
const imageConfig = NODE_DEFAULT_SIZE[CanvasNodeType.Image];
const requestedImageSize = nodeSizeFromRatio(generationConfig.size || "auto", imageConfig.width, imageConfig.height);
const imageSize = useReferenceImages && !requestedImageSize
? { width: node.width, height: node.height }
: fitNodeSize(uploadedImage.width, uploadedImage.height, requestedImageSize?.width || imageConfig.width, requestedImageSize?.height || imageConfig.height);
const generationMetadata = savedImageMetadata?.generationType
? { generationType: savedImageMetadata.generationType, model: generationConfig.model, size: generationConfig.size, quality: generationConfig.quality, transparentBackground: generationConfig.transparentBackground, count: savedImageMetadata.count || 1, references: savedImageMetadata.references }
? {
generationType: savedImageMetadata.generationType,
model: generationConfig.model,
size: generationConfig.size,
quality: generationConfig.quality,
transparentBackground: generationConfig.transparentBackground,
count: savedImageMetadata.count || 1,
references: savedImageMetadata.references,
}
: buildImageGenerationMetadata(useReferenceImages ? "edit" : "generation", generationConfig, 1, retryImages);
setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, type: CanvasNodeType.Image, position: { x: item.position.x + item.width / 2 - imageSize.width / 2, y: item.position.y + item.height / 2 - imageSize.height / 2 }, width: imageSize.width, height: imageSize.height, metadata: { ...item.metadata, ...imageMetadata(uploadedImage), prompt: mediaPrompt, ...generationMetadata, ...styleMetadata } } : item)));
setNodes((current) =>
current.map((item) =>
item.id === node.id
? {
...item,
metadata: { ...item.metadata, prompt: mediaPrompt, ...generationMetadata, ...styleMetadata },
}
: item,
),
);
await runAndConsumeRetry({
projectId,
nodeId: node.id,
mode: "image",
prompt: mediaPrompt,
config: generationConfig,
referenceImages: useReferenceImages ? retryImages : [],
signal: controller.signal,
metadata: {
retry: true,
sourceNodeId: sourceNode.id,
resolvedCharacterVersions: context?.resolvedCharacterVersions || [],
promptTemplateOperation: node.metadata?.promptTemplateOperation,
promptTemplateVariables: node.metadata?.promptTemplateVariables,
...styleMetadata,
},
});
} catch (error) {
if (isGenerationCanceled(error)) return;
const failure = generationFailureMetadata(error, retryPromptSource);
@@ -229,7 +366,7 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, addedSkil
setRunningNodeId(null);
}
},
[addedSkills, assets, bindGenerationTask, connectionsRef, domainProjectId, effectiveConfig, finishGenerationRequest, isAiConfigReady, message, nodesRef, projectId, setNodes, setRunningNodeId, startGenerationRequest],
[addedSkills, applyGenerationTaskResult, assets, bindGenerationTask, connectionsRef, domainProjectId, effectiveConfig, finishGenerationRequest, isAiConfigReady, message, nodesRef, projectId, setNodes, setRunningNodeId, startGenerationRequest],
);
}