tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(canvas): 视频生成 - 仅使用输入框提示词并保留结构化媒体引用

Browse Source
This commit is contained in:
ddCat
2026-08-08 08:38:11 +08:00
parent 444f8910fe
commit 8d4f656ae9
4 changed files with 25 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
backend/internal/service/provider.go
View File

@@ -162,8 +162,12 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string
if strings.TrimSpace(input.Prompt) == "" { if strings.TrimSpace(input.Prompt) == "" {
input.Prompt = fallbackPrompt input.Prompt = fallbackPrompt
} }
if input.Mode == "" && strings.HasPrefix(taskType, "video_") {
input.Mode = "video"
}
promptTemplateOperation := metadataString(input.Metadata, "promptTemplateOperation") promptTemplateOperation := metadataString(input.Metadata, "promptTemplateOperation")
if promptTemplateOperation != "" { // 视频节点的最终 Prompt 只取输入框内容，不能被分镜模板替换；图片和文本仍沿用模板能力。
if input.Mode != "video" && promptTemplateOperation != "" {
values := metadataStringValues(input.Metadata["promptTemplateVariables"]) values := metadataStringValues(input.Metadata["promptTemplateVariables"])
compiled, compileErr := s.compilePrompt(userID, promptTemplateOperation, values) compiled, compileErr := s.compilePrompt(userID, promptTemplateOperation, values)
if compileErr != nil { if compileErr != nil {
@@ -174,15 +178,12 @@ func (s *Service) processCanvasGenerationTask(ctx context.Context, userID string
if strings.TrimSpace(input.Prompt) == "" { if strings.TrimSpace(input.Prompt) == "" {
return nil, errors.New("prompt is required") return nil, errors.New("prompt is required")
} }
if input.Mode == "" && strings.HasPrefix(taskType, "video_") {
input.Mode = "video"
}
config, err := s.resolveProviderConfig(input.Config) config, err := s.resolveProviderConfig(input.Config)
if err != nil { if err != nil {
return nil, err return nil, err
} }
input.Config = config input.Config = config
if (input.Mode == "image" || input.Mode == "video") && input.Metadata != nil { if input.Mode == "image" && input.Metadata != nil {
if err := s.applyGenerationStyleProfile(userID, taskProjectID, &input); err != nil { if err := s.applyGenerationStyleProfile(userID, taskProjectID, &input); err != nil {
return nil, err return nil, err
} }

14
web/src/components/canvas/canvas-node-generation.ts
View File

@@ -55,13 +55,13 @@ export type NodeGenerationInput = {
character?: CharacterGenerationReference; character?: CharacterGenerationReference;
}; };
export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string): NodeGenerationContext { export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[], prompt: string, promptOnly = false): NodeGenerationContext {
const inputs = buildNodeGenerationInputs(nodeId, nodes, connections); const inputs = buildNodeGenerationInputs(nodeId, nodes, connections);
const sourceNode = nodes.find((node) => node.id === nodeId); const sourceNode = nodes.find((node) => node.id === nodeId);
const storyboardInputs = getConnectedStoryboardRows(nodeId, nodes, connections); const storyboardInputs = getConnectedStoryboardRows(nodeId, nodes, connections);
const hasExplicitNodeMention = /@\[node:[^\]]+\]/.test(normalizeLegacyNodeMentions(prompt, inputs)); const hasExplicitNodeMention = /@\[node:[^\]]+\]/.test(normalizeLegacyNodeMentions(prompt, inputs));
if ((sourceNode?.type === CanvasNodeType.Config && Boolean(sourceNode.metadata?.composerContent?.trim())) || hasExplicitNodeMention) { if ((sourceNode?.type === CanvasNodeType.Config && Boolean(sourceNode.metadata?.composerContent?.trim())) || hasExplicitNodeMention) {
return buildComposerGenerationContext(inputs, prompt, [sourceNode?.metadata?.videoStartFrameNodeId, sourceNode?.metadata?.videoEndFrameNodeId].filter((id): id is string => Boolean(id))); return buildComposerGenerationContext(inputs, prompt, [sourceNode?.metadata?.videoStartFrameNodeId, sourceNode?.metadata?.videoEndFrameNodeId].filter((id): id is string => Boolean(id)), promptOnly);
} }
const isStoryboardMedia = sourceNode?.type === CanvasNodeType.Image || sourceNode?.type === CanvasNodeType.Video; const isStoryboardMedia = sourceNode?.type === CanvasNodeType.Image || sourceNode?.type === CanvasNodeType.Video;
@@ -77,7 +77,7 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData
const referenceAudios = inputs.map((input) => input.audio).filter((audio): audio is ReferenceAudio => Boolean(audio)); const referenceAudios = inputs.map((input) => input.audio).filter((audio): audio is ReferenceAudio => Boolean(audio));
return { return {
prompt: upstreamText ? `${basePrompt}\n\n${upstreamText}` : basePrompt, prompt: promptOnly ? prompt : upstreamText ? `${basePrompt}\n\n${upstreamText}` : basePrompt,
referenceImages, referenceImages,
referenceVideos, referenceVideos,
referenceAudios, referenceAudios,
@@ -109,7 +109,7 @@ function removeTrailingInputBlocks(prompt: string, inputs: NodeGenerationInput[]
return next; return next;
} }
function buildComposerGenerationContext(inputs: NodeGenerationInput[], prompt: string, videoFrameNodeIds: string[] = []): NodeGenerationContext { function buildComposerGenerationContext(inputs: NodeGenerationInput[], prompt: string, videoFrameNodeIds: string[] = [], promptOnly = false): NodeGenerationContext {
const normalizedPrompt = normalizeLegacyNodeMentions(prompt, inputs); const normalizedPrompt = normalizeLegacyNodeMentions(prompt, inputs);
const inputByNodeId = new Map(inputs.map((input) => [input.nodeId, input])); const inputByNodeId = new Map(inputs.map((input) => [input.nodeId, input]));
const selectedInputs: NodeGenerationInput[] = []; const selectedInputs: NodeGenerationInput[] = [];
@@ -140,7 +140,7 @@ function buildComposerGenerationContext(inputs: NodeGenerationInput[], prompt: s
} }
nextPrompt += normalizedPrompt.slice(lastIndex); nextPrompt += normalizedPrompt.slice(lastIndex);
if (textBlocks.length) nextPrompt = `${nextPrompt.trim()}\n\n${textBlocks.join("\n\n")}`; if (textBlocks.length && !promptOnly) nextPrompt = `${nextPrompt.trim()}\n\n${textBlocks.join("\n\n")}`;
// 首尾帧是结构化生成参数，不受提示词中的 @ 引用筛选影响。 // 首尾帧是结构化生成参数，不受提示词中的 @ 引用筛选影响。
const selectedNodeIds = new Set(selectedInputs.map((input) => input.nodeId)); const selectedNodeIds = new Set(selectedInputs.map((input) => input.nodeId));
videoFrameNodeIds.forEach((nodeId) => { videoFrameNodeIds.forEach((nodeId) => {
@@ -286,7 +286,7 @@ export function buildNodeResponseMessages(context: NodeGenerationContext): AiTex
]; ];
} }
export async function hydrateNodeGenerationContext(context: NodeGenerationContext, projectId: string, domainProjectId?: string, mode?: CanvasGenerationMode, includeCharacterVoiceSamples = false) { export async function hydrateNodeGenerationContext(context: NodeGenerationContext, projectId: string, domainProjectId?: string, mode?: CanvasGenerationMode, includeCharacterVoiceSamples = false, includeCharacterPrompt = true) {
const { imageToDataUrl } = await import("@/services/image-storage"); const { imageToDataUrl } = await import("@/services/image-storage");
let referenceImages = await Promise.all( let referenceImages = await Promise.all(
context.referenceImages.map(async (image) => { context.referenceImages.map(async (image) => {
@@ -365,7 +365,7 @@ export async function hydrateNodeGenerationContext(context: NodeGenerationContex
const voiceBlocks = mode === "video" ? resolvedCharacterVoices.map(compileResolvedVoicePrompt) : []; const voiceBlocks = mode === "video" ? resolvedCharacterVoices.map(compileResolvedVoicePrompt) : [];
return { return {
...context, ...context,
prompt: [context.prompt.trim(), ...characterBlocks, ...voiceBlocks].filter(Boolean).join("\n\n"), prompt: includeCharacterPrompt ? [context.prompt.trim(), ...characterBlocks, ...voiceBlocks].filter(Boolean).join("\n\n") : context.prompt,
referenceImages, referenceImages,
referenceAudios, referenceAudios,
resolvedCharacterVersions, resolvedCharacterVersions,

9
web/src/pages/canvas/use-canvas-generation-executor.ts
View File

@@ -124,13 +124,16 @@ export function useCanvasGenerationExecutor({
} }
let rawGenerationContext: Awaited<ReturnType<typeof hydrateNodeGenerationContext>>; let rawGenerationContext: Awaited<ReturnType<typeof hydrateNodeGenerationContext>>;
// 视频文本只保留输入框内容；连接的媒体仍作为结构化参考传递。
const promptOnly = mode === "video";
try { try {
rawGenerationContext = await hydrateNodeGenerationContext( rawGenerationContext = await hydrateNodeGenerationContext(
buildNodeGenerationContext(nodeId, nodesRef.current, connectionsRef.current, editingTextNode ? `请根据要求修改以下文本。\n\n原文：\n${sourceTextContent}\n\n修改要求：\n${prompt}` : generationPrompt), buildNodeGenerationContext(nodeId, nodesRef.current, connectionsRef.current, editingTextNode ? `请根据要求修改以下文本。\n\n原文：\n${sourceTextContent}\n\n修改要求：\n${prompt}` : generationPrompt, promptOnly),
projectId, projectId,
domainProjectId, domainProjectId,
mode, mode,
mode === "video" && supportsVideoReferenceAudio(generationConfig), mode === "video" && supportsVideoReferenceAudio(generationConfig),
!promptOnly,
); );
} catch (error) { } catch (error) {
const errorDetails = generationErrorMessage(error); const errorDetails = generationErrorMessage(error);
@@ -143,10 +146,10 @@ export function useCanvasGenerationExecutor({
return; return;
} }
const expandedPrompt = expandSkillMentions(rawGenerationContext.prompt, addedSkills); const expandedPrompt = promptOnly ? rawGenerationContext.prompt : expandSkillMentions(rawGenerationContext.prompt, addedSkills);
let effectivePrompt = expandedPrompt.trim(); let effectivePrompt = expandedPrompt.trim();
let styleMetadata = {}; let styleMetadata = {};
if (mode === "image" || mode === "video") { if (mode === "image") {
try { try {
const styleRuntime = resolveCanvasStyleExecution(nodesRef.current, sourceNode, effectivePrompt, generationConfig, mode); const styleRuntime = resolveCanvasStyleExecution(nodesRef.current, sourceNode, effectivePrompt, generationConfig, mode);
if (styleRuntime) { if (styleRuntime) {

11
web/src/pages/canvas/use-canvas-generation-retry.ts
View File

@@ -86,15 +86,16 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, addedSkil
} }
let rawContext: Awaited<ReturnType<typeof hydrateNodeGenerationContext>> | null; let rawContext: Awaited<ReturnType<typeof hydrateNodeGenerationContext>> | null;
try { try {
const baseContext = buildNodeGenerationContext(sourceNode.id, nodesRef.current, connectionsRef.current, retryContextPrompt); const promptOnly = retryMode === "video";
rawContext = hasSavedImageMetadata && !baseContext.characterReferences.length ? null : await hydrateNodeGenerationContext(baseContext, projectId, domainProjectId, retryMode, retryMode === "video" && supportsVideoReferenceAudio(generationConfig)); const baseContext = buildNodeGenerationContext(sourceNode.id, nodesRef.current, connectionsRef.current, retryContextPrompt, promptOnly);
rawContext = hasSavedImageMetadata && !baseContext.characterReferences.length ? null : await hydrateNodeGenerationContext(baseContext, projectId, domainProjectId, retryMode, retryMode === "video" && supportsVideoReferenceAudio(generationConfig), !promptOnly);
} catch (error) { } catch (error) {
const failure = generationFailureMetadata(error, retryPromptSource); const failure = generationFailureMetadata(error, retryPromptSource);
message.error(failure.errorDetails); message.error(failure.errorDetails);
setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, status: NODE_STATUS_ERROR, ...failure } } : item))); setNodes((current) => current.map((item) => (item.id === node.id ? { ...item, metadata: { ...item.metadata, status: NODE_STATUS_ERROR, ...failure } } : item)));
return; return;
} }
const context = rawContext ? { ...rawContext, prompt: expandSkillMentions(rawContext.prompt, addedSkills) } : null; const context = rawContext ? { ...rawContext, prompt: retryMode === "video" ? rawContext.prompt : expandSkillMentions(rawContext.prompt, addedSkills) } : null;
const prompt = (context?.characterReferences.length ? context.prompt : savedImageMetadata?.prompt || context?.prompt || "").trim(); const prompt = (context?.characterReferences.length ? context.prompt : savedImageMetadata?.prompt || context?.prompt || "").trim();
if (!prompt) { if (!prompt) {
message.warning("找不到提示词，无法重试"); message.warning("找不到提示词，无法重试");
@@ -102,9 +103,9 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, addedSkil
} }
let mediaPrompt = prompt; let mediaPrompt = prompt;
let styleMetadata = {}; let styleMetadata = {};
if (node.type === CanvasNodeType.Image || node.type === CanvasNodeType.Video) { if (node.type === CanvasNodeType.Image) {
try { try {
const runtime = resolveCanvasStyleExecution(nodesRef.current, sourceNode, prompt, generationConfig, node.type === CanvasNodeType.Image ? "image" : "video"); const runtime = resolveCanvasStyleExecution(nodesRef.current, sourceNode, prompt, generationConfig, "image");
if (runtime) { if (runtime) {
mediaPrompt = runtime.prompt; mediaPrompt = runtime.prompt;
styleMetadata = { styleProfileJson: runtime.profileJson, styleExecutionPlan: runtime.plan }; styleMetadata = { styleProfileJson: runtime.profileJson, styleExecutionPlan: runtime.plan };