tt/ttstd-infinite-canvas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(*): 生成任务 - 修复任务可见性、分镜重试与角色视频引用

Browse Source
This commit is contained in:
ddCat
2026-07-27 15:29:36 +08:00
parent dd4af80b98
commit f5a9a0f0e1
13 changed files with 204 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/internal/service/provider_test.go
View File

@@ -276,7 +276,7 @@ func TestRunVideoTaskUsesNewAPIForAnyVideoModel(t *testing.T) {
}
}
func TestRunVideoTaskUsesNestedResultURLBeforeContentEndpoint(t *testing.T) {
func TestRunVideoTaskUsesNestedURLBeforeResultURL(t *testing.T) {
t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true")
paths := make([]string, 0, 3)
var server *httptest.Server
@@ -288,7 +288,7 @@ func TestRunVideoTaskUsesNestedResultURLBeforeContentEndpoint(t *testing.T) {
_, _ = w.Write([]byte(`{"data":{"task_id":"video-1","status":"queued"}}`))
case "GET /v1/videos/video-1":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"code":"success","data":{"task_id":"video-1","status":"SUCCESS","result_url":"` + server.URL + `/v1/videos/video-1/content","data":{"status":"completed","video_url":"` + server.URL + `/files/video.mp4"}}}`))
_, _ = w.Write([]byte(`{"code":"success","data":{"task_id":"video-1","status":"SUCCESS","result_url":"` + server.URL + `/v1/videos/video-1/content","data":{"status":"completed","url":"` + server.URL + `/files/video.mp4"}}}`))
case "GET /files/video.mp4":
if authorization := r.Header.Get("Authorization"); authorization != "" {
t.Errorf("file Authorization = %q, want empty", authorization)

128
web/src/components/canvas/canvas-node-generation.ts
View File

@@ -3,7 +3,7 @@ import { imageReferenceLabel } from "@/lib/image-reference-prompt";
import { seedanceReferenceLabel } from "@/lib/seedance-video";
import type { ReferenceImage } from "@/types/image";
import type { ReferenceAudio, ReferenceVideo } from "@/types/media";
import { CanvasNodeType, type CanvasConnection, type CanvasNodeData } from "@/types/canvas";
import { CanvasNodeType, type CanvasConnection, type CanvasGenerationMode, type CanvasNodeData } from "@/types/canvas";
import { getGenerationResourceNodes } from "@/lib/canvas/canvas-resource-references";
import { resolveCanvasDrawingReference } from "@/lib/canvas/canvas-drawing-reference";
import { compileCharacterReferencePrompt } from "@/lib/canvas/canvas-character-reference";
@@ -20,6 +20,11 @@ export type ResolvedCharacterVoice = {
versionId: string;
characterName: string;
voiceKey: string;
sampleResourceId?: string;
language?: string;
voiceAge?: string;
timbre?: string;
deliveryInstructions?: string;
instructions: string;
};
@@ -54,7 +59,8 @@ export function buildNodeGenerationContext(nodeId: string, nodes: CanvasNodeData
const inputs = buildNodeGenerationInputs(nodeId, nodes, connections);
const sourceNode = nodes.find((node) => node.id === nodeId);
const storyboardInputs = getConnectedStoryboardRows(nodeId, nodes, connections);
if (sourceNode?.type === CanvasNodeType.Config && Boolean(sourceNode.metadata?.composerContent?.trim())) {
const hasExplicitNodeMention = /@\[node:[^\]]+\]/.test(normalizeLegacyNodeMentions(prompt, inputs));
if ((sourceNode?.type === CanvasNodeType.Config && Boolean(sourceNode.metadata?.composerContent?.trim())) || hasExplicitNodeMention) {
return buildComposerGenerationContext(inputs, prompt);
}
@@ -104,6 +110,7 @@ function removeTrailingInputBlocks(prompt: string, inputs: NodeGenerationInput[]
}
function buildComposerGenerationContext(inputs: NodeGenerationInput[], prompt: string): NodeGenerationContext {
const normalizedPrompt = normalizeLegacyNodeMentions(prompt, inputs);
const inputByNodeId = new Map(inputs.map((input) => [input.nodeId, input]));
const selectedInputs: NodeGenerationInput[] = [];
const labelByNodeId = new Map<string, string>();
@@ -113,10 +120,10 @@ function buildComposerGenerationContext(inputs: NodeGenerationInput[], prompt: s
let lastIndex = 0;
let nextPrompt = "";
for (const match of prompt.matchAll(/@\[node:([^\]]+)\]/g)) {
for (const match of normalizedPrompt.matchAll(/@\[node:([^\]]+)\]/g)) {
if (match.index === undefined) continue;
hasToken = true;
nextPrompt += prompt.slice(lastIndex, match.index);
nextPrompt += normalizedPrompt.slice(lastIndex, match.index);
const input = inputByNodeId.get(match[1]);
if (input) {
let label = labelByNodeId.get(input.nodeId);
@@ -132,7 +139,7 @@ function buildComposerGenerationContext(inputs: NodeGenerationInput[], prompt: s
lastIndex = match.index + match[0].length;
}
nextPrompt += prompt.slice(lastIndex);
nextPrompt += normalizedPrompt.slice(lastIndex);
if (textBlocks.length) nextPrompt = `${nextPrompt.trim()}\n\n${textBlocks.join("\n\n")}`;
const referenceImages = selectedInputs.map((input) => input.image).filter((image): image is ReferenceImage => Boolean(image));
const referenceVideos = selectedInputs.map((input) => input.video).filter((video): video is ReferenceVideo => Boolean(video));
@@ -170,6 +177,39 @@ function buildComposerGenerationContext(inputs: NodeGenerationInput[], prompt: s
};
}
// 旧画布保存的是 @角色1 等显示标签；生成时升级为稳定节点 Token，避免标题或排序变化后引用错位。
function normalizeLegacyNodeMentions(prompt: string, inputs: NodeGenerationInput[]) {
const counts = { image: 0, drawing: 0, video: 0, audio: 0, text: 0, character: 0 };
const labels = inputs.map((input) => {
const kind = input.sourceKind === "drawing" ? "drawing" : input.type;
return { label: generationLabel(kind, counts[kind]++), nodeId: input.nodeId };
}).sort((a, b) => b.label.length - a.label.length);
let next = prompt;
labels.forEach(({ label, nodeId }) => {
const token = `@${label}`;
let cursor = 0;
let result = "";
while (cursor < next.length) {
const found = next.indexOf(token, cursor);
if (found < 0) {
result += next.slice(cursor);
break;
}
const end = found + token.length;
result += next.slice(cursor, found);
result += hasMentionBoundary(next, end) ? `@[node:${nodeId}]` : token;
cursor = end;
}
next = result;
});
return next;
}
function hasMentionBoundary(value: string, index: number) {
const char = value[index];
return !char || /\s|[,.!?;:，。！？；：、)\]}】）]/.test(char);
}
export function buildNodeGenerationInputs(nodeId: string, nodes: CanvasNodeData[], connections: CanvasConnection[]): NodeGenerationInput[] {
const resourceNodes = getGenerationResourceNodes(nodeId, nodes, connections);
return resourceNodes.flatMap((node): NodeGenerationInput[] => {
@@ -238,7 +278,7 @@ export function buildNodeResponseMessages(context: NodeGenerationContext): AiTex
];
}
export async function hydrateNodeGenerationContext(context: NodeGenerationContext, projectId: string, domainProjectId?: string) {
export async function hydrateNodeGenerationContext(context: NodeGenerationContext, projectId: string, domainProjectId?: string, mode?: CanvasGenerationMode, includeCharacterVoiceSamples = false) {
const { imageToDataUrl } = await import("@/services/image-storage");
let referenceImages = await Promise.all(
context.referenceImages.map(async (image) => {
@@ -249,7 +289,7 @@ export async function hydrateNodeGenerationContext(context: NodeGenerationContex
if (!context.characterReferences.length) return { ...context, referenceImages };
if (!domainProjectId) throw new Error("角色引用未关联短剧项目，无法解析角色版本");
const { getProjectCharacter } = await import("@/services/api/projects");
const { resourceStorageKey } = await import("@/services/api/resources");
const { resourceFileUrl, resourceIdFromStorageKey, resourceStorageKey } = await import("@/services/api/resources");
const details = await Promise.all(context.characterReferences.map((reference) => getProjectCharacter(domainProjectId, reference.assetId)));
const remainingBudget = 9 - referenceImages.length;
const selected = details.flatMap((detail) => {
@@ -274,8 +314,56 @@ export async function hydrateNodeGenerationContext(context: NodeGenerationContex
referenceImages = [...referenceImages, ...hydratedCharacterImages];
const characterBlocks = details.map((detail) => compileCharacterReferencePrompt(detail.asset.title, detail.character.definition));
const resolvedCharacterVersions = details.map((detail) => ({ assetId: detail.asset.id, versionId: detail.character.versionId }));
const resolvedCharacterVoices = details.flatMap((detail): ResolvedCharacterVoice[] => detail.character.voice ? [{ assetId: detail.asset.id, versionId: detail.character.versionId, characterName: detail.asset.title, voiceKey: detail.character.voice.profile.voiceKey, instructions: compileResolvedVoiceInstructions(detail.character.definition, detail.character.voice.instructions) }] : []);
return { ...context, prompt: [context.prompt.trim(), ...characterBlocks].filter(Boolean).join("\n\n"), referenceImages, resolvedCharacterVersions, resolvedCharacterVoices };
const resolvedCharacterVoices = details.flatMap((detail): ResolvedCharacterVoice[] => {
const voice = detail.character.voice;
if (!voice) return [];
const language = stringField(detail.character.definition.voiceLanguage) || stringField(voice.profile.language);
const voiceAge = stringField(detail.character.definition.voiceAge);
const timbre = stringField(detail.character.definition.voiceTimbre) || stringField(voice.profile.timbre);
const deliveryInstructions = stringField(voice.instructions);
const sampleResourceId = stringField(voice.profile.sampleResourceId);
return [{
assetId: detail.asset.id,
versionId: detail.character.versionId,
characterName: detail.asset.title,
voiceKey: stringField(voice.profile.voiceKey),
sampleResourceId: sampleResourceId || undefined,
language: language || undefined,
voiceAge: voiceAge || undefined,
timbre: timbre || undefined,
deliveryInstructions: deliveryInstructions || undefined,
instructions: [language && `语言与口音：${language}`, voiceAge && `声音年龄感：${voiceAge}`, timbre && `音色气质：${timbre}`, deliveryInstructions].filter(Boolean).join("；"),
}];
});
const usedAudioResourceIds = new Set(context.referenceAudios.map((audio) => resourceIdFromStorageKey(audio.storageKey)).filter(Boolean));
const voiceSamples: ResolvedCharacterVoice[] = [];
// 视频模型接收声音样本；独立配音任务仍通过 voiceKey 选音色，不能把两种协议混用。
if (mode === "video" && includeCharacterVoiceSamples) {
resolvedCharacterVoices.forEach((voice) => {
if (!voice.sampleResourceId || usedAudioResourceIds.has(voice.sampleResourceId)) return;
usedAudioResourceIds.add(voice.sampleResourceId);
voiceSamples.push(voice);
});
}
if (context.referenceAudios.length + voiceSamples.length > 3) throw new Error(`当前模型参考音频容量不足：已连接 ${context.referenceAudios.length} 个音频，角色声音样本还需要 ${voiceSamples.length} 个名额`);
const characterVoiceAudios = voiceSamples.map((voice) => ({
id: `character-voice-${voice.assetId}`,
name: `${voice.characterName}-声音样本.mp3`,
type: "audio/mpeg",
url: resourceFileUrl(voice.sampleResourceId!),
storageKey: resourceStorageKey(voice.sampleResourceId!),
} satisfies ReferenceAudio));
const referenceAudios = [...context.referenceAudios, ...characterVoiceAudios];
const voiceBlocks = mode === "video" ? resolvedCharacterVoices.map(compileResolvedVoicePrompt) : [];
return {
...context,
prompt: [context.prompt.trim(), ...characterBlocks, ...voiceBlocks].filter(Boolean).join("\n\n"),
referenceImages,
referenceAudios,
resolvedCharacterVersions,
resolvedCharacterVoices,
audioCount: referenceAudios.length,
};
}
function readNodeTextInput(node: CanvasNodeData) {
@@ -293,16 +381,18 @@ function preferredCharacterRepresentation(representations: Array<{ id: string; r
return ["turnaround_sheet", "primary", "front", "side", "back"].map((role) => representations.find((item) => item.role === role)).find(Boolean);
}
function compileResolvedVoiceInstructions(definition: Record<string, unknown>, bindingInstructions: string) {
const fields: Array<[string, unknown]> = [
["语言与口音", definition.voiceLanguage],
["声音年龄感", definition.voiceAge],
["音色气质", definition.voiceTimbre],
];
const characterInstructions = fields
.map(([label, value]) => typeof value === "string" && value.trim() ? `${label}：${value.trim()}` : "")
.filter(Boolean);
return [...characterInstructions, bindingInstructions.trim()].filter(Boolean).join("；");
function compileResolvedVoicePrompt(voice: ResolvedCharacterVoice) {
return [
`【角色声音：${voice.characterName}】`,
voice.language && `语言与口音：${voice.language}`,
voice.voiceAge && `声音年龄感：${voice.voiceAge}`,
voice.timbre && `音色气质：${voice.timbre}`,
voice.deliveryInstructions && `表演与朗读要求：${voice.deliveryInstructions}`,
].filter(Boolean).join("\n");
}
function stringField(value: unknown) {
return typeof value === "string" ? value.trim() : "";
}
function readSkillInput(node: CanvasNodeData) {

9
web/src/components/canvas/canvas-node-prompt-panel.tsx
View File

@@ -1,5 +1,5 @@
import { useCallback, useEffect, useMemo, useState, type ReactNode } from "react";
import { ArrowUp, AtSign, Boxes, FileText, ImageIcon, ImagePlus, Maximize2, Music2, Pencil, Square, Video } from "lucide-react";
import { ArrowUp, AtSign, Boxes, FileText, ImageIcon, ImagePlus, Maximize2, Music2, Pencil, Square, UserRound, Video } from "lucide-react";
import { Button, Modal, Tooltip } from "antd";
import { ModelPicker } from "@/components/model-picker";
@@ -16,7 +16,7 @@ import { CanvasVideoSettingsPopover } from "./canvas-video-settings-popover";
import { CanvasVideoPromptTools } from "./canvas-video-prompt-tools";
import { CanvasPresetPicker, type CanvasPromptPreset } from "./canvas-preset-picker";
import { CanvasNodeType, type CanvasGenerationMode, type CanvasNodeData, type CanvasNodeMetadata, type CanvasWorkspaceMode } from "@/types/canvas";
import type { CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
import { canvasResourceMentionToken, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
export type CanvasNodeGenerationMode = CanvasGenerationMode;
@@ -94,7 +94,7 @@ export function CanvasNodePromptPanel({ node, isRunning, onPromptChange, onConfi
};
const insertPromptReference = (reference: CanvasResourceReference) => {
const insertText = `@${reference.label} `;
const insertText = `${canvasResourceMentionToken(reference)} `;
const pendingMentionMatch = /@[^\s@，。！？、,.!?;:]*\s*$/.exec(prompt);
if (pendingMentionMatch) {
const prefix = prompt.slice(0, pendingMentionMatch.index).replace(/\s*$/, "");
@@ -342,8 +342,9 @@ function ConnectedReferenceShelf({ references, theme, onInsert }: { references:
function ReferenceThumbnail({ reference }: { reference: CanvasResourceReference }) {
if (reference.kind === "image" && reference.previewUrl) return <img src={reference.previewUrl} alt="" className="size-full object-cover" />;
if (reference.kind === "video" && reference.previewUrl) return <video src={reference.previewUrl} className="size-full bg-black object-cover" muted preload="metadata" />;
if (reference.kind === "character" && reference.previewUrl) return <img src={reference.previewUrl} alt="" className="size-full bg-black/5 object-contain" />;
const Icon = reference.sourceType === CanvasNodeType.Drawing ? Pencil : reference.kind === "audio" ? Music2 : reference.kind === "video" ? Video : reference.kind === "image" ? ImageIcon : FileText;
const Icon = reference.sourceType === CanvasNodeType.Drawing ? Pencil : reference.kind === "character" ? UserRound : reference.kind === "audio" ? Music2 : reference.kind === "video" ? Video : reference.kind === "image" ? ImageIcon : FileText;
return (
<span className="grid size-full place-items-center bg-black/10 text-current dark:bg-white/10">
<Icon className="size-3.5 opacity-75" />

23
web/src/components/canvas/canvas-resource-mention-textarea.tsx
View File

@@ -5,7 +5,7 @@ import { FileText, Image as ImageIcon, Music2, Pencil, Sparkles, UserRound, Vide
import { canvasThemes } from "@/lib/canvas-theme";
import { useThemeStore } from "@/stores/use-theme-store";
import type { CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
import { canvasResourceMentionToken, type CanvasResourceReference } from "@/lib/canvas/canvas-resource-references";
import { CanvasNodeType } from "@/types/canvas";
type MentionState = {
@@ -142,7 +142,7 @@ export const CanvasResourceMentionTextarea = forwardRef<HTMLTextAreaElement, Pro
if (!mention) return;
const selection = useRichEditor ? getEditableSelection(editorRef.current) : null;
const end = selection?.end ?? textareaRef.current?.selectionStart ?? value.length;
const insertText = `@${reference.label} `;
const insertText = `${canvasResourceMentionToken(reference)} `;
const next = `${value.slice(0, mention.start)}${insertText}${value.slice(end)}`;
closeMention();
updateValue(next, mention.start + insertText.length);
@@ -371,9 +371,9 @@ function createInlineMentionChip(reference: CanvasResourceReference, token: stri
}
function createInlinePreview(reference: CanvasResourceReference) {
if ((reference.kind === "image" || reference.kind === "video") && reference.previewUrl) {
const media = document.createElement(reference.kind === "image" ? "img" : "video");
media.className = `size-[1.18em] shrink-0 rounded-[0.24em] ${reference.kind === "video" ? "bg-black " : ""}object-cover`;
if ((reference.kind === "image" || reference.kind === "video" || reference.kind === "character") && reference.previewUrl) {
const media = document.createElement(reference.kind === "video" ? "video" : "img");
media.className = `size-[1.18em] shrink-0 rounded-[0.24em] ${reference.kind === "video" ? "bg-black object-cover" : reference.kind === "character" ? "bg-black/5 object-contain" : "object-cover"}`;
media.setAttribute("src", reference.previewUrl);
media.setAttribute("alt", "");
if (media instanceof HTMLVideoElement) {
@@ -384,7 +384,7 @@ function createInlinePreview(reference: CanvasResourceReference) {
}
const fallback = document.createElement("span");
fallback.className = "grid size-[1.18em] shrink-0 place-items-center rounded-[0.24em] bg-current/10";
fallback.textContent = reference.sourceType === CanvasNodeType.Drawing ? "✎" : reference.kind === "character" ? "角" : reference.kind === "audio" ? "♪" : reference.kind === "video" ? "▶" : reference.kind === "image" ? "□" : reference.kind === "skill" ? "✦" : "T";
fallback.textContent = reference.sourceType === CanvasNodeType.Drawing ? "✎" : reference.kind === "audio" ? "♪" : reference.kind === "video" ? "▶" : reference.kind === "image" ? "□" : reference.kind === "skill" ? "✦" : "";
return fallback;
}
@@ -449,6 +449,7 @@ function MentionMenu({ anchor, references, activeIndex, theme, onSelect }: { anc
function ReferencePreview({ reference }: { reference: CanvasResourceReference }) {
if (reference.kind === "image" && reference.previewUrl) return <img src={reference.previewUrl} alt="" className="size-9 rounded-md object-cover" />;
if (reference.kind === "video" && reference.previewUrl) return <video src={reference.previewUrl} className="size-9 rounded-md bg-black object-cover" muted preload="metadata" />;
if (reference.kind === "character" && reference.previewUrl) return <img src={reference.previewUrl} alt="" className="size-9 rounded-md bg-black/5 object-contain" />;
if (reference.kind === "skill") {
return (
<span className="grid size-9 shrink-0 place-items-center rounded-md bg-cyan-500/12 text-cyan-600 dark:text-cyan-200">
@@ -466,7 +467,12 @@ function ReferencePreview({ reference }: { reference: CanvasResourceReference })
function splitMentionText(value: string, references: CanvasResourceReference[]) {
if (!references.length || !value) return value ? [{ type: "text", text: value } as MentionTextPart] : [];
const referenceByToken = new Map(references.map((reference) => [`@${reference.label}`, reference]));
const referenceByToken = new Map<string, { reference: CanvasResourceReference; serializedToken: string }>();
references.forEach((reference) => {
const serializedToken = canvasResourceMentionToken(reference);
referenceByToken.set(serializedToken, { reference, serializedToken });
referenceByToken.set(`@${reference.label}`, { reference, serializedToken });
});
const tokens = [...referenceByToken.keys()].sort((a, b) => b.length - a.length);
const parts: MentionTextPart[] = [];
let index = 0;
@@ -479,7 +485,8 @@ function splitMentionText(value: string, references: CanvasResourceReference[])
index = end;
continue;
}
parts.push({ type: "mention", token, reference: referenceByToken.get(token)! });
const matched = referenceByToken.get(token)!;
parts.push({ type: "mention", token: matched.serializedToken, reference: matched.reference });
index += token.length;
}
return parts;

7
web/src/components/canvas/canvas-workspace-overlays.tsx
View File

@@ -1,6 +1,6 @@
import { motion, useReducedMotion } from "motion/react";
import { useLayoutEffect, useRef, useState, type CSSProperties, type ReactNode, type RefObject } from "react";
import { ChevronRight, Image as ImageIcon, List, Music2, Pencil, Settings2, Video, WandSparkles, X } from "lucide-react";
import { ChevronRight, Clapperboard, Image as ImageIcon, List, Music2, Pencil, Settings2, Video, WandSparkles, X } from "lucide-react";
import { SpotlightSurface } from "@/components/ui/aceternity/spotlight-surface";
import { canvasThemes } from "@/lib/canvas-theme";
@@ -123,12 +123,12 @@ export function CanvasNodePanelOverlay({ node, viewport, containerRef, panelWidt
);
}
export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, containerRef, canCreateDrawing, onCreate, onClose }: { pending: PendingConnectionCreate; viewport: ViewportTransform; viewportSize: { width: number; height: number }; containerRef: RefObject<HTMLDivElement | null>; canCreateDrawing: boolean; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing) => void; onClose: () => void }) {
export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, containerRef, canCreateDrawing, onCreate, onClose }: { pending: PendingConnectionCreate; viewport: ViewportTransform; viewportSize: { width: number; height: number }; containerRef: RefObject<HTMLDivElement | null>; canCreateDrawing: boolean; onCreate: (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Config | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing) => void; onClose: () => void }) {
const theme = canvasThemes[useThemeStore((state) => state.theme)];
const reducedMotion = useReducedMotion();
const menuRef = useRef<HTMLDivElement>(null);
const menuWidth = 248;
const menuHeight = canCreateDrawing ? 376 : 332;
const menuHeight = canCreateDrawing ? 420 : 376;
const gap = 12;
const initialPosition = getConnectionMenuPosition(pending.position, viewport, viewportSize, menuWidth, menuHeight, gap);
@@ -170,6 +170,7 @@ export function CanvasConnectionCreateMenu({ pending, viewport, viewportSize, co
</div>
<div className="grid gap-1">
<ConnectionCreateOption motionEnabled={!reducedMotion} icon={<List className="size-4" />} title="文本生成" onClick={() => onCreate(CanvasNodeType.Text)} />
<ConnectionCreateOption motionEnabled={!reducedMotion} icon={<Clapperboard className="size-4" />} title="分镜脚本" onClick={() => onCreate(CanvasNodeType.Script)} />
<ConnectionCreateOption motionEnabled={!reducedMotion} icon={<ImageIcon className="size-4" />} title="图片生成" onClick={() => onCreate(CanvasNodeType.Image)} />
{canCreateDrawing ? <ConnectionCreateOption motionEnabled={!reducedMotion} icon={<Pencil className="size-4" />} title="绘图" onClick={() => onCreate(CanvasNodeType.Drawing)} /> : null}
<ConnectionCreateOption motionEnabled={!reducedMotion} icon={<Video className="size-4" />} title="视频生成" onClick={() => onCreate(CanvasNodeType.Video)} />

7
web/src/lib/canvas/canvas-project-generation.ts
View File

@@ -5,6 +5,7 @@ import { getMediaBlob, resolveMediaUrl } from "@/services/file-storage";
import { resourceIdFromStorageKey, resourceStorageKey, uploadResourceFile } from "@/services/api/resources";
import { NODE_DEFAULT_SIZE } from "@/constant/canvas";
import { normalizeVideoDuration, normalizeVideoResolution } from "@/lib/video-generation-options";
import { isSeedanceVideoConfig } from "@/lib/seedance-video";
import { imageMetadata, parseBackendGenerationResult } from "@/lib/canvas/canvas-generation-task-sync";
import type { CanvasNodeGenerationMode } from "@/components/canvas/canvas-node-prompt-panel";
import { CanvasNodeType, type CanvasAssistantSession, type CanvasConnection, type CanvasImageGenerationType, type CanvasNodeData, type CanvasNodeMetadata, type CanvasVideoEditOperation } from "@/types/canvas";
@@ -353,6 +354,12 @@ export function buildGenerationConfig(config: AiConfig, node: CanvasNodeData | u
};
}
export function supportsVideoReferenceAudio(config: AiConfig) {
const interfaceType = resolveModelRequestConfig(config, config.model).interfaceType;
if (interfaceType === "newapi-channel-2") return false;
return interfaceType === "newapi-channel-1" || isSeedanceVideoConfig(config);
}
export function resetInterruptedGeneration(nodes: CanvasNodeData[]) {
const configHeight = NODE_DEFAULT_SIZE[CanvasNodeType.Config].height;
return nodes.map((node) => {

5
web/src/lib/canvas/canvas-resource-references.ts
View File

@@ -19,6 +19,11 @@ export type CanvasResourceReference = {
skill?: UpdreamSkill;
};
export function canvasResourceMentionToken(reference: CanvasResourceReference) {
if (reference.kind === "skill" && reference.skill?.dir) return `@[skill:${reference.skill.dir}]`;
return `@[node:${reference.nodeId}]`;
}
export function buildCanvasResourceReferences(nodes: CanvasNodeData[], connections: CanvasConnection[], contextNodeId?: string | null) {
const contextNodes = contextNodeId ? getMentionResourceNodes(contextNodeId, nodes, connections) : [];
const globalReferences = labelResourceNodes(nodes.filter(isResourceNode), false);

2
web/src/pages/canvas/canvas-media-generation-executors.ts
View File

@@ -79,7 +79,7 @@ export async function executeVideoGeneration({
startGenerationRequest(videoId, nodeId, nodeId, controller);
try {
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: videoId, mode: "video", prompt: effectivePrompt, config: generationConfig, referenceImages: generationContext.referenceImages, referenceVideos: generationContext.referenceVideos, referenceAudios: generationContext.referenceAudios, signal: controller.signal, metadata: { sourceNodeId: nodeId, resolvedCharacterVersions: generationContext.resolvedCharacterVersions, ...videoGenerationMetadata }, onTaskCreated: (task) => bindGenerationTask(videoId, task) });
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: videoId, mode: "video", prompt: effectivePrompt, config: generationConfig, referenceImages: generationContext.referenceImages, referenceVideos: generationContext.referenceVideos, referenceAudios: generationContext.referenceAudios, signal: controller.signal, metadata: { sourceNodeId: nodeId, resolvedCharacterVersions: generationContext.resolvedCharacterVersions, resolvedCharacterVoices: generationContext.resolvedCharacterVoices, ...videoGenerationMetadata }, onTaskCreated: (task) => bindGenerationTask(videoId, task) });
if (!result.video?.dataUrl) throw new Error("后端任务没有返回视频");
const video = await storeGeneratedVideo({ url: result.video.dataUrl, mimeType: result.video.mimeType || "video/mp4" });
const videoSize = fitNodeSize(video.width || spec.width, video.height || spec.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);

13
web/src/pages/canvas/project.tsx
View File

@@ -1291,7 +1291,18 @@ function InfiniteCanvasPage() {
setHoveredNodeId((current) => (current === nodeId ? null : current));
hideNodeToolbar();
}, [hideNodeToolbar]);
const retryCanvasNode = useCallback((node: CanvasNodeData) => { void handleRetryNode(node); }, [handleRetryNode]);
const retryCanvasNode = useCallback((node: CanvasNodeData) => {
if (node.type === CanvasNodeType.Script) {
const prompt = (node.metadata?.composerContent || node.metadata?.prompt || "").trim();
if (!prompt) {
message.warning("分镜脚本缺少剧情内容，无法重试");
return;
}
void generateScriptRows(node.id, prompt);
return;
}
void handleRetryNode(node);
}, [generateScriptRows, handleRetryNode, message]);
const openCanvasNodeTaskDetails = useCallback((node: CanvasNodeData) => { void openNodeTaskDetails(node); }, [openNodeTaskDetails]);
const openCanvasNodeVersions = useCallback((node: CanvasNodeData) => setVersionCompareRootId(node.metadata?.versionOfNodeId || node.id), []);
const viewCanvasNodeImage = useCallback((node: CanvasNodeData) => setPreviewNodeId(node.id), []);

22
web/src/pages/canvas/use-canvas-connection-controller.ts
View File

@@ -103,11 +103,15 @@ export function useCanvasConnectionController({
setContextMenu(null);
}, [connectionsRef, message, nodesRef, setConnections, setContextMenu, setNodes]);
const createConnectedNode = useCallback(async (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Config | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing, pending: PendingConnectionCreate) => {
const createConnectedNode = useCallback(async (type: CanvasNodeType.Image | CanvasNodeType.Text | CanvasNodeType.Script | CanvasNodeType.Config | CanvasNodeType.Video | CanvasNodeType.Audio | CanvasNodeType.Drawing, pending: PendingConnectionCreate) => {
const storyboardRow = type === CanvasNodeType.Video ? storyboardRowFromHandle(nodesRef.current, pending.connection.nodeId, pending.connection.handleId) : undefined;
const videoPrompt = storyboardRow ? (storyboardRow.videoMotionPrompt || storyboardRow.plotDescription).trim() : "";
const sourceNode = pending.connection.handleType === "source" ? nodesRef.current.find((node) => node.id === pending.connection.nodeId) : undefined;
const scriptPrompt = type === CanvasNodeType.Script && sourceNode?.type === CanvasNodeType.Text ? (sourceNode.metadata?.content || sourceNode.metadata?.prompt || "").trim() : "";
const metadata = type === CanvasNodeType.Config
? { model: effectiveConfig.imageModel || effectiveConfig.model, size: effectiveConfig.size, count: getGenerationCount(effectiveConfig.canvasImageCount || effectiveConfig.count) }
: type === CanvasNodeType.Script && scriptPrompt
? { prompt: scriptPrompt, composerContent: scriptPrompt }
: type === CanvasNodeType.Video && storyboardRow
? { prompt: videoPrompt, composerContent: videoPrompt, generationMode: "video" as const, videoEditOperation: "text_to_video" as const, workflowKind: "shot" as const, workflowTitle: `镜头 ${storyboardRow.shotNumber} 视频`, shotIndex: storyboardRow.shotNumber, seconds: String(storyboardRow.durationSeconds), status: NODE_STATUS_IDLE }
: undefined;
@@ -119,9 +123,9 @@ export function useCanvasConnectionController({
return;
}
if (type === CanvasNodeType.Drawing) {
const sourceNode = nodesRef.current.find((node) => node.id === pending.connection.nodeId);
const sourceUrl = sourceNode?.type === CanvasNodeType.Image ? sourceNode.metadata?.content : "";
if (pending.connection.handleType !== "source" || !sourceNode || !sourceUrl || !newNode.metadata?.drawingId) {
const drawingSourceNode = nodesRef.current.find((node) => node.id === pending.connection.nodeId);
const sourceUrl = drawingSourceNode?.type === CanvasNodeType.Image ? drawingSourceNode.metadata?.content : "";
if (pending.connection.handleType !== "source" || !drawingSourceNode || !sourceUrl || !newNode.metadata?.drawingId) {
message.error("只有已有图片内容的输出连线可以创建绘图");
return;
}
@@ -130,11 +134,11 @@ export function useCanvasConnectionController({
try {
const saved = await createCanvasDrawingFromImage(projectId, newNode.metadata.drawingId, {
url: sourceUrl,
storageKey: sourceNode.metadata?.storageKey,
name: sourceNode.title || "来源图片",
mimeType: sourceNode.metadata?.mimeType,
storageKey: drawingSourceNode.metadata?.storageKey,
name: drawingSourceNode.title || "来源图片",
mimeType: drawingSourceNode.metadata?.mimeType,
});
newNode.title = `${sourceNode.title || "图片"} · 绘图`;
newNode.title = `${drawingSourceNode.title || "图片"} · 绘图`;
newNode.metadata = {
...newNode.metadata,
drawingRevision: saved.revision,
@@ -155,7 +159,7 @@ export function useCanvasConnectionController({
setSelectedNodeIds(new Set([newNode.id]));
setSelectedConnectionId(null);
if (type === CanvasNodeType.Drawing) setDrawingNodeId(newNode.id);
else if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Audio) setDialogNodeId(newNode.id);
else if (type !== CanvasNodeType.Text && type !== CanvasNodeType.Script && type !== CanvasNodeType.Audio) setDialogNodeId(newNode.id);
closeConnectionCreateMenu();
setConnecting(null);
}, [closeConnectionCreateMenu, effectiveConfig.canvasImageCount, effectiveConfig.count, effectiveConfig.imageModel, effectiveConfig.model, effectiveConfig.size, message, nodesRef, projectId, setConnecting, setConnections, setDialogNodeId, setDrawingNodeId, setNodes, setSelectedConnectionId, setSelectedNodeIds]);

4
web/src/pages/canvas/use-canvas-generation-executor.ts
View File

@@ -3,7 +3,7 @@ import { App } from "antd";
import { buildNodeGenerationContext, hydrateNodeGenerationContext } from "@/components/canvas/canvas-node-generation";
import type { CanvasNodeGenerationMode } from "@/components/canvas/canvas-node-prompt-panel";
import { buildGenerationConfig, isGenerationCanceled } from "@/lib/canvas/canvas-project-generation";
import { buildGenerationConfig, isGenerationCanceled, supportsVideoReferenceAudio } from "@/lib/canvas/canvas-project-generation";
import { isGenerationTaskCapacityError } from "@/lib/canvas/canvas-generation-batch";
import { expandSkillMentions } from "@/lib/canvas/canvas-skill-mentions";
import { generationFailureMetadata } from "@/lib/generation-error";
@@ -110,6 +110,8 @@ export function useCanvasGenerationExecutor({
buildNodeGenerationContext(nodeId, nodesRef.current, connectionsRef.current, editingTextNode ? `请根据要求修改以下文本。\n\n原文：\n${sourceTextContent}\n\n修改要求：\n${prompt}` : prompt),
projectId,
domainProjectId,
mode,
mode === "video" && supportsVideoReferenceAudio(generationConfig),
);
} catch (error) {
const errorDetails = error instanceof Error ? error.message : "生成任务准备失败";

20
web/src/pages/canvas/use-canvas-generation-retry.ts
View File

@@ -19,6 +19,7 @@ import {
resolveStoredReferenceImages,
runBackendCanvasGenerationTask,
sourceNodeReferenceImages,
supportsVideoReferenceAudio,
} from "@/lib/canvas/canvas-project-generation";
import { expandSkillMentions } from "@/lib/canvas/canvas-skill-mentions";
import { generationFailureMetadata, unchangedModeratedPrompt } from "@/lib/generation-error";
@@ -57,11 +58,15 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, activated
return useCallback(
async (node: CanvasNodeData) => {
const retryMode = retryModeForNode(node.type);
if (!retryMode) {
message.warning("当前节点不能使用通用生成重试");
return;
}
const sourceNode = findRetrySourceNode(node.id, nodesRef.current, connectionsRef.current) || node;
const batchRoot = node.metadata?.batchRootId ? nodesRef.current.find((item) => item.id === node.metadata?.batchRootId) : null;
const savedImageMetadata = node.type === CanvasNodeType.Image ? { ...batchRoot?.metadata, ...node.metadata } : undefined;
const hasSavedImageMetadata = Boolean(savedImageMetadata?.generationType);
const retryMode: CanvasNodeGenerationMode = node.type === CanvasNodeType.Text ? "text" : node.type === CanvasNodeType.Video ? "video" : node.type === CanvasNodeType.Audio ? "audio" : "image";
let generationConfig =
hasSavedImageMetadata && savedImageMetadata
? { ...effectiveConfig, model: savedImageMetadata.model || effectiveConfig.imageModel || effectiveConfig.model, quality: savedImageMetadata.quality || effectiveConfig.quality, size: savedImageMetadata.size || effectiveConfig.size, transparentBackground: (savedImageMetadata.transparentBackground || effectiveConfig.transparentBackground) === "true" ? "true" : "false", count: "1" }
@@ -79,7 +84,7 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, activated
let rawContext: Awaited<ReturnType<typeof hydrateNodeGenerationContext>> | null;
try {
const baseContext = buildNodeGenerationContext(sourceNode.id, nodesRef.current, connectionsRef.current, retryPromptSource);
rawContext = hasSavedImageMetadata && !baseContext.characterReferences.length ? null : await hydrateNodeGenerationContext(baseContext, projectId, domainProjectId);
rawContext = hasSavedImageMetadata && !baseContext.characterReferences.length ? null : await hydrateNodeGenerationContext(baseContext, projectId, domainProjectId, retryMode, retryMode === "video" && supportsVideoReferenceAudio(generationConfig));
} catch (error) {
const failure = generationFailureMetadata(error, retryPromptSource);
message.error(failure.errorDetails);
@@ -141,7 +146,7 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, activated
}
if (node.type === CanvasNodeType.Video) {
const videoGenerationMetadata = buildVideoGenerationMetadata(node, videoContext);
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "video", prompt, config: generationConfig, referenceImages: videoContext?.referenceImages || [], referenceVideos: videoContext?.referenceVideos || [], referenceAudios: videoContext?.referenceAudios || [], signal: controller.signal, metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context?.resolvedCharacterVersions || [], ...videoGenerationMetadata }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
const result = await runBackendCanvasGenerationTask({ projectId, nodeId: node.id, mode: "video", prompt, config: generationConfig, referenceImages: videoContext?.referenceImages || [], referenceVideos: videoContext?.referenceVideos || [], referenceAudios: videoContext?.referenceAudios || [], signal: controller.signal, metadata: { retry: true, sourceNodeId: sourceNode.id, resolvedCharacterVersions: context?.resolvedCharacterVersions || [], resolvedCharacterVoices: context?.resolvedCharacterVoices || [], ...videoGenerationMetadata }, onTaskCreated: (task) => bindGenerationTask(node.id, task) });
if (!result.video?.dataUrl) throw new Error("后端任务没有返回视频");
const video = await storeGeneratedVideo({ url: result.video.dataUrl, mimeType: result.video.mimeType || "video/mp4" });
const videoSize = fitNodeSize(video.width || node.width, video.height || node.height, VIDEO_NODE_MAX_WIDTH, VIDEO_NODE_MAX_HEIGHT);
@@ -204,6 +209,15 @@ export function useCanvasGenerationRetry({ projectId, domainProjectId, activated
);
}
// 生成类型必须由明确的节点契约决定，未知节点不能降级成图片任务。
function retryModeForNode(type: CanvasNodeType): CanvasNodeGenerationMode | null {
if (type === CanvasNodeType.Text) return "text";
if (type === CanvasNodeType.Image) return "image";
if (type === CanvasNodeType.Video) return "video";
if (type === CanvasNodeType.Audio) return "audio";
return null;
}
function markMissingReferences(nodeId: string, setNodes: Dispatch<SetStateAction<CanvasNodeData[]>>) {
setNodes((current) => current.map((item) => (item.id === nodeId ? { ...item, metadata: { ...item.metadata, status: NODE_STATUS_ERROR, errorDetails: "参考图片已丢失，无法继续重试" } } : item)));
}

13
web/src/pages/tasks/index.tsx
View File

@@ -17,10 +17,10 @@ import { formatCredits } from "@/constant/credits";
import { listProjects, type ProjectSummary } from "@/services/api/projects";
const taskTableClassName = "app-data-table";
type TaskStatusFilter = "failed" | "active" | "succeeded";
type TaskStatusFilter = "all" | "failed" | "active" | "succeeded";
function taskStatusFilter(value: string | null): TaskStatusFilter {
return value === "active" || value === "succeeded" ? value : "failed";
return value === "failed" || value === "active" || value === "succeeded" ? value : "all";
}
export default function TasksPage() {
@@ -61,6 +61,7 @@ export default function TasksPage() {
return { label: projectName ? `${project.title || "未命名画布"} · ${projectName}` : project.title || "未命名画布", value: project.id };
}), [domainProjectNameById, projects]);
const filteredTasks = useMemo(() => tasks.filter((task) => {
if (statusFilter === "all") return true;
if (statusFilter === "active") return task.status === "queued" || task.status === "running";
if (statusFilter === "failed") return task.status === "failed" || task.status === "cancelled";
if (statusFilter === "succeeded") return task.status === "succeeded";
@@ -178,6 +179,10 @@ export default function TasksPage() {
try {
const next = action === "retry" ? await retryGenerationTask(id) : await cancelGenerationTask(id);
setTasks((items) => items.map((item) => (item.id === id ? next : item)));
if (action === "retry") {
setStatusFilter("active");
setPage(1);
}
message.success(action === "retry" ? "任务已重新入队" : "任务已取消");
} catch (error) {
message.error(error instanceof Error ? error.message : "操作失败");
@@ -223,6 +228,8 @@ export default function TasksPage() {
});
setTasks((items) => [task, ...items]);
}
setStatusFilter("active");
setPage(1);
setCreateOpen(false);
form.resetFields();
message.success("任务已创建");
@@ -364,6 +371,7 @@ export default function TasksPage() {
value={statusFilter}
onChange={(value) => { setStatusFilter(value as typeof statusFilter); setPage(1); }}
options={[
{ label: `全部 ${tasks.length}`, value: "all" },
{ label: `需要处理 ${tasks.filter((task) => task.status === "failed" || task.status === "cancelled").length}`, value: "failed" },
{ label: `运行中 ${tasks.filter((task) => task.status === "queued" || task.status === "running").length}`, value: "active" },
{ label: `已完成 ${tasks.filter((task) => task.status === "succeeded").length}`, value: "succeeded" },
@@ -536,6 +544,7 @@ function taskAttentionReason(task: GenerationTask) {
}
function taskEmptyState(status: TaskStatusFilter) {
if (status === "all") return { title: "还没有任务", description: "新提交的生成会在这里显示状态和实时进度。" };
if (status === "active") return { title: "没有运行中的任务", description: "新提交的生成会在这里显示排队状态和实时进度。" };
if (status === "succeeded") return { title: "还没有已完成任务", description: "生成成功后，结果预览和积分消耗会保留在这里。" };
return { title: "目前没有需要处理的任务", description: "失败或取消的生成会出现在这里，并提供原因和可用操作。" };