From d3a029732f6aa6be07957237b331ebcd112034ac Mon Sep 17 00:00:00 2001 From: QAyong Date: Mon, 17 Aug 2026 19:46:50 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat(canvas):=20=E5=AF=BC=E6=BC=94=E5=8F=B0?= =?UTF-8?q?=20-=20=E5=AE=8C=E5=96=84=E9=AA=A8=E9=AA=BC=E6=8E=A7=E5=88=B6?= =?UTF-8?q?=E3=80=81=E5=AF=B9=E8=B1=A1=E7=AE=A1=E7=90=86=E4=B8=8E=E9=9D=A2?= =?UTF-8?q?=E6=9D=BF=E4=BA=A4=E4=BA=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../director/canvas-director-workbench.tsx | 132 ++++++++++++++++-- .../canvas/director/director-sequencer.tsx | 15 +- .../canvas/director/director-viewport.tsx | 93 ++++++++++-- web/src/lib/canvas/director/director-scene.ts | 18 ++- web/src/pages/canvas/project.tsx | 1 + .../canvas/use-director-workbench-store.ts | 5 +- web/src/types/director.ts | 6 +- 7 files changed, 240 insertions(+), 30 deletions(-) diff --git a/web/src/components/canvas/director/canvas-director-workbench.tsx b/web/src/components/canvas/director/canvas-director-workbench.tsx index 6c90d570c..4aa627374 100644 --- a/web/src/components/canvas/director/canvas-director-workbench.tsx +++ b/web/src/components/canvas/director/canvas-director-workbench.tsx @@ -1,7 +1,9 @@ -import { App, Button, ColorPicker, Input, InputNumber, Select, Switch } from "antd"; +import { App, Button, ColorPicker, Dropdown, Input, InputNumber, Select, Slider, Switch } from "antd"; +import type { MenuProps } from "antd"; import { Box, BoxSelect, Camera, Circle, Cuboid, FileUp, Focus, Image as ImageIcon, LampDesk, Lightbulb, Plus, Redo2, Save, Trash2, Undo2, UserRound, Video, X } from "lucide-react"; import { useCallback, useEffect, useMemo, useRef, useState, type ReactElement, type ReactNode } from "react"; import { nanoid } from "nanoid"; +import { Euler, Quaternion } from "three"; import type { AnimationClip } from "three"; import { DirectorViewport, type DirectorViewportHandle } from "@/components/canvas/director/director-viewport"; @@ -17,7 +19,7 @@ import { useThemeStore } from "@/stores/use-theme-store"; import type { CanvasNodeData } from "@/types/canvas"; import type { DirectorCamera, DirectorCameraMove, DirectorHumanoidBone, DirectorLight, DirectorObject, DirectorPose, DirectorQuat, DirectorRig, DirectorScene, DirectorSceneOutput, DirectorShot, DirectorShotSize, DirectorTransform, DirectorVec3 } from "@/types/director"; -export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onChange, onApply }: { open: boolean; scene: DirectorScene | null; imageNodes: CanvasNodeData[]; onClose: () => void; onChange: (scene: DirectorScene) => void; onApply: (output: DirectorSceneOutput) => Promise }) { +export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onChange, onApply, onDeleteImageNode }: { open: boolean; scene: DirectorScene | null; imageNodes: CanvasNodeData[]; onClose: () => void; onChange: (scene: DirectorScene) => void; onApply: (output: DirectorSceneOutput) => Promise; onDeleteImageNode: (nodeId: string) => void }) { const { message } = App.useApp(); const theme = canvasThemes[useThemeStore((state) => state.theme)]; const viewportRef = useRef(null); @@ -36,6 +38,7 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh const selectedBone = useDirectorWorkbenchStore((state) => state.selectedBone); const autoKey = useDirectorWorkbenchStore((state) => state.autoKey); const sequencerHeight = useDirectorWorkbenchStore((state) => state.sequencerHeight); + const sequencerVisible = useDirectorWorkbenchStore((state) => state.sequencerVisible); const setSelectedObjectId = useDirectorWorkbenchStore((state) => state.setSelectedObjectId); const setSelectedLightId = useDirectorWorkbenchStore((state) => state.setSelectedLightId); const setTransformMode = useDirectorWorkbenchStore((state) => state.setTransformMode); @@ -45,6 +48,7 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh const setSelectedBone = useDirectorWorkbenchStore((state) => state.setSelectedBone); const setAutoKey = useDirectorWorkbenchStore((state) => state.setAutoKey); const setSequencerHeight = useDirectorWorkbenchStore((state) => state.setSequencerHeight); + const setSequencerVisible = useDirectorWorkbenchStore((state) => state.setSequencerVisible); const resetWorkbench = useDirectorWorkbenchStore((state) => state.reset); const assets = useAssetStore((state) => state.assets); const addAsset = useAssetStore((state) => state.addAsset); @@ -110,6 +114,30 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh const updateObject = (id: string, patch: Partial) => commit((current) => ({ ...current, objects: current.objects.map((item) => (item.id === id ? { ...item, ...patch } : item)) })); const updateLight = (id: string, patch: Partial) => commit((current) => ({ ...current, lights: current.lights.map((item) => (item.id === id ? { ...item, ...patch } : item)) })); const updateShot = (id: string, patch: Partial) => commit((current) => ({ ...current, shots: current.shots.map((item) => (item.id === id ? { ...item, ...patch } : item)) })); + const removeObject = (id: string) => { + commit((current) => ({ ...current, objects: current.objects.filter((item) => item.id !== id) })); + if (selectedObjectId === id) { + setSelectedObjectId(null); + setSelectedBone(null); + } + }; + const removeLight = (id: string) => { + commit((current) => ({ ...current, lights: current.lights.filter((item) => item.id !== id) })); + if (selectedLightId === id) setSelectedLightId(null); + }; + const removeCamera = (id: string) => { + if (!draft || draft.cameras.length <= 1) { + message.warning("至少保留一台摄影机"); + return; + } + const fallback = draft.cameras.find((item) => item.id !== id); + if (!fallback) return; + commit((current) => ({ + ...current, + cameras: current.cameras.filter((item) => item.id !== id), + shots: current.shots.map((shot) => shot.cameraId === id ? { ...shot, cameraId: fallback.id } : shot), + })); + }; const addPrimitive = (primitive: DirectorObject["primitive"], name: string) => { const object = createDirectorObject(primitive, name); @@ -152,12 +180,29 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh if (activeShot) updateShot(activeShot.id, { cameraId: camera.id }); }; - const addLight = () => { - const light = createDirectorLight("point", `灯光 ${draft?.lights.length ? draft.lights.length + 1 : 1}`, [2, 3, 2], 1.5); + const addLight = (type: DirectorLight["type"] = "point", label = "灯光", position: DirectorVec3 = [2, 3, 2], intensity = 1.5) => { + const light = createDirectorLight(type, `${label} ${draft?.lights.length ? draft.lights.length + 1 : 1}`, position, intensity); commit((current) => ({ ...current, lights: [...current.lights, light] })); setSelectedLightId(light.id); }; + const addCameraMenuItems: MenuProps["items"] = [ + { key: "camera", icon: , label: "添加摄影机", onClick: addCamera }, + ]; + const addLightMenuItems: MenuProps["items"] = [ + { key: "directional", icon: , label: "方向光", onClick: () => addLight("directional", "方向光", [4, 6, 4], 2.4) }, + { key: "point", icon: , label: "点光源", onClick: () => addLight("point", "点光源") }, + { key: "spot", icon: , label: "聚光灯", onClick: () => addLight("spot", "聚光灯", [2, 4, 2], 2) }, + { key: "ambient", icon: , label: "环境光", onClick: () => addLight("ambient", "环境光", [0, 0, 0], 0.65) }, + ]; + const addObjectMenuItems: MenuProps["items"] = [ + { key: "actor", icon: , label: "演员", onClick: addActor }, + { key: "box", icon: , label: "立方体", onClick: () => addPrimitive("box", "立方体") }, + { key: "sphere", icon: , label: "球体", onClick: () => addPrimitive("sphere", "球体") }, + { key: "cylinder", icon: , label: "圆柱", onClick: () => addPrimitive("cylinder", "圆柱") }, + { key: "model", icon: , label: "上传模型", onClick: () => modelInputRef.current?.click() }, + ]; + const addShot = () => { if (!activeCamera) return; const shot: DirectorShot = { id: nanoid(), name: `镜头 ${(draft?.shots.length || 0) + 1}`, cameraId: activeCamera.id, duration: 5, fps: 24, shotSize: "medium", cameraMove: "static", prompt: "" }; @@ -291,14 +336,14 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh
@@ -320,20 +365,28 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh
- setPlaying(!playing)} onPlayheadChange={setPlayhead} onAutoKeyChange={setAutoKey} onHeightChange={setSequencerHeight} onSelectObject={setSelectedObjectId} onSelectBone={setSelectedBone} onRecordKeyframe={recordSelectedKeyframe} onAddShot={addShot} onSelectShot={(id) => { commit((current) => ({ ...current, activeShotId: id })); setPlayhead(0); }} /> + setPlaying(!playing)} onPlayheadChange={setPlayhead} onAutoKeyChange={setAutoKey} onHeightChange={setSequencerHeight} onVisibilityChange={setSequencerVisible} onSelectObject={setSelectedObjectId} onSelectBone={setSelectedBone} onRecordKeyframe={recordSelectedKeyframe} onAddShot={addShot} onSelectShot={(id) => { commit((current) => ({ ...current, activeShotId: id })); setPlayhead(0); }} /> ); } -function ObjectInspector({ object, playhead, selectedBone, onSelectBone, onUpdate, onAddKeyframe, onDelete }: { object: DirectorObject; playhead: number; selectedBone: string | null; onSelectBone: (bone: string | null) => void; onUpdate: (patch: Partial) => void; onAddKeyframe: () => void; onDelete: () => void }) { +function ObjectInspector({ object, playhead, selectedBone, autoKey, onSelectBone, onUpdate, onAddKeyframe, onDelete }: { object: DirectorObject; playhead: number; selectedBone: string | null; autoKey: boolean; onSelectBone: (bone: string | null) => void; onUpdate: (patch: Partial) => void; onAddKeyframe: () => void; onDelete: () => void }) { const motionClips = object.motionClips || []; const activeMotionClip = motionClips.find((clip) => clip.id === object.activeMotionClipId); const mappedBones = Object.keys(object.rig?.boneMap || {}) as DirectorHumanoidBone[]; + const selectedBoneId = selectedBone as DirectorHumanoidBone | null; + const selectedBoneRotation = selectedBoneId ? object.boneOverrides?.[selectedBoneId] || [0, 0, 0, 1] as DirectorQuat : null; const updateActiveMotion = (patch: Partial[number]>) => activeMotionClip && onUpdate({ motionClips: motionClips.map((clip) => clip.id === activeMotionClip.id ? { ...clip, ...patch } : clip) }); + const updateSelectedBoneRotation = (rotation: DirectorQuat) => { + if (!selectedBoneId) return; + const patch: Partial = { boneOverrides: { ...object.boneOverrides, [selectedBoneId]: rotation } }; + if (autoKey) patch.boneTracks = upsertDirectorBoneKeyframe(object.boneTracks || [], selectedBoneId, playhead, rotation); + onUpdate(patch); + }; const applyPose = (pose: DirectorPose) => onUpdate({ pose, activeMotionClipId: undefined, boneOverrides: {} }); return onUpdate({ name })} onDelete={onDelete}> onUpdate({ transform })} /> @@ -348,6 +401,7 @@ function ObjectInspector({ object, playhead, selectedBone, onSelectBone, onUpdat
角色绑定{object.rig?.status === "ready" ? `${mappedBones.length} 根骨骼` : "等待模型"}
{motionClips.length ? <> ({ label: directorBoneLabel(bone), value: bone }))} onChange={(bone) => onSelectBone(bone || null)} /> : null} + {selectedBoneId && selectedBoneRotation ? : null} : null} onUpdate({ visible })} /> onUpdate({ castShadow })} /> @@ -379,13 +433,61 @@ function TransformFields({ transform, onChange }: { transform: DirectorTransform return <> onChange({ ...transform, position })} /> onChange({ ...transform, rotation })} /> onChange({ ...transform, scale })} />; } +function BoneRotationFields({ rotation, onChange }: { rotation: DirectorQuat; onChange: (rotation: DirectorQuat) => void }) { + const initialDegrees = useMemo(() => { + const euler = new Euler().setFromQuaternion(new Quaternion(...rotation), "XYZ"); + return [euler.x, euler.y, euler.z].map((value) => Number(((value * 180) / Math.PI).toFixed(1))) as DirectorVec3; + }, [rotation]); + const [degrees, setDegrees] = useState(initialDegrees); + const lastEmittedRotation = useRef(null); + useEffect(() => { + if (lastEmittedRotation.current && sameDirectorQuaternion(rotation, lastEmittedRotation.current)) { + lastEmittedRotation.current = null; + return; + } + setDegrees(initialDegrees); + }, [initialDegrees, rotation]); + const updateAxis = (index: number, value: number) => { + const next = degrees.map((entry, entryIndex) => entryIndex === index ? value : entry) as DirectorVec3; + const radians = next.map((entry) => (entry * Math.PI) / 180) as DirectorVec3; + const nextRotation = new Quaternion().setFromEuler(new Euler(radians[0], radians[1], radians[2], "XYZ")).toArray() as DirectorQuat; + setDegrees(next); + lastEmittedRotation.current = nextRotation; + onChange(nextRotation); + }; + return
+ {degrees.map((value, index) =>
+ {["X", "Y", "Z"][index]} + updateAxis(index, Array.isArray(next) ? next[0] ?? 0 : next)} /> + {value.toFixed(1)}° +
)} +
; +} + +function sameDirectorQuaternion(left: DirectorQuat, right: DirectorQuat) { + const directDistance = left.reduce((sum, value, index) => sum + Math.abs(value - right[index]), 0); + const inverseDistance = left.reduce((sum, value, index) => sum + Math.abs(value + right[index]), 0); + return Math.min(directDistance, inverseDistance) < 0.0001; +} + function Vec3Field({ label, value, step = 0.1, onChange }: { label: string; value: DirectorVec3; step?: number; onChange: (value: DirectorVec3) => void }) { return
{value.map((item, index) => onChange(value.map((entry, itemIndex) => itemIndex === index ? next || 0 : entry) as DirectorVec3)} />)}
; } function Field({ label, children }: { label: string; children: ReactNode }) { return ; } function PanelTitle({ title, action }: { title: string; action?: ReactNode }) { return
{title}{action}
; } -function SceneRow({ active, icon, label, onClick }: { active?: boolean; icon: ReactElement; label: string; onClick: () => void }) { return ; } +function SceneRow({ active, icon, label, onClick, onDelete }: { active?: boolean; icon: ReactElement; label: string; onClick: () => void; onDelete?: () => void }) { + return
+ + {onDelete ? : null} +
; +} +function AddMenuButton({ label, items }: { label: string; items: MenuProps["items"] }) { + return ; +} function QuickAdd({ label, icon, onClick }: { label: string; icon: ReactElement; onClick: () => void }) { return ; } function IconButton({ label, disabled, children, onClick }: { label: string; disabled?: boolean; children: ReactNode; onClick: () => void }) { return ; } const poseOptions: Array<{ label: string; value: DirectorPose }> = [ diff --git a/web/src/components/canvas/director/director-sequencer.tsx b/web/src/components/canvas/director/director-sequencer.tsx index 9aa48a722..446608dfe 100644 --- a/web/src/components/canvas/director/director-sequencer.tsx +++ b/web/src/components/canvas/director/director-sequencer.tsx @@ -1,4 +1,4 @@ -import { ChevronDown, ChevronRight, KeyRound, Magnet, Pause, Play, Plus, Rows3, ZoomIn, ZoomOut } from "lucide-react"; +import { ChevronDown, ChevronRight, ChevronUp, KeyRound, Magnet, Pause, Play, Plus, Rows3, ZoomIn, ZoomOut } from "lucide-react"; import { useMemo, useRef, useState, type CSSProperties, type ReactNode } from "react"; import { directorBoneLabel } from "@/lib/canvas/director/director-scene"; @@ -15,10 +15,12 @@ type DirectorSequencerProps = { playing: boolean; autoKey: boolean; height: number; + visible: boolean; onPlayToggle: () => void; onPlayheadChange: (time: number) => void; onAutoKeyChange: (value: boolean) => void; onHeightChange: (height: number) => void; + onVisibilityChange: (visible: boolean) => void; onSelectObject: (id: string | null) => void; onSelectBone: (bone: string | null) => void; onRecordKeyframe: () => void; @@ -28,7 +30,7 @@ type DirectorSequencerProps = { type TrackKey = { id: string; time: number; color?: string }; -export function DirectorSequencer({ scene, shot, camera, objects, selectedObjectId, selectedBone, playhead, playing, autoKey, height, onPlayToggle, onPlayheadChange, onAutoKeyChange, onHeightChange, onSelectObject, onSelectBone, onRecordKeyframe, onAddShot, onSelectShot }: DirectorSequencerProps) { +export function DirectorSequencer({ scene, shot, camera, objects, selectedObjectId, selectedBone, playhead, playing, autoKey, height, visible, onPlayToggle, onPlayheadChange, onAutoKeyChange, onHeightChange, onVisibilityChange, onSelectObject, onSelectBone, onRecordKeyframe, onAddShot, onSelectShot }: DirectorSequencerProps) { const [expanded, setExpanded] = useState>({}); const [snapEnabled, setSnapEnabled] = useState(true); const [showDetails, setShowDetails] = useState(true); @@ -60,6 +62,14 @@ export function DirectorSequencer({ scene, shot, camera, objects, selectedObject const cameraKeys = camera?.keyframes.map((key) => ({ id: key.id, time: key.time, color: "#78a9ff" })) || []; + if (!visible) { + return
+
+ +
+
; + } + return (
@@ -79,6 +89,7 @@ export function DirectorSequencer({ scene, shot, camera, objects, selectedObject + diff --git a/web/src/components/canvas/director/director-viewport.tsx b/web/src/components/canvas/director/director-viewport.tsx index ca58f61fc..b6dbd308c 100644 --- a/web/src/components/canvas/director/director-viewport.tsx +++ b/web/src/components/canvas/director/director-viewport.tsx @@ -1,7 +1,7 @@ -import { Canvas, useFrame, useThree } from "@react-three/fiber"; +import { Canvas, useFrame, useThree, type ThreeEvent } from "@react-three/fiber"; import { Grid, OrbitControls, TransformControls } from "@react-three/drei"; import { forwardRef, Suspense, useCallback, useEffect, useImperativeHandle, useMemo, useRef, useState } from "react"; -import { AnimationClip, AnimationMixer, Box3, Bone, Color, Group, LoopOnce, LoopRepeat, Mesh, MeshBasicMaterial, MeshDepthMaterial, MeshNormalMaterial, MeshStandardMaterial, Object3D, PerspectiveCamera, Quaternion, Scene, SkeletonHelper, Texture, TextureLoader, Vector3, WebGLRenderer } from "three"; +import { AnimationClip, AnimationMixer, Box3, Bone, Camera, Color, Group, LoopOnce, LoopRepeat, Mesh, MeshBasicMaterial, MeshDepthMaterial, MeshNormalMaterial, MeshStandardMaterial, Object3D, OrthographicCamera, PerspectiveCamera, Quaternion, Scene, SkeletonHelper, Texture, TextureLoader, Vector3, WebGLRenderer } from "three"; import type { Material } from "three"; import { GLTFLoader, SkeletonUtils } from "three-stdlib"; @@ -205,6 +205,21 @@ function DirectorModel({ object, selected, selectedBone, playhead, onSelectBone, const motion = object.motionClips?.find((item) => item.id === object.activeMotionClipId); const activeAnimation = motion ? animations.find((item) => item.name === motion.sourceAnimation) : undefined; const modelUrl = object.kind === "actor" || object.primitive === "character" ? DIRECTOR_DEFAULT_ACTOR_URL : object.url; + const handleModelPointerDown = useCallback((event: ThreeEvent) => { + if (!selected || !rig) return; + let nearest: { bone: string; distance: number } | null = null; + Object.entries(rig.boneMap).forEach(([bone, name]) => { + if (!name) return; + const target = model?.getObjectByName(name); + if (!target) return; + const distance = event.ray.distanceSqToPoint(target.getWorldPosition(new Vector3())); + if (distance <= 0.12 ** 2 && (!nearest || distance < nearest.distance)) nearest = { bone, distance }; + }); + if (!nearest) return; + // 模型表面可能先于关节控制球被射线命中，用最近骨骼保证点击仍可选中。 + event.stopPropagation(); + onSelectBone(nearest.bone); + }, [model, onSelectBone, rig, selected]); useEffect(() => { onActorRigReadyRef.current = onActorRigReady; }, [onActorRigReady]); @@ -276,10 +291,14 @@ function DirectorModel({ object, selected, selectedBone, playhead, onSelectBone, }); if (!model) return ; - return + return {selected && helper ? : null} - {selected && rig ? Object.entries(rig.boneMap).filter(([, name]) => Boolean(name)).map(([bone, name]) => onSelectBone(bone)} />) : null} + {selected && rig ? Object.entries(rig.boneMap).filter(([, name]) => Boolean(name)).map(([bone, name]) => { + const fingerGroup = directorFingerGroup(bone); + const selectedFingerGroup = directorFingerGroup(selectedBone); + return onSelectBone(bone)} />; + }) : null} {selected && selectedBoneObject ? onBoneTransform(selectedBone!, selectedBoneObject.quaternion.toArray() as DirectorQuat)} /> : null} ; } @@ -303,17 +322,55 @@ function LoadingBone({ from, to, color }: { from: DirectorVec3; to: DirectorVec3 return ; } -function BoneController({ bone, model, selected, onSelect }: { bone: Object3D | undefined; model: Object3D; selected: boolean; onSelect: () => void }) { +function BoneController({ bone, selected, dimmed, onSelect }: { bone: Object3D | undefined; selected: boolean; dimmed: boolean; onSelect: () => void }) { const ref = useRef(null); + const visibleRef = useRef(null); + const hitRef = useRef(null); const world = useMemo(() => new Vector3(), []); + const local = useMemo(() => new Vector3(), []); + const cameraWorld = useMemo(() => new Vector3(), []); + const { camera, size } = useThree(); useFrame(() => { if (!bone || !ref.current) return; + const parent = ref.current.parent; + if (!parent) return; + // 控制点与模型根节点是兄弟关系，必须转换到控制点父级坐标，否则点击区域会与骨骼错位。 bone.getWorldPosition(world); - model.worldToLocal(world); - ref.current.position.copy(world); + camera.getWorldPosition(cameraWorld); + local.copy(world); + parent.worldToLocal(local); + ref.current.position.copy(local); + const distance = cameraWorld.distanceTo(world); + const visualPixels = selected ? 5 : dimmed ? 2.5 : 3.5; + const hitPixels = selected ? 12 : 10; + visibleRef.current?.scale.setScalar(screenPixelsToWorldRadius(camera, distance, visualPixels, size.height)); + hitRef.current?.scale.setScalar(screenPixelsToWorldRadius(camera, distance, hitPixels, size.height)); }); if (!bone) return null; - return { event.stopPropagation(); onSelect(); }}>; + const handlePointerDown = (event: ThreeEvent) => { event.stopPropagation(); onSelect(); }; + return + + + + + {/* 可视点保持小尺寸，透明球只负责提供稳定的点击面积。 */} + + + + + ; +} + +function screenPixelsToWorldRadius(camera: Camera, distance: number, pixels: number, viewportHeight: number) { + const height = Math.max(1, viewportHeight); + if (camera instanceof PerspectiveCamera) return (pixels * 2 * Math.max(0.01, distance) * Math.tan((camera.fov * Math.PI) / 360)) / (height * Math.max(0.01, camera.zoom)); + if (camera instanceof OrthographicCamera) return (pixels * (camera.top - camera.bottom)) / (height * Math.max(0.01, camera.zoom)); + return pixels * 0.001; +} + +function directorFingerGroup(bone: string | null) { + const match = bone?.match(/^(left|right)(Thumb|Index|Middle|Ring|Pinky)\d$/); + return match ? `${match[1]}${match[2]}` : null; } function normalizeModel(root: Object3D, castShadow: boolean, receiveShadow: boolean) { @@ -367,10 +424,26 @@ function readRigRestRotations(root: Object3D, rig: DirectorRig) { function inferDirectorRig(root: Object3D, animationNames: string[]): DirectorRig { const names = new Map(); root.traverse((child) => { if (child instanceof Bone) names.set(normalizeBoneName(child.name), child.name); }); + const fingerPatterns = (side: "left" | "right", finger: "thumb" | "index" | "middle" | "ring" | "pinky", segment: 1 | 2 | 3) => [ + new RegExp(`^mixamorig${side}hand${finger}${segment}$`), + new RegExp(`^${side}hand${finger}${segment}$`), + new RegExp(`^${side}${finger}${segment}$`), + new RegExp(`^${finger}0?${segment}${side === "left" ? "l" : "r"}$`), + ]; const patterns: Record = { root: [/^root$/, /armature/], hips: [/hips|pelvis/, /mixamorig.*hip/], spine: [/spine1?$|lowerback/], chest: [/spine2|chest|upperback/], neck: [/neck/], head: [/head/], - leftShoulder: [/leftshoulder|shoulder_l|mixamorigleftshoulder/], leftUpperArm: [/leftupperarm|leftarm|upperarm_l|mixamorigleftarm/], leftLowerArm: [/leftforearm|leftlowerarm|forearm_l|mixamorigleftforearm/], leftHand: [/lefthand|hand_l|mixamoriglefthand/], - rightShoulder: [/rightshoulder|shoulder_r|mixamorigrightshoulder/], rightUpperArm: [/rightupperarm|rightarm|upperarm_r|mixamorigrightarm/], rightLowerArm: [/rightforearm|rightlowerarm|forearm_r|mixamorigrightforearm/], rightHand: [/righthand|hand_r|mixamorigrighthand/], + leftShoulder: [/leftshoulder|shoulder_l|mixamorigleftshoulder/], leftUpperArm: [/leftupperarm|leftarm|upperarm_l|mixamorigleftarm/], leftLowerArm: [/leftforearm|leftlowerarm|forearm_l|mixamorigleftforearm/], leftHand: [/^lefthand$/, /^handl$/, /^mixamoriglefthand$/], + leftThumb1: fingerPatterns("left", "thumb", 1), leftThumb2: fingerPatterns("left", "thumb", 2), leftThumb3: fingerPatterns("left", "thumb", 3), + leftIndex1: fingerPatterns("left", "index", 1), leftIndex2: fingerPatterns("left", "index", 2), leftIndex3: fingerPatterns("left", "index", 3), + leftMiddle1: fingerPatterns("left", "middle", 1), leftMiddle2: fingerPatterns("left", "middle", 2), leftMiddle3: fingerPatterns("left", "middle", 3), + leftRing1: fingerPatterns("left", "ring", 1), leftRing2: fingerPatterns("left", "ring", 2), leftRing3: fingerPatterns("left", "ring", 3), + leftPinky1: fingerPatterns("left", "pinky", 1), leftPinky2: fingerPatterns("left", "pinky", 2), leftPinky3: fingerPatterns("left", "pinky", 3), + rightShoulder: [/rightshoulder|shoulder_r|mixamorigrightshoulder/], rightUpperArm: [/rightupperarm|rightarm|upperarm_r|mixamorigrightarm/], rightLowerArm: [/rightforearm|rightlowerarm|forearm_r|mixamorigrightforearm/], rightHand: [/^righthand$/, /^handr$/, /^mixamorigrighthand$/], + rightThumb1: fingerPatterns("right", "thumb", 1), rightThumb2: fingerPatterns("right", "thumb", 2), rightThumb3: fingerPatterns("right", "thumb", 3), + rightIndex1: fingerPatterns("right", "index", 1), rightIndex2: fingerPatterns("right", "index", 2), rightIndex3: fingerPatterns("right", "index", 3), + rightMiddle1: fingerPatterns("right", "middle", 1), rightMiddle2: fingerPatterns("right", "middle", 2), rightMiddle3: fingerPatterns("right", "middle", 3), + rightRing1: fingerPatterns("right", "ring", 1), rightRing2: fingerPatterns("right", "ring", 2), rightRing3: fingerPatterns("right", "ring", 3), + rightPinky1: fingerPatterns("right", "pinky", 1), rightPinky2: fingerPatterns("right", "pinky", 2), rightPinky3: fingerPatterns("right", "pinky", 3), leftUpperLeg: [/leftupleg|leftthigh|thigh_l|mixamorigleftupleg/], leftLowerLeg: [/leftleg|leftcalf|calf_l|mixamorigleftleg/], leftFoot: [/leftfoot|foot_l|mixamorigleftfoot/], rightUpperLeg: [/rightupleg|rightthigh|thigh_r|mixamorigrightupleg/], rightLowerLeg: [/rightleg|rightcalf|calf_r|mixamorigrightleg/], rightFoot: [/rightfoot|foot_r|mixamorigrightfoot/], }; const boneMap = Object.fromEntries(Object.entries(patterns).map(([bone, candidates]) => [bone, candidates.map((pattern) => [...names.entries()].find(([normalized]) => pattern.test(normalized))?.[1]).find(Boolean)]).filter(([, name]) => Boolean(name))) as DirectorRig["boneMap"]; diff --git a/web/src/lib/canvas/director/director-scene.ts b/web/src/lib/canvas/director/director-scene.ts index 7b26fb6c7..58a19c4ca 100644 --- a/web/src/lib/canvas/director/director-scene.ts +++ b/web/src/lib/canvas/director/director-scene.ts @@ -122,7 +122,23 @@ export function interpolateDirectorBoneRotation(base: DirectorQuat, keyframes: D } export function directorBoneLabel(bone: string) { - return ({ hips: "骨盆", spine: "脊柱", chest: "胸腔", neck: "颈部", head: "头部", leftShoulder: "左肩", leftUpperArm: "左上臂", leftLowerArm: "左前臂", leftHand: "左手", rightShoulder: "右肩", rightUpperArm: "右上臂", rightLowerArm: "右前臂", rightHand: "右手", leftUpperLeg: "左大腿", leftLowerLeg: "左小腿", leftFoot: "左脚", rightUpperLeg: "右大腿", rightLowerLeg: "右小腿", rightFoot: "右脚" } as Record)[bone] || bone; + return ({ + hips: "骨盆", spine: "脊柱", chest: "胸腔", neck: "颈部", head: "头部", + leftShoulder: "左肩", leftUpperArm: "左上臂", leftLowerArm: "左前臂", leftHand: "左手", + rightShoulder: "右肩", rightUpperArm: "右上臂", rightLowerArm: "右前臂", rightHand: "右手", + leftUpperLeg: "左大腿", leftLowerLeg: "左小腿", leftFoot: "左脚", + rightUpperLeg: "右大腿", rightLowerLeg: "右小腿", rightFoot: "右脚", + leftThumb1: "左拇指·根", leftThumb2: "左拇指·中", leftThumb3: "左拇指·尖", + leftIndex1: "左食指·根", leftIndex2: "左食指·中", leftIndex3: "左食指·尖", + leftMiddle1: "左中指·根", leftMiddle2: "左中指·中", leftMiddle3: "左中指·尖", + leftRing1: "左无名指·根", leftRing2: "左无名指·中", leftRing3: "左无名指·尖", + leftPinky1: "左小指·根", leftPinky2: "左小指·中", leftPinky3: "左小指·尖", + rightThumb1: "右拇指·根", rightThumb2: "右拇指·中", rightThumb3: "右拇指·尖", + rightIndex1: "右食指·根", rightIndex2: "右食指·中", rightIndex3: "右食指·尖", + rightMiddle1: "右中指·根", rightMiddle2: "右中指·中", rightMiddle3: "右中指·尖", + rightRing1: "右无名指·根", rightRing2: "右无名指·中", rightRing3: "右无名指·尖", + rightPinky1: "右小指·根", rightPinky2: "右小指·中", rightPinky3: "右小指·尖", + } as Record)[bone] || bone; } export function directorPoseLabel(pose: DirectorPose) { diff --git a/web/src/pages/canvas/project.tsx b/web/src/pages/canvas/project.tsx index 68cce559c..f8577b134 100644 --- a/web/src/pages/canvas/project.tsx +++ b/web/src/pages/canvas/project.tsx @@ -2227,6 +2227,7 @@ function InfiniteCanvasPage() { onClose={() => setDirectorNodeId(null)} onChange={saveDirectorScene} onApply={applyDirectorOutput} + onDeleteImageNode={(nodeId) => deleteNodes(new Set([nodeId]))} /> ) : null} diff --git a/web/src/stores/canvas/use-director-workbench-store.ts b/web/src/stores/canvas/use-director-workbench-store.ts index a8c70d7b4..ea538d90a 100644 --- a/web/src/stores/canvas/use-director-workbench-store.ts +++ b/web/src/stores/canvas/use-director-workbench-store.ts @@ -12,6 +12,7 @@ type DirectorWorkbenchStore = { playing: boolean; autoKey: boolean; sequencerHeight: number; + sequencerVisible: boolean; setSelectedObjectId: (id: string | null) => void; setSelectedBone: (bone: string | null) => void; setSelectedLightId: (id: string | null) => void; @@ -21,10 +22,11 @@ type DirectorWorkbenchStore = { setPlaying: (playing: boolean) => void; setAutoKey: (autoKey: boolean) => void; setSequencerHeight: (height: number) => void; + setSequencerVisible: (visible: boolean) => void; reset: () => void; }; -const initialState = { selectedObjectId: null, selectedBone: null, selectedLightId: null, transformMode: "translate" as const, renderMode: "beauty" as const, playhead: 0, playing: false, autoKey: true, sequencerHeight: 300 }; +const initialState = { selectedObjectId: null, selectedBone: null, selectedLightId: null, transformMode: "translate" as const, renderMode: "beauty" as const, playhead: 0, playing: false, autoKey: true, sequencerHeight: 300, sequencerVisible: true }; export const useDirectorWorkbenchStore = create((set) => ({ ...initialState, @@ -37,5 +39,6 @@ export const useDirectorWorkbenchStore = create((set) => setPlaying: (playing) => set({ playing }), setAutoKey: (autoKey) => set({ autoKey }), setSequencerHeight: (sequencerHeight) => set({ sequencerHeight: Math.max(180, Math.min(620, sequencerHeight)) }), + setSequencerVisible: (sequencerVisible) => set({ sequencerVisible }), reset: () => set(initialState), })); diff --git a/web/src/types/director.ts b/web/src/types/director.ts index c45a5e4ec..5bb21e45d 100644 --- a/web/src/types/director.ts +++ b/web/src/types/director.ts @@ -21,7 +21,11 @@ export type DirectorKeyframe = { easing?: "step" | "linear" | "smooth"; }; -export type DirectorHumanoidBone = "root" | "hips" | "spine" | "chest" | "neck" | "head" | "leftShoulder" | "leftUpperArm" | "leftLowerArm" | "leftHand" | "rightShoulder" | "rightUpperArm" | "rightLowerArm" | "rightHand" | "leftUpperLeg" | "leftLowerLeg" | "leftFoot" | "rightUpperLeg" | "rightLowerLeg" | "rightFoot"; +export type DirectorFingerBone = + | "leftThumb1" | "leftThumb2" | "leftThumb3" | "leftIndex1" | "leftIndex2" | "leftIndex3" | "leftMiddle1" | "leftMiddle2" | "leftMiddle3" | "leftRing1" | "leftRing2" | "leftRing3" | "leftPinky1" | "leftPinky2" | "leftPinky3" + | "rightThumb1" | "rightThumb2" | "rightThumb3" | "rightIndex1" | "rightIndex2" | "rightIndex3" | "rightMiddle1" | "rightMiddle2" | "rightMiddle3" | "rightRing1" | "rightRing2" | "rightRing3" | "rightPinky1" | "rightPinky2" | "rightPinky3"; + +export type DirectorHumanoidBone = "root" | "hips" | "spine" | "chest" | "neck" | "head" | "leftShoulder" | "leftUpperArm" | "leftLowerArm" | "leftHand" | "rightShoulder" | "rightUpperArm" | "rightLowerArm" | "rightHand" | "leftUpperLeg" | "leftLowerLeg" | "leftFoot" | "rightUpperLeg" | "rightLowerLeg" | "rightFoot" | DirectorFingerBone; export type DirectorBoneKeyframe = { id: string; From bda114b29eaca178daf52da0785ad84f1b9f40c5 Mon Sep 17 00:00:00 2001 From: ddCat <2695092045@qq.com> Date: Tue, 18 Aug 2026 15:15:36 +0800 Subject: [PATCH 2/4] =?UTF-8?q?fix(runtime):=20=E6=9C=AC=E5=9C=B0=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E6=81=A2=E5=A4=8D=20-=20=E6=81=A2=E5=A4=8D=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E7=94=BB=E5=B8=83=E5=92=8C=E5=BB=B6=E8=BF=9F=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/internal/repository/repository.go | 22 ++- .../repository/task_provider_poll_test.go | 51 +++++++ backend/internal/service/provider.go | 5 +- backend/internal/service/provider_test.go | 57 +++++++ .../internal/service/runtime_policy_test.go | 73 +++++++++ backend/internal/service/service.go | 22 ++- .../canvas/canvas-local-agent-panel.tsx | 57 +++---- web/src/services/api/generation-task.ts | 12 +- .../stores/canvas/use-canvas-agent-store.ts | 47 +++++- .../stores/use-local-dreamina-model-store.ts | 116 +++++++++++--- web/test/canvas-local-runtime.test.ts | 67 ++++++++ web/test/generation-task-local.test.ts | 134 ++++++++++++++++ web/test/local-dreamina-model-catalog.test.ts | 143 ++++++++++++++++++ 13 files changed, 739 insertions(+), 67 deletions(-) create mode 100644 backend/internal/repository/task_provider_poll_test.go diff --git a/backend/internal/repository/repository.go b/backend/internal/repository/repository.go index 981730b81..8a6071a7d 100644 --- a/backend/internal/repository/repository.go +++ b/backend/internal/repository/repository.go @@ -294,7 +294,7 @@ func (r *Repository) ClaimNextTask(owner string, leaseDuration time.Duration) (* now := time.Now() leaseExpiresAt := now.Add(leaseDuration) err := r.db.Transaction(func(tx *gorm.DB) error { - query := tx.Where("status = ? OR (status = ? AND (lease_expires_at IS NULL OR lease_expires_at <= ?))", model.TaskStatusQueued, model.TaskStatusRunning, now). + query := tx.Where("(status = ? OR (status = ? AND (lease_expires_at IS NULL OR lease_expires_at <= ?))) AND (next_poll_at IS NULL OR next_poll_at <= ?)", model.TaskStatusQueued, model.TaskStatusRunning, now, now). Order("created_at asc").Limit(1) if r.Dialect() == "postgres" { query = query.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}) @@ -309,7 +309,7 @@ func (r *Repository) ClaimNextTask(owner string, leaseDuration time.Duration) (* } claim := tx.Model(&model.Task{}).Where("id = ?", task.ID) if r.Dialect() != "postgres" { - claim = claim.Where("status = ? OR (status = ? AND (lease_expires_at IS NULL OR lease_expires_at <= ?))", model.TaskStatusQueued, model.TaskStatusRunning, now) + claim = claim.Where("(status = ? OR (status = ? AND (lease_expires_at IS NULL OR lease_expires_at <= ?))) AND (next_poll_at IS NULL OR next_poll_at <= ?)", model.TaskStatusQueued, model.TaskStatusRunning, now, now) } updated := claim. Updates(map[string]any{ @@ -320,6 +320,7 @@ func (r *Repository) ClaimNextTask(owner string, leaseDuration time.Duration) (* "started_at": gorm.Expr("COALESCE(started_at, ?)", now), "lease_owner": owner, "lease_expires_at": leaseExpiresAt, + "next_poll_at": nil, "updated_at": now, }) if updated.Error != nil { @@ -358,6 +359,23 @@ func (r *Repository) UpdateTaskProviderState(id string, providerRequestID string return r.db.Model(&model.Task{}).Where("id = ?", id).Updates(updates).Error } +func (r *Repository) DeferRunningTaskForProviderPoll(id string, owner string, stage string, delay time.Duration) error { + now := time.Now() + result := r.db.Model(&model.Task{}). + Where("id = ? AND status = ? AND lease_owner = ?", id, model.TaskStatusRunning, owner). + Updates(map[string]any{ + "stage": stage, "error": "", "completed_at": nil, "next_poll_at": now.Add(delay), + "lease_owner": "", "lease_expires_at": nil, "updated_at": now, + }) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return ErrTaskStateConflict + } + return nil +} + // 人工恢复仅锁定失败任务；旧 worker 的租约可覆盖，但未过期的人工恢复租约不能并发抢占。 func (r *Repository) ClaimFailedTaskProviderRecovery(id string, userID string, owner string, leaseDuration time.Duration) error { now := time.Now() diff --git a/backend/internal/repository/task_provider_poll_test.go b/backend/internal/repository/task_provider_poll_test.go new file mode 100644 index 000000000..76b0b84e8 --- /dev/null +++ b/backend/internal/repository/task_provider_poll_test.go @@ -0,0 +1,51 @@ +package repository + +import ( + "testing" + "time" + + "infinite-canvas/backend/internal/model" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestDeferredProviderPollKeepsOriginalTaskIdentityWithoutImmediateReclaim(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:provider-poll-defer?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.Task{}); err != nil { + t.Fatal(err) + } + task := model.Task{ + ID: "task-1", UserID: "user-1", Type: "canvas_video", Status: model.TaskStatusRunning, + LeaseOwner: "worker-1", LeaseExpiresAt: ptrTime(time.Now().Add(time.Minute)), ProviderRequestID: "provider-task-1", + } + if err := db.Create(&task).Error; err != nil { + t.Fatal(err) + } + repo := New(db) + if err := repo.DeferRunningTaskForProviderPoll(task.ID, task.LeaseOwner, "后台仍在生成", time.Minute); err != nil { + t.Fatal(err) + } + claimed, err := repo.ClaimNextTask("worker-2", 45*time.Second) + if err != nil { + t.Fatal(err) + } + if claimed != nil { + t.Fatalf("task reclaimed before next poll: %#v", claimed) + } + if err := db.Model(&model.Task{}).Where("id = ?", task.ID).Update("next_poll_at", time.Now().Add(-time.Second)).Error; err != nil { + t.Fatal(err) + } + claimed, err = repo.ClaimNextTask("worker-2", 45*time.Second) + if err != nil { + t.Fatal(err) + } + if claimed == nil || claimed.ID != task.ID || claimed.ProviderRequestID != task.ProviderRequestID { + t.Fatalf("reclaimed task = %#v", claimed) + } +} + +func ptrTime(value time.Time) *time.Time { return &value } diff --git a/backend/internal/service/provider.go b/backend/internal/service/provider.go index 46b9cd6cb..496192c16 100644 --- a/backend/internal/service/provider.go +++ b/backend/internal/service/provider.go @@ -1854,7 +1854,10 @@ func runNewAPIChannel2VideoTask(ctx context.Context, input canvasGenerationInput return nil, err } } - return nil, fmt.Errorf("NewAPI Video Generations 视频生成超时（任务 %s）", id) + if err := ctx.Err(); err != nil { + return nil, err + } + return nil, context.DeadlineExceeded } // 单次查询只读取既有上游任务，不创建新任务；自动轮询和人工恢复共用这条安全边界。 diff --git a/backend/internal/service/provider_test.go b/backend/internal/service/provider_test.go index 981843ea8..5dd6ee521 100644 --- a/backend/internal/service/provider_test.go +++ b/backend/internal/service/provider_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "io" "mime/multipart" "net/http" @@ -12,6 +13,9 @@ import ( "strconv" "strings" "testing" + "time" + + "infinite-canvas/backend/internal/model" ) const testReferenceImageDataURL = "data:image/png;base64,aGVsbG8=" @@ -1306,6 +1310,59 @@ func TestRunNewAPIChannel2VideoTaskDownloadsTemporaryResult(t *testing.T) { } } +func TestRunNewAPIChannel2VideoTaskResumesOriginalProviderTaskWithoutAnotherPost(t *testing.T) { + t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true") + paths := make([]string, 0, 2) + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.Method+" "+r.URL.Path) + switch r.Method + " " + r.URL.Path { + case "GET /v1/video/generations/existing-provider-task": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"code":"success","data":{"task_id":"existing-provider-task","status":"SUCCESS","result_url":"` + server.URL + `/video.mp4"}}`)) + case "GET /video.mp4": + w.Header().Set("Content-Type", "video/mp4") + _, _ = w.Write([]byte("video")) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + input := canvasGenerationInput{ + Mode: "video", + Config: providerConfig{BaseURL: server.URL, APIKey: "test-key", Model: "video-model", InterfaceType: string(model.ChannelInterfaceNewAPIChannel2)}, + } + inputJSON, err := json.Marshal(input) + if err != nil { + t.Fatal(err) + } + ctx := withProviderAnalytics(context.Background(), nil, model.Task{ + ID: "task-1", Type: "canvas_video", ProviderRequestID: "existing-provider-task", InputJSON: string(inputJSON), + }) + result, err := runNewAPIChannel2VideoTask(ctx, input) + if err != nil { + t.Fatalf("runNewAPIChannel2VideoTask() error = %v", err) + } + if result["video"] == nil { + t.Fatalf("result = %#v", result) + } + want := "GET /v1/video/generations/existing-provider-task,GET /video.mp4" + if got := strings.Join(paths, ","); got != want { + t.Fatalf("paths = %q, want %q", got, want) + } +} + +func TestRunNewAPIChannel2VideoTaskReturnsTypedDeadlineWhenPollingWindowEnds(t *testing.T) { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + ctx = withProviderAnalytics(ctx, nil, model.Task{ID: "task-1", Type: "canvas_video", ProviderRequestID: "existing-provider-task"}) + _, err := runNewAPIChannel2VideoTask(ctx, canvasGenerationInput{}) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("runNewAPIChannel2VideoTask() error = %v, want context deadline exceeded", err) + } +} + func TestRunGeminiVeoVideoTaskUsesLongRunningOperation(t *testing.T) { t.Setenv("CANVAS_ALLOW_PRIVATE_UPSTREAMS", "true") paths := make([]string, 0, 3) diff --git a/backend/internal/service/runtime_policy_test.go b/backend/internal/service/runtime_policy_test.go index ae34d3f0b..8a85de63a 100644 --- a/backend/internal/service/runtime_policy_test.go +++ b/backend/internal/service/runtime_policy_test.go @@ -1,7 +1,10 @@ package service import ( + "context" + "encoding/json" "testing" + "time" "infinite-canvas/backend/internal/model" "infinite-canvas/backend/internal/repository" @@ -10,6 +13,76 @@ import ( "gorm.io/gorm" ) +func TestVideoTaskTimeoutHasFiveMinuteSafetyFloor(t *testing.T) { + policy := defaultRuntimePolicy().Task + policy.VideoTimeoutMinutes = 1 + policy.ImageTimeoutMinutes = 1 + if got := taskExecutionTimeoutWithPolicy("canvas_video", policy); got != 5*time.Minute { + t.Fatalf("video timeout = %s, want 5m", got) + } + if got := taskExecutionTimeoutWithPolicy("canvas_image", policy); got != time.Minute { + t.Fatalf("image timeout = %s, want 1m", got) + } +} + +func TestOnlyResumableNewAPIChannel2VideoDeadlinesStayRunning(t *testing.T) { + svc := &Service{} + input, err := json.Marshal(canvasGenerationInput{Mode: "video", Config: providerConfig{BaseURL: "https://example.com", InterfaceType: string(model.ChannelInterfaceNewAPIChannel2)}}) + if err != nil { + t.Fatal(err) + } + base := model.Task{ID: "task-1", Type: "canvas_video", ProviderRequestID: "provider-task-1"} + if !svc.shouldDeferVideoProviderTask(base, string(input), context.DeadlineExceeded) { + t.Fatal("resumable NewAPI Channel 2 deadline should remain running") + } + if svc.shouldDeferVideoProviderTask(model.Task{ID: "task-2", Type: "canvas_video"}, string(input), context.DeadlineExceeded) { + t.Fatal("missing provider task id must not be deferred") + } + if svc.shouldDeferVideoProviderTask(base, string(input), context.Canceled) { + t.Fatal("explicit cancellation must not be deferred") + } + other, err := json.Marshal(canvasGenerationInput{Mode: "video", Config: providerConfig{BaseURL: "https://example.com", InterfaceType: string(model.ChannelInterfaceNewAPIVideo)}}) + if err != nil { + t.Fatal(err) + } + if svc.shouldDeferVideoProviderTask(base, string(other), context.DeadlineExceeded) { + t.Fatal("providers without a verified query-only resume contract must fail closed") + } +} + +func TestResumableVideoDeadlineUsesResolvedSystemChannelProtocol(t *testing.T) { + db, err := gorm.Open(sqlite.Open("file:"+newID()+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.ModelChannel{}, &model.ChannelModel{}); err != nil { + t.Fatal(err) + } + channel := model.ModelChannel{ + ID: "channel-video", UserID: "admin", Scope: model.ChannelScopeSystem, Enabled: true, Name: "Video", + BaseURL: "https://example.com", APIKey: "test-key", APIFormat: "openai", ModelsJSON: `["video-model"]`, + } + channelModel := model.ChannelModel{ + ID: "model-video", ChannelID: channel.ID, ModelKey: "video-model", Capability: "video", + Protocol: model.ChannelInterfaceNewAPIChannel2, Enabled: true, PriceConfigured: true, + } + if err := db.Create(&channel).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&channelModel).Error; err != nil { + t.Fatal(err) + } + input, err := json.Marshal(canvasGenerationInput{Mode: "video", Config: providerConfig{ChannelID: channel.ID, Model: "video-model"}}) + if err != nil { + t.Fatal(err) + } + svc := New(repository.New(db), t.TempDir()) + task := model.Task{ID: "task-system", Type: "canvas_video", ProviderRequestID: "provider-task-1"} + if !svc.shouldDeferVideoProviderTask(task, string(input), context.DeadlineExceeded) { + t.Fatal("system-channel task must use the resolved model protocol when deciding query-only recovery") + } +} + func TestRuntimePolicyDefaultsAndSelfUseModeValidate(t *testing.T) { if err := validateRuntimePolicy(defaultRuntimePolicy()); err != nil { t.Fatalf("default runtime policy error = %v", err) diff --git a/backend/internal/service/service.go b/backend/internal/service/service.go index dd93883c9..691ced9cc 100644 --- a/backend/internal/service/service.go +++ b/backend/internal/service/service.go @@ -990,6 +990,14 @@ func (s *Service) processClaimedTask(task *model.Task) error { return nil } if errors.Is(err, context.DeadlineExceeded) { + decryptedInput, decryptErr := s.decryptTaskInputJSON(task.InputJSON) + if decryptErr == nil && s.shouldDeferVideoProviderTask(*task, decryptedInput, err) { + if deferErr := s.repo.DeferRunningTaskForProviderPoll(task.ID, task.LeaseOwner, "后台仍在生成", 15*time.Second); deferErr != nil { + return deferErr + } + _ = s.log(task.UserID, task.ID, "info", "前台等待结束，上游视频仍在生成，将继续回查原任务", task.PollStage) + return nil + } err = errors.New(taskTimeoutMessage(task.Type)) } task.Status = model.TaskStatusFailed @@ -1079,7 +1087,7 @@ func taskExecutionTimeoutWithPolicy(taskType string, policy RuntimeTaskPolicy) t case taskType == "agent_storyboard" || taskType == "agent_storyboard_rows": return time.Duration(policy.StoryboardTimeoutMinutes) * time.Minute case strings.HasPrefix(taskType, "canvas_video") || strings.HasPrefix(taskType, "video_"): - return time.Duration(policy.VideoTimeoutMinutes) * time.Minute + return max(time.Duration(policy.VideoTimeoutMinutes)*time.Minute, 5*time.Minute) case strings.HasPrefix(taskType, "canvas_image"): return time.Duration(policy.ImageTimeoutMinutes) * time.Minute case strings.HasPrefix(taskType, "canvas_audio"): @@ -1091,6 +1099,18 @@ func taskExecutionTimeoutWithPolicy(taskType string, policy RuntimeTaskPolicy) t } } +func (s *Service) shouldDeferVideoProviderTask(task model.Task, decryptedInput string, err error) bool { + if !errors.Is(err, context.DeadlineExceeded) || strings.TrimSpace(task.ProviderRequestID) == "" || (!strings.HasPrefix(task.Type, "canvas_video") && !strings.HasPrefix(task.Type, "video_")) { + return false + } + var input canvasGenerationInput + if json.Unmarshal([]byte(decryptedInput), &input) != nil { + return false + } + resolved, resolveErr := s.resolveProviderConfig(input.Config) + return resolveErr == nil && resolved.InterfaceType == string(model.ChannelInterfaceNewAPIChannel2) +} + func taskTimeoutMessage(taskType string) string { if strings.HasPrefix(taskType, "canvas_video") || strings.HasPrefix(taskType, "video_") { return "视频生成等待超时，请稍后到任务中心查看或重试。" diff --git a/web/src/components/canvas/canvas-local-agent-panel.tsx b/web/src/components/canvas/canvas-local-agent-panel.tsx index 0693c5eea..7d42da2f4 100644 --- a/web/src/components/canvas/canvas-local-agent-panel.tsx +++ b/web/src/components/canvas/canvas-local-agent-panel.tsx @@ -10,7 +10,18 @@ import { createClientId } from "@/lib/client-id"; import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store"; import { useThemeStore } from "@/stores/use-theme-store"; import { useUserStore } from "@/stores/use-user-store"; -import { useCanvasAgentStore, type AgentAttachment, type AgentChatItem, type AgentEventLog, type AgentPanelTab, type AgentPendingToolCall, type AgentThreadSummary } from "@/stores/canvas/use-canvas-agent-store"; +import { + canvasAgentConnectionStatusText, + canvasAgentConnectionStartingPatch, + canvasAgentTransientDisconnectPatch, + useCanvasAgentStore, + type AgentAttachment, + type AgentChatItem, + type AgentEventLog, + type AgentPanelTab, + type AgentPendingToolCall, + type AgentThreadSummary, +} from "@/stores/canvas/use-canvas-agent-store"; import { previewCanvasAgentOps, summarizeCanvasAgentOps, type CanvasAgentOp, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops"; import { isProjectAgentReadTool, isProjectAgentToolName, runProjectAgentTool } from "@/services/api/project-agent-tools"; import { AgentChatComposer, AgentChatMessage, AgentPanelTabs, AgentPendingToolCard, AgentWorkingMessage, type CanvasAgentChatAttachment } from "./canvas-agent-chat-ui"; @@ -236,7 +247,7 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({ } errorLoggedRef.current = true; connectedRef.current = false; - clearAgentSession({ activity: wasConnected ? "正在重连" : "连接失败", connected: false, connectError: text }); + setAgentState(canvasAgentTransientDisconnectPatch(wasConnected ? "正在重连" : "连接失败", text)); await waitForCanvasRuntimeReconnect(controller.signal); } } @@ -402,28 +413,15 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({ if (connected) syncState(clientIdRef.current, restored); }; - const toggleAgentConnection = async () => { + const toggleAgentConnection = () => { if (enabled) { connectionControllerRef.current?.abort(); - clearAgentSession({ enabled: false, connected: false, activity: "离线", connectError: "" }); + pendingToolRef.current = null; + setAgentState({ enabled: false, connected: false, activity: "离线", connectError: "", waiting: false, sending: false, pendingTool: null }); return; } - const controller = new AbortController(); - connectionControllerRef.current = controller; - setAgentState({ connected: false, activity: "连接中", connectError: "", activeTab: "setup" }); - try { - await prepareCanvasRuntimeConnection(useLocalRuntimeStore, controller.signal); - if (controller.signal.aborted) return; - errorLoggedRef.current = false; - setAgentState({ enabled: true, connected: false, activity: "连接中", connectError: "", activeTab: "setup" }); - } catch (error) { - if (controller.signal.aborted) return; - const text = error instanceof Error ? error.message : "本机 Runtime 连接失败"; - setAgentState({ connectError: text }); - if (!headless) message.warning(text); - } finally { - if (connectionControllerRef.current === controller) connectionControllerRef.current = null; - } + errorLoggedRef.current = false; + setAgentState(canvasAgentConnectionStartingPatch()); }; useEffect(() => { @@ -432,21 +430,6 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({ void toggleAgentConnection(); }, [autoConnect, connected, enabled]); - function clearAgentSession(patch: Parameters[0] = {}) { - setAgentState({ - messages: [], - threads: [], - activeThreadId: "", - workspacePath: "", - loadingThreads: false, - waiting: false, - sending: false, - pendingTool: null, - ...patch, - }); - pendingToolRef.current = null; - } - const startNewThread = async () => { const projectId = snapshotRef.current.projectId; if (!connected || !projectId) return; @@ -786,8 +769,8 @@ function AgentConnectView({ connectError: string; onToggleEnabled: () => void; }) { - const statusText = connectError ? "连接失败" : connected ? activity : enabled ? "连接中" : "未连接"; - const statusColor = connectError ? "#dc2626" : connected ? "#16a34a" : enabled ? "#d97706" : theme.node.muted; + const statusText = canvasAgentConnectionStatusText({ enabled, connected, activity, connectError }); + const statusColor = statusText === "连接失败" ? "#dc2626" : connected ? "#16a34a" : enabled ? "#d97706" : theme.node.muted; return (
diff --git a/web/src/services/api/generation-task.ts b/web/src/services/api/generation-task.ts index eb4b08463..b2372c560 100644 --- a/web/src/services/api/generation-task.ts +++ b/web/src/services/api/generation-task.ts @@ -7,6 +7,7 @@ import { isLocalDreaminaBackgroundTask, localDreaminaTaskId, projectLocalDreamin import { modelCapabilityConfigFor } from "@/lib/model-capabilities"; import { grokImagePromptLimitError } from "@/lib/grok-image-prompt-limit"; import { resolveModelRequestConfig, type AiConfig } from "@/stores/use-config-store"; +import { useLocalDreaminaModelStore } from "@/stores/use-local-dreamina-model-store"; import type { ReferenceImage } from "@/types/image"; import type { ReferenceAudio, ReferenceVideo } from "@/types/media"; @@ -46,6 +47,7 @@ export type GenerationTaskDependencies = { runLocal: (input: LocalDreaminaGenerationInput, signal?: AbortSignal, onTaskUpdate?: (task: LocalDreaminaGenerationTask) => void) => ReturnType; createId: () => string; now: () => string; + ensureLocalDreaminaReady?: (signal?: AbortSignal) => Promise; }; const defaultDependencies: GenerationTaskDependencies = { @@ -54,6 +56,7 @@ const defaultDependencies: GenerationTaskDependencies = { runLocal: (input, signal, onTaskUpdate) => runLocalDreaminaGenerationTask(input, { onTaskUpdate }, signal), createId: () => crypto.randomUUID(), now: () => new Date().toISOString(), + ensureLocalDreaminaReady: (signal) => useLocalDreaminaModelStore.getState().ensureReady(signal), }; type PreparedGenerationReferences = { @@ -88,6 +91,8 @@ export async function runBackendGenerationTask( throwIfAborted(signal); assertClientPromptLimit(mode, prompt, config, metadata); if (isLocalDreaminaModel(config.model)) { + await dependencies.ensureLocalDreaminaReady?.(signal); + throwIfAborted(signal); return await runLocalDreaminaGeneration( { projectId, mode, prompt, config, referenceImages, referenceVideos, referenceAudios, mask, signal, metadata, onTaskUpdate, localIdempotencyKey, localResumeOnly, clientOperationId, retryOf, attemptGroupId }, dependencies, @@ -104,6 +109,8 @@ export async function runBackendGenerationTaskBatch(options: BackendGenerationTa assertClientPromptLimit(options.mode, options.prompt, options.config, options.metadata); if (options.retryContextsByBatchIndex && options.retryContextsByBatchIndex.length !== count) throw new Error("生成重试批次任务数量不匹配"); if (isLocalDreaminaModel(options.config.model)) { + await dependencies.ensureLocalDreaminaReady?.(options.signal); + throwIfAborted(options.signal); return Promise.allSettled( Array.from({ length: count }, (_, batchIndex) => { const retryContext = options.retryContextsByBatchIndex?.[batchIndex]; @@ -191,7 +198,8 @@ async function runLocalDreaminaGeneration(options: BackendGenerationTaskOptions, }, ); const completedAt = dependencies.now(); - options.onTaskUpdate?.({ ...task, status: "succeeded", progress: 100, stage: "local_cli_succeeded", resultJson: JSON.stringify(result), completedAt, updatedAt: completedAt }); + latestPublicTask = { ...latestPublicTask, status: "succeeded", progress: 100, stage: "local_cli_succeeded", resultJson: JSON.stringify(result), completedAt, updatedAt: completedAt }; + options.onTaskUpdate?.(latestPublicTask); return result; } catch (error) { const completedAt = dependencies.now(); @@ -200,7 +208,7 @@ async function runLocalDreaminaGeneration(options: BackendGenerationTaskOptions, const localErrorCode = error instanceof LocalDreaminaGenerationClientError ? error.code : undefined; if (!(cancelled && isLocalDreaminaBackgroundTask(latestPublicTask))) { options.onTaskUpdate?.({ - ...task, + ...latestPublicTask, status: cancelled ? "cancelled" : "failed", stage: cancelled ? "local_cli_cancelled" : "local_cli_failed", completedAt, diff --git a/web/src/stores/canvas/use-canvas-agent-store.ts b/web/src/stores/canvas/use-canvas-agent-store.ts index 88e4984e8..b9f8a0be2 100644 --- a/web/src/stores/canvas/use-canvas-agent-store.ts +++ b/web/src/stores/canvas/use-canvas-agent-store.ts @@ -35,10 +35,50 @@ type CanvasAgentStore = { clearEventLogs: () => void; }; +const CANVAS_AGENT_ENABLED_STORAGE_KEY = "canvas-agent-enabled"; + +type CanvasAgentPreferenceStorage = { + getItem(key: string): string | null; + setItem(key: string, value: string): void; +}; + +export function readCanvasAgentEnabledPreference(storage: Pick | undefined = browserPreferenceStorage()) { + try { + return storage?.getItem(CANVAS_AGENT_ENABLED_STORAGE_KEY) === "true"; + } catch { + return false; + } +} + +export function writeCanvasAgentEnabledPreference(enabled: boolean, storage: Pick | undefined = browserPreferenceStorage()) { + try { + storage?.setItem(CANVAS_AGENT_ENABLED_STORAGE_KEY, String(enabled)); + } catch { + // 浏览器隐私模式可能拒绝本地偏好写入；连接本身仍可继续。 + } +} + +export function canvasAgentConnectionStartingPatch() { + return { enabled: true, connected: false, activity: "连接中", connectError: "", activeTab: "setup" as const }; +} + +export function canvasAgentTransientDisconnectPatch(activity: string, connectError: string) { + return { enabled: true, connected: false, activity, connectError, waiting: false, sending: false }; +} + +export function canvasAgentConnectionStatusText({ enabled, connected, activity, connectError }: Pick) { + if (enabled && !connected && activity === "正在重连") return activity; + return connectError ? "连接失败" : connected ? activity : enabled ? "连接中" : "未连接"; +} + +function browserPreferenceStorage(): CanvasAgentPreferenceStorage | undefined { + return typeof window === "undefined" ? undefined : window.localStorage; +} + export const useCanvasAgentStore = create((set) => ({ width: typeof window === "undefined" ? 440 : Number(localStorage.getItem("canvas-agent-panel-width")) || 440, connected: false, - enabled: false, + enabled: readCanvasAgentEnabledPreference(), prompt: "", attachments: [], sending: false, @@ -54,7 +94,10 @@ export const useCanvasAgentStore = create((set) => ({ activity: "就绪", connectError: "", pendingTool: null, - setAgentState: (patch) => set(patch), + setAgentState: (patch) => { + if (typeof patch.enabled === "boolean") writeCanvasAgentEnabledPreference(patch.enabled); + set(patch); + }, addMessage: (item) => set((state) => ({ messages: [...state.messages.slice(-120), item] })), addEventLog: (item) => set((state) => ({ eventLogs: [...state.eventLogs.slice(-160), item] })), clearEventLogs: () => set({ eventLogs: [] }), diff --git a/web/src/stores/use-local-dreamina-model-store.ts b/web/src/stores/use-local-dreamina-model-store.ts index c19dbcfce..18b1b62bf 100644 --- a/web/src/stores/use-local-dreamina-model-store.ts +++ b/web/src/stores/use-local-dreamina-model-store.ts @@ -4,31 +4,103 @@ import { create } from "zustand"; import { getDreaminaModelCatalogSnapshotWithSessionRecovery, type DreaminaLocalModel } from "@/services/local-dreamina-model-catalog"; import { getLocalRuntimeSessionClient, useLocalRuntimeStore } from "@/stores/use-local-runtime-store"; -type State = { state: "idle" | "loading" | "ready" | "error"; models: DreaminaLocalModel[]; cacheScope?: string; sync(signal?: AbortSignal): Promise }; +type CatalogSnapshot = Awaited>; +type RuntimeCatalogState = { connection: string; modules: Array<{ id: string; scopes: string[] }> }; +type Dependencies = { + getRuntimeState(): RuntimeCatalogState; + getClient(): Parameters[0]; + loadSnapshot(client: Parameters[0], signal: AbortSignal): Promise; + requestTimeoutMs?: number; +}; +type State = { + state: "idle" | "loading" | "ready" | "error"; + models: DreaminaLocalModel[]; + cacheScope?: string; + sync(signal?: AbortSignal): Promise; + ensureReady(signal?: AbortSignal): Promise; +}; export const dreaminaModelCacheScopeKey = ({ accountBinding, sessionEpoch }: { accountBinding: string; sessionEpoch: string | number }) => `${accountBinding}:${sessionEpoch}`; -export const useLocalDreaminaModelStore = create((set, get) => ({ - state: "idle", - models: [], - cacheScope: undefined, - async sync(signal) { - const runtime = useLocalRuntimeStore.getState(); - const available = runtime.connection === "connected" && runtime.modules.some((module) => module.id === "dreamina" && module.scopes.includes("dreamina:models")); - if (!available) { - set({ state: "idle", models: [], cacheScope: undefined }); - return; - } - const client = getLocalRuntimeSessionClient(); - try { - const snapshot = await getDreaminaModelCatalogSnapshotWithSessionRecovery(client, signal); - const cacheScope = dreaminaModelCacheScopeKey(snapshot); - set({ state: "ready", models: snapshot.models, cacheScope }); - } catch { - if (!signal?.aborted) set({ state: "error", models: [], cacheScope: undefined }); - } - }, -})); +const DEFAULT_CATALOG_TIMEOUT_MS = 8_000; + +export function createLocalDreaminaModelStore(dependencies: Dependencies) { + let requestRevision = 0; + let activeRequest: { revision: number; controller: AbortController; promise: Promise } | null = null; + return create((set, get) => { + const load = (force: boolean, signal?: AbortSignal) => { + const runtime = dependencies.getRuntimeState(); + const available = runtime.connection === "connected" && runtime.modules.some((module) => module.id === "dreamina" && module.scopes.includes("dreamina:models")); + if (!available) { + requestRevision++; + activeRequest?.controller.abort(); + activeRequest = null; + set({ state: "idle", models: [], cacheScope: undefined }); + return Promise.reject(new Error("即梦本机模型目录尚未就绪")); + } + const current = get(); + if (!force && current.state === "ready" && current.models.length) return Promise.resolve(current.models); + if (force && activeRequest) { + requestRevision++; + activeRequest.controller.abort(); + activeRequest = null; + } + if (!activeRequest) { + set({ state: "loading" }); + const revision = ++requestRevision; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), Math.max(1, dependencies.requestTimeoutMs ?? DEFAULT_CATALOG_TIMEOUT_MS)); + const promise = dependencies + .loadSnapshot(dependencies.getClient(), controller.signal) + .then((snapshot) => { + if (revision === requestRevision) set({ state: "ready", models: snapshot.models, cacheScope: dreaminaModelCacheScopeKey(snapshot) }); + return snapshot.models; + }) + .catch((error) => { + if (revision === requestRevision) set({ state: "error", models: [], cacheScope: undefined }); + throw error; + }) + .finally(() => { + clearTimeout(timer); + if (activeRequest?.revision === revision) activeRequest = null; + }); + activeRequest = { revision, controller, promise }; + } + return waitForCatalog(activeRequest.promise, signal); + }; + return { + state: "idle", + models: [], + cacheScope: undefined, + async sync(signal) { + try { + await load(true, signal); + } catch { + // 页面引导读取失败只投影 store 状态；生成入口的 ensureReady 会明确失败。 + } + }, + ensureReady(signal) { + return load(false, signal); + }, + }; + }); +} + +function waitForCatalog(promise: Promise, signal?: AbortSignal): Promise { + if (!signal) return promise; + if (signal.aborted) return Promise.reject(new DOMException("Aborted", "AbortError")); + return new Promise((resolve, reject) => { + const abort = () => reject(new DOMException("Aborted", "AbortError")); + signal.addEventListener("abort", abort, { once: true }); + promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort)); + }); +} + +export const useLocalDreaminaModelStore = createLocalDreaminaModelStore({ + getRuntimeState: () => useLocalRuntimeStore.getState(), + getClient: getLocalRuntimeSessionClient, + loadSnapshot: (client, signal) => getDreaminaModelCatalogSnapshotWithSessionRecovery(client, signal), +}); export function useLocalDreaminaModelBootstrap() { const connection = useLocalRuntimeStore((state) => state.connection); diff --git a/web/test/canvas-local-runtime.test.ts b/web/test/canvas-local-runtime.test.ts index c504393d4..898b6951d 100644 --- a/web/test/canvas-local-runtime.test.ts +++ b/web/test/canvas-local-runtime.test.ts @@ -98,6 +98,73 @@ test("Canvas Agent UI store contains no Runtime endpoint, bearer, or session fie expect("session" in state).toBe(false); }); +test("Canvas Agent reconnect intent persists while transient disconnects preserve the active conversation", async () => { + const module = await import("../src/stores/canvas/use-canvas-agent-store").catch(() => ({})); + const connectionStartingPatch = ( + module as { + canvasAgentConnectionStartingPatch?: () => Record; + } + ).canvasAgentConnectionStartingPatch; + const transientDisconnectPatch = ( + module as { + canvasAgentTransientDisconnectPatch?: (activity: string, connectError: string) => Record; + } + ).canvasAgentTransientDisconnectPatch; + const writeEnabled = ( + module as { + writeCanvasAgentEnabledPreference?: (enabled: boolean, storage: { setItem(key: string, value: string): void }) => void; + } + ).writeCanvasAgentEnabledPreference; + const readEnabled = ( + module as { + readCanvasAgentEnabledPreference?: (storage: { getItem(key: string): string | null }) => boolean; + } + ).readCanvasAgentEnabledPreference; + + expect(typeof connectionStartingPatch).toBe("function"); + expect(typeof transientDisconnectPatch).toBe("function"); + expect(typeof writeEnabled).toBe("function"); + expect(typeof readEnabled).toBe("function"); + if (!connectionStartingPatch || !transientDisconnectPatch || !writeEnabled || !readEnabled) return; + + const durable = { + messages: [{ id: "message-1", role: "assistant", text: "kept" }], + threads: [{ id: "thread-1", preview: "kept" }], + activeThreadId: "thread-1", + workspacePath: "D:/workspace", + pendingTool: { requestId: "tool-1", name: "canvas_get_state" }, + }; + const starting = { ...durable, ...connectionStartingPatch() }; + expect(starting).toMatchObject({ enabled: true, connected: false, activity: "连接中", ...durable }); + + const reconnecting = { ...starting, ...transientDisconnectPatch("正在重连", "连接已断开") }; + expect(reconnecting).toMatchObject({ enabled: true, connected: false, activity: "正在重连", connectError: "连接已断开", ...durable }); + + const values = new Map(); + const storage = { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => values.set(key, value), + }; + writeEnabled(true, storage); + expect(readEnabled(storage)).toBe(true); + writeEnabled(false, storage); + expect(readEnabled(storage)).toBe(false); +}); + +test("Canvas Agent status shows reconnecting instead of a stale connection failure", async () => { + const module = await import("../src/stores/canvas/use-canvas-agent-store").catch(() => ({})); + const statusText = ( + module as { + canvasAgentConnectionStatusText?: (state: { enabled: boolean; connected: boolean; activity: string; connectError: string }) => string; + } + ).canvasAgentConnectionStatusText; + expect(typeof statusText).toBe("function"); + if (!statusText) return; + + expect(statusText({ enabled: true, connected: false, activity: "正在重连", connectError: "连接已断开" })).toBe("正在重连"); + expect(statusText({ enabled: true, connected: false, activity: "连接失败", connectError: "首次连接失败" })).toBe("连接失败"); +}); + test("Canvas connection reuses the shared Runtime store and requires the canvas module", async () => { const module = await import("../src/lib/canvas/local-runtime-connection"); const prepare = ( diff --git a/web/test/generation-task-local.test.ts b/web/test/generation-task-local.test.ts index 3b1ef7662..0fc2b12b1 100644 --- a/web/test/generation-task-local.test.ts +++ b/web/test/generation-task-local.test.ts @@ -335,6 +335,140 @@ test("shared Create and Canvas task projection exposes queued, submitted, genera expect(distinctUpdates[4]?.resultJson).toContain('"mode":"video"'); }); +test("the first local Dreamina submission waits for catalog readiness before invoking the paid Runtime path", async () => { + const order: string[] = []; + let release!: () => void; + const readyGate = new Promise((resolve) => { + release = resolve; + }); + const pending = runBackendGenerationTask( + { + mode: "video", + prompt: "first visit", + config: { ...defaultConfig, model: "local:dreamina-cli:seedance2.0mini", videoSeconds: "4", vquality: "720" }, + }, + { + createTask: async () => { + throw new Error("must not create backend task"); + }, + waitTask: async () => { + throw new Error("must not wait backend task"); + }, + ensureLocalDreaminaReady: async () => { + order.push("catalog-start"); + await readyGate; + order.push("catalog-ready"); + }, + runLocal: async () => { + order.push("paid-runtime"); + return { mode: "video", video: { dataUrl: "data:video/mp4;base64,AAAA", mimeType: "video/mp4", bytes: 3 } }; + }, + createId: () => "dreamina-first-ready-0001", + now: () => "2026-08-18T00:00:00.000Z", + }, + ); + + await Promise.resolve(); + expect(order).toEqual(["catalog-start"]); + release(); + await pending; + expect(order).toEqual(["catalog-start", "catalog-ready", "paid-runtime"]); +}); + +test("the first local Dreamina batch shares one catalog readiness gate before any paid Runtime call", async () => { + const order: string[] = []; + let release!: () => void; + const readyGate = new Promise((resolve) => { + release = resolve; + }); + const pending = runBackendGenerationTaskBatch( + { + mode: "image", + prompt: "first batch", + config: { ...defaultConfig, model: "local:dreamina-cli:5.0Pro", count: "1" }, + count: 3, + }, + { + createTask: async () => { + throw new Error("must not create backend task"); + }, + waitTask: async () => { + throw new Error("must not wait backend task"); + }, + ensureLocalDreaminaReady: async () => { + order.push("catalog-start"); + await readyGate; + order.push("catalog-ready"); + }, + runLocal: async () => { + order.push("paid-runtime"); + return { mode: "image", images: [{ dataUrl: "data:image/png;base64,AAAA", mimeType: "image/png", bytes: 3 }] }; + }, + createId: () => `dreamina-first-batch-${order.length}`, + now: () => "2026-08-18T00:00:00.000Z", + }, + ); + + await Promise.resolve(); + expect(order).toEqual(["catalog-start"]); + release(); + const settled = await pending; + expect(settled.every((entry) => entry.status === "fulfilled")).toBe(true); + expect(order).toEqual(["catalog-start", "catalog-ready", "paid-runtime", "paid-runtime", "paid-runtime"]); +}); + +test("local Dreamina terminal projection keeps the latest receipt, observation, and durable outputs", async () => { + const updates: GenerationTask[] = []; + const timestamp = "2026-08-18T00:00:00.000Z"; + await runBackendGenerationTask( + { + mode: "video", + prompt: "late result", + config: { ...defaultConfig, model: "local:dreamina-cli:seedance2.0mini", videoSeconds: "4", vquality: "720" }, + onTaskUpdate: (task) => updates.push(task), + }, + { + createTask: async () => { + throw new Error("must not create backend task"); + }, + waitTask: async () => { + throw new Error("must not wait backend task"); + }, + runLocal: async (input, _signal, onTaskUpdate) => { + onTaskUpdate?.({ + id: input.idempotencyKey!, + provider: "dreamina-cli", + mode: "video", + operation: "text2video", + model: "seedance2.0mini", + status: "succeeded", + stage: "succeeded", + progress: 100, + receiptRecorded: true, + officialStatus: "completed", + providerObservation: { source: "query_result", status: "completed", observedAt: timestamp }, + outputs: [{ outputIndex: 0, mediaType: "video", materializedAssetId: "asset-late-0001" }], + result: { mode: "video", video: { dataUrl: "data:video/mp4;base64,AAAA", mimeType: "video/mp4", bytes: 3 } }, + createdAt: timestamp, + updatedAt: timestamp, + }); + return { mode: "video", video: { dataUrl: "data:video/mp4;base64,AAAA", mimeType: "video/mp4", bytes: 3 } }; + }, + createId: () => "dreamina-latest-context-0001", + now: () => timestamp, + }, + ); + + expect(updates.at(-1)).toMatchObject({ + status: "succeeded", + stage: "local_cli_succeeded", + receiptRecorded: true, + officialStatus: "completed", + outputs: [{ outputIndex: 0, mediaType: "video", materializedAssetId: "asset-late-0001" }], + }); + expect(updates.at(-1)?.resultJson).toContain('"mode":"video"'); +}); + test("an explicitly cancelled Dreamina task stays cancelled instead of being projected as failed", async () => { const updates: Array<{ status: string; stage?: string }> = []; const timestamp = "2026-08-12T00:00:00.000Z"; diff --git a/web/test/local-dreamina-model-catalog.test.ts b/web/test/local-dreamina-model-catalog.test.ts index dcc29dce9..62e278394 100644 --- a/web/test/local-dreamina-model-catalog.test.ts +++ b/web/test/local-dreamina-model-catalog.test.ts @@ -213,6 +213,128 @@ test("effective config projects an asynchronously arriving Dreamina catalog with expect(defaultConfig.channels.some((channel) => channel.id === "local:dreamina-cli")).toBe(false); }); +test("Dreamina catalog readiness is loading immediately and shares one in-flight bootstrap", async () => { + const module = await import("../src/stores/use-local-dreamina-model-store").catch(() => ({})); + const createStore = ( + module as { + createLocalDreaminaModelStore?: (dependencies: Record) => { + getState(): { + state: string; + models: Array<{ id: string }>; + ensureReady(signal?: AbortSignal): Promise>; + }; + }; + } + ).createLocalDreaminaModelStore; + expect(typeof createStore).toBe("function"); + if (!createStore) return; + + let requests = 0; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const store = createStore({ + getRuntimeState: () => ({ connection: "connected", modules: [{ id: "dreamina", scopes: ["dreamina:models"] }] }), + getClient: () => ({ request: async () => catalogResponse() }), + loadSnapshot: async () => { + requests += 1; + await gate; + return { + accountBinding: "a".repeat(64), + sessionEpoch: 3, + models: [ + { + provider: "dreamina-cli", + id: "seedance2.0mini", + displayName: "seedance2.0mini", + modality: "video", + operations: ["text-to-video"], + adapterSupported: true, + accountEntitlement: "unknown", + currentlyObservedAvailable: "unknown", + settings: { aliases: [], aspects: ["16:9"], maxReferenceImages: 9, minDuration: 4, maxDuration: 15, tiers: ["720p"] }, + source: "runtime-execution-contract", + }, + ], + }; + }, + }); + + const first = store.getState().ensureReady(); + const second = store.getState().ensureReady(); + expect(store.getState().state).toBe("loading"); + expect(requests).toBe(1); + release(); + const [firstModels, secondModels] = await Promise.all([first, second]); + expect(firstModels.map((item) => item.id)).toEqual(["seedance2.0mini"]); + expect(secondModels).toEqual(firstModels); + expect(store.getState()).toMatchObject({ state: "ready", cacheScope: `${"a".repeat(64)}:3` }); + expect(requests).toBe(1); +}); + +test("Dreamina catalog timeout releases a hung shared request so generation can recover without reload", async () => { + const { createLocalDreaminaModelStore } = await import("../src/stores/use-local-dreamina-model-store"); + let requests = 0; + const store = createLocalDreaminaModelStore({ + getRuntimeState: () => ({ connection: "connected", modules: [{ id: "dreamina", scopes: ["dreamina:models"] }] }), + getClient: () => ({ request: async () => catalogResponse() }), + requestTimeoutMs: 10, + loadSnapshot: async (_client, signal) => { + requests += 1; + if (requests > 1) return catalogSnapshot("seedance-recovered", 4); + if (!signal) return await new Promise>(() => undefined); + return await new Promise>((_resolve, reject) => { + signal.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError")), { once: true }); + }); + }, + }); + + const caller = new AbortController(); + const first = store.getState().ensureReady(caller.signal); + caller.abort(); + await expect(first).rejects.toMatchObject({ name: "AbortError" }); + await Bun.sleep(20); + + const recovered = await Promise.race([ + store.getState().ensureReady(), + Bun.sleep(40).then(() => "stuck" as const), + ]); + expect(recovered).not.toBe("stuck"); + expect(recovered).toMatchObject([{ id: "seedance-recovered" }]); + expect(store.getState()).toMatchObject({ state: "ready", cacheScope: `${"a".repeat(64)}:4` }); + expect(requests).toBe(2); +}); + +test("Dreamina catalog forced resync ignores a late result from the previous Runtime scope", async () => { + const { createLocalDreaminaModelStore } = await import("../src/stores/use-local-dreamina-model-store"); + let requests = 0; + let releaseOld!: (snapshot: ReturnType) => void; + const oldRequest = new Promise>((resolve) => { + releaseOld = resolve; + }); + const store = createLocalDreaminaModelStore({ + getRuntimeState: () => ({ connection: "connected", modules: [{ id: "dreamina", scopes: ["dreamina:models"] }] }), + getClient: () => ({ request: async () => catalogResponse() }), + requestTimeoutMs: 100, + loadSnapshot: async () => { + requests += 1; + return requests === 1 ? oldRequest : catalogSnapshot("seedance-current", 5); + }, + }); + + const oldSync = store.getState().sync(); + const currentSync = store.getState().sync(); + const currentResult = await Promise.race([currentSync.then(() => "ready" as const), Bun.sleep(40).then(() => "stuck" as const)]); + expect(currentResult).toBe("ready"); + expect(requests).toBe(2); + expect(store.getState()).toMatchObject({ state: "ready", cacheScope: `${"a".repeat(64)}:5`, models: [{ id: "seedance-current" }] }); + + releaseOld(catalogSnapshot("seedance-stale", 3)); + await oldSync; + expect(store.getState()).toMatchObject({ state: "ready", cacheScope: `${"a".repeat(64)}:5`, models: [{ id: "seedance-current" }] }); +}); + test("effective config removes custom channels when administrators disable them", async () => { const { createModelChannel, effectiveConfigForCustomChannels, normalizeConfigSnapshot } = await import("../src/stores/use-config-store"); const config = normalizeConfigSnapshot({ @@ -264,3 +386,24 @@ function catalogResponse() { { status: 200, headers: { "content-type": "application/json" } }, ); } + +function catalogSnapshot(id: string, sessionEpoch: number) { + return { + accountBinding: "a".repeat(64), + sessionEpoch, + models: [ + { + provider: "dreamina-cli" as const, + id, + displayName: id, + modality: "video" as const, + operations: ["text-to-video" as const], + adapterSupported: true, + accountEntitlement: "unknown" as const, + currentlyObservedAvailable: "unknown" as const, + settings: { aliases: [], aspects: ["16:9"], maxReferenceImages: 9, minDuration: 4, maxDuration: 15, tiers: ["720p"] }, + source: "runtime-execution-contract" as const, + }, + ], + }; +} From f62cc5ef3ea11262e281a32689327148af5cf21d Mon Sep 17 00:00:00 2001 From: ddCat <2695092045@qq.com> Date: Tue, 18 Aug 2026 15:43:25 +0800 Subject: [PATCH 3/4] =?UTF-8?q?style(web):=20=E5=8D=B3=E6=A2=A6=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E7=9B=AE=E5=BD=95=E6=B5=8B=E8=AF=95=20-=20=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=E6=B5=8B=E8=AF=95=E4=BB=A3=E7=A0=81=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/test/local-dreamina-model-catalog.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/web/test/local-dreamina-model-catalog.test.ts b/web/test/local-dreamina-model-catalog.test.ts index 62e278394..f6ea34712 100644 --- a/web/test/local-dreamina-model-catalog.test.ts +++ b/web/test/local-dreamina-model-catalog.test.ts @@ -296,10 +296,7 @@ test("Dreamina catalog timeout releases a hung shared request so generation can await expect(first).rejects.toMatchObject({ name: "AbortError" }); await Bun.sleep(20); - const recovered = await Promise.race([ - store.getState().ensureReady(), - Bun.sleep(40).then(() => "stuck" as const), - ]); + const recovered = await Promise.race([store.getState().ensureReady(), Bun.sleep(40).then(() => "stuck" as const)]); expect(recovered).not.toBe("stuck"); expect(recovered).toMatchObject([{ id: "seedance-recovered" }]); expect(store.getState()).toMatchObject({ state: "ready", cacheScope: `${"a".repeat(64)}:4` }); From 474f1e94ab464e57c08c1e0af20906b2924fda95 Mon Sep 17 00:00:00 2001 From: ddCat <2695092045@qq.com> Date: Tue, 18 Aug 2026 20:27:26 +0800 Subject: [PATCH 4/4] =?UTF-8?q?fix(canvas):=20=E5=AF=BC=E6=BC=94=E5=8F=B0?= =?UTF-8?q?=20-=20=E4=BF=AE=E5=A4=8D=E9=AA=A8=E9=AA=BC=E5=91=BD=E4=B8=AD?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E6=94=B6=E7=AA=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/canvas/director/director-viewport.tsx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/web/src/components/canvas/director/director-viewport.tsx b/web/src/components/canvas/director/director-viewport.tsx index b6dbd308c..1c2e19603 100644 --- a/web/src/components/canvas/director/director-viewport.tsx +++ b/web/src/components/canvas/director/director-viewport.tsx @@ -207,18 +207,22 @@ function DirectorModel({ object, selected, selectedBone, playhead, onSelectBone, const modelUrl = object.kind === "actor" || object.primitive === "character" ? DIRECTOR_DEFAULT_ACTOR_URL : object.url; const handleModelPointerDown = useCallback((event: ThreeEvent) => { if (!selected || !rig) return; - let nearest: { bone: string; distance: number } | null = null; + let nearestBone: string | null = null; + let nearestDistance = Number.POSITIVE_INFINITY; Object.entries(rig.boneMap).forEach(([bone, name]) => { if (!name) return; const target = model?.getObjectByName(name); if (!target) return; const distance = event.ray.distanceSqToPoint(target.getWorldPosition(new Vector3())); - if (distance <= 0.12 ** 2 && (!nearest || distance < nearest.distance)) nearest = { bone, distance }; + if (distance <= 0.12 ** 2 && distance < nearestDistance) { + nearestBone = bone; + nearestDistance = distance; + } }); - if (!nearest) return; + if (!nearestBone) return; // 模型表面可能先于关节控制球被射线命中，用最近骨骼保证点击仍可选中。 event.stopPropagation(); - onSelectBone(nearest.bone); + onSelectBone(nearestBone); }, [model, onSelectBone, rig, selected]); useEffect(() => { onActorRigReadyRef.current = onActorRigReady; }, [onActorRigReady]);