diff --git a/web/src/components/canvas/director/canvas-director-workbench.tsx b/web/src/components/canvas/director/canvas-director-workbench.tsx index 70d86738b..8226b0ff6 100644 --- a/web/src/components/canvas/director/canvas-director-workbench.tsx +++ b/web/src/components/canvas/director/canvas-director-workbench.tsx @@ -1,18 +1,21 @@ -import { App, Button, ColorPicker, Input, InputNumber, Select, Slider, Switch } from "antd"; -import { Box, BoxSelect, Camera, Circle, Cuboid, FileUp, Focus, Image as ImageIcon, LampDesk, Lightbulb, Move3D, Pause, Play, Plus, Redo2, Rotate3D, Save, Scaling, Trash2, Undo2, UserRound, Video, X } from "lucide-react"; +import { App, Button, ColorPicker, Input, InputNumber, Select, Switch } from "antd"; +import { Box, BoxSelect, Camera, Circle, Cuboid, FileUp, Focus, Image as ImageIcon, LampDesk, Lightbulb, Plus, Redo2, Save, Trash2, Undo2, UserRound, Video, X } from "lucide-react"; import { useCallback, useEffect, useMemo, useRef, useState, type ReactElement, type ReactNode } from "react"; import { nanoid } from "nanoid"; +import type { AnimationClip } from "three"; import { DirectorViewport, type DirectorViewportHandle } from "@/components/canvas/director/director-viewport"; +import { DirectorViewportDock } from "@/components/canvas/director/director-viewport-dock"; +import { DirectorSequencer } from "@/components/canvas/director/director-sequencer"; import { canvasThemes } from "@/lib/canvas-theme"; import { compileDirectorPrompt } from "@/lib/canvas/director/director-prompt-compiler"; -import { createDirectorBillboard, createDirectorCamera, createDirectorLight, createDirectorModel, createDirectorObject, touchDirectorScene, upsertDirectorKeyframe } from "@/lib/canvas/director/director-scene"; +import { createDirectorActor, createDirectorBillboard, createDirectorCamera, createDirectorLight, createDirectorModel, createDirectorObject, DIRECTOR_ACTOR_COLORS, directorBoneLabel, directorPoseLabel, touchDirectorScene, upsertDirectorBoneKeyframe, upsertDirectorKeyframe } from "@/lib/canvas/director/director-scene"; import { uploadMediaFile } from "@/services/file-storage"; import { useAssetStore, type ModelAsset } from "@/stores/use-asset-store"; import { useDirectorWorkbenchStore } from "@/stores/canvas/use-director-workbench-store"; import { useThemeStore } from "@/stores/use-theme-store"; import type { CanvasNodeData } from "@/types/canvas"; -import type { DirectorCamera, DirectorCameraMove, DirectorLight, DirectorObject, DirectorScene, DirectorSceneOutput, DirectorShot, DirectorShotSize, DirectorTransform, DirectorVec3 } from "@/types/director"; +import type { DirectorCamera, DirectorCameraMove, DirectorHumanoidBone, DirectorLight, DirectorObject, DirectorPose, DirectorQuat, DirectorRig, DirectorScene, DirectorSceneOutput, DirectorShot, DirectorShotSize, DirectorTransform, DirectorVec3 } from "@/types/director"; export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onChange, onApply }: { open: boolean; scene: DirectorScene | null; imageNodes: CanvasNodeData[]; onClose: () => void; onChange: (scene: DirectorScene) => void; onApply: (output: DirectorSceneOutput) => Promise }) { const { message } = App.useApp(); @@ -23,18 +26,25 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh const [history, setHistory] = useState([]); const [future, setFuture] = useState([]); const [saving, setSaving] = useState(false); + const [recording, setRecording] = useState(false); const selectedObjectId = useDirectorWorkbenchStore((state) => state.selectedObjectId); const selectedLightId = useDirectorWorkbenchStore((state) => state.selectedLightId); const transformMode = useDirectorWorkbenchStore((state) => state.transformMode); const renderMode = useDirectorWorkbenchStore((state) => state.renderMode); const playhead = useDirectorWorkbenchStore((state) => state.playhead); const playing = useDirectorWorkbenchStore((state) => state.playing); + const selectedBone = useDirectorWorkbenchStore((state) => state.selectedBone); + const autoKey = useDirectorWorkbenchStore((state) => state.autoKey); + const sequencerHeight = useDirectorWorkbenchStore((state) => state.sequencerHeight); const setSelectedObjectId = useDirectorWorkbenchStore((state) => state.setSelectedObjectId); const setSelectedLightId = useDirectorWorkbenchStore((state) => state.setSelectedLightId); const setTransformMode = useDirectorWorkbenchStore((state) => state.setTransformMode); const setRenderMode = useDirectorWorkbenchStore((state) => state.setRenderMode); const setPlayhead = useDirectorWorkbenchStore((state) => state.setPlayhead); const setPlaying = useDirectorWorkbenchStore((state) => state.setPlaying); + const setSelectedBone = useDirectorWorkbenchStore((state) => state.setSelectedBone); + const setAutoKey = useDirectorWorkbenchStore((state) => state.setAutoKey); + const setSequencerHeight = useDirectorWorkbenchStore((state) => state.setSequencerHeight); const resetWorkbench = useDirectorWorkbenchStore((state) => state.reset); const assets = useAssetStore((state) => state.assets); const addAsset = useAssetStore((state) => state.addAsset); @@ -42,7 +52,9 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh useEffect(() => { if (!open || !scene) return; - setDraft(structuredClone(scene)); + const next = structuredClone(scene); + next.shots = next.shots.map((shot) => ({ ...shot, fps: shot.fps || 24 })); + setDraft(next); setHistory([]); setFuture([]); resetWorkbench(); @@ -105,6 +117,13 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh setSelectedObjectId(object.id); }; + const addActor = () => { + const actorCount = draft?.objects.filter((item) => item.kind === "actor").length || 0; + const actor = createDirectorActor(`演员 ${actorCount + 1}`, [actorCount * 0.8, 0, 0], DIRECTOR_ACTOR_COLORS[actorCount % DIRECTOR_ACTOR_COLORS.length]); + commit((current) => ({ ...current, objects: [...current.objects, actor] })); + setSelectedObjectId(actor.id); + }; + const addModelAsset = (asset: ModelAsset) => { const object = createDirectorModel({ name: asset.title, assetId: asset.id, storageKey: asset.data.storageKey, url: asset.data.url, mimeType: asset.data.mimeType }); commit((current) => ({ ...current, objects: [...current.objects, object] })); @@ -141,7 +160,7 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh const addShot = () => { if (!activeCamera) return; - const shot: DirectorShot = { id: nanoid(), name: `镜头 ${(draft?.shots.length || 0) + 1}`, cameraId: activeCamera.id, duration: 5, shotSize: "medium", cameraMove: "static", prompt: "" }; + const shot: DirectorShot = { id: nanoid(), name: `镜头 ${(draft?.shots.length || 0) + 1}`, cameraId: activeCamera.id, duration: 5, fps: 24, shotSize: "medium", cameraMove: "static", prompt: "" }; commit((current) => ({ ...current, shots: [...current.shots, shot], activeShotId: shot.id })); setPlayhead(0); }; @@ -156,6 +175,46 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh commit((current) => ({ ...current, cameras: current.cameras.map((item) => item.id === activeCamera.id ? { ...item, keyframes: upsertDirectorKeyframe(item.keyframes, playhead, item.transform) } : item) })); }; + const recordSelectedKeyframe = () => { + if (selectedObject && selectedBone) { + const rotation = selectedObject.boneOverrides?.[selectedBone as DirectorHumanoidBone] || [0, 0, 0, 1] as DirectorQuat; + updateObject(selectedObject.id, { boneTracks: upsertDirectorBoneKeyframe(selectedObject.boneTracks || [], selectedBone as DirectorHumanoidBone, playhead, rotation) }); + return; + } + if (selectedObject) addObjectKeyframe(); + else addCameraKeyframe(); + }; + + const handleObjectTransform = useCallback((id: string, transform: DirectorTransform) => { + commit((current) => ({ + ...current, + objects: current.objects.map((item) => item.id === id ? { ...item, transform, keyframes: autoKey ? upsertDirectorKeyframe(item.keyframes, playhead, transform) : item.keyframes } : item), + })); + }, [autoKey, commit, playhead]); + + const handleBoneTransform = useCallback((id: string, bone: string, rotation: DirectorQuat) => { + commit((current) => ({ + ...current, + objects: current.objects.map((item) => item.id === id ? { + ...item, + boneOverrides: { ...item.boneOverrides, [bone]: rotation }, + boneTracks: autoKey ? upsertDirectorBoneKeyframe(item.boneTracks || [], bone as DirectorHumanoidBone, playhead, rotation) : item.boneTracks, + } : item), + })); + }, [autoKey, commit, playhead]); + + const handleActorRigReady = useCallback((id: string, rig: DirectorRig, animations: AnimationClip[]) => { + replaceWithoutHistory((current) => ({ + ...current, + objects: current.objects.map((item) => { + if (item.id !== id) return item; + const existing = item.motionClips || []; + const motionClips = existing.length ? existing : animations.map((clip) => ({ id: nanoid(), name: clip.name || "动作片段", sourceAnimation: clip.name, start: 0, duration: Math.max(0.1, clip.duration), playbackRate: 1, loop: true })); + return { ...item, rig, motionClips }; + }), + })); + }, [replaceWithoutHistory]); + const applyCameraMove = () => { if (!activeCamera || !activeShot) return; const start = activeCamera.transform; @@ -177,14 +236,12 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh setSaving(true); try { const beauty = await viewportRef.current.capture("beauty"); - const depth = await viewportRef.current.capture("depth"); - const normal = await viewportRef.current.capture("normal"); const prompt = compileDirectorPrompt(draft, activeShot); const next = touchDirectorScene(draft); setDraft(next); onChange(next); - await onApply({ scene: next, shot: activeShot, prompt, beauty, depth, normal }); - message.success("镜头已回写画布"); + await onApply({ scene: next, shot: activeShot, prompt, beauty }); + message.success("导演台构图已回写画布"); } catch (error) { message.error(error instanceof Error ? error.message : "导演台输出失败"); } finally { @@ -192,6 +249,29 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh } }; + const exportClayVideo = async () => { + if (!draft || !activeShot || !viewportRef.current || recording) return; + setRecording(true); + const wasPlaying = playing; + const previousPlayhead = playhead; + setPlayhead(0); + setPlaying(true); + try { + await new Promise((resolve) => requestAnimationFrame(() => resolve())); + const clayVideo = await viewportRef.current.recordVideo(activeShot.duration, activeShot.fps); + const next = touchDirectorScene(draft); + onChange(next); + await onApply({ scene: next, shot: activeShot, prompt: compileDirectorPrompt(next, activeShot), beauty: await viewportRef.current.capture("beauty"), clayVideo, clayVideoMimeType: clayVideo.type }); + message.success("白膜视频已回写画布"); + } catch (error) { + message.error(error instanceof Error ? error.message : "白膜视频导出失败"); + } finally { + setPlaying(wasPlaying); + setPlayhead(previousPlayhead); + setRecording(false); + } + }; + if (!open || !draft || !activeShot) return null; return ( @@ -202,12 +282,9 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh - - setTransformMode("translate")}> - setTransformMode("rotate")}> - setTransformMode("scale")}>
- +
@@ -216,7 +293,7 @@ export function CanvasDirectorWorkbench({ open, scene, imageNodes, onClose, onCh
- updateObject(id, { transform })} /> +
{activeShot.name} · {activeCamera?.name || "无摄影机"} · {activeShot.duration}s
+ addPrimitive("box", "立方体")} onAddLight={addLight} onAddCamera={addCamera} onAlignCamera={alignCameraToView} />
-
-
- setPlaying(!playing)}>{playing ? : } - {playhead.toFixed(1)}s - - -
-
- {draft.shots.map((shot, index) => )} - -
-
+ setPlaying(!playing)} onPlayheadChange={setPlayhead} onAutoKeyChange={setAutoKey} onHeightChange={setSequencerHeight} onSelectObject={setSelectedObjectId} onSelectBone={setSelectedBone} onRecordKeyframe={recordSelectedKeyframe} onAddShot={addShot} onSelectShot={(id) => { commit((current) => ({ ...current, activeShotId: id })); setPlayhead(0); }} /> ); } -function ObjectInspector({ object, playhead, onUpdate, onAddKeyframe, onDelete }: { object: DirectorObject; playhead: number; onUpdate: (patch: Partial) => void; onAddKeyframe: () => void; onDelete: () => void }) { - return onUpdate({ name })} onDelete={onDelete}> onUpdate({ transform })} /> onUpdate({ color })} />{object.primitive === "character" ? ({ label: clip.name, value: clip.id }))]} onChange={(activeMotionClipId) => onUpdate({ activeMotionClipId: activeMotionClipId || undefined })} />{activeMotionClip ?
updateActiveMotion({ playbackRate: playbackRate || 1 })} /> updateActiveMotion({ loop })} />
: null} :
模型加载后会显示可用动作 Clip
} + {mappedBones.length ? onUpdate({ type })} /> onUpdate({ transform: { ...light.transform, position } })} /> onUpdate({ color })} /> onUpdate({ intensity: value || 0 })} /> onUpdate({ castShadow })} />; } -function ShotInspector({ shot, camera, cameras, onUpdateShot, onUpdateCamera, onAddCameraKeyframe, onApplyCameraMove, onAlignCameraToView }: { shot: DirectorShot; camera: DirectorCamera | null; cameras: DirectorScene["cameras"]; onUpdateShot: (patch: Partial) => void; onUpdateCamera: (patch: Partial) => void; onAddCameraKeyframe: () => void; onApplyCameraMove: () => void; onAlignCameraToView: () => void }) { - return onUpdateShot({ name })}> onUpdateShot({ shotSize })} /> ({ label: item.name, value: item.id }))} onChange={(cameraId) => onUpdateShot({ cameraId })} /> +
({ label: `${fps} fps`, value: fps }))} onChange={(fps: 24 | 25 | 30) => onUpdateShot({ fps })} />
+ onSelectShot(event.target.value)}> + {scene.shots.map((item, index) => )} + + + + + + + + + + + + +
+
+ +
轨道
+
+ {ticks.map((tick) => {tick}s)} + +
+ + undefined}> + + + undefined}> + + + {camera ? { onSelectObject(null); onSelectBone(null); }}> + + + : null} + {actorObjects.map((object) => { + const isExpanded = expanded[object.id] ?? object.id === selectedObjectId; + const activeClip = object.motionClips?.find((clip) => clip.id === object.activeMotionClipId); + const boneTrackKeys = object.boneTracks?.flatMap((track) => track.keyframes.map((key) => ({ id: `${track.bone}-${key.id}`, time: key.time, color: "#f0b36a" }))) || []; + const transformKeys = object.keyframes.map((key) => ({ id: key.id, time: key.time, color: "#61d2ad" })); + return
+ : } selected={selectedObjectId === object.id && !selectedBone} keys={transformKeys} onClick={() => { onSelectObject(object.id); onSelectBone(null); setExpanded((current) => ({ ...current, [object.id]: !isExpanded })); }}> + + + {isExpanded && showDetails ? <> + onSelectObject(object.id)}> + + + onSelectObject(object.id)}> + + + {object.boneTracks?.map((track) => { + const keys = track.keyframes.map((key) => ({ id: key.id, time: key.time, color: "#f0b36a" })); + return { onSelectObject(object.id); onSelectBone(track.bone); }}>; + })} + : null} +
; + })} + {objects.filter((object) => !actorObjects.some((actor) => actor.id === object.id)).map((object) => { + const keys = object.keyframes.map((key) => ({ id: key.id, time: key.time, color: "#b8c0ca" })); + return { onSelectObject(object.id); onSelectBone(null); }}>; + })} +
+
+ + ); +} + +function SequencerRow({ label, icon, selected, indent, children, onClick }: { label: string; icon: ReactNode; selected: boolean; indent?: boolean; keys?: TrackKey[]; children: ReactNode; onClick: () => void }) { + return
+ +
{children}
+
; +} + +function TrackKeys({ duration, keys }: { duration: number; keys: TrackKey[] }) { + return
{keys.map((key) => )}
; +} + +function TrackBar({ duration, color, label, start = 0, clipDuration }: { duration: number; color: string; label: string; start?: number; clipDuration?: number }) { + return
{label}
; +} + +function formatTime(time: number) { + return `${time.toFixed(2)}s`; +} diff --git a/web/src/components/canvas/director/director-viewport-dock.tsx b/web/src/components/canvas/director/director-viewport-dock.tsx new file mode 100644 index 000000000..b3fe09782 --- /dev/null +++ b/web/src/components/canvas/director/director-viewport-dock.tsx @@ -0,0 +1,49 @@ +import { Tooltip } from "antd"; +import { Bone, Box, Camera, Crosshair, Lightbulb, Move3D, Palette, Rotate3D, Scaling, UserRound } from "lucide-react"; +import type { ReactNode } from "react"; + +import type { DirectorRenderMode } from "@/types/director"; + +type DirectorViewportDockProps = { + transformMode: "translate" | "rotate" | "scale"; + renderMode: DirectorRenderMode; + onTransformModeChange: (mode: DirectorViewportDockProps["transformMode"]) => void; + onRenderModeChange: (mode: DirectorRenderMode) => void; + onAddActor: () => void; + onAddBox: () => void; + onAddLight: () => void; + onAddCamera: () => void; + onAlignCamera: () => void; +}; + +export function DirectorViewportDock({ transformMode, renderMode, onTransformModeChange, onRenderModeChange, onAddActor, onAddBox, onAddLight, onAddCamera, onAlignCamera }: DirectorViewportDockProps) { + return ( + + ); +} + +function DockButton({ label, active, children, onClick }: { label: string; active?: boolean; children: ReactNode; onClick: () => void }) { + return ( + + + + ); +} + +function DockDivider() { + return ; +} diff --git a/web/src/components/canvas/director/director-viewport.tsx b/web/src/components/canvas/director/director-viewport.tsx index 20b52041e..f2bb78be4 100644 --- a/web/src/components/canvas/director/director-viewport.tsx +++ b/web/src/components/canvas/director/director-viewport.tsx @@ -1,34 +1,42 @@ -import { Canvas, useThree } from "@react-three/fiber"; +import { Canvas, useFrame, useThree } from "@react-three/fiber"; import { Grid, OrbitControls, TransformControls } from "@react-three/drei"; -import { forwardRef, Suspense, useEffect, useImperativeHandle, useMemo, useRef, useState } from "react"; -import { Box3, Color, Group, Mesh, MeshDepthMaterial, MeshNormalMaterial, Object3D, PerspectiveCamera, Scene, Texture, TextureLoader, Vector3, WebGLRenderer } from "three"; -import { GLTFLoader } from "three-stdlib"; +import { forwardRef, Suspense, useCallback, useEffect, useImperativeHandle, useMemo, useRef, useState } from "react"; +import { AnimationClip, AnimationMixer, Box3, Bone, Color, Group, LoopOnce, LoopRepeat, Mesh, MeshBasicMaterial, MeshDepthMaterial, MeshNormalMaterial, MeshStandardMaterial, Object3D, PerspectiveCamera, Quaternion, Scene, SkeletonHelper, Texture, TextureLoader, Vector3, WebGLRenderer } from "three"; +import type { Material } from "three"; +import { GLTFLoader, SkeletonUtils } from "three-stdlib"; -import { interpolateDirectorTransform } from "@/lib/canvas/director/director-scene"; +import { directorPoseBoneDeltas, interpolateDirectorBoneRotation, interpolateDirectorTransform } from "@/lib/canvas/director/director-scene"; import { resolveMediaUrl } from "@/services/file-storage"; -import type { DirectorCamera, DirectorLight, DirectorObject, DirectorRenderMode, DirectorScene, DirectorTransform } from "@/types/director"; +import type { DirectorCamera, DirectorHumanoidBone, DirectorLight, DirectorObject, DirectorQuat, DirectorRenderMode, DirectorRig, DirectorScene, DirectorTransform, DirectorVec3 } from "@/types/director"; export type DirectorViewportHandle = { capture: (mode: DirectorRenderMode) => Promise; + recordVideo: (duration: number, fps: number) => Promise; readCameraTransform: () => DirectorTransform | null; }; type DirectorViewportProps = { scene: DirectorScene; selectedObjectId: string | null; + selectedBone: string | null; transformMode: "translate" | "rotate" | "scale"; renderMode: DirectorRenderMode; playhead: number; onSelectObject: (id: string | null) => void; + onSelectBone: (bone: string | null) => void; onObjectTransform: (id: string, transform: DirectorTransform) => void; + onBoneTransform: (id: string, bone: string, rotation: DirectorQuat) => void; + onActorRigReady: (id: string, rig: DirectorRig, animations: AnimationClip[]) => void; }; -type CaptureContext = { gl: WebGLRenderer; scene: Scene; camera: PerspectiveCamera }; +type CaptureContext = { gl: WebGLRenderer; scene: Scene; camera: PerspectiveCamera; suspendDisplayMaterialOverride: () => () => void }; export const DirectorViewport = forwardRef(function DirectorViewport(props, ref) { const captureContext = useRef(null); + const onCaptureContext = useCallback((context: CaptureContext) => { captureContext.current = context; }, []); useImperativeHandle(ref, () => ({ capture: (mode) => captureFrame(captureContext.current, mode), + recordVideo: (duration, fps) => recordCanvas(captureContext.current, duration, fps), readCameraTransform: () => { const camera = captureContext.current?.camera; return camera ? { position: camera.position.toArray() as DirectorTransform["position"], rotation: [camera.rotation.x, camera.rotation.y, camera.rotation.z], scale: [1, 1, 1] } : null; @@ -45,21 +53,30 @@ export const DirectorViewport = forwardRef props.onSelectObject(null)} > - (captureContext.current = context)} /> + ); }); -function DirectorSceneContent({ scene, selectedObjectId, transformMode, renderMode, playhead, onSelectObject, onObjectTransform, onCaptureContext }: DirectorViewportProps & { onCaptureContext: (context: CaptureContext) => void }) { +function DirectorSceneContent({ scene, selectedObjectId, selectedBone, transformMode, renderMode, playhead, onSelectObject, onSelectBone, onObjectTransform, onBoneTransform, onActorRigReady, onCaptureContext }: DirectorViewportProps & { onCaptureContext: (context: CaptureContext) => void }) { const { gl, camera, scene: threeScene, invalidate } = useThree(); const [transforming, setTransforming] = useState(false); + const displayClayRestoreRef = useRef<(() => void) | null>(null); const shot = scene.shots.find((item) => item.id === scene.activeShotId) || scene.shots[0]; const activeCamera = scene.cameras.find((item) => item.id === shot?.cameraId) || scene.cameras[0]; + const suspendDisplayMaterialOverride = useCallback(() => { + const suspended = Boolean(displayClayRestoreRef.current); + displayClayRestoreRef.current?.(); + displayClayRestoreRef.current = null; + return () => { + if (suspended) displayClayRestoreRef.current = applyClaySceneMaterials(threeScene); + }; + }, [threeScene]); useEffect(() => { - onCaptureContext({ gl, camera: camera as PerspectiveCamera, scene: threeScene }); - }, [camera, gl, onCaptureContext, threeScene]); + onCaptureContext({ gl, camera: camera as PerspectiveCamera, scene: threeScene, suspendDisplayMaterialOverride }); + }, [camera, gl, onCaptureContext, suspendDisplayMaterialOverride, threeScene]); useEffect(() => { threeScene.background = new Color(scene.background); @@ -67,14 +84,17 @@ function DirectorSceneContent({ scene, selectedObjectId, transformMode, renderMo }, [invalidate, scene.background, threeScene]); useEffect(() => { - const material = renderMode === "depth" ? new MeshDepthMaterial() : renderMode === "normal" ? new MeshNormalMaterial() : null; + const material = renderMode === "depth" ? new MeshDepthMaterial() : renderMode === "normal" ? new MeshNormalMaterial() : renderMode === "pose" ? new MeshBasicMaterial({ color: "#ffffff", wireframe: true }) : null; + if (renderMode === "clay") displayClayRestoreRef.current = applyClaySceneMaterials(threeScene); threeScene.overrideMaterial = material; invalidate(); return () => { if (threeScene.overrideMaterial === material) threeScene.overrideMaterial = null; + displayClayRestoreRef.current?.(); + displayClayRestoreRef.current = null; material?.dispose(); }; - }, [invalidate, renderMode, threeScene]); + }, [invalidate, renderMode, scene.objects, threeScene]); return ( <> @@ -91,11 +111,15 @@ function DirectorSceneContent({ scene, selectedObjectId, transformMode, renderMo key={object.id} object={object} selected={selectedObjectId === object.id} + selectedBone={selectedObjectId === object.id ? selectedBone : null} transformMode={transformMode} playhead={playhead} onSelect={() => onSelectObject(object.id)} + onSelectBone={(bone) => { onSelectObject(object.id); onSelectBone(bone); }} onTransforming={setTransforming} onTransform={(transform) => onObjectTransform(object.id, transform)} + onBoneTransform={(bone, rotation) => onBoneTransform(object.id, bone, rotation)} + onActorRigReady={(rig, animations) => onActorRigReady(object.id, rig, animations)} /> ))} @@ -121,7 +145,7 @@ function CameraSync({ camera, playhead }: { camera?: DirectorCamera; playhead: n return null; } -function DirectorObjectView({ object, selected, transformMode, playhead, onSelect, onTransforming, onTransform }: { object: DirectorObject; selected: boolean; transformMode: DirectorViewportProps["transformMode"]; playhead: number; onSelect: () => void; onTransforming: (value: boolean) => void; onTransform: (transform: DirectorTransform) => void }) { +function DirectorObjectView({ object, selected, selectedBone, transformMode, playhead, onSelect, onSelectBone, onTransforming, onTransform, onBoneTransform, onActorRigReady }: { object: DirectorObject; selected: boolean; selectedBone: string | null; transformMode: DirectorViewportProps["transformMode"]; playhead: number; onSelect: () => void; onSelectBone: (bone: string | null) => void; onTransforming: (value: boolean) => void; onTransform: (transform: DirectorTransform) => void; onBoneTransform: (bone: string, rotation: DirectorQuat) => void; onActorRigReady: (rig: DirectorRig, animations: AnimationClip[]) => void }) { const groupRef = useRef(null); const transform = interpolateDirectorTransform(object.transform, object.keyframes, playhead); const content = ( @@ -135,7 +159,7 @@ function DirectorObjectView({ object, selected, transformMode, playhead, onSelec onSelect(); }} > - + ); if (!selected) return content; @@ -156,8 +180,8 @@ function DirectorObjectView({ object, selected, transformMode, playhead, onSelec ); } -function DirectorObjectVisual({ object, selected }: { object: DirectorObject; selected: boolean }) { - if (object.kind === "model" && object.url) return ; +function DirectorObjectVisual({ object, selected, selectedBone, playhead, onSelectBone, onBoneTransform, onActorRigReady }: { object: DirectorObject; selected: boolean; selectedBone: string | null; playhead: number; onSelectBone: (bone: string | null) => void; onBoneTransform: (bone: string, rotation: DirectorQuat) => void; onActorRigReady: (rig: DirectorRig, animations: AnimationClip[]) => void }) { + if ((object.kind === "model" || object.kind === "actor") && object.url) return ; if (object.kind === "billboard" && object.url) return ; if (object.primitive === "character") return ; const material = ; @@ -188,35 +212,210 @@ function Limb({ position, rotation, color, length = 0.68 }: { position: [number, return ; } -function DirectorModel({ object, selected }: { object: DirectorObject; selected: boolean }) { +function DirectorModel({ object, selected, selectedBone, playhead, onSelectBone, onBoneTransform, onActorRigReady }: { object: DirectorObject; selected: boolean; selectedBone: string | null; playhead: number; onSelectBone: (bone: string | null) => void; onBoneTransform: (bone: string, rotation: DirectorQuat) => void; onActorRigReady: (rig: DirectorRig, animations: AnimationClip[]) => void }) { const [model, setModel] = useState(null); + const [animations, setAnimations] = useState([]); + const [rig, setRig] = useState(null); + const [restRotations, setRestRotations] = useState>>({}); + const mixerRef = useRef(null); + const onActorRigReadyRef = useRef(onActorRigReady); + const invalidate = useThree((state) => state.invalidate); + const helper = useMemo(() => model ? new SkeletonHelper(model) : null, [model]); + const selectedBoneObject = selectedBone && rig?.boneMap[selectedBone as DirectorHumanoidBone] ? model?.getObjectByName(rig.boneMap[selectedBone as DirectorHumanoidBone]!) : null; + const motion = object.motionClips?.find((item) => item.id === object.activeMotionClipId); + const activeAnimation = motion ? animations.find((item) => item.name === motion.sourceAnimation) : undefined; + + useEffect(() => { onActorRigReadyRef.current = onActorRigReady; }, [onActorRigReady]); + useEffect(() => { let active = true; const loader = new GLTFLoader(); void resolveMediaUrl(object.storageKey, object.url).then((url) => loader.load(url, (gltf) => { if (!active) return; - const next = gltf.scene.clone(true); - const bounds = new Box3().setFromObject(next); - const size = bounds.getSize(new Vector3()); - const maxSize = Math.max(size.x, size.y, size.z, 0.001); - next.scale.multiplyScalar(2 / maxSize); - const centered = new Box3().setFromObject(next); - const center = centered.getCenter(new Vector3()); - next.position.sub(center); - next.position.y -= centered.min.y - center.y; - next.traverse((child) => { - const mesh = child as Mesh; - if (!mesh.isMesh) return; - mesh.castShadow = object.castShadow; - mesh.receiveShadow = object.receiveShadow; - }); - applyHumanoidPose(next, object.pose || "stand"); + const next = SkeletonUtils.clone(gltf.scene); + normalizeModel(next, object.castShadow, object.receiveShadow); + const nextRig = inferDirectorRig(next, gltf.animations.map((clip) => clip.name)); + if (object.kind === "actor") applyActorReferenceMaterial(next, object.color); + mixerRef.current = new AnimationMixer(next); + setRig(nextRig); + setRestRotations(readRigRestRotations(next, nextRig)); + setAnimations(gltf.animations); setModel(next); + onActorRigReadyRef.current(nextRig, gltf.animations); }, undefined, () => active && setModel(null))); - return () => { active = false; }; - }, [object.castShadow, object.pose, object.receiveShadow, object.storageKey, object.url]); - if (!model) return ; - return ; + return () => { + active = false; + mixerRef.current?.stopAllAction(); + mixerRef.current = null; + }; + }, [object.kind, object.storageKey, object.url]); + + useEffect(() => { + if (!model) return; + model.traverse((child) => { + const mesh = child as Mesh; + if (!mesh.isMesh) return; + mesh.castShadow = object.castShadow; + mesh.receiveShadow = object.receiveShadow; + }); + invalidate(); + }, [invalidate, model, object.castShadow, object.receiveShadow]); + + useEffect(() => { + if (!model || object.kind !== "actor") return; + updateActorReferenceColor(model, object.color); + invalidate(); + }, [invalidate, model, object.color, object.kind]); + + useEffect(() => { + if (!model || !mixerRef.current) return; + const mixer = mixerRef.current; + mixer.stopAllAction(); + if (!activeAnimation) return; + mixer.clipAction(activeAnimation).setLoop(motion?.loop ? LoopRepeat : LoopOnce, motion?.loop ? Infinity : 1).play(); + return () => mixer.stopAllAction(); + }, [activeAnimation, model, motion?.loop]); + + useEffect(() => { + if (!model || !mixerRef.current) return; + if (activeAnimation && motion) { + const localTime = Math.max(0, playhead - motion.start) * motion.playbackRate; + const clipDuration = motion.duration || activeAnimation.duration; + mixerRef.current.setTime(motion.loop && clipDuration > 0 ? localTime % clipDuration : localTime); + } + applyDirectorBoneTracks(model, object, playhead, rig, restRotations, Boolean(activeAnimation && motion)); + helper?.updateMatrixWorld(true); + invalidate(); + }, [activeAnimation, helper, invalidate, model, motion, object.boneOverrides, object.boneTracks, object.pose, playhead, restRotations, rig]); + + useFrame(() => { + if (selectedBoneObject && selected) selectedBoneObject.updateMatrixWorld(true); + }); + + if (!model) return ; + return + + {selected && helper ? : null} + {selected && rig ? Object.entries(rig.boneMap).filter(([, name]) => Boolean(name)).map(([bone, name]) => onSelectBone(bone)} />) : null} + {selected && selectedBoneObject ? { event.stopPropagation(); }} onMouseUp={() => onBoneTransform(selectedBone!, selectedBoneObject.quaternion.toArray() as DirectorQuat)} /> : null} + ; +} + +function DirectorMannequin({ color, selected }: { color: string; selected: boolean }) { + const resolvedColor = selected ? new Color(color).lerp(new Color("#78a9ff"), 0.18).getStyle() : color; + const joints: DirectorVec3[] = [[0, 1.72, 0], [0, 1.48, 0], [-0.3, 1.4, 0], [0.3, 1.4, 0], [-0.32, 1.05, 0], [0.32, 1.05, 0], [-0.33, 0.72, 0], [0.33, 0.72, 0], [-0.13, 0.88, 0], [0.13, 0.88, 0], [-0.13, 0.46, 0], [0.13, 0.46, 0], [-0.13, 0.05, 0], [0.13, 0.05, 0]]; + const bones: Array<[DirectorVec3, DirectorVec3]> = [[joints[0], joints[1]], [joints[1], joints[2]], [joints[1], joints[3]], [joints[2], joints[4]], [joints[4], joints[6]], [joints[3], joints[5]], [joints[5], joints[7]], [joints[1], [0, 0.92, 0]], [[0, 0.92, 0], joints[8]], [[0, 0.92, 0], joints[9]], [joints[8], joints[10]], [joints[10], joints[12]], [joints[9], joints[11]], [joints[11], joints[13]]]; + return + {bones.map(([from, to], index) => )} + {joints.map((position, index) => )} + ; +} + +function LoadingBone({ from, to, color }: { from: DirectorVec3; to: DirectorVec3; color: string }) { + const start = useMemo(() => new Vector3(...from), [from]); + const end = useMemo(() => new Vector3(...to), [to]); + const direction = useMemo(() => end.clone().sub(start), [end, start]); + const midpoint = useMemo(() => start.clone().add(end).multiplyScalar(0.5), [end, start]); + const rotation = useMemo(() => new Quaternion().setFromUnitVectors(new Vector3(0, 1, 0), direction.clone().normalize()), [direction]); + return ; +} + +function BoneController({ bone, model, selected, onSelect }: { bone: Object3D | undefined; model: Object3D; selected: boolean; onSelect: () => void }) { + const ref = useRef(null); + const world = useMemo(() => new Vector3(), []); + useFrame(() => { + if (!bone || !ref.current) return; + bone.getWorldPosition(world); + model.worldToLocal(world); + ref.current.position.copy(world); + }); + if (!bone) return null; + return { event.stopPropagation(); onSelect(); }}>; +} + +function normalizeModel(root: Object3D, castShadow: boolean, receiveShadow: boolean) { + root.updateMatrixWorld(true); + const bounds = new Box3().setFromObject(root, true); + const size = bounds.getSize(new Vector3()); + const maxSize = Math.max(size.x, size.y, size.z, 0.001); + root.scale.multiplyScalar(2 / maxSize); + root.updateMatrixWorld(true); + const centered = new Box3().setFromObject(root, true); + const center = centered.getCenter(new Vector3()); + root.position.sub(center); + root.position.y -= centered.min.y - center.y; + root.traverse((child) => { + const mesh = child as Mesh; + if (!mesh.isMesh) return; + mesh.castShadow = castShadow; + mesh.receiveShadow = receiveShadow; + }); +} + +function applyActorReferenceMaterial(root: Object3D, color: string) { + const material = new MeshStandardMaterial({ color, roughness: 0.74, metalness: 0.02 }); + root.traverse((child) => { + const mesh = child as Mesh; + if (!mesh.isMesh) return; + mesh.material = material; + mesh.userData.directorActor = true; + mesh.userData.directorActorMaterial = material; + }); +} + +function updateActorReferenceColor(root: Object3D, color: string) { + root.traverse((child) => { + const mesh = child as Mesh; + if (!mesh.isMesh || !mesh.userData.directorActor) return; + const materials = Array.isArray(mesh.material) ? mesh.material : [mesh.material]; + materials.forEach((material) => { + if (material instanceof MeshStandardMaterial) material.color.set(color); + }); + }); +} + +function readRigRestRotations(root: Object3D, rig: DirectorRig) { + return Object.fromEntries(Object.entries(rig.boneMap).flatMap(([bone, name]) => { + const target = name ? root.getObjectByName(name) : null; + return target ? [[bone, target.quaternion.toArray() as DirectorQuat]] : []; + })) as Partial>; +} + +function inferDirectorRig(root: Object3D, animationNames: string[]): DirectorRig { + const names = new Map(); + root.traverse((child) => { if (child instanceof Bone) names.set(normalizeBoneName(child.name), child.name); }); + const patterns: Record = { + root: [/^root$/, /armature/], hips: [/hips|pelvis/, /mixamorig.*hip/], spine: [/spine1?$|lowerback/], chest: [/spine2|chest|upperback/], neck: [/neck/], head: [/head/], + leftShoulder: [/leftshoulder|shoulder_l|mixamorigleftshoulder/], leftUpperArm: [/leftupperarm|leftarm|upperarm_l|mixamorigleftarm/], leftLowerArm: [/leftforearm|leftlowerarm|forearm_l|mixamorigleftforearm/], leftHand: [/lefthand|hand_l|mixamoriglefthand/], + rightShoulder: [/rightshoulder|shoulder_r|mixamorigrightshoulder/], rightUpperArm: [/rightupperarm|rightarm|upperarm_r|mixamorigrightarm/], rightLowerArm: [/rightforearm|rightlowerarm|forearm_r|mixamorigrightforearm/], rightHand: [/righthand|hand_r|mixamorigrighthand/], + leftUpperLeg: [/leftupleg|leftthigh|thigh_l|mixamorigleftupleg/], leftLowerLeg: [/leftleg|leftcalf|calf_l|mixamorigleftleg/], leftFoot: [/leftfoot|foot_l|mixamorigleftfoot/], rightUpperLeg: [/rightupleg|rightthigh|thigh_r|mixamorigrightupleg/], rightLowerLeg: [/rightleg|rightcalf|calf_r|mixamorigrightleg/], rightFoot: [/rightfoot|foot_r|mixamorigrightfoot/], + }; + const boneMap = Object.fromEntries(Object.entries(patterns).map(([bone, candidates]) => [bone, candidates.map((pattern) => [...names.entries()].find(([normalized]) => pattern.test(normalized))?.[1]).find(Boolean)]).filter(([, name]) => Boolean(name))) as DirectorRig["boneMap"]; + return { status: Object.keys(boneMap).length >= 8 ? "ready" : "unmapped", boneMap, animationNames }; +} + +function normalizeBoneName(name: string) { + return name.toLowerCase().replace(/[^a-z0-9]/g, ""); +} + +function applyDirectorBoneTracks(model: Object3D, object: DirectorObject, playhead: number, rig: DirectorRig | null, restRotations: Partial>, hasActiveMotion: boolean) { + if (!rig) return; + const poseDeltas = hasActiveMotion ? {} : directorPoseBoneDeltas(object.pose || "stand"); + Object.entries(rig.boneMap).forEach(([bone, name]) => { + const target = name ? model.getObjectByName(name) : null; + if (!target) return; + const humanoidBone = bone as DirectorHumanoidBone; + if (!hasActiveMotion) { + const rest = restRotations[humanoidBone]; + if (rest) target.quaternion.copy(new Quaternion(...rest)); + const delta = poseDeltas[humanoidBone]; + if (delta) target.quaternion.multiply(new Quaternion(...delta)); + } + const override = object.boneOverrides?.[bone as DirectorHumanoidBone]; + const track = object.boneTracks?.find((item) => item.bone === bone); + const rotation = track ? interpolateDirectorBoneRotation(override || target.quaternion.toArray() as DirectorQuat, track.keyframes, playhead) : override; + if (rotation) target.quaternion.copy(new Quaternion(...rotation)); + }); } function DirectorBillboard({ object, selected }: { object: DirectorObject; selected: boolean }) { @@ -246,42 +445,73 @@ function characterPose(pose: NonNullable): Record<"leftA if (pose === "walk") return { leftArm: [0.35, 0, 0.18], rightArm: [-0.35, 0, -0.18], leftLeg: [-0.24, 0, 0.08], rightLeg: [0.24, 0, -0.08] }; if (pose === "run") return { leftArm: [0.7, 0, 0.35], rightArm: [-0.7, 0, -0.35], leftLeg: [-0.55, 0, 0.12], rightLeg: [0.55, 0, -0.12] }; if (pose === "sit") return { leftArm: [0.15, 0, 0.05], rightArm: [0.15, 0, -0.05], leftLeg: [1.1, 0, 0], rightLeg: [1.1, 0, 0] }; - if (pose === "action") return { leftArm: [1.1, 0.2, 0.5], rightArm: [-0.7, -0.2, -0.5], leftLeg: [-0.35, 0, 0.2], rightLeg: [0.45, 0, -0.15] }; + if (pose === "fight" || pose === "throw" || pose === "push") return { leftArm: [1.1, 0.2, 0.5], rightArm: [-0.7, -0.2, -0.5], leftLeg: [-0.35, 0, 0.2], rightLeg: [0.45, 0, -0.15] }; return { leftArm: [0, 0, 0.08], rightArm: [0, 0, -0.08], leftLeg: [0, 0, 0], rightLeg: [0, 0, 0] }; } -function applyHumanoidPose(root: Object3D, pose: NonNullable) { - const rotations = humanoidBoneRotations(pose); - root.traverse((child) => { - const name = child.name.toLowerCase().replace(/[^a-z0-9]/g, ""); - const entry = rotations.find(([pattern]) => pattern.test(name)); - if (entry) child.rotation.set(...entry[1]); - }); -} - -function humanoidBoneRotations(pose: NonNullable): Array<[RegExp, [number, number, number]]> { - const values = characterPose(pose); - return [ - [/left(upper)?arm|upperarmleft|mixamorigleftarm/, values.leftArm], - [/right(upper)?arm|upperarmright|mixamorigrightarm/, values.rightArm], - [/left(up)?leg|upperlegleft|mixamorigleftupleg/, values.leftLeg], - [/right(up)?leg|upperlegright|mixamorigrightupleg/, values.rightLeg], - [/spine|chest/, pose === "action" ? [0.08, 0.18, -0.08] : [0, 0, 0]], - ]; -} - async function captureFrame(context: CaptureContext | null, mode: DirectorRenderMode) { if (!context) throw new Error("3D 视口尚未就绪"); const { gl, scene, camera } = context; + const resumeDisplayMaterialOverride = context.suspendDisplayMaterialOverride(); const previous = scene.overrideMaterial; - const override = mode === "depth" ? new MeshDepthMaterial() : mode === "normal" ? new MeshNormalMaterial() : null; - scene.overrideMaterial = override; - gl.render(scene, camera); - const blob = await canvasToBlob(gl.domElement); - scene.overrideMaterial = previous; - override?.dispose(); - gl.render(scene, camera); - return blob; + const override = mode === "depth" ? new MeshDepthMaterial() : mode === "normal" ? new MeshNormalMaterial() : mode === "pose" ? new MeshBasicMaterial({ color: "#ffffff", wireframe: true }) : null; + const restoreClayMaterials = mode === "clay" ? applyClaySceneMaterials(scene) : null; + try { + scene.overrideMaterial = override; + gl.render(scene, camera); + return await canvasToBlob(gl.domElement); + } finally { + scene.overrideMaterial = previous; + restoreClayMaterials?.(); + override?.dispose(); + resumeDisplayMaterialOverride(); + gl.render(scene, camera); + } +} + +async function recordCanvas(context: CaptureContext | null, duration: number, fps: number) { + if (!context) throw new Error("3D 视口尚未就绪"); + if (!context.gl.domElement.captureStream || typeof MediaRecorder === "undefined") throw new Error("当前浏览器不支持视频录制，请导出帧序列"); + const resumeDisplayMaterialOverride = context.suspendDisplayMaterialOverride(); + const previousMaterial = context.scene.overrideMaterial; + const restoreClayMaterials = applyClaySceneMaterials(context.scene); + context.scene.overrideMaterial = null; + context.gl.render(context.scene, context.camera); + const stream = context.gl.domElement.captureStream(fps); + const mimeType = ["video/webm;codecs=vp9", "video/webm;codecs=vp8", "video/webm"].find((type) => MediaRecorder.isTypeSupported(type)) || ""; + const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); + const chunks: Blob[] = []; + const result = new Promise((resolve, reject) => { + recorder.ondataavailable = (event) => { if (event.data.size) chunks.push(event.data); }; + recorder.onerror = () => reject(new Error("白膜视频录制失败")); + recorder.onstop = () => resolve(new Blob(chunks, { type: recorder.mimeType || "video/webm" })); + }); + recorder.start(); + window.setTimeout(() => recorder.stop(), Math.max(250, duration * 1000 + 120)); + try { + return await result; + } finally { + stream.getTracks().forEach((track) => track.stop()); + restoreClayMaterials(); + context.scene.overrideMaterial = previousMaterial; + resumeDisplayMaterialOverride(); + context.gl.render(context.scene, context.camera); + } +} + +function applyClaySceneMaterials(scene: Scene) { + const clayMaterial = new MeshStandardMaterial({ color: "#d6d9dd", roughness: 0.88, metalness: 0 }); + const originals: Array<{ mesh: Mesh; material: Material | Material[] }> = []; + scene.traverse((child) => { + const mesh = child as Mesh; + if (!mesh.isMesh || mesh.userData.directorActor) return; + originals.push({ mesh, material: mesh.material }); + mesh.material = clayMaterial; + }); + return () => { + originals.forEach(({ mesh, material }) => { mesh.material = material; }); + clayMaterial.dispose(); + }; } function canvasToBlob(canvas: HTMLCanvasElement) { diff --git a/web/src/lib/canvas/director/director-prompt-compiler.ts b/web/src/lib/canvas/director/director-prompt-compiler.ts index ffacebf2d..42834169b 100644 --- a/web/src/lib/canvas/director/director-prompt-compiler.ts +++ b/web/src/lib/canvas/director/director-prompt-compiler.ts @@ -1,4 +1,5 @@ import type { DirectorCamera, DirectorObject, DirectorScene, DirectorShot } from "@/types/director"; +import { directorColorLabel, directorPoseLabel } from "@/lib/canvas/director/director-scene"; const shotSizeLabels: Record = { extreme_wide: "大远景", @@ -25,10 +26,12 @@ const cameraMoveLabels: Record = { export function compileDirectorPrompt(scene: DirectorScene, shot: DirectorShot) { const camera = scene.cameras.find((item) => item.id === shot.cameraId) || scene.cameras[0]; const visibleObjects = scene.objects.filter((item) => item.visible); + const actors = visibleObjects.filter((item) => item.kind === "actor"); return [ shot.prompt.trim(), `镜头设计：${shotSizeLabels[shot.shotSize]}，${cameraMoveLabels[shot.cameraMove]}，时长 ${formatNumber(shot.duration)} 秒。`, camera ? cameraPrompt(camera) : "", + actors.length ? `角色颜色映射：${actors.map((actor) => `${directorColorLabel(actor.color)}人偶（${actor.color}）代表${actor.name}`).join("；")}。生成视频时严格按颜色识别角色，不交换人物身份。` : "", visibleObjects.length ? `空间调度：${visibleObjects.map(objectPrompt).join("；")}。` : "", `灯光：${scene.lights.map((light) => `${light.name}${formatNumber(light.intensity)}强度${light.color}`).join("，")}。`, "保持角色、道具、空间方向、光线方向和镜头轴线连续，遵循真实摄影机透视与物理遮挡。", @@ -48,12 +51,9 @@ function cameraPrompt(camera: DirectorCamera) { function objectPrompt(object: DirectorObject) { const [x, y, z] = object.transform.position; - const pose = object.pose ? `，姿势${poseLabel(object.pose)}` : ""; - return `${object.name}位于 (${formatNumber(x)}, ${formatNumber(y)}, ${formatNumber(z)})${pose}`; -} - -function poseLabel(pose: NonNullable) { - return { neutral: "自然", stand: "站立", walk: "行走", run: "奔跑", sit: "坐姿", action: "动作姿态" }[pose]; + const pose = object.pose ? `，姿势${directorPoseLabel(object.pose)}` : ""; + const color = object.kind === "actor" ? `，${directorColorLabel(object.color)}参考人偶` : ""; + return `${object.name}${color}位于 (${formatNumber(x)}, ${formatNumber(y)}, ${formatNumber(z)})${pose}`; } function formatNumber(value: number) { diff --git a/web/src/lib/canvas/director/director-scene.ts b/web/src/lib/canvas/director/director-scene.ts index c1b37df85..e35fe932e 100644 --- a/web/src/lib/canvas/director/director-scene.ts +++ b/web/src/lib/canvas/director/director-scene.ts @@ -1,6 +1,10 @@ import { nanoid } from "nanoid"; +import { Color, Euler, Quaternion } from "three"; -import type { DirectorCamera, DirectorKeyframe, DirectorLight, DirectorObject, DirectorScene, DirectorTransform, DirectorVec3 } from "@/types/director"; +import type { DirectorBoneKeyframe, DirectorBoneTrack, DirectorCamera, DirectorHumanoidBone, DirectorKeyframe, DirectorLight, DirectorObject, DirectorPose, DirectorQuat, DirectorScene, DirectorTransform, DirectorVec3 } from "@/types/director"; + +export const DIRECTOR_DEFAULT_ACTOR_URL = "https://cdn.jsdelivr.net/gh/mrdoob/three.js@r185/examples/models/gltf/Soldier.glb"; +export const DIRECTOR_ACTOR_COLORS = ["#f1f3f5", "#202329", "#2f7de1", "#d84949", "#dfae3f", "#34a276"] as const; export const directorIdentityTransform = (position: DirectorVec3 = [0, 0, 0]): DirectorTransform => ({ position, rotation: [0, 0, 0], scale: [1, 1, 1] }); @@ -16,12 +20,12 @@ export function createDirectorScene(title = "未命名场景"): DirectorScene { environmentIntensity: 0.7, gridVisible: true, objects: [ - createDirectorObject("character", "演员", [0, 0.9, 0], "#9aa6b2"), + createDirectorActor("演员 1", [0, 0, 0]), createDirectorObject("box", "前景道具", [1.8, 0.45, 0.8], "#8b7355"), ], cameras: [camera], lights: [createDirectorLight("directional", "主光", [4, 6, 4], 2.4), createDirectorLight("directional", "轮廓光", [-4, 3, -2], 1.1), createDirectorLight("ambient", "环境光", [0, 0, 0], 0.65)], - shots: [{ id: shotId, name: "镜头 1", cameraId: camera.id, duration: 5, shotSize: "medium", cameraMove: "static", prompt: "" }], + shots: [{ id: shotId, name: "镜头 1", cameraId: camera.id, duration: 5, fps: 24, shotSize: "medium", cameraMove: "static", prompt: "" }], activeShotId: shotId, createdAt: now, updatedAt: now, @@ -44,6 +48,21 @@ export function createDirectorObject(primitive: DirectorObject["primitive"] = "b }; } +export function createDirectorActor(name = "演员", position: DirectorVec3 = [0, 0, 0], color = DIRECTOR_ACTOR_COLORS[0]): DirectorObject { + return { + ...createDirectorObject("box", name, position, color), + kind: "actor", + primitive: undefined, + url: DIRECTOR_DEFAULT_ACTOR_URL, + mimeType: "model/gltf-binary", + pose: "stand", + rig: { status: "unmapped", boneMap: {}, animationNames: [] }, + motionClips: [], + boneOverrides: {}, + boneTracks: [], + }; +} + export function createDirectorModel(input: Pick): DirectorObject { return { ...createDirectorObject("box", input.name, [0, 0, 0]), ...input, kind: "model", primitive: undefined }; } @@ -70,19 +89,99 @@ export function upsertDirectorKeyframe(keyframes: DirectorKeyframe[], time: numb return next.toSorted((a, b) => a.time - b.time); } +export function upsertDirectorBoneKeyframe(tracks: DirectorBoneTrack[], bone: DirectorHumanoidBone, time: number, rotation: DirectorQuat) { + const track = tracks.find((item) => item.bone === bone); + const nextKeyframes = upsertBoneKeyframe(track?.keyframes || [], time, rotation); + return track ? tracks.map((item) => item.bone === bone ? { ...item, keyframes: nextKeyframes } : item) : [...tracks, { bone, keyframes: nextKeyframes }]; +} + +function upsertBoneKeyframe(keyframes: DirectorBoneKeyframe[], time: number, rotation: DirectorQuat) { + const current = keyframes.find((item) => Math.abs(item.time - time) < 0.001); + const next = current ? keyframes.map((item) => item.id === current.id ? { ...item, rotation } : item) : [...keyframes, { id: nanoid(), time, rotation }]; + return next.toSorted((a, b) => a.time - b.time); +} + export function interpolateDirectorTransform(base: DirectorTransform, keyframes: DirectorKeyframe[], time: number): DirectorTransform { if (!keyframes.length) return base; const previous = [...keyframes].reverse().find((item) => item.time <= time) || keyframes[0]; const next = keyframes.find((item) => item.time >= time) || keyframes[keyframes.length - 1]; if (previous.id === next.id) return previous.transform; const progress = Math.max(0, Math.min(1, (time - previous.time) / Math.max(next.time - previous.time, 0.001))); + const rotation = new Quaternion().setFromEuler(new Euler(...previous.transform.rotation)).slerp(new Quaternion().setFromEuler(new Euler(...next.transform.rotation)), progress); return { position: lerpVec3(previous.transform.position, next.transform.position, progress), - rotation: lerpVec3(previous.transform.rotation, next.transform.rotation, progress), + rotation: new Euler().setFromQuaternion(rotation).toArray().slice(0, 3) as DirectorVec3, scale: lerpVec3(previous.transform.scale, next.transform.scale, progress), }; } +export function interpolateDirectorBoneRotation(base: DirectorQuat, keyframes: DirectorBoneKeyframe[], time: number): DirectorQuat { + if (!keyframes.length) return base; + const previous = [...keyframes].reverse().find((item) => item.time <= time) || keyframes[0]; + const next = keyframes.find((item) => item.time >= time) || keyframes[keyframes.length - 1]; + if (previous.id === next.id) return previous.rotation; + const progress = Math.max(0, Math.min(1, (time - previous.time) / Math.max(next.time - previous.time, 0.001))); + return new Quaternion(...previous.rotation).slerp(new Quaternion(...next.rotation), progress).toArray() as DirectorQuat; +} + +export function directorBoneLabel(bone: string) { + return ({ hips: "骨盆", spine: "脊柱", chest: "胸腔", neck: "颈部", head: "头部", leftShoulder: "左肩", leftUpperArm: "左上臂", leftLowerArm: "左前臂", leftHand: "左手", rightShoulder: "右肩", rightUpperArm: "右上臂", rightLowerArm: "右前臂", rightHand: "右手", leftUpperLeg: "左大腿", leftLowerLeg: "左小腿", leftFoot: "左脚", rightUpperLeg: "右大腿", rightLowerLeg: "右小腿", rightFoot: "右脚" } as Record)[bone] || bone; +} + +export function directorPoseLabel(pose: DirectorPose) { + return ({ neutral: "自然", stand: "站立", t_pose: "T 型", walk: "行走", run: "跑步", sit: "坐姿", squat: "蹲下", kneel_single: "单膝跪", kneel_double: "双膝跪", hands_hips: "叉腰", lean: "倚靠", bow: "鞠躬", think: "思考", fight: "格斗", kick: "踢球", throw: "投掷", push: "推进", wave: "招手", reach: "伸手", arms_crossed: "抱臂", phone: "看手机" } as Record)[pose]; +} + +export function directorPoseBoneDeltas(pose: DirectorPose): Partial> { + // Soldier 的左右上臂局部 Z 轴方向一致，正向旋转才会把两侧手臂从 T Pose 放下。 + const armsDown = { leftUpperArm: poseQuaternion(0, 0, 1.28), rightUpperArm: poseQuaternion(0, 0, 1.28) }; + const poses: Record>> = { + neutral: armsDown, + stand: armsDown, + t_pose: {}, + walk: { ...armsDown, leftUpperArm: poseQuaternion(0.36, 0, 1.2), rightUpperArm: poseQuaternion(-0.36, 0, 1.2), leftUpperLeg: poseQuaternion(-0.32, 0, 0), rightUpperLeg: poseQuaternion(0.32, 0, 0) }, + run: { ...armsDown, leftUpperArm: poseQuaternion(0.75, 0, 1.05), rightUpperArm: poseQuaternion(-0.75, 0, 1.05), leftLowerArm: poseQuaternion(-0.7, 0, 0), rightLowerArm: poseQuaternion(-0.7, 0, 0), leftUpperLeg: poseQuaternion(-0.65, 0, 0), rightUpperLeg: poseQuaternion(0.55, 0, 0), rightLowerLeg: poseQuaternion(0.8, 0, 0) }, + sit: { ...armsDown, leftUpperLeg: poseQuaternion(-1.35, 0, 0), rightUpperLeg: poseQuaternion(-1.35, 0, 0), leftLowerLeg: poseQuaternion(1.25, 0, 0), rightLowerLeg: poseQuaternion(1.25, 0, 0) }, + squat: { ...armsDown, hips: poseQuaternion(0.25, 0, 0), leftUpperLeg: poseQuaternion(-0.75, 0, 0), rightUpperLeg: poseQuaternion(-0.75, 0, 0), leftLowerLeg: poseQuaternion(1.2, 0, 0), rightLowerLeg: poseQuaternion(1.2, 0, 0) }, + kneel_single: { ...armsDown, leftUpperLeg: poseQuaternion(-0.95, 0, 0), leftLowerLeg: poseQuaternion(1.45, 0, 0), rightUpperLeg: poseQuaternion(-0.35, 0, 0), rightLowerLeg: poseQuaternion(0.75, 0, 0) }, + kneel_double: { ...armsDown, leftUpperLeg: poseQuaternion(-0.55, 0, 0), rightUpperLeg: poseQuaternion(-0.55, 0, 0), leftLowerLeg: poseQuaternion(1.45, 0, 0), rightLowerLeg: poseQuaternion(1.45, 0, 0) }, + hands_hips: { leftUpperArm: poseQuaternion(0, 0, 0.75), rightUpperArm: poseQuaternion(0, 0, 0.75), leftLowerArm: poseQuaternion(-0.1, 0.2, -1.5), rightLowerArm: poseQuaternion(-0.1, -0.2, 1.5) }, + lean: { ...armsDown, hips: poseQuaternion(0, 0, 0.18), spine: poseQuaternion(0, 0, -0.12), head: poseQuaternion(0, 0, -0.08) }, + bow: { ...armsDown, hips: poseQuaternion(0.5, 0, 0), spine: poseQuaternion(0.28, 0, 0), head: poseQuaternion(-0.18, 0, 0) }, + think: { ...armsDown, rightUpperArm: poseQuaternion(-0.25, 0, 0.55), rightLowerArm: poseQuaternion(-1.35, 0, 0.3), head: poseQuaternion(0.05, -0.22, 0) }, + fight: { leftUpperArm: poseQuaternion(-0.65, 0, 0.7), rightUpperArm: poseQuaternion(-0.55, 0, 0.65), leftLowerArm: poseQuaternion(-1.2, 0, 0), rightLowerArm: poseQuaternion(-1.25, 0, 0), chest: poseQuaternion(0, 0.2, 0), leftUpperLeg: poseQuaternion(-0.15, 0, 0), rightUpperLeg: poseQuaternion(0.2, 0, 0) }, + kick: { ...armsDown, leftUpperArm: poseQuaternion(0.3, 0, 1.1), rightUpperArm: poseQuaternion(-0.3, 0, 1.1), rightUpperLeg: poseQuaternion(-1.1, 0, 0), rightLowerLeg: poseQuaternion(0.35, 0, 0) }, + throw: { leftUpperArm: poseQuaternion(-0.35, 0.2, 0.35), rightUpperArm: poseQuaternion(-1.2, 0, 0.25), rightLowerArm: poseQuaternion(-1.05, 0, 0), chest: poseQuaternion(0, -0.3, 0) }, + push: { leftUpperArm: poseQuaternion(-0.9, 0, 0.3), rightUpperArm: poseQuaternion(-0.9, 0, 0.3), leftLowerArm: poseQuaternion(-0.35, 0, 0), rightLowerArm: poseQuaternion(-0.35, 0, 0), chest: poseQuaternion(0.15, 0, 0) }, + wave: { ...armsDown, rightUpperArm: poseQuaternion(0, 0, -0.35), rightLowerArm: poseQuaternion(0, 0, -1.45), rightHand: poseQuaternion(0, 0, -0.3) }, + reach: { ...armsDown, rightUpperArm: poseQuaternion(-1.35, 0, -0.05), rightLowerArm: poseQuaternion(-0.1, 0, 0) }, + arms_crossed: { leftUpperArm: poseQuaternion(-0.65, 0, 0.65), rightUpperArm: poseQuaternion(-0.65, 0, 0.65), leftLowerArm: poseQuaternion(-1.2, 0.15, -0.4), rightLowerArm: poseQuaternion(-1.2, -0.15, 0.4) }, + phone: { ...armsDown, leftUpperArm: poseQuaternion(-0.45, 0, 0.95), rightUpperArm: poseQuaternion(-0.45, 0, 0.95), leftLowerArm: poseQuaternion(-1.1, 0, 0.15), rightLowerArm: poseQuaternion(-1.1, 0, -0.15), head: poseQuaternion(0.28, 0, 0) }, + }; + return poses[pose]; +} + +export function directorColorLabel(value: string) { + const hsl = { h: 0, s: 0, l: 0 }; + new Color(value).getHSL(hsl); + if (hsl.l >= 0.86 && hsl.s <= 0.2) return "白色"; + if (hsl.l <= 0.18) return "黑色"; + if (hsl.s <= 0.16) return hsl.l >= 0.52 ? "浅灰色" : "深灰色"; + const hue = hsl.h * 360; + if (hue < 18 || hue >= 345) return "红色"; + if (hue < 48) return "橙色"; + if (hue < 72) return "黄色"; + if (hue < 165) return "绿色"; + if (hue < 195) return "青色"; + if (hue < 255) return "蓝色"; + if (hue < 290) return "紫色"; + return "粉色"; +} + +function poseQuaternion(x: number, y: number, z: number): DirectorQuat { + return new Quaternion().setFromEuler(new Euler(x, y, z)).toArray() as DirectorQuat; +} + function lerpVec3(from: DirectorVec3, to: DirectorVec3, progress: number): DirectorVec3 { return from.map((value, index) => value + (to[index] - value) * progress) as DirectorVec3; } diff --git a/web/src/pages/canvas/use-canvas-director.ts b/web/src/pages/canvas/use-canvas-director.ts index 587a1a7a6..ff8c6cbc0 100644 --- a/web/src/pages/canvas/use-canvas-director.ts +++ b/web/src/pages/canvas/use-canvas-director.ts @@ -2,11 +2,12 @@ import { useCallback, type Dispatch, type SetStateAction } from "react"; import { App } from "antd"; import { nanoid } from "nanoid"; -import { imageMetadata } from "@/lib/canvas/canvas-generation-task-sync"; +import { imageMetadata, videoMetadata } from "@/lib/canvas/canvas-generation-task-sync"; import { fitNodeSize } from "@/lib/canvas/canvas-node-size"; import { createCanvasNode } from "@/lib/canvas/canvas-project-domain"; import { createDirectorScene } from "@/lib/canvas/director/director-scene"; import { uploadImage } from "@/services/image-storage"; +import { uploadMediaFile } from "@/services/file-storage"; import { CanvasNodeType, type CanvasConnection, type CanvasNodeData, type CanvasNodeMetadata, type Position } from "@/types/canvas"; import type { DirectorScene, DirectorSceneOutput } from "@/types/director"; @@ -94,50 +95,76 @@ export function useCanvasDirector({ const applyDirectorOutput = useCallback(async (output: DirectorSceneOutput) => { const sourceNode = nodesRef.current.find((item) => item.id === directorNodeId); if (!sourceNode) throw new Error("镜头节点不存在"); - const uploads = await Promise.all([uploadImage(output.beauty), uploadImage(output.depth), uploadImage(output.normal)]); - const labels = ["导演台构图", "导演台深度", "导演台法线"]; - const existingIds = [sourceNode.metadata?.directorPreviewNodeId, sourceNode.metadata?.directorDepthNodeId, sourceNode.metadata?.directorNormalNodeId]; + const [image, videoUpload] = await Promise.all([ + uploadImage(output.beauty), + output.clayVideo ? uploadMediaFile(output.clayVideo, "director-clay") : Promise.resolve(null), + ]); + const previewId = sourceNode.metadata?.directorPreviewNodeId || `image-director-${Date.now()}`; + const previewSize = fitNodeSize(image.width, image.height); const nextNodes = [...nodesRef.current]; - const outputIds: string[] = []; - uploads.forEach((image, index) => { - const id = existingIds[index] || `image-director-${Date.now()}-${index}`; - const size = fitNodeSize(image.width, image.height); - const node: CanvasNodeData = { - id, - type: CanvasNodeType.Image, - title: `${sourceNode.title} · ${labels[index]}`, - position: { x: sourceNode.position.x - 3 * (size.width + 36) + index * (size.width + 36), y: sourceNode.position.y }, - width: size.width, - height: size.height, - metadata: { ...imageMetadata(image), prompt: output.prompt, workflowKind: "reference_set", assetTags: [labels[index], `镜头:${sourceNode.title}`] }, + const previewIndex = nextNodes.findIndex((item) => item.id === previewId); + const existingPreview = previewIndex >= 0 ? nextNodes[previewIndex] : null; + const previewNode: CanvasNodeData = { + ...existingPreview, + id: previewId, + type: CanvasNodeType.Image, + title: `${sourceNode.title} · 导演台构图`, + position: existingPreview?.position || { x: sourceNode.position.x - previewSize.width - 36, y: sourceNode.position.y }, + width: previewSize.width, + height: previewSize.height, + metadata: { ...existingPreview?.metadata, ...imageMetadata(image), prompt: output.prompt, workflowKind: "reference_set", assetTags: ["导演台构图", `镜头:${sourceNode.title}`] }, + }; + if (previewIndex >= 0) nextNodes[previewIndex] = previewNode; + else nextNodes.push(previewNode); + + let clayVideoId = sourceNode.metadata?.directorClayVideoNodeId; + if (videoUpload) { + clayVideoId ||= `video-director-clay-${Date.now()}`; + const videoIndex = nextNodes.findIndex((item) => item.id === clayVideoId); + const existingVideo = videoIndex >= 0 ? nextNodes[videoIndex] : null; + const videoNode: CanvasNodeData = { + ...existingVideo, + id: clayVideoId, + type: CanvasNodeType.Video, + title: `${sourceNode.title} · 白膜视频`, + position: existingVideo?.position || { x: sourceNode.position.x, y: sourceNode.position.y + sourceNode.height + 48 }, + width: existingVideo?.width || 360, + height: existingVideo?.height || 220, + metadata: { ...existingVideo?.metadata, ...videoMetadata(videoUpload), prompt: output.prompt, workflowKind: "reference_video", assetTags: ["导演台白膜", `镜头:${sourceNode.title}`] }, }; - const currentIndex = nextNodes.findIndex((item) => item.id === id); - if (currentIndex >= 0) nextNodes[currentIndex] = node; - else nextNodes.push(node); - outputIds.push(id); - }); + if (videoIndex >= 0) nextNodes[videoIndex] = videoNode; + else nextNodes.push(videoNode); + } + const nextConnections = [...connectionsRef.current]; - outputIds.forEach((id) => { + [previewId, videoUpload ? clayVideoId : null].filter((id): id is string => Boolean(id)).forEach((id) => { if (!nextConnections.some((connection) => connection.fromNodeId === id && connection.toNodeId === sourceNode.id)) nextConnections.push({ id: nanoid(), fromNodeId: id, toNodeId: sourceNode.id }); }); + const retiredReferenceIds = new Set([sourceNode.metadata?.directorDepthNodeId, sourceNode.metadata?.directorNormalNodeId].filter(Boolean)); + const referenceAssetNodeIds = Array.from(new Set([ + ...(sourceNode.metadata?.referenceAssetNodeIds || []).filter((id) => !retiredReferenceIds.has(id)), + previewId, + ...(clayVideoId ? [clayVideoId] : []), + ])); const directorMetadata: Partial = { directorSceneId: output.scene.id, directorShotId: output.shot.id, - directorPreviewNodeId: outputIds[0], - directorDepthNodeId: outputIds[1], - directorNormalNodeId: outputIds[2], + directorPreviewNodeId: previewId, + directorDepthNodeId: undefined, + directorNormalNodeId: undefined, + directorClayVideoNodeId: clayVideoId, composerContent: output.prompt, prompt: output.prompt, videoCameraMoveId: output.shot.cameraMove, videoCameraMovePrompt: output.prompt, - referenceAssetNodeIds: Array.from(new Set([...(sourceNode.metadata?.referenceAssetNodeIds || []), ...outputIds])), + referenceAssetNodeIds, }; const finalizedNodes = nextNodes.map((item) => item.id === sourceNode.id ? { ...item, metadata: { ...item.metadata, ...directorMetadata } } : item); nodesRef.current = finalizedNodes; connectionsRef.current = nextConnections; setNodes(finalizedNodes); setConnections(nextConnections); - saveDirectorScene({ ...output.scene, shots: output.scene.shots.map((shot) => shot.id === output.shot.id ? { ...shot, previewNodeId: outputIds[0], depthNodeId: outputIds[1], normalNodeId: outputIds[2] } : shot) }); + saveDirectorScene({ ...output.scene, shots: output.scene.shots.map((shot) => shot.id === output.shot.id ? { ...shot, previewNodeId: previewId, depthNodeId: undefined, normalNodeId: undefined } : shot) }); }, [connectionsRef, directorNodeId, nodesRef, saveDirectorScene, setConnections, setNodes]); return { applyDirectorOutput, createDirectorShot, openDirectorWorkbench, saveDirectorScene }; diff --git a/web/src/stores/canvas/use-director-workbench-store.ts b/web/src/stores/canvas/use-director-workbench-store.ts index 565a67034..a8c70d7b4 100644 --- a/web/src/stores/canvas/use-director-workbench-store.ts +++ b/web/src/stores/canvas/use-director-workbench-store.ts @@ -4,29 +4,38 @@ import type { DirectorRenderMode } from "@/types/director"; type DirectorWorkbenchStore = { selectedObjectId: string | null; + selectedBone: string | null; selectedLightId: string | null; transformMode: "translate" | "rotate" | "scale"; renderMode: DirectorRenderMode; playhead: number; playing: boolean; + autoKey: boolean; + sequencerHeight: number; setSelectedObjectId: (id: string | null) => void; + setSelectedBone: (bone: string | null) => void; setSelectedLightId: (id: string | null) => void; setTransformMode: (mode: DirectorWorkbenchStore["transformMode"]) => void; setRenderMode: (mode: DirectorRenderMode) => void; setPlayhead: (time: number) => void; setPlaying: (playing: boolean) => void; + setAutoKey: (autoKey: boolean) => void; + setSequencerHeight: (height: number) => void; reset: () => void; }; -const initialState = { selectedObjectId: null, selectedLightId: null, transformMode: "translate" as const, renderMode: "beauty" as const, playhead: 0, playing: false }; +const initialState = { selectedObjectId: null, selectedBone: null, selectedLightId: null, transformMode: "translate" as const, renderMode: "beauty" as const, playhead: 0, playing: false, autoKey: true, sequencerHeight: 300 }; export const useDirectorWorkbenchStore = create((set) => ({ ...initialState, - setSelectedObjectId: (selectedObjectId) => set({ selectedObjectId, selectedLightId: null }), - setSelectedLightId: (selectedLightId) => set({ selectedLightId, selectedObjectId: null }), + setSelectedObjectId: (selectedObjectId) => set({ selectedObjectId, selectedBone: null, selectedLightId: null }), + setSelectedBone: (selectedBone) => set({ selectedBone }), + setSelectedLightId: (selectedLightId) => set({ selectedLightId, selectedObjectId: null, selectedBone: null }), setTransformMode: (transformMode) => set({ transformMode }), setRenderMode: (renderMode) => set({ renderMode }), setPlayhead: (playhead) => set({ playhead }), setPlaying: (playing) => set({ playing }), + setAutoKey: (autoKey) => set({ autoKey }), + setSequencerHeight: (sequencerHeight) => set({ sequencerHeight: Math.max(180, Math.min(620, sequencerHeight)) }), reset: () => set(initialState), })); diff --git a/web/src/styles/globals.css b/web/src/styles/globals.css index 4ef6e1d23..c669d9b2c 100644 --- a/web/src/styles/globals.css +++ b/web/src/styles/globals.css @@ -341,6 +341,17 @@ --cn-spatial-elevated: var(--color-neutral-0); --cn-spatial-toolbar-border: var(--color-neutral-200); + --director-sequencer-surface: var(--color-neutral-900); + --director-sequencer-surface-raised: var(--color-neutral-800); + --director-sequencer-border: rgba(255, 255, 255, 0.11); + --director-sequencer-muted: rgba(255, 255, 255, 0.48); + --director-sequencer-grid: rgba(255, 255, 255, 0.055); + --director-dock-surface: rgba(17, 19, 21, 0.92); + --director-dock-fg: rgba(255, 255, 255, 0.62); + --director-dock-fg-strong: rgba(255, 255, 255, 0.96); + --director-control-hover: rgba(255, 255, 255, 0.09); + --director-swatch-inset: rgba(0, 0, 0, 0.18); + /* 节点类型色（引用 Primitive 节点色板，对应 #97 决策1 色带方案）*/ --cn-type-image: var(--palette-node-image); --cn-type-video: var(--palette-node-video); @@ -457,6 +468,17 @@ --cn-spatial-elevated: var(--color-neutral-900); --cn-spatial-toolbar-border: rgba(255, 255, 255, 0.1); + --director-sequencer-surface: #111315; + --director-sequencer-surface-raised: #191c1f; + --director-sequencer-border: rgba(255, 255, 255, 0.11); + --director-sequencer-muted: rgba(255, 255, 255, 0.48); + --director-sequencer-grid: rgba(255, 255, 255, 0.055); + --director-dock-surface: rgba(17, 19, 21, 0.92); + --director-dock-fg: rgba(255, 255, 255, 0.62); + --director-dock-fg-strong: rgba(255, 255, 255, 0.96); + --director-control-hover: rgba(255, 255, 255, 0.09); + --director-swatch-inset: rgba(0, 0, 0, 0.18); + /* --- Shadow dark 覆盖（暗色下黑影不可见，改用白色微影制造抬升感，审计 D4/P2-D2）--- 组件 token（--node-shadow/--dock-shadow/--panel-shadow/--modal-shadow）均引用 var(--shadow-*)，此处覆盖自动级联。--cn-spatial-shadow（内联 boxShadow 用）已有 @@ -527,6 +549,17 @@ --panel-shadow: var(--shadow-lg); --panel-section-gap: var(--space-3); /* 12px */ + /* Director workbench / Sequencer */ + --director-sequencer-min-height: 180px; + --director-sequencer-label-width: 208px; + --director-sequencer-row-height: 30px; + --director-sequencer-base-width: 860px; + --director-sequencer-shot-select-width: 144px; + --director-dock-button-size: var(--space-10); + --director-dock-bottom: var(--space-3); + --director-pose-button-height: var(--space-8); + --director-color-swatch-size: var(--space-6); + /* Canvas 布局（画布页浮层定位，对应审计 P0-8.1/P1-2.1）*/ --canvas-sidebar-width: 232px; /* 项目侧栏宽度（短剧模式）*/ --canvas-panel-width: min(332px, calc(100vw - 24px)); /* 浮动面板默认宽度（活动任务等）*/ @@ -5137,6 +5170,357 @@ html:not(.dark) .creation-model-picker-menu .canvas-model-picker-option > span:f } /* 创作入口与空间导航在同一块画布上，避免视觉断层。 */ + +.director-sequencer { + position: relative; + color: rgba(255, 255, 255, 0.86); + font-size: var(--fs-caption); +} + +.director-viewport-dock { + position: absolute; + z-index: var(--z-toolbar); + bottom: var(--director-dock-bottom); + left: 50%; + display: flex; + max-width: calc(100% - var(--space-6)); + align-items: center; + gap: var(--space-1); + padding: var(--space-1-half); + overflow-x: auto; + border: var(--stroke-1) solid var(--director-sequencer-border); + border-radius: var(--r-lg); + background: var(--director-dock-surface); + box-shadow: var(--shadow-xl); + color: var(--director-dock-fg); + transform: translateX(-50%); + backdrop-filter: blur(var(--space-4)); +} + +.director-viewport-dock-button { + display: grid; + width: var(--director-dock-button-size); + height: var(--director-dock-button-size); + flex: 0 0 var(--director-dock-button-size); + place-items: center; + border-radius: var(--r-md); + transition: background var(--motion-dur-fast) var(--motion-ease-out), color var(--motion-dur-fast) var(--motion-ease-out), transform var(--motion-dur-fast) var(--motion-ease-out); +} + +.director-viewport-dock-button > svg { + width: var(--space-5); + height: var(--space-5); +} + +.director-viewport-dock-button:hover { + background: var(--director-control-hover); + color: var(--director-dock-fg-strong); +} + +.director-viewport-dock-button:active { + transform: translateY(1px); +} + +.director-viewport-dock-button.is-active { + background: var(--workspace-accent); + color: var(--color-neutral-0); + box-shadow: var(--shadow-glow-sm); +} + +.director-viewport-dock-divider { + width: var(--stroke-1); + height: var(--space-6); + flex: 0 0 var(--stroke-1); + margin: 0 var(--space-half); + background: var(--director-sequencer-border); +} + +.director-actor-colors { + display: flex; + min-width: 0; + align-items: center; + gap: var(--space-1-half); +} + +.director-actor-color { + width: var(--director-color-swatch-size); + height: var(--director-color-swatch-size); + flex: 0 0 var(--director-color-swatch-size); + border: var(--stroke-1) solid var(--director-sequencer-border); + border-radius: var(--r-full); + box-shadow: inset 0 0 0 var(--stroke-1) var(--director-swatch-inset); +} + +.director-actor-color.is-active { + outline: var(--stroke-2) solid var(--workspace-accent); + outline-offset: var(--space-half); +} + +.director-pose-section { + display: grid; + gap: var(--space-2); +} + +.director-inspector-section-title { + display: flex; + align-items: center; + justify-content: space-between; + color: var(--cn-muted); + font-size: var(--fs-label); +} + +.director-inspector-section-title > span:last-child { + color: var(--workspace-accent); +} + +.director-pose-grid { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: var(--space-1); +} + +.director-pose-button { + min-width: 0; + height: var(--director-pose-button-height); + padding: 0 var(--space-1); + overflow: hidden; + border: var(--stroke-1) solid var(--cn-stroke); + border-radius: var(--r-sm); + color: var(--cn-muted); + font-size: var(--fs-tiny); + text-overflow: ellipsis; + white-space: nowrap; + transition: border-color var(--motion-dur-fast) var(--motion-ease-out), background var(--motion-dur-fast) var(--motion-ease-out), color var(--motion-dur-fast) var(--motion-ease-out); +} + +.director-pose-button:hover { + border-color: var(--workspace-accent); + color: var(--cn-text); +} + +.director-pose-button.is-active { + border-color: var(--workspace-accent); + background: var(--workspace-accent-soft); + color: var(--workspace-accent); +} + +@media (prefers-reduced-motion: reduce) { + .director-viewport-dock-button, + .director-pose-button { + transition: none; + } +} + +.director-sequencer-resizer { + position: absolute; + z-index: 2; + inset: -4px 0 auto; + height: 8px; + cursor: ns-resize; +} + +.director-sequencer-resizer::after { + content: ""; + position: absolute; + top: 3px; + left: 50%; + width: 44px; + height: 2px; + border-radius: var(--r-full); + background: rgba(255, 255, 255, 0.28); + transform: translateX(-50%); + opacity: 0; + transition: opacity var(--motion-dur-fast) var(--motion-ease-out); +} + +.director-sequencer:hover .director-sequencer-resizer::after, +.director-sequencer-resizer:focus-visible::after { + opacity: 1; +} + +.director-sequencer-body { + position: relative; + height: calc(100% - 40px); + min-height: 0; +} + +.director-sequencer-grid { + position: relative; + display: grid; + min-width: calc(var(--director-sequencer-base-width) * var(--director-sequencer-track-scale, 1)); + grid-template-columns: var(--director-sequencer-label-width) minmax(640px, 1fr); +} + +.director-sequencer-global-playhead { + position: absolute; + z-index: 1; + top: 28px; + bottom: 0; + right: 0; + left: var(--director-sequencer-label-width); + pointer-events: none; +} + +.director-sequencer-global-playhead > i { + position: absolute; + top: 0; + bottom: 0; + width: 1px; + background: #f0b36a; +} + +.director-sequencer-label, +.director-sequencer-track { + min-width: 0; + height: var(--director-sequencer-row-height); + border-bottom: 1px solid var(--director-sequencer-border); +} + +.director-sequencer-row { + display: contents; +} + +.director-sequencer-label { + display: flex; + align-items: center; + gap: var(--space-2); + width: var(--director-sequencer-label-width); + padding: 0 var(--space-3); + color: rgba(255, 255, 255, 0.64); + text-align: left; + white-space: nowrap; + transition: background var(--motion-dur-fast) var(--motion-ease-out), color var(--motion-dur-fast) var(--motion-ease-out); +} + +.director-sequencer-label:hover, +.director-sequencer-row.is-selected > .director-sequencer-label { + background: rgba(255, 255, 255, 0.075); + color: rgba(255, 255, 255, 0.96); +} + +.director-sequencer-label.is-indent { + padding-left: var(--space-6); + color: rgba(255, 255, 255, 0.46); +} + +.director-sequencer-ruler-label, +.director-sequencer-ruler { + height: 28px; + border-bottom-color: var(--director-sequencer-border); + background: var(--director-sequencer-surface-raised); +} + +.director-sequencer-ruler-label { + padding-left: var(--space-3); + color: rgba(255, 255, 255, 0.42); + font-size: var(--fs-micro); + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.director-sequencer-ruler { + position: relative; + cursor: ew-resize; + background-image: repeating-linear-gradient(to right, transparent 0, transparent calc(20% - 1px), var(--director-sequencer-grid) 20%); +} + +.director-sequencer-tick { + position: absolute; + top: var(--space-2); + color: rgba(255, 255, 255, 0.38); + font-size: var(--fs-micro); + transform: translateX(-50%); +} + +.director-sequencer-playhead { + position: absolute; + top: 0; + bottom: 0; + width: 1px; + background: #f0b36a; + pointer-events: none; +} + +.director-sequencer-track { + position: relative; + overflow: hidden; + background-image: repeating-linear-gradient(to right, transparent 0, transparent calc(20% - 1px), var(--director-sequencer-grid) 20%); +} + +.director-sequencer-track-content { + position: absolute; + inset: 0; +} + +.director-sequencer-key { + position: absolute; + top: 50%; + width: 8px; + height: 8px; + border-radius: var(--r-xs); + box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.38); + transform: translate(-50%, -50%) rotate(45deg); +} + +.director-sequencer-clip { + position: absolute; + top: 5px; + bottom: 5px; + display: flex; + align-items: center; + min-width: 30px; + padding: 0 var(--space-2); + overflow: hidden; + border: 1px solid; + border-radius: var(--r-xs); + font-size: var(--fs-micro); +} + +.director-sequencer-transport, +.director-sequencer-icon, +.director-sequencer-tool { + display: inline-flex; + align-items: center; + justify-content: center; + height: var(--space-8); + border-radius: var(--r-sm); + color: rgba(255, 255, 255, 0.62); + transition: background var(--motion-dur-fast) var(--motion-ease-out), color var(--motion-dur-fast) var(--motion-ease-out); +} + +.director-sequencer-transport, +.director-sequencer-icon { + width: var(--space-8); +} + +.director-sequencer-tool { + gap: var(--space-1); + padding: 0 var(--space-2); + font-size: var(--fs-micro); +} + +.director-sequencer-shot-select { + width: var(--director-sequencer-shot-select-width); + height: var(--space-8); + padding: 0 var(--space-2); + overflow: hidden; + border: 1px solid var(--director-sequencer-border); + border-radius: var(--r-sm); + background: var(--director-sequencer-surface-raised); + color: rgba(255, 255, 255, 0.72); + font-size: var(--fs-micro); + text-overflow: ellipsis; + white-space: nowrap; +} + +.director-sequencer-transport:hover, +.director-sequencer-icon:hover, +.director-sequencer-tool:hover, +.director-sequencer-icon.is-active, +.director-sequencer-tool.is-active { + background: rgba(255, 255, 255, 0.1); + color: #ffffff; +} .app-workspace-shell.is-creation-workspace, .app-workspace-shell.is-creation-workspace .app-workspace-sidebar, .app-workspace-shell.is-creation-workspace .app-workspace-stage { diff --git a/web/src/types/canvas.ts b/web/src/types/canvas.ts index 707af086d..da62163e0 100644 --- a/web/src/types/canvas.ts +++ b/web/src/types/canvas.ts @@ -224,6 +224,7 @@ export type CanvasNodeMetadata = { directorPreviewNodeId?: string; directorDepthNodeId?: string; directorNormalNodeId?: string; + directorClayVideoNodeId?: string; skillId?: string; skillVersion?: number; skillSnapshot?: CanvasSkillSnapshot; diff --git a/web/src/types/director.ts b/web/src/types/director.ts index 5906de776..c45a5e4ec 100644 --- a/web/src/types/director.ts +++ b/web/src/types/director.ts @@ -1,4 +1,5 @@ export type DirectorVec3 = [number, number, number]; +export type DirectorQuat = [number, number, number, number]; export type DirectorTransform = { position: DirectorVec3; @@ -7,16 +8,47 @@ export type DirectorTransform = { }; export type DirectorPrimitiveKind = "box" | "sphere" | "cylinder" | "plane" | "character"; -export type DirectorObjectKind = "primitive" | "model" | "billboard"; -export type DirectorPose = "neutral" | "stand" | "walk" | "run" | "sit" | "action"; +export type DirectorObjectKind = "primitive" | "model" | "actor" | "billboard"; +export type DirectorPose = "neutral" | "stand" | "t_pose" | "walk" | "run" | "sit" | "squat" | "kneel_single" | "kneel_double" | "hands_hips" | "lean" | "bow" | "think" | "fight" | "kick" | "throw" | "push" | "wave" | "reach" | "arms_crossed" | "phone"; export type DirectorCameraMove = "static" | "push_in" | "pull_out" | "pan_left" | "pan_right" | "tilt_up" | "tilt_down" | "orbit_left" | "orbit_right" | "handheld"; export type DirectorShotSize = "extreme_wide" | "wide" | "full" | "medium" | "close_up" | "extreme_close_up"; -export type DirectorRenderMode = "beauty" | "depth" | "normal"; +export type DirectorRenderMode = "beauty" | "clay" | "depth" | "normal" | "pose"; export type DirectorKeyframe = { id: string; time: number; transform: DirectorTransform; + easing?: "step" | "linear" | "smooth"; +}; + +export type DirectorHumanoidBone = "root" | "hips" | "spine" | "chest" | "neck" | "head" | "leftShoulder" | "leftUpperArm" | "leftLowerArm" | "leftHand" | "rightShoulder" | "rightUpperArm" | "rightLowerArm" | "rightHand" | "leftUpperLeg" | "leftLowerLeg" | "leftFoot" | "rightUpperLeg" | "rightLowerLeg" | "rightFoot"; + +export type DirectorBoneKeyframe = { + id: string; + time: number; + rotation: DirectorQuat; + easing?: "step" | "linear" | "smooth"; +}; + +export type DirectorBoneTrack = { + bone: DirectorHumanoidBone; + keyframes: DirectorBoneKeyframe[]; +}; + +export type DirectorRig = { + status: "unmapped" | "ready"; + boneMap: Partial>; + animationNames: string[]; +}; + +export type DirectorMotionClip = { + id: string; + name: string; + sourceAnimation: string; + start: number; + duration: number; + playbackRate: number; + loop: boolean; }; export type DirectorObject = { @@ -30,6 +62,11 @@ export type DirectorObject = { castShadow: boolean; receiveShadow: boolean; pose?: DirectorPose; + rig?: DirectorRig; + motionClips?: DirectorMotionClip[]; + activeMotionClipId?: string; + boneOverrides?: Partial>; + boneTracks?: DirectorBoneTrack[]; sourceNodeId?: string; assetId?: string; storageKey?: string; @@ -69,6 +106,7 @@ export type DirectorShot = { name: string; cameraId: string; duration: number; + fps: 24 | 25 | 30; shotSize: DirectorShotSize; cameraMove: DirectorCameraMove; prompt: string; @@ -98,6 +136,8 @@ export type DirectorSceneOutput = { shot: DirectorShot; prompt: string; beauty: Blob; - depth: Blob; - normal: Blob; + depth?: Blob; + normal?: Blob; + clayVideo?: Blob; + clayVideoMimeType?: string; };