import { Canvas, useFrame, useThree } from "@react-three/fiber"; import { PerspectiveCamera as DreiPerspectiveCamera } from "@react-three/drei"; import { memo, useLayoutEffect, useMemo, useRef, useState } from "react"; import { PerspectiveCamera as ThreePerspectiveCamera, Raycaster, Scene, Vector2 } from "three"; import { useDirectorStore } from "../store/directorStore"; import { usePlaybackStore } from "../runtime/playbackStore"; import { setMonitorRenderContext } from "../io/monitorRenderBridge"; import { SceneRoot } from "./SceneRoot"; import { ViewportBackground } from "./ViewportBackground"; import { PlaybackTransformSync } from "./PlaybackTransformSync"; import { renderSceneWithBokeh } from "./viewportBokeh"; import { resolveCameraViewRenderRect } from "./viewportAspectFrame"; import { formatLensInfo, getCameraRenderQuaternion, getCameraViewSnapshotFromShot, getWorldCameraViewSnapshot, } from "../schema/cameraGeometry"; import { resolveLensAtTime } from "../runtime/playbackTransforms"; import type { DirectorCameraShot, SceneSettings } from "../schema/directorProject"; import type { CameraKeyframeValue, CameraTimelineTrack } from "../schema/timeline"; import { evaluateCameraViewAtTime, getActiveCameraIdAtTime } from "../schema/timeline"; /** * 镜头监视器:**机位画面的唯一渲染源**。 * * - 导演视角下:它是一块独立的小窗(`right` 偏移悬浮在画布右上角),随时显示当前机位看到的内容; * - 机位视角下:它直接顶到主视口那块画幅的位置上(`.lens-monitor.is-viewport-camera`,尺寸取 * `resolveCameraViewRenderRect` 算出的矩形),**机位视角显示的就是这块 canvas 的像素** —— * 同一份渲染、同一个画幅,所以「机位视角 == 监视器 == 导出」不可能再出现取景 / 虚化不一致, * 主视口那边也不必重复跑一遍完整的景深 composer + 深度 pass(省掉约一半开销)。 * * 复用 SceneRoot 渲染同一份场景(不带编辑器辅助),按传感器宽高比裁切画幅、 * 用 ViewportBackground 渲染与导出一致的全景背景,通过共享的 renderSceneWithBokeh 反映景深虚化。 * * 「当前机位」= 时间轴此刻正在显示的机位(多机位自动切换 / 过渡时会变),与视频导出用同一个 * getActiveCameraIdAtTime 解析;位姿、镜头(焦段/光圈/对焦/景深强度)、画幅全部取自这台机位, * 保证监视器所见与导出的机位视角严格一致。 * * 刻意做成独立的第二个 Canvas:景深后处理只走这一条管线,主视口 / 截图不必各写一套。 */ function LensMonitorCamera({ camera, sceneSettings, sceneRef, camRef, cameraTracks, }: { camera: DirectorCameraShot; sceneSettings: SceneSettings; sceneRef: React.MutableRefObject; camRef: React.MutableRefObject; cameraTracks: CameraTimelineTrack[]; }) { const { gl, scene } = useThree(); const cameraRef = useRef(null); // 把监视器自己的场景 / 渲染相机暴露给外层 DOM 事件,供「触屏对焦」射线拾取使用 sceneRef.current = scene; const cameras = useDirectorStore((state) => state.project.cameras); const transitionClips = useDirectorStore((state) => state.project.timeline.transitionClips); const globalTransition = useDirectorStore((state) => state.project.timeline.cameraTransition); const isExporting = usePlaybackStore((state) => state.isExporting); // playhead 自己订阅(而不是从外层当 prop 传进来):外壳组件不必因为这个每秒变几十次的值 // 去 reconcile 整棵场景树(见 LensMonitorCanvas 的 memo)。 const playhead = usePlaybackStore((state) => state.playhead); // 拖动播放头期间走草稿档:景深照常渲染(移焦 / 变焦要看得到),只把模糊采样点数降到最低档 const isScrubbing = usePlaybackStore((state) => state.isScrubbing); // 关键帧感知:拖播放头时,监视器上的对焦距离 / 光圈 / 焦段 / 景深强度都按关键帧插值, // 让「移焦 / 渐入渐出虚化 / 变焦」在监视器上所见即所得(与导出管线一致的 resolveLensAtTime)。 const lens = useMemo( () => resolveLensAtTime(camera, playhead, cameraTracks), [camera, playhead, cameraTracks] ); // 与导出共用 evaluateCameraViewAtTime:让监视器随运镜关键帧 / 多机位过渡实时动起来, // 渲染的位姿(含场景变换下的世界坐标换算)与导出的机位视角严格一致。 // 无运镜轨道时回落到机位静态视角,行为和以前一样。 const worldView = useMemo(() => { const getStaticPose = (cameraId: string): CameraKeyframeValue | null => { const shot = cameras.find((item) => item.id === cameraId); return shot ? getWorldCameraViewSnapshot(getCameraViewSnapshotFromShot(shot), sceneSettings) : null; }; const view = evaluateCameraViewAtTime( transitionClips, cameraTracks, globalTransition, camera.id, playhead, getStaticPose ); if (!view) return getWorldCameraViewSnapshot(getCameraViewSnapshotFromShot(camera), sceneSettings); return getWorldCameraViewSnapshot( { fov: view.pose.fov, position: view.pose.position, lookRotation: view.pose.lookRotation }, sceneSettings ); }, [camera, cameras, cameraTracks, transitionClips, globalTransition, playhead, sceneSettings]); const sensorAspect = lens.sensor.widthMm / lens.sensor.heightMm; // 把监视器的渲染三件套注册给视频导出:机位视角导出直接用这块画布 / 这颗相机 / 这个场景出片, // 因此导出的画面与监视器所见是同一份渲染,逐像素一致。 useLayoutEffect(() => { const renderCamera = cameraRef.current; if (!renderCamera) return undefined; setMonitorRenderContext({ gl, scene, camera: renderCamera }); return () => setMonitorRenderContext(null); }, [gl, scene]); useFrame(() => { const renderCamera = cameraRef.current; if (!renderCamera || !worldView) return; // 把当前机位的世界位姿 + 由焦段/画幅换算的 FOV 写到渲染相机上,并按传感器比例裁切画幅 renderCamera.position.set(...worldView.position); renderCamera.quaternion.copy(getCameraRenderQuaternion(worldView.lookRotation)); if (Math.abs(renderCamera.fov - lens.fov) > 0.001) { renderCamera.fov = lens.fov; } if (Math.abs(renderCamera.aspect - sensorAspect) > 1e-4) { renderCamera.aspect = sensorAspect; } renderCamera.updateProjectionMatrix(); camRef.current = renderCamera; // 导出期间由 videoExport 在这块画布上逐帧渲染(同一颗 gl / 同一个场景 / 同一颗相机), // 这里跳过,避免和导出循环双重渲染、也避免争用同一份景深合成器缓存。 if (isExporting) return; // 与主视口 / 视频导出共用同一份景深渲染(renderSceneWithBokeh):监视器不再自建一套 // EffectComposer —— 两套实现很容易漂移,导出与监视器的取景 / 虚化就会对不上。 // 合成器按 gl 缓存、画布尺寸变化会自动重建;不支持后处理时回退普通渲染,监视器永远可用。 // 拖动播放头时传 draft:景深仍然整套跑(拖动期间「移焦 / 变焦 / 虚化强弱」必须看得见), // 只是把模糊采样点数压到最低档省开销,松手立刻回到自适应档。 renderSceneWithBokeh(gl, scene, renderCamera, lens, undefined, { draft: isScrubbing }); }, 1); return ( ); } /** * 监视器那块 3D 画布:整棵场景树(SceneRoot 会把当前工程所有物体铺进来)。 * * 单独抽成 memo 组件的原因:拖动播放头时 playhead 每秒变几十次,如果这棵树长在外壳组件里, * 每动一下都要把整棵场景树 reconcile 一遍 —— 那是拖动卡顿的一大来源。这里 props 只在 * 「换机位 / 改场景 / 换全景图」这类真实编辑时变,播放头由 LensMonitorCamera 自己订阅。 */ const LensMonitorCanvas = memo(function LensMonitorCanvas({ camera, sceneSettings, panoramaAsset, cameraTracks, sceneRef, camRef, initialCamera, }: { camera: DirectorCameraShot; sceneSettings: SceneSettings; panoramaAsset: React.ComponentProps["panoramaAsset"]; cameraTracks: CameraTimelineTrack[]; sceneRef: React.MutableRefObject; camRef: React.MutableRefObject; /** 画布的初始相机参数:只取首次渲染那一刻的值(位姿 / fov 由 LensMonitorCamera 每帧写) */ initialCamera: { fov: number; position: [number, number, number] }; }) { return ( {/* 与主视口 / 导出共用同一份背景实现:有全景图时渲染全景(equirect 走 scene.background、 backdrop 走穹顶网格),否则用背景色。少了它,监视器会和导出画面差一个背景。 */} {/* 与主视口 / 视频导出共用同一份时间轴变换:把物体 / 角色关键帧的插值写进监视器场景。 SceneRoot 自身只取 item.transform 和人物轨迹,不应用时间轴轨道 —— 少了这一层, 监视器里的物体就不会随关键帧运动,物体到相机的深度和导出不同, 清晰平面落在别处,看起来就是「对焦距离和导出对不上」。 */} ); }); /** 鼠标每像素对应的角度增量(度) */ const LOOK_DRAG_SENSITIVITY = 0.3; /** 俯仰角夹到 ±89°,避免越过天顶后镜头翻转 */ const PITCH_LIMIT = 89; /** 区分「单击对焦」与「拖拽瞄准」的位移阈值(像素) */ const CLICK_THRESHOLD_PX = 4; export function LensMonitor({ rightOffset = 12 }: { rightOffset?: number }) { const sceneSettings = useDirectorStore((state) => state.project.scene); const assets = useDirectorStore((state) => state.project.assets); const panoramaAssetId = useDirectorStore((state) => state.project.panoramaAssetId); const panoramaAsset = assets.find((item) => item.id === panoramaAssetId); const activeCamera = useDirectorStore((state) => state.project.cameras.find((item) => item.id === state.project.activeCameraId) ); const updateCameraLookRotationAtTime = useDirectorStore((state) => state.updateCameraLookRotationAtTime); const updateCameraFocusDistanceAtTime = useDirectorStore((state) => state.updateCameraFocusDistanceAtTime); const beginUndoBatch = useDirectorStore((state) => state.beginUndoBatch); const endUndoBatch = useDirectorStore((state) => state.endUndoBatch); const playhead = usePlaybackStore((state) => state.playhead); const isPlaying = usePlaybackStore((state) => state.isPlaying); const isScrubbing = usePlaybackStore((state) => state.isScrubbing); const isExporting = usePlaybackStore((state) => state.isExporting); const cameras = useDirectorStore((state) => state.project.cameras); const transitionClips = useDirectorStore((state) => state.project.timeline.transitionClips); const cameraTracks = useDirectorStore((state) => state.project.timeline.cameraTracks); // 当前时刻「正在显示」的机位:与机位视角(ActiveCameraPlaybackSync)用同一套选取逻辑 —— // 播放 / 导出 / scrub 时按时间轴关键帧自动切(也与导出的 getActiveCameraIdAtTime 一致), // 静止编辑时沿用手动选中的机位。机位视角和监视器因此永远显示同一台机位,不会出现 // 「主视口是手选的 A 机位、监视器却是时间轴上的 B 机位」这种画面 / 虚化都对不上的情况。 const autoCut = isPlaying || isExporting || isScrubbing; const displayCameraId = useMemo( () => autoCut ? getActiveCameraIdAtTime(transitionClips, cameraTracks, activeCamera?.id ?? null, playhead) : (activeCamera?.id ?? null), [autoCut, transitionClips, cameraTracks, activeCamera?.id, playhead] ); const camera = cameras.find((item) => item.id === displayCameraId) ?? activeCamera; // 小字镜头信息也按播放头解析(与画面渲染同一份 resolveLensAtTime),点击/拖到关键帧时数字同步刷新 const lens = camera ? resolveLensAtTime(camera, playhead, cameraTracks) : null; const sensorAspect = lens ? lens.sensor.widthMm / lens.sensor.heightMm : 16 / 9; const viewMode = useDirectorStore((state) => state.viewMode); // 机位视角下,这块监视器直接顶到主视口「机位画面」的位置上,成为那块画幅本身: // 机位视角显示的**就是这块 canvas 的像素**(同一份渲染、同一个画幅), // 于是「机位视角 == 镜头监视器 == 导出」天然一致,不会再出现取景 / 虚化两边对不上; // 主视口那边同时不再重复渲染一遍完整景深管线(少一半开销,之前又卡又糊的主因)。 const rootRef = useRef(null); const [viewportFrame, setViewportFrame] = useState<{ left: number; top: number; width: number; height: number } | null>( null ); useLayoutEffect(() => { // 画幅是相对「画布」等比适配出来的,而这块监视器的父元素正是 .canvas-frame(position: relative) const frame = rootRef.current?.parentElement ?? null; if (!frame) return undefined; let animationFrameId = 0; const measure = () => { const rect = resolveCameraViewRenderRect(frame.clientWidth, frame.clientHeight, sensorAspect); setViewportFrame((current) => current && rect && current.left === rect.left && current.top === rect.top && current.width === rect.width && current.height === rect.height ? current : rect ); }; const scheduleMeasure = () => { cancelAnimationFrame(animationFrameId); animationFrameId = requestAnimationFrame(measure); }; measure(); window.addEventListener("resize", scheduleMeasure); if (typeof ResizeObserver === "undefined") { return () => { cancelAnimationFrame(animationFrameId); window.removeEventListener("resize", scheduleMeasure); }; } const resizeObserver = new ResizeObserver(scheduleMeasure); resizeObserver.observe(frame); return () => { cancelAnimationFrame(animationFrameId); window.removeEventListener("resize", scheduleMeasure); resizeObserver.disconnect(); }; }, [sensorAspect]); const viewportCameraFrame = viewMode === "camera" && camera ? viewportFrame : null; // 画布初始相机参数只在首次渲染那一刻取一次:位姿 / fov 每帧都由 LensMonitorCamera 写, // 保持这个对象引用稳定, 才不会因为播放头变化被重新应用 camera props。 const initialCanvasCameraRef = useRef<{ fov: number; position: [number, number, number] } | null>(null); initialCanvasCameraRef.current ??= { fov: lens?.fov ?? 50, position: [0, 1, 5] }; const initialCanvasCamera = initialCanvasCameraRef.current; const [dragging, setDragging] = useState(false); const dragRef = useRef<{ startX: number; startY: number; startLook: [number, number, number]; rollMode: boolean; } | null>(null); // 触屏对焦需要的监视器场景 / 渲染相机引用(由 LensMonitorCamera 每帧回填) const monitorSceneRef = useRef(null); const monitorCamRef = useRef(null); const stageRef = useRef(null); // 在监视器画面上「按住拖动」改变机位注视角度:普通拖动改水平角 + 俯仰角;Shift+拖动改横滚角。 // 用基础机位的 lookRotation 作为起点累加位移,避免逐帧累积误差;写入走 updateCameraLookRotationAtTime, // 由它决定写基础机位还是最近关键帧(关键帧感知),并在拖拽起止用 undo 批次合并成单条撤销记录。 const handlePointerDown = (event: React.PointerEvent) => { if (!camera || event.button !== 0) return; event.currentTarget.setPointerCapture(event.pointerId); dragRef.current = { startX: event.clientX, startY: event.clientY, startLook: [...camera.lookRotation] as [number, number, number], rollMode: event.shiftKey, }; setDragging(true); beginUndoBatch(); }; const handlePointerMove = (event: React.PointerEvent) => { const drag = dragRef.current; if (!drag || !camera) return; const dx = event.clientX - drag.startX; const dy = event.clientY - drag.startY; const [pitch, yaw, roll] = drag.startLook; const next: [number, number, number] = drag.rollMode ? [pitch, yaw, roll + dx * LOOK_DRAG_SENSITIVITY] : [ Math.max(-PITCH_LIMIT, Math.min(PITCH_LIMIT, pitch - dy * LOOK_DRAG_SENSITIVITY)), yaw + dx * LOOK_DRAG_SENSITIVITY, roll, ]; updateCameraLookRotationAtTime(camera.id, next, playhead); }; // 监视器触屏对焦:在已选中机位的前提下,单击画面某点(非拖拽)时, // 从渲染相机发出射线拾取场景,取命中点到相机的距离写入对焦距离。 const handleTouchFocus = (clientX: number, clientY: number) => { if (!camera) return; const scene = monitorSceneRef.current; const renderCamera = monitorCamRef.current; const stage = stageRef.current; if (!scene || !renderCamera || !stage) return; const rect = stage.getBoundingClientRect(); const ndcX = ((clientX - rect.left) / rect.width) * 2 - 1; const ndcY = -((clientY - rect.top) / rect.height) * 2 + 1; const raycaster = new Raycaster(); raycaster.setFromCamera(new Vector2(ndcX, ndcY), renderCamera); const hits = raycaster.intersectObjects(scene.children, true); if (hits.length === 0) return; const distance = Math.max(0.1, Math.min(100, hits[0].distance)); updateCameraFocusDistanceAtTime(camera.id, distance, playhead); }; const endDrag = (event: React.PointerEvent) => { const drag = dragRef.current; if (!drag) return; if (event.currentTarget.hasPointerCapture(event.pointerId)) { event.currentTarget.releasePointerCapture(event.pointerId); } // 位移极小且无右键 => 视为单击,触发触屏对焦;否则只是结束拖拽瞄准 const moved = Math.hypot(event.clientX - drag.startX, event.clientY - drag.startY); const wasClick = moved <= CLICK_THRESHOLD_PX && event.button === 0; dragRef.current = null; setDragging(false); endUndoBatch(); if (wasClick) handleTouchFocus(event.clientX, event.clientY); }; return (
{camera && lens ? ( ) : (
无激活机位
)}
{lens ? formatLensInfo(lens) : "无激活机位"}
); }