Files
ttstd-3d-director/src/editor/canvas/LensMonitor.tsx
tongtongstudio ada16fa3ec fix(timeline): 修复拖动播放头时画布不实时更新
拖动期间改用 performance.now() 时间片节流替代 requestAnimationFrame,避免在窗口被遮挡或后台节流时整段拖动不写 playhead。同时草稿档不再跳过景深渲染,仅降低模糊采样点数以保留移焦与变焦预览。
2026-09-21 10:19:43 +08:00

436 lines
22 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { Canvas, useFrame, useThree } from "@react-three/fiber";
import { PerspectiveCamera as DreiPerspectiveCamera } from "@react-three/drei";
import { memo, useLayoutEffect, useMemo, useRef, useState } from "react";
import { PerspectiveCamera as ThreePerspectiveCamera, Raycaster, Scene, Vector2 } from "three";
import { useDirectorStore } from "../store/directorStore";
import { usePlaybackStore } from "../runtime/playbackStore";
import { setMonitorRenderContext } from "../io/monitorRenderBridge";
import { SceneRoot } from "./SceneRoot";
import { ViewportBackground } from "./ViewportBackground";
import { PlaybackTransformSync } from "./PlaybackTransformSync";
import { renderSceneWithBokeh } from "./viewportBokeh";
import { resolveCameraViewRenderRect } from "./viewportAspectFrame";
import {
formatLensInfo,
getCameraRenderQuaternion,
getCameraViewSnapshotFromShot,
getWorldCameraViewSnapshot,
} from "../schema/cameraGeometry";
import { resolveLensAtTime } from "../runtime/playbackTransforms";
import type { DirectorCameraShot, SceneSettings } from "../schema/directorProject";
import type { CameraKeyframeValue, CameraTimelineTrack } from "../schema/timeline";
import { evaluateCameraViewAtTime, getActiveCameraIdAtTime } from "../schema/timeline";
/**
* 镜头监视器:**机位画面的唯一渲染源**。
*
* - 导演视角下:它是一块独立的小窗(`right` 偏移悬浮在画布右上角),随时显示当前机位看到的内容;
* - 机位视角下:它直接顶到主视口那块画幅的位置上(`.lens-monitor.is-viewport-camera`,尺寸取
* `resolveCameraViewRenderRect` 算出的矩形),**机位视角显示的就是这块 canvas 的像素** ——
* 同一份渲染、同一个画幅,所以「机位视角 == 监视器 == 导出」不可能再出现取景 / 虚化不一致,
* 主视口那边也不必重复跑一遍完整的景深 composer + 深度 pass(省掉约一半开销)。
*
* 复用 SceneRoot 渲染同一份场景(不带编辑器辅助),按传感器宽高比裁切画幅、
* 用 ViewportBackground 渲染与导出一致的全景背景,通过共享的 renderSceneWithBokeh 反映景深虚化。
*
* 「当前机位」= 时间轴此刻正在显示的机位(多机位自动切换 / 过渡时会变),与视频导出用同一个
* getActiveCameraIdAtTime 解析;位姿、镜头(焦段/光圈/对焦/景深强度)、画幅全部取自这台机位,
* 保证监视器所见与导出的机位视角严格一致。
*
* 刻意做成独立的第二个 Canvas:景深后处理只走这一条管线,主视口 / 截图不必各写一套。
*/
function LensMonitorCamera({
camera,
sceneSettings,
sceneRef,
camRef,
cameraTracks,
}: {
camera: DirectorCameraShot;
sceneSettings: SceneSettings;
sceneRef: React.MutableRefObject<Scene | null>;
camRef: React.MutableRefObject<ThreePerspectiveCamera | null>;
cameraTracks: CameraTimelineTrack[];
}) {
const { gl, scene } = useThree();
const cameraRef = useRef<ThreePerspectiveCamera>(null);
// 把监视器自己的场景 / 渲染相机暴露给外层 DOM 事件,供「触屏对焦」射线拾取使用
sceneRef.current = scene;
const cameras = useDirectorStore((state) => state.project.cameras);
const transitionClips = useDirectorStore((state) => state.project.timeline.transitionClips);
const globalTransition = useDirectorStore((state) => state.project.timeline.cameraTransition);
const isExporting = usePlaybackStore((state) => state.isExporting);
// playhead 自己订阅(而不是从外层当 prop 传进来):外壳组件不必因为这个每秒变几十次的值
// 去 reconcile 整棵场景树(见 LensMonitorCanvas 的 memo)。
const playhead = usePlaybackStore((state) => state.playhead);
// 拖动播放头期间走草稿档:景深照常渲染(移焦 / 变焦要看得到),只把模糊采样点数降到最低档
const isScrubbing = usePlaybackStore((state) => state.isScrubbing);
// 关键帧感知:拖播放头时,监视器上的对焦距离 / 光圈 / 焦段 / 景深强度都按关键帧插值,
// 让「移焦 / 渐入渐出虚化 / 变焦」在监视器上所见即所得(与导出管线一致的 resolveLensAtTime)。
const lens = useMemo(
() => resolveLensAtTime(camera, playhead, cameraTracks),
[camera, playhead, cameraTracks]
);
// 与导出共用 evaluateCameraViewAtTime:让监视器随运镜关键帧 / 多机位过渡实时动起来,
// 渲染的位姿(含场景变换下的世界坐标换算)与导出的机位视角严格一致。
// 无运镜轨道时回落到机位静态视角,行为和以前一样。
const worldView = useMemo(() => {
const getStaticPose = (cameraId: string): CameraKeyframeValue | null => {
const shot = cameras.find((item) => item.id === cameraId);
return shot ? getWorldCameraViewSnapshot(getCameraViewSnapshotFromShot(shot), sceneSettings) : null;
};
const view = evaluateCameraViewAtTime(
transitionClips,
cameraTracks,
globalTransition,
camera.id,
playhead,
getStaticPose
);
if (!view) return getWorldCameraViewSnapshot(getCameraViewSnapshotFromShot(camera), sceneSettings);
return getWorldCameraViewSnapshot(
{ fov: view.pose.fov, position: view.pose.position, lookRotation: view.pose.lookRotation },
sceneSettings
);
}, [camera, cameras, cameraTracks, transitionClips, globalTransition, playhead, sceneSettings]);
const sensorAspect = lens.sensor.widthMm / lens.sensor.heightMm;
// 把监视器的渲染三件套注册给视频导出:机位视角导出直接用这块画布 / 这颗相机 / 这个场景出片,
// 因此导出的画面与监视器所见是同一份渲染,逐像素一致。
useLayoutEffect(() => {
const renderCamera = cameraRef.current;
if (!renderCamera) return undefined;
setMonitorRenderContext({ gl, scene, camera: renderCamera });
return () => setMonitorRenderContext(null);
}, [gl, scene]);
useFrame(() => {
const renderCamera = cameraRef.current;
if (!renderCamera || !worldView) return;
// 把当前机位的世界位姿 + 由焦段/画幅换算的 FOV 写到渲染相机上,并按传感器比例裁切画幅
renderCamera.position.set(...worldView.position);
renderCamera.quaternion.copy(getCameraRenderQuaternion(worldView.lookRotation));
if (Math.abs(renderCamera.fov - lens.fov) > 0.001) {
renderCamera.fov = lens.fov;
}
if (Math.abs(renderCamera.aspect - sensorAspect) > 1e-4) {
renderCamera.aspect = sensorAspect;
}
renderCamera.updateProjectionMatrix();
camRef.current = renderCamera;
// 导出期间由 videoExport 在这块画布上逐帧渲染(同一颗 gl / 同一个场景 / 同一颗相机),
// 这里跳过,避免和导出循环双重渲染、也避免争用同一份景深合成器缓存。
if (isExporting) return;
// 与主视口 / 视频导出共用同一份景深渲染(renderSceneWithBokeh):监视器不再自建一套
// EffectComposer —— 两套实现很容易漂移,导出与监视器的取景 / 虚化就会对不上。
// 合成器按 gl 缓存、画布尺寸变化会自动重建;不支持后处理时回退普通渲染,监视器永远可用。
// 拖动播放头时传 draft:景深仍然整套跑(拖动期间「移焦 / 变焦 / 虚化强弱」必须看得见),
// 只是把模糊采样点数压到最低档省开销,松手立刻回到自适应档。
renderSceneWithBokeh(gl, scene, renderCamera, lens, undefined, { draft: isScrubbing });
}, 1);
return (
<DreiPerspectiveCamera
ref={cameraRef}
makeDefault
fov={lens.fov}
position={worldView ? worldView.position : [0, 1, 5]}
quaternion={worldView ? (getCameraRenderQuaternion(worldView.lookRotation).toArray() as [number, number, number, number]) : undefined}
/>
);
}
/**
* 监视器那块 3D 画布:整棵场景树(SceneRoot 会把当前工程所有物体铺进来)。
*
* 单独抽成 memo 组件的原因:拖动播放头时 playhead 每秒变几十次,如果这棵树长在外壳组件里,
* 每动一下都要把整棵场景树 reconcile 一遍 —— 那是拖动卡顿的一大来源。这里 props 只在
* 「换机位 / 改场景 / 换全景图」这类真实编辑时变,播放头由 LensMonitorCamera 自己订阅。
*/
const LensMonitorCanvas = memo(function LensMonitorCanvas({
camera,
sceneSettings,
panoramaAsset,
cameraTracks,
sceneRef,
camRef,
initialCamera,
}: {
camera: DirectorCameraShot;
sceneSettings: SceneSettings;
panoramaAsset: React.ComponentProps<typeof ViewportBackground>["panoramaAsset"];
cameraTracks: CameraTimelineTrack[];
sceneRef: React.MutableRefObject<Scene | null>;
camRef: React.MutableRefObject<ThreePerspectiveCamera | null>;
/** 画布的初始相机参数:只取首次渲染那一刻的值(位姿 / fov 由 LensMonitorCamera 每帧写) */
initialCamera: { fov: number; position: [number, number, number] };
}) {
return (
<Canvas
className="lens-monitor-canvas"
// 这块画布在机位视角下就是主视口那块画幅(尺寸跟着画幅走),所以 dpr 上限收一档:
// 大画布上跑到 2x 会让「景深 composer + 深度 pass」的像素量翻倍,是之前发卡的主因;
// 1.5x 在高分屏上肉眼几乎无差别,开销却少 ~45%。导出不受影响(导出自己设 1x + 导出分辨率)。
dpr={[1, 1.5]}
// preserveDrawingBuffer:机位视角导出就是把这块画布的内容逐帧读进 VideoFrame,
// 不保留绘画缓冲会读到空帧。
gl={{ antialias: true, preserveDrawingBuffer: true }}
camera={initialCamera}
>
<color attach="background" args={[sceneSettings.backgroundColor]} />
{/* 与主视口 / 导出共用同一份背景实现:有全景图时渲染全景(equirect 走 scene.background、
backdrop 走穹顶网格),否则用背景色。少了它,监视器会和导出画面差一个背景。 */}
<ViewportBackground
backgroundColor={sceneSettings.backgroundColor}
panoramaAsset={panoramaAsset}
panoramaRadius={sceneSettings.panoramaRadius}
panoramaYaw={sceneSettings.panoramaYaw}
/>
<ambientLight intensity={1.15} />
<directionalLight intensity={1.2} position={[8, 10, 6]} />
{/* 与主视口 / 视频导出共用同一份时间轴变换:把物体 / 角色关键帧的插值写进监视器场景。
SceneRoot 自身只取 item.transform 和人物轨迹,不应用时间轴轨道 —— 少了这一层,
监视器里的物体就不会随关键帧运动,物体到相机的深度和导出不同,
清晰平面落在别处,看起来就是「对焦距离和导出对不上」。 */}
<PlaybackTransformSync />
<LensMonitorCamera
camera={camera}
sceneSettings={sceneSettings}
sceneRef={sceneRef}
camRef={camRef}
cameraTracks={cameraTracks}
/>
<SceneRoot editorHelpers={false} />
</Canvas>
);
});
/** 鼠标每像素对应的角度增量(度) */
const LOOK_DRAG_SENSITIVITY = 0.3;
/** 俯仰角夹到 ±89°,避免越过天顶后镜头翻转 */
const PITCH_LIMIT = 89;
/** 区分「单击对焦」与「拖拽瞄准」的位移阈值(像素) */
const CLICK_THRESHOLD_PX = 4;
export function LensMonitor({ rightOffset = 12 }: { rightOffset?: number }) {
const sceneSettings = useDirectorStore((state) => state.project.scene);
const assets = useDirectorStore((state) => state.project.assets);
const panoramaAssetId = useDirectorStore((state) => state.project.panoramaAssetId);
const panoramaAsset = assets.find((item) => item.id === panoramaAssetId);
const activeCamera = useDirectorStore((state) =>
state.project.cameras.find((item) => item.id === state.project.activeCameraId)
);
const updateCameraLookRotationAtTime = useDirectorStore((state) => state.updateCameraLookRotationAtTime);
const updateCameraFocusDistanceAtTime = useDirectorStore((state) => state.updateCameraFocusDistanceAtTime);
const beginUndoBatch = useDirectorStore((state) => state.beginUndoBatch);
const endUndoBatch = useDirectorStore((state) => state.endUndoBatch);
const playhead = usePlaybackStore((state) => state.playhead);
const isPlaying = usePlaybackStore((state) => state.isPlaying);
const isScrubbing = usePlaybackStore((state) => state.isScrubbing);
const isExporting = usePlaybackStore((state) => state.isExporting);
const cameras = useDirectorStore((state) => state.project.cameras);
const transitionClips = useDirectorStore((state) => state.project.timeline.transitionClips);
const cameraTracks = useDirectorStore((state) => state.project.timeline.cameraTracks);
// 当前时刻「正在显示」的机位:与机位视角(ActiveCameraPlaybackSync)用同一套选取逻辑 ——
// 播放 / 导出 / scrub 时按时间轴关键帧自动切(也与导出的 getActiveCameraIdAtTime 一致),
// 静止编辑时沿用手动选中的机位。机位视角和监视器因此永远显示同一台机位,不会出现
// 「主视口是手选的 A 机位、监视器却是时间轴上的 B 机位」这种画面 / 虚化都对不上的情况。
const autoCut = isPlaying || isExporting || isScrubbing;
const displayCameraId = useMemo(
() =>
autoCut
? getActiveCameraIdAtTime(transitionClips, cameraTracks, activeCamera?.id ?? null, playhead)
: (activeCamera?.id ?? null),
[autoCut, transitionClips, cameraTracks, activeCamera?.id, playhead]
);
const camera = cameras.find((item) => item.id === displayCameraId) ?? activeCamera;
// 小字镜头信息也按播放头解析(与画面渲染同一份 resolveLensAtTime),点击/拖到关键帧时数字同步刷新
const lens = camera ? resolveLensAtTime(camera, playhead, cameraTracks) : null;
const sensorAspect = lens ? lens.sensor.widthMm / lens.sensor.heightMm : 16 / 9;
const viewMode = useDirectorStore((state) => state.viewMode);
// 机位视角下,这块监视器直接顶到主视口「机位画面」的位置上,成为那块画幅本身:
// 机位视角显示的**就是这块 canvas 的像素**(同一份渲染、同一个画幅),
// 于是「机位视角 == 镜头监视器 == 导出」天然一致,不会再出现取景 / 虚化两边对不上;
// 主视口那边同时不再重复渲染一遍完整景深管线(少一半开销,之前又卡又糊的主因)。
const rootRef = useRef<HTMLDivElement | null>(null);
const [viewportFrame, setViewportFrame] = useState<{ left: number; top: number; width: number; height: number } | null>(
null
);
useLayoutEffect(() => {
// 画幅是相对「画布」等比适配出来的,而这块监视器的父元素正是 .canvas-frame(position: relative)
const frame = rootRef.current?.parentElement ?? null;
if (!frame) return undefined;
let animationFrameId = 0;
const measure = () => {
const rect = resolveCameraViewRenderRect(frame.clientWidth, frame.clientHeight, sensorAspect);
setViewportFrame((current) =>
current && rect && current.left === rect.left && current.top === rect.top && current.width === rect.width && current.height === rect.height
? current
: rect
);
};
const scheduleMeasure = () => {
cancelAnimationFrame(animationFrameId);
animationFrameId = requestAnimationFrame(measure);
};
measure();
window.addEventListener("resize", scheduleMeasure);
if (typeof ResizeObserver === "undefined") {
return () => {
cancelAnimationFrame(animationFrameId);
window.removeEventListener("resize", scheduleMeasure);
};
}
const resizeObserver = new ResizeObserver(scheduleMeasure);
resizeObserver.observe(frame);
return () => {
cancelAnimationFrame(animationFrameId);
window.removeEventListener("resize", scheduleMeasure);
resizeObserver.disconnect();
};
}, [sensorAspect]);
const viewportCameraFrame = viewMode === "camera" && camera ? viewportFrame : null;
// 画布初始相机参数只在首次渲染那一刻取一次:位姿 / fov 每帧都由 LensMonitorCamera 写,
// 保持这个对象引用稳定,<Canvas> 才不会因为播放头变化被重新应用 camera props。
const initialCanvasCameraRef = useRef<{ fov: number; position: [number, number, number] } | null>(null);
initialCanvasCameraRef.current ??= { fov: lens?.fov ?? 50, position: [0, 1, 5] };
const initialCanvasCamera = initialCanvasCameraRef.current;
const [dragging, setDragging] = useState(false);
const dragRef = useRef<{
startX: number;
startY: number;
startLook: [number, number, number];
rollMode: boolean;
} | null>(null);
// 触屏对焦需要的监视器场景 / 渲染相机引用(由 LensMonitorCamera 每帧回填)
const monitorSceneRef = useRef<Scene | null>(null);
const monitorCamRef = useRef<ThreePerspectiveCamera | null>(null);
const stageRef = useRef<HTMLDivElement | null>(null);
// 在监视器画面上「按住拖动」改变机位注视角度:普通拖动改水平角 + 俯仰角;Shift+拖动改横滚角。
// 用基础机位的 lookRotation 作为起点累加位移,避免逐帧累积误差;写入走 updateCameraLookRotationAtTime,
// 由它决定写基础机位还是最近关键帧(关键帧感知),并在拖拽起止用 undo 批次合并成单条撤销记录。
const handlePointerDown = (event: React.PointerEvent<HTMLDivElement>) => {
if (!camera || event.button !== 0) return;
event.currentTarget.setPointerCapture(event.pointerId);
dragRef.current = {
startX: event.clientX,
startY: event.clientY,
startLook: [...camera.lookRotation] as [number, number, number],
rollMode: event.shiftKey,
};
setDragging(true);
beginUndoBatch();
};
const handlePointerMove = (event: React.PointerEvent<HTMLDivElement>) => {
const drag = dragRef.current;
if (!drag || !camera) return;
const dx = event.clientX - drag.startX;
const dy = event.clientY - drag.startY;
const [pitch, yaw, roll] = drag.startLook;
const next: [number, number, number] = drag.rollMode
? [pitch, yaw, roll + dx * LOOK_DRAG_SENSITIVITY]
: [
Math.max(-PITCH_LIMIT, Math.min(PITCH_LIMIT, pitch - dy * LOOK_DRAG_SENSITIVITY)),
yaw + dx * LOOK_DRAG_SENSITIVITY,
roll,
];
updateCameraLookRotationAtTime(camera.id, next, playhead);
};
// 监视器触屏对焦:在已选中机位的前提下,单击画面某点(非拖拽)时,
// 从渲染相机发出射线拾取场景,取命中点到相机的距离写入对焦距离。
const handleTouchFocus = (clientX: number, clientY: number) => {
if (!camera) return;
const scene = monitorSceneRef.current;
const renderCamera = monitorCamRef.current;
const stage = stageRef.current;
if (!scene || !renderCamera || !stage) return;
const rect = stage.getBoundingClientRect();
const ndcX = ((clientX - rect.left) / rect.width) * 2 - 1;
const ndcY = -((clientY - rect.top) / rect.height) * 2 + 1;
const raycaster = new Raycaster();
raycaster.setFromCamera(new Vector2(ndcX, ndcY), renderCamera);
const hits = raycaster.intersectObjects(scene.children, true);
if (hits.length === 0) return;
const distance = Math.max(0.1, Math.min(100, hits[0].distance));
updateCameraFocusDistanceAtTime(camera.id, distance, playhead);
};
const endDrag = (event: React.PointerEvent<HTMLDivElement>) => {
const drag = dragRef.current;
if (!drag) return;
if (event.currentTarget.hasPointerCapture(event.pointerId)) {
event.currentTarget.releasePointerCapture(event.pointerId);
}
// 位移极小且无右键 => 视为单击,触发触屏对焦;否则只是结束拖拽瞄准
const moved = Math.hypot(event.clientX - drag.startX, event.clientY - drag.startY);
const wasClick = moved <= CLICK_THRESHOLD_PX && event.button === 0;
dragRef.current = null;
setDragging(false);
endUndoBatch();
if (wasClick) handleTouchFocus(event.clientX, event.clientY);
};
return (
<div
ref={rootRef}
className={`lens-monitor${viewportCameraFrame ? " is-viewport-camera" : ""}`}
aria-label="镜头监视器"
style={
viewportCameraFrame
? {
left: `${viewportCameraFrame.left}px`,
top: `${viewportCameraFrame.top}px`,
width: `${viewportCameraFrame.width}px`,
}
: { right: `${rightOffset}px` }
}
>
<div
className="lens-monitor-stage"
ref={stageRef}
style={{
aspectRatio: String(sensorAspect),
// 机位视角下高度也要精确落在画幅上(宽高由同一个矩形算出来,避免 aspect 取整误差留下细缝)
height: viewportCameraFrame ? `${viewportCameraFrame.height}px` : undefined,
cursor: dragging ? "grabbing" : "grab",
}}
title="拖拽改变机位注视角度(Shift+拖拽旋转横滚角);单击画面中某点使其对焦清晰"
onPointerDown={handlePointerDown}
onPointerMove={handlePointerMove}
onPointerUp={endDrag}
onPointerCancel={endDrag}
>
{camera && lens ? (
<LensMonitorCanvas
camera={camera}
cameraTracks={cameraTracks}
camRef={monitorCamRef}
initialCamera={initialCanvasCamera}
panoramaAsset={panoramaAsset}
sceneRef={monitorSceneRef}
sceneSettings={sceneSettings}
/>
) : (
<div className="lens-monitor-empty">无激活机位</div>
)}
</div>
<div className="lens-monitor-info">
{lens ? formatLensInfo(lens) : "无激活机位"}
</div>
</div>
);
}