feat(camera): 机位视角增加景深虚化与镜头参数编辑

- 截图 / 视频导出在机位视角下走 bokeh 后处理渲染,按传感器画幅裁切
- 镜头监视器复用同一套解析与渲染,隐藏编辑器辅助元素
- 镜头面板支持焦段、画幅、光圈、对焦、虚化编辑,并按播放头写入关键帧
- 位置 / 角度编辑同步进选中关键帧,锁定物体时回填实测对焦距离
- 轨迹点支持整体锁定与隐藏,角色姿势关键帧按播放头插值
This commit is contained in:
tongtong committed 2026-09-19 21:11:47 +08:00
1 parent 91fcb6841a
commit 7e28c73643
23 files changed
+1968 -229

No files matched your search

+356
View File
@@ -0,0 +1,356 @@
import { Canvas, useFrame, useThree } from "@react-three/fiber";
import { PerspectiveCamera as DreiPerspectiveCamera } from "@react-three/drei";
import { useEffect, useMemo, useRef, useState } from "react";
import { PerspectiveCamera as ThreePerspectiveCamera, Raycaster, Scene, Vector2 } from "three";
import { useDirectorStore } from "../store/directorStore";
import { usePlaybackStore } from "../runtime/playbackStore";
import { SceneRoot } from "./SceneRoot";
import { ViewportBackground } from "./ViewportBackground";
import { PlaybackTransformSync } from "./PlaybackTransformSync";
import { renderSceneWithBokeh } from "./viewportBokeh";
import {
formatLensInfo,
getCameraRenderQuaternion,
getCameraViewSnapshotFromShot,
getWorldCameraViewSnapshot,
} from "../schema/cameraGeometry";
import { resolveLensAtTime } from "../runtime/playbackTransforms";
import type { DirectorCameraShot, SceneSettings } from "../schema/directorProject";
import type { CameraKeyframeValue, CameraTimelineTrack } from "../schema/timeline";
import { evaluateCameraViewAtTime, getActiveCameraIdAtTime } from "../schema/timeline";
/**
* 镜头监视器:独立于主视口(即便主视口是「导演视角」也照常显示当前机位看到的内容)。
* 复用 SceneRoot 渲染同一份场景(不带编辑器辅助),并按传感器宽高比裁切画幅、
* 用 ViewportBackground 渲染与主视口 / 导出一致的全景背景,通过共享的 renderSceneWithBokeh
* 反映光圈带来的景深虚化。UI 上的小字显示当前镜头参数。
*
* 「当前机位」= 时间轴此刻正在显示的机位(多机位自动切换 / 过渡时会变),与视频导出用同一个
* getActiveCameraIdAtTime 解析;位姿、镜头(焦段/光圈/对焦/景深强度)、画幅全部取自这台机位,
* 保证监视器所见与导出的机位视角严格一致。
*
* 刻意做成独立的第二个 Canvas:景深后处理只作用在监视器上,不影响主视口 / 截图的渲染管线。
*/
function LensMonitorCamera({
camera,
sceneSettings,
sceneRef,
camRef,
playhead,
cameraTracks,
}: {
camera: DirectorCameraShot;
sceneSettings: SceneSettings;
sceneRef: React.MutableRefObject<Scene | null>;
camRef: React.MutableRefObject<ThreePerspectiveCamera | null>;
playhead: number;
cameraTracks: CameraTimelineTrack[];
}) {
const { gl, scene } = useThree();
const cameraRef = useRef<ThreePerspectiveCamera>(null);
// 把监视器自己的场景 / 渲染相机暴露给外层 DOM 事件,供「触屏对焦」射线拾取使用
sceneRef.current = scene;
const cameras = useDirectorStore((state) => state.project.cameras);
const transitionClips = useDirectorStore((state) => state.project.timeline.transitionClips);
const globalTransition = useDirectorStore((state) => state.project.timeline.cameraTransition);
// 关键帧感知:拖播放头时,监视器上的对焦距离 / 光圈 / 焦段 / 景深强度都按关键帧插值,
// 让「移焦 / 渐入渐出虚化 / 变焦」在监视器上所见即所得(与导出管线一致的 resolveLensAtTime)。
const lens = useMemo(
() => resolveLensAtTime(camera, playhead, cameraTracks),
[camera, playhead, cameraTracks]
);
// 与导出共用 evaluateCameraViewAtTime:让监视器随运镜关键帧 / 多机位过渡实时动起来,
// 渲染的位姿(含场景变换下的世界坐标换算)与导出的机位视角严格一致。
// 无运镜轨道时回落到机位静态视角,行为和以前一样。
const worldView = useMemo(() => {
const getStaticPose = (cameraId: string): CameraKeyframeValue | null => {
const shot = cameras.find((item) => item.id === cameraId);
return shot ? getWorldCameraViewSnapshot(getCameraViewSnapshotFromShot(shot), sceneSettings) : null;
};
const view = evaluateCameraViewAtTime(
transitionClips,
cameraTracks,
globalTransition,
camera.id,
playhead,
getStaticPose
);
if (!view) return getWorldCameraViewSnapshot(getCameraViewSnapshotFromShot(camera), sceneSettings);
return getWorldCameraViewSnapshot(
{ fov: view.pose.fov, position: view.pose.position, lookRotation: view.pose.lookRotation },
sceneSettings
);
}, [camera, cameras, cameraTracks, transitionClips, globalTransition, playhead, sceneSettings]);
const sensorAspect = lens.sensor.widthMm / lens.sensor.heightMm;
useFrame(() => {
const renderCamera = cameraRef.current;
if (!renderCamera || !worldView) return;
// 把当前机位的世界位姿 + 由焦段/画幅换算的 FOV 写到渲染相机上,并按传感器比例裁切画幅
renderCamera.position.set(...worldView.position);
renderCamera.quaternion.copy(getCameraRenderQuaternion(worldView.lookRotation));
if (Math.abs(renderCamera.fov - lens.fov) > 0.001) {
renderCamera.fov = lens.fov;
}
if (Math.abs(renderCamera.aspect - sensorAspect) > 1e-4) {
renderCamera.aspect = sensorAspect;
}
renderCamera.updateProjectionMatrix();
camRef.current = renderCamera;
// 与主视口 / 视频导出共用同一份景深渲染(renderSceneWithBokeh):监视器不再自建一套
// EffectComposer —— 两套实现很容易漂移,导出与监视器的取景 / 虚化就会对不上。
// 合成器按 gl 缓存、画布尺寸变化会自动重建;不支持后处理时回退普通渲染,监视器永远可用。
renderSceneWithBokeh(gl, scene, renderCamera, lens);
}, 1);
return (
<DreiPerspectiveCamera
ref={cameraRef}
makeDefault
fov={lens.fov}
position={worldView ? worldView.position : [0, 1, 5]}
quaternion={worldView ? (getCameraRenderQuaternion(worldView.lookRotation).toArray() as [number, number, number, number]) : undefined}
/>
);
}
/** 鼠标每像素对应的角度增量(度) */
const LOOK_DRAG_SENSITIVITY = 0.3;
/** 俯仰角夹到 ±89°,避免越过天顶后镜头翻转 */
const PITCH_LIMIT = 89;
/** 区分「单击对焦」与「拖拽瞄准」的位移阈值(像素) */
const CLICK_THRESHOLD_PX = 4;
/**
* 机位视角下监视器的「镜像」:主视口此刻渲染的就是机位画面,监视器没必要再开一个 Canvas
* 把整个场景重渲一遍(第二份场景 + 第二套景深后处理,既卡,又容易和主视口取景 / 虚化漂移)。
* 这里直接把主视口画布按「传感器画幅」居中裁一块拷过来,逐像素与主视口一致。
*
* fov 是垂直的:视口比传感器画幅更宽时按宽度裁,垂直取景保持不变 ——
* 结果与监视器自己用传感器画幅渲染完全一致。
*/
function MonitorMirror({ sensorAspect }: { sensorAspect: number }) {
const canvasRef = useRef<HTMLCanvasElement | null>(null);
useEffect(() => {
let frame = 0;
const draw = () => {
frame = requestAnimationFrame(draw);
const target = canvasRef.current;
const source = document.querySelector<HTMLCanvasElement>(".director-canvas canvas");
if (!target || !source) return;
// 输出尺寸跟随容器(CSS 像素)
const cssWidth = target.clientWidth;
const cssHeight = target.clientHeight;
if (!cssWidth || !cssHeight) return;
if (target.width !== cssWidth || target.height !== cssHeight) {
target.width = cssWidth;
target.height = cssHeight;
}
const sourceWidth = source.width;
const sourceHeight = source.height;
if (!sourceWidth || !sourceHeight) return;
let cropWidth = sourceWidth;
let cropHeight = Math.round(sourceWidth / sensorAspect);
if (cropHeight > sourceHeight) {
cropHeight = sourceHeight;
cropWidth = Math.round(sourceHeight * sensorAspect);
}
const context = target.getContext("2d");
if (!context) return;
context.drawImage(
source,
Math.round((sourceWidth - cropWidth) / 2),
Math.round((sourceHeight - cropHeight) / 2),
cropWidth,
cropHeight,
0,
0,
target.width,
target.height
);
};
frame = requestAnimationFrame(draw);
return () => cancelAnimationFrame(frame);
}, [sensorAspect]);
return <canvas className="lens-monitor-canvas" ref={canvasRef} />;
}
export function LensMonitor({ rightOffset = 12 }: { rightOffset?: number }) {
const sceneSettings = useDirectorStore((state) => state.project.scene);
const viewMode = useDirectorStore((state) => state.viewMode);
const assets = useDirectorStore((state) => state.project.assets);
const panoramaAssetId = useDirectorStore((state) => state.project.panoramaAssetId);
const panoramaAsset = assets.find((item) => item.id === panoramaAssetId);
const activeCamera = useDirectorStore((state) =>
state.project.cameras.find((item) => item.id === state.project.activeCameraId)
);
const updateCameraLookRotationAtTime = useDirectorStore((state) => state.updateCameraLookRotationAtTime);
const updateCameraFocusDistanceAtTime = useDirectorStore((state) => state.updateCameraFocusDistanceAtTime);
const beginUndoBatch = useDirectorStore((state) => state.beginUndoBatch);
const endUndoBatch = useDirectorStore((state) => state.endUndoBatch);
const playhead = usePlaybackStore((state) => state.playhead);
const cameras = useDirectorStore((state) => state.project.cameras);
const transitionClips = useDirectorStore((state) => state.project.timeline.transitionClips);
const cameraTracks = useDirectorStore((state) => state.project.timeline.cameraTracks);
// 当前时刻「正在显示」的机位:与视频导出用同一个 getActiveCameraIdAtTime 解析。
// 多机位自动切换 / 过渡时,监视器画面就是这台机位;它生效的镜头(焦段 / 光圈 / 对焦 / 景深强度)
// 和画幅也必须取这台机位 —— 否则会出现「画面是 B 机位、虚化却用了激活的 A 机位」,
// 导出(按显示机位解析镜头)就会和监视器对不上。
const displayCameraId = useMemo(
() => getActiveCameraIdAtTime(transitionClips, cameraTracks, activeCamera?.id ?? null, playhead),
[transitionClips, cameraTracks, activeCamera?.id, playhead]
);
const camera = cameras.find((item) => item.id === displayCameraId) ?? activeCamera;
// 小字镜头信息也按播放头解析(与画面渲染同一份 resolveLensAtTime),点击/拖到关键帧时数字同步刷新
const lens = camera ? resolveLensAtTime(camera, playhead, cameraTracks) : null;
const sensorAspect = lens ? lens.sensor.widthMm / lens.sensor.heightMm : 16 / 9;
const [dragging, setDragging] = useState(false);
const dragRef = useRef<{
startX: number;
startY: number;
startLook: [number, number, number];
rollMode: boolean;
} | null>(null);
// 触屏对焦需要的监视器场景 / 渲染相机引用(由 LensMonitorCamera 每帧回填)
const monitorSceneRef = useRef<Scene | null>(null);
const monitorCamRef = useRef<ThreePerspectiveCamera | null>(null);
const stageRef = useRef<HTMLDivElement | null>(null);
// 在监视器画面上「按住拖动」改变机位注视角度:普通拖动改水平角 + 俯仰角;Shift+拖动改横滚角。
// 用基础机位的 lookRotation 作为起点累加位移,避免逐帧累积误差;写入走 updateCameraLookRotationAtTime,
// 由它决定写基础机位还是最近关键帧(关键帧感知),并在拖拽起止用 undo 批次合并成单条撤销记录。
const handlePointerDown = (event: React.PointerEvent<HTMLDivElement>) => {
if (!camera || event.button !== 0) return;
event.currentTarget.setPointerCapture(event.pointerId);
dragRef.current = {
startX: event.clientX,
startY: event.clientY,
startLook: [...camera.lookRotation] as [number, number, number],
rollMode: event.shiftKey,
};
setDragging(true);
beginUndoBatch();
};
const handlePointerMove = (event: React.PointerEvent<HTMLDivElement>) => {
const drag = dragRef.current;
if (!drag || !camera) return;
const dx = event.clientX - drag.startX;
const dy = event.clientY - drag.startY;
const [pitch, yaw, roll] = drag.startLook;
const next: [number, number, number] = drag.rollMode
? [pitch, yaw, roll + dx * LOOK_DRAG_SENSITIVITY]
: [
Math.max(-PITCH_LIMIT, Math.min(PITCH_LIMIT, pitch - dy * LOOK_DRAG_SENSITIVITY)),
yaw + dx * LOOK_DRAG_SENSITIVITY,
roll,
];
updateCameraLookRotationAtTime(camera.id, next, playhead);
};
// 监视器触屏对焦:在已选中机位的前提下,单击画面某点(非拖拽)时,
// 从渲染相机发出射线拾取场景,取命中点到相机的距离写入对焦距离。
const handleTouchFocus = (clientX: number, clientY: number) => {
if (!camera) return;
const scene = monitorSceneRef.current;
const renderCamera = monitorCamRef.current;
const stage = stageRef.current;
if (!scene || !renderCamera || !stage) return;
const rect = stage.getBoundingClientRect();
const ndcX = ((clientX - rect.left) / rect.width) * 2 - 1;
const ndcY = -((clientY - rect.top) / rect.height) * 2 + 1;
const raycaster = new Raycaster();
raycaster.setFromCamera(new Vector2(ndcX, ndcY), renderCamera);
const hits = raycaster.intersectObjects(scene.children, true);
if (hits.length === 0) return;
const distance = Math.max(0.1, Math.min(100, hits[0].distance));
updateCameraFocusDistanceAtTime(camera.id, distance, playhead);
};
const endDrag = (event: React.PointerEvent<HTMLDivElement>) => {
const drag = dragRef.current;
if (!drag) return;
if (event.currentTarget.hasPointerCapture(event.pointerId)) {
event.currentTarget.releasePointerCapture(event.pointerId);
}
// 位移极小且无右键 => 视为单击,触发触屏对焦;否则只是结束拖拽瞄准
const moved = Math.hypot(event.clientX - drag.startX, event.clientY - drag.startY);
const wasClick = moved <= CLICK_THRESHOLD_PX && event.button === 0;
dragRef.current = null;
setDragging(false);
endUndoBatch();
if (wasClick) handleTouchFocus(event.clientX, event.clientY);
};
return (
<div className="lens-monitor" aria-label="镜头监视器" style={{ right: `${rightOffset}px` }}>
<div
className="lens-monitor-stage"
ref={stageRef}
style={{ aspectRatio: String(sensorAspect), cursor: dragging ? "grabbing" : "grab" }}
title="拖拽改变机位注视角度(Shift+拖拽旋转横滚角);单击画面中某点使其对焦清晰"
onPointerDown={handlePointerDown}
onPointerMove={handlePointerMove}
onPointerUp={endDrag}
onPointerCancel={endDrag}
>
{camera && lens ? (
// 机位视角下主视口本身就是机位画面 → 监视器直接镜像它,不再开第二个 Canvas 重渲整个场景
//(省掉一份场景渲染 + 一套景深后处理,而且与主视口逐像素一致)。
// 其它视角(例如导演视角下想同时看机位画面)才需要监视器自己渲染。
viewMode === "camera" ? (
<MonitorMirror sensorAspect={sensorAspect} />
) : (
<Canvas
className="lens-monitor-canvas"
dpr={[1, 1.5]}
gl={{ antialias: true }}
camera={{ fov: lens.fov, position: [0, 1, 5] }}
>
<color attach="background" args={[sceneSettings.backgroundColor]} />
{/* 与主视口 / 导出共用同一份背景实现:有全景图时渲染全景(equirect 走 scene.background、
backdrop 走穹顶网格),否则用背景色。少了它,监视器会和导出画面差一个背景。 */}
<ViewportBackground
backgroundColor={sceneSettings.backgroundColor}
panoramaAsset={panoramaAsset}
panoramaRadius={sceneSettings.panoramaRadius}
panoramaYaw={sceneSettings.panoramaYaw}
/>
<ambientLight intensity={1.15} />
<directionalLight intensity={1.2} position={[8, 10, 6]} />
{/* 与主视口 / 视频导出共用同一份时间轴变换:把物体 / 角色关键帧的插值写进监视器场景。
SceneRoot 自身只取 item.transform 和人物轨迹,不应用时间轴轨道 —— 少了这一层,
监视器里的物体就不会随关键帧运动,物体到相机的深度和导出不同,
清晰平面落在别处,看起来就是「对焦距离和导出对不上」。 */}
<PlaybackTransformSync />
<LensMonitorCamera
camera={camera}
sceneSettings={sceneSettings}
sceneRef={monitorSceneRef}
camRef={monitorCamRef}
playhead={playhead}
cameraTracks={cameraTracks}
/>
<SceneRoot editorHelpers={false} />
</Canvas>
)
) : (
<div className="lens-monitor-empty">无激活机位</div>
)}
</div>
<div className="lens-monitor-info">
{lens ? formatLensInfo(lens) : "无激活机位"}
</div>
</div>
);
}