fix(canvas): 3D 导演台 - 修复白膜视频导出渲染循环逐帧崩溃 (#439)
导出白膜视频得到的是 0 秒视频：文件实测仅 1 帧（ffprobe nb_read_frames=1，duration=0.001s），却无任何报错并照常回写画布。 根因：applyClaySceneMaterials 把场景内所有非 actor mesh 的材质换成 clay MeshStandardMaterial，drei Grid 的 GridMaterial（带 uniforms 的 ShaderMaterial）也在其中。Grid 组件内部 useFrame 每帧读取 material.uniforms.worldCamProjPosition，材质被换后 uniforms 为 undefined，渲染循环逐帧抛 TypeError，R3F 不再产出新帧， captureStream 只录到启动前手动 gl.render 的那一帧；5 秒后 setTimeout 照常 stop，残缺视频被当作成功结果。开启轨迹显示 （drei Line 同为 uniforms 材质）或视口白膜显示模式时同样触发。 修复： - applyClaySceneMaterials 跳过带 uniforms 的 ShaderMaterial， 并抽到 lib/canvas/director/director-clay-materials.ts 补回归测试； - recordCanvas 加固：录制期间监听 window 未捕获错误，首个渲染 错误立即中止并报错；录制结束用 <video> 探测真实时长（Chrome MediaRecorder 的 webm 无时长头，需 seek 到末尾），低于预期一半 报「时长异常」，杜绝残缺视频再被静默回写。
This commit is contained in:
VicoHu
@@ -10,7 +10,7 @@
|
||||
"pretest": "bun test test/tiptap-dependency-versions.test.ts test/admin-ui-regressions.test.ts test/announcement-prompt-regressions.test.ts test/password-reset.test.ts test/oss-settings.test.ts test/channel-model-editor-form.test.ts test/channel-model-price-tier-form.test.ts test/admin-logical-models.test.ts && bun test test/canvas-grid-viewport.test.tsx",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test:canvas": "bun test test/canvas-media-performance.test.ts test/canvas-node-title-interaction.test.ts test/canvas-spatial-index.test.ts",
|
||||
"test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-pricing.test.ts test/model-protocols.test.ts test/model-capabilities.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-generation-mentions.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-frame.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts",
|
||||
"test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-pricing.test.ts test/model-protocols.test.ts test/model-capabilities.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-generation-mentions.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-frame.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-clay-materials.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts",
|
||||
"test:director:e2e": "bun scripts/director-p0-chrome-e2e.mjs",
|
||||
"start": "vite preview --host 0.0.0.0 --port 3000",
|
||||
"format": "prettier --write .",
|
||||
|
||||
@@ -6,6 +6,7 @@ import type { Material } from "three";
|
||||
import { GLTFLoader, SkeletonUtils } from "three-stdlib";
|
||||
|
||||
import { resolveDirectorBoneRotation } from "@/lib/canvas/director/director-animation-semantics";
|
||||
import { applyClaySceneMaterials } from "@/lib/canvas/director/director-clay-materials";
|
||||
import { createDirectorTransaction, installDirectorTerminalListeners } from "@/lib/canvas/director/director-gesture-transaction";
|
||||
import { emptyDirectorPlacementIntent, finiteDirectorGroundPoint, type DirectorGroundPoint, type DirectorPlacementIntent } from "@/lib/canvas/director/director-placement";
|
||||
import { directorDiagnosticObjectKind } from "@/lib/canvas/director/director-diagnostics";
|
||||
@@ -1135,16 +1136,30 @@ async function recordCanvas(context: CaptureContext | null, duration: number, fp
|
||||
const mimeType = ["video/webm;codecs=vp9", "video/webm;codecs=vp8", "video/webm"].find((type) => MediaRecorder.isTypeSupported(type)) || "";
|
||||
const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined);
|
||||
const chunks: Blob[] = [];
|
||||
// captureStream 依赖渲染循环持续产出新帧；循环里任何未捕获异常都会让剩余录制变成空帧，
|
||||
// 与其 5 秒后静默产出残缺视频回写画布，不如捕获到首个错误就立刻中止并报错。
|
||||
let renderError: Error | null = null;
|
||||
const onRenderError = () => {
|
||||
renderError ??= new Error("白膜视频录制期间发生渲染错误，请重试");
|
||||
if (recorder.state !== "inactive") recorder.stop();
|
||||
};
|
||||
window.addEventListener("error", onRenderError);
|
||||
const result = new Promise<Blob>((resolve, reject) => {
|
||||
recorder.ondataavailable = (event) => { if (event.data.size) chunks.push(event.data); };
|
||||
recorder.onerror = () => reject(new Error("白膜视频录制失败"));
|
||||
recorder.onstop = () => resolve(new Blob(chunks, { type: recorder.mimeType || "video/webm" }));
|
||||
});
|
||||
recorder.start();
|
||||
window.setTimeout(() => recorder.stop(), Math.max(250, duration * 1000 + 120));
|
||||
recorder.start(250);
|
||||
const stopTimer = window.setTimeout(() => { if (recorder.state !== "inactive") recorder.stop(); }, Math.max(250, duration * 1000 + 120));
|
||||
try {
|
||||
return await result;
|
||||
const blob = await result;
|
||||
if (renderError) throw renderError;
|
||||
const recorded = await probeRecordedDuration(blob);
|
||||
if (!Number.isFinite(recorded) || recorded < Math.max(0.25, duration * 0.5)) throw new Error("白膜视频时长异常，录制可能不完整，请重试");
|
||||
return blob;
|
||||
} finally {
|
||||
window.clearTimeout(stopTimer);
|
||||
window.removeEventListener("error", onRenderError);
|
||||
stream.getTracks().forEach((track) => track.stop());
|
||||
restoreClayMaterials();
|
||||
context.scene.overrideMaterial = previousMaterial;
|
||||
@@ -1153,19 +1168,30 @@ async function recordCanvas(context: CaptureContext | null, duration: number, fp
|
||||
}
|
||||
}
|
||||
|
||||
function applyClaySceneMaterials(scene: Scene) {
|
||||
const clayMaterial = new MeshStandardMaterial({ color: "#d6d9dd", roughness: 0.88, metalness: 0 });
|
||||
const originals: Array<{ mesh: Mesh; material: Material | Material[] }> = [];
|
||||
scene.traverse((child) => {
|
||||
const mesh = child as Mesh;
|
||||
if (!mesh.isMesh || mesh.userData.directorActor) return;
|
||||
originals.push({ mesh, material: mesh.material });
|
||||
mesh.material = clayMaterial;
|
||||
});
|
||||
return () => {
|
||||
originals.forEach(({ mesh, material }) => { mesh.material = material; });
|
||||
clayMaterial.dispose();
|
||||
};
|
||||
async function probeRecordedDuration(blob: Blob) {
|
||||
// Chrome MediaRecorder 产出的 webm 不带时长头，loaded metadata 时 duration 是 Infinity，
|
||||
// 只有 seek 到末尾触发收尾后 duration 才是真实值；这是校验录制完整性的唯一途径。
|
||||
const url = URL.createObjectURL(blob);
|
||||
const video = document.createElement("video");
|
||||
video.muted = true;
|
||||
video.preload = "metadata";
|
||||
try {
|
||||
video.src = url;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
video.onloadedmetadata = () => resolve();
|
||||
video.onerror = () => reject(new Error("白膜视频无法解析"));
|
||||
});
|
||||
if (video.duration !== Infinity) return video.duration;
|
||||
await new Promise<void>((resolve) => {
|
||||
const finish = () => { video.removeEventListener("seeked", finish); resolve(); };
|
||||
video.addEventListener("seeked", finish);
|
||||
video.currentTime = 1e6;
|
||||
window.setTimeout(finish, 1000);
|
||||
});
|
||||
return video.duration;
|
||||
} finally {
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
}
|
||||
|
||||
function canvasToBlob(canvas: HTMLCanvasElement) {
|
||||
|
||||
29
web/src/lib/canvas/director/director-clay-materials.ts
Normal file
29
web/src/lib/canvas/director/director-clay-materials.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import { Mesh, MeshStandardMaterial, Scene, type Material } from "three";
|
||||
|
||||
/**
|
||||
* 将场景整体切换为白膜材质，返回恢复函数；恢复时销毁共享的 clay 材质。
|
||||
*
|
||||
* 两类 mesh 必须跳过，否则切换本身会破坏渲染：
|
||||
* - directorActor mesh 的材质由展示层按展示状态自行管理；
|
||||
* - 带 uniforms 的 ShaderMaterial（drei Grid/Line 等）由组件自身 useFrame 逐帧读写 uniforms，
|
||||
* 换成 clay 后渲染循环会逐帧抛错、画布冻结，白膜录制只剩启动前手动渲染的首帧。
|
||||
*/
|
||||
export function applyClaySceneMaterials(scene: Scene) {
|
||||
const clayMaterial = new MeshStandardMaterial({ color: "#d6d9dd", roughness: 0.88, metalness: 0 });
|
||||
const originals: Array<{ mesh: Mesh; material: Material | Material[] }> = [];
|
||||
scene.traverse((child) => {
|
||||
const mesh = child as Mesh;
|
||||
if (!mesh.isMesh || mesh.userData.directorActor) return;
|
||||
const material = mesh.material;
|
||||
const shaderBacked = Array.isArray(material) ? material.some((item) => "uniforms" in item) : "uniforms" in material;
|
||||
if (shaderBacked) return;
|
||||
originals.push({ mesh, material });
|
||||
mesh.material = clayMaterial;
|
||||
});
|
||||
return () => {
|
||||
originals.forEach(({ mesh, material }) => {
|
||||
mesh.material = material;
|
||||
});
|
||||
clayMaterial.dispose();
|
||||
};
|
||||
}
|
||||
56
web/test/director-clay-materials.test.ts
Normal file
56
web/test/director-clay-materials.test.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { Line, LineBasicMaterial, Mesh, MeshBasicMaterial, MeshStandardMaterial, PlaneGeometry, Scene, ShaderMaterial, type Material } from "three";
|
||||
|
||||
import { applyClaySceneMaterials } from "../src/lib/canvas/director/director-clay-materials";
|
||||
|
||||
const geometry = new PlaneGeometry(1, 1);
|
||||
|
||||
describe("导演台白膜材质切换", () => {
|
||||
test("标准材质 mesh 换成 clay，恢复后还原为原材质", () => {
|
||||
const scene = new Scene();
|
||||
const original = new MeshStandardMaterial({ color: "#ff0000" });
|
||||
const mesh = new Mesh(geometry, original);
|
||||
scene.add(mesh);
|
||||
const restore = applyClaySceneMaterials(scene);
|
||||
expect(mesh.material).not.toBe(original);
|
||||
expect(mesh.material).toBeInstanceOf(MeshStandardMaterial);
|
||||
restore();
|
||||
expect(mesh.material).toBe(original);
|
||||
});
|
||||
|
||||
test("带 uniforms 的 shader 材质保持原样（Grid 逐帧崩溃回归）", () => {
|
||||
const scene = new Scene();
|
||||
const shader = new ShaderMaterial({ uniforms: { worldCamProjPosition: { value: null } } });
|
||||
const grid = new Mesh(geometry, shader);
|
||||
scene.add(grid);
|
||||
const restore = applyClaySceneMaterials(scene);
|
||||
expect(grid.material).toBe(shader);
|
||||
restore();
|
||||
expect(grid.material).toBe(shader);
|
||||
});
|
||||
|
||||
test("directorActor mesh 与非 Mesh 对象不受影响", () => {
|
||||
const scene = new Scene();
|
||||
const actorMaterial = new MeshBasicMaterial();
|
||||
const actor = new Mesh(geometry, actorMaterial);
|
||||
actor.userData.directorActor = true;
|
||||
const line = new Line(geometry, new LineBasicMaterial());
|
||||
scene.add(actor, line);
|
||||
const restore = applyClaySceneMaterials(scene);
|
||||
expect(actor.material).toBe(actorMaterial);
|
||||
expect(line.material).toBeInstanceOf(LineBasicMaterial);
|
||||
restore();
|
||||
expect(actor.material).toBe(actorMaterial);
|
||||
});
|
||||
|
||||
test("材质数组混入 uniforms 材质时整组跳过", () => {
|
||||
const scene = new Scene();
|
||||
const materials: Material[] = [new MeshStandardMaterial(), new ShaderMaterial({ uniforms: {} })];
|
||||
const mesh = new Mesh(geometry, materials);
|
||||
scene.add(mesh);
|
||||
const restore = applyClaySceneMaterials(scene);
|
||||
expect(mesh.material).toBe(materials);
|
||||
restore();
|
||||
expect(mesh.material).toBe(materials);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user