diff --git a/web/package.json b/web/package.json index f02f269fc..071481a7e 100644 --- a/web/package.json +++ b/web/package.json @@ -10,7 +10,7 @@ "pretest": "bun test test/tiptap-dependency-versions.test.ts test/admin-ui-regressions.test.ts test/announcement-prompt-regressions.test.ts test/password-reset.test.ts test/oss-settings.test.ts test/channel-model-editor-form.test.ts test/channel-model-price-tier-form.test.ts test/admin-logical-models.test.ts && bun test test/canvas-grid-viewport.test.tsx", "typecheck": "tsc --noEmit", "test:canvas": "bun test test/canvas-media-performance.test.ts test/canvas-node-title-interaction.test.ts test/canvas-spatial-index.test.ts", - "test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-pricing.test.ts test/model-protocols.test.ts test/model-capabilities.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-generation-mentions.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-frame.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts", + "test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-pricing.test.ts test/model-protocols.test.ts test/model-capabilities.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-generation-mentions.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-frame.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-clay-materials.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts", "test:director:e2e": "bun scripts/director-p0-chrome-e2e.mjs", "start": "vite preview --host 0.0.0.0 --port 3000", "format": "prettier --write .", diff --git a/web/src/components/canvas/director/director-viewport.tsx b/web/src/components/canvas/director/director-viewport.tsx index aaa18b072..32fcf625f 100644 --- a/web/src/components/canvas/director/director-viewport.tsx +++ b/web/src/components/canvas/director/director-viewport.tsx @@ -6,6 +6,7 @@ import type { Material } from "three"; import { GLTFLoader, SkeletonUtils } from "three-stdlib"; import { resolveDirectorBoneRotation } from "@/lib/canvas/director/director-animation-semantics"; +import { applyClaySceneMaterials } from "@/lib/canvas/director/director-clay-materials"; import { createDirectorTransaction, installDirectorTerminalListeners } from "@/lib/canvas/director/director-gesture-transaction"; import { emptyDirectorPlacementIntent, finiteDirectorGroundPoint, type DirectorGroundPoint, type DirectorPlacementIntent } from "@/lib/canvas/director/director-placement"; import { directorDiagnosticObjectKind } from "@/lib/canvas/director/director-diagnostics"; @@ -1135,16 +1136,30 @@ async function recordCanvas(context: CaptureContext | null, duration: number, fp const mimeType = ["video/webm;codecs=vp9", "video/webm;codecs=vp8", "video/webm"].find((type) => MediaRecorder.isTypeSupported(type)) || ""; const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); const chunks: Blob[] = []; + // captureStream 依赖渲染循环持续产出新帧;循环里任何未捕获异常都会让剩余录制变成空帧, + // 与其 5 秒后静默产出残缺视频回写画布,不如捕获到首个错误就立刻中止并报错。 + let renderError: Error | null = null; + const onRenderError = () => { + renderError ??= new Error("白膜视频录制期间发生渲染错误,请重试"); + if (recorder.state !== "inactive") recorder.stop(); + }; + window.addEventListener("error", onRenderError); const result = new Promise((resolve, reject) => { recorder.ondataavailable = (event) => { if (event.data.size) chunks.push(event.data); }; recorder.onerror = () => reject(new Error("白膜视频录制失败")); recorder.onstop = () => resolve(new Blob(chunks, { type: recorder.mimeType || "video/webm" })); }); - recorder.start(); - window.setTimeout(() => recorder.stop(), Math.max(250, duration * 1000 + 120)); + recorder.start(250); + const stopTimer = window.setTimeout(() => { if (recorder.state !== "inactive") recorder.stop(); }, Math.max(250, duration * 1000 + 120)); try { - return await result; + const blob = await result; + if (renderError) throw renderError; + const recorded = await probeRecordedDuration(blob); + if (!Number.isFinite(recorded) || recorded < Math.max(0.25, duration * 0.5)) throw new Error("白膜视频时长异常,录制可能不完整,请重试"); + return blob; } finally { + window.clearTimeout(stopTimer); + window.removeEventListener("error", onRenderError); stream.getTracks().forEach((track) => track.stop()); restoreClayMaterials(); context.scene.overrideMaterial = previousMaterial; @@ -1153,19 +1168,30 @@ async function recordCanvas(context: CaptureContext | null, duration: number, fp } } -function applyClaySceneMaterials(scene: Scene) { - const clayMaterial = new MeshStandardMaterial({ color: "#d6d9dd", roughness: 0.88, metalness: 0 }); - const originals: Array<{ mesh: Mesh; material: Material | Material[] }> = []; - scene.traverse((child) => { - const mesh = child as Mesh; - if (!mesh.isMesh || mesh.userData.directorActor) return; - originals.push({ mesh, material: mesh.material }); - mesh.material = clayMaterial; - }); - return () => { - originals.forEach(({ mesh, material }) => { mesh.material = material; }); - clayMaterial.dispose(); - }; +async function probeRecordedDuration(blob: Blob) { + // Chrome MediaRecorder 产出的 webm 不带时长头,loaded metadata 时 duration 是 Infinity, + // 只有 seek 到末尾触发收尾后 duration 才是真实值;这是校验录制完整性的唯一途径。 + const url = URL.createObjectURL(blob); + const video = document.createElement("video"); + video.muted = true; + video.preload = "metadata"; + try { + video.src = url; + await new Promise((resolve, reject) => { + video.onloadedmetadata = () => resolve(); + video.onerror = () => reject(new Error("白膜视频无法解析")); + }); + if (video.duration !== Infinity) return video.duration; + await new Promise((resolve) => { + const finish = () => { video.removeEventListener("seeked", finish); resolve(); }; + video.addEventListener("seeked", finish); + video.currentTime = 1e6; + window.setTimeout(finish, 1000); + }); + return video.duration; + } finally { + URL.revokeObjectURL(url); + } } function canvasToBlob(canvas: HTMLCanvasElement) { diff --git a/web/src/lib/canvas/director/director-clay-materials.ts b/web/src/lib/canvas/director/director-clay-materials.ts new file mode 100644 index 000000000..2b189752e --- /dev/null +++ b/web/src/lib/canvas/director/director-clay-materials.ts @@ -0,0 +1,29 @@ +import { Mesh, MeshStandardMaterial, Scene, type Material } from "three"; + +/** + * 将场景整体切换为白膜材质,返回恢复函数;恢复时销毁共享的 clay 材质。 + * + * 两类 mesh 必须跳过,否则切换本身会破坏渲染: + * - directorActor mesh 的材质由展示层按展示状态自行管理; + * - 带 uniforms 的 ShaderMaterial(drei Grid/Line 等)由组件自身 useFrame 逐帧读写 uniforms, + * 换成 clay 后渲染循环会逐帧抛错、画布冻结,白膜录制只剩启动前手动渲染的首帧。 + */ +export function applyClaySceneMaterials(scene: Scene) { + const clayMaterial = new MeshStandardMaterial({ color: "#d6d9dd", roughness: 0.88, metalness: 0 }); + const originals: Array<{ mesh: Mesh; material: Material | Material[] }> = []; + scene.traverse((child) => { + const mesh = child as Mesh; + if (!mesh.isMesh || mesh.userData.directorActor) return; + const material = mesh.material; + const shaderBacked = Array.isArray(material) ? material.some((item) => "uniforms" in item) : "uniforms" in material; + if (shaderBacked) return; + originals.push({ mesh, material }); + mesh.material = clayMaterial; + }); + return () => { + originals.forEach(({ mesh, material }) => { + mesh.material = material; + }); + clayMaterial.dispose(); + }; +} diff --git a/web/test/director-clay-materials.test.ts b/web/test/director-clay-materials.test.ts new file mode 100644 index 000000000..f1135e262 --- /dev/null +++ b/web/test/director-clay-materials.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, test } from "bun:test"; +import { Line, LineBasicMaterial, Mesh, MeshBasicMaterial, MeshStandardMaterial, PlaneGeometry, Scene, ShaderMaterial, type Material } from "three"; + +import { applyClaySceneMaterials } from "../src/lib/canvas/director/director-clay-materials"; + +const geometry = new PlaneGeometry(1, 1); + +describe("导演台白膜材质切换", () => { + test("标准材质 mesh 换成 clay,恢复后还原为原材质", () => { + const scene = new Scene(); + const original = new MeshStandardMaterial({ color: "#ff0000" }); + const mesh = new Mesh(geometry, original); + scene.add(mesh); + const restore = applyClaySceneMaterials(scene); + expect(mesh.material).not.toBe(original); + expect(mesh.material).toBeInstanceOf(MeshStandardMaterial); + restore(); + expect(mesh.material).toBe(original); + }); + + test("带 uniforms 的 shader 材质保持原样(Grid 逐帧崩溃回归)", () => { + const scene = new Scene(); + const shader = new ShaderMaterial({ uniforms: { worldCamProjPosition: { value: null } } }); + const grid = new Mesh(geometry, shader); + scene.add(grid); + const restore = applyClaySceneMaterials(scene); + expect(grid.material).toBe(shader); + restore(); + expect(grid.material).toBe(shader); + }); + + test("directorActor mesh 与非 Mesh 对象不受影响", () => { + const scene = new Scene(); + const actorMaterial = new MeshBasicMaterial(); + const actor = new Mesh(geometry, actorMaterial); + actor.userData.directorActor = true; + const line = new Line(geometry, new LineBasicMaterial()); + scene.add(actor, line); + const restore = applyClaySceneMaterials(scene); + expect(actor.material).toBe(actorMaterial); + expect(line.material).toBeInstanceOf(LineBasicMaterial); + restore(); + expect(actor.material).toBe(actorMaterial); + }); + + test("材质数组混入 uniforms 材质时整组跳过", () => { + const scene = new Scene(); + const materials: Material[] = [new MeshStandardMaterial(), new ShaderMaterial({ uniforms: {} })]; + const mesh = new Mesh(geometry, materials); + scene.add(mesh); + const restore = applyClaySceneMaterials(scene); + expect(mesh.material).toBe(materials); + restore(); + expect(mesh.material).toBe(materials); + }); +});