diff --git a/backend/internal/service/canvas_share.go b/backend/internal/service/canvas_share.go index 7413309f5..6ca57333e 100644 --- a/backend/internal/service/canvas_share.go +++ b/backend/internal/service/canvas_share.go @@ -41,7 +41,7 @@ var publicCanvasMetadataKeys = map[string]bool{ "audioVoice": true, "audioFormat": true, "audioSpeed": true, "audioInstructions": true, "naturalWidth": true, "naturalHeight": true, "freeResize": true, "isBatchRoot": true, "batchRootId": true, "batchChildIds": true, "batchUsesReferenceImages": true, "primaryImageId": true, - "imageBatchExpanded": true, "mimeType": true, "bytes": true, "durationMs": true, "assetTags": true, + "imageBatchExpanded": true, "mimeType": true, "bytes": true, "durationMs": true, "hasAudio": true, "assetTags": true, "workflowKind": true, "workflowTitle": true, "workflowDescription": true, "shotIndex": true, "sceneId": true, "characterIds": true, "referenceSetId": true, "referenceAssetNodeIds": true, "assetBindings": true, "characterName": true, "characterPrompt": true, "characterAliases": true, "characterView": true, "characterViewNodeIds": true, diff --git a/docs/content/docs/progress/pending-test.mdx b/docs/content/docs/progress/pending-test.mdx index fbf8c6733..b8265f145 100644 --- a/docs/content/docs/progress/pending-test.mdx +++ b/docs/content/docs/progress/pending-test.mdx @@ -14,6 +14,12 @@ description: 影策已实现功能的手工验证清单与未确认边界 - [ ] 在存在图片、视频和文本长任务时发送 SIGTERM,确认 readiness 立即返回 503、新任务被拒绝、已领取任务在宽限期内完成,超时后租约可由重启实例恢复。 - [ ] 构建带版本参数的前后端镜像,确认 `/api/system/version`、管理后台版本、OCI Commit 和 Release Tag 一致。 +## 画布视频静态首帧 + +- [x] 普通上传与生成视频会在入库时串行截取并持久化 400px JPEG 首帧;历史视频仅在可见、未选中且浏览器空闲时单并发补图,选中媒体或离开视口会取消后台解码。 +- [x] 未选中视频只挂载静态图片,单选后才创建真实播放器;专项测试、TypeScript 检查、生产构建通过,本地画布刷新后 3 个历史视频首帧仍可显示。 +- [ ] 用户在包含本地上传、模型生成和外部无 CORS 视频的真实画布中确认首帧画面、刷新持久化与降级图标;连续平移、缩放和切换视频时确认最多只有一个播放器且没有明显闪烁。 + ## 自由空白画布与画布外观 - [x] 新建画布引导新增“自由空白画布”主入口;进入后显示中央空白起始框和加号,不创建占位节点。 diff --git a/web/package.json b/web/package.json index ce09e839a..0eb42fff0 100644 --- a/web/package.json +++ b/web/package.json @@ -9,6 +9,7 @@ "build": "bun run build:bridge && tsc --noEmit && vite build", "pretest": "bun test test/admin-ui-regressions.test.ts", "typecheck": "tsc --noEmit", + "test:canvas": "bun test test/canvas-media-performance.test.ts test/canvas-node-title-interaction.test.ts", "test": "bun test test/api-request.test.ts test/video-api-contracts.test.ts test/image-api-contracts.test.ts test/channel-model-catalog.test.ts test/model-protocols.test.ts test/model-capabilities.test.ts test/agent-prompt-cache.test.ts test/canvas-agent-context.test.ts test/canvas-batch-connection.test.ts test/canvas-character-reference.test.ts test/canvas-document.test.ts test/canvas-drawing-engine.test.ts test/canvas-emotion-edit-plan.test.ts test/canvas-extension-nodes.test.ts test/canvas-folder.test.ts test/canvas-generation-layout.test.ts test/canvas-generation-submission.test.ts test/canvas-image-batch-retry.test.ts test/canvas-keyboard.test.ts test/canvas-node-generation-mentions.test.ts test/canvas-node-search.test.ts test/canvas-leafer-viewport.test.ts test/canvas-media-download.test.ts test/canvas-local-runtime.test.ts test/canvas-model-policy.test.ts test/canvas-resource-mention-editor.test.ts test/canvas-resource-references.test.ts test/canvas-storyboard-assets.test.ts test/canvas-storyboard-references.test.ts test/canvas-node-config-reset.test.ts test/canvas-node-registry.test.ts test/canvas-video-frame.test.ts test/canvas-video-regeneration.test.ts test/canvas-video-timeline-segments.test.ts test/create-canvas-handoff.test.ts test/create-library-button.test.ts test/creation-references.test.ts test/director-transform-semantics.test.ts test/director-bone-precedence.test.ts test/director-gesture-transaction.test.ts test/director-transaction-lifecycle.test.ts test/director-session-authority.test.ts test/director-placement.test.ts test/director-resources.test.ts test/director-recovery.test.ts test/director-node-preview.test.ts test/generation-asset-repository.node.test.ts test/generation-storage-consistency.test.ts test/generation-task-durability.node.test.ts test/generation-task-local.test.ts test/generation-task-materializer.test.ts test/task-media-fallback.test.ts test/user-scoped-generation-persistence.test.ts test/local-channel-permission.test.ts test/local-channel-runtime-projection.node.test.mjs test/local-channel-task-center.node.test.mjs test/local-channel-ui.node.test.mjs test/local-cli-settings.test.ts test/local-dreamina-cli.test.ts test/local-dreamina-generation.test.ts test/local-dreamina-model-catalog.test.ts test/local-runtime-bootstrap.test.ts test/local-runtime-connection.test.ts test/local-runtime.test.ts test/client-id.test.ts test/grok-image-prompt-limit.test.ts test/image-resolution-tiers.test.ts test/gemini-image.test.ts test/libtv-import.test.ts test/timeline-to-ffmpeg.test.ts test/canvas-video-segment-args.test.ts test/tapnow-import.test.ts test/prompt-optimizer-drawer.test.ts test/director-save-coordinator.test.ts test/director-save-wiring.test.ts test/director-diagnostics.test.ts test/director-repro-runtime.test.ts test/director-diagnostics-recorder.test.ts test/director-keyframe-editing.test.ts test/director-shortcuts.test.ts test/director-keyframe-delete-wiring.test.ts test/director-templates.test.ts test/director-modes.test.ts test/director-template-mode-wiring.test.ts test/director-view-modes.test.ts && bun test test/local-dreamina-cross-runtime.test.ts", "test:director:e2e": "bun scripts/director-p0-chrome-e2e.mjs", "start": "vite preview --host 0.0.0.0 --port 3000", diff --git a/web/src/components/canvas/asset-picker-modal.tsx b/web/src/components/canvas/asset-picker-modal.tsx index 801dc150c..9a7f90ec2 100644 --- a/web/src/components/canvas/asset-picker-modal.tsx +++ b/web/src/components/canvas/asset-picker-modal.tsx @@ -10,7 +10,7 @@ type InsertableAsset = Extract; coverUrl?: string; visualStatus: string; voiceStatus: string; voiceName?: string; voiceProfile?: { name: string; provider: string; language: string; timbre: string }; voiceInstructions?: string }; @@ -75,7 +75,7 @@ export function assetPickerItemsToInsertPayloads(ids: string[], items: AssetLibr function localAssetToInsertPayload(asset: InsertableAsset): InsertAssetPayload { if (asset.kind === "text") return { kind: "text", content: asset.data.content, title: asset.title, assetId: asset.id }; if (asset.kind === "audio") return { kind: "audio", url: asset.data.url, storageKey: asset.data.storageKey, title: asset.title, durationMs: asset.data.durationMs, bytes: asset.data.bytes, mimeType: asset.data.mimeType, assetId: asset.id }; - if (asset.kind === "video") return { kind: "video", url: asset.data.url, storageKey: asset.data.storageKey, title: asset.title, width: asset.data.width, height: asset.data.height, durationMs: asset.data.durationMs, bytes: asset.data.bytes, mimeType: asset.data.mimeType, assetId: asset.id }; + if (asset.kind === "video") return { kind: "video", url: asset.data.url, storageKey: asset.data.storageKey, title: asset.title, width: asset.data.width, height: asset.data.height, durationMs: asset.data.durationMs, hasAudio: asset.data.hasAudio, bytes: asset.data.bytes, mimeType: asset.data.mimeType, assetId: asset.id }; return { kind: "image", dataUrl: asset.data.dataUrl, storageKey: asset.data.storageKey, title: asset.title, width: asset.data.width, height: asset.data.height, bytes: asset.data.bytes, mimeType: asset.data.mimeType, assetId: asset.id }; } diff --git a/web/src/components/canvas/canvas-audio-player.tsx b/web/src/components/canvas/canvas-audio-player.tsx new file mode 100644 index 000000000..43efa0e84 --- /dev/null +++ b/web/src/components/canvas/canvas-audio-player.tsx @@ -0,0 +1,133 @@ +import { useCallback, useEffect, useMemo, useRef, useSyncExternalStore } from "react"; +import { AlertCircle, LoaderCircle, Pause, Play } from "lucide-react"; + +import type { CanvasTheme } from "@/lib/canvas-theme"; +import { useCanvasNodeActions } from "./canvas-node-action-context"; +import { getCanvasAudioPlaybackSnapshot, seekCanvasAudio, stopCanvasAudio, subscribeCanvasAudioNode, toggleCanvasAudio } from "@/services/canvas-audio-playback"; +import type { CanvasNodeData } from "@/types/canvas"; + +type CanvasAudioPlayerProps = { + node: CanvasNodeData; + theme: CanvasTheme; +}; + +export function CanvasAudioPlayer({ node, theme }: CanvasAudioPlayerProps) { + const { updateMetadata } = useCanvasNodeActions(); + const subscribe = useCallback((listener: () => void) => subscribeCanvasAudioNode(node.id, listener), [node.id]); + const getSnapshot = useCallback(() => getCanvasAudioPlaybackSnapshot(node.id), [node.id]); + const snapshot = useSyncExternalStore(subscribe, getSnapshot, getSnapshot); + const source = useMemo( + () => ({ + nodeId: node.id, + content: node.metadata?.content || "", + storageKey: node.metadata?.storageKey, + mimeType: node.metadata?.mimeType, + durationMs: node.metadata?.durationMs, + }), + [node.id, node.metadata?.content, node.metadata?.durationMs, node.metadata?.mimeType, node.metadata?.storageKey], + ); + const waveform = useMemo(() => createStableWaveform(node.id), [node.id]); + const syncedDurationRef = useRef(0); + const durationMs = snapshot.durationMs || node.metadata?.durationMs || 0; + const progress = durationMs > 0 ? Math.min(1, Math.max(0, snapshot.currentTimeMs / durationMs)) : 0; + + useEffect(() => { + if (!snapshot.durationMs || snapshot.durationMs === node.metadata?.durationMs || syncedDurationRef.current === snapshot.durationMs) return; + syncedDurationRef.current = snapshot.durationMs; + updateMetadata?.(node.id, { durationMs: snapshot.durationMs }); + }, [node.id, node.metadata?.durationMs, snapshot.durationMs, updateMetadata]); + + useEffect(() => () => stopCanvasAudio(node.id), [node.id]); + + const stopCanvasEvent = (event: React.SyntheticEvent) => event.stopPropagation(); + const togglePlayback = () => { + void toggleCanvasAudio(source); + }; + const seek = (event: React.ChangeEvent) => { + seekCanvasAudio(node.id, Number(event.target.value), source); + }; + const label = node.title || "音频"; + const isPlaying = snapshot.phase === "playing"; + const isLoading = snapshot.phase === "loading"; + const isError = snapshot.phase === "error"; + + return ( +
+
+ +
+
+ {formatAudioTime(snapshot.currentTimeMs)}/{durationMs ? formatAudioTime(durationMs) : "--:--"} +
+ +
+
+ + + + {isError ? ( +
+
+ ) : null} +
+ ); +} + +function createStableWaveform(nodeId: string) { + let seed = 2166136261; + for (let index = 0; index < nodeId.length; index += 1) { + seed ^= nodeId.charCodeAt(index); + seed = Math.imul(seed, 16777619); + } + return Array.from({ length: 36 }, (_, index) => { + seed = Math.imul(seed ^ index, 16777619); + const normalized = ((seed >>> 0) % 100) / 100; + return 0.28 + normalized * 0.62; + }); +} + +function formatAudioTime(timeMs: number) { + const totalSeconds = Math.max(0, Math.floor(timeMs / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return `${minutes}:${seconds.toString().padStart(2, "0")}`; +} diff --git a/web/src/components/canvas/canvas-config-composer.tsx b/web/src/components/canvas/canvas-config-composer.tsx index c2126b9fa..fda57e78f 100644 --- a/web/src/components/canvas/canvas-config-composer.tsx +++ b/web/src/components/canvas/canvas-config-composer.tsx @@ -331,7 +331,7 @@ function ResourcePreview({ candidate }: { candidate: ComposerCandidate }) { ); } if (input.type === "image" && input.image) return ; - if (input.type === "video" && input.video) return