From 8652ee74c686f6d78ab61cf32d863f769b69ddfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=88=B1=E7=AC=91=E7=9A=84=E6=AF=9B=E6=AF=9B=E8=99=AB?= <315515767@qq.com> Date: Fri, 7 Aug 2026 15:06:56 +0800 Subject: [PATCH] =?UTF-8?q?feat(web):=20=E5=AE=9E=E6=97=B6=E5=AF=B9?= =?UTF-8?q?=E8=AF=9D=20-=20=E7=AC=AC=E4=B8=80=E9=98=B6=E6=AE=B5=E8=BE=93?= =?UTF-8?q?=E5=85=A5=E8=A1=8C=E5=86=85=E8=81=94=E8=AF=AD=E9=9F=B3=E5=BD=95?= =?UTF-8?q?=E5=88=B6=E4=B8=8E=E6=B5=8F=E8=A7=88=E5=99=A8=20STT=20=E8=BD=AC?= =?UTF-8?q?=E5=86=99=20(#165)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: zhaoyue --- .../canvas/canvas-assistant-panel.tsx | 5 + .../canvas/canvas-local-agent-panel.tsx | 11 +- .../components/conversation/IMPLEMENTATION.md | 109 ++++++++++ web/src/components/conversation/README.md | 100 +++++++++ .../conversation/audio-waveform.tsx | 69 +++++++ web/src/components/conversation/index.ts | 3 + .../conversation/voice-recording-button.tsx | 49 +++++ .../conversation/voice-recording-inline.tsx | 191 +++++++++++++++++ web/src/hooks/use-speech-recognition.ts | 163 +++++++++++++++ web/src/hooks/use-voice-recording.ts | 193 ++++++++++++++++++ web/src/pages/create/index.tsx | 5 + web/src/pages/test-voice-recording.tsx | 107 ++++++++++ web/src/router.tsx | 2 + 13 files changed, 1006 insertions(+), 1 deletion(-) create mode 100644 web/src/components/conversation/IMPLEMENTATION.md create mode 100644 web/src/components/conversation/README.md create mode 100644 web/src/components/conversation/audio-waveform.tsx create mode 100644 web/src/components/conversation/index.ts create mode 100644 web/src/components/conversation/voice-recording-button.tsx create mode 100644 web/src/components/conversation/voice-recording-inline.tsx create mode 100644 web/src/hooks/use-speech-recognition.ts create mode 100644 web/src/hooks/use-voice-recording.ts create mode 100644 web/src/pages/test-voice-recording.tsx diff --git a/web/src/components/canvas/canvas-assistant-panel.tsx b/web/src/components/canvas/canvas-assistant-panel.tsx index fc09ee1da..7a16c8ba8 100644 --- a/web/src/components/canvas/canvas-assistant-panel.tsx +++ b/web/src/components/canvas/canvas-assistant-panel.tsx @@ -17,6 +17,7 @@ import { navigateToSettings } from "@/lib/settings-navigation"; import { cinematicAgentSessionOpsJson, createCinematicAgentSession, isAgentSessionPollingAbort, resumeCinematicAgentSession } from "@/lib/canvas/canvas-agent-session"; import { summarizeCanvasContext } from "@/lib/canvas/canvas-context-summary"; import { AgentChatComposer, AgentChatMessage, AgentPanelTabs, AgentWorkingMessage, type CanvasAgentChatMessage, type CanvasAgentMode } from "./canvas-agent-chat-ui"; +import { VoiceRecordingButton } from "@/components/conversation/voice-recording-button"; import { AgentChatEmptyState, AgentPanelChrome } from "./canvas-agent-panel-chrome"; import { CanvasLocalAgentPanel } from "./canvas-local-agent-panel"; import { NODE_DEFAULT_SIZE } from "@/constant/canvas"; @@ -781,6 +782,10 @@ export function CanvasAssistantPanel({ nodes, selectedNodeIds, snapshot, project onAddFiles={addImagesToCanvas} left={ <> + setPrompt((prev) => (prev.trim() ? `${prev} ${text}` : text))} + /> updateConfig("textModel", model)} /> {cinematicEntryActive ? 影视项目 : null} diff --git a/web/src/components/canvas/canvas-local-agent-panel.tsx b/web/src/components/canvas/canvas-local-agent-panel.tsx index 0dd2bb2fc..18fc8521d 100644 --- a/web/src/components/canvas/canvas-local-agent-panel.tsx +++ b/web/src/components/canvas/canvas-local-agent-panel.tsx @@ -13,6 +13,7 @@ import { useCanvasAgentStore, type AgentAttachment, type AgentChatItem, type Age import { previewCanvasAgentOps, summarizeCanvasAgentOps, type CanvasAgentOp, type CanvasAgentSnapshot } from "@/lib/canvas/canvas-agent-ops"; import { isProjectAgentReadTool, isProjectAgentToolName, runProjectAgentTool } from "@/services/api/project-agent-tools"; import { AgentChatComposer, AgentChatMessage, AgentPanelTabs, AgentPendingToolCard, AgentWorkingMessage, type CanvasAgentChatAttachment } from "./canvas-agent-chat-ui"; +import { VoiceRecordingButton } from "@/components/conversation/voice-recording-button"; import { AgentChatEmptyState } from "./canvas-agent-panel-chrome"; const PANEL_MOTION_SECONDS = 0.5; @@ -562,7 +563,15 @@ export const CanvasLocalAgentPanel = memo(function CanvasLocalAgentPanel({ snaps onSubmit={sendPrompt} onAddFiles={addAttachments} onRemoveAttachment={removeAttachment} - left={attachments.length ? {formatBytes(attachmentPayloadBytes(attachments))} / 30MB : null} + left={ + <> + setAgentState({ prompt: prompt.trim() ? `${prompt} ${text}` : text })} + /> + {attachments.length ? {formatBytes(attachmentPayloadBytes(attachments))} / 30MB : null} + + } /> )} diff --git a/web/src/components/conversation/IMPLEMENTATION.md b/web/src/components/conversation/IMPLEMENTATION.md new file mode 100644 index 000000000..75e93ebbc --- /dev/null +++ b/web/src/components/conversation/IMPLEMENTATION.md @@ -0,0 +1,109 @@ +# 实时对话功能实现(第一阶段 MVP) + +## 项目概述 + +实现第一阶段(MVP):**本地语音录制 + 浏览器语音识别转写**。点击"实时对话"按钮 → 输入行内展开波形录制条 → 停止后自动转写为文字 → 文字填入输入框或直接发送。 + +第一阶段**不接入语音大模型、不依赖后端与 API Key**,转写由浏览器内置 Web Speech API 完成,零配置可用。 + +--- + +## 交互流程 + +1. 点击"实时对话"按钮(`VoiceRecordingButton`) +2. 输入行内展开 `VoiceRecordingInline` 录制条(不弹窗),自动开始录音 +3. 显示波形动画(AnalyserNode 实时音量)+ 时长 +4. 点击停止 → 停止录音并触发浏览器语音识别 +5. 识别成功 → 短暂显示"转写完成" → 回调文本,由调用方填入输入框或直接发送 +6. 识别失败(无语音 / 权限拒绝 / 网络异常)→ 录制条内提示并支持重试 + +--- + +## 技术架构 + +``` +用户点击"实时对话"按钮 + │ + ▼ +VoiceRecordingButton(输入行内展开录制条,不弹窗) + │ + ▼ +VoiceRecordingInline 内联波形录制条 + ├── useVoiceRecording:MediaRecorder 录制 + AnalyserNode 实时波形 + ├── AudioWaveform:SVG 波形动画 + 时长显示 + └── useSpeechRecognition:Web Speech API 实时转写 + │ + ▼ +点击停止 → speech.stop() 返回文本 → onTranscribed(文本) → 填入输入框 +``` + +## 前端实现 + +### 组件 + +| 文件 | 说明 | +|------|------| +| `components/conversation/voice-recording-button.tsx` | 语音输入按钮:点击后输入行内展开录制条(局部状态,不弹窗) | +| `components/conversation/voice-recording-inline.tsx` | 内联波形录制条:波形 + 时长 + 停止/取消,停止后自动转写 | +| `components/conversation/audio-waveform.tsx` | SVG 波形可视化组件 | +| `components/conversation/index.ts` | 组件统一导出 | + +### Hook + +| 文件 | 说明 | +|------|------| +| `hooks/use-voice-recording.ts` | 语音录制逻辑(MediaRecorder + AnalyserNode,波形/时长/停止/取消) | +| `hooks/use-speech-recognition.ts` | 浏览器语音识别(Web Speech API,start/stop/cancel,返回文本与错误) | + +## 接入位置 + +- 创建页主输入行:`web/src/pages/create/index.tsx`(`CreationComposer` 的 `creation-chat-controls`) +- 画布助手面板:`canvas-assistant-panel.tsx`、`canvas-local-agent-panel.tsx`(输入行 left 插槽) +- 测试页:`web/src/pages/test-voice-recording.tsx`(路由 `/test-voice-recording`) + +## 使用示例 + +```tsx +import { VoiceRecordingButton } from "@/components/conversation/voice-recording-button"; + + setPrompt((prev) => (prev.trim() ? `${prev} ${text}` : text))} +/> +``` + +## 浏览器支持 + +- 语音识别使用 Web Speech API(`SpeechRecognition` / `webkitSpeechRecognition`),Chrome / Edge 支持 +- 不支持时录制条内提示"当前浏览器不支持语音识别,请使用 Chrome 或 Edge 浏览器",并可取消 +- 识别语言默认 `zh-CN`,可通过 `useSpeechRecognition({ lang })` 调整 + +## 已删除内容 + +以下第二阶段实验代码已全部删除: + +### 前端 + +- `components/conversation/realtime-conversation.tsx`、`realtime-conversation-button.tsx`、`voice-recording-overlay.tsx` +- `hooks/use-websocket.ts`、`stores/conversation-mode-store.ts` +- `services/api/stt.ts`(后端 Whisper 转写客户端) + +### 后端 + +- `service/stt.go`(OpenAI Whisper 转写)、`model/stt.go`(`stt_results` 表) +- `handler/routes.go` 中 `RegisterSTTRoutes`、`main.go` 中路由注册 +- `database/schema.go` 中 `STTResult` 模型注册、`repository/repository.go` 中 `CreateSTTResult` + +## 验证情况 + +- 后端 `go build ./...`:通过 +- 前端 `npx tsc --noEmit`:无本次新增类型错误(上游遗留 style profile / image.ts 错误除外) +- 浏览器手动测试(真实麦克风 + 语音识别)未运行,需在 Chrome/Edge 下验证 + +## 下一步 + +- [x] 内联波形录制 + 浏览器语音识别自动转写(第一阶段 MVP) +- [x] 移除第二阶段实验代码(WebSocket / WebRTC / Realtime API) +- [ ] 语音播放功能(TTS) +- [ ] 语音消息发送和显示 +- [ ] 真正实时对话(WebSocket + WebRTC + Realtime API,后续阶段) diff --git a/web/src/components/conversation/README.md b/web/src/components/conversation/README.md new file mode 100644 index 000000000..f2431b2c5 --- /dev/null +++ b/web/src/components/conversation/README.md @@ -0,0 +1,100 @@ +# 实时对话功能组件(第一阶段 MVP) +本目录实现第一阶段(MVP):**本地语音录制 + 浏览器语音识别转写**。点击"实时对话"按钮 → 输入行内展开波形录制条 → 停止后自动转写为文字 → 文字填入输入框或直接发送。 + +## 零配置说明 + +第一阶段**不接入语音大模型、不依赖后端与 API Key**:录音完成后直接使用浏览器内置 Web Speech API(`SpeechRecognition`)识别,转写全程在前端完成。 + +- 支持的浏览器:Chrome / Edge(Chromium 内核) +- 不支持时录制条内会明确提示,并可取消 +- 识别语言默认 `zh-CN`,见 `useSpeechRecognition` 的 `lang` 参数 + +--- + +## 组件列表 + +### 1. `VoiceRecordingButton` - 语音输入按钮 + +点击后输入行内展开波形录制条(不弹窗),自动开始录音;停止后自动转写并回调文本。使用局部状态,多个输入行可独立使用。 + +**使用示例:** +```tsx +import { VoiceRecordingButton } from "@/components/conversation/voice-recording-button"; + + setPrompt((prev) => (prev.trim() ? `${prev} ${text}` : text))} +/> +``` + +### 2. `VoiceRecordingInline` - 内联波形录制条 + +输入行内展示的录制控件:波形动画 + 时长 + 停止/取消按钮;停止后自动转写,失败可重试。`VoiceRecordingButton` 默认使用本组件。 + +**使用示例:** +```tsx +import { VoiceRecordingInline } from "@/components/conversation/voice-recording-inline"; + + setPrompt((prev) => (prev.trim() ? `${prev} ${text}` : text))} + onCancel={() => setOpen(false)} +/> +``` + +### 3. `AudioWaveform` - 音频波形可视化 + +SVG 波形可视化组件,支持实时数据更新。 + +--- + +## Hook + +### `useVoiceRecording` - 语音录制 Hook + +封装 MediaRecorder API 和 AnalyserNode,提供录制状态、波形数据、控制方法(供波形动画与时长展示)。 + +### `useSpeechRecognition` - 浏览器语音识别 Hook + +封装 Web Speech API(`SpeechRecognition`/`webkitSpeechRecognition`),提供 `start()` / `stop(): Promise` / `cancel()`,返回是否支持、识别错误。转写不依赖后端。 + +--- + +## 完整集成示例 + +```tsx +import { useState } from "react"; +import { VoiceRecordingButton } from "@/components/conversation/voice-recording-button"; + +function ChatInput() { + const [prompt, setPrompt] = useState(""); + + const handleTranscribed = (text: string) => { + // 转写结果填入输入框,用户确认后发送 + setPrompt((current) => (current.trim() ? `${current} ${text}` : text)); + }; + + return ( +
+