feat(web): 实时对话 - 第一阶段输入行内联语音录制与浏览器 STT 转写 (#165)
Co-authored-by: zhaoyue <zhaoyue@rongshantech.com>
This commit is contained in:
爱笑的毛毛虫
163
web/src/hooks/use-speech-recognition.ts
Normal file
163
web/src/hooks/use-speech-recognition.ts
Normal file
@@ -0,0 +1,163 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
|
||||
type SpeechRecognitionAlternative = { transcript: string; confidence: number };
|
||||
type SpeechRecognitionResult = { isFinal: boolean; length: number; [index: number]: SpeechRecognitionAlternative };
|
||||
type SpeechRecognitionResultList = { length: number; item(index: number): SpeechRecognitionResult; [index: number]: SpeechRecognitionResult };
|
||||
|
||||
type SpeechRecognitionLike = {
|
||||
lang: string;
|
||||
continuous: boolean;
|
||||
interimResults: boolean;
|
||||
maxAlternatives: number;
|
||||
onstart: (() => void) | null;
|
||||
onresult: ((event: { results: SpeechRecognitionResultList }) => void) | null;
|
||||
onerror: ((event: { error: string }) => void) | null;
|
||||
onend: (() => void) | null;
|
||||
start: () => void;
|
||||
stop: () => void;
|
||||
abort: () => void;
|
||||
};
|
||||
|
||||
type SpeechRecognitionConstructor = new () => SpeechRecognitionLike;
|
||||
|
||||
function getSpeechRecognition(): SpeechRecognitionConstructor | null {
|
||||
if (typeof window === "undefined") return null;
|
||||
const globalWithSpeech = window as unknown as {
|
||||
SpeechRecognition?: SpeechRecognitionConstructor;
|
||||
webkitSpeechRecognition?: SpeechRecognitionConstructor;
|
||||
};
|
||||
return globalWithSpeech.SpeechRecognition || globalWithSpeech.webkitSpeechRecognition || null;
|
||||
}
|
||||
|
||||
export type SpeechRecognitionError = {
|
||||
code: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
type UseSpeechRecognitionOptions = {
|
||||
/** 语言代码，默认 zh-CN */
|
||||
lang?: string;
|
||||
};
|
||||
|
||||
export type UseSpeechRecognitionReturn = {
|
||||
/** 浏览器是否支持 Web Speech API */
|
||||
supported: boolean;
|
||||
/** 识别错误（权限、网络、不支持等） */
|
||||
error: SpeechRecognitionError | null;
|
||||
/** 开始识别，挂载后自动调用 */
|
||||
start: () => void;
|
||||
/** 停止识别并返回已累积的最终文本 */
|
||||
stop: () => Promise<string>;
|
||||
/** 取消识别并清空累积文本 */
|
||||
cancel: () => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* 浏览器内置语音识别 Hook（Web Speech API）
|
||||
* 第一阶段 MVP 使用：不依赖后端、模型渠道或 API Key，零配置可用
|
||||
*/
|
||||
export function useSpeechRecognition(options: UseSpeechRecognitionOptions = {}): UseSpeechRecognitionReturn {
|
||||
const { lang = "zh-CN" } = options;
|
||||
const recognitionRef = useRef<SpeechRecognitionLike | null>(null);
|
||||
const finalTextRef = useRef("");
|
||||
const stopResolveRef = useRef<((text: string) => void) | null>(null);
|
||||
const stopTimerRef = useRef<number | null>(null);
|
||||
const [supported] = useState(() => Boolean(getSpeechRecognition()));
|
||||
const [error, setError] = useState<SpeechRecognitionError | null>(null);
|
||||
|
||||
const start = useCallback(() => {
|
||||
const Constructor = getSpeechRecognition();
|
||||
if (!Constructor) {
|
||||
setError({ code: "unsupported", message: "当前浏览器不支持语音识别，请使用 Chrome 或 Edge" });
|
||||
return;
|
||||
}
|
||||
finalTextRef.current = "";
|
||||
const recognition = new Constructor();
|
||||
recognition.lang = lang;
|
||||
recognition.continuous = true;
|
||||
recognition.interimResults = false;
|
||||
recognition.maxAlternatives = 1;
|
||||
recognition.onstart = () => setError(null);
|
||||
recognition.onresult = (event) => {
|
||||
let text = finalTextRef.current;
|
||||
for (let i = 0; i < event.results.length; i += 1) {
|
||||
const result = event.results[i];
|
||||
if (result.isFinal && result[0]?.transcript) {
|
||||
text = [text, result[0].transcript.trim()].filter(Boolean).join(" ");
|
||||
}
|
||||
}
|
||||
finalTextRef.current = text;
|
||||
};
|
||||
recognition.onerror = (event) => {
|
||||
if (event.error === "not-allowed" || event.error === "service-not-allowed") {
|
||||
setError({ code: event.error, message: "麦克风权限被拒绝，请在浏览器设置中允许访问麦克风" });
|
||||
} else if (event.error === "no-speech") {
|
||||
setError({ code: event.error, message: "未检测到语音，请重试" });
|
||||
} else if (event.error === "network") {
|
||||
setError({ code: event.error, message: "语音识别服务不可用，请检查网络后重试" });
|
||||
} else {
|
||||
setError({ code: event.error, message: `语音识别失败（${event.error}）` });
|
||||
}
|
||||
};
|
||||
recognition.onend = () => {
|
||||
if (stopTimerRef.current !== null) {
|
||||
window.clearTimeout(stopTimerRef.current);
|
||||
stopTimerRef.current = null;
|
||||
}
|
||||
const resolve = stopResolveRef.current;
|
||||
stopResolveRef.current = null;
|
||||
resolve?.(finalTextRef.current);
|
||||
};
|
||||
recognitionRef.current = recognition;
|
||||
recognition.start();
|
||||
}, [lang]);
|
||||
|
||||
const stop = useCallback((): Promise<string> => {
|
||||
return new Promise((resolve) => {
|
||||
const recognition = recognitionRef.current;
|
||||
if (!recognition) {
|
||||
resolve(finalTextRef.current);
|
||||
return;
|
||||
}
|
||||
stopResolveRef.current = resolve;
|
||||
// 兜底：部分浏览器 stop() 后 onend 可能不触发，超时后返回已识别文本
|
||||
stopTimerRef.current = window.setTimeout(() => {
|
||||
if (stopResolveRef.current) {
|
||||
stopResolveRef.current = null;
|
||||
resolve(finalTextRef.current);
|
||||
}
|
||||
}, 1500);
|
||||
try {
|
||||
recognition.stop();
|
||||
} catch {
|
||||
if (stopTimerRef.current !== null) {
|
||||
window.clearTimeout(stopTimerRef.current);
|
||||
stopTimerRef.current = null;
|
||||
}
|
||||
stopResolveRef.current = null;
|
||||
resolve(finalTextRef.current);
|
||||
}
|
||||
});
|
||||
}, []);
|
||||
|
||||
const cancel = useCallback(() => {
|
||||
if (stopTimerRef.current !== null) {
|
||||
window.clearTimeout(stopTimerRef.current);
|
||||
stopTimerRef.current = null;
|
||||
}
|
||||
stopResolveRef.current = null;
|
||||
const recognition = recognitionRef.current;
|
||||
recognitionRef.current = null;
|
||||
if (recognition) {
|
||||
recognition.onend = null;
|
||||
try {
|
||||
recognition.abort();
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
finalTextRef.current = "";
|
||||
}, []);
|
||||
|
||||
useEffect(() => cancel, [cancel]);
|
||||
|
||||
return { supported, error, start, stop, cancel };
|
||||
}
|
||||
193
web/src/hooks/use-voice-recording.ts
Normal file
193
web/src/hooks/use-voice-recording.ts
Normal file
@@ -0,0 +1,193 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
|
||||
export type VoiceRecordingState = "idle" | "recording" | "paused";
|
||||
|
||||
type UseVoiceRecordingOptions = {
|
||||
/** 最大录音时长（秒），默认 60 秒 */
|
||||
maxDuration?: number;
|
||||
};
|
||||
|
||||
export type UseVoiceRecordingReturn = {
|
||||
state: VoiceRecordingState;
|
||||
waveform: number[];
|
||||
duration: number;
|
||||
error: string | null;
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<Blob | null>;
|
||||
cancel: () => void;
|
||||
reset: () => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* 语音录制 Hook
|
||||
* 使用 MediaRecorder API 录制音频，AnalyserNode 获取实时波形数据
|
||||
*/
|
||||
export function useVoiceRecording(options: UseVoiceRecordingOptions = {}): UseVoiceRecordingReturn {
|
||||
const { maxDuration = 60 } = options;
|
||||
const [state, setState] = useState<VoiceRecordingState>("idle");
|
||||
const [waveform, setWaveform] = useState<number[]>([]);
|
||||
const [duration, setDuration] = useState(0);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const analyserRef = useRef<AnalyserNode | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const chunksRef = useRef<Blob[]>([]);
|
||||
const animationFrameRef = useRef<number | null>(null);
|
||||
const timerRef = useRef<number | null>(null);
|
||||
const startTimeRef = useRef<number>(0);
|
||||
|
||||
// 清理资源
|
||||
const cleanup = useCallback(() => {
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current);
|
||||
animationFrameRef.current = null;
|
||||
}
|
||||
if (timerRef.current) {
|
||||
window.clearInterval(timerRef.current);
|
||||
timerRef.current = null;
|
||||
}
|
||||
if (streamRef.current) {
|
||||
streamRef.current.getTracks().forEach((track) => track.stop());
|
||||
streamRef.current = null;
|
||||
}
|
||||
if (audioContextRef.current) {
|
||||
void audioContextRef.current.close();
|
||||
audioContextRef.current = null;
|
||||
}
|
||||
analyserRef.current = null;
|
||||
mediaRecorderRef.current = null;
|
||||
}, []);
|
||||
|
||||
// 更新波形数据
|
||||
const updateWaveform = useCallback(() => {
|
||||
const analyser = analyserRef.current;
|
||||
if (!analyser) return;
|
||||
|
||||
const bufferLength = analyser.frequencyBinCount;
|
||||
const dataArray = new Uint8Array(bufferLength);
|
||||
analyser.getByteFrequencyData(dataArray);
|
||||
|
||||
// 取 32 个频段的平均值，用于波形显示
|
||||
const step = Math.floor(bufferLength / 32);
|
||||
const values: number[] = [];
|
||||
for (let i = 0; i < 32; i++) {
|
||||
let sum = 0;
|
||||
for (let j = 0; j < step; j++) {
|
||||
sum += dataArray[i * step + j];
|
||||
}
|
||||
values.push(sum / step / 255);
|
||||
}
|
||||
setWaveform(values);
|
||||
|
||||
animationFrameRef.current = requestAnimationFrame(updateWaveform);
|
||||
}, []);
|
||||
|
||||
// 开始录制
|
||||
const start = useCallback(async () => {
|
||||
setError(null);
|
||||
setWaveform([]);
|
||||
setDuration(0);
|
||||
chunksRef.current = [];
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
streamRef.current = stream;
|
||||
|
||||
const audioContext = new AudioContext();
|
||||
audioContextRef.current = audioContext;
|
||||
|
||||
const source = audioContext.createMediaStreamSource(stream);
|
||||
const analyser = audioContext.createAnalyser();
|
||||
analyser.fftSize = 256;
|
||||
source.connect(analyser);
|
||||
analyserRef.current = analyser;
|
||||
|
||||
const mediaRecorder = new MediaRecorder(stream);
|
||||
mediaRecorderRef.current = mediaRecorder;
|
||||
|
||||
mediaRecorder.ondataavailable = (event) => {
|
||||
if (event.data.size > 0) {
|
||||
chunksRef.current.push(event.data);
|
||||
}
|
||||
};
|
||||
|
||||
mediaRecorder.start(100);
|
||||
startTimeRef.current = Date.now();
|
||||
setState("recording");
|
||||
|
||||
// 启动波形更新
|
||||
animationFrameRef.current = requestAnimationFrame(updateWaveform);
|
||||
|
||||
// 启动计时器
|
||||
timerRef.current = window.setInterval(() => {
|
||||
const elapsed = (Date.now() - startTimeRef.current) / 1000;
|
||||
setDuration(elapsed);
|
||||
if (elapsed >= maxDuration) {
|
||||
// 达到最大时长，自动停止
|
||||
mediaRecorder.stop();
|
||||
}
|
||||
}, 100);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "无法访问麦克风";
|
||||
setError(message);
|
||||
setState("idle");
|
||||
cleanup();
|
||||
}
|
||||
|
||||
}, [cleanup, maxDuration, updateWaveform]);
|
||||
|
||||
// 停止录制
|
||||
const stop = useCallback((): Promise<Blob | null> => {
|
||||
return new Promise((resolve) => {
|
||||
const mediaRecorder = mediaRecorderRef.current;
|
||||
if (!mediaRecorder || mediaRecorder.state === "inactive") {
|
||||
cleanup();
|
||||
setState("idle");
|
||||
resolve(null);
|
||||
return;
|
||||
}
|
||||
|
||||
mediaRecorder.onstop = () => {
|
||||
const blob = chunksRef.current.length > 0 ? new Blob(chunksRef.current, { type: "audio/webm" }) : null;
|
||||
cleanup();
|
||||
setState("idle");
|
||||
resolve(blob);
|
||||
};
|
||||
|
||||
mediaRecorder.stop();
|
||||
});
|
||||
}, [cleanup]);
|
||||
|
||||
// 取消录制
|
||||
const cancel = useCallback(() => {
|
||||
chunksRef.current = [];
|
||||
cleanup();
|
||||
setState("idle");
|
||||
setWaveform([]);
|
||||
setDuration(0);
|
||||
}, [cleanup]);
|
||||
|
||||
// 重置
|
||||
const reset = useCallback(() => {
|
||||
cancel();
|
||||
setError(null);
|
||||
}, [cancel]);
|
||||
|
||||
// 组件卸载时清理
|
||||
useEffect(() => {
|
||||
return () => cleanup();
|
||||
}, [cleanup]);
|
||||
|
||||
return {
|
||||
state,
|
||||
waveform,
|
||||
duration,
|
||||
error,
|
||||
start,
|
||||
stop,
|
||||
cancel,
|
||||
reset,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user