362b9f238a
在 Android、Web 及 Flutter 控制端新增帧率选择 UI，被控端按屏幕刷新率生成支持的帧率档位并上报，允许在保持分辨率不变的情况下仅切换帧率。 另附带更新信号服务器的数据库配置。
333 lines
14 KiB
JavaScript
333 lines
14 KiB
JavaScript
import { encodeControlMessage, decodeControlMessage, Action } from '../proto/controlMessage';
|
||
|
||
// 对应 Android 端 WebRtcClient / Flutter webrtc_controller.dart。
|
||
// 作为 OFFER 方：仅接收远端视频（recvonly）+ 创建控制用 DataChannel，
|
||
// 并将屏幕触摸/按键转换为 protobuf 控制指令经 DataChannel 发送给被控端。
|
||
const DATA_CHANNEL_LABEL = 'control_channel';
|
||
|
||
export class WebRtcController {
|
||
constructor({ iceServers, deviceId, targetDeviceId, signaling, onIceState, onDataChannelState, onStream, onStats, onError, onResolutionReport }) {
|
||
this.iceServers = iceServers;
|
||
this.deviceId = deviceId;
|
||
this.targetDeviceId = targetDeviceId;
|
||
this.signaling = signaling;
|
||
this.onIceState = onIceState;
|
||
this.onDataChannelState = onDataChannelState;
|
||
this.onStream = onStream;
|
||
this.onStats = onStats;
|
||
this.onError = onError;
|
||
this.onResolutionReport = onResolutionReport;
|
||
|
||
this.pc = null;
|
||
this.dataChannel = null;
|
||
this._statsTimer = null;
|
||
this._prevBytes = 0;
|
||
this._prevTs = 0;
|
||
this._prevSentBytes = 0;
|
||
this._prevSentTs = 0;
|
||
this._connectedAt = 0;
|
||
this._gatheredRelay = false;
|
||
this._gatheredSrflx = false;
|
||
this._gatheredHost = false;
|
||
|
||
// 关键修复：浏览器要求 addIceCandidate 必须在 setRemoteDescription(answer) 之后调用。
|
||
// 被控端会在一瞬间突发大量候选，若此时远端描述尚未设置则会整批失败被丢弃，导致 ICE 无法配对。
|
||
// 因此先把远端候选缓存起来，等 answer 设置完成后再统一 flush。
|
||
this._remoteDescSet = false;
|
||
this._pendingCandidates = [];
|
||
}
|
||
|
||
async createOffer(authType = null, authValue = null) {
|
||
this.pc = new RTCPeerConnection({ iceServers: this.iceServers });
|
||
|
||
this.pc.onicecandidate = (e) => {
|
||
if (e.candidate) {
|
||
if (e.candidate.candidate.includes('typ relay')) this._gatheredRelay = true;
|
||
else if (e.candidate.candidate.includes('typ srflx')) this._gatheredSrflx = true;
|
||
else if (e.candidate.candidate.includes('typ host')) this._gatheredHost = true;
|
||
this.signaling.sendIceCandidate(e.candidate, this.targetDeviceId);
|
||
} else {
|
||
console.info('[ICE] 本端候选收集完成 relay=%s srflx=%s host=%s',
|
||
this._gatheredRelay, this._gatheredSrflx, this._gatheredHost);
|
||
}
|
||
};
|
||
// 关键诊断：TURN/STUN 分配失败时浏览器会触发该事件，原代码未捕获导致原因被吞掉。
|
||
this.pc.onicecandidateerror = (e) => {
|
||
const url = e.url || '';
|
||
console.warn('[ICE 候选错误] url=%s code=%s text=%s', url, e.errorCode, e.errorText);
|
||
};
|
||
this.pc.oniceconnectionstatechange = () => {
|
||
const s = this.pc.iceConnectionState;
|
||
if (s === 'connected') this._connectedAt = Date.now();
|
||
this.onIceState && this.onIceState(s);
|
||
};
|
||
this.pc.onconnectionstatechange = () => {
|
||
const s = this.pc.connectionState;
|
||
if (s === 'failed') {
|
||
// ICE 彻底失败：通常是双方没有可达的候选路径（需 TURN 中继或同一网络）。
|
||
const relayInfo = this._gatheredRelay
|
||
? '本端已拿到 TURN 中继候选'
|
||
: '本端未拿到任何 TURN 中继候选（中继分配很可能失败，见上方 TURN 错误）';
|
||
this.onError && this.onError('WebRTC 连接失败（connectionState=failed）。iceConnectionState='
|
||
+ this.pc.iceConnectionState + '；' + relayInfo
|
||
+ '。请确认浏览器能访问 TURN 服务器 175.178.213.60:3478（UDP/TCP 至少一种可达），且被控端与本机网络可互通。');
|
||
} else if (s === 'connecting') {
|
||
this.onError && this.onError('');
|
||
}
|
||
this.onIceState && this.onIceState(s);
|
||
};
|
||
this.pc.ontrack = (e) => {
|
||
// 某些浏览器/协商场景下 event.streams 可能为空，需回退到用 track 自行构造 MediaStream，
|
||
// 否则 store.remoteStream 始终为空 -> 视频元素拿不到流 -> 黑屏（用户会误以为无法控制）。
|
||
let stream = e.streams && e.streams[0];
|
||
if (!stream) {
|
||
stream = new MediaStream([e.track]);
|
||
}
|
||
this.onStream && this.onStream(stream);
|
||
e.track.onunmute = () => {
|
||
if (e.track.readyState === 'live') {
|
||
this.onStream && this.onStream(stream);
|
||
}
|
||
};
|
||
};
|
||
this.pc.ondatachannel = (e) => this.setupDataChannel(e.channel);
|
||
|
||
// 仅接收被控端屏幕视频
|
||
this.pc.addTransceiver('video', { direction: 'recvonly' });
|
||
|
||
// 控制用 DataChannel：非可靠、无序，降低延迟（与被控端协商一致）
|
||
const dc = this.pc.createDataChannel(DATA_CHANNEL_LABEL, { ordered: false, maxRetransmits: 0 });
|
||
this.setupDataChannel(dc);
|
||
|
||
const offer = await this.pc.createOffer();
|
||
// 与 WebRTCControlled 的 optimizeSdp 保持一致：将 m=video 行的编解码器
|
||
// 重排为 VP8/VP9 优先、H264 兜底，确保各控制端（浏览器/桌面/Chromium/Linux）
|
||
// 都能稳定解码，避免 H264 High Profile 无法解码而黑屏。
|
||
offer.sdp = this.preferVideoCodecs(offer.sdp);
|
||
await this.pc.setLocalDescription(offer);
|
||
this.signaling.sendOffer(offer.sdp, this.targetDeviceId, authType, authValue);
|
||
|
||
this._startStats();
|
||
}
|
||
|
||
/**
|
||
* 重排 Offer SDP 中 m=video 行的视频编解码器顺序：VP8/VP9 优先，H264 兜底。
|
||
* 与 Android 被控端 WebRtcClient.optimizeSdp 的编解码优先级保持一致，
|
||
* 保证协商出的编码格式控制端一定可解码（编码/解码一致性）。
|
||
* @param {string} sdp 原始 SDP
|
||
* @returns {string} 重排后的 SDP
|
||
*/
|
||
preferVideoCodecs(sdp) {
|
||
const lines = sdp.split('\r\n');
|
||
const result = [];
|
||
|
||
// 先收集各类型视频负载的 payload type
|
||
const vp8 = [];
|
||
const vp9 = [];
|
||
const h264 = [];
|
||
for (const line of lines) {
|
||
const t = line.trim();
|
||
if (t.startsWith('a=rtpmap:')) {
|
||
const payload = t.split(':')[1].split(' ')[0];
|
||
if (t.includes('VP8/90000')) vp8.push(payload);
|
||
else if (t.includes('VP9/90000')) vp9.push(payload);
|
||
else if (t.includes('H264/90000')) h264.push(payload);
|
||
}
|
||
}
|
||
const ordered = [...vp8, ...vp9, ...h264];
|
||
if (ordered.length === 0) return sdp;
|
||
|
||
for (const line of lines) {
|
||
const t = line.trim();
|
||
if (t.startsWith('m=video')) {
|
||
const parts = t.split(' ');
|
||
if (parts.length > 3) {
|
||
const head = [parts[0], parts[1], parts[2]].join(' ');
|
||
const rest = parts.slice(3).filter((p) => !ordered.includes(p));
|
||
result.push([head, ...ordered, ...rest].join(' '));
|
||
continue;
|
||
}
|
||
}
|
||
result.push(line);
|
||
}
|
||
return result.join('\r\n');
|
||
}
|
||
|
||
setupDataChannel(dc) {
|
||
this.dataChannel = dc;
|
||
dc.onopen = () => this.onDataChannelState && this.onDataChannelState(true);
|
||
dc.onclose = () => this.onDataChannelState && this.onDataChannelState(false);
|
||
dc.onmessage = (e) => {
|
||
try {
|
||
// 被控端发来的是二进制 protobuf（目前仅 REPORT_RESOLUTION 上报）。
|
||
const bytes = e.data instanceof ArrayBuffer ? new Uint8Array(e.data) : e.data;
|
||
const msg = decodeControlMessage(bytes);
|
||
if (msg && msg.action === Action.REPORT_RESOLUTION) {
|
||
this.onResolutionReport && this.onResolutionReport({
|
||
width: msg.width | 0,
|
||
height: msg.height | 0,
|
||
fps: msg.fps | 0,
|
||
// 被控端按屏幕刷新率筛选出的帧率档位（供帧率下拉框使用）
|
||
supportedFps: Array.isArray(msg.supportedFps) ? msg.supportedFps.map((f) => f | 0) : [],
|
||
});
|
||
}
|
||
} catch (err) {
|
||
console.debug('[DataChannel] 解码消息失败（非分辨率上报或格式异常）', err);
|
||
}
|
||
};
|
||
}
|
||
|
||
async handleAnswer(sdp) {
|
||
if (!this.pc) return;
|
||
try {
|
||
await this.pc.setRemoteDescription({ type: 'answer', sdp });
|
||
// 诊断：确认视频收发方向，便于区分“编解码器问题”与“连接方向问题”
|
||
const vt = this.pc.getTransceivers
|
||
? this.pc.getTransceivers().find((t) => t.receiver && t.receiver.track && t.receiver.track.kind === 'video')
|
||
: null;
|
||
console.info('[协商完成] video transceiver currentDirection=', vt ? vt.currentDirection : 'n/a');
|
||
} catch (e) {
|
||
this.onError && this.onError('设置远端描述(ANSWER)失败: ' + (e?.message || e));
|
||
throw e;
|
||
}
|
||
// 远端描述已就绪，flush 之前缓存的候选。
|
||
this._remoteDescSet = true;
|
||
const pending = this._pendingCandidates;
|
||
this._pendingCandidates = [];
|
||
for (const c of pending) {
|
||
try {
|
||
await this.pc.addIceCandidate(c);
|
||
} catch (e) {
|
||
console.warn('flush addIceCandidate 失败', e);
|
||
}
|
||
}
|
||
}
|
||
|
||
async handleIceCandidate(payload) {
|
||
if (!this.pc) return;
|
||
const cand = { candidate: payload.candidate, sdpMid: payload.sdpMid, sdpMLineIndex: payload.sdpMLineIndex };
|
||
if (!this._remoteDescSet) {
|
||
// 远端描述尚未设置，先缓存，避免整批候选被浏览器丢弃。
|
||
this._pendingCandidates.push(cand);
|
||
return;
|
||
}
|
||
try {
|
||
await this.pc.addIceCandidate(cand);
|
||
} catch (e) {
|
||
console.warn('addIceCandidate 失败', e);
|
||
}
|
||
}
|
||
|
||
sendControlMessage(fields) {
|
||
if (!this.dataChannel || this.dataChannel.readyState !== 'open') return false;
|
||
const bytes = encodeControlMessage(fields);
|
||
this.dataChannel.send(bytes);
|
||
return true;
|
||
}
|
||
|
||
sendTouch(x, y) { return this.sendControlMessage({ action: 1, x, y }); } // TOUCH
|
||
sendSwipe(x1, y1, x2, y2, duration) { return this.sendControlMessage({ action: 2, x1, y1, x2, y2, duration }); } // SWIPE
|
||
sendKey(keyCode) { return this.sendControlMessage({ action: 3, keyCode, keyAction: 0 }); } // KEY
|
||
sendLongPress(x, y) { return this.sendControlMessage({ action: 4, x, y }); } // LONG_PRESS
|
||
sendMotionEvent(action, x, y) { return this.sendControlMessage({ action: 5, motionAction: action, x, y }); } // MOTION_EVENT
|
||
sendResolutionChange(width, height, fps) { return this.sendControlMessage({ action: 6, width, height, fps }); } // SET_RESOLUTION
|
||
|
||
_startStats() {
|
||
this._statsTimer = setInterval(async () => {
|
||
if (!this.pc) return;
|
||
try {
|
||
this.onStats && this.onStats(await this.collectStats());
|
||
} catch { /* ignore */ }
|
||
}, 1000);
|
||
}
|
||
|
||
async collectStats() {
|
||
const stats = await this.pc.getStats();
|
||
let width = '-', height = '-', fps = '-', codec = '-';
|
||
let bytesReceived = 0, bytesSent = 0;
|
||
let jitter = 0, framesDropped = 0, packetsLost = 0, packetsReceived = 0;
|
||
let framesDecoded = 0, totalDecodeTime = 0, rtt = null;
|
||
const codecs = {};
|
||
stats.forEach((r) => {
|
||
if (r.type === 'inbound-rtp' && r.kind === 'video') {
|
||
width = r.frameWidth ?? '-';
|
||
height = r.frameHeight ?? '-';
|
||
fps = r.framesPerSecond ?? '-';
|
||
if (r.codecId && codecs[r.codecId]) codec = codecs[r.codecId];
|
||
jitter = r.jitter ?? 0; // 接收抖动（秒）
|
||
framesDropped = r.framesDropped ?? 0; // 累计丢帧数
|
||
packetsLost = r.packetsLost ?? 0; // 累计丢包数
|
||
packetsReceived = r.packetsReceived ?? 0; // 累计接收包数
|
||
framesDecoded = r.framesDecoded ?? 0; // 累计解码帧数
|
||
totalDecodeTime = r.totalDecodeTime ?? 0; // 累计解码耗时（秒）
|
||
} else if (r.type === 'codec' && r.mimeType && r.mimeType.startsWith('video/')) {
|
||
codecs[r.id] = r.mimeType.substring(6);
|
||
} else if (r.type === 'candidate-pair' && r.nominated) {
|
||
bytesReceived = r.bytesReceived ?? 0;
|
||
bytesSent = r.bytesSent ?? 0;
|
||
if (typeof r.currentRoundTripTime === 'number') rtt = r.currentRoundTripTime; // 往返时延（秒）
|
||
}
|
||
});
|
||
const now = Date.now();
|
||
// 下载速率
|
||
let downSpeed = '-';
|
||
if (this._prevTs) {
|
||
const dt = (now - this._prevTs) / 1000;
|
||
if (dt > 0) {
|
||
const bps = (bytesReceived - this._prevBytes) / dt;
|
||
downSpeed = formatSpeed(bps);
|
||
}
|
||
}
|
||
this._prevBytes = bytesReceived;
|
||
this._prevTs = now;
|
||
// 上传速率
|
||
let upSpeed = '-';
|
||
if (this._prevSentTs) {
|
||
const dt = (now - this._prevSentTs) / 1000;
|
||
if (dt > 0) {
|
||
const bps = (bytesSent - this._prevSentBytes) / dt;
|
||
upSpeed = formatSpeed(bps);
|
||
}
|
||
}
|
||
this._prevSentBytes = bytesSent;
|
||
this._prevSentTs = now;
|
||
let duration = '-';
|
||
if (this._connectedAt) {
|
||
const secs = Math.floor((now - this._connectedAt) / 1000);
|
||
duration = String(Math.floor(secs / 60)).padStart(2, '0') + ':' + String(secs % 60).padStart(2, '0');
|
||
}
|
||
// 延迟（往返时延）
|
||
const latency = rtt != null ? (rtt * 1000).toFixed(0) + ' ms' : '-';
|
||
// 抖动
|
||
const jitterMs = jitter ? (jitter * 1000).toFixed(1) + ' ms' : '-';
|
||
// 解码耗时（平均每帧解码时间）
|
||
const decodeMs = (framesDecoded > 0 && totalDecodeTime > 0)
|
||
? (totalDecodeTime / framesDecoded * 1000).toFixed(1) + ' ms' : '-';
|
||
// 丢包率
|
||
const totalPackets = packetsLost + packetsReceived;
|
||
const lossRate = totalPackets > 0 ? (packetsLost / totalPackets * 100).toFixed(1) + '%' : '-';
|
||
return {
|
||
width, height, fps, codec, downSpeed, upSpeed, duration,
|
||
latency, jitter: jitterMs, framesDropped, decodeMs, packetLoss: lossRate,
|
||
dcState: this.dataChannel ? this.dataChannel.readyState : 'none',
|
||
};
|
||
}
|
||
|
||
async close() {
|
||
if (this._statsTimer) clearInterval(this._statsTimer);
|
||
if (this.dataChannel) { try { this.dataChannel.close(); } catch { /* ignore */ } }
|
||
if (this.pc) { try { await this.pc.close(); } catch { /* ignore */ } }
|
||
this.dataChannel = null;
|
||
this.pc = null;
|
||
this._remoteDescSet = false;
|
||
this._pendingCandidates = [];
|
||
}
|
||
}
|
||
|
||
// 将字节/秒格式化为可读速率
|
||
function formatSpeed(bps) {
|
||
if (!bps || bps <= 0) return '0 B/s';
|
||
return bps >= 1048576 ? (bps / 1048576).toFixed(1) + ' MB/s'
|
||
: bps >= 1024 ? (bps / 1024).toFixed(1) + ' KB/s' : bps.toFixed(0) + ' B/s';
|
||
}
|