TongTongStudio
778c37b8db
- 修复 MediaCodec 回调需在 configure 前设置导致的异常 - 修复解码器线程安全问题及包乱序/重复处理 - 增加关键帧前重发 SPS/PPS 以防首帧丢失导致黑屏 - 支持分辨率切换时重置解码器 - 修复 Flutter 控制端连接类型选项 UI 显示拥挤问题
556 lines
19 KiB
Dart
556 lines
19 KiB
Dart
import 'package:flutter/foundation.dart';
|
||
import 'package:flutter_webrtc/flutter_webrtc.dart';
|
||
|
||
import 'self_codec_decoder.dart';
|
||
|
||
import '../config/ice_servers.dart';
|
||
import '../models/signal_message.dart';
|
||
import '../proto/control_message.pb.dart';
|
||
import '../signaling/signaling_client.dart';
|
||
import '../utils/control_commands.dart';
|
||
|
||
/// 封装 WebRTC 连接逻辑（对应 Android 端 WebRtcClient）。
|
||
///
|
||
/// 职责：
|
||
/// - 创建 PeerConnection（recvonly 视频 + 控制用 DataChannel）
|
||
/// - 创建 Offer 并通过信令发送
|
||
/// - 处理 Answer / ICE 候选
|
||
/// - 通过 DataChannel 发送控制指令（触摸/滑动/按键）
|
||
/// - 采集并解析连接统计信息
|
||
///
|
||
/// 本类同时兼容 Android 与 iOS（基于 flutter_webrtc）。
|
||
class WebRtcController {
|
||
final SignalingClient signaling;
|
||
final String deviceId;
|
||
final String targetDeviceId;
|
||
final String? authType;
|
||
final String? authValue;
|
||
|
||
RTCPeerConnection? _pc;
|
||
RTCDataChannel? _dataChannel;
|
||
RTCDataChannel? _videoDataChannel;
|
||
SelfCodecDecoder? _selfCodecDecoder;
|
||
int _currentStreamMode = streamModeWebRtc;
|
||
bool _creatingDecoder = false;
|
||
|
||
/// 远端视频渲染器，由调用方持有并显示。
|
||
final RTCVideoRenderer renderer = RTCVideoRenderer();
|
||
|
||
void Function()? onConnectionEstablished;
|
||
void Function(String error)? onConnectionFailed;
|
||
void Function()? onDisconnected;
|
||
|
||
/// ICE 连接断开（如网络中断、被控端退出），用于提示用户并返回连接设置。
|
||
void Function(String message)? onIceDisconnected;
|
||
|
||
/// 远端视频流就绪（renderer 已绑定视频轨道）。
|
||
void Function(RTCVideoRenderer renderer)? onRemoteStream;
|
||
|
||
/// 自编码解码纹理已就绪（textureId >= 0）。
|
||
void Function(int)? onSelfCodecReady;
|
||
|
||
/// 自编码解码器已释放（连接断开时）。
|
||
void Function()? onSelfCodecLost;
|
||
|
||
/// 被控端上报当前生效的串流模式。
|
||
void Function(int mode)? onStreamModeReport;
|
||
|
||
/// 被控端上报分辨率（自编码解码器输出尺寸变化时也会触发）。
|
||
void Function(int width, int height)? onResolutionReported;
|
||
|
||
/// 当前平台（Windows / Linux / Web / iOS 等）不支持原生硬解时回调。
|
||
void Function()? onSelfCodecNotSupported;
|
||
|
||
static const String kVideoChannelLabel = 'video_channel';
|
||
static const int streamModeWebRtc = 0;
|
||
static const int streamModeSelfCodec = 1;
|
||
|
||
WebRtcController({
|
||
required this.signaling,
|
||
required this.deviceId,
|
||
required this.targetDeviceId,
|
||
this.authType,
|
||
this.authValue,
|
||
});
|
||
|
||
/// 初始化渲染器、PeerConnection，并创建 Offer。
|
||
Future<void> initialize() async {
|
||
await renderer.initialize();
|
||
|
||
final configuration = {
|
||
'iceServers': kIceServers,
|
||
'sdpSemantics': 'unified-plan',
|
||
'iceCandidatePoolSize': 10,
|
||
// 增强复杂网络下的稳定性，参考 Android 端配置。
|
||
'continualGatheringPolicy': 'gatherContinually',
|
||
'iceTransportsType': 'all',
|
||
'tcpCandidatePolicy': 'enabled',
|
||
};
|
||
|
||
_pc = await createPeerConnection(configuration);
|
||
|
||
_pc!.onIceCandidate = _onIceCandidate;
|
||
_pc!.onIceConnectionState = _onIceConnectionState;
|
||
_pc!.onTrack = _onTrack;
|
||
_pc!.onDataChannel = _onDataChannel;
|
||
|
||
// 仅接收远端视频（recvonly）。
|
||
await _pc!.addTransceiver(
|
||
kind: RTCRtpMediaType.RTCRtpMediaTypeVideo,
|
||
init: RTCRtpTransceiverInit(direction: TransceiverDirection.RecvOnly),
|
||
);
|
||
|
||
// 创建控制用 DataChannel。
|
||
// 使用非可靠、无序模式以降低延迟，解决“不跟手”问题。
|
||
final dcInit = RTCDataChannelInit()
|
||
..ordered = false
|
||
..maxRetransmits = 0;
|
||
_dataChannel = await _pc!.createDataChannel(kDataChannelLabel, dcInit);
|
||
_setupDataChannel(_dataChannel!);
|
||
|
||
// 创建自编码视频透传通道（无序、不可靠，类似 RTP，最低延迟）。
|
||
// 被控端仅在收到 SET_STREAM_MODE=自编码 后才经此通道推送 H.264 裸流。
|
||
final videoInit = RTCDataChannelInit()
|
||
..ordered = false
|
||
..maxRetransmits = 0;
|
||
_videoDataChannel = await _pc!.createDataChannel(
|
||
kVideoChannelLabel,
|
||
videoInit,
|
||
);
|
||
_setupVideoDataChannel(_videoDataChannel!);
|
||
|
||
// 创建并发送 Offer。
|
||
final constraints = {
|
||
'mandatory': {
|
||
'OfferToReceiveVideo': true,
|
||
'OfferToReceiveAudio': false,
|
||
},
|
||
'optional': [],
|
||
};
|
||
final offer = await _pc!.createOffer(constraints);
|
||
await _pc!.setLocalDescription(offer);
|
||
_sendOffer(offer.sdp!, authType: authType, authValue: authValue);
|
||
}
|
||
|
||
void _onTrack(RTCTrackEvent event) {
|
||
if (event.track.kind == 'video' && event.streams.isNotEmpty) {
|
||
renderer.srcObject = event.streams[0];
|
||
onRemoteStream?.call(renderer);
|
||
}
|
||
}
|
||
|
||
void _onDataChannel(RTCDataChannel channel) {
|
||
// 被控端也可能主动创建 DataChannel，按标签统一处理。
|
||
if (channel.label == kDataChannelLabel) {
|
||
_dataChannel ??= channel;
|
||
_setupDataChannel(channel);
|
||
} else if (channel.label == kVideoChannelLabel) {
|
||
_videoDataChannel ??= channel;
|
||
_setupVideoDataChannel(channel);
|
||
}
|
||
}
|
||
|
||
void _setupDataChannel(RTCDataChannel channel) {
|
||
channel.onMessage = (RTCDataChannelMessage message) {
|
||
if (message.isBinary) {
|
||
try {
|
||
final command = ControlMessage.fromBuffer(message.binary);
|
||
_handleControlMessage(command);
|
||
} catch (e) {
|
||
// ignore: avoid_print
|
||
print('DataChannel message decode failed: $e');
|
||
}
|
||
} else {
|
||
// ignore: avoid_print
|
||
print('DataChannel text message: ${message.text}');
|
||
}
|
||
};
|
||
channel.onDataChannelState = (RTCDataChannelState state) {
|
||
// ignore: avoid_print
|
||
print('DataChannel state: $state');
|
||
};
|
||
}
|
||
|
||
void _setupVideoDataChannel(RTCDataChannel channel) {
|
||
channel.onMessage = (RTCDataChannelMessage message) {
|
||
if (message.isBinary) {
|
||
_selfCodecDecoder?.feed(message.binary);
|
||
}
|
||
};
|
||
channel.onDataChannelState = (RTCDataChannelState state) {
|
||
if (state == RTCDataChannelState.RTCDataChannelOpen) {
|
||
_onVideoChannelOpen();
|
||
} else if (state == RTCDataChannelState.RTCDataChannelClosed) {
|
||
_selfCodecDecoder?.setEnabled(false);
|
||
}
|
||
};
|
||
}
|
||
|
||
void _onVideoChannelOpen() {
|
||
_ensureSelfCodecDecoder().then((_) {
|
||
_selfCodecDecoder?.setEnabled(_currentStreamMode == streamModeSelfCodec);
|
||
});
|
||
}
|
||
|
||
Future<void> _ensureSelfCodecDecoder() async {
|
||
if (_selfCodecDecoder != null || _creatingDecoder) return;
|
||
_creatingDecoder = true;
|
||
try {
|
||
final decoder = SelfCodecDecoder(
|
||
onSize: (w, h) => onResolutionReported?.call(w, h),
|
||
onError: (err) => debugPrint('自编码解码器: $err'),
|
||
);
|
||
final textureId = await decoder.create();
|
||
if (textureId != null) {
|
||
_selfCodecDecoder = decoder;
|
||
onSelfCodecReady?.call(textureId);
|
||
} else {
|
||
// 当前平台（Windows / Linux / Web / iOS 等）无原生硬解能力，回退 WebRTC 媒体流。
|
||
onSelfCodecNotSupported?.call();
|
||
}
|
||
} finally {
|
||
_creatingDecoder = false;
|
||
}
|
||
}
|
||
|
||
void _handleControlMessage(ControlMessage msg) {
|
||
switch (msg.action) {
|
||
case Action.REPORT_STREAM_MODE:
|
||
_currentStreamMode = msg.streamMode;
|
||
if (msg.streamMode == streamModeSelfCodec) {
|
||
_ensureSelfCodecDecoder().then((_) {
|
||
_selfCodecDecoder?.setEnabled(true);
|
||
});
|
||
} else {
|
||
_selfCodecDecoder?.setEnabled(false);
|
||
}
|
||
onStreamModeReport?.call(msg.streamMode);
|
||
break;
|
||
case Action.REPORT_RESOLUTION:
|
||
onResolutionReported?.call(msg.width, msg.height);
|
||
break;
|
||
default:
|
||
// ignore: avoid_print
|
||
print('DataChannel message: ${msg.action}');
|
||
break;
|
||
}
|
||
}
|
||
|
||
/// 请求被控端切换屏幕串流模式（SET_STREAM_MODE 指令）。
|
||
/// [mode] 为 0=WebRTC 全托管 / 1=自编码。
|
||
Future<void> sendStreamMode(int mode) async {
|
||
_currentStreamMode = mode;
|
||
if (mode == streamModeSelfCodec) {
|
||
await _ensureSelfCodecDecoder();
|
||
await _selfCodecDecoder?.setEnabled(true);
|
||
} else {
|
||
await _selfCodecDecoder?.setEnabled(false);
|
||
}
|
||
sendControlCommand(ControlCommands.streamMode(mode));
|
||
}
|
||
|
||
void _onIceCandidate(RTCIceCandidate candidate) {
|
||
final payload = {
|
||
'sdpMid': candidate.sdpMid,
|
||
'sdpMLineIndex': candidate.sdpMLineIndex,
|
||
'candidate': candidate.candidate,
|
||
};
|
||
final msg = SignalMessage.withPayload(
|
||
type: 'ICE_CANDIDATE',
|
||
fromDeviceId: deviceId,
|
||
toDeviceId: targetDeviceId,
|
||
deviceType: 'CONTROLLER',
|
||
payload: payload,
|
||
);
|
||
signaling.send(msg);
|
||
}
|
||
|
||
void _onIceConnectionState(RTCIceConnectionState state) {
|
||
// ignore: avoid_print
|
||
print('ICE connection state: $state');
|
||
if (state == RTCIceConnectionState.RTCIceConnectionStateConnected) {
|
||
_connectedAt = DateTime.now().millisecondsSinceEpoch;
|
||
onConnectionEstablished?.call();
|
||
} else if (state == RTCIceConnectionState.RTCIceConnectionStateDisconnected) {
|
||
onIceDisconnected?.call('ICE 连接已断开');
|
||
} else if (state == RTCIceConnectionState.RTCIceConnectionStateFailed) {
|
||
onConnectionFailed?.call('ICE 连接失败');
|
||
onDisconnected?.call();
|
||
}
|
||
}
|
||
|
||
void _sendOffer(String sdp, {String? authType, String? authValue}) {
|
||
final payload = {'sdp': sdp};
|
||
final msg = SignalMessage.withPayload(
|
||
type: 'OFFER',
|
||
fromDeviceId: deviceId,
|
||
toDeviceId: targetDeviceId,
|
||
deviceType: 'CONTROLLER',
|
||
payload: payload,
|
||
authType: authType,
|
||
authValue: authValue,
|
||
);
|
||
signaling.send(msg);
|
||
}
|
||
|
||
/// 处理来自信令服务器的 Answer。
|
||
Future<void> handleAnswer(String sdp) async {
|
||
final answer = RTCSessionDescription(sdp, 'answer');
|
||
await _pc?.setRemoteDescription(answer);
|
||
}
|
||
|
||
/// 处理来自信令服务器的 ICE 候选。
|
||
Future<void> handleIceCandidate(Map<String, dynamic> payload) async {
|
||
final candidate = RTCIceCandidate(
|
||
payload['candidate'] as String,
|
||
payload['sdpMid'] as String?,
|
||
payload['sdpMLineIndex'] as int?,
|
||
);
|
||
await _pc?.addCandidate(candidate);
|
||
}
|
||
|
||
/// 通过 DataChannel 发送控制指令（protobuf 二进制）。
|
||
void sendControlCommand(ControlMessage command) {
|
||
if (_dataChannel?.state == RTCDataChannelState.RTCDataChannelOpen) {
|
||
debugPrint('WebRtcController: Sending command -> ${command.action}');
|
||
_dataChannel!.send(RTCDataChannelMessage.fromBinary(command.writeToBuffer()));
|
||
}
|
||
}
|
||
|
||
/// 请求被控端切换屏幕采集分辨率（SET_RESOLUTION 指令）。
|
||
void sendResolutionChange(int width, int height, int fps) {
|
||
sendControlCommand(ControlCommands.setResolution(width, height, fps));
|
||
}
|
||
|
||
/// 上一次统计采样的字节数与时间戳，用于计算每秒网速。
|
||
int _prevBytesReceived = 0;
|
||
int _prevBytesSent = 0;
|
||
int _prevStatsTimestamp = 0;
|
||
|
||
/// 连接建立时的时间戳（毫秒），用于计算连接时长。
|
||
int? _connectedAt;
|
||
|
||
/// 采集连接统计信息，回调格式化后的文本（对应 Android 端 updateStats）。
|
||
///
|
||
/// 包含：分辨率/帧率/延迟/解码格式、抖动、丢包率、丢帧、
|
||
/// 每秒上下行网速、连接类型(P2P/TURN)+候选协议、连接时长、
|
||
/// 控制 DataChannel 状态与收发消息数。
|
||
Future<String> getStatsText() async {
|
||
if (_pc == null) return '';
|
||
final reports = await _pc!.getStats();
|
||
|
||
dynamic width = '-';
|
||
dynamic height = '-';
|
||
dynamic fps = '-';
|
||
dynamic delay = '-';
|
||
dynamic codec = '-';
|
||
dynamic jitter = '-';
|
||
dynamic lossRate = '-';
|
||
dynamic framesDropped = '-';
|
||
|
||
int bytesReceived = 0;
|
||
int bytesSent = 0;
|
||
String? localCandidateId;
|
||
String? remoteCandidateId;
|
||
bool hasNominatedPair = false;
|
||
|
||
// 控制 DataChannel 统计（仅统计与对端的控制通道）。
|
||
int dcMessagesSent = 0;
|
||
int dcMessagesReceived = 0;
|
||
|
||
for (final report in reports) {
|
||
final values = report.values;
|
||
final type = report.type;
|
||
if (type == 'inbound-rtp' && values['kind'] == 'video') {
|
||
width = values['frameWidth'] ?? '-';
|
||
height = values['frameHeight'] ?? '-';
|
||
fps = values['framesPerSecond'] ?? '-';
|
||
|
||
final j = values['jitter'];
|
||
if (j is num) jitter = (j * 1000).toStringAsFixed(0);
|
||
|
||
final pr = values['packetsReceived'];
|
||
final pl = values['packetsLost'];
|
||
if (pr is num && pl is num) {
|
||
final total = pr + pl;
|
||
lossRate = total > 0 ? ((pl / total) * 100).toStringAsFixed(1) : '0.0';
|
||
}
|
||
|
||
final fd = values['framesDropped'];
|
||
if (fd is num) framesDropped = fd;
|
||
|
||
final codecId = values['codecId'];
|
||
if (codecId != null) {
|
||
final codecReport = reports.where((r) => r.id == codecId).firstOrNull;
|
||
final mime = codecReport?.values['mimeType'];
|
||
if (mime is String && mime.startsWith('video/')) {
|
||
codec = mime.substring(6);
|
||
}
|
||
}
|
||
} else if (type == 'candidate-pair') {
|
||
if (values['nominated'] == true) {
|
||
hasNominatedPair = true;
|
||
final rtt = values['currentRoundTripTime'];
|
||
if (rtt is num) {
|
||
delay = (rtt * 1000).toStringAsFixed(0);
|
||
}
|
||
// candidate-pair 的 bytesReceived/bytesSent 为整条连接（视频+DataChannel+ICE）的累计值。
|
||
final br = values['bytesReceived'];
|
||
final bs = values['bytesSent'];
|
||
if (br is num) bytesReceived = br.toInt();
|
||
if (bs is num) bytesSent = bs.toInt();
|
||
final lc = values['localCandidateId'];
|
||
final rc = values['remoteCandidateId'];
|
||
if (lc is String) localCandidateId = lc;
|
||
if (rc is String) remoteCandidateId = rc;
|
||
}
|
||
} else if (type == 'data-channel') {
|
||
if (values['label'] == kDataChannelLabel) {
|
||
final ms = values['messagesSent'];
|
||
final mr = values['messagesReceived'];
|
||
if (ms is num) dcMessagesSent = ms.toInt();
|
||
if (mr is num) dcMessagesReceived = mr.toInt();
|
||
}
|
||
}
|
||
}
|
||
|
||
// 计算每秒网速（与上次采样差值）。
|
||
final now = DateTime.now().millisecondsSinceEpoch;
|
||
String downSpeed = '-';
|
||
String upSpeed = '-';
|
||
if (_prevStatsTimestamp != 0) {
|
||
final dt = (now - _prevStatsTimestamp) / 1000.0;
|
||
if (dt > 0) {
|
||
final downBps = (bytesReceived - _prevBytesReceived) / dt;
|
||
final upBps = (bytesSent - _prevBytesSent) / dt;
|
||
downSpeed = _formatSpeed(downBps);
|
||
upSpeed = _formatSpeed(upBps);
|
||
}
|
||
}
|
||
_prevBytesReceived = bytesReceived;
|
||
_prevBytesSent = bytesSent;
|
||
_prevStatsTimestamp = now;
|
||
|
||
// 连接类型（P2P/TURN）与候选协议（udp/tcp）。
|
||
String connType = '-';
|
||
String protocol = '-';
|
||
if (hasNominatedPair) {
|
||
connType =
|
||
_resolveConnectionType(reports, localCandidateId, remoteCandidateId);
|
||
protocol = _resolveProtocol(reports, localCandidateId, remoteCandidateId);
|
||
}
|
||
|
||
// 连接时长。
|
||
String duration = '-';
|
||
if (_connectedAt != null) {
|
||
final secs = ((now - _connectedAt!) / 1000).floor();
|
||
final m = (secs ~/ 60).toString().padLeft(2, '0');
|
||
final s = (secs % 60).toString().padLeft(2, '0');
|
||
duration = '$m:$s';
|
||
}
|
||
|
||
// 控制 DataChannel 状态。
|
||
final dcState = _dataChannel?.state;
|
||
final dcStatus = dcState == RTCDataChannelState.RTCDataChannelOpen
|
||
? '已连接'
|
||
: (dcState == null ? '未连接' : dcState.name);
|
||
|
||
return '分辨率: ${width}x$height 帧率: $fps 延迟: $delay ms\n'
|
||
'解码格式: $codec 抖动: $jitter ms 丢包: $lossRate%\n'
|
||
'丢帧: $framesDropped ↓下载: $downSpeed ↑上传: $upSpeed\n'
|
||
'连接类型: $connType ($protocol) 时长: $duration\n'
|
||
'控制通道: $dcStatus 收发: $dcMessagesSent/$dcMessagesReceived';
|
||
}
|
||
|
||
/// 依据选定候选对的本地/远端候选类型，返回 P2P / TURN 中继描述。
|
||
String _resolveConnectionType(
|
||
List<StatsReport> reports,
|
||
String? localCandidateId,
|
||
String? remoteCandidateId,
|
||
) {
|
||
String? typeOf(String? id) {
|
||
if (id == null) return null;
|
||
final report = reports.where((r) => r.id == id).firstOrNull;
|
||
final t = report?.values['candidateType'];
|
||
return t is String ? t : null;
|
||
}
|
||
|
||
final local = typeOf(localCandidateId);
|
||
final remote = typeOf(remoteCandidateId);
|
||
final isRelay = local == 'relay' || remote == 'relay';
|
||
|
||
if (isRelay) {
|
||
// 进一步用本地候选具体类型细化（优先说明本端是否经 TURN 中继）。
|
||
if (local == 'relay') return 'TURN 中继 (本端经服务器)';
|
||
return 'TURN 中继';
|
||
}
|
||
// host: 同一局域网直连; srflx/prflx: 经 NAT 打洞的 P2P。
|
||
return 'P2P 直连';
|
||
}
|
||
|
||
/// 返回选定候选对的传输协议（udp/tcp）。
|
||
String _resolveProtocol(
|
||
List<StatsReport> reports,
|
||
String? localCandidateId,
|
||
String? remoteCandidateId,
|
||
) {
|
||
String protoOf(String? id) {
|
||
if (id == null) return '-';
|
||
final report = reports.where((r) => r.id == id).firstOrNull;
|
||
final p = report?.values['protocol'];
|
||
return p is String ? p : '-';
|
||
}
|
||
|
||
final local = protoOf(localCandidateId);
|
||
final remote = protoOf(remoteCandidateId);
|
||
if (local != '-' && remote != '-' && local != remote) {
|
||
return '$local/$remote';
|
||
}
|
||
return local != '-' ? local : remote;
|
||
}
|
||
|
||
/// 将字节/秒格式化为人类可读字符串（B/s、KB/s、MB/s）。
|
||
String _formatSpeed(double bytesPerSecond) {
|
||
if (bytesPerSecond >= 1024 * 1024) {
|
||
return '${(bytesPerSecond / (1024 * 1024)).toStringAsFixed(1)} MB/s';
|
||
} else if (bytesPerSecond >= 1024) {
|
||
return '${(bytesPerSecond / 1024).toStringAsFixed(1)} KB/s';
|
||
}
|
||
return '${bytesPerSecond.toStringAsFixed(0)} B/s';
|
||
}
|
||
|
||
/// 释放所有资源。
|
||
Future<void> close() async {
|
||
try {
|
||
await _dataChannel?.close();
|
||
} catch (_) {}
|
||
try {
|
||
await _pc?.close();
|
||
} catch (_) {}
|
||
try {
|
||
await renderer.dispose();
|
||
} catch (_) {}
|
||
_dataChannel = null;
|
||
try {
|
||
await _videoDataChannel?.close();
|
||
} catch (_) {}
|
||
_videoDataChannel = null;
|
||
try {
|
||
await _selfCodecDecoder?.dispose();
|
||
} catch (_) {}
|
||
_selfCodecDecoder = null;
|
||
_currentStreamMode = streamModeWebRtc;
|
||
onSelfCodecLost?.call();
|
||
onSelfCodecReady = null;
|
||
onStreamModeReport = null;
|
||
onResolutionReported = null;
|
||
onSelfCodecNotSupported = null;
|
||
_pc = null;
|
||
// 重置网速统计基线，避免下次连接首帧显示错误的瞬时速率。
|
||
_prevBytesReceived = 0;
|
||
_prevBytesSent = 0;
|
||
_prevStatsTimestamp = 0;
|
||
_connectedAt = null;
|
||
}
|
||
}
|