tt/VibeCoding
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(streaming): 添加自编码串流模式切换功能

Browse Source
This commit is contained in:
tongtongstudio
2026-07-23 12:01:08 +08:00
parent a1c29246d7
commit e56723a010
20 changed files with 1684 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
webrtc_controller_flutter/lib/controller/remote_controller.dart
View File

@@ -45,6 +45,18 @@ class RemoteController {
/// 统计信息刷新（每秒一次）。
void Function(String stats)? onStats;
/// 自编码解码纹理已就绪（textureId >= 0）。
void Function(int textureId)? onSelfCodecReady;
/// 被控端上报当前生效的串流模式。
void Function(int mode)? onStreamModeReport;
/// 被控端/解码器上报分辨率。
void Function(int width, int height)? onResolutionReported;
/// 当前平台不支持原生硬解时回调。
void Function()? onSelfCodecNotSupported;
RemoteController({
required this.serverUrl,
required this.deviceId,
@@ -92,6 +104,10 @@ class RemoteController {
};
_webRtc!.onIceDisconnected = (message) => onIceDisconnected?.call(message);
_webRtc!.onRemoteStream = (renderer) => onRemoteStream?.call(renderer);
_webRtc!.onSelfCodecReady = (id) => onSelfCodecReady?.call(id);
_webRtc!.onStreamModeReport = (mode) => onStreamModeReport?.call(mode);
_webRtc!.onResolutionReported = (w, h) => onResolutionReported?.call(w, h);
_webRtc!.onSelfCodecNotSupported = () => onSelfCodecNotSupported?.call();
_webRtc!.initialize().catchError((e) {
onStatusChanged?.call('状态: 连接失败 - $e');
});
@@ -155,6 +171,10 @@ class RemoteController {
_webRtc?.sendResolutionChange(width, height, fps);
}
/// 请求被控端切换屏幕串流模式（0=WebRTC 全托管 / 1=自编码）。
Future<void> sendStreamMode(int mode) =>
_webRtc?.sendStreamMode(mode) ?? Future.value();
/// 断开连接并释放资源。
Future<void> disconnect() async {
_statsTimer?.cancel();

99
webrtc_controller_flutter/lib/main.dart
View File

@@ -8,6 +8,7 @@ import 'config/ice_servers.dart';
import 'controller/remote_controller.dart';
import 'utils/control_commands.dart';
import 'utils/device_utils.dart';
import 'webrtc/self_codec_decoder.dart';
import 'widgets/remote_touch_view.dart';
void main() {
@@ -55,6 +56,12 @@ class _ControllerHomeState extends State<ControllerHome> {
String _stats = '';
double _videoAspect = 16 / 9;
/// 当前串流模式：0=WebRTC 全托管；1=自编码（自建 MediaCodec 解码）。
int _streamMode = SelfCodecDecoder.streamModeWebRtc;
int? _selfCodecTextureId;
bool _selfCodecReady = false;
bool _selfCodecSupported = true;
/// 分辨率预设：width 为长边像素；0 表示被控端原生分辨率；
/// height 传 0（由被控端按屏幕宽高比计算），fps 传 0（沿用当前帧率）。
int _selectedResolution = 0;
@@ -217,6 +224,28 @@ class _ControllerHomeState extends State<ControllerHome> {
renderer.addListener(_onRendererUpdate);
};
_controller!.onStats = (stats) => setState(() => _stats = stats);
_controller!.onSelfCodecReady = (textureId) {
if (mounted) {
setState(() {
_selfCodecTextureId = textureId;
_selfCodecReady = true;
});
}
};
_controller!.onStreamModeReport = (mode) {
if (mounted) setState(() => _streamMode = mode);
};
_controller!.onResolutionReported = (w, h) {
if (w > 0 && h > 0 && mounted) {
final aspect = w / h;
setState(() => _videoAspect = aspect);
_resizeWindowToAspect(aspect);
}
};
_controller!.onSelfCodecNotSupported = () {
if (mounted) setState(() => _selfCodecSupported = false);
_showAlert('当前平台不支持自编码硬解，已回退到 WebRTC 媒体流。');
};
_controller!.connect(authType: authType, authValue: authValue);
}
@@ -300,6 +329,19 @@ class _ControllerHomeState extends State<ControllerHome> {
);
}
/// 切换串流模式：WebRTC 全托管 <-> 自编码（自建 MediaCodec 解码）。
void _toggleStreamMode() {
if (!_selfCodecSupported) {
_showAlert('当前平台不支持自编码硬解。');
return;
}
final next = _streamMode == SelfCodecDecoder.streamModeSelfCodec
? SelfCodecDecoder.streamModeWebRtc
: SelfCodecDecoder.streamModeSelfCodec;
setState(() => _streamMode = next);
_controller?.sendStreamMode(next);
}
Future<void> _disconnect() async {
await _controller?.disconnect();
_controller = null;
@@ -310,6 +352,10 @@ class _ControllerHomeState extends State<ControllerHome> {
_renderer = null;
_status = '状态: 已停止';
_stats = '';
_streamMode = SelfCodecDecoder.streamModeWebRtc;
_selfCodecTextureId = null;
_selfCodecReady = false;
_selfCodecSupported = true;
});
}
@@ -412,6 +458,33 @@ class _ControllerHomeState extends State<ControllerHome> {
],
),
),
// 自编码串流开关：仅 Android 等支持原生硬解的平台可用。
CupertinoButton(
padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 10),
onPressed: _selfCodecSupported ? _toggleStreamMode : null,
child: Row(
mainAxisSize: MainAxisSize.min,
children: [
Text(
'自编码',
style: TextStyle(
color: _selfCodecSupported
? CupertinoColors.white
: CupertinoColors.white.withValues(alpha: 0.4),
fontSize: 14,
),
),
const SizedBox(width: 6),
CupertinoSwitch(
value: _streamMode == SelfCodecDecoder.streamModeSelfCodec,
onChanged: _selfCodecSupported
? (_) => _toggleStreamMode()
: null,
activeTrackColor: CupertinoColors.activeBlue,
),
],
),
),
// 分隔线
Container(
width: 1,
@@ -444,7 +517,31 @@ class _ControllerHomeState extends State<ControllerHome> {
child: Stack(
children: [
Container(color: Colors.black),
if (_renderer != null)
if (_streamMode == SelfCodecDecoder.streamModeSelfCodec &&
_selfCodecReady &&
_selfCodecTextureId != null)
Center(
child: AspectRatio(
aspectRatio: _videoAspect,
child: Stack(
fit: StackFit.expand,
children: [
Texture(textureId: _selfCodecTextureId!),
RemoteTouchView(
onTouch: (x, y) {},
onSwipe: (x1, y1, x2, y2, d) {},
onLongPress: (x, y) {},
onKey: (k, a) =>
_controller?.sendControlCommand(ControlCommands.key(k, a)),
onMotionEvent: (a, x, y) => _controller?.sendControlCommand(
ControlCommands.motionEvent(a, x, y),
),
),
],
),
),
)
else if (_renderer != null)
Center(
child: AspectRatio(
aspectRatio: _videoAspect,

14
webrtc_controller_flutter/lib/proto/control_message.pb.dart
View File

@@ -39,6 +39,7 @@ class ControlMessage extends $pb.GeneratedMessage {
$core.int? width,
$core.int? height,
$core.int? fps,
$core.int? streamMode,
}) {
final result = create();
if (action != null) result.action = action;
@@ -55,6 +56,7 @@ class ControlMessage extends $pb.GeneratedMessage {
if (width != null) result.width = width;
if (height != null) result.height = height;
if (fps != null) result.fps = fps;
if (streamMode != null) result.streamMode = streamMode;
return result;
}
@@ -86,6 +88,7 @@ class ControlMessage extends $pb.GeneratedMessage {
..aI(12, _omitFieldNames ? '' : 'width')
..aI(13, _omitFieldNames ? '' : 'height')
..aI(14, _omitFieldNames ? '' : 'fps')
..aI(15, _omitFieldNames ? '' : 'stream_mode')
..hasRequiredFields = false;
@$core.Deprecated('See https://github.com/google/protobuf.dart/issues/998.')
@@ -240,6 +243,17 @@ class ControlMessage extends $pb.GeneratedMessage {
$core.bool hasFps() => $_has(13);
@$pb.TagNumber(14)
void clearFps() => $_clearField(14);
/// 串流模式（SET_STREAM_MODE / REPORT_STREAM_MODE）：
/// 0 = WebRTC 全托管；1 = 自编码（自建 MediaCodec 编解码 + video DataChannel）
@$pb.TagNumber(15)
$core.int get streamMode => $_getIZ(14);
@$pb.TagNumber(15)
set streamMode($core.int value) => $_setSignedInt32(14, value);
@$pb.TagNumber(15)
$core.bool hasStreamMode() => $_has(14);
@$pb.TagNumber(15)
void clearStreamMode() => $_clearField(15);
}
const $core.bool _omitFieldNames =

8
webrtc_controller_flutter/lib/proto/control_message.pbenum.dart
View File

@@ -29,6 +29,10 @@ class Action extends $pb.ProtobufEnum {
Action._(6, _omitEnumNames ? '' : 'SET_RESOLUTION');
static const Action REPORT_RESOLUTION =
Action._(7, _omitEnumNames ? '' : 'REPORT_RESOLUTION');
static const Action SET_STREAM_MODE =
Action._(8, _omitEnumNames ? '' : 'SET_STREAM_MODE');
static const Action REPORT_STREAM_MODE =
Action._(9, _omitEnumNames ? '' : 'REPORT_STREAM_MODE');
static const $core.List<Action> values = <Action>[
ACTION_UNKNOWN,
@@ -39,10 +43,12 @@ class Action extends $pb.ProtobufEnum {
MOTION_EVENT,
SET_RESOLUTION,
REPORT_RESOLUTION,
SET_STREAM_MODE,
REPORT_STREAM_MODE,
];
static final $core.List<Action?> _byValue =
$pb.ProtobufEnum.$_initByValueList(values, 7);
$pb.ProtobufEnum.$_initByValueList(values, 9);
static Action? valueOf($core.int value) =>
value < 0 || value >= _byValue.length ? null : _byValue[value];

7
webrtc_controller_flutter/lib/utils/control_commands.dart
View File

@@ -68,4 +68,11 @@ class ControlCommands {
height: height,
fps: fps,
);
/// 串流模式切换指令。
/// [mode] 为 0=WebRTC 全托管 / 1=自编码（自建 MediaCodec 编解码 + video DataChannel 裸流透传）。
static ControlMessage streamMode(int mode) => ControlMessage(
action: Action.SET_STREAM_MODE,
streamMode: mode,
);
}

118
webrtc_controller_flutter/lib/webrtc/self_codec_decoder.dart Normal file
View File

@@ -0,0 +1,118 @@
import 'dart:async';
import 'package:flutter/services.dart';
import 'package:flutter/widgets.dart';
/// 自编码解码器（控制端）。
///
/// 接收来自 `video` DataChannel 的二进制分片（由被控端 [SelfCodecEncoder] 产生），
/// 通过原生 Android `MediaCodec` 硬解码，并渲染到 Flutter 纹理（`Texture`）。
///
/// 为保证与 `WebRTCControlled` 的编码器在二进制协议上完全一致，协议的
/// 分片重组、H.264 参数集解析与硬解、纹理渲染全部在原生侧完成；
/// 本类仅负责与原生侧通信。对应 Android 端 `SelfCodecDecoder`。
///
/// 注意：原生解码器目前仅在 Android 平台实现。其它平台（Windows / Linux /
/// Web / iOS）调用 [create] 会失败并回调 [onError]，调用方应回退到
/// WebRTC 默认媒体流。
class SelfCodecDecoder {
static const MethodChannel _channel =
MethodChannel('com.ttstd.fluttercontroller/self_codec');
/// 串流模式常量（与 ControlMessage.stream_mode 对齐）。
static const int streamModeWebRtc = 0;
static const int streamModeSelfCodec = 1;
int? _textureId;
bool _disposed = false;
bool _handlerSet = false;
/// 解码输出尺寸变化（宽/高，单位像素）。
final void Function(int width, int height)? onSize;
/// 错误回调（如当前平台不支持硬解）。
final void Function(String error)? onError;
SelfCodecDecoder({this.onSize, this.onError}) {
_channel.setMethodCallHandler(_handleNativeCall);
_handlerSet = true;
}
Future<dynamic> _handleNativeCall(MethodCall call) async {
switch (call.method) {
case 'onSize':
final args = call.arguments;
final w = args is Map ? (args['width'] as int? ?? 0) : 0;
final h = args is Map ? (args['height'] as int? ?? 0) : 0;
if (w > 0 && h > 0) onSize?.call(w, h);
break;
case 'onError':
final msg = call.arguments is String ? call.arguments as String : '';
if (msg.isNotEmpty) onError?.call(msg);
break;
}
}
/// 创建解码纹理。成功返回纹理 id（>=0），失败返回 null。
Future<int?> create() async {
if (_disposed) return null;
try {
final id = await _channel.invokeMethod<int>('create');
if (id == null || id < 0) return null;
_textureId = id;
return _textureId;
} on PlatformException catch (e) {
onError?.call('创建自编码解码纹理失败: ${e.message}');
return null;
} catch (e) {
onError?.call('创建自编码解码纹理失败: $e');
return null;
}
}
/// 已创建的纹理 id；未创建或已释放时为 null。
int? get textureId => _textureId;
/// 用于渲染解码画面的纹理控件。
Widget build(BuildContext context) {
final id = _textureId;
if (id == null) return const SizedBox.shrink();
return Texture(textureId: id);
}
/// 喂入来自 video DataChannel 的二进制分片。
Future<void> feed(Uint8List chunk) async {
if (_disposed || _textureId == null) return;
try {
await _channel.invokeMethod<void>('feed', chunk);
} catch (_) {
// 解码未就绪或被释放时忽略。
}
}
/// 启用/停用解码（仅自编码模式为 true）。
Future<void> setEnabled(bool enabled) async {
if (_disposed || _textureId == null) return;
try {
await _channel.invokeMethod<void>('setEnabled', enabled);
} catch (_) {
// 忽略。
}
}
/// 释放原生解码资源。重复调用安全。
Future<void> dispose() async {
if (_disposed) return;
_disposed = true;
if (_handlerSet) {
_channel.setMethodCallHandler(null);
_handlerSet = false;
}
try {
await _channel.invokeMethod<void>('dispose');
} catch (_) {
// 忽略。
}
_textureId = null;
}
}

136
webrtc_controller_flutter/lib/webrtc/webrtc_controller.dart
View File

@@ -1,6 +1,8 @@
import 'package:flutter/foundation.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart';
import 'self_codec_decoder.dart';
import '../config/ice_servers.dart';
import '../models/signal_message.dart';
import '../proto/control_message.pb.dart';
@@ -26,6 +28,10 @@ class WebRtcController {
RTCPeerConnection? _pc;
RTCDataChannel? _dataChannel;
RTCDataChannel? _videoDataChannel;
SelfCodecDecoder? _selfCodecDecoder;
int _currentStreamMode = streamModeWebRtc;
bool _creatingDecoder = false;
/// 远端视频渲染器，由调用方持有并显示。
final RTCVideoRenderer renderer = RTCVideoRenderer();
@@ -40,6 +46,25 @@ class WebRtcController {
/// 远端视频流就绪（renderer 已绑定视频轨道）。
void Function(RTCVideoRenderer renderer)? onRemoteStream;
/// 自编码解码纹理已就绪（textureId >= 0）。
void Function(int)? onSelfCodecReady;
/// 自编码解码器已释放（连接断开时）。
void Function()? onSelfCodecLost;
/// 被控端上报当前生效的串流模式。
void Function(int mode)? onStreamModeReport;
/// 被控端上报分辨率（自编码解码器输出尺寸变化时也会触发）。
void Function(int width, int height)? onResolutionReported;
/// 当前平台（Windows / Linux / Web / iOS 等）不支持原生硬解时回调。
void Function()? onSelfCodecNotSupported;
static const String kVideoChannelLabel = 'video_channel';
static const int streamModeWebRtc = 0;
static const int streamModeSelfCodec = 1;
WebRtcController({
required this.signaling,
required this.deviceId,
@@ -83,6 +108,17 @@ class WebRtcController {
_dataChannel = await _pc!.createDataChannel(kDataChannelLabel, dcInit);
_setupDataChannel(_dataChannel!);
// 创建自编码视频透传通道（无序、不可靠，类似 RTP，最低延迟）。
// 被控端仅在收到 SET_STREAM_MODE=自编码 后才经此通道推送 H.264 裸流。
final videoInit = RTCDataChannelInit()
..ordered = false
..maxRetransmits = 0;
_videoDataChannel = await _pc!.createDataChannel(
kVideoChannelLabel,
videoInit,
);
_setupVideoDataChannel(_videoDataChannel!);
// 创建并发送 Offer。
final constraints = {
'mandatory': {
@@ -104,18 +140,22 @@ class WebRtcController {
}
void _onDataChannel(RTCDataChannel channel) {
// 被控端也可能主动创建 DataChannel，统一处理。
_setupDataChannel(channel);
// 被控端也可能主动创建 DataChannel，按标签统一处理。
if (channel.label == kDataChannelLabel) {
_dataChannel ??= channel;
_setupDataChannel(channel);
} else if (channel.label == kVideoChannelLabel) {
_videoDataChannel ??= channel;
_setupVideoDataChannel(channel);
}
}
void _setupDataChannel(RTCDataChannel channel) {
channel.onMessage = (RTCDataChannelMessage message) {
// 处理来自被控端的消息（本控制端主要发送，此处仅做日志记录）。
if (message.isBinary) {
try {
final command = ControlMessage.fromBuffer(message.binary);
// ignore: avoid_print
print('DataChannel message: ${command.action}');
_handleControlMessage(command);
} catch (e) {
// ignore: avoid_print
print('DataChannel message decode failed: $e');
@@ -131,6 +171,78 @@ class WebRtcController {
};
}
void _setupVideoDataChannel(RTCDataChannel channel) {
channel.onMessage = (RTCDataChannelMessage message) {
if (message.isBinary) {
_selfCodecDecoder?.feed(message.binary);
}
};
channel.onDataChannelState = (RTCDataChannelState state) {
if (state == RTCDataChannelState.RTCDataChannelOpen) {
_onVideoChannelOpen();
} else if (state == RTCDataChannelState.RTCDataChannelClosed) {
_selfCodecDecoder?.setEnabled(false);
}
};
}
void _onVideoChannelOpen() {
_ensureSelfCodecDecoder().then((_) {
_selfCodecDecoder?.setEnabled(_currentStreamMode == streamModeSelfCodec);
});
}
Future<void> _ensureSelfCodecDecoder() async {
if (_selfCodecDecoder != null || _creatingDecoder) return;
_creatingDecoder = true;
try {
final decoder = SelfCodecDecoder(
onSize: (w, h) => onResolutionReported?.call(w, h),
onError: (err) => debugPrint('自编码解码器: $err'),
);
final textureId = await decoder.create();
if (textureId != null) {
_selfCodecDecoder = decoder;
onSelfCodecReady?.call(textureId);
} else {
// 当前平台（Windows / Linux / Web / iOS 等）无原生硬解能力，回退 WebRTC 媒体流。
onSelfCodecNotSupported?.call();
}
} finally {
_creatingDecoder = false;
}
}
void _handleControlMessage(ControlMessage msg) {
switch (msg.action) {
case Action.REPORT_STREAM_MODE:
_currentStreamMode = msg.streamMode;
_selfCodecDecoder?.setEnabled(msg.streamMode == streamModeSelfCodec);
onStreamModeReport?.call(msg.streamMode);
break;
case Action.REPORT_RESOLUTION:
onResolutionReported?.call(msg.width, msg.height);
break;
default:
// ignore: avoid_print
print('DataChannel message: ${msg.action}');
break;
}
}
/// 请求被控端切换屏幕串流模式（SET_STREAM_MODE 指令）。
/// [mode] 为 0=WebRTC 全托管 / 1=自编码。
Future<void> sendStreamMode(int mode) async {
_currentStreamMode = mode;
if (mode == streamModeSelfCodec) {
await _ensureSelfCodecDecoder();
await _selfCodecDecoder?.setEnabled(true);
} else {
await _selfCodecDecoder?.setEnabled(false);
}
sendControlCommand(ControlCommands.streamMode(mode));
}
void _onIceCandidate(RTCIceCandidate candidate) {
final payload = {
'sdpMid': candidate.sdpMid,
@@ -413,6 +525,20 @@ class WebRtcController {
await renderer.dispose();
} catch (_) {}
_dataChannel = null;
try {
await _videoDataChannel?.close();
} catch (_) {}
_videoDataChannel = null;
try {
await _selfCodecDecoder?.dispose();
} catch (_) {}
_selfCodecDecoder = null;
_currentStreamMode = streamModeWebRtc;
onSelfCodecLost?.call();
onSelfCodecReady = null;
onStreamModeReport = null;
onResolutionReported = null;
onSelfCodecNotSupported = null;
_pc = null;
// 重置网速统计基线，避免下次连接首帧显示错误的瞬时速率。
_prevBytesReceived = 0;