From e56723a0105a11295e9d6033c13bddba4ada47ff Mon Sep 17 00:00:00 2001 From: tongtongstudio Date: Thu, 23 Jul 2026 12:01:08 +0800 Subject: [PATCH] =?UTF-8?q?feat(streaming):=20=E6=B7=BB=E5=8A=A0=E8=87=AA?= =?UTF-8?q?=E7=BC=96=E7=A0=81=E4=B8=B2=E6=B5=81=E6=A8=A1=E5=BC=8F=E5=88=87?= =?UTF-8?q?=E6=8D=A2=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controlled/input/InputCommandHandler.java | 18 + .../service/ScreenCaptureService.java | 127 +++++++ .../controlled/webrtc/SelfCodecEncoder.java | 245 ++++++++++++ .../ttstd/controlled/webrtc/WebRtcClient.java | 68 ++++ .../app/src/main/proto/control_message.proto | 3 + .../com/ttstd/controller/MainActivity.java | 91 +++++ .../controller/webrtc/SelfCodecDecoder.java | 260 +++++++++++++ .../ttstd/controller/webrtc/WebRtcClient.java | 74 +++- .../app/src/main/proto/control_message.proto | 3 + .../app/src/main/res/layout/activity_main.xml | 39 +- .../ttstd/fluttercontroller/MainActivity.kt | 9 +- .../SelfCodecDecoderPlugin.kt | 355 ++++++++++++++++++ .../lib/controller/remote_controller.dart | 20 + webrtc_controller_flutter/lib/main.dart | 99 ++++- .../lib/proto/control_message.pb.dart | 14 + .../lib/proto/control_message.pbenum.dart | 8 +- .../lib/utils/control_commands.dart | 7 + .../lib/webrtc/self_codec_decoder.dart | 118 ++++++ .../lib/webrtc/webrtc_controller.dart | 136 ++++++- .../proto/control_message.proto | 6 + 20 files changed, 1684 insertions(+), 16 deletions(-) create mode 100644 WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/SelfCodecEncoder.java create mode 100644 WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/SelfCodecDecoder.java create mode 100644 webrtc_controller_flutter/android/app/src/main/kotlin/com/ttstd/fluttercontroller/SelfCodecDecoderPlugin.kt create mode 100644 webrtc_controller_flutter/lib/webrtc/self_codec_decoder.dart diff --git a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputCommandHandler.java b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputCommandHandler.java index 43f51c0..b1c9fab 100644 --- a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputCommandHandler.java +++ b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputCommandHandler.java @@ -23,7 +23,13 @@ public class InputCommandHandler { void onResolutionRequested(int width, int height, int fps); } + /** 控制端请求切换屏幕串流模式（WebRTC / 自编码）时的回调。 */ + public interface StreamModeListener { + void onStreamModeRequested(int mode); + } + private ResolutionRequestListener resolutionListener; + private StreamModeListener streamModeListener; public InputCommandHandler(int screenWidth, int screenHeight) { this(screenWidth, screenHeight, new SystemInputUtils()); @@ -46,6 +52,10 @@ public class InputCommandHandler { this.resolutionListener = listener; } + public void setStreamModeListener(StreamModeListener listener) { + this.streamModeListener = listener; + } + public void handleCommand(byte[] data) { try { ControlMessage command = ControlMessage.parseFrom(data); @@ -76,6 +86,14 @@ public class InputCommandHandler { Log.w(TAG, "SET_RESOLUTION received but no listener registered"); } break; + case SET_STREAM_MODE: + // 串流模式切换不依赖输入执行器，转发给屏幕采集服务处理。 + if (streamModeListener != null) { + streamModeListener.onStreamModeRequested(command.getStreamMode()); + } else { + Log.w(TAG, "SET_STREAM_MODE received but no listener registered"); + } + break; default: Log.w(TAG, "Unknown action: " + action); } diff --git a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/service/ScreenCaptureService.java b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/service/ScreenCaptureService.java index ff43ba3..c865f02 100644 --- a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/service/ScreenCaptureService.java +++ b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/service/ScreenCaptureService.java @@ -40,6 +40,9 @@ import com.ttstd.controlled.signaling.SignalMessage; import com.ttstd.controlled.utils.AuthSettings; import com.ttstd.controlled.signaling.WebSocketClient; import com.ttstd.controlled.webrtc.WebRtcClient; +import com.ttstd.controlled.webrtc.SelfCodecEncoder; +import com.ttstd.control.Action; +import com.ttstd.control.ControlMessage; import com.google.gson.Gson; import com.google.gson.JsonObject; @@ -96,6 +99,13 @@ public class ScreenCaptureService extends Service { /** 标记服务是否正在关闭，避免重复停止/重复弹窗。 */ private boolean isShuttingDown = false; + // ---- 屏幕串流"自编码"模式 ---- + private int resultCode; // 屏幕捕获授权结果码（供自编码模式重新获取 MediaProjection） + private Intent resultDataIntent; // 屏幕捕获授权 Intent（同上） + private int currentStreamMode = SelfCodecEncoder.STREAM_MODE_WEBRTC; + private MediaProjection selfMediaProjection; // 自编码模式独占的 MediaProjection + private SelfCodecEncoder selfEncoder; // 自编码 H264 编码器 + @Override public void onCreate() { super.onCreate(); @@ -123,6 +133,10 @@ public class ScreenCaptureService extends Service { String serverUrl = intent.getStringExtra(EXTRA_SERVER_URL); deviceId = intent.getStringExtra(EXTRA_DEVICE_ID); + // 保存屏幕捕获授权结果，供"自编码"模式重新获取 MediaProjection 使用。 + this.resultCode = resultCode; + this.resultDataIntent = resultData; + if (resultCode != Activity.RESULT_OK || resultData == null || serverUrl == null || deviceId == null) { Log.e(TAG, "Invalid service parameters"); stopSelf(); @@ -186,6 +200,7 @@ public class ScreenCaptureService extends Service { inputHandler = new InputCommandHandler(realWidth, realHeight, createInputExecutor()); // 控制端发来的 SET_RESOLUTION 指令转交本服务处理（坐标映射与输入执行器无关）。 inputHandler.setResolutionRequestListener(this::onRemoteResolutionRequested); + inputHandler.setStreamModeListener(this::onStreamModeRequested); // 开启屏幕捕获，使用计算出的 capture 分辨率 startScreenCapture(resultCode, resultData, serverUrl, deviceId, captureWidth, captureHeight, fps); @@ -317,6 +332,110 @@ public class ScreenCaptureService extends Service { return new int[]{ w, h }; } + // ---- 屏幕串流"自编码"模式 ---- + + /** 控制端经 SET_STREAM_MODE 请求切换串流模式。 */ + private void onStreamModeRequested(int mode) { + setStreamMode(mode); + } + + /** + * 切换屏幕串流模式： + * - {@link SelfCodecEncoder#STREAM_MODE_WEBRTC}：使用 WebRTC 默认媒体轨道（ScreenCapturerAndroid）。 + * - {@link SelfCodecEncoder#STREAM_MODE_SELF_CODEC}：暂停 WebRTC 采集，改用自建 MediaCodec 编码， + * 经 DataChannel 的 video_channel 透传裸 H.264 码流，由控制端自建解码器渲染。 + */ + public void setStreamMode(int mode) { + if (mode == currentStreamMode) { + return; + } + if (mode == SelfCodecEncoder.STREAM_MODE_SELF_CODEC) { + // 切到自编码：先停 WebRTC 默认屏幕采集（释放其 MediaProjection），再启动自编码。 + if (webRtcClient != null) { + webRtcClient.stopCapture(); + } + startSelfCodecEncoder(); + } else { + // 切回 WebRTC：停止自编码，恢复默认屏幕采集。 + stopSelfCodecEncoder(); + if (webRtcClient != null) { + webRtcClient.startCapture(); + } + } + currentStreamMode = mode; + reportStreamMode(mode); + } + + /** 返回当前串流模式。 */ + public int getCurrentStreamMode() { + return currentStreamMode; + } + + private void startSelfCodecEncoder() { + if (selfEncoder != null) { + Log.w(TAG, "self codec encoder already running"); + return; + } + if (webRtcClient == null || !webRtcClient.isVideoDataChannelOpen()) { + Log.w(TAG, "startSelfCodecEncoder: video channel not ready yet"); + } + MediaProjectionManager mpm = + (MediaProjectionManager) getSystemService(Context.MEDIA_PROJECTION_SERVICE); + try { + selfMediaProjection = mpm.getMediaProjection(resultCode, resultDataIntent); + } catch (Exception e) { + Log.e(TAG, "getMediaProjection for self codec failed", e); + return; + } + int dpi = getResources().getDisplayMetrics().densityDpi; + int[] res = getCurrentCaptureResolution(); + selfEncoder = new SelfCodecEncoder( + selfMediaProjection, res[0], res[1], res[2], dpi, + webRtcClient != null ? webRtcClient.getVideoDataChannel() : null, + new SelfCodecEncoder.Callback() { + @Override + public void onEncoderStarted(int w, int h, int f) { + Log.i(TAG, "self codec encoder started: " + w + "x" + h); + } + + @Override + public void onEncoderError(String error) { + Log.e(TAG, "self codec encoder error: " + error); + // 出错时回退到 WebRTC 模式 + mainHandler.post(() -> setStreamMode(SelfCodecEncoder.STREAM_MODE_WEBRTC)); + } + }); + selfEncoder.start(); + } + + private void stopSelfCodecEncoder() { + if (selfEncoder != null) { + selfEncoder.release(); + selfEncoder = null; + } + if (selfMediaProjection != null) { + try { + selfMediaProjection.stop(); + } catch (Exception ignored) { + } + selfMediaProjection = null; + } + } + + /** 经 control DataChannel 向控制端回报当前生效的串流模式。 */ + private void reportStreamMode(int mode) { + if (webRtcClient == null) return; + try { + ControlMessage msg = ControlMessage.newBuilder() + .setAction(Action.REPORT_STREAM_MODE) + .setStreamMode(mode) + .build(); + webRtcClient.sendControlCommand(msg); + } catch (Exception e) { + Log.e(TAG, "reportStreamMode failed", e); + } + } + /** * 状态回调。由已 bind 的 MainActivity 注册，用于把连接状态同步到界面。 */ @@ -357,6 +476,14 @@ public class ScreenCaptureService extends Service { screenCapturer.stopCapture(); screenCapturer.dispose(); } + if (selfEncoder != null) { + selfEncoder.release(); + selfEncoder = null; + } + if (selfMediaProjection != null) { + selfMediaProjection.stop(); + selfMediaProjection = null; + } if (webRtcClient != null) { webRtcClient.close(); } diff --git a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/SelfCodecEncoder.java b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/SelfCodecEncoder.java new file mode 100644 index 0000000..f48607e --- /dev/null +++ b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/SelfCodecEncoder.java @@ -0,0 +1,245 @@ +package com.ttstd.controlled.webrtc; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaFormat; +import android.media.projection.MediaProjection; +import android.hardware.display.VirtualDisplay; +import android.hardware.display.DisplayManager; +import android.os.Bundle; +import android.util.Log; +import android.view.Surface; + +import org.webrtc.DataChannel; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** + * 自建屏幕编码通道：用 MediaProjection 的 VirtualDisplay 把屏幕投影到 MediaCodec + * 的输入 Surface，编码为 H.264 裸流后，经 WebRTC 的 video DataChannel 分片透传给 + * 控制端（控制端使用自建 MediaCodec 解码渲染）。绕开 WebRTC 默认媒体轨道，使编码/ + * 解码参数完全可控（码率、关键帧、低延迟调优）。 + */ +public class SelfCodecEncoder { + + public static final int STREAM_MODE_WEBRTC = 0; // WebRTC 内置媒体流 + public static final int STREAM_MODE_SELF_CODEC = 1; // 自编码（本类） + + // ---- 自定义二进制协议（与控制端 SelfCodecDecoder 保持一致）---- + private static final byte MAGIC = (byte) 0xAB; + private static final int CHUNK_SIZE = 16384; // 每个 DataChannel 分片大小 + private static final int UNIT_TYPE_CONFIG = 1; // SPS/PPS 等参数集 + private static final int UNIT_TYPE_FRAME = 2; // 编码帧 + + private static final String TAG = "SelfCodecEncoder"; + + private final MediaProjection mediaProjection; + private final int width; + private final int height; + private final int fps; + private final int dpi; + private final DataChannel videoChannel; + private final Callback callback; + + private MediaCodec encoder; + private Surface inputSurface; + private VirtualDisplay virtualDisplay; + private boolean running = false; + private int frameSeq = 0; + private final Object sendLock = new Object(); + + public interface Callback { + void onEncoderStarted(int w, int h, int fps); + + void onEncoderError(String error); + } + + public SelfCodecEncoder(MediaProjection mp, int width, int height, int fps, int dpi, + DataChannel videoChannel, Callback cb) { + this.mediaProjection = mp; + this.width = width; + this.height = height; + this.fps = fps; + this.dpi = dpi; + this.videoChannel = videoChannel; + this.callback = cb; + } + + public void start() { + if (running) return; + try { + MediaFormat format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height); + int bitrate = computeBitrate(width, height, fps); + format.setInteger(MediaFormat.KEY_BIT_RATE, bitrate); + format.setInteger(MediaFormat.KEY_FRAME_RATE, fps); + format.setInteger(MediaFormat.KEY_CAPTURE_RATE, fps); + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); + // 关键帧间隔：1 秒一个 IDR，便于断流后快速恢复；也可经 requestKeyFrame 即时强求 + format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1); + + encoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC); + encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + inputSurface = encoder.createInputSurface(); + encoder.setCallback(encoderCallback); + encoder.start(); + + virtualDisplay = mediaProjection.createVirtualDisplay( + "SelfCodec", width, height, dpi, + DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, inputSurface, null, null); + + running = true; + if (callback != null) callback.onEncoderStarted(width, height, fps); + Log.i(TAG, "self codec encoder started: " + width + "x" + height + "@" + fps + ", bitrate=" + bitrate); + } catch (Exception e) { + Log.e(TAG, "start encoder failed", e); + release(); + if (callback != null) callback.onEncoderError(e.getMessage()); + } + } + + private int computeBitrate(int w, int h, int fps) { + int b = (int) (w * h * fps * 0.12); + return Math.max(500_000, Math.min(8_000_000, b)); + } + + private final MediaCodec.Callback encoderCallback = new MediaCodec.Callback() { + @Override + public void onInputBufferAvailable(MediaCodec mc, int index) { + // 使用 Surface 输入，编码器自动从输入 Surface 取帧，无需手动喂数据 + } + + @Override + public void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { + // 主流设备在此给出 SPS/PPS，作为 CONFIG 单元发送 + sendCsds(format); + } + + @Override + public void onOutputBufferAvailable(MediaCodec mc, int index, MediaCodec.BufferInfo info) { + try { + ByteBuffer buf = mc.getOutputBuffer(index); + if (buf == null) { + mc.releaseOutputBuffer(index, false); + return; + } + byte[] data = new byte[info.size]; + buf.position(info.offset); + buf.get(data); + boolean isKey = (info.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0; + boolean isConfig = (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0; + long pts = info.presentationTimeUs; + mc.releaseOutputBuffer(index, false); + + if (isConfig) { + sendUnit(UNIT_TYPE_CONFIG, 0, false, data); + } else { + sendUnit(UNIT_TYPE_FRAME, pts, isKey, data); + } + } catch (Exception e) { + Log.e(TAG, "onOutputBufferAvailable error", e); + } + } + + @Override + public void onError(MediaCodec mc, MediaCodec.CodecException e) { + Log.e(TAG, "encoder error", e); + if (callback != null) callback.onEncoderError(e.getMessage()); + } + }; + + private void sendCsds(MediaFormat format) { + if (format == null) return; + ByteBuffer sps = format.getByteBuffer("csd-0"); + ByteBuffer pps = format.getByteBuffer("csd-1"); + if (sps != null) sendUnit(UNIT_TYPE_CONFIG, 0, false, toBytes(sps)); + if (pps != null) sendUnit(UNIT_TYPE_CONFIG, 0, false, toBytes(pps)); + } + + private static byte[] toBytes(ByteBuffer b) { + ByteBuffer d = b.duplicate(); + byte[] out = new byte[d.remaining()]; + d.get(out); + return out; + } + + /** 组装一个"单元"（类型 + pts + isKey + 数据），再切片经 videoChannel 发送。 */ + private void sendUnit(byte unitType, long pts, boolean isKey, byte[] data) { + ByteBuffer unit = ByteBuffer.allocate(1 + 4 + 1 + 4 + data.length).order(ByteOrder.BIG_ENDIAN); + unit.put(unitType); + unit.putInt((int) (pts & 0xFFFFFFFFL)); + unit.put((byte) (isKey ? 1 : 0)); + unit.putInt(data.length); + unit.put(data); + byte[] unitBytes = unit.array(); + + int totalChunks = (unitBytes.length + CHUNK_SIZE - 1) / CHUNK_SIZE; + if (totalChunks == 0) totalChunks = 1; + int seq; + synchronized (sendLock) { + seq = frameSeq++; + } + for (int i = 0; i < totalChunks; i++) { + int off = i * CHUNK_SIZE; + int len = Math.min(CHUNK_SIZE, unitBytes.length - off); + byte[] chunk = new byte[len]; + System.arraycopy(unitBytes, off, chunk, 0, len); + sendChunk(seq, (short) totalChunks, (short) i, chunk); + } + } + + private void sendChunk(int seq, short total, short idx, byte[] chunk) { + if (videoChannel == null || videoChannel.state() != DataChannel.State.OPEN) return; + ByteBuffer msg = ByteBuffer.allocate(1 + 4 + 2 + 2 + 4 + chunk.length).order(ByteOrder.BIG_ENDIAN); + msg.put(MAGIC); + msg.putInt(seq); + msg.putShort(total); + msg.putShort(idx); + msg.putInt(chunk.length); + msg.put(chunk); + msg.flip(); + try { + videoChannel.send(new DataChannel.Buffer(msg, true)); + } catch (Exception e) { + Log.e(TAG, "send video chunk failed", e); + } + } + + /** 即时请求一个关键帧（强求 IDR）。 */ + public void requestKeyFrame() { + if (encoder == null) return; + try { + Bundle params = new Bundle(); + params.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0); + encoder.setParameters(params); + } catch (Exception e) { + Log.w(TAG, "requestKeyFrame failed", e); + } + } + + public void release() { + running = false; + if (virtualDisplay != null) { + try { + virtualDisplay.release(); + } catch (Exception ignored) { + } + virtualDisplay = null; + } + if (encoder != null) { + try { + encoder.stop(); + } catch (Exception ignored) { + } + try { + encoder.release(); + } catch (Exception ignored) { + } + encoder = null; + } + if (inputSurface != null) { + inputSurface.release(); + inputSurface = null; + } + } +} diff --git a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/WebRtcClient.java b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/WebRtcClient.java index 5366bfc..05b89ec 100644 --- a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/WebRtcClient.java +++ b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/WebRtcClient.java @@ -40,6 +40,7 @@ public class WebRtcClient { private static final String VIDEO_TRACK_ID = "screen_track"; private static final String STREAM_ID = "screen_stream"; private static final String DATA_CHANNEL_LABEL = "control_channel"; + private static final String VIDEO_CHANNEL_LABEL = "video_channel"; // 自编码视频透传通道（控制端创建） // 码率优化参数（单位 kbps）。 private static final int MIN_BITRATE_KBPS = 1000; // 1 Mbps @@ -54,6 +55,7 @@ public class WebRtcClient { private PeerConnectionFactory peerConnectionFactory; private PeerConnection peerConnection; private DataChannel dataChannel; + private DataChannel videoDataChannel; // 自编码视频透传（控制端创建的 video_channel） private VideoSource videoSource; private VideoTrack videoTrack; private org.webrtc.RtpSender videoSender; @@ -419,6 +421,17 @@ public class WebRtcClient { dataChannel.dispose(); dataChannel = null; } + if (videoDataChannel != null) { + try { + videoDataChannel.close(); + } catch (Exception e) { + } + try { + videoDataChannel.dispose(); + } catch (Exception e) { + } + videoDataChannel = null; + } if (peerConnection != null) { peerConnection.close(); peerConnection.dispose(); @@ -426,7 +439,62 @@ public class WebRtcClient { } } + // ---- 屏幕串流"自编码"模式相关 ---- + + /** 返回控制端建立的 video DataChannel（用于自编码视频透传），可能为空。 */ + public DataChannel getVideoDataChannel() { + return videoDataChannel; + } + + /** video DataChannel 是否已打开。 */ + public boolean isVideoDataChannelOpen() { + return videoDataChannel != null && videoDataChannel.state() == DataChannel.State.OPEN; + } + + /** 暂停 WebRTC 默认屏幕采集（切到自编码模式时调用）。 */ + public void stopCapture() { + if (videoCapturer != null) { + try { + videoCapturer.stopCapture(); + } catch (InterruptedException e) { + Log.w(TAG, "stopCapture interrupted", e); + } + } + } + + /** 恢复 WebRTC 默认屏幕采集（从自编码模式切回时调用）。 */ + public void startCapture() { + if (videoCapturer != null) { + try { + videoCapturer.startCapture(videoWidth, videoHeight, videoFps); + } catch (RuntimeException e) { + Log.e(TAG, "startCapture failed", e); + } + } + } + + /** 经 control DataChannel 发送控制类 protobuf 消息（如串流模式回报）。 */ + public void sendControlCommand(ControlMessage message) { + if (dataChannel == null || dataChannel.state() != DataChannel.State.OPEN) { + Log.d(TAG, "sendControlCommand skipped: dataChannel not open"); + return; + } + try { + ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray()); + dataChannel.send(new DataChannel.Buffer(buffer, true)); + } catch (Exception e) { + Log.e(TAG, "sendControlCommand failed", e); + } + } + private void handleDataChannel(DataChannel dc) { + // 区分控制端建立的两条通道：video_channel 用于本端自编码视频透传（仅发送）， + // 其余（control_channel）用于指令收发。 + if (VIDEO_CHANNEL_LABEL.equals(dc.label())) { + this.videoDataChannel = dc; + Log.d(TAG, "Video data channel received: " + dc.label()); + return; + } // 保存控制端建立的通道引用（替代原先本端自建的通道）。 this.dataChannel = dc; dc.registerObserver(new DataChannel.Observer() { diff --git a/WebRTCControlled/app/src/main/proto/control_message.proto b/WebRTCControlled/app/src/main/proto/control_message.proto index 99c6720..d53fc96 100644 --- a/WebRTCControlled/app/src/main/proto/control_message.proto +++ b/WebRTCControlled/app/src/main/proto/control_message.proto @@ -15,6 +15,8 @@ enum Action { MOTION_EVENT = 5; SET_RESOLUTION = 6; // 控制端请求被控端切换屏幕采集分辨率 REPORT_RESOLUTION = 7; // 被控端上报当前实际采集分辨率（含连接建立后的初始值） + SET_STREAM_MODE = 8; // 控制端请求被控端切换屏幕串流模式 + REPORT_STREAM_MODE = 9; // 被控端上报当前生效的串流模式 } // 通过 RTCDataChannel 传输的控制指令（protobuf 二进制）。 @@ -48,4 +50,5 @@ message ControlMessage { int32 width = 12; int32 height = 13; int32 fps = 14; + int32 stream_mode = 15; // 串流模式：0=WebRTC 内置媒体流，1=自编码(MediaCodec 硬编 + DataChannel 透传) } diff --git a/WebRTCController/app/src/main/java/com/ttstd/controller/MainActivity.java b/WebRTCController/app/src/main/java/com/ttstd/controller/MainActivity.java index 462d275..cb8d1ec 100644 --- a/WebRTCController/app/src/main/java/com/ttstd/controller/MainActivity.java +++ b/WebRTCController/app/src/main/java/com/ttstd/controller/MainActivity.java @@ -16,6 +16,10 @@ import android.widget.AdapterView; import android.widget.ArrayAdapter; import android.widget.Spinner; import android.widget.RadioGroup; +import android.widget.Switch; +import android.view.SurfaceView; +import android.view.SurfaceHolder; +import com.ttstd.controller.webrtc.SelfCodecDecoder; import androidx.appcompat.app.AlertDialog; import java.util.ArrayList; @@ -58,6 +62,14 @@ public class MainActivity extends AppCompatActivity { private TextView tvStats; private Spinner spinnerResolution; + // 屏幕串流"自编码"模式相关 UI 与解码器 + private Switch switchStreamMode; + private TextView tvStreamMode; + private SurfaceView selfCodecSurface; + private SelfCodecDecoder selfCodecDecoder; + private SurfaceHolder.Callback surfaceCallback; + private boolean programmaticSwitch = false; + private WebSocketClient wsClient; private WebRtcClient webRtcClient; private EglBase eglBase; @@ -162,6 +174,8 @@ public class MainActivity extends AppCompatActivity { setupResolutionSpinner(); updateUI(false); + + setupSelfCodecUi(); } private void adjustVideoSize(int videoWidth, int videoHeight, int rotation) { @@ -309,6 +323,8 @@ public class MainActivity extends AppCompatActivity { private void initWebRtcAndConnect() { webRtcClient = new WebRtcClient(this, wsClient, myDeviceId); webRtcClient.initialize(eglBase); + webRtcClient.setSelfCodecDecoder(selfCodecDecoder); + webRtcClient.setStreamModeReportListener(mode -> runOnUiThread(() -> syncStreamModeUi(mode))); webRtcClient.setConnectionListener(new WebRtcClient.ConnectionListener() { @Override public void onConnectionEstablished() { @@ -598,8 +614,83 @@ public class MainActivity extends AppCompatActivity { if (connected) { statsHandler.post(statsRunnable); + if (switchStreamMode != null) switchStreamMode.setEnabled(true); } else { statsHandler.removeCallbacks(statsRunnable); + if (switchStreamMode != null) { + switchStreamMode.setEnabled(false); + programmaticSwitch = true; + switchStreamMode.setChecked(false); + programmaticSwitch = false; + } + if (tvStreamMode != null) tvStreamMode.setText("串流: WebRTC"); + if (selfCodecDecoder != null) selfCodecDecoder.release(); + } + } + + /** 初始化"自编码"串流模式的 UI 与解码器。 */ + private void setupSelfCodecUi() { + switchStreamMode = findViewById(R.id.switch_stream_mode); + tvStreamMode = findViewById(R.id.tv_stream_mode); + selfCodecSurface = findViewById(R.id.self_codec_surface); + selfCodecDecoder = new SelfCodecDecoder(); + surfaceCallback = new SurfaceHolder.Callback() { + @Override + public void surfaceCreated(SurfaceHolder holder) { + if (selfCodecDecoder != null) { + selfCodecDecoder.setSurface(holder.getSurface()); + } + } + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { + } + + @Override + public void surfaceDestroyed(SurfaceHolder holder) { + } + }; + selfCodecSurface.getHolder().addCallback(surfaceCallback); + switchStreamMode.setEnabled(false); + switchStreamMode.setOnCheckedChangeListener((buttonView, isChecked) -> { + if (programmaticSwitch) return; + applyStreamMode(isChecked); + }); + } + + /** 用户切换串流模式：发送信令并切换本地视图。 */ + private void applyStreamMode(boolean self) { + if (webRtcClient != null && webRtcClient.isDataChannelOpen()) { + webRtcClient.sendStreamMode(self ? WebRtcClient.STREAM_MODE_SELF_CODEC + : WebRtcClient.STREAM_MODE_WEBRTC); + } + tvStreamMode.setText(self ? "串流: 自编码" : "串流: WebRTC"); + if (self) { + selfCodecSurface.setVisibility(View.VISIBLE); + remoteVideoView.setVisibility(View.GONE); + selfCodecDecoder.setEnabled(true); + } else { + selfCodecSurface.setVisibility(View.GONE); + remoteVideoView.setVisibility(View.VISIBLE); + selfCodecDecoder.setEnabled(false); + } + } + + /** 收到被控端回报的当前模式：同步 UI 与视图（防止切换失败不一致）。 */ + private void syncStreamModeUi(int mode) { + boolean self = (mode == WebRtcClient.STREAM_MODE_SELF_CODEC); + programmaticSwitch = true; + switchStreamMode.setChecked(self); + programmaticSwitch = false; + tvStreamMode.setText(self ? "串流: 自编码" : "串流: WebRTC"); + if (self) { + selfCodecSurface.setVisibility(View.VISIBLE); + remoteVideoView.setVisibility(View.GONE); + selfCodecDecoder.setEnabled(true); + } else { + selfCodecSurface.setVisibility(View.GONE); + remoteVideoView.setVisibility(View.VISIBLE); + selfCodecDecoder.setEnabled(false); } } diff --git a/WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/SelfCodecDecoder.java b/WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/SelfCodecDecoder.java new file mode 100644 index 0000000..1c269bc --- /dev/null +++ b/WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/SelfCodecDecoder.java @@ -0,0 +1,260 @@ +package com.ttstd.controller.webrtc; + +import android.media.MediaCodec; +import android.media.MediaFormat; +import android.util.Log; +import android.view.Surface; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.Map; + +/** + * 自建屏幕解码通道：从 WebRTC 的 video DataChannel 收取被控端发来的 H.264 裸流分片， + * 重组后使用 MediaCodec 解码并渲染到指定 Surface。与 WebRTC 默认媒体轨道解耦， + * 配合被控端 SelfCodecEncoder 使用。 + */ +public class SelfCodecDecoder { + + private static final String TAG = "SelfCodecDecoder"; + + // ---- 必须与被控端 SelfCodecEncoder 保持一致 ---- + private static final byte MAGIC = (byte) 0xAB; + private static final int CHUNK_SIZE = 16384; + private static final int UNIT_TYPE_CONFIG = 1; + private static final int UNIT_TYPE_FRAME = 2; + private static final int PLACEHOLDER_W = 1920; + private static final int PLACEHOLDER_H = 1080; + + private MediaCodec decoder; + private Surface surface; + private boolean configured = false; + private boolean enabled = false; + + // 待喂入的参数集（SPS/PPS 等），以 CODEC_CONFIG 形式输入解码器 + private final ArrayDeque pendingConfigs = new ArrayDeque<>(); + // 待解码帧 + private final ArrayDeque frameQueue = new ArrayDeque<>(); + // 解码器可用的输入缓冲下标 + private final ArrayDeque inputQueue = new ArrayDeque<>(); + // 分片重组：frameSeq -> 该单元的若干分片 + private final Map assembling = new HashMap<>(); + private int lastSeq = -1; + + private static class Frame { + byte[] data; + long pts; + int flags; + } + + private static class ChunkBuffer { + int total; + byte[][] chunks; + int received; + } + + public void setSurface(Surface s) { + this.surface = s; + tryConfigure(); + } + + public void setEnabled(boolean e) { + enabled = e; + if (!e) { + releaseDecoder(); + } + } + + /** 收到 video DataChannel 的二进制消息（一条分片）。 */ + public void onBinaryMessage(ByteBuffer raw) { + if (!enabled) return; + if (raw == null) return; + ByteBuffer b = raw.duplicate(); + if (b.remaining() < 13) return; + byte magic = b.get(); + if (magic != MAGIC) return; + int seq = b.getInt(); + short total = b.getShort(); + short idx = b.getShort(); + int len = b.getInt(); + if (b.remaining() < len) return; + byte[] chunk = new byte[len]; + b.get(chunk); + + ChunkBuffer cb = assembling.get(seq); + if (cb == null) { + cb = new ChunkBuffer(); + cb.total = total; + cb.chunks = new byte[total][]; + cb.received = 0; + assembling.put(seq, cb); + if (assembling.size() > 24) { + int min = Integer.MAX_VALUE; + for (int k : assembling.keySet()) { + if (k < min) min = k; + } + assembling.remove(min); + } + } + if (idx >= 0 && idx < total && cb.chunks[idx] == null) { + cb.chunks[idx] = chunk; + cb.received++; + } + if (cb.received == total) { + assembling.remove(seq); + if (seq > lastSeq) lastSeq = seq; + byte[] unit = join(cb); + if (unit != null) handleUnit(unit); + } + } + + private byte[] join(ChunkBuffer cb) { + int totalLen = 0; + for (byte[] c : cb.chunks) { + if (c != null) totalLen += c.length; + } + ByteBuffer out = ByteBuffer.allocate(totalLen); + for (byte[] c : cb.chunks) { + if (c != null) out.put(c); + } + return out.array(); + } + + private void handleUnit(byte[] unit) { + ByteBuffer b = ByteBuffer.wrap(unit); + byte type = b.get(); + long pts = b.getInt() & 0xFFFFFFFFL; + byte isKey = b.get(); + int len = b.getInt(); + byte[] data = new byte[len]; + b.get(data); + + if (type == UNIT_TYPE_CONFIG) { + if (configured) { + // 已配置：忽略重复/初始参数集，避免重复以 CODEC_CONFIG 喂入报错 + return; + } + synchronized (pendingConfigs) { + pendingConfigs.add(data); + } + tryConfigure(); + } else if (type == UNIT_TYPE_FRAME) { + Frame f = new Frame(); + f.data = data; + f.pts = pts; + f.flags = (isKey != 0) ? MediaCodec.BUFFER_FLAG_KEY_FRAME : 0; + synchronized (frameQueue) { + frameQueue.add(f); + } + feed(); + } + } + + private void tryConfigure() { + if (configured) return; + if (surface == null) return; + synchronized (pendingConfigs) { + if (pendingConfigs.isEmpty()) return; + } + try { + MediaFormat format = MediaFormat.createVideoFormat( + MediaFormat.MIMETYPE_VIDEO_AVC, PLACEHOLDER_W, PLACEHOLDER_H); + decoder = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC); + decoder.configure(format, surface, null, 0); + decoder.setCallback(decoderCallback); + decoder.start(); + configured = true; + Log.i(TAG, "self codec decoder configured"); + feed(); + } catch (Exception e) { + Log.e(TAG, "decoder configure failed", e); + } + } + + private final MediaCodec.Callback decoderCallback = new MediaCodec.Callback() { + @Override + public void onInputBufferAvailable(MediaCodec mc, int index) { + inputQueue.add(index); + feed(); + } + + @Override + public void onOutputBufferAvailable(MediaCodec mc, int index, MediaCodec.BufferInfo info) { + mc.releaseOutputBuffer(index, true); + } + + @Override + public void onError(MediaCodec mc, MediaCodec.CodecException e) { + Log.e(TAG, "decoder error", e); + } + + @Override + public void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { + Log.d(TAG, "decoder output format changed"); + } + }; + + private void feed() { + if (decoder == null || !configured) return; + synchronized (pendingConfigs) { + while (!pendingConfigs.isEmpty() && !inputQueue.isEmpty()) { + byte[] cfg = pendingConfigs.poll(); + int idx = inputQueue.poll(); + try { + ByteBuffer ib = decoder.getInputBuffer(idx); + ib.clear(); + ib.put(cfg); + decoder.queueInputBuffer(idx, 0, cfg.length, 0, MediaCodec.BUFFER_FLAG_CODEC_CONFIG); + } catch (Exception e) { + Log.e(TAG, "queue config failed", e); + pendingConfigs.addFirst(cfg); + inputQueue.addFirst(idx); + break; + } + } + } + synchronized (frameQueue) { + while (!frameQueue.isEmpty() && !inputQueue.isEmpty()) { + Frame f = frameQueue.poll(); + int idx = inputQueue.poll(); + try { + ByteBuffer ib = decoder.getInputBuffer(idx); + ib.clear(); + ib.put(f.data); + decoder.queueInputBuffer(idx, 0, f.data.length, f.pts, f.flags); + } catch (Exception e) { + Log.e(TAG, "queue frame failed", e); + frameQueue.addFirst(f); + inputQueue.addFirst(idx); + break; + } + } + } + } + + private void releaseDecoder() { + if (decoder != null) { + try { + decoder.stop(); + } catch (Exception ignored) { + } + try { + decoder.release(); + } catch (Exception ignored) { + } + decoder = null; + } + configured = false; + pendingConfigs.clear(); + frameQueue.clear(); + inputQueue.clear(); + assembling.clear(); + } + + public void release() { + releaseDecoder(); + surface = null; + } +} diff --git a/WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/WebRtcClient.java b/WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/WebRtcClient.java index d82fca3..ff9f108 100644 --- a/WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/WebRtcClient.java +++ b/WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/WebRtcClient.java @@ -35,7 +35,10 @@ import java.util.List; public class WebRtcClient { private static final String TAG = "WebRtcClient"; - private static final String DATA_CHANNEL_LABEL = "control_channel"; + public static final String DATA_CHANNEL_LABEL = "control_channel"; + public static final String VIDEO_CHANNEL_LABEL = "video_channel"; // 自编码视频透传通道 + public static final int STREAM_MODE_WEBRTC = 0; // WebRTC 内置媒体流 + public static final int STREAM_MODE_SELF_CODEC = 1; // 自编码（MediaCodec + DataChannel） private final Context context; private final WebSocketClient wsClient; @@ -47,6 +50,9 @@ public class WebRtcClient { private PeerConnectionFactory peerConnectionFactory; private PeerConnection peerConnection; private DataChannel dataChannel; + private DataChannel videoDataChannel; // 自编码视频透传通道（本端创建） + private SelfCodecDecoder selfCodecDecoder; + private StreamModeReportListener streamModeReportListener; private EglBase eglBase; private String currentControlledId; private SurfaceViewRenderer remoteSurfaceView; @@ -177,6 +183,14 @@ public class WebRtcClient { dataChannel = peerConnection.createDataChannel(DATA_CHANNEL_LABEL, dcInit); handleDataChannel(dataChannel); + // 新建"自编码"视频透传通道（无序、不可靠，以最低延迟传输裸 H.264 码流）。 + // 必须在 setLocalDescription 之前创建，才会进入 SDP。 + DataChannel.Init vInit = new DataChannel.Init(); + vInit.ordered = false; + vInit.maxRetransmits = 0; + videoDataChannel = peerConnection.createDataChannel(VIDEO_CHANNEL_LABEL, vInit); + setupVideoDataChannel(videoDataChannel); + // 保存远端视频渲染器 this.remoteSurfaceView = remoteRenderer; @@ -270,6 +284,10 @@ public class WebRtcClient { dataChannel.close(); dataChannel.dispose(); } + if (videoDataChannel != null) { + videoDataChannel.close(); + videoDataChannel.dispose(); + } if (peerConnection != null) { peerConnection.close(); peerConnection.dispose(); @@ -300,13 +318,65 @@ public class WebRtcClient { @Override public void onMessage(DataChannel.Buffer buffer) { + if (buffer == null || buffer.data == null) return; byte[] data = new byte[buffer.data.remaining()]; buffer.data.get(data); - Log.d(TAG, "DataChannel message received, bytes=" + data.length); + // 若收到被控端回报的串流模式，单独处理。 + try { + ControlMessage msg = ControlMessage.parseFrom(data); + if (msg.getAction() == Action.REPORT_STREAM_MODE) { + if (streamModeReportListener != null) { + streamModeReportListener.onStreamModeReported(msg.getStreamMode()); + } + return; + } + } catch (Exception ignored) { + } } }); } + private void setupVideoDataChannel(DataChannel dc) { + dc.registerObserver(new DataChannel.Observer() { + @Override + public void onBufferedAmountChange(long previousAmount) { + } + + @Override + public void onStateChange() { + Log.d(TAG, "Video DataChannel state: " + dc.state()); + } + + @Override + public void onMessage(DataChannel.Buffer buffer) { + if (selfCodecDecoder != null && buffer != null && buffer.data != null) { + selfCodecDecoder.onBinaryMessage(buffer.data); + } + } + }); + } + + public void setSelfCodecDecoder(SelfCodecDecoder decoder) { + this.selfCodecDecoder = decoder; + } + + public void setStreamModeReportListener(StreamModeReportListener l) { + this.streamModeReportListener = l; + } + + public boolean isVideoDataChannelOpen() { + return videoDataChannel != null && videoDataChannel.state() == DataChannel.State.OPEN; + } + + /** 请求被控端切换屏幕串流模式（WebRTC / 自编码）。 */ + public void sendStreamMode(int mode) { + ControlMessage msg = ControlMessage.newBuilder() + .setAction(Action.SET_STREAM_MODE) + .setStreamMode(mode) + .build(); + sendControlCommand(msg); + } + private void sendOffer(String sdp, String controlledId) { Log.e(TAG, "sendOffer: " ); SignalMessage msg = new SignalMessage(); diff --git a/WebRTCController/app/src/main/proto/control_message.proto b/WebRTCController/app/src/main/proto/control_message.proto index 99c6720..d53fc96 100644 --- a/WebRTCController/app/src/main/proto/control_message.proto +++ b/WebRTCController/app/src/main/proto/control_message.proto @@ -15,6 +15,8 @@ enum Action { MOTION_EVENT = 5; SET_RESOLUTION = 6; // 控制端请求被控端切换屏幕采集分辨率 REPORT_RESOLUTION = 7; // 被控端上报当前实际采集分辨率（含连接建立后的初始值） + SET_STREAM_MODE = 8; // 控制端请求被控端切换屏幕串流模式 + REPORT_STREAM_MODE = 9; // 被控端上报当前生效的串流模式 } // 通过 RTCDataChannel 传输的控制指令（protobuf 二进制）。 @@ -48,4 +50,5 @@ message ControlMessage { int32 width = 12; int32 height = 13; int32 fps = 14; + int32 stream_mode = 15; // 串流模式：0=WebRTC 内置媒体流，1=自编码(MediaCodec 硬编 + DataChannel 透传) } diff --git a/WebRTCController/app/src/main/res/layout/activity_main.xml b/WebRTCController/app/src/main/res/layout/activity_main.xml index 47e8885..215f890 100644 --- a/WebRTCController/app/src/main/res/layout/activity_main.xml +++ b/WebRTCController/app/src/main/res/layout/activity_main.xml @@ -97,11 +97,20 @@ android:visibility="gone"> - + + + + + + + + + diff --git a/webrtc_controller_flutter/android/app/src/main/kotlin/com/ttstd/fluttercontroller/MainActivity.kt b/webrtc_controller_flutter/android/app/src/main/kotlin/com/ttstd/fluttercontroller/MainActivity.kt index b180782..a66bdaf 100644 --- a/webrtc_controller_flutter/android/app/src/main/kotlin/com/ttstd/fluttercontroller/MainActivity.kt +++ b/webrtc_controller_flutter/android/app/src/main/kotlin/com/ttstd/fluttercontroller/MainActivity.kt @@ -1,5 +1,12 @@ package com.ttstd.fluttercontroller import io.flutter.embedding.android.FlutterActivity +import io.flutter.embedding.engine.FlutterEngine -class MainActivity : FlutterActivity() +class MainActivity : FlutterActivity() { + override fun configureFlutterEngine(flutterEngine: FlutterEngine) { + super.configureFlutterEngine(flutterEngine) + // 注册自编码解码器插件（仅 Android 平台提供硬解能力）。 + flutterEngine.plugins.add(SelfCodecDecoderPlugin()) + } +} diff --git a/webrtc_controller_flutter/android/app/src/main/kotlin/com/ttstd/fluttercontroller/SelfCodecDecoderPlugin.kt b/webrtc_controller_flutter/android/app/src/main/kotlin/com/ttstd/fluttercontroller/SelfCodecDecoderPlugin.kt new file mode 100644 index 0000000..f2934a7 --- /dev/null +++ b/webrtc_controller_flutter/android/app/src/main/kotlin/com/ttstd/fluttercontroller/SelfCodecDecoderPlugin.kt @@ -0,0 +1,355 @@ +package com.ttstd.fluttercontroller + +import android.graphics.SurfaceTexture +import android.media.MediaCodec +import android.media.MediaFormat +import android.os.Handler +import android.os.Looper +import android.util.Log +import android.view.Surface +import io.flutter.embedding.engine.plugins.FlutterPlugin +import io.flutter.embedding.engine.plugins.activity.ActivityAware +import io.flutter.embedding.engine.plugins.activity.ActivityPluginBinding +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel +import io.flutter.view.TextureRegistry +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder + +private const val CHANNEL = "com.ttstd.fluttercontroller/self_codec" +private const val TAG = "SelfCodecDecoderPlugin" + +private const val MAGIC: Byte = 0xAB.toByte() +private const val PROTOCOL_VERSION = 1 +private const val HEADER_SIZE = 13 +private const val UNIT_TYPE_CONFIG = 1 +private const val UNIT_TYPE_FRAME = 2 +private const val PLACEHOLDER_W = 1920 +private const val PLACEHOLDER_H = 1080 + +/// 单个单元的分片重组缓冲。 +private class ChunkBuffer(val seq: Int, val total: Int) { + private val chunks = arrayOfNulls(total) + var received = 0 + private set + + fun add(idx: Int, data: ByteArray) { + if (idx in 0 until total && chunks[idx] == null) { + chunks[idx] = data + received++ + } + } + + fun isComplete(): Boolean = received == total + + fun join(): ByteArray { + val out = ByteArrayOutputStream() + for (c in chunks) if (c != null) out.write(c) + return out.toByteArray() + } +} + +private class Frame(val data: ByteArray, val pts: Long, val key: Boolean) + +/// 自编码解码器（控制端原生实现）。 +/// +/// 对应 WebRTCController 的 SelfCodecDecoder.java：从 `video` DataChannel 接收 +/// 二进制分片，重组为完整单元，使用 Android MediaCodec 硬解 H.264， +/// 并渲染到 Flutter 纹理（TextureRegistry.SurfaceTextureEntry）。 +class SelfCodecDecoderPlugin : + FlutterPlugin, ActivityAware, MethodChannel.MethodCallHandler { + + private var methodChannel: MethodChannel? = null + private var textureRegistry: TextureRegistry? = null + + private var textureEntry: TextureRegistry.SurfaceTextureEntry? = null + private var surfaceTexture: SurfaceTexture? = null + private var surface: Surface? = null + + private var decoder: MediaCodec? = null + private var configured = false + private var enabled = false + private val configLock = Any() + + private val chunkBuffers = HashMap() + private val pendingConfigs = ArrayList() + private val frameQueue = ArrayList() + private val inputQueue = ArrayList() + + private val mainHandler = Handler(Looper.getMainLooper()) + + override fun onAttachedToEngine(binding: FlutterPlugin.FlutterPluginBinding) { + textureRegistry = binding.textureRegistry + methodChannel = MethodChannel(binding.binaryMessenger, CHANNEL) + methodChannel?.setMethodCallHandler(this) + } + + override fun onDetachedFromEngine(binding: FlutterPlugin.FlutterPluginBinding) { + methodChannel?.setMethodCallHandler(null) + methodChannel = null + textureRegistry = null + } + + override fun onAttachedToActivity(binding: ActivityPluginBinding) {} + + override fun onDetachedFromActivityForConfigChanges() {} + + override fun onReattachedToActivityForConfigChanges(binding: ActivityPluginBinding) {} + + override fun onDetachedFromActivity() {} + + override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) { + when (call.method) { + "create" -> { + try { + val id = createInternal() + if (id >= 0) result.success(id) else result.error("E_CREATE", "failed", null) + } catch (e: Exception) { + Log.e(TAG, "create error", e) + result.error("E_CREATE", e.message, null) + } + } + "feed" -> { + val bytes = call.arguments as? ByteArray + if (bytes != null) { + try { + onBinaryMessage(ByteBuffer.wrap(bytes).order(ByteOrder.BIG_ENDIAN)) + } catch (e: Exception) { + Log.e(TAG, "feed error", e) + } + } + result.success(null) + } + "setEnabled" -> { + enabled = call.arguments as? Boolean ?: false + if (!enabled) releaseDecoder() + result.success(null) + } + "dispose" -> { + releaseAll() + result.success(null) + } + else -> result.notImplemented() + } + } + + private fun createInternal(): Int { + if (textureEntry != null) return textureEntry!!.id().toInt() + val entry = textureRegistry?.createSurfaceTexture() ?: return -1 + textureEntry = entry + val st = entry.surfaceTexture() + surfaceTexture = st + st.setDefaultBufferSize(PLACEHOLDER_W, PLACEHOLDER_H) + st.setOnFrameAvailableListener { texture: SurfaceTexture -> + // 解码器将帧渲染到 Surface 后触发，这里更新纹理内容供 Flutter 显示。 + try { + texture.updateTexImage() + } catch (e: Exception) { + Log.w(TAG, "updateTexImage failed", e) + } + } + surface = Surface(st) + return entry.id().toInt() + } + + private fun onBinaryMessage(buffer: ByteBuffer) { + if (!enabled) return + while (buffer.remaining() >= HEADER_SIZE) { + val start = buffer.position() + val magic = buffer.get() + val seq = buffer.int + val total = buffer.short.toInt() + val idx = buffer.short.toInt() + val len = buffer.int + if (magic != MAGIC) { + Log.w(TAG, "bad magic byte: $magic") + return + } + if (len < 0 || buffer.remaining() < len) { + buffer.position(start) + break + } + val chunkData = ByteArray(len) + buffer.get(chunkData) + handleChunk(seq, total, idx, chunkData) + } + } + + private fun handleChunk(seq: Int, total: Int, idx: Int, data: ByteArray) { + var buf = chunkBuffers[seq] + if (buf == null) { + buf = ChunkBuffer(seq, total) + chunkBuffers[seq] = buf + } + buf.add(idx, data) + if (buf.isComplete()) { + chunkBuffers.remove(seq) + handleUnit(buf.join()) + } + } + + private fun handleUnit(unit: ByteArray) { + if (unit.size < 10) return + val bb = ByteBuffer.wrap(unit).order(ByteOrder.BIG_ENDIAN) + val type = bb.get().toInt() and 0xFF + val pts = bb.int.toLong() and 0xFFFFFFFFL + val isKey = bb.get().toInt() != 0 + val len = bb.int + if (len < 0 || bb.remaining() < len) return + val data = ByteArray(len) + bb.get(data) + when (type) { + UNIT_TYPE_CONFIG -> handleConfig(data) + UNIT_TYPE_FRAME -> handleFrame(data, pts, isKey) + } + } + + private fun handleConfig(config: ByteArray) { + synchronized(configLock) { + if (configured) return // 忽略重复参数集 + pendingConfigs.add(config) + tryConfigure() + } + } + + private fun handleFrame(frame: ByteArray, pts: Long, key: Boolean) { + synchronized(configLock) { + frameQueue.add(Frame(frame, pts, key)) + tryConfigure() + feed() + } + } + + private fun tryConfigure() { + synchronized(configLock) { + if (configured || surface == null || pendingConfigs.isEmpty()) return + val format = MediaFormat.createVideoFormat( + MediaFormat.MIMETYPE_VIDEO_AVC, PLACEHOLDER_W, PLACEHOLDER_H) + var codec: MediaCodec? = null + try { + codec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC) + codec.setCallback(object : MediaCodec.Callback() { + override fun onInputBufferAvailable(codec: MediaCodec, index: Int) { + synchronized(configLock) { + inputQueue.add(index) + feed() + } + } + + override fun onOutputBufferAvailable( + codec: MediaCodec, index: Int, info: MediaCodec.BufferInfo) { + try { + codec.releaseOutputBuffer(index, true) + } catch (e: Exception) { + Log.w(TAG, "releaseOutputBuffer failed", e) + } + } + + override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) { + Log.e(TAG, "decoder error", e) + } + + override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) { + val w = format.getInteger(MediaFormat.KEY_WIDTH, 0) + val h = format.getInteger(MediaFormat.KEY_HEIGHT, 0) + if (w > 0 && h > 0) { + surfaceTexture?.setDefaultBufferSize(w, h) + mainHandler.post { + methodChannel?.invokeMethod( + "onSize", mapOf("width" to w, "height" to h)) + } + } + } + }) + codec.configure(format, surface, null, 0) + codec.start() + decoder = codec + configured = true + feed() + } catch (e: Exception) { + Log.e(TAG, "configure failed", e) + try { + codec?.release() + } catch (_: Exception) { + } + decoder = null + configured = false + } + } + } + + private fun feed() { + synchronized(configLock) { + if (!configured) return + // 先喂入参数集（SPS / PPS）。 + while (pendingConfigs.isNotEmpty() && inputQueue.isNotEmpty()) { + val config = pendingConfigs.removeAt(0) + val index = inputQueue.removeAt(0) + val inBuf = decoder?.getInputBuffer(index) ?: return + try { + inBuf.clear() + inBuf.put(config) + decoder?.queueInputBuffer( + index, 0, config.size, 0, MediaCodec.BUFFER_FLAG_CODEC_CONFIG) + } catch (e: Exception) { + Log.w(TAG, "queue config failed", e) + } + } + // 再喂入视频帧。 + while (frameQueue.isNotEmpty() && inputQueue.isNotEmpty()) { + val frame = frameQueue.removeAt(0) + val index = inputQueue.removeAt(0) + val inBuf = decoder?.getInputBuffer(index) ?: return + try { + inBuf.clear() + inBuf.put(frame.data) + val flags = if (frame.key) MediaCodec.BUFFER_FLAG_KEY_FRAME else 0 + decoder?.queueInputBuffer( + index, 0, frame.data.size, frame.pts, flags) + } catch (e: Exception) { + Log.w(TAG, "queue frame failed", e) + } + } + } + } + + private fun releaseDecoder() { + synchronized(configLock) { + try { + decoder?.stop() + } catch (_: Exception) { + } + try { + decoder?.release() + } catch (_: Exception) { + } + decoder = null + configured = false + pendingConfigs.clear() + frameQueue.clear() + inputQueue.clear() + } + } + + private fun releaseAll() { + releaseDecoder() + try { + surface?.release() + } catch (_: Exception) { + } + surface = null + try { + surfaceTexture?.release() + } catch (_: Exception) { + } + surfaceTexture = null + try { + textureEntry?.release() + } catch (_: Exception) { + } + textureEntry = null + chunkBuffers.clear() + enabled = false + } +} diff --git a/webrtc_controller_flutter/lib/controller/remote_controller.dart b/webrtc_controller_flutter/lib/controller/remote_controller.dart index 16aa271..1352d08 100644 --- a/webrtc_controller_flutter/lib/controller/remote_controller.dart +++ b/webrtc_controller_flutter/lib/controller/remote_controller.dart @@ -45,6 +45,18 @@ class RemoteController { /// 统计信息刷新（每秒一次）。 void Function(String stats)? onStats; + /// 自编码解码纹理已就绪（textureId >= 0）。 + void Function(int textureId)? onSelfCodecReady; + + /// 被控端上报当前生效的串流模式。 + void Function(int mode)? onStreamModeReport; + + /// 被控端/解码器上报分辨率。 + void Function(int width, int height)? onResolutionReported; + + /// 当前平台不支持原生硬解时回调。 + void Function()? onSelfCodecNotSupported; + RemoteController({ required this.serverUrl, required this.deviceId, @@ -92,6 +104,10 @@ class RemoteController { }; _webRtc!.onIceDisconnected = (message) => onIceDisconnected?.call(message); _webRtc!.onRemoteStream = (renderer) => onRemoteStream?.call(renderer); + _webRtc!.onSelfCodecReady = (id) => onSelfCodecReady?.call(id); + _webRtc!.onStreamModeReport = (mode) => onStreamModeReport?.call(mode); + _webRtc!.onResolutionReported = (w, h) => onResolutionReported?.call(w, h); + _webRtc!.onSelfCodecNotSupported = () => onSelfCodecNotSupported?.call(); _webRtc!.initialize().catchError((e) { onStatusChanged?.call('状态: 连接失败 - $e'); }); @@ -155,6 +171,10 @@ class RemoteController { _webRtc?.sendResolutionChange(width, height, fps); } + /// 请求被控端切换屏幕串流模式（0=WebRTC 全托管 / 1=自编码）。 + Future sendStreamMode(int mode) => + _webRtc?.sendStreamMode(mode) ?? Future.value(); + /// 断开连接并释放资源。 Future disconnect() async { _statsTimer?.cancel(); diff --git a/webrtc_controller_flutter/lib/main.dart b/webrtc_controller_flutter/lib/main.dart index 3bb0474..192a127 100644 --- a/webrtc_controller_flutter/lib/main.dart +++ b/webrtc_controller_flutter/lib/main.dart @@ -8,6 +8,7 @@ import 'config/ice_servers.dart'; import 'controller/remote_controller.dart'; import 'utils/control_commands.dart'; import 'utils/device_utils.dart'; +import 'webrtc/self_codec_decoder.dart'; import 'widgets/remote_touch_view.dart'; void main() { @@ -55,6 +56,12 @@ class _ControllerHomeState extends State { String _stats = ''; double _videoAspect = 16 / 9; + /// 当前串流模式：0=WebRTC 全托管；1=自编码（自建 MediaCodec 解码）。 + int _streamMode = SelfCodecDecoder.streamModeWebRtc; + int? _selfCodecTextureId; + bool _selfCodecReady = false; + bool _selfCodecSupported = true; + /// 分辨率预设：width 为长边像素；0 表示被控端原生分辨率； /// height 传 0（由被控端按屏幕宽高比计算），fps 传 0（沿用当前帧率）。 int _selectedResolution = 0; @@ -217,6 +224,28 @@ class _ControllerHomeState extends State { renderer.addListener(_onRendererUpdate); }; _controller!.onStats = (stats) => setState(() => _stats = stats); + _controller!.onSelfCodecReady = (textureId) { + if (mounted) { + setState(() { + _selfCodecTextureId = textureId; + _selfCodecReady = true; + }); + } + }; + _controller!.onStreamModeReport = (mode) { + if (mounted) setState(() => _streamMode = mode); + }; + _controller!.onResolutionReported = (w, h) { + if (w > 0 && h > 0 && mounted) { + final aspect = w / h; + setState(() => _videoAspect = aspect); + _resizeWindowToAspect(aspect); + } + }; + _controller!.onSelfCodecNotSupported = () { + if (mounted) setState(() => _selfCodecSupported = false); + _showAlert('当前平台不支持自编码硬解，已回退到 WebRTC 媒体流。'); + }; _controller!.connect(authType: authType, authValue: authValue); } @@ -300,6 +329,19 @@ class _ControllerHomeState extends State { ); } + /// 切换串流模式：WebRTC 全托管 <-> 自编码（自建 MediaCodec 解码）。 + void _toggleStreamMode() { + if (!_selfCodecSupported) { + _showAlert('当前平台不支持自编码硬解。'); + return; + } + final next = _streamMode == SelfCodecDecoder.streamModeSelfCodec + ? SelfCodecDecoder.streamModeWebRtc + : SelfCodecDecoder.streamModeSelfCodec; + setState(() => _streamMode = next); + _controller?.sendStreamMode(next); + } + Future _disconnect() async { await _controller?.disconnect(); _controller = null; @@ -310,6 +352,10 @@ class _ControllerHomeState extends State { _renderer = null; _status = '状态: 已停止'; _stats = ''; + _streamMode = SelfCodecDecoder.streamModeWebRtc; + _selfCodecTextureId = null; + _selfCodecReady = false; + _selfCodecSupported = true; }); } @@ -412,6 +458,33 @@ class _ControllerHomeState extends State { ], ), ), + // 自编码串流开关：仅 Android 等支持原生硬解的平台可用。 + CupertinoButton( + padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 10), + onPressed: _selfCodecSupported ? _toggleStreamMode : null, + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Text( + '自编码', + style: TextStyle( + color: _selfCodecSupported + ? CupertinoColors.white + : CupertinoColors.white.withValues(alpha: 0.4), + fontSize: 14, + ), + ), + const SizedBox(width: 6), + CupertinoSwitch( + value: _streamMode == SelfCodecDecoder.streamModeSelfCodec, + onChanged: _selfCodecSupported + ? (_) => _toggleStreamMode() + : null, + activeTrackColor: CupertinoColors.activeBlue, + ), + ], + ), + ), // 分隔线 Container( width: 1, @@ -444,7 +517,31 @@ class _ControllerHomeState extends State { child: Stack( children: [ Container(color: Colors.black), - if (_renderer != null) + if (_streamMode == SelfCodecDecoder.streamModeSelfCodec && + _selfCodecReady && + _selfCodecTextureId != null) + Center( + child: AspectRatio( + aspectRatio: _videoAspect, + child: Stack( + fit: StackFit.expand, + children: [ + Texture(textureId: _selfCodecTextureId!), + RemoteTouchView( + onTouch: (x, y) {}, + onSwipe: (x1, y1, x2, y2, d) {}, + onLongPress: (x, y) {}, + onKey: (k, a) => + _controller?.sendControlCommand(ControlCommands.key(k, a)), + onMotionEvent: (a, x, y) => _controller?.sendControlCommand( + ControlCommands.motionEvent(a, x, y), + ), + ), + ], + ), + ), + ) + else if (_renderer != null) Center( child: AspectRatio( aspectRatio: _videoAspect, diff --git a/webrtc_controller_flutter/lib/proto/control_message.pb.dart b/webrtc_controller_flutter/lib/proto/control_message.pb.dart index d2c7538..d5487b6 100644 --- a/webrtc_controller_flutter/lib/proto/control_message.pb.dart +++ b/webrtc_controller_flutter/lib/proto/control_message.pb.dart @@ -39,6 +39,7 @@ class ControlMessage extends $pb.GeneratedMessage { $core.int? width, $core.int? height, $core.int? fps, + $core.int? streamMode, }) { final result = create(); if (action != null) result.action = action; @@ -55,6 +56,7 @@ class ControlMessage extends $pb.GeneratedMessage { if (width != null) result.width = width; if (height != null) result.height = height; if (fps != null) result.fps = fps; + if (streamMode != null) result.streamMode = streamMode; return result; } @@ -86,6 +88,7 @@ class ControlMessage extends $pb.GeneratedMessage { ..aI(12, _omitFieldNames ? '' : 'width') ..aI(13, _omitFieldNames ? '' : 'height') ..aI(14, _omitFieldNames ? '' : 'fps') + ..aI(15, _omitFieldNames ? '' : 'stream_mode') ..hasRequiredFields = false; @$core.Deprecated('See https://github.com/google/protobuf.dart/issues/998.') @@ -240,6 +243,17 @@ class ControlMessage extends $pb.GeneratedMessage { $core.bool hasFps() => $_has(13); @$pb.TagNumber(14) void clearFps() => $_clearField(14); + + /// 串流模式（SET_STREAM_MODE / REPORT_STREAM_MODE）： + /// 0 = WebRTC 全托管；1 = 自编码（自建 MediaCodec 编解码 + video DataChannel） + @$pb.TagNumber(15) + $core.int get streamMode => $_getIZ(14); + @$pb.TagNumber(15) + set streamMode($core.int value) => $_setSignedInt32(14, value); + @$pb.TagNumber(15) + $core.bool hasStreamMode() => $_has(14); + @$pb.TagNumber(15) + void clearStreamMode() => $_clearField(15); } const $core.bool _omitFieldNames = diff --git a/webrtc_controller_flutter/lib/proto/control_message.pbenum.dart b/webrtc_controller_flutter/lib/proto/control_message.pbenum.dart index 0acf70d..15baa1e 100644 --- a/webrtc_controller_flutter/lib/proto/control_message.pbenum.dart +++ b/webrtc_controller_flutter/lib/proto/control_message.pbenum.dart @@ -29,6 +29,10 @@ class Action extends $pb.ProtobufEnum { Action._(6, _omitEnumNames ? '' : 'SET_RESOLUTION'); static const Action REPORT_RESOLUTION = Action._(7, _omitEnumNames ? '' : 'REPORT_RESOLUTION'); + static const Action SET_STREAM_MODE = + Action._(8, _omitEnumNames ? '' : 'SET_STREAM_MODE'); + static const Action REPORT_STREAM_MODE = + Action._(9, _omitEnumNames ? '' : 'REPORT_STREAM_MODE'); static const $core.List values = [ ACTION_UNKNOWN, @@ -39,10 +43,12 @@ class Action extends $pb.ProtobufEnum { MOTION_EVENT, SET_RESOLUTION, REPORT_RESOLUTION, + SET_STREAM_MODE, + REPORT_STREAM_MODE, ]; static final $core.List _byValue = - $pb.ProtobufEnum.$_initByValueList(values, 7); + $pb.ProtobufEnum.$_initByValueList(values, 9); static Action? valueOf($core.int value) => value < 0 || value >= _byValue.length ? null : _byValue[value]; diff --git a/webrtc_controller_flutter/lib/utils/control_commands.dart b/webrtc_controller_flutter/lib/utils/control_commands.dart index 0a37111..97ec7c8 100644 --- a/webrtc_controller_flutter/lib/utils/control_commands.dart +++ b/webrtc_controller_flutter/lib/utils/control_commands.dart @@ -68,4 +68,11 @@ class ControlCommands { height: height, fps: fps, ); + + /// 串流模式切换指令。 + /// [mode] 为 0=WebRTC 全托管 / 1=自编码（自建 MediaCodec 编解码 + video DataChannel 裸流透传）。 + static ControlMessage streamMode(int mode) => ControlMessage( + action: Action.SET_STREAM_MODE, + streamMode: mode, + ); } diff --git a/webrtc_controller_flutter/lib/webrtc/self_codec_decoder.dart b/webrtc_controller_flutter/lib/webrtc/self_codec_decoder.dart new file mode 100644 index 0000000..a6f31dc --- /dev/null +++ b/webrtc_controller_flutter/lib/webrtc/self_codec_decoder.dart @@ -0,0 +1,118 @@ +import 'dart:async'; + +import 'package:flutter/services.dart'; +import 'package:flutter/widgets.dart'; + +/// 自编码解码器（控制端）。 +/// +/// 接收来自 `video` DataChannel 的二进制分片（由被控端 [SelfCodecEncoder] 产生）， +/// 通过原生 Android `MediaCodec` 硬解码，并渲染到 Flutter 纹理（`Texture`）。 +/// +/// 为保证与 `WebRTCControlled` 的编码器在二进制协议上完全一致，协议的 +/// 分片重组、H.264 参数集解析与硬解、纹理渲染全部在原生侧完成； +/// 本类仅负责与原生侧通信。对应 Android 端 `SelfCodecDecoder`。 +/// +/// 注意：原生解码器目前仅在 Android 平台实现。其它平台（Windows / Linux / +/// Web / iOS）调用 [create] 会失败并回调 [onError]，调用方应回退到 +/// WebRTC 默认媒体流。 +class SelfCodecDecoder { + static const MethodChannel _channel = + MethodChannel('com.ttstd.fluttercontroller/self_codec'); + + /// 串流模式常量（与 ControlMessage.stream_mode 对齐）。 + static const int streamModeWebRtc = 0; + static const int streamModeSelfCodec = 1; + + int? _textureId; + bool _disposed = false; + bool _handlerSet = false; + + /// 解码输出尺寸变化（宽/高，单位像素）。 + final void Function(int width, int height)? onSize; + + /// 错误回调（如当前平台不支持硬解）。 + final void Function(String error)? onError; + + SelfCodecDecoder({this.onSize, this.onError}) { + _channel.setMethodCallHandler(_handleNativeCall); + _handlerSet = true; + } + + Future _handleNativeCall(MethodCall call) async { + switch (call.method) { + case 'onSize': + final args = call.arguments; + final w = args is Map ? (args['width'] as int? ?? 0) : 0; + final h = args is Map ? (args['height'] as int? ?? 0) : 0; + if (w > 0 && h > 0) onSize?.call(w, h); + break; + case 'onError': + final msg = call.arguments is String ? call.arguments as String : ''; + if (msg.isNotEmpty) onError?.call(msg); + break; + } + } + + /// 创建解码纹理。成功返回纹理 id（>=0），失败返回 null。 + Future create() async { + if (_disposed) return null; + try { + final id = await _channel.invokeMethod('create'); + if (id == null || id < 0) return null; + _textureId = id; + return _textureId; + } on PlatformException catch (e) { + onError?.call('创建自编码解码纹理失败: ${e.message}'); + return null; + } catch (e) { + onError?.call('创建自编码解码纹理失败: $e'); + return null; + } + } + + /// 已创建的纹理 id；未创建或已释放时为 null。 + int? get textureId => _textureId; + + /// 用于渲染解码画面的纹理控件。 + Widget build(BuildContext context) { + final id = _textureId; + if (id == null) return const SizedBox.shrink(); + return Texture(textureId: id); + } + + /// 喂入来自 video DataChannel 的二进制分片。 + Future feed(Uint8List chunk) async { + if (_disposed || _textureId == null) return; + try { + await _channel.invokeMethod('feed', chunk); + } catch (_) { + // 解码未就绪或被释放时忽略。 + } + } + + /// 启用/停用解码（仅自编码模式为 true）。 + Future setEnabled(bool enabled) async { + if (_disposed || _textureId == null) return; + try { + await _channel.invokeMethod('setEnabled', enabled); + } catch (_) { + // 忽略。 + } + } + + /// 释放原生解码资源。重复调用安全。 + Future dispose() async { + if (_disposed) return; + _disposed = true; + if (_handlerSet) { + _channel.setMethodCallHandler(null); + _handlerSet = false; + } + try { + await _channel.invokeMethod('dispose'); + } catch (_) { + // 忽略。 + } + _textureId = null; + } +} diff --git a/webrtc_controller_flutter/lib/webrtc/webrtc_controller.dart b/webrtc_controller_flutter/lib/webrtc/webrtc_controller.dart index cb51618..562db58 100644 --- a/webrtc_controller_flutter/lib/webrtc/webrtc_controller.dart +++ b/webrtc_controller_flutter/lib/webrtc/webrtc_controller.dart @@ -1,6 +1,8 @@ import 'package:flutter/foundation.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart'; +import 'self_codec_decoder.dart'; + import '../config/ice_servers.dart'; import '../models/signal_message.dart'; import '../proto/control_message.pb.dart'; @@ -26,6 +28,10 @@ class WebRtcController { RTCPeerConnection? _pc; RTCDataChannel? _dataChannel; + RTCDataChannel? _videoDataChannel; + SelfCodecDecoder? _selfCodecDecoder; + int _currentStreamMode = streamModeWebRtc; + bool _creatingDecoder = false; /// 远端视频渲染器，由调用方持有并显示。 final RTCVideoRenderer renderer = RTCVideoRenderer(); @@ -40,6 +46,25 @@ class WebRtcController { /// 远端视频流就绪（renderer 已绑定视频轨道）。 void Function(RTCVideoRenderer renderer)? onRemoteStream; + /// 自编码解码纹理已就绪（textureId >= 0）。 + void Function(int)? onSelfCodecReady; + + /// 自编码解码器已释放（连接断开时）。 + void Function()? onSelfCodecLost; + + /// 被控端上报当前生效的串流模式。 + void Function(int mode)? onStreamModeReport; + + /// 被控端上报分辨率（自编码解码器输出尺寸变化时也会触发）。 + void Function(int width, int height)? onResolutionReported; + + /// 当前平台（Windows / Linux / Web / iOS 等）不支持原生硬解时回调。 + void Function()? onSelfCodecNotSupported; + + static const String kVideoChannelLabel = 'video_channel'; + static const int streamModeWebRtc = 0; + static const int streamModeSelfCodec = 1; + WebRtcController({ required this.signaling, required this.deviceId, @@ -83,6 +108,17 @@ class WebRtcController { _dataChannel = await _pc!.createDataChannel(kDataChannelLabel, dcInit); _setupDataChannel(_dataChannel!); + // 创建自编码视频透传通道（无序、不可靠，类似 RTP，最低延迟）。 + // 被控端仅在收到 SET_STREAM_MODE=自编码 后才经此通道推送 H.264 裸流。 + final videoInit = RTCDataChannelInit() + ..ordered = false + ..maxRetransmits = 0; + _videoDataChannel = await _pc!.createDataChannel( + kVideoChannelLabel, + videoInit, + ); + _setupVideoDataChannel(_videoDataChannel!); + // 创建并发送 Offer。 final constraints = { 'mandatory': { @@ -104,18 +140,22 @@ class WebRtcController { } void _onDataChannel(RTCDataChannel channel) { - // 被控端也可能主动创建 DataChannel，统一处理。 - _setupDataChannel(channel); + // 被控端也可能主动创建 DataChannel，按标签统一处理。 + if (channel.label == kDataChannelLabel) { + _dataChannel ??= channel; + _setupDataChannel(channel); + } else if (channel.label == kVideoChannelLabel) { + _videoDataChannel ??= channel; + _setupVideoDataChannel(channel); + } } void _setupDataChannel(RTCDataChannel channel) { channel.onMessage = (RTCDataChannelMessage message) { - // 处理来自被控端的消息（本控制端主要发送，此处仅做日志记录）。 if (message.isBinary) { try { final command = ControlMessage.fromBuffer(message.binary); - // ignore: avoid_print - print('DataChannel message: ${command.action}'); + _handleControlMessage(command); } catch (e) { // ignore: avoid_print print('DataChannel message decode failed: $e'); @@ -131,6 +171,78 @@ class WebRtcController { }; } + void _setupVideoDataChannel(RTCDataChannel channel) { + channel.onMessage = (RTCDataChannelMessage message) { + if (message.isBinary) { + _selfCodecDecoder?.feed(message.binary); + } + }; + channel.onDataChannelState = (RTCDataChannelState state) { + if (state == RTCDataChannelState.RTCDataChannelOpen) { + _onVideoChannelOpen(); + } else if (state == RTCDataChannelState.RTCDataChannelClosed) { + _selfCodecDecoder?.setEnabled(false); + } + }; + } + + void _onVideoChannelOpen() { + _ensureSelfCodecDecoder().then((_) { + _selfCodecDecoder?.setEnabled(_currentStreamMode == streamModeSelfCodec); + }); + } + + Future _ensureSelfCodecDecoder() async { + if (_selfCodecDecoder != null || _creatingDecoder) return; + _creatingDecoder = true; + try { + final decoder = SelfCodecDecoder( + onSize: (w, h) => onResolutionReported?.call(w, h), + onError: (err) => debugPrint('自编码解码器: $err'), + ); + final textureId = await decoder.create(); + if (textureId != null) { + _selfCodecDecoder = decoder; + onSelfCodecReady?.call(textureId); + } else { + // 当前平台（Windows / Linux / Web / iOS 等）无原生硬解能力，回退 WebRTC 媒体流。 + onSelfCodecNotSupported?.call(); + } + } finally { + _creatingDecoder = false; + } + } + + void _handleControlMessage(ControlMessage msg) { + switch (msg.action) { + case Action.REPORT_STREAM_MODE: + _currentStreamMode = msg.streamMode; + _selfCodecDecoder?.setEnabled(msg.streamMode == streamModeSelfCodec); + onStreamModeReport?.call(msg.streamMode); + break; + case Action.REPORT_RESOLUTION: + onResolutionReported?.call(msg.width, msg.height); + break; + default: + // ignore: avoid_print + print('DataChannel message: ${msg.action}'); + break; + } + } + + /// 请求被控端切换屏幕串流模式（SET_STREAM_MODE 指令）。 + /// [mode] 为 0=WebRTC 全托管 / 1=自编码。 + Future sendStreamMode(int mode) async { + _currentStreamMode = mode; + if (mode == streamModeSelfCodec) { + await _ensureSelfCodecDecoder(); + await _selfCodecDecoder?.setEnabled(true); + } else { + await _selfCodecDecoder?.setEnabled(false); + } + sendControlCommand(ControlCommands.streamMode(mode)); + } + void _onIceCandidate(RTCIceCandidate candidate) { final payload = { 'sdpMid': candidate.sdpMid, @@ -413,6 +525,20 @@ class WebRtcController { await renderer.dispose(); } catch (_) {} _dataChannel = null; + try { + await _videoDataChannel?.close(); + } catch (_) {} + _videoDataChannel = null; + try { + await _selfCodecDecoder?.dispose(); + } catch (_) {} + _selfCodecDecoder = null; + _currentStreamMode = streamModeWebRtc; + onSelfCodecLost?.call(); + onSelfCodecReady = null; + onStreamModeReport = null; + onResolutionReported = null; + onSelfCodecNotSupported = null; _pc = null; // 重置网速统计基线，避免下次连接首帧显示错误的瞬时速率。 _prevBytesReceived = 0; diff --git a/webrtc_controller_flutter/proto/control_message.proto b/webrtc_controller_flutter/proto/control_message.proto index 99c6720..a98e3e7 100644 --- a/webrtc_controller_flutter/proto/control_message.proto +++ b/webrtc_controller_flutter/proto/control_message.proto @@ -15,6 +15,8 @@ enum Action { MOTION_EVENT = 5; SET_RESOLUTION = 6; // 控制端请求被控端切换屏幕采集分辨率 REPORT_RESOLUTION = 7; // 被控端上报当前实际采集分辨率（含连接建立后的初始值） + SET_STREAM_MODE = 8; // 控制端请求被控端切换屏幕串流模式（WebRTC / 自编码） + REPORT_STREAM_MODE = 9; // 被控端上报当前生效的串流模式 } // 通过 RTCDataChannel 传输的控制指令（protobuf 二进制）。 @@ -48,4 +50,8 @@ message ControlMessage { int32 width = 12; int32 height = 13; int32 fps = 14; + + // 串流模式（SET_STREAM_MODE / REPORT_STREAM_MODE）： + // 0 = WebRTC 全托管；1 = 自编码（自建 MediaCodec 编解码 + video DataChannel 裸流透传） + int32 stream_mode = 15; }