feat(streaming): 添加自编码串流模式切换功能

WebRTCController/app/src/main/java/com/ttstd/controller/MainActivity.java

@@ -16,6 +16,10 @@ import android.widget.AdapterView;
 import android.widget.ArrayAdapter;
 import android.widget.Spinner;
 import android.widget.RadioGroup;
+import android.widget.Switch;
+import android.view.SurfaceView;
+import android.view.SurfaceHolder;
+import com.ttstd.controller.webrtc.SelfCodecDecoder;
 import androidx.appcompat.app.AlertDialog;
 
 import java.util.ArrayList;
@@ -58,6 +62,14 @@ public class MainActivity extends AppCompatActivity {
     private TextView tvStats;
     private Spinner spinnerResolution;
 
+    // 屏幕串流"自编码"模式相关 UI 与解码器
+    private Switch switchStreamMode;
+    private TextView tvStreamMode;
+    private SurfaceView selfCodecSurface;
+    private SelfCodecDecoder selfCodecDecoder;
+    private SurfaceHolder.Callback surfaceCallback;
+    private boolean programmaticSwitch = false;
+
     private WebSocketClient wsClient;
     private WebRtcClient webRtcClient;
     private EglBase eglBase;
@@ -162,6 +174,8 @@ public class MainActivity extends AppCompatActivity {
 
         setupResolutionSpinner();
         updateUI(false);
+
+        setupSelfCodecUi();
     }
 
     private void adjustVideoSize(int videoWidth, int videoHeight, int rotation) {
@@ -309,6 +323,8 @@ public class MainActivity extends AppCompatActivity {
     private void initWebRtcAndConnect() {
         webRtcClient = new WebRtcClient(this, wsClient, myDeviceId);
         webRtcClient.initialize(eglBase);
+        webRtcClient.setSelfCodecDecoder(selfCodecDecoder);
+        webRtcClient.setStreamModeReportListener(mode -> runOnUiThread(() -> syncStreamModeUi(mode)));
         webRtcClient.setConnectionListener(new WebRtcClient.ConnectionListener() {
             @Override
             public void onConnectionEstablished() {
@@ -598,8 +614,83 @@ public class MainActivity extends AppCompatActivity {
 
         if (connected) {
             statsHandler.post(statsRunnable);
+            if (switchStreamMode != null) switchStreamMode.setEnabled(true);
         } else {
             statsHandler.removeCallbacks(statsRunnable);
+            if (switchStreamMode != null) {
+                switchStreamMode.setEnabled(false);
+                programmaticSwitch = true;
+                switchStreamMode.setChecked(false);
+                programmaticSwitch = false;
+            }
+            if (tvStreamMode != null) tvStreamMode.setText("串流: WebRTC");
+            if (selfCodecDecoder != null) selfCodecDecoder.release();
+        }
+    }
+
+    /** 初始化"自编码"串流模式的 UI 与解码器。 */
+    private void setupSelfCodecUi() {
+        switchStreamMode = findViewById(R.id.switch_stream_mode);
+        tvStreamMode = findViewById(R.id.tv_stream_mode);
+        selfCodecSurface = findViewById(R.id.self_codec_surface);
+        selfCodecDecoder = new SelfCodecDecoder();
+        surfaceCallback = new SurfaceHolder.Callback() {
+            @Override
+            public void surfaceCreated(SurfaceHolder holder) {
+                if (selfCodecDecoder != null) {
+                    selfCodecDecoder.setSurface(holder.getSurface());
+                }
+            }
+
+            @Override
+            public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+            }
+
+            @Override
+            public void surfaceDestroyed(SurfaceHolder holder) {
+            }
+        };
+        selfCodecSurface.getHolder().addCallback(surfaceCallback);
+        switchStreamMode.setEnabled(false);
+        switchStreamMode.setOnCheckedChangeListener((buttonView, isChecked) -> {
+            if (programmaticSwitch) return;
+            applyStreamMode(isChecked);
+        });
+    }
+
+    /** 用户切换串流模式：发送信令并切换本地视图。 */
+    private void applyStreamMode(boolean self) {
+        if (webRtcClient != null && webRtcClient.isDataChannelOpen()) {
+            webRtcClient.sendStreamMode(self ? WebRtcClient.STREAM_MODE_SELF_CODEC
+                    : WebRtcClient.STREAM_MODE_WEBRTC);
+        }
+        tvStreamMode.setText(self ? "串流: 自编码" : "串流: WebRTC");
+        if (self) {
+            selfCodecSurface.setVisibility(View.VISIBLE);
+            remoteVideoView.setVisibility(View.GONE);
+            selfCodecDecoder.setEnabled(true);
+        } else {
+            selfCodecSurface.setVisibility(View.GONE);
+            remoteVideoView.setVisibility(View.VISIBLE);
+            selfCodecDecoder.setEnabled(false);
+        }
+    }
+
+    /** 收到被控端回报的当前模式：同步 UI 与视图（防止切换失败不一致）。 */
+    private void syncStreamModeUi(int mode) {
+        boolean self = (mode == WebRtcClient.STREAM_MODE_SELF_CODEC);
+        programmaticSwitch = true;
+        switchStreamMode.setChecked(self);
+        programmaticSwitch = false;
+        tvStreamMode.setText(self ? "串流: 自编码" : "串流: WebRTC");
+        if (self) {
+            selfCodecSurface.setVisibility(View.VISIBLE);
+            remoteVideoView.setVisibility(View.GONE);
+            selfCodecDecoder.setEnabled(true);
+        } else {
+            selfCodecSurface.setVisibility(View.GONE);
+            remoteVideoView.setVisibility(View.VISIBLE);
+            selfCodecDecoder.setEnabled(false);
         }
     }
 

WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/SelfCodecDecoder.java

@@ -0,0 +1,260 @@
+package com.ttstd.controller.webrtc;
+
+import android.media.MediaCodec;
+import android.media.MediaFormat;
+import android.util.Log;
+import android.view.Surface;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayDeque;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * 自建屏幕解码通道：从 WebRTC 的 video DataChannel 收取被控端发来的 H.264 裸流分片，
+ * 重组后使用 MediaCodec 解码并渲染到指定 Surface。与 WebRTC 默认媒体轨道解耦，
+ * 配合被控端 SelfCodecEncoder 使用。
+ */
+public class SelfCodecDecoder {
+
+    private static final String TAG = "SelfCodecDecoder";
+
+    // ---- 必须与被控端 SelfCodecEncoder 保持一致 ----
+    private static final byte MAGIC = (byte) 0xAB;
+    private static final int CHUNK_SIZE = 16384;
+    private static final int UNIT_TYPE_CONFIG = 1;
+    private static final int UNIT_TYPE_FRAME = 2;
+    private static final int PLACEHOLDER_W = 1920;
+    private static final int PLACEHOLDER_H = 1080;
+
+    private MediaCodec decoder;
+    private Surface surface;
+    private boolean configured = false;
+    private boolean enabled = false;
+
+    // 待喂入的参数集（SPS/PPS 等），以 CODEC_CONFIG 形式输入解码器
+    private final ArrayDeque<byte[]> pendingConfigs = new ArrayDeque<>();
+    // 待解码帧
+    private final ArrayDeque<Frame> frameQueue = new ArrayDeque<>();
+    // 解码器可用的输入缓冲下标
+    private final ArrayDeque<Integer> inputQueue = new ArrayDeque<>();
+    // 分片重组：frameSeq -> 该单元的若干分片
+    private final Map<Integer, ChunkBuffer> assembling = new HashMap<>();
+    private int lastSeq = -1;
+
+    private static class Frame {
+        byte[] data;
+        long pts;
+        int flags;
+    }
+
+    private static class ChunkBuffer {
+        int total;
+        byte[][] chunks;
+        int received;
+    }
+
+    public void setSurface(Surface s) {
+        this.surface = s;
+        tryConfigure();
+    }
+
+    public void setEnabled(boolean e) {
+        enabled = e;
+        if (!e) {
+            releaseDecoder();
+        }
+    }
+
+    /** 收到 video DataChannel 的二进制消息（一条分片）。 */
+    public void onBinaryMessage(ByteBuffer raw) {
+        if (!enabled) return;
+        if (raw == null) return;
+        ByteBuffer b = raw.duplicate();
+        if (b.remaining() < 13) return;
+        byte magic = b.get();
+        if (magic != MAGIC) return;
+        int seq = b.getInt();
+        short total = b.getShort();
+        short idx = b.getShort();
+        int len = b.getInt();
+        if (b.remaining() < len) return;
+        byte[] chunk = new byte[len];
+        b.get(chunk);
+
+        ChunkBuffer cb = assembling.get(seq);
+        if (cb == null) {
+            cb = new ChunkBuffer();
+            cb.total = total;
+            cb.chunks = new byte[total][];
+            cb.received = 0;
+            assembling.put(seq, cb);
+            if (assembling.size() > 24) {
+                int min = Integer.MAX_VALUE;
+                for (int k : assembling.keySet()) {
+                    if (k < min) min = k;
+                }
+                assembling.remove(min);
+            }
+        }
+        if (idx >= 0 && idx < total && cb.chunks[idx] == null) {
+            cb.chunks[idx] = chunk;
+            cb.received++;
+        }
+        if (cb.received == total) {
+            assembling.remove(seq);
+            if (seq > lastSeq) lastSeq = seq;
+            byte[] unit = join(cb);
+            if (unit != null) handleUnit(unit);
+        }
+    }
+
+    private byte[] join(ChunkBuffer cb) {
+        int totalLen = 0;
+        for (byte[] c : cb.chunks) {
+            if (c != null) totalLen += c.length;
+        }
+        ByteBuffer out = ByteBuffer.allocate(totalLen);
+        for (byte[] c : cb.chunks) {
+            if (c != null) out.put(c);
+        }
+        return out.array();
+    }
+
+    private void handleUnit(byte[] unit) {
+        ByteBuffer b = ByteBuffer.wrap(unit);
+        byte type = b.get();
+        long pts = b.getInt() & 0xFFFFFFFFL;
+        byte isKey = b.get();
+        int len = b.getInt();
+        byte[] data = new byte[len];
+        b.get(data);
+
+        if (type == UNIT_TYPE_CONFIG) {
+            if (configured) {
+                // 已配置：忽略重复/初始参数集，避免重复以 CODEC_CONFIG 喂入报错
+                return;
+            }
+            synchronized (pendingConfigs) {
+                pendingConfigs.add(data);
+            }
+            tryConfigure();
+        } else if (type == UNIT_TYPE_FRAME) {
+            Frame f = new Frame();
+            f.data = data;
+            f.pts = pts;
+            f.flags = (isKey != 0) ? MediaCodec.BUFFER_FLAG_KEY_FRAME : 0;
+            synchronized (frameQueue) {
+                frameQueue.add(f);
+            }
+            feed();
+        }
+    }
+
+    private void tryConfigure() {
+        if (configured) return;
+        if (surface == null) return;
+        synchronized (pendingConfigs) {
+            if (pendingConfigs.isEmpty()) return;
+        }
+        try {
+            MediaFormat format = MediaFormat.createVideoFormat(
+                    MediaFormat.MIMETYPE_VIDEO_AVC, PLACEHOLDER_W, PLACEHOLDER_H);
+            decoder = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
+            decoder.configure(format, surface, null, 0);
+            decoder.setCallback(decoderCallback);
+            decoder.start();
+            configured = true;
+            Log.i(TAG, "self codec decoder configured");
+            feed();
+        } catch (Exception e) {
+            Log.e(TAG, "decoder configure failed", e);
+        }
+    }
+
+    private final MediaCodec.Callback decoderCallback = new MediaCodec.Callback() {
+        @Override
+        public void onInputBufferAvailable(MediaCodec mc, int index) {
+            inputQueue.add(index);
+            feed();
+        }
+
+        @Override
+        public void onOutputBufferAvailable(MediaCodec mc, int index, MediaCodec.BufferInfo info) {
+            mc.releaseOutputBuffer(index, true);
+        }
+
+        @Override
+        public void onError(MediaCodec mc, MediaCodec.CodecException e) {
+            Log.e(TAG, "decoder error", e);
+        }
+
+        @Override
+        public void onOutputFormatChanged(MediaCodec mc, MediaFormat format) {
+            Log.d(TAG, "decoder output format changed");
+        }
+    };
+
+    private void feed() {
+        if (decoder == null || !configured) return;
+        synchronized (pendingConfigs) {
+            while (!pendingConfigs.isEmpty() && !inputQueue.isEmpty()) {
+                byte[] cfg = pendingConfigs.poll();
+                int idx = inputQueue.poll();
+                try {
+                    ByteBuffer ib = decoder.getInputBuffer(idx);
+                    ib.clear();
+                    ib.put(cfg);
+                    decoder.queueInputBuffer(idx, 0, cfg.length, 0, MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
+                } catch (Exception e) {
+                    Log.e(TAG, "queue config failed", e);
+                    pendingConfigs.addFirst(cfg);
+                    inputQueue.addFirst(idx);
+                    break;
+                }
+            }
+        }
+        synchronized (frameQueue) {
+            while (!frameQueue.isEmpty() && !inputQueue.isEmpty()) {
+                Frame f = frameQueue.poll();
+                int idx = inputQueue.poll();
+                try {
+                    ByteBuffer ib = decoder.getInputBuffer(idx);
+                    ib.clear();
+                    ib.put(f.data);
+                    decoder.queueInputBuffer(idx, 0, f.data.length, f.pts, f.flags);
+                } catch (Exception e) {
+                    Log.e(TAG, "queue frame failed", e);
+                    frameQueue.addFirst(f);
+                    inputQueue.addFirst(idx);
+                    break;
+                }
+            }
+        }
+    }
+
+    private void releaseDecoder() {
+        if (decoder != null) {
+            try {
+                decoder.stop();
+            } catch (Exception ignored) {
+            }
+            try {
+                decoder.release();
+            } catch (Exception ignored) {
+            }
+            decoder = null;
+        }
+        configured = false;
+        pendingConfigs.clear();
+        frameQueue.clear();
+        inputQueue.clear();
+        assembling.clear();
+    }
+
+    public void release() {
+        releaseDecoder();
+        surface = null;
+    }
+}

WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/WebRtcClient.java

@@ -35,7 +35,10 @@ import java.util.List;
 public class WebRtcClient {
 
     private static final String TAG = "WebRtcClient";
-    private static final String DATA_CHANNEL_LABEL = "control_channel";
+    public static final String DATA_CHANNEL_LABEL = "control_channel";
+    public static final String VIDEO_CHANNEL_LABEL = "video_channel"; // 自编码视频透传通道
+    public static final int STREAM_MODE_WEBRTC = 0;     // WebRTC 内置媒体流
+    public static final int STREAM_MODE_SELF_CODEC = 1;  // 自编码（MediaCodec + DataChannel）
 
     private final Context context;
     private final WebSocketClient wsClient;
@@ -47,6 +50,9 @@ public class WebRtcClient {
     private PeerConnectionFactory peerConnectionFactory;
     private PeerConnection peerConnection;
     private DataChannel dataChannel;
+    private DataChannel videoDataChannel; // 自编码视频透传通道（本端创建）
+    private SelfCodecDecoder selfCodecDecoder;
+    private StreamModeReportListener streamModeReportListener;
     private EglBase eglBase;
     private String currentControlledId;
     private SurfaceViewRenderer remoteSurfaceView;
@@ -177,6 +183,14 @@ public class WebRtcClient {
         dataChannel = peerConnection.createDataChannel(DATA_CHANNEL_LABEL, dcInit);
         handleDataChannel(dataChannel);
 
+        // 新建"自编码"视频透传通道（无序、不可靠，以最低延迟传输裸 H.264 码流）。
+        // 必须在 setLocalDescription 之前创建，才会进入 SDP。
+        DataChannel.Init vInit = new DataChannel.Init();
+        vInit.ordered = false;
+        vInit.maxRetransmits = 0;
+        videoDataChannel = peerConnection.createDataChannel(VIDEO_CHANNEL_LABEL, vInit);
+        setupVideoDataChannel(videoDataChannel);
+
         // 保存远端视频渲染器
         this.remoteSurfaceView = remoteRenderer;
 
@@ -270,6 +284,10 @@ public class WebRtcClient {
             dataChannel.close();
             dataChannel.dispose();
         }
+        if (videoDataChannel != null) {
+            videoDataChannel.close();
+            videoDataChannel.dispose();
+        }
         if (peerConnection != null) {
             peerConnection.close();
             peerConnection.dispose();
@@ -300,13 +318,65 @@ public class WebRtcClient {
 
             @Override
             public void onMessage(DataChannel.Buffer buffer) {
+                if (buffer == null || buffer.data == null) return;
                 byte[] data = new byte[buffer.data.remaining()];
                 buffer.data.get(data);
-                Log.d(TAG, "DataChannel message received, bytes=" + data.length);
+                // 若收到被控端回报的串流模式，单独处理。
+                try {
+                    ControlMessage msg = ControlMessage.parseFrom(data);
+                    if (msg.getAction() == Action.REPORT_STREAM_MODE) {
+                        if (streamModeReportListener != null) {
+                            streamModeReportListener.onStreamModeReported(msg.getStreamMode());
+                        }
+                        return;
+                    }
+                } catch (Exception ignored) {
+                }
             }
         });
     }
 
+    private void setupVideoDataChannel(DataChannel dc) {
+        dc.registerObserver(new DataChannel.Observer() {
+            @Override
+            public void onBufferedAmountChange(long previousAmount) {
+            }
+
+            @Override
+            public void onStateChange() {
+                Log.d(TAG, "Video DataChannel state: " + dc.state());
+            }
+
+            @Override
+            public void onMessage(DataChannel.Buffer buffer) {
+                if (selfCodecDecoder != null && buffer != null && buffer.data != null) {
+                    selfCodecDecoder.onBinaryMessage(buffer.data);
+                }
+            }
+        });
+    }
+
+    public void setSelfCodecDecoder(SelfCodecDecoder decoder) {
+        this.selfCodecDecoder = decoder;
+    }
+
+    public void setStreamModeReportListener(StreamModeReportListener l) {
+        this.streamModeReportListener = l;
+    }
+
+    public boolean isVideoDataChannelOpen() {
+        return videoDataChannel != null && videoDataChannel.state() == DataChannel.State.OPEN;
+    }
+
+    /** 请求被控端切换屏幕串流模式（WebRTC / 自编码）。 */
+    public void sendStreamMode(int mode) {
+        ControlMessage msg = ControlMessage.newBuilder()
+                .setAction(Action.SET_STREAM_MODE)
+                .setStreamMode(mode)
+                .build();
+        sendControlCommand(msg);
+    }
+
     private void sendOffer(String sdp, String controlledId) {
         Log.e(TAG, "sendOffer: " );
         SignalMessage msg = new SignalMessage();

WebRTCController/app/src/main/proto/control_message.proto

@@ -15,6 +15,8 @@ enum Action {
   MOTION_EVENT = 5;
   SET_RESOLUTION = 6; // 控制端请求被控端切换屏幕采集分辨率
   REPORT_RESOLUTION = 7; // 被控端上报当前实际采集分辨率（含连接建立后的初始值）
+  SET_STREAM_MODE = 8;      // 控制端请求被控端切换屏幕串流模式
+  REPORT_STREAM_MODE = 9;   // 被控端上报当前生效的串流模式
 }
 
 // 通过 RTCDataChannel 传输的控制指令（protobuf 二进制）。
@@ -48,4 +50,5 @@ message ControlMessage {
   int32 width = 12;
   int32 height = 13;
   int32 fps = 14;
+  int32 stream_mode = 15; // 串流模式：0=WebRTC 内置媒体流，1=自编码(MediaCodec 硬编 + DataChannel 透传)
 }

WebRTCController/app/src/main/res/layout/activity_main.xml

@@ -97,11 +97,20 @@
         android:visibility="gone">
 
         <!-- 远端视频渲染 -->
-        <org.webrtc.SurfaceViewRenderer
-            android:id="@+id/remote_video_view"
-            android:layout_width="match_parent"
-            android:layout_height="match_parent"
-            android:layout_gravity="center" />
+    <org.webrtc.SurfaceViewRenderer
+        android:id="@+id/remote_video_view"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:layout_gravity="center" />
+
+    <!-- 自编码（自建 MediaCodec）屏幕串流渲染层，默认隐藏，切到自编码模式时显示 -->
+    <SurfaceView
+        android:id="@+id/self_codec_surface"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:layout_gravity="center"
+        android:visibility="gone" />
+
 
         <!-- 触摸控制层（覆盖在视频上方） -->
         <com.ttstd.controller.view.RemoteTouchView
@@ -137,12 +146,30 @@
                 android:textSize="12sp" />
 
             <Spinner
-                android:id="@+id/spinner_resolution"
+                android:id="@+rid/spinner_resolution"
                 android:layout_width="wrap_content"
                 android:layout_height="wrap_content"
                 android:layout_marginTop="4dp"
                 android:backgroundTint="#80FFFFFF" />
 
+            <Switch
+                android:id="@+rid/switch_stream_mode"
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:layout_marginTop="6dp"
+                android:text="自编码串流"
+                android:textColor="@android:color/white"
+                android:textSize="12sp" />
+
+            <TextView
+                android:id="@+rid/tv_stream_mode"
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:layout_marginTop="2dp"
+                android:text="串流: WebRTC"
+                android:textColor="@android:color/white"
+                android:textSize="12sp" />
+
         </LinearLayout>
 
     </FrameLayout>