tt/VibeCoding
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(streaming): 添加自编码串流模式切换功能

Browse Source
This commit is contained in:
tongtongstudio
2026-07-23 12:01:08 +08:00
parent a1c29246d7
commit e56723a010
20 changed files with 1684 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputCommandHandler.java
View File

@@ -23,7 +23,13 @@ public class InputCommandHandler {
void onResolutionRequested(int width, int height, int fps);
}
/** 控制端请求切换屏幕串流模式（WebRTC / 自编码）时的回调。 */
public interface StreamModeListener {
void onStreamModeRequested(int mode);
}
private ResolutionRequestListener resolutionListener;
private StreamModeListener streamModeListener;
public InputCommandHandler(int screenWidth, int screenHeight) {
this(screenWidth, screenHeight, new SystemInputUtils());
@@ -46,6 +52,10 @@ public class InputCommandHandler {
this.resolutionListener = listener;
}
public void setStreamModeListener(StreamModeListener listener) {
this.streamModeListener = listener;
}
public void handleCommand(byte[] data) {
try {
ControlMessage command = ControlMessage.parseFrom(data);
@@ -76,6 +86,14 @@ public class InputCommandHandler {
Log.w(TAG, "SET_RESOLUTION received but no listener registered");
}
break;
case SET_STREAM_MODE:
// 串流模式切换不依赖输入执行器，转发给屏幕采集服务处理。
if (streamModeListener != null) {
streamModeListener.onStreamModeRequested(command.getStreamMode());
} else {
Log.w(TAG, "SET_STREAM_MODE received but no listener registered");
}
break;
default:
Log.w(TAG, "Unknown action: " + action);
}

127
WebRTCControlled/app/src/main/java/com/ttstd/controlled/service/ScreenCaptureService.java
View File

@@ -40,6 +40,9 @@ import com.ttstd.controlled.signaling.SignalMessage;
import com.ttstd.controlled.utils.AuthSettings;
import com.ttstd.controlled.signaling.WebSocketClient;
import com.ttstd.controlled.webrtc.WebRtcClient;
import com.ttstd.controlled.webrtc.SelfCodecEncoder;
import com.ttstd.control.Action;
import com.ttstd.control.ControlMessage;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
@@ -96,6 +99,13 @@ public class ScreenCaptureService extends Service {
/** 标记服务是否正在关闭，避免重复停止/重复弹窗。 */
private boolean isShuttingDown = false;
// ---- 屏幕串流"自编码"模式 ----
private int resultCode; // 屏幕捕获授权结果码（供自编码模式重新获取 MediaProjection）
private Intent resultDataIntent; // 屏幕捕获授权 Intent（同上）
private int currentStreamMode = SelfCodecEncoder.STREAM_MODE_WEBRTC;
private MediaProjection selfMediaProjection; // 自编码模式独占的 MediaProjection
private SelfCodecEncoder selfEncoder; // 自编码 H264 编码器
@Override
public void onCreate() {
super.onCreate();
@@ -123,6 +133,10 @@ public class ScreenCaptureService extends Service {
String serverUrl = intent.getStringExtra(EXTRA_SERVER_URL);
deviceId = intent.getStringExtra(EXTRA_DEVICE_ID);
// 保存屏幕捕获授权结果，供"自编码"模式重新获取 MediaProjection 使用。
this.resultCode = resultCode;
this.resultDataIntent = resultData;
if (resultCode != Activity.RESULT_OK || resultData == null || serverUrl == null || deviceId == null) {
Log.e(TAG, "Invalid service parameters");
stopSelf();
@@ -186,6 +200,7 @@ public class ScreenCaptureService extends Service {
inputHandler = new InputCommandHandler(realWidth, realHeight, createInputExecutor());
// 控制端发来的 SET_RESOLUTION 指令转交本服务处理（坐标映射与输入执行器无关）。
inputHandler.setResolutionRequestListener(this::onRemoteResolutionRequested);
inputHandler.setStreamModeListener(this::onStreamModeRequested);
// 开启屏幕捕获，使用计算出的 capture 分辨率
startScreenCapture(resultCode, resultData, serverUrl, deviceId, captureWidth, captureHeight, fps);
@@ -317,6 +332,110 @@ public class ScreenCaptureService extends Service {
return new int[]{ w, h };
}
// ---- 屏幕串流"自编码"模式 ----
/** 控制端经 SET_STREAM_MODE 请求切换串流模式。 */
private void onStreamModeRequested(int mode) {
setStreamMode(mode);
}
/**
* 切换屏幕串流模式：
* - {@link SelfCodecEncoder#STREAM_MODE_WEBRTC}：使用 WebRTC 默认媒体轨道（ScreenCapturerAndroid）。
* - {@link SelfCodecEncoder#STREAM_MODE_SELF_CODEC}：暂停 WebRTC 采集，改用自建 MediaCodec 编码，
* 经 DataChannel 的 video_channel 透传裸 H.264 码流，由控制端自建解码器渲染。
*/
public void setStreamMode(int mode) {
if (mode == currentStreamMode) {
return;
}
if (mode == SelfCodecEncoder.STREAM_MODE_SELF_CODEC) {
// 切到自编码：先停 WebRTC 默认屏幕采集（释放其 MediaProjection），再启动自编码。
if (webRtcClient != null) {
webRtcClient.stopCapture();
}
startSelfCodecEncoder();
} else {
// 切回 WebRTC：停止自编码，恢复默认屏幕采集。
stopSelfCodecEncoder();
if (webRtcClient != null) {
webRtcClient.startCapture();
}
}
currentStreamMode = mode;
reportStreamMode(mode);
}
/** 返回当前串流模式。 */
public int getCurrentStreamMode() {
return currentStreamMode;
}
private void startSelfCodecEncoder() {
if (selfEncoder != null) {
Log.w(TAG, "self codec encoder already running");
return;
}
if (webRtcClient == null || !webRtcClient.isVideoDataChannelOpen()) {
Log.w(TAG, "startSelfCodecEncoder: video channel not ready yet");
}
MediaProjectionManager mpm =
(MediaProjectionManager) getSystemService(Context.MEDIA_PROJECTION_SERVICE);
try {
selfMediaProjection = mpm.getMediaProjection(resultCode, resultDataIntent);
} catch (Exception e) {
Log.e(TAG, "getMediaProjection for self codec failed", e);
return;
}
int dpi = getResources().getDisplayMetrics().densityDpi;
int[] res = getCurrentCaptureResolution();
selfEncoder = new SelfCodecEncoder(
selfMediaProjection, res[0], res[1], res[2], dpi,
webRtcClient != null ? webRtcClient.getVideoDataChannel() : null,
new SelfCodecEncoder.Callback() {
@Override
public void onEncoderStarted(int w, int h, int f) {
Log.i(TAG, "self codec encoder started: " + w + "x" + h);
}
@Override
public void onEncoderError(String error) {
Log.e(TAG, "self codec encoder error: " + error);
// 出错时回退到 WebRTC 模式
mainHandler.post(() -> setStreamMode(SelfCodecEncoder.STREAM_MODE_WEBRTC));
}
});
selfEncoder.start();
}
private void stopSelfCodecEncoder() {
if (selfEncoder != null) {
selfEncoder.release();
selfEncoder = null;
}
if (selfMediaProjection != null) {
try {
selfMediaProjection.stop();
} catch (Exception ignored) {
}
selfMediaProjection = null;
}
}
/** 经 control DataChannel 向控制端回报当前生效的串流模式。 */
private void reportStreamMode(int mode) {
if (webRtcClient == null) return;
try {
ControlMessage msg = ControlMessage.newBuilder()
.setAction(Action.REPORT_STREAM_MODE)
.setStreamMode(mode)
.build();
webRtcClient.sendControlCommand(msg);
} catch (Exception e) {
Log.e(TAG, "reportStreamMode failed", e);
}
}
/**
* 状态回调。由已 bind 的 MainActivity 注册，用于把连接状态同步到界面。
*/
@@ -357,6 +476,14 @@ public class ScreenCaptureService extends Service {
screenCapturer.stopCapture();
screenCapturer.dispose();
}
if (selfEncoder != null) {
selfEncoder.release();
selfEncoder = null;
}
if (selfMediaProjection != null) {
selfMediaProjection.stop();
selfMediaProjection = null;
}
if (webRtcClient != null) {
webRtcClient.close();
}

245
WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/SelfCodecEncoder.java Normal file
View File

@@ -0,0 +1,245 @@
package com.ttstd.controlled.webrtc;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.media.projection.MediaProjection;
import android.hardware.display.VirtualDisplay;
import android.hardware.display.DisplayManager;
import android.os.Bundle;
import android.util.Log;
import android.view.Surface;
import org.webrtc.DataChannel;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/**
* 自建屏幕编码通道：用 MediaProjection 的 VirtualDisplay 把屏幕投影到 MediaCodec
* 的输入 Surface，编码为 H.264 裸流后，经 WebRTC 的 video DataChannel 分片透传给
* 控制端（控制端使用自建 MediaCodec 解码渲染）。绕开 WebRTC 默认媒体轨道，使编码/
* 解码参数完全可控（码率、关键帧、低延迟调优）。
*/
public class SelfCodecEncoder {
public static final int STREAM_MODE_WEBRTC = 0; // WebRTC 内置媒体流
public static final int STREAM_MODE_SELF_CODEC = 1; // 自编码（本类）
// ---- 自定义二进制协议（与控制端 SelfCodecDecoder 保持一致）----
private static final byte MAGIC = (byte) 0xAB;
private static final int CHUNK_SIZE = 16384; // 每个 DataChannel 分片大小
private static final int UNIT_TYPE_CONFIG = 1; // SPS/PPS 等参数集
private static final int UNIT_TYPE_FRAME = 2; // 编码帧
private static final String TAG = "SelfCodecEncoder";
private final MediaProjection mediaProjection;
private final int width;
private final int height;
private final int fps;
private final int dpi;
private final DataChannel videoChannel;
private final Callback callback;
private MediaCodec encoder;
private Surface inputSurface;
private VirtualDisplay virtualDisplay;
private boolean running = false;
private int frameSeq = 0;
private final Object sendLock = new Object();
public interface Callback {
void onEncoderStarted(int w, int h, int fps);
void onEncoderError(String error);
}
public SelfCodecEncoder(MediaProjection mp, int width, int height, int fps, int dpi,
DataChannel videoChannel, Callback cb) {
this.mediaProjection = mp;
this.width = width;
this.height = height;
this.fps = fps;
this.dpi = dpi;
this.videoChannel = videoChannel;
this.callback = cb;
}
public void start() {
if (running) return;
try {
MediaFormat format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height);
int bitrate = computeBitrate(width, height, fps);
format.setInteger(MediaFormat.KEY_BIT_RATE, bitrate);
format.setInteger(MediaFormat.KEY_FRAME_RATE, fps);
format.setInteger(MediaFormat.KEY_CAPTURE_RATE, fps);
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
// 关键帧间隔：1 秒一个 IDR，便于断流后快速恢复；也可经 requestKeyFrame 即时强求
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
encoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
inputSurface = encoder.createInputSurface();
encoder.setCallback(encoderCallback);
encoder.start();
virtualDisplay = mediaProjection.createVirtualDisplay(
"SelfCodec", width, height, dpi,
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, inputSurface, null, null);
running = true;
if (callback != null) callback.onEncoderStarted(width, height, fps);
Log.i(TAG, "self codec encoder started: " + width + "x" + height + "@" + fps + ", bitrate=" + bitrate);
} catch (Exception e) {
Log.e(TAG, "start encoder failed", e);
release();
if (callback != null) callback.onEncoderError(e.getMessage());
}
}
private int computeBitrate(int w, int h, int fps) {
int b = (int) (w * h * fps * 0.12);
return Math.max(500_000, Math.min(8_000_000, b));
}
private final MediaCodec.Callback encoderCallback = new MediaCodec.Callback() {
@Override
public void onInputBufferAvailable(MediaCodec mc, int index) {
// 使用 Surface 输入，编码器自动从输入 Surface 取帧，无需手动喂数据
}
@Override
public void onOutputFormatChanged(MediaCodec mc, MediaFormat format) {
// 主流设备在此给出 SPS/PPS，作为 CONFIG 单元发送
sendCsds(format);
}
@Override
public void onOutputBufferAvailable(MediaCodec mc, int index, MediaCodec.BufferInfo info) {
try {
ByteBuffer buf = mc.getOutputBuffer(index);
if (buf == null) {
mc.releaseOutputBuffer(index, false);
return;
}
byte[] data = new byte[info.size];
buf.position(info.offset);
buf.get(data);
boolean isKey = (info.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0;
boolean isConfig = (info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0;
long pts = info.presentationTimeUs;
mc.releaseOutputBuffer(index, false);
if (isConfig) {
sendUnit(UNIT_TYPE_CONFIG, 0, false, data);
} else {
sendUnit(UNIT_TYPE_FRAME, pts, isKey, data);
}
} catch (Exception e) {
Log.e(TAG, "onOutputBufferAvailable error", e);
}
}
@Override
public void onError(MediaCodec mc, MediaCodec.CodecException e) {
Log.e(TAG, "encoder error", e);
if (callback != null) callback.onEncoderError(e.getMessage());
}
};
private void sendCsds(MediaFormat format) {
if (format == null) return;
ByteBuffer sps = format.getByteBuffer("csd-0");
ByteBuffer pps = format.getByteBuffer("csd-1");
if (sps != null) sendUnit(UNIT_TYPE_CONFIG, 0, false, toBytes(sps));
if (pps != null) sendUnit(UNIT_TYPE_CONFIG, 0, false, toBytes(pps));
}
private static byte[] toBytes(ByteBuffer b) {
ByteBuffer d = b.duplicate();
byte[] out = new byte[d.remaining()];
d.get(out);
return out;
}
/** 组装一个"单元"（类型 + pts + isKey + 数据），再切片经 videoChannel 发送。 */
private void sendUnit(byte unitType, long pts, boolean isKey, byte[] data) {
ByteBuffer unit = ByteBuffer.allocate(1 + 4 + 1 + 4 + data.length).order(ByteOrder.BIG_ENDIAN);
unit.put(unitType);
unit.putInt((int) (pts & 0xFFFFFFFFL));
unit.put((byte) (isKey ? 1 : 0));
unit.putInt(data.length);
unit.put(data);
byte[] unitBytes = unit.array();
int totalChunks = (unitBytes.length + CHUNK_SIZE - 1) / CHUNK_SIZE;
if (totalChunks == 0) totalChunks = 1;
int seq;
synchronized (sendLock) {
seq = frameSeq++;
}
for (int i = 0; i < totalChunks; i++) {
int off = i * CHUNK_SIZE;
int len = Math.min(CHUNK_SIZE, unitBytes.length - off);
byte[] chunk = new byte[len];
System.arraycopy(unitBytes, off, chunk, 0, len);
sendChunk(seq, (short) totalChunks, (short) i, chunk);
}
}
private void sendChunk(int seq, short total, short idx, byte[] chunk) {
if (videoChannel == null || videoChannel.state() != DataChannel.State.OPEN) return;
ByteBuffer msg = ByteBuffer.allocate(1 + 4 + 2 + 2 + 4 + chunk.length).order(ByteOrder.BIG_ENDIAN);
msg.put(MAGIC);
msg.putInt(seq);
msg.putShort(total);
msg.putShort(idx);
msg.putInt(chunk.length);
msg.put(chunk);
msg.flip();
try {
videoChannel.send(new DataChannel.Buffer(msg, true));
} catch (Exception e) {
Log.e(TAG, "send video chunk failed", e);
}
}
/** 即时请求一个关键帧（强求 IDR）。 */
public void requestKeyFrame() {
if (encoder == null) return;
try {
Bundle params = new Bundle();
params.putInt(MediaCodec.PARAMETER_KEY_REQUEST_SYNC_FRAME, 0);
encoder.setParameters(params);
} catch (Exception e) {
Log.w(TAG, "requestKeyFrame failed", e);
}
}
public void release() {
running = false;
if (virtualDisplay != null) {
try {
virtualDisplay.release();
} catch (Exception ignored) {
}
virtualDisplay = null;
}
if (encoder != null) {
try {
encoder.stop();
} catch (Exception ignored) {
}
try {
encoder.release();
} catch (Exception ignored) {
}
encoder = null;
}
if (inputSurface != null) {
inputSurface.release();
inputSurface = null;
}
}
}

68
WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/WebRtcClient.java
View File

@@ -40,6 +40,7 @@ public class WebRtcClient {
private static final String VIDEO_TRACK_ID = "screen_track";
private static final String STREAM_ID = "screen_stream";
private static final String DATA_CHANNEL_LABEL = "control_channel";
private static final String VIDEO_CHANNEL_LABEL = "video_channel"; // 自编码视频透传通道（控制端创建）
// 码率优化参数（单位 kbps）。
private static final int MIN_BITRATE_KBPS = 1000; // 1 Mbps
@@ -54,6 +55,7 @@ public class WebRtcClient {
private PeerConnectionFactory peerConnectionFactory;
private PeerConnection peerConnection;
private DataChannel dataChannel;
private DataChannel videoDataChannel; // 自编码视频透传（控制端创建的 video_channel）
private VideoSource videoSource;
private VideoTrack videoTrack;
private org.webrtc.RtpSender videoSender;
@@ -419,6 +421,17 @@ public class WebRtcClient {
dataChannel.dispose();
dataChannel = null;
}
if (videoDataChannel != null) {
try {
videoDataChannel.close();
} catch (Exception e) {
}
try {
videoDataChannel.dispose();
} catch (Exception e) {
}
videoDataChannel = null;
}
if (peerConnection != null) {
peerConnection.close();
peerConnection.dispose();
@@ -426,7 +439,62 @@ public class WebRtcClient {
}
}
// ---- 屏幕串流"自编码"模式相关 ----
/** 返回控制端建立的 video DataChannel（用于自编码视频透传），可能为空。 */
public DataChannel getVideoDataChannel() {
return videoDataChannel;
}
/** video DataChannel 是否已打开。 */
public boolean isVideoDataChannelOpen() {
return videoDataChannel != null && videoDataChannel.state() == DataChannel.State.OPEN;
}
/** 暂停 WebRTC 默认屏幕采集（切到自编码模式时调用）。 */
public void stopCapture() {
if (videoCapturer != null) {
try {
videoCapturer.stopCapture();
} catch (InterruptedException e) {
Log.w(TAG, "stopCapture interrupted", e);
}
}
}
/** 恢复 WebRTC 默认屏幕采集（从自编码模式切回时调用）。 */
public void startCapture() {
if (videoCapturer != null) {
try {
videoCapturer.startCapture(videoWidth, videoHeight, videoFps);
} catch (RuntimeException e) {
Log.e(TAG, "startCapture failed", e);
}
}
}
/** 经 control DataChannel 发送控制类 protobuf 消息（如串流模式回报）。 */
public void sendControlCommand(ControlMessage message) {
if (dataChannel == null || dataChannel.state() != DataChannel.State.OPEN) {
Log.d(TAG, "sendControlCommand skipped: dataChannel not open");
return;
}
try {
ByteBuffer buffer = ByteBuffer.wrap(message.toByteArray());
dataChannel.send(new DataChannel.Buffer(buffer, true));
} catch (Exception e) {
Log.e(TAG, "sendControlCommand failed", e);
}
}
private void handleDataChannel(DataChannel dc) {
// 区分控制端建立的两条通道：video_channel 用于本端自编码视频透传（仅发送），
// 其余（control_channel）用于指令收发。
if (VIDEO_CHANNEL_LABEL.equals(dc.label())) {
this.videoDataChannel = dc;
Log.d(TAG, "Video data channel received: " + dc.label());
return;
}
// 保存控制端建立的通道引用（替代原先本端自建的通道）。
this.dataChannel = dc;
dc.registerObserver(new DataChannel.Observer() {

3
WebRTCControlled/app/src/main/proto/control_message.proto
View File

@@ -15,6 +15,8 @@ enum Action {
MOTION_EVENT = 5;
SET_RESOLUTION = 6; // 控制端请求被控端切换屏幕采集分辨率
REPORT_RESOLUTION = 7; // 被控端上报当前实际采集分辨率（含连接建立后的初始值）
SET_STREAM_MODE = 8; // 控制端请求被控端切换屏幕串流模式
REPORT_STREAM_MODE = 9; // 被控端上报当前生效的串流模式
}
// 通过 RTCDataChannel 传输的控制指令（protobuf 二进制）。
@@ -48,4 +50,5 @@ message ControlMessage {
int32 width = 12;
int32 height = 13;
int32 fps = 14;
int32 stream_mode = 15; // 串流模式：0=WebRTC 内置媒体流，1=自编码(MediaCodec 硬编 + DataChannel 透传)
}