tt/VibeCoding
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复自编码模式黑屏及并发问题

Browse Source 
- 修复 MediaCodec 回调需在 configure 前设置导致的异常
- 修复解码器线程安全问题及包乱序/重复处理
- 增加关键帧前重发 SPS/PPS 以防首帧丢失导致黑屏
- 支持分辨率切换时重置解码器
- 修复 Flutter 控制端连接类型选项 UI 显示拥挤问题
This commit is contained in:
TongTongStudio
2026-07-24 09:31:58 +08:00
parent f8048e1331
commit 778c37b8db
9 changed files with 374 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/SelfCodecDecoder.java
View File

@@ -2,6 +2,7 @@ package com.ttstd.controller.webrtc;
import android.media.MediaCodec;
import android.media.MediaFormat;
import android.os.Build;
import android.util.Log;
import android.view.Surface;
@@ -33,6 +34,10 @@ public class SelfCodecDecoder {
private boolean configured = false;
private boolean enabled = false;
// 实际宽高（由被控端 REPORT_RESOLUTION 上报），若未上报则使用 PLACEHOLDER
private int videoWidth = PLACEHOLDER_W;
private int videoHeight = PLACEHOLDER_H;
// 待喂入的参数集（SPS/PPS 等），以 CODEC_CONFIG 形式输入解码器
private final ArrayDeque<byte[]> pendingConfigs = new ArrayDeque<>();
// 待解码帧
@@ -67,6 +72,23 @@ public class SelfCodecDecoder {
}
}
/**
* 更新当前视频流的实际宽高（由外部 WebRtcClient 收到被控端上报后调用）。
* 如果宽高发生变化且解码器已配置，则重置解码器以应用新分辨率，避免黑屏或花屏。
*/
public void updateResolution(int w, int h) {
if (w <= 0 || h <= 0) return;
if (this.videoWidth != w || this.videoHeight != h) {
Log.i(TAG, "Resolution update: " + videoWidth + "x" + videoHeight + " -> " + w + "x" + h);
this.videoWidth = w;
this.videoHeight = h;
if (configured) {
// 分辨率变化时需重置解码器以重新 configure
releaseDecoder();
}
}
}
/** 收到 video DataChannel 的二进制消息（一条分片）。 */
public void onBinaryMessage(ByteBuffer raw) {
if (!enabled) return;
@@ -76,6 +98,11 @@ public class SelfCodecDecoder {
byte magic = b.get();
if (magic != MAGIC) return;
int seq = b.getInt();
// 关键修复：增加包序号过期检查（处理包乱序或重复）。
// 实时视频流中，如果收到了序号更旧的包，直接丢弃以保证延迟和状态一致性。
if (lastSeq != -1 && seq <= lastSeq && (lastSeq - seq) < 1000) {
return;
}
short total = b.getShort();
short idx = b.getShort();
int len = b.getInt();
@@ -83,30 +110,32 @@ public class SelfCodecDecoder {
byte[] chunk = new byte[len];
b.get(chunk);
ChunkBuffer cb = assembling.get(seq);
if (cb == null) {
cb = new ChunkBuffer();
cb.total = total;
cb.chunks = new byte[total][];
cb.received = 0;
assembling.put(seq, cb);
if (assembling.size() > 24) {
int min = Integer.MAX_VALUE;
for (int k : assembling.keySet()) {
if (k < min) min = k;
synchronized (assembling) {
ChunkBuffer cb = assembling.get(seq);
if (cb == null) {
cb = new ChunkBuffer();
cb.total = total;
cb.chunks = new byte[total][];
cb.received = 0;
assembling.put(seq, cb);
if (assembling.size() > 24) {
int min = Integer.MAX_VALUE;
for (int k : assembling.keySet()) {
if (k < min) min = k;
}
assembling.remove(min);
}
assembling.remove(min);
}
}
if (idx >= 0 && idx < total && cb.chunks[idx] == null) {
cb.chunks[idx] = chunk;
cb.received++;
}
if (cb.received == total) {
assembling.remove(seq);
if (seq > lastSeq) lastSeq = seq;
byte[] unit = join(cb);
if (unit != null) handleUnit(unit);
if (idx >= 0 && idx < total && cb.chunks[idx] == null) {
cb.chunks[idx] = chunk;
cb.received++;
}
if (cb.received == total) {
assembling.remove(seq);
if (seq > lastSeq) lastSeq = seq;
byte[] unit = join(cb);
if (unit != null) handleUnit(unit);
}
}
}
@@ -132,14 +161,25 @@ public class SelfCodecDecoder {
b.get(data);
if (type == UNIT_TYPE_CONFIG) {
if (configured) {
// 已配置：忽略重复/初始参数集，避免重复以 CODEC_CONFIG 喂入报错
return;
}
synchronized (pendingConfigs) {
pendingConfigs.add(data);
// 仅在尚未包含该配置时添加，避免关键帧前的重复重发堆积。
// 关键：不因为 configured 而 return，确保 SPS 和 PPS 都能被加入队列。
boolean exists = false;
for (byte[] c : pendingConfigs) {
if (java.util.Arrays.equals(c, data)) {
exists = true;
break;
}
}
if (!exists) {
pendingConfigs.add(data);
}
}
if (!configured) {
tryConfigure();
} else {
feed(); // 已配置时也尝试喂入，以应对中途可能的参数集更新
}
tryConfigure();
} else if (type == UNIT_TYPE_FRAME) {
Frame f = new Frame();
f.data = data;
@@ -160,29 +200,50 @@ public class SelfCodecDecoder {
}
try {
MediaFormat format = MediaFormat.createVideoFormat(
MediaFormat.MIMETYPE_VIDEO_AVC, PLACEHOLDER_W, PLACEHOLDER_H);
MediaFormat.MIMETYPE_VIDEO_AVC, videoWidth, videoHeight);
// 关键：对部分芯片组（如 MTK/Exynos），低延迟标志有助于减少首帧黑屏时间（API 30+）
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
format.setInteger(MediaFormat.KEY_LOW_LATENCY, 1);
}
decoder = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
decoder.configure(format, surface, null, 0);
// 异步回调必须在 configure() 之前设置，否则抛 IllegalStateException，
// 解码器将永远无法进入工作状态（黑屏）。
decoder.setCallback(decoderCallback);
decoder.configure(format, surface, null, 0);
decoder.start();
configured = true;
Log.i(TAG, "self codec decoder configured");
feed();
} catch (Exception e) {
Log.e(TAG, "decoder configure failed", e);
// 清理半初始化的解码器，等待下一个 CONFIG 到达后重试
if (decoder != null) {
try {
decoder.release();
} catch (Exception ignored) {
}
decoder = null;
}
configured = false;
}
}
private final MediaCodec.Callback decoderCallback = new MediaCodec.Callback() {
@Override
public void onInputBufferAvailable(MediaCodec mc, int index) {
inputQueue.add(index);
synchronized (inputQueue) {
inputQueue.add(index);
}
feed();
}
@Override
public void onOutputBufferAvailable(MediaCodec mc, int index, MediaCodec.BufferInfo info) {
mc.releaseOutputBuffer(index, true);
try {
mc.releaseOutputBuffer(index, true);
} catch (IllegalStateException e) {
Log.w(TAG, "onOutputBufferAvailable: codec might be stopped/released", e);
}
}
@Override
@@ -199,42 +260,51 @@ public class SelfCodecDecoder {
private void feed() {
if (decoder == null || !configured) return;
synchronized (pendingConfigs) {
while (!pendingConfigs.isEmpty() && !inputQueue.isEmpty()) {
byte[] cfg = pendingConfigs.poll();
int idx = inputQueue.poll();
try {
ByteBuffer ib = decoder.getInputBuffer(idx);
ib.clear();
ib.put(cfg);
decoder.queueInputBuffer(idx, 0, cfg.length, 0, MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
} catch (Exception e) {
Log.e(TAG, "queue config failed", e);
pendingConfigs.addFirst(cfg);
inputQueue.addFirst(idx);
break;
synchronized (inputQueue) {
while (!pendingConfigs.isEmpty() && !inputQueue.isEmpty()) {
byte[] cfg = pendingConfigs.poll();
int idx = inputQueue.poll();
try {
ByteBuffer ib = decoder.getInputBuffer(idx);
ib.clear();
ib.put(cfg);
decoder.queueInputBuffer(idx, 0, cfg.length, 0, MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
} catch (Exception e) {
Log.e(TAG, "queue config failed", e);
pendingConfigs.addFirst(cfg);
inputQueue.addFirst(idx);
break;
}
}
}
}
synchronized (frameQueue) {
while (!frameQueue.isEmpty() && !inputQueue.isEmpty()) {
Frame f = frameQueue.poll();
int idx = inputQueue.poll();
try {
ByteBuffer ib = decoder.getInputBuffer(idx);
ib.clear();
ib.put(f.data);
decoder.queueInputBuffer(idx, 0, f.data.length, f.pts, f.flags);
} catch (Exception e) {
Log.e(TAG, "queue frame failed", e);
frameQueue.addFirst(f);
inputQueue.addFirst(idx);
break;
synchronized (inputQueue) {
while (!frameQueue.isEmpty() && !inputQueue.isEmpty()) {
Frame f = frameQueue.poll();
int idx = inputQueue.poll();
try {
ByteBuffer ib = decoder.getInputBuffer(idx);
ib.clear();
ib.put(f.data);
decoder.queueInputBuffer(idx, 0, f.data.length, f.pts, f.flags);
} catch (Exception e) {
Log.e(TAG, "queue frame failed", e);
frameQueue.addFirst(f);
inputQueue.addFirst(idx);
break;
}
}
}
}
}
private void releaseDecoder() {
configured = false;
synchronized (assembling) {
lastSeq = -1; // 释放时重置序号，避免重连后首帧被误判为过期
assembling.clear();
}
if (decoder != null) {
try {
decoder.stop();
@@ -246,11 +316,15 @@ public class SelfCodecDecoder {
}
decoder = null;
}
configured = false;
pendingConfigs.clear();
frameQueue.clear();
inputQueue.clear();
assembling.clear();
synchronized (pendingConfigs) {
pendingConfigs.clear();
}
synchronized (frameQueue) {
frameQueue.clear();
}
synchronized (inputQueue) {
inputQueue.clear();
}
}
public void release() {

17
WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/WebRtcClient.java
View File

@@ -318,6 +318,15 @@ public class WebRtcClient {
}
private void handleDataChannel(DataChannel dc) {
if (dc == null) return;
// 关键修复：区分控制端创建的两条通道。
// 虽然本端是创建方，但 PeerObserver.onDataChannel 仍可能在某些 WebRTC 协商逻辑中被触发。
if (VIDEO_CHANNEL_LABEL.equals(dc.label())) {
this.videoDataChannel = dc;
setupVideoDataChannel(dc);
return;
}
this.dataChannel = dc;
dc.registerObserver(new DataChannel.Observer() {
@Override
public void onBufferedAmountChange(long previousAmount) {}
@@ -337,14 +346,18 @@ public class WebRtcClient {
if (buffer == null || buffer.data == null) return;
byte[] data = new byte[buffer.data.remaining()];
buffer.data.get(data);
// 若收到被控端回报的串流模式，单独处理。
// 收到被控端回报的消息，按 Action 分发处理。
try {
ControlMessage msg = ControlMessage.parseFrom(data);
if (msg.getAction() == Action.REPORT_STREAM_MODE) {
if (streamModeReportListener != null) {
streamModeReportListener.onStreamModeReported(msg.getStreamMode());
}
return;
} else if (msg.getAction() == Action.REPORT_RESOLUTION) {
Log.i(TAG, "Received resolution report: " + msg.getWidth() + "x" + msg.getHeight());
if (selfCodecDecoder != null) {
selfCodecDecoder.updateResolution(msg.getWidth(), msg.getHeight());
}
}
} catch (Exception ignored) {
}