tt/VibeCoding
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 增加预览并远程显示画面

Browse Source
This commit is contained in:
tongtongstudio
2026-07-14 00:23:36 +08:00
parent c3b816415a
commit b2af050e5d
21 changed files with 709 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
WebRTCControlled/app/src/main/AndroidManifest.xml
View File

@@ -1,5 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
android:sharedUserId="android.uid.system">
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
@@ -12,6 +13,7 @@
android:allowBackup="true"
android:icon="@mipmap/ic_launcher"
android:label="@string/app_name"
android:networkSecurityConfig="@xml/network_security_config"
android:supportsRtl="true"
android:theme="@style/Theme.MaterialComponents.DayNight">

75
WebRTCControlled/app/src/main/java/com/tt/controlled/MainActivity.java
View File

@@ -1,12 +1,15 @@
package com.tt.controlled;
import android.Manifest;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.content.ServiceConnection;
import android.content.pm.PackageManager;
import android.media.projection.MediaProjectionManager;
import android.os.Build;
import android.os.Bundle;
import android.os.IBinder;
import android.provider.Settings;
import android.util.Log;
import android.view.View;
@@ -22,6 +25,10 @@ import androidx.core.app.ActivityCompat;
import androidx.core.content.ContextCompat;
import com.tt.controlled.service.ScreenCaptureService;
import com.tt.controlled.webrtc.WebRtcClient;
import org.webrtc.RendererCommon;
import org.webrtc.SurfaceViewRenderer;
import java.util.UUID;
@@ -36,8 +43,28 @@ public class MainActivity extends AppCompatActivity {
private Button btnStart;
private Button btnStop;
private TextView tvStatus;
private SurfaceViewRenderer localView;
private boolean isServiceRunning = false;
private ScreenCaptureService screenCaptureService;
private boolean isBound = false;
private boolean isLocalViewInitialized = false;
private final ServiceConnection serviceConnection = new ServiceConnection() {
@Override
public void onServiceConnected(ComponentName name, IBinder service) {
ScreenCaptureService.LocalBinder binder = (ScreenCaptureService.LocalBinder) service;
screenCaptureService = binder.getService();
isBound = true;
setupLocalPreview();
}
@Override
public void onServiceDisconnected(ComponentName name) {
isBound = false;
screenCaptureService = null;
}
};
@Override
protected void onCreate(Bundle savedInstanceState) {
@@ -49,11 +76,12 @@ public class MainActivity extends AppCompatActivity {
btnStart = findViewById(R.id.btn_start);
btnStop = findViewById(R.id.btn_stop);
tvStatus = findViewById(R.id.tv_status);
localView = findViewById(R.id.local_view);
// 默认服务器地址
etServerUrl.setText("ws://192.168.1.100:8080/ws/signal");
etServerUrl.setText("ws://192.168.100.222:8080/ws/signal");
// 生成设备ID
etDeviceId.setText("controlled_" + UUID.randomUUID().toString().substring(0, 8));
// etDeviceId.setText("controlled_" + UUID.randomUUID().toString().substring(0, 8));
btnStart.setOnClickListener(v -> startScreenSharing());
btnStop.setOnClickListener(v -> stopScreenSharing());
@@ -97,6 +125,8 @@ public class MainActivity extends AppCompatActivity {
startService(serviceIntent);
}
bindService(serviceIntent, serviceConnection, BIND_AUTO_CREATE);
updateUI(true);
Toast.makeText(this, "屏幕共享已启动", Toast.LENGTH_SHORT).show();
} else {
@@ -118,12 +148,53 @@ public class MainActivity extends AppCompatActivity {
}
private void stopScreenSharing() {
if (isBound) {
unbindService(serviceConnection);
isBound = false;
}
Intent serviceIntent = new Intent(this, ScreenCaptureService.class);
stopService(serviceIntent);
if (localView != null) {
localView.release();
isLocalViewInitialized = false;
}
updateUI(false);
Toast.makeText(this, "屏幕共享已停止", Toast.LENGTH_SHORT).show();
}
private void setupLocalPreview() {
if (isBound && screenCaptureService != null && !isLocalViewInitialized) {
WebRtcClient webRtcClient = screenCaptureService.getWebRtcClient();
if (webRtcClient != null && webRtcClient.getEglContext() != null) {
try {
localView.init(webRtcClient.getEglContext(), null);
localView.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT);
localView.setMirror(false);
webRtcClient.setLocalSink(localView);
isLocalViewInitialized = true;
} catch (Exception e) {
Log.e(TAG, "Error initializing local view", e);
}
} else {
// 如果 webRtcClient 还没准备好，稍后重试
localView.postDelayed(this::setupLocalPreview, 500);
}
}
}
@Override
protected void onDestroy() {
super.onDestroy();
if (isBound) {
unbindService(serviceConnection);
isBound = false;
}
if (localView != null) {
localView.release();
isLocalViewInitialized = false;
}
}
private void updateUI(boolean running) {
isServiceRunning = running;
btnStart.setEnabled(!running);

45
WebRTCControlled/app/src/main/java/com/tt/controlled/service/ScreenCaptureService.java
View File

@@ -11,6 +11,7 @@ import android.content.Intent;
import android.content.pm.ServiceInfo;
import android.media.projection.MediaProjection;
import android.media.projection.MediaProjectionManager;
import android.os.Binder;
import android.os.Build;
import android.os.IBinder;
import android.util.DisplayMetrics;
@@ -85,20 +86,50 @@ public class ScreenCaptureService extends Service {
return START_NOT_STICKY;
}
// 获取屏幕尺寸
// 获取屏幕真实尺寸和刷新率
WindowManager wm = (WindowManager) getSystemService(Context.WINDOW_SERVICE);
DisplayMetrics dm = new DisplayMetrics();
wm.getDefaultDisplay().getMetrics(dm);
inputHandler = new InputCommandHandler(dm.widthPixels, dm.heightPixels);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.JELLY_BEAN_MR1) {
wm.getDefaultDisplay().getRealMetrics(dm);
} else {
wm.getDefaultDisplay().getMetrics(dm);
}
int width = dm.widthPixels;
int height = dm.heightPixels;
float refreshRate = 30;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
try {
refreshRate = wm.getDefaultDisplay().getRefreshRate();
} catch (Exception ignored) {}
}
int fps = Math.round(refreshRate);
if (fps > 60) fps = 60; // 限制最大 60fps
if (fps <= 0) fps = 30;
startScreenCapture(resultCode, resultData, serverUrl, deviceId);
Log.i(TAG, "Screen resolution: " + width + "x" + height + " @ " + fps + "fps");
inputHandler = new InputCommandHandler(width, height);
startScreenCapture(resultCode, resultData, serverUrl, deviceId, width, height, fps);
return START_NOT_STICKY;
}
private final IBinder binder = new LocalBinder();
public class LocalBinder extends Binder {
public ScreenCaptureService getService() {
return ScreenCaptureService.this;
}
}
@Nullable
@Override
public IBinder onBind(Intent intent) {
return null;
return binder;
}
public WebRtcClient getWebRtcClient() {
return webRtcClient;
}
@Override
@@ -122,7 +153,7 @@ public class ScreenCaptureService extends Service {
}
}
private void startScreenCapture(int resultCode, Intent resultData, String serverUrl, String deviceId) {
private void startScreenCapture(int resultCode, Intent resultData, String serverUrl, String deviceId, int width, int height, int fps) {
// 初始化 MediaProjection
MediaProjectionManager projectionManager =
(MediaProjectionManager) getSystemService(Context.MEDIA_PROJECTION_SERVICE);
@@ -166,7 +197,7 @@ public class ScreenCaptureService extends Service {
});
webRtcClient.setVideoCapturer(screenCapturer);
webRtcClient.setVideoCapturer(screenCapturer, width, height, fps);
}
private void handleSignalMessage(SignalMessage message) {

3
WebRTCControlled/app/src/main/java/com/tt/controlled/signaling/WebSocketClient.java
View File

@@ -5,6 +5,7 @@ import android.os.Looper;
import android.util.Log;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import okhttp3.OkHttpClient;
import okhttp3.Request;
@@ -18,7 +19,7 @@ public class WebSocketClient {
private final String serverUrl;
private final String deviceId;
private final Gson gson = new Gson();
private final Gson gson = new GsonBuilder().disableHtmlEscaping().create();
private final Handler mainHandler = new Handler(Looper.getMainLooper());
private OkHttpClient client;
private WebSocket webSocket;

174
WebRTCControlled/app/src/main/java/com/tt/controlled/webrtc/WebRtcClient.java
View File

@@ -4,11 +4,13 @@ import android.content.Context;
import android.util.Log;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonObject;
import com.tt.controlled.signaling.SignalMessage;
import com.tt.controlled.signaling.WebSocketClient;
import org.webrtc.DataChannel;
import org.webrtc.DefaultVideoDecoderFactory;
import org.webrtc.DefaultVideoEncoderFactory;
import org.webrtc.EglBase;
import org.webrtc.IceCandidate;
@@ -17,13 +19,17 @@ import org.webrtc.MediaStream;
import org.webrtc.PeerConnection;
import org.webrtc.PeerConnectionFactory;
import org.webrtc.SessionDescription;
import org.webrtc.SurfaceTextureHelper;
import org.webrtc.VideoCapturer;
import org.webrtc.VideoCodecInfo;
import org.webrtc.VideoSink;
import org.webrtc.VideoSource;
import org.webrtc.VideoTrack;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class WebRtcClient {
@@ -36,7 +42,7 @@ public class WebRtcClient {
private final Context context;
private final WebSocketClient wsClient;
private final String deviceId;
private final Gson gson = new Gson();
private final Gson gson = new GsonBuilder().disableHtmlEscaping().create();
private PeerConnectionFactory peerConnectionFactory;
private PeerConnection peerConnection;
@@ -65,30 +71,77 @@ public class WebRtcClient {
this.eglBase = eglBase;
PeerConnectionFactory.InitializationOptions initOptions =
PeerConnectionFactory.InitializationOptions.builder(context)
.setFieldTrials("WebRTC-H264HighProfile/Enabled/")
.setFieldTrials("WebRTC-H264HighProfile/Enabled/WebRTC-Video-HwAcceleration/Enabled/")
.createInitializationOptions();
PeerConnectionFactory.initialize(initOptions);
DefaultVideoEncoderFactory encoderFactory = new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true);
DefaultVideoDecoderFactory decoderFactory = new DefaultVideoDecoderFactory(eglBase.getEglBaseContext());
// 打印所有支持的编码格式
Log.i(TAG, "Supported Encoders:");
for (VideoCodecInfo info : encoderFactory.getSupportedCodecs()) {
Log.i(TAG, " - Codec: " + info.name + ", params: " + info.params);
}
PeerConnectionFactory.Options options = new PeerConnectionFactory.Options();
peerConnectionFactory = PeerConnectionFactory.builder()
.setOptions(options)
.setVideoEncoderFactory(new DefaultVideoEncoderFactory(eglBase.getEglBaseContext(), true, true))
.setVideoEncoderFactory(encoderFactory)
.setVideoDecoderFactory(decoderFactory)
.createPeerConnectionFactory();
}
public void setVideoCapturer(VideoCapturer capturer) {
VideoSource videoSource = peerConnectionFactory.createVideoSource(false);
capturer.initialize(null, context, videoSource.getCapturerObserver());
capturer.startCapture(1280, 720, 30);
public void setVideoCapturer(VideoCapturer capturer, int width, int height, int fps) {
SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread", eglBase.getEglBaseContext());
// 对于屏幕共享，isScreencast 设为 true，这会优化细节保留
VideoSource videoSource = peerConnectionFactory.createVideoSource(true);
capturer.initialize(surfaceTextureHelper, context, videoSource.getCapturerObserver());
capturer.startCapture(width, height, fps);
videoTrack = peerConnectionFactory.createVideoTrack(VIDEO_TRACK_ID, videoSource);
if (localSink != null) {
videoTrack.addSink(localSink);
}
}
private VideoSink localSink;
public void setLocalSink(VideoSink sink) {
this.localSink = sink;
if (videoTrack != null && sink != null) {
videoTrack.addSink(sink);
}
}
public EglBase.Context getEglContext() {
return eglBase != null ? eglBase.getEglBaseContext() : null;
}
public void createPeerConnectionAndAnswer(String offerSdp, String controllerId) {
if (peerConnection != null) {
Log.d(TAG, "Closing previous peer connection");
closeCurrentConnection();
}
this.currentControllerId = controllerId;
PeerConnection.RTCConfiguration config = new PeerConnection.RTCConfiguration(new ArrayList<>());
List<PeerConnection.IceServer> iceServers = new ArrayList<>();
// 建议：如果你有 TURN 服务器，请务必在此处添加，例如：
// iceServers.add(PeerConnection.IceServer.builder("turn:your_turn_server").setUsername("user").setPassword("pass").createIceServer());
iceServers.add(PeerConnection.IceServer.builder("turn:175.178.213.60:3478").setUsername("fanhuitong").setPassword("Fan19961207..").createIceServer());
iceServers.add(PeerConnection.IceServer.builder("turn:192.168.5.224:3478").setUsername("tt").setPassword("fht").createIceServer());
iceServers.add(PeerConnection.IceServer.builder("stun:175.178.213.60:3478").createIceServer());
iceServers.add(PeerConnection.IceServer.builder("stun:192.168.5.224:3478").createIceServer());
iceServers.add(PeerConnection.IceServer.builder("stun:stun.l.google.com:19302").createIceServer());
iceServers.add(PeerConnection.IceServer.builder("stun:stun1.l.google.com:19302").createIceServer());
iceServers.add(PeerConnection.IceServer.builder("stun:stun2.l.google.com:19302").createIceServer());
PeerConnection.RTCConfiguration config = new PeerConnection.RTCConfiguration(iceServers);
config.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN;
config.continualGatheringPolicy = PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY;
config.iceCandidatePoolSize = 10;
// 建议启用，以便在复杂网络下有更好的表现
config.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.ENABLED;
PeerObserver peerObserver = new PeerObserver(new PeerObserver.PeerEventListener() {
@Override
@@ -108,18 +161,18 @@ public class WebRtcClient {
}
@Override
public void onAddStream(MediaStream stream) {}
public void onAddStream(MediaStream stream) {
}
@Override
public void onRemoveStream(MediaStream stream) {}
public void onRemoveStream(MediaStream stream) {
}
});
peerConnection = peerConnectionFactory.createPeerConnection(config, peerObserver);
if (videoTrack != null) {
MediaStream stream = peerConnectionFactory.createLocalMediaStream(STREAM_ID);
stream.addTrack(videoTrack);
peerConnection.addStream(stream);
peerConnection.addTrack(videoTrack, Collections.singletonList(STREAM_ID));
}
// 创建本地 DataChannel
@@ -132,7 +185,8 @@ public class WebRtcClient {
SessionDescription remoteSdp = new SessionDescription(SessionDescription.Type.OFFER, offerSdp);
peerConnection.setRemoteDescription(new AppSdpObserver(new AppSdpObserver.SdpEventListener() {
@Override
public void onCreateSuccess(SessionDescription sdp) {}
public void onCreateSuccess(SessionDescription sdp) {
}
@Override
public void onSetSuccess() {
@@ -143,31 +197,47 @@ public class WebRtcClient {
public void onCreateSuccess(SessionDescription sdp) {
peerConnection.setLocalDescription(new AppSdpObserver(new AppSdpObserver.SdpEventListener() {
@Override
public void onCreateSuccess(SessionDescription s) {}
public void onCreateSuccess(SessionDescription s) {
}
@Override
public void onSetSuccess() {
sendAnswer(sdp.description, controllerId);
// 尝试通过 SDP Munging 提高码率和优化编码
String modifiedSdp = optimizeSdp(sdp.description);
sendAnswer(modifiedSdp, controllerId);
}
@Override
public void onCreateFailure(String error) {}
public void onCreateFailure(String error) {
}
@Override
public void onSetFailure(String error) {}
public void onSetFailure(String error) {
}
}), sdp);
}
@Override
public void onSetSuccess() {}
public void onSetSuccess() {
}
@Override
public void onCreateFailure(String error) {}
public void onCreateFailure(String error) {
}
@Override
public void onSetFailure(String error) {}
public void onSetFailure(String error) {
}
}), constraints);
}
@Override
public void onCreateFailure(String error) {}
public void onCreateFailure(String error) {
}
@Override
public void onSetFailure(String error) {}
public void onSetFailure(String error) {
}
}), remoteSdp);
}
@@ -186,23 +256,35 @@ public class WebRtcClient {
}
public void close() {
closeCurrentConnection();
if (peerConnectionFactory != null) {
peerConnectionFactory.dispose();
peerConnectionFactory = null;
}
}
private void closeCurrentConnection() {
if (dataChannel != null) {
try {
dataChannel.unregisterObserver();
} catch (Exception e) {
}
dataChannel.close();
dataChannel.dispose();
dataChannel = null;
}
if (peerConnection != null) {
peerConnection.close();
peerConnection.dispose();
}
if (peerConnectionFactory != null) {
peerConnectionFactory.dispose();
peerConnection = null;
}
}
private void handleDataChannel(DataChannel dc) {
dc.registerObserver(new DataChannel.Observer() {
@Override
public void onBufferedAmountChange(long previousAmount) {}
public void onBufferedAmountChange(long previousAmount) {
}
@Override
public void onStateChange() {
@@ -222,6 +304,44 @@ public class WebRtcClient {
});
}
private String optimizeSdp(String sdp) {
String[] lines = sdp.split("\r\n");
StringBuilder newSdp = new StringBuilder();
// 优先使用 H264
String h264Payload = null;
// 首先找到 H264 的 payload type
for (String line : lines) {
if (line.startsWith("a=rtpmap:") && line.contains("H264/90000")) {
h264Payload = line.split(":")[1].split(" ")[0];
break;
}
}
for (String line : lines) {
if (line.startsWith("m=video") && h264Payload != null) {
// 将 H264 payload 移到最前面以优先使用硬件加速编码
String[] parts = line.split(" ");
StringBuilder mLine = new StringBuilder(parts[0] + " " + parts[1] + " " + parts[2]);
mLine.append(" ").append(h264Payload);
for (int i = 3; i < parts.length; i++) {
if (!parts[i].equals(h264Payload)) {
mLine.append(" ").append(parts[i]);
}
}
newSdp.append(mLine).append("\r\n");
} else if (h264Payload != null && line.startsWith("a=fmtp:" + h264Payload)) {
// 为 H264 添加起步码率和最大码率限制 (单位: kbps)
// 提高起步码率能显著改善初始画质
newSdp.append(line).append(";x-google-start-bitrate=5000;x-google-max-bitrate=50000;x-google-min-bitrate=2000\r\n");
} else {
newSdp.append(line).append("\r\n");
}
}
return newSdp.toString();
}
private void sendAnswer(String sdp, String controllerId) {
SignalMessage msg = new SignalMessage();
msg.setType("ANSWER");

38
WebRTCControlled/app/src/main/res/layout/activity_main.xml
View File

@@ -8,60 +8,68 @@
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="24dp"
android:text="WebRTC 被控端"
android:textSize="24sp"
android:textStyle="bold"
android:layout_marginBottom="24dp"/>
android:textStyle="bold" />
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="信令服务器地址:"
android:textSize="14sp"/>
android:textSize="14sp" />
<EditText
android:id="@+id/et_server_url"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="textUri"
android:hint="ws://192.168.1.100:8080/ws/signal"
android:layout_marginBottom="16dp"/>
android:layout_marginBottom="16dp"
android:hint="ws://192.168.100.222:8080/ws/signal"
android:inputType="textUri" />
<TextView
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="设备ID:"
android:textSize="14sp"/>
android:textSize="14sp" />
<EditText
android:id="@+id/et_device_id"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:inputType="text"
android:layout_marginBottom="24dp"
android:hint="设备ID"
android:layout_marginBottom="24dp"/>
android:inputType="text"
android:text="981964879" />
<TextView
android:id="@+id/tv_status"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginBottom="24dp"
android:text="状态: 已停止"
android:textSize="16sp"
android:textStyle="bold"
android:layout_marginBottom="24dp"/>
android:textStyle="bold" />
<Button
android:id="@+id/btn_start"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:text="开始屏幕共享"
android:layout_marginBottom="8dp"/>
android:layout_marginBottom="8dp"
android:text="开始屏幕共享" />
<Button
android:id="@+id/btn_stop"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:text="停止屏幕共享"
android:enabled="false"/>
android:layout_marginBottom="16dp"
android:enabled="false"
android:text="停止屏幕共享" />
<org.webrtc.SurfaceViewRenderer
android:id="@+id/local_view"
android:layout_width="match_parent"
android:layout_height="0dp"
android:layout_weight="1" />
</LinearLayout>

8
WebRTCControlled/app/src/main/res/xml/network_security_config.xml Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<network-security-config>
<base-config cleartextTrafficPermitted="true">
<trust-anchors>
<certificates src="system" />
</trust-anchors>
</base-config>
</network-security-config>