tt/VibeCoding
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 优化不跟手的问题

Browse Source
This commit is contained in:
tongtongstudio
2026-07-14 19:49:01 +08:00
parent 590d14c85e
commit 5cc47f9fb3
8 changed files with 156 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputCommandHandler.java
View File

@@ -42,6 +42,9 @@ public class InputCommandHandler {
case "LONG_PRESS":
handleLongPress(command);
break;
case "MOTION_EVENT":
handleMotionEvent(command);
break;
default:
Log.w(TAG, "Unknown action: " + action);
}
@@ -50,6 +53,15 @@ public class InputCommandHandler {
}
}
private void handleMotionEvent(JsonObject command) {
int action = command.get("motionAction").getAsInt();
float relX = command.get("x").getAsFloat();
float relY = command.get("y").getAsFloat();
int x = (int) (relX * screenWidth);
int y = (int) (relY * screenHeight);
inputExecutor.injectMotionEvent(action, x, y);
}
private void handleTouch(JsonObject command) {
float relX = command.get("x").getAsFloat();
float relY = command.get("y").getAsFloat();

1
WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputExecutor.java
View File

@@ -8,4 +8,5 @@ public interface InputExecutor {
void injectLongPress(int x, int y);
void injectSwipe(int x1, int y1, int x2, int y2, long duration);
void injectKeyEvent(int keyCode);
void injectMotionEvent(int action, int x, int y);
}

8
WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/ShellInputUtils.java
View File

@@ -29,6 +29,14 @@ public class ShellInputUtils implements InputExecutor {
executeShellCommand(String.format(Locale.US, "input keyevent %d", keyCode));
}
@Override
public void injectMotionEvent(int action, int x, int y) {
// Shell 命令不支持高效的连续手势追踪，这里仅作兼容处理
if (action == 0) { // ACTION_DOWN
injectTap(x, y);
}
}
private void executeShellCommand(String command) {
try {
Log.d(TAG, "Executing shell command: " + command);

10
WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/SystemInputUtils.java
View File

@@ -19,6 +19,7 @@ public class SystemInputUtils implements InputExecutor {
private Object mInputManager;
private Method mInjectInputEventMethod;
private static final int INJECT_INPUT_EVENT_MODE_WAIT_FOR_FINISH = 2;
private long mDownTime;
public SystemInputUtils() {
try {
@@ -73,6 +74,15 @@ public class SystemInputUtils implements InputExecutor {
injectKeyEvent(now, SystemClock.uptimeMillis(), KeyEvent.ACTION_UP, keyCode);
}
@Override
public void injectMotionEvent(int action, int x, int y) {
long now = SystemClock.uptimeMillis();
if (action == MotionEvent.ACTION_DOWN) {
mDownTime = now;
}
injectMotionEvent(mDownTime, now, action, x, y);
}
private void injectMotionEvent(long downTime, long eventTime, int action, float x, float y) {
MotionEvent event = MotionEvent.obtain(downTime, eventTime, action, x, y, 0);
event.setSource(InputDevice.SOURCE_TOUCHSCREEN);

30
WebRTCControlled/app/src/main/java/com/ttstd/controlled/service/ScreenCaptureService.java
View File

@@ -93,9 +93,23 @@ public class ScreenCaptureService extends Service {
} else {
wm.getDefaultDisplay().getMetrics(dm);
}
int width = dm.widthPixels;
int height = dm.heightPixels;
int realWidth = dm.widthPixels;
int realHeight = dm.heightPixels;
// 分辨率缩放逻辑：如果设备分辨率大于 1080p，则等比例缩小到 1080p 以内
int captureWidth = realWidth;
int captureHeight = realHeight;
int maxResolution = 1920; // 1080p 的长边
if (Math.max(realWidth, realHeight) > maxResolution) {
float scale = (float) maxResolution / Math.max(realWidth, realHeight);
captureWidth = Math.round(realWidth * scale);
captureHeight = Math.round(realHeight * scale);
// 确保是偶数，部分编码器要求宽高为偶数
if (captureWidth % 2 != 0) captureWidth--;
if (captureHeight % 2 != 0) captureHeight--;
}
float refreshRate = 30;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
try {
@@ -106,10 +120,14 @@ public class ScreenCaptureService extends Service {
if (fps > 60) fps = 60; // 限制最大 60fps
if (fps <= 0) fps = 30;
Log.i(TAG, "Screen resolution: " + width + "x" + height + " @ " + fps + "fps");
inputHandler = new InputCommandHandler(width, height);
Log.i(TAG, "Real resolution: " + realWidth + "x" + realHeight);
Log.i(TAG, "Capture resolution: " + captureWidth + "x" + captureHeight + " @ " + fps + "fps");
startScreenCapture(resultCode, resultData, serverUrl, deviceId, width, height, fps);
// 输入处理器必须使用真实分辨率进行坐标映射
inputHandler = new InputCommandHandler(realWidth, realHeight);
// 开启屏幕捕获，使用计算出的 capture 分辨率
startScreenCapture(resultCode, resultData, serverUrl, deviceId, captureWidth, captureHeight, fps);
return START_NOT_STICKY;
}

90
WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/WebRtcClient.java
View File

@@ -208,6 +208,11 @@ public class WebRtcClient {
peerConnection.createAnswer(new AppSdpObserver(new AppSdpObserver.SdpEventListener() {
@Override
public void onCreateSuccess(SessionDescription sdp) {
// 【关键修复】在 setLocalDescription 之前修改 SDP
// 这样本地和远端都会统一使用优化后的 H264 配置
String modifiedSdpDescription = optimizeSdp(sdp.description);
SessionDescription modifiedSdp = new SessionDescription(sdp.type, modifiedSdpDescription);
peerConnection.setLocalDescription(new AppSdpObserver(new AppSdpObserver.SdpEventListener() {
@Override
public void onCreateSuccess(SessionDescription s) {
@@ -215,19 +220,20 @@ public class WebRtcClient {
@Override
public void onSetSuccess() {
// 尝试通过 SDP Munging 提高码率和优化编码
String modifiedSdp = optimizeSdp(sdp.description);
sendAnswer(modifiedSdp, controllerId);
Log.e(TAG, "onSetSuccess: modifiedSdp = " + modifiedSdpDescription);
sendAnswer(modifiedSdpDescription, controllerId);
}
@Override
public void onCreateFailure(String error) {
Log.e(TAG, "setLocalDescription onCreateFailure: " + error);
}
@Override
public void onSetFailure(String error) {
Log.e(TAG, "setLocalDescription onSetFailure: " + error);
}
}), sdp);
}), modifiedSdp);
}
@Override
@@ -236,6 +242,7 @@ public class WebRtcClient {
@Override
public void onCreateFailure(String error) {
Log.e(TAG, "createAnswer onCreateFailure: " + error);
}
@Override
@@ -250,6 +257,7 @@ public class WebRtcClient {
@Override
public void onSetFailure(String error) {
Log.e(TAG, "setRemoteDescription onSetFailure: " + error);
}
}), remoteSdp);
}
@@ -318,38 +326,68 @@ public class WebRtcClient {
}
private String optimizeSdp(String sdp) {
String[] lines = sdp.split("\r\n");
// 使用 \n 分割以提高兼容性，并处理可能的 \r
String[] lines = sdp.split("\n");
StringBuilder newSdp = new StringBuilder();
// 优先使用 H264
String h264Payload = null;
// 首先找到 H264 的 payload type
List<String> h264Payloads = new ArrayList<>();
// 首先找到所有的 H264 payload type (可能包含多个 profile)
for (String line : lines) {
if (line.startsWith("a=rtpmap:") && line.contains("H264/90000")) {
h264Payload = line.split(":")[1].split(" ")[0];
break;
String trimmedLine = line.trim();
if (trimmedLine.startsWith("a=rtpmap:") && trimmedLine.contains("H264/90000")) {
String payload = trimmedLine.split(":")[1].split(" ")[0];
h264Payloads.add(payload);
}
}
if (h264Payloads.isEmpty()) {
return sdp;
}
for (String line : lines) {
if (line.startsWith("m=video") && h264Payload != null) {
// 将 H264 payload 移到最前面以优先使用硬件加速编码
String[] parts = line.split(" ");
StringBuilder mLine = new StringBuilder(parts[0] + " " + parts[1] + " " + parts[2]);
mLine.append(" ").append(h264Payload);
for (int i = 3; i < parts.length; i++) {
if (!parts[i].equals(h264Payload)) {
mLine.append(" ").append(parts[i]);
String trimmedLine = line.trim();
if (trimmedLine.isEmpty()) continue;
if (trimmedLine.startsWith("m=video")) {
// 将所有 H264 payload 移到最前面以优先使用
String[] parts = trimmedLine.split(" ");
if (parts.length > 3) {
StringBuilder mLine = new StringBuilder(parts[0] + " " + parts[1] + " " + parts[2]);
// 先添加所有的 H264 payload
for (String h264 : h264Payloads) {
mLine.append(" ").append(h264);
}
// 再添加其余的 payload
for (int i = 3; i < parts.length; i++) {
if (!h264Payloads.contains(parts[i])) {
mLine.append(" ").append(parts[i]);
}
}
newSdp.append(mLine).append("\r\n");
} else {
newSdp.append(trimmedLine).append("\r\n");
}
} else if (trimmedLine.startsWith("a=fmtp:")) {
boolean isH264 = false;
for (String h264 : h264Payloads) {
if (trimmedLine.startsWith("a=fmtp:" + h264)) {
isH264 = true;
break;
}
}
newSdp.append(mLine).append("\r\n");
} else if (h264Payload != null && line.startsWith("a=fmtp:" + h264Payload)) {
// 为 H264 添加起步码率和最大码率限制 (单位: kbps)
// 提高起步码率能显著改善初始画质
newSdp.append(line).append(";x-google-start-bitrate=5000;x-google-max-bitrate=50000;x-google-min-bitrate=2000\r\n");
if (isH264) {
// 为 H264 注入码率优化参数
String bonus = ";x-google-start-bitrate=5000;x-google-max-bitrate=50000;x-google-min-bitrate=2000";
if (!trimmedLine.contains("x-google-start-bitrate")) {
newSdp.append(trimmedLine).append(bonus).append("\r\n");
} else {
newSdp.append(trimmedLine).append("\r\n");
}
} else {
newSdp.append(trimmedLine).append("\r\n");
}
} else {
newSdp.append(line).append("\r\n");
newSdp.append(trimmedLine).append("\r\n");
}
}
return newSdp.toString();

23
WebRTCController/app/src/main/java/com/ttstd/controller/MainActivity.java
View File

@@ -111,12 +111,12 @@ public class MainActivity extends AppCompatActivity {
touchOverlay.setTouchEventListener(new RemoteTouchView.TouchEventListener() {
@Override
public void onTouch(float relX, float relY) {
sendTouchCommand(relX, relY);
// 已通过 onMotionEvent 处理
}
@Override
public void onSwipe(float relX1, float relY1, float relX2, float relY2, long duration) {
sendSwipeCommand(relX1, relY1, relX2, relY2, duration);
// 已通过 onMotionEvent 处理
}
@Override
@@ -126,7 +126,13 @@ public class MainActivity extends AppCompatActivity {
@Override
public void onLongPress(float relX, float relY) {
sendLongPressCommand(relX, relY);
// 如果需要高层级的长按，可以保留，但 raw 模式下远程系统会自动识别
// sendLongPressCommand(relX, relY);
}
@Override
public void onMotionEvent(int action, float relX, float relY) {
sendMotionEventCommand(action, relX, relY);
}
});
@@ -361,6 +367,17 @@ public class MainActivity extends AppCompatActivity {
}
}
private void sendMotionEventCommand(int action, float relX, float relY) {
if (webRtcClient != null) {
JsonObject cmd = new JsonObject();
cmd.addProperty("action", "MOTION_EVENT");
cmd.addProperty("motionAction", action);
cmd.addProperty("x", relX);
cmd.addProperty("y", relY);
webRtcClient.sendControlCommand(cmd.toString());
}
}
private void disconnect() {
if (webRtcClient != null) {
webRtcClient.close();

28
WebRTCController/app/src/main/java/com/ttstd/controller/view/RemoteTouchView.java
View File

@@ -25,8 +25,6 @@ public class RemoteTouchView extends SurfaceView {
private float lastTouchX;
private float lastTouchY;
private float lastAbsX;
private float lastAbsY;
private long swipeStartTime;
private boolean isLongPressed = false;
private GestureDetector gestureDetector;
@@ -36,6 +34,7 @@ public class RemoteTouchView extends SurfaceView {
void onSwipe(float relX1, float relY1, float relX2, float relY2, long duration);
void onKeyEvent(int keyCode, int action);
void onLongPress(float relX, float relY);
void onMotionEvent(int action, float relX, float relY);
}
public RemoteTouchView(Context context) {
@@ -101,7 +100,6 @@ public class RemoteTouchView extends SurfaceView {
float absX = event.getX();
float absY = event.getY();
Log.d(TAG, "onTouchEvent: absX=" + absX + ", absY=" + absY);
float relX = absX / getWidth();
float relY = absY / getHeight();
@@ -110,18 +108,20 @@ public class RemoteTouchView extends SurfaceView {
relX = Math.max(0, Math.min(1, relX));
relY = Math.max(0, Math.min(1, relY));
switch (event.getActionMasked()) {
int action = event.getActionMasked();
// 发送原始 MotionEvent 实现实时跟手
listener.onMotionEvent(action, relX, relY);
switch (action) {
case MotionEvent.ACTION_DOWN:
isLongPressed = false;
lastTouchX = relX;
lastTouchY = relY;
lastAbsX = absX;
lastAbsY = absY;
swipeStartTime = System.currentTimeMillis();
return true;
case MotionEvent.ACTION_MOVE:
// 仅返回 true 以持续接收事件，不更新起始坐标，确保 onSwipe 计算的是从按下点开始的位移
return true;
case MotionEvent.ACTION_UP:
@@ -132,14 +132,11 @@ public class RemoteTouchView extends SurfaceView {
float dx = Math.abs(relX - lastTouchX);
float dy = Math.abs(relY - lastTouchY);
// 保留旧的回调以防万一，但 MainActivity 现在应该忽略它们
if (duration < 200 && dx < 0.02 && dy < 0.02) {
// 点击事件
listener.onTouch(lastTouchX, lastTouchY);
Log.d(TAG, "Touch at: " + lastTouchX + ", " + lastTouchY + " (abs: " + absX + ", " + absY + ")");
} else {
// 滑动事件
listener.onSwipe(lastTouchX, lastTouchY, relX, relY, duration);
Log.d(TAG, "Swipe from (" + lastTouchX + "," + lastTouchY + ") [abs: " + lastAbsX + "," + lastAbsY + "] to (" + relX + "," + relY + ") [abs: " + absX + "," + absY + "]");
}
return true;
}
@@ -173,4 +170,13 @@ public class RemoteTouchView extends SurfaceView {
cmd.addProperty("y", relY);
return cmd.toString();
}
public String createMotionEventCommand(int action, float relX, float relY) {
JsonObject cmd = new JsonObject();
cmd.addProperty("action", "MOTION_EVENT");
cmd.addProperty("motionAction", action);
cmd.addProperty("x", relX);
cmd.addProperty("y", relY);
return cmd.toString();
}
}