feat: 优化不跟手的问题

WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputCommandHandler.java

@@ -42,6 +42,9 @@ public class InputCommandHandler {
                 case "LONG_PRESS":
                     handleLongPress(command);
                     break;
+                case "MOTION_EVENT":
+                    handleMotionEvent(command);
+                    break;
                 default:
                     Log.w(TAG, "Unknown action: " + action);
             }
@@ -50,6 +53,15 @@ public class InputCommandHandler {
         }
     }
 
+    private void handleMotionEvent(JsonObject command) {
+        int action = command.get("motionAction").getAsInt();
+        float relX = command.get("x").getAsFloat();
+        float relY = command.get("y").getAsFloat();
+        int x = (int) (relX * screenWidth);
+        int y = (int) (relY * screenHeight);
+        inputExecutor.injectMotionEvent(action, x, y);
+    }
+
     private void handleTouch(JsonObject command) {
         float relX = command.get("x").getAsFloat();
         float relY = command.get("y").getAsFloat();

WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputExecutor.java

@@ -8,4 +8,5 @@ public interface InputExecutor {
     void injectLongPress(int x, int y);
     void injectSwipe(int x1, int y1, int x2, int y2, long duration);
     void injectKeyEvent(int keyCode);
+    void injectMotionEvent(int action, int x, int y);
 }

WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/ShellInputUtils.java

@@ -29,6 +29,14 @@ public class ShellInputUtils implements InputExecutor {
         executeShellCommand(String.format(Locale.US, "input keyevent %d", keyCode));
     }
 
+    @Override
+    public void injectMotionEvent(int action, int x, int y) {
+        // Shell 命令不支持高效的连续手势追踪，这里仅作兼容处理
+        if (action == 0) { // ACTION_DOWN
+            injectTap(x, y);
+        }
+    }
+
     private void executeShellCommand(String command) {
         try {
             Log.d(TAG, "Executing shell command: " + command);

WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/SystemInputUtils.java

@@ -19,6 +19,7 @@ public class SystemInputUtils implements InputExecutor {
     private Object mInputManager;
     private Method mInjectInputEventMethod;
     private static final int INJECT_INPUT_EVENT_MODE_WAIT_FOR_FINISH = 2;
+    private long mDownTime;
 
     public SystemInputUtils() {
         try {
@@ -73,6 +74,15 @@ public class SystemInputUtils implements InputExecutor {
         injectKeyEvent(now, SystemClock.uptimeMillis(), KeyEvent.ACTION_UP, keyCode);
     }
 
+    @Override
+    public void injectMotionEvent(int action, int x, int y) {
+        long now = SystemClock.uptimeMillis();
+        if (action == MotionEvent.ACTION_DOWN) {
+            mDownTime = now;
+        }
+        injectMotionEvent(mDownTime, now, action, x, y);
+    }
+
     private void injectMotionEvent(long downTime, long eventTime, int action, float x, float y) {
         MotionEvent event = MotionEvent.obtain(downTime, eventTime, action, x, y, 0);
         event.setSource(InputDevice.SOURCE_TOUCHSCREEN);

WebRTCControlled/app/src/main/java/com/ttstd/controlled/service/ScreenCaptureService.java

@@ -93,8 +93,22 @@ public class ScreenCaptureService extends Service {
         } else {
             wm.getDefaultDisplay().getMetrics(dm);
         }
-        int width = dm.widthPixels;
+        int realWidth = dm.widthPixels;
-        int height = dm.heightPixels;
+        int realHeight = dm.heightPixels;
+
+        // 分辨率缩放逻辑：如果设备分辨率大于 1080p，则等比例缩小到 1080p 以内
+        int captureWidth = realWidth;
+        int captureHeight = realHeight;
+        int maxResolution = 1920; // 1080p 的长边
+
+        if (Math.max(realWidth, realHeight) > maxResolution) {
+            float scale = (float) maxResolution / Math.max(realWidth, realHeight);
+            captureWidth = Math.round(realWidth * scale);
+            captureHeight = Math.round(realHeight * scale);
+            // 确保是偶数，部分编码器要求宽高为偶数
+            if (captureWidth % 2 != 0) captureWidth--;
+            if (captureHeight % 2 != 0) captureHeight--;
+        }
 
         float refreshRate = 30;
         if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
@@ -106,10 +120,14 @@ public class ScreenCaptureService extends Service {
         if (fps > 60) fps = 60; // 限制最大 60fps
         if (fps <= 0) fps = 30;
 
-        Log.i(TAG, "Screen resolution: " + width + "x" + height + " @ " + fps + "fps");
+        Log.i(TAG, "Real resolution: " + realWidth + "x" + realHeight);
-        inputHandler = new InputCommandHandler(width, height);
+        Log.i(TAG, "Capture resolution: " + captureWidth + "x" + captureHeight + " @ " + fps + "fps");
 
-        startScreenCapture(resultCode, resultData, serverUrl, deviceId, width, height, fps);
+        // 输入处理器必须使用真实分辨率进行坐标映射
+        inputHandler = new InputCommandHandler(realWidth, realHeight);
+
+        // 开启屏幕捕获，使用计算出的 capture 分辨率
+        startScreenCapture(resultCode, resultData, serverUrl, deviceId, captureWidth, captureHeight, fps);
         return START_NOT_STICKY;
     }
 

WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/WebRtcClient.java

@@ -208,6 +208,11 @@ public class WebRtcClient {
                 peerConnection.createAnswer(new AppSdpObserver(new AppSdpObserver.SdpEventListener() {
                     @Override
                     public void onCreateSuccess(SessionDescription sdp) {
+                        // 【关键修复】在 setLocalDescription 之前修改 SDP
+                        // 这样本地和远端都会统一使用优化后的 H264 配置
+                        String modifiedSdpDescription = optimizeSdp(sdp.description);
+                        SessionDescription modifiedSdp = new SessionDescription(sdp.type, modifiedSdpDescription);
+
                         peerConnection.setLocalDescription(new AppSdpObserver(new AppSdpObserver.SdpEventListener() {
                             @Override
                             public void onCreateSuccess(SessionDescription s) {
@@ -215,19 +220,20 @@ public class WebRtcClient {
 
                             @Override
                             public void onSetSuccess() {
-                                // 尝试通过 SDP Munging 提高码率和优化编码
+                                Log.e(TAG, "onSetSuccess: modifiedSdp = " + modifiedSdpDescription);
-                                String modifiedSdp = optimizeSdp(sdp.description);
+                                sendAnswer(modifiedSdpDescription, controllerId);
-                                sendAnswer(modifiedSdp, controllerId);
                             }
 
                             @Override
                             public void onCreateFailure(String error) {
+                                Log.e(TAG, "setLocalDescription onCreateFailure: " + error);
                             }
 
                             @Override
                             public void onSetFailure(String error) {
+                                Log.e(TAG, "setLocalDescription onSetFailure: " + error);
                             }
-                        }), sdp);
+                        }), modifiedSdp);
                     }
 
                     @Override
@@ -236,6 +242,7 @@ public class WebRtcClient {
 
                     @Override
                     public void onCreateFailure(String error) {
+                        Log.e(TAG, "createAnswer onCreateFailure: " + error);
                     }
 
                     @Override
@@ -250,6 +257,7 @@ public class WebRtcClient {
 
             @Override
             public void onSetFailure(String error) {
+                Log.e(TAG, "setRemoteDescription onSetFailure: " + error);
             }
         }), remoteSdp);
     }
@@ -318,38 +326,68 @@ public class WebRtcClient {
     }
 
     private String optimizeSdp(String sdp) {
-        String[] lines = sdp.split("\r\n");
+        // 使用 \n 分割以提高兼容性，并处理可能的 \r
+        String[] lines = sdp.split("\n");
         StringBuilder newSdp = new StringBuilder();
 
-        // 优先使用 H264
+        List<String> h264Payloads = new ArrayList<>();
-        String h264Payload = null;
+        // 首先找到所有的 H264 payload type (可能包含多个 profile)
-
-        // 首先找到 H264 的 payload type
         for (String line : lines) {
-            if (line.startsWith("a=rtpmap:") && line.contains("H264/90000")) {
+            String trimmedLine = line.trim();
-                h264Payload = line.split(":")[1].split(" ")[0];
+            if (trimmedLine.startsWith("a=rtpmap:") && trimmedLine.contains("H264/90000")) {
-                break;
+                String payload = trimmedLine.split(":")[1].split(" ")[0];
+                h264Payloads.add(payload);
             }
         }
 
+        if (h264Payloads.isEmpty()) {
+            return sdp;
+        }
+
         for (String line : lines) {
-            if (line.startsWith("m=video") && h264Payload != null) {
+            String trimmedLine = line.trim();
-                // 将 H264 payload 移到最前面以优先使用硬件加速编码
+            if (trimmedLine.isEmpty()) continue;
-                String[] parts = line.split(" ");
+
+            if (trimmedLine.startsWith("m=video")) {
+                // 将所有 H264 payload 移到最前面以优先使用
+                String[] parts = trimmedLine.split(" ");
+                if (parts.length > 3) {
                     StringBuilder mLine = new StringBuilder(parts[0] + " " + parts[1] + " " + parts[2]);
-                mLine.append(" ").append(h264Payload);
+                    // 先添加所有的 H264 payload
+                    for (String h264 : h264Payloads) {
+                        mLine.append(" ").append(h264);
+                    }
+                    // 再添加其余的 payload
                     for (int i = 3; i < parts.length; i++) {
-                    if (!parts[i].equals(h264Payload)) {
+                        if (!h264Payloads.contains(parts[i])) {
                             mLine.append(" ").append(parts[i]);
                         }
                     }
                     newSdp.append(mLine).append("\r\n");
-            } else if (h264Payload != null && line.startsWith("a=fmtp:" + h264Payload)) {
-                // 为 H264 添加起步码率和最大码率限制 (单位: kbps)
-                // 提高起步码率能显著改善初始画质
-                newSdp.append(line).append(";x-google-start-bitrate=5000;x-google-max-bitrate=50000;x-google-min-bitrate=2000\r\n");
                 } else {
-                newSdp.append(line).append("\r\n");
+                    newSdp.append(trimmedLine).append("\r\n");
+                }
+            } else if (trimmedLine.startsWith("a=fmtp:")) {
+                boolean isH264 = false;
+                for (String h264 : h264Payloads) {
+                    if (trimmedLine.startsWith("a=fmtp:" + h264)) {
+                        isH264 = true;
+                        break;
+                    }
+                }
+                if (isH264) {
+                    // 为 H264 注入码率优化参数
+                    String bonus = ";x-google-start-bitrate=5000;x-google-max-bitrate=50000;x-google-min-bitrate=2000";
+                    if (!trimmedLine.contains("x-google-start-bitrate")) {
+                        newSdp.append(trimmedLine).append(bonus).append("\r\n");
+                    } else {
+                        newSdp.append(trimmedLine).append("\r\n");
+                    }
+                } else {
+                    newSdp.append(trimmedLine).append("\r\n");
+                }
+            } else {
+                newSdp.append(trimmedLine).append("\r\n");
             }
         }
         return newSdp.toString();

WebRTCController/app/src/main/java/com/ttstd/controller/MainActivity.java

@@ -111,12 +111,12 @@ public class MainActivity extends AppCompatActivity {
         touchOverlay.setTouchEventListener(new RemoteTouchView.TouchEventListener() {
             @Override
             public void onTouch(float relX, float relY) {
-                sendTouchCommand(relX, relY);
+                // 已通过 onMotionEvent 处理
             }
 
             @Override
             public void onSwipe(float relX1, float relY1, float relX2, float relY2, long duration) {
-                sendSwipeCommand(relX1, relY1, relX2, relY2, duration);
+                // 已通过 onMotionEvent 处理
             }
 
             @Override
@@ -126,7 +126,13 @@ public class MainActivity extends AppCompatActivity {
 
             @Override
             public void onLongPress(float relX, float relY) {
-                sendLongPressCommand(relX, relY);
+                // 如果需要高层级的长按，可以保留，但 raw 模式下远程系统会自动识别
+                // sendLongPressCommand(relX, relY);
+            }
+
+            @Override
+            public void onMotionEvent(int action, float relX, float relY) {
+                sendMotionEventCommand(action, relX, relY);
             }
         });
 
@@ -361,6 +367,17 @@ public class MainActivity extends AppCompatActivity {
         }
     }
 
+    private void sendMotionEventCommand(int action, float relX, float relY) {
+        if (webRtcClient != null) {
+            JsonObject cmd = new JsonObject();
+            cmd.addProperty("action", "MOTION_EVENT");
+            cmd.addProperty("motionAction", action);
+            cmd.addProperty("x", relX);
+            cmd.addProperty("y", relY);
+            webRtcClient.sendControlCommand(cmd.toString());
+        }
+    }
+
     private void disconnect() {
         if (webRtcClient != null) {
             webRtcClient.close();

WebRTCController/app/src/main/java/com/ttstd/controller/view/RemoteTouchView.java

@@ -25,8 +25,6 @@ public class RemoteTouchView extends SurfaceView {
 
     private float lastTouchX;
     private float lastTouchY;
-    private float lastAbsX;
-    private float lastAbsY;
     private long swipeStartTime;
     private boolean isLongPressed = false;
     private GestureDetector gestureDetector;
@@ -36,6 +34,7 @@ public class RemoteTouchView extends SurfaceView {
         void onSwipe(float relX1, float relY1, float relX2, float relY2, long duration);
         void onKeyEvent(int keyCode, int action);
         void onLongPress(float relX, float relY);
+        void onMotionEvent(int action, float relX, float relY);
     }
 
     public RemoteTouchView(Context context) {
@@ -101,7 +100,6 @@ public class RemoteTouchView extends SurfaceView {
 
         float absX = event.getX();
         float absY = event.getY();
-        Log.d(TAG, "onTouchEvent: absX=" + absX + ", absY=" + absY);
 
         float relX = absX / getWidth();
         float relY = absY / getHeight();
@@ -110,18 +108,20 @@ public class RemoteTouchView extends SurfaceView {
         relX = Math.max(0, Math.min(1, relX));
         relY = Math.max(0, Math.min(1, relY));
 
-        switch (event.getActionMasked()) {
+        int action = event.getActionMasked();
+        
+        // 发送原始 MotionEvent 实现实时跟手
+        listener.onMotionEvent(action, relX, relY);
+
+        switch (action) {
             case MotionEvent.ACTION_DOWN:
                 isLongPressed = false;
                 lastTouchX = relX;
                 lastTouchY = relY;
-                lastAbsX = absX;
-                lastAbsY = absY;
                 swipeStartTime = System.currentTimeMillis();
                 return true;
 
             case MotionEvent.ACTION_MOVE:
-                // 仅返回 true 以持续接收事件，不更新起始坐标，确保 onSwipe 计算的是从按下点开始的位移
                 return true;
 
             case MotionEvent.ACTION_UP:
@@ -132,14 +132,11 @@ public class RemoteTouchView extends SurfaceView {
                 float dx = Math.abs(relX - lastTouchX);
                 float dy = Math.abs(relY - lastTouchY);
 
+                // 保留旧的回调以防万一，但 MainActivity 现在应该忽略它们
                 if (duration < 200 && dx < 0.02 && dy < 0.02) {
-                    // 点击事件
                     listener.onTouch(lastTouchX, lastTouchY);
-                    Log.d(TAG, "Touch at: " + lastTouchX + ", " + lastTouchY + " (abs: " + absX + ", " + absY + ")");
                 } else {
-                    // 滑动事件
                     listener.onSwipe(lastTouchX, lastTouchY, relX, relY, duration);
-                    Log.d(TAG, "Swipe from (" + lastTouchX + "," + lastTouchY + ") [abs: " + lastAbsX + "," + lastAbsY + "] to (" + relX + "," + relY + ") [abs: " + absX + "," + absY + "]");
                 }
                 return true;
         }
@@ -173,4 +170,13 @@ public class RemoteTouchView extends SurfaceView {
         cmd.addProperty("y", relY);
         return cmd.toString();
     }
+
+    public String createMotionEventCommand(int action, float relX, float relY) {
+        JsonObject cmd = new JsonObject();
+        cmd.addProperty("action", "MOTION_EVENT");
+        cmd.addProperty("motionAction", action);
+        cmd.addProperty("x", relX);
+        cmd.addProperty("y", relY);
+        return cmd.toString();
+    }
 }