diff --git a/WebRTCControlled/app/build.gradle b/WebRTCControlled/app/build.gradle index 187c170..6d9d1f4 100644 --- a/WebRTCControlled/app/build.gradle +++ b/WebRTCControlled/app/build.gradle @@ -1,5 +1,6 @@ plugins { id 'com.android.application' + id 'com.google.protobuf' } def releaseTime() { @@ -124,6 +125,19 @@ android { } +protobuf { + protoc { + artifact = 'com.google.protobuf:protoc:3.25.1' + } + generateProtoTasks { + all().each { task -> + task.builtins { + java {} + } + } + } +} + dependencies { implementation 'androidx.appcompat:appcompat:1.6.1' implementation 'com.google.android.material:material:1.11.0' @@ -137,6 +151,8 @@ dependencies { // implementation 'org.webrtc:google-webrtc:1.0.32006' // OkHttp for WebSocket implementation 'com.squareup.okhttp3:okhttp:4.12.0' - // Gson for JSON + // Gson for JSON(信令消息仍使用 JSON) implementation 'com.google.code.gson:gson:2.10.1' + // Protobuf(DataChannel 控制指令二进制) + implementation 'com.google.protobuf:protobuf-java:3.25.1' } diff --git a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputCommandHandler.java b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputCommandHandler.java index 60907cc..dd3ef7a 100644 --- a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputCommandHandler.java +++ b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/input/InputCommandHandler.java @@ -2,17 +2,18 @@ package com.ttstd.controlled.input; import android.util.Log; -import com.google.gson.Gson; -import com.google.gson.JsonObject; +import com.google.protobuf.InvalidProtocolBufferException; +import com.ttstd.control.Action; +import com.ttstd.control.ControlMessage; /** * 处理控制端发来的输入指令(触摸、按键)。 - * 解析指令并委托给 {@link InputExecutor} 接口执行。 + * 指令以 protobuf 二进制({@link ControlMessage})通过 DataChannel 传输, + * 解析后委托给 {@link InputExecutor} 接口执行。 */ public class InputCommandHandler { private static final String TAG = "InputCommandHandler"; - private final Gson gson = new Gson(); private final int screenWidth; private final int screenHeight; private final InputExecutor inputExecutor; @@ -24,47 +25,47 @@ public class InputCommandHandler { this.inputExecutor = new SystemInputUtils(); } - public void handleCommand(String commandJson) { + public void handleCommand(byte[] data) { try { - JsonObject command = gson.fromJson(commandJson, JsonObject.class); - String action = command.get("action").getAsString(); + ControlMessage command = ControlMessage.parseFrom(data); + Action action = command.getAction(); switch (action) { - case "TOUCH": + case TOUCH: handleTouch(command); break; - case "KEY": + case KEY: handleKey(command); break; - case "SWIPE": + case SWIPE: handleSwipe(command); break; - case "LONG_PRESS": + case LONG_PRESS: handleLongPress(command); break; - case "MOTION_EVENT": + case MOTION_EVENT: handleMotionEvent(command); break; default: Log.w(TAG, "Unknown action: " + action); } - } catch (Exception e) { - Log.e(TAG, "Error handling command: " + e.getMessage(), e); + } catch (InvalidProtocolBufferException e) { + Log.e(TAG, "Error parsing command: " + e.getMessage(), e); } } - private void handleMotionEvent(JsonObject command) { - int action = command.get("motionAction").getAsInt(); - float relX = command.get("x").getAsFloat(); - float relY = command.get("y").getAsFloat(); + private void handleMotionEvent(ControlMessage command) { + int action = command.getMotionAction(); + float relX = (float) command.getX(); + float relY = (float) command.getY(); int x = (int) (relX * screenWidth); int y = (int) (relY * screenHeight); inputExecutor.injectMotionEvent(action, x, y); } - private void handleTouch(JsonObject command) { - float relX = command.get("x").getAsFloat(); - float relY = command.get("y").getAsFloat(); + private void handleTouch(ControlMessage command) { + float relX = (float) command.getX(); + float relY = (float) command.getY(); int x = (int) (relX * screenWidth); int y = (int) (relY * screenHeight); @@ -72,18 +73,18 @@ public class InputCommandHandler { inputExecutor.injectTap(x, y); } - private void handleKey(JsonObject command) { - int keyCode = command.get("keyCode").getAsInt(); + private void handleKey(ControlMessage command) { + int keyCode = command.getKeyCode(); Log.d(TAG, "Key press: " + keyCode); inputExecutor.injectKeyEvent(keyCode); } - private void handleSwipe(JsonObject command) { - float relX1 = command.get("x1").getAsFloat(); - float relY1 = command.get("y1").getAsFloat(); - float relX2 = command.get("x2").getAsFloat(); - float relY2 = command.get("y2").getAsFloat(); - long duration = command.has("duration") ? command.get("duration").getAsLong() : 300; + private void handleSwipe(ControlMessage command) { + float relX1 = (float) command.getX1(); + float relY1 = (float) command.getY1(); + float relX2 = (float) command.getX2(); + float relY2 = (float) command.getY2(); + long duration = command.getDuration(); int x1 = (int) (relX1 * screenWidth); int y1 = (int) (relY1 * screenHeight); @@ -94,9 +95,9 @@ public class InputCommandHandler { inputExecutor.injectSwipe(x1, y1, x2, y2, duration); } - private void handleLongPress(JsonObject command) { - float relX = command.get("x").getAsFloat(); - float relY = command.get("y").getAsFloat(); + private void handleLongPress(ControlMessage command) { + float relX = (float) command.getX(); + float relY = (float) command.getY(); int x = (int) (relX * screenWidth); int y = (int) (relY * screenHeight); diff --git a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/service/ScreenCaptureService.java b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/service/ScreenCaptureService.java index dd91978..8ba4dff 100644 --- a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/service/ScreenCaptureService.java +++ b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/service/ScreenCaptureService.java @@ -207,7 +207,7 @@ public class ScreenCaptureService extends Service { // 初始化 WebRTC webRtcClient = new WebRtcClient(this, wsClient, deviceId); webRtcClient.initialize(eglBase); - webRtcClient.setInputCallback(commandJson -> inputHandler.handleCommand(commandJson)); + webRtcClient.setInputCallback(commandBytes -> inputHandler.handleCommand(commandBytes)); // 初始化屏幕捕获 screenCapturer = new ScreenCapturerAndroid(resultData, new MediaProjection.Callback() { diff --git a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/WebRtcClient.java b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/WebRtcClient.java index f551900..98214ba 100644 --- a/WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/WebRtcClient.java +++ b/WebRTCControlled/app/src/main/java/com/ttstd/controlled/webrtc/WebRtcClient.java @@ -59,7 +59,7 @@ public class WebRtcClient { private InputCommandCallback inputCallback; public interface InputCommandCallback { - void onCommandReceived(String commandJson); + void onCommandReceived(byte[] data); } public WebRtcClient(Context context, WebSocketClient wsClient, String deviceId) { @@ -316,10 +316,9 @@ public class WebRtcClient { public void onMessage(DataChannel.Buffer buffer) { byte[] data = new byte[buffer.data.remaining()]; buffer.data.get(data); - String message = new String(data, StandardCharsets.UTF_8); - Log.d(TAG, "DataChannel message: " + message); + Log.d(TAG, "DataChannel message received, bytes=" + data.length); if (inputCallback != null) { - inputCallback.onCommandReceived(message); + inputCallback.onCommandReceived(data); } } }); diff --git a/WebRTCControlled/app/src/main/proto/control_message.proto b/WebRTCControlled/app/src/main/proto/control_message.proto new file mode 100644 index 0000000..e72212d --- /dev/null +++ b/WebRTCControlled/app/src/main/proto/control_message.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package com.ttstd.control; + +option java_package = "com.ttstd.control"; +option java_multiple_files = true; + +// 控制指令类型,对应原 JSON 字段 action。 +enum Action { + ACTION_UNKNOWN = 0; + TOUCH = 1; + SWIPE = 2; + KEY = 3; + LONG_PRESS = 4; + MOTION_EVENT = 5; +} + +// 通过 RTCDataChannel 传输的控制指令(protobuf 二进制)。 +// 坐标 x/y/x1/y1/x2/y2 均为相对屏幕的百分比,取值范围 0.0 ~ 1.0。 +message ControlMessage { + // 指令类型 + Action action = 1; + + // 单点坐标(TOUCH / LONG_PRESS / MOTION_EVENT) + double x = 2; + double y = 3; + + // 滑动起止坐标(SWIPE) + double x1 = 4; + double y1 = 5; + double x2 = 6; + double y2 = 7; + int64 duration = 8; + + // 按键(KEY):key_action 0=按下 1=抬起(对应 Android KeyEvent ACTION_DOWN/UP) + int32 key_code = 9; + int32 key_action = 10; + + // 原始动作(MOTION_EVENT):对应 Android MotionEvent ACTION_*(0=DOWN 1=UP 2=MOVE) + int32 motion_action = 11; +} diff --git a/WebRTCControlled/build.gradle b/WebRTCControlled/build.gradle index 05c39aa..76c8b63 100644 --- a/WebRTCControlled/build.gradle +++ b/WebRTCControlled/build.gradle @@ -1,5 +1,6 @@ // Top-level build file plugins { id 'com.android.application' version '8.1.4' apply false + id 'com.google.protobuf' version '0.9.4' apply false } apply from: "config.gradle" diff --git a/WebRTCController/app/build.gradle b/WebRTCController/app/build.gradle index 5c1656b..b0cbe65 100644 --- a/WebRTCController/app/build.gradle +++ b/WebRTCController/app/build.gradle @@ -1,5 +1,6 @@ plugins { id 'com.android.application' + id 'com.google.protobuf' } def releaseTime() { @@ -116,6 +117,19 @@ android { } } +protobuf { + protoc { + artifact = 'com.google.protobuf:protoc:3.25.1' + } + generateProtoTasks { + all().each { task -> + task.builtins { + java {} + } + } + } +} + dependencies { implementation 'androidx.appcompat:appcompat:1.6.1' implementation 'com.google.android.material:material:1.11.0' @@ -127,6 +141,8 @@ dependencies { // OkHttp for WebSocket implementation 'com.squareup.okhttp3:okhttp:4.12.0' - // Gson for JSON + // Gson for JSON(信令消息仍使用 JSON) implementation 'com.google.code.gson:gson:2.10.1' + // Protobuf(DataChannel 控制指令二进制) + implementation 'com.google.protobuf:protobuf-java:3.25.1' } diff --git a/WebRTCController/app/src/main/java/com/ttstd/controller/MainActivity.java b/WebRTCController/app/src/main/java/com/ttstd/controller/MainActivity.java index 25f507d..becfb5b 100644 --- a/WebRTCController/app/src/main/java/com/ttstd/controller/MainActivity.java +++ b/WebRTCController/app/src/main/java/com/ttstd/controller/MainActivity.java @@ -16,6 +16,8 @@ import androidx.appcompat.app.AppCompatActivity; import com.google.gson.Gson; import com.google.gson.JsonObject; +import com.ttstd.control.Action; +import com.ttstd.control.ControlMessage; import com.ttstd.controller.signaling.SignalMessage; import com.ttstd.controller.signaling.WebSocketClient; import com.ttstd.controller.view.RemoteTouchView; @@ -326,55 +328,60 @@ public class MainActivity extends AppCompatActivity { private void sendTouchCommand(float relX, float relY) { if (webRtcClient != null) { - JsonObject cmd = new JsonObject(); - cmd.addProperty("action", "TOUCH"); - cmd.addProperty("x", relX); - cmd.addProperty("y", relY); - webRtcClient.sendControlCommand(cmd.toString()); + ControlMessage msg = ControlMessage.newBuilder() + .setAction(Action.TOUCH) + .setX(relX) + .setY(relY) + .build(); + webRtcClient.sendControlCommand(msg); } } private void sendSwipeCommand(float relX1, float relY1, float relX2, float relY2, long duration) { if (webRtcClient != null) { - JsonObject cmd = new JsonObject(); - cmd.addProperty("action", "SWIPE"); - cmd.addProperty("x1", relX1); - cmd.addProperty("y1", relY1); - cmd.addProperty("x2", relX2); - cmd.addProperty("y2", relY2); - cmd.addProperty("duration", duration); - webRtcClient.sendControlCommand(cmd.toString()); + ControlMessage msg = ControlMessage.newBuilder() + .setAction(Action.SWIPE) + .setX1(relX1) + .setY1(relY1) + .setX2(relX2) + .setY2(relY2) + .setDuration(duration) + .build(); + webRtcClient.sendControlCommand(msg); } } private void sendKeyCommand(int keyCode, int action) { if (webRtcClient != null) { - JsonObject cmd = new JsonObject(); - cmd.addProperty("action", "KEY"); - cmd.addProperty("keyCode", keyCode); - cmd.addProperty("keyAction", action); - webRtcClient.sendControlCommand(cmd.toString()); + ControlMessage msg = ControlMessage.newBuilder() + .setAction(Action.KEY) + .setKeyCode(keyCode) + .setKeyAction(action) + .build(); + webRtcClient.sendControlCommand(msg); } } private void sendLongPressCommand(float relX, float relY) { if (webRtcClient != null) { - JsonObject cmd = new JsonObject(); - cmd.addProperty("action", "LONG_PRESS"); - cmd.addProperty("x", relX); - cmd.addProperty("y", relY); - webRtcClient.sendControlCommand(cmd.toString()); + ControlMessage msg = ControlMessage.newBuilder() + .setAction(Action.LONG_PRESS) + .setX(relX) + .setY(relY) + .build(); + webRtcClient.sendControlCommand(msg); } } private void sendMotionEventCommand(int action, float relX, float relY) { if (webRtcClient != null) { - JsonObject cmd = new JsonObject(); - cmd.addProperty("action", "MOTION_EVENT"); - cmd.addProperty("motionAction", action); - cmd.addProperty("x", relX); - cmd.addProperty("y", relY); - webRtcClient.sendControlCommand(cmd.toString()); + ControlMessage msg = ControlMessage.newBuilder() + .setAction(Action.MOTION_EVENT) + .setMotionAction(action) + .setX(relX) + .setY(relY) + .build(); + webRtcClient.sendControlCommand(msg); } } diff --git a/WebRTCController/app/src/main/java/com/ttstd/controller/view/RemoteTouchView.java b/WebRTCController/app/src/main/java/com/ttstd/controller/view/RemoteTouchView.java index a0fd901..9fbef43 100644 --- a/WebRTCController/app/src/main/java/com/ttstd/controller/view/RemoteTouchView.java +++ b/WebRTCController/app/src/main/java/com/ttstd/controller/view/RemoteTouchView.java @@ -9,8 +9,8 @@ import android.view.SurfaceView; import androidx.annotation.NonNull; -import com.google.gson.Gson; -import com.google.gson.JsonObject; +import com.ttstd.control.Action; +import com.ttstd.control.ControlMessage; /** * 自定义SurfaceView,用于捕获触摸事件并转换为远端输入指令。 @@ -20,7 +20,6 @@ import com.google.gson.JsonObject; public class RemoteTouchView extends SurfaceView { private static final String TAG = "RemoteTouchView"; - private final Gson gson = new Gson(); private TouchEventListener listener; private float lastTouchX; @@ -144,39 +143,39 @@ public class RemoteTouchView extends SurfaceView { return super.onTouchEvent(event); } - public String createTouchCommand(float relX, float relY) { - JsonObject cmd = new JsonObject(); - cmd.addProperty("action", "TOUCH"); - cmd.addProperty("x", relX); - cmd.addProperty("y", relY); - return cmd.toString(); + public ControlMessage createTouchCommand(float relX, float relY) { + return ControlMessage.newBuilder() + .setAction(Action.TOUCH) + .setX(relX) + .setY(relY) + .build(); } - public String createSwipeCommand(float relX1, float relY1, float relX2, float relY2, long duration) { - JsonObject cmd = new JsonObject(); - cmd.addProperty("action", "SWIPE"); - cmd.addProperty("x1", relX1); - cmd.addProperty("y1", relY1); - cmd.addProperty("x2", relX2); - cmd.addProperty("y2", relY2); - cmd.addProperty("duration", duration); - return cmd.toString(); + public ControlMessage createSwipeCommand(float relX1, float relY1, float relX2, float relY2, long duration) { + return ControlMessage.newBuilder() + .setAction(Action.SWIPE) + .setX1(relX1) + .setY1(relY1) + .setX2(relX2) + .setY2(relY2) + .setDuration(duration) + .build(); } - public String createLongPressCommand(float relX, float relY) { - JsonObject cmd = new JsonObject(); - cmd.addProperty("action", "LONG_PRESS"); - cmd.addProperty("x", relX); - cmd.addProperty("y", relY); - return cmd.toString(); + public ControlMessage createLongPressCommand(float relX, float relY) { + return ControlMessage.newBuilder() + .setAction(Action.LONG_PRESS) + .setX(relX) + .setY(relY) + .build(); } - public String createMotionEventCommand(int action, float relX, float relY) { - JsonObject cmd = new JsonObject(); - cmd.addProperty("action", "MOTION_EVENT"); - cmd.addProperty("motionAction", action); - cmd.addProperty("x", relX); - cmd.addProperty("y", relY); - return cmd.toString(); + public ControlMessage createMotionEventCommand(int action, float relX, float relY) { + return ControlMessage.newBuilder() + .setAction(Action.MOTION_EVENT) + .setMotionAction(action) + .setX(relX) + .setY(relY) + .build(); } } diff --git a/WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/WebRtcClient.java b/WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/WebRtcClient.java index abc4e5d..667b2fb 100644 --- a/WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/WebRtcClient.java +++ b/WebRTCController/app/src/main/java/com/ttstd/controller/webrtc/WebRtcClient.java @@ -7,6 +7,7 @@ import com.google.gson.Gson; import com.google.gson.JsonObject; import com.ttstd.controller.signaling.SignalMessage; import com.ttstd.controller.signaling.WebSocketClient; +import com.ttstd.control.ControlMessage; import org.webrtc.DataChannel; import org.webrtc.DefaultVideoDecoderFactory; @@ -27,7 +28,6 @@ import org.webrtc.VideoEncoderFactory; import org.webrtc.VideoTrack; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -231,10 +231,10 @@ public class WebRtcClient { } } - public void sendControlCommand(String commandJson) { + public void sendControlCommand(ControlMessage message) { if (dataChannel != null && dataChannel.state() == DataChannel.State.OPEN) { DataChannel.Buffer buffer = new DataChannel.Buffer( - ByteBuffer.wrap(commandJson.getBytes(StandardCharsets.UTF_8)), false); + ByteBuffer.wrap(message.toByteArray()), false); dataChannel.send(buffer); } } @@ -279,8 +279,7 @@ public class WebRtcClient { public void onMessage(DataChannel.Buffer buffer) { byte[] data = new byte[buffer.data.remaining()]; buffer.data.get(data); - String message = new String(data, StandardCharsets.UTF_8); - Log.d(TAG, "DataChannel message: " + message); + Log.d(TAG, "DataChannel message received, bytes=" + data.length); } }); } diff --git a/WebRTCController/app/src/main/proto/control_message.proto b/WebRTCController/app/src/main/proto/control_message.proto new file mode 100644 index 0000000..e72212d --- /dev/null +++ b/WebRTCController/app/src/main/proto/control_message.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package com.ttstd.control; + +option java_package = "com.ttstd.control"; +option java_multiple_files = true; + +// 控制指令类型,对应原 JSON 字段 action。 +enum Action { + ACTION_UNKNOWN = 0; + TOUCH = 1; + SWIPE = 2; + KEY = 3; + LONG_PRESS = 4; + MOTION_EVENT = 5; +} + +// 通过 RTCDataChannel 传输的控制指令(protobuf 二进制)。 +// 坐标 x/y/x1/y1/x2/y2 均为相对屏幕的百分比,取值范围 0.0 ~ 1.0。 +message ControlMessage { + // 指令类型 + Action action = 1; + + // 单点坐标(TOUCH / LONG_PRESS / MOTION_EVENT) + double x = 2; + double y = 3; + + // 滑动起止坐标(SWIPE) + double x1 = 4; + double y1 = 5; + double x2 = 6; + double y2 = 7; + int64 duration = 8; + + // 按键(KEY):key_action 0=按下 1=抬起(对应 Android KeyEvent ACTION_DOWN/UP) + int32 key_code = 9; + int32 key_action = 10; + + // 原始动作(MOTION_EVENT):对应 Android MotionEvent ACTION_*(0=DOWN 1=UP 2=MOVE) + int32 motion_action = 11; +} diff --git a/WebRTCController/build.gradle b/WebRTCController/build.gradle index 05c39aa..76c8b63 100644 --- a/WebRTCController/build.gradle +++ b/WebRTCController/build.gradle @@ -1,5 +1,6 @@ // Top-level build file plugins { id 'com.android.application' version '8.1.4' apply false + id 'com.google.protobuf' version '0.9.4' apply false } apply from: "config.gradle" diff --git a/webrtc_controller_flutter/lib/controller/remote_controller.dart b/webrtc_controller_flutter/lib/controller/remote_controller.dart index 64592ed..d0196ba 100644 --- a/webrtc_controller_flutter/lib/controller/remote_controller.dart +++ b/webrtc_controller_flutter/lib/controller/remote_controller.dart @@ -4,6 +4,7 @@ import 'dart:convert'; import 'package:flutter_webrtc/flutter_webrtc.dart'; import '../models/signal_message.dart'; +import '../proto/control_message.pb.dart'; import '../signaling/signaling_client.dart'; import '../webrtc/webrtc_controller.dart'; @@ -113,9 +114,9 @@ class RemoteController { }); } - /// 发送控制指令(JSON 字符串)。 - void sendControlCommand(String commandJson) { - _webRtc?.sendControlCommand(commandJson); + /// 发送控制指令(protobuf 二进制)。 + void sendControlCommand(ControlMessage command) { + _webRtc?.sendControlCommand(command); } /// 断开连接并释放资源。 diff --git a/webrtc_controller_flutter/lib/proto/control_message.pb.dart b/webrtc_controller_flutter/lib/proto/control_message.pb.dart new file mode 100644 index 0000000..1a87ee6 --- /dev/null +++ b/webrtc_controller_flutter/lib/proto/control_message.pb.dart @@ -0,0 +1,209 @@ +// This is a generated file - do not edit. +// +// Generated from control_message.proto. + +// @dart = 3.3 + +// ignore_for_file: annotate_overrides, camel_case_types, comment_references +// ignore_for_file: constant_identifier_names +// ignore_for_file: curly_braces_in_flow_control_structures +// ignore_for_file: deprecated_member_use_from_same_package, library_prefixes +// ignore_for_file: non_constant_identifier_names, prefer_relative_imports + +import 'dart:core' as $core; + +import 'package:fixnum/fixnum.dart' as $fixnum; +import 'package:protobuf/protobuf.dart' as $pb; + +import 'control_message.pbenum.dart'; + +export 'package:protobuf/protobuf.dart' show GeneratedMessageGenericExtensions; + +export 'control_message.pbenum.dart'; + +/// 通过 RTCDataChannel 传输的控制指令(protobuf 二进制)。 +/// 坐标 x/y/x1/y1/x2/y2 均为相对屏幕的百分比,取值范围 0.0 ~ 1.0。 +class ControlMessage extends $pb.GeneratedMessage { + factory ControlMessage({ + Action? action, + $core.double? x, + $core.double? y, + $core.double? x1, + $core.double? y1, + $core.double? x2, + $core.double? y2, + $fixnum.Int64? duration, + $core.int? keyCode, + $core.int? keyAction, + $core.int? motionAction, + }) { + final result = create(); + if (action != null) result.action = action; + if (x != null) result.x = x; + if (y != null) result.y = y; + if (x1 != null) result.x1 = x1; + if (y1 != null) result.y1 = y1; + if (x2 != null) result.x2 = x2; + if (y2 != null) result.y2 = y2; + if (duration != null) result.duration = duration; + if (keyCode != null) result.keyCode = keyCode; + if (keyAction != null) result.keyAction = keyAction; + if (motionAction != null) result.motionAction = motionAction; + return result; + } + + ControlMessage._(); + + factory ControlMessage.fromBuffer($core.List<$core.int> data, + [$pb.ExtensionRegistry registry = $pb.ExtensionRegistry.EMPTY]) => + create()..mergeFromBuffer(data, registry); + factory ControlMessage.fromJson($core.String json, + [$pb.ExtensionRegistry registry = $pb.ExtensionRegistry.EMPTY]) => + create()..mergeFromJson(json, registry); + + static final $pb.BuilderInfo _i = $pb.BuilderInfo( + _omitMessageNames ? '' : 'ControlMessage', + package: + const $pb.PackageName(_omitMessageNames ? '' : 'com.ttstd.control'), + createEmptyInstance: create) + ..aE(1, _omitFieldNames ? '' : 'action', enumValues: Action.values) + ..aD(2, _omitFieldNames ? '' : 'x') + ..aD(3, _omitFieldNames ? '' : 'y') + ..aD(4, _omitFieldNames ? '' : 'x1') + ..aD(5, _omitFieldNames ? '' : 'y1') + ..aD(6, _omitFieldNames ? '' : 'x2') + ..aD(7, _omitFieldNames ? '' : 'y2') + ..aInt64(8, _omitFieldNames ? '' : 'duration') + ..aI(9, _omitFieldNames ? '' : 'keyCode') + ..aI(10, _omitFieldNames ? '' : 'keyAction') + ..aI(11, _omitFieldNames ? '' : 'motionAction') + ..hasRequiredFields = false; + + @$core.Deprecated('See https://github.com/google/protobuf.dart/issues/998.') + ControlMessage clone() => deepCopy(); + @$core.Deprecated('See https://github.com/google/protobuf.dart/issues/998.') + ControlMessage copyWith(void Function(ControlMessage) updates) => + super.copyWith((message) => updates(message as ControlMessage)) + as ControlMessage; + + @$core.override + $pb.BuilderInfo get info_ => _i; + + @$core.pragma('dart2js:noInline') + static ControlMessage create() => ControlMessage._(); + @$core.override + ControlMessage createEmptyInstance() => create(); + @$core.pragma('dart2js:noInline') + static ControlMessage getDefault() => _defaultInstance ??= + $pb.GeneratedMessage.$_defaultFor(create); + static ControlMessage? _defaultInstance; + + /// 指令类型 + @$pb.TagNumber(1) + Action get action => $_getN(0); + @$pb.TagNumber(1) + set action(Action value) => $_setField(1, value); + @$pb.TagNumber(1) + $core.bool hasAction() => $_has(0); + @$pb.TagNumber(1) + void clearAction() => $_clearField(1); + + /// 单点坐标(TOUCH / LONG_PRESS / MOTION_EVENT) + @$pb.TagNumber(2) + $core.double get x => $_getN(1); + @$pb.TagNumber(2) + set x($core.double value) => $_setDouble(1, value); + @$pb.TagNumber(2) + $core.bool hasX() => $_has(1); + @$pb.TagNumber(2) + void clearX() => $_clearField(2); + + @$pb.TagNumber(3) + $core.double get y => $_getN(2); + @$pb.TagNumber(3) + set y($core.double value) => $_setDouble(2, value); + @$pb.TagNumber(3) + $core.bool hasY() => $_has(2); + @$pb.TagNumber(3) + void clearY() => $_clearField(3); + + /// 滑动起止坐标(SWIPE) + @$pb.TagNumber(4) + $core.double get x1 => $_getN(3); + @$pb.TagNumber(4) + set x1($core.double value) => $_setDouble(3, value); + @$pb.TagNumber(4) + $core.bool hasX1() => $_has(3); + @$pb.TagNumber(4) + void clearX1() => $_clearField(4); + + @$pb.TagNumber(5) + $core.double get y1 => $_getN(4); + @$pb.TagNumber(5) + set y1($core.double value) => $_setDouble(4, value); + @$pb.TagNumber(5) + $core.bool hasY1() => $_has(4); + @$pb.TagNumber(5) + void clearY1() => $_clearField(5); + + @$pb.TagNumber(6) + $core.double get x2 => $_getN(5); + @$pb.TagNumber(6) + set x2($core.double value) => $_setDouble(5, value); + @$pb.TagNumber(6) + $core.bool hasX2() => $_has(5); + @$pb.TagNumber(6) + void clearX2() => $_clearField(6); + + @$pb.TagNumber(7) + $core.double get y2 => $_getN(6); + @$pb.TagNumber(7) + set y2($core.double value) => $_setDouble(6, value); + @$pb.TagNumber(7) + $core.bool hasY2() => $_has(6); + @$pb.TagNumber(7) + void clearY2() => $_clearField(7); + + @$pb.TagNumber(8) + $fixnum.Int64 get duration => $_getI64(7); + @$pb.TagNumber(8) + set duration($fixnum.Int64 value) => $_setInt64(7, value); + @$pb.TagNumber(8) + $core.bool hasDuration() => $_has(7); + @$pb.TagNumber(8) + void clearDuration() => $_clearField(8); + + /// 按键(KEY):key_action 0=按下 1=抬起(对应 Android KeyEvent ACTION_DOWN/UP) + @$pb.TagNumber(9) + $core.int get keyCode => $_getIZ(8); + @$pb.TagNumber(9) + set keyCode($core.int value) => $_setSignedInt32(8, value); + @$pb.TagNumber(9) + $core.bool hasKeyCode() => $_has(8); + @$pb.TagNumber(9) + void clearKeyCode() => $_clearField(9); + + @$pb.TagNumber(10) + $core.int get keyAction => $_getIZ(9); + @$pb.TagNumber(10) + set keyAction($core.int value) => $_setSignedInt32(9, value); + @$pb.TagNumber(10) + $core.bool hasKeyAction() => $_has(9); + @$pb.TagNumber(10) + void clearKeyAction() => $_clearField(10); + + /// 原始动作(MOTION_EVENT):对应 Android MotionEvent ACTION_*(0=DOWN 1=UP 2=MOVE) + @$pb.TagNumber(11) + $core.int get motionAction => $_getIZ(10); + @$pb.TagNumber(11) + set motionAction($core.int value) => $_setSignedInt32(10, value); + @$pb.TagNumber(11) + $core.bool hasMotionAction() => $_has(10); + @$pb.TagNumber(11) + void clearMotionAction() => $_clearField(11); +} + +const $core.bool _omitFieldNames = + $core.bool.fromEnvironment('protobuf.omit_field_names'); +const $core.bool _omitMessageNames = + $core.bool.fromEnvironment('protobuf.omit_message_names'); diff --git a/webrtc_controller_flutter/lib/proto/control_message.pbenum.dart b/webrtc_controller_flutter/lib/proto/control_message.pbenum.dart new file mode 100644 index 0000000..895dd81 --- /dev/null +++ b/webrtc_controller_flutter/lib/proto/control_message.pbenum.dart @@ -0,0 +1,47 @@ +// This is a generated file - do not edit. +// +// Generated from control_message.proto. + +// @dart = 3.3 + +// ignore_for_file: annotate_overrides, camel_case_types, comment_references +// ignore_for_file: constant_identifier_names +// ignore_for_file: curly_braces_in_flow_control_structures +// ignore_for_file: deprecated_member_use_from_same_package, library_prefixes +// ignore_for_file: non_constant_identifier_names, prefer_relative_imports + +import 'dart:core' as $core; + +import 'package:protobuf/protobuf.dart' as $pb; + +/// 控制指令类型,对应原 JSON 字段 action。 +class Action extends $pb.ProtobufEnum { + static const Action ACTION_UNKNOWN = + Action._(0, _omitEnumNames ? '' : 'ACTION_UNKNOWN'); + static const Action TOUCH = Action._(1, _omitEnumNames ? '' : 'TOUCH'); + static const Action SWIPE = Action._(2, _omitEnumNames ? '' : 'SWIPE'); + static const Action KEY = Action._(3, _omitEnumNames ? '' : 'KEY'); + static const Action LONG_PRESS = + Action._(4, _omitEnumNames ? '' : 'LONG_PRESS'); + static const Action MOTION_EVENT = + Action._(5, _omitEnumNames ? '' : 'MOTION_EVENT'); + + static const $core.List values = [ + ACTION_UNKNOWN, + TOUCH, + SWIPE, + KEY, + LONG_PRESS, + MOTION_EVENT, + ]; + + static final $core.List _byValue = + $pb.ProtobufEnum.$_initByValueList(values, 5); + static Action? valueOf($core.int value) => + value < 0 || value >= _byValue.length ? null : _byValue[value]; + + const Action._(super.value, super.name); +} + +const $core.bool _omitEnumNames = + $core.bool.fromEnvironment('protobuf.omit_enum_names'); diff --git a/webrtc_controller_flutter/lib/proto/control_message.pbjson.dart b/webrtc_controller_flutter/lib/proto/control_message.pbjson.dart new file mode 100644 index 0000000..a6aa41c --- /dev/null +++ b/webrtc_controller_flutter/lib/proto/control_message.pbjson.dart @@ -0,0 +1,68 @@ +// This is a generated file - do not edit. +// +// Generated from control_message.proto. + +// @dart = 3.3 + +// ignore_for_file: annotate_overrides, camel_case_types, comment_references +// ignore_for_file: constant_identifier_names +// ignore_for_file: curly_braces_in_flow_control_structures +// ignore_for_file: deprecated_member_use_from_same_package, library_prefixes +// ignore_for_file: non_constant_identifier_names, prefer_relative_imports +// ignore_for_file: unused_import + +import 'dart:convert' as $convert; +import 'dart:core' as $core; +import 'dart:typed_data' as $typed_data; + +@$core.Deprecated('Use actionDescriptor instead') +const Action$json = { + '1': 'Action', + '2': [ + {'1': 'ACTION_UNKNOWN', '2': 0}, + {'1': 'TOUCH', '2': 1}, + {'1': 'SWIPE', '2': 2}, + {'1': 'KEY', '2': 3}, + {'1': 'LONG_PRESS', '2': 4}, + {'1': 'MOTION_EVENT', '2': 5}, + ], +}; + +/// Descriptor for `Action`. Decode as a `google.protobuf.EnumDescriptorProto`. +final $typed_data.Uint8List actionDescriptor = $convert.base64Decode( + 'CgZBY3Rpb24SEgoOQUNUSU9OX1VOS05PV04QABIJCgVUT1VDSBABEgkKBVNXSVBFEAISBwoDS0' + 'VZEAMSDgoKTE9OR19QUkVTUxAEEhAKDE1PVElPTl9FVkVOVBAF'); + +@$core.Deprecated('Use controlMessageDescriptor instead') +const ControlMessage$json = { + '1': 'ControlMessage', + '2': [ + { + '1': 'action', + '3': 1, + '4': 1, + '5': 14, + '6': '.com.ttstd.control.Action', + '10': 'action' + }, + {'1': 'x', '3': 2, '4': 1, '5': 1, '10': 'x'}, + {'1': 'y', '3': 3, '4': 1, '5': 1, '10': 'y'}, + {'1': 'x1', '3': 4, '4': 1, '5': 1, '10': 'x1'}, + {'1': 'y1', '3': 5, '4': 1, '5': 1, '10': 'y1'}, + {'1': 'x2', '3': 6, '4': 1, '5': 1, '10': 'x2'}, + {'1': 'y2', '3': 7, '4': 1, '5': 1, '10': 'y2'}, + {'1': 'duration', '3': 8, '4': 1, '5': 3, '10': 'duration'}, + {'1': 'key_code', '3': 9, '4': 1, '5': 5, '10': 'keyCode'}, + {'1': 'key_action', '3': 10, '4': 1, '5': 5, '10': 'keyAction'}, + {'1': 'motion_action', '3': 11, '4': 1, '5': 5, '10': 'motionAction'}, + ], +}; + +/// Descriptor for `ControlMessage`. Decode as a `google.protobuf.DescriptorProto`. +final $typed_data.Uint8List controlMessageDescriptor = $convert.base64Decode( + 'Cg5Db250cm9sTWVzc2FnZRIxCgZhY3Rpb24YASABKA4yGS5jb20udHRzdGQuY29udHJvbC5BY3' + 'Rpb25SBmFjdGlvbhIMCgF4GAIgASgBUgF4EgwKAXkYAyABKAFSAXkSDgoCeDEYBCABKAFSAngx' + 'Eg4KAnkxGAUgASgBUgJ5MRIOCgJ4MhgGIAEoAVICeDISDgoCeTIYByABKAFSAnkyEhoKCGR1cm' + 'F0aW9uGAggASgDUghkdXJhdGlvbhIZCghrZXlfY29kZRgJIAEoBVIHa2V5Q29kZRIdCgprZXlf' + 'YWN0aW9uGAogASgFUglrZXlBY3Rpb24SIwoNbW90aW9uX2FjdGlvbhgLIAEoBVIMbW90aW9uQW' + 'N0aW9u'); diff --git a/webrtc_controller_flutter/lib/utils/control_commands.dart b/webrtc_controller_flutter/lib/utils/control_commands.dart index 9308527..ea7d69d 100644 --- a/webrtc_controller_flutter/lib/utils/control_commands.dart +++ b/webrtc_controller_flutter/lib/utils/control_commands.dart @@ -1,57 +1,59 @@ -import 'dart:convert'; +import 'package:fixnum/fixnum.dart'; +import 'package:webrtc_controller_flutter/proto/control_message.pb.dart'; /// 控制指令构造工具,对应 Android 端 RemoteTouchView 的指令格式。 /// -/// 指令通过 DataChannel 以 JSON 字符串发送,被控端(WebRTCControlled) -/// 的 InputCommandHandler 负责解析执行。 +/// 指令通过 DataChannel 以 protobuf 二进制([ControlMessage])发送, +/// 被控端(WebRTCControlled)的 InputCommandHandler 负责解析执行。 class ControlCommands { ControlCommands._(); /// 单击指令。坐标 x/y 为相对于屏幕的百分比(0.0 ~ 1.0)。 - static String touch(double x, double y) => jsonEncode({ - 'action': 'TOUCH', - 'x': x, - 'y': y, - }); + static ControlMessage touch(double x, double y) => ControlMessage( + action: Action.TOUCH, + x: x, + y: y, + ); /// 滑动指令。坐标为相对百分比,duration 为毫秒。 - static String swipe( + static ControlMessage swipe( double x1, double y1, double x2, double y2, int durationMs, ) => - jsonEncode({ - 'action': 'SWIPE', - 'x1': x1, - 'y1': y1, - 'x2': x2, - 'y2': y2, - 'duration': durationMs, - }); + ControlMessage( + action: Action.SWIPE, + x1: x1, + y1: y1, + x2: x2, + y2: y2, + duration: Int64(durationMs), + ); /// 按键指令。action: 0=按下, 1=抬起(对应 Android KeyEvent.ACTION_DOWN/UP)。 - static String key(int keyCode, int action) => jsonEncode({ - 'action': 'KEY', - 'keyCode': keyCode, - 'keyAction': action, - }); + static ControlMessage key(int keyCode, int action) => ControlMessage( + action: Action.KEY, + keyCode: keyCode, + keyAction: action, + ); /// 长按指令。坐标 x/y 为相对于屏幕的百分比(0.0 ~ 1.0)。 /// 对应 Android 端 RemoteTouchView.createLongPressCommand。 - static String longPress(double x, double y) => jsonEncode({ - 'action': 'LONG_PRESS', - 'x': x, - 'y': y, - }); + static ControlMessage longPress(double x, double y) => ControlMessage( + action: Action.LONG_PRESS, + x: x, + y: y, + ); /// 原始 MotionEvent 指令,用于实现“实时跟手”。 /// action: 0=DOWN, 1=UP, 2=MOVE(对应 Android MotionEvent.ACTION_XXX)。 - static String motionEvent(int action, double x, double y) => jsonEncode({ - 'action': 'MOTION_EVENT', - 'motionAction': action, - 'x': x, - 'y': y, - }); + static ControlMessage motionEvent(int action, double x, double y) => + ControlMessage( + action: Action.MOTION_EVENT, + motionAction: action, + x: x, + y: y, + ); } diff --git a/webrtc_controller_flutter/lib/webrtc/webrtc_controller.dart b/webrtc_controller_flutter/lib/webrtc/webrtc_controller.dart index 8cd88cf..d42e1a7 100644 --- a/webrtc_controller_flutter/lib/webrtc/webrtc_controller.dart +++ b/webrtc_controller_flutter/lib/webrtc/webrtc_controller.dart @@ -3,6 +3,7 @@ import 'package:flutter_webrtc/flutter_webrtc.dart'; import '../config/ice_servers.dart'; import '../models/signal_message.dart'; +import '../proto/control_message.pb.dart'; import '../signaling/signaling_client.dart'; /// 封装 WebRTC 连接逻辑(对应 Android 端 WebRtcClient)。 @@ -105,8 +106,19 @@ class WebRtcController { void _setupDataChannel(RTCDataChannel channel) { channel.onMessage = (RTCDataChannelMessage message) { // 处理来自被控端的消息(本控制端主要发送,此处仅做日志记录)。 - // ignore: avoid_print - print('DataChannel message: ${message.text}'); + if (message.isBinary) { + try { + final command = ControlMessage.fromBuffer(message.binary); + // ignore: avoid_print + print('DataChannel message: ${command.action}'); + } catch (e) { + // ignore: avoid_print + print('DataChannel message decode failed: $e'); + } + } else { + // ignore: avoid_print + print('DataChannel text message: ${message.text}'); + } }; channel.onDataChannelState = (RTCDataChannelState state) { // ignore: avoid_print @@ -171,11 +183,11 @@ class WebRtcController { await _pc?.addCandidate(candidate); } - /// 通过 DataChannel 发送控制指令(JSON 字符串)。 - void sendControlCommand(String commandJson) { + /// 通过 DataChannel 发送控制指令(protobuf 二进制)。 + void sendControlCommand(ControlMessage command) { if (_dataChannel?.state == RTCDataChannelState.RTCDataChannelOpen) { - debugPrint('WebRtcController: Sending command -> $commandJson'); - _dataChannel!.send(RTCDataChannelMessage(commandJson)); + debugPrint('WebRtcController: Sending command -> ${command.action}'); + _dataChannel!.send(RTCDataChannelMessage.fromBinary(command.writeToBuffer())); } } diff --git a/webrtc_controller_flutter/proto/control_message.proto b/webrtc_controller_flutter/proto/control_message.proto new file mode 100644 index 0000000..e72212d --- /dev/null +++ b/webrtc_controller_flutter/proto/control_message.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package com.ttstd.control; + +option java_package = "com.ttstd.control"; +option java_multiple_files = true; + +// 控制指令类型,对应原 JSON 字段 action。 +enum Action { + ACTION_UNKNOWN = 0; + TOUCH = 1; + SWIPE = 2; + KEY = 3; + LONG_PRESS = 4; + MOTION_EVENT = 5; +} + +// 通过 RTCDataChannel 传输的控制指令(protobuf 二进制)。 +// 坐标 x/y/x1/y1/x2/y2 均为相对屏幕的百分比,取值范围 0.0 ~ 1.0。 +message ControlMessage { + // 指令类型 + Action action = 1; + + // 单点坐标(TOUCH / LONG_PRESS / MOTION_EVENT) + double x = 2; + double y = 3; + + // 滑动起止坐标(SWIPE) + double x1 = 4; + double y1 = 5; + double x2 = 6; + double y2 = 7; + int64 duration = 8; + + // 按键(KEY):key_action 0=按下 1=抬起(对应 Android KeyEvent ACTION_DOWN/UP) + int32 key_code = 9; + int32 key_action = 10; + + // 原始动作(MOTION_EVENT):对应 Android MotionEvent ACTION_*(0=DOWN 1=UP 2=MOVE) + int32 motion_action = 11; +} diff --git a/webrtc_controller_flutter/pubspec.lock b/webrtc_controller_flutter/pubspec.lock index c11932e..965a81f 100644 --- a/webrtc_controller_flutter/pubspec.lock +++ b/webrtc_controller_flutter/pubspec.lock @@ -98,7 +98,7 @@ packages: source: hosted version: "2.2.0" fixnum: - dependency: transitive + dependency: "direct main" description: name: fixnum sha256: b6dc7065e46c974bc7c5f143080a6764ec7a4be6da1285ececdc37be96de53be @@ -323,6 +323,14 @@ packages: url: "https://pub.flutter-io.cn" source: hosted version: "2.1.8" + protobuf: + dependency: "direct main" + description: + name: protobuf + sha256: "75ec242d22e950bdcc79ee38dd520ce4ee0bc491d7fadc4ea47694604d22bf06" + url: "https://pub.flutter-io.cn" + source: hosted + version: "6.0.0" pub_semver: dependency: transitive description: diff --git a/webrtc_controller_flutter/pubspec.yaml b/webrtc_controller_flutter/pubspec.yaml index 2982b85..5319dcd 100644 --- a/webrtc_controller_flutter/pubspec.yaml +++ b/webrtc_controller_flutter/pubspec.yaml @@ -43,6 +43,8 @@ dependencies: # 设备 ID 生成 uuid: ^4.4.0 + protobuf: ^6.0.0 + fixnum: ^1.1.1 dev_dependencies: flutter_test: