From 41a514e9ccb5c9d56ce9ecdc894a978fe6f3dc43 Mon Sep 17 00:00:00 2001 From: TongTongStudio Date: Wed, 26 Aug 2026 06:16:44 +0800 Subject: [PATCH] =?UTF-8?q?feat(connection):=20=E6=B7=BB=E5=8A=A0=E8=BF=9C?= =?UTF-8?q?=E7=A8=8B=E6=8E=A7=E5=88=B6=E8=B7=AF=E7=94=B1=E5=B9=B6=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E7=89=88=E6=9C=AC=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 /connection 路由，从全局选中设备 SN 读取并渲染远程控制页 - 首页镜像操作改为静默下发 mirror 命令并跳转远程控制页 - 更新 AGENTS.md 版本管理策略为使用最新稳定版本 - 调整 pubspec.lock 中 hooks 与 record_use 版本 --- AGENTS.md | 12 +- lib/app/router/app_router.dart | 14 + .../connection/data/connection_providers.dart | 41 +++ .../connection/data/protobuf_codec.dart | 139 +++++++++ .../connection/data/signaling_client.dart | 86 ++++++ .../connection/data/webrtc_orchestrator.dart | 283 ++++++++++++++++++ .../connection/domain/connection_models.dart | 89 ++++++ .../presentation/connection_controller.dart | 153 ++++++++++ .../presentation/remote_control_page.dart | 260 ++++++++++++++++ lib/features/home/presentation/home_tab.dart | 14 +- pubspec.lock | 8 +- 11 files changed, 1087 insertions(+), 12 deletions(-) create mode 100644 lib/features/connection/data/connection_providers.dart create mode 100644 lib/features/connection/data/protobuf_codec.dart create mode 100644 lib/features/connection/data/signaling_client.dart create mode 100644 lib/features/connection/data/webrtc_orchestrator.dart create mode 100644 lib/features/connection/domain/connection_models.dart create mode 100644 lib/features/connection/presentation/connection_controller.dart create mode 100644 lib/features/connection/presentation/remote_control_page.dart diff --git a/AGENTS.md b/AGENTS.md index 5df64aa..31f276f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,8 +6,8 @@ ## 1. 技术栈 -- **语言：** Dart（SDK `^3.12.2`，严格空安全） -- **框架：** Flutter（**锁定版本，禁止使用 `flutter upgrade` 自由升级**，详见 [§11 版本管理](#11-版本管理)） +- **语言：** Dart（严格空安全，使用最新稳定版本） +- **框架：** Flutter（使用最新稳定版本） - **状态管理：** `flutter_riverpod`（v2.x，注解 + 代码生成） - **路由：** `go_router`（v14.x，含鉴权重定向，详见 [§9 路由鉴权](#9-路由鉴权)） - **网络请求：** `dio`（配合 `json_annotation` + `freezed` 进行 JSON 序列化） @@ -16,8 +16,6 @@ - **依赖注入：** Riverpod Providers（不使用 GetIt） - **国际化：** `flutter_localizations`（l10n / ARB 格式） -> ⚠️ **版本一致性约束**：不同开发者执行 `flutter upgrade` 的时间不同会导致 SDK 版本漂移，引发构建差异。Flutter 与 Dart SDK 版本必须以 CI / `.fvmrc` 为准，禁止在本地随意升级。 - --- ## 2. 目录结构 @@ -225,9 +223,9 @@ assets/ ## 11. 版本管理 -- **锁定 Flutter / Dart 版本**：在 `README` 与 CI 中明确记录 `flutter --version` 对应的版本号。 -- **推荐使用 FVM**：项目根放置 `.fvmrc`，团队成员统一 `fvm use`，禁止本地自由 `flutter upgrade`。 -- `pubspec.yaml` 的 `environment.sdk` 已锁定为 `^3.12.2`，依赖版本尽量用 caret 范围并定期 `flutter pub outdated` 审查。 +- **使用最新稳定版本**：保持 Flutter 与 Dart SDK 为最新稳定版，可随时执行 `flutter upgrade` 获取更新。 +- **依赖版本**：`pubspec.yaml` 的依赖版本尽量使用 caret 范围，并定期通过 `flutter pub outdated` 审查与升级到最新兼容版本。 +- **环境约束**：保持 `environment.sdk` 为合理的 Dart SDK 下限，确保与最新稳定版兼容。 --- diff --git a/lib/app/router/app_router.dart b/lib/app/router/app_router.dart index 8f4aa2f..51a1ab0 100644 --- a/lib/app/router/app_router.dart +++ b/lib/app/router/app_router.dart @@ -30,6 +30,7 @@ import '../../features/device/domain/device_models.dart'; import '../../features/device/presentation/device_bind_page.dart'; import '../../features/device/presentation/device_manage_page.dart'; import '../../features/device/presentation/device_scan_page.dart'; +import '../../features/connection/presentation/remote_control_page.dart'; /// 应用路由定义（GoRouter）。 /// @@ -140,6 +141,10 @@ GoRouter buildAppRouter() { initialIndex: (state.extra is int) ? state.extra as int : 0, ), ), + GoRoute( + path: '/connection', + builder: (context, state) => _RemoteControlRoute(), + ), StatefulShellRoute.indexedStack( builder: (context, state, shell) => HomePage(child: shell), branches: [ @@ -203,3 +208,12 @@ class _ScreenshotsRoute extends ConsumerWidget { return ScreenshotPage(screenshots: list, sn: sn ?? '', initialIndex: index); } } + +/// /connection 路由包装：从全局选中的设备 SN 读取后渲染远程控制页。 +class _RemoteControlRoute extends ConsumerWidget { + @override + Widget build(BuildContext context, WidgetRef ref) { + final sn = ref.watch(selectedDeviceSnProvider) ?? ''; + return RemoteControlPage(sn: sn); + } +} diff --git a/lib/features/connection/data/connection_providers.dart b/lib/features/connection/data/connection_providers.dart new file mode 100644 index 0000000..57cbd82 --- /dev/null +++ b/lib/features/connection/data/connection_providers.dart @@ -0,0 +1,41 @@ +import 'package:flutter_riverpod/flutter_riverpod.dart'; + +import '../../../app/constants/app_constants.dart'; +import '../../../core/storage/token_storage.dart'; +import '../presentation/connection_controller.dart'; +import 'signaling_client.dart'; + +/// 信令 WebSocket 地址（从 baseUrl 推导）。 +/// +/// baseUrl 形如 http://host:port/api/，信令端点固定为 /ws/signal（后端新增）。 +String get signalServerUrl { + final base = AppConstants.kBaseUrl; // e.g. http://192.168.100.244:8000/api/ + try { + final uri = Uri.parse(base); + final scheme = uri.scheme == 'https' ? 'wss' : 'ws'; + return '$scheme://${uri.authority}/ws/signal'; + } catch (_) { + return 'ws://localhost:8000/ws/signal'; + } +} + +/// 信令客户端（会话级单例）。 +final signalingClientProvider = Provider((ref) { + final client = SignalingClient(); + ref.onDispose(client.dispose); + return client; +}); + +/// client 端标识（用于信令 fromDeviceId）。 +/// +/// 实际身份由后端从 JWT 解析，这里仅作客户端本地标识，取设备 ID 或固定值均可。 +final clientNameProvider = Provider((ref) { + final deviceId = TokenStorage.deviceId; + return deviceId.isNotEmpty ? deviceId : 'client'; +}); + +/// 远程控制控制器（按目标设备 SN 隔离，管理连接状态机）。 +final connectionControllerProvider = + NotifierProvider( + ConnectionController.new, +); diff --git a/lib/features/connection/data/protobuf_codec.dart b/lib/features/connection/data/protobuf_codec.dart new file mode 100644 index 0000000..7a241bd --- /dev/null +++ b/lib/features/connection/data/protobuf_codec.dart @@ -0,0 +1,139 @@ +import 'dart:typed_data'; + +/// 控制指令 Action 枚举（与控制端 proto 的 Action 字段号一致）。 +class ActionType { + static const unknown = 0; + static const touch = 1; + static const swipe = 2; + static const key = 3; + static const longPress = 4; + static const motionEvent = 5; + static const setResolution = 6; + static const reportResolution = 7; + static const setStreamMode = 8; + static const reportStreamMode = 9; +} + +/// protobuf wire format 编码器。 +/// +/// 与控制端 `com.ttstd.control.ControlMessage`（protobuf 3）字段号完全对齐： +/// action=1, x=2, y=3, x1=4, y1=5, x2=6, y2=7, duration=8, +/// key_code=9, key_action=10, motion_action=11, width=12, height=13, +/// fps=14, stream_mode=15, supported_fps=16(repeated int32)。 +/// +/// 坐标均为相对屏幕百分比（0.0~1.0），被控端乘以真实分辨率还原像素。 +class ControlMessageEncoder { + final int action; + + // 单点坐标 + final double x; + final double y; + + // 滑动起止 + final double x1; + final double y1; + final double x2; + final double y2; + final int duration; + + // 按键 + final int keyCode; + final int keyAction; + + // 原始动作 + final int motionAction; + + // 分辨率/帧率 + final int width; + final int height; + final int fps; + final int streamMode; + + const ControlMessageEncoder({ + required this.action, + this.x = 0, + this.y = 0, + this.x1 = 0, + this.y1 = 0, + this.x2 = 0, + this.y2 = 0, + this.duration = 0, + this.keyCode = 0, + this.keyAction = 0, + this.motionAction = 0, + this.width = 0, + this.height = 0, + this.fps = 0, + this.streamMode = 0, + }); + + /// 编码为 protobuf 二进制字节数组。 + Uint8List encode() { + final out = BytesBuilder(copy: false); + + // 1: action (enum=int32) + _writeVarintField(out, 1, action); + + // 2: x, 3: y (double, wire type 1) + if (x != 0) _writeFixed64Field(out, 2, x); + if (y != 0) _writeFixed64Field(out, 3, y); + + // 4-7: swipe 起止 + if (x1 != 0) _writeFixed64Field(out, 4, x1); + if (y1 != 0) _writeFixed64Field(out, 5, y1); + if (x2 != 0) _writeFixed64Field(out, 6, x2); + if (y2 != 0) _writeFixed64Field(out, 7, y2); + + // 8: duration (int64) + if (duration != 0) _writeVarintField(out, 8, duration); + + // 9: key_code, 10: key_action (int32) + if (keyCode != 0) _writeVarintField(out, 9, keyCode); + if (keyAction != 0) _writeVarintField(out, 10, keyAction); + + // 11: motion_action + if (motionAction != 0) _writeVarintField(out, 11, motionAction); + + // 12-14: width/height/fps + if (width != 0) _writeVarintField(out, 12, width); + if (height != 0) _writeVarintField(out, 13, height); + if (fps != 0) _writeVarintField(out, 14, fps); + + // 15: stream_mode + if (streamMode != 0) _writeVarintField(out, 15, streamMode); + + return out.toBytes(); + } + + // ---- protobuf wire format 原语 ---- + + void _writeVarintField(BytesBuilder out, int field, int value) { + _writeTag(out, field, 0); // wire type 0 = varint + _writeVarint(out, value); + } + + void _writeFixed64Field(BytesBuilder out, int field, double value) { + _writeTag(out, field, 1); // wire type 1 = 64-bit (double) + // protobuf 的 fixed64/double 必须为 little-endian，否则设备端解析出错误值（如 NaN/Infinity） + final bd = ByteData(8); + bd.setFloat64(0, value, Endian.little); + out.add(bd.buffer.asUint8List()); + } + + void _writeTag(BytesBuilder out, int field, int wireType) { + _writeVarint(out, (field << 3) | wireType); + } + + void _writeVarint(BytesBuilder out, int value) { + // 转成无符号 64 位表示，配合逻辑右移 >>>，避免负数算术右移导致死循环（OOM）。 + var v = value.toUnsigned(64); + while (true) { + if ((v & ~0x7F) == 0) { + out.addByte(v & 0xFF); + break; + } + out.addByte((v & 0x7F) | 0x80); + v = v >>> 7; + } + } +} diff --git a/lib/features/connection/data/signaling_client.dart b/lib/features/connection/data/signaling_client.dart new file mode 100644 index 0000000..6465b96 --- /dev/null +++ b/lib/features/connection/data/signaling_client.dart @@ -0,0 +1,86 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:flutter/foundation.dart'; +import 'package:web_socket_channel/web_socket_channel.dart'; +import 'package:web_socket_channel/io.dart'; + +import '../../../core/storage/token_storage.dart'; +import '../domain/connection_models.dart'; + +/// WebRTC 信令 WebSocket 客户端（控制端）。 +/// +/// 遵循平台原有用户逻辑：控制端为家属端 client，握手时携带 +/// `Authorization: Bearer `，由后端信令服务识别 client 身份。 +class SignalingClient { + WebSocketChannel? _channel; + StreamSubscription? _subscription; + + /// 是否已连接（可复用，供上层判断）。 + bool get isConnected => _channel != null; + + /// 消息回调：上游处理 OFFER/ANSWER/ICE/REJECTED/OFFLINE 等。 + void Function(SignalMessage message)? onMessage; + + /// 连接状态回调。 + void Function(bool connected)? onConnectionChanged; + + void Function(String error)? onError; + + /// 建立信令连接。 + /// + /// [serverUrl] WebSocket 地址（如 ws://host:port/ws/signal）。 + /// [targetSn] 目标设备 SN（作为 query 参数传给服务端做绑定校验）。 + Future connect(String serverUrl, {String? targetSn}) async { + final token = TokenStorage.accessToken ?? ''; + + final uri = Uri.parse('$serverUrl?sn=$targetSn'); + _channel = IOWebSocketChannel.connect( + uri, + headers: {'Authorization': 'Bearer $token'}, + pingInterval: const Duration(seconds: 20), + ); + + _subscription = _channel!.stream.listen( + (data) => _handleRaw(data), + onError: (Object e) { + debugPrint('[webrtc] 信令错误: $e'); + onError?.call('信令连接异常'); + }, + onDone: () { + debugPrint('[webrtc] 信令连接关闭(serverUrl=$serverUrl)'); + onConnectionChanged?.call(false); + }, + ); + debugPrint('[webrtc] 信令已发起连接: $uri'); + } + + void _handleRaw(dynamic data) { + if (data is String) { + try { + final json = jsonDecode(data) as Map; + final msg = SignalMessage.fromJson(json); + debugPrint('[webrtc] 信令收到: type=${msg.type}, from=${msg.fromDeviceId}, to=${msg.toDeviceId}'); + onMessage?.call(msg); + } catch (e) { + debugPrint('[webrtc] 信令解析失败: $e'); + } + } + } + + /// 发送信令消息。 + void send(SignalMessage message) { + if (_channel == null) return; + _channel!.sink.add(jsonEncode(message.toJson())); + } + + /// 主动关闭连接。 + void dispose() { + _subscription?.cancel(); + _subscription = null; + try { + _channel?.sink.close(); + } catch (_) {} + _channel = null; + } +} diff --git a/lib/features/connection/data/webrtc_orchestrator.dart b/lib/features/connection/data/webrtc_orchestrator.dart new file mode 100644 index 0000000..213afd4 --- /dev/null +++ b/lib/features/connection/data/webrtc_orchestrator.dart @@ -0,0 +1,283 @@ +import 'dart:async'; +import 'dart:convert'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter_webrtc/flutter_webrtc.dart'; + +import 'protobuf_codec.dart'; +import 'signaling_client.dart'; +import '../domain/connection_models.dart'; + +/// WebRTC 编排器（控制端 / Offer 方）。 +/// +/// - 只收屏幕画面（RECV_ONLY video track）。 +/// - 创建 `control_channel` DataChannel，用于向被控端发送触摸/按键指令。 +/// - 与信令客户端配合完成 OFFER/ANSWER/ICE 握手。 +class WebRtcOrchestrator { + static const _dataChannelLabel = 'control_channel'; + // ICE 服务器（与 webrtc_controller_flutter 的 ice_servers.dart 完全一致）。 + static const _iceServers = [ + {'urls': 'stun:stun.l.google.com:19302'}, + {'urls': 'stun:175.178.213.60:3478'}, + { + 'urls': [ + 'turns:175.178.213.60:5349', + 'turn:175.178.213.60:3478?transport=tcp', + 'turn:175.178.213.60:3478', + ], + 'username': 'fanhuitong', + 'credential': 'Fan19961907..', + }, + {'urls': 'stun:47.242.112.133:3478'}, + { + 'urls': 'turn:47.242.112.133:3478', + 'username': 'ttstd', + 'credential': 'fanhuitong', + }, + { + 'urls': [ + 'stun:192.168.5.224:3478', + 'turn:192.168.5.224:3478', + ], + 'username': 'tt', + 'credential': 'fht', + }, + ]; + + final SignalingClient _signaling; + final String _targetSn; + final String _clientName; + + RTCPeerConnection? _pc; + RTCDataChannel? _controlChannel; + + /// 远程屏幕渲染器（供页面 RTCVideoView 使用）。 + final RTCVideoRenderer remoteRenderer = RTCVideoRenderer(); + + /// 接收到的被控端报告消息（如 REPORT_RESOLUTION）。 + void Function(Uint8List data)? onControlMessage; + + /// 连接状态变化（用于 UI 显示 "已连接"）。 + void Function(bool connected)? onConnectionState; + + /// 是否已建立媒体连接。 + bool get isConnected => + _pc?.connectionState == RTCPeerConnectionState.RTCPeerConnectionStateConnected; + + WebRtcOrchestrator({ + required SignalingClient signaling, + required String targetSn, + required String clientName, + }) : _signaling = signaling, + _targetSn = targetSn, + _clientName = clientName; // ignore: prefer_initializing_formals + + /// 发起连接：创建 PeerConnection + DataChannel + 发送 OFFER。 + Future start() async { + await _initPeerConnection(); + + // 创建 control_channel（有序、可靠，保证指令不丢） + final dataChannelInit = RTCDataChannelInit() + ..ordered = true + ..protocol = _dataChannelLabel; + _controlChannel = await _pc!.createDataChannel(_dataChannelLabel, dataChannelInit); + _controlChannel!.onMessage = (RTCDataChannelMessage message) { + if (message.isBinary) { + onControlMessage?.call(message.binary); + } + }; + _controlChannel!.onDataChannelState = (RTCDataChannelState state) { + debugPrint('[webrtc] control_channel state: $state'); + }; + + // 创建 Offer + final offer = await _pc!.createOffer(); + await _pc!.setLocalDescription(offer); + + // 通过信令发送 OFFER（带 username） + _signaling.send(SignalMessage( + type: SignalType.offer, + fromDeviceId: _clientName, + toDeviceId: _targetSn, + deviceType: 'CONTROLLER', + payload: jsonEncode({ + 'sdp': offer.sdp, + 'username': _clientName, + }), + )); + debugPrint('[webrtc] OFFER 已发送 → $_targetSn'); + } + + Future _initPeerConnection() async { + final config = { + 'iceServers': _iceServers, + 'sdpSemantics': 'unified-plan', + }; + _pc = await createPeerConnection(config); + + // 只收屏幕：添加一个 RECV_ONLY video transceiver + final transceiverInit = RTCRtpTransceiverInit(); + transceiverInit.direction = TransceiverDirection.RecvOnly; + await _pc!.addTransceiver( + kind: RTCRtpMediaType.RTCRtpMediaTypeVideo, + init: transceiverInit, + ); + + _pc!.onTrack = (RTCTrackEvent event) { + debugPrint('[webrtc] 收到远程屏幕轨道'); + final stream = event.streams.isNotEmpty ? event.streams.first : null; + if (stream != null) { + // RTCVideoRenderer 必须先 initialize() 再设置 srcObject，否则抛 + // "Call initialize before setting the stream" 异常，导致画面不渲染。 + remoteRenderer.initialize().then((_) { + remoteRenderer.srcObject = stream; + }).catchError((e) { + debugPrint('[webrtc] renderer initialize error: $e'); + }); + onConnectionState?.call(true); + } + }; + + _pc!.onConnectionState = (RTCPeerConnectionState state) { + debugPrint('[webrtc] connection state: ${state.name}'); + if (state == RTCPeerConnectionState.RTCPeerConnectionStateConnected) { + onConnectionState?.call(true); + } else if (state == RTCPeerConnectionState.RTCPeerConnectionStateDisconnected || + state == RTCPeerConnectionState.RTCPeerConnectionStateFailed || + state == RTCPeerConnectionState.RTCPeerConnectionStateClosed) { + onConnectionState?.call(false); + } + }; + + _pc!.onIceCandidate = (RTCIceCandidate candidate) { + _signaling.send(SignalMessage( + type: SignalType.iceCandidate, + fromDeviceId: _clientName, + toDeviceId: _targetSn, + deviceType: 'CONTROLLER', + payload: jsonEncode({ + 'sdpMid': candidate.sdpMid, + 'sdpMLineIndex': candidate.sdpMLineIndex, + 'candidate': candidate.candidate, + }), + )); + }; + } + + /// 处理信令消息：ANSWER / ICE_CANDIDATE。 + Future handleSignal(SignalMessage message) async { + if (_pc == null) { + debugPrint('[webrtc] handleSignal skipped: pc not ready, type=${message.type}'); + return; + } + switch (message.type) { + case SignalType.answer: + final payload = jsonDecode(message.payload ?? '{}') as Map; + final sdp = payload['sdp'] as String?; + if (sdp != null) { + debugPrint('[webrtc] 收到 ANSWER，setRemoteDescription'); + await _pc!.setRemoteDescription(RTCSessionDescription(sdp, 'answer')); + } + break; + case SignalType.iceCandidate: + final payload = jsonDecode(message.payload ?? '{}') as Map; + final candidate = RTCIceCandidate( + payload['candidate'] as String? ?? '', + payload['sdpMid'] as String? ?? '', + (payload['sdpMLineIndex'] as num?)?.toInt() ?? 0, + ); + await _pc!.addCandidate(candidate); + break; + } + } + + /// 发送控制指令（protobuf 二进制）。 + void sendCommand(Uint8List bytes) { + if (_controlChannel != null && + _controlChannel!.state == RTCDataChannelState.RTCDataChannelOpen) { + _controlChannel!.send(RTCDataChannelMessage.fromBinary(bytes)); + } + } + + /// 发送一次点击（相对坐标 0~1）。 + void sendTap(double relX, double relY) { + final bytes = ControlMessageEncoder( + action: ActionType.touch, + x: relX, + y: relY, + ).encode(); + sendCommand(bytes); + } + + /// 发送长按。 + void sendLongPress(double relX, double relY) { + final bytes = ControlMessageEncoder( + action: ActionType.longPress, + x: relX, + y: relY, + ).encode(); + sendCommand(bytes); + } + + /// 发送滑动。 + void sendSwipe(double relX1, double relY1, double relX2, double relY2, {int duration = 300}) { + final bytes = ControlMessageEncoder( + action: ActionType.swipe, + x1: relX1, + y1: relY1, + x2: relX2, + y2: relY2, + duration: duration, + ).encode(); + sendCommand(bytes); + } + + /// 发送实时触摸事件（MOTION_EVENT，跟手用）。action：0=DOWN,1=UP,2=MOVE。 + void sendMotionEvent(int action, double relX, double relY) { + final bytes = ControlMessageEncoder( + action: ActionType.motionEvent, + x: relX, + y: relY, + motionAction: action, + ).encode(); + sendCommand(bytes); + } + + /// 发送按键（keyAction：0=down，1=up；-1=点按）。 + void sendKey(int keyCode, {int keyAction = -1}) { + final bytes = ControlMessageEncoder( + action: ActionType.key, + keyCode: keyCode, + keyAction: keyAction, + ).encode(); + sendCommand(bytes); + } + + /// 请求切换被控端采集分辨率。 + void sendSetResolution({int width = 0, int height = 0, int fps = 0}) { + final bytes = ControlMessageEncoder( + action: ActionType.setResolution, + width: width, + height: height, + fps: fps, + ).encode(); + sendCommand(bytes); + } + + /// 关闭连接。 + Future dispose() async { + try { + if (_controlChannel != null) { + await _controlChannel!.close(); + } + if (_pc != null) { + await _pc!.close(); + } + await remoteRenderer.dispose(); + } catch (e) { + debugPrint('[webrtc] dispose error: $e'); + } + _controlChannel = null; + _pc = null; + } +} diff --git a/lib/features/connection/domain/connection_models.dart b/lib/features/connection/domain/connection_models.dart new file mode 100644 index 0000000..f979e4a --- /dev/null +++ b/lib/features/connection/domain/connection_models.dart @@ -0,0 +1,89 @@ +import 'package:flutter/foundation.dart'; + +/// WebRTC 远程控制会话状态。 +enum ConnectionState { + idle, + connecting, + connected, + ended, + error, +} + +/// 信令消息类型（与后端 /ws/signal 及移动端 SignalMessage 对齐）。 +class SignalType { + static const offer = 'OFFER'; + static const answer = 'ANSWER'; + static const iceCandidate = 'ICE_CANDIDATE'; + static const registerSuccess = 'REGISTER_SUCCESS'; + static const connectionRejected = 'CONNECTION_REJECTED'; + static const targetOffline = 'TARGET_OFFLINE'; + static const requestTimeout = 'REQUEST_TIMEOUT'; + static const forceLogout = 'FORCE_LOGOUT'; + static const ping = 'PING'; +} + +/// 信令消息模型。 +@immutable +class SignalMessage { + final String type; + final String? fromDeviceId; + final String? toDeviceId; + final String? deviceType; + final String? payload; + + const SignalMessage({ + required this.type, + this.fromDeviceId, + this.toDeviceId, + this.deviceType, + this.payload, + }); + + factory SignalMessage.fromJson(Map json) { + return SignalMessage( + type: json['type'] as String? ?? '', + fromDeviceId: json['fromDeviceId'] as String?, + toDeviceId: json['toDeviceId'] as String?, + deviceType: json['deviceType'] as String?, + payload: json['payload'] as String?, + ); + } + + Map toJson() { + return { + 'type': type, + if (fromDeviceId != null) 'fromDeviceId': fromDeviceId, + if (toDeviceId != null) 'toDeviceId': toDeviceId, + if (deviceType != null) 'deviceType': deviceType, + if (payload != null) 'payload': payload, + }; + } +} + +/// 远程控制会话会话上下文（当前控制的目标设备与连接身份）。 +@immutable +class ConnectionSession { + final String targetSn; + final String clientName; + final ConnectionState state; + final String? errorMessage; + + const ConnectionSession({ + required this.targetSn, + required this.clientName, + this.state = ConnectionState.idle, + this.errorMessage, + }); + + ConnectionSession copyWith({ + ConnectionState? state, + String? errorMessage, + }) { + return ConnectionSession( + targetSn: targetSn, + clientName: clientName, + state: state ?? this.state, + errorMessage: errorMessage ?? this.errorMessage, + ); + } +} diff --git a/lib/features/connection/presentation/connection_controller.dart b/lib/features/connection/presentation/connection_controller.dart new file mode 100644 index 0000000..3ebfec6 --- /dev/null +++ b/lib/features/connection/presentation/connection_controller.dart @@ -0,0 +1,153 @@ +import 'package:flutter/foundation.dart'; +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:flutter_webrtc/flutter_webrtc.dart'; + +import '../data/connection_providers.dart'; +import '../data/webrtc_orchestrator.dart'; +import '../domain/connection_models.dart'; + +/// 远程控制会话状态（供 UI 渲染）。 +class ConnectionSessionState { + final ConnectionState state; + final String? errorMessage; + final bool hasVideo; + + const ConnectionSessionState({ + this.state = ConnectionState.idle, + this.errorMessage, + this.hasVideo = false, + }); + + ConnectionSessionState copyWith({ + ConnectionState? state, + String? errorMessage, + bool? hasVideo, + }) { + return ConnectionSessionState( + state: state ?? this.state, + errorMessage: errorMessage ?? this.errorMessage, + hasVideo: hasVideo ?? this.hasVideo, + ); + } +} + +/// 远程控制控制器：管理连接状态机（idle → connecting → connected → ended）。 +class ConnectionController extends Notifier { + WebRtcOrchestrator? _orchestrator; + + @override + ConnectionSessionState build() { + return const ConnectionSessionState(); + } + + /// 目标设备 SN（页面初始化时设置）。 + String? _targetSn; + set targetSn(String? value) => _targetSn = value; + + /// 远程屏幕渲染器（供页面 RTCVideoView 渲染）。 + RTCVideoRenderer? get remoteRenderer => _orchestrator?.remoteRenderer; + + /// 发起连接。 + Future connect() async { + final sn = _targetSn; + if (sn == null || sn.isEmpty) { + state = const ConnectionSessionState( + state: ConnectionState.error, + errorMessage: '缺少目标设备', + ); + return; + } + + final signaling = ref.read(signalingClientProvider); + final clientName = ref.read(clientNameProvider); + + state = const ConnectionSessionState(state: ConnectionState.connecting); + + _orchestrator = WebRtcOrchestrator( + signaling: signaling, + targetSn: sn, + clientName: clientName, + ); + _orchestrator!.onConnectionState = (connected) { + _onRtcState(connected); + }; + + signaling.onMessage = (message) { + // 先透传给 WebRTC 编排器处理（ANSWER/ICE 等），确保握手能继续 + _orchestrator?.handleSignal(message); + // 仅"明确被拒"才进入错误态；TARGET_OFFLINE 可能因握手时序/目标会话短暂未就绪 + // 而误报（如设备端刚连上信令、正在应答），不立即判定失败，避免闪断"设备不在线"。 + if (message.type == SignalType.connectionRejected) { + state = ConnectionSessionState( + state: ConnectionState.error, + errorMessage: '连接被拒绝', + ); + } + }; + signaling.onError = (error) { + state = ConnectionSessionState( + state: ConnectionState.error, + errorMessage: error, + ); + }; + + try { + await signaling.connect(signalServerUrl, targetSn: sn); + await _orchestrator!.start(); + } catch (e) { + state = ConnectionSessionState( + state: ConnectionState.error, + errorMessage: '连接失败: $e', + ); + } + } + + void _onRtcState(bool connected) { + if (connected) { + state = const ConnectionSessionState( + state: ConnectionState.connected, + hasVideo: true, + ); + } else if (state.state == ConnectionState.connected) { + state = const ConnectionSessionState(state: ConnectionState.ended); + } + } + + /// 发送点击（相对坐标 0~1）。 + void sendTap(double relX, double relY) { + debugPrint('[touch] TAP rel=(${relX.toStringAsFixed(3)}, ${relY.toStringAsFixed(3)})'); + _orchestrator?.sendTap(relX, relY); + } + + void sendLongPress(double relX, double relY) { + debugPrint('[touch] LONG_PRESS rel=(${relX.toStringAsFixed(3)}, ${relY.toStringAsFixed(3)})'); + _orchestrator?.sendLongPress(relX, relY); + } + + void sendSwipe(double relX1, double relY1, double relX2, double relY2) { + debugPrint('[touch] SWIPE rel=(${relX1.toStringAsFixed(3)}, ${relY1.toStringAsFixed(3)}) -> (${relX2.toStringAsFixed(3)}, ${relY2.toStringAsFixed(3)})'); + _orchestrator?.sendSwipe(relX1, relY1, relX2, relY2); + } + + /// 发送实时触摸事件（MOTION_EVENT，跟手用）。action：0=DOWN,1=UP,2=MOVE。 + void sendMotionEvent(int action, double relX, double relY) { + debugPrint('[touch] MOTION action=$action rel=(${relX.toStringAsFixed(3)}, ${relY.toStringAsFixed(3)})'); + _orchestrator?.sendMotionEvent(action, relX, relY); + } + + void sendKey(int keyCode) { + _orchestrator?.sendKey(keyCode); + } + + void sendSetResolution({int width = 0, int height = 0, int fps = 0}) { + _orchestrator?.sendSetResolution(width: width, height: height, fps: fps); + } + + /// 关闭并清理。 + Future disconnect() async { + await _orchestrator?.dispose(); + _orchestrator = null; + ref.read(signalingClientProvider).dispose(); + state = const ConnectionSessionState(state: ConnectionState.ended); + } +} diff --git a/lib/features/connection/presentation/remote_control_page.dart b/lib/features/connection/presentation/remote_control_page.dart new file mode 100644 index 0000000..3df1364 --- /dev/null +++ b/lib/features/connection/presentation/remote_control_page.dart @@ -0,0 +1,260 @@ +import 'package:cupertino_ui/cupertino_ui.dart'; +import 'package:flutter_webrtc/flutter_webrtc.dart'; +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:material_ui/material_ui.dart'; + +import '../data/connection_providers.dart'; +import '../domain/connection_models.dart' as connection; +import 'connection_controller.dart'; + +/// 远程控制页面：实时观看老人设备屏幕，并通过手势向设备下发控制指令。 +/// +/// 使用说明： +/// - 单指点击 = 点击 +/// - 长按 = 长按 +/// - 单指拖动 = 滑动 +/// 坐标归一化为 0.0~1.0（相对屏幕），由被控端还原像素。 +class RemoteControlPage extends ConsumerStatefulWidget { + final String sn; + + const RemoteControlPage({super.key, required this.sn}); + + @override + ConsumerState createState() => _RemoteControlPageState(); +} + +class _RemoteControlPageState extends ConsumerState { + @override + void initState() { + super.initState(); + final controller = ref.read(connectionControllerProvider.notifier); + controller.targetSn = widget.sn; + // 等一帧，确保 provider 已就绪 + WidgetsBinding.instance.addPostFrameCallback((_) { + controller.connect(); + }); + } + + @override + Widget build(BuildContext context) { + final session = ref.watch(connectionControllerProvider); + final renderer = ref.read(connectionControllerProvider.notifier).remoteRenderer; + final connected = session.state == connection.ConnectionState.connected; + + return CupertinoPageScaffold( + navigationBar: CupertinoNavigationBar( + middle: const Text('远程控制'), + trailing: CupertinoButton( + padding: EdgeInsets.zero, + onPressed: () { + ref.read(connectionControllerProvider.notifier).disconnect(); + Navigator.of(context).maybePop(); + }, + child: const Text('退出'), + ), + ), + child: SafeArea( + child: Column( + children: [ + Expanded( + child: _buildVideo(renderer, connected, session), + ), + _buildControlsBar(session), + ], + ), + ), + ); + } + + Widget _buildVideo( + RTCVideoRenderer? renderer, + bool connected, + ConnectionSessionState session, + ) { + if (session.state == connection.ConnectionState.connecting) { + return const Center(child: CupertinoActivityIndicator()); + } + if (session.state == connection.ConnectionState.error) { + return Center( + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + const Icon(CupertinoIcons.exclamationmark_circle, + color: CupertinoColors.systemRed, size: 48), + const SizedBox(height: 12), + Text(session.errorMessage ?? '连接失败', textAlign: TextAlign.center), + ], + ), + ); + } + if (!connected || renderer == null) { + return const Center(child: Text('正在等待画面...')); + } + // 远程画面：用 LayoutBuilder 获取视频区域真实尺寸（坐标基准）， + // 用 Listener 捕获原始指针事件（避免 GestureDetector 手势竞争导致坐标错乱）。 + return LayoutBuilder( + builder: (context, constraints) { + final vw = constraints.maxWidth; + final vh = constraints.maxHeight; + return Listener( + behavior: HitTestBehavior.opaque, + onPointerDown: (e) => _onPointerDown(e.localPosition, vw, vh), + onPointerMove: (e) => _onPointerMove(e.localPosition, vw, vh), + onPointerUp: (e) => _onPointerUp(e.localPosition, vw, vh), + onPointerCancel: (e) => _onPointerCancel(), + child: Container( + color: Colors.black, + alignment: Alignment.center, + child: RTCVideoView( + renderer, + objectFit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain, + ), + ), + ); + }, + ); + } + + // ---- 基于 Listener 的手势处理（参考源实现 remote_touch_view.dart）---- + + /// 触摸起点（本地坐标）与按下时间。 + Offset? _downPos; + DateTime? _downTime; + /// 长按超时（400ms 后视为长按）。 + static const _longPressTimeout = Duration(milliseconds: 400); + /// 点击判定阈值：相对坐标 2%。 + static const _tapSlop = 0.02; + /// 实时 MOTION_EVENT 采样节流（16ms）。 + static const _sampleIntervalMs = 16; + DateTime? _lastMoveSample; + + /// 归一化：本地坐标 / 视频区域尺寸，clamp 0~1。 + double _rx(double x, double w) => w <= 0 ? 0 : (x / w).clamp(0.0, 1.0); + double _ry(double y, double h) => h <= 0 ? 0 : (y / h).clamp(0.0, 1.0); + + void _onPointerDown(Offset local, double vw, double vh) { + _downPos = local; + _downTime = DateTime.now(); + _lastMoveSample = null; + ref.read(connectionControllerProvider.notifier) + .sendMotionEvent(0, _rx(local.dx, vw), _ry(local.dy, vh)); // ACTION_DOWN + } + + void _onPointerMove(Offset local, double vw, double vh) { + final down = _downPos; + if (down == null) return; + // 节流采样，减少指令洪泛 + final now = DateTime.now(); + final last = _lastMoveSample; + if (last != null && now.difference(last).inMilliseconds < _sampleIntervalMs) return; + _lastMoveSample = now; + ref.read(connectionControllerProvider.notifier) + .sendMotionEvent(2, _rx(local.dx, vw), _ry(local.dy, vh)); // ACTION_MOVE + } + + void _onPointerUp(Offset local, double vw, double vh) { + final down = _downPos; + final downTime = _downTime; + _downPos = null; + _downTime = null; + _lastMoveSample = null; + if (down == null || vw <= 0 || vh <= 0) return; + + final controller = ref.read(connectionControllerProvider.notifier); + final duration = DateTime.now().difference(downTime ?? DateTime.now()); + final dx = (_rx(local.dx, vw) - _rx(down.dx, vw)).abs(); + final dy = (_ry(local.dy, vh) - _ry(down.dy, vh)).abs(); + + // 先补发 ACTION_UP，结束触摸 + controller.sendMotionEvent(1, _rx(local.dx, vw), _ry(local.dy, vh)); + + // 位移很小且时间短 → 点击；否则 → 滑动；长按由按下时长判定 + if (duration >= _longPressTimeout && dx < _tapSlop && dy < _tapSlop) { + controller.sendLongPress(_rx(down.dx, vw), _ry(down.dy, vh)); + } else if (dx < _tapSlop && dy < _tapSlop) { + controller.sendTap(_rx(down.dx, vw), _ry(down.dy, vh)); + } else { + controller.sendSwipe( + _rx(down.dx, vw), + _ry(down.dy, vh), + _rx(local.dx, vw), + _ry(local.dy, vh), + ); + } + } + + void _onPointerCancel() { + _downPos = null; + _downTime = null; + _lastMoveSample = null; + } + + Widget _buildControlsBar(ConnectionSessionState session) { + final connected = session.state == connection.ConnectionState.connected; + return Container( + padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 10), + decoration: BoxDecoration( + color: CupertinoColors.systemGroupedBackground, + border: Border(top: BorderSide(color: CupertinoColors.separator, width: 0.5)), + ), + child: Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + _CtrlButton( + icon: CupertinoIcons.speaker_slash_fill, + label: '静音', + enabled: connected, + onTap: () {}, + ), + _CtrlButton( + icon: CupertinoIcons.home, + label: '主页', + enabled: connected, + onTap: () => ref.read(connectionControllerProvider.notifier).sendKey(3), // KEYCODE_HOME + ), + _CtrlButton( + icon: CupertinoIcons.arrow_left, + label: '返回', + enabled: connected, + onTap: () => ref.read(connectionControllerProvider.notifier).sendKey(4), // KEYCODE_BACK + ), + _CtrlButton( + icon: CupertinoIcons.rectangle_grid_2x2, + label: '最近', + enabled: connected, + onTap: () => ref.read(connectionControllerProvider.notifier).sendKey(187), // KEYCODE_APP_SWITCH + ), + ], + ), + ); + } +} + +class _CtrlButton extends StatelessWidget { + final IconData icon; + final String label; + final bool enabled; + final VoidCallback onTap; + + const _CtrlButton({ + required this.icon, + required this.label, + required this.enabled, + required this.onTap, + }); + + @override + Widget build(BuildContext context) { + return GestureDetector( + onTap: enabled ? onTap : null, + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Icon(icon, size: 26, color: enabled ? CupertinoColors.activeBlue : CupertinoColors.systemGrey3), + const SizedBox(height: 4), + Text(label, style: const TextStyle(fontSize: 12)), + ], + ), + ); + } +} diff --git a/lib/features/home/presentation/home_tab.dart b/lib/features/home/presentation/home_tab.dart index b8a2a91..f6b6b4a 100644 --- a/lib/features/home/presentation/home_tab.dart +++ b/lib/features/home/presentation/home_tab.dart @@ -822,7 +822,19 @@ class HomeTab extends ConsumerWidget { label: l10n.cfMirror, size: 50, iconSize: 20, - onTap: () => _runDeviceOp(context, ref, l10n, sn, 'mirror'), + // 远程控制：静默触发设备端启动屏幕采集（HTTP mirror 命令，不弹窗）， + // 直接进入 WebRTC 远程控制页建立实时画面与控制。 + onTap: () async { + if (sn != null && sn.isNotEmpty) { + // 静默下发 mirror，触发设备端启动屏幕采集；失败也不阻断进入控制页 + await ref + .read(deviceRepositoryProvider) + .sendDeviceOp(sn, 'mirror'); + } + if (context.mounted) { + context.push('/connection'); + } + }, ), ), Expanded( diff --git a/pubspec.lock b/pubspec.lock index a569f06..370abb1 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -529,10 +529,10 @@ packages: dependency: transitive description: name: hooks - sha256: eaac480a35ec0814146c2c48d96aaa829e0e44a7662c88ae84c9edf4bc35651f + sha256: "9a62a50b50b769a737bc0a8ff381f333529df3ab746b2f6b02e83760231455ba" url: "https://pub.dev" source: hosted - version: "2.2.0" + version: "2.0.2" http: dependency: transitive description: @@ -1017,10 +1017,10 @@ packages: dependency: transitive description: name: record_use - sha256: "1cb8564af8d43b464294411db9217f5ec04891c6f22ee2c32d73ae05e88a6bd2" + sha256: "2551bd8eecfe95d14ae75f6021ad0248be5c27f138c2ec12fcb52b500b3ba1ed" url: "https://pub.dev" source: hosted - version: "1.1.1" + version: "0.6.0" riverpod: dependency: transitive description: