feat(device): 远程协助增加控制按键、串流统计与分辨率帧率切换

This commit is contained in:
tongtong committed 2026-11-13 17:57:04 +08:00
1 parent fd3615407c
commit aec3317180
2 files changed
+382 -6

No files matched your search

@@ -37,8 +37,105 @@ export function readTouchMode(data: ArrayBuffer): TouchMode {
return 0;
}
export type ResolutionReport = {
width: number;
height: number;
fps: number;
supportedFps: number[];
};
/** 解析被控端 REPORT_RESOLUTION;非该消息返回 undefined。 */
export function readResolutionReport(data: ArrayBuffer): ResolutionReport | undefined {
const bytes = new Uint8Array(data);
let position = 0;
let action = 0;
let width = 0;
let height = 0;
let fps = 0;
const supportedFps: number[] = [];
function readVarint() {
let value = 0;
for (let shift = 0; shift <= 28 && position < bytes.length; shift += 7) {
const byte = bytes[position++];
value += (byte & 127) * 2 ** shift;
if ((byte & 128) === 0) return value;
}
throw new Error("Invalid resolution report");
}
try {
while (position < bytes.length) {
const tag = readVarint();
const field = tag >>> 3;
switch (tag & 7) {
case 0: {
const value = readVarint();
if (field === 1) action = value;
else if (field === 12) width = value;
else if (field === 13) height = value;
else if (field === 14) fps = value;
else if (field === 16) supportedFps.push(value);
break;
}
case 2: {
const length = readVarint();
const end = position + length;
if (end > bytes.length) return undefined;
if (field === 16) {
while (position < end) supportedFps.push(readVarint());
}
position = end;
break;
}
default:
return undefined;
}
}
} catch {
return undefined;
}
return action === 7 && width > 0 && height > 0 ? { width, height, fps, supportedFps } : undefined;
}
/** 编码 SET_RESOLUTION;width<=0 表示原始分辨率,fps<=0 表示沿用当前帧率。 */
export function encodeResolution(width: number, height: number, fps: number) {
const bytes = new Uint8Array(32);
let offset = 0;
function varint(field: number, value: number) {
bytes[offset++] = field << 3;
while (value >= 128) {
bytes[offset++] = (value & 127) | 128;
value >>>= 7;
}
bytes[offset++] = value;
}
varint(1, 6);
varint(12, width);
varint(13, height);
varint(14, fps);
return bytes.slice(0, offset);
}
export type Point = { x: number; y: number };
/** 编码 KEY 指令;keyAction 传 -1 表示按下并立即抬起。 */
export function encodeKey(keyCode: number, keyAction: number) {
const bytes = new Uint8Array(16);
let offset = 0;
function varint(field: number, value: number) {
bytes[offset++] = field << 3;
while (value >= 128 || value < 0) {
bytes[offset++] = (value & 127) | 128;
value >>>= 7;
}
bytes[offset++] = value;
}
varint(1, 3);
varint(9, keyCode);
varint(10, keyAction);
return bytes.slice(0, offset);
}
export function encodeTouch(action: number, start: Point, end?: Point, detail?: number) {
const bytes = new Uint8Array(48);
const view = new DataView(bytes.buffer);
+285 -6
View File
@@ -27,6 +27,44 @@
</el-space>
<p>管理员可直接连接在线设备,无需老人逐次确认。</p>
<p role="status">{{ status }}</p>
<div v-if="stats" class="stats-bar">
<span>
分辨率
<b>{{ stats.resolution }}</b>
</span>
<span>
帧率
<b>{{ stats.fps }}</b>
</span>
<span>
编码
<b>{{ stats.codec }}</b>
</span>
<span>
延迟
<b>{{ stats.latency }}</b>
</span>
<span>
抖动
<b>{{ stats.jitter }}</b>
</span>
<span>
丢帧
<b>{{ stats.framesDropped }}</b>
</span>
<span>
丢包
<b>{{ stats.packetLoss }}</b>
</span>
<span>
下载
<b>{{ stats.downSpeed }}</b>
</span>
<span>
时长
<b>{{ stats.duration }}</b>
</span>
</div>
<video
ref="video"
class="remote-screen"
@@ -39,6 +77,45 @@
@pointercancel="pointerEnd($event, true)"
@lostpointercapture="pointerEnd($event, true)"
/>
<div class="stream-controls">
<span>分辨率</span>
<el-select
v-model="selectedWidth"
:disabled="!channelOpen"
style="width: 130px"
@change="changeResolution"
>
<el-option
v-for="preset in resolutions"
:key="preset.value"
:label="preset.label"
:value="preset.value"
/>
</el-select>
<span>帧率</span>
<el-select
v-model="selectedFps"
:disabled="!channelOpen"
style="width: 110px"
@change="changeFps"
>
<el-option v-if="selectedFps === 0" :value="0" label="--" disabled />
<el-option v-for="fps in fpsOptions" :key="fps" :label="`${fps} fps`" :value="fps" />
</el-select>
</div>
<div class="key-row">
<el-button
v-for="key in keys"
:key="key.code"
:disabled="!channelOpen"
@pointerdown.prevent="keyDown(key.code)"
@pointerup.prevent="keyUp(key.code)"
@pointerleave.prevent="keyUp(key.code)"
@pointercancel.prevent="keyUp(key.code)"
>
{{ key.label }}
</el-button>
</div>
</el-card>
</div>
</template>
@@ -48,8 +125,31 @@ import { onBeforeUnmount, onDeactivated, ref } from "vue";
import DeviceAPI from "@/api/device";
import type { DeviceItem } from "@/api/device/types";
import { openRemoteControlSignal } from "@/api/remote-control";
import { encodeTouch, readTouchMode } from "./controlProtocol";
import type { Point, TouchMode } from "./controlProtocol";
import {
encodeKey,
encodeResolution,
encodeTouch,
readResolutionReport,
readTouchMode,
} from "./controlProtocol";
import type { Point, ResolutionReport, TouchMode } from "./controlProtocol";
const resolutions = [
{ label: "原始画质", value: 0 },
{ label: "1080P", value: 1920 },
{ label: "720P", value: 1280 },
{ label: "480P", value: 854 },
];
const keys = [
{ label: "主页", code: 3 },
{ label: "返回", code: 4 },
{ label: "多任务", code: 187 },
{ label: "菜单", code: 82 },
{ label: "音量 +", code: 24 },
{ label: "音量 -", code: 25 },
{ label: "电源", code: 26 },
];
interface Signal {
type: string;
@@ -63,6 +163,12 @@ const sn = ref("");
const devices = ref<DeviceItem[]>([]);
const searching = ref(false);
const active = ref(false);
const channelOpen = ref(false);
const stats = ref<Record<string, string>>();
const selectedWidth = ref(0);
const selectedFps = ref(0);
const fpsOptions = ref([15, 24, 30, 60]);
let currentResolution: { width: number; height: number } | undefined;
const status = ref("请选择要协助的设备");
const video = ref<HTMLVideoElement>();
let searchVersion = 0;
@@ -71,7 +177,12 @@ let socket: WebSocket | undefined;
let peer: RTCPeerConnection | undefined;
let channel: RTCDataChannel | undefined;
let timer: ReturnType<typeof setTimeout> | undefined;
let statsTimer: ReturnType<typeof setInterval> | undefined;
let connectedAt = 0;
let lastStatsAt = 0;
let lastBytes = 0;
let mode: TouchMode = 0;
const pressedKeys = new Set<number>();
let gesture:
{ pointer: number; start: Point; time: number; lastMove: number; mode: TouchMode } | undefined;
@@ -97,8 +208,20 @@ function loadOptions(open: boolean) {
function stop(reason: string) {
version++;
clearTimeout(timer);
clearInterval(statsTimer);
statsTimer = undefined;
stats.value = undefined;
connectedAt = 0;
lastStatsAt = 0;
lastBytes = 0;
currentResolution = undefined;
selectedWidth.value = 0;
selectedFps.value = 0;
fpsOptions.value = [15, 24, 30, 60];
gesture = undefined;
mode = 0;
pressedKeys.clear();
channelOpen.value = false;
channel?.close();
channel = undefined;
peer?.close();
@@ -150,10 +273,7 @@ async function start() {
if (message.type === "REGISTER_SUCCESS" && !peer) {
const relay = JSON.parse(message.payload || "{}") as RTCIceServer;
const connection = new RTCPeerConnection({
iceServers: [
{ urls: "stun:stun.l.google.com:19302" },
...(relay.urls ? [relay] : []),
],
iceServers: [{ urls: "stun:47.242.112.133:3478" }, ...(relay.urls ? [relay] : [])],
});
peer = connection;
connection.addTransceiver("video", { direction: "recvonly" });
@@ -163,9 +283,17 @@ async function start() {
});
channel = control;
control.binaryType = "arraybuffer";
control.onopen = () => {
if (version === attempt) channelOpen.value = true;
};
control.onclose = () => {
if (version === attempt) channelOpen.value = false;
};
control.onmessage = (report) => {
if (version === attempt && report.data instanceof ArrayBuffer) {
mode = readTouchMode(report.data) || mode;
const resolution = readResolutionReport(report.data);
if (resolution) applyResolution(resolution);
}
};
connection.ontrack = ({ streams, track }) => {
@@ -177,6 +305,11 @@ async function start() {
if (connection.connectionState === "connected") {
clearTimeout(timer);
status.value = "协助已连接";
connectedAt = Date.now();
lastStatsAt = 0;
lastBytes = 0;
clearInterval(statsTimer);
statsTimer = setInterval(() => void collectStats(attempt), 1000);
} else if (["failed", "closed"].includes(connection.connectionState)) {
stop("远程连接已断开");
}
@@ -233,6 +366,105 @@ async function start() {
}
}
function applyResolution(report: ResolutionReport) {
currentResolution = { width: report.width, height: report.height };
const longEdge = Math.max(report.width, report.height);
const preset = resolutions.find((item) => item.value === longEdge);
if (preset) selectedWidth.value = preset.value;
if (report.supportedFps.length) fpsOptions.value = report.supportedFps;
if (report.fps > 0 && fpsOptions.value.includes(report.fps)) selectedFps.value = report.fps;
}
function changeResolution() {
if (channel?.readyState !== "open") return;
// height/fps 传 0:由被控端按屏幕宽高比计算高度并沿用当前帧率。
channel.send(encodeResolution(selectedWidth.value, 0, 0));
}
function changeFps() {
if (channel?.readyState !== "open" || !currentResolution) return;
channel.send(
encodeResolution(currentResolution.width, currentResolution.height, selectedFps.value),
);
}
async function collectStats(attempt: number) {
const connection = peer;
if (!connection || version !== attempt) return;
try {
const report = await connection.getStats();
if (version !== attempt) return;
let width: number | string = "-";
let height: number | string = "-";
let fps: number | string = "-";
let codecId = "";
let jitter = 0;
let dropped = 0;
let lost = 0;
let received = 0;
let bytes = 0;
let rtt: number | null = null;
const codecs = new Map<string, string>();
report.forEach((entry) => {
if (entry.type === "codec" && entry.mimeType?.startsWith("video/")) {
codecs.set(entry.id, entry.mimeType.slice(6));
}
});
report.forEach((entry) => {
if (entry.type === "inbound-rtp" && entry.kind === "video") {
width = entry.frameWidth ?? "-";
height = entry.frameHeight ?? "-";
fps = entry.framesPerSecond ?? "-";
codecId = entry.codecId ?? "";
jitter = entry.jitter ?? 0;
dropped = entry.framesDropped ?? 0;
lost = entry.packetsLost ?? 0;
received = entry.packetsReceived ?? 0;
} else if (entry.type === "candidate-pair" && entry.nominated) {
bytes = entry.bytesReceived ?? 0;
if (typeof entry.currentRoundTripTime === "number") rtt = entry.currentRoundTripTime;
}
});
const now = Date.now();
let downSpeed = "0 B/s";
if (lastStatsAt) {
const elapsed = (now - lastStatsAt) / 1000;
if (elapsed > 0) downSpeed = formatSpeed((bytes - lastBytes) / elapsed);
}
lastStatsAt = now;
lastBytes = bytes;
const totalPackets = lost + received;
stats.value = {
resolution: `${width}×${height}`,
fps: fps === "-" ? "-" : `${fps} fps`,
codec: codecs.get(codecId) ?? "-",
latency: rtt == null ? "-" : `${(rtt * 1000).toFixed(0)} ms`,
jitter: jitter ? `${(jitter * 1000).toFixed(1)} ms` : "-",
framesDropped: String(dropped),
packetLoss: totalPackets ? `${((lost / totalPackets) * 100).toFixed(1)}%` : "-",
downSpeed,
duration: formatDuration(now - connectedAt),
};
} catch {
// 统计失败不影响协助
}
}
function formatSpeed(bytesPerSecond: number) {
if (!bytesPerSecond || bytesPerSecond <= 0) return "0 B/s";
return bytesPerSecond >= 1048576
? `${(bytesPerSecond / 1048576).toFixed(1)} MB/s`
: bytesPerSecond >= 1024
? `${(bytesPerSecond / 1024).toFixed(1)} KB/s`
: `${bytesPerSecond.toFixed(0)} B/s`;
}
function formatDuration(milliseconds: number) {
const seconds = Math.max(0, Math.floor(milliseconds / 1000));
const minutes = String(Math.floor(seconds / 60)).padStart(2, "0");
return `${minutes}:${String(seconds % 60).padStart(2, "0")}`;
}
function point(event: PointerEvent, clamp: boolean): Point | undefined {
const element = video.value;
if (!element?.videoWidth || !element.videoHeight) return;
@@ -251,6 +483,22 @@ function sendTouch(action: number, start: Point, end?: Point, detail?: number) {
if (channel?.readyState === "open") channel.send(encodeTouch(action, start, end, detail));
}
function sendKey(keyCode: number, keyAction: number) {
if (channel?.readyState === "open") channel.send(encodeKey(keyCode, keyAction));
}
function keyDown(keyCode: number) {
if (pressedKeys.has(keyCode) || channel?.readyState !== "open") return;
pressedKeys.add(keyCode);
// 实时触摸模式按真实按下/抬起发送;Root/Shell 模式会把每个按键事件都执行一次,只在抬起时发一次点击。
if (mode === 2) sendKey(keyCode, 0);
}
function keyUp(keyCode: number) {
if (!pressedKeys.delete(keyCode) || channel?.readyState !== "open") return;
sendKey(keyCode, mode === 2 ? 1 : -1);
}
function pointerDown(event: PointerEvent) {
if (gesture || !mode || event.button !== 0 || channel?.readyState !== "open") return;
const start = point(event, false);
@@ -305,4 +553,35 @@ onBeforeUnmount(() => stop("已离开远程协助页面"));
object-fit: contain;
background: #111;
}
.stats-bar {
display: flex;
flex-wrap: wrap;
gap: 4px 18px;
margin: 8px 0;
font-size: 13px;
color: #606266;
}
.stats-bar b {
font-weight: 500;
color: #303133;
}
.stream-controls {
display: flex;
flex-wrap: wrap;
gap: 8px;
align-items: center;
margin-top: 12px;
font-size: 13px;
color: #606266;
}
.key-row {
display: flex;
flex-wrap: wrap;
gap: 8px;
margin-top: 12px;
}
</style>