feat(device): 远程协助增加控制按键、串流统计与分辨率帧率切换
This commit is contained in:
1 parent
fd3615407c
commit
aec3317180
2 files changed
+382
-6
No files matched your search
@@ -37,8 +37,105 @@ export function readTouchMode(data: ArrayBuffer): TouchMode {
|
||||
return 0;
|
||||
}
|
||||
|
||||
export type ResolutionReport = {
|
||||
width: number;
|
||||
height: number;
|
||||
fps: number;
|
||||
supportedFps: number[];
|
||||
};
|
||||
|
||||
/** 解析被控端 REPORT_RESOLUTION;非该消息返回 undefined。 */
|
||||
export function readResolutionReport(data: ArrayBuffer): ResolutionReport | undefined {
|
||||
const bytes = new Uint8Array(data);
|
||||
let position = 0;
|
||||
let action = 0;
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
let fps = 0;
|
||||
const supportedFps: number[] = [];
|
||||
function readVarint() {
|
||||
let value = 0;
|
||||
for (let shift = 0; shift <= 28 && position < bytes.length; shift += 7) {
|
||||
const byte = bytes[position++];
|
||||
value += (byte & 127) * 2 ** shift;
|
||||
if ((byte & 128) === 0) return value;
|
||||
}
|
||||
throw new Error("Invalid resolution report");
|
||||
}
|
||||
|
||||
try {
|
||||
while (position < bytes.length) {
|
||||
const tag = readVarint();
|
||||
const field = tag >>> 3;
|
||||
switch (tag & 7) {
|
||||
case 0: {
|
||||
const value = readVarint();
|
||||
if (field === 1) action = value;
|
||||
else if (field === 12) width = value;
|
||||
else if (field === 13) height = value;
|
||||
else if (field === 14) fps = value;
|
||||
else if (field === 16) supportedFps.push(value);
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
const length = readVarint();
|
||||
const end = position + length;
|
||||
if (end > bytes.length) return undefined;
|
||||
if (field === 16) {
|
||||
while (position < end) supportedFps.push(readVarint());
|
||||
}
|
||||
position = end;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
return action === 7 && width > 0 && height > 0 ? { width, height, fps, supportedFps } : undefined;
|
||||
}
|
||||
|
||||
/** 编码 SET_RESOLUTION;width<=0 表示原始分辨率,fps<=0 表示沿用当前帧率。 */
|
||||
export function encodeResolution(width: number, height: number, fps: number) {
|
||||
const bytes = new Uint8Array(32);
|
||||
let offset = 0;
|
||||
function varint(field: number, value: number) {
|
||||
bytes[offset++] = field << 3;
|
||||
while (value >= 128) {
|
||||
bytes[offset++] = (value & 127) | 128;
|
||||
value >>>= 7;
|
||||
}
|
||||
bytes[offset++] = value;
|
||||
}
|
||||
varint(1, 6);
|
||||
varint(12, width);
|
||||
varint(13, height);
|
||||
varint(14, fps);
|
||||
return bytes.slice(0, offset);
|
||||
}
|
||||
|
||||
export type Point = { x: number; y: number };
|
||||
|
||||
/** 编码 KEY 指令;keyAction 传 -1 表示按下并立即抬起。 */
|
||||
export function encodeKey(keyCode: number, keyAction: number) {
|
||||
const bytes = new Uint8Array(16);
|
||||
let offset = 0;
|
||||
function varint(field: number, value: number) {
|
||||
bytes[offset++] = field << 3;
|
||||
while (value >= 128 || value < 0) {
|
||||
bytes[offset++] = (value & 127) | 128;
|
||||
value >>>= 7;
|
||||
}
|
||||
bytes[offset++] = value;
|
||||
}
|
||||
varint(1, 3);
|
||||
varint(9, keyCode);
|
||||
varint(10, keyAction);
|
||||
return bytes.slice(0, offset);
|
||||
}
|
||||
|
||||
export function encodeTouch(action: number, start: Point, end?: Point, detail?: number) {
|
||||
const bytes = new Uint8Array(48);
|
||||
const view = new DataView(bytes.buffer);
|
||||
|
||||
@@ -27,6 +27,44 @@
|
||||
</el-space>
|
||||
<p>管理员可直接连接在线设备,无需老人逐次确认。</p>
|
||||
<p role="status">{{ status }}</p>
|
||||
<div v-if="stats" class="stats-bar">
|
||||
<span>
|
||||
分辨率
|
||||
<b>{{ stats.resolution }}</b>
|
||||
</span>
|
||||
<span>
|
||||
帧率
|
||||
<b>{{ stats.fps }}</b>
|
||||
</span>
|
||||
<span>
|
||||
编码
|
||||
<b>{{ stats.codec }}</b>
|
||||
</span>
|
||||
<span>
|
||||
延迟
|
||||
<b>{{ stats.latency }}</b>
|
||||
</span>
|
||||
<span>
|
||||
抖动
|
||||
<b>{{ stats.jitter }}</b>
|
||||
</span>
|
||||
<span>
|
||||
丢帧
|
||||
<b>{{ stats.framesDropped }}</b>
|
||||
</span>
|
||||
<span>
|
||||
丢包
|
||||
<b>{{ stats.packetLoss }}</b>
|
||||
</span>
|
||||
<span>
|
||||
下载
|
||||
<b>{{ stats.downSpeed }}</b>
|
||||
</span>
|
||||
<span>
|
||||
时长
|
||||
<b>{{ stats.duration }}</b>
|
||||
</span>
|
||||
</div>
|
||||
<video
|
||||
ref="video"
|
||||
class="remote-screen"
|
||||
@@ -39,6 +77,45 @@
|
||||
@pointercancel="pointerEnd($event, true)"
|
||||
@lostpointercapture="pointerEnd($event, true)"
|
||||
/>
|
||||
<div class="stream-controls">
|
||||
<span>分辨率</span>
|
||||
<el-select
|
||||
v-model="selectedWidth"
|
||||
:disabled="!channelOpen"
|
||||
style="width: 130px"
|
||||
@change="changeResolution"
|
||||
>
|
||||
<el-option
|
||||
v-for="preset in resolutions"
|
||||
:key="preset.value"
|
||||
:label="preset.label"
|
||||
:value="preset.value"
|
||||
/>
|
||||
</el-select>
|
||||
<span>帧率</span>
|
||||
<el-select
|
||||
v-model="selectedFps"
|
||||
:disabled="!channelOpen"
|
||||
style="width: 110px"
|
||||
@change="changeFps"
|
||||
>
|
||||
<el-option v-if="selectedFps === 0" :value="0" label="--" disabled />
|
||||
<el-option v-for="fps in fpsOptions" :key="fps" :label="`${fps} fps`" :value="fps" />
|
||||
</el-select>
|
||||
</div>
|
||||
<div class="key-row">
|
||||
<el-button
|
||||
v-for="key in keys"
|
||||
:key="key.code"
|
||||
:disabled="!channelOpen"
|
||||
@pointerdown.prevent="keyDown(key.code)"
|
||||
@pointerup.prevent="keyUp(key.code)"
|
||||
@pointerleave.prevent="keyUp(key.code)"
|
||||
@pointercancel.prevent="keyUp(key.code)"
|
||||
>
|
||||
{{ key.label }}
|
||||
</el-button>
|
||||
</div>
|
||||
</el-card>
|
||||
</div>
|
||||
</template>
|
||||
@@ -48,8 +125,31 @@ import { onBeforeUnmount, onDeactivated, ref } from "vue";
|
||||
import DeviceAPI from "@/api/device";
|
||||
import type { DeviceItem } from "@/api/device/types";
|
||||
import { openRemoteControlSignal } from "@/api/remote-control";
|
||||
import { encodeTouch, readTouchMode } from "./controlProtocol";
|
||||
import type { Point, TouchMode } from "./controlProtocol";
|
||||
import {
|
||||
encodeKey,
|
||||
encodeResolution,
|
||||
encodeTouch,
|
||||
readResolutionReport,
|
||||
readTouchMode,
|
||||
} from "./controlProtocol";
|
||||
import type { Point, ResolutionReport, TouchMode } from "./controlProtocol";
|
||||
|
||||
const resolutions = [
|
||||
{ label: "原始画质", value: 0 },
|
||||
{ label: "1080P", value: 1920 },
|
||||
{ label: "720P", value: 1280 },
|
||||
{ label: "480P", value: 854 },
|
||||
];
|
||||
|
||||
const keys = [
|
||||
{ label: "主页", code: 3 },
|
||||
{ label: "返回", code: 4 },
|
||||
{ label: "多任务", code: 187 },
|
||||
{ label: "菜单", code: 82 },
|
||||
{ label: "音量 +", code: 24 },
|
||||
{ label: "音量 -", code: 25 },
|
||||
{ label: "电源", code: 26 },
|
||||
];
|
||||
|
||||
interface Signal {
|
||||
type: string;
|
||||
@@ -63,6 +163,12 @@ const sn = ref("");
|
||||
const devices = ref<DeviceItem[]>([]);
|
||||
const searching = ref(false);
|
||||
const active = ref(false);
|
||||
const channelOpen = ref(false);
|
||||
const stats = ref<Record<string, string>>();
|
||||
const selectedWidth = ref(0);
|
||||
const selectedFps = ref(0);
|
||||
const fpsOptions = ref([15, 24, 30, 60]);
|
||||
let currentResolution: { width: number; height: number } | undefined;
|
||||
const status = ref("请选择要协助的设备");
|
||||
const video = ref<HTMLVideoElement>();
|
||||
let searchVersion = 0;
|
||||
@@ -71,7 +177,12 @@ let socket: WebSocket | undefined;
|
||||
let peer: RTCPeerConnection | undefined;
|
||||
let channel: RTCDataChannel | undefined;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let statsTimer: ReturnType<typeof setInterval> | undefined;
|
||||
let connectedAt = 0;
|
||||
let lastStatsAt = 0;
|
||||
let lastBytes = 0;
|
||||
let mode: TouchMode = 0;
|
||||
const pressedKeys = new Set<number>();
|
||||
let gesture:
|
||||
{ pointer: number; start: Point; time: number; lastMove: number; mode: TouchMode } | undefined;
|
||||
|
||||
@@ -97,8 +208,20 @@ function loadOptions(open: boolean) {
|
||||
function stop(reason: string) {
|
||||
version++;
|
||||
clearTimeout(timer);
|
||||
clearInterval(statsTimer);
|
||||
statsTimer = undefined;
|
||||
stats.value = undefined;
|
||||
connectedAt = 0;
|
||||
lastStatsAt = 0;
|
||||
lastBytes = 0;
|
||||
currentResolution = undefined;
|
||||
selectedWidth.value = 0;
|
||||
selectedFps.value = 0;
|
||||
fpsOptions.value = [15, 24, 30, 60];
|
||||
gesture = undefined;
|
||||
mode = 0;
|
||||
pressedKeys.clear();
|
||||
channelOpen.value = false;
|
||||
channel?.close();
|
||||
channel = undefined;
|
||||
peer?.close();
|
||||
@@ -150,10 +273,7 @@ async function start() {
|
||||
if (message.type === "REGISTER_SUCCESS" && !peer) {
|
||||
const relay = JSON.parse(message.payload || "{}") as RTCIceServer;
|
||||
const connection = new RTCPeerConnection({
|
||||
iceServers: [
|
||||
{ urls: "stun:stun.l.google.com:19302" },
|
||||
...(relay.urls ? [relay] : []),
|
||||
],
|
||||
iceServers: [{ urls: "stun:47.242.112.133:3478" }, ...(relay.urls ? [relay] : [])],
|
||||
});
|
||||
peer = connection;
|
||||
connection.addTransceiver("video", { direction: "recvonly" });
|
||||
@@ -163,9 +283,17 @@ async function start() {
|
||||
});
|
||||
channel = control;
|
||||
control.binaryType = "arraybuffer";
|
||||
control.onopen = () => {
|
||||
if (version === attempt) channelOpen.value = true;
|
||||
};
|
||||
control.onclose = () => {
|
||||
if (version === attempt) channelOpen.value = false;
|
||||
};
|
||||
control.onmessage = (report) => {
|
||||
if (version === attempt && report.data instanceof ArrayBuffer) {
|
||||
mode = readTouchMode(report.data) || mode;
|
||||
const resolution = readResolutionReport(report.data);
|
||||
if (resolution) applyResolution(resolution);
|
||||
}
|
||||
};
|
||||
connection.ontrack = ({ streams, track }) => {
|
||||
@@ -177,6 +305,11 @@ async function start() {
|
||||
if (connection.connectionState === "connected") {
|
||||
clearTimeout(timer);
|
||||
status.value = "协助已连接";
|
||||
connectedAt = Date.now();
|
||||
lastStatsAt = 0;
|
||||
lastBytes = 0;
|
||||
clearInterval(statsTimer);
|
||||
statsTimer = setInterval(() => void collectStats(attempt), 1000);
|
||||
} else if (["failed", "closed"].includes(connection.connectionState)) {
|
||||
stop("远程连接已断开");
|
||||
}
|
||||
@@ -233,6 +366,105 @@ async function start() {
|
||||
}
|
||||
}
|
||||
|
||||
function applyResolution(report: ResolutionReport) {
|
||||
currentResolution = { width: report.width, height: report.height };
|
||||
const longEdge = Math.max(report.width, report.height);
|
||||
const preset = resolutions.find((item) => item.value === longEdge);
|
||||
if (preset) selectedWidth.value = preset.value;
|
||||
if (report.supportedFps.length) fpsOptions.value = report.supportedFps;
|
||||
if (report.fps > 0 && fpsOptions.value.includes(report.fps)) selectedFps.value = report.fps;
|
||||
}
|
||||
|
||||
function changeResolution() {
|
||||
if (channel?.readyState !== "open") return;
|
||||
// height/fps 传 0:由被控端按屏幕宽高比计算高度并沿用当前帧率。
|
||||
channel.send(encodeResolution(selectedWidth.value, 0, 0));
|
||||
}
|
||||
|
||||
function changeFps() {
|
||||
if (channel?.readyState !== "open" || !currentResolution) return;
|
||||
channel.send(
|
||||
encodeResolution(currentResolution.width, currentResolution.height, selectedFps.value),
|
||||
);
|
||||
}
|
||||
|
||||
async function collectStats(attempt: number) {
|
||||
const connection = peer;
|
||||
if (!connection || version !== attempt) return;
|
||||
try {
|
||||
const report = await connection.getStats();
|
||||
if (version !== attempt) return;
|
||||
let width: number | string = "-";
|
||||
let height: number | string = "-";
|
||||
let fps: number | string = "-";
|
||||
let codecId = "";
|
||||
let jitter = 0;
|
||||
let dropped = 0;
|
||||
let lost = 0;
|
||||
let received = 0;
|
||||
let bytes = 0;
|
||||
let rtt: number | null = null;
|
||||
const codecs = new Map<string, string>();
|
||||
report.forEach((entry) => {
|
||||
if (entry.type === "codec" && entry.mimeType?.startsWith("video/")) {
|
||||
codecs.set(entry.id, entry.mimeType.slice(6));
|
||||
}
|
||||
});
|
||||
report.forEach((entry) => {
|
||||
if (entry.type === "inbound-rtp" && entry.kind === "video") {
|
||||
width = entry.frameWidth ?? "-";
|
||||
height = entry.frameHeight ?? "-";
|
||||
fps = entry.framesPerSecond ?? "-";
|
||||
codecId = entry.codecId ?? "";
|
||||
jitter = entry.jitter ?? 0;
|
||||
dropped = entry.framesDropped ?? 0;
|
||||
lost = entry.packetsLost ?? 0;
|
||||
received = entry.packetsReceived ?? 0;
|
||||
} else if (entry.type === "candidate-pair" && entry.nominated) {
|
||||
bytes = entry.bytesReceived ?? 0;
|
||||
if (typeof entry.currentRoundTripTime === "number") rtt = entry.currentRoundTripTime;
|
||||
}
|
||||
});
|
||||
const now = Date.now();
|
||||
let downSpeed = "0 B/s";
|
||||
if (lastStatsAt) {
|
||||
const elapsed = (now - lastStatsAt) / 1000;
|
||||
if (elapsed > 0) downSpeed = formatSpeed((bytes - lastBytes) / elapsed);
|
||||
}
|
||||
lastStatsAt = now;
|
||||
lastBytes = bytes;
|
||||
const totalPackets = lost + received;
|
||||
stats.value = {
|
||||
resolution: `${width}×${height}`,
|
||||
fps: fps === "-" ? "-" : `${fps} fps`,
|
||||
codec: codecs.get(codecId) ?? "-",
|
||||
latency: rtt == null ? "-" : `${(rtt * 1000).toFixed(0)} ms`,
|
||||
jitter: jitter ? `${(jitter * 1000).toFixed(1)} ms` : "-",
|
||||
framesDropped: String(dropped),
|
||||
packetLoss: totalPackets ? `${((lost / totalPackets) * 100).toFixed(1)}%` : "-",
|
||||
downSpeed,
|
||||
duration: formatDuration(now - connectedAt),
|
||||
};
|
||||
} catch {
|
||||
// 统计失败不影响协助
|
||||
}
|
||||
}
|
||||
|
||||
function formatSpeed(bytesPerSecond: number) {
|
||||
if (!bytesPerSecond || bytesPerSecond <= 0) return "0 B/s";
|
||||
return bytesPerSecond >= 1048576
|
||||
? `${(bytesPerSecond / 1048576).toFixed(1)} MB/s`
|
||||
: bytesPerSecond >= 1024
|
||||
? `${(bytesPerSecond / 1024).toFixed(1)} KB/s`
|
||||
: `${bytesPerSecond.toFixed(0)} B/s`;
|
||||
}
|
||||
|
||||
function formatDuration(milliseconds: number) {
|
||||
const seconds = Math.max(0, Math.floor(milliseconds / 1000));
|
||||
const minutes = String(Math.floor(seconds / 60)).padStart(2, "0");
|
||||
return `${minutes}:${String(seconds % 60).padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
function point(event: PointerEvent, clamp: boolean): Point | undefined {
|
||||
const element = video.value;
|
||||
if (!element?.videoWidth || !element.videoHeight) return;
|
||||
@@ -251,6 +483,22 @@ function sendTouch(action: number, start: Point, end?: Point, detail?: number) {
|
||||
if (channel?.readyState === "open") channel.send(encodeTouch(action, start, end, detail));
|
||||
}
|
||||
|
||||
function sendKey(keyCode: number, keyAction: number) {
|
||||
if (channel?.readyState === "open") channel.send(encodeKey(keyCode, keyAction));
|
||||
}
|
||||
|
||||
function keyDown(keyCode: number) {
|
||||
if (pressedKeys.has(keyCode) || channel?.readyState !== "open") return;
|
||||
pressedKeys.add(keyCode);
|
||||
// 实时触摸模式按真实按下/抬起发送;Root/Shell 模式会把每个按键事件都执行一次,只在抬起时发一次点击。
|
||||
if (mode === 2) sendKey(keyCode, 0);
|
||||
}
|
||||
|
||||
function keyUp(keyCode: number) {
|
||||
if (!pressedKeys.delete(keyCode) || channel?.readyState !== "open") return;
|
||||
sendKey(keyCode, mode === 2 ? 1 : -1);
|
||||
}
|
||||
|
||||
function pointerDown(event: PointerEvent) {
|
||||
if (gesture || !mode || event.button !== 0 || channel?.readyState !== "open") return;
|
||||
const start = point(event, false);
|
||||
@@ -305,4 +553,35 @@ onBeforeUnmount(() => stop("已离开远程协助页面"));
|
||||
object-fit: contain;
|
||||
background: #111;
|
||||
}
|
||||
|
||||
.stats-bar {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px 18px;
|
||||
margin: 8px 0;
|
||||
font-size: 13px;
|
||||
color: #606266;
|
||||
}
|
||||
|
||||
.stats-bar b {
|
||||
font-weight: 500;
|
||||
color: #303133;
|
||||
}
|
||||
|
||||
.stream-controls {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
margin-top: 12px;
|
||||
font-size: 13px;
|
||||
color: #606266;
|
||||
}
|
||||
|
||||
.key-row {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-top: 12px;
|
||||
}
|
||||
</style>
|
||||
Reference in new issue
Block a user