feat: add control center, multiple relay nodes and P2P-first transport

This commit is contained in:
tongtong committed 2026-11-12 16:56:06 +08:00
1 parent 0988e9c363
commit 2abfb63da9
44 files changed
+3477 -67

No files matched your search

+19 -2
View File
@@ -14,6 +14,7 @@ use tokio_tungstenite::{
MaybeTlsStream, WebSocketStream,
};
use url::Url;
pub mod transport;
pub type Ws = WebSocketStream<MaybeTlsStream<TcpStream>>;
pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
@@ -112,7 +113,7 @@ pub async fn dial(url: Url, token: &str, connect_address: Option<&str>) -> Resul
}
}
#[derive(Debug, Deserialize, Serialize)]
#[derive(Clone, Deserialize, Serialize)]
pub struct Control {
#[serde(rename = "type")]
pub kind: String,
@@ -122,13 +123,29 @@ pub struct Control {
pub lease: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub session: Option<String>,
#[serde(default)]
pub protocol: u8,
#[serde(default)]
pub signal_key: Option<String>,
#[serde(default)]
pub ice_servers: Vec<String>,
#[serde(default)]
pub p2p_timeout_ms: u64,
#[serde(default)]
pub public_only: bool,
#[serde(default)]
pub routes: Vec<transport::Route>,
#[serde(default)]
pub sdp: Option<String>,
#[serde(default)]
pub node: Option<String>,
}
pub async fn next_control(ws: &mut Ws) -> Result<Control> {
loop {
match ws.next().await.context("control connection closed")?? {
Message::Text(text) => {
if text.len() > 16 * 1024 {
if text.len() > 128 * 1024 {
bail!("control message too large");
}
return Ok(serde_json::from_str(&text)?);
+582
View File
@@ -0,0 +1,582 @@
//! v2 signaling, P2P-first data transport and ordered relay fallback.
use crate::{dial, next_control, relay, Control, NetworkArgs, Ws, IO_TIMEOUT};
use anyhow::{bail, Context, Result};
use bytes::Bytes;
use futures_util::SinkExt;
use serde::{Deserialize, Serialize};
use std::{
collections::BTreeMap,
path::PathBuf,
sync::{
atomic::{AtomicU64, Ordering},
Arc, Mutex,
},
time::Duration,
};
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpStream,
sync::{mpsc, watch, Notify},
time::{interval, timeout},
};
use tokio_tungstenite::tungstenite::Message;
use webrtc::{
api::APIBuilder,
data_channel::{data_channel_message::DataChannelMessage, RTCDataChannel},
ice_transport::ice_server::RTCIceServer,
peer_connection::{
configuration::RTCConfiguration, sdp::session_description::RTCSessionDescription,
RTCPeerConnection,
},
};
#[derive(Clone, Deserialize, Serialize)]
pub struct Route {
pub id: String,
pub url: String,
#[serde(default)]
pub connect_address: Option<String>,
pub ticket: String,
}
impl std::fmt::Debug for Route {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Route")
.field("id", &self.id)
.field("url", &self.url)
.finish_non_exhaustive()
}
}
pub async fn send(ws: &mut Ws, value: serde_json::Value) -> Result<()> {
timeout(
IO_TIMEOUT,
ws.send(Message::Text(serde_json::to_string(&value)?.into())),
)
.await??;
Ok(())
}
async fn monitor_signal(mut ws: Ws) -> Result<()> {
let mut heartbeat = interval(Duration::from_secs(20));
loop {
tokio::select! {
result=next_control(&mut ws)=>{result?;},
_=heartbeat.tick()=>{timeout(IO_TIMEOUT,ws.send(Message::Ping(Bytes::new()))).await??;}
}
}
}
struct Channel {
dc: Arc<RTCDataChannel>,
incoming: mpsc::Receiver<Bytes>,
opened: watch::Receiver<bool>,
closed: watch::Receiver<bool>,
writable: Arc<Notify>,
}
async fn channel(dc: Arc<RTCDataChannel>) -> Channel {
let (tx, rx) = mpsc::channel(64);
let (opened_tx, opened) = watch::channel(false);
let (closed_tx, closed) = watch::channel(false);
dc.on_open(Box::new(move || {
let tx = opened_tx.clone();
Box::pin(async move {
let _ = tx.send(true);
})
}));
let close = closed_tx.clone();
dc.on_close(Box::new(move || {
let tx = close.clone();
Box::pin(async move {
let _ = tx.send(true);
})
}));
let close = closed_tx.clone();
dc.on_error(Box::new(move |_| {
let tx = close.clone();
Box::pin(async move {
let _ = tx.send(true);
})
}));
dc.on_message(Box::new(move |message: DataChannelMessage| {
let tx = tx.clone();
let closed = closed_tx.clone();
Box::pin(async move {
if message.is_string || message.data.len() > 16384 {
let _ = closed.send(true);
return;
}
if tx.send(message.data).await.is_err() {
let _ = closed.send(true);
}
})
}));
let writable = Arc::new(Notify::new());
let notify = writable.clone();
dc.set_buffered_amount_low_threshold(256 * 1024).await;
dc.on_buffered_amount_low(Box::new(move || {
let notify = notify.clone();
Box::pin(async move {
notify.notify_one();
})
}))
.await;
Channel {
dc,
incoming: rx,
opened,
closed,
writable,
}
}
impl Channel {
async fn ready(&mut self) -> Result<()> {
loop {
if *self.opened.borrow() {
return Ok(());
}
if *self.closed.borrow() {
bail!("P2P data channel closed before opening")
}
tokio::select! {
result=self.opened.changed()=>{result.context("P2P open event closed")?;},
result=self.closed.changed()=>{result.context("P2P close event closed")?;}
}
}
}
async fn run(mut self, tcp: TcpStream) -> Result<()> {
tcp.set_nodelay(true)?;
let (mut read, mut write) = tcp.into_split();
let dc = self.dc.clone();
let writable = self.writable.clone();
let outbound = async {
let mut buffer = vec![0; 16384];
loop {
let size = read.read(&mut buffer).await?;
if size == 0 {
return Ok::<(), anyhow::Error>(());
}
while dc.buffered_amount().await > 1024 * 1024 {
timeout(IO_TIMEOUT, writable.notified())
.await
.context("P2P send backpressure timed out")?;
}
let sent = timeout(
IO_TIMEOUT,
dc.send(&Bytes::copy_from_slice(&buffer[..size])),
)
.await??;
if sent != size {
bail!("incomplete P2P message write")
}
}
};
let inbound = async {
while let Some(data) = self.incoming.recv().await {
timeout(IO_TIMEOUT, write.write_all(&data)).await??;
}
Ok::<(), anyhow::Error>(())
};
tokio::select! {result=outbound=>result,result=inbound=>result,_=self.closed.changed()=>Ok(())}
}
}
struct Peer {
pc: Arc<RTCPeerConnection>,
channels: mpsc::Receiver<Channel>,
public_only: bool,
}
impl Drop for Peer {
fn drop(&mut self) {
let pc = self.pc.clone();
tokio::spawn(async move {
let _ = pc.close().await;
});
}
}
impl Peer {
async fn new(offerer: bool, manifest: &Control) -> Result<Self> {
let api = APIBuilder::new().build();
let ice_servers = manifest
.ice_servers
.iter()
.map(|url| RTCIceServer {
urls: vec![url.clone()],
..Default::default()
})
.collect();
let pc = Arc::new(
api.new_peer_connection(RTCConfiguration {
ice_servers,
..Default::default()
})
.await?,
);
let (tx, rx) = mpsc::channel(1);
if offerer {
let dc = pc.create_data_channel("ads-adb", None).await?;
tx.send(channel(dc).await)
.await
.map_err(|_| anyhow::anyhow!("P2P channel queue closed"))?;
} else {
pc.on_data_channel(Box::new(move |dc| {
let tx = tx.clone();
Box::pin(async move {
if dc.label() != "ads-adb" {
let _ = dc.close().await;
return;
}
let _ = tx.try_send(channel(dc).await);
})
}));
}
Ok(Self {
pc,
channels: rx,
public_only: manifest.public_only,
})
}
async fn description(&self, offer: bool) -> Result<String> {
let mut complete = self.pc.gathering_complete_promise().await;
let sdp = if offer {
self.pc.create_offer(None).await?
} else {
self.pc.create_answer(None).await?
};
self.pc.set_local_description(sdp).await?;
// STUN failures must not stall fallback; a partial SDP still has host candidates.
let _ = timeout(Duration::from_millis(2200), complete.recv()).await;
let mut sdp = self
.pc
.local_description()
.await
.context("missing local SDP")?
.sdp;
if self.public_only {
sdp = sdp
.lines()
.filter(|line| !line.starts_with("a=candidate:") || line.contains(" typ srflx"))
.collect::<Vec<_>>()
.join("\r\n")
+ "\r\n";
}
Ok(sdp)
}
async fn take_channel(&mut self) -> Result<Channel> {
let mut channel = self
.channels
.recv()
.await
.context("P2P channel not received")?;
channel.ready().await?;
Ok(channel)
}
}
async fn node_dial(network: &NetworkArgs, route: &Route) -> Result<Ws> {
let target = NetworkArgs {
server: route.url.clone(),
connect_address: route.connect_address.clone(),
allow_insecure: network.allow_insecure,
};
timeout(
Duration::from_secs(4),
dial(
target.url("/v2/tunnel", &[])?,
&route.ticket,
target.connect_address.as_deref(),
),
)
.await
.context("relay node connection timed out")?
}
async fn run_ws(tcp: TcpStream, data: Ws, signal: Ws) -> Result<()> {
tokio::select! {result=relay(tcp,data)=>result,result=monitor_signal(signal)=>{result?;bail!("controller signaling closed")}}
}
async fn run_p2p(tcp: TcpStream, data: Channel, signal: Ws, _peer: Peer) -> Result<()> {
tokio::select! {result=data.run(tcp)=>result,result=monitor_signal(signal)=>{result?;bail!("controller signaling closed")}}
}
/// The controller carries SDP and route decisions only, never TCP payloads.
pub async fn client_session(
tcp: TcpStream,
mut signal: Ws,
network: &NetworkArgs,
mode: &str,
status: &StreamStatus,
) -> Result<()> {
let manifest = timeout(Duration::from_secs(15), next_control(&mut signal)).await??;
if manifest.kind != "session" {
bail!("expected session manifest")
}
if mode != "relay" {
let attempt = async {
let mut peer = Peer::new(true, &manifest).await?;
let offer = peer.description(true).await?;
send(&mut signal, serde_json::json!({"type":"offer","sdp":offer})).await?;
let answer = next_control(&mut signal).await?;
if answer.kind != "answer" {
bail!("peer did not accept P2P offer")
}
peer.pc
.set_remote_description(RTCSessionDescription::answer(
answer.sdp.context("missing answer SDP")?,
)?)
.await?;
let channel = peer.take_channel().await?;
send(&mut signal, serde_json::json!({"type":"p2p-ready"})).await?;
let ready = next_control(&mut signal).await?;
if ready.kind != "p2p-ready" {
bail!("peer did not commit P2P")
}
Ok::<_, anyhow::Error>((channel, peer))
};
match timeout(
Duration::from_millis(manifest.p2p_timeout_ms.clamp(1000, 30000)),
attempt,
)
.await
{
Ok(Ok((channel, peer))) => {
status.set("p2p", "");
return run_p2p(tcp, channel, signal, peer).await;
}
_ => {
tracing::info!("P2P unavailable; trying relay nodes");
}
}
}
for route in &manifest.routes {
send(
&mut signal,
serde_json::json!({"type":"relay","node":route.id}),
)
.await?;
let response = timeout(Duration::from_secs(7), async {
loop {
let reply = next_control(&mut signal).await?;
// A timed-out P2P answer or an earlier node reply can arrive late.
if reply.node.as_deref() == Some(route.id.as_str())
&& (reply.kind == "route-ready" || reply.kind == "route-error")
{
return Ok::<_, anyhow::Error>(reply.kind == "route-ready");
}
}
})
.await;
if !matches!(response, Ok(Ok(true))) {
continue;
}
match node_dial(network, route).await {
Ok(data) => {
send(
&mut signal,
serde_json::json!({"type":"route-ready","node":route.id}),
)
.await?;
status.set("relay", &route.id);
return run_ws(tcp, data, signal).await;
}
Err(_) => {
send(
&mut signal,
serde_json::json!({"type":"route-error","node":route.id}),
)
.await?;
}
}
}
bail!("P2P unavailable and all configured relay nodes failed")
}
pub async fn agent_session(tcp: TcpStream, mut signal: Ws, network: &NetworkArgs) -> Result<()> {
let manifest = timeout(Duration::from_secs(15), next_control(&mut signal)).await??;
if manifest.kind != "session" {
bail!("expected agent session manifest")
}
let mut peer: Option<Peer> = None;
let mut pending: Option<(String, Ws)> = None;
loop {
let message = next_control(&mut signal).await?;
match message.kind.as_str() {
"offer" => {
let attempt = async {
let value = Peer::new(false, &manifest).await?;
value
.pc
.set_remote_description(RTCSessionDescription::offer(
message.sdp.context("missing offer SDP")?,
)?)
.await?;
let answer = value.description(false).await?;
Ok::<_, anyhow::Error>((value, answer))
};
match timeout(Duration::from_secs(5), attempt).await {
Ok(Ok((value, answer))) => {
peer = Some(value);
send(
&mut signal,
serde_json::json!({"type":"answer","sdp":answer}),
)
.await?;
}
_ => {
send(
&mut signal,
serde_json::json!({"type":"route-error","node":"p2p"}),
)
.await?;
}
}
}
"p2p-ready" => {
if let Some(mut value) = peer.take() {
match timeout(Duration::from_secs(2), value.take_channel()).await {
Ok(Ok(channel)) => {
send(&mut signal, serde_json::json!({"type":"p2p-ready"})).await?;
return run_p2p(tcp, channel, signal, value).await;
}
_ => {
send(
&mut signal,
serde_json::json!({"type":"route-error","node":"p2p"}),
)
.await?;
}
}
}
}
"relay" => {
peer = None;
pending = None;
let id = message.node.context("missing relay node")?;
if let Some(route) = manifest.routes.iter().find(|route| route.id == id) {
match node_dial(network, route).await {
Ok(data) => {
pending = Some((id.clone(), data));
send(
&mut signal,
serde_json::json!({"type":"route-ready","node":id}),
)
.await?;
}
Err(_) => {
send(
&mut signal,
serde_json::json!({"type":"route-error","node":id}),
)
.await?;
}
}
} else {
bail!("unknown relay node")
}
}
"route-ready" => {
if let Some((id, data)) = pending.take() {
if message.node.as_deref() != Some(id.as_str()) {
bail!("relay commit does not match selected node")
}
return run_ws(tcp, data, signal).await;
}
}
"route-error" => {
pending = None;
}
_ => bail!("unexpected session signal"),
}
}
}
#[derive(Clone, Serialize)]
struct RouteState {
transport: String,
node: String,
}
pub struct TransportStatus {
file: Option<PathBuf>,
next: AtomicU64,
streams: Mutex<BTreeMap<u64, RouteState>>,
}
pub struct StreamStatus {
owner: Arc<TransportStatus>,
id: u64,
}
impl TransportStatus {
pub fn new(file: Option<PathBuf>) -> Arc<Self> {
Arc::new(Self {
file,
next: AtomicU64::new(1),
streams: Mutex::new(BTreeMap::new()),
})
}
pub fn stream(self: &Arc<Self>) -> StreamStatus {
let id = self.next.fetch_add(1, Ordering::Relaxed);
let value = StreamStatus {
owner: self.clone(),
id,
};
value.set("negotiating", "");
value
}
fn publish(&self, streams: &BTreeMap<u64, RouteState>) {
if let Some(file) = &self.file {
let p2p = streams.values().filter(|v| v.transport == "p2p").count();
let relay = streams
.values()
.filter(|v| v.transport == "relay" || v.transport == "compat")
.count();
let transport = if p2p > 0 && relay > 0 {
"mixed"
} else if p2p > 0 {
"p2p"
} else if relay > 0 {
"relay"
} else if streams.is_empty() {
"idle"
} else {
"negotiating"
};
let nodes = streams
.values()
.filter(|v| !v.node.is_empty())
.map(|v| v.node.clone())
.collect::<std::collections::BTreeSet<_>>();
let body = serde_json::json!({"transport":transport,"p2p_streams":p2p,"relay_streams":relay,"active_streams":streams.len(),"nodes":nodes});
if let Some(parent) = file.parent() {
let _ = std::fs::create_dir_all(parent);
}
let temp = file.with_extension("tmp");
if std::fs::write(&temp, body.to_string()).is_ok() {
let _ = std::fs::rename(temp, file);
}
}
}
}
impl StreamStatus {
pub fn set(&self, transport: &str, node: &str) {
let mut streams = self.owner.streams.lock().unwrap_or_else(|e| e.into_inner());
streams.insert(
self.id,
RouteState {
transport: transport.into(),
node: node.into(),
},
);
self.owner.publish(&streams);
if transport != "negotiating" {
tracing::info!(transport, node, "transport selected");
}
}
}
impl Drop for StreamStatus {
fn drop(&mut self) {
let mut streams = self.owner.streams.lock().unwrap_or_else(|e| e.into_inner());
streams.remove(&self.id);
self.owner.publish(&streams);
}
}