feat: add control center, multiple relay nodes and P2P-first transport
This commit is contained in:
1 parent
0988e9c363
commit
2abfb63da9
44 files changed
+3477
-67
No files matched your search
@@ -0,0 +1,343 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"android-device-streaming/server/internal/relay"
|
||||
"github.com/coder/websocket"
|
||||
)
|
||||
|
||||
type NodeConfig struct {
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url"`
|
||||
ConnectAddress string `json:"connect_address,omitempty"`
|
||||
Priority int `json:"priority"`
|
||||
}
|
||||
type clusterConfig struct {
|
||||
Nodes []NodeConfig `json:"nodes"`
|
||||
ICEServers []string `json:"ice_servers"`
|
||||
P2PTimeoutMS int `json:"p2p_timeout_ms"`
|
||||
}
|
||||
type nodeHealth struct {
|
||||
Healthy bool `json:"healthy"`
|
||||
Sessions int `json:"sessions"`
|
||||
Capacity int `json:"capacity"`
|
||||
}
|
||||
type cluster struct {
|
||||
config clusterConfig
|
||||
secret string
|
||||
mu sync.Mutex
|
||||
health map[string]nodeHealth
|
||||
signalingBytes atomic.Int64
|
||||
}
|
||||
type route struct {
|
||||
ID string `json:"id"`
|
||||
URL string `json:"url"`
|
||||
ConnectAddress string `json:"connect_address,omitempty"`
|
||||
Ticket string `json:"ticket"`
|
||||
}
|
||||
|
||||
func loadCluster(path, secret string, allowInsecure bool) (*cluster, error) {
|
||||
if len(secret) < 32 {
|
||||
return nil, errors.New("cluster mode requires ADS_RELAY_SECRET (32+ characters)")
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var config clusterConfig
|
||||
if err = json.Unmarshal(data, &config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.ICEServers == nil {
|
||||
config.ICEServers = []string{}
|
||||
}
|
||||
if len(config.Nodes) == 0 || len(config.Nodes) > 32 {
|
||||
return nil, errors.New("configure 1–32 relay nodes")
|
||||
}
|
||||
if config.P2PTimeoutMS == 0 {
|
||||
config.P2PTimeoutMS = 10000
|
||||
}
|
||||
if config.P2PTimeoutMS < 1000 || config.P2PTimeoutMS > 30000 {
|
||||
return nil, errors.New("p2p_timeout_ms must be 1000–30000")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, node := range config.Nodes {
|
||||
u, e := url.Parse(node.URL)
|
||||
if e != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Scheme != "wss" && !(allowInsecure && u.Scheme == "ws")) || !validID(node.ID) || seen[node.ID] {
|
||||
return nil, errors.New("invalid or duplicate relay node")
|
||||
}
|
||||
if node.ConnectAddress != "" {
|
||||
if _, _, e = net.SplitHostPort(node.ConnectAddress); e != nil {
|
||||
return nil, errors.New("invalid node connect_address")
|
||||
}
|
||||
}
|
||||
seen[node.ID] = true
|
||||
}
|
||||
for _, ice := range config.ICEServers {
|
||||
if !strings.HasPrefix(ice, "stun:") {
|
||||
return nil, errors.New("only STUN URLs are accepted; fallback uses WSS relay nodes")
|
||||
}
|
||||
}
|
||||
return &cluster{config: config, secret: secret, health: map[string]nodeHealth{}}, nil
|
||||
}
|
||||
|
||||
func nodeHTTP(node NodeConfig, path string) (*http.Client, string) {
|
||||
u, _ := url.Parse(node.URL)
|
||||
if u.Scheme == "wss" {
|
||||
u.Scheme = "https"
|
||||
} else {
|
||||
u.Scheme = "http"
|
||||
}
|
||||
u.Path = strings.TrimRight(u.Path, "/") + path
|
||||
transport := &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}, Proxy: nil}
|
||||
if node.ConnectAddress != "" {
|
||||
transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
return (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, network, node.ConnectAddress)
|
||||
}
|
||||
}
|
||||
return &http.Client{Transport: transport, Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}, u.String()
|
||||
}
|
||||
|
||||
func (c *cluster) monitor(ctx context.Context) {
|
||||
probe := func() {
|
||||
var wg sync.WaitGroup
|
||||
for _, node := range c.config.Nodes {
|
||||
wg.Add(1)
|
||||
go func(node NodeConfig) {
|
||||
defer wg.Done()
|
||||
client, address := nodeHTTP(node, "/healthz")
|
||||
defer client.CloseIdleConnections()
|
||||
req, _ := http.NewRequestWithContext(ctx, "GET", address, nil)
|
||||
response, err := client.Do(req)
|
||||
health := nodeHealth{}
|
||||
if err == nil {
|
||||
defer response.Body.Close()
|
||||
var body struct {
|
||||
ID string `json:"id"`
|
||||
Sessions int `json:"sessions"`
|
||||
Capacity int `json:"capacity"`
|
||||
}
|
||||
if response.StatusCode == 200 && json.NewDecoder(io.LimitReader(response.Body, 4096)).Decode(&body) == nil && body.ID == node.ID && body.Capacity > 0 {
|
||||
health = nodeHealth{true, body.Sessions, body.Capacity}
|
||||
}
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.health[node.ID] = health
|
||||
c.mu.Unlock()
|
||||
}(node)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
probe()
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
probe()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cluster) routes(session, lease, side string) []route {
|
||||
nodes := append([]NodeConfig(nil), c.config.Nodes...)
|
||||
c.mu.Lock()
|
||||
sort.SliceStable(nodes, func(i, j int) bool {
|
||||
a, b := c.health[nodes[i].ID], c.health[nodes[j].ID]
|
||||
if a.Healthy != b.Healthy {
|
||||
return a.Healthy
|
||||
}
|
||||
if nodes[i].Priority != nodes[j].Priority {
|
||||
return nodes[i].Priority < nodes[j].Priority
|
||||
}
|
||||
return float64(a.Sessions)/float64(max(a.Capacity, 1)) < float64(b.Sessions)/float64(max(b.Capacity, 1))
|
||||
})
|
||||
c.mu.Unlock()
|
||||
result := make([]route, 0, len(nodes))
|
||||
for _, node := range nodes {
|
||||
result = append(result, route{ID: node.ID, URL: node.URL, ConnectAddress: node.ConnectAddress,
|
||||
Ticket: relay.Sign(c.secret, relay.Ticket{Session: session, Lease: lease, Node: node.ID, Side: side, Expires: time.Now().Add(time.Minute).Unix()})})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (c *cluster) revoke(lease string) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"lease": lease})
|
||||
var wg sync.WaitGroup
|
||||
for _, node := range c.config.Nodes {
|
||||
wg.Add(1)
|
||||
go func(node NodeConfig) {
|
||||
defer wg.Done()
|
||||
client, address := nodeHTTP(node, "/internal/revoke")
|
||||
defer client.CloseIdleConnections()
|
||||
request, _ := http.NewRequest("POST", address, bytes.NewReader(body))
|
||||
request.Header.Set("Authorization", "Bearer "+c.secret)
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
if response, err := client.Do(request); err == nil {
|
||||
response.Body.Close()
|
||||
}
|
||||
}(node)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (b *broker) nodesEndpoint(w http.ResponseWriter, r *http.Request) {
|
||||
if b.cluster == nil {
|
||||
http.Error(w, "cluster not configured", 404)
|
||||
return
|
||||
}
|
||||
b.cluster.mu.Lock()
|
||||
defer b.cluster.mu.Unlock()
|
||||
type node struct {
|
||||
NodeConfig
|
||||
nodeHealth
|
||||
}
|
||||
nodes := []node{}
|
||||
for _, configured := range b.cluster.config.Nodes {
|
||||
nodes = append(nodes, node{configured, b.cluster.health[configured.ID]})
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(map[string]any{"nodes": nodes, "signaling_bytes": b.cluster.signalingBytes.Load(), "data_bytes": 0})
|
||||
}
|
||||
|
||||
func (b *broker) clusterConnectEndpoint(w http.ResponseWriter, r *http.Request) {
|
||||
if b.cluster == nil {
|
||||
http.Error(w, "cluster not configured", 404)
|
||||
return
|
||||
}
|
||||
id, leaseID := r.URL.Query().Get("device"), r.URL.Query().Get("lease")
|
||||
b.mu.Lock()
|
||||
a, l := b.agents[id], b.leases[id]
|
||||
if a == nil || !a.v2 || a.ws == nil || l == nil || l.ws == nil || l.id != leaseID {
|
||||
b.mu.Unlock()
|
||||
http.Error(w, "no active v2 lease", 409)
|
||||
return
|
||||
}
|
||||
count := 0
|
||||
for _, s := range b.sessions {
|
||||
if s.agent == a {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count >= 32 {
|
||||
b.mu.Unlock()
|
||||
http.Error(w, "stream limit reached", 429)
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithCancel(b.ctx)
|
||||
s := &session{id: newID(), agent: a, lease: l, ready: make(chan *websocket.Conn, 1), ctx: ctx, cancel: cancel, v2: true, signalKey: newID()}
|
||||
b.sessions[s.id] = s
|
||||
b.mu.Unlock()
|
||||
defer func() { cancel(); b.mu.Lock(); delete(b.sessions, s.id); b.mu.Unlock() }()
|
||||
client, err := accept(w, r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer client.CloseNow()
|
||||
publicOnly := r.URL.Query().Get("public_only") == "1"
|
||||
manifest := func(side string) map[string]any {
|
||||
return map[string]any{"type": "session", "protocol": 2, "session": s.id,
|
||||
"ice_servers": b.cluster.config.ICEServers, "p2p_timeout_ms": b.cluster.config.P2PTimeoutMS, "public_only": publicOnly, "routes": b.cluster.routes(s.id, l.id, side)}
|
||||
}
|
||||
if writeJSON(ctx, client, manifest("client")) != nil {
|
||||
return
|
||||
}
|
||||
if writeJSON(ctx, a.ws, map[string]any{"type": "open_v2", "session": s.id, "signal_key": s.signalKey}) != nil {
|
||||
return
|
||||
}
|
||||
timer := time.NewTimer(15 * time.Second)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case peer := <-s.ready:
|
||||
defer peer.CloseNow()
|
||||
if writeJSON(ctx, peer, manifest("agent")) != nil {
|
||||
return
|
||||
}
|
||||
b.bridgeSignals(ctx, client, peer)
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
|
||||
func (b *broker) signalEndpoint(w http.ResponseWriter, r *http.Request) {
|
||||
b.mu.Lock()
|
||||
s := b.sessions[r.URL.Query().Get("session")]
|
||||
if s == nil || !s.v2 || s.claimed || s.ctx.Err() != nil || b.agents[s.agent.id] != s.agent ||
|
||||
subtle.ConstantTimeCompare([]byte(s.signalKey), []byte(r.URL.Query().Get("key"))) != 1 {
|
||||
b.mu.Unlock()
|
||||
http.Error(w, "invalid signaling ticket", 404)
|
||||
return
|
||||
}
|
||||
s.claimed = true
|
||||
b.mu.Unlock()
|
||||
ws, err := accept(w, r)
|
||||
if err != nil {
|
||||
s.cancel()
|
||||
return
|
||||
}
|
||||
defer ws.CloseNow()
|
||||
select {
|
||||
case s.ready <- ws:
|
||||
case <-s.ctx.Done():
|
||||
return
|
||||
}
|
||||
<-s.ctx.Done()
|
||||
}
|
||||
|
||||
func (b *broker) bridgeSignals(parent context.Context, a, z *websocket.Conn) {
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
defer cancel()
|
||||
copyOne := func(dst, src *websocket.Conn) {
|
||||
defer cancel()
|
||||
for {
|
||||
typ, data, err := src.Read(ctx)
|
||||
if err != nil || typ != websocket.MessageText {
|
||||
return
|
||||
}
|
||||
var message struct {
|
||||
Type string `json:"type"`
|
||||
}
|
||||
if json.Unmarshal(data, &message) != nil {
|
||||
return
|
||||
}
|
||||
switch message.Type {
|
||||
case "offer", "answer", "relay", "route-ready", "route-error", "p2p-ready":
|
||||
default:
|
||||
return
|
||||
}
|
||||
deadline, stop := context.WithTimeout(ctx, 10*time.Second)
|
||||
err = dst.Write(deadline, typ, data)
|
||||
stop()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
b.cluster.signalingBytes.Add(int64(len(data)))
|
||||
}
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() { defer wg.Done(); copyOne(a, z) }()
|
||||
go func() { defer wg.Done(); copyOne(z, a) }()
|
||||
wg.Wait()
|
||||
}
|
||||
Reference in new issue
Block a user