Files
AndroidDeviceStreaming/server/cluster.go
T

344 lines
9.7 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"bytes"
"context"
"crypto/subtle"
"crypto/tls"
"encoding/json"
"errors"
"io"
"net"
"net/http"
"net/url"
"os"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"android-device-streaming/server/internal/relay"
"github.com/coder/websocket"
)
type NodeConfig struct {
ID string `json:"id"`
URL string `json:"url"`
ConnectAddress string `json:"connect_address,omitempty"`
Priority int `json:"priority"`
}
type clusterConfig struct {
Nodes []NodeConfig `json:"nodes"`
ICEServers []string `json:"ice_servers"`
P2PTimeoutMS int `json:"p2p_timeout_ms"`
}
type nodeHealth struct {
Healthy bool `json:"healthy"`
Sessions int `json:"sessions"`
Capacity int `json:"capacity"`
}
type cluster struct {
config clusterConfig
secret string
mu sync.Mutex
health map[string]nodeHealth
signalingBytes atomic.Int64
}
type route struct {
ID string `json:"id"`
URL string `json:"url"`
ConnectAddress string `json:"connect_address,omitempty"`
Ticket string `json:"ticket"`
}
func loadCluster(path, secret string, allowInsecure bool) (*cluster, error) {
if len(secret) < 32 {
return nil, errors.New("cluster mode requires ADS_RELAY_SECRET (32+ characters)")
}
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var config clusterConfig
if err = json.Unmarshal(data, &config); err != nil {
return nil, err
}
if config.ICEServers == nil {
config.ICEServers = []string{}
}
if len(config.Nodes) == 0 || len(config.Nodes) > 32 {
return nil, errors.New("configure 1–32 relay nodes")
}
if config.P2PTimeoutMS == 0 {
config.P2PTimeoutMS = 10000
}
if config.P2PTimeoutMS < 1000 || config.P2PTimeoutMS > 30000 {
return nil, errors.New("p2p_timeout_ms must be 1000–30000")
}
seen := map[string]bool{}
for _, node := range config.Nodes {
u, e := url.Parse(node.URL)
if e != nil || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Scheme != "wss" && !(allowInsecure && u.Scheme == "ws")) || !validID(node.ID) || seen[node.ID] {
return nil, errors.New("invalid or duplicate relay node")
}
if node.ConnectAddress != "" {
if _, _, e = net.SplitHostPort(node.ConnectAddress); e != nil {
return nil, errors.New("invalid node connect_address")
}
}
seen[node.ID] = true
}
for _, ice := range config.ICEServers {
if !strings.HasPrefix(ice, "stun:") {
return nil, errors.New("only STUN URLs are accepted; fallback uses WSS relay nodes")
}
}
return &cluster{config: config, secret: secret, health: map[string]nodeHealth{}}, nil
}
func nodeHTTP(node NodeConfig, path string) (*http.Client, string) {
u, _ := url.Parse(node.URL)
if u.Scheme == "wss" {
u.Scheme = "https"
} else {
u.Scheme = "http"
}
u.Path = strings.TrimRight(u.Path, "/") + path
transport := &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}, Proxy: nil}
if node.ConnectAddress != "" {
transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
return (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, network, node.ConnectAddress)
}
}
return &http.Client{Transport: transport, Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}, u.String()
}
func (c *cluster) monitor(ctx context.Context) {
probe := func() {
var wg sync.WaitGroup
for _, node := range c.config.Nodes {
wg.Add(1)
go func(node NodeConfig) {
defer wg.Done()
client, address := nodeHTTP(node, "/healthz")
defer client.CloseIdleConnections()
req, _ := http.NewRequestWithContext(ctx, "GET", address, nil)
response, err := client.Do(req)
health := nodeHealth{}
if err == nil {
defer response.Body.Close()
var body struct {
ID string `json:"id"`
Sessions int `json:"sessions"`
Capacity int `json:"capacity"`
}
if response.StatusCode == 200 && json.NewDecoder(io.LimitReader(response.Body, 4096)).Decode(&body) == nil && body.ID == node.ID && body.Capacity > 0 {
health = nodeHealth{true, body.Sessions, body.Capacity}
}
}
c.mu.Lock()
c.health[node.ID] = health
c.mu.Unlock()
}(node)
}
wg.Wait()
}
probe()
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
probe()
}
}
}
func (c *cluster) routes(session, lease, side string) []route {
nodes := append([]NodeConfig(nil), c.config.Nodes...)
c.mu.Lock()
sort.SliceStable(nodes, func(i, j int) bool {
a, b := c.health[nodes[i].ID], c.health[nodes[j].ID]
if a.Healthy != b.Healthy {
return a.Healthy
}
if nodes[i].Priority != nodes[j].Priority {
return nodes[i].Priority < nodes[j].Priority
}
return float64(a.Sessions)/float64(max(a.Capacity, 1)) < float64(b.Sessions)/float64(max(b.Capacity, 1))
})
c.mu.Unlock()
result := make([]route, 0, len(nodes))
for _, node := range nodes {
result = append(result, route{ID: node.ID, URL: node.URL, ConnectAddress: node.ConnectAddress,
Ticket: relay.Sign(c.secret, relay.Ticket{Session: session, Lease: lease, Node: node.ID, Side: side, Expires: time.Now().Add(time.Minute).Unix()})})
}
return result
}
func (c *cluster) revoke(lease string) {
if c == nil {
return
}
body, _ := json.Marshal(map[string]string{"lease": lease})
var wg sync.WaitGroup
for _, node := range c.config.Nodes {
wg.Add(1)
go func(node NodeConfig) {
defer wg.Done()
client, address := nodeHTTP(node, "/internal/revoke")
defer client.CloseIdleConnections()
request, _ := http.NewRequest("POST", address, bytes.NewReader(body))
request.Header.Set("Authorization", "Bearer "+c.secret)
request.Header.Set("Content-Type", "application/json")
if response, err := client.Do(request); err == nil {
response.Body.Close()
}
}(node)
}
wg.Wait()
}
func (b *broker) nodesEndpoint(w http.ResponseWriter, r *http.Request) {
if b.cluster == nil {
http.Error(w, "cluster not configured", 404)
return
}
b.cluster.mu.Lock()
defer b.cluster.mu.Unlock()
type node struct {
NodeConfig
nodeHealth
}
nodes := []node{}
for _, configured := range b.cluster.config.Nodes {
nodes = append(nodes, node{configured, b.cluster.health[configured.ID]})
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]any{"nodes": nodes, "signaling_bytes": b.cluster.signalingBytes.Load(), "data_bytes": 0})
}
func (b *broker) clusterConnectEndpoint(w http.ResponseWriter, r *http.Request) {
if b.cluster == nil {
http.Error(w, "cluster not configured", 404)
return
}
id, leaseID := r.URL.Query().Get("device"), r.URL.Query().Get("lease")
b.mu.Lock()
a, l := b.agents[id], b.leases[id]
if a == nil || !a.v2 || a.ws == nil || l == nil || l.ws == nil || l.id != leaseID {
b.mu.Unlock()
http.Error(w, "no active v2 lease", 409)
return
}
count := 0
for _, s := range b.sessions {
if s.agent == a {
count++
}
}
if count >= 32 {
b.mu.Unlock()
http.Error(w, "stream limit reached", 429)
return
}
ctx, cancel := context.WithCancel(b.ctx)
s := &session{id: newID(), agent: a, lease: l, ready: make(chan *websocket.Conn, 1), ctx: ctx, cancel: cancel, v2: true, signalKey: newID()}
b.sessions[s.id] = s
b.mu.Unlock()
defer func() { cancel(); b.mu.Lock(); delete(b.sessions, s.id); b.mu.Unlock() }()
client, err := accept(w, r)
if err != nil {
return
}
defer client.CloseNow()
publicOnly := r.URL.Query().Get("public_only") == "1"
manifest := func(side string) map[string]any {
return map[string]any{"type": "session", "protocol": 2, "session": s.id,
"ice_servers": b.cluster.config.ICEServers, "p2p_timeout_ms": b.cluster.config.P2PTimeoutMS, "public_only": publicOnly, "routes": b.cluster.routes(s.id, l.id, side)}
}
if writeJSON(ctx, client, manifest("client")) != nil {
return
}
if writeJSON(ctx, a.ws, map[string]any{"type": "open_v2", "session": s.id, "signal_key": s.signalKey}) != nil {
return
}
timer := time.NewTimer(15 * time.Second)
defer timer.Stop()
select {
case peer := <-s.ready:
defer peer.CloseNow()
if writeJSON(ctx, peer, manifest("agent")) != nil {
return
}
b.bridgeSignals(ctx, client, peer)
case <-ctx.Done():
case <-timer.C:
}
}
func (b *broker) signalEndpoint(w http.ResponseWriter, r *http.Request) {
b.mu.Lock()
s := b.sessions[r.URL.Query().Get("session")]
if s == nil || !s.v2 || s.claimed || s.ctx.Err() != nil || b.agents[s.agent.id] != s.agent ||
subtle.ConstantTimeCompare([]byte(s.signalKey), []byte(r.URL.Query().Get("key"))) != 1 {
b.mu.Unlock()
http.Error(w, "invalid signaling ticket", 404)
return
}
s.claimed = true
b.mu.Unlock()
ws, err := accept(w, r)
if err != nil {
s.cancel()
return
}
defer ws.CloseNow()
select {
case s.ready <- ws:
case <-s.ctx.Done():
return
}
<-s.ctx.Done()
}
func (b *broker) bridgeSignals(parent context.Context, a, z *websocket.Conn) {
ctx, cancel := context.WithCancel(parent)
defer cancel()
copyOne := func(dst, src *websocket.Conn) {
defer cancel()
for {
typ, data, err := src.Read(ctx)
if err != nil || typ != websocket.MessageText {
return
}
var message struct {
Type string `json:"type"`
}
if json.Unmarshal(data, &message) != nil {
return
}
switch message.Type {
case "offer", "answer", "relay", "route-ready", "route-error", "p2p-ready":
default:
return
}
deadline, stop := context.WithTimeout(ctx, 10*time.Second)
err = dst.Write(deadline, typ, data)
stop()
if err != nil {
return
}
b.cluster.signalingBytes.Add(int64(len(data)))
}
}
var wg sync.WaitGroup
wg.Add(2)
go func() { defer wg.Done(); copyOne(a, z) }()
go func() { defer wg.Done(); copyOne(z, a) }()
wg.Wait()
}