78 lines
5.2 KiB
Python
78 lines
5.2 KiB
Python
"""Real-binary P2P and two-node failover test; loopback, no credentials required."""
|
|
from e2e import *
|
|
|
|
def main():
|
|
suffix = ".exe" if os.name == "nt" else ""
|
|
logs = ROOT / "work" / "cluster-e2e"; logs.mkdir(parents=True, exist_ok=True)
|
|
processes, handles = [], []
|
|
tokens = {"ADS_AGENT_TOKEN": secrets.token_hex(32), "ADS_CLIENT_TOKEN": secrets.token_hex(32), "ADS_RELAY_SECRET": secrets.token_hex(32)}
|
|
ports = [free_port() for _ in range(4)]
|
|
api = f"http://127.0.0.1:{ports[0]}"
|
|
status = logs / "status.json"
|
|
config = logs / "cluster.json"
|
|
config.write_text(json.dumps({"nodes":[{"id":f"relay-{i}","url":f"ws://127.0.0.1:{ports[i]}","priority":i} for i in (1,2)], "ice_servers":[], "p2p_timeout_ms":8000}))
|
|
def launch(name, executable, args):
|
|
binary = (ROOT / ("target/debug" if executable in ("ads-agent","ads-client") else "bin") / (executable+suffix)).resolve()
|
|
handle = open(logs / (name+".log"), "wb"); handles.append(handle)
|
|
env=os.environ.copy();env.pop("ADS_CONNECT_ADDRESS",None);env.update(tokens)
|
|
p=subprocess.Popen([str(binary),*args],env=env,stdout=handle,stderr=handle,creationflags=subprocess.CREATE_NO_WINDOW if os.name=="nt" else 0);processes.append(p);return p
|
|
def stop(p):
|
|
if p.poll() is None:p.terminate();p.wait(timeout=10)
|
|
def get(path):
|
|
req=urllib.request.Request(api+path,headers={"Authorization":"Bearer "+tokens["ADS_CLIENT_TOKEN"]})
|
|
with urllib.request.urlopen(req,timeout=2) as response:return json.load(response)
|
|
def health(i):
|
|
with urllib.request.urlopen(f"http://127.0.0.1:{ports[i]}/healthz",timeout=2) as response:return json.load(response)
|
|
def route():
|
|
try:return json.loads(status.read_text())
|
|
except (OSError,ValueError):return {}
|
|
with Phone(("127.0.0.1",0),Echo) as phone:
|
|
threading.Thread(target=phone.serve_forever,daemon=True).start()
|
|
try:
|
|
nodes=[launch(f"relay-{i}","ads-relay",["--id",f"relay-{i}","--listen",f"127.0.0.1:{ports[i]}"]) for i in (1,2)]
|
|
launch("controller","ads-server",["--listen",f"127.0.0.1:{ports[0]}","--cluster-config",str(config),"--allow-insecure-nodes"])
|
|
wait_until(lambda:get('/v1/devices')==[],"controller")
|
|
launch("agent","ads-agent",["--server",api.replace('http:','ws:'),"--allow-insecure","--device","test-phone","--target",f"127.0.0.1:{phone.server_address[1]}"])
|
|
wait_until(lambda:len(get('/v1/devices'))==1,"agent")
|
|
def client(mode,public=False):
|
|
if status.exists():status.unlink()
|
|
p=launch("client-"+mode+str(public),"ads-client",["--server",api.replace('http:','ws:'),"--allow-insecure","--device","test-phone","--listen",f"127.0.0.1:{ports[3]}","--no-auto-connect","--transport",mode,"--status-file",str(status)]+(["--p2p-public-only"] if public else []))
|
|
wait_until(lambda:any(d['busy'] for d in get('/v1/devices')),"lease")
|
|
return p
|
|
def transfer(expected,node=None):
|
|
with socket.create_connection(("127.0.0.1",ports[3]),timeout=5) as sock:
|
|
sock.settimeout(25)
|
|
payload=secrets.token_bytes(2*1024*1024)
|
|
with concurrent.futures.ThreadPoolExecutor() as pool:
|
|
writer=pool.submit(sock.sendall,payload)
|
|
data=bytearray()
|
|
while len(data)<len(payload):
|
|
chunk=sock.recv(65536);assert chunk,"premature close";data.extend(chunk)
|
|
writer.result(timeout=5)
|
|
assert data==payload
|
|
wait_until(lambda:route().get('transport')==expected,"route status")
|
|
if node:assert node in route()['nodes'],route()
|
|
assert get('/v2/nodes')['data_bytes']==0
|
|
def release(p):
|
|
stop(p);wait_until(lambda:not any(d['busy'] for d in get('/v1/devices')),"release")
|
|
wait_until(lambda:health(2)['sessions']==0,"revocation")
|
|
p=client('auto');transfer('p2p');assert health(1)['bytes']==0 and health(2)['bytes']==0;release(p)
|
|
print('PASS: P2P 2 MiB integrity; both relay nodes carried zero data',flush=True)
|
|
p=client('relay');transfer('relay','relay-1')
|
|
print('PASS: forced relay-1',flush=True)
|
|
with socket.create_connection(("127.0.0.1",ports[3]),timeout=5) as active:
|
|
active.settimeout(10);active.sendall(b'before-node-failure');assert active.recv(64)==b'before-node-failure'
|
|
stop(nodes[0])
|
|
try:assert active.recv(64)==b'',"failed node kept stream open"
|
|
except ConnectionResetError:pass
|
|
transfer('relay','relay-2');release(p)
|
|
print('PASS: active node failure closes old stream; same client lease reconnects via relay-2; revocation clears streams',flush=True)
|
|
p=client('auto',True);transfer('relay','relay-2');release(p)
|
|
print('PASS: unreachable P2P (no public candidates) falls back to relay-2',flush=True)
|
|
finally:
|
|
for p in reversed(processes):stop(p)
|
|
for h in handles:h.close()
|
|
phone.shutdown()
|
|
|
|
if __name__=='__main__':main()
|