#!/usr/bin/env python3 """Grade-4: closed-loop POSTs during rolling API replacement.""" from __future__ import annotations import base64 import json import os import subprocess import threading import time import uuid import urllib.error import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parents[1] BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") REPLICAS = os.environ.get("API_REPLICAS", "2") def call(method, path, body=None, token=None, headers=None): h = {"Accept": "application/json"} if token: h["Authorization"] = "Bearer " + token data = None if body is not None: data = json.dumps(body).encode("utf-8") h["Content-Type"] = "application/json" if headers: h.update(headers) req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) try: resp = urllib.request.urlopen(req, timeout=5) except urllib.error.HTTPError as err: resp = err except Exception as exc: return {"status": 0, "error": type(exc).__name__} raw = resp.read() js = None if raw and resp.headers.get_content_type() == "application/json": js = json.loads(raw.decode("utf-8")) return {"status": resp.code, "json": js, "instance": resp.headers.get("X-Instance-Id", "")} def api_ids(): out = subprocess.check_output( ["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], cwd=ROOT, text=True ) return out.split() def hostname(cid): return subprocess.check_output( ["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True ).strip() def replace_replica(old_id, known): """Stop+remove one replica, create a fresh one, wait until the LB routes to it.""" subprocess.check_call(["docker", "stop", "-t", "15", old_id], stdout=subprocess.DEVNULL) subprocess.check_call(["docker", "rm", old_id], stdout=subprocess.DEVNULL) # --no-recreate: the surviving replica keeps serving even if the image/config changed. subprocess.check_call( ["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={REPLICAS}", "api"], cwd=ROOT, ) new = [cid for cid in api_ids() if cid not in known] if len(new) != 1: raise RuntimeError(f"expected one new replica, got {new}") host = hostname(new[0]) deadline = time.monotonic() + 90 while time.monotonic() < deadline: r = call("GET", "/health/ready") if r.get("status") == 200 and r.get("instance") == host: return new[0], host time.sleep(0.5) raise RuntimeError(f"new replica {host} not in LB rotation after 90s") def bootstrap(): uname = "roll_" + uuid.uuid4().hex[:10] pw = "T3st-" + uuid.uuid4().hex user = call("POST", "/api/v1/users", {"username": uname, "display_name": "Roll", "password": pw}) assert user["status"] == 201, user cred = base64.b64encode(f"{uname}:{pw}".encode()).decode() tok = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})["json"]["token"] peer_u = "rollb_" + uuid.uuid4().hex[:10] peer = call("POST", "/api/v1/users", {"username": peer_u, "display_name": "RollB", "password": pw})["json"] chat = call( "POST", "/api/v1/chats", {"title": "roll", "member_ids": [user["json"]["id"], peer["id"]]}, token=tok, )["json"] return tok, chat["id"] def main(): tok, chat_id = bootstrap() stats = {"ok": 0, "err": 0, "err_by_status": {}, "latencies_ms": []} stop = threading.Event() def worker(): while not stop.is_set(): key = uuid.uuid4().hex t0 = time.monotonic() r = call( "POST", f"/api/v1/chats/{chat_id}/messages", {"text": "roll-" + key}, token=tok, headers={"Idempotency-Key": key}, ) dt = (time.monotonic() - t0) * 1000 if r.get("status") == 201: stats["ok"] += 1 stats["latencies_ms"].append(dt) else: stats["err"] += 1 msg = ((r.get("json") or {}).get("error") or {}).get("message", "") code = f"{r.get('status') or r.get('error')} {msg}".strip() stats["err_by_status"][code] = stats["err_by_status"].get(code, 0) + 1 time.sleep(0.05) ids = api_ids() known = set(ids) replaced = [] threads = [threading.Thread(target=worker, daemon=True) for _ in range(4)] started = time.monotonic() for th in threads: th.start() time.sleep(2) # warm-up # Rolling: replace each original replica in turn; the next one is stopped # only after the LB already routes requests to the fresh container. for cid in ids: old_host = hostname(cid) new_id, new_host = replace_replica(cid, known) known.add(new_id) replaced.append({"old": old_host, "new": new_host}) time.sleep(2) time.sleep(2) stop.set() for th in threads: th.join(timeout=5) duration = time.monotonic() - started lats = sorted(stats["latencies_ms"]) def pct(p): if not lats: return None return round(lats[min(len(lats) - 1, int(len(lats) * p))], 2) summary = { "successful": stats["ok"], "errors": stats["err"], "errors_by_status": stats["err_by_status"], "p50_ms": pct(0.50), "p95_ms": pct(0.95), "p99_ms": pct(0.99), "duration_s": round(duration, 1), "replicas_rolled": len(replaced), "replaced": replaced, } out = ROOT / "evidence" / "rolling-load.json" out.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") print(json.dumps(summary, ensure_ascii=False)) print("WROTE", out) if __name__ == "__main__": main()