#!/usr/bin/env python3 """What the load balancer thinks right now, mapped to real containers. python scripts/lbctl.py status # table: HAProxy slot, IP, UP/DOWN/MAINT, container, X-Instance-Id python scripts/lbctl.py wait-up 2 # wait until exactly 2 API replicas are UP in HAProxy python scripts/lbctl.py replicas # api containers of this compose project: name, hostname, state Reads HAProxy's runtime API (`show servers state api`) through `docker compose exec lb socat`, so no admin port is published. Container hostname = X-Instance-Id of that replica. Prints no secrets. Used by student-tests and bench scripts as a library as well. """ from __future__ import annotations import argparse import json import subprocess import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] # srv_op_state of HAProxy (src/server.h): 0 stopped, 1 starting, 2 running, 3 stopping. OP_STATE = {0: "DOWN", 1: "STARTING", 2: "UP", 3: "STOPPING"} ADMIN_RMAINT = 0x20 # slot without a resolved address (no running container behind it) def run(*args: str, timeout: float = 60) -> str: result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding="utf-8", timeout=timeout) if result.returncode != 0: raise RuntimeError(f"{' '.join(args[:3])}... failed: {result.stderr.strip()[:300]}") return result.stdout def replicas(all_states: bool = True) -> list[dict]: """API containers of this compose project (running and stopped).""" ids = run("docker", "compose", "ps", "-q", *(["-a"] if all_states else []), "api").split() if not ids: return [] data = json.loads(run("docker", "inspect", *ids)) result = [] for item in data: networks = item["NetworkSettings"]["Networks"] or {} result.append({ "id": item["Id"][:12], "name": item["Name"].lstrip("/"), "hostname": item["Config"]["Hostname"], "state": item["State"]["Status"], "health": (item["State"].get("Health") or {}).get("Status", "-"), "exit_code": item["State"]["ExitCode"], "started_at": item["State"]["StartedAt"], "pid": item["State"]["Pid"], "ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")), # compose network name without the project prefix -> IP, e.g. {"edge": ..., "data": ...} "networks": {name.split("_", 1)[-1]: n["IPAddress"] for name, n in networks.items() if n.get("IPAddress")}, }) return sorted(result, key=lambda r: r["name"]) def lb_servers() -> list[dict]: raw = run("docker", "compose", "exec", "-T", "lb", "sh", "-c", 'echo "show servers state api" | socat stdio /var/lib/haproxy/admin.sock') by_ip = {ip: r for r in replicas() for ip in r["ips"]} servers = [] for line in raw.splitlines(): parts = line.split() if len(parts) < 7 or not parts[0].isdigit(): continue name, addr, op, admin = parts[3], parts[4], int(parts[5]), int(parts[6]) if admin & ADMIN_RMAINT or addr == "-": state = "MAINT" else: state = OP_STATE.get(op, str(op)) container = by_ip.get(addr) servers.append({"slot": name, "addr": addr, "state": state, "container": container["name"] if container else None, "instance_id": container["hostname"] if container else None}) return servers def up_instances() -> set[str]: return {s["instance_id"] or s["addr"] for s in lb_servers() if s["state"] == "UP"} def wait_up(count: int, timeout: float = 60) -> float: """Seconds until exactly `count` servers are UP (raises TimeoutError otherwise).""" started = time.monotonic() while True: try: if len(up_instances()) == count: return time.monotonic() - started except RuntimeError: pass # lb restarting if time.monotonic() - started > timeout: raise TimeoutError(f"HAProxy does not report {count} UP API replicas after {timeout:.0f} s") time.sleep(0.25) def main() -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) sub = parser.add_subparsers(dest="command", required=True) sub.add_parser("status") sub.add_parser("replicas") wait = sub.add_parser("wait-up") wait.add_argument("count", type=int) wait.add_argument("--timeout", type=float, default=60) args = parser.parse_args() if args.command == "status": for s in lb_servers(): if s["state"] != "MAINT" or s["container"]: print(f"{s['slot']:6} {s['addr']:15} {s['state']:8} {s['container'] or '-':24} X-Instance-Id={s['instance_id'] or '-'}") print(f"(other slots: MAINT, no container)") elif args.command == "replicas": for r in replicas(): print(f"{r['name']:24} X-Instance-Id={r['hostname']} {r['state']:8} health={r['health']:9} " f"pid={r['pid']} started={r['started_at'][:23]} ip={','.join(r['ips']) or '-'}") else: took = wait_up(args.count, args.timeout) print(f"{args.count} API replica(s) UP in HAProxy after {took:.1f} s: {sorted(up_instances())}") return 0 if __name__ == "__main__": sys.exit(main())