#!/usr/bin/env python3 """Background HTTP traffic through the LB for failover / rolling-update experiments (level 4). Closed loop, N worker threads, each: POST a message (Idempotency-Key) + GET the last page. An uncertain POST (timeout, connection error, 5xx) is retried with the SAME key and body, exactly as a correct client must do; the history is checked afterwards for losses and duplicates. Every request is recorded as (t, kind, status, instance, latency). This is a correctness/availability probe, not a capacity benchmark (see bench/track1 for that). python bench/loadgen.py --duration 30 --workers 4 # prints a summary JSON """ from __future__ import annotations import argparse import base64 import json import math import os import threading import time import uuid from collections import Counter, defaultdict import urllib.error import urllib.request BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") class Http: """Minimal stdlib client with keep-alive per thread (http.client under urllib is not).""" def __init__(self, base_url: str, timeout: float): import http.client from urllib.parse import urlsplit parts = urlsplit(base_url) self.host, self.port = parts.hostname, parts.port or 80 self.timeout = timeout self.factory = lambda: http.client.HTTPConnection(self.host, self.port, timeout=self.timeout) self.conn = self.factory() def request(self, method: str, path: str, body: bytes | None = None, headers: dict | None = None): try: self.conn.request(method, path, body=body, headers=headers or {}) response = self.conn.getresponse() data = response.read() if response.getheader("connection", "").lower() == "close": self.conn.close() self.conn = self.factory() return response.status, dict(response.getheaders()), data except Exception: self.conn.close() self.conn = self.factory() raise def call(method: str, path: str, body=None, headers=None, timeout=10.0): data = json.dumps(body).encode() if body is not None else None h = {"Content-Type": "application/json", **(headers or {})} req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method) try: with urllib.request.urlopen(req, timeout=timeout) as r: return r.status, json.loads(r.read() or b"null") except urllib.error.HTTPError as e: return e.code, json.loads(e.read() or b"null") def make_user(label: str) -> dict: username = f"{label}_{uuid.uuid4().hex[:12]}" password = "Load-password-" + uuid.uuid4().hex status, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password}) assert status == 201, (status, user) cred = base64.b64encode(f"{username}:{password}".encode()).decode() status, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) assert status == 201, (status, session) return {"id": user["id"], "token": session["token"]} class Load: def __init__(self, workers: int = 4, timeout: float = 3.0, pause: float = 0.0): self.workers = workers self.timeout = timeout self.pause = pause self.alice, self.bob = make_user("load"), make_user("load") status, chat = call("POST", "/api/v1/chats", {"title": "load", "member_ids": [self.alice["id"], self.bob["id"]]}, headers={"Authorization": "Bearer " + self.alice["token"]}) assert status == 201, chat self.chat = chat self.stop_event = threading.Event() self.lock = threading.Lock() self.events: list[tuple[float, str, int, str, float]] = [] self.confirmed: dict[str, dict] = {} self.retries = 0 self.t0 = time.monotonic() self.threads = [threading.Thread(target=self._worker, args=(i,), daemon=True) for i in range(workers)] def now(self) -> float: return time.monotonic() - self.t0 def _record(self, kind: str, status: int, instance: str, latency: float) -> None: with self.lock: self.events.append((round(self.now(), 4), kind, status, instance, round(latency * 1000, 2))) def _worker(self, index: int) -> None: http = Http(BASE_URL, self.timeout) auth = {"Authorization": "Bearer " + self.alice["token"]} path = f"/api/v1/chats/{self.chat['id']}/messages" n, key = 0, uuid.uuid4().hex text = f"w{index}-0-{key[:6]}" while not self.stop_event.is_set(): started = time.monotonic() try: status, headers, data = http.request("POST", path, json.dumps({"text": text}).encode(), {**auth, "Idempotency-Key": key, "Content-Type": "application/json"}) instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-" except Exception: status, instance, data = 0, "-", b"" self._record("POST", status, instance, time.monotonic() - started) if status == 201: msg = json.loads(data) with self.lock: self.confirmed[msg["id"]] = msg n += 1 key = uuid.uuid4().hex text = f"w{index}-{n}-{key[:6]}" elif status in (0, 502, 503, 504, 500): with self.lock: self.retries += 1 # same key + same text on the next iteration started = time.monotonic() try: status, headers, _ = http.request("GET", path + "?limit=5", None, auth) instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-" except Exception: status, instance = 0, "-" self._record("GET", status, instance, time.monotonic() - started) if self.pause: time.sleep(self.pause) def start(self) -> "Load": self.t0 = time.monotonic() for t in self.threads: t.start() return self def stop(self) -> None: self.stop_event.set() for t in self.threads: t.join(timeout=15) def verify_history(self) -> dict: """Walk the whole history with the second user: no confirmed message lost, no duplicate.""" items, cursor = [], None while True: q = "?limit=100" + (f"&cursor={cursor}" if cursor else "") status, page = call("GET", f"/api/v1/chats/{self.chat['id']}/messages{q}", headers={"Authorization": "Bearer " + self.bob["token"]}, timeout=30) assert status == 200, page items += page["items"] cursor = page["next_cursor"] if not cursor: break ids = [m["id"] for m in items] texts = [m["text"] for m in items] lost = [i for i in self.confirmed if i not in set(ids)] changed = [m for m in items if m["id"] in self.confirmed and self.confirmed[m["id"]] != m] return {"history_messages": len(items), "confirmed_messages": len(self.confirmed), "lost_confirmed": len(lost), "altered_confirmed": len(changed), "duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)), "unconfirmed_but_stored": len(set(ids) - set(self.confirmed))} # ------------------------------------------------------------------ summaries def window(self, start: float, end: float) -> dict: ev = [e for e in self.events if start <= e[0] < end] ok = [e for e in ev if e[2] in (200, 201)] lat = sorted(e[4] for e in ok) def pct(p: float): return lat[max(0, math.ceil(p * len(lat)) - 1)] if lat else None span = max(end - start, 1e-9) return {"seconds": round(span, 2), "requests": len(ev), "ok": len(ok), "failed": len(ev) - len(ok), "ok_rps": round(len(ok) / span, 1), "error_ratio": round((len(ev) - len(ok)) / len(ev), 5) if ev else None, "statuses": dict(Counter(e[2] for e in ev)), "instances": dict(Counter(e[3] for e in ok)), "p50_ms": pct(.5), "p95_ms": pct(.95), "p99_ms": pct(.99)} def per_second(self) -> list[dict]: buckets: dict[int, list] = defaultdict(list) for e in self.events: buckets[int(e[0])].append(e) rows = [] for second in sorted(buckets): ev = buckets[second] rows.append({"second": second, "ok": sum(1 for e in ev if e[2] in (200, 201)), "failed": sum(1 for e in ev if e[2] not in (200, 201)), "instances": "|".join(f"{k}:{v}" for k, v in sorted(Counter(e[3] for e in ev).items()))}) return rows def main() -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--duration", type=float, default=20) parser.add_argument("--workers", type=int, default=4) args = parser.parse_args() load = Load(args.workers).start() time.sleep(args.duration) load.stop() print(json.dumps({"total": load.window(0, load.now()), "history": load.verify_history(), "client_retries_same_key": load.retries}, ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": raise SystemExit(main())