#!/usr/bin/env python3 """Reproducible, rate-controlled GET benchmark for the lab3 scaling track.""" import argparse from collections import defaultdict import http.client import json import math import os from pathlib import Path from queue import Full, Queue import statistics import subprocess import sys import tempfile import threading import time from urllib.parse import urlsplit from urllib.request import urlopen ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client # noqa: E402 - the repository's public bootstrap client def topology(mode, replicas): if mode not in ("fixed", "growth") or replicas not in (1, 2, 4): raise ValueError("mode must be fixed/growth and replicas must be 1/2/4") return replicas, (1 / replicas if mode == "fixed" else 0.5), (512 // replicas if mode == "fixed" else 256) def validate_matrix(rows): if not rows: raise ValueError("no benchmark rows") rates = {row["rate"] for row in rows} if len(rates) < 3: raise ValueError("need at least three offered rates") profiles = {(row["duration"], row["warmup"], row["dataset_messages"]) for row in rows} if len(profiles) != 1: raise ValueError("workload profile changed across measurements") for mode in ("fixed", "growth"): for replicas in (1, 2, 4): _, cpu, memory = topology(mode, replicas) for rate in rates: repeats = [row["repeat"] for row in rows if (row["mode"], row["replicas"], row["rate"]) == (mode, replicas, rate)] if sorted(repeats) != [1, 2, 3]: raise ValueError(f"need three repeats for {mode}/{replicas}/{rate}") for row in rows: if (row["mode"], row["replicas"]) == (mode, replicas) and (row["api_cpu_each"], row["api_memory_mib_each"]) != (cpu, memory): raise ValueError("incorrect API resource quota") def percentile(values, fraction): if not values: return None ordered = sorted(values) return round(ordered[math.ceil(fraction * len(ordered)) - 1], 3) def command(*args): return subprocess.run(args, cwd=ROOT, text=True, capture_output=True, check=True).stdout.strip() def compose(*args, override=None): prefix = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"] if override: prefix += ["-f", str(override)] return command(*prefix, *args) def pool_snapshot(base_url, replicas): found = {} for _ in range(8 * replicas): with urlopen(base_url + "/health/ready", timeout=4) as response: headers = response.headers found[headers["X-Instance-Id"]] = { "requests": int(headers["X-Pool-Requests"]), "queued": int(headers["X-Pool-Queued"]), "wait_ms": int(headers["X-Pool-Wait-Ms"]), "size": int(headers["X-Pool-Size"]), "available": int(headers["X-Pool-Available"]), } if len(found) == replicas: break return found def db_snapshot(override): sql = ("SELECT pg_database_size(current_database()),xact_commit,xact_rollback," "blks_read,blks_hit,temp_files,deadlocks,(SELECT count(*) FROM messages) " "FROM pg_stat_database WHERE datname=current_database()") values = command("docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", str(override), "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql) return dict(zip(("database_bytes", "commits", "rollbacks", "blocks_read", "blocks_hit", "temp_files", "deadlocks", "messages"), map(int, values.split("|")))) def docker_ids(override): ids = {} for service in ("api", "postgres", "lb"): ids[service] = compose("ps", "-q", service, override=override).splitlines() return ids def stats_sample(ids): all_ids = [id for service_ids in ids.values() for id in service_ids] if not all_ids: return {} output = command("docker", "stats", "--no-stream", "--format", "{{json .}}", *all_ids) by_id = {} for line in output.splitlines(): item = json.loads(line) by_id[item["ID"]] = item result = {} for service, service_ids in ids.items(): cpu = [] memory = [] for id in service_ids: item = by_id.get(id[:12]) or by_id.get(id) if not item: raise ValueError("Docker stats omitted a benchmark container") cpu.append(float(item["CPUPerc"].rstrip("%"))) amount = item["MemUsage"].split("/")[0].strip() value, unit = float("".join(c for c in amount if c.isdigit() or c == ".")), "".join(c for c in amount if c.isalpha()) memory.append(round(value * {"B": 1 / 2**20, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024, "kB": 1000 / 2**20, "MB": 10**6 / 2**20, "GB": 10**9 / 2**20}[unit], 3)) result[service] = {"cpu_percent": round(sum(cpu), 3), "memory_mib": round(sum(memory), 3)} return result def open_loop(base_url, path, token, rate, duration, workers=64): """Schedule arrivals by clock; latency includes queue wait and HTTP time.""" address = urlsplit(base_url) if address.scheme != "http": raise ValueError("this local lab3 benchmark expects HTTP") jobs = Queue(maxsize=256) latencies = [] errors = defaultdict(int) lock = threading.Lock() starts = time.monotonic() cpu_start = time.process_time() count = math.ceil(rate * duration) def worker(): connection = None while True: due = jobs.get() if due is None: jobs.task_done() break try: if connection is None: connection = http.client.HTTPConnection(address.hostname, address.port or 80, timeout=4) connection.request("GET", path, headers={"Authorization": "Bearer " + token, "Accept": "application/json"}) response = connection.getresponse() payload = response.read() if response.status != 200: with lock: errors[f"http_{response.status}"] += 1 elif len(json.loads(payload)["items"]) != 20: with lock: errors["wrong_page"] += 1 else: with lock: latencies.append((time.monotonic() - due) * 1000) except (OSError, TimeoutError, ValueError, KeyError, http.client.HTTPException): with lock: errors["transport_or_response"] += 1 if connection: connection.close() connection = None finally: jobs.task_done() if connection: connection.close() threads = [threading.Thread(target=worker, daemon=True) for _ in range(workers)] for thread in threads: thread.start() dropped = 0 for index in range(count): due = starts + index / rate delay = due - time.monotonic() if delay > 0: time.sleep(delay) try: jobs.put_nowait(due) except Full: dropped += 1 jobs.join() for _ in threads: jobs.put(None) for thread in threads: thread.join() elapsed = time.monotonic() - starts error_count = sum(errors.values()) + dropped return { "offered_rate": rate, "scheduled_requests": count, "successful_requests": len(latencies), "failed_requests": error_count, "error_ratio": round(error_count / count, 6), "successful_rps": round(len(latencies) / elapsed, 3), "elapsed_seconds": round(elapsed, 3), "successful_latency_ms": {f"p{p}": percentile(latencies, p / 100) for p in (50, 95, 99)}, "errors_by_type": dict(errors), "queue_dropped": dropped, "client_cpu_seconds": round(time.process_time() - cpu_start, 3), } def fixture(base_url, size, fixture_file): if fixture_file.exists(): saved = json.loads(fixture_file.read_text()) if saved["base_url"] != base_url or saved["dataset_messages"] != size: raise ValueError("saved fixture URL or size differs from requested profile") Client(base_url).call("GET", saved["path"], actor={"token": saved["token"]}) return saved["token"], saved["path"] client = Client(base_url) alice, _, _, chat = client.bootstrap() for index in range(size): client.message(alice, chat, f"synthetic-scale-{index}") path = "/api/v1/chats/" + chat["id"] + "/messages?limit=20" fixture_file.parent.mkdir(parents=True, exist_ok=True) with os.fdopen(os.open(fixture_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as file: file.write(json.dumps({"base_url": base_url, "dataset_messages": size, "token": alice["token"], "path": path}) + "\n") return alice["token"], path def run(args): full_matrix = args.modes == ["fixed", "growth"] and args.replicas == [1, 2, 4] if args.repeats != 3 or (full_matrix and len(args.rates) < 3) or args.dataset_messages < 20: raise ValueError("require 3 repeats, at least 3 rates for a full matrix, and at least 20 fixture messages") args.output.parent.mkdir(parents=True, exist_ok=True) if args.output.exists(): raise ValueError(f"output already exists: {args.output}") token = path = None rows = [] with tempfile.TemporaryDirectory() as temporary: override = Path(temporary) / "scale.yaml" for mode in args.modes: for replicas in args.replicas: _, cpu, memory = topology(mode, replicas) override.write_text("services:\n api:\n cpus: " + str(cpu) + "\n mem_limit: " + str(memory) + "m\n") compose("up", "-d", "--build", "--wait", "--scale", f"api={replicas}", override=override) if token is None: token, path = fixture(args.base_url, args.dataset_messages, args.fixture) ids = docker_ids(override) for rate in args.rates: for repeat in range(1, args.repeats + 1): open_loop(args.base_url, path, token, rate, args.warmup) pool_before = pool_snapshot(args.base_url, replicas) db_before = db_snapshot(override) samples = [] sampling_errors = [] stop = threading.Event() def sample(): while not stop.is_set(): try: samples.append(stats_sample(ids)) except (subprocess.CalledProcessError, ValueError, KeyError) as error: sampling_errors.append(type(error).__name__) return stop.wait(1) sampler = threading.Thread(target=sample, daemon=True) sampler.start() result = open_loop(args.base_url, path, token, rate, args.duration) stop.set() sampler.join() if sampling_errors or not samples: raise RuntimeError(f"Docker CPU/RAM sampler failed: {sampling_errors}") db_after = db_snapshot(override) if db_after["messages"] != db_before["messages"]: raise RuntimeError("message dataset changed during read-only benchmark") pool_after = pool_snapshot(args.base_url, replicas) pool_delta = { "observed_replicas": len(set(pool_before) & set(pool_after)), "requests_queued": sum(pool_after[i]["queued"] - pool_before[i]["queued"] for i in pool_before.keys() & pool_after.keys()), "wait_ms": sum(pool_after[i]["wait_ms"] - pool_before[i]["wait_ms"] for i in pool_before.keys() & pool_after.keys()), "max_size": max((v["size"] for v in pool_after.values()), default=0), } services = {name: { "avg_cpu_percent": round(statistics.mean(s[name]["cpu_percent"] for s in samples if name in s), 3), "max_memory_mib": round(max(s[name]["memory_mib"] for s in samples if name in s), 3), } for name in ("api", "postgres", "lb") if any(name in s for s in samples)} row = { "mode": mode, "replicas": replicas, "rate": rate, "repeat": repeat, "duration": args.duration, "warmup": args.warmup, "dataset_messages": args.dataset_messages, "api_cpu_each": cpu, "api_memory_mib_each": memory, "pool_max_size": 8, **result, "docker": services, "pool": pool_delta, "postgres_delta": {key: db_after[key] - db_before[key] for key in db_before if key != "database_bytes"}, "database_bytes": db_after["database_bytes"], "database_messages": db_after["messages"], } rows.append(row) with args.output.open("a") as output: output.write(json.dumps(row, ensure_ascii=False) + "\n") print(f"{mode} replicas={replicas} rate={rate} repeat={repeat} rps={result['successful_rps']} errors={result['error_ratio']}", flush=True) if full_matrix: validate_matrix(rows) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default="http://127.0.0.1:8080") parser.add_argument("--rates", type=lambda value: [int(x) for x in value.split(",")], default=[100, 250, 500]) parser.add_argument("--modes", type=lambda value: value.split(","), default=["fixed", "growth"]) parser.add_argument("--replicas", type=lambda value: [int(x) for x in value.split(",")], default=[1, 2, 4]) parser.add_argument("--duration", type=int, default=8) parser.add_argument("--warmup", type=int, default=2) parser.add_argument("--repeats", type=int, default=3) parser.add_argument("--dataset-messages", type=int, default=1000) parser.add_argument("--fixture", type=Path, default=ROOT / "secrets/scale_fixture.json") parser.add_argument("--output", type=Path, default=ROOT / "evidence/scale/baseline.jsonl") args = parser.parse_args() if (not args.rates or min(args.rates) <= 0 or not args.modes or not args.replicas or any(mode not in ("fixed", "growth") for mode in args.modes) or any(replica not in (1, 2, 4) for replica in args.replicas) or args.duration <= 0 or args.warmup <= 0): raise ValueError("rates and durations must be positive") run(args) if __name__ == "__main__": main()