#!/usr/bin/env python3 """Reproducible Track 1 cache benchmark (Python standard library only). See evidence/README.md for the evidence index, evidence/track1-cache-consistency.md for the recorded methodology and per-repetition results, and REPORT.md (section "Track 1") for the surrounding context. Usage: --help. """ import argparse import base64 import concurrent.futures import json import os import platform import sys import threading import time import urllib.error import urllib.request import uuid from pathlib import Path SEED_MESSAGE_COUNT = 1 SAFE_ENVIRONMENT = ( "APP_ENV", "CACHE_MISS_DELAY_MS", "CACHE_MODE", "IMAGE_VERSION", "API_IMAGE_VERSION", "POSTGRES_IMAGE_VERSION", "REDIS_IMAGE_VERSION", ) class HttpFailure(RuntimeError): """An HTTP failure whose representation cannot contain request secrets.""" def __init__(self, status: int | None, kind: str = "http_error") -> None: super().__init__(kind) self.status = status self.kind = kind def request_json(base_url: str, method: str, path: str, *, token: str | None = None, basic: tuple[str, str] | None = None, body: object | None = None, timeout: float) -> object: data = None if body is None else json.dumps(body).encode("utf-8") headers = {"Accept": "application/json"} if data is not None: headers["Content-Type"] = "application/json" if token is not None: headers["Authorization"] = f"Bearer {token}" if basic is not None: encoded = base64.b64encode(f"{basic[0]}:{basic[1]}".encode("utf-8")).decode("ascii") headers["Authorization"] = f"Basic {encoded}" request = urllib.request.Request(f"{base_url.rstrip('/')}{path}", method=method, headers=headers, data=data) try: with urllib.request.urlopen(request, timeout=timeout) as response: raw = response.read() return None if not raw else json.loads(raw.decode("utf-8")) except urllib.error.HTTPError as error: raise HttpFailure(error.code) from None except (urllib.error.URLError, TimeoutError, OSError, ValueError) as error: raise HttpFailure(None, type(error).__name__) from None def percentile(values: list[float], fraction: float) -> float: if not values: return 0.0 ordered = sorted(values) index = max(0, min(len(ordered) - 1, int(len(ordered) * fraction + 0.999999) - 1)) return round(ordered[index], 3) def require_object(value: object, name: str) -> dict[str, object]: if not isinstance(value, dict): raise RuntimeError(f"{name} response was not an object") return value def require_string(value: object, name: str) -> str: if not isinstance(value, str) or not value: raise RuntimeError(f"{name} response did not contain a string") return value def create_user(base_url: str, suffix: str, role: str, password: str, timeout: float) -> tuple[str, str]: username = f"track1_{role}_{suffix}" response = require_object(request_json(base_url, "POST", "/api/v1/users", body={"username": username, "display_name": role, "password": password}, timeout=timeout), "user") return username, require_string(response.get("id"), "user id") def login(base_url: str, username: str, password: str, timeout: float) -> str: response = require_object(request_json(base_url, "POST", "/api/v1/auth/sessions", basic=(username, password), timeout=timeout), "session") return require_string(response.get("token"), "session token") def create_chat(base_url: str, token: str, member_ids: list[str], title: str, timeout: float) -> str: response = require_object(request_json(base_url, "POST", "/api/v1/chats", token=token, body={"title": title, "member_ids": member_ids}, timeout=timeout), "chat") return require_string(response.get("id"), "chat id") def post_message(base_url: str, token: str, chat_id: str, text: str, timeout: float) -> dict[str, object]: return require_object(request_json(base_url, "POST", f"/api/v1/chats/{chat_id}/messages", token=token, body={"text": text}, timeout=timeout), "message") def history(base_url: str, token: str, chat_id: str, timeout: float) -> dict[str, object]: return require_object(request_json(base_url, "GET", f"/api/v1/chats/{chat_id}/messages?limit=50", token=token, timeout=timeout), "history") def counter(snapshot: dict[str, object], *names: str) -> int: counters = snapshot.get("counters", {}) if not isinstance(counters, dict): return 0 return sum(int(counters.get(name, 0)) for name in names) def metrics_delta(before: dict[str, object], after: dict[str, object]) -> dict[str, int]: return {"history_sql": counter(after, "messages.history", "history_sql") - counter(before, "messages.history", "history_sql"), "cache_hit": counter(after, "cache_hit") - counter(before, "cache_hit"), "cache_miss": counter(after, "cache_miss") - counter(before, "cache_miss"), "cache_lock_wait": counter(after, "cache_lock_wait") - counter(before, "cache_lock_wait")} def run_barrier_gets(base_url: str, token: str, chat_id: str, concurrency: int, timeout: float) -> tuple[list[float], dict[str, int]]: barrier = threading.Barrier(concurrency) def one() -> tuple[float | None, int | None, str | None]: try: barrier.wait(timeout=timeout + 5) started = time.perf_counter() history(base_url, token, chat_id, timeout) return time.perf_counter() - started, None, None except HttpFailure as error: return None, error.status, error.kind except (threading.BrokenBarrierError, TimeoutError): return None, None, "barrier_timeout" except Exception: return None, None, "request_error" with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: outcomes = list(pool.map(lambda _: one(), range(concurrency))) latencies = [seconds * 1000 for seconds, _, _ in outcomes if seconds is not None] errors: dict[str, int] = {} for _, status, kind in outcomes: if kind is not None: key = f"http_{status}" if status is not None else kind errors[key] = errors.get(key, 0) + 1 return latencies, errors def safe_metadata(args: argparse.Namespace) -> dict[str, object]: supplied: dict[str, str] = {} for item in args.runtime_version: if "=" not in item: continue key, value = item.split("=", 1) lowered = key.lower() if (key and value and len(key) <= 64 and len(value) <= 128 and all(character.isalnum() or character in "_.-" for character in key) and not any(term in lowered for term in ("token", "pass", "secret", "url"))): supplied[key] = value environment = {key: os.environ[key] for key in SAFE_ENVIRONMENT if os.environ.get(key)} return {"python": platform.python_version(), "platform": platform.platform(aliased=True), "supplied": supplied, "environment": environment} def run(args: argparse.Namespace) -> dict[str, object]: metrics_token = Path(args.metrics_token_file).read_text(encoding="utf-8").strip() if not metrics_token: raise RuntimeError("metrics token file is empty") if args.concurrency < 1 or args.repetitions < 1: raise RuntimeError("concurrency and repetitions must be positive") suffix = uuid.uuid4().hex[:12] password = f"Track1-only-{uuid.uuid4().hex[:20]}" owner_name, owner_id = create_user(args.base_url, suffix, "owner", password, args.timeout) reader_name, reader_id = create_user(args.base_url, suffix, "reader", password, args.timeout) owner_token = login(args.base_url, owner_name, password, args.timeout) reader_token = login(args.base_url, reader_name, password, args.timeout) member_ids = [owner_id, reader_id] warm_chat = create_chat(args.base_url, owner_token, member_ids, "track1-warm", args.timeout) for index in range(SEED_MESSAGE_COUNT): post_message(args.base_url, owner_token, warm_chat, f"track1 warm seed {index}", args.timeout) history(args.base_url, reader_token, warm_chat, args.timeout) consistency_chat = create_chat(args.base_url, owner_token, member_ids, "track1-consistency", args.timeout) post_message(args.base_url, owner_token, consistency_chat, "track1 old page", args.timeout) history(args.base_url, reader_token, consistency_chat, args.timeout) new_message = post_message(args.base_url, owner_token, consistency_chat, "track1 read-after-write", args.timeout) new_id = new_message.get("id") consistency_result: dict[str, object] = {"asserted": False, "stale_observed": False} try: after_write = history(args.base_url, reader_token, consistency_chat, args.timeout) items = after_write.get("items", []) present = isinstance(items, list) and any(isinstance(item, dict) and item.get("id") == new_id for item in items) consistency_result["asserted"] = bool(present) consistency_result["stale_observed"] = not present if args.mode == "protected" and not present: raise RuntimeError("protected read-after-write assertion failed") except HttpFailure: if args.mode == "protected": raise consistency_result["stale_observed"] = True repetitions: list[dict[str, object]] = [] for repetition in range(1, args.repetitions + 1): cold_chat = create_chat(args.base_url, owner_token, member_ids, f"track1-cold-{repetition}", args.timeout) for index in range(SEED_MESSAGE_COUNT): post_message(args.base_url, owner_token, cold_chat, f"track1 cold {repetition} seed {index}", args.timeout) request_json(args.base_url, "POST", "/internal/metrics/reset", token=metrics_token, timeout=args.timeout) before = require_object(request_json(args.base_url, "GET", "/internal/metrics", token=metrics_token, timeout=args.timeout), "metrics") latencies, errors = run_barrier_gets(args.base_url, reader_token, cold_chat, args.concurrency, args.timeout) after = require_object(request_json(args.base_url, "GET", "/internal/metrics", token=metrics_token, timeout=args.timeout), "metrics") repetitions.append({"repetition": repetition, "successful_requests": len(latencies), "error_counts": errors, "metrics_delta": metrics_delta(before, after), "latency_ms": {"p50": percentile(latencies, .50), "p95": percentile(latencies, .95), "p99": percentile(latencies, .99)}}) totals = {"history_sql": 0, "cache_hit": 0, "cache_miss": 0, "cache_lock_wait": 0} errors_total: dict[str, int] = {} for repetition in repetitions: metrics = repetition["metrics_delta"] if isinstance(metrics, dict): for key in totals: totals[key] += int(metrics.get(key, 0)) errors = repetition["error_counts"] if isinstance(errors, dict): for key, value in errors.items(): errors_total[key] = errors_total.get(key, 0) + int(value) return {"schema_version": 1, "mode": args.mode, "workload": {"concurrency": args.concurrency, "repetitions": args.repetitions, "seed_messages": SEED_MESSAGE_COUNT, "barrier": "threading.Barrier before each GET", "warmup": "one separate chat page before measurements", "read_after_write": consistency_result}, "totals": {"metrics_delta": totals, "error_counts": errors_total}, "repetitions": repetitions, "runtime": safe_metadata(args)} def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", default="http://localhost:8080") parser.add_argument("--mode", choices=("naive", "protected"), required=True) parser.add_argument("--concurrency", type=int, default=32) parser.add_argument("--repetitions", type=int, default=5) parser.add_argument("--metrics-token-file", required=True) parser.add_argument("--output", required=True) parser.add_argument("--timeout", type=float, default=30.0) parser.add_argument("--runtime-version", action="append", default=[], metavar="NAME=VALUE", help="safe image/runtime metadata (repeatable; no secrets)") return parser.parse_args() def main() -> None: args = parse_args() result = run(args) output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") print(f"wrote {output}") if __name__ == "__main__": try: main() except (HttpFailure, OSError, RuntimeError) as error: print(f"benchmark failed: {error}", file=sys.stderr) raise SystemExit(2)