Files
mtusi2026lab2/scripts/track1_cache_benchmark.py
T
ilya.tyunin 575f8ee386
Skeleton contracts / contracts (pull_request) Canceled after 0s
feat: lab2 done
2026-10-09 15:48:03 +03:00

284 lines
13 KiB
Python
Executable File

#!/usr/bin/env python3
"""Reproducible Track 1 cache benchmark (Python standard library only).
See evidence/README.md for the evidence index, evidence/track1-cache-consistency.md
for the recorded methodology and per-repetition results, and REPORT.md (section
"Track 1") for the surrounding context. Usage: --help.
"""
import argparse
import base64
import concurrent.futures
import json
import os
import platform
import sys
import threading
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
SEED_MESSAGE_COUNT = 1
SAFE_ENVIRONMENT = (
"APP_ENV",
"CACHE_MISS_DELAY_MS",
"CACHE_MODE",
"IMAGE_VERSION",
"API_IMAGE_VERSION",
"POSTGRES_IMAGE_VERSION",
"REDIS_IMAGE_VERSION",
)
class HttpFailure(RuntimeError):
"""An HTTP failure whose representation cannot contain request secrets."""
def __init__(self, status: int | None, kind: str = "http_error") -> None:
super().__init__(kind)
self.status = status
self.kind = kind
def request_json(base_url: str, method: str, path: str, *, token: str | None = None,
basic: tuple[str, str] | None = None, body: object | None = None,
timeout: float) -> object:
data = None if body is None else json.dumps(body).encode("utf-8")
headers = {"Accept": "application/json"}
if data is not None:
headers["Content-Type"] = "application/json"
if token is not None:
headers["Authorization"] = f"Bearer {token}"
if basic is not None:
encoded = base64.b64encode(f"{basic[0]}:{basic[1]}".encode("utf-8")).decode("ascii")
headers["Authorization"] = f"Basic {encoded}"
request = urllib.request.Request(f"{base_url.rstrip('/')}{path}", method=method,
headers=headers, data=data)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
raw = response.read()
return None if not raw else json.loads(raw.decode("utf-8"))
except urllib.error.HTTPError as error:
raise HttpFailure(error.code) from None
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as error:
raise HttpFailure(None, type(error).__name__) from None
def percentile(values: list[float], fraction: float) -> float:
if not values:
return 0.0
ordered = sorted(values)
index = max(0, min(len(ordered) - 1, int(len(ordered) * fraction + 0.999999) - 1))
return round(ordered[index], 3)
def require_object(value: object, name: str) -> dict[str, object]:
if not isinstance(value, dict):
raise RuntimeError(f"{name} response was not an object")
return value
def require_string(value: object, name: str) -> str:
if not isinstance(value, str) or not value:
raise RuntimeError(f"{name} response did not contain a string")
return value
def create_user(base_url: str, suffix: str, role: str, password: str, timeout: float) -> tuple[str, str]:
username = f"track1_{role}_{suffix}"
response = require_object(request_json(base_url, "POST", "/api/v1/users",
body={"username": username, "display_name": role, "password": password}, timeout=timeout), "user")
return username, require_string(response.get("id"), "user id")
def login(base_url: str, username: str, password: str, timeout: float) -> str:
response = require_object(request_json(base_url, "POST", "/api/v1/auth/sessions",
basic=(username, password), timeout=timeout), "session")
return require_string(response.get("token"), "session token")
def create_chat(base_url: str, token: str, member_ids: list[str], title: str, timeout: float) -> str:
response = require_object(request_json(base_url, "POST", "/api/v1/chats", token=token,
body={"title": title, "member_ids": member_ids}, timeout=timeout), "chat")
return require_string(response.get("id"), "chat id")
def post_message(base_url: str, token: str, chat_id: str, text: str, timeout: float) -> dict[str, object]:
return require_object(request_json(base_url, "POST", f"/api/v1/chats/{chat_id}/messages",
token=token, body={"text": text}, timeout=timeout), "message")
def history(base_url: str, token: str, chat_id: str, timeout: float) -> dict[str, object]:
return require_object(request_json(base_url, "GET", f"/api/v1/chats/{chat_id}/messages?limit=50",
token=token, timeout=timeout), "history")
def counter(snapshot: dict[str, object], *names: str) -> int:
counters = snapshot.get("counters", {})
if not isinstance(counters, dict):
return 0
return sum(int(counters.get(name, 0)) for name in names)
def metrics_delta(before: dict[str, object], after: dict[str, object]) -> dict[str, int]:
return {"history_sql": counter(after, "messages.history", "history_sql") - counter(before, "messages.history", "history_sql"),
"cache_hit": counter(after, "cache_hit") - counter(before, "cache_hit"),
"cache_miss": counter(after, "cache_miss") - counter(before, "cache_miss"),
"cache_lock_wait": counter(after, "cache_lock_wait") - counter(before, "cache_lock_wait")}
def run_barrier_gets(base_url: str, token: str, chat_id: str, concurrency: int,
timeout: float) -> tuple[list[float], dict[str, int]]:
barrier = threading.Barrier(concurrency)
def one() -> tuple[float | None, int | None, str | None]:
try:
barrier.wait(timeout=timeout + 5)
started = time.perf_counter()
history(base_url, token, chat_id, timeout)
return time.perf_counter() - started, None, None
except HttpFailure as error:
return None, error.status, error.kind
except (threading.BrokenBarrierError, TimeoutError):
return None, None, "barrier_timeout"
except Exception:
return None, None, "request_error"
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
outcomes = list(pool.map(lambda _: one(), range(concurrency)))
latencies = [seconds * 1000 for seconds, _, _ in outcomes if seconds is not None]
errors: dict[str, int] = {}
for _, status, kind in outcomes:
if kind is not None:
key = f"http_{status}" if status is not None else kind
errors[key] = errors.get(key, 0) + 1
return latencies, errors
def safe_metadata(args: argparse.Namespace) -> dict[str, object]:
supplied: dict[str, str] = {}
for item in args.runtime_version:
if "=" not in item:
continue
key, value = item.split("=", 1)
lowered = key.lower()
if (key and value and len(key) <= 64 and len(value) <= 128
and all(character.isalnum() or character in "_.-" for character in key)
and not any(term in lowered for term in ("token", "pass", "secret", "url"))):
supplied[key] = value
environment = {key: os.environ[key] for key in SAFE_ENVIRONMENT if os.environ.get(key)}
return {"python": platform.python_version(), "platform": platform.platform(aliased=True),
"supplied": supplied, "environment": environment}
def run(args: argparse.Namespace) -> dict[str, object]:
metrics_token = Path(args.metrics_token_file).read_text(encoding="utf-8").strip()
if not metrics_token:
raise RuntimeError("metrics token file is empty")
if args.concurrency < 1 or args.repetitions < 1:
raise RuntimeError("concurrency and repetitions must be positive")
suffix = uuid.uuid4().hex[:12]
password = f"Track1-only-{uuid.uuid4().hex[:20]}"
owner_name, owner_id = create_user(args.base_url, suffix, "owner", password, args.timeout)
reader_name, reader_id = create_user(args.base_url, suffix, "reader", password, args.timeout)
owner_token = login(args.base_url, owner_name, password, args.timeout)
reader_token = login(args.base_url, reader_name, password, args.timeout)
member_ids = [owner_id, reader_id]
warm_chat = create_chat(args.base_url, owner_token, member_ids, "track1-warm", args.timeout)
for index in range(SEED_MESSAGE_COUNT):
post_message(args.base_url, owner_token, warm_chat, f"track1 warm seed {index}", args.timeout)
history(args.base_url, reader_token, warm_chat, args.timeout)
consistency_chat = create_chat(args.base_url, owner_token, member_ids, "track1-consistency", args.timeout)
post_message(args.base_url, owner_token, consistency_chat, "track1 old page", args.timeout)
history(args.base_url, reader_token, consistency_chat, args.timeout)
new_message = post_message(args.base_url, owner_token, consistency_chat,
"track1 read-after-write", args.timeout)
new_id = new_message.get("id")
consistency_result: dict[str, object] = {"asserted": False, "stale_observed": False}
try:
after_write = history(args.base_url, reader_token, consistency_chat, args.timeout)
items = after_write.get("items", [])
present = isinstance(items, list) and any(isinstance(item, dict) and item.get("id") == new_id for item in items)
consistency_result["asserted"] = bool(present)
consistency_result["stale_observed"] = not present
if args.mode == "protected" and not present:
raise RuntimeError("protected read-after-write assertion failed")
except HttpFailure:
if args.mode == "protected":
raise
consistency_result["stale_observed"] = True
repetitions: list[dict[str, object]] = []
for repetition in range(1, args.repetitions + 1):
cold_chat = create_chat(args.base_url, owner_token, member_ids,
f"track1-cold-{repetition}", args.timeout)
for index in range(SEED_MESSAGE_COUNT):
post_message(args.base_url, owner_token, cold_chat,
f"track1 cold {repetition} seed {index}", args.timeout)
request_json(args.base_url, "POST", "/internal/metrics/reset", token=metrics_token, timeout=args.timeout)
before = require_object(request_json(args.base_url, "GET", "/internal/metrics",
token=metrics_token, timeout=args.timeout), "metrics")
latencies, errors = run_barrier_gets(args.base_url, reader_token, cold_chat,
args.concurrency, args.timeout)
after = require_object(request_json(args.base_url, "GET", "/internal/metrics",
token=metrics_token, timeout=args.timeout), "metrics")
repetitions.append({"repetition": repetition, "successful_requests": len(latencies),
"error_counts": errors, "metrics_delta": metrics_delta(before, after),
"latency_ms": {"p50": percentile(latencies, .50), "p95": percentile(latencies, .95),
"p99": percentile(latencies, .99)}})
totals = {"history_sql": 0, "cache_hit": 0, "cache_miss": 0, "cache_lock_wait": 0}
errors_total: dict[str, int] = {}
for repetition in repetitions:
metrics = repetition["metrics_delta"]
if isinstance(metrics, dict):
for key in totals:
totals[key] += int(metrics.get(key, 0))
errors = repetition["error_counts"]
if isinstance(errors, dict):
for key, value in errors.items():
errors_total[key] = errors_total.get(key, 0) + int(value)
return {"schema_version": 1, "mode": args.mode,
"workload": {"concurrency": args.concurrency, "repetitions": args.repetitions,
"seed_messages": SEED_MESSAGE_COUNT,
"barrier": "threading.Barrier before each GET",
"warmup": "one separate chat page before measurements",
"read_after_write": consistency_result},
"totals": {"metrics_delta": totals, "error_counts": errors_total},
"repetitions": repetitions, "runtime": safe_metadata(args)}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--mode", choices=("naive", "protected"), required=True)
parser.add_argument("--concurrency", type=int, default=32)
parser.add_argument("--repetitions", type=int, default=5)
parser.add_argument("--metrics-token-file", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--timeout", type=float, default=30.0)
parser.add_argument("--runtime-version", action="append", default=[], metavar="NAME=VALUE",
help="safe image/runtime metadata (repeatable; no secrets)")
return parser.parse_args()
def main() -> None:
args = parse_args()
result = run(args)
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(f"wrote {output}")
if __name__ == "__main__":
try:
main()
except (HttpFailure, OSError, RuntimeError) as error:
print(f"benchmark failed: {error}", file=sys.stderr)
raise SystemExit(2)