Files
mtusi2026lab4/bench/reconcile_cost.py
T

157 lines
7.2 KiB
Python

#!/usr/bin/env python3
"""Track 2 experiment: cost of one reconciler pass as a function of the number of attachments.
Hypothesis: a pass is O(N) with ~N/1000 S3 list requests + N/1000 DB pages and constant memory
(merge-join of two key-ordered streams); --verify-checksums adds reading every byte (O(total size)).
Method: synthetic, CONSISTENT attachments (row 'ready' + 1 KiB object with the matching size and
SHA-256, filename 'bench-reconcile') are added in steps up to the largest N; after each step the
reconciler runs in dry-run 3 times (median reported), plus page-size and checksum variants.
Everything synthetic is deleted at the end (objects first, then rows).
python bench/reconcile_cost.py --steps 1000 10000 50000 --output evidence/reconcile
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import platform
import statistics
import subprocess
import sys
import time
import uuid
from pathlib import Path
import httpx2 as httpx
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
PASSWORD = "Bench-password-Ω-12345"
OBJECT = bytes(range(256)) * 4 # 1 KiB, the same content for every object
SHA = hashlib.sha256(OBJECT).hexdigest()
PUT_SCRIPT = """
import asyncio, sys
from messenger.config import Settings, s3_settings
from messenger.storage import Storage
DATA = bytes(range(256)) * 4
async def main():
s = Storage(Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings()))
await s.open()
keys = sys.stdin.read().split()
sem = asyncio.Semaphore(32)
async def one(k):
async with sem:
await s.{op}(k{args})
await asyncio.gather(*(one(k) for k in keys))
await s.close()
print(len(keys))
asyncio.run(main())
"""
def compose(*args: str, stdin: str | None = None, timeout: float = 1800, check: bool = True) -> subprocess.CompletedProcess:
return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True,
encoding="utf-8", timeout=timeout, check=check)
def psql(sql: str) -> str:
"""SQL through stdin: a 5000-row INSERT does not fit into a Windows command line (32 KiB)."""
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v",
"ON_ERROR_STOP=1", "-f", "-", stdin=sql).stdout.strip()
def s3_bulk(op: str, keys: list[str]) -> None:
script = PUT_SCRIPT.replace("{op}", op).replace("{args}", ", DATA, 'image/png', {}" if op == "put" else "")
compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "python", "reconciler", "-c", script, stdin="\n".join(keys))
def owner_and_chat() -> tuple[str, str]:
import base64
c = httpx.Client(base_url=BASE, timeout=30)
name = "rc_" + uuid.uuid4().hex[:10]
c.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
token = c.post("/api/v1/auth/sessions", headers={
"Authorization": "Basic " + base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()}).json()["token"]
me = c.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
chat = c.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
json={"title": "bench", "member_ids": [me["id"]]}).json()
return me["id"], chat["id"]
def add(n: int, owner: str, chat: str) -> None:
keys = ["o/" + uuid.uuid4().hex for _ in range(n)]
for i in range(0, n, 5000):
part = keys[i:i + 5000]
values = ",".join(f"('{uuid.uuid4()}','{k}')" for k in part)
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
"object_key, status, created_at, updated_at) "
f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, "
f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', "
f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)")
s3_bulk("put", keys)
def reconcile(*args: str) -> dict:
out = compose("run", "--rm", "--no-deps", "-T", "reconciler", *args).stdout
return json.loads(out)
def measure(label: str, args: list[str], repeats: int) -> dict:
runs = [reconcile(*args) for _ in range(repeats)]
durations = [r["duration_seconds"] for r in runs]
last = runs[-1]
return {"label": label, "args": args, "runs": repeats, "duration_s_median": statistics.median(durations),
"duration_s_all": durations, "objects": last["scanned"]["objects"], "rows": last["scanned"]["rows"],
"findings": {k: v for k, v in last["findings"].items() if v}, "cost": last["cost"]}
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--steps", type=int, nargs="+", default=[1000, 10000, 50000])
p.add_argument("--repeats", type=int, default=3)
p.add_argument("--output", default="evidence/reconcile")
args = p.parse_args()
out = ROOT / args.output
out.mkdir(parents=True, exist_ok=True)
owner, chat = owner_and_chat()
present = 0
results = []
steps = sorted(args.steps)
variant_step = steps[min(1, len(steps) - 1)]
try:
for target in sorted(args.steps):
started = time.perf_counter()
add(target - present, owner, chat)
present = target
print(f"populated {present} synthetic attachments in {time.perf_counter() - started:.1f} s", flush=True)
results.append({"synthetic": present, **measure("dry-run, page 1000", [], args.repeats)})
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
if present == variant_step: # page-size and checksum variants at one size
for label, extra in (("dry-run, page 100", ["--page-size", "100"]),
("dry-run, --verify-checksums", ["--verify-checksums"])):
results.append({"synthetic": present, **measure(label, extra, args.repeats)})
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
finally:
keys = psql("SELECT object_key FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = "
f"'{chat}'").split()
if keys:
s3_bulk("delete", keys)
psql(f"DELETE FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = '{chat}'")
print(f"removed {len(keys)} synthetic attachments", flush=True)
meta = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
text=True).stdout.strip(),
"object_bytes": len(OBJECT), "reconciler_limits": "cpus 1.0, memory 256m (compose)",
"note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)"}
(out / "cost.json").write_text(json.dumps({"meta": meta, "results": results}, ensure_ascii=False, indent=1),
encoding="utf-8")
return 0
if __name__ == "__main__":
sys.exit(main())