forked from mtusi2026/mtusi2026lab4
- python -m messenger.reconcile: merge-join листинга S3 и attachments по object_key; dry-run read-only конструктивно (default_transaction_read_only + S3 без записи), --apply: orphan/stale intent удаляются, ready-строки только в карантин (обратимо), перепроверка под FOR UPDATE, предохранитель от массового удаления; - attachment_ids хранится в строке сообщения (история остаётся одним index range scan), message_attachments — для FK и обратного поиска; - семафор полной проверки изображений (IMAGE_VERIFY_CONCURRENCY); - тесты: вложения (56), отказ/пауза S3 и деградированный режим LB, failover загрузки, приватность bucket, FAULT_POINTS после PUT/после COMMIT, janitor, reconciler; - бенчмарки памяти загрузок и стоимости обхода reconciler. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
156 lines
7.1 KiB
Python
156 lines
7.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Track 2 experiment: cost of one reconciler pass as a function of the number of attachments.
|
|
|
|
Hypothesis: a pass is O(N) with ~N/1000 S3 list requests + N/1000 DB pages and constant memory
|
|
(merge-join of two key-ordered streams); --verify-checksums adds reading every byte (O(total size)).
|
|
|
|
Method: synthetic, CONSISTENT attachments (row 'ready' + 1 KiB object with the matching size and
|
|
SHA-256, filename 'bench-reconcile') are added in steps up to the largest N; after each step the
|
|
reconciler runs in dry-run 3 times (median reported), plus page-size and checksum variants.
|
|
Everything synthetic is deleted at the end (objects first, then rows).
|
|
|
|
python bench/reconcile_cost.py --steps 1000 10000 50000 --output evidence/reconcile
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import httpx2 as httpx
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
|
PASSWORD = "Bench-password-Ω-12345"
|
|
OBJECT = bytes(range(256)) * 4 # 1 KiB, the same content for every object
|
|
SHA = hashlib.sha256(OBJECT).hexdigest()
|
|
|
|
PUT_SCRIPT = """
|
|
import asyncio, sys
|
|
from messenger.config import Settings, s3_settings
|
|
from messenger.storage import Storage
|
|
DATA = bytes(range(256)) * 4
|
|
async def main():
|
|
s = Storage(Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings()))
|
|
await s.open()
|
|
keys = sys.stdin.read().split()
|
|
sem = asyncio.Semaphore(32)
|
|
async def one(k):
|
|
async with sem:
|
|
await s.{op}(k{args})
|
|
await asyncio.gather(*(one(k) for k in keys))
|
|
await s.close()
|
|
print(len(keys))
|
|
asyncio.run(main())
|
|
"""
|
|
|
|
|
|
def compose(*args: str, stdin: str | None = None, timeout: float = 1800, check: bool = True) -> subprocess.CompletedProcess:
|
|
return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True,
|
|
encoding="utf-8", timeout=timeout, check=check)
|
|
|
|
|
|
def psql(sql: str) -> str:
|
|
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v",
|
|
"ON_ERROR_STOP=1", "-c", sql).stdout.strip()
|
|
|
|
|
|
def s3_bulk(op: str, keys: list[str]) -> None:
|
|
script = PUT_SCRIPT.replace("{op}", op).replace("{args}", ", DATA, 'image/png', {}" if op == "put" else "")
|
|
compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "python", "reconciler", "-c", script, stdin="\n".join(keys))
|
|
|
|
|
|
def owner_and_chat() -> tuple[str, str]:
|
|
import base64
|
|
c = httpx.Client(base_url=BASE, timeout=30)
|
|
name = "rc_" + uuid.uuid4().hex[:10]
|
|
c.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
|
|
token = c.post("/api/v1/auth/sessions", headers={
|
|
"Authorization": "Basic " + base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()}).json()["token"]
|
|
me = c.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
|
|
chat = c.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
|
|
json={"title": "bench", "member_ids": [me["id"]]}).json()
|
|
return me["id"], chat["id"]
|
|
|
|
|
|
def add(n: int, owner: str, chat: str) -> None:
|
|
keys = ["o/" + uuid.uuid4().hex for _ in range(n)]
|
|
for i in range(0, n, 5000):
|
|
part = keys[i:i + 5000]
|
|
values = ",".join(f"('{uuid.uuid4()}','{k}')" for k in part)
|
|
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
|
|
"object_key, status, created_at, updated_at) "
|
|
f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, "
|
|
f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', "
|
|
f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)")
|
|
s3_bulk("put", keys)
|
|
|
|
|
|
def reconcile(*args: str) -> dict:
|
|
out = compose("run", "--rm", "--no-deps", "-T", "reconciler", *args).stdout
|
|
return json.loads(out)
|
|
|
|
|
|
def measure(label: str, args: list[str], repeats: int) -> dict:
|
|
runs = [reconcile(*args) for _ in range(repeats)]
|
|
durations = [r["duration_seconds"] for r in runs]
|
|
last = runs[-1]
|
|
return {"label": label, "args": args, "runs": repeats, "duration_s_median": statistics.median(durations),
|
|
"duration_s_all": durations, "objects": last["scanned"]["objects"], "rows": last["scanned"]["rows"],
|
|
"findings": {k: v for k, v in last["findings"].items() if v}, "cost": last["cost"]}
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--steps", type=int, nargs="+", default=[1000, 10000, 50000])
|
|
p.add_argument("--repeats", type=int, default=3)
|
|
p.add_argument("--output", default="evidence/reconcile")
|
|
args = p.parse_args()
|
|
out = ROOT / args.output
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
owner, chat = owner_and_chat()
|
|
present = 0
|
|
results = []
|
|
steps = sorted(args.steps)
|
|
variant_step = steps[min(1, len(steps) - 1)]
|
|
try:
|
|
for target in sorted(args.steps):
|
|
started = time.perf_counter()
|
|
add(target - present, owner, chat)
|
|
present = target
|
|
print(f"populated {present} synthetic attachments in {time.perf_counter() - started:.1f} s", flush=True)
|
|
results.append({"synthetic": present, **measure("dry-run, page 1000", [], args.repeats)})
|
|
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
|
|
if present == variant_step: # page-size and checksum variants at one size
|
|
for label, extra in (("dry-run, page 100", ["--page-size", "100"]),
|
|
("dry-run, --verify-checksums", ["--verify-checksums"])):
|
|
results.append({"synthetic": present, **measure(label, extra, args.repeats)})
|
|
print(json.dumps(results[-1], ensure_ascii=False), flush=True)
|
|
finally:
|
|
keys = psql("SELECT object_key FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = "
|
|
f"'{chat}'").split()
|
|
if keys:
|
|
s3_bulk("delete", keys)
|
|
psql(f"DELETE FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = '{chat}'")
|
|
print(f"removed {len(keys)} synthetic attachments", flush=True)
|
|
meta = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
|
|
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
|
|
text=True).stdout.strip(),
|
|
"object_bytes": len(OBJECT), "reconciler_limits": "cpus 1.0, memory 256m (compose)",
|
|
"note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)"}
|
|
(out / "cost.json").write_text(json.dumps({"meta": meta, "results": results}, ensure_ascii=False, indent=1),
|
|
encoding="utf-8")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|