reconcile-inject.sh and reconcile_check.py come from the prepared kit: the first breaks either side on purpose (an object without a row, a removed object, another size, the same size with other bytes, an old intent with and without its object), the second is an oracle outside the API that classifies every row and object from psql and rc, hashes objects with --deep and reports ready attachments lost against a saved set. reconcile_track.py demo uploads through the balancer, links some attachments to messages, injects each class on both sides with old and fresh variants, and checks that a deep dry-run finds exactly what was injected and changes nothing, that apply fixes exactly that and a second apply does nothing, that broken rows and their message links survive in quarantine with metadata 200 and content 404, that a restored object heals, that a second pass is skipped while the lock is held, and that passes in a loop during 75 s of uploads lose nothing. Intents are injected 300 s old, between the reconciler grace and the API sweep grace, so only the reconciler removes them. reconcile_track.py cost snapshots docker stats of the whole host, pins S3, PostgreSQL and the reconciler to their own CPUs, fills the stand to each N with seed_storage.py run in the API image (parallel PutObject and COPY), and records three dry runs per N and mode with a median table. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
592 lines
21 KiB
Python
592 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import http.client
|
|
import json
|
|
import os
|
|
import statistics
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import zlib
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
client = __import__("client")
|
|
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
OUT = ROOT / "evidence" / "reconcile"
|
|
GRACE = 60
|
|
FLAGS = ("--grace", str(GRACE), "--skew", "0")
|
|
INTENT_AGE = 300
|
|
LOCK_KEY = 20260004
|
|
FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale")
|
|
|
|
|
|
def run(cmd, env=None, stdin=None):
|
|
started = time.monotonic()
|
|
done = subprocess.run(
|
|
cmd,
|
|
cwd=ROOT,
|
|
input=stdin,
|
|
capture_output=True,
|
|
check=False,
|
|
env=dict(os.environ, **(env or {})),
|
|
)
|
|
out, err = done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace")
|
|
return done.returncode, out, err, round(time.monotonic() - started, 3)
|
|
|
|
|
|
def must(cmd, env=None, stdin=None):
|
|
code, out, err, _ = run(cmd, env, stdin)
|
|
if code:
|
|
raise SystemExit(f"failed: {' '.join(cmd[:7])}\n{err[-800:]}")
|
|
return out
|
|
|
|
|
|
def compose(*args, env=None, stdin=None):
|
|
return must(["docker", "compose", *args], env, stdin)
|
|
|
|
|
|
def psql(sql):
|
|
return compose(
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
"-qAt",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-c",
|
|
sql,
|
|
).strip()
|
|
|
|
|
|
def reconcile(*flags, env=None):
|
|
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", *flags]
|
|
code, out, err, wall = run(cmd, env)
|
|
if code not in (0, 1) or not out.strip():
|
|
raise SystemExit(f"reconciler failed ({code}):\n{err[-800:]}")
|
|
report = json.loads(out)
|
|
report.update(exit_code=code, wall_seconds=wall)
|
|
return report
|
|
|
|
|
|
def oracle(*flags):
|
|
code, out, err, _ = run([sys.executable, "scripts/reconcile_check.py", *flags])
|
|
if not out.strip():
|
|
raise SystemExit(f"reconcile_check failed ({code}):\n{err[-800:]}")
|
|
return json.loads(out)
|
|
|
|
|
|
def inject(kind, count, ids=(), age=None):
|
|
env = {"INJECT_IDS": ",".join(ids)} if ids else {}
|
|
if age is not None:
|
|
env["INJECT_AGE_SECONDS"] = str(age)
|
|
out = must(["sh", "scripts/reconcile-inject.sh", kind, str(count)], env)
|
|
return [line.split()[1] for line in out.splitlines() if line.strip()]
|
|
|
|
|
|
def head():
|
|
return must(["git", "rev-parse", "--short", "HEAD"]).strip()
|
|
|
|
|
|
def save(name, data):
|
|
path = OUT / name
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
|
return path
|
|
|
|
|
|
class Checks:
|
|
def __init__(self):
|
|
self.items = []
|
|
|
|
def __call__(self, name, ok, **detail):
|
|
self.items.append(dict(name=name, ok=bool(ok), **detail))
|
|
sys.stderr.write(("PASS " if ok else "FAIL ") + name + "\n")
|
|
|
|
@property
|
|
def ok(self):
|
|
return all(item["ok"] for item in self.items)
|
|
|
|
|
|
def chunk(tag, data):
|
|
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data))
|
|
|
|
|
|
def tiny_png():
|
|
raw = b"".join(b"\x00" + os.urandom(24) for _ in range(8))
|
|
header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
|
|
return (
|
|
b"\x89PNG\r\n\x1a\n"
|
|
+ chunk(b"IHDR", header)
|
|
+ chunk(b"IDAT", zlib.compress(raw))
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
|
|
|
|
def pick(found, names):
|
|
return {name: found.get(name, 0) for name in names}
|
|
|
|
|
|
def faults(c, check, pre):
|
|
alice, bob, _, chat = c.bootstrap()
|
|
ids = [c.attachment(alice, chat)[0]["id"] for _ in range(18)]
|
|
linked = ids[:6]
|
|
for part in (linked[:3], linked[3:]):
|
|
c.message(alice, chat, "", attachments=part)
|
|
targets = {
|
|
"missing": [linked[0], linked[1], ids[6]],
|
|
"corrupt": [linked[2], ids[7], ids[8]],
|
|
"bitflip": [linked[3], ids[9], ids[10]],
|
|
}
|
|
damaged = {i for group in targets.values() for i in group}
|
|
ready = OUT / "ready-before.json"
|
|
oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready))
|
|
untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged}
|
|
save("ready-untouched.json", untouched)
|
|
started = time.monotonic()
|
|
injected = {
|
|
"orphan_stale": inject("orphan", 3),
|
|
"missing": inject("missing", 3, targets["missing"]),
|
|
"size_mismatch": inject("corrupt", 3, targets["corrupt"]),
|
|
"sha_mismatch": inject("bitflip", 3, targets["bitflip"]),
|
|
"pending_stale": inject("stale-pending", 3, age=INTENT_AGE)
|
|
+ inject("pending-object", 3, age=INTENT_AGE),
|
|
}
|
|
time.sleep(max(0.0, GRACE + 5 - (time.monotonic() - started)))
|
|
injected["orphan_fresh"] = inject("orphan", 2)
|
|
injected["pending_fresh"] = inject("stale-pending", 2, age=0)
|
|
expected = {name: len(keys) for name, keys in injected.items()}
|
|
expected.update(quarantined=pre["counts"]["quarantined"])
|
|
before = oracle("--grace", str(GRACE), "--deep", "--output", str(OUT / "check-before.json"))
|
|
dry = reconcile("--dry-run", "--deep", *FLAGS)
|
|
save("dry-run.json", dry)
|
|
after_dry = oracle("--grace", str(GRACE), "--deep")
|
|
check(
|
|
"dry-run находит каждый внедрённый класс",
|
|
pick(dry["found"], expected) == expected,
|
|
expected=expected,
|
|
found=dry["found"],
|
|
)
|
|
check(
|
|
"dry-run ничего не меняет",
|
|
not any(dry["fixed"].values())
|
|
and after_dry["counts"] == before["counts"]
|
|
and (after_dry["objects"], after_dry["rows"]) == (before["objects"], before["rows"]),
|
|
objects=[before["objects"], after_dry["objects"]],
|
|
rows=[before["rows"], after_dry["rows"]],
|
|
)
|
|
first = reconcile("--apply", "--deep", *FLAGS)
|
|
save("apply-1.json", first)
|
|
check(
|
|
"apply исправляет ровно найденное dry-run",
|
|
pick(first["fixed"], FIXABLE) == pick(dry["found"], FIXABLE) and not first["errors"],
|
|
dry_run=pick(dry["found"], FIXABLE),
|
|
fixed=pick(first["fixed"], FIXABLE),
|
|
)
|
|
second = reconcile("--apply", "--deep", *FLAGS)
|
|
save("apply-2.json", second)
|
|
check(
|
|
"повторный apply — 0 действий",
|
|
not any(second["fixed"].values()),
|
|
fixed=second["fixed"],
|
|
found=second["found"],
|
|
)
|
|
after = oracle(
|
|
"--grace",
|
|
str(GRACE),
|
|
"--deep",
|
|
"--expect-ready",
|
|
str(OUT / "ready-untouched.json"),
|
|
"--output",
|
|
str(OUT / "check-after.json"),
|
|
)
|
|
tolerated = ("orphan_fresh", "pending_fresh")
|
|
check(
|
|
"независимая проверка после apply: нарушений нет, готовые не потеряны",
|
|
after["ok"]
|
|
and not after["counts"]["lost_ready"]
|
|
and pick(after["counts"], tolerated) == pick(before["counts"], tolerated)
|
|
and after["counts"]["quarantined"] == pre["counts"]["quarantined"] + 9,
|
|
counts=after["counts"],
|
|
)
|
|
marks = dict(
|
|
line.split("|") for line in psql("SELECT id, quarantine_reason FROM attachments").split()
|
|
)
|
|
links = psql("SELECT attachment_id FROM message_attachments").split()
|
|
kept = {i: marks.get(i) for i in sorted(damaged)}
|
|
check(
|
|
"строки с расхождениями и ссылки сообщений сохранены",
|
|
all(kept.values()) and sum(i in linked for i in links) == len(linked),
|
|
quarantine=kept,
|
|
message_links=sum(i in linked for i in links),
|
|
)
|
|
meta = c.call("GET", f"/api/v1/attachments/{linked[0]}", 200, actor=bob, schema="Attachment")
|
|
blocked = c.call("GET", f"/api/v1/attachments/{linked[2]}/content", 404, actor=bob)
|
|
intact = c.call("GET", f"/api/v1/attachments/{linked[4]}/content", 200, actor=bob)
|
|
check(
|
|
"карантин: метаданные 200, /content 404; нетронутое скачивается",
|
|
meta["status"] == 200 and blocked["status"] == 404 and intact["bytes"] == FIXTURE,
|
|
)
|
|
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
|
compose(
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"-T",
|
|
"s3-cli",
|
|
"pipe",
|
|
f"root/{bucket}/attachments/{linked[0]}",
|
|
stdin=FIXTURE,
|
|
)
|
|
healed = reconcile("--apply", *FLAGS)
|
|
save("healed.json", healed)
|
|
restored = c.call("GET", f"/api/v1/attachments/{linked[0]}/content", 200, actor=bob)
|
|
check(
|
|
"восстановленный объект выходит из карантина",
|
|
healed["fixed"]["healed"] == 1 and restored["bytes"] == FIXTURE,
|
|
fixed=healed["fixed"],
|
|
)
|
|
return {"injected": injected, "targets": targets, "linked": linked}
|
|
|
|
|
|
def single(check):
|
|
holder = subprocess.Popen(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-c",
|
|
f"SELECT pg_advisory_lock({LOCK_KEY}), pg_sleep(15)",
|
|
],
|
|
cwd=ROOT,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
try:
|
|
time.sleep(3)
|
|
busy = reconcile("--apply", *FLAGS)
|
|
finally:
|
|
holder.wait()
|
|
save("lock-busy.json", busy)
|
|
check(
|
|
"второй одновременный проход пропущен",
|
|
busy["skipped"] and busy["exit_code"] == 0 and not any(busy["fixed"].values()),
|
|
)
|
|
|
|
|
|
def uploader(base, token, chat, *, deadline, results, lock):
|
|
host = urlsplit(base).netloc
|
|
conn = http.client.HTTPConnection(host, timeout=30)
|
|
headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
|
|
sent = 0
|
|
while time.monotonic() < deadline:
|
|
path = f"/api/v1/chats/{chat}/attachments?filename=load.png"
|
|
try:
|
|
conn.request("POST", path, tiny_png(), {**headers, "Content-Type": "image/png"})
|
|
response = conn.getresponse()
|
|
status, data = response.status, response.read()
|
|
if status == 201 and sent % 10 == 0:
|
|
body = json.dumps({"text": "", "attachment_ids": [json.loads(data)["id"]]})
|
|
extra = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}
|
|
conn.request("POST", f"/api/v1/chats/{chat}/messages", body, {**headers, **extra})
|
|
linked = conn.getresponse()
|
|
linked.read()
|
|
status = 201 if linked.status == 201 else -linked.status
|
|
except (OSError, http.client.HTTPException):
|
|
conn.close()
|
|
conn = http.client.HTTPConnection(host, timeout=30)
|
|
status, data = 0, b""
|
|
with lock:
|
|
results.append((status, json.loads(data) if status == 201 else None))
|
|
sent += 1
|
|
time.sleep(0.05)
|
|
|
|
|
|
def under_load(c, check, base, seconds, threads):
|
|
alice, _, _, chat = c.bootstrap()
|
|
ready = OUT / "ready-load.json"
|
|
left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"]
|
|
leftover = {
|
|
"orphan_stale": left["orphan_fresh"] + left["orphan_stale"],
|
|
"pending_stale": left["pending_fresh"]
|
|
+ left["pending_stale_with_object"]
|
|
+ left["pending_stale_without_object"],
|
|
}
|
|
orphans = inject("orphan", 3)
|
|
intents = inject("stale-pending", 3, age=INTENT_AGE)
|
|
results, lock = [], threading.Lock()
|
|
deadline = time.monotonic() + seconds
|
|
workers = [
|
|
threading.Thread(
|
|
target=uploader,
|
|
args=(base, alice["token"], chat["id"]),
|
|
kwargs={"deadline": deadline, "results": results, "lock": lock},
|
|
)
|
|
for _ in range(threads)
|
|
]
|
|
for worker in workers:
|
|
worker.start()
|
|
runs = []
|
|
while time.monotonic() < deadline:
|
|
runs.append(reconcile("--apply", *FLAGS))
|
|
for worker in workers:
|
|
worker.join()
|
|
runs.append(reconcile("--apply", *FLAGS))
|
|
created = {att["id"]: att["sha256"] for status, att in results if att}
|
|
expected = dict(json.loads(ready.read_text()), **created)
|
|
save("ready-load.json", expected)
|
|
after = oracle(
|
|
"--grace",
|
|
str(GRACE),
|
|
"--deep",
|
|
"--expect-ready",
|
|
str(ready),
|
|
"--output",
|
|
str(OUT / "check-under-load.json"),
|
|
)
|
|
statuses = {}
|
|
for status, _ in results:
|
|
statuses[str(status)] = statuses.get(str(status), 0) + 1
|
|
fixed = {name: sum(r["fixed"][name] for r in runs) for name in FIXABLE}
|
|
seen = {name: sum(r["found"][name] for r in runs) for name in ("orphan_fresh", "pending_fresh")}
|
|
summary = {
|
|
"seconds": seconds,
|
|
"upload_threads": threads,
|
|
"uploads": statuses,
|
|
"created": len(created),
|
|
"linked_every": 10,
|
|
"reconciler_runs": len(runs),
|
|
"run_seconds": [r["cost"]["seconds"] for r in runs],
|
|
"fixed_total": fixed,
|
|
"fresh_seen_total": seen,
|
|
"errors": sum(r["errors"] for r in runs),
|
|
"skipped": sum(1 for r in runs if r["skipped"]),
|
|
"injected_before_load": {"orphan": len(orphans), "pending_stale": len(intents)},
|
|
"left_from_faults": leftover,
|
|
"check_after": {k: after[k] for k in ("objects", "rows", "ready_healthy", "counts", "ok")},
|
|
}
|
|
save("under-load.json", summary)
|
|
check(
|
|
"под нагрузкой: исправлены только внедрённые, ошибок нет",
|
|
fixed
|
|
== {
|
|
**dict.fromkeys(FIXABLE, 0),
|
|
"orphan_stale": leftover["orphan_stale"] + len(orphans),
|
|
"pending_stale": leftover["pending_stale"] + len(intents),
|
|
}
|
|
and not summary["errors"]
|
|
and not summary["skipped"],
|
|
fixed=fixed,
|
|
runs=len(runs),
|
|
)
|
|
check(
|
|
"под нагрузкой: все 201 готовы и совпадают по SHA-256, потерь 0",
|
|
after["ok"] and not after["counts"]["lost_ready"] and set(statuses) == {"201"},
|
|
uploads=statuses,
|
|
lost_ready=after["counts"]["lost_ready"],
|
|
)
|
|
|
|
|
|
def demo(args):
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
c = client.Client(args.base_url, timeout=30)
|
|
check = Checks()
|
|
pre = oracle("--grace", str(GRACE))
|
|
dirty = {k: v for k, v in pre["counts"].items() if v and k != "quarantined"}
|
|
if dirty:
|
|
raise SystemExit(f"стенд не в исходном состоянии {dirty}: make reconcile ARGS=--apply")
|
|
started = time.monotonic()
|
|
result = {"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head()}
|
|
result["faults"] = faults(c, check, pre)
|
|
single(check)
|
|
under_load(c, check, args.base_url, args.load_seconds, args.load_threads)
|
|
result.update(seconds=round(time.monotonic() - started, 1), checks=check.items, ok=check.ok)
|
|
save("demo.json", result)
|
|
print(json.dumps({"ok": check.ok, "checks": len(check.items)}, ensure_ascii=False))
|
|
sys.exit(0 if check.ok else 1)
|
|
|
|
|
|
def pins(args):
|
|
cpus = int(must(["docker", "info", "--format", "{{.NCPU}}"]).strip())
|
|
if args.no_pin or cpus < 8:
|
|
return {"S3_MEMORY": args.s3_memory}
|
|
return {
|
|
"S3_CPUSET": str(cpus - 4),
|
|
"PG_CPUSET": f"{cpus - 3}-{cpus - 2}",
|
|
"RECONCILE_CPUSET": str(cpus - 1),
|
|
"S3_MEMORY": args.s3_memory,
|
|
}
|
|
|
|
|
|
def limits():
|
|
ids = compose("ps", "-q", "s3", "postgres").split()
|
|
fmt = (
|
|
"{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}"
|
|
" cpuset={{.HostConfig.CpusetCpus}}"
|
|
)
|
|
return must(["docker", "inspect", "-f", fmt, *ids]).strip().splitlines()
|
|
|
|
|
|
def seed(count, env):
|
|
if count <= 0:
|
|
return {"added": 0}
|
|
env = dict(env, RECONCILE_CPUSET="", RECONCILE_CPUS="4", RECONCILE_MEMORY="512m")
|
|
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
|
|
out = compose(
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"-T",
|
|
"-v",
|
|
tools,
|
|
"--entrypoint",
|
|
"python",
|
|
"reconciler",
|
|
"/tools/seed_storage.py",
|
|
"--count",
|
|
str(count),
|
|
env=env,
|
|
)
|
|
return json.loads(out.strip().splitlines()[-1])
|
|
|
|
|
|
def median(runs, path):
|
|
values = []
|
|
for r in runs:
|
|
value = r
|
|
for key in path:
|
|
value = value[key]
|
|
values.append(value)
|
|
return statistics.median(values)
|
|
|
|
|
|
def summarize(results):
|
|
rows = []
|
|
for (n, mode), runs in sorted(results.items()):
|
|
seconds = median(runs, ("cost", "seconds"))
|
|
rows.append(
|
|
{
|
|
"objects": n,
|
|
"mode": mode,
|
|
"runs": len(runs),
|
|
"seconds_median": seconds,
|
|
"seconds_all": [r["cost"]["seconds"] for r in runs],
|
|
"wall_seconds_median": median(runs, ("wall_seconds",)),
|
|
"us_per_object": round(seconds / n * 1e6, 1),
|
|
"s3_list_wait_median": median(runs, ("cost", "waited_seconds", "s3_list")),
|
|
"sql_wait_median": median(runs, ("cost", "waited_seconds", "sql")),
|
|
"requests": runs[0]["cost"]["requests"],
|
|
"max_rss_mib_median": median(runs, ("cost", "max_rss_mib")),
|
|
"scanned": runs[0]["scanned"],
|
|
"found_nonzero": {k: v for k, v in runs[0]["found"].items() if v},
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def table(rows):
|
|
head = (
|
|
"objects mode runs t_med,s wall,s us/obj list,s sql,s lists sqls gets"
|
|
" rss,MiB"
|
|
)
|
|
lines = [head]
|
|
for r in rows:
|
|
q = r["requests"]
|
|
lines.append(
|
|
f"{r['objects']:>7} {r['mode']:<9} {r['runs']:>4} {r['seconds_median']:>7.3f} "
|
|
f"{r['wall_seconds_median']:>6.2f} {r['us_per_object']:>6.1f} "
|
|
f"{r['s3_list_wait_median']:>6.3f} {r['sql_wait_median']:>5.3f} {q['s3_list']:>5} "
|
|
f"{q['sql']:>4} {q['s3_get']:>4} {r['max_rss_mib_median']:>7.1f}"
|
|
)
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def cost(args):
|
|
out = OUT / "cost"
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
stats = must(
|
|
[
|
|
"docker",
|
|
"stats",
|
|
"--no-stream",
|
|
"--all",
|
|
"--format",
|
|
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}",
|
|
]
|
|
)
|
|
snapshot = f"date: {stamp}\n$ docker stats --no-stream --all\n{stats}"
|
|
(out / "host-docker-stats.txt").write_text(snapshot)
|
|
rows = int(psql("SELECT count(*) FROM attachments"))
|
|
if rows > args.sizes[0]:
|
|
raise SystemExit(f"на стенде уже {rows} вложений: make reset-data && make up && make test")
|
|
env = pins(args)
|
|
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres", env=env)
|
|
plan = {"date": stamp, "commit": head(), "pins": env, "limits": limits(), "seeds": []}
|
|
results = {}
|
|
try:
|
|
for n in args.sizes:
|
|
plan["seeds"].append(dict(seed(n - rows, env), target=n))
|
|
rows = n
|
|
time.sleep(args.settle)
|
|
modes = [("shallow", ["--dry-run"])]
|
|
if n <= args.deep_max:
|
|
modes.append(("deep", ["--dry-run", "--deep"]))
|
|
if n == args.page_compare:
|
|
modes.append(("page100", ["--dry-run", "--page-size", "100"]))
|
|
for mode, flags in modes:
|
|
for attempt in range(1, args.repeats + 1):
|
|
report = reconcile(*flags, "--samples", "3", env=env)
|
|
(out / f"n{n}-{mode}-{attempt}.json").write_text(json.dumps(report, indent=2))
|
|
results.setdefault((n, mode), []).append(report)
|
|
sys.stderr.write(f"{n} {mode} #{attempt}: {report['cost']['seconds']} s\n")
|
|
finally:
|
|
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres")
|
|
summary = summarize(results)
|
|
save("cost/summary.json", {**plan, "results": summary})
|
|
(out / "summary.txt").write_text(f"date: {stamp} pins: {env}\n" + table(summary))
|
|
sys.stdout.write(table(summary))
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description="Lab4 track 2: reconciler demo and cost of a pass")
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
d = sub.add_parser("demo", help="fault injection, dry-run/apply/repeat, safety, under load")
|
|
d.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
|
|
d.add_argument("--load-seconds", type=int, default=75)
|
|
d.add_argument("--load-threads", type=int, default=3)
|
|
k = sub.add_parser("cost", help="cost of a pass for N objects, repeated")
|
|
k.add_argument("--sizes", type=int, nargs="+", default=[1000, 10000, 100000])
|
|
k.add_argument("--repeats", type=int, default=3)
|
|
k.add_argument("--deep-max", type=int, default=100000)
|
|
k.add_argument("--page-compare", type=int, default=10000)
|
|
k.add_argument("--settle", type=float, default=10)
|
|
k.add_argument("--s3-memory", default="1g")
|
|
k.add_argument("--no-pin", action="store_true")
|
|
args = p.parse_args()
|
|
if args.command == "demo":
|
|
demo(args)
|
|
else:
|
|
cost(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|