forked from mtusi2026/mtusi2026lab4
The first cost run stalled while filling the bucket to 100k objects: RustFS keeps object metadata in memory, at about 45k objects it hit its 1 GiB limit, the cgroup kept reclaiming and PutObject slowed from about 240 to 11 objects per second. The S3 CPU limit is now taken from S3_CPUS (default 1, as before), and the cost run gives RustFS 2 pinned CPUs and 3 GiB for every point, so all N are measured with the same limits; seeding uses 16 parallel writes. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
595 lines
21 KiB
Python
595 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import http.client
|
|
import json
|
|
import os
|
|
import statistics
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import zlib
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
client = __import__("client")
|
|
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
OUT = ROOT / "evidence" / "reconcile"
|
|
GRACE = 60
|
|
FLAGS = ("--grace", str(GRACE), "--skew", "0")
|
|
INTENT_AGE = 300
|
|
LOCK_KEY = 20260004
|
|
FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale")
|
|
|
|
|
|
def run(cmd, env=None, stdin=None):
|
|
started = time.monotonic()
|
|
done = subprocess.run(
|
|
cmd,
|
|
cwd=ROOT,
|
|
input=stdin,
|
|
capture_output=True,
|
|
check=False,
|
|
env=dict(os.environ, **(env or {})),
|
|
)
|
|
out, err = done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace")
|
|
return done.returncode, out, err, round(time.monotonic() - started, 3)
|
|
|
|
|
|
def must(cmd, env=None, stdin=None):
|
|
code, out, err, _ = run(cmd, env, stdin)
|
|
if code:
|
|
raise SystemExit(f"failed: {' '.join(cmd[:7])}\n{err[-800:]}")
|
|
return out
|
|
|
|
|
|
def compose(*args, env=None, stdin=None):
|
|
return must(["docker", "compose", *args], env, stdin)
|
|
|
|
|
|
def psql(sql):
|
|
return compose(
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
"-qAt",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-c",
|
|
sql,
|
|
).strip()
|
|
|
|
|
|
def reconcile(*flags, env=None):
|
|
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", *flags]
|
|
code, out, err, wall = run(cmd, env)
|
|
if code not in (0, 1) or not out.strip():
|
|
raise SystemExit(f"reconciler failed ({code}):\n{err[-800:]}")
|
|
report = json.loads(out)
|
|
report.update(exit_code=code, wall_seconds=wall)
|
|
return report
|
|
|
|
|
|
def oracle(*flags):
|
|
code, out, err, _ = run([sys.executable, "scripts/reconcile_check.py", *flags])
|
|
if not out.strip():
|
|
raise SystemExit(f"reconcile_check failed ({code}):\n{err[-800:]}")
|
|
return json.loads(out)
|
|
|
|
|
|
def inject(kind, count, ids=(), age=None):
|
|
env = {"INJECT_IDS": ",".join(ids)} if ids else {}
|
|
if age is not None:
|
|
env["INJECT_AGE_SECONDS"] = str(age)
|
|
out = must(["sh", "scripts/reconcile-inject.sh", kind, str(count)], env)
|
|
return [line.split()[1] for line in out.splitlines() if line.strip()]
|
|
|
|
|
|
def head():
|
|
return must(["git", "rev-parse", "--short", "HEAD"]).strip()
|
|
|
|
|
|
def save(name, data):
|
|
path = OUT / name
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
|
return path
|
|
|
|
|
|
class Checks:
|
|
def __init__(self):
|
|
self.items = []
|
|
|
|
def __call__(self, name, ok, **detail):
|
|
self.items.append(dict(name=name, ok=bool(ok), **detail))
|
|
sys.stderr.write(("PASS " if ok else "FAIL ") + name + "\n")
|
|
|
|
@property
|
|
def ok(self):
|
|
return all(item["ok"] for item in self.items)
|
|
|
|
|
|
def chunk(tag, data):
|
|
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data))
|
|
|
|
|
|
def tiny_png():
|
|
raw = b"".join(b"\x00" + os.urandom(24) for _ in range(8))
|
|
header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
|
|
return (
|
|
b"\x89PNG\r\n\x1a\n"
|
|
+ chunk(b"IHDR", header)
|
|
+ chunk(b"IDAT", zlib.compress(raw))
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
|
|
|
|
def pick(found, names):
|
|
return {name: found.get(name, 0) for name in names}
|
|
|
|
|
|
def faults(c, check, pre):
|
|
alice, bob, _, chat = c.bootstrap()
|
|
ids = [c.attachment(alice, chat)[0]["id"] for _ in range(18)]
|
|
linked = ids[:6]
|
|
for part in (linked[:3], linked[3:]):
|
|
c.message(alice, chat, "", attachments=part)
|
|
targets = {
|
|
"missing": [linked[0], linked[1], ids[6]],
|
|
"corrupt": [linked[2], ids[7], ids[8]],
|
|
"bitflip": [linked[3], ids[9], ids[10]],
|
|
}
|
|
damaged = {i for group in targets.values() for i in group}
|
|
ready = OUT / "ready-before.json"
|
|
oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready))
|
|
untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged}
|
|
save("ready-untouched.json", untouched)
|
|
started = time.monotonic()
|
|
injected = {
|
|
"orphan_stale": inject("orphan", 3),
|
|
"missing": inject("missing", 3, targets["missing"]),
|
|
"size_mismatch": inject("corrupt", 3, targets["corrupt"]),
|
|
"sha_mismatch": inject("bitflip", 3, targets["bitflip"]),
|
|
"pending_stale": inject("stale-pending", 3, age=INTENT_AGE)
|
|
+ inject("pending-object", 3, age=INTENT_AGE),
|
|
}
|
|
time.sleep(max(0.0, GRACE + 5 - (time.monotonic() - started)))
|
|
injected["orphan_fresh"] = inject("orphan", 2)
|
|
injected["pending_fresh"] = inject("stale-pending", 2, age=0)
|
|
expected = {name: len(keys) for name, keys in injected.items()}
|
|
expected.update(quarantined=pre["counts"]["quarantined"])
|
|
before = oracle("--grace", str(GRACE), "--deep", "--output", str(OUT / "check-before.json"))
|
|
dry = reconcile("--dry-run", "--deep", *FLAGS)
|
|
save("dry-run.json", dry)
|
|
after_dry = oracle("--grace", str(GRACE), "--deep")
|
|
check(
|
|
"dry-run находит каждый внедрённый класс",
|
|
pick(dry["found"], expected) == expected,
|
|
expected=expected,
|
|
found=dry["found"],
|
|
)
|
|
check(
|
|
"dry-run ничего не меняет",
|
|
not any(dry["fixed"].values())
|
|
and after_dry["counts"] == before["counts"]
|
|
and (after_dry["objects"], after_dry["rows"]) == (before["objects"], before["rows"]),
|
|
objects=[before["objects"], after_dry["objects"]],
|
|
rows=[before["rows"], after_dry["rows"]],
|
|
)
|
|
first = reconcile("--apply", "--deep", *FLAGS)
|
|
save("apply-1.json", first)
|
|
check(
|
|
"apply исправляет ровно найденное dry-run",
|
|
pick(first["fixed"], FIXABLE) == pick(dry["found"], FIXABLE) and not first["errors"],
|
|
dry_run=pick(dry["found"], FIXABLE),
|
|
fixed=pick(first["fixed"], FIXABLE),
|
|
)
|
|
second = reconcile("--apply", "--deep", *FLAGS)
|
|
save("apply-2.json", second)
|
|
check(
|
|
"повторный apply — 0 действий",
|
|
not any(second["fixed"].values()),
|
|
fixed=second["fixed"],
|
|
found=second["found"],
|
|
)
|
|
after = oracle(
|
|
"--grace",
|
|
str(GRACE),
|
|
"--deep",
|
|
"--expect-ready",
|
|
str(OUT / "ready-untouched.json"),
|
|
"--output",
|
|
str(OUT / "check-after.json"),
|
|
)
|
|
tolerated = ("orphan_fresh", "pending_fresh")
|
|
check(
|
|
"независимая проверка после apply: нарушений нет, готовые не потеряны",
|
|
after["ok"]
|
|
and not after["counts"]["lost_ready"]
|
|
and pick(after["counts"], tolerated) == pick(before["counts"], tolerated)
|
|
and after["counts"]["quarantined"] == pre["counts"]["quarantined"] + 9,
|
|
counts=after["counts"],
|
|
)
|
|
marks = dict(
|
|
line.split("|") for line in psql("SELECT id, quarantine_reason FROM attachments").split()
|
|
)
|
|
links = psql("SELECT attachment_id FROM message_attachments").split()
|
|
kept = {i: marks.get(i) for i in sorted(damaged)}
|
|
check(
|
|
"строки с расхождениями и ссылки сообщений сохранены",
|
|
all(kept.values()) and sum(i in linked for i in links) == len(linked),
|
|
quarantine=kept,
|
|
message_links=sum(i in linked for i in links),
|
|
)
|
|
meta = c.call("GET", f"/api/v1/attachments/{linked[0]}", 200, actor=bob, schema="Attachment")
|
|
blocked = c.call("GET", f"/api/v1/attachments/{linked[2]}/content", 404, actor=bob)
|
|
intact = c.call("GET", f"/api/v1/attachments/{linked[4]}/content", 200, actor=bob)
|
|
check(
|
|
"карантин: метаданные 200, /content 404; нетронутое скачивается",
|
|
meta["status"] == 200 and blocked["status"] == 404 and intact["bytes"] == FIXTURE,
|
|
)
|
|
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
|
compose(
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"-T",
|
|
"s3-cli",
|
|
"pipe",
|
|
f"root/{bucket}/attachments/{linked[0]}",
|
|
stdin=FIXTURE,
|
|
)
|
|
healed = reconcile("--apply", *FLAGS)
|
|
save("healed.json", healed)
|
|
restored = c.call("GET", f"/api/v1/attachments/{linked[0]}/content", 200, actor=bob)
|
|
check(
|
|
"восстановленный объект выходит из карантина",
|
|
healed["fixed"]["healed"] == 1 and restored["bytes"] == FIXTURE,
|
|
fixed=healed["fixed"],
|
|
)
|
|
return {"injected": injected, "targets": targets, "linked": linked}
|
|
|
|
|
|
def single(check):
|
|
holder = subprocess.Popen(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-c",
|
|
f"SELECT pg_advisory_lock({LOCK_KEY}), pg_sleep(15)",
|
|
],
|
|
cwd=ROOT,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
try:
|
|
time.sleep(3)
|
|
busy = reconcile("--apply", *FLAGS)
|
|
finally:
|
|
holder.wait()
|
|
save("lock-busy.json", busy)
|
|
check(
|
|
"второй одновременный проход пропущен",
|
|
busy["skipped"] and busy["exit_code"] == 0 and not any(busy["fixed"].values()),
|
|
)
|
|
|
|
|
|
def uploader(base, token, chat, *, deadline, results, lock):
|
|
host = urlsplit(base).netloc
|
|
conn = http.client.HTTPConnection(host, timeout=30)
|
|
headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
|
|
sent = 0
|
|
while time.monotonic() < deadline:
|
|
path = f"/api/v1/chats/{chat}/attachments?filename=load.png"
|
|
try:
|
|
conn.request("POST", path, tiny_png(), {**headers, "Content-Type": "image/png"})
|
|
response = conn.getresponse()
|
|
status, data = response.status, response.read()
|
|
if status == 201 and sent % 10 == 0:
|
|
body = json.dumps({"text": "", "attachment_ids": [json.loads(data)["id"]]})
|
|
extra = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}
|
|
conn.request("POST", f"/api/v1/chats/{chat}/messages", body, {**headers, **extra})
|
|
linked = conn.getresponse()
|
|
linked.read()
|
|
status = 201 if linked.status == 201 else -linked.status
|
|
except (OSError, http.client.HTTPException):
|
|
conn.close()
|
|
conn = http.client.HTTPConnection(host, timeout=30)
|
|
status, data = 0, b""
|
|
with lock:
|
|
results.append((status, json.loads(data) if status == 201 else None))
|
|
sent += 1
|
|
time.sleep(0.05)
|
|
|
|
|
|
def under_load(c, check, base, seconds, threads):
|
|
alice, _, _, chat = c.bootstrap()
|
|
ready = OUT / "ready-load.json"
|
|
left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"]
|
|
leftover = {
|
|
"orphan_stale": left["orphan_fresh"] + left["orphan_stale"],
|
|
"pending_stale": left["pending_fresh"]
|
|
+ left["pending_stale_with_object"]
|
|
+ left["pending_stale_without_object"],
|
|
}
|
|
orphans = inject("orphan", 3)
|
|
intents = inject("stale-pending", 3, age=INTENT_AGE)
|
|
results, lock = [], threading.Lock()
|
|
deadline = time.monotonic() + seconds
|
|
workers = [
|
|
threading.Thread(
|
|
target=uploader,
|
|
args=(base, alice["token"], chat["id"]),
|
|
kwargs={"deadline": deadline, "results": results, "lock": lock},
|
|
)
|
|
for _ in range(threads)
|
|
]
|
|
for worker in workers:
|
|
worker.start()
|
|
runs = []
|
|
while time.monotonic() < deadline:
|
|
runs.append(reconcile("--apply", *FLAGS))
|
|
for worker in workers:
|
|
worker.join()
|
|
runs.append(reconcile("--apply", *FLAGS))
|
|
created = {att["id"]: att["sha256"] for status, att in results if att}
|
|
expected = dict(json.loads(ready.read_text()), **created)
|
|
save("ready-load.json", expected)
|
|
after = oracle(
|
|
"--grace",
|
|
str(GRACE),
|
|
"--deep",
|
|
"--expect-ready",
|
|
str(ready),
|
|
"--output",
|
|
str(OUT / "check-under-load.json"),
|
|
)
|
|
statuses = {}
|
|
for status, _ in results:
|
|
statuses[str(status)] = statuses.get(str(status), 0) + 1
|
|
fixed = {name: sum(r["fixed"][name] for r in runs) for name in FIXABLE}
|
|
seen = {name: sum(r["found"][name] for r in runs) for name in ("orphan_fresh", "pending_fresh")}
|
|
summary = {
|
|
"seconds": seconds,
|
|
"upload_threads": threads,
|
|
"uploads": statuses,
|
|
"created": len(created),
|
|
"linked_every": 10,
|
|
"reconciler_runs": len(runs),
|
|
"run_seconds": [r["cost"]["seconds"] for r in runs],
|
|
"fixed_total": fixed,
|
|
"fresh_seen_total": seen,
|
|
"errors": sum(r["errors"] for r in runs),
|
|
"skipped": sum(1 for r in runs if r["skipped"]),
|
|
"injected_before_load": {"orphan": len(orphans), "pending_stale": len(intents)},
|
|
"left_from_faults": leftover,
|
|
"check_after": {k: after[k] for k in ("objects", "rows", "ready_healthy", "counts", "ok")},
|
|
}
|
|
save("under-load.json", summary)
|
|
check(
|
|
"под нагрузкой: исправлены только внедрённые, ошибок нет",
|
|
fixed
|
|
== {
|
|
**dict.fromkeys(FIXABLE, 0),
|
|
"orphan_stale": leftover["orphan_stale"] + len(orphans),
|
|
"pending_stale": leftover["pending_stale"] + len(intents),
|
|
}
|
|
and not summary["errors"]
|
|
and not summary["skipped"],
|
|
fixed=fixed,
|
|
runs=len(runs),
|
|
)
|
|
check(
|
|
"под нагрузкой: все 201 готовы и совпадают по SHA-256, потерь 0",
|
|
after["ok"] and not after["counts"]["lost_ready"] and set(statuses) == {"201"},
|
|
uploads=statuses,
|
|
lost_ready=after["counts"]["lost_ready"],
|
|
)
|
|
|
|
|
|
def demo(args):
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
c = client.Client(args.base_url, timeout=30)
|
|
check = Checks()
|
|
pre = oracle("--grace", str(GRACE))
|
|
dirty = {k: v for k, v in pre["counts"].items() if v and k != "quarantined"}
|
|
if dirty:
|
|
raise SystemExit(f"стенд не в исходном состоянии {dirty}: make reconcile ARGS=--apply")
|
|
started = time.monotonic()
|
|
result = {"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head()}
|
|
result["faults"] = faults(c, check, pre)
|
|
single(check)
|
|
under_load(c, check, args.base_url, args.load_seconds, args.load_threads)
|
|
result.update(seconds=round(time.monotonic() - started, 1), checks=check.items, ok=check.ok)
|
|
save("demo.json", result)
|
|
print(json.dumps({"ok": check.ok, "checks": len(check.items)}, ensure_ascii=False))
|
|
sys.exit(0 if check.ok else 1)
|
|
|
|
|
|
def pins(args):
|
|
cpus = int(must(["docker", "info", "--format", "{{.NCPU}}"]).strip())
|
|
sized = {"S3_CPUS": "2", "S3_MEMORY": args.s3_memory}
|
|
if args.no_pin or cpus < 8:
|
|
return sized
|
|
return dict(
|
|
sized,
|
|
S3_CPUSET=f"{cpus - 5}-{cpus - 4}",
|
|
PG_CPUSET=f"{cpus - 3}-{cpus - 2}",
|
|
RECONCILE_CPUSET=str(cpus - 1),
|
|
)
|
|
|
|
|
|
def limits():
|
|
ids = compose("ps", "-q", "s3", "postgres").split()
|
|
fmt = (
|
|
"{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}"
|
|
" cpuset={{.HostConfig.CpusetCpus}}"
|
|
)
|
|
return must(["docker", "inspect", "-f", fmt, *ids]).strip().splitlines()
|
|
|
|
|
|
def seed(count, env):
|
|
if count <= 0:
|
|
return {"added": 0}
|
|
env = dict(env, RECONCILE_CPUSET="", RECONCILE_CPUS="4", RECONCILE_MEMORY="512m")
|
|
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
|
|
out = compose(
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"-T",
|
|
"-v",
|
|
tools,
|
|
"--entrypoint",
|
|
"python",
|
|
"reconciler",
|
|
"/tools/seed_storage.py",
|
|
"--count",
|
|
str(count),
|
|
"--concurrency",
|
|
"16",
|
|
env=env,
|
|
)
|
|
return json.loads(out.strip().splitlines()[-1])
|
|
|
|
|
|
def median(runs, path):
|
|
values = []
|
|
for r in runs:
|
|
value = r
|
|
for key in path:
|
|
value = value[key]
|
|
values.append(value)
|
|
return statistics.median(values)
|
|
|
|
|
|
def summarize(results):
|
|
rows = []
|
|
for (n, mode), runs in sorted(results.items()):
|
|
seconds = median(runs, ("cost", "seconds"))
|
|
rows.append(
|
|
{
|
|
"objects": n,
|
|
"mode": mode,
|
|
"runs": len(runs),
|
|
"seconds_median": seconds,
|
|
"seconds_all": [r["cost"]["seconds"] for r in runs],
|
|
"wall_seconds_median": median(runs, ("wall_seconds",)),
|
|
"us_per_object": round(seconds / n * 1e6, 1),
|
|
"s3_list_wait_median": median(runs, ("cost", "waited_seconds", "s3_list")),
|
|
"sql_wait_median": median(runs, ("cost", "waited_seconds", "sql")),
|
|
"requests": runs[0]["cost"]["requests"],
|
|
"max_rss_mib_median": median(runs, ("cost", "max_rss_mib")),
|
|
"scanned": runs[0]["scanned"],
|
|
"found_nonzero": {k: v for k, v in runs[0]["found"].items() if v},
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def table(rows):
|
|
head = (
|
|
"objects mode runs t_med,s wall,s us/obj list,s sql,s lists sqls gets"
|
|
" rss,MiB"
|
|
)
|
|
lines = [head]
|
|
for r in rows:
|
|
q = r["requests"]
|
|
lines.append(
|
|
f"{r['objects']:>7} {r['mode']:<9} {r['runs']:>4} {r['seconds_median']:>7.3f} "
|
|
f"{r['wall_seconds_median']:>6.2f} {r['us_per_object']:>6.1f} "
|
|
f"{r['s3_list_wait_median']:>6.3f} {r['sql_wait_median']:>5.3f} {q['s3_list']:>5} "
|
|
f"{q['sql']:>4} {q['s3_get']:>4} {r['max_rss_mib_median']:>7.1f}"
|
|
)
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def cost(args):
|
|
out = OUT / "cost"
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
stats = must(
|
|
[
|
|
"docker",
|
|
"stats",
|
|
"--no-stream",
|
|
"--all",
|
|
"--format",
|
|
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}",
|
|
]
|
|
)
|
|
snapshot = f"date: {stamp}\n$ docker stats --no-stream --all\n{stats}"
|
|
(out / "host-docker-stats.txt").write_text(snapshot)
|
|
rows = int(psql("SELECT count(*) FROM attachments"))
|
|
if rows > args.sizes[0]:
|
|
raise SystemExit(f"на стенде уже {rows} вложений: make reset-data && make up && make test")
|
|
env = pins(args)
|
|
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres", env=env)
|
|
plan = {"date": stamp, "commit": head(), "pins": env, "limits": limits(), "seeds": []}
|
|
results = {}
|
|
try:
|
|
for n in args.sizes:
|
|
plan["seeds"].append(dict(seed(n - rows, env), target=n))
|
|
rows = n
|
|
time.sleep(args.settle)
|
|
modes = [("shallow", ["--dry-run"])]
|
|
if n <= args.deep_max:
|
|
modes.append(("deep", ["--dry-run", "--deep"]))
|
|
if n == args.page_compare:
|
|
modes.append(("page100", ["--dry-run", "--page-size", "100"]))
|
|
for mode, flags in modes:
|
|
for attempt in range(1, args.repeats + 1):
|
|
report = reconcile(*flags, "--samples", "3", env=env)
|
|
(out / f"n{n}-{mode}-{attempt}.json").write_text(json.dumps(report, indent=2))
|
|
results.setdefault((n, mode), []).append(report)
|
|
sys.stderr.write(f"{n} {mode} #{attempt}: {report['cost']['seconds']} s\n")
|
|
finally:
|
|
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres")
|
|
summary = summarize(results)
|
|
save("cost/summary.json", {**plan, "results": summary})
|
|
(out / "summary.txt").write_text(f"date: {stamp} pins: {env}\n" + table(summary))
|
|
sys.stdout.write(table(summary))
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description="Lab4 track 2: reconciler demo and cost of a pass")
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
d = sub.add_parser("demo", help="fault injection, dry-run/apply/repeat, safety, under load")
|
|
d.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
|
|
d.add_argument("--load-seconds", type=int, default=75)
|
|
d.add_argument("--load-threads", type=int, default=3)
|
|
k = sub.add_parser("cost", help="cost of a pass for N objects, repeated")
|
|
k.add_argument("--sizes", type=int, nargs="+", default=[1000, 10000, 100000])
|
|
k.add_argument("--repeats", type=int, default=3)
|
|
k.add_argument("--deep-max", type=int, default=100000)
|
|
k.add_argument("--page-compare", type=int, default=10000)
|
|
k.add_argument("--settle", type=float, default=10)
|
|
k.add_argument("--s3-memory", default="3g")
|
|
k.add_argument("--no-pin", action="store_true")
|
|
args = p.parse_args()
|
|
if args.command == "demo":
|
|
demo(args)
|
|
else:
|
|
cost(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|