#!/usr/bin/env python3 import argparse import http.client import json import os import re import statistics import struct import subprocess import sys import threading import time import uuid import zlib from pathlib import Path from urllib.parse import urlsplit ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) client = __import__("client") FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes() GRACE = 60 FLAGS = ("--grace", str(GRACE), "--skew", "0") INTENT_AGE = 300 LOCK_KEY = 20260004 FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale") USER_HZ = 100 BUSY = (0, 1, 2, 5, 6, 7) def run(cmd, env=None, stdin=None): started = time.monotonic() done = subprocess.run( cmd, cwd=ROOT, input=stdin, capture_output=True, check=False, env=dict(os.environ, **(env or {})), ) out, err = done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace") return done.returncode, out, err, round(time.monotonic() - started, 3) def must(cmd, env=None, stdin=None): code, out, err, _ = run(cmd, env, stdin) if code: raise SystemExit(f"failed: {' '.join(cmd[:7])}\n{err[-800:]}") return out def compose(*args, env=None, stdin=None): return must(["docker", "compose", *args], env, stdin) def psql(sql): return compose( "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt", "-U", "messenger", "-d", "messenger", "-c", sql, ).strip() def reconcile(*flags, env=None): cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", *flags] code, out, err, wall = run(cmd, env) if code not in (0, 1) or not out.strip(): raise SystemExit(f"reconciler failed ({code}):\n{err[-800:]}") report = json.loads(out) audit = [json.loads(line) for line in err.splitlines() if '"audit.' in line] report.update(exit_code=code, wall_seconds=wall, audit=audit) return report def oracle(*flags): code, out, err, _ = run([sys.executable, "scripts/reconcile_check.py", *flags]) if not out.strip(): raise SystemExit(f"reconcile_check failed ({code}):\n{err[-800:]}") return json.loads(out) def inject(kind, count, ids=(), age=None): env = {"INJECT_IDS": ",".join(ids)} if ids else {} if age is not None: env["INJECT_AGE_SECONDS"] = str(age) out = must(["sh", "scripts/reconcile-inject.sh", kind, str(count)], env) return [line.split()[1] for line in out.splitlines() if line.strip()] def head(): return must(["git", "rev-parse", "HEAD"]).strip() def evidence_root(): return ROOT / os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile") def save(name, data, *, stamp=True): path = evidence_root() / name path.parent.mkdir(parents=True, exist_ok=True) stamped = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head(), **data} path.write_text(json.dumps(stamped if stamp else data, ensure_ascii=False, indent=2) + "\n") return path class Checks: def __init__(self): self.items = [] def __call__(self, name, ok, **detail): self.items.append(dict(name=name, ok=bool(ok), **detail)) sys.stderr.write(("PASS " if ok else "FAIL ") + name + "\n") @property def ok(self): return all(item["ok"] for item in self.items) def chunk(tag, data): return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data)) def tiny_png(): raw = b"".join(b"\x00" + os.urandom(24) for _ in range(8)) header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0) return ( b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"") ) def pick(found, names): return {name: found.get(name, 0) for name in names} def faults(c, check, pre): alice, bob, _, chat = c.bootstrap() ids = [c.attachment(alice, chat)[0]["id"] for _ in range(18)] linked = ids[:6] for part in (linked[:3], linked[3:]): c.message(alice, chat, "", attachments=part) targets = { "missing": [linked[0], linked[1], ids[6]], "corrupt": [linked[2], ids[7], ids[8]], "bitflip": [linked[3], ids[9], ids[10]], } damaged = {i for group in targets.values() for i in group} ready = evidence_root() / "ready-before.json" oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready)) untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged} save("ready-untouched.json", untouched, stamp=False) started = time.monotonic() injected = { "orphan_stale": inject("orphan", 3), "missing": inject("missing", 3, targets["missing"]), "size_mismatch": inject("corrupt", 3, targets["corrupt"]), "sha_mismatch": inject("bitflip", 3, targets["bitflip"]), "pending_stale": inject("stale-pending", 3, age=INTENT_AGE) + inject("pending-object", 3, age=INTENT_AGE), } time.sleep(max(0.0, GRACE + 5 - (time.monotonic() - started))) injected["orphan_fresh"] = inject("orphan", 2) injected["pending_fresh"] = inject("stale-pending", 2, age=0) expected = {name: len(keys) for name, keys in injected.items()} expected.update(quarantined=pre["counts"]["quarantined"]) before = oracle( "--grace", str(GRACE), "--deep", "--output", str(evidence_root() / "check-before.json") ) dry = reconcile("--dry-run", "--deep", *FLAGS) save("dry-run.json", dry) after_dry = oracle("--grace", str(GRACE), "--deep") named = {k: sorted(s.rsplit("/", 1)[-1] for s in dry["samples"].get(k, [])) for k in injected} check( "dry-run находит каждый внедрённый класс, и найдено ровно внедрённое", pick(dry["found"], expected) == expected and named == {name: sorted(ids) for name, ids in injected.items()}, expected=expected, found=dry["found"], ) check( "dry-run ничего не меняет", not any(dry["fixed"].values()) and after_dry["counts"] == before["counts"] and (after_dry["objects"], after_dry["rows"]) == (before["objects"], before["rows"]), objects=[before["objects"], after_dry["objects"]], rows=[before["rows"], after_dry["rows"]], ) first = reconcile("--apply", "--deep", *FLAGS) save("apply-1.json", first) discarded = sorted(e["attachment_id"] for e in first["audit"] if e["reason"] == "reconcile") check( "apply исправляет ровно найденное dry-run, удаление намерений — в журнале аудита", pick(first["fixed"], FIXABLE) == pick(dry["found"], FIXABLE) and not first["errors"] and discarded == sorted(injected["pending_stale"]), dry_run=pick(dry["found"], FIXABLE), fixed=pick(first["fixed"], FIXABLE), audit_discarded=len(discarded), ) second = reconcile("--apply", "--deep", *FLAGS) save("apply-2.json", second) check( "повторный apply — 0 действий", not any(second["fixed"].values()), fixed=second["fixed"], found=second["found"], ) after = oracle( "--grace", str(GRACE), "--deep", "--expect-ready", str(evidence_root() / "ready-untouched.json"), "--output", str(evidence_root() / "check-after.json"), ) tolerated = ("orphan_fresh", "pending_fresh") check( "независимая проверка после apply: нарушений нет, готовые не потеряны", after["ok"] and not after["counts"]["lost_ready"] and pick(after["counts"], tolerated) == pick(before["counts"], tolerated) and after["counts"]["quarantined"] == pre["counts"]["quarantined"] + 9, counts=after["counts"], ) marks = dict( line.split("|") for line in psql("SELECT id, quarantine_reason FROM attachments").split() ) links = psql("SELECT attachment_id FROM message_attachments").split() kept = {i: marks.get(i) for i in sorted(damaged)} check( "строки с расхождениями и ссылки сообщений сохранены", all(kept.values()) and sum(i in linked for i in links) == len(linked), quarantine=kept, message_links=sum(i in linked for i in links), ) meta = c.call("GET", f"/api/v1/attachments/{linked[0]}", 200, actor=bob, schema="Attachment") blocked = c.call("GET", f"/api/v1/attachments/{linked[2]}/content", 404, actor=bob) intact = c.call("GET", f"/api/v1/attachments/{linked[4]}/content", 200, actor=bob) check( "карантин: метаданные 200, /content 404; нетронутое скачивается", meta["status"] == 200 and blocked["status"] == 404 and intact["bytes"] == FIXTURE, ) bucket = os.environ.get("S3_BUCKET", "messenger-attachments") for attachment, data in ((linked[0], FIXTURE), (linked[1], os.urandom(len(FIXTURE)))): compose( "run", "--rm", "--no-deps", "-T", "s3-cli", "pipe", f"root/{bucket}/attachments/{attachment}", stdin=data, ) healed = reconcile("--apply", *FLAGS) save("healed.json", healed) restored = c.call("GET", f"/api/v1/attachments/{linked[0]}/content", 200, actor=bob) foreign = c.call("GET", f"/api/v1/attachments/{linked[1]}/content", 404, actor=bob) check( "вернули те же байты — карантин снят; другие байты той же длины — остался", healed["fixed"]["healed"] == 1 and healed["samples"].get("healed") == [linked[0]] and linked[1] in healed["samples"].get("quarantined", []) and restored["bytes"] == FIXTURE and foreign["status"] == 404, fixed=healed["fixed"], s3_get=healed["cost"]["requests"]["s3_get"], ) return {"injected": injected, "targets": targets, "linked": linked} def single(check): holder = subprocess.Popen( [ "docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c", f"SELECT pg_advisory_lock({LOCK_KEY}), pg_sleep(15)", ], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) try: time.sleep(3) busy = reconcile("--apply", *FLAGS) finally: holder.wait() save("lock-busy.json", busy) check( "второй одновременный проход пропущен", busy["skipped"] and busy["exit_code"] == 0 and not any(busy["fixed"].values()), ) def healthy(service, seconds=90): container = compose("ps", "-q", service).strip() probe = ["docker", "inspect", "-f", "{{.State.Health.Status}}", container] deadline = time.monotonic() + seconds while must(probe).strip() != "healthy": if time.monotonic() > deadline: raise SystemExit(f"{service} is not healthy after {seconds} s") time.sleep(1) def interrupt(check): name = f"lab4-reconcile-sigterm-{os.getpid()}" compose("pause", "s3") try: cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "--name", name, "reconciler"] runner = subprocess.Popen( [*cmd, "--apply", *FLAGS], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) time.sleep(4) state = must(["docker", "inspect", "-f", "{{.State.Status}} {{.HostConfig.Init}}", name]) started = time.monotonic() must(["docker", "stop", name]) stopped = round(time.monotonic() - started, 2) code = runner.wait(timeout=60) finally: compose("unpause", "s3") healthy("s3") after = reconcile("--dry-run", *FLAGS) status, init = state.split() report = { "s3": "paused", "state_before_stop": status, "init": init, "stop_seconds": stopped, "exit_code": code, "next_pass_skipped": after["skipped"], } save("sigterm.json", report) check( "SIGTERM: идущий проход останавливается за секунды, блокировка освобождена", status == "running" and code == 143 and stopped < 5 and not after["skipped"], **report, ) def outage(check): before = oracle("--grace", str(GRACE)) compose("stop", "s3") try: cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", "--apply"] code, out, err, wall = run([*cmd, *FLAGS]) finally: compose("up", "-d", "--wait", "--no-deps", "s3") after = oracle("--grace", str(GRACE)) log = [line for line in err.splitlines() if "reconcile." in line] same = all(after[k] == before[k] for k in ("objects", "rows", "counts")) save("s3-down.json", {"exit_code": code, "wall_seconds": wall, "stdout": out, "log": log}) check( "S3 остановлен: код 1 за ограниченное время, состояние не изменилось", code == 1 and wall < 30 and not out.strip() and same, exit_code=code, wall_seconds=wall, ) def uploader(base, token, chat, *, deadline, results, lock): host = urlsplit(base).netloc conn = http.client.HTTPConnection(host, timeout=30) headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"} sent = 0 while time.monotonic() < deadline: path = f"/api/v1/chats/{chat}/attachments?filename=load.png" try: conn.request("POST", path, tiny_png(), {**headers, "Content-Type": "image/png"}) response = conn.getresponse() status, data = response.status, response.read() if status == 201 and sent % 10 == 0: body = json.dumps({"text": "", "attachment_ids": [json.loads(data)["id"]]}) extra = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex} conn.request("POST", f"/api/v1/chats/{chat}/messages", body, {**headers, **extra}) linked = conn.getresponse() linked.read() status = 201 if linked.status == 201 else -linked.status except (OSError, http.client.HTTPException): conn.close() conn = http.client.HTTPConnection(host, timeout=30) status, data = 0, b"" with lock: results.append((status, json.loads(data) if status == 201 else None)) sent += 1 time.sleep(0.05) def under_load(c, check, base, seconds, threads): alice, _, _, chat = c.bootstrap() ready = evidence_root() / "ready-load.json" left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"] leftover = { "orphan_stale": left["orphan_fresh"] + left["orphan_stale"], "pending_stale": left["pending_fresh"] + left["pending_stale_with_object"] + left["pending_stale_without_object"], } orphans = inject("orphan", 3) intents = inject("stale-pending", 3, age=INTENT_AGE) results, lock = [], threading.Lock() deadline = time.monotonic() + seconds workers = [ threading.Thread( target=uploader, args=(base, alice["token"], chat["id"]), kwargs={"deadline": deadline, "results": results, "lock": lock}, ) for _ in range(threads) ] for worker in workers: worker.start() runs = [] while time.monotonic() < deadline: runs.append(reconcile("--apply", *FLAGS)) for worker in workers: worker.join() runs.append(reconcile("--apply", *FLAGS)) created = {att["id"]: att["sha256"] for status, att in results if att} expected = dict(json.loads(ready.read_text()), **created) save("ready-load.json", expected, stamp=False) after = oracle( "--grace", str(GRACE), "--deep", "--expect-ready", str(ready), "--output", str(evidence_root() / "check-under-load.json"), ) statuses = {} for status, _ in results: statuses[str(status)] = statuses.get(str(status), 0) + 1 fixed = {name: sum(r["fixed"][name] for r in runs) for name in FIXABLE} rechecked = {} for r in runs: for name, count in r["rechecked"].items(): rechecked[name] = rechecked.get(name, 0) + count seen = {name: sum(r["found"][name] for r in runs) for name in ("orphan_fresh", "pending_fresh")} summary = { "seconds": seconds, "upload_threads": threads, "uploads": statuses, "created": len(created), "linked_every": 10, "reconciler_runs": len(runs), "run_seconds": [r["cost"]["seconds"] for r in runs], "fixed_total": fixed, "fresh_seen_total": seen, "rechecked_total": rechecked, "errors": sum(r["errors"] for r in runs), "skipped": sum(1 for r in runs if r["skipped"]), "injected_before_load": {"orphan": len(orphans), "pending_stale": len(intents)}, "left_from_faults": leftover, "check_after": {k: after[k] for k in ("objects", "rows", "ready_healthy", "counts", "ok")}, } save("under-load.json", summary) check( "под нагрузкой: исправлены только внедрённые, ошибок нет", fixed == { **dict.fromkeys(FIXABLE, 0), "orphan_stale": leftover["orphan_stale"] + len(orphans), "pending_stale": leftover["pending_stale"] + len(intents), } and not summary["errors"] and not summary["skipped"], fixed=fixed, runs=len(runs), ) check( "под нагрузкой: все 201 готовы и совпадают по SHA-256, потерь 0", after["ok"] and not after["counts"]["lost_ready"] and set(statuses) == {"201"}, uploads=statuses, lost_ready=after["counts"]["lost_ready"], ) def demo(args): evidence_root().mkdir(parents=True, exist_ok=False) c = client.Client(args.base_url, timeout=30) check = Checks() pre = oracle("--grace", str(GRACE)) dirty = {k: v for k, v in pre["counts"].items() if v and k != "quarantined"} if dirty: raise SystemExit(f"стенд не в исходном состоянии {dirty}: make reconcile ARGS=--apply") started = time.monotonic() result = {"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head()} result["faults"] = faults(c, check, pre) single(check) interrupt(check) outage(check) under_load(c, check, args.base_url, args.load_seconds, args.load_threads) result.update(seconds=round(time.monotonic() - started, 1), checks=check.items, ok=check.ok) save("demo.json", result) print(json.dumps({"ok": check.ok, "checks": len(check.items)}, ensure_ascii=False)) sys.exit(0 if check.ok else 1) def pins(args): cpus = int(must(["docker", "info", "--format", "{{.NCPU}}"]).strip()) sized = {"S3_CPUS": "2", "S3_MEMORY": args.s3_memory} if args.no_pin or cpus < 8: return sized return dict( sized, S3_CPUSET=f"{cpus - 5}-{cpus - 4}", PG_CPUSET=f"{cpus - 3}-{cpus - 2}", RECONCILE_CPUSET=str(cpus - 1), ) def cpu_set(env): cpus = set() for name in ("S3_CPUSET", "PG_CPUSET", "RECONCILE_CPUSET"): for part in filter(None, env.get(name, "").split(",")): low, _, high = part.partition("-") cpus.update(range(int(low), int(high or low) + 1)) return cpus def limits(): ids = compose("ps", "-q", "s3", "postgres").split() fmt = ( "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}" " cpuset={{.HostConfig.CpusetCpus}}" ) return must(["docker", "inspect", "-f", fmt, *ids]).strip().splitlines() def host(title): stats = must( [ "docker", "stats", "--no-stream", "--all", "--format", "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}", ] ) running = must(["docker", "ps", "--format", "{{.Names}} {{.Image}} {{.Status}}"]) stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) load = " ".join(f"{v:.2f}" for v in os.getloadavg()) return ( f"== {title} date: {stamp} commit: {head()} host loadavg: {load}\n" f"$ docker stats --no-stream --all\n{stats}\n$ docker ps\n{running}\n" ) def usage(text): return int(re.search(r"^usage_usec (\d+)$", text, re.M).group(1)) / 1e6 def sample(): vm = compose( "exec", "-T", "postgres", "cat", "/proc/uptime", "/proc/stat", "/sys/fs/cgroup/cpu.stat" ) s3 = compose("exec", "-T", "s3", "cat", "/sys/fs/cgroup/cpu.stat") busy = {} for line in vm.splitlines()[1:]: name, *values = line.split() or [""] if name.startswith("cpu") and name[3:].isdigit(): busy[int(name[3:])] = sum(int(values[i]) for i in BUSY) / USER_HZ return float(vm.split()[0]), busy, usage(vm) + usage(s3) def screen(first, last, own, pinned, args, *, started=True): window = max(last[0] - first[0], 0.01) busy = {cpu: spent - first[1].get(cpu, 0.0) for cpu, spent in last[1].items()} ours = last[2] - first[2] + own other = sum(busy.values()) - ours - args.start_other * started on_pinned = sum(busy.get(cpu, 0.0) for cpu in pinned) - ours - args.start_pinned * started other, on_pinned = max(0.0, other / window), max(0.0, on_pinned / window) if pinned else 0.0 return { "window_seconds": round(window, 2), "ours_cores": round(ours / window, 2), "other_cores": round(other, 2), "other_on_pinned_cores": round(on_pinned, 2), "host_loadavg_1m": round(os.getloadavg()[0], 2), "disturbed": other > args.max_other or on_pinned > args.max_pinned, } def wait_quiet(args, pinned): deadline, calm = time.monotonic() + args.quiet_timeout, 0 while calm < 3 and time.monotonic() < deadline: first = sample() time.sleep(5) quiet = screen(first, sample(), 0.0, pinned, args, started=False) calm = 0 if quiet["disturbed"] else calm + 1 sys.stderr.write(f"waiting for quiet: {json.dumps(quiet)}\n") return calm == 3 def measure(flags, env, args, pinned): rejected = [] for attempt in range(args.retries + 1): first = sample() report = reconcile(*flags, "--samples", "3", env=env) report["screen"] = screen(first, sample(), report["cost"]["cpu_seconds"], pinned, args) if not report["screen"]["disturbed"]: return report, rejected rejected.append(report) if attempt < args.retries and not wait_quiet(args, pinned): break return None, rejected def seed(count, env): if count <= 0: return {"added": 0} env = dict(env, API_CPUS="4", API_MEMORY="512m") tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro" out = compose( "run", "--rm", "--no-deps", "-T", "-v", tools, "--entrypoint", "python", "api", "/tools/seed_storage.py", "--count", str(count), "--concurrency", "4", env=env, ) records = [json.loads(line) for line in out.strip().splitlines()] return {**records[-1], "progress": records[:-1]} def median(runs, path): values = [] for r in runs: value = r for key in path: value = value[key] values.append(value) return statistics.median(values) def summarize(results, rejected): rows = [] for (n, mode), runs in sorted(results.items()): seconds = median(runs, ("cost", "seconds")) rows.append( { "objects": n, "mode": mode, "runs": len(runs), "seconds_median": seconds, "seconds_all": [r["cost"]["seconds"] for r in runs], "wall_seconds_median": median(runs, ("wall_seconds",)), "us_per_object": round(seconds / n * 1e6, 1), "cpu_seconds_median": median(runs, ("cost", "cpu_seconds")), "s3_list_wait_median": median(runs, ("cost", "waited_seconds", "s3_list")), "sql_wait_median": median(runs, ("cost", "waited_seconds", "sql")), "requests": runs[0]["cost"]["requests"], "max_rss_mib_median": median(runs, ("cost", "max_rss_mib")), "other_cores_max": max(r["screen"]["other_cores"] for r in runs), "other_on_pinned_cores_max": max( r["screen"]["other_on_pinned_cores"] for r in runs ), "disturbed_kept": sum(r["screen"]["disturbed"] for r in runs), "rejected": rejected.get((n, mode), 0), "scanned": runs[0]["scanned"], "found_nonzero": {k: v for k, v in runs[0]["found"].items() if v}, } ) return rows def table(rows): head = ( "objects mode runs t_med,s wall,s us/obj cpu,s list,s sql,s lists sqls" " gets rss,MiB other pinned rejected" ) lines = [head] for r in rows: q = r["requests"] lines.append( f"{r['objects']:>7} {r['mode']:<9} {r['runs']:>4} {r['seconds_median']:>7.3f} " f"{r['wall_seconds_median']:>6.2f} {r['us_per_object']:>6.1f} " f"{r['cpu_seconds_median']:>5.2f} {r['s3_list_wait_median']:>6.3f} " f"{r['sql_wait_median']:>5.3f} {q['s3_list']:>5} {q['sql']:>4} {q['s3_get']:>5} " f"{r['max_rss_mib_median']:>7.1f} {r['other_cores_max']:>5.2f} " f"{r['other_on_pinned_cores_max']:>6.2f} {r['rejected']:>8}" ) return "\n".join(lines) + "\n" def cost(args): out = evidence_root() / "cost" out.mkdir(parents=True, exist_ok=False) (out / "disturbed").mkdir() rows = int(psql("SELECT count(*) FROM attachments")) if rows > args.sizes[0]: raise SystemExit(f"на стенде уже {rows} вложений: используйте новый отдельный project") snapshot = out / "host-docker-stats.txt" snapshot.write_text(host("before the series")) env = pins(args) pinned = cpu_set(env) compose("up", "-d", "--wait", "--no-deps", "s3", "postgres", env=env) stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) plan = {"date": stamp, "commit": head(), "pins": env, "limits": limits(), "seeds": []} plan["screen"] = { "max_other_cores": args.max_other, "max_other_on_pinned_cores": args.max_pinned, "start_allowance_core_seconds": {"vm": args.start_other, "pinned": args.start_pinned}, "retries": args.retries, } results, rejected = {}, {} try: for n in args.sizes: plan["seeds"].append(dict(seed(n - rows, env), target=n)) rows = n time.sleep(args.settle) save("cost/plan.json", plan) if not wait_quiet(args, pinned): raise SystemExit("no quiet measurement window before timeout") modes = [("shallow", ["--dry-run"])] if n <= args.deep_max: modes.append(("deep", ["--dry-run", "--deep"])) if n == args.page_compare: modes.append(("page100", ["--dry-run", "--page-size", "100"])) for mode, flags in modes: for attempt in range(1, args.repeats + 1): report, dropped = measure(flags, env, args, pinned) name = f"n{n}-{mode}-{attempt}" for number, bad in enumerate(dropped, 1): save(f"cost/disturbed/{name}-try{number}.json", bad) rejected[(n, mode)] = rejected.get((n, mode), 0) + len(dropped) if report is None: raise SystemExit(f"{name}: no clean run after retries; rejected preserved") if ( report["exit_code"] or report["errors"] or report["skipped"] or report["scanned"] != {"objects": n, "rows": n} or any(report["found"].values()) ): save(f"cost/invalid/{name}.json", report) raise SystemExit(f"{name}: inconsistent dataset or failed pass") save(f"cost/{name}.json", report) results.setdefault((n, mode), []).append(report) save("cost/summary.json", {**plan, "results": summarize(results, rejected)}) sys.stderr.write(f"{name}: {report['cost']['seconds']} s {report['screen']}\n") finally: save("cost/summary.json", {**plan, "results": summarize(results, rejected)}) compose("up", "-d", "--wait", "--no-deps", "s3", "postgres") with snapshot.open("a") as handle: handle.write("\n" + host("after the series")) summary = summarize(results, rejected) save("cost/summary.json", {**plan, "results": summary}) (out / "summary.txt").write_text( f"date: {stamp} commit: {plan['commit']} pins: {env}\n" + table(summary) ) sys.stdout.write(table(summary)) def main(): p = argparse.ArgumentParser(description="Lab4 track 2: reconciler demo and cost of a pass") sub = p.add_subparsers(dest="command", required=True) d = sub.add_parser("demo", help="fault injection, dry-run/apply/repeat, safety, under load") d.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) d.add_argument("--load-seconds", type=int, default=75) d.add_argument("--load-threads", type=int, default=3) k = sub.add_parser("cost", help="cost of a pass for N objects, repeated") k.add_argument("--sizes", type=int, nargs="+", default=[1000, 10000, 30000, 100000]) k.add_argument("--repeats", type=int, default=3) k.add_argument("--deep-max", type=int, default=100000) k.add_argument("--page-compare", type=int, default=10000) k.add_argument("--settle", type=float, default=10) k.add_argument("--s3-memory", default="3g") k.add_argument("--no-pin", action="store_true") k.add_argument("--max-other", type=float, default=2.0, help="foreign CPU in the VM, cores") k.add_argument("--max-pinned", type=float, default=0.25, help="foreign CPU on pinned cores") k.add_argument("--start-other", type=float, default=1.5, help="container start, core-s") k.add_argument("--start-pinned", type=float, default=0.5, help="the same on pinned cores") k.add_argument("--retries", type=int, default=3, help="reruns of a disturbed run") k.add_argument("--quiet-timeout", type=float, default=900.0) for command in (d, k): command.add_argument( "--out", default=os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile") ) args = p.parse_args() os.environ["RECONCILE_EVIDENCE_DIR"] = args.out if args.command == "demo": demo(args) else: cost(args) if __name__ == "__main__": main()