Files
mtusi2026lab4/scripts/reconcile_track.py
T
ArtyomandClaude Opus 5.5 2891b37e75 Split the S3 keys of the API and the reconciler, drop root from one-offs
The API key now has only GetObject, PutObject and DeleteObject on the
attachment objects: the server never lists the bucket, never runs
multipart uploads and has its region set. The reconciler gets its own
user messenger-reconciler with ListBucket, GetObject and DeleteObject,
created by s3-init from a secret of its own, so the API replicas can no
longer list the bucket. The cost seeding writes objects and therefore
runs in a one-off API container under the API key. s3-check proves the
API key cannot list and checks the reconciler key; its snapshot drops
dated fields with a JSON filter instead of grep, which left a dangling
comma, and its first line carries the date and the commit.

s3-init and s3-cli run as uid 65534, and migrate and reconciler get an
init process, so SIGTERM reaches python and docker stop ends a pass in
a fraction of a second instead of waiting for SIGKILL. limits.py and CI
check the users of the one-off services. Stack tests move a 3 MiB file
up and down through the balancer at 256 and 192 KiB/s, longer than the
request timeout, and check 401 before the declared size.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
2026-10-30 08:53:39 +03:00

726 lines
27 KiB
Python

#!/usr/bin/env python3
import argparse
import http.client
import json
import os
import re
import statistics
import struct
import subprocess
import sys
import threading
import time
import uuid
import zlib
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
client = __import__("client")
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
OUT = ROOT / "evidence" / "reconcile"
GRACE = 60
FLAGS = ("--grace", str(GRACE), "--skew", "0")
INTENT_AGE = 300
LOCK_KEY = 20260004
FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale")
USER_HZ = 100
BUSY = (0, 1, 2, 5, 6, 7)
def run(cmd, env=None, stdin=None):
started = time.monotonic()
done = subprocess.run(
cmd,
cwd=ROOT,
input=stdin,
capture_output=True,
check=False,
env=dict(os.environ, **(env or {})),
)
out, err = done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace")
return done.returncode, out, err, round(time.monotonic() - started, 3)
def must(cmd, env=None, stdin=None):
code, out, err, _ = run(cmd, env, stdin)
if code:
raise SystemExit(f"failed: {' '.join(cmd[:7])}\n{err[-800:]}")
return out
def compose(*args, env=None, stdin=None):
return must(["docker", "compose", *args], env, stdin)
def psql(sql):
return compose(
"exec",
"-T",
"postgres",
"psql",
"-v",
"ON_ERROR_STOP=1",
"-qAt",
"-U",
"messenger",
"-d",
"messenger",
"-c",
sql,
).strip()
def reconcile(*flags, env=None):
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", *flags]
code, out, err, wall = run(cmd, env)
if code not in (0, 1) or not out.strip():
raise SystemExit(f"reconciler failed ({code}):\n{err[-800:]}")
report = json.loads(out)
report.update(exit_code=code, wall_seconds=wall)
return report
def oracle(*flags):
code, out, err, _ = run([sys.executable, "scripts/reconcile_check.py", *flags])
if not out.strip():
raise SystemExit(f"reconcile_check failed ({code}):\n{err[-800:]}")
return json.loads(out)
def inject(kind, count, ids=(), age=None):
env = {"INJECT_IDS": ",".join(ids)} if ids else {}
if age is not None:
env["INJECT_AGE_SECONDS"] = str(age)
out = must(["sh", "scripts/reconcile-inject.sh", kind, str(count)], env)
return [line.split()[1] for line in out.splitlines() if line.strip()]
def head():
return must(["git", "rev-parse", "--short", "HEAD"]).strip()
def save(name, data):
path = OUT / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
return path
class Checks:
def __init__(self):
self.items = []
def __call__(self, name, ok, **detail):
self.items.append(dict(name=name, ok=bool(ok), **detail))
sys.stderr.write(("PASS " if ok else "FAIL ") + name + "\n")
@property
def ok(self):
return all(item["ok"] for item in self.items)
def chunk(tag, data):
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data))
def tiny_png():
raw = b"".join(b"\x00" + os.urandom(24) for _ in range(8))
header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(raw))
+ chunk(b"IEND", b"")
)
def pick(found, names):
return {name: found.get(name, 0) for name in names}
def faults(c, check, pre):
alice, bob, _, chat = c.bootstrap()
ids = [c.attachment(alice, chat)[0]["id"] for _ in range(18)]
linked = ids[:6]
for part in (linked[:3], linked[3:]):
c.message(alice, chat, "", attachments=part)
targets = {
"missing": [linked[0], linked[1], ids[6]],
"corrupt": [linked[2], ids[7], ids[8]],
"bitflip": [linked[3], ids[9], ids[10]],
}
damaged = {i for group in targets.values() for i in group}
ready = OUT / "ready-before.json"
oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready))
untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged}
save("ready-untouched.json", untouched)
started = time.monotonic()
injected = {
"orphan_stale": inject("orphan", 3),
"missing": inject("missing", 3, targets["missing"]),
"size_mismatch": inject("corrupt", 3, targets["corrupt"]),
"sha_mismatch": inject("bitflip", 3, targets["bitflip"]),
"pending_stale": inject("stale-pending", 3, age=INTENT_AGE)
+ inject("pending-object", 3, age=INTENT_AGE),
}
time.sleep(max(0.0, GRACE + 5 - (time.monotonic() - started)))
injected["orphan_fresh"] = inject("orphan", 2)
injected["pending_fresh"] = inject("stale-pending", 2, age=0)
expected = {name: len(keys) for name, keys in injected.items()}
expected.update(quarantined=pre["counts"]["quarantined"])
before = oracle("--grace", str(GRACE), "--deep", "--output", str(OUT / "check-before.json"))
dry = reconcile("--dry-run", "--deep", *FLAGS)
save("dry-run.json", dry)
after_dry = oracle("--grace", str(GRACE), "--deep")
check(
"dry-run находит каждый внедрённый класс",
pick(dry["found"], expected) == expected,
expected=expected,
found=dry["found"],
)
check(
"dry-run ничего не меняет",
not any(dry["fixed"].values())
and after_dry["counts"] == before["counts"]
and (after_dry["objects"], after_dry["rows"]) == (before["objects"], before["rows"]),
objects=[before["objects"], after_dry["objects"]],
rows=[before["rows"], after_dry["rows"]],
)
first = reconcile("--apply", "--deep", *FLAGS)
save("apply-1.json", first)
check(
"apply исправляет ровно найденное dry-run",
pick(first["fixed"], FIXABLE) == pick(dry["found"], FIXABLE) and not first["errors"],
dry_run=pick(dry["found"], FIXABLE),
fixed=pick(first["fixed"], FIXABLE),
)
second = reconcile("--apply", "--deep", *FLAGS)
save("apply-2.json", second)
check(
"повторный apply — 0 действий",
not any(second["fixed"].values()),
fixed=second["fixed"],
found=second["found"],
)
after = oracle(
"--grace",
str(GRACE),
"--deep",
"--expect-ready",
str(OUT / "ready-untouched.json"),
"--output",
str(OUT / "check-after.json"),
)
tolerated = ("orphan_fresh", "pending_fresh")
check(
"независимая проверка после apply: нарушений нет, готовые не потеряны",
after["ok"]
and not after["counts"]["lost_ready"]
and pick(after["counts"], tolerated) == pick(before["counts"], tolerated)
and after["counts"]["quarantined"] == pre["counts"]["quarantined"] + 9,
counts=after["counts"],
)
marks = dict(
line.split("|") for line in psql("SELECT id, quarantine_reason FROM attachments").split()
)
links = psql("SELECT attachment_id FROM message_attachments").split()
kept = {i: marks.get(i) for i in sorted(damaged)}
check(
"строки с расхождениями и ссылки сообщений сохранены",
all(kept.values()) and sum(i in linked for i in links) == len(linked),
quarantine=kept,
message_links=sum(i in linked for i in links),
)
meta = c.call("GET", f"/api/v1/attachments/{linked[0]}", 200, actor=bob, schema="Attachment")
blocked = c.call("GET", f"/api/v1/attachments/{linked[2]}/content", 404, actor=bob)
intact = c.call("GET", f"/api/v1/attachments/{linked[4]}/content", 200, actor=bob)
check(
"карантин: метаданные 200, /content 404; нетронутое скачивается",
meta["status"] == 200 and blocked["status"] == 404 and intact["bytes"] == FIXTURE,
)
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
compose(
"run",
"--rm",
"--no-deps",
"-T",
"s3-cli",
"pipe",
f"root/{bucket}/attachments/{linked[0]}",
stdin=FIXTURE,
)
healed = reconcile("--apply", *FLAGS)
save("healed.json", healed)
restored = c.call("GET", f"/api/v1/attachments/{linked[0]}/content", 200, actor=bob)
check(
"восстановленный объект выходит из карантина",
healed["fixed"]["healed"] == 1 and restored["bytes"] == FIXTURE,
fixed=healed["fixed"],
)
return {"injected": injected, "targets": targets, "linked": linked}
def single(check):
holder = subprocess.Popen(
[
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-c",
f"SELECT pg_advisory_lock({LOCK_KEY}), pg_sleep(15)",
],
cwd=ROOT,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
time.sleep(3)
busy = reconcile("--apply", *FLAGS)
finally:
holder.wait()
save("lock-busy.json", busy)
check(
"второй одновременный проход пропущен",
busy["skipped"] and busy["exit_code"] == 0 and not any(busy["fixed"].values()),
)
def outage(check):
before = oracle("--grace", str(GRACE))
compose("stop", "s3")
try:
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", "--apply"]
code, out, err, wall = run([*cmd, *FLAGS])
finally:
compose("up", "-d", "--wait", "--no-deps", "s3")
after = oracle("--grace", str(GRACE))
log = [line for line in err.splitlines() if "reconcile." in line]
same = all(after[k] == before[k] for k in ("objects", "rows", "counts"))
save("s3-down.json", {"exit_code": code, "wall_seconds": wall, "stdout": out, "log": log})
check(
"S3 остановлен: код 1 за ограниченное время, состояние не изменилось",
code == 1 and wall < 30 and not out.strip() and same,
exit_code=code,
wall_seconds=wall,
)
def uploader(base, token, chat, *, deadline, results, lock):
host = urlsplit(base).netloc
conn = http.client.HTTPConnection(host, timeout=30)
headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
sent = 0
while time.monotonic() < deadline:
path = f"/api/v1/chats/{chat}/attachments?filename=load.png"
try:
conn.request("POST", path, tiny_png(), {**headers, "Content-Type": "image/png"})
response = conn.getresponse()
status, data = response.status, response.read()
if status == 201 and sent % 10 == 0:
body = json.dumps({"text": "", "attachment_ids": [json.loads(data)["id"]]})
extra = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}
conn.request("POST", f"/api/v1/chats/{chat}/messages", body, {**headers, **extra})
linked = conn.getresponse()
linked.read()
status = 201 if linked.status == 201 else -linked.status
except (OSError, http.client.HTTPException):
conn.close()
conn = http.client.HTTPConnection(host, timeout=30)
status, data = 0, b""
with lock:
results.append((status, json.loads(data) if status == 201 else None))
sent += 1
time.sleep(0.05)
def under_load(c, check, base, seconds, threads):
alice, _, _, chat = c.bootstrap()
ready = OUT / "ready-load.json"
left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"]
leftover = {
"orphan_stale": left["orphan_fresh"] + left["orphan_stale"],
"pending_stale": left["pending_fresh"]
+ left["pending_stale_with_object"]
+ left["pending_stale_without_object"],
}
orphans = inject("orphan", 3)
intents = inject("stale-pending", 3, age=INTENT_AGE)
results, lock = [], threading.Lock()
deadline = time.monotonic() + seconds
workers = [
threading.Thread(
target=uploader,
args=(base, alice["token"], chat["id"]),
kwargs={"deadline": deadline, "results": results, "lock": lock},
)
for _ in range(threads)
]
for worker in workers:
worker.start()
runs = []
while time.monotonic() < deadline:
runs.append(reconcile("--apply", *FLAGS))
for worker in workers:
worker.join()
runs.append(reconcile("--apply", *FLAGS))
created = {att["id"]: att["sha256"] for status, att in results if att}
expected = dict(json.loads(ready.read_text()), **created)
save("ready-load.json", expected)
after = oracle(
"--grace",
str(GRACE),
"--deep",
"--expect-ready",
str(ready),
"--output",
str(OUT / "check-under-load.json"),
)
statuses = {}
for status, _ in results:
statuses[str(status)] = statuses.get(str(status), 0) + 1
fixed = {name: sum(r["fixed"][name] for r in runs) for name in FIXABLE}
seen = {name: sum(r["found"][name] for r in runs) for name in ("orphan_fresh", "pending_fresh")}
summary = {
"seconds": seconds,
"upload_threads": threads,
"uploads": statuses,
"created": len(created),
"linked_every": 10,
"reconciler_runs": len(runs),
"run_seconds": [r["cost"]["seconds"] for r in runs],
"fixed_total": fixed,
"fresh_seen_total": seen,
"errors": sum(r["errors"] for r in runs),
"skipped": sum(1 for r in runs if r["skipped"]),
"injected_before_load": {"orphan": len(orphans), "pending_stale": len(intents)},
"left_from_faults": leftover,
"check_after": {k: after[k] for k in ("objects", "rows", "ready_healthy", "counts", "ok")},
}
save("under-load.json", summary)
check(
"под нагрузкой: исправлены только внедрённые, ошибок нет",
fixed
== {
**dict.fromkeys(FIXABLE, 0),
"orphan_stale": leftover["orphan_stale"] + len(orphans),
"pending_stale": leftover["pending_stale"] + len(intents),
}
and not summary["errors"]
and not summary["skipped"],
fixed=fixed,
runs=len(runs),
)
check(
"под нагрузкой: все 201 готовы и совпадают по SHA-256, потерь 0",
after["ok"] and not after["counts"]["lost_ready"] and set(statuses) == {"201"},
uploads=statuses,
lost_ready=after["counts"]["lost_ready"],
)
def demo(args):
OUT.mkdir(parents=True, exist_ok=True)
c = client.Client(args.base_url, timeout=30)
check = Checks()
pre = oracle("--grace", str(GRACE))
dirty = {k: v for k, v in pre["counts"].items() if v and k != "quarantined"}
if dirty:
raise SystemExit(f"стенд не в исходном состоянии {dirty}: make reconcile ARGS=--apply")
started = time.monotonic()
result = {"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head()}
result["faults"] = faults(c, check, pre)
single(check)
outage(check)
under_load(c, check, args.base_url, args.load_seconds, args.load_threads)
result.update(seconds=round(time.monotonic() - started, 1), checks=check.items, ok=check.ok)
save("demo.json", result)
print(json.dumps({"ok": check.ok, "checks": len(check.items)}, ensure_ascii=False))
sys.exit(0 if check.ok else 1)
def pins(args):
cpus = int(must(["docker", "info", "--format", "{{.NCPU}}"]).strip())
sized = {"S3_CPUS": "2", "S3_MEMORY": args.s3_memory}
if args.no_pin or cpus < 8:
return sized
return dict(
sized,
S3_CPUSET=f"{cpus - 5}-{cpus - 4}",
PG_CPUSET=f"{cpus - 3}-{cpus - 2}",
RECONCILE_CPUSET=str(cpus - 1),
)
def cpu_set(env):
cpus = set()
for name in ("S3_CPUSET", "PG_CPUSET", "RECONCILE_CPUSET"):
for part in filter(None, env.get(name, "").split(",")):
low, _, high = part.partition("-")
cpus.update(range(int(low), int(high or low) + 1))
return cpus
def limits():
ids = compose("ps", "-q", "s3", "postgres").split()
fmt = (
"{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}"
" cpuset={{.HostConfig.CpusetCpus}}"
)
return must(["docker", "inspect", "-f", fmt, *ids]).strip().splitlines()
def host(title):
stats = must(
[
"docker",
"stats",
"--no-stream",
"--all",
"--format",
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}",
]
)
running = must(["docker", "ps", "--format", "{{.Names}} {{.Image}} {{.Status}}"])
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
load = " ".join(f"{v:.2f}" for v in os.getloadavg())
return (
f"== {title} date: {stamp} commit: {head()} host loadavg: {load}\n"
f"$ docker stats --no-stream --all\n{stats}\n$ docker ps\n{running}\n"
)
def usage(text):
return int(re.search(r"^usage_usec (\d+)$", text, re.M).group(1)) / 1e6
def sample():
vm = compose(
"exec", "-T", "postgres", "cat", "/proc/uptime", "/proc/stat", "/sys/fs/cgroup/cpu.stat"
)
s3 = compose("exec", "-T", "s3", "cat", "/sys/fs/cgroup/cpu.stat")
busy = {}
for line in vm.splitlines()[1:]:
name, *values = line.split() or [""]
if name.startswith("cpu") and name[3:].isdigit():
busy[int(name[3:])] = sum(int(values[i]) for i in BUSY) / USER_HZ
return float(vm.split()[0]), busy, usage(vm) + usage(s3)
def screen(first, last, own, pinned, args, *, started=True):
window = max(last[0] - first[0], 0.01)
busy = {cpu: spent - first[1].get(cpu, 0.0) for cpu, spent in last[1].items()}
ours = last[2] - first[2] + own
other = sum(busy.values()) - ours - args.start_other * started
on_pinned = sum(busy.get(cpu, 0.0) for cpu in pinned) - ours - args.start_pinned * started
other, on_pinned = max(0.0, other / window), max(0.0, on_pinned / window) if pinned else 0.0
return {
"window_seconds": round(window, 2),
"ours_cores": round(ours / window, 2),
"other_cores": round(other, 2),
"other_on_pinned_cores": round(on_pinned, 2),
"host_loadavg_1m": round(os.getloadavg()[0], 2),
"disturbed": other > args.max_other or on_pinned > args.max_pinned,
}
def wait_quiet(args, pinned):
deadline, calm = time.monotonic() + args.quiet_timeout, 0
while calm < 3 and time.monotonic() < deadline:
first = sample()
time.sleep(5)
quiet = screen(first, sample(), 0.0, pinned, args, started=False)
calm = 0 if quiet["disturbed"] else calm + 1
sys.stderr.write(f"waiting for quiet: {json.dumps(quiet)}\n")
def measure(flags, env, args, pinned):
rejected = []
for _ in range(args.retries + 1):
first = sample()
report = reconcile(*flags, "--samples", "3", env=env)
report["screen"] = screen(first, sample(), report["cost"]["cpu_seconds"], pinned, args)
if not report["screen"]["disturbed"]:
break
rejected.append(report)
wait_quiet(args, pinned)
return report, rejected
def seed(count, env):
if count <= 0:
return {"added": 0}
env = dict(env, API_CPUS="4", API_MEMORY="512m")
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
out = compose(
"run",
"--rm",
"--no-deps",
"-T",
"-v",
tools,
"--entrypoint",
"python",
"api",
"/tools/seed_storage.py",
"--count",
str(count),
"--concurrency",
"4",
env=env,
)
return json.loads(out.strip().splitlines()[-1])
def median(runs, path):
values = []
for r in runs:
value = r
for key in path:
value = value[key]
values.append(value)
return statistics.median(values)
def summarize(results, rejected):
rows = []
for (n, mode), runs in sorted(results.items()):
seconds = median(runs, ("cost", "seconds"))
rows.append(
{
"objects": n,
"mode": mode,
"runs": len(runs),
"seconds_median": seconds,
"seconds_all": [r["cost"]["seconds"] for r in runs],
"wall_seconds_median": median(runs, ("wall_seconds",)),
"us_per_object": round(seconds / n * 1e6, 1),
"cpu_seconds_median": median(runs, ("cost", "cpu_seconds")),
"s3_list_wait_median": median(runs, ("cost", "waited_seconds", "s3_list")),
"sql_wait_median": median(runs, ("cost", "waited_seconds", "sql")),
"requests": runs[0]["cost"]["requests"],
"max_rss_mib_median": median(runs, ("cost", "max_rss_mib")),
"other_cores_max": max(r["screen"]["other_cores"] for r in runs),
"other_on_pinned_cores_max": max(
r["screen"]["other_on_pinned_cores"] for r in runs
),
"disturbed_kept": sum(r["screen"]["disturbed"] for r in runs),
"rejected": rejected.get((n, mode), 0),
"scanned": runs[0]["scanned"],
"found_nonzero": {k: v for k, v in runs[0]["found"].items() if v},
}
)
return rows
def table(rows):
head = (
"objects mode runs t_med,s wall,s us/obj cpu,s list,s sql,s lists sqls"
" gets rss,MiB other pinned rejected"
)
lines = [head]
for r in rows:
q = r["requests"]
lines.append(
f"{r['objects']:>7} {r['mode']:<9} {r['runs']:>4} {r['seconds_median']:>7.3f} "
f"{r['wall_seconds_median']:>6.2f} {r['us_per_object']:>6.1f} "
f"{r['cpu_seconds_median']:>5.2f} {r['s3_list_wait_median']:>6.3f} "
f"{r['sql_wait_median']:>5.3f} {q['s3_list']:>5} {q['sql']:>4} {q['s3_get']:>5} "
f"{r['max_rss_mib_median']:>7.1f} {r['other_cores_max']:>5.2f} "
f"{r['other_on_pinned_cores_max']:>6.2f} {r['rejected']:>8}"
)
return "\n".join(lines) + "\n"
def cost(args):
out = OUT / "cost"
out.mkdir(parents=True, exist_ok=True)
rows = int(psql("SELECT count(*) FROM attachments"))
if rows > args.sizes[0]:
raise SystemExit(f"на стенде уже {rows} вложений: make reset-data && make up && make test")
snapshot = out / "host-docker-stats.txt"
snapshot.write_text(host("before the series"))
env = pins(args)
pinned = cpu_set(env)
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres", env=env)
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
plan = {"date": stamp, "commit": head(), "pins": env, "limits": limits(), "seeds": []}
plan["screen"] = {
"max_other_cores": args.max_other,
"max_other_on_pinned_cores": args.max_pinned,
"start_allowance_core_seconds": {"vm": args.start_other, "pinned": args.start_pinned},
"retries": args.retries,
}
results, rejected = {}, {}
try:
for n in args.sizes:
plan["seeds"].append(dict(seed(n - rows, env), target=n))
rows = n
time.sleep(args.settle)
wait_quiet(args, pinned)
modes = [("shallow", ["--dry-run"])]
if n <= args.deep_max:
modes.append(("deep", ["--dry-run", "--deep"]))
if n == args.page_compare:
modes.append(("page100", ["--dry-run", "--page-size", "100"]))
for mode, flags in modes:
for attempt in range(1, args.repeats + 1):
report, dropped = measure(flags, env, args, pinned)
name = f"n{n}-{mode}-{attempt}"
for number, bad in enumerate(dropped, 1):
save(f"cost/disturbed/{name}-try{number}.json", bad)
rejected[(n, mode)] = rejected.get((n, mode), 0) + len(dropped)
(out / f"{name}.json").write_text(json.dumps(report, indent=2) + "\n")
results.setdefault((n, mode), []).append(report)
sys.stderr.write(f"{name}: {report['cost']['seconds']} s {report['screen']}\n")
finally:
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres")
with snapshot.open("a") as handle:
handle.write("\n" + host("after the series"))
summary = summarize(results, rejected)
save("cost/summary.json", {**plan, "results": summary})
(out / "summary.txt").write_text(
f"date: {stamp} commit: {plan['commit']} pins: {env}\n" + table(summary)
)
sys.stdout.write(table(summary))
def main():
p = argparse.ArgumentParser(description="Lab4 track 2: reconciler demo and cost of a pass")
sub = p.add_subparsers(dest="command", required=True)
d = sub.add_parser("demo", help="fault injection, dry-run/apply/repeat, safety, under load")
d.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
d.add_argument("--load-seconds", type=int, default=75)
d.add_argument("--load-threads", type=int, default=3)
k = sub.add_parser("cost", help="cost of a pass for N objects, repeated")
k.add_argument("--sizes", type=int, nargs="+", default=[1000, 10000, 30000])
k.add_argument("--repeats", type=int, default=3)
k.add_argument("--deep-max", type=int, default=30000)
k.add_argument("--page-compare", type=int, default=10000)
k.add_argument("--settle", type=float, default=10)
k.add_argument("--s3-memory", default="3g")
k.add_argument("--no-pin", action="store_true")
k.add_argument("--max-other", type=float, default=2.0, help="foreign CPU in the VM, cores")
k.add_argument("--max-pinned", type=float, default=0.25, help="foreign CPU on pinned cores")
k.add_argument("--start-other", type=float, default=1.5, help="container start, core-s")
k.add_argument("--start-pinned", type=float, default=0.5, help="the same on pinned cores")
k.add_argument("--retries", type=int, default=3, help="reruns of a disturbed run")
k.add_argument("--quiet-timeout", type=float, default=900.0)
args = p.parse_args()
if args.command == "demo":
demo(args)
else:
cost(args)
if __name__ == "__main__":
main()