forked from mtusi2026/mtusi2026lab2
824 lines
31 KiB
Python
824 lines
31 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import http.client
|
|
import json
|
|
import os
|
|
import re
|
|
import statistics
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import zlib
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
client = __import__("client")
|
|
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
GRACE = 60
|
|
FLAGS = ("--grace", str(GRACE), "--skew", "0")
|
|
INTENT_AGE = 300
|
|
LOCK_KEY = 20260004
|
|
FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale")
|
|
USER_HZ = 100
|
|
BUSY = (0, 1, 2, 5, 6, 7)
|
|
|
|
|
|
def run(cmd, env=None, stdin=None):
|
|
started = time.monotonic()
|
|
done = subprocess.run(
|
|
cmd,
|
|
cwd=ROOT,
|
|
input=stdin,
|
|
capture_output=True,
|
|
check=False,
|
|
env=dict(os.environ, **(env or {})),
|
|
)
|
|
out, err = done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace")
|
|
return done.returncode, out, err, round(time.monotonic() - started, 3)
|
|
|
|
|
|
def must(cmd, env=None, stdin=None):
|
|
code, out, err, _ = run(cmd, env, stdin)
|
|
if code:
|
|
raise SystemExit(f"failed: {' '.join(cmd[:7])}\n{err[-800:]}")
|
|
return out
|
|
|
|
|
|
def compose(*args, env=None, stdin=None):
|
|
return must(["docker", "compose", *args], env, stdin)
|
|
|
|
|
|
def psql(sql):
|
|
return compose(
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
"-qAt",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-c",
|
|
sql,
|
|
).strip()
|
|
|
|
|
|
def reconcile(*flags, env=None):
|
|
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", *flags]
|
|
code, out, err, wall = run(cmd, env)
|
|
if code not in (0, 1) or not out.strip():
|
|
raise SystemExit(f"reconciler failed ({code}):\n{err[-800:]}")
|
|
report = json.loads(out)
|
|
audit = [json.loads(line) for line in err.splitlines() if '"audit.' in line]
|
|
report.update(exit_code=code, wall_seconds=wall, audit=audit)
|
|
return report
|
|
|
|
|
|
def oracle(*flags):
|
|
code, out, err, _ = run([sys.executable, "scripts/reconcile_check.py", *flags])
|
|
if not out.strip():
|
|
raise SystemExit(f"reconcile_check failed ({code}):\n{err[-800:]}")
|
|
return json.loads(out)
|
|
|
|
|
|
def inject(kind, count, ids=(), age=None):
|
|
env = {"INJECT_IDS": ",".join(ids)} if ids else {}
|
|
if age is not None:
|
|
env["INJECT_AGE_SECONDS"] = str(age)
|
|
out = must(["sh", "scripts/reconcile-inject.sh", kind, str(count)], env)
|
|
return [line.split()[1] for line in out.splitlines() if line.strip()]
|
|
|
|
|
|
def head():
|
|
return must(["git", "rev-parse", "HEAD"]).strip()
|
|
|
|
|
|
def evidence_root():
|
|
return ROOT / os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
|
|
|
|
|
|
def save(name, data, *, stamp=True):
|
|
path = evidence_root() / name
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
stamped = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head(), **data}
|
|
path.write_text(json.dumps(stamped if stamp else data, ensure_ascii=False, indent=2) + "\n")
|
|
return path
|
|
|
|
|
|
class Checks:
|
|
def __init__(self):
|
|
self.items = []
|
|
|
|
def __call__(self, name, ok, **detail):
|
|
self.items.append(dict(name=name, ok=bool(ok), **detail))
|
|
sys.stderr.write(("PASS " if ok else "FAIL ") + name + "\n")
|
|
|
|
@property
|
|
def ok(self):
|
|
return all(item["ok"] for item in self.items)
|
|
|
|
|
|
def chunk(tag, data):
|
|
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data))
|
|
|
|
|
|
def tiny_png():
|
|
raw = b"".join(b"\x00" + os.urandom(24) for _ in range(8))
|
|
header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)
|
|
return (
|
|
b"\x89PNG\r\n\x1a\n"
|
|
+ chunk(b"IHDR", header)
|
|
+ chunk(b"IDAT", zlib.compress(raw))
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
|
|
|
|
def pick(found, names):
|
|
return {name: found.get(name, 0) for name in names}
|
|
|
|
|
|
def faults(c, check, pre):
|
|
alice, bob, _, chat = c.bootstrap()
|
|
ids = [c.attachment(alice, chat)[0]["id"] for _ in range(18)]
|
|
linked = ids[:6]
|
|
for part in (linked[:3], linked[3:]):
|
|
c.message(alice, chat, "", attachments=part)
|
|
targets = {
|
|
"missing": [linked[0], linked[1], ids[6]],
|
|
"corrupt": [linked[2], ids[7], ids[8]],
|
|
"bitflip": [linked[3], ids[9], ids[10]],
|
|
}
|
|
damaged = {i for group in targets.values() for i in group}
|
|
ready = evidence_root() / "ready-before.json"
|
|
oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready))
|
|
untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged}
|
|
save("ready-untouched.json", untouched, stamp=False)
|
|
started = time.monotonic()
|
|
injected = {
|
|
"orphan_stale": inject("orphan", 3),
|
|
"missing": inject("missing", 3, targets["missing"]),
|
|
"size_mismatch": inject("corrupt", 3, targets["corrupt"]),
|
|
"sha_mismatch": inject("bitflip", 3, targets["bitflip"]),
|
|
"pending_stale": inject("stale-pending", 3, age=INTENT_AGE)
|
|
+ inject("pending-object", 3, age=INTENT_AGE),
|
|
}
|
|
time.sleep(max(0.0, GRACE + 5 - (time.monotonic() - started)))
|
|
injected["orphan_fresh"] = inject("orphan", 2)
|
|
injected["pending_fresh"] = inject("stale-pending", 2, age=0)
|
|
expected = {name: len(keys) for name, keys in injected.items()}
|
|
expected.update(quarantined=pre["counts"]["quarantined"])
|
|
before = oracle(
|
|
"--grace", str(GRACE), "--deep", "--output", str(evidence_root() / "check-before.json")
|
|
)
|
|
dry = reconcile("--dry-run", "--deep", *FLAGS)
|
|
save("dry-run.json", dry)
|
|
after_dry = oracle("--grace", str(GRACE), "--deep")
|
|
named = {k: sorted(s.rsplit("/", 1)[-1] for s in dry["samples"].get(k, [])) for k in injected}
|
|
check(
|
|
"dry-run находит каждый внедрённый класс, и найдено ровно внедрённое",
|
|
pick(dry["found"], expected) == expected
|
|
and named == {name: sorted(ids) for name, ids in injected.items()},
|
|
expected=expected,
|
|
found=dry["found"],
|
|
)
|
|
check(
|
|
"dry-run ничего не меняет",
|
|
not any(dry["fixed"].values())
|
|
and after_dry["counts"] == before["counts"]
|
|
and (after_dry["objects"], after_dry["rows"]) == (before["objects"], before["rows"]),
|
|
objects=[before["objects"], after_dry["objects"]],
|
|
rows=[before["rows"], after_dry["rows"]],
|
|
)
|
|
first = reconcile("--apply", "--deep", *FLAGS)
|
|
save("apply-1.json", first)
|
|
discarded = sorted(e["attachment_id"] for e in first["audit"] if e["reason"] == "reconcile")
|
|
check(
|
|
"apply исправляет ровно найденное dry-run, удаление намерений — в журнале аудита",
|
|
pick(first["fixed"], FIXABLE) == pick(dry["found"], FIXABLE)
|
|
and not first["errors"]
|
|
and discarded == sorted(injected["pending_stale"]),
|
|
dry_run=pick(dry["found"], FIXABLE),
|
|
fixed=pick(first["fixed"], FIXABLE),
|
|
audit_discarded=len(discarded),
|
|
)
|
|
second = reconcile("--apply", "--deep", *FLAGS)
|
|
save("apply-2.json", second)
|
|
check(
|
|
"повторный apply — 0 действий",
|
|
not any(second["fixed"].values()),
|
|
fixed=second["fixed"],
|
|
found=second["found"],
|
|
)
|
|
after = oracle(
|
|
"--grace",
|
|
str(GRACE),
|
|
"--deep",
|
|
"--expect-ready",
|
|
str(evidence_root() / "ready-untouched.json"),
|
|
"--output",
|
|
str(evidence_root() / "check-after.json"),
|
|
)
|
|
tolerated = ("orphan_fresh", "pending_fresh")
|
|
check(
|
|
"независимая проверка после apply: нарушений нет, готовые не потеряны",
|
|
after["ok"]
|
|
and not after["counts"]["lost_ready"]
|
|
and pick(after["counts"], tolerated) == pick(before["counts"], tolerated)
|
|
and after["counts"]["quarantined"] == pre["counts"]["quarantined"] + 9,
|
|
counts=after["counts"],
|
|
)
|
|
marks = dict(
|
|
line.split("|") for line in psql("SELECT id, quarantine_reason FROM attachments").split()
|
|
)
|
|
links = psql("SELECT attachment_id FROM message_attachments").split()
|
|
kept = {i: marks.get(i) for i in sorted(damaged)}
|
|
check(
|
|
"строки с расхождениями и ссылки сообщений сохранены",
|
|
all(kept.values()) and sum(i in linked for i in links) == len(linked),
|
|
quarantine=kept,
|
|
message_links=sum(i in linked for i in links),
|
|
)
|
|
meta = c.call("GET", f"/api/v1/attachments/{linked[0]}", 200, actor=bob, schema="Attachment")
|
|
blocked = c.call("GET", f"/api/v1/attachments/{linked[2]}/content", 404, actor=bob)
|
|
intact = c.call("GET", f"/api/v1/attachments/{linked[4]}/content", 200, actor=bob)
|
|
check(
|
|
"карантин: метаданные 200, /content 404; нетронутое скачивается",
|
|
meta["status"] == 200 and blocked["status"] == 404 and intact["bytes"] == FIXTURE,
|
|
)
|
|
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
|
for attachment, data in ((linked[0], FIXTURE), (linked[1], os.urandom(len(FIXTURE)))):
|
|
compose(
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"-T",
|
|
"s3-cli",
|
|
"pipe",
|
|
f"root/{bucket}/attachments/{attachment}",
|
|
stdin=data,
|
|
)
|
|
healed = reconcile("--apply", *FLAGS)
|
|
save("healed.json", healed)
|
|
restored = c.call("GET", f"/api/v1/attachments/{linked[0]}/content", 200, actor=bob)
|
|
foreign = c.call("GET", f"/api/v1/attachments/{linked[1]}/content", 404, actor=bob)
|
|
check(
|
|
"вернули те же байты — карантин снят; другие байты той же длины — остался",
|
|
healed["fixed"]["healed"] == 1
|
|
and healed["samples"].get("healed") == [linked[0]]
|
|
and linked[1] in healed["samples"].get("quarantined", [])
|
|
and restored["bytes"] == FIXTURE
|
|
and foreign["status"] == 404,
|
|
fixed=healed["fixed"],
|
|
s3_get=healed["cost"]["requests"]["s3_get"],
|
|
)
|
|
return {"injected": injected, "targets": targets, "linked": linked}
|
|
|
|
|
|
def single(check):
|
|
holder = subprocess.Popen(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-c",
|
|
f"SELECT pg_advisory_lock({LOCK_KEY}), pg_sleep(15)",
|
|
],
|
|
cwd=ROOT,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
try:
|
|
time.sleep(3)
|
|
busy = reconcile("--apply", *FLAGS)
|
|
finally:
|
|
holder.wait()
|
|
save("lock-busy.json", busy)
|
|
check(
|
|
"второй одновременный проход пропущен",
|
|
busy["skipped"] and busy["exit_code"] == 0 and not any(busy["fixed"].values()),
|
|
)
|
|
|
|
|
|
def healthy(service, seconds=90):
|
|
container = compose("ps", "-q", service).strip()
|
|
probe = ["docker", "inspect", "-f", "{{.State.Health.Status}}", container]
|
|
deadline = time.monotonic() + seconds
|
|
while must(probe).strip() != "healthy":
|
|
if time.monotonic() > deadline:
|
|
raise SystemExit(f"{service} is not healthy after {seconds} s")
|
|
time.sleep(1)
|
|
|
|
|
|
def interrupt(check):
|
|
name = f"lab4-reconcile-sigterm-{os.getpid()}"
|
|
compose("pause", "s3")
|
|
try:
|
|
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "--name", name, "reconciler"]
|
|
runner = subprocess.Popen(
|
|
[*cmd, "--apply", *FLAGS],
|
|
cwd=ROOT,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
time.sleep(4)
|
|
state = must(["docker", "inspect", "-f", "{{.State.Status}} {{.HostConfig.Init}}", name])
|
|
started = time.monotonic()
|
|
must(["docker", "stop", name])
|
|
stopped = round(time.monotonic() - started, 2)
|
|
code = runner.wait(timeout=60)
|
|
finally:
|
|
compose("unpause", "s3")
|
|
healthy("s3")
|
|
after = reconcile("--dry-run", *FLAGS)
|
|
status, init = state.split()
|
|
report = {
|
|
"s3": "paused",
|
|
"state_before_stop": status,
|
|
"init": init,
|
|
"stop_seconds": stopped,
|
|
"exit_code": code,
|
|
"next_pass_skipped": after["skipped"],
|
|
}
|
|
save("sigterm.json", report)
|
|
check(
|
|
"SIGTERM: идущий проход останавливается за секунды, блокировка освобождена",
|
|
status == "running" and code == 143 and stopped < 5 and not after["skipped"],
|
|
**report,
|
|
)
|
|
|
|
|
|
def outage(check):
|
|
before = oracle("--grace", str(GRACE))
|
|
compose("stop", "s3")
|
|
try:
|
|
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", "--apply"]
|
|
code, out, err, wall = run([*cmd, *FLAGS])
|
|
finally:
|
|
compose("up", "-d", "--wait", "--no-deps", "s3")
|
|
after = oracle("--grace", str(GRACE))
|
|
log = [line for line in err.splitlines() if "reconcile." in line]
|
|
same = all(after[k] == before[k] for k in ("objects", "rows", "counts"))
|
|
save("s3-down.json", {"exit_code": code, "wall_seconds": wall, "stdout": out, "log": log})
|
|
check(
|
|
"S3 остановлен: код 1 за ограниченное время, состояние не изменилось",
|
|
code == 1 and wall < 30 and not out.strip() and same,
|
|
exit_code=code,
|
|
wall_seconds=wall,
|
|
)
|
|
|
|
|
|
def uploader(base, token, chat, *, deadline, results, lock):
|
|
host = urlsplit(base).netloc
|
|
conn = http.client.HTTPConnection(host, timeout=30)
|
|
headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
|
|
sent = 0
|
|
while time.monotonic() < deadline:
|
|
path = f"/api/v1/chats/{chat}/attachments?filename=load.png"
|
|
try:
|
|
conn.request("POST", path, tiny_png(), {**headers, "Content-Type": "image/png"})
|
|
response = conn.getresponse()
|
|
status, data = response.status, response.read()
|
|
if status == 201 and sent % 10 == 0:
|
|
body = json.dumps({"text": "", "attachment_ids": [json.loads(data)["id"]]})
|
|
extra = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}
|
|
conn.request("POST", f"/api/v1/chats/{chat}/messages", body, {**headers, **extra})
|
|
linked = conn.getresponse()
|
|
linked.read()
|
|
status = 201 if linked.status == 201 else -linked.status
|
|
except (OSError, http.client.HTTPException):
|
|
conn.close()
|
|
conn = http.client.HTTPConnection(host, timeout=30)
|
|
status, data = 0, b""
|
|
with lock:
|
|
results.append((status, json.loads(data) if status == 201 else None))
|
|
sent += 1
|
|
time.sleep(0.05)
|
|
|
|
|
|
def under_load(c, check, base, seconds, threads):
|
|
alice, _, _, chat = c.bootstrap()
|
|
ready = evidence_root() / "ready-load.json"
|
|
left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"]
|
|
leftover = {
|
|
"orphan_stale": left["orphan_fresh"] + left["orphan_stale"],
|
|
"pending_stale": left["pending_fresh"]
|
|
+ left["pending_stale_with_object"]
|
|
+ left["pending_stale_without_object"],
|
|
}
|
|
orphans = inject("orphan", 3)
|
|
intents = inject("stale-pending", 3, age=INTENT_AGE)
|
|
results, lock = [], threading.Lock()
|
|
deadline = time.monotonic() + seconds
|
|
workers = [
|
|
threading.Thread(
|
|
target=uploader,
|
|
args=(base, alice["token"], chat["id"]),
|
|
kwargs={"deadline": deadline, "results": results, "lock": lock},
|
|
)
|
|
for _ in range(threads)
|
|
]
|
|
for worker in workers:
|
|
worker.start()
|
|
runs = []
|
|
while time.monotonic() < deadline:
|
|
runs.append(reconcile("--apply", *FLAGS))
|
|
for worker in workers:
|
|
worker.join()
|
|
runs.append(reconcile("--apply", *FLAGS))
|
|
created = {att["id"]: att["sha256"] for status, att in results if att}
|
|
expected = dict(json.loads(ready.read_text()), **created)
|
|
save("ready-load.json", expected, stamp=False)
|
|
after = oracle(
|
|
"--grace",
|
|
str(GRACE),
|
|
"--deep",
|
|
"--expect-ready",
|
|
str(ready),
|
|
"--output",
|
|
str(evidence_root() / "check-under-load.json"),
|
|
)
|
|
statuses = {}
|
|
for status, _ in results:
|
|
statuses[str(status)] = statuses.get(str(status), 0) + 1
|
|
fixed = {name: sum(r["fixed"][name] for r in runs) for name in FIXABLE}
|
|
rechecked = {}
|
|
for r in runs:
|
|
for name, count in r["rechecked"].items():
|
|
rechecked[name] = rechecked.get(name, 0) + count
|
|
seen = {name: sum(r["found"][name] for r in runs) for name in ("orphan_fresh", "pending_fresh")}
|
|
summary = {
|
|
"seconds": seconds,
|
|
"upload_threads": threads,
|
|
"uploads": statuses,
|
|
"created": len(created),
|
|
"linked_every": 10,
|
|
"reconciler_runs": len(runs),
|
|
"run_seconds": [r["cost"]["seconds"] for r in runs],
|
|
"fixed_total": fixed,
|
|
"fresh_seen_total": seen,
|
|
"rechecked_total": rechecked,
|
|
"errors": sum(r["errors"] for r in runs),
|
|
"skipped": sum(1 for r in runs if r["skipped"]),
|
|
"injected_before_load": {"orphan": len(orphans), "pending_stale": len(intents)},
|
|
"left_from_faults": leftover,
|
|
"check_after": {k: after[k] for k in ("objects", "rows", "ready_healthy", "counts", "ok")},
|
|
}
|
|
save("under-load.json", summary)
|
|
check(
|
|
"под нагрузкой: исправлены только внедрённые, ошибок нет",
|
|
fixed
|
|
== {
|
|
**dict.fromkeys(FIXABLE, 0),
|
|
"orphan_stale": leftover["orphan_stale"] + len(orphans),
|
|
"pending_stale": leftover["pending_stale"] + len(intents),
|
|
}
|
|
and not summary["errors"]
|
|
and not summary["skipped"],
|
|
fixed=fixed,
|
|
runs=len(runs),
|
|
)
|
|
check(
|
|
"под нагрузкой: все 201 готовы и совпадают по SHA-256, потерь 0",
|
|
after["ok"] and not after["counts"]["lost_ready"] and set(statuses) == {"201"},
|
|
uploads=statuses,
|
|
lost_ready=after["counts"]["lost_ready"],
|
|
)
|
|
|
|
|
|
def demo(args):
|
|
evidence_root().mkdir(parents=True, exist_ok=False)
|
|
c = client.Client(args.base_url, timeout=30)
|
|
check = Checks()
|
|
pre = oracle("--grace", str(GRACE))
|
|
dirty = {k: v for k, v in pre["counts"].items() if v and k != "quarantined"}
|
|
if dirty:
|
|
raise SystemExit(f"стенд не в исходном состоянии {dirty}: make reconcile ARGS=--apply")
|
|
started = time.monotonic()
|
|
result = {"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head()}
|
|
result["faults"] = faults(c, check, pre)
|
|
single(check)
|
|
interrupt(check)
|
|
outage(check)
|
|
under_load(c, check, args.base_url, args.load_seconds, args.load_threads)
|
|
result.update(seconds=round(time.monotonic() - started, 1), checks=check.items, ok=check.ok)
|
|
save("demo.json", result)
|
|
print(json.dumps({"ok": check.ok, "checks": len(check.items)}, ensure_ascii=False))
|
|
sys.exit(0 if check.ok else 1)
|
|
|
|
|
|
def pins(args):
|
|
cpus = int(must(["docker", "info", "--format", "{{.NCPU}}"]).strip())
|
|
sized = {"S3_CPUS": "2", "S3_MEMORY": args.s3_memory}
|
|
if args.no_pin or cpus < 8:
|
|
return sized
|
|
return dict(
|
|
sized,
|
|
S3_CPUSET=f"{cpus - 5}-{cpus - 4}",
|
|
PG_CPUSET=f"{cpus - 3}-{cpus - 2}",
|
|
RECONCILE_CPUSET=str(cpus - 1),
|
|
)
|
|
|
|
|
|
def cpu_set(env):
|
|
cpus = set()
|
|
for name in ("S3_CPUSET", "PG_CPUSET", "RECONCILE_CPUSET"):
|
|
for part in filter(None, env.get(name, "").split(",")):
|
|
low, _, high = part.partition("-")
|
|
cpus.update(range(int(low), int(high or low) + 1))
|
|
return cpus
|
|
|
|
|
|
def limits():
|
|
ids = compose("ps", "-q", "s3", "postgres").split()
|
|
fmt = (
|
|
"{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}"
|
|
" cpuset={{.HostConfig.CpusetCpus}}"
|
|
)
|
|
return must(["docker", "inspect", "-f", fmt, *ids]).strip().splitlines()
|
|
|
|
|
|
def host(title):
|
|
stats = must(
|
|
[
|
|
"docker",
|
|
"stats",
|
|
"--no-stream",
|
|
"--all",
|
|
"--format",
|
|
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}",
|
|
]
|
|
)
|
|
running = must(["docker", "ps", "--format", "{{.Names}} {{.Image}} {{.Status}}"])
|
|
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
load = " ".join(f"{v:.2f}" for v in os.getloadavg())
|
|
return (
|
|
f"== {title} date: {stamp} commit: {head()} host loadavg: {load}\n"
|
|
f"$ docker stats --no-stream --all\n{stats}\n$ docker ps\n{running}\n"
|
|
)
|
|
|
|
|
|
def usage(text):
|
|
return int(re.search(r"^usage_usec (\d+)$", text, re.M).group(1)) / 1e6
|
|
|
|
|
|
def sample():
|
|
vm = compose(
|
|
"exec", "-T", "postgres", "cat", "/proc/uptime", "/proc/stat", "/sys/fs/cgroup/cpu.stat"
|
|
)
|
|
s3 = compose("exec", "-T", "s3", "cat", "/sys/fs/cgroup/cpu.stat")
|
|
busy = {}
|
|
for line in vm.splitlines()[1:]:
|
|
name, *values = line.split() or [""]
|
|
if name.startswith("cpu") and name[3:].isdigit():
|
|
busy[int(name[3:])] = sum(int(values[i]) for i in BUSY) / USER_HZ
|
|
return float(vm.split()[0]), busy, usage(vm) + usage(s3)
|
|
|
|
|
|
def screen(first, last, own, pinned, args, *, started=True):
|
|
window = max(last[0] - first[0], 0.01)
|
|
busy = {cpu: spent - first[1].get(cpu, 0.0) for cpu, spent in last[1].items()}
|
|
ours = last[2] - first[2] + own
|
|
other = sum(busy.values()) - ours - args.start_other * started
|
|
on_pinned = sum(busy.get(cpu, 0.0) for cpu in pinned) - ours - args.start_pinned * started
|
|
other, on_pinned = max(0.0, other / window), max(0.0, on_pinned / window) if pinned else 0.0
|
|
return {
|
|
"window_seconds": round(window, 2),
|
|
"ours_cores": round(ours / window, 2),
|
|
"other_cores": round(other, 2),
|
|
"other_on_pinned_cores": round(on_pinned, 2),
|
|
"host_loadavg_1m": round(os.getloadavg()[0], 2),
|
|
"disturbed": other > args.max_other or on_pinned > args.max_pinned,
|
|
}
|
|
|
|
|
|
def wait_quiet(args, pinned):
|
|
deadline, calm = time.monotonic() + args.quiet_timeout, 0
|
|
while calm < 3 and time.monotonic() < deadline:
|
|
first = sample()
|
|
time.sleep(5)
|
|
quiet = screen(first, sample(), 0.0, pinned, args, started=False)
|
|
calm = 0 if quiet["disturbed"] else calm + 1
|
|
sys.stderr.write(f"waiting for quiet: {json.dumps(quiet)}\n")
|
|
return calm == 3
|
|
|
|
|
|
def measure(flags, env, args, pinned):
|
|
rejected = []
|
|
for attempt in range(args.retries + 1):
|
|
first = sample()
|
|
report = reconcile(*flags, "--samples", "3", env=env)
|
|
report["screen"] = screen(first, sample(), report["cost"]["cpu_seconds"], pinned, args)
|
|
if not report["screen"]["disturbed"]:
|
|
return report, rejected
|
|
rejected.append(report)
|
|
if attempt < args.retries and not wait_quiet(args, pinned):
|
|
break
|
|
return None, rejected
|
|
|
|
|
|
def seed(count, env):
|
|
if count <= 0:
|
|
return {"added": 0}
|
|
env = dict(env, API_CPUS="4", API_MEMORY="512m")
|
|
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
|
|
out = compose(
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"-T",
|
|
"-v",
|
|
tools,
|
|
"--entrypoint",
|
|
"python",
|
|
"api",
|
|
"/tools/seed_storage.py",
|
|
"--count",
|
|
str(count),
|
|
"--concurrency",
|
|
"4",
|
|
env=env,
|
|
)
|
|
records = [json.loads(line) for line in out.strip().splitlines()]
|
|
return {**records[-1], "progress": records[:-1]}
|
|
|
|
|
|
def median(runs, path):
|
|
values = []
|
|
for r in runs:
|
|
value = r
|
|
for key in path:
|
|
value = value[key]
|
|
values.append(value)
|
|
return statistics.median(values)
|
|
|
|
|
|
def summarize(results, rejected):
|
|
rows = []
|
|
for (n, mode), runs in sorted(results.items()):
|
|
seconds = median(runs, ("cost", "seconds"))
|
|
rows.append(
|
|
{
|
|
"objects": n,
|
|
"mode": mode,
|
|
"runs": len(runs),
|
|
"seconds_median": seconds,
|
|
"seconds_all": [r["cost"]["seconds"] for r in runs],
|
|
"wall_seconds_median": median(runs, ("wall_seconds",)),
|
|
"us_per_object": round(seconds / n * 1e6, 1),
|
|
"cpu_seconds_median": median(runs, ("cost", "cpu_seconds")),
|
|
"s3_list_wait_median": median(runs, ("cost", "waited_seconds", "s3_list")),
|
|
"sql_wait_median": median(runs, ("cost", "waited_seconds", "sql")),
|
|
"requests": runs[0]["cost"]["requests"],
|
|
"max_rss_mib_median": median(runs, ("cost", "max_rss_mib")),
|
|
"other_cores_max": max(r["screen"]["other_cores"] for r in runs),
|
|
"other_on_pinned_cores_max": max(
|
|
r["screen"]["other_on_pinned_cores"] for r in runs
|
|
),
|
|
"disturbed_kept": sum(r["screen"]["disturbed"] for r in runs),
|
|
"rejected": rejected.get((n, mode), 0),
|
|
"scanned": runs[0]["scanned"],
|
|
"found_nonzero": {k: v for k, v in runs[0]["found"].items() if v},
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def table(rows):
|
|
head = (
|
|
"objects mode runs t_med,s wall,s us/obj cpu,s list,s sql,s lists sqls"
|
|
" gets rss,MiB other pinned rejected"
|
|
)
|
|
lines = [head]
|
|
for r in rows:
|
|
q = r["requests"]
|
|
lines.append(
|
|
f"{r['objects']:>7} {r['mode']:<9} {r['runs']:>4} {r['seconds_median']:>7.3f} "
|
|
f"{r['wall_seconds_median']:>6.2f} {r['us_per_object']:>6.1f} "
|
|
f"{r['cpu_seconds_median']:>5.2f} {r['s3_list_wait_median']:>6.3f} "
|
|
f"{r['sql_wait_median']:>5.3f} {q['s3_list']:>5} {q['sql']:>4} {q['s3_get']:>5} "
|
|
f"{r['max_rss_mib_median']:>7.1f} {r['other_cores_max']:>5.2f} "
|
|
f"{r['other_on_pinned_cores_max']:>6.2f} {r['rejected']:>8}"
|
|
)
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def cost(args):
|
|
out = evidence_root() / "cost"
|
|
out.mkdir(parents=True, exist_ok=False)
|
|
(out / "disturbed").mkdir()
|
|
rows = int(psql("SELECT count(*) FROM attachments"))
|
|
if rows > args.sizes[0]:
|
|
raise SystemExit(f"на стенде уже {rows} вложений: используйте новый отдельный project")
|
|
snapshot = out / "host-docker-stats.txt"
|
|
snapshot.write_text(host("before the series"))
|
|
env = pins(args)
|
|
pinned = cpu_set(env)
|
|
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres", env=env)
|
|
stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
plan = {"date": stamp, "commit": head(), "pins": env, "limits": limits(), "seeds": []}
|
|
plan["screen"] = {
|
|
"max_other_cores": args.max_other,
|
|
"max_other_on_pinned_cores": args.max_pinned,
|
|
"start_allowance_core_seconds": {"vm": args.start_other, "pinned": args.start_pinned},
|
|
"retries": args.retries,
|
|
}
|
|
results, rejected = {}, {}
|
|
try:
|
|
for n in args.sizes:
|
|
plan["seeds"].append(dict(seed(n - rows, env), target=n))
|
|
rows = n
|
|
time.sleep(args.settle)
|
|
save("cost/plan.json", plan)
|
|
if not wait_quiet(args, pinned):
|
|
raise SystemExit("no quiet measurement window before timeout")
|
|
modes = [("shallow", ["--dry-run"])]
|
|
if n <= args.deep_max:
|
|
modes.append(("deep", ["--dry-run", "--deep"]))
|
|
if n == args.page_compare:
|
|
modes.append(("page100", ["--dry-run", "--page-size", "100"]))
|
|
for mode, flags in modes:
|
|
for attempt in range(1, args.repeats + 1):
|
|
report, dropped = measure(flags, env, args, pinned)
|
|
name = f"n{n}-{mode}-{attempt}"
|
|
for number, bad in enumerate(dropped, 1):
|
|
save(f"cost/disturbed/{name}-try{number}.json", bad)
|
|
rejected[(n, mode)] = rejected.get((n, mode), 0) + len(dropped)
|
|
if report is None:
|
|
raise SystemExit(f"{name}: no clean run after retries; rejected preserved")
|
|
if (
|
|
report["exit_code"]
|
|
or report["errors"]
|
|
or report["skipped"]
|
|
or report["scanned"] != {"objects": n, "rows": n}
|
|
or any(report["found"].values())
|
|
):
|
|
save(f"cost/invalid/{name}.json", report)
|
|
raise SystemExit(f"{name}: inconsistent dataset or failed pass")
|
|
save(f"cost/{name}.json", report)
|
|
results.setdefault((n, mode), []).append(report)
|
|
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
|
|
sys.stderr.write(f"{name}: {report['cost']['seconds']} s {report['screen']}\n")
|
|
finally:
|
|
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
|
|
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres")
|
|
with snapshot.open("a") as handle:
|
|
handle.write("\n" + host("after the series"))
|
|
summary = summarize(results, rejected)
|
|
save("cost/summary.json", {**plan, "results": summary})
|
|
(out / "summary.txt").write_text(
|
|
f"date: {stamp} commit: {plan['commit']} pins: {env}\n" + table(summary)
|
|
)
|
|
sys.stdout.write(table(summary))
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description="Lab4 track 2: reconciler demo and cost of a pass")
|
|
sub = p.add_subparsers(dest="command", required=True)
|
|
d = sub.add_parser("demo", help="fault injection, dry-run/apply/repeat, safety, under load")
|
|
d.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
|
|
d.add_argument("--load-seconds", type=int, default=75)
|
|
d.add_argument("--load-threads", type=int, default=3)
|
|
k = sub.add_parser("cost", help="cost of a pass for N objects, repeated")
|
|
k.add_argument("--sizes", type=int, nargs="+", default=[1000, 10000, 30000, 100000])
|
|
k.add_argument("--repeats", type=int, default=3)
|
|
k.add_argument("--deep-max", type=int, default=100000)
|
|
k.add_argument("--page-compare", type=int, default=10000)
|
|
k.add_argument("--settle", type=float, default=10)
|
|
k.add_argument("--s3-memory", default="3g")
|
|
k.add_argument("--no-pin", action="store_true")
|
|
k.add_argument("--max-other", type=float, default=2.0, help="foreign CPU in the VM, cores")
|
|
k.add_argument("--max-pinned", type=float, default=0.25, help="foreign CPU on pinned cores")
|
|
k.add_argument("--start-other", type=float, default=1.5, help="container start, core-s")
|
|
k.add_argument("--start-pinned", type=float, default=0.5, help="the same on pinned cores")
|
|
k.add_argument("--retries", type=int, default=3, help="reruns of a disturbed run")
|
|
k.add_argument("--quiet-timeout", type=float, default=900.0)
|
|
for command in (d, k):
|
|
command.add_argument(
|
|
"--out", default=os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
|
|
)
|
|
args = p.parse_args()
|
|
os.environ["RECONCILE_EVIDENCE_DIR"] = args.out
|
|
if args.command == "demo":
|
|
demo(args)
|
|
else:
|
|
cost(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|