forked from mtusi2026/mtusi2026lab4
lab4: память загрузок и S3 под лимитом, документация защиты
- тело загрузки: один буфер ровно Content-Length, без копий до S3 и djpeg; - PNG: zlib получает вход кусками по 64 KiB (unconsumed_tail копировал остаток IDAT); - djpeg, убитый OOM-killer'ом, -> 503 (ресурсы сервера), а не 415; - api: memswap_limit = memory (жёсткий лимит без swap); S3: GOMEMLIMIT и лимиты одновременных загрузок SeaweedFS, память 1g; - docs: CONSISTENCY.md, DEFENSE.md, WINDOWS.md, DATA_MODEL.md под lab4; scripts/demo_attachments.py для сценария защиты; CI и collect_evidence под lab4; - bench/reconcile_faults.py: отчёты трека 2 (ошибки с обеих сторон, прерванный apply). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a2d867bec6
commit
4ea6924db1
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 2 evidence: inject faults on both sides, run the reconciler, keep every JSON report.
|
||||
|
||||
Scenario A (all fault kinds; the same as student-tests/test_reconcile.py, but the reports are saved):
|
||||
S3 side: old orphan objects, object deleted under a USED ready row, object truncated under an
|
||||
unused ready row, same-size corruption (found only with --verify-checksums);
|
||||
DB side: stale intents (with / without object), quarantine mark on a healthy row;
|
||||
fresh: orphan + intent younger than --grace.
|
||||
Runs: dry-run (+ snapshot of both stores before/after) -> apply -> apply again -> restore of the
|
||||
lost object from a "backup" -> apply (quarantine lifted).
|
||||
Scenario B (interrupted apply): 400 old orphans, S3 is paused while --apply deletes them ->
|
||||
the run stops with an error, nothing else is damaged; the next run finishes the rest; the
|
||||
third run finds nothing.
|
||||
|
||||
python bench/reconcile_faults.py --output evidence/reconcile
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
import imagegen # noqa: E402
|
||||
from conftest import (Api, attachment_row, compose, findings_by_key, psql, s3_delete, s3_exec, s3_keys, # noqa: E402
|
||||
s3_put)
|
||||
from test_reconcile import insert_intent, snapshot # noqa: E402
|
||||
|
||||
GRACE = ["--grace", "30", "--allow-small-grace"]
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
|
||||
|
||||
def reconcile(*args: str) -> tuple[int, dict]:
|
||||
r = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "1000", *args, check=False, timeout=600)
|
||||
return r.returncode, json.loads(r.stdout)
|
||||
|
||||
|
||||
def save(out: Path, name: str, data) -> None:
|
||||
(out / name).write_text(json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
|
||||
|
||||
def ours(report: dict, keys: dict[str, str]) -> dict:
|
||||
"""The findings of this scenario's keys only (the stand also holds other test data)."""
|
||||
f = findings_by_key(report)
|
||||
return {label: ({"kind": f[k]["kind"], "result": f[k]["result"], "used": f[k]["used"]} if k in f else None)
|
||||
for label, k in keys.items()}
|
||||
|
||||
|
||||
def scenario_a(out: Path) -> dict:
|
||||
api = Api()
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
chat = api.chat(alice, bob)
|
||||
up = {n: api.upload(alice, chat, imagegen.png(40 + i, 30)).json() for i, n in enumerate("ABCDEF")}
|
||||
data = {n: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content
|
||||
for n, a in up.items()}
|
||||
key = {n: attachment_row(a["id"])["object_key"] for n, a in up.items()}
|
||||
api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]})
|
||||
keys = {"healthy used A": key["A"], "healthy unused B": key["B"], "used, object deleted C": key["C"],
|
||||
"unused, object truncated D": key["D"], "quarantined but healthy E": key["E"],
|
||||
"same-size corruption F": key["F"]}
|
||||
orphan = "o/" + uuid.uuid4().hex
|
||||
stale_with, stale_without = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
|
||||
s3_put(orphan, SAMPLE)
|
||||
s3_put(stale_with, SAMPLE)
|
||||
insert_intent(chat, alice, stale_with, SAMPLE, 600)
|
||||
insert_intent(chat, alice, stale_without, SAMPLE, 600)
|
||||
s3_delete(key["C"])
|
||||
s3_put(key["D"], data["D"][: len(data["D"]) // 2])
|
||||
s3_put(key["F"], data["F"][:200] + bytes([data["F"][200] ^ 1]) + data["F"][201:])
|
||||
psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' WHERE id = '{up['E']['id']}'")
|
||||
keys.update({"old orphan object": orphan, "stale intent + object": stale_with, "stale intent, no object": stale_without})
|
||||
time.sleep(32)
|
||||
fresh_orphan, fresh_intent = "o/" + uuid.uuid4().hex, "o/" + uuid.uuid4().hex
|
||||
s3_put(fresh_orphan, SAMPLE)
|
||||
s3_put(fresh_intent, SAMPLE)
|
||||
insert_intent(chat, alice, fresh_intent, SAMPLE, 0)
|
||||
keys.update({"fresh orphan (in-flight)": fresh_orphan, "fresh intent (in-flight)": fresh_intent})
|
||||
|
||||
before = snapshot()
|
||||
_, dry = reconcile("--verify-checksums", *GRACE)
|
||||
after = snapshot()
|
||||
_, apply1 = reconcile("--apply", "--verify-checksums", *GRACE)
|
||||
_, apply2 = reconcile("--apply", "--verify-checksums", *GRACE)
|
||||
s3_put(key["C"], data["C"]) # the lost object restored from a backup
|
||||
_, apply3 = reconcile("--apply", "--verify-checksums", *GRACE)
|
||||
for name, report in (("faults-1-dry-run.json", dry), ("faults-2-apply.json", apply1),
|
||||
("faults-3-apply-again.json", apply2), ("faults-4-after-restore.json", apply3)):
|
||||
save(out, name, report)
|
||||
rows = {label: attachment_row(up[n]["id"]) for label, n in (("C", "C"), ("D", "D"), ("E", "E"), ("F", "F"), ("A", "A"), ("B", "B"))}
|
||||
content = {n: api.client.get(f"/api/v1/attachments/{up[n]['id']}/content",
|
||||
headers={"Authorization": "Bearer " + bob["token"]}).status_code for n in "ABCDEF"}
|
||||
present = s3_keys()
|
||||
summary = {
|
||||
"dry_run_changed_nothing": before == after,
|
||||
"per_key": {"dry-run": ours(dry, keys), "apply": ours(apply1, keys), "apply again": ours(apply2, keys),
|
||||
"after restore of C": ours(apply3, keys)},
|
||||
"objects_present_after": {label: k in present for label, k in keys.items()},
|
||||
"rows_after": {k: (v and {"status": v["status"], "lost_reason": v["lost_reason"]}) for k, v in rows.items()},
|
||||
"content_status_after": content,
|
||||
"durations_s": [dry["duration_seconds"], apply1["duration_seconds"], apply2["duration_seconds"],
|
||||
apply3["duration_seconds"]],
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
|
||||
return summary
|
||||
|
||||
|
||||
def scenario_b(out: Path, count: int = 400) -> dict:
|
||||
keys = ["o/" + uuid.uuid4().hex for _ in range(count)]
|
||||
s3_exec("ks = sys.stdin.read().split()\n"
|
||||
"sem = asyncio.Semaphore(32)\n"
|
||||
"async def one(k):\n"
|
||||
" async with sem:\n"
|
||||
" await s.put(k, b'orphan', 'image/png', {})\n"
|
||||
"await asyncio.gather(*(one(k) for k in ks))", stdin="\n".join(keys))
|
||||
time.sleep(32)
|
||||
result: dict = {}
|
||||
|
||||
def run_apply() -> None:
|
||||
result["code"], result["report"] = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
|
||||
|
||||
t = threading.Thread(target=run_apply)
|
||||
t.start()
|
||||
# pause S3 as soon as deletions are under way (the reconciler is planning/listing first)
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
left = len(s3_keys() & set(keys))
|
||||
if left < count:
|
||||
break
|
||||
time.sleep(0.2)
|
||||
subprocess.run(["docker", "compose", "pause", "s3"], cwd=ROOT, check=True, capture_output=True)
|
||||
t.join()
|
||||
subprocess.run(["docker", "compose", "unpause", "s3"], cwd=ROOT, check=True, capture_output=True)
|
||||
time.sleep(2)
|
||||
left_after_interrupt = len(s3_keys() & set(keys))
|
||||
code2, second = reconcile("--apply", "--max-orphan-fraction", "1", *GRACE)
|
||||
code3, third = reconcile("--apply", *GRACE)
|
||||
save(out, "interrupted-1-apply.json", result["report"])
|
||||
save(out, "interrupted-2-rerun.json", second)
|
||||
save(out, "interrupted-3-third.json", third)
|
||||
summary = {
|
||||
"orphans_injected": count,
|
||||
"interrupted_run": {"exit_code": result["code"], "error": result["report"].get("error"),
|
||||
"applied": result["report"]["actions"]["applied"]},
|
||||
"orphans_left_after_interrupt": left_after_interrupt,
|
||||
"rerun": {"exit_code": code2, "applied": second["actions"]["applied"], "error": second["error"]},
|
||||
"third": {"exit_code": code3, "planned": third["actions"]["planned"]},
|
||||
"orphans_left_at_end": len(s3_keys() & set(keys)),
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=1), flush=True)
|
||||
return summary
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--output", default="evidence/reconcile")
|
||||
args = p.parse_args()
|
||||
out = ROOT / args.output
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
a = scenario_a(out)
|
||||
b = scenario_b(out)
|
||||
commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, text=True).stdout.strip()
|
||||
save(out, "faults-summary.json", {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "commit": commit,
|
||||
"grace_seconds": 30, "scenario_a": a, "scenario_b_interrupted": b})
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+36
-7
@@ -57,6 +57,14 @@ def cg(cid: str, name: str) -> int:
|
||||
return int(sh("docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}").split()[0])
|
||||
|
||||
|
||||
def memory_stat(cid: str, key: str) -> int:
|
||||
for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat").splitlines():
|
||||
name, value = line.split()
|
||||
if name == key:
|
||||
return int(value)
|
||||
return 0
|
||||
|
||||
|
||||
def vmhwm(cid: str) -> int:
|
||||
for line in sh("docker", "exec", cid, "cat", "/proc/1/status").splitlines():
|
||||
if line.startswith("VmHWM:"):
|
||||
@@ -64,6 +72,16 @@ def vmhwm(cid: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def container_events() -> dict[str, str]:
|
||||
"""restart count + OOM-kill events per container (cgroup memory.events oom_kill counter)."""
|
||||
out = {}
|
||||
for cid in sh("docker", "compose", "ps", "-q").split():
|
||||
name, restarts = sh("docker", "inspect", "-f", "{{.Name}} {{.RestartCount}}", cid).split()
|
||||
events = dict(line.split() for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.events").splitlines())
|
||||
out[name.lstrip("/")] = f"restarts={restarts} oom_kill={events.get('oom_kill', '?')}"
|
||||
return out
|
||||
|
||||
|
||||
def recreate(env: dict) -> None:
|
||||
sh("docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api", env=env, timeout=300)
|
||||
sh(sys.executable, "scripts/lbctl.py", "wait-up", str(len(replicas())), "--timeout", "90") # LB sees them UP
|
||||
@@ -129,7 +147,9 @@ def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict:
|
||||
results[i] = (r.status_code, time.perf_counter() - started, r.headers.get("x-instance-id", "-"))
|
||||
c.close()
|
||||
|
||||
samplers = [Sampler(cid) for cid in ids]
|
||||
s3_id = sh("docker", "compose", "ps", "-q", "s3").strip()
|
||||
restarts_before = container_events()
|
||||
samplers = [Sampler(cid) for cid in ids] + [Sampler(s3_id)]
|
||||
for s in samplers:
|
||||
s.start()
|
||||
time.sleep(0.5)
|
||||
@@ -144,7 +164,7 @@ def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict:
|
||||
for s in samplers:
|
||||
s.stop()
|
||||
per_replica = {}
|
||||
for cid, s in zip(ids, samplers):
|
||||
for cid, s in zip(ids, samplers[:-1]):
|
||||
hostname = sh("docker", "inspect", "-f", "{{.Config.Hostname}}", cid).strip()
|
||||
per_replica[hostname] = {
|
||||
"idle_mib": round(idle[cid] / MiB, 1),
|
||||
@@ -162,6 +182,11 @@ def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict:
|
||||
codes[str(r[0])] = codes.get(str(r[0]), 0) + 1
|
||||
summary = {
|
||||
"scenario": name, "settings": env, "clients": len(files),
|
||||
"s3_sampled_max_mib": round(max(samplers[-1].values or [0]) / MiB, 1),
|
||||
"s3_anon_after_mib": round(memory_stat(s3_id, "anon") / MiB, 1),
|
||||
"s3_limit_mib": round(cg(s3_id, "memory.max") / MiB, 1),
|
||||
# OOM kills / restarts DURING the scenario (any container of the stand): must be empty
|
||||
"incidents": {k: v for k, v in container_events().items() if v != restarts_before.get(k)},
|
||||
"file_mib_each": round(statistics.mean(len(d) for d, _ in files) / MiB, 2),
|
||||
"types": sorted({t for _, t in files}), "status_codes": codes, "wall_seconds": round(wall, 2),
|
||||
"latency_s": {"p50": round(lat[len(lat) // 2], 2), "max": round(lat[-1], 2)} if lat else None,
|
||||
@@ -179,16 +204,20 @@ def main() -> int:
|
||||
out = ROOT / args.output
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
print("generating files ...", flush=True)
|
||||
pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(args.clients)]
|
||||
pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(2 * args.clients)]
|
||||
jpegs = [(big_jpeg(8000, 5000, i), "image/jpeg") for i in range(args.clients)]
|
||||
base = {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}
|
||||
n, n2 = args.clients, 2 * args.clients
|
||||
scenarios = [
|
||||
("png-9.5MiB x16, slots=4 (default)", base, pngs),
|
||||
("png-9.5MiB x16, slots=16 (no bound)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs),
|
||||
("jpeg-40Mpx-progressive x16, slots=4, verify=1 (default)", base, jpegs),
|
||||
("jpeg-40Mpx-progressive x16, slots=4, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs),
|
||||
(f"png-9.4MiB x{n}, slots=4 (default)", base, pngs[:n]),
|
||||
(f"png-9.4MiB x{n2}, slots=4 (default)", base, pngs),
|
||||
(f"png-9.4MiB x{n2}, slots=16 (bound raised)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs),
|
||||
(f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=1 (default)", base, jpegs),
|
||||
(f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs),
|
||||
]
|
||||
results = []
|
||||
sh("docker", "compose", "restart", "s3") # S3 memory counters start from an idle process
|
||||
sh("docker", "compose", "up", "-d", "--wait", "s3")
|
||||
try:
|
||||
for name, env, files in scenarios:
|
||||
results.append(run_scenario(name, env, files))
|
||||
|
||||
Reference in New Issue
Block a user