diff --git a/openspec/changes/add-lab4-storage-reconciler/tasks.md b/openspec/changes/add-lab4-storage-reconciler/tasks.md index 04c9511c..cbec9e35 100644 --- a/openspec/changes/add-lab4-storage-reconciler/tasks.md +++ b/openspec/changes/add-lab4-storage-reconciler/tasks.md @@ -26,10 +26,10 @@ ## 5. Обвязка трека -- [ ] 5.1 `scripts/reconcile-inject.sh` и `scripts/reconcile_check.py` из заготовки -- [ ] 5.2 `student-tests/tools/seed_storage.py`: наполнение N объектов и строк в образе API -- [ ] 5.3 `scripts/reconcile_track.py`: `demo` (внедрение, dry-run, apply ×2, безопасность, `healed`, нагрузка) и `cost` (снимок хоста, закрепление CPU, матрица N × режим × 3) -- [ ] 5.4 Проверка группы: `make check` (Python 3.9, JSON), `make lint` +- [x] 5.1 `scripts/reconcile-inject.sh` и `scripts/reconcile_check.py` из заготовки +- [x] 5.2 `student-tests/tools/seed_storage.py`: наполнение N объектов и строк в образе API +- [x] 5.3 `scripts/reconcile_track.py`: `demo` (внедрение, dry-run, apply ×2, безопасность, `healed`, нагрузка) и `cost` (снимок хоста, закрепление CPU, матрица N × режим × 3) +- [x] 5.4 Проверка группы: `make check` (Python 3.9, JSON), `make lint` ## 6. Измерения diff --git a/scripts/reconcile-inject.sh b/scripts/reconcile-inject.sh new file mode 100755 index 00000000..05fac513 --- /dev/null +++ b/scripts/reconcile-inject.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +kind="${1:?usage: reconcile-inject.sh orphan|missing|corrupt|bitflip|stale-pending|pending-object|multipart [count]}" +count="${2:-1}" +age="${INJECT_AGE_SECONDS:-7200}" +prefix="${S3_PREFIX:-attachments/}" +sql() { docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -qAt -F ' ' -U "${PGUSER:-messenger}" -d "${PGDATABASE:-messenger}" -c "$1"; } +s3() { docker compose run --rm --no-deps -T -e ROWS="$1" s3-cli sh -c "$2"; } +ready() { sql "SELECT id, object_key, size_bytes FROM attachments WHERE status = 'ready' ${INJECT_IDS:+AND id = ANY('{$INJECT_IDS}'::uuid[])} ORDER BY random() LIMIT $count"; } +pending() { + sql "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status, created_at, updated_at) + SELECT g.id, s.chat_id, s.owner_id, 'injected.png', 'image/png', 2048, repeat('0', 64), 1, 1, '$prefix' || g.id, 'pending', + clock_timestamp() - make_interval(secs => $age), clock_timestamp() - make_interval(secs => $age) + FROM (SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1) s, + (SELECT gen_random_uuid() AS id FROM generate_series(1, $count)) g + RETURNING id, object_key" +} +fresh() { sql "SELECT gen_random_uuid(), '$prefix' || gen_random_uuid() FROM generate_series(1, $count)"; } +put_random='printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; head -c "${size:-2048}" /dev/urandom | rc pipe "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done' +case "$kind" in + orphan) rows=$(fresh); s3 "$rows" "$put_random" | sed "s/^/orphan /" ;; + missing) rows=$(ready); s3 "$rows" 'printf "%s\n" "$ROWS" | while read -r id key size; do [ -n "$key" ] || continue; rc object remove --force "root/$S3_BUCKET/$key" >/dev/null; echo "$id $key"; done' | sed "s/^/missing /" ;; + corrupt) rows=$(ready | awk '{print $1, $2, int($3 / 2) + 1}'); s3 "$rows" "$put_random" | sed "s/^/corrupt /" ;; + bitflip) rows=$(ready); s3 "$rows" "$put_random" | sed "s/^/bitflip /" ;; + stale-pending) pending | sed "s/^/stale-pending /" ;; + pending-object) rows=$(pending); s3 "$rows" "$put_random" | sed "s/^/pending-object /" ;; + multipart) + fresh | while read -r _ key; do + docker compose exec -T -e KEY="$key" -e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" s3 sh -c 'u="http://127.0.0.1:9000/$S3_BUCKET/$KEY"; a="$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"; s="aws:amz:${RUSTFS_REGION:-us-east-1}:s3" + upload=$(curl -fsS --aws-sigv4 "$s" --user "$a" -X POST "$u?uploads" | sed -n "s:.*\(.*\).*:\1:p") + head -c 4096 /dev/urandom | curl -fsS --aws-sigv4 "$s" --user "$a" -X PUT --data-binary @- "$u?partNumber=1&uploadId=$upload" -o /dev/null' &2; exit 2 ;; +esac diff --git a/scripts/reconcile_check.py b/scripts/reconcile_check.py new file mode 100755 index 00000000..8712c5ad --- /dev/null +++ b/scripts/reconcile_check.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +import argparse +import datetime as dt +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SQL = ( + "SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256," + " floor(extract(epoch FROM clock_timestamp() - a.created_at))::bigint," + " EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id)," + " (to_jsonb(a) ->> 'quarantined_at') IS NOT NULL" + " FROM attachments a ORDER BY a.object_key" +) +HASH = ( + 'while read -r k; do printf "%s %s\\n" "$k" ' + '"$(rc object show "root/$S3_BUCKET/$k" | sha256sum | cut -c1-64)"; done' +) +CLASSES = ( + "orphan_stale", + "orphan_fresh", + "missing_referenced", + "missing_unreferenced", + "size_mismatch", + "sha_mismatch", + "pending_stale_with_object", + "pending_stale_without_object", + "pending_fresh", + "quarantined", + "incomplete_stale", + "incomplete_fresh", + "lost_ready", +) +TOLERATED = {"orphan_fresh", "pending_fresh", "incomplete_fresh", "quarantined"} +BROKEN = ( + "missing_referenced", + "missing_unreferenced", + "size_mismatch", + "sha_mismatch", + "quarantined", +) + + +def timed(cmd, stdin=None): + started = time.monotonic() + out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False) + if out.returncode: + raise SystemExit(f"command failed: {' '.join(cmd[:6])}\n{out.stderr[-500:]}") + return out.stdout, round(time.monotonic() - started, 3) + + +def s3(*args, stdin=None): + return timed(["docker", "compose", "run", "--rm", "--no-deps", "-T", "s3-cli", *args], stdin) + + +def age(stamp, now): + return now - dt.datetime.fromisoformat(stamp[:19] + "+00:00").timestamp() + + +def load(bucket, prefix, deep): + listing, list_seconds = s3("object", "list", "--recursive", "--json", f"root/{bucket}/{prefix}") + uploads, uploads_seconds = s3( + "object", "list", "--incomplete", "--recursive", "--json", f"root/{bucket}" + ) + user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger") + psql = ["docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt"] + rows_raw, pg_seconds = timed([*psql, "-F", "\t", "-U", user, "-d", db, "-c", SQL]) + objects = {} + for item in json.loads(listing).get("items", []): + key = item["key"] if item["key"].startswith(prefix) else prefix + item["key"] + objects[key] = item + rows = {} + for line in rows_raw.splitlines(): + rid, key, status, size, sha, row_age, referenced, quarantined = line.split("\t") + rows[key] = { + "id": rid, + "status": status, + "size": int(size), + "sha256": sha, + "age": int(row_age), + "referenced": referenced == "t", + "quarantined": quarantined == "t", + } + hashes, deep_seconds = {}, 0.0 + if deep: + present = "".join( + f"{k}\n" for k, r in rows.items() if r["status"] == "ready" and k in objects + ) + out, deep_seconds = s3("sh", "-c", HASH, stdin=present) + hashes = dict(line.split(" ", 1) for line in out.splitlines() if " " in line) + seconds = { + "s3_list": list_seconds, + "s3_incomplete": uploads_seconds, + "pg_query": pg_seconds, + "deep_hash": deep_seconds, + } + return objects, rows, hashes, json.loads(uploads).get("data", {}).get("items", []), seconds + + +def row_class(key, row, objects, hashes, grace): + if row["quarantined"]: + return "quarantined" + if row["status"] == "pending": + stale = "pending_stale_with_object" if key in objects else "pending_stale_without_object" + return "pending_fresh" if row["age"] <= grace else stale + if key not in objects: + return "missing_referenced" if row["referenced"] else "missing_unreferenced" + if objects[key]["size_bytes"] != row["size"]: + return "size_mismatch" + return "sha_mismatch" if hashes and hashes.get(key) != row["sha256"] else None + + +def classify(objects, rows, hashes, uploads, grace): + now = time.time() + classes = {name: [] for name in CLASSES} + for key, item in objects.items(): + if key not in rows: + classes[ + "orphan_stale" if age(item["last_modified"], now) > grace else "orphan_fresh" + ].append(key) + for key, row in rows.items(): + name = row_class(key, row, objects, hashes, grace) + if name: + classes[name].append(key if name.startswith("orphan") else row["id"]) + for item in uploads: + classes[ + "incomplete_stale" if age(item["initiated"], now) > grace else "incomplete_fresh" + ].append(item["key"]) + return classes + + +def main(): + p = argparse.ArgumentParser( + description="Independent PG <-> S3 invariant check for lab4 attachments" + ) + p.add_argument( + "--grace", type=int, default=int(os.environ.get("RECONCILE_GRACE_SECONDS", "3600")) + ) + p.add_argument( + "--deep", action="store_true", help="stream every present object and compare sha256" + ) + p.add_argument( + "--save-ready", metavar="FILE", help="store ids and sha256 of healthy ready attachments" + ) + p.add_argument( + "--expect-ready", + metavar="FILE", + help="fail if any attachment from FILE is no longer healthy", + ) + p.add_argument("--limit", type=int, default=20, help="max ids per class in the report") + p.add_argument("--output", metavar="FILE") + args = p.parse_args() + started = time.monotonic() + prefix = os.environ.get("S3_PREFIX", "attachments/") + objects, rows, hashes, uploads, seconds = load( + os.environ.get("S3_BUCKET", "messenger-attachments"), prefix, args.deep + ) + classes = classify(objects, rows, hashes, uploads, args.grace) + bad = {i for c in BROKEN for i in classes[c]} + healthy = { + r["id"]: r["sha256"] + for k, r in rows.items() + if r["status"] == "ready" and k in objects and r["id"] not in bad + } + if args.expect_ready: + expected = json.loads(Path(args.expect_ready).read_text()) + classes["lost_ready"] = sorted(i for i, sha in expected.items() if healthy.get(i) != sha) + if args.save_ready: + Path(args.save_ready).write_text(json.dumps(healthy, sort_keys=True)) + violations = [c for c in CLASSES if classes[c] and c not in TOLERATED] + seconds["total"] = round(time.monotonic() - started, 3) + report = { + "checked_at": dt.datetime.now(dt.timezone.utc) + .isoformat(timespec="seconds") + .replace("+00:00", "Z"), + "grace_seconds": args.grace, + "deep": args.deep, + "objects": len(objects), + "rows": len(rows), + "ready_healthy": len(healthy), + "counts": {c: len(v) for c, v in classes.items()}, + "samples": {c: sorted(v)[: args.limit] for c, v in classes.items() if v}, + "seconds": seconds, + "violations": violations, + "ok": not violations, + } + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).write_text(text + "\n") + print(text) + sys.exit(0 if report["ok"] else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/reconcile_track.py b/scripts/reconcile_track.py new file mode 100644 index 00000000..1819985b --- /dev/null +++ b/scripts/reconcile_track.py @@ -0,0 +1,591 @@ +#!/usr/bin/env python3 +import argparse +import http.client +import json +import os +import statistics +import struct +import subprocess +import sys +import threading +import time +import uuid +import zlib +from pathlib import Path +from urllib.parse import urlsplit + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +client = __import__("client") +FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes() +OUT = ROOT / "evidence" / "reconcile" +GRACE = 60 +FLAGS = ("--grace", str(GRACE), "--skew", "0") +INTENT_AGE = 300 +LOCK_KEY = 20260004 +FIXABLE = ("orphan_stale", "missing", "size_mismatch", "sha_mismatch", "pending_stale") + + +def run(cmd, env=None, stdin=None): + started = time.monotonic() + done = subprocess.run( + cmd, + cwd=ROOT, + input=stdin, + capture_output=True, + check=False, + env=dict(os.environ, **(env or {})), + ) + out, err = done.stdout.decode(errors="replace"), done.stderr.decode(errors="replace") + return done.returncode, out, err, round(time.monotonic() - started, 3) + + +def must(cmd, env=None, stdin=None): + code, out, err, _ = run(cmd, env, stdin) + if code: + raise SystemExit(f"failed: {' '.join(cmd[:7])}\n{err[-800:]}") + return out + + +def compose(*args, env=None, stdin=None): + return must(["docker", "compose", *args], env, stdin) + + +def psql(sql): + return compose( + "exec", + "-T", + "postgres", + "psql", + "-v", + "ON_ERROR_STOP=1", + "-qAt", + "-U", + "messenger", + "-d", + "messenger", + "-c", + sql, + ).strip() + + +def reconcile(*flags, env=None): + cmd = ["docker", "compose", "run", "--rm", "--no-deps", "-T", "reconciler", *flags] + code, out, err, wall = run(cmd, env) + if code not in (0, 1) or not out.strip(): + raise SystemExit(f"reconciler failed ({code}):\n{err[-800:]}") + report = json.loads(out) + report.update(exit_code=code, wall_seconds=wall) + return report + + +def oracle(*flags): + code, out, err, _ = run([sys.executable, "scripts/reconcile_check.py", *flags]) + if not out.strip(): + raise SystemExit(f"reconcile_check failed ({code}):\n{err[-800:]}") + return json.loads(out) + + +def inject(kind, count, ids=(), age=None): + env = {"INJECT_IDS": ",".join(ids)} if ids else {} + if age is not None: + env["INJECT_AGE_SECONDS"] = str(age) + out = must(["sh", "scripts/reconcile-inject.sh", kind, str(count)], env) + return [line.split()[1] for line in out.splitlines() if line.strip()] + + +def head(): + return must(["git", "rev-parse", "--short", "HEAD"]).strip() + + +def save(name, data): + path = OUT / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n") + return path + + +class Checks: + def __init__(self): + self.items = [] + + def __call__(self, name, ok, **detail): + self.items.append(dict(name=name, ok=bool(ok), **detail)) + sys.stderr.write(("PASS " if ok else "FAIL ") + name + "\n") + + @property + def ok(self): + return all(item["ok"] for item in self.items) + + +def chunk(tag, data): + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data)) + + +def tiny_png(): + raw = b"".join(b"\x00" + os.urandom(24) for _ in range(8)) + header = struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", zlib.compress(raw)) + + chunk(b"IEND", b"") + ) + + +def pick(found, names): + return {name: found.get(name, 0) for name in names} + + +def faults(c, check, pre): + alice, bob, _, chat = c.bootstrap() + ids = [c.attachment(alice, chat)[0]["id"] for _ in range(18)] + linked = ids[:6] + for part in (linked[:3], linked[3:]): + c.message(alice, chat, "", attachments=part) + targets = { + "missing": [linked[0], linked[1], ids[6]], + "corrupt": [linked[2], ids[7], ids[8]], + "bitflip": [linked[3], ids[9], ids[10]], + } + damaged = {i for group in targets.values() for i in group} + ready = OUT / "ready-before.json" + oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready)) + untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged} + save("ready-untouched.json", untouched) + started = time.monotonic() + injected = { + "orphan_stale": inject("orphan", 3), + "missing": inject("missing", 3, targets["missing"]), + "size_mismatch": inject("corrupt", 3, targets["corrupt"]), + "sha_mismatch": inject("bitflip", 3, targets["bitflip"]), + "pending_stale": inject("stale-pending", 3, age=INTENT_AGE) + + inject("pending-object", 3, age=INTENT_AGE), + } + time.sleep(max(0.0, GRACE + 5 - (time.monotonic() - started))) + injected["orphan_fresh"] = inject("orphan", 2) + injected["pending_fresh"] = inject("stale-pending", 2, age=0) + expected = {name: len(keys) for name, keys in injected.items()} + expected.update(quarantined=pre["counts"]["quarantined"]) + before = oracle("--grace", str(GRACE), "--deep", "--output", str(OUT / "check-before.json")) + dry = reconcile("--dry-run", "--deep", *FLAGS) + save("dry-run.json", dry) + after_dry = oracle("--grace", str(GRACE), "--deep") + check( + "dry-run находит каждый внедрённый класс", + pick(dry["found"], expected) == expected, + expected=expected, + found=dry["found"], + ) + check( + "dry-run ничего не меняет", + not any(dry["fixed"].values()) + and after_dry["counts"] == before["counts"] + and (after_dry["objects"], after_dry["rows"]) == (before["objects"], before["rows"]), + objects=[before["objects"], after_dry["objects"]], + rows=[before["rows"], after_dry["rows"]], + ) + first = reconcile("--apply", "--deep", *FLAGS) + save("apply-1.json", first) + check( + "apply исправляет ровно найденное dry-run", + pick(first["fixed"], FIXABLE) == pick(dry["found"], FIXABLE) and not first["errors"], + dry_run=pick(dry["found"], FIXABLE), + fixed=pick(first["fixed"], FIXABLE), + ) + second = reconcile("--apply", "--deep", *FLAGS) + save("apply-2.json", second) + check( + "повторный apply — 0 действий", + not any(second["fixed"].values()), + fixed=second["fixed"], + found=second["found"], + ) + after = oracle( + "--grace", + str(GRACE), + "--deep", + "--expect-ready", + str(OUT / "ready-untouched.json"), + "--output", + str(OUT / "check-after.json"), + ) + tolerated = ("orphan_fresh", "pending_fresh") + check( + "независимая проверка после apply: нарушений нет, готовые не потеряны", + after["ok"] + and not after["counts"]["lost_ready"] + and pick(after["counts"], tolerated) == pick(before["counts"], tolerated) + and after["counts"]["quarantined"] == pre["counts"]["quarantined"] + 9, + counts=after["counts"], + ) + marks = dict( + line.split("|") for line in psql("SELECT id, quarantine_reason FROM attachments").split() + ) + links = psql("SELECT attachment_id FROM message_attachments").split() + kept = {i: marks.get(i) for i in sorted(damaged)} + check( + "строки с расхождениями и ссылки сообщений сохранены", + all(kept.values()) and sum(i in linked for i in links) == len(linked), + quarantine=kept, + message_links=sum(i in linked for i in links), + ) + meta = c.call("GET", f"/api/v1/attachments/{linked[0]}", 200, actor=bob, schema="Attachment") + blocked = c.call("GET", f"/api/v1/attachments/{linked[2]}/content", 404, actor=bob) + intact = c.call("GET", f"/api/v1/attachments/{linked[4]}/content", 200, actor=bob) + check( + "карантин: метаданные 200, /content 404; нетронутое скачивается", + meta["status"] == 200 and blocked["status"] == 404 and intact["bytes"] == FIXTURE, + ) + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + compose( + "run", + "--rm", + "--no-deps", + "-T", + "s3-cli", + "pipe", + f"root/{bucket}/attachments/{linked[0]}", + stdin=FIXTURE, + ) + healed = reconcile("--apply", *FLAGS) + save("healed.json", healed) + restored = c.call("GET", f"/api/v1/attachments/{linked[0]}/content", 200, actor=bob) + check( + "восстановленный объект выходит из карантина", + healed["fixed"]["healed"] == 1 and restored["bytes"] == FIXTURE, + fixed=healed["fixed"], + ) + return {"injected": injected, "targets": targets, "linked": linked} + + +def single(check): + holder = subprocess.Popen( + [ + "docker", + "compose", + "exec", + "-T", + "postgres", + "psql", + "-U", + "messenger", + "-d", + "messenger", + "-c", + f"SELECT pg_advisory_lock({LOCK_KEY}), pg_sleep(15)", + ], + cwd=ROOT, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + time.sleep(3) + busy = reconcile("--apply", *FLAGS) + finally: + holder.wait() + save("lock-busy.json", busy) + check( + "второй одновременный проход пропущен", + busy["skipped"] and busy["exit_code"] == 0 and not any(busy["fixed"].values()), + ) + + +def uploader(base, token, chat, *, deadline, results, lock): + host = urlsplit(base).netloc + conn = http.client.HTTPConnection(host, timeout=30) + headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"} + sent = 0 + while time.monotonic() < deadline: + path = f"/api/v1/chats/{chat}/attachments?filename=load.png" + try: + conn.request("POST", path, tiny_png(), {**headers, "Content-Type": "image/png"}) + response = conn.getresponse() + status, data = response.status, response.read() + if status == 201 and sent % 10 == 0: + body = json.dumps({"text": "", "attachment_ids": [json.loads(data)["id"]]}) + extra = {"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex} + conn.request("POST", f"/api/v1/chats/{chat}/messages", body, {**headers, **extra}) + linked = conn.getresponse() + linked.read() + status = 201 if linked.status == 201 else -linked.status + except (OSError, http.client.HTTPException): + conn.close() + conn = http.client.HTTPConnection(host, timeout=30) + status, data = 0, b"" + with lock: + results.append((status, json.loads(data) if status == 201 else None)) + sent += 1 + time.sleep(0.05) + + +def under_load(c, check, base, seconds, threads): + alice, _, _, chat = c.bootstrap() + ready = OUT / "ready-load.json" + left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"] + leftover = { + "orphan_stale": left["orphan_fresh"] + left["orphan_stale"], + "pending_stale": left["pending_fresh"] + + left["pending_stale_with_object"] + + left["pending_stale_without_object"], + } + orphans = inject("orphan", 3) + intents = inject("stale-pending", 3, age=INTENT_AGE) + results, lock = [], threading.Lock() + deadline = time.monotonic() + seconds + workers = [ + threading.Thread( + target=uploader, + args=(base, alice["token"], chat["id"]), + kwargs={"deadline": deadline, "results": results, "lock": lock}, + ) + for _ in range(threads) + ] + for worker in workers: + worker.start() + runs = [] + while time.monotonic() < deadline: + runs.append(reconcile("--apply", *FLAGS)) + for worker in workers: + worker.join() + runs.append(reconcile("--apply", *FLAGS)) + created = {att["id"]: att["sha256"] for status, att in results if att} + expected = dict(json.loads(ready.read_text()), **created) + save("ready-load.json", expected) + after = oracle( + "--grace", + str(GRACE), + "--deep", + "--expect-ready", + str(ready), + "--output", + str(OUT / "check-under-load.json"), + ) + statuses = {} + for status, _ in results: + statuses[str(status)] = statuses.get(str(status), 0) + 1 + fixed = {name: sum(r["fixed"][name] for r in runs) for name in FIXABLE} + seen = {name: sum(r["found"][name] for r in runs) for name in ("orphan_fresh", "pending_fresh")} + summary = { + "seconds": seconds, + "upload_threads": threads, + "uploads": statuses, + "created": len(created), + "linked_every": 10, + "reconciler_runs": len(runs), + "run_seconds": [r["cost"]["seconds"] for r in runs], + "fixed_total": fixed, + "fresh_seen_total": seen, + "errors": sum(r["errors"] for r in runs), + "skipped": sum(1 for r in runs if r["skipped"]), + "injected_before_load": {"orphan": len(orphans), "pending_stale": len(intents)}, + "left_from_faults": leftover, + "check_after": {k: after[k] for k in ("objects", "rows", "ready_healthy", "counts", "ok")}, + } + save("under-load.json", summary) + check( + "под нагрузкой: исправлены только внедрённые, ошибок нет", + fixed + == { + **dict.fromkeys(FIXABLE, 0), + "orphan_stale": leftover["orphan_stale"] + len(orphans), + "pending_stale": leftover["pending_stale"] + len(intents), + } + and not summary["errors"] + and not summary["skipped"], + fixed=fixed, + runs=len(runs), + ) + check( + "под нагрузкой: все 201 готовы и совпадают по SHA-256, потерь 0", + after["ok"] and not after["counts"]["lost_ready"] and set(statuses) == {"201"}, + uploads=statuses, + lost_ready=after["counts"]["lost_ready"], + ) + + +def demo(args): + OUT.mkdir(parents=True, exist_ok=True) + c = client.Client(args.base_url, timeout=30) + check = Checks() + pre = oracle("--grace", str(GRACE)) + dirty = {k: v for k, v in pre["counts"].items() if v and k != "quarantined"} + if dirty: + raise SystemExit(f"стенд не в исходном состоянии {dirty}: make reconcile ARGS=--apply") + started = time.monotonic() + result = {"started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head()} + result["faults"] = faults(c, check, pre) + single(check) + under_load(c, check, args.base_url, args.load_seconds, args.load_threads) + result.update(seconds=round(time.monotonic() - started, 1), checks=check.items, ok=check.ok) + save("demo.json", result) + print(json.dumps({"ok": check.ok, "checks": len(check.items)}, ensure_ascii=False)) + sys.exit(0 if check.ok else 1) + + +def pins(args): + cpus = int(must(["docker", "info", "--format", "{{.NCPU}}"]).strip()) + if args.no_pin or cpus < 8: + return {"S3_MEMORY": args.s3_memory} + return { + "S3_CPUSET": str(cpus - 4), + "PG_CPUSET": f"{cpus - 3}-{cpus - 2}", + "RECONCILE_CPUSET": str(cpus - 1), + "S3_MEMORY": args.s3_memory, + } + + +def limits(): + ids = compose("ps", "-q", "s3", "postgres").split() + fmt = ( + "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}" + " cpuset={{.HostConfig.CpusetCpus}}" + ) + return must(["docker", "inspect", "-f", fmt, *ids]).strip().splitlines() + + +def seed(count, env): + if count <= 0: + return {"added": 0} + env = dict(env, RECONCILE_CPUSET="", RECONCILE_CPUS="4", RECONCILE_MEMORY="512m") + tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro" + out = compose( + "run", + "--rm", + "--no-deps", + "-T", + "-v", + tools, + "--entrypoint", + "python", + "reconciler", + "/tools/seed_storage.py", + "--count", + str(count), + env=env, + ) + return json.loads(out.strip().splitlines()[-1]) + + +def median(runs, path): + values = [] + for r in runs: + value = r + for key in path: + value = value[key] + values.append(value) + return statistics.median(values) + + +def summarize(results): + rows = [] + for (n, mode), runs in sorted(results.items()): + seconds = median(runs, ("cost", "seconds")) + rows.append( + { + "objects": n, + "mode": mode, + "runs": len(runs), + "seconds_median": seconds, + "seconds_all": [r["cost"]["seconds"] for r in runs], + "wall_seconds_median": median(runs, ("wall_seconds",)), + "us_per_object": round(seconds / n * 1e6, 1), + "s3_list_wait_median": median(runs, ("cost", "waited_seconds", "s3_list")), + "sql_wait_median": median(runs, ("cost", "waited_seconds", "sql")), + "requests": runs[0]["cost"]["requests"], + "max_rss_mib_median": median(runs, ("cost", "max_rss_mib")), + "scanned": runs[0]["scanned"], + "found_nonzero": {k: v for k, v in runs[0]["found"].items() if v}, + } + ) + return rows + + +def table(rows): + head = ( + "objects mode runs t_med,s wall,s us/obj list,s sql,s lists sqls gets" + " rss,MiB" + ) + lines = [head] + for r in rows: + q = r["requests"] + lines.append( + f"{r['objects']:>7} {r['mode']:<9} {r['runs']:>4} {r['seconds_median']:>7.3f} " + f"{r['wall_seconds_median']:>6.2f} {r['us_per_object']:>6.1f} " + f"{r['s3_list_wait_median']:>6.3f} {r['sql_wait_median']:>5.3f} {q['s3_list']:>5} " + f"{q['sql']:>4} {q['s3_get']:>4} {r['max_rss_mib_median']:>7.1f}" + ) + return "\n".join(lines) + "\n" + + +def cost(args): + out = OUT / "cost" + out.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + stats = must( + [ + "docker", + "stats", + "--no-stream", + "--all", + "--format", + "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.PIDs}}", + ] + ) + snapshot = f"date: {stamp}\n$ docker stats --no-stream --all\n{stats}" + (out / "host-docker-stats.txt").write_text(snapshot) + rows = int(psql("SELECT count(*) FROM attachments")) + if rows > args.sizes[0]: + raise SystemExit(f"на стенде уже {rows} вложений: make reset-data && make up && make test") + env = pins(args) + compose("up", "-d", "--wait", "--no-deps", "s3", "postgres", env=env) + plan = {"date": stamp, "commit": head(), "pins": env, "limits": limits(), "seeds": []} + results = {} + try: + for n in args.sizes: + plan["seeds"].append(dict(seed(n - rows, env), target=n)) + rows = n + time.sleep(args.settle) + modes = [("shallow", ["--dry-run"])] + if n <= args.deep_max: + modes.append(("deep", ["--dry-run", "--deep"])) + if n == args.page_compare: + modes.append(("page100", ["--dry-run", "--page-size", "100"])) + for mode, flags in modes: + for attempt in range(1, args.repeats + 1): + report = reconcile(*flags, "--samples", "3", env=env) + (out / f"n{n}-{mode}-{attempt}.json").write_text(json.dumps(report, indent=2)) + results.setdefault((n, mode), []).append(report) + sys.stderr.write(f"{n} {mode} #{attempt}: {report['cost']['seconds']} s\n") + finally: + compose("up", "-d", "--wait", "--no-deps", "s3", "postgres") + summary = summarize(results) + save("cost/summary.json", {**plan, "results": summary}) + (out / "summary.txt").write_text(f"date: {stamp} pins: {env}\n" + table(summary)) + sys.stdout.write(table(summary)) + + +def main(): + p = argparse.ArgumentParser(description="Lab4 track 2: reconciler demo and cost of a pass") + sub = p.add_subparsers(dest="command", required=True) + d = sub.add_parser("demo", help="fault injection, dry-run/apply/repeat, safety, under load") + d.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) + d.add_argument("--load-seconds", type=int, default=75) + d.add_argument("--load-threads", type=int, default=3) + k = sub.add_parser("cost", help="cost of a pass for N objects, repeated") + k.add_argument("--sizes", type=int, nargs="+", default=[1000, 10000, 100000]) + k.add_argument("--repeats", type=int, default=3) + k.add_argument("--deep-max", type=int, default=100000) + k.add_argument("--page-compare", type=int, default=10000) + k.add_argument("--settle", type=float, default=10) + k.add_argument("--s3-memory", default="1g") + k.add_argument("--no-pin", action="store_true") + args = p.parse_args() + if args.command == "demo": + demo(args) + else: + cost(args) + + +if __name__ == "__main__": + main() diff --git a/student-tests/tools/seed_storage.py b/student-tests/tools/seed_storage.py new file mode 100644 index 00000000..01bc6ccc --- /dev/null +++ b/student-tests/tools/seed_storage.py @@ -0,0 +1,112 @@ +import argparse +import asyncio +import hashlib +import json +import time +from datetime import UTC, datetime, timedelta +from io import BytesIO +from uuid import uuid4 + +import asyncpg +from PIL import Image + +from messenger.attachments.config import AttachmentsSettings +from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client +from messenger.shared.config import PlatformSettings + +COLUMNS = ( + "id", + "chat_id", + "owner_id", + "filename", + "content_type", + "size_bytes", + "sha256", + "width", + "height", + "object_key", + "status", + "created_at", + "updated_at", +) + + +def picture() -> bytes: + buffer = BytesIO() + Image.effect_noise((8, 8), 64).convert("RGB").save(buffer, "PNG") + return buffer.getvalue() + + +async def seed(count: int, concurrency: int) -> dict[str, object]: + platform, settings = PlatformSettings(), AttachmentsSettings() + body = picture() + digest = hashlib.sha256(body).hexdigest() + connection = await asyncpg.connect(platform.database_url.get_secret_value()) + try: + member = await connection.fetchrow("SELECT chat_id, user_id FROM chat_members LIMIT 1") + if member is None: + raise SystemExit("нет ни одного чата: сначала make test") + ids = [uuid4() for _ in range(count)] + pending = iter(ids) + started = time.monotonic() + async with s3_client( + endpoint=settings.s3_endpoint, + region=settings.s3_region, + access_key_id=settings.s3_access_key_id.get_secret_value(), + secret_access_key=settings.s3_secret_access_key.get_secret_value(), + connect_timeout=settings.s3_connect_timeout_seconds, + read_timeout=settings.s3_read_timeout_seconds, + max_attempts=3, + ) as client: + storage = S3ObjectStorage(client, settings.s3_bucket) + + async def worker() -> None: + for attachment_id in pending: + await storage.put(f"attachments/{attachment_id}", body, "image/png") + + await asyncio.gather(*(worker() for _ in range(concurrency))) + stored = time.monotonic() + stamp = datetime.now(UTC) - timedelta(hours=2) + records = [ + ( + attachment_id, + member["chat_id"], + member["user_id"], + f"seed-{number}.png", + "image/png", + len(body), + digest, + 8, + 8, + f"attachments/{attachment_id}", + "ready", + stamp, + stamp, + ) + for number, attachment_id in enumerate(ids) + ] + await connection.copy_records_to_table("attachments", records=records, columns=COLUMNS) + total = await connection.fetchval("SELECT count(*) FROM attachments") + finally: + await connection.close() + return { + "added": count, + "rows_total": total, + "object_bytes": len(body), + "seconds": { + "s3_put": round(stored - started, 3), + "pg_copy": round(time.monotonic() - stored, 3), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Наполнить bucket и attachments N вложениями") + parser.add_argument("--count", type=int, required=True) + parser.add_argument("--concurrency", type=int, default=32) + args = parser.parse_args() + print(json.dumps(asyncio.run(seed(args.count, args.concurrency)))) + + +if __name__ == "__main__": + main()