Files
mtusi2026lab4/scripts/reconcile_experiment.py
vest1ter 44ae7b2472 ЛР4 на 5: вложения в MinIO, отказоустойчивость, трек 2 (сверка S3 и PostgreSQL)
Уровень 3:
- код lab3 перенесён; MinIO (сборка релиза 2025-10-15, по digest) в общем Compose,
  идемпотентный bootstrap: приватные bucket, отдельный пользователь API с ограниченной политикой
- метаданные вложений в PostgreSQL (миграции 0003, 0004), байты в S3; ключ назначает сервер
- upload PNG/JPEG: лимиты до чтения/декодирования, MIME по содержимому, проверка целостности,
  безопасный filename; 201 только после PUT и COMMIT; download потоком через API
- attachment_ids в сообщениях (только файлы своего чата, FK в БД)
- 503 API больше не повторяется в HAProxy (отказ общей зависимости), S3 не входит в readiness

Уровень 4:
- 45 тестов вложений: размер, повреждение, подмена MIME, чужой id, файл другого чата, отказы S3/БД
- компенсация при сбое COMMIT; слоты загрузки (8 на реплику) против неограниченной памяти
- эксперименты: пик RAM при 10-40 одновременных загрузках, оборванная загрузка и сироты;
  исправлены удержание соединения БД во время загрузки и таймаут отправки в S3

Уровень 5, трек 2:
- reconciler: слияние листинга S3 и курсора БД, 7 категорий расхождений, dry-run, grace period,
  перепроверка перед действием, тормоз --max-deletes, advisory lock; сервис reconciler в Compose
- 8 тестов сверки; эксперимент с 70 внедрёнными ошибками на 10 000 объектах (точное обнаружение,
  безопасное исправление, идемпотентность) и замер стоимости обхода на 1k/10k/50k
2026-11-12 17:38:00 +03:00

320 lines
16 KiB
Python

#!/usr/bin/env python3
"""Трек 2: эксперименты сверки хранилищ на отдельных базе messenger_bench и bucket messenger-attachments-bench.
python scripts/reconcile_experiment.py faults --objects 10000 # внедрение ошибок с обеих сторон
python scripts/reconcile_experiment.py cost --sizes 1000,10000,50000
Данные: N вложений по 1 KiB (случайные байты), половина прикреплена к сообщениям (по 10 на сообщение).
Сверка запускается так же, как в работе: `docker compose run reconciler python -m maintenance.reconcile`
(контейнер с лимитом 0.25 CPU / 128 MiB), с переопределёнными базой и bucket. Хост — только для генерации
данных и внедрения ошибок (venv, настройки из .env). Рабочие данные не затрагиваются.
"""
import argparse
import hashlib
import json
import logging
import os
import random
import statistics
import subprocess
import sys
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BENCH_DB = "messenger_bench"
BENCH_BUCKET = "messenger-attachments-bench"
os.chdir(ROOT)
os.environ["APP_CONFIG__DB__NAME"] = BENCH_DB
os.environ["APP_CONFIG__S3__BUCKET"] = BENCH_BUCKET
sys.path.insert(0, str(ROOT / "app"))
from alembic import command # noqa: E402
from alembic.config import Config # noqa: E402
from minio.deleteobjects import DeleteObject # noqa: E402
from sqlalchemy import create_engine, delete, func, insert, select, text, update # noqa: E402
from core import s3 # noqa: E402
from core.config import settings # noqa: E402
from core.db import SessionLocal, build_url, engine # noqa: E402
from models import Attachment, Chat, ChatMember, Message, MessageAttachment, User # noqa: E402
logging.getLogger("urllib3").setLevel(logging.ERROR)
OBJECT_SIZE = 1024
PREFIX = "originals/"
# --- данные ---
def recreate_database():
engine.dispose()
admin = create_engine(build_url().set(database="postgres"), isolation_level="AUTOCOMMIT")
with admin.connect() as connection:
connection.execute(text(f'DROP DATABASE IF EXISTS "{BENCH_DB}" WITH (FORCE)'))
connection.execute(text(f'CREATE DATABASE "{BENCH_DB}"'))
admin.dispose()
command.upgrade(Config(str(ROOT / "app" / "alembic.ini")), "head")
def empty_bucket():
client = s3.get_client()
batch = []
for stored in s3.list_objects(""):
batch.append(DeleteObject(stored.key))
if len(batch) == 1000:
list(client.remove_objects(BENCH_BUCKET, batch))
batch = []
if batch:
list(client.remove_objects(BENCH_BUCKET, batch))
def put_many(items):
"""items: [(key, bytes)] — параллельно, 16 потоков."""
def put(item):
key, data = item
s3.put_object(key, data, "image/png", hashlib.sha256(data).hexdigest())
with ThreadPoolExecutor(16) as pool:
list(pool.map(put, items))
def seed(n: int) -> dict:
started = time.monotonic()
recreate_database()
empty_bucket()
owner, other, chat = uuid.uuid4(), uuid.uuid4(), uuid.uuid4()
blobs = {uuid.uuid4(): os.urandom(OBJECT_SIZE) for _ in range(n)}
ids = list(blobs)
with SessionLocal() as db:
db.execute(insert(User), [{"id": owner, "username": "bench_owner", "display_name": "o", "password_hash": "x"},
{"id": other, "username": "bench_other", "display_name": "x", "password_hash": "x"}])
db.execute(insert(Chat), [{"id": chat, "title": "bench"}])
db.execute(insert(ChatMember), [{"chat_id": chat, "user_id": owner}, {"chat_id": chat, "user_id": other}])
rows = [{"id": i, "chat_id": chat, "owner_id": owner, "filename": "f.png", "content_type": "image/png",
"size_bytes": OBJECT_SIZE, "sha256": hashlib.sha256(blobs[i]).hexdigest(),
"object_key": f"{PREFIX}{i}"} for i in ids]
for start in range(0, n, 5000):
db.execute(insert(Attachment), rows[start:start + 5000])
# Первая половина прикреплена к сообщениям, по 10 вложений на сообщение.
referenced = ids[: n // 2]
messages, links = [], []
for start in range(0, len(referenced), 10):
message_id = uuid.uuid4()
messages.append({"id": message_id, "chat_id": chat, "sender_id": owner, "text": ""})
links += [{"message_id": message_id, "attachment_id": a, "chat_id": chat, "position": p}
for p, a in enumerate(referenced[start:start + 10])]
for start in range(0, len(messages), 5000):
db.execute(insert(Message), messages[start:start + 5000])
for start in range(0, len(links), 5000):
db.execute(insert(MessageAttachment), links[start:start + 5000])
db.commit()
db.execute(text("ANALYZE"))
db.commit()
put_many([(f"{PREFIX}{i}", data) for i, data in blobs.items()])
return {"objects": n, "referenced": n // 2, "chat": chat, "owner": owner, "ids": ids,
"seed_seconds": round(time.monotonic() - started, 1)}
def snapshot() -> dict:
with SessionLocal() as db:
rows = db.scalar(select(func.count()).select_from(Attachment))
referenced = db.scalar(select(func.count(func.distinct(MessageAttachment.attachment_id))))
return {"objects": sum(1 for _ in s3.list_objects(PREFIX)), "rows": rows, "referenced_rows": referenced}
def existing_keys() -> set:
return {stored.key for stored in s3.list_objects(PREFIX)}
# --- запуск сверки так же, как в работе ---
def reconcile(*flags) -> dict:
out = subprocess.run(
["docker", "compose", "run", "--rm", "--no-deps",
"-e", f"APP_CONFIG__DB__NAME={BENCH_DB}", "-e", f"APP_CONFIG__S3__BUCKET={BENCH_BUCKET}",
"reconciler", "python", "-m", "maintenance.reconcile", "--samples", "100000", *flags],
cwd=ROOT, capture_output=True, text=True)
lines = [line for line in out.stdout.splitlines() if line.startswith("{")]
if not lines:
raise SystemExit(f"сверка не вернула отчёт (exit {out.returncode}): {out.stderr[-500:]}")
return json.loads(lines[-1])
def brief(report: dict) -> dict:
keep = ("mode", "grace_seconds", "verify_checksums", "objects_scanned", "rows_scanned", "ok", "issues", "actions",
"seconds", "bytes_read", "peak_rss_mib", "aborted")
return {k: report[k] for k in keep if k in report}
# --- эксперимент 1: внедрение ошибок ---
def faults(args):
grace = args.grace
base = seed(args.objects)
ids, chat, owner = base["ids"], base["chat"], base["owner"]
random.seed(42)
referenced, free = ids[: len(ids) // 2], ids[len(ids) // 2:]
pick_ref = random.sample(referenced, 20)
pick_free = random.sample(free, 40)
injected = {
"orphan_object": set(), # 10 PUT без строки + 10 строк удалено из БД
"missing_object": set(), # объект удалён, строка не прикреплена
"missing_object_referenced": set(),
"size_mismatch": set(),
"checksum_mismatch": set(),
"orphan_object_fresh": set(),
"missing_object_fresh": set(),
}
report = {"objects": args.objects, "grace_seconds": grace, "seed_seconds": base["seed_seconds"]}
# Базовые данные должны стать «старыми» относительно grace.
time.sleep(grace + 2)
report["baseline"] = snapshot()
report["baseline_reconcile"] = brief(reconcile("--grace-seconds", str(grace)))
# Старые ошибки: S3-сторона и БД-сторона.
extra = [f"{PREFIX}{uuid.uuid4()}" for _ in range(10)]
put_many([(key, os.urandom(OBJECT_SIZE)) for key in extra])
injected["orphan_object"] |= set(extra)
with SessionLocal() as db:
lost_rows = pick_free[:10] # строки потеряны, объекты остались
db.execute(delete(Attachment).where(Attachment.id.in_(lost_rows)))
injected["orphan_object"] |= {f"{PREFIX}{i}" for i in lost_rows}
size_rows = pick_free[10:15] + pick_ref[10:15]
db.execute(update(Attachment).where(Attachment.id.in_(size_rows)).values(size_bytes=OBJECT_SIZE + 1))
injected["size_mismatch"] |= {str(i) for i in size_rows}
db.commit()
for i in pick_free[15:25]:
s3.delete_object(f"{PREFIX}{i}")
injected["missing_object"].add(str(i))
for i in pick_ref[:10]:
s3.delete_object(f"{PREFIX}{i}")
injected["missing_object_referenced"].add(str(i))
checksum_rows = pick_free[25:30] + pick_ref[15:20]
put_many([(f"{PREFIX}{i}", os.urandom(OBJECT_SIZE)) for i in checksum_rows]) # тот же размер, другие байты
injected["checksum_mismatch"] |= {str(i) for i in checksum_rows}
time.sleep(grace + 2)
# Свежие: загрузка «между PUT и COMMIT» и строка новее листинга.
fresh_objects = [f"{PREFIX}{uuid.uuid4()}" for _ in range(10)]
put_many([(key, os.urandom(OBJECT_SIZE)) for key in fresh_objects])
injected["orphan_object_fresh"] |= set(fresh_objects)
fresh_rows = [uuid.uuid4() for _ in range(10)]
with SessionLocal() as db:
db.execute(insert(Attachment), [{"id": i, "chat_id": chat, "owner_id": owner, "filename": "f.png",
"content_type": "image/png", "size_bytes": OBJECT_SIZE, "sha256": "0" * 64,
"object_key": f"{PREFIX}{i}"} for i in fresh_rows])
db.commit()
injected["missing_object_fresh"] |= {str(i) for i in fresh_rows}
report["injected"] = {k: len(v) for k, v in injected.items()}
report["after_injection"] = before = snapshot()
keys_before = existing_keys()
# Быстрые шаги идут подряд, пока «свежие» расхождения моложе grace (каждый запуск ~5 с);
# глубокая проверка (чтение каждого объекта, ~80 с на 10k) — после apply: она меняет только checksum_mismatch.
g = ("--grace-seconds", str(grace))
shallow = reconcile(*g)
detected = {k: set(v) for k, v in shallow["samples"].items()}
report["dry_run_shallow"] = brief(shallow)
report["state_unchanged_after_dry_run"] = snapshot() == before
report["apply_with_brake"] = brief(reconcile(*g, "--apply", "--max-deletes", "10"))
report["state_unchanged_after_brake"] = snapshot() == before
report["apply"] = brief(reconcile(*g, "--apply"))
after = snapshot()
keys_after = existing_keys()
report["after_apply"] = after
report["apply_again"] = brief(reconcile(*g, "--apply"))
report["fresh_still_fresh_at_apply"] = (report["apply_again"]["issues"]["orphan_object_fresh"] == 10
and report["apply_again"]["issues"]["missing_object_fresh"] == 10)
deep = reconcile(*g, "--verify-checksums")
report["dry_run_deep"] = brief(deep)
detected["checksum_mismatch"] = set(deep["samples"].get("checksum_mismatch", []))
report["detection_exact"] = {k: detected.get(k, set()) == injected[k] for k in injected}
report["safety"] = {
# Ни одна строка, прикреплённая к сообщению, не удалена.
"referenced_rows_kept": after["referenced_rows"] == before["referenced_rows"],
# Удалены ровно сироты; свежие объекты и объекты со строками на месте.
"deleted_objects_are_exactly_orphans": keys_before - keys_after == injected["orphan_object"],
"fresh_objects_kept": injected["orphan_object_fresh"] <= keys_after,
"rows_deleted_exactly_missing_free": before["rows"] - after["rows"] == len(injected["missing_object"]),
}
# Когда grace истёк, «свежие» расхождения становятся обычными и исправляются тем же путём.
time.sleep(grace + 2)
report["after_grace_apply"] = brief(reconcile(*g, "--apply"))
report["final_dry_run"] = brief(reconcile(*g, "--verify-checksums"))
report["final"] = snapshot()
final = report["final_dry_run"]["issues"]
ok = (all(report["detection_exact"].values()) and report["state_unchanged_after_dry_run"]
and report["fresh_still_fresh_at_apply"]
and "aborted" in report["apply_with_brake"] and report["state_unchanged_after_brake"]
and report["apply"]["actions"] == {"objects_deleted": 20, "rows_deleted": 10, "skipped_on_recheck": 0}
and report["apply_again"]["actions"] == {"objects_deleted": 0, "rows_deleted": 0, "skipped_on_recheck": 0}
and all(report["safety"].values())
and report["after_grace_apply"]["actions"]["objects_deleted"] == 10
and report["after_grace_apply"]["actions"]["rows_deleted"] == 10
and final == {"orphan_object": 0, "orphan_object_fresh": 0, "missing_object": 0,
"missing_object_referenced": 10, "missing_object_fresh": 0,
"size_mismatch": 10, "checksum_mismatch": 10})
report["result"] = "PASS" if ok else "FAIL"
Path(args.output or ROOT / "evidence" / "reconcile-faults.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2, default=str) + "\n", encoding="utf-8")
print(json.dumps({k: report[k] for k in ("injected", "detection_exact", "safety", "result")}, ensure_ascii=False))
return 0 if ok else 1
# --- эксперимент 2: стоимость обхода ---
def plan_for_rows() -> str:
with SessionLocal() as db:
rows = db.execute(text("EXPLAIN SELECT object_key FROM attachments WHERE object_key LIKE 'originals/%' "
"ORDER BY object_key")).scalars().all()
return rows[0]
def cost(args):
report = {"object_size_bytes": OBJECT_SIZE, "reconciler_limits": "0.25 CPU, 128 MiB", "runs": []}
for n in [int(x) for x in args.sizes.split(",")]:
base = seed(n)
entry = {"objects": n, "seed_seconds": base["seed_seconds"], "rows_plan": plan_for_rows(),
"s3_list_requests": -(-n // 1000), "shallow": [], "deep": []}
for _ in range(args.repeats):
entry["shallow"].append(brief(reconcile("--grace-seconds", "0")))
for _ in range(args.deep_repeats):
entry["deep"].append(brief(reconcile("--grace-seconds", "0", "--verify-checksums")))
for mode in ("shallow", "deep"):
runs = entry[mode]
if runs:
seconds = statistics.median(r["seconds"]["scan"] for r in runs)
entry[f"{mode}_median_seconds"] = round(seconds, 3)
entry[f"{mode}_objects_per_second"] = round(n / seconds)
entry[f"{mode}_peak_rss_mib"] = max(r["peak_rss_mib"] for r in runs)
entry[f"{mode}_all_ok"] = all(r["ok"] == n and sum(r["issues"].values()) == 0 for r in runs)
report["runs"].append(entry)
print(json.dumps({k: v for k, v in entry.items() if k not in ("shallow", "deep")}, ensure_ascii=False))
Path(args.output or ROOT / "evidence" / "reconcile-cost.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def main():
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="cmd", required=True)
f = sub.add_parser("faults")
f.add_argument("--objects", type=int, default=10000)
f.add_argument("--grace", type=int, default=20)
f.add_argument("--output")
c = sub.add_parser("cost")
c.add_argument("--sizes", default="1000,10000,50000")
c.add_argument("--repeats", type=int, default=3)
c.add_argument("--deep-repeats", type=int, default=2)
c.add_argument("--output")
args = parser.parse_args()
return faults(args) if args.cmd == "faults" else cost(args)
if __name__ == "__main__":
sys.exit(main() or 0)