forked from mtusi2026/mtusi2026lab4
Уровень 3: - код lab3 перенесён; MinIO (сборка релиза 2025-10-15, по digest) в общем Compose, идемпотентный bootstrap: приватные bucket, отдельный пользователь API с ограниченной политикой - метаданные вложений в PostgreSQL (миграции 0003, 0004), байты в S3; ключ назначает сервер - upload PNG/JPEG: лимиты до чтения/декодирования, MIME по содержимому, проверка целостности, безопасный filename; 201 только после PUT и COMMIT; download потоком через API - attachment_ids в сообщениях (только файлы своего чата, FK в БД) - 503 API больше не повторяется в HAProxy (отказ общей зависимости), S3 не входит в readiness Уровень 4: - 45 тестов вложений: размер, повреждение, подмена MIME, чужой id, файл другого чата, отказы S3/БД - компенсация при сбое COMMIT; слоты загрузки (8 на реплику) против неограниченной памяти - эксперименты: пик RAM при 10-40 одновременных загрузках, оборванная загрузка и сироты; исправлены удержание соединения БД во время загрузки и таймаут отправки в S3 Уровень 5, трек 2: - reconciler: слияние листинга S3 и курсора БД, 7 категорий расхождений, dry-run, grace period, перепроверка перед действием, тормоз --max-deletes, advisory lock; сервис reconciler в Compose - 8 тестов сверки; эксперимент с 70 внедрёнными ошибками на 10 000 объектах (точное обнаружение, безопасное исправление, идемпотентность) и замер стоимости обхода на 1k/10k/50k
320 lines
16 KiB
Python
320 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Трек 2: эксперименты сверки хранилищ на отдельных базе messenger_bench и bucket messenger-attachments-bench.
|
|
|
|
python scripts/reconcile_experiment.py faults --objects 10000 # внедрение ошибок с обеих сторон
|
|
python scripts/reconcile_experiment.py cost --sizes 1000,10000,50000
|
|
|
|
Данные: N вложений по 1 KiB (случайные байты), половина прикреплена к сообщениям (по 10 на сообщение).
|
|
Сверка запускается так же, как в работе: `docker compose run reconciler python -m maintenance.reconcile`
|
|
(контейнер с лимитом 0.25 CPU / 128 MiB), с переопределёнными базой и bucket. Хост — только для генерации
|
|
данных и внедрения ошибок (venv, настройки из .env). Рабочие данные не затрагиваются.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import random
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
BENCH_DB = "messenger_bench"
|
|
BENCH_BUCKET = "messenger-attachments-bench"
|
|
os.chdir(ROOT)
|
|
os.environ["APP_CONFIG__DB__NAME"] = BENCH_DB
|
|
os.environ["APP_CONFIG__S3__BUCKET"] = BENCH_BUCKET
|
|
sys.path.insert(0, str(ROOT / "app"))
|
|
|
|
from alembic import command # noqa: E402
|
|
from alembic.config import Config # noqa: E402
|
|
from minio.deleteobjects import DeleteObject # noqa: E402
|
|
from sqlalchemy import create_engine, delete, func, insert, select, text, update # noqa: E402
|
|
|
|
from core import s3 # noqa: E402
|
|
from core.config import settings # noqa: E402
|
|
from core.db import SessionLocal, build_url, engine # noqa: E402
|
|
from models import Attachment, Chat, ChatMember, Message, MessageAttachment, User # noqa: E402
|
|
|
|
logging.getLogger("urllib3").setLevel(logging.ERROR)
|
|
OBJECT_SIZE = 1024
|
|
PREFIX = "originals/"
|
|
|
|
|
|
# --- данные ---
|
|
|
|
def recreate_database():
|
|
engine.dispose()
|
|
admin = create_engine(build_url().set(database="postgres"), isolation_level="AUTOCOMMIT")
|
|
with admin.connect() as connection:
|
|
connection.execute(text(f'DROP DATABASE IF EXISTS "{BENCH_DB}" WITH (FORCE)'))
|
|
connection.execute(text(f'CREATE DATABASE "{BENCH_DB}"'))
|
|
admin.dispose()
|
|
command.upgrade(Config(str(ROOT / "app" / "alembic.ini")), "head")
|
|
|
|
|
|
def empty_bucket():
|
|
client = s3.get_client()
|
|
batch = []
|
|
for stored in s3.list_objects(""):
|
|
batch.append(DeleteObject(stored.key))
|
|
if len(batch) == 1000:
|
|
list(client.remove_objects(BENCH_BUCKET, batch))
|
|
batch = []
|
|
if batch:
|
|
list(client.remove_objects(BENCH_BUCKET, batch))
|
|
|
|
|
|
def put_many(items):
|
|
"""items: [(key, bytes)] — параллельно, 16 потоков."""
|
|
def put(item):
|
|
key, data = item
|
|
s3.put_object(key, data, "image/png", hashlib.sha256(data).hexdigest())
|
|
with ThreadPoolExecutor(16) as pool:
|
|
list(pool.map(put, items))
|
|
|
|
|
|
def seed(n: int) -> dict:
|
|
started = time.monotonic()
|
|
recreate_database()
|
|
empty_bucket()
|
|
owner, other, chat = uuid.uuid4(), uuid.uuid4(), uuid.uuid4()
|
|
blobs = {uuid.uuid4(): os.urandom(OBJECT_SIZE) for _ in range(n)}
|
|
ids = list(blobs)
|
|
with SessionLocal() as db:
|
|
db.execute(insert(User), [{"id": owner, "username": "bench_owner", "display_name": "o", "password_hash": "x"},
|
|
{"id": other, "username": "bench_other", "display_name": "x", "password_hash": "x"}])
|
|
db.execute(insert(Chat), [{"id": chat, "title": "bench"}])
|
|
db.execute(insert(ChatMember), [{"chat_id": chat, "user_id": owner}, {"chat_id": chat, "user_id": other}])
|
|
rows = [{"id": i, "chat_id": chat, "owner_id": owner, "filename": "f.png", "content_type": "image/png",
|
|
"size_bytes": OBJECT_SIZE, "sha256": hashlib.sha256(blobs[i]).hexdigest(),
|
|
"object_key": f"{PREFIX}{i}"} for i in ids]
|
|
for start in range(0, n, 5000):
|
|
db.execute(insert(Attachment), rows[start:start + 5000])
|
|
# Первая половина прикреплена к сообщениям, по 10 вложений на сообщение.
|
|
referenced = ids[: n // 2]
|
|
messages, links = [], []
|
|
for start in range(0, len(referenced), 10):
|
|
message_id = uuid.uuid4()
|
|
messages.append({"id": message_id, "chat_id": chat, "sender_id": owner, "text": ""})
|
|
links += [{"message_id": message_id, "attachment_id": a, "chat_id": chat, "position": p}
|
|
for p, a in enumerate(referenced[start:start + 10])]
|
|
for start in range(0, len(messages), 5000):
|
|
db.execute(insert(Message), messages[start:start + 5000])
|
|
for start in range(0, len(links), 5000):
|
|
db.execute(insert(MessageAttachment), links[start:start + 5000])
|
|
db.commit()
|
|
db.execute(text("ANALYZE"))
|
|
db.commit()
|
|
put_many([(f"{PREFIX}{i}", data) for i, data in blobs.items()])
|
|
return {"objects": n, "referenced": n // 2, "chat": chat, "owner": owner, "ids": ids,
|
|
"seed_seconds": round(time.monotonic() - started, 1)}
|
|
|
|
|
|
def snapshot() -> dict:
|
|
with SessionLocal() as db:
|
|
rows = db.scalar(select(func.count()).select_from(Attachment))
|
|
referenced = db.scalar(select(func.count(func.distinct(MessageAttachment.attachment_id))))
|
|
return {"objects": sum(1 for _ in s3.list_objects(PREFIX)), "rows": rows, "referenced_rows": referenced}
|
|
|
|
|
|
def existing_keys() -> set:
|
|
return {stored.key for stored in s3.list_objects(PREFIX)}
|
|
|
|
|
|
# --- запуск сверки так же, как в работе ---
|
|
|
|
def reconcile(*flags) -> dict:
|
|
out = subprocess.run(
|
|
["docker", "compose", "run", "--rm", "--no-deps",
|
|
"-e", f"APP_CONFIG__DB__NAME={BENCH_DB}", "-e", f"APP_CONFIG__S3__BUCKET={BENCH_BUCKET}",
|
|
"reconciler", "python", "-m", "maintenance.reconcile", "--samples", "100000", *flags],
|
|
cwd=ROOT, capture_output=True, text=True)
|
|
lines = [line for line in out.stdout.splitlines() if line.startswith("{")]
|
|
if not lines:
|
|
raise SystemExit(f"сверка не вернула отчёт (exit {out.returncode}): {out.stderr[-500:]}")
|
|
return json.loads(lines[-1])
|
|
|
|
|
|
def brief(report: dict) -> dict:
|
|
keep = ("mode", "grace_seconds", "verify_checksums", "objects_scanned", "rows_scanned", "ok", "issues", "actions",
|
|
"seconds", "bytes_read", "peak_rss_mib", "aborted")
|
|
return {k: report[k] for k in keep if k in report}
|
|
|
|
|
|
# --- эксперимент 1: внедрение ошибок ---
|
|
|
|
def faults(args):
|
|
grace = args.grace
|
|
base = seed(args.objects)
|
|
ids, chat, owner = base["ids"], base["chat"], base["owner"]
|
|
random.seed(42)
|
|
referenced, free = ids[: len(ids) // 2], ids[len(ids) // 2:]
|
|
pick_ref = random.sample(referenced, 20)
|
|
pick_free = random.sample(free, 40)
|
|
injected = {
|
|
"orphan_object": set(), # 10 PUT без строки + 10 строк удалено из БД
|
|
"missing_object": set(), # объект удалён, строка не прикреплена
|
|
"missing_object_referenced": set(),
|
|
"size_mismatch": set(),
|
|
"checksum_mismatch": set(),
|
|
"orphan_object_fresh": set(),
|
|
"missing_object_fresh": set(),
|
|
}
|
|
report = {"objects": args.objects, "grace_seconds": grace, "seed_seconds": base["seed_seconds"]}
|
|
# Базовые данные должны стать «старыми» относительно grace.
|
|
time.sleep(grace + 2)
|
|
report["baseline"] = snapshot()
|
|
report["baseline_reconcile"] = brief(reconcile("--grace-seconds", str(grace)))
|
|
|
|
# Старые ошибки: S3-сторона и БД-сторона.
|
|
extra = [f"{PREFIX}{uuid.uuid4()}" for _ in range(10)]
|
|
put_many([(key, os.urandom(OBJECT_SIZE)) for key in extra])
|
|
injected["orphan_object"] |= set(extra)
|
|
with SessionLocal() as db:
|
|
lost_rows = pick_free[:10] # строки потеряны, объекты остались
|
|
db.execute(delete(Attachment).where(Attachment.id.in_(lost_rows)))
|
|
injected["orphan_object"] |= {f"{PREFIX}{i}" for i in lost_rows}
|
|
size_rows = pick_free[10:15] + pick_ref[10:15]
|
|
db.execute(update(Attachment).where(Attachment.id.in_(size_rows)).values(size_bytes=OBJECT_SIZE + 1))
|
|
injected["size_mismatch"] |= {str(i) for i in size_rows}
|
|
db.commit()
|
|
for i in pick_free[15:25]:
|
|
s3.delete_object(f"{PREFIX}{i}")
|
|
injected["missing_object"].add(str(i))
|
|
for i in pick_ref[:10]:
|
|
s3.delete_object(f"{PREFIX}{i}")
|
|
injected["missing_object_referenced"].add(str(i))
|
|
checksum_rows = pick_free[25:30] + pick_ref[15:20]
|
|
put_many([(f"{PREFIX}{i}", os.urandom(OBJECT_SIZE)) for i in checksum_rows]) # тот же размер, другие байты
|
|
injected["checksum_mismatch"] |= {str(i) for i in checksum_rows}
|
|
time.sleep(grace + 2)
|
|
|
|
# Свежие: загрузка «между PUT и COMMIT» и строка новее листинга.
|
|
fresh_objects = [f"{PREFIX}{uuid.uuid4()}" for _ in range(10)]
|
|
put_many([(key, os.urandom(OBJECT_SIZE)) for key in fresh_objects])
|
|
injected["orphan_object_fresh"] |= set(fresh_objects)
|
|
fresh_rows = [uuid.uuid4() for _ in range(10)]
|
|
with SessionLocal() as db:
|
|
db.execute(insert(Attachment), [{"id": i, "chat_id": chat, "owner_id": owner, "filename": "f.png",
|
|
"content_type": "image/png", "size_bytes": OBJECT_SIZE, "sha256": "0" * 64,
|
|
"object_key": f"{PREFIX}{i}"} for i in fresh_rows])
|
|
db.commit()
|
|
injected["missing_object_fresh"] |= {str(i) for i in fresh_rows}
|
|
report["injected"] = {k: len(v) for k, v in injected.items()}
|
|
report["after_injection"] = before = snapshot()
|
|
keys_before = existing_keys()
|
|
|
|
# Быстрые шаги идут подряд, пока «свежие» расхождения моложе grace (каждый запуск ~5 с);
|
|
# глубокая проверка (чтение каждого объекта, ~80 с на 10k) — после apply: она меняет только checksum_mismatch.
|
|
g = ("--grace-seconds", str(grace))
|
|
shallow = reconcile(*g)
|
|
detected = {k: set(v) for k, v in shallow["samples"].items()}
|
|
report["dry_run_shallow"] = brief(shallow)
|
|
report["state_unchanged_after_dry_run"] = snapshot() == before
|
|
|
|
report["apply_with_brake"] = brief(reconcile(*g, "--apply", "--max-deletes", "10"))
|
|
report["state_unchanged_after_brake"] = snapshot() == before
|
|
|
|
report["apply"] = brief(reconcile(*g, "--apply"))
|
|
after = snapshot()
|
|
keys_after = existing_keys()
|
|
report["after_apply"] = after
|
|
report["apply_again"] = brief(reconcile(*g, "--apply"))
|
|
report["fresh_still_fresh_at_apply"] = (report["apply_again"]["issues"]["orphan_object_fresh"] == 10
|
|
and report["apply_again"]["issues"]["missing_object_fresh"] == 10)
|
|
deep = reconcile(*g, "--verify-checksums")
|
|
report["dry_run_deep"] = brief(deep)
|
|
detected["checksum_mismatch"] = set(deep["samples"].get("checksum_mismatch", []))
|
|
report["detection_exact"] = {k: detected.get(k, set()) == injected[k] for k in injected}
|
|
report["safety"] = {
|
|
# Ни одна строка, прикреплённая к сообщению, не удалена.
|
|
"referenced_rows_kept": after["referenced_rows"] == before["referenced_rows"],
|
|
# Удалены ровно сироты; свежие объекты и объекты со строками на месте.
|
|
"deleted_objects_are_exactly_orphans": keys_before - keys_after == injected["orphan_object"],
|
|
"fresh_objects_kept": injected["orphan_object_fresh"] <= keys_after,
|
|
"rows_deleted_exactly_missing_free": before["rows"] - after["rows"] == len(injected["missing_object"]),
|
|
}
|
|
|
|
# Когда grace истёк, «свежие» расхождения становятся обычными и исправляются тем же путём.
|
|
time.sleep(grace + 2)
|
|
report["after_grace_apply"] = brief(reconcile(*g, "--apply"))
|
|
report["final_dry_run"] = brief(reconcile(*g, "--verify-checksums"))
|
|
report["final"] = snapshot()
|
|
|
|
final = report["final_dry_run"]["issues"]
|
|
ok = (all(report["detection_exact"].values()) and report["state_unchanged_after_dry_run"]
|
|
and report["fresh_still_fresh_at_apply"]
|
|
and "aborted" in report["apply_with_brake"] and report["state_unchanged_after_brake"]
|
|
and report["apply"]["actions"] == {"objects_deleted": 20, "rows_deleted": 10, "skipped_on_recheck": 0}
|
|
and report["apply_again"]["actions"] == {"objects_deleted": 0, "rows_deleted": 0, "skipped_on_recheck": 0}
|
|
and all(report["safety"].values())
|
|
and report["after_grace_apply"]["actions"]["objects_deleted"] == 10
|
|
and report["after_grace_apply"]["actions"]["rows_deleted"] == 10
|
|
and final == {"orphan_object": 0, "orphan_object_fresh": 0, "missing_object": 0,
|
|
"missing_object_referenced": 10, "missing_object_fresh": 0,
|
|
"size_mismatch": 10, "checksum_mismatch": 10})
|
|
report["result"] = "PASS" if ok else "FAIL"
|
|
Path(args.output or ROOT / "evidence" / "reconcile-faults.json").write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2, default=str) + "\n", encoding="utf-8")
|
|
print(json.dumps({k: report[k] for k in ("injected", "detection_exact", "safety", "result")}, ensure_ascii=False))
|
|
return 0 if ok else 1
|
|
|
|
|
|
# --- эксперимент 2: стоимость обхода ---
|
|
|
|
def plan_for_rows() -> str:
|
|
with SessionLocal() as db:
|
|
rows = db.execute(text("EXPLAIN SELECT object_key FROM attachments WHERE object_key LIKE 'originals/%' "
|
|
"ORDER BY object_key")).scalars().all()
|
|
return rows[0]
|
|
|
|
|
|
def cost(args):
|
|
report = {"object_size_bytes": OBJECT_SIZE, "reconciler_limits": "0.25 CPU, 128 MiB", "runs": []}
|
|
for n in [int(x) for x in args.sizes.split(",")]:
|
|
base = seed(n)
|
|
entry = {"objects": n, "seed_seconds": base["seed_seconds"], "rows_plan": plan_for_rows(),
|
|
"s3_list_requests": -(-n // 1000), "shallow": [], "deep": []}
|
|
for _ in range(args.repeats):
|
|
entry["shallow"].append(brief(reconcile("--grace-seconds", "0")))
|
|
for _ in range(args.deep_repeats):
|
|
entry["deep"].append(brief(reconcile("--grace-seconds", "0", "--verify-checksums")))
|
|
for mode in ("shallow", "deep"):
|
|
runs = entry[mode]
|
|
if runs:
|
|
seconds = statistics.median(r["seconds"]["scan"] for r in runs)
|
|
entry[f"{mode}_median_seconds"] = round(seconds, 3)
|
|
entry[f"{mode}_objects_per_second"] = round(n / seconds)
|
|
entry[f"{mode}_peak_rss_mib"] = max(r["peak_rss_mib"] for r in runs)
|
|
entry[f"{mode}_all_ok"] = all(r["ok"] == n and sum(r["issues"].values()) == 0 for r in runs)
|
|
report["runs"].append(entry)
|
|
print(json.dumps({k: v for k, v in entry.items() if k not in ("shallow", "deep")}, ensure_ascii=False))
|
|
Path(args.output or ROOT / "evidence" / "reconcile-cost.json").write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
sub = parser.add_subparsers(dest="cmd", required=True)
|
|
f = sub.add_parser("faults")
|
|
f.add_argument("--objects", type=int, default=10000)
|
|
f.add_argument("--grace", type=int, default=20)
|
|
f.add_argument("--output")
|
|
c = sub.add_parser("cost")
|
|
c.add_argument("--sizes", default="1000,10000,50000")
|
|
c.add_argument("--repeats", type=int, default=3)
|
|
c.add_argument("--deep-repeats", type=int, default=2)
|
|
c.add_argument("--output")
|
|
args = parser.parse_args()
|
|
return faults(args) if args.cmd == "faults" else cost(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main() or 0)
|