Files
mtusi2026lab5/student-tests/consistency_faults.py
T
zemiles 13bec4ab8b ЛР 5 — БПИ2303 — Лазарев — на 4
Асинхронная обработка вложений: RabbitMQ 4.1 (quorum-очереди, publisher
confirms, manual ack, DLQ), transactional outbox, отдельный worker.

- upload: оригинал в staging S3, затем одной транзакцией queued + jobs +
  outbox -> 202; API к брокеру не подключается
- relay: публикация outbox с confirm, reaper истёкших аренд, redrive
- worker: claim по строке jobs, аренда с heartbeat и fencing по
  lease_token, transform в подпроцессе с RLIMIT_AS/RLIMIT_CPU/таймаутом,
  thumbnail 256, crop 128, WebP 1280 по детерминированным ключам
- retry с backoff и jitter, постоянные/временные ошибки, реестр failed,
  ops replay (ready не повторяется), DLQ для невалидных сообщений
- janitor: очистка staging и сирот; миграция 004 с backfill вложений lab4
- тесты отказов: окна COMMIT/publish, крахи worker до/после S3 и ack,
  заморозка worker, всплеск загрузок, права брокера; evidence/ и REPORT.md
2026-11-13 16:28:18 +03:00

180 lines
8.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Отказы между записью в S3 и PostgreSQL и очистка janitor (уровень 4).
Требует API с FAULT_INJECTION=1 (учебный хук, по умолчанию выключен):
FAULT_INJECTION=1 docker compose --env-file .env up -d --no-deps --wait api
python3 student-tests/consistency_faults.py --output evidence/consistency-faults.json
docker compose --env-file .env up -d --no-deps --wait api # вернуть FAULT_INJECTION=0
Сценарии (каждый — реальный HTTP upload через LB):
db-fail-after-s3-put PUT прошёл, «COMMIT» ready упал -> 503; компенсация удаляет объект
и pending-строку -> ничего не остаётся;
crash-after-s3-put процесс API умирает (os._exit) после PUT -> pending + объект;
crash-after-pending процесс умирает до PUT -> pending без объекта;
обрыв клиента Content-Length 5 MiB, отправлен 1 MiB, сокет закрыт -> ничего;
orphan pending-строку удаляем вручную -> объект без строки;
missing объект ready-вложения удаляем в S3 -> только отчёт, download = 503.
Затем janitor: dry-run и боевой проход с grace 3600 (свежее не трогает), ожидание
grace-seconds, dry-run -> боевой -> повторный (идемпотентность). Контрольное ready-вложение
должно остаться доступным побайтно.
"""
import argparse
import hashlib
import json
import socket
import subprocess
import sys
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
log = []
def sh(*cmd, timeout=120):
r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=timeout)
if r.returncode != 0:
raise SystemExit("command failed: %s\n%s" % (" ".join(cmd[:4]), r.stderr[-500:]))
return r.stdout
def psql(sql):
return sh("docker", "compose", "exec", "-T", "db", "sh", "-c",
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "$0"', sql).strip()
def janitor(grace, dry_run):
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "janitor", "python", "/app/src/janitor.py",
"--once", "--grace-seconds", str(grace)] + (["--dry-run"] if dry_run else [])
out = sh(*cmd)
report = json.loads([line for line in out.splitlines() if line.startswith("{")][-1])
keep = ("dry_run", "grace_seconds", "rows_by_status", "objects_total", "abandoned_pending",
"orphan_objects", "missing_objects", "deleted_objects", "deleted_rows", "errors", "duration_ms")
return {k: report.get(k) for k in keep}
def step(name, **data):
entry = dict(step=name, t=round(time.time(), 1), **data)
log.append(entry)
print(json.dumps(entry, ensure_ascii=False), flush=True)
return entry
def wait_api_healthy():
deadline = time.time() + 90
while time.time() < deadline:
states = sh("docker", "compose", "ps", "-a", "--format", "{{.Health}}", "api").split()
if states and all(s == "healthy" for s in states):
time.sleep(3) # HAProxy rise
return
time.sleep(1)
raise SystemExit("api did not become healthy")
def upload(c, actor, chat, fault=None, expected=201):
h = {"Content-Type": "image/png"}
if fault:
h["X-Fault-Inject"] = fault
try:
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=f.png" % chat["id"], expected,
actor=actor, raw=PNG, headers=h, schema="Attachment" if expected == 201 else None)
return r["status"], r["json"]
except Exception as exc: # обрыв соединения при crash-* тоже допустимый исход
return type(exc).__name__, str(exc)[:120]
def interrupted_upload(base, token, chat_id):
host, port = base.split("//")[1].split(":")
s = socket.create_connection((host, int(port)), timeout=10)
s.sendall(("POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: %s\r\n"
"Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n"
% (chat_id, host, token, 5 * 1024 * 1024)).encode())
s.sendall(PNG[:8] + b"\0" * (1024 * 1024 - 8))
s.close() # клиент «пропал» на середине
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--base-url", default="http://localhost:8080")
ap.add_argument("--grace-seconds", type=int, default=20)
ap.add_argument("--output")
args = ap.parse_args()
env = sh("docker", "compose", "exec", "-T", "api", "printenv", "FAULT_INJECTION").strip()
check(env == "1", "start api with FAULT_INJECTION=1 first (see docstring)")
c = Client(args.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
started = psql("SELECT now()")
base = janitor(3600, True)
step("baseline janitor dry-run", report=base)
_, control = upload(c, alice, chat)
_, victim = upload(c, alice, chat)
step("control + victim uploaded (ready)", control=control["id"], victim=victim["id"])
st, body = upload(c, alice, chat, "db-fail-after-s3-put", expected=503)
after_comp = janitor(3600, True)
step("db-fail-after-s3-put -> compensation", status=st, report=after_comp,
leftover_pending=after_comp["rows_by_status"].get("pending", 0) - base["rows_by_status"].get("pending", 0),
leftover_objects=after_comp["objects_total"] - base["objects_total"] - 2)
for fault in ("crash-after-s3-put", "crash-after-s3-put", "crash-after-pending"):
st, body = upload(c, alice, chat, fault, expected=[502, 503])
step(fault, client_saw=st)
wait_api_healthy()
interrupted_upload(args.base_url, alice["token"], chat["id"])
time.sleep(2)
step("client aborted upload at 1 of 5 MiB", report=janitor(3600, True))
pending = psql("SELECT id || ' ' || (object_key IS NOT NULL) FROM attachments WHERE status='pending' "
"AND created_at >= '%s' ORDER BY created_at" % started).splitlines()
step("pending rows left by crashes", pending=len(pending))
check(len(pending) == 3, "expected 3 pending rows from crashes")
orphan_id = pending[0].split()[0]
psql("DELETE FROM attachments WHERE id = '%s'" % orphan_id)
step("orphan injected: pending row deleted, object kept", row=orphan_id)
victim_key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % victim["id"])
sh("docker", "compose", "--profile", "tools", "run", "--rm", "--entrypoint", "python", "s3-check", "-c",
"import sys; sys.argv=['x']; exec(open('/app/student-tests/s3_private_check.py').read().split('results = []')[0]);"
"client('api').remove_object(BUCKET, %r)" % victim_key)
dl = c.call("GET", "/api/v1/attachments/%s/content" % victim["id"], 503, actor=bob)
step("missing injected: victim object deleted in S3", download_status=dl["status"])
fresh_dry = janitor(3600, True)
fresh_run = janitor(3600, False)
step("janitor grace=3600 (all fresh): dry-run", report=fresh_dry)
step("janitor grace=3600 (all fresh): real run deletes nothing", report=fresh_run)
check(fresh_run["deleted_objects"] == 0 and fresh_run["deleted_rows"] == 0, "fresh data deleted")
time.sleep(args.grace_seconds + 2)
dry = janitor(args.grace_seconds, True)
step("after grace: dry-run", report=dry)
real = janitor(args.grace_seconds, False)
step("after grace: real run", report=real)
again = janitor(args.grace_seconds, False)
step("after grace: repeated run (idempotent)", report=again)
d = c.call("GET", "/api/v1/attachments/%s/content" % control["id"], actor=bob)
control_ok = hashlib.sha256(d["bytes"]).hexdigest() == control["sha256"]
still = psql("SELECT count(*) FROM attachments WHERE status='pending' AND created_at >= '%s'" % started)
step("control attachment still intact", sha256_ok=control_ok, pending_left=int(still))
ok = (log[2]["leftover_pending"] == 0 and log[2]["leftover_objects"] == 0
and dry["abandoned_pending"] >= 2 and dry["orphan_objects"] >= 1 and dry["missing_objects"] >= 1
and dry["deleted_objects"] == 0
and real["deleted_rows"] >= 2 and again["abandoned_pending"] == 0 and again["orphan_objects"] == 0
and again["deleted_objects"] == 0 and again["missing_objects"] >= 1
and control_ok and int(still) == 0)
print(json.dumps({"pass": ok}))
if args.output:
Path(args.output).write_text(json.dumps({"pass": ok, "steps": log}, ensure_ascii=False, indent=1))
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main())