forked from mtusi2026/mtusi2026lab5
Асинхронная обработка вложений: RabbitMQ 4.1 (quorum-очереди, publisher confirms, manual ack, DLQ), transactional outbox, отдельный worker. - upload: оригинал в staging S3, затем одной транзакцией queued + jobs + outbox -> 202; API к брокеру не подключается - relay: публикация outbox с confirm, reaper истёкших аренд, redrive - worker: claim по строке jobs, аренда с heartbeat и fencing по lease_token, transform в подпроцессе с RLIMIT_AS/RLIMIT_CPU/таймаутом, thumbnail 256, crop 128, WebP 1280 по детерминированным ключам - retry с backoff и jitter, постоянные/временные ошибки, реестр failed, ops replay (ready не повторяется), DLQ для невалидных сообщений - janitor: очистка staging и сирот; миграция 004 с backfill вложений lab4 - тесты отказов: окна COMMIT/publish, крахи worker до/после S3 и ack, заморозка worker, всплеск загрузок, права брокера; evidence/ и REPORT.md
180 lines
8.8 KiB
Python
180 lines
8.8 KiB
Python
#!/usr/bin/env python3
|
||
"""Отказы между записью в S3 и PostgreSQL и очистка janitor (уровень 4).
|
||
|
||
Требует API с FAULT_INJECTION=1 (учебный хук, по умолчанию выключен):
|
||
FAULT_INJECTION=1 docker compose --env-file .env up -d --no-deps --wait api
|
||
python3 student-tests/consistency_faults.py --output evidence/consistency-faults.json
|
||
docker compose --env-file .env up -d --no-deps --wait api # вернуть FAULT_INJECTION=0
|
||
|
||
Сценарии (каждый — реальный HTTP upload через LB):
|
||
db-fail-after-s3-put PUT прошёл, «COMMIT» ready упал -> 503; компенсация удаляет объект
|
||
и pending-строку -> ничего не остаётся;
|
||
crash-after-s3-put процесс API умирает (os._exit) после PUT -> pending + объект;
|
||
crash-after-pending процесс умирает до PUT -> pending без объекта;
|
||
обрыв клиента Content-Length 5 MiB, отправлен 1 MiB, сокет закрыт -> ничего;
|
||
orphan pending-строку удаляем вручную -> объект без строки;
|
||
missing объект ready-вложения удаляем в S3 -> только отчёт, download = 503.
|
||
Затем janitor: dry-run и боевой проход с grace 3600 (свежее не трогает), ожидание
|
||
grace-seconds, dry-run -> боевой -> повторный (идемпотентность). Контрольное ready-вложение
|
||
должно остаться доступным побайтно.
|
||
"""
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / "tests"))
|
||
from client import Client, check # noqa: E402
|
||
|
||
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||
log = []
|
||
|
||
|
||
def sh(*cmd, timeout=120):
|
||
r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=timeout)
|
||
if r.returncode != 0:
|
||
raise SystemExit("command failed: %s\n%s" % (" ".join(cmd[:4]), r.stderr[-500:]))
|
||
return r.stdout
|
||
|
||
|
||
def psql(sql):
|
||
return sh("docker", "compose", "exec", "-T", "db", "sh", "-c",
|
||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "$0"', sql).strip()
|
||
|
||
|
||
def janitor(grace, dry_run):
|
||
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "janitor", "python", "/app/src/janitor.py",
|
||
"--once", "--grace-seconds", str(grace)] + (["--dry-run"] if dry_run else [])
|
||
out = sh(*cmd)
|
||
report = json.loads([line for line in out.splitlines() if line.startswith("{")][-1])
|
||
keep = ("dry_run", "grace_seconds", "rows_by_status", "objects_total", "abandoned_pending",
|
||
"orphan_objects", "missing_objects", "deleted_objects", "deleted_rows", "errors", "duration_ms")
|
||
return {k: report.get(k) for k in keep}
|
||
|
||
|
||
def step(name, **data):
|
||
entry = dict(step=name, t=round(time.time(), 1), **data)
|
||
log.append(entry)
|
||
print(json.dumps(entry, ensure_ascii=False), flush=True)
|
||
return entry
|
||
|
||
|
||
def wait_api_healthy():
|
||
deadline = time.time() + 90
|
||
while time.time() < deadline:
|
||
states = sh("docker", "compose", "ps", "-a", "--format", "{{.Health}}", "api").split()
|
||
if states and all(s == "healthy" for s in states):
|
||
time.sleep(3) # HAProxy rise
|
||
return
|
||
time.sleep(1)
|
||
raise SystemExit("api did not become healthy")
|
||
|
||
|
||
def upload(c, actor, chat, fault=None, expected=201):
|
||
h = {"Content-Type": "image/png"}
|
||
if fault:
|
||
h["X-Fault-Inject"] = fault
|
||
try:
|
||
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=f.png" % chat["id"], expected,
|
||
actor=actor, raw=PNG, headers=h, schema="Attachment" if expected == 201 else None)
|
||
return r["status"], r["json"]
|
||
except Exception as exc: # обрыв соединения при crash-* тоже допустимый исход
|
||
return type(exc).__name__, str(exc)[:120]
|
||
|
||
|
||
def interrupted_upload(base, token, chat_id):
|
||
host, port = base.split("//")[1].split(":")
|
||
s = socket.create_connection((host, int(port)), timeout=10)
|
||
s.sendall(("POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: %s\r\n"
|
||
"Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n"
|
||
% (chat_id, host, token, 5 * 1024 * 1024)).encode())
|
||
s.sendall(PNG[:8] + b"\0" * (1024 * 1024 - 8))
|
||
s.close() # клиент «пропал» на середине
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||
ap.add_argument("--grace-seconds", type=int, default=20)
|
||
ap.add_argument("--output")
|
||
args = ap.parse_args()
|
||
env = sh("docker", "compose", "exec", "-T", "api", "printenv", "FAULT_INJECTION").strip()
|
||
check(env == "1", "start api with FAULT_INJECTION=1 first (see docstring)")
|
||
c = Client(args.base_url, timeout=30)
|
||
alice, bob, _eve, chat = c.bootstrap()
|
||
started = psql("SELECT now()")
|
||
base = janitor(3600, True)
|
||
step("baseline janitor dry-run", report=base)
|
||
|
||
_, control = upload(c, alice, chat)
|
||
_, victim = upload(c, alice, chat)
|
||
step("control + victim uploaded (ready)", control=control["id"], victim=victim["id"])
|
||
|
||
st, body = upload(c, alice, chat, "db-fail-after-s3-put", expected=503)
|
||
after_comp = janitor(3600, True)
|
||
step("db-fail-after-s3-put -> compensation", status=st, report=after_comp,
|
||
leftover_pending=after_comp["rows_by_status"].get("pending", 0) - base["rows_by_status"].get("pending", 0),
|
||
leftover_objects=after_comp["objects_total"] - base["objects_total"] - 2)
|
||
|
||
for fault in ("crash-after-s3-put", "crash-after-s3-put", "crash-after-pending"):
|
||
st, body = upload(c, alice, chat, fault, expected=[502, 503])
|
||
step(fault, client_saw=st)
|
||
wait_api_healthy()
|
||
|
||
interrupted_upload(args.base_url, alice["token"], chat["id"])
|
||
time.sleep(2)
|
||
step("client aborted upload at 1 of 5 MiB", report=janitor(3600, True))
|
||
|
||
pending = psql("SELECT id || ' ' || (object_key IS NOT NULL) FROM attachments WHERE status='pending' "
|
||
"AND created_at >= '%s' ORDER BY created_at" % started).splitlines()
|
||
step("pending rows left by crashes", pending=len(pending))
|
||
check(len(pending) == 3, "expected 3 pending rows from crashes")
|
||
orphan_id = pending[0].split()[0]
|
||
psql("DELETE FROM attachments WHERE id = '%s'" % orphan_id)
|
||
step("orphan injected: pending row deleted, object kept", row=orphan_id)
|
||
victim_key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % victim["id"])
|
||
sh("docker", "compose", "--profile", "tools", "run", "--rm", "--entrypoint", "python", "s3-check", "-c",
|
||
"import sys; sys.argv=['x']; exec(open('/app/student-tests/s3_private_check.py').read().split('results = []')[0]);"
|
||
"client('api').remove_object(BUCKET, %r)" % victim_key)
|
||
dl = c.call("GET", "/api/v1/attachments/%s/content" % victim["id"], 503, actor=bob)
|
||
step("missing injected: victim object deleted in S3", download_status=dl["status"])
|
||
|
||
fresh_dry = janitor(3600, True)
|
||
fresh_run = janitor(3600, False)
|
||
step("janitor grace=3600 (all fresh): dry-run", report=fresh_dry)
|
||
step("janitor grace=3600 (all fresh): real run deletes nothing", report=fresh_run)
|
||
check(fresh_run["deleted_objects"] == 0 and fresh_run["deleted_rows"] == 0, "fresh data deleted")
|
||
|
||
time.sleep(args.grace_seconds + 2)
|
||
dry = janitor(args.grace_seconds, True)
|
||
step("after grace: dry-run", report=dry)
|
||
real = janitor(args.grace_seconds, False)
|
||
step("after grace: real run", report=real)
|
||
again = janitor(args.grace_seconds, False)
|
||
step("after grace: repeated run (idempotent)", report=again)
|
||
d = c.call("GET", "/api/v1/attachments/%s/content" % control["id"], actor=bob)
|
||
control_ok = hashlib.sha256(d["bytes"]).hexdigest() == control["sha256"]
|
||
still = psql("SELECT count(*) FROM attachments WHERE status='pending' AND created_at >= '%s'" % started)
|
||
step("control attachment still intact", sha256_ok=control_ok, pending_left=int(still))
|
||
|
||
ok = (log[2]["leftover_pending"] == 0 and log[2]["leftover_objects"] == 0
|
||
and dry["abandoned_pending"] >= 2 and dry["orphan_objects"] >= 1 and dry["missing_objects"] >= 1
|
||
and dry["deleted_objects"] == 0
|
||
and real["deleted_rows"] >= 2 and again["abandoned_pending"] == 0 and again["orphan_objects"] == 0
|
||
and again["deleted_objects"] == 0 and again["missing_objects"] >= 1
|
||
and control_ok and int(still) == 0)
|
||
print(json.dumps({"pass": ok}))
|
||
if args.output:
|
||
Path(args.output).write_text(json.dumps({"pass": ok, "steps": log}, ensure_ascii=False, indent=1))
|
||
return 0 if ok else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|