Files
mtusi2026lab5/student-tests/outbox_gap.py
T
zemiles 13bec4ab8b ЛР 5 — БПИ2303 — Лазарев — на 4
Асинхронная обработка вложений: RabbitMQ 4.1 (quorum-очереди, publisher
confirms, manual ack, DLQ), transactional outbox, отдельный worker.

- upload: оригинал в staging S3, затем одной транзакцией queued + jobs +
  outbox -> 202; API к брокеру не подключается
- relay: публикация outbox с confirm, reaper истёкших аренд, redrive
- worker: claim по строке jobs, аренда с heartbeat и fencing по
  lease_token, transform в подпроцессе с RLIMIT_AS/RLIMIT_CPU/таймаутом,
  thumbnail 256, crop 128, WebP 1280 по детерминированным ключам
- retry с backoff и jitter, постоянные/временные ошибки, реестр failed,
  ops replay (ready не повторяется), DLQ для невалидных сообщений
- janitor: очистка staging и сирот; миграция 004 с backfill вложений lab4
- тесты отказов: окна COMMIT/publish, крахи worker до/после S3 и ack,
  заморозка worker, всплеск загрузок, права брокера; evidence/ и REPORT.md
2026-11-13 16:28:18 +03:00

143 lines
8.3 KiB
Python

#!/usr/bin/env python3
"""Уровень 4: разрыв PostgreSQL <-> broker закрыт transactional outbox. Окна отказа:
python3 student-tests/outbox_gap.py --output evidence/outbox-gap.json
A. relay остановлен: COMMIT есть, publish нет -> outbox не опубликован, очередь пуста;
relay запущен -> задание доходит до ready без ручного создания нового задания;
B. RELAY_FAULT=crash-before-publish: relay выбрал строки outbox и умер до publish
(внедрённый crash между COMMIT и publish) -> рестарт контейнера -> ready;
C. RELAY_FAULT=crash-after-publish: publish + confirm прошли, published_at не записан ->
после рестарта сообщение опубликовано ещё раз -> worker гасит дубль, вариантов 3;
D. API crash-after-commit (FAULT_INJECTION=1): клиент не получил 202, но задание принято
и доходит до ready;
E. API crash-after-staging: оригинал в staging, задания нет (202 не было) -> pending-строку
и staging-объект убирает janitor после grace.
"""
import argparse
import subprocess
import time
from lab5lib import (Client, Failure, Log, SAMPLE_PNG, check, compose, events, get_att, invariants, ops,
queues, recreate, sql, upload, wait_status, wait_until)
def restart_count(service):
ids = compose("ps", "-q", "-a", service).stdout.split()
out = subprocess.run(["docker", "inspect", "-f", "{{.RestartCount}}"] + ids, capture_output=True, text=True)
return [int(x) for x in out.stdout.split()]
def find_by_filename(name):
rows = sql("SELECT id, status, staging_key IS NOT NULL FROM attachments WHERE filename = '%s'" % name)
return rows[0] if rows else None
def raw_upload(c, actor, chat, name, fault):
try:
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], name), [502, 503, 504],
actor=actor, raw=SAMPLE_PNG, headers={"Content-Type": "image/png", "X-Fault-Inject": fault})
return "http %d" % r["status"]
except Failure as exc:
return str(exc)
def main():
p = argparse.ArgumentParser()
p.add_argument("--base-url", default=None)
p.add_argument("--output")
args = p.parse_args()
log = Log("outbox: COMMIT/publish gap and API crash windows")
c = Client(args.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
res = {}
try:
# A ---------------------------------------------------------------------------
compose("stop", "relay")
att = upload(c, alice, chat, SAMPLE_PNG, "gap-relay-stopped.png")
time.sleep(3)
info = ops("inspect", att["id"])
q = queues()["attachments.process.v1"]
check(info["outbox"][0]["published_at"] is None and get_att(c, bob, att["id"])["status"] == "queued",
"A: must stay unpublished")
log.step("A: relay stopped, upload 202", attachment_id=att["id"], outbox=info["outbox"], queue=q)
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "relay")
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 60)
info = ops("inspect", att["id"])
check(ready["status"] == "ready" and len(info["jobs"]) == 1, "A: not ready / extra job")
res["A"] = {"attachment_id": att["id"], "events": events(info), "outbox": info["outbox"],
"jobs": len(info["jobs"]), "invariants": invariants(info)}
log.step("A: relay started -> ready, same single job", **res["A"])
# B ---------------------------------------------------------------------------
recreate("relay", {"RELAY_FAULT": "crash-before-publish"})
rc0 = restart_count("relay")
att = upload(c, alice, chat, SAMPLE_PNG, "gap-crash-before-publish.png")
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 90)
info = ops("inspect", att["id"])
logs = compose("logs", "--no-log-prefix", "relay").stdout
check(ready["status"] == "ready" and "FAULT INJECTION crash-before-publish" in logs, "B")
res["B"] = {"attachment_id": att["id"], "relay_restart_count": [rc0, restart_count("relay")],
"events": events(info), "outbox": info["outbox"], "invariants": invariants(info)}
log.step("B: relay crashed between COMMIT and publish -> restarted -> ready", **res["B"])
# C ---------------------------------------------------------------------------
recreate("relay", {"RELAY_FAULT": "crash-after-publish"})
att = upload(c, alice, chat, SAMPLE_PNG, "gap-crash-after-publish.png")
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 90)
info = wait_until(lambda: (lambda i: i if "duplicate_ignored" in events(i) or
"busy_ignored" in events(i) else None)(ops("inspect", att["id"])),
60, 1, "duplicate handled")
inv = invariants(info)
check(ready["status"] == "ready" and inv["variants_ok"] and inv["final_objects_ok"], "C")
res["C"] = {"attachment_id": att["id"], "relay_restart_count": restart_count("relay"),
"events": events(info), "outbox": info["outbox"], "invariants": inv}
log.step("C: published twice (crash before marking) -> duplicate is a no-op", **res["C"])
recreate("relay")
# D ---------------------------------------------------------------------------
recreate("api", {"FAULT_INJECTION": "1"})
name = "gap-api-crash-after-commit-%d.png" % time.time()
outcome = raw_upload(c, alice, chat, name, "crash-after-commit")
row = wait_until(lambda: find_by_filename(name), 20, what="attachment row")
ready = wait_status(c, bob, row[0], {"ready", "failed"}, 90)
info = ops("inspect", row[0])
check(ready["status"] == "ready", "D")
res["D"] = {"attachment_id": row[0], "client_saw": outcome, "events": events(info),
"invariants": invariants(info)}
log.step("D: API died after COMMIT before 202 -> work still completed", **res["D"])
# E ---------------------------------------------------------------------------
name = "gap-api-crash-after-staging-%d.png" % time.time()
outcome = raw_upload(c, alice, chat, name, "crash-after-staging")
row = wait_until(lambda: find_by_filename(name), 20, what="pending row")
jobs_n = sql("SELECT count(*) FROM jobs WHERE attachment_id = '%s'" % row[0])[0][0]
check(row[1] == "pending" and jobs_n == "0", "E: expected pending row without job")
c.call("GET", "/api/v1/attachments/" + row[0], 404, actor=bob)
log.step("E: API died after staging before COMMIT", attachment_id=row[0], client_saw=outcome,
status=row[1], jobs=int(jobs_n), api_get=404)
time.sleep(3)
dry = compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once", "--dry-run",
"--grace-seconds", "2").stdout
real = compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once",
"--grace-seconds", "2").stdout
import json as _j
dry, real = _j.loads(dry.strip().splitlines()[-1]), _j.loads(real.strip().splitlines()[-1])
gone = find_by_filename(name) is None
check(row[0] in dry["abandoned_ids"] and gone, "E: janitor did not clean")
res["E"] = {"attachment_id": row[0], "janitor_dry_run": {k: dry[k] for k in ("abandoned_pending", "orphan_objects",
"deleted_objects", "deleted_rows")},
"janitor": {k: real[k] for k in ("abandoned_pending", "orphan_objects", "deleted_objects",
"deleted_rows", "missing_objects")}, "row_gone": gone}
log.step("E: janitor removed pending row and staging object", **res["E"])
finally:
recreate("relay")
recreate("api")
log.step("relay and api restored to .env configuration")
log.save(args.output, results=res)
print("PASS: outbox closes COMMIT/publish gap; API crash windows handled")
if __name__ == "__main__":
main()