forked from mtusi2026/mtusi2026lab5
Асинхронная обработка вложений: RabbitMQ 4.1 (quorum-очереди, publisher confirms, manual ack, DLQ), transactional outbox, отдельный worker. - upload: оригинал в staging S3, затем одной транзакцией queued + jobs + outbox -> 202; API к брокеру не подключается - relay: публикация outbox с confirm, reaper истёкших аренд, redrive - worker: claim по строке jobs, аренда с heartbeat и fencing по lease_token, transform в подпроцессе с RLIMIT_AS/RLIMIT_CPU/таймаутом, thumbnail 256, crop 128, WebP 1280 по детерминированным ключам - retry с backoff и jitter, постоянные/временные ошибки, реестр failed, ops replay (ready не повторяется), DLQ для невалидных сообщений - janitor: очистка staging и сирот; миграция 004 с backfill вложений lab4 - тесты отказов: окна COMMIT/publish, крахи worker до/после S3 и ack, заморозка worker, всплеск загрузок, права брокера; evidence/ и REPORT.md
143 lines
8.3 KiB
Python
143 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Уровень 4: разрыв PostgreSQL <-> broker закрыт transactional outbox. Окна отказа:
|
|
|
|
python3 student-tests/outbox_gap.py --output evidence/outbox-gap.json
|
|
|
|
A. relay остановлен: COMMIT есть, publish нет -> outbox не опубликован, очередь пуста;
|
|
relay запущен -> задание доходит до ready без ручного создания нового задания;
|
|
B. RELAY_FAULT=crash-before-publish: relay выбрал строки outbox и умер до publish
|
|
(внедрённый crash между COMMIT и publish) -> рестарт контейнера -> ready;
|
|
C. RELAY_FAULT=crash-after-publish: publish + confirm прошли, published_at не записан ->
|
|
после рестарта сообщение опубликовано ещё раз -> worker гасит дубль, вариантов 3;
|
|
D. API crash-after-commit (FAULT_INJECTION=1): клиент не получил 202, но задание принято
|
|
и доходит до ready;
|
|
E. API crash-after-staging: оригинал в staging, задания нет (202 не было) -> pending-строку
|
|
и staging-объект убирает janitor после grace.
|
|
"""
|
|
import argparse
|
|
import subprocess
|
|
import time
|
|
|
|
from lab5lib import (Client, Failure, Log, SAMPLE_PNG, check, compose, events, get_att, invariants, ops,
|
|
queues, recreate, sql, upload, wait_status, wait_until)
|
|
|
|
|
|
def restart_count(service):
|
|
ids = compose("ps", "-q", "-a", service).stdout.split()
|
|
out = subprocess.run(["docker", "inspect", "-f", "{{.RestartCount}}"] + ids, capture_output=True, text=True)
|
|
return [int(x) for x in out.stdout.split()]
|
|
|
|
|
|
def find_by_filename(name):
|
|
rows = sql("SELECT id, status, staging_key IS NOT NULL FROM attachments WHERE filename = '%s'" % name)
|
|
return rows[0] if rows else None
|
|
|
|
|
|
def raw_upload(c, actor, chat, name, fault):
|
|
try:
|
|
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], name), [502, 503, 504],
|
|
actor=actor, raw=SAMPLE_PNG, headers={"Content-Type": "image/png", "X-Fault-Inject": fault})
|
|
return "http %d" % r["status"]
|
|
except Failure as exc:
|
|
return str(exc)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--base-url", default=None)
|
|
p.add_argument("--output")
|
|
args = p.parse_args()
|
|
log = Log("outbox: COMMIT/publish gap and API crash windows")
|
|
c = Client(args.base_url, timeout=30)
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
res = {}
|
|
try:
|
|
# A ---------------------------------------------------------------------------
|
|
compose("stop", "relay")
|
|
att = upload(c, alice, chat, SAMPLE_PNG, "gap-relay-stopped.png")
|
|
time.sleep(3)
|
|
info = ops("inspect", att["id"])
|
|
q = queues()["attachments.process.v1"]
|
|
check(info["outbox"][0]["published_at"] is None and get_att(c, bob, att["id"])["status"] == "queued",
|
|
"A: must stay unpublished")
|
|
log.step("A: relay stopped, upload 202", attachment_id=att["id"], outbox=info["outbox"], queue=q)
|
|
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "relay")
|
|
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 60)
|
|
info = ops("inspect", att["id"])
|
|
check(ready["status"] == "ready" and len(info["jobs"]) == 1, "A: not ready / extra job")
|
|
res["A"] = {"attachment_id": att["id"], "events": events(info), "outbox": info["outbox"],
|
|
"jobs": len(info["jobs"]), "invariants": invariants(info)}
|
|
log.step("A: relay started -> ready, same single job", **res["A"])
|
|
|
|
# B ---------------------------------------------------------------------------
|
|
recreate("relay", {"RELAY_FAULT": "crash-before-publish"})
|
|
rc0 = restart_count("relay")
|
|
att = upload(c, alice, chat, SAMPLE_PNG, "gap-crash-before-publish.png")
|
|
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 90)
|
|
info = ops("inspect", att["id"])
|
|
logs = compose("logs", "--no-log-prefix", "relay").stdout
|
|
check(ready["status"] == "ready" and "FAULT INJECTION crash-before-publish" in logs, "B")
|
|
res["B"] = {"attachment_id": att["id"], "relay_restart_count": [rc0, restart_count("relay")],
|
|
"events": events(info), "outbox": info["outbox"], "invariants": invariants(info)}
|
|
log.step("B: relay crashed between COMMIT and publish -> restarted -> ready", **res["B"])
|
|
|
|
# C ---------------------------------------------------------------------------
|
|
recreate("relay", {"RELAY_FAULT": "crash-after-publish"})
|
|
att = upload(c, alice, chat, SAMPLE_PNG, "gap-crash-after-publish.png")
|
|
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 90)
|
|
info = wait_until(lambda: (lambda i: i if "duplicate_ignored" in events(i) or
|
|
"busy_ignored" in events(i) else None)(ops("inspect", att["id"])),
|
|
60, 1, "duplicate handled")
|
|
inv = invariants(info)
|
|
check(ready["status"] == "ready" and inv["variants_ok"] and inv["final_objects_ok"], "C")
|
|
res["C"] = {"attachment_id": att["id"], "relay_restart_count": restart_count("relay"),
|
|
"events": events(info), "outbox": info["outbox"], "invariants": inv}
|
|
log.step("C: published twice (crash before marking) -> duplicate is a no-op", **res["C"])
|
|
recreate("relay")
|
|
|
|
# D ---------------------------------------------------------------------------
|
|
recreate("api", {"FAULT_INJECTION": "1"})
|
|
name = "gap-api-crash-after-commit-%d.png" % time.time()
|
|
outcome = raw_upload(c, alice, chat, name, "crash-after-commit")
|
|
row = wait_until(lambda: find_by_filename(name), 20, what="attachment row")
|
|
ready = wait_status(c, bob, row[0], {"ready", "failed"}, 90)
|
|
info = ops("inspect", row[0])
|
|
check(ready["status"] == "ready", "D")
|
|
res["D"] = {"attachment_id": row[0], "client_saw": outcome, "events": events(info),
|
|
"invariants": invariants(info)}
|
|
log.step("D: API died after COMMIT before 202 -> work still completed", **res["D"])
|
|
|
|
# E ---------------------------------------------------------------------------
|
|
name = "gap-api-crash-after-staging-%d.png" % time.time()
|
|
outcome = raw_upload(c, alice, chat, name, "crash-after-staging")
|
|
row = wait_until(lambda: find_by_filename(name), 20, what="pending row")
|
|
jobs_n = sql("SELECT count(*) FROM jobs WHERE attachment_id = '%s'" % row[0])[0][0]
|
|
check(row[1] == "pending" and jobs_n == "0", "E: expected pending row without job")
|
|
c.call("GET", "/api/v1/attachments/" + row[0], 404, actor=bob)
|
|
log.step("E: API died after staging before COMMIT", attachment_id=row[0], client_saw=outcome,
|
|
status=row[1], jobs=int(jobs_n), api_get=404)
|
|
time.sleep(3)
|
|
dry = compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once", "--dry-run",
|
|
"--grace-seconds", "2").stdout
|
|
real = compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once",
|
|
"--grace-seconds", "2").stdout
|
|
import json as _j
|
|
dry, real = _j.loads(dry.strip().splitlines()[-1]), _j.loads(real.strip().splitlines()[-1])
|
|
gone = find_by_filename(name) is None
|
|
check(row[0] in dry["abandoned_ids"] and gone, "E: janitor did not clean")
|
|
res["E"] = {"attachment_id": row[0], "janitor_dry_run": {k: dry[k] for k in ("abandoned_pending", "orphan_objects",
|
|
"deleted_objects", "deleted_rows")},
|
|
"janitor": {k: real[k] for k in ("abandoned_pending", "orphan_objects", "deleted_objects",
|
|
"deleted_rows", "missing_objects")}, "row_gone": gone}
|
|
log.step("E: janitor removed pending row and staging object", **res["E"])
|
|
finally:
|
|
recreate("relay")
|
|
recreate("api")
|
|
log.step("relay and api restored to .env configuration")
|
|
log.save(args.output, results=res)
|
|
print("PASS: outbox closes COMMIT/publish gap; API crash windows handled")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|