Files
mtusi2026lab5/student-tests/worker_faults.py
T
zemiles 13bec4ab8b ЛР 5 — БПИ2303 — Лазарев — на 4
Асинхронная обработка вложений: RabbitMQ 4.1 (quorum-очереди, publisher
confirms, manual ack, DLQ), transactional outbox, отдельный worker.

- upload: оригинал в staging S3, затем одной транзакцией queued + jobs +
  outbox -> 202; API к брокеру не подключается
- relay: публикация outbox с confirm, reaper истёкших аренд, redrive
- worker: claim по строке jobs, аренда с heartbeat и fencing по
  lease_token, transform в подпроцессе с RLIMIT_AS/RLIMIT_CPU/таймаутом,
  thumbnail 256, crop 128, WebP 1280 по детерминированным ключам
- retry с backoff и jitter, постоянные/временные ошибки, реестр failed,
  ops replay (ready не повторяется), DLQ для невалидных сообщений
- janitor: очистка staging и сирот; миграция 004 с backfill вложений lab4
- тесты отказов: окна COMMIT/publish, крахи worker до/после S3 и ack,
  заморозка worker, всплеск загрузок, права брокера; evidence/ и REPORT.md
2026-11-13 16:28:18 +03:00

148 lines
9.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Уровень 4: отказы worker, retry/backoff, failed + replay, DLQ.
python3 student-tests/worker_faults.py --output evidence/worker-faults.json
На время теста worker пересоздаются с WORKER_FAULT_INJECTION=1 и JOB_RETRY_BASE_SECONDS=1
(короче ожидание backoff); в конце возвращаются к значениям из .env.
Точка отказа задаётся именем файла fault-<point>.png (src/worker.py).
crash-after-claim процесс умер в processing до любых эффектов
crash-after-s3-put все объекты записаны в S3, COMMIT ready не сделан, ack не отправлен
crash-after-commit ready зафиксирован, ack не отправлен (и staging не удалён)
transient-2 временная ошибка в попытках 1–2: backoff с jitter, ready с 3-й
transient-always временная ошибка до replay: 5 попыток -> failed(retry_exhausted)
-> content 409, прикрепление 409 -> ops replay -> ready; повторный replay отклонён
staging tamper staging подменён до обработки -> failed(staging_corrupted) без повторов
poison message невалидное сообщение -> reject -> DLQ -> ops dlq --purge-invalid
Инварианты после каждого: 3 варианта, 4 конечных объекта, staging убран (с janitor).
"""
import argparse
import json
import subprocess
from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, events, get_att, invariants, ops, png, queues,
recreate, upload, wait_status, wait_until)
FAULT_ENV = {"WORKER_FAULT_INJECTION": "1", "JOB_RETRY_BASE_SECONDS": "1", "JOB_RETRY_MAX_SECONDS": "8"}
def inspect_until(att_id, pred, timeout=60, what="state"):
return wait_until(lambda: (lambda i: i if pred(i) else None)(ops("inspect", att_id)), timeout, 1, what)
def main():
p = argparse.ArgumentParser()
p.add_argument("--base-url", default=None)
p.add_argument("--output")
args = p.parse_args()
log = Log("worker faults, retries, failed registry, replay, DLQ")
c = Client(args.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
recreate("worker", FAULT_ENV)
log.step("workers recreated with fault injection", env=FAULT_ENV)
report = {}
try:
# --- крахи процесса в разных окнах -------------------------------------------------
for point in ("crash-after-claim", "crash-after-s3-put", "crash-after-commit"):
att = upload(c, alice, chat, SAMPLE_PNG, "fault-%s.png" % point)
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
check(ready["status"] == "ready", point + ": not ready")
if point == "crash-after-commit":
# ready уже был зафиксирован; redelivery после рестарта -> no-op ack.
inspect_until(att["id"], lambda i: "duplicate_ignored" in events(i), 60, "redelivery no-op")
compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once")
info = ops("inspect", att["id"])
inv = invariants(info)
check(inv["variants_ok"] and inv["final_objects_ok"] and not inv["staging_left"], point + ": invariants")
report[point] = {"attachment_id": att["id"], "events": events(info), "invariants": inv}
log.step(point, events=events(info), invariants=inv)
ids = compose("ps", "-q", "worker").stdout.split()
rc = subprocess.run(["docker", "inspect", "-f", "{{.Name}} {{.RestartCount}} {{.State.Status}}"] + ids,
capture_output=True, text=True).stdout.split("\n")
log.step("worker containers after crashes (restart: unless-stopped)", containers=[x for x in rc if x])
# --- временные ошибки: backoff + jitter ---------------------------------------------
att = upload(c, alice, chat, SAMPLE_PNG, "fault-transient-2.png")
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
check(ready["status"] == "ready", "transient-2 not ready")
info = ops("inspect", att["id"])
delays = [e["detail"]["delay_seconds"] for e in info["events"] if e["event"] == "retry_scheduled"]
claims = [e["at"] for e in info["events"] if e["event"] == "claimed"]
bounds = [[0.5 * min(8, 2 ** (n - 1)), min(8, 2 ** (n - 1))] for n in (1, 2)]
check(len(delays) == 2 and all(lo <= d <= hi for d, (lo, hi) in zip(delays, bounds)), "backoff bounds")
inv = invariants(info)
check(inv["attempt"] == 3 and inv["variants_ok"], "transient-2 attempts")
report["transient-2"] = {"attachment_id": att["id"], "delays": delays, "bounds": bounds, "claims_at": claims,
"invariants": inv}
log.step("transient-2: retried with backoff", delays=delays, bounds=bounds, invariants=inv)
# --- исчерпание попыток -> failed -> replay ----------------------------------------
att = upload(c, alice, chat, SAMPLE_PNG, "fault-transient-always.png")
failed = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=120)
check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", "expected retry_exhausted")
r409 = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob)
m409 = c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice,
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": "f-" + att["id"]})
registry = ops("failed")
in_registry = [r for r in registry["failed"] if r["attachment_id"] == att["id"]]
check(in_registry and in_registry[0]["replayable"], "failed registry")
info = ops("inspect", att["id"])
log.step("transient-always: failed after max attempts", error_code=failed["error_code"],
attempts=info["jobs"][0]["attempt"], content=r409["status"], attach=m409["status"],
registry_entry=in_registry[0])
dry = ops("replay", att["id"], "--dry-run")
rep = ops("replay", att["id"], "--reason", "test: S3 restored")
check(rep["replay"][0]["result"] == "replayed", "replay")
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
check(ready["status"] == "ready", "not ready after replay")
again = ops("replay", att["id"])
check(again["replay"][0]["result"] == "refused_status_ready", "replay of ready must be refused")
info = ops("inspect", att["id"])
inv = invariants(info)
check(inv["variants_ok"] and inv["final_objects_ok"], "replay invariants")
report["transient-always+replay"] = {"attachment_id": att["id"], "dry_run": dry["replay"][0]["result"],
"replay": rep["replay"][0]["result"],
"second_replay": again["replay"][0]["result"],
"replay_count": info["jobs"][0]["replay_count"],
"events": events(info), "invariants": inv}
log.step("replay -> ready; replay of ready refused", **report["transient-always+replay"])
# --- подмена staging: постоянная ошибка без повторов --------------------------------
compose("stop", "worker")
att = upload(c, alice, chat, png(50, 40), "tamper.png")
info = ops("inspect", att["id"])
key = info["attachment"]["staging_key"]
compose("--profile", "tools", "run", "--rm", "-T", "checks", "/app/student-tests/s3_tamper.py", key)
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker", env=FAULT_ENV)
failed = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
info = ops("inspect", att["id"])
check(failed["error_code"] == "staging_corrupted" and info["jobs"][0]["attempt"] == 1, "tamper -> permanent")
report["staging-tamper"] = {"attachment_id": att["id"], "error_code": failed["error_code"],
"attempts": info["jobs"][0]["attempt"], "events": events(info)}
log.step("tampered staging -> failed without retries", **report["staging-tamper"])
# --- poison message -> DLQ ----------------------------------------------------------
before = queues()["attachments.dlq"]["messages"]
ops("publish-raw", json.dumps({"schema_version": 1, "hello": "not a job"}))
q = wait_until(lambda: (lambda x: x if x["messages"] > before else None)(queues()["attachments.dlq"]),
30, what="message in DLQ")
listed = ops("dlq")
purged = ops("dlq", "--purge-invalid")
# Счётчики quorum-очереди в rabbitmqctl обновляются с задержкой в секунды.
after = wait_until(lambda: (lambda n: "0" if n == 0 else None)(queues()["attachments.dlq"]["messages"]),
20, what="empty DLQ") and 0
report["poison"] = {"dlq_before": before, "dlq_after_publish": q["messages"], "listed": listed["dlq_messages"],
"purged": [m["action"] for m in purged["dlq_messages"]], "dlq_after_purge": after}
log.step("poison message dead-lettered and purged", **report["poison"])
check(after == 0, "DLQ not empty after purge")
finally:
recreate("worker")
log.step("workers restored to .env configuration")
log.save(args.output, results=report)
print("PASS: worker crashes, backoff, retry_exhausted -> replay, staging tamper, DLQ")
if __name__ == "__main__":
main()