forked from mtusi2026/mtusi2026lab5
- F01: janitor под блокировкой jobs+attachments перепроверяет условие, снимает staging_key и только после COMMIT удаляет объект; replay перечитывает staging_key отдельным запросом под блокировкой вложения (в READ COMMITTED соединённые колонки могли остаться из старого снимка); тест трёх порядков, включая чередование из аудита (учебная пауза JANITOR_PAUSE_AFTER_SELECT_SECONDS) - F11: pipeline_version пишется при создании вложения; API не стартует с неподдерживаемой версией - F12: строгий разбор по jobs.schema.json (bool, RFC 3339, traceparent, tracestate); 17 случаев совпадают с jsonschema Draft 2020-12 - F13: единый срок задания на чтение staging, transform и запись; после срока аренда не продлевается, работа прерывается как processing_timeout
173 lines
12 KiB
Python
173 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""Уровень 4: отказы worker, retry/backoff, failed + replay, DLQ.
|
||
|
||
python3 student-tests/worker_faults.py --output evidence/worker-faults.json
|
||
|
||
На время теста worker пересоздаются с WORKER_FAULT_INJECTION=1 и JOB_RETRY_BASE_SECONDS=1
|
||
(короче ожидание backoff); в конце возвращаются к значениям из .env.
|
||
Точка отказа задаётся именем файла fault-<point>.png (src/worker.py).
|
||
|
||
crash-after-claim процесс умер в processing до любых эффектов
|
||
crash-after-s3-put все объекты записаны в S3, COMMIT ready не сделан, ack не отправлен
|
||
crash-after-commit ready зафиксирован, ack не отправлен (и staging не удалён)
|
||
transient-2 временная ошибка в попытках 1–2: backoff с jitter, ready с 3-й
|
||
bug непредусмотренное исключение в 1-й попытке -> processing_failed -> retry -> ready
|
||
transient-always временная ошибка до replay: 5 попыток -> failed(retry_exhausted)
|
||
-> content 409, прикрепление 409 -> ops replay -> ready; повторный replay отклонён
|
||
staging tamper staging подменён до обработки -> failed(staging_corrupted) без повторов
|
||
poison message невалидное сообщение -> reject -> DLQ -> ops dlq --purge-invalid
|
||
Инварианты после каждого: 3 варианта, 4 конечных объекта, staging убран (с janitor).
|
||
"""
|
||
import argparse
|
||
import json
|
||
import subprocess
|
||
import time
|
||
|
||
from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, events, get_att, invariants, ops, png, queues,
|
||
recreate, upload, wait_status, wait_until)
|
||
|
||
FAULT_ENV = {"WORKER_FAULT_INJECTION": "1", "JOB_RETRY_BASE_SECONDS": "1", "JOB_RETRY_MAX_SECONDS": "8"}
|
||
|
||
|
||
def inspect_until(att_id, pred, timeout=60, what="state"):
|
||
return wait_until(lambda: (lambda i: i if pred(i) else None)(ops("inspect", att_id)), timeout, 1, what)
|
||
|
||
|
||
def main():
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument("--base-url", default=None)
|
||
p.add_argument("--output")
|
||
args = p.parse_args()
|
||
log = Log("worker faults, retries, failed registry, replay, DLQ")
|
||
c = Client(args.base_url, timeout=30)
|
||
alice, bob, _eve, chat = c.bootstrap()
|
||
recreate("worker", FAULT_ENV)
|
||
log.step("workers recreated with fault injection", env=FAULT_ENV)
|
||
report = {}
|
||
try:
|
||
# --- крахи процесса в разных окнах -------------------------------------------------
|
||
for point in ("crash-after-claim", "crash-after-s3-put", "crash-after-commit"):
|
||
att = upload(c, alice, chat, SAMPLE_PNG, "fault-%s.png" % point)
|
||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
|
||
check(ready["status"] == "ready", point + ": not ready")
|
||
if point == "crash-after-commit":
|
||
# ready уже был зафиксирован; redelivery после рестарта -> no-op ack.
|
||
inspect_until(att["id"], lambda i: "duplicate_ignored" in events(i), 60, "redelivery no-op")
|
||
compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once")
|
||
info = ops("inspect", att["id"])
|
||
inv = invariants(info)
|
||
check(inv["variants_ok"] and inv["final_objects_ok"] and not inv["staging_left"], point + ": invariants")
|
||
report[point] = {"attachment_id": att["id"], "events": events(info), "invariants": inv}
|
||
log.step(point, events=events(info), invariants=inv)
|
||
ids = compose("ps", "-q", "worker").stdout.split()
|
||
rc = subprocess.run(["docker", "inspect", "-f", "{{.Name}} {{.RestartCount}} {{.State.Status}}"] + ids,
|
||
capture_output=True, text=True).stdout.split("\n")
|
||
log.step("worker containers after crashes (restart: unless-stopped)", containers=[x for x in rc if x])
|
||
|
||
# --- временные ошибки: backoff + jitter ---------------------------------------------
|
||
att = upload(c, alice, chat, SAMPLE_PNG, "fault-transient-2.png")
|
||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
|
||
check(ready["status"] == "ready", "transient-2 not ready")
|
||
info = ops("inspect", att["id"])
|
||
delays = [e["detail"]["delay_seconds"] for e in info["events"] if e["event"] == "retry_scheduled"]
|
||
claims = [e["at"] for e in info["events"] if e["event"] == "claimed"]
|
||
bounds = [[0.5 * min(8, 2 ** (n - 1)), min(8, 2 ** (n - 1))] for n in (1, 2)]
|
||
check(len(delays) == 2 and all(lo <= d <= hi for d, (lo, hi) in zip(delays, bounds)), "backoff bounds")
|
||
inv = invariants(info)
|
||
check(inv["attempt"] == 3 and inv["variants_ok"], "transient-2 attempts")
|
||
report["transient-2"] = {"attachment_id": att["id"], "delays": delays, "bounds": bounds, "claims_at": claims,
|
||
"invariants": inv}
|
||
log.step("transient-2: retried with backoff", delays=delays, bounds=bounds, invariants=inv)
|
||
|
||
# --- непредусмотренное исключение: штатный retry, а не «висящий» processing ----------
|
||
att = upload(c, alice, chat, SAMPLE_PNG, "fault-bug.png")
|
||
t0 = time.monotonic()
|
||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
|
||
took = round(time.monotonic() - t0, 2)
|
||
info = ops("inspect", att["id"])
|
||
codes = [e["detail"]["error_code"] for e in info["events"] if e["event"] == "retry_scheduled"]
|
||
check(ready["status"] == "ready" and codes == ["processing_failed"] and took < 15, "bug: retry path")
|
||
report["bug"] = {"attachment_id": att["id"], "retry_error_codes": codes, "seconds_to_ready": took,
|
||
"events": events(info), "invariants": invariants(info)}
|
||
log.step("unexpected exception -> processing_failed -> retry -> ready", **report["bug"])
|
||
|
||
# --- исчерпание попыток -> failed -> replay ----------------------------------------
|
||
att = upload(c, alice, chat, SAMPLE_PNG, "fault-transient-always.png")
|
||
failed = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=120)
|
||
check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", "expected retry_exhausted")
|
||
r409 = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob)
|
||
m409 = c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice,
|
||
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": "f-" + att["id"]})
|
||
registry = ops("failed")
|
||
in_registry = [r for r in registry["failed"] if r["attachment_id"] == att["id"]]
|
||
check(in_registry and in_registry[0]["replayable"], "failed registry")
|
||
info = ops("inspect", att["id"])
|
||
log.step("transient-always: failed after max attempts", error_code=failed["error_code"],
|
||
attempts=info["jobs"][0]["attempt"], content=r409["status"], attach=m409["status"],
|
||
registry_entry=in_registry[0])
|
||
dry = ops("replay", att["id"], "--dry-run")
|
||
rep = ops("replay", att["id"], "--reason", "test: S3 restored")
|
||
check(rep["replay"][0]["result"] == "replayed", "replay")
|
||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
|
||
check(ready["status"] == "ready", "not ready after replay")
|
||
again = ops("replay", att["id"])
|
||
check(again["replay"][0]["result"] == "refused_status_ready", "replay of ready must be refused")
|
||
info = ops("inspect", att["id"])
|
||
inv = invariants(info)
|
||
check(inv["variants_ok"] and inv["final_objects_ok"], "replay invariants")
|
||
report["transient-always+replay"] = {"attachment_id": att["id"], "dry_run": dry["replay"][0]["result"],
|
||
"replay": rep["replay"][0]["result"],
|
||
"second_replay": again["replay"][0]["result"],
|
||
"replay_count": info["jobs"][0]["replay_count"],
|
||
"events": events(info), "invariants": inv}
|
||
log.step("replay -> ready; replay of ready refused", **report["transient-always+replay"])
|
||
|
||
# --- подмена staging: постоянная ошибка без повторов --------------------------------
|
||
compose("stop", "worker")
|
||
att = upload(c, alice, chat, png(50, 40), "tamper.png")
|
||
info = ops("inspect", att["id"])
|
||
key = info["attachment"]["staging_key"]
|
||
compose("--profile", "tools", "run", "--rm", "-T", "checks", "/app/student-tests/s3_tamper.py", key)
|
||
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker", env=FAULT_ENV)
|
||
failed = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
|
||
info = ops("inspect", att["id"])
|
||
check(failed["error_code"] == "staging_corrupted" and info["jobs"][0]["attempt"] == 1, "tamper -> permanent")
|
||
report["staging-tamper"] = {"attachment_id": att["id"], "error_code": failed["error_code"],
|
||
"attempts": info["jobs"][0]["attempt"], "events": events(info)}
|
||
log.step("tampered staging -> failed without retries", **report["staging-tamper"])
|
||
|
||
# --- poison message -> DLQ ----------------------------------------------------------
|
||
before = queues()["attachments.dlq"]["messages"]
|
||
ops("publish-raw", json.dumps({"schema_version": 1, "hello": "not a job"}))
|
||
q = wait_until(lambda: (lambda x: x if x["messages"] > before else None)(queues()["attachments.dlq"]),
|
||
30, what="message in DLQ")
|
||
listed = ops("dlq")
|
||
purged = ops("dlq", "--purge-invalid")
|
||
# Счётчики quorum-очереди в rabbitmqctl обновляются с задержкой в секунды.
|
||
after = wait_until(lambda: (lambda n: "0" if n == 0 else None)(queues()["attachments.dlq"]["messages"]),
|
||
20, what="empty DLQ") and 0
|
||
report["poison"] = {"dlq_before": before, "dlq_after_publish": q["messages"], "listed": listed["dlq_messages"],
|
||
"purged": [m["action"] for m in purged["dlq_messages"]], "dlq_after_purge": after}
|
||
log.step("poison message dead-lettered and purged", **report["poison"])
|
||
check(after == 0, "DLQ not empty after purge")
|
||
|
||
# --- жёсткий срок задания (аудит F13): JOB_TIMEOUT 5 с, «долгая» работа 8 с ----------
|
||
recreate("worker", dict(FAULT_ENV, JOB_TIMEOUT_SECONDS="5", FAULT_SLOW_SECONDS="8"))
|
||
att = upload(c, alice, chat, SAMPLE_PNG, "fault-slow.png")
|
||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
|
||
info = ops("inspect", att["id"])
|
||
codes = [e["detail"]["error_code"] for e in info["events"] if e["event"] == "retry_scheduled"]
|
||
check(ready["status"] == "ready" and codes == ["processing_timeout"], "deadline: %s" % codes)
|
||
report["deadline"] = {"attachment_id": att["id"], "retry_error_codes": codes, "events": events(info),
|
||
"invariants": invariants(info)}
|
||
log.step("job deadline exceeded -> processing_timeout -> retry -> ready", **report["deadline"])
|
||
finally:
|
||
recreate("worker")
|
||
log.step("workers restored to .env configuration")
|
||
log.save(args.output, results=report)
|
||
print("PASS: worker crashes, backoff, retry_exhausted -> replay, staging tamper, DLQ")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|