Files
mtusi2026lab5/student-tests/worker_faults.py
T
zemiles a71b445019 Аудит F01, F11, F12, F13: гонка janitor/replay, версия pipeline, схема сообщения, срок задания
- F01: janitor под блокировкой jobs+attachments перепроверяет условие, снимает
  staging_key и только после COMMIT удаляет объект; replay перечитывает
  staging_key отдельным запросом под блокировкой вложения (в READ COMMITTED
  соединённые колонки могли остаться из старого снимка); тест трёх порядков,
  включая чередование из аудита (учебная пауза JANITOR_PAUSE_AFTER_SELECT_SECONDS)
- F11: pipeline_version пишется при создании вложения; API не стартует с
  неподдерживаемой версией
- F12: строгий разбор по jobs.schema.json (bool, RFC 3339, traceparent,
  tracestate); 17 случаев совпадают с jsonschema Draft 2020-12
- F13: единый срок задания на чтение staging, transform и запись; после срока
  аренда не продлевается, работа прерывается как processing_timeout
2026-11-13 20:56:05 +03:00

173 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Уровень 4: отказы worker, retry/backoff, failed + replay, DLQ.
python3 student-tests/worker_faults.py --output evidence/worker-faults.json
На время теста worker пересоздаются с WORKER_FAULT_INJECTION=1 и JOB_RETRY_BASE_SECONDS=1
(короче ожидание backoff); в конце возвращаются к значениям из .env.
Точка отказа задаётся именем файла fault-<point>.png (src/worker.py).
crash-after-claim процесс умер в processing до любых эффектов
crash-after-s3-put все объекты записаны в S3, COMMIT ready не сделан, ack не отправлен
crash-after-commit ready зафиксирован, ack не отправлен (и staging не удалён)
transient-2 временная ошибка в попытках 1–2: backoff с jitter, ready с 3-й
bug непредусмотренное исключение в 1-й попытке -> processing_failed -> retry -> ready
transient-always временная ошибка до replay: 5 попыток -> failed(retry_exhausted)
-> content 409, прикрепление 409 -> ops replay -> ready; повторный replay отклонён
staging tamper staging подменён до обработки -> failed(staging_corrupted) без повторов
poison message невалидное сообщение -> reject -> DLQ -> ops dlq --purge-invalid
Инварианты после каждого: 3 варианта, 4 конечных объекта, staging убран (с janitor).
"""
import argparse
import json
import subprocess
import time
from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, events, get_att, invariants, ops, png, queues,
recreate, upload, wait_status, wait_until)
FAULT_ENV = {"WORKER_FAULT_INJECTION": "1", "JOB_RETRY_BASE_SECONDS": "1", "JOB_RETRY_MAX_SECONDS": "8"}
def inspect_until(att_id, pred, timeout=60, what="state"):
return wait_until(lambda: (lambda i: i if pred(i) else None)(ops("inspect", att_id)), timeout, 1, what)
def main():
p = argparse.ArgumentParser()
p.add_argument("--base-url", default=None)
p.add_argument("--output")
args = p.parse_args()
log = Log("worker faults, retries, failed registry, replay, DLQ")
c = Client(args.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
recreate("worker", FAULT_ENV)
log.step("workers recreated with fault injection", env=FAULT_ENV)
report = {}
try:
# --- крахи процесса в разных окнах -------------------------------------------------
for point in ("crash-after-claim", "crash-after-s3-put", "crash-after-commit"):
att = upload(c, alice, chat, SAMPLE_PNG, "fault-%s.png" % point)
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
check(ready["status"] == "ready", point + ": not ready")
if point == "crash-after-commit":
# ready уже был зафиксирован; redelivery после рестарта -> no-op ack.
inspect_until(att["id"], lambda i: "duplicate_ignored" in events(i), 60, "redelivery no-op")
compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once")
info = ops("inspect", att["id"])
inv = invariants(info)
check(inv["variants_ok"] and inv["final_objects_ok"] and not inv["staging_left"], point + ": invariants")
report[point] = {"attachment_id": att["id"], "events": events(info), "invariants": inv}
log.step(point, events=events(info), invariants=inv)
ids = compose("ps", "-q", "worker").stdout.split()
rc = subprocess.run(["docker", "inspect", "-f", "{{.Name}} {{.RestartCount}} {{.State.Status}}"] + ids,
capture_output=True, text=True).stdout.split("\n")
log.step("worker containers after crashes (restart: unless-stopped)", containers=[x for x in rc if x])
# --- временные ошибки: backoff + jitter ---------------------------------------------
att = upload(c, alice, chat, SAMPLE_PNG, "fault-transient-2.png")
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
check(ready["status"] == "ready", "transient-2 not ready")
info = ops("inspect", att["id"])
delays = [e["detail"]["delay_seconds"] for e in info["events"] if e["event"] == "retry_scheduled"]
claims = [e["at"] for e in info["events"] if e["event"] == "claimed"]
bounds = [[0.5 * min(8, 2 ** (n - 1)), min(8, 2 ** (n - 1))] for n in (1, 2)]
check(len(delays) == 2 and all(lo <= d <= hi for d, (lo, hi) in zip(delays, bounds)), "backoff bounds")
inv = invariants(info)
check(inv["attempt"] == 3 and inv["variants_ok"], "transient-2 attempts")
report["transient-2"] = {"attachment_id": att["id"], "delays": delays, "bounds": bounds, "claims_at": claims,
"invariants": inv}
log.step("transient-2: retried with backoff", delays=delays, bounds=bounds, invariants=inv)
# --- непредусмотренное исключение: штатный retry, а не «висящий» processing ----------
att = upload(c, alice, chat, SAMPLE_PNG, "fault-bug.png")
t0 = time.monotonic()
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
took = round(time.monotonic() - t0, 2)
info = ops("inspect", att["id"])
codes = [e["detail"]["error_code"] for e in info["events"] if e["event"] == "retry_scheduled"]
check(ready["status"] == "ready" and codes == ["processing_failed"] and took < 15, "bug: retry path")
report["bug"] = {"attachment_id": att["id"], "retry_error_codes": codes, "seconds_to_ready": took,
"events": events(info), "invariants": invariants(info)}
log.step("unexpected exception -> processing_failed -> retry -> ready", **report["bug"])
# --- исчерпание попыток -> failed -> replay ----------------------------------------
att = upload(c, alice, chat, SAMPLE_PNG, "fault-transient-always.png")
failed = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=120)
check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", "expected retry_exhausted")
r409 = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob)
m409 = c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice,
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": "f-" + att["id"]})
registry = ops("failed")
in_registry = [r for r in registry["failed"] if r["attachment_id"] == att["id"]]
check(in_registry and in_registry[0]["replayable"], "failed registry")
info = ops("inspect", att["id"])
log.step("transient-always: failed after max attempts", error_code=failed["error_code"],
attempts=info["jobs"][0]["attempt"], content=r409["status"], attach=m409["status"],
registry_entry=in_registry[0])
dry = ops("replay", att["id"], "--dry-run")
rep = ops("replay", att["id"], "--reason", "test: S3 restored")
check(rep["replay"][0]["result"] == "replayed", "replay")
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
check(ready["status"] == "ready", "not ready after replay")
again = ops("replay", att["id"])
check(again["replay"][0]["result"] == "refused_status_ready", "replay of ready must be refused")
info = ops("inspect", att["id"])
inv = invariants(info)
check(inv["variants_ok"] and inv["final_objects_ok"], "replay invariants")
report["transient-always+replay"] = {"attachment_id": att["id"], "dry_run": dry["replay"][0]["result"],
"replay": rep["replay"][0]["result"],
"second_replay": again["replay"][0]["result"],
"replay_count": info["jobs"][0]["replay_count"],
"events": events(info), "invariants": inv}
log.step("replay -> ready; replay of ready refused", **report["transient-always+replay"])
# --- подмена staging: постоянная ошибка без повторов --------------------------------
compose("stop", "worker")
att = upload(c, alice, chat, png(50, 40), "tamper.png")
info = ops("inspect", att["id"])
key = info["attachment"]["staging_key"]
compose("--profile", "tools", "run", "--rm", "-T", "checks", "/app/student-tests/s3_tamper.py", key)
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker", env=FAULT_ENV)
failed = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
info = ops("inspect", att["id"])
check(failed["error_code"] == "staging_corrupted" and info["jobs"][0]["attempt"] == 1, "tamper -> permanent")
report["staging-tamper"] = {"attachment_id": att["id"], "error_code": failed["error_code"],
"attempts": info["jobs"][0]["attempt"], "events": events(info)}
log.step("tampered staging -> failed without retries", **report["staging-tamper"])
# --- poison message -> DLQ ----------------------------------------------------------
before = queues()["attachments.dlq"]["messages"]
ops("publish-raw", json.dumps({"schema_version": 1, "hello": "not a job"}))
q = wait_until(lambda: (lambda x: x if x["messages"] > before else None)(queues()["attachments.dlq"]),
30, what="message in DLQ")
listed = ops("dlq")
purged = ops("dlq", "--purge-invalid")
# Счётчики quorum-очереди в rabbitmqctl обновляются с задержкой в секунды.
after = wait_until(lambda: (lambda n: "0" if n == 0 else None)(queues()["attachments.dlq"]["messages"]),
20, what="empty DLQ") and 0
report["poison"] = {"dlq_before": before, "dlq_after_publish": q["messages"], "listed": listed["dlq_messages"],
"purged": [m["action"] for m in purged["dlq_messages"]], "dlq_after_purge": after}
log.step("poison message dead-lettered and purged", **report["poison"])
check(after == 0, "DLQ not empty after purge")
# --- жёсткий срок задания (аудит F13): JOB_TIMEOUT 5 с, «долгая» работа 8 с ----------
recreate("worker", dict(FAULT_ENV, JOB_TIMEOUT_SECONDS="5", FAULT_SLOW_SECONDS="8"))
att = upload(c, alice, chat, SAMPLE_PNG, "fault-slow.png")
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
info = ops("inspect", att["id"])
codes = [e["detail"]["error_code"] for e in info["events"] if e["event"] == "retry_scheduled"]
check(ready["status"] == "ready" and codes == ["processing_timeout"], "deadline: %s" % codes)
report["deadline"] = {"attachment_id": att["id"], "retry_error_codes": codes, "events": events(info),
"invariants": invariants(info)}
log.step("job deadline exceeded -> processing_timeout -> retry -> ready", **report["deadline"])
finally:
recreate("worker")
log.step("workers restored to .env configuration")
log.save(args.output, results=report)
print("PASS: worker crashes, backoff, retry_exhausted -> replay, staging tamper, DLQ")
if __name__ == "__main__":
main()