Files
mtusi2026lab7/student-tests/worker_faults.py
T
zemiles 9cc80d2529
Application / app (pull_request) Canceled after 0s
Skeleton contracts / contracts (pull_request) Canceled after 0s
ЛР 7 — БПИ2303 — Лазарев — на 3
Перенесена реализация lab6; бизнес-API без изменений, транспорт — HTTPS.

- Внешний HTTPS (public-ca) на 127.0.0.1:8443; HSTS, TLS >= 1.2.
- Обязательное mTLS LB -> каждая API-реплика: API проверяет цепочку
  internal-ca и SAN клиента (allowlist: lb); health только на 127.0.0.1.
- TLS с проверкой сервера ко всем зависимостям: PostgreSQL verify-full
  (pg_hba только hostssl), HTTPS S3, AMQPS, OTLP/HTTPS с client cert,
  mTLS Collector -> Prometheus/Loki/Tempo, Grafana -> datasources, scrape.
- OpenBao 2.7.2 не в dev-mode: Raft на volume, init/unseal оператором,
  auth/cert с ролью и policy на сервис, TTL 15 мин + renew/re-login.
- Отдельные роли PostgreSQL с грантами по таблицам; разные ключи S3 и
  пользователи брокера выдаются из OpenBao.
- Argon2id (m=19 MiB, t=2, p=1, замер на стенде) и rehash при входе.
- Случайный X-Instance-Id; hardening контейнеров (non-root, cap_drop ALL,
  no-new-privileges, read-only, лимиты); TCP_NODELAY на сокете API.
- Модель угроз, trust matrix, отрицательные тесты TLS/mTLS, API, OpenBao,
  hardening; evidence, в т.ч. прогон из чистого clone.
2026-11-18 23:59:35 +03:00

173 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Уровень 4: отказы worker, retry/backoff, failed + replay, DLQ.
python3 student-tests/worker_faults.py --output evidence/worker-faults.json
На время теста worker пересоздаются с WORKER_FAULT_INJECTION=1 и JOB_RETRY_BASE_SECONDS=1
(короче ожидание backoff); в конце возвращаются к значениям из .env.
Точка отказа задаётся именем файла fault-<point>.png (src/worker.py).
crash-after-claim процесс умер в processing до любых эффектов
crash-after-s3-put все объекты записаны в S3, COMMIT ready не сделан, ack не отправлен
crash-after-commit ready зафиксирован, ack не отправлен (и staging не удалён)
transient-2 временная ошибка в попытках 1–2: backoff с jitter, ready с 3-й
bug непредусмотренное исключение в 1-й попытке -> processing_failed -> retry -> ready
transient-always временная ошибка до replay: 5 попыток -> failed(retry_exhausted)
-> content 409, прикрепление 409 -> ops replay -> ready; повторный replay отклонён
staging tamper staging подменён до обработки -> failed(staging_corrupted) без повторов
poison message невалидное сообщение -> reject -> DLQ -> ops dlq --purge-invalid
Инварианты после каждого: 3 варианта, 4 конечных объекта, staging убран (с janitor).
"""
import argparse
import json
import subprocess
import time
from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, events, get_att, invariants, ops, png, queues,
recreate, upload, wait_status, wait_until)
FAULT_ENV = {"WORKER_FAULT_INJECTION": "1", "JOB_RETRY_BASE_SECONDS": "1", "JOB_RETRY_MAX_SECONDS": "8"}
def inspect_until(att_id, pred, timeout=60, what="state"):
return wait_until(lambda: (lambda i: i if pred(i) else None)(ops("inspect", att_id)), timeout, 1, what)
def main():
p = argparse.ArgumentParser()
p.add_argument("--base-url", default=None)
p.add_argument("--output")
args = p.parse_args()
log = Log("worker faults, retries, failed registry, replay, DLQ")
c = Client(args.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
recreate("worker", FAULT_ENV)
log.step("workers recreated with fault injection", env=FAULT_ENV)
report = {}
try:
# --- крахи процесса в разных окнах -------------------------------------------------
for point in ("crash-after-claim", "crash-after-s3-put", "crash-after-commit"):
att = upload(c, alice, chat, SAMPLE_PNG, "fault-%s.png" % point)
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
check(ready["status"] == "ready", point + ": not ready")
if point == "crash-after-commit":
# ready уже был зафиксирован; redelivery после рестарта -> no-op ack.
inspect_until(att["id"], lambda i: "duplicate_ignored" in events(i), 60, "redelivery no-op")
compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once")
info = ops("inspect", att["id"])
inv = invariants(info)
check(inv["variants_ok"] and inv["final_objects_ok"] and not inv["staging_left"], point + ": invariants")
report[point] = {"attachment_id": att["id"], "events": events(info), "invariants": inv}
log.step(point, events=events(info), invariants=inv)
ids = compose("ps", "-q", "worker").stdout.split()
rc = subprocess.run(["docker", "inspect", "-f", "{{.Name}} {{.RestartCount}} {{.State.Status}}"] + ids,
capture_output=True, text=True).stdout.split("\n")
log.step("worker containers after crashes (restart: unless-stopped)", containers=[x for x in rc if x])
# --- временные ошибки: backoff + jitter ---------------------------------------------
att = upload(c, alice, chat, SAMPLE_PNG, "fault-transient-2.png")
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
check(ready["status"] == "ready", "transient-2 not ready")
info = ops("inspect", att["id"])
delays = [e["detail"]["delay_seconds"] for e in info["events"] if e["event"] == "retry_scheduled"]
claims = [e["at"] for e in info["events"] if e["event"] == "claimed"]
bounds = [[0.5 * min(8, 2 ** (n - 1)), min(8, 2 ** (n - 1))] for n in (1, 2)]
check(len(delays) == 2 and all(lo <= d <= hi for d, (lo, hi) in zip(delays, bounds)), "backoff bounds")
inv = invariants(info)
check(inv["attempt"] == 3 and inv["variants_ok"], "transient-2 attempts")
report["transient-2"] = {"attachment_id": att["id"], "delays": delays, "bounds": bounds, "claims_at": claims,
"invariants": inv}
log.step("transient-2: retried with backoff", delays=delays, bounds=bounds, invariants=inv)
# --- непредусмотренное исключение: штатный retry, а не «висящий» processing ----------
att = upload(c, alice, chat, SAMPLE_PNG, "fault-bug.png")
t0 = time.monotonic()
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
took = round(time.monotonic() - t0, 2)
info = ops("inspect", att["id"])
codes = [e["detail"]["error_code"] for e in info["events"] if e["event"] == "retry_scheduled"]
check(ready["status"] == "ready" and codes == ["processing_failed"] and took < 15, "bug: retry path")
report["bug"] = {"attachment_id": att["id"], "retry_error_codes": codes, "seconds_to_ready": took,
"events": events(info), "invariants": invariants(info)}
log.step("unexpected exception -> processing_failed -> retry -> ready", **report["bug"])
# --- исчерпание попыток -> failed -> replay ----------------------------------------
att = upload(c, alice, chat, SAMPLE_PNG, "fault-transient-always.png")
failed = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=120)
check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", "expected retry_exhausted")
r409 = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob)
m409 = c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice,
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": "f-" + att["id"]})
registry = ops("failed")
in_registry = [r for r in registry["failed"] if r["attachment_id"] == att["id"]]
check(in_registry and in_registry[0]["replayable"], "failed registry")
info = ops("inspect", att["id"])
log.step("transient-always: failed after max attempts", error_code=failed["error_code"],
attempts=info["jobs"][0]["attempt"], content=r409["status"], attach=m409["status"],
registry_entry=in_registry[0])
dry = ops("replay", att["id"], "--dry-run")
rep = ops("replay", att["id"], "--reason", "test: S3 restored")
check(rep["replay"][0]["result"] == "replayed", "replay")
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
check(ready["status"] == "ready", "not ready after replay")
again = ops("replay", att["id"])
check(again["replay"][0]["result"] == "refused_status_ready", "replay of ready must be refused")
info = ops("inspect", att["id"])
inv = invariants(info)
check(inv["variants_ok"] and inv["final_objects_ok"], "replay invariants")
report["transient-always+replay"] = {"attachment_id": att["id"], "dry_run": dry["replay"][0]["result"],
"replay": rep["replay"][0]["result"],
"second_replay": again["replay"][0]["result"],
"replay_count": info["jobs"][0]["replay_count"],
"events": events(info), "invariants": inv}
log.step("replay -> ready; replay of ready refused", **report["transient-always+replay"])
# --- подмена staging: постоянная ошибка без повторов --------------------------------
compose("stop", "worker")
att = upload(c, alice, chat, png(50, 40), "tamper.png")
info = ops("inspect", att["id"])
key = info["attachment"]["staging_key"]
compose("--profile", "tools", "run", "--rm", "-T", "checks", "/app/student-tests/s3_tamper.py", key)
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker", env=FAULT_ENV)
failed = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
info = ops("inspect", att["id"])
check(failed["error_code"] == "staging_corrupted" and info["jobs"][0]["attempt"] == 1, "tamper -> permanent")
report["staging-tamper"] = {"attachment_id": att["id"], "error_code": failed["error_code"],
"attempts": info["jobs"][0]["attempt"], "events": events(info)}
log.step("tampered staging -> failed without retries", **report["staging-tamper"])
# --- poison message -> DLQ ----------------------------------------------------------
before = queues()["attachments.dlq"]["messages"]
ops("publish-raw", json.dumps({"schema_version": 1, "hello": "not a job"}))
q = wait_until(lambda: (lambda x: x if x["messages"] > before else None)(queues()["attachments.dlq"]),
30, what="message in DLQ")
listed = ops("dlq")
purged = ops("dlq", "--purge-invalid")
# Счётчики quorum-очереди в rabbitmqctl обновляются с задержкой в секунды.
after = wait_until(lambda: (lambda n: "0" if n == 0 else None)(queues()["attachments.dlq"]["messages"]),
20, what="empty DLQ") and 0
report["poison"] = {"dlq_before": before, "dlq_after_publish": q["messages"], "listed": listed["dlq_messages"],
"purged": [m["action"] for m in purged["dlq_messages"]], "dlq_after_purge": after}
log.step("poison message dead-lettered and purged", **report["poison"])
check(after == 0, "DLQ not empty after purge")
# --- жёсткий срок задания (аудит F13): JOB_TIMEOUT 5 с, «долгая» работа 8 с ----------
recreate("worker", dict(FAULT_ENV, JOB_TIMEOUT_SECONDS="5", FAULT_SLOW_SECONDS="8"))
att = upload(c, alice, chat, SAMPLE_PNG, "fault-slow.png")
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
info = ops("inspect", att["id"])
codes = [e["detail"]["error_code"] for e in info["events"] if e["event"] == "retry_scheduled"]
check(ready["status"] == "ready" and codes == ["processing_timeout"], "deadline: %s" % codes)
report["deadline"] = {"attachment_id": att["id"], "retry_error_codes": codes, "events": events(info),
"invariants": invariants(info)}
log.step("job deadline exceeded -> processing_timeout -> retry -> ready", **report["deadline"])
finally:
recreate("worker")
log.step("workers restored to .env configuration")
log.save(args.output, results=report)
print("PASS: worker crashes, backoff, retry_exhausted -> replay, staging tamper, DLQ")
if __name__ == "__main__":
main()