Files
mtusi2026lab5/scripts/check-failures.py
T

286 lines
11 KiB
Python

#!/usr/bin/env python3
"""Grade-4 failure scenarios for the lab5 attachment pipeline.
Each scenario is explicit and reproducible:
1. worker-stopped accept: 202 while no worker, resume -> ready;
2. outbox gap: broker down at COMMIT -> job stays unpublished -> API restart
-> broker returns -> ready (no new task needed);
3. retry/replay: repeated transient failures -> failed(retry_exhausted) ->
operator replay after recovery -> ready;
4. redelivery of a ready job is a no-op (variants not duplicated);
5. lease recovery: a stuck 'processing' job is requeued by the watchdog.
Run from the repository root (docker CLI + running lab5 stand).
"""
from __future__ import annotations
import argparse
import base64
import io
import json
import os
import struct
import subprocess
import time
import urllib.error
import urllib.request
import uuid
import zlib
BASE = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
def _png_chunk(tag, data):
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
def make_png(width=320, height=200):
raw = bytearray()
pixels = os.urandom(width * height * 3)
for row in range(height):
raw.append(0)
raw += pixels[row * width * 3 : (row + 1) * width * 3]
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + _png_chunk(b"IHDR", ihdr) + _png_chunk(b"IDAT", zlib.compress(bytes(raw), 0)) + _png_chunk(b"IEND", b"")
def http(method, path, token=None, body=None, raw=None, content_type=None):
headers = {"Accept": "application/json"}
data = raw
if body is not None:
data = json.dumps(body).encode()
headers["Content-Type"] = "application/json"
if content_type:
headers["Content-Type"] = content_type
if token:
headers["Authorization"] = "Bearer " + token
request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
try:
with urllib.request.urlopen(request, timeout=20) as response:
return response.status, response.read()
except urllib.error.HTTPError as error:
return error.code, error.read()
except (urllib.error.URLError, TimeoutError, OSError):
return 0, b""
def compose(*args, check=True):
return subprocess.run(["docker", "compose", *args], check=check, capture_output=True, text=True)
def psql(sql):
result = subprocess.run(
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-tAc", sql],
capture_output=True, text=True,
)
return result.stdout.strip()
def service_running(name: str) -> bool:
result = subprocess.run(
["docker", "compose", "ps", "--status", "running", "--format", "{{.Service}}"],
capture_output=True, text=True,
)
return name in result.stdout.split()
def storage_reachable() -> bool:
# Raw TCP probe (bypasses the app's circuit breaker, which cannot be used to
# decide whether the store is really down).
result = subprocess.run(
["docker", "compose", "exec", "-T", "api", "python", "-c",
"import socket,sys; socket.create_connection(('s3',8333),1); sys.exit(0)"],
capture_output=True, text=True, timeout=30,
)
return result.returncode == 0
def wait_storage_down(timeout: float = 60.0) -> None:
deadline = time.time() + timeout
while time.time() < deadline:
if not storage_reachable():
return
time.sleep(1)
raise SystemExit("S3 still reachable; cannot run the failure scenario")
def wait_ready(timeout=90.0):
deadline = time.time() + timeout
while time.time() < deadline:
if http("GET", "/health/ready")[0] == 200:
return
time.sleep(1)
raise SystemExit("stand did not become ready")
def new_actor():
username = "fail_" + uuid.uuid4().hex[:8]
password = "T3st-" + uuid.uuid4().hex
http("POST", "/api/v1/users", body={"username": username, "display_name": "Fail", "password": password})
credential = base64.b64encode(f"{username}:{password}".encode()).decode()
request = urllib.request.Request(
BASE + "/api/v1/auth/sessions",
headers={"Authorization": "Basic " + credential, "Accept": "application/json"},
method="POST",
)
with urllib.request.urlopen(request, timeout=20) as response:
token = json.loads(response.read())["token"]
user_id = json.loads(http("GET", "/api/v1/users/me", token=token)[1])["id"]
chat = json.loads(http("POST", "/api/v1/chats", token=token, body={"title": "fail", "member_ids": [user_id]})[1])
return token, chat["id"]
def upload(token, chat_id, data=None):
data = data if data is not None else make_png()
status, body = http("POST", f"/api/v1/chats/{chat_id}/attachments?filename=x.png", token=token, raw=data, content_type="image/png")
assert status == 202, (status, body)
return json.loads(body)["id"]
def status_of(token, attachment_id):
return json.loads(http("GET", f"/api/v1/attachments/{attachment_id}", token=token)[1])
def wait_status(token, attachment_id, wanted, timeout=120.0):
deadline = time.time() + timeout
last = None
while time.time() < deadline:
last = status_of(token, attachment_id)
if last["status"] == wanted:
return last
time.sleep(0.5)
raise SystemExit(f"attachment {attachment_id} did not reach {wanted}: {last}")
def scenario_worker_stopped(log):
log("\n== scenario 1: accept while worker stopped, resume -> ready ==")
compose("stop", "worker")
try:
token, chat_id = new_actor()
att_id = upload(token, chat_id)
time.sleep(2)
state = status_of(token, att_id)
log(f"with worker stopped: status={state['status']} (expected queued)")
assert state["status"] == "queued", state
compose("restart", "api")
wait_ready()
log("API restarted; work still queued (durable)")
assert status_of(token, att_id)["status"] == "queued"
finally:
compose("start", "worker")
ready = wait_status(token, att_id, "ready")
names = sorted(v["name"] for v in ready["variants"])
log(f"worker resumed: status=ready variants={names}")
assert {"thumbnail", "crop", "optimized"} <= set(names)
def scenario_outbox_gap(log):
log("\n== scenario 2: COMMIT/publish gap (broker down), API restart, then broker ==")
compose("stop", "broker")
try:
token, chat_id = new_actor()
att_id = upload(token, chat_id)
time.sleep(2)
job = psql(f"select published_at from attachment_jobs where attachment_id='{att_id}'")
log(f"broker down: status={status_of(token, att_id)['status']} published_at={job!r} (expected empty)")
assert status_of(token, att_id)["status"] in {"queued", "processing"}
compose("restart", "api")
wait_ready()
log("API restarted with broker still down; intent preserved in PostgreSQL")
finally:
compose("start", "broker")
ready = wait_status(token, att_id, "ready", timeout=180)
log(f"broker back: status={ready['status']} (outbox republished without new task)")
def scenario_retry_replay(log):
log("\n== scenario 3: transient failures -> failed(retry_exhausted) -> replay ==")
compose("stop", "worker")
while service_running("worker"):
time.sleep(0.5)
token, chat_id = new_actor()
att_id = upload(token, chat_id)
original_key = psql(f"select staging_key from attachments where id='{att_id}'")
# Fault injection: the durable original is (temporarily) unavailable, so the
# worker's fetch fails and is retried with backoff until attempts are spent.
psql(f"update attachments set staging_key='staging/missing-{att_id}' where id='{att_id}'")
log("injected: staging object points to a missing key (transient fetch failure)")
compose("start", "worker")
failed = wait_status(token, att_id, "failed", timeout=180)
log(f"status=failed error_code={failed['error_code']} (expected retry_exhausted)")
assert failed["error_code"] == "retry_exhausted", failed
# Operator repairs the cause and replays the failed job explicitly.
psql(f"update attachments set staging_key='{original_key}' where id='{att_id}'")
job_id = att_id_job(att_id)
replayed = subprocess.run(
["docker", "compose", "exec", "-T", "api", "python", "-m", "src.replay", job_id],
capture_output=True, text=True,
).stdout.strip()
log(f"explicit replay after recovery: {replayed}")
ready = wait_status(token, att_id, "ready", timeout=120)
log(f"after replay: status=ready variants={sorted(v['name'] for v in ready['variants'])}")
def att_id_job(attachment_id):
return psql(f"select job_id from attachment_jobs where attachment_id='{attachment_id}'")
def scenario_redelivery(log):
log("\n== scenario 4: redelivery of a ready job is a no-op ==")
token, chat_id = new_actor()
att_id = upload(token, chat_id)
ready = wait_status(token, att_id, "ready")
before = sorted((v["name"], v["sha256"]) for v in ready["variants"])
# Republish the finished job (duplicate delivery).
psql(f"update attachment_jobs set status='pending', published_at=null, available_at=now() where attachment_id='{att_id}'")
log("re-queued the same job (duplicate delivery)")
time.sleep(6)
after = status_of(token, att_id)
after_variants = sorted((v["name"], v["sha256"]) for v in after["variants"])
log(f"status={after['status']} variants={[n for n,_ in after_variants]} (must be unchanged)")
assert after["status"] == "ready"
assert after_variants == before, "duplicate delivery changed variants"
def scenario_lease_recovery(log):
log("\n== scenario 5: stuck processing -> lease recovery ==")
compose("stop", "worker")
token, chat_id = new_actor()
att_id = upload(token, chat_id)
job_id = att_id_job(att_id)
# Simulate a worker that died mid-processing with an already-expired lease.
psql(f"update attachments set status='processing' where id='{att_id}'")
psql(f"update attachment_jobs set status='processing', lease_until=now() - interval '5 seconds' where job_id='{job_id}'")
log("marked attachment+job processing with an expired lease")
compose("start", "worker")
ready = wait_status(token, att_id, "ready", timeout=120)
log(f"watchdog requeued and finished the job: status={ready['status']}")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--only", choices=["1", "2", "3", "4", "5"], help="run one scenario")
args = parser.parse_args()
wait_ready()
def log(message):
print(message, flush=True)
scenarios = {
"1": scenario_worker_stopped,
"2": scenario_outbox_gap,
"3": scenario_retry_replay,
"4": scenario_redelivery,
"5": scenario_lease_recovery,
}
if args.only:
scenarios[args.only](log)
else:
for name in ("1", "2", "3", "4", "5"):
scenarios[name](log)
print("\nPASS: lab5 grade-4 failure scenarios" + (f" (scenario {args.only})" if args.only else ""))
if __name__ == "__main__":
main()