286 lines
11 KiB
Python
286 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Grade-4 failure scenarios for the lab5 attachment pipeline.
|
|
|
|
Each scenario is explicit and reproducible:
|
|
1. worker-stopped accept: 202 while no worker, resume -> ready;
|
|
2. outbox gap: broker down at COMMIT -> job stays unpublished -> API restart
|
|
-> broker returns -> ready (no new task needed);
|
|
3. retry/replay: repeated transient failures -> failed(retry_exhausted) ->
|
|
operator replay after recovery -> ready;
|
|
4. redelivery of a ready job is a no-op (variants not duplicated);
|
|
5. lease recovery: a stuck 'processing' job is requeued by the watchdog.
|
|
|
|
Run from the repository root (docker CLI + running lab5 stand).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import io
|
|
import json
|
|
import os
|
|
import struct
|
|
import subprocess
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
import zlib
|
|
|
|
BASE = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
|
|
|
|
|
|
def _png_chunk(tag, data):
|
|
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
|
|
|
|
|
|
def make_png(width=320, height=200):
|
|
raw = bytearray()
|
|
pixels = os.urandom(width * height * 3)
|
|
for row in range(height):
|
|
raw.append(0)
|
|
raw += pixels[row * width * 3 : (row + 1) * width * 3]
|
|
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
|
return b"\x89PNG\r\n\x1a\n" + _png_chunk(b"IHDR", ihdr) + _png_chunk(b"IDAT", zlib.compress(bytes(raw), 0)) + _png_chunk(b"IEND", b"")
|
|
|
|
|
|
def http(method, path, token=None, body=None, raw=None, content_type=None):
|
|
headers = {"Accept": "application/json"}
|
|
data = raw
|
|
if body is not None:
|
|
data = json.dumps(body).encode()
|
|
headers["Content-Type"] = "application/json"
|
|
if content_type:
|
|
headers["Content-Type"] = content_type
|
|
if token:
|
|
headers["Authorization"] = "Bearer " + token
|
|
request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=20) as response:
|
|
return response.status, response.read()
|
|
except urllib.error.HTTPError as error:
|
|
return error.code, error.read()
|
|
except (urllib.error.URLError, TimeoutError, OSError):
|
|
return 0, b""
|
|
|
|
|
|
def compose(*args, check=True):
|
|
return subprocess.run(["docker", "compose", *args], check=check, capture_output=True, text=True)
|
|
|
|
|
|
def psql(sql):
|
|
result = subprocess.run(
|
|
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-tAc", sql],
|
|
capture_output=True, text=True,
|
|
)
|
|
return result.stdout.strip()
|
|
|
|
|
|
def service_running(name: str) -> bool:
|
|
result = subprocess.run(
|
|
["docker", "compose", "ps", "--status", "running", "--format", "{{.Service}}"],
|
|
capture_output=True, text=True,
|
|
)
|
|
return name in result.stdout.split()
|
|
|
|
|
|
def storage_reachable() -> bool:
|
|
# Raw TCP probe (bypasses the app's circuit breaker, which cannot be used to
|
|
# decide whether the store is really down).
|
|
result = subprocess.run(
|
|
["docker", "compose", "exec", "-T", "api", "python", "-c",
|
|
"import socket,sys; socket.create_connection(('s3',8333),1); sys.exit(0)"],
|
|
capture_output=True, text=True, timeout=30,
|
|
)
|
|
return result.returncode == 0
|
|
|
|
|
|
def wait_storage_down(timeout: float = 60.0) -> None:
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
if not storage_reachable():
|
|
return
|
|
time.sleep(1)
|
|
raise SystemExit("S3 still reachable; cannot run the failure scenario")
|
|
|
|
|
|
def wait_ready(timeout=90.0):
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
if http("GET", "/health/ready")[0] == 200:
|
|
return
|
|
time.sleep(1)
|
|
raise SystemExit("stand did not become ready")
|
|
|
|
|
|
def new_actor():
|
|
username = "fail_" + uuid.uuid4().hex[:8]
|
|
password = "T3st-" + uuid.uuid4().hex
|
|
http("POST", "/api/v1/users", body={"username": username, "display_name": "Fail", "password": password})
|
|
credential = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
request = urllib.request.Request(
|
|
BASE + "/api/v1/auth/sessions",
|
|
headers={"Authorization": "Basic " + credential, "Accept": "application/json"},
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(request, timeout=20) as response:
|
|
token = json.loads(response.read())["token"]
|
|
user_id = json.loads(http("GET", "/api/v1/users/me", token=token)[1])["id"]
|
|
chat = json.loads(http("POST", "/api/v1/chats", token=token, body={"title": "fail", "member_ids": [user_id]})[1])
|
|
return token, chat["id"]
|
|
|
|
|
|
def upload(token, chat_id, data=None):
|
|
data = data if data is not None else make_png()
|
|
status, body = http("POST", f"/api/v1/chats/{chat_id}/attachments?filename=x.png", token=token, raw=data, content_type="image/png")
|
|
assert status == 202, (status, body)
|
|
return json.loads(body)["id"]
|
|
|
|
|
|
def status_of(token, attachment_id):
|
|
return json.loads(http("GET", f"/api/v1/attachments/{attachment_id}", token=token)[1])
|
|
|
|
|
|
def wait_status(token, attachment_id, wanted, timeout=120.0):
|
|
deadline = time.time() + timeout
|
|
last = None
|
|
while time.time() < deadline:
|
|
last = status_of(token, attachment_id)
|
|
if last["status"] == wanted:
|
|
return last
|
|
time.sleep(0.5)
|
|
raise SystemExit(f"attachment {attachment_id} did not reach {wanted}: {last}")
|
|
|
|
|
|
def scenario_worker_stopped(log):
|
|
log("\n== scenario 1: accept while worker stopped, resume -> ready ==")
|
|
compose("stop", "worker")
|
|
try:
|
|
token, chat_id = new_actor()
|
|
att_id = upload(token, chat_id)
|
|
time.sleep(2)
|
|
state = status_of(token, att_id)
|
|
log(f"with worker stopped: status={state['status']} (expected queued)")
|
|
assert state["status"] == "queued", state
|
|
compose("restart", "api")
|
|
wait_ready()
|
|
log("API restarted; work still queued (durable)")
|
|
assert status_of(token, att_id)["status"] == "queued"
|
|
finally:
|
|
compose("start", "worker")
|
|
ready = wait_status(token, att_id, "ready")
|
|
names = sorted(v["name"] for v in ready["variants"])
|
|
log(f"worker resumed: status=ready variants={names}")
|
|
assert {"thumbnail", "crop", "optimized"} <= set(names)
|
|
|
|
|
|
def scenario_outbox_gap(log):
|
|
log("\n== scenario 2: COMMIT/publish gap (broker down), API restart, then broker ==")
|
|
compose("stop", "broker")
|
|
try:
|
|
token, chat_id = new_actor()
|
|
att_id = upload(token, chat_id)
|
|
time.sleep(2)
|
|
job = psql(f"select published_at from attachment_jobs where attachment_id='{att_id}'")
|
|
log(f"broker down: status={status_of(token, att_id)['status']} published_at={job!r} (expected empty)")
|
|
assert status_of(token, att_id)["status"] in {"queued", "processing"}
|
|
compose("restart", "api")
|
|
wait_ready()
|
|
log("API restarted with broker still down; intent preserved in PostgreSQL")
|
|
finally:
|
|
compose("start", "broker")
|
|
ready = wait_status(token, att_id, "ready", timeout=180)
|
|
log(f"broker back: status={ready['status']} (outbox republished without new task)")
|
|
|
|
|
|
def scenario_retry_replay(log):
|
|
log("\n== scenario 3: transient failures -> failed(retry_exhausted) -> replay ==")
|
|
compose("stop", "worker")
|
|
while service_running("worker"):
|
|
time.sleep(0.5)
|
|
token, chat_id = new_actor()
|
|
att_id = upload(token, chat_id)
|
|
original_key = psql(f"select staging_key from attachments where id='{att_id}'")
|
|
# Fault injection: the durable original is (temporarily) unavailable, so the
|
|
# worker's fetch fails and is retried with backoff until attempts are spent.
|
|
psql(f"update attachments set staging_key='staging/missing-{att_id}' where id='{att_id}'")
|
|
log("injected: staging object points to a missing key (transient fetch failure)")
|
|
compose("start", "worker")
|
|
failed = wait_status(token, att_id, "failed", timeout=180)
|
|
log(f"status=failed error_code={failed['error_code']} (expected retry_exhausted)")
|
|
assert failed["error_code"] == "retry_exhausted", failed
|
|
# Operator repairs the cause and replays the failed job explicitly.
|
|
psql(f"update attachments set staging_key='{original_key}' where id='{att_id}'")
|
|
job_id = att_id_job(att_id)
|
|
replayed = subprocess.run(
|
|
["docker", "compose", "exec", "-T", "api", "python", "-m", "src.replay", job_id],
|
|
capture_output=True, text=True,
|
|
).stdout.strip()
|
|
log(f"explicit replay after recovery: {replayed}")
|
|
ready = wait_status(token, att_id, "ready", timeout=120)
|
|
log(f"after replay: status=ready variants={sorted(v['name'] for v in ready['variants'])}")
|
|
|
|
|
|
def att_id_job(attachment_id):
|
|
return psql(f"select job_id from attachment_jobs where attachment_id='{attachment_id}'")
|
|
|
|
|
|
def scenario_redelivery(log):
|
|
log("\n== scenario 4: redelivery of a ready job is a no-op ==")
|
|
token, chat_id = new_actor()
|
|
att_id = upload(token, chat_id)
|
|
ready = wait_status(token, att_id, "ready")
|
|
before = sorted((v["name"], v["sha256"]) for v in ready["variants"])
|
|
# Republish the finished job (duplicate delivery).
|
|
psql(f"update attachment_jobs set status='pending', published_at=null, available_at=now() where attachment_id='{att_id}'")
|
|
log("re-queued the same job (duplicate delivery)")
|
|
time.sleep(6)
|
|
after = status_of(token, att_id)
|
|
after_variants = sorted((v["name"], v["sha256"]) for v in after["variants"])
|
|
log(f"status={after['status']} variants={[n for n,_ in after_variants]} (must be unchanged)")
|
|
assert after["status"] == "ready"
|
|
assert after_variants == before, "duplicate delivery changed variants"
|
|
|
|
|
|
def scenario_lease_recovery(log):
|
|
log("\n== scenario 5: stuck processing -> lease recovery ==")
|
|
compose("stop", "worker")
|
|
token, chat_id = new_actor()
|
|
att_id = upload(token, chat_id)
|
|
job_id = att_id_job(att_id)
|
|
# Simulate a worker that died mid-processing with an already-expired lease.
|
|
psql(f"update attachments set status='processing' where id='{att_id}'")
|
|
psql(f"update attachment_jobs set status='processing', lease_until=now() - interval '5 seconds' where job_id='{job_id}'")
|
|
log("marked attachment+job processing with an expired lease")
|
|
compose("start", "worker")
|
|
ready = wait_status(token, att_id, "ready", timeout=120)
|
|
log(f"watchdog requeued and finished the job: status={ready['status']}")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--only", choices=["1", "2", "3", "4", "5"], help="run one scenario")
|
|
args = parser.parse_args()
|
|
wait_ready()
|
|
|
|
def log(message):
|
|
print(message, flush=True)
|
|
|
|
scenarios = {
|
|
"1": scenario_worker_stopped,
|
|
"2": scenario_outbox_gap,
|
|
"3": scenario_retry_replay,
|
|
"4": scenario_redelivery,
|
|
"5": scenario_lease_recovery,
|
|
}
|
|
if args.only:
|
|
scenarios[args.only](log)
|
|
else:
|
|
for name in ("1", "2", "3", "4", "5"):
|
|
scenarios[name](log)
|
|
print("\nPASS: lab5 grade-4 failure scenarios" + (f" (scenario {args.only})" if args.only else ""))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|