Files
mtusi2026lab5-fork/student-tests/level4_failures.py
T
2026-11-07 19:45:45 +03:00

177 lines
8.0 KiB
Python

#!/usr/bin/env python3
"""Lab5 level-4 failure scenarios against a RUNNING compose stack (stdlib only; run from the repo root).
python3 student-tests/level4_failures.py all # or: worker_down | commit_publish_gap | crash_after_s3 | replay | s3_outage
Each scenario drives `docker compose` (never `down -v`: volumes stay) and prints PASS/FAIL with the observed facts.
"""
import base64, json, os, subprocess, sys, time, urllib.error, urllib.request, uuid
from pathlib import Path
BASE = os.getenv("BASE_URL", "http://localhost:8080")
PNG = (Path(__file__).resolve().parents[1] / "tests/fixtures/sample.png").read_bytes()
DC = ["docker", "compose", "--env-file", ".env"]
def sh(*args, env=None, check=True):
r = subprocess.run(list(args), capture_output=True, text=True, env={**os.environ, **(env or {})})
if check and r.returncode:
raise SystemExit(f"command failed: {' '.join(args)}\n{r.stderr[-500:]}")
return r.stdout.strip()
def sql(query):
return sh(*DC, "exec", "-T", "postgres", "sh", "-c", f'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -c "{query}"')
def http(method, path, token=None, body=None, headers=None, raw=None):
h = dict(headers or {})
if token:
h["Authorization"] = "Bearer " + token
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
if body is not None:
h["Content-Type"] = "application/json"
req = urllib.request.Request(BASE + path, data=data, method=method, headers=h)
try:
with urllib.request.urlopen(req, timeout=30) as r:
payload = r.read()
return r.status, (json.loads(payload) if payload and "json" in r.headers.get("Content-Type", "") else payload)
except urllib.error.HTTPError as e:
payload = e.read()
try:
return e.code, json.loads(payload)
except ValueError:
return e.code, payload
def setup():
name = "u" + uuid.uuid4().hex[:10]
pw = "correct-horse-battery"
st, user = http("POST", "/api/v1/users", body={"username": name, "display_name": "Tester", "password": pw})
assert st == 201, (st, user)
cred = base64.b64encode(f"{name}:{pw}".encode()).decode()
st, sess = http("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
token = sess["token"]
st, chat = http("POST", "/api/v1/chats", token, {"title": "lab5", "member_ids": [user["id"]]})
return token, chat["id"]
def upload(token, chat):
st, att = http("POST", f"/api/v1/chats/{chat}/attachments?filename=sample.png", token, raw=PNG, headers={"Content-Type": "image/png"})
assert st == 202 and att["status"] == "queued", (st, att)
return att["id"]
def wait(token, aid, want, timeout=90):
end, last = time.time() + timeout, None
while time.time() < end:
st, att = http("GET", f"/api/v1/attachments/{aid}", token)
last = att
if st == 200 and att["status"] == want:
return att
time.sleep(1)
raise AssertionError(f"attachment {aid} did not reach {want}; last={last}")
def wait_ready_api(timeout=90):
end = time.time() + timeout
while time.time() < end:
if http("GET", "/health/ready")[0] == 200:
return
time.sleep(1)
raise AssertionError("API not ready")
def check_variants(att):
v = {x["name"]: x for x in att["variants"]}
assert (v["thumbnail"]["width"], v["thumbnail"]["height"], v["thumbnail"]["content_type"]) == (256, 160, "image/jpeg"), v["thumbnail"]
assert (v["crop"]["width"], v["crop"]["height"]) == (128, 128), v["crop"]
assert v["optimized"]["content_type"] == "image/webp"
def events(aid):
return sql(f"SELECT string_agg(event, ',' ORDER BY id) FROM job_events e JOIN attachment_jobs j USING (job_id) WHERE j.attachment_id='{aid}'")
def worker_down():
token, chat = setup()
sh(*DC, "stop", "worker")
aid = upload(token, chat)
st, _ = http("GET", f"/api/v1/attachments/{aid}/content", token)
assert st == 409, st # content of a queued attachment
st, msg = http("POST", f"/api/v1/chats/{chat}/messages", token, {"text": "hi", "attachment_ids": [aid]}, {"Idempotency-Key": "k1"})
assert st == 201, (st, msg) # other features keep working; queued attachment can be attached
sh(*DC, "restart", "api"); wait_ready_api()
assert http("GET", f"/api/v1/attachments/{aid}", token)[1]["status"] == "queued" # accepted work survived the API restart
sh(*DC, "up", "-d", "worker")
att = wait(token, aid, "ready"); check_variants(att)
print("PASS worker_down:", aid, "events:", events(aid))
def commit_publish_gap():
token, chat = setup()
sh(*DC, "up", "-d", "dispatcher", env={"FAULT_DISPATCHER_EXIT_BEFORE_PUBLISH": "1"}, check=True)
sh(*DC, "exec", "-T", "dispatcher", "rm", "-f", "/tmp/dispatcher_fault_done", check=False)
sh(*DC, "restart", "dispatcher") # fresh container process: the fault fires once, on the first outbox batch
time.sleep(3)
aid = upload(token, chat) # COMMIT happens; the dispatcher dies before publishing
att = wait(token, aid, "ready", 90); check_variants(att)
sh(*DC, "up", "-d", "dispatcher") # back to the normal configuration (.env: fault off)
print("PASS commit_publish_gap:", aid, "events:", events(aid))
def crash_after_s3():
token, chat = setup()
env = {"FAULT_WORKER_EXIT_AFTER_S3": "1", "JOB_TIMEOUT_SECONDS": "10", "LEASE_GRACE_SECONDS": "5"}
sh(*DC, "up", "-d", "worker", env=env)
time.sleep(5)
aid = upload(token, chat)
att = wait(token, aid, "ready", 150); check_variants(att)
n = sql(f"SELECT count(*) FROM attachment_variants WHERE attachment_id='{aid}'")
assert n == "3", n # the retry completed the SAME result, no second set of variants
sh(*DC, "up", "-d", "worker") # restore normal worker config
print("PASS crash_after_s3:", aid, "events:", events(aid))
def replay():
token, chat = setup()
sh(*DC, "stop", "worker")
aid = upload(token, chat)
good = sql(f"SELECT sha256 FROM attachments WHERE id='{aid}'")
sql(f"UPDATE attachments SET sha256=repeat('0',64) WHERE id='{aid}'") # simulate corrupted/tampered staging bytes
sh(*DC, "up", "-d", "worker")
att = wait(token, aid, "failed", 60)
assert att["error_code"] == "integrity_error", att # permanent error: failed at once, no retries
st, _ = http("POST", f"/api/v1/chats/{chat}/messages", token, {"text": "", "attachment_ids": [aid]}, {"Idempotency-Key": "k2"})
assert st == 409, st
sql(f"UPDATE attachments SET sha256='{good}' WHERE id='{aid}'") # operator fixes the cause
job = sql(f"SELECT job_id FROM attachment_jobs WHERE attachment_id='{aid}'")
print(sh(*DC, "exec", "-T", "worker", "python", "scripts/replay_failed.py", "--job-id", job, "--reason", "student-test"))
att = wait(token, aid, "ready", 60); check_variants(att)
again = sh(*DC, "exec", "-T", "worker", "python", "scripts/replay_failed.py", "--job-id", job, check=False)
assert "REFUSED" in again, again # replay of a successful job is refused
print("PASS replay:", aid, "events:", events(aid))
def s3_outage():
token, chat = setup()
sh(*DC, "stop", "worker")
aid = upload(token, chat)
sh(*DC, "stop", "s3")
sh(*DC, "up", "-d", "--no-deps", "worker")
time.sleep(15)
ev = events(aid)
assert "retry_scheduled" in ev, ev # temporary error => backoff with jitter, not failed
sh(*DC, "start", "s3"); wait_ready_api()
att = wait(token, aid, "ready", 120); check_variants(att)
print("PASS s3_outage:", aid, "events:", events(aid))
SCENARIOS = {"worker_down": worker_down, "commit_publish_gap": commit_publish_gap, "crash_after_s3": crash_after_s3, "replay": replay, "s3_outage": s3_outage}
if __name__ == "__main__":
which = sys.argv[1] if len(sys.argv) > 1 else "all"
for name, fn in SCENARIOS.items():
if which in ("all", name):
fn()