forked from mtusi2026/mtusi2026lab5
177 lines
8.0 KiB
Python
177 lines
8.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Lab5 level-4 failure scenarios against a RUNNING compose stack (stdlib only; run from the repo root).
|
|
|
|
python3 student-tests/level4_failures.py all # or: worker_down | commit_publish_gap | crash_after_s3 | replay | s3_outage
|
|
|
|
Each scenario drives `docker compose` (never `down -v`: volumes stay) and prints PASS/FAIL with the observed facts.
|
|
"""
|
|
import base64, json, os, subprocess, sys, time, urllib.error, urllib.request, uuid
|
|
from pathlib import Path
|
|
|
|
BASE = os.getenv("BASE_URL", "http://localhost:8080")
|
|
PNG = (Path(__file__).resolve().parents[1] / "tests/fixtures/sample.png").read_bytes()
|
|
DC = ["docker", "compose", "--env-file", ".env"]
|
|
|
|
|
|
def sh(*args, env=None, check=True):
|
|
r = subprocess.run(list(args), capture_output=True, text=True, env={**os.environ, **(env or {})})
|
|
if check and r.returncode:
|
|
raise SystemExit(f"command failed: {' '.join(args)}\n{r.stderr[-500:]}")
|
|
return r.stdout.strip()
|
|
|
|
|
|
def sql(query):
|
|
return sh(*DC, "exec", "-T", "postgres", "sh", "-c", f'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -c "{query}"')
|
|
|
|
|
|
def http(method, path, token=None, body=None, headers=None, raw=None):
|
|
h = dict(headers or {})
|
|
if token:
|
|
h["Authorization"] = "Bearer " + token
|
|
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
|
|
if body is not None:
|
|
h["Content-Type"] = "application/json"
|
|
req = urllib.request.Request(BASE + path, data=data, method=method, headers=h)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
payload = r.read()
|
|
return r.status, (json.loads(payload) if payload and "json" in r.headers.get("Content-Type", "") else payload)
|
|
except urllib.error.HTTPError as e:
|
|
payload = e.read()
|
|
try:
|
|
return e.code, json.loads(payload)
|
|
except ValueError:
|
|
return e.code, payload
|
|
|
|
|
|
def setup():
|
|
name = "u" + uuid.uuid4().hex[:10]
|
|
pw = "correct-horse-battery"
|
|
st, user = http("POST", "/api/v1/users", body={"username": name, "display_name": "Tester", "password": pw})
|
|
assert st == 201, (st, user)
|
|
cred = base64.b64encode(f"{name}:{pw}".encode()).decode()
|
|
st, sess = http("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
|
token = sess["token"]
|
|
st, chat = http("POST", "/api/v1/chats", token, {"title": "lab5", "member_ids": [user["id"]]})
|
|
return token, chat["id"]
|
|
|
|
|
|
def upload(token, chat):
|
|
st, att = http("POST", f"/api/v1/chats/{chat}/attachments?filename=sample.png", token, raw=PNG, headers={"Content-Type": "image/png"})
|
|
assert st == 202 and att["status"] == "queued", (st, att)
|
|
return att["id"]
|
|
|
|
|
|
def wait(token, aid, want, timeout=90):
|
|
end, last = time.time() + timeout, None
|
|
while time.time() < end:
|
|
st, att = http("GET", f"/api/v1/attachments/{aid}", token)
|
|
last = att
|
|
if st == 200 and att["status"] == want:
|
|
return att
|
|
time.sleep(1)
|
|
raise AssertionError(f"attachment {aid} did not reach {want}; last={last}")
|
|
|
|
|
|
def wait_ready_api(timeout=90):
|
|
end = time.time() + timeout
|
|
while time.time() < end:
|
|
if http("GET", "/health/ready")[0] == 200:
|
|
return
|
|
time.sleep(1)
|
|
raise AssertionError("API not ready")
|
|
|
|
|
|
def check_variants(att):
|
|
v = {x["name"]: x for x in att["variants"]}
|
|
assert (v["thumbnail"]["width"], v["thumbnail"]["height"], v["thumbnail"]["content_type"]) == (256, 160, "image/jpeg"), v["thumbnail"]
|
|
assert (v["crop"]["width"], v["crop"]["height"]) == (128, 128), v["crop"]
|
|
assert v["optimized"]["content_type"] == "image/webp"
|
|
|
|
|
|
def events(aid):
|
|
return sql(f"SELECT string_agg(event, ',' ORDER BY id) FROM job_events e JOIN attachment_jobs j USING (job_id) WHERE j.attachment_id='{aid}'")
|
|
|
|
|
|
def worker_down():
|
|
token, chat = setup()
|
|
sh(*DC, "stop", "worker")
|
|
aid = upload(token, chat)
|
|
st, _ = http("GET", f"/api/v1/attachments/{aid}/content", token)
|
|
assert st == 409, st # content of a queued attachment
|
|
st, msg = http("POST", f"/api/v1/chats/{chat}/messages", token, {"text": "hi", "attachment_ids": [aid]}, {"Idempotency-Key": "k1"})
|
|
assert st == 201, (st, msg) # other features keep working; queued attachment can be attached
|
|
sh(*DC, "restart", "api"); wait_ready_api()
|
|
assert http("GET", f"/api/v1/attachments/{aid}", token)[1]["status"] == "queued" # accepted work survived the API restart
|
|
sh(*DC, "up", "-d", "worker")
|
|
att = wait(token, aid, "ready"); check_variants(att)
|
|
print("PASS worker_down:", aid, "events:", events(aid))
|
|
|
|
|
|
def commit_publish_gap():
|
|
token, chat = setup()
|
|
sh(*DC, "up", "-d", "dispatcher", env={"FAULT_DISPATCHER_EXIT_BEFORE_PUBLISH": "1"}, check=True)
|
|
sh(*DC, "exec", "-T", "dispatcher", "rm", "-f", "/tmp/dispatcher_fault_done", check=False)
|
|
sh(*DC, "restart", "dispatcher") # fresh container process: the fault fires once, on the first outbox batch
|
|
time.sleep(3)
|
|
aid = upload(token, chat) # COMMIT happens; the dispatcher dies before publishing
|
|
att = wait(token, aid, "ready", 90); check_variants(att)
|
|
sh(*DC, "up", "-d", "dispatcher") # back to the normal configuration (.env: fault off)
|
|
print("PASS commit_publish_gap:", aid, "events:", events(aid))
|
|
|
|
|
|
def crash_after_s3():
|
|
token, chat = setup()
|
|
env = {"FAULT_WORKER_EXIT_AFTER_S3": "1", "JOB_TIMEOUT_SECONDS": "10", "LEASE_GRACE_SECONDS": "5"}
|
|
sh(*DC, "up", "-d", "worker", env=env)
|
|
time.sleep(5)
|
|
aid = upload(token, chat)
|
|
att = wait(token, aid, "ready", 150); check_variants(att)
|
|
n = sql(f"SELECT count(*) FROM attachment_variants WHERE attachment_id='{aid}'")
|
|
assert n == "3", n # the retry completed the SAME result, no second set of variants
|
|
sh(*DC, "up", "-d", "worker") # restore normal worker config
|
|
print("PASS crash_after_s3:", aid, "events:", events(aid))
|
|
|
|
|
|
def replay():
|
|
token, chat = setup()
|
|
sh(*DC, "stop", "worker")
|
|
aid = upload(token, chat)
|
|
good = sql(f"SELECT sha256 FROM attachments WHERE id='{aid}'")
|
|
sql(f"UPDATE attachments SET sha256=repeat('0',64) WHERE id='{aid}'") # simulate corrupted/tampered staging bytes
|
|
sh(*DC, "up", "-d", "worker")
|
|
att = wait(token, aid, "failed", 60)
|
|
assert att["error_code"] == "integrity_error", att # permanent error: failed at once, no retries
|
|
st, _ = http("POST", f"/api/v1/chats/{chat}/messages", token, {"text": "", "attachment_ids": [aid]}, {"Idempotency-Key": "k2"})
|
|
assert st == 409, st
|
|
sql(f"UPDATE attachments SET sha256='{good}' WHERE id='{aid}'") # operator fixes the cause
|
|
job = sql(f"SELECT job_id FROM attachment_jobs WHERE attachment_id='{aid}'")
|
|
print(sh(*DC, "exec", "-T", "worker", "python", "scripts/replay_failed.py", "--job-id", job, "--reason", "student-test"))
|
|
att = wait(token, aid, "ready", 60); check_variants(att)
|
|
again = sh(*DC, "exec", "-T", "worker", "python", "scripts/replay_failed.py", "--job-id", job, check=False)
|
|
assert "REFUSED" in again, again # replay of a successful job is refused
|
|
print("PASS replay:", aid, "events:", events(aid))
|
|
|
|
|
|
def s3_outage():
|
|
token, chat = setup()
|
|
sh(*DC, "stop", "worker")
|
|
aid = upload(token, chat)
|
|
sh(*DC, "stop", "s3")
|
|
sh(*DC, "up", "-d", "--no-deps", "worker")
|
|
time.sleep(15)
|
|
ev = events(aid)
|
|
assert "retry_scheduled" in ev, ev # temporary error => backoff with jitter, not failed
|
|
sh(*DC, "start", "s3"); wait_ready_api()
|
|
att = wait(token, aid, "ready", 120); check_variants(att)
|
|
print("PASS s3_outage:", aid, "events:", events(aid))
|
|
|
|
|
|
SCENARIOS = {"worker_down": worker_down, "commit_publish_gap": commit_publish_gap, "crash_after_s3": crash_after_s3, "replay": replay, "s3_outage": s3_outage}
|
|
|
|
if __name__ == "__main__":
|
|
which = sys.argv[1] if len(sys.argv) > 1 else "all"
|
|
for name, fn in SCENARIOS.items():
|
|
if which in ("all", name):
|
|
fn()
|