Files
mtusi2026lab6/tests/async_resilience.py
T

105 lines
5.5 KiB
Python

#!/usr/bin/env python3
"""Grade-3 asynchronous durability scenario against the local Compose stack."""
import hashlib
import subprocess
import time
from pathlib import Path
from client import Client, arguments, check, run
from image_probe import jpeg_size
ROOT = Path(__file__).resolve().parents[1]
def compose(*args):
command = ["docker", "compose"]
if (ROOT / ".env").exists():
command += ["--env-file", ".env"]
command += list(args)
subprocess.run(command, cwd=ROOT, check=True,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def sql(statement):
command = ["docker", "compose"]
if (ROOT / ".env").exists():
command += ["--env-file", ".env"]
command += ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-Atc", statement]
return subprocess.run(command, cwd=ROOT, check=True, text=True,
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.strip()
def main():
args = arguments("Lab5 worker-stop and broker-restart durability scenario")
client = Client(args.base_url, args.ca_file)
compose("stop", "worker")
try:
alice, bob, _, chat = client.bootstrap()
attachment, original = client.attachment(alice, chat)
linked = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
check(linked["attachment_ids"] == [attachment["id"]], "queued attachment was not linkable")
client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", 409, actor=bob)
# Accepted staging, DB intent and durable queue must survive both API and
# broker restart while no worker is available.
compose("restart", "api", "broker")
compose("up", "-d", "--wait", "api", "broker")
compose("restart", "dispatcher")
snapshot = client.call("GET", "/api/v1/attachments/" + attachment["id"], actor=bob, schema="Attachment")["json"]
check(snapshot["status"] in ("queued", "processing"), "accepted job disappeared or failed during restart")
compose("start", "worker")
ready = client.wait_ready(bob, snapshot)
downloaded = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
check(downloaded["bytes"] == original, "original changed after asynchronous recovery")
variants = [item for item in ready["variants"] if item["name"] == "thumbnail"]
check(len(variants) == 1, "recovery did not produce exactly one thumbnail")
variant = variants[0]
thumbnail = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", actor=bob)
check(hashlib.sha256(thumbnail["bytes"]).hexdigest() == variant["sha256"], "thumbnail checksum mismatch")
check(jpeg_size(thumbnail["bytes"]) == (256, 160), "thumbnail dimensions mismatch")
# Force a redelivery of the same logical job. Deterministic keys and the
# variant primary key must turn it into the same effect, not a duplicate.
attachment_id = attachment["id"]
sql("UPDATE attachment_jobs SET status='queued', published_at=NULL, error_code=NULL WHERE attachment_id='" + attachment_id + "'")
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
if sql("SELECT status FROM attachment_jobs WHERE attachment_id='" + attachment_id + "'") == "succeeded":
break
time.sleep(0.2)
else:
raise AssertionError("redelivered job did not finish")
count = sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='" + attachment_id + "' AND name='thumbnail'")
check(count == "1", "redelivery created duplicate variants")
after_duplicate = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=bob, schema="Attachment")["json"]
repeated = [item for item in after_duplicate["variants"] if item["name"] == "thumbnail"]
check(len(repeated) == 1 and repeated[0]["sha256"] == variant["sha256"], "redelivery changed the logical result")
# A transient storage failure is retried a bounded number of times and
# ends with a stable, non-sensitive API error code.
compose("stop", "worker")
failed_attachment, _ = client.attachment(alice, chat)
failed_id = failed_attachment["id"]
client.message(alice, chat, "", attachments=[failed_id])
sql("UPDATE attachments SET staging_key='staging/" + failed_id + "/missing' WHERE id='" + failed_id + "'")
compose("start", "worker")
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
failed_snapshot = client.call("GET", "/api/v1/attachments/" + failed_id, actor=bob, schema="Attachment")["json"]
if failed_snapshot["status"] == "failed":
break
time.sleep(0.2)
else:
raise AssertionError("job did not reach failed after bounded retries")
check(failed_snapshot["error_code"] == "retry_exhausted", "unsafe or unexpected final error code")
retry_state = sql("SELECT attempt || ':' || status FROM attachment_jobs WHERE attachment_id='" + failed_id + "'")
check(retry_state == "5:failed", "job did not stop at configured retry limit")
client.call("GET", "/api/v1/attachments/" + failed_id + "/content", 409, actor=bob)
print("PASS: durable recovery, idempotent redelivery and bounded final failure verified.")
finally:
compose("up", "-d", "api", "broker", "dispatcher", "worker")
if __name__ == "__main__":
run(main)