forked from mtusi2026/mtusi2026lab6
105 lines
5.5 KiB
Python
105 lines
5.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Grade-3 asynchronous durability scenario against the local Compose stack."""
|
|
import hashlib
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from client import Client, arguments, check, run
|
|
from image_probe import jpeg_size
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def compose(*args):
|
|
command = ["docker", "compose"]
|
|
if (ROOT / ".env").exists():
|
|
command += ["--env-file", ".env"]
|
|
command += list(args)
|
|
subprocess.run(command, cwd=ROOT, check=True,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
|
|
|
|
def sql(statement):
|
|
command = ["docker", "compose"]
|
|
if (ROOT / ".env").exists():
|
|
command += ["--env-file", ".env"]
|
|
command += ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-Atc", statement]
|
|
return subprocess.run(command, cwd=ROOT, check=True, text=True,
|
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.strip()
|
|
|
|
|
|
def main():
|
|
args = arguments("Lab5 worker-stop and broker-restart durability scenario")
|
|
client = Client(args.base_url, args.ca_file)
|
|
compose("stop", "worker")
|
|
try:
|
|
alice, bob, _, chat = client.bootstrap()
|
|
attachment, original = client.attachment(alice, chat)
|
|
linked = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
|
check(linked["attachment_ids"] == [attachment["id"]], "queued attachment was not linkable")
|
|
client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", 409, actor=bob)
|
|
|
|
# Accepted staging, DB intent and durable queue must survive both API and
|
|
# broker restart while no worker is available.
|
|
compose("restart", "api", "broker")
|
|
compose("up", "-d", "--wait", "api", "broker")
|
|
compose("restart", "dispatcher")
|
|
snapshot = client.call("GET", "/api/v1/attachments/" + attachment["id"], actor=bob, schema="Attachment")["json"]
|
|
check(snapshot["status"] in ("queued", "processing"), "accepted job disappeared or failed during restart")
|
|
|
|
compose("start", "worker")
|
|
ready = client.wait_ready(bob, snapshot)
|
|
downloaded = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
|
|
check(downloaded["bytes"] == original, "original changed after asynchronous recovery")
|
|
variants = [item for item in ready["variants"] if item["name"] == "thumbnail"]
|
|
check(len(variants) == 1, "recovery did not produce exactly one thumbnail")
|
|
variant = variants[0]
|
|
thumbnail = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", actor=bob)
|
|
check(hashlib.sha256(thumbnail["bytes"]).hexdigest() == variant["sha256"], "thumbnail checksum mismatch")
|
|
check(jpeg_size(thumbnail["bytes"]) == (256, 160), "thumbnail dimensions mismatch")
|
|
|
|
# Force a redelivery of the same logical job. Deterministic keys and the
|
|
# variant primary key must turn it into the same effect, not a duplicate.
|
|
attachment_id = attachment["id"]
|
|
sql("UPDATE attachment_jobs SET status='queued', published_at=NULL, error_code=NULL WHERE attachment_id='" + attachment_id + "'")
|
|
deadline = time.monotonic() + 15
|
|
while time.monotonic() < deadline:
|
|
if sql("SELECT status FROM attachment_jobs WHERE attachment_id='" + attachment_id + "'") == "succeeded":
|
|
break
|
|
time.sleep(0.2)
|
|
else:
|
|
raise AssertionError("redelivered job did not finish")
|
|
count = sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='" + attachment_id + "' AND name='thumbnail'")
|
|
check(count == "1", "redelivery created duplicate variants")
|
|
after_duplicate = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=bob, schema="Attachment")["json"]
|
|
repeated = [item for item in after_duplicate["variants"] if item["name"] == "thumbnail"]
|
|
check(len(repeated) == 1 and repeated[0]["sha256"] == variant["sha256"], "redelivery changed the logical result")
|
|
# A transient storage failure is retried a bounded number of times and
|
|
# ends with a stable, non-sensitive API error code.
|
|
compose("stop", "worker")
|
|
failed_attachment, _ = client.attachment(alice, chat)
|
|
failed_id = failed_attachment["id"]
|
|
client.message(alice, chat, "", attachments=[failed_id])
|
|
sql("UPDATE attachments SET staging_key='staging/" + failed_id + "/missing' WHERE id='" + failed_id + "'")
|
|
compose("start", "worker")
|
|
deadline = time.monotonic() + 20
|
|
while time.monotonic() < deadline:
|
|
failed_snapshot = client.call("GET", "/api/v1/attachments/" + failed_id, actor=bob, schema="Attachment")["json"]
|
|
if failed_snapshot["status"] == "failed":
|
|
break
|
|
time.sleep(0.2)
|
|
else:
|
|
raise AssertionError("job did not reach failed after bounded retries")
|
|
check(failed_snapshot["error_code"] == "retry_exhausted", "unsafe or unexpected final error code")
|
|
retry_state = sql("SELECT attempt || ':' || status FROM attachment_jobs WHERE attachment_id='" + failed_id + "'")
|
|
check(retry_state == "5:failed", "job did not stop at configured retry limit")
|
|
client.call("GET", "/api/v1/attachments/" + failed_id + "/content", 409, actor=bob)
|
|
|
|
print("PASS: durable recovery, idempotent redelivery and bounded final failure verified.")
|
|
finally:
|
|
compose("up", "-d", "api", "broker", "dispatcher", "worker")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main) |