forked from mtusi2026/mtusi2026lab5
188 lines
7.9 KiB
Python
188 lines
7.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Prove the producer and worker crash boundaries on a local Compose stand."""
|
|
import hashlib
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, check
|
|
from image_probe import jpeg_size
|
|
|
|
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
|
|
|
|
def docker(*args, timeout=45, expected=0):
|
|
result = subprocess.run(
|
|
["docker", *args],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
)
|
|
check(result.returncode == expected, f"docker {args[0]} returned {result.returncode}, expected {expected}")
|
|
return (result.stdout if expected == 0 else result.stdout + result.stderr).strip()
|
|
|
|
|
|
def compose(*args, **kwargs):
|
|
return docker("compose", *args, **kwargs)
|
|
|
|
|
|
def sql(query):
|
|
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-Atc", query)
|
|
|
|
|
|
def metadata_state(attachment_id):
|
|
return sql(
|
|
"SELECT a.status, j.attempts, EXISTS "
|
|
"(SELECT 1 FROM attachment_outbox o WHERE o.job_id = j.id) "
|
|
"FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id "
|
|
f"WHERE a.id = '{uuid.UUID(attachment_id)}'"
|
|
)
|
|
|
|
|
|
def await_api(client):
|
|
deadline = time.monotonic() + 30
|
|
while True:
|
|
try:
|
|
client.call("GET", "/health/ready")
|
|
return
|
|
except Failure:
|
|
check(time.monotonic() < deadline, "API did not recover")
|
|
time.sleep(0.25)
|
|
|
|
|
|
def upload(client, actor, chat):
|
|
return client.call(
|
|
"POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png",
|
|
202, actor=actor, raw=FIXTURE, headers={"Content-Type": "image/png"}, schema="Attachment",
|
|
)
|
|
|
|
|
|
def verify_result(client, actor, queued):
|
|
attachment = client.wait_ready(actor, queued, wait_seconds=75)
|
|
path = f"/api/v1/attachments/{queued['id']}/content"
|
|
original = client.call("GET", path, actor=actor)["bytes"]
|
|
check(original == FIXTURE, "original bytes changed")
|
|
|
|
variants = {variant["name"]: variant for variant in attachment["variants"]}
|
|
check(set(variants) == {"thumbnail", "crop", "optimized"}, "variant set differs")
|
|
for name, variant in variants.items():
|
|
response = client.call("GET", f"{path}?variant={name}", actor=actor)
|
|
data = response["bytes"]
|
|
check(hashlib.sha256(data).hexdigest() == variant["sha256"], f"{name} checksum differs")
|
|
check(len(data) == variant["size_bytes"], f"{name} size differs")
|
|
check(response["headers"].get_content_type() == variant["content_type"], f"{name} MIME differs")
|
|
if name == "optimized":
|
|
check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP")
|
|
check((variant["width"], variant["height"]) == (320, 200), "optimized dimensions differ")
|
|
else:
|
|
expected = (128, 128) if name == "crop" else (256, 160)
|
|
check(jpeg_size(data) == expected, f"{name} dimensions differ")
|
|
|
|
|
|
def producer_crash(client, actor, reader, chat):
|
|
compose("stop", "worker")
|
|
compose("stop", "broker")
|
|
try:
|
|
result = upload(client, actor, chat)
|
|
queued = result["json"]
|
|
check(metadata_state(queued["id"]) == "queued|0|t", "outbox did not commit with the upload")
|
|
|
|
instance = result["headers"]["X-Instance-Id"]
|
|
matches = [cid for cid in compose("ps", "-q", "api").splitlines() if cid.startswith(instance)]
|
|
check(len(matches) == 1, "uploading API replica was not identified")
|
|
replica = matches[0]
|
|
before = docker("inspect", "-f", "{{.State.StartedAt}}", replica)
|
|
docker("kill", "--signal", "KILL", replica)
|
|
docker("start", replica)
|
|
await_api(client)
|
|
check(before != docker("inspect", "-f", "{{.State.StartedAt}}", replica), "uploading API did not restart")
|
|
check(metadata_state(queued["id"]) == "queued|0|t", "pending outbox lost after API crash")
|
|
print(f"PASS: producer SIGKILL after PostgreSQL commit and before publish; outbox survived; id={queued['id']}", flush=True)
|
|
|
|
compose("start", "broker", "worker")
|
|
verify_result(client, reader, queued)
|
|
print(f"PASS: outbox recovered the same attachment with all three variants; id={queued['id']}", flush=True)
|
|
finally:
|
|
compose("start", "broker", "api", "worker")
|
|
|
|
|
|
def worker_crash(client, actor, reader, chat):
|
|
compose("stop", "worker")
|
|
temporary = "lab5-worker-fault-" + uuid.uuid4().hex[:12]
|
|
try:
|
|
queued = upload(client, actor, chat)["json"]
|
|
compose(
|
|
"run", "--rm", "--no-deps", "-T", "--name", temporary,
|
|
"-e", f"LAB5_TEST_CRASH_AFTER_OBJECTS={queued['id']}",
|
|
"-e", "JOB_TIMEOUT_SECONDS=15", "worker",
|
|
timeout=70, expected=86,
|
|
)
|
|
check(metadata_state(queued["id"]).startswith("processing|1|"), "worker did not stop before committing results")
|
|
client.call("GET", f"/api/v1/attachments/{queued['id']}/content", 409, actor=reader)
|
|
print(f"PASS: worker exited 86 after writing S3 objects, before result commit and ack; id={queued['id']}", flush=True)
|
|
|
|
compose("start", "worker")
|
|
verify_result(client, reader, queued)
|
|
check(metadata_state(queued["id"]).startswith("ready|2|"), "recovery did not use exactly one new attempt")
|
|
print(f"PASS: redelivery recovered the same id, original and three variants on attempt 2; id={queued['id']}", flush=True)
|
|
finally:
|
|
subprocess.run(["docker", "rm", "-f", temporary], capture_output=True, cwd=ROOT)
|
|
compose("start", "worker")
|
|
|
|
|
|
def operator_replay(client, actor, reader, chat):
|
|
compose("stop", "worker")
|
|
temporary = "lab5-worker-replay-" + uuid.uuid4().hex[:12]
|
|
try:
|
|
queued = upload(client, actor, chat)["json"]
|
|
job = sql(f"SELECT id FROM attachment_jobs WHERE attachment_id = '{uuid.UUID(queued['id'])}'")
|
|
job = str(uuid.UUID(job))
|
|
compose("stop", "s3")
|
|
compose(
|
|
"run", "-d", "--rm", "--no-deps", "--name", temporary,
|
|
"-e", "JOB_MAX_ATTEMPTS=1", "worker",
|
|
)
|
|
|
|
deadline = time.monotonic() + 40
|
|
while not metadata_state(queued["id"]).startswith("failed|1|"):
|
|
check(time.monotonic() < deadline, "one-attempt worker did not record the storage failure")
|
|
time.sleep(0.25)
|
|
|
|
docker("stop", "--time", "6", temporary)
|
|
compose("up", "-d", "--wait", "s3")
|
|
await_api(client)
|
|
|
|
command = ("run", "--rm", "--no-deps", "-T", "worker", "attachment-jobs")
|
|
listed = compose(*command, "failed")
|
|
check(any(line.startswith(job + "\t") for line in listed.splitlines()), "failed job was not listed")
|
|
check(f"QUEUED {job}" in compose(*command, "replay", job, "local-verification"), "replay was not queued")
|
|
check(metadata_state(queued["id"]) == "queued|1|t", "operator replay was not committed with outbox")
|
|
compose("start", "worker")
|
|
verify_result(client, reader, queued)
|
|
|
|
refused = compose(*command, "replay", job, "local-verification", expected=2)
|
|
check("REJECTED" in refused, "successful job accepted a duplicate replay")
|
|
check(metadata_state(queued["id"]).startswith("ready|2|"), "successful result changed on duplicate replay")
|
|
print(f"PASS: failed job listed, replayed with same id on attempt 2, and replay refused after ready; id={queued['id']}", flush=True)
|
|
finally:
|
|
subprocess.run(["docker", "rm", "-f", temporary], capture_output=True, cwd=ROOT)
|
|
compose("start", "s3", "worker")
|
|
|
|
|
|
def main():
|
|
client = Client(timeout=15)
|
|
check(client.base.startswith(("http://localhost:", "http://127.0.0.1:")), "use a local Compose stand")
|
|
actor, reader, _, chat = client.bootstrap()
|
|
producer_crash(client, actor, reader, chat)
|
|
worker_crash(client, actor, reader, chat)
|
|
operator_replay(client, actor, reader, chat)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|