Completed for a grade of four
This commit is contained in:
@@ -0,0 +1,187 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove the producer and worker crash boundaries on a local Compose stand."""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check
|
||||
from image_probe import jpeg_size
|
||||
|
||||
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def docker(*args, timeout=45, expected=0):
|
||||
result = subprocess.run(
|
||||
["docker", *args],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
check(result.returncode == expected, f"docker {args[0]} returned {result.returncode}, expected {expected}")
|
||||
return (result.stdout if expected == 0 else result.stdout + result.stderr).strip()
|
||||
|
||||
|
||||
def compose(*args, **kwargs):
|
||||
return docker("compose", *args, **kwargs)
|
||||
|
||||
|
||||
def sql(query):
|
||||
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-Atc", query)
|
||||
|
||||
|
||||
def metadata_state(attachment_id):
|
||||
return sql(
|
||||
"SELECT a.status, j.attempts, EXISTS "
|
||||
"(SELECT 1 FROM attachment_outbox o WHERE o.job_id = j.id) "
|
||||
"FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id "
|
||||
f"WHERE a.id = '{uuid.UUID(attachment_id)}'"
|
||||
)
|
||||
|
||||
|
||||
def await_api(client):
|
||||
deadline = time.monotonic() + 30
|
||||
while True:
|
||||
try:
|
||||
client.call("GET", "/health/ready")
|
||||
return
|
||||
except Failure:
|
||||
check(time.monotonic() < deadline, "API did not recover")
|
||||
time.sleep(0.25)
|
||||
|
||||
|
||||
def upload(client, actor, chat):
|
||||
return client.call(
|
||||
"POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png",
|
||||
202, actor=actor, raw=FIXTURE, headers={"Content-Type": "image/png"}, schema="Attachment",
|
||||
)
|
||||
|
||||
|
||||
def verify_result(client, actor, queued):
|
||||
attachment = client.wait_ready(actor, queued, wait_seconds=75)
|
||||
path = f"/api/v1/attachments/{queued['id']}/content"
|
||||
original = client.call("GET", path, actor=actor)["bytes"]
|
||||
check(original == FIXTURE, "original bytes changed")
|
||||
|
||||
variants = {variant["name"]: variant for variant in attachment["variants"]}
|
||||
check(set(variants) == {"thumbnail", "crop", "optimized"}, "variant set differs")
|
||||
for name, variant in variants.items():
|
||||
response = client.call("GET", f"{path}?variant={name}", actor=actor)
|
||||
data = response["bytes"]
|
||||
check(hashlib.sha256(data).hexdigest() == variant["sha256"], f"{name} checksum differs")
|
||||
check(len(data) == variant["size_bytes"], f"{name} size differs")
|
||||
check(response["headers"].get_content_type() == variant["content_type"], f"{name} MIME differs")
|
||||
if name == "optimized":
|
||||
check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP")
|
||||
check((variant["width"], variant["height"]) == (320, 200), "optimized dimensions differ")
|
||||
else:
|
||||
expected = (128, 128) if name == "crop" else (256, 160)
|
||||
check(jpeg_size(data) == expected, f"{name} dimensions differ")
|
||||
|
||||
|
||||
def producer_crash(client, actor, reader, chat):
|
||||
compose("stop", "worker")
|
||||
compose("stop", "broker")
|
||||
try:
|
||||
result = upload(client, actor, chat)
|
||||
queued = result["json"]
|
||||
check(metadata_state(queued["id"]) == "queued|0|t", "outbox did not commit with the upload")
|
||||
|
||||
instance = result["headers"]["X-Instance-Id"]
|
||||
matches = [cid for cid in compose("ps", "-q", "api").splitlines() if cid.startswith(instance)]
|
||||
check(len(matches) == 1, "uploading API replica was not identified")
|
||||
replica = matches[0]
|
||||
before = docker("inspect", "-f", "{{.State.StartedAt}}", replica)
|
||||
docker("kill", "--signal", "KILL", replica)
|
||||
docker("start", replica)
|
||||
await_api(client)
|
||||
check(before != docker("inspect", "-f", "{{.State.StartedAt}}", replica), "uploading API did not restart")
|
||||
check(metadata_state(queued["id"]) == "queued|0|t", "pending outbox lost after API crash")
|
||||
print(f"PASS: producer SIGKILL after PostgreSQL commit and before publish; outbox survived; id={queued['id']}", flush=True)
|
||||
|
||||
compose("start", "broker", "worker")
|
||||
verify_result(client, reader, queued)
|
||||
print(f"PASS: outbox recovered the same attachment with all three variants; id={queued['id']}", flush=True)
|
||||
finally:
|
||||
compose("start", "broker", "api", "worker")
|
||||
|
||||
|
||||
def worker_crash(client, actor, reader, chat):
|
||||
compose("stop", "worker")
|
||||
temporary = "lab5-worker-fault-" + uuid.uuid4().hex[:12]
|
||||
try:
|
||||
queued = upload(client, actor, chat)["json"]
|
||||
compose(
|
||||
"run", "--rm", "--no-deps", "-T", "--name", temporary,
|
||||
"-e", f"LAB5_TEST_CRASH_AFTER_OBJECTS={queued['id']}",
|
||||
"-e", "JOB_TIMEOUT_SECONDS=15", "worker",
|
||||
timeout=70, expected=86,
|
||||
)
|
||||
check(metadata_state(queued["id"]).startswith("processing|1|"), "worker did not stop before committing results")
|
||||
client.call("GET", f"/api/v1/attachments/{queued['id']}/content", 409, actor=reader)
|
||||
print(f"PASS: worker exited 86 after writing S3 objects, before result commit and ack; id={queued['id']}", flush=True)
|
||||
|
||||
compose("start", "worker")
|
||||
verify_result(client, reader, queued)
|
||||
check(metadata_state(queued["id"]).startswith("ready|2|"), "recovery did not use exactly one new attempt")
|
||||
print(f"PASS: redelivery recovered the same id, original and three variants on attempt 2; id={queued['id']}", flush=True)
|
||||
finally:
|
||||
subprocess.run(["docker", "rm", "-f", temporary], capture_output=True, cwd=ROOT)
|
||||
compose("start", "worker")
|
||||
|
||||
|
||||
def operator_replay(client, actor, reader, chat):
|
||||
compose("stop", "worker")
|
||||
temporary = "lab5-worker-replay-" + uuid.uuid4().hex[:12]
|
||||
try:
|
||||
queued = upload(client, actor, chat)["json"]
|
||||
job = sql(f"SELECT id FROM attachment_jobs WHERE attachment_id = '{uuid.UUID(queued['id'])}'")
|
||||
job = str(uuid.UUID(job))
|
||||
compose("stop", "s3")
|
||||
compose(
|
||||
"run", "-d", "--rm", "--no-deps", "--name", temporary,
|
||||
"-e", "JOB_MAX_ATTEMPTS=1", "worker",
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + 40
|
||||
while not metadata_state(queued["id"]).startswith("failed|1|"):
|
||||
check(time.monotonic() < deadline, "one-attempt worker did not record the storage failure")
|
||||
time.sleep(0.25)
|
||||
|
||||
docker("stop", "--time", "6", temporary)
|
||||
compose("up", "-d", "--wait", "s3")
|
||||
await_api(client)
|
||||
|
||||
command = ("run", "--rm", "--no-deps", "-T", "worker", "attachment-jobs")
|
||||
listed = compose(*command, "failed")
|
||||
check(any(line.startswith(job + "\t") for line in listed.splitlines()), "failed job was not listed")
|
||||
check(f"QUEUED {job}" in compose(*command, "replay", job, "local-verification"), "replay was not queued")
|
||||
check(metadata_state(queued["id"]) == "queued|1|t", "operator replay was not committed with outbox")
|
||||
compose("start", "worker")
|
||||
verify_result(client, reader, queued)
|
||||
|
||||
refused = compose(*command, "replay", job, "local-verification", expected=2)
|
||||
check("REJECTED" in refused, "successful job accepted a duplicate replay")
|
||||
check(metadata_state(queued["id"]).startswith("ready|2|"), "successful result changed on duplicate replay")
|
||||
print(f"PASS: failed job listed, replayed with same id on attempt 2, and replay refused after ready; id={queued['id']}", flush=True)
|
||||
finally:
|
||||
subprocess.run(["docker", "rm", "-f", temporary], capture_output=True, cwd=ROOT)
|
||||
compose("start", "s3", "worker")
|
||||
|
||||
|
||||
def main():
|
||||
client = Client(timeout=15)
|
||||
check(client.base.startswith(("http://localhost:", "http://127.0.0.1:")), "use a local Compose stand")
|
||||
actor, reader, _, chat = client.bootstrap()
|
||||
producer_crash(client, actor, reader, chat)
|
||||
worker_crash(client, actor, reader, chat)
|
||||
operator_replay(client, actor, reader, chat)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -54,7 +54,7 @@ def main():
|
||||
thumbnail = client.call('GET', path + '/content?variant=thumbnail', actor=bob)['bytes']
|
||||
check(downloaded == original, 'original changed')
|
||||
check(jpeg_size(thumbnail) == (256, 160), 'incorrect thumbnail dimensions')
|
||||
variant = attachment['variants'][0]
|
||||
variant = next(item for item in attachment['variants'] if item['name'] == 'thumbnail')
|
||||
check(variant['sha256'] == hashlib.sha256(thumbnail).hexdigest(), 'thumbnail checksum mismatch')
|
||||
client.call('GET', path, 404, actor=eve)
|
||||
print(f"PASS: same accepted job reached ready; original unchanged; thumbnail JPEG 256x160 with matching checksum; attachment_id={attachment['id']}", flush=True)
|
||||
|
||||
@@ -12,7 +12,7 @@ if mc anonymous set download "app/$S3_BUCKET" >/dev/null 2>&1; then exit 1; fi
|
||||
printf test > /tmp/test-object
|
||||
if mc cp /tmp/test-object "app/$S3_BUCKET/forbidden-prefix/object" >/dev/null 2>&1; then exit 1; fi
|
||||
mc stat "app/$S3_BUCKET" >/dev/null
|
||||
printf "PASS: app cannot administer MinIO, create buckets, change policy or write outside originals/\n"
|
||||
printf "PASS: app cannot administer MinIO, create buckets, change policy or write to an unrelated prefix\n"
|
||||
'
|
||||
if docker compose exec -T lb wget -q -T 2 -O /dev/null http://s3:9001; then
|
||||
echo 'FAIL: console reachable from shared network'
|
||||
|
||||
Reference in New Issue
Block a user