Completed for a grade of four

This commit is contained in:
2026-11-19 16:48:52 +03:00
parent 248f8a5cb0
commit be2abbb734
50 changed files with 1831 additions and 532 deletions
+187
View File
@@ -0,0 +1,187 @@
#!/usr/bin/env python3
"""Prove the producer and worker crash boundaries on a local Compose stand."""
import hashlib
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check
from image_probe import jpeg_size
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
def docker(*args, timeout=45, expected=0):
result = subprocess.run(
["docker", *args],
cwd=ROOT,
capture_output=True,
text=True,
timeout=timeout,
)
check(result.returncode == expected, f"docker {args[0]} returned {result.returncode}, expected {expected}")
return (result.stdout if expected == 0 else result.stdout + result.stderr).strip()
def compose(*args, **kwargs):
return docker("compose", *args, **kwargs)
def sql(query):
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-Atc", query)
def metadata_state(attachment_id):
return sql(
"SELECT a.status, j.attempts, EXISTS "
"(SELECT 1 FROM attachment_outbox o WHERE o.job_id = j.id) "
"FROM attachments a JOIN attachment_jobs j ON j.attachment_id = a.id "
f"WHERE a.id = '{uuid.UUID(attachment_id)}'"
)
def await_api(client):
deadline = time.monotonic() + 30
while True:
try:
client.call("GET", "/health/ready")
return
except Failure:
check(time.monotonic() < deadline, "API did not recover")
time.sleep(0.25)
def upload(client, actor, chat):
return client.call(
"POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png",
202, actor=actor, raw=FIXTURE, headers={"Content-Type": "image/png"}, schema="Attachment",
)
def verify_result(client, actor, queued):
attachment = client.wait_ready(actor, queued, wait_seconds=75)
path = f"/api/v1/attachments/{queued['id']}/content"
original = client.call("GET", path, actor=actor)["bytes"]
check(original == FIXTURE, "original bytes changed")
variants = {variant["name"]: variant for variant in attachment["variants"]}
check(set(variants) == {"thumbnail", "crop", "optimized"}, "variant set differs")
for name, variant in variants.items():
response = client.call("GET", f"{path}?variant={name}", actor=actor)
data = response["bytes"]
check(hashlib.sha256(data).hexdigest() == variant["sha256"], f"{name} checksum differs")
check(len(data) == variant["size_bytes"], f"{name} size differs")
check(response["headers"].get_content_type() == variant["content_type"], f"{name} MIME differs")
if name == "optimized":
check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP")
check((variant["width"], variant["height"]) == (320, 200), "optimized dimensions differ")
else:
expected = (128, 128) if name == "crop" else (256, 160)
check(jpeg_size(data) == expected, f"{name} dimensions differ")
def producer_crash(client, actor, reader, chat):
compose("stop", "worker")
compose("stop", "broker")
try:
result = upload(client, actor, chat)
queued = result["json"]
check(metadata_state(queued["id"]) == "queued|0|t", "outbox did not commit with the upload")
instance = result["headers"]["X-Instance-Id"]
matches = [cid for cid in compose("ps", "-q", "api").splitlines() if cid.startswith(instance)]
check(len(matches) == 1, "uploading API replica was not identified")
replica = matches[0]
before = docker("inspect", "-f", "{{.State.StartedAt}}", replica)
docker("kill", "--signal", "KILL", replica)
docker("start", replica)
await_api(client)
check(before != docker("inspect", "-f", "{{.State.StartedAt}}", replica), "uploading API did not restart")
check(metadata_state(queued["id"]) == "queued|0|t", "pending outbox lost after API crash")
print(f"PASS: producer SIGKILL after PostgreSQL commit and before publish; outbox survived; id={queued['id']}", flush=True)
compose("start", "broker", "worker")
verify_result(client, reader, queued)
print(f"PASS: outbox recovered the same attachment with all three variants; id={queued['id']}", flush=True)
finally:
compose("start", "broker", "api", "worker")
def worker_crash(client, actor, reader, chat):
compose("stop", "worker")
temporary = "lab5-worker-fault-" + uuid.uuid4().hex[:12]
try:
queued = upload(client, actor, chat)["json"]
compose(
"run", "--rm", "--no-deps", "-T", "--name", temporary,
"-e", f"LAB5_TEST_CRASH_AFTER_OBJECTS={queued['id']}",
"-e", "JOB_TIMEOUT_SECONDS=15", "worker",
timeout=70, expected=86,
)
check(metadata_state(queued["id"]).startswith("processing|1|"), "worker did not stop before committing results")
client.call("GET", f"/api/v1/attachments/{queued['id']}/content", 409, actor=reader)
print(f"PASS: worker exited 86 after writing S3 objects, before result commit and ack; id={queued['id']}", flush=True)
compose("start", "worker")
verify_result(client, reader, queued)
check(metadata_state(queued["id"]).startswith("ready|2|"), "recovery did not use exactly one new attempt")
print(f"PASS: redelivery recovered the same id, original and three variants on attempt 2; id={queued['id']}", flush=True)
finally:
subprocess.run(["docker", "rm", "-f", temporary], capture_output=True, cwd=ROOT)
compose("start", "worker")
def operator_replay(client, actor, reader, chat):
compose("stop", "worker")
temporary = "lab5-worker-replay-" + uuid.uuid4().hex[:12]
try:
queued = upload(client, actor, chat)["json"]
job = sql(f"SELECT id FROM attachment_jobs WHERE attachment_id = '{uuid.UUID(queued['id'])}'")
job = str(uuid.UUID(job))
compose("stop", "s3")
compose(
"run", "-d", "--rm", "--no-deps", "--name", temporary,
"-e", "JOB_MAX_ATTEMPTS=1", "worker",
)
deadline = time.monotonic() + 40
while not metadata_state(queued["id"]).startswith("failed|1|"):
check(time.monotonic() < deadline, "one-attempt worker did not record the storage failure")
time.sleep(0.25)
docker("stop", "--time", "6", temporary)
compose("up", "-d", "--wait", "s3")
await_api(client)
command = ("run", "--rm", "--no-deps", "-T", "worker", "attachment-jobs")
listed = compose(*command, "failed")
check(any(line.startswith(job + "\t") for line in listed.splitlines()), "failed job was not listed")
check(f"QUEUED {job}" in compose(*command, "replay", job, "local-verification"), "replay was not queued")
check(metadata_state(queued["id"]) == "queued|1|t", "operator replay was not committed with outbox")
compose("start", "worker")
verify_result(client, reader, queued)
refused = compose(*command, "replay", job, "local-verification", expected=2)
check("REJECTED" in refused, "successful job accepted a duplicate replay")
check(metadata_state(queued["id"]).startswith("ready|2|"), "successful result changed on duplicate replay")
print(f"PASS: failed job listed, replayed with same id on attempt 2, and replay refused after ready; id={queued['id']}", flush=True)
finally:
subprocess.run(["docker", "rm", "-f", temporary], capture_output=True, cwd=ROOT)
compose("start", "s3", "worker")
def main():
client = Client(timeout=15)
check(client.base.startswith(("http://localhost:", "http://127.0.0.1:")), "use a local Compose stand")
actor, reader, _, chat = client.bootstrap()
producer_crash(client, actor, reader, chat)
worker_crash(client, actor, reader, chat)
operator_replay(client, actor, reader, chat)
if __name__ == "__main__":
main()
+1 -1
View File
@@ -54,7 +54,7 @@ def main():
thumbnail = client.call('GET', path + '/content?variant=thumbnail', actor=bob)['bytes']
check(downloaded == original, 'original changed')
check(jpeg_size(thumbnail) == (256, 160), 'incorrect thumbnail dimensions')
variant = attachment['variants'][0]
variant = next(item for item in attachment['variants'] if item['name'] == 'thumbnail')
check(variant['sha256'] == hashlib.sha256(thumbnail).hexdigest(), 'thumbnail checksum mismatch')
client.call('GET', path, 404, actor=eve)
print(f"PASS: same accepted job reached ready; original unchanged; thumbnail JPEG 256x160 with matching checksum; attachment_id={attachment['id']}", flush=True)
+1 -1
View File
@@ -12,7 +12,7 @@ if mc anonymous set download "app/$S3_BUCKET" >/dev/null 2>&1; then exit 1; fi
printf test > /tmp/test-object
if mc cp /tmp/test-object "app/$S3_BUCKET/forbidden-prefix/object" >/dev/null 2>&1; then exit 1; fi
mc stat "app/$S3_BUCKET" >/dev/null
printf "PASS: app cannot administer MinIO, create buckets, change policy or write outside originals/\n"
printf "PASS: app cannot administer MinIO, create buckets, change policy or write to an unrelated prefix\n"
'
if docker compose exec -T lb wget -q -T 2 -O /dev/null http://s3:9001; then
echo 'FAIL: console reachable from shared network'