forked from mtusi2026/mtusi2026lab5
feat(lab5): durable async attachment pipeline through grade 4
- RabbitMQ quorum queue, persistent messages and publisher confirms - S3 staging plus PostgreSQL attachments/jobs outbox transaction - worker variants (thumbnail, crop, optimized WebP), DB lease and lease sweeper - jittered retry, permanent failures, explicit replay and admission cap - deterministic crash hooks, resilience script and evidence for grade 4 - async-aware custom tests, oversized pixel case, docs and changelog
This commit is contained in:
@@ -0,0 +1,761 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Drive the lab5 grade-4 failure scenarios and write sanitized evidence.
|
||||
|
||||
This is an operator script for the local Compose stand, not part of the public
|
||||
black-box test suite. It deliberately restarts the worker, RabbitMQ and the API
|
||||
replicas, so run it only against the disposable lab environment.
|
||||
|
||||
Scenarios:
|
||||
1. worker stopped -> 202 queued -> message link -> worker start -> ready
|
||||
2. RabbitMQ down -> commit/outbox gap -> broker start -> ready
|
||||
3. crash after durable effect, before ack -> redelivery -> same result
|
||||
4. transient failure -> jittered retry -> ready on attempt 2
|
||||
5. permanent failure -> failed registry -> explicit replay -> ready
|
||||
6. crash after claim -> lease expiry -> sweeper requeue -> ready
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
WORKER = "mtusi2026lab-worker-1"
|
||||
|
||||
|
||||
def run(cmd, *, env=None, timeout=180, check_code=True):
|
||||
merged = os.environ.copy()
|
||||
if env:
|
||||
merged.update(env)
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=ROOT,
|
||||
env=merged,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=timeout,
|
||||
)
|
||||
if check_code and proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"command failed ({proc.returncode}): {' '.join(shlex.quote(x) for x in cmd)}\n"
|
||||
f"stdout:\n{proc.stdout[-4000:]}\nstderr:\n{proc.stderr[-4000:]}"
|
||||
)
|
||||
return proc
|
||||
|
||||
|
||||
def compose(*args, env=None, timeout=180, check_code=True):
|
||||
return run(COMPOSE + list(args), env=env, timeout=timeout, check_code=check_code)
|
||||
|
||||
|
||||
def compose_capture(*args, env=None, timeout=180):
|
||||
return compose(*args, env=env, timeout=timeout).stdout
|
||||
|
||||
|
||||
def psql(sql):
|
||||
command = (
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -F "|" -v ON_ERROR_STOP=1 -c '
|
||||
+ shlex.quote(sql)
|
||||
)
|
||||
out = compose_capture("exec", "-T", "postgres", "sh", "-c", command)
|
||||
return [line for line in out.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def parse_ts(value):
|
||||
value = value.strip()
|
||||
if value.endswith("+00"):
|
||||
value += ":00"
|
||||
return datetime.fromisoformat(value.replace(" ", "T"))
|
||||
|
||||
|
||||
def inspect(container_id):
|
||||
try:
|
||||
out = run(["docker", "inspect", container_id]).stdout
|
||||
return json.loads(out)[0]
|
||||
except (RuntimeError, json.JSONDecodeError):
|
||||
return {}
|
||||
|
||||
|
||||
def service_ids(service):
|
||||
ids = compose_capture("ps", "-q", service).split()
|
||||
return [item for item in ids if item]
|
||||
|
||||
|
||||
def wait_healthy(service, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
ids = service_ids(service)
|
||||
if ids:
|
||||
states = [inspect(item) for item in ids]
|
||||
if all(
|
||||
state.get("State", {}).get("Health", {}).get("Status") == "healthy"
|
||||
for state in states
|
||||
):
|
||||
return ids
|
||||
time.sleep(1)
|
||||
raise Failure(f"{service} did not become healthy within {timeout}s")
|
||||
|
||||
|
||||
def wait_stopped(service, timeout=30):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if not service_ids(service):
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise Failure(f"{service} did not stop within {timeout}s")
|
||||
|
||||
|
||||
def recreate_worker(extra_env=None):
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-build",
|
||||
"--force-recreate",
|
||||
"worker",
|
||||
env=extra_env or {},
|
||||
timeout=240,
|
||||
)
|
||||
wait_healthy("worker")
|
||||
|
||||
|
||||
def recreate_api(extra_env=None):
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-build",
|
||||
"--force-recreate",
|
||||
"api",
|
||||
env=extra_env or {},
|
||||
timeout=240,
|
||||
)
|
||||
wait_healthy("api")
|
||||
# HAProxy resolves api through Docker DNS; give its health checks a moment
|
||||
# to converge after both replica addresses change.
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
def restore_worker():
|
||||
recreate_worker()
|
||||
|
||||
|
||||
def restore_api():
|
||||
recreate_api()
|
||||
|
||||
|
||||
def job_row(attachment_id):
|
||||
rows = psql(
|
||||
"SELECT j.job_id, j.status, j.attempt, j.error_code, j.created_at, "
|
||||
"j.processing_started_at, j.finished_at, j.published_at, j.replay_count, "
|
||||
"a.staging_key, a.object_key "
|
||||
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id "
|
||||
f"WHERE j.attachment_id = '{attachment_id}'"
|
||||
)
|
||||
if len(rows) != 1:
|
||||
raise Failure(f"expected one job row for {attachment_id}, got {len(rows)}")
|
||||
parts = rows[0].split("|")
|
||||
return {
|
||||
"job_id": parts[0],
|
||||
"status": parts[1],
|
||||
"attempt": int(parts[2]),
|
||||
"error_code": None if parts[3] == "" else parts[3],
|
||||
"created_at": parse_ts(parts[4]),
|
||||
"processing_started_at": None if parts[5] == "" else parse_ts(parts[5]),
|
||||
"finished_at": None if parts[6] == "" else parse_ts(parts[6]),
|
||||
"published_at": None if parts[7] == "" else parse_ts(parts[7]),
|
||||
"replay_count": int(parts[8]),
|
||||
"staging_key": parts[9],
|
||||
"object_key": parts[10],
|
||||
}
|
||||
|
||||
|
||||
def wait_job_status(attachment_id, wanted, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
last = job_row(attachment_id)
|
||||
if last["status"] == wanted:
|
||||
return last
|
||||
if wanted != "failed" and last["status"] == "failed":
|
||||
raise Failure(f"job {attachment_id} failed unexpectedly: {last['error_code']}")
|
||||
time.sleep(0.4)
|
||||
raise Failure(f"job {attachment_id} did not reach {wanted}; last={last}")
|
||||
|
||||
|
||||
def s3_variant_keys(prefix):
|
||||
script = (
|
||||
'export HOME=/tmp; '
|
||||
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null; '
|
||||
'rc ls -r --json --no-progress "app/$S3_BUCKET/$PREFIX"'
|
||||
)
|
||||
proc = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-e",
|
||||
f"PREFIX={prefix}",
|
||||
"--entrypoint",
|
||||
"sh",
|
||||
"s3-init",
|
||||
"-c",
|
||||
script,
|
||||
check_code=False,
|
||||
timeout=120,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise Failure(f"rc ls failed: {proc.stderr[-1000:]}")
|
||||
payload = json.loads(proc.stdout)
|
||||
return sorted(item["key"] for item in payload.get("items", []))
|
||||
|
||||
|
||||
def s3_object_exists(key):
|
||||
script = (
|
||||
'export HOME=/tmp; '
|
||||
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null; '
|
||||
'rc head --quiet "app/$S3_BUCKET/$KEY" >/dev/null'
|
||||
)
|
||||
proc = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-e",
|
||||
f"KEY={key}",
|
||||
"--entrypoint",
|
||||
"sh",
|
||||
"s3-init",
|
||||
"-c",
|
||||
script,
|
||||
check_code=False,
|
||||
timeout=120,
|
||||
)
|
||||
return proc.returncode == 0
|
||||
|
||||
|
||||
def jpeg_size(data):
|
||||
if not data.startswith(b"\xff\xd8"):
|
||||
raise Failure("variant is not a JPEG")
|
||||
index = 2
|
||||
while index + 9 < len(data):
|
||||
if data[index] != 0xFF:
|
||||
index += 1
|
||||
continue
|
||||
marker = data[index + 1]
|
||||
index += 2
|
||||
if marker in (0xD8, 0xD9):
|
||||
continue
|
||||
length = int.from_bytes(data[index : index + 2], "big")
|
||||
if marker in (0xC0, 0xC1, 0xC2):
|
||||
height = int.from_bytes(data[index + 3 : index + 5], "big")
|
||||
width = int.from_bytes(data[index + 5 : index + 7], "big")
|
||||
return width, height
|
||||
index += length
|
||||
raise Failure("JPEG dimensions not found")
|
||||
|
||||
|
||||
def png_size(data):
|
||||
if not data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
raise Failure("variant is not a PNG")
|
||||
width = int.from_bytes(data[16:20], "big")
|
||||
height = int.from_bytes(data[20:24], "big")
|
||||
return width, height
|
||||
|
||||
|
||||
def verify_ready_attachment(client, actor, attachment, original):
|
||||
check(attachment["status"] == "ready", "attachment is not ready")
|
||||
check(attachment["error_code"] is None, "ready attachment has an error_code")
|
||||
variants = {item["name"]: item for item in attachment["variants"]}
|
||||
check(set(variants) == {"thumbnail", "crop", "optimized"}, "wrong variant set")
|
||||
check(
|
||||
(variants["thumbnail"]["width"], variants["thumbnail"]["height"]) == (256, 160),
|
||||
"thumbnail is not 256x160 for the 320x200 fixture",
|
||||
)
|
||||
check(
|
||||
(variants["crop"]["width"], variants["crop"]["height"]) == (128, 128),
|
||||
"crop is not 128x128",
|
||||
)
|
||||
check(
|
||||
(variants["optimized"]["width"], variants["optimized"]["height"]) == (320, 200),
|
||||
"optimized variant upscaled the small fixture",
|
||||
)
|
||||
|
||||
downloaded = client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}/content",
|
||||
actor=actor,
|
||||
)["bytes"]
|
||||
check(downloaded == original, "original download differs from the upload")
|
||||
check(
|
||||
hashlib.sha256(downloaded).hexdigest() == attachment["sha256"],
|
||||
"original download checksum differs",
|
||||
)
|
||||
|
||||
checks = {
|
||||
"original": (downloaded, "image/png", png_size),
|
||||
"thumbnail": (
|
||||
client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}/content?variant=thumbnail",
|
||||
actor=actor,
|
||||
)["bytes"],
|
||||
"image/jpeg",
|
||||
jpeg_size,
|
||||
),
|
||||
"crop": (
|
||||
client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}/content?variant=crop",
|
||||
actor=actor,
|
||||
)["bytes"],
|
||||
"image/jpeg",
|
||||
jpeg_size,
|
||||
),
|
||||
"optimized": (
|
||||
client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}/content?variant=optimized",
|
||||
actor=actor,
|
||||
)["bytes"],
|
||||
"image/webp",
|
||||
None,
|
||||
),
|
||||
}
|
||||
observed = {}
|
||||
for name, (data, media_type, size_parser) in checks.items():
|
||||
if media_type == "image/jpeg":
|
||||
check(size_parser(data) == (
|
||||
variants[name]["width"],
|
||||
variants[name]["height"],
|
||||
), f"{name} bytes do not match metadata dimensions")
|
||||
if media_type == "image/webp":
|
||||
check(data.startswith(b"RIFF") and data[8:12] == b"WEBP", "optimized is not WebP")
|
||||
if name != "original":
|
||||
check(
|
||||
hashlib.sha256(data).hexdigest() == variants[name]["sha256"],
|
||||
f"{name} checksum differs",
|
||||
)
|
||||
check(len(data) == variants[name]["size_bytes"], f"{name} size differs")
|
||||
observed[name] = {
|
||||
"content_type": media_type,
|
||||
"size_bytes": len(data),
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
"width": variants[name]["width"] if name != "original" else 320,
|
||||
"height": variants[name]["height"] if name != "original" else 200,
|
||||
}
|
||||
return observed
|
||||
|
||||
|
||||
def upload(client, actor, chat, content, name):
|
||||
return client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/attachments?filename={name}",
|
||||
202,
|
||||
actor=actor,
|
||||
raw=content,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)["json"]
|
||||
|
||||
|
||||
def attach_message(client, actor, chat, attachment_id, expected=201):
|
||||
return client.call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
expected,
|
||||
actor=actor,
|
||||
body={"text": "", "attachment_ids": [attachment_id]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
|
||||
def worker_logs():
|
||||
return compose_capture("logs", "--no-color", "--tail=160", "worker")
|
||||
|
||||
|
||||
def relevant_logs(text, tokens):
|
||||
lines = []
|
||||
for line in text.splitlines():
|
||||
if any(token and token in line for token in tokens):
|
||||
lines.append(line)
|
||||
return lines
|
||||
|
||||
|
||||
def scenario_worker_stopped(client, actor, chat, content, evidence):
|
||||
compose("stop", "worker")
|
||||
wait_stopped("worker")
|
||||
attachment = upload(client, actor, chat, content, "grade4-worker-stopped.png")
|
||||
check(attachment["status"] == "queued", "upload did not return queued")
|
||||
attach_message(client, actor, chat, attachment["id"])
|
||||
content_status = client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}/content",
|
||||
409,
|
||||
actor=actor,
|
||||
)["json"]
|
||||
check(content_status["error"]["code"] == "conflict", "queued content is not 409")
|
||||
compose("start", "worker")
|
||||
wait_healthy("worker")
|
||||
ready = client.wait_ready(actor, attachment, wait_seconds=60)
|
||||
observed = verify_ready_attachment(client, actor, ready, content)
|
||||
row = job_row(attachment["id"])
|
||||
check(row["status"] == "ready" and row["staging_key"] == "", "ready row/staging cleanup mismatch")
|
||||
evidence["worker_stopped"] = {
|
||||
"attachment_id": attachment["id"],
|
||||
"queued_status": attachment["status"],
|
||||
"content_while_queued": 409,
|
||||
"ready_after_worker_start": True,
|
||||
"queue_seconds": round(
|
||||
(row["processing_started_at"] - row["created_at"]).total_seconds(), 3
|
||||
),
|
||||
"evidence": observed,
|
||||
"row": {
|
||||
"attempt": row["attempt"],
|
||||
"published": row["published_at"] is not None,
|
||||
"staging_cleared": row["staging_key"] == "",
|
||||
},
|
||||
}
|
||||
return attachment["id"]
|
||||
|
||||
|
||||
def scenario_broker_outage(client, actor, chat, content, evidence):
|
||||
compose("stop", "rabbitmq")
|
||||
wait_stopped("rabbitmq")
|
||||
attachment = upload(client, actor, chat, content, "grade4-broker-down.png")
|
||||
row = job_row(attachment["id"])
|
||||
check(row["status"] == "queued", "job is not queued during broker outage")
|
||||
check(row["published_at"] is None, "job unexpectedly published without broker")
|
||||
time.sleep(3)
|
||||
row_again = job_row(attachment["id"])
|
||||
check(row_again["published_at"] is None, "outbox published while broker was down")
|
||||
healthy = client.call("GET", "/health/ready", schema="Health")
|
||||
check(healthy["json"]["status"] == "ok", "API readiness depends on RabbitMQ")
|
||||
compose("start", "rabbitmq")
|
||||
wait_healthy("rabbitmq")
|
||||
ready = client.wait_ready(actor, attachment, wait_seconds=90)
|
||||
verify_ready_attachment(client, actor, ready, content)
|
||||
row_ready = job_row(attachment["id"])
|
||||
check(row_ready["published_at"] is not None, "job was not published after broker recovery")
|
||||
evidence["broker_outage"] = {
|
||||
"attachment_id": attachment["id"],
|
||||
"202_accepted_without_broker": attachment["status"] == "queued",
|
||||
"unpublished_during_outage": True,
|
||||
"api_ready_during_broker_outage": healthy["json"]["status"] == "ok",
|
||||
"published_after_recovery": True,
|
||||
"attempt": row_ready["attempt"],
|
||||
"queue_seconds": round(
|
||||
(row_ready["processing_started_at"] - row_ready["created_at"]).total_seconds(), 3
|
||||
),
|
||||
}
|
||||
return attachment["id"]
|
||||
|
||||
|
||||
def scenario_crash_after_effect(client, actor, chat, content, evidence):
|
||||
recreate_worker({"WORKER_CRASH_AFTER_EFFECT": "true"})
|
||||
worker_id = service_ids("worker")[0]
|
||||
restarts_before = inspect(worker_id)["RestartCount"]
|
||||
attachment = upload(client, actor, chat, content, "grade4-crash-after-effect.png")
|
||||
ready = client.wait_ready(actor, attachment, wait_seconds=90)
|
||||
observed = verify_ready_attachment(client, actor, ready, content)
|
||||
deadline = time.monotonic() + 30
|
||||
restarts_after = restarts_before
|
||||
while time.monotonic() < deadline and restarts_after <= restarts_before:
|
||||
time.sleep(0.5)
|
||||
ids = service_ids("worker")
|
||||
if not ids:
|
||||
continue
|
||||
restarts_after = inspect(ids[0]).get("RestartCount", restarts_after)
|
||||
row = job_row(attachment["id"])
|
||||
keys = s3_variant_keys(row["object_key"])
|
||||
check(len(keys) == 3, f"expected exactly three variant objects, got {len(keys)}")
|
||||
logs = relevant_logs(
|
||||
worker_logs(),
|
||||
[attachment["id"], row["job_id"], "fault injection", "attachment ready"],
|
||||
)
|
||||
check(any("after durable effect before ack" in line for line in logs), "crash hook did not log")
|
||||
evidence["crash_after_effect"] = {
|
||||
"attachment_id": attachment["id"],
|
||||
"worker_restart_count_before": restarts_before,
|
||||
"worker_restart_count_after": restarts_after,
|
||||
"ready_after_redelivery": True,
|
||||
"variant_objects": len(keys),
|
||||
"evidence": observed,
|
||||
"log_excerpt": logs,
|
||||
}
|
||||
restore_worker()
|
||||
return attachment["id"]
|
||||
|
||||
|
||||
def scenario_transient_retry(client, actor, chat, content, evidence):
|
||||
recreate_worker(
|
||||
{
|
||||
"WORKER_FAULT_MODE": "transient-once",
|
||||
"JOB_RETRY_BASE_SECONDS": "2",
|
||||
"JOB_RETRY_MAX_SECONDS": "8",
|
||||
}
|
||||
)
|
||||
attachment = upload(client, actor, chat, content, "grade4-transient-retry.png")
|
||||
ready = client.wait_ready(actor, attachment, wait_seconds=90)
|
||||
verify_ready_attachment(client, actor, ready, content)
|
||||
row = job_row(attachment["id"])
|
||||
check(row["status"] == "ready", "transient retry did not finish ready")
|
||||
check(row["attempt"] >= 2, "transient retry did not use a second attempt")
|
||||
logs = relevant_logs(
|
||||
worker_logs(),
|
||||
[attachment["id"], row["job_id"], "transient job failure", "processing attachment"],
|
||||
)
|
||||
check(any("transient job failure" in line for line in logs), "retry log missing")
|
||||
evidence["transient_retry"] = {
|
||||
"attachment_id": attachment["id"],
|
||||
"attempts": row["attempt"],
|
||||
"status": row["status"],
|
||||
"log_excerpt": logs,
|
||||
}
|
||||
restore_worker()
|
||||
return attachment["id"]
|
||||
|
||||
|
||||
def scenario_permanent_failure_and_replay(client, actor, chat, content, evidence):
|
||||
recreate_worker({"WORKER_FAULT_MODE": "permanent-once"})
|
||||
attachment = upload(client, actor, chat, content, "grade4-permanent-failure.png")
|
||||
failed = wait_job_status(attachment["id"], "failed", timeout=45)
|
||||
check(failed["error_code"] == "invalid_image", "unexpected permanent error code")
|
||||
snapshot = client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}",
|
||||
actor=actor,
|
||||
schema="Attachment",
|
||||
)["json"]
|
||||
check(snapshot["status"] == "failed", "attachment did not expose failed")
|
||||
client.call(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment['id']}/content",
|
||||
409,
|
||||
actor=actor,
|
||||
)
|
||||
conflict = attach_message(client, actor, chat, attachment["id"], expected=409)["json"]
|
||||
check(conflict["error"]["code"] == "conflict", "failed attachment was attachable")
|
||||
failure_log = relevant_logs(
|
||||
worker_logs(),
|
||||
[attachment["id"], failed["job_id"], "permanent job failure", "invalid_image"],
|
||||
)
|
||||
|
||||
restore_worker()
|
||||
replay = compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"worker",
|
||||
"/usr/local/bin/worker",
|
||||
"-replay",
|
||||
attachment["id"],
|
||||
check_code=False,
|
||||
)
|
||||
check(replay.returncode == 0, f"replay command failed: {replay.stderr[-1000:]}")
|
||||
ready = client.wait_ready(actor, attachment, wait_seconds=90)
|
||||
observed = verify_ready_attachment(client, actor, ready, content)
|
||||
row = job_row(attachment["id"])
|
||||
check(row["status"] == "ready" and row["replay_count"] >= 1, "replay did not finish ready")
|
||||
keys = s3_variant_keys(row["object_key"])
|
||||
check(len(keys) == 3, f"replay created a second variant set: {keys}")
|
||||
check(s3_object_exists(row["object_key"]), "original object is missing after replay")
|
||||
after_replay_log = relevant_logs(
|
||||
worker_logs(),
|
||||
[attachment["id"], row["job_id"], "replay", "attachment ready"],
|
||||
)
|
||||
logs = failure_log + after_replay_log
|
||||
evidence["permanent_failure_replay"] = {
|
||||
"attachment_id": attachment["id"],
|
||||
"failed_code": failed["error_code"],
|
||||
"failed_content_http": 409,
|
||||
"failed_message_link_http": 409,
|
||||
"replay_exit": replay.returncode,
|
||||
"replay_output": (replay.stdout + replay.stderr).strip().splitlines(),
|
||||
"replay_count": row["replay_count"],
|
||||
"variant_objects_after_replay": len(keys),
|
||||
"original_object_present": True,
|
||||
"evidence": observed,
|
||||
"log_excerpt": logs,
|
||||
}
|
||||
return attachment["id"]
|
||||
|
||||
|
||||
def scenario_lease_recovery(client, actor, chat, content, evidence):
|
||||
compose("stop", "worker")
|
||||
wait_stopped("worker")
|
||||
attachment = upload(client, actor, chat, content, "grade4-lease-recovery.png")
|
||||
# The one-shot worker claims the job, then exits before any side effect.
|
||||
# A short lease keeps the demonstration quick without changing the default.
|
||||
one_shot = compose(
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-e",
|
||||
"WORKER_CRASH_AFTER_CLAIM=true",
|
||||
"-e",
|
||||
"JOB_LEASE_SECONDS=10",
|
||||
"worker",
|
||||
check_code=False,
|
||||
timeout=120,
|
||||
)
|
||||
check(one_shot.returncode == 10, f"crash-after-claim exit={one_shot.returncode}")
|
||||
claimed = job_row(attachment["id"])
|
||||
check(
|
||||
claimed["status"] == "processing" and claimed["processing_started_at"] is not None,
|
||||
"job was not left in processing",
|
||||
)
|
||||
recreate_worker({"JOB_LEASE_SECONDS": "10", "LEASE_SWEEP_SECONDS": "2"})
|
||||
ready = client.wait_ready(actor, attachment, wait_seconds=90)
|
||||
observed = verify_ready_attachment(client, actor, ready, content)
|
||||
row = job_row(attachment["id"])
|
||||
check(row["status"] == "ready", "lease recovery did not finish ready")
|
||||
logs = relevant_logs(
|
||||
worker_logs(),
|
||||
[attachment["id"], row["job_id"], "recovered expired", "processing attachment"],
|
||||
)
|
||||
check(any("recovered expired job leases" in line for line in logs), "sweeper log missing")
|
||||
evidence["lease_recovery"] = {
|
||||
"attachment_id": attachment["id"],
|
||||
"one_shot_exit": one_shot.returncode,
|
||||
"processing_before_expiry": True,
|
||||
"attempts": row["attempt"],
|
||||
"ready_after_sweep": True,
|
||||
"evidence": observed,
|
||||
"log_excerpt": logs,
|
||||
}
|
||||
restore_worker()
|
||||
return attachment["id"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||
parser.add_argument("--out", type=Path, default=ROOT / "evidence/lab5-grade4.json")
|
||||
parser.add_argument("--markdown", type=Path, default=ROOT / "evidence/lab5-grade4.md")
|
||||
parser.add_argument(
|
||||
"--scenario",
|
||||
choices=(
|
||||
"all",
|
||||
"worker-stopped",
|
||||
"broker-outage",
|
||||
"crash-after-effect",
|
||||
"transient-retry",
|
||||
"permanent-failure-replay",
|
||||
"lease-recovery",
|
||||
),
|
||||
default="all",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
client = Client(args.base_url)
|
||||
actor, _, _, chat = client.bootstrap()
|
||||
evidence = {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"fixture": {
|
||||
"bytes": len(content),
|
||||
"sha256": hashlib.sha256(content).hexdigest(),
|
||||
"width": 320,
|
||||
"height": 200,
|
||||
},
|
||||
"scenarios": {},
|
||||
}
|
||||
|
||||
selected = [
|
||||
("worker-stopped", scenario_worker_stopped),
|
||||
("broker-outage", scenario_broker_outage),
|
||||
("crash-after-effect", scenario_crash_after_effect),
|
||||
("transient-retry", scenario_transient_retry),
|
||||
("permanent-failure-replay", scenario_permanent_failure_and_replay),
|
||||
("lease-recovery", scenario_lease_recovery),
|
||||
]
|
||||
try:
|
||||
for name, function in selected:
|
||||
if args.scenario not in ("all", name):
|
||||
continue
|
||||
print(f"== {name}", flush=True)
|
||||
function(client, actor, chat, content, evidence["scenarios"])
|
||||
print(f"== {name}: OK", flush=True)
|
||||
finally:
|
||||
restore_worker()
|
||||
restore_api()
|
||||
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(json.dumps(evidence, indent=2) + "\n")
|
||||
|
||||
lines = [
|
||||
"# Lab5 grade-4 runtime evidence",
|
||||
"",
|
||||
f"Generated: `{evidence['generated_at']}`",
|
||||
"",
|
||||
"Synthetic fixture: 320x200 PNG, public contract fixture; no tokens or credentials are stored.",
|
||||
"",
|
||||
"| Scenario | Observation |",
|
||||
"| --- | --- |",
|
||||
]
|
||||
s = evidence["scenarios"]
|
||||
if "worker_stopped" in s:
|
||||
lines.append(
|
||||
"| Worker stopped during upload | 202 queued, content 409, message linked while queued, "
|
||||
f"ready after worker start; queue {s['worker_stopped']['queue_seconds']} s, "
|
||||
f"attempt {s['worker_stopped']['row']['attempt']} |"
|
||||
)
|
||||
if "broker_outage" in s:
|
||||
lines.append(
|
||||
"| Broker outage (COMMIT/publish gap) | 202 accepted with `published_at IS NULL`, "
|
||||
"API readiness stayed 200, job published and ready after RabbitMQ restart |"
|
||||
)
|
||||
if "crash_after_effect" in s:
|
||||
lines.append(
|
||||
"| Crash after effect, before ack | worker restart count "
|
||||
f"{s['crash_after_effect']['worker_restart_count_before']} -> "
|
||||
f"{s['crash_after_effect']['worker_restart_count_after']}, "
|
||||
f"{s['crash_after_effect']['variant_objects']} deterministic variant objects, ready |"
|
||||
)
|
||||
if "transient_retry" in s:
|
||||
lines.append(
|
||||
"| Transient error + jittered retry | failed once, attempt "
|
||||
f"{s['transient_retry']['attempts']}, ready |"
|
||||
)
|
||||
if "permanent_failure_replay" in s:
|
||||
lines.append(
|
||||
"| Permanent error + explicit replay | failed `invalid_image`, content/link 409, "
|
||||
f"replay_count {s['permanent_failure_replay']['replay_count']}, ready with "
|
||||
f"{s['permanent_failure_replay']['variant_objects_after_replay']} variant objects |"
|
||||
)
|
||||
if "lease_recovery" in s:
|
||||
lines.append(
|
||||
"| Crash after claim / expired lease | one-shot exit 10, processing, sweeper recovered, "
|
||||
f"attempt {s['lease_recovery']['attempts']}, ready |"
|
||||
)
|
||||
lines.extend(["", "## Sanitized log excerpts", ""])
|
||||
for name, payload in s.items():
|
||||
excerpt = payload.get("log_excerpt")
|
||||
if not excerpt:
|
||||
continue
|
||||
lines.append(f"### {name}")
|
||||
lines.append("")
|
||||
lines.append("```text")
|
||||
lines.extend(line for line in excerpt[:12])
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
args.markdown.write_text("\n".join(lines) + "\n")
|
||||
print(json.dumps({"ok": True, "out": str(args.out), "markdown": str(args.markdown)}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (Failure, RuntimeError, subprocess.TimeoutExpired) as exc:
|
||||
print("FAIL:", exc)
|
||||
raise SystemExit(1)
|
||||
+3
-1
@@ -1,4 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env restart postgres redis rustfs haproxy api
|
||||
# Lab5: restart the broker and worker as well, with all volumes preserved. The
|
||||
# persistence test then proves an already accepted job still completes.
|
||||
docker compose --env-file .env restart postgres redis rustfs rabbitmq haproxy api worker
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Verify the lab4 durability claim: upload through the load balancer, stop the
|
||||
# API replica that served the upload, then download the same attachment through
|
||||
# Verify the lab5 durability claim: accept an upload through the load balancer
|
||||
# (202 after durable staging plus outbox), let the worker make it ready, stop
|
||||
# the API replica that accepted it, then download the same attachment through
|
||||
# another replica and compare the SHA-256. The stopped replica is restored on
|
||||
# exit so the stand is left as it was found.
|
||||
set -eu
|
||||
|
||||
Reference in New Issue
Block a user