Files

762 lines
26 KiB
Python

#!/usr/bin/env python3
"""Drive the durable-pipeline failure scenarios and write sanitized evidence.
This is an operator script for the local Compose stand, not part of the public
black-box test suite. It deliberately restarts the worker, RabbitMQ and the API
replicas, so run it only against the disposable stand.
Scenarios:
1. worker stopped -> 202 queued -> message link -> worker start -> ready
2. RabbitMQ down -> commit/outbox gap -> broker start -> ready
3. crash after durable effect, before ack -> redelivery -> same result
4. transient failure -> jittered retry -> ready on attempt 2
5. permanent failure -> failed registry -> explicit replay -> ready
6. crash after claim -> lease expiry -> sweeper requeue -> ready
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import shlex
import subprocess
import sys
import time
import uuid
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
COMPOSE = ["docker", "compose", "--env-file", ".env"]
WORKER = "mtusi2026lab-worker-1"
def run(cmd, *, env=None, timeout=180, check_code=True):
merged = os.environ.copy()
if env:
merged.update(env)
proc = subprocess.run(
cmd,
cwd=ROOT,
env=merged,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=timeout,
)
if check_code and proc.returncode != 0:
raise RuntimeError(
f"command failed ({proc.returncode}): {' '.join(shlex.quote(x) for x in cmd)}\n"
f"stdout:\n{proc.stdout[-4000:]}\nstderr:\n{proc.stderr[-4000:]}"
)
return proc
def compose(*args, env=None, timeout=180, check_code=True):
return run(COMPOSE + list(args), env=env, timeout=timeout, check_code=check_code)
def compose_capture(*args, env=None, timeout=180):
return compose(*args, env=env, timeout=timeout).stdout
def psql(sql):
command = (
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -F "|" -v ON_ERROR_STOP=1 -c '
+ shlex.quote(sql)
)
out = compose_capture("exec", "-T", "postgres", "sh", "-c", command)
return [line for line in out.splitlines() if line.strip()]
def parse_ts(value):
value = value.strip()
if value.endswith("+00"):
value += ":00"
return datetime.fromisoformat(value.replace(" ", "T"))
def inspect(container_id):
try:
out = run(["docker", "inspect", container_id]).stdout
return json.loads(out)[0]
except (RuntimeError, json.JSONDecodeError):
return {}
def service_ids(service):
ids = compose_capture("ps", "-q", service).split()
return [item for item in ids if item]
def wait_healthy(service, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
ids = service_ids(service)
if ids:
states = [inspect(item) for item in ids]
if all(
state.get("State", {}).get("Health", {}).get("Status") == "healthy"
for state in states
):
return ids
time.sleep(1)
raise Failure(f"{service} did not become healthy within {timeout}s")
def wait_stopped(service, timeout=30):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if not service_ids(service):
return
time.sleep(0.5)
raise Failure(f"{service} did not stop within {timeout}s")
def recreate_worker(extra_env=None):
compose(
"up",
"-d",
"--no-build",
"--force-recreate",
"worker",
env=extra_env or {},
timeout=240,
)
wait_healthy("worker")
def recreate_api(extra_env=None):
compose(
"up",
"-d",
"--no-build",
"--force-recreate",
"api",
env=extra_env or {},
timeout=240,
)
wait_healthy("api")
# HAProxy resolves api through Docker DNS; give its health checks a moment
# to converge after both replica addresses change.
time.sleep(3)
def restore_worker():
recreate_worker()
def restore_api():
recreate_api()
def job_row(attachment_id):
rows = psql(
"SELECT j.job_id, j.status, j.attempt, j.error_code, j.created_at, "
"j.processing_started_at, j.finished_at, j.published_at, j.replay_count, "
"a.staging_key, a.object_key "
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id "
f"WHERE j.attachment_id = '{attachment_id}'"
)
if len(rows) != 1:
raise Failure(f"expected one job row for {attachment_id}, got {len(rows)}")
parts = rows[0].split("|")
return {
"job_id": parts[0],
"status": parts[1],
"attempt": int(parts[2]),
"error_code": None if parts[3] == "" else parts[3],
"created_at": parse_ts(parts[4]),
"processing_started_at": None if parts[5] == "" else parse_ts(parts[5]),
"finished_at": None if parts[6] == "" else parse_ts(parts[6]),
"published_at": None if parts[7] == "" else parse_ts(parts[7]),
"replay_count": int(parts[8]),
"staging_key": parts[9],
"object_key": parts[10],
}
def wait_job_status(attachment_id, wanted, timeout=90):
deadline = time.monotonic() + timeout
last = None
while time.monotonic() < deadline:
last = job_row(attachment_id)
if last["status"] == wanted:
return last
if wanted != "failed" and last["status"] == "failed":
raise Failure(f"job {attachment_id} failed unexpectedly: {last['error_code']}")
time.sleep(0.4)
raise Failure(f"job {attachment_id} did not reach {wanted}; last={last}")
def s3_variant_keys(prefix):
script = (
'export HOME=/tmp; '
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null; '
'rc ls -r --json --no-progress "app/$S3_BUCKET/$PREFIX"'
)
proc = compose(
"run",
"--rm",
"--no-deps",
"-e",
f"PREFIX={prefix}",
"--entrypoint",
"sh",
"s3-init",
"-c",
script,
check_code=False,
timeout=120,
)
if proc.returncode != 0:
raise Failure(f"rc ls failed: {proc.stderr[-1000:]}")
payload = json.loads(proc.stdout)
return sorted(item["key"] for item in payload.get("items", []))
def s3_object_exists(key):
script = (
'export HOME=/tmp; '
'rc alias set app "$S3_ENDPOINT" "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null; '
'rc head --quiet "app/$S3_BUCKET/$KEY" >/dev/null'
)
proc = compose(
"run",
"--rm",
"--no-deps",
"-e",
f"KEY={key}",
"--entrypoint",
"sh",
"s3-init",
"-c",
script,
check_code=False,
timeout=120,
)
return proc.returncode == 0
def jpeg_size(data):
if not data.startswith(b"\xff\xd8"):
raise Failure("variant is not a JPEG")
index = 2
while index + 9 < len(data):
if data[index] != 0xFF:
index += 1
continue
marker = data[index + 1]
index += 2
if marker in (0xD8, 0xD9):
continue
length = int.from_bytes(data[index : index + 2], "big")
if marker in (0xC0, 0xC1, 0xC2):
height = int.from_bytes(data[index + 3 : index + 5], "big")
width = int.from_bytes(data[index + 5 : index + 7], "big")
return width, height
index += length
raise Failure("JPEG dimensions not found")
def png_size(data):
if not data.startswith(b"\x89PNG\r\n\x1a\n"):
raise Failure("variant is not a PNG")
width = int.from_bytes(data[16:20], "big")
height = int.from_bytes(data[20:24], "big")
return width, height
def verify_ready_attachment(client, actor, attachment, original):
check(attachment["status"] == "ready", "attachment is not ready")
check(attachment["error_code"] is None, "ready attachment has an error_code")
variants = {item["name"]: item for item in attachment["variants"]}
check(set(variants) == {"thumbnail", "crop", "optimized"}, "wrong variant set")
check(
(variants["thumbnail"]["width"], variants["thumbnail"]["height"]) == (256, 160),
"thumbnail is not 256x160 for the 320x200 fixture",
)
check(
(variants["crop"]["width"], variants["crop"]["height"]) == (128, 128),
"crop is not 128x128",
)
check(
(variants["optimized"]["width"], variants["optimized"]["height"]) == (320, 200),
"optimized variant upscaled the small fixture",
)
downloaded = client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}/content",
actor=actor,
)["bytes"]
check(downloaded == original, "original download differs from the upload")
check(
hashlib.sha256(downloaded).hexdigest() == attachment["sha256"],
"original download checksum differs",
)
checks = {
"original": (downloaded, "image/png", png_size),
"thumbnail": (
client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}/content?variant=thumbnail",
actor=actor,
)["bytes"],
"image/jpeg",
jpeg_size,
),
"crop": (
client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}/content?variant=crop",
actor=actor,
)["bytes"],
"image/jpeg",
jpeg_size,
),
"optimized": (
client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}/content?variant=optimized",
actor=actor,
)["bytes"],
"image/webp",
None,
),
}
observed = {}
for name, (data, media_type, size_parser) in checks.items():
if media_type == "image/jpeg":
check(size_parser(data) == (
variants[name]["width"],
variants[name]["height"],
), f"{name} bytes do not match metadata dimensions")
if media_type == "image/webp":
check(data.startswith(b"RIFF") and data[8:12] == b"WEBP", "optimized is not WebP")
if name != "original":
check(
hashlib.sha256(data).hexdigest() == variants[name]["sha256"],
f"{name} checksum differs",
)
check(len(data) == variants[name]["size_bytes"], f"{name} size differs")
observed[name] = {
"content_type": media_type,
"size_bytes": len(data),
"sha256": hashlib.sha256(data).hexdigest(),
"width": variants[name]["width"] if name != "original" else 320,
"height": variants[name]["height"] if name != "original" else 200,
}
return observed
def upload(client, actor, chat, content, name):
return client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename={name}",
202,
actor=actor,
raw=content,
headers={"Content-Type": "image/png"},
schema="Attachment",
)["json"]
def attach_message(client, actor, chat, attachment_id, expected=201):
return client.call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
expected,
actor=actor,
body={"text": "", "attachment_ids": [attachment_id]},
headers={"Idempotency-Key": uuid.uuid4().hex},
)
def worker_logs():
return compose_capture("logs", "--no-color", "--tail=160", "worker")
def relevant_logs(text, tokens):
lines = []
for line in text.splitlines():
if any(token and token in line for token in tokens):
lines.append(line)
return lines
def scenario_worker_stopped(client, actor, chat, content, evidence):
compose("stop", "worker")
wait_stopped("worker")
attachment = upload(client, actor, chat, content, "grade4-worker-stopped.png")
check(attachment["status"] == "queued", "upload did not return queued")
attach_message(client, actor, chat, attachment["id"])
content_status = client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}/content",
409,
actor=actor,
)["json"]
check(content_status["error"]["code"] == "conflict", "queued content is not 409")
compose("start", "worker")
wait_healthy("worker")
ready = client.wait_ready(actor, attachment, wait_seconds=60)
observed = verify_ready_attachment(client, actor, ready, content)
row = job_row(attachment["id"])
check(row["status"] == "ready" and row["staging_key"] == "", "ready row/staging cleanup mismatch")
evidence["worker_stopped"] = {
"attachment_id": attachment["id"],
"queued_status": attachment["status"],
"content_while_queued": 409,
"ready_after_worker_start": True,
"queue_seconds": round(
(row["processing_started_at"] - row["created_at"]).total_seconds(), 3
),
"evidence": observed,
"row": {
"attempt": row["attempt"],
"published": row["published_at"] is not None,
"staging_cleared": row["staging_key"] == "",
},
}
return attachment["id"]
def scenario_broker_outage(client, actor, chat, content, evidence):
compose("stop", "rabbitmq")
wait_stopped("rabbitmq")
attachment = upload(client, actor, chat, content, "grade4-broker-down.png")
row = job_row(attachment["id"])
check(row["status"] == "queued", "job is not queued during broker outage")
check(row["published_at"] is None, "job unexpectedly published without broker")
time.sleep(3)
row_again = job_row(attachment["id"])
check(row_again["published_at"] is None, "outbox published while broker was down")
healthy = client.call("GET", "/health/ready", schema="Health")
check(healthy["json"]["status"] == "ok", "API readiness depends on RabbitMQ")
compose("start", "rabbitmq")
wait_healthy("rabbitmq")
ready = client.wait_ready(actor, attachment, wait_seconds=90)
verify_ready_attachment(client, actor, ready, content)
row_ready = job_row(attachment["id"])
check(row_ready["published_at"] is not None, "job was not published after broker recovery")
evidence["broker_outage"] = {
"attachment_id": attachment["id"],
"202_accepted_without_broker": attachment["status"] == "queued",
"unpublished_during_outage": True,
"api_ready_during_broker_outage": healthy["json"]["status"] == "ok",
"published_after_recovery": True,
"attempt": row_ready["attempt"],
"queue_seconds": round(
(row_ready["processing_started_at"] - row_ready["created_at"]).total_seconds(), 3
),
}
return attachment["id"]
def scenario_crash_after_effect(client, actor, chat, content, evidence):
recreate_worker({"WORKER_CRASH_AFTER_EFFECT": "true"})
worker_id = service_ids("worker")[0]
restarts_before = inspect(worker_id)["RestartCount"]
attachment = upload(client, actor, chat, content, "grade4-crash-after-effect.png")
ready = client.wait_ready(actor, attachment, wait_seconds=90)
observed = verify_ready_attachment(client, actor, ready, content)
deadline = time.monotonic() + 30
restarts_after = restarts_before
while time.monotonic() < deadline and restarts_after <= restarts_before:
time.sleep(0.5)
ids = service_ids("worker")
if not ids:
continue
restarts_after = inspect(ids[0]).get("RestartCount", restarts_after)
row = job_row(attachment["id"])
keys = s3_variant_keys(row["object_key"])
check(len(keys) == 3, f"expected exactly three variant objects, got {len(keys)}")
logs = relevant_logs(
worker_logs(),
[attachment["id"], row["job_id"], "fault injection", "attachment ready"],
)
check(any("after durable effect before ack" in line for line in logs), "crash hook did not log")
evidence["crash_after_effect"] = {
"attachment_id": attachment["id"],
"worker_restart_count_before": restarts_before,
"worker_restart_count_after": restarts_after,
"ready_after_redelivery": True,
"variant_objects": len(keys),
"evidence": observed,
"log_excerpt": logs,
}
restore_worker()
return attachment["id"]
def scenario_transient_retry(client, actor, chat, content, evidence):
recreate_worker(
{
"WORKER_FAULT_MODE": "transient-once",
"JOB_RETRY_BASE_SECONDS": "2",
"JOB_RETRY_MAX_SECONDS": "8",
}
)
attachment = upload(client, actor, chat, content, "grade4-transient-retry.png")
ready = client.wait_ready(actor, attachment, wait_seconds=90)
verify_ready_attachment(client, actor, ready, content)
row = job_row(attachment["id"])
check(row["status"] == "ready", "transient retry did not finish ready")
check(row["attempt"] >= 2, "transient retry did not use a second attempt")
logs = relevant_logs(
worker_logs(),
[attachment["id"], row["job_id"], "transient job failure", "processing attachment"],
)
check(any("transient job failure" in line for line in logs), "retry log missing")
evidence["transient_retry"] = {
"attachment_id": attachment["id"],
"attempts": row["attempt"],
"status": row["status"],
"log_excerpt": logs,
}
restore_worker()
return attachment["id"]
def scenario_permanent_failure_and_replay(client, actor, chat, content, evidence):
recreate_worker({"WORKER_FAULT_MODE": "permanent-once"})
attachment = upload(client, actor, chat, content, "grade4-permanent-failure.png")
failed = wait_job_status(attachment["id"], "failed", timeout=45)
check(failed["error_code"] == "invalid_image", "unexpected permanent error code")
snapshot = client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}",
actor=actor,
schema="Attachment",
)["json"]
check(snapshot["status"] == "failed", "attachment did not expose failed")
client.call(
"GET",
f"/api/v1/attachments/{attachment['id']}/content",
409,
actor=actor,
)
conflict = attach_message(client, actor, chat, attachment["id"], expected=409)["json"]
check(conflict["error"]["code"] == "conflict", "failed attachment was attachable")
failure_log = relevant_logs(
worker_logs(),
[attachment["id"], failed["job_id"], "permanent job failure", "invalid_image"],
)
restore_worker()
replay = compose(
"exec",
"-T",
"worker",
"/usr/local/bin/worker",
"-replay",
attachment["id"],
check_code=False,
)
check(replay.returncode == 0, f"replay command failed: {replay.stderr[-1000:]}")
ready = client.wait_ready(actor, attachment, wait_seconds=90)
observed = verify_ready_attachment(client, actor, ready, content)
row = job_row(attachment["id"])
check(row["status"] == "ready" and row["replay_count"] >= 1, "replay did not finish ready")
keys = s3_variant_keys(row["object_key"])
check(len(keys) == 3, f"replay created a second variant set: {keys}")
check(s3_object_exists(row["object_key"]), "original object is missing after replay")
after_replay_log = relevant_logs(
worker_logs(),
[attachment["id"], row["job_id"], "replay", "attachment ready"],
)
logs = failure_log + after_replay_log
evidence["permanent_failure_replay"] = {
"attachment_id": attachment["id"],
"failed_code": failed["error_code"],
"failed_content_http": 409,
"failed_message_link_http": 409,
"replay_exit": replay.returncode,
"replay_output": (replay.stdout + replay.stderr).strip().splitlines(),
"replay_count": row["replay_count"],
"variant_objects_after_replay": len(keys),
"original_object_present": True,
"evidence": observed,
"log_excerpt": logs,
}
return attachment["id"]
def scenario_lease_recovery(client, actor, chat, content, evidence):
compose("stop", "worker")
wait_stopped("worker")
attachment = upload(client, actor, chat, content, "grade4-lease-recovery.png")
# The one-shot worker claims the job, then exits before any side effect.
# A short lease keeps the demonstration quick without changing the default.
one_shot = compose(
"run",
"--rm",
"--no-deps",
"-e",
"WORKER_CRASH_AFTER_CLAIM=true",
"-e",
"JOB_LEASE_SECONDS=10",
"worker",
check_code=False,
timeout=120,
)
check(one_shot.returncode == 10, f"crash-after-claim exit={one_shot.returncode}")
claimed = job_row(attachment["id"])
check(
claimed["status"] == "processing" and claimed["processing_started_at"] is not None,
"job was not left in processing",
)
recreate_worker({"JOB_LEASE_SECONDS": "10", "LEASE_SWEEP_SECONDS": "2"})
ready = client.wait_ready(actor, attachment, wait_seconds=90)
observed = verify_ready_attachment(client, actor, ready, content)
row = job_row(attachment["id"])
check(row["status"] == "ready", "lease recovery did not finish ready")
logs = relevant_logs(
worker_logs(),
[attachment["id"], row["job_id"], "recovered expired", "processing attachment"],
)
check(any("recovered expired job leases" in line for line in logs), "sweeper log missing")
evidence["lease_recovery"] = {
"attachment_id": attachment["id"],
"one_shot_exit": one_shot.returncode,
"processing_before_expiry": True,
"attempts": row["attempt"],
"ready_after_sweep": True,
"evidence": observed,
"log_excerpt": logs,
}
restore_worker()
return attachment["id"]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--out", type=Path, default=ROOT / "evidence/lab5-grade4.json")
parser.add_argument("--markdown", type=Path, default=ROOT / "evidence/lab5-grade4.md")
parser.add_argument(
"--scenario",
choices=(
"all",
"worker-stopped",
"broker-outage",
"crash-after-effect",
"transient-retry",
"permanent-failure-replay",
"lease-recovery",
),
default="all",
)
args = parser.parse_args()
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
client = Client(args.base_url)
actor, _, _, chat = client.bootstrap()
evidence = {
"generated_at": datetime.now(timezone.utc).isoformat(),
"fixture": {
"bytes": len(content),
"sha256": hashlib.sha256(content).hexdigest(),
"width": 320,
"height": 200,
},
"scenarios": {},
}
selected = [
("worker-stopped", scenario_worker_stopped),
("broker-outage", scenario_broker_outage),
("crash-after-effect", scenario_crash_after_effect),
("transient-retry", scenario_transient_retry),
("permanent-failure-replay", scenario_permanent_failure_and_replay),
("lease-recovery", scenario_lease_recovery),
]
try:
for name, function in selected:
if args.scenario not in ("all", name):
continue
print(f"== {name}", flush=True)
function(client, actor, chat, content, evidence["scenarios"])
print(f"== {name}: OK", flush=True)
finally:
restore_worker()
restore_api()
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(json.dumps(evidence, indent=2) + "\n")
lines = [
"# Lab5 grade-4 runtime evidence",
"",
f"Generated: `{evidence['generated_at']}`",
"",
"Synthetic fixture: 320x200 PNG, public contract fixture; no tokens or credentials are stored.",
"",
"| Scenario | Observation |",
"| --- | --- |",
]
s = evidence["scenarios"]
if "worker_stopped" in s:
lines.append(
"| Worker stopped during upload | 202 queued, content 409, message linked while queued, "
f"ready after worker start; queue {s['worker_stopped']['queue_seconds']} s, "
f"attempt {s['worker_stopped']['row']['attempt']} |"
)
if "broker_outage" in s:
lines.append(
"| Broker outage (COMMIT/publish gap) | 202 accepted with `published_at IS NULL`, "
"API readiness stayed 200, job published and ready after RabbitMQ restart |"
)
if "crash_after_effect" in s:
lines.append(
"| Crash after effect, before ack | worker restart count "
f"{s['crash_after_effect']['worker_restart_count_before']} -> "
f"{s['crash_after_effect']['worker_restart_count_after']}, "
f"{s['crash_after_effect']['variant_objects']} deterministic variant objects, ready |"
)
if "transient_retry" in s:
lines.append(
"| Transient error + jittered retry | failed once, attempt "
f"{s['transient_retry']['attempts']}, ready |"
)
if "permanent_failure_replay" in s:
lines.append(
"| Permanent error + explicit replay | failed `invalid_image`, content/link 409, "
f"replay_count {s['permanent_failure_replay']['replay_count']}, ready with "
f"{s['permanent_failure_replay']['variant_objects_after_replay']} variant objects |"
)
if "lease_recovery" in s:
lines.append(
"| Crash after claim / expired lease | one-shot exit 10, processing, sweeper recovered, "
f"attempt {s['lease_recovery']['attempts']}, ready |"
)
lines.extend(["", "## Sanitized log excerpts", ""])
for name, payload in s.items():
excerpt = payload.get("log_excerpt")
if not excerpt:
continue
lines.append(f"### {name}")
lines.append("")
lines.append("```text")
lines.extend(line for line in excerpt[:12])
lines.append("```")
lines.append("")
args.markdown.write_text("\n".join(lines) + "\n")
print(json.dumps({"ok": True, "out": str(args.out), "markdown": str(args.markdown)}))
if __name__ == "__main__":
try:
main()
except (Failure, RuntimeError, subprocess.TimeoutExpired) as exc:
print("FAIL:", exc)
raise SystemExit(1)