Files
Artyom 12db77133f Forward the lab 5 foundation while preserving lab 6 telemetry
Source: immutable lab5 snapshot f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071; forward the allowed diff from 64d5c67. Carry SQLAlchemy, Alembic, reliability fixes, versioned jobs, infrastructure and student tests without changing protected coursework files.

Validation: make check, validate, lint; 515 student tests passed with one Linux-only skip; public smoke with telemetry enabled and repeat migration passed. Stop the owned stack and retain volumes. Preserve the existing OTel work in progress; grade implementation remains a separate phase.
2026-11-06 12:29:35 +03:00

1139 lines
50 KiB
Python

#!/usr/bin/env python3
import argparse
import hashlib
import json
import os
import shlex
import struct
import subprocess
import sys
import time
import urllib.request
import uuid
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
import make_images
from client import Client, Failure, check
from image_probe import jpeg_size
FIXTURE = ROOT / "tests/fixtures/sample.png"
HEAVY = "png-at-pixel-limit-8000x5000.png"
EXCHANGE, ROUTING_KEY = "attachments", "attachment.process.v1"
QUEUE, DLQ = "attachments.process", "attachments.process.dlq"
RABBITMQADMIN = (
'RABBITMQADMIN_USERNAME=messenger-ops RABBITMQADMIN_PASSWORD="$(cat /run/secrets/'
'rabbitmq_ops_password)" exec rabbitmqadmin --non-interactive -V messenger "$@"'
)
PSQL = 'exec psql -X -q -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"'
ERRORS = (Failure, OSError, ValueError, KeyError, subprocess.TimeoutExpired)
SCENARIOS = (
("smoke_pipeline", 3, "upload, ready, variants match metadata"),
("worker_down_broker_restart", 3, "worker down, API and broker restart"),
("duplicate_after_ready", 3, "redelivery of a finished job"),
("poison_image", 3, "permanent error without retries"),
("s3_outage_recovers", 3, "transient error with retry"),
("retry_exhausted", 3, "transient error until JOB_MAX_ATTEMPTS"),
("image_matrix", 4, "limits and variants"),
("crash_commit_publish", 4, "after COMMIT, before publish"),
("crash_publish_confirm", 4, "after publish, before confirm"),
("crash_after_s3_before_commit", 4, "after S3 write, before COMMIT"),
("crash_after_commit_before_ack", 4, "after COMMIT, before ack"),
("lease_expiry_fencing", 4, "stalled worker, lease expiry"),
("replay_failed", 4, "operator replay"),
("malformed_message_dead_lettered", 4, "message that is not a job goes to the DLQ"),
("worker_limits", 4, "render time and memory limits, 40 Mpx in the worker limit"),
("retry_backoff_jitter", 4, "backoff with equal jitter between attempts"),
("failed_retention", 4, "failed staging kept until retention, then replay expires"),
("crash_before_side_effect", 5, "after claim, before side effects"),
("worker_kill_mid_task", 5, "SIGKILL during processing"),
("duplicate_in_flight", 5, "two copies of one job in flight"),
("reorder_attempts", 5, "attempt 2 delivered before attempt 1"),
("reorder_versions", 5, "v2 delivered before v1"),
("stale_after_new_version", 5, "v1 job after v2 is published"),
)
QUERIES = """
attachment:
SELECT status, published_version, error_code, staging_key FROM attachments WHERE id = :'id'
jobs:
SELECT job_id, pipeline_version, status, attempt FROM jobs
WHERE attachment_id = :'id' ORDER BY pipeline_version
outbox:
SELECT o.payload, o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id)
WHERE j.attachment_id = :'id' ORDER BY o.id
variants:
SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants
WHERE attachment_id = :'id' ORDER BY pipeline_version, name
events:
SELECT kind, attempt FROM job_events WHERE attachment_id = :'id' ORDER BY id
backoff:
SELECT attempt, (detail->>'delay_seconds')::float8 AS delay FROM job_events
WHERE attachment_id = :'id' AND kind = 'retry_scheduled' ORDER BY id
nonterminal:
SELECT id, status FROM attachments
WHERE id = ANY(:'ids'::uuid[]) AND status NOT IN ('ready', 'failed')
known:
SELECT id FROM attachments WHERE id = ANY(:'ids'::uuid[])
duplicate_effects:
SELECT attachment_id, pipeline_version, name, count(*) AS copies FROM attachment_variants
WHERE attachment_id = ANY(:'ids'::uuid[]) GROUP BY 1, 2, 3 HAVING count(*) > 1
attempts:
SELECT job_id, attempt FROM jobs WHERE attachment_id = ANY(:'ids'::uuid[]) AND attempt > :max
pending_outbox:
SELECT o.job_id FROM outbox o JOIN jobs j USING (job_id)
WHERE j.attachment_id = ANY(:'ids'::uuid[]) AND o.published_at IS NULL
unpublished_version:
SELECT a.id, a.published_version, max(j.pipeline_version) AS done FROM attachments a
JOIN jobs j ON j.attachment_id = a.id AND j.status = 'done' WHERE a.id = ANY(:'ids'::uuid[])
GROUP BY a.id, a.published_version HAVING a.published_version <> max(j.pipeline_version)
staging_left:
SELECT id FROM attachments
WHERE id = ANY(:'ids'::uuid[]) AND status = 'ready' AND staging_key IS NOT NULL
object_keys:
SELECT object_key AS key FROM attachment_variants WHERE attachment_id = ANY(:'ids'::uuid[])
UNION ALL SELECT original_key FROM attachments
WHERE id = ANY(:'ids'::uuid[]) AND original_key IS NOT NULL
UNION ALL SELECT staging_key FROM attachments
WHERE id = ANY(:'ids'::uuid[]) AND staging_key IS NOT NULL
columns:
SELECT table_name, column_name FROM information_schema.columns
WHERE table_schema = current_schema()
"""
SQL = {
name.strip(): " ".join(body.split())
for name, body in (block.split(":\n", 1) for block in QUERIES.strip().split("\n\n"))
}
REQUIRED_COLUMNS = {
"attachments": "id status published_version error_code staging_key original_key",
"attachment_variants": "attachment_id pipeline_version name sha256 size_bytes object_key",
"jobs": "job_id attachment_id pipeline_version status attempt",
"outbox": "job_id payload published_at",
"job_events": "job_id attachment_id kind attempt",
}
def now():
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
def run(cmd, stdin=None, env=None, timeout=180):
done = subprocess.run(
cmd, input=stdin, capture_output=True, text=True, env=env, timeout=timeout, check=False
)
if done.returncode:
raise Failure(f"{' '.join(cmd[:7])}: exit {done.returncode}: {done.stderr.strip()[-400:]}")
return done.stdout
def poll(probe, timeout, what, interval=0.5):
deadline = time.monotonic() + timeout
while True:
value = probe()
if value:
return value
if time.monotonic() > deadline:
raise Failure(f"timeout {timeout}s waiting for {what}")
time.sleep(interval)
def http_ready(base_url):
try:
with urllib.request.urlopen(base_url.rstrip("/") + "/health/ready", timeout=3) as response:
return response.status == 200
except OSError:
return False
def image_size(data, content_type):
if content_type == "image/jpeg":
return jpeg_size(data)
if content_type == "image/png":
return struct.unpack(">II", data[16:24])
check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP")
if data[12:16] == b"VP8 ":
width, height = struct.unpack("<HH", data[26:30])
return width & 0x3FFF, height & 0x3FFF
if data[12:16] == b"VP8L":
bits = int.from_bytes(data[21:25], "little")
return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
return int.from_bytes(data[24:27], "little") + 1, int.from_bytes(data[27:30], "little") + 1
class Stack:
def __init__(self, args):
self.args = args
self.base = shlex.split(args.compose)
self.faults = shlex.split(args.compose_faults)
def dc(self, *argv, faults=False, stdin=None, env=None, timeout=180):
command = [*(self.faults if faults else self.base), *argv]
return run(command, stdin, dict(os.environ, **(env or {})), timeout)
def up(self, *services, faults=False, env=None):
wait = ["up", "-d", "--no-deps", "--no-recreate", "--wait", "--wait-timeout", "90"]
if faults or env:
wait.remove("--no-recreate")
self.dc(*wait, *services, faults=faults, env=env)
def launch(self, service, point="", **overrides):
variable = "WORKER_FAULT_POINT" if service == self.args.worker else "RELAY_FAULT_POINT"
self.up(service, faults=True, env={variable: point, **overrides})
def cgroup(self, service, name):
return self.dc("exec", "-T", service, "cat", "/sys/fs/cgroup/" + name).strip()
def restore(self):
self.dc("up", "-d", "--no-deps", "--wait", self.args.relay, self.args.worker)
def stop(self, *services):
self.dc("stop", "-t", "10", *services)
def kill(self, service):
self.dc("kill", "-s", "KILL", service)
def restart(self, *services):
self.dc("restart", "-t", "20", *services)
self.up(*services)
def state(self, service):
text = self.dc("ps", "-a", "--format", "json", service).strip()
rows = (
json.loads(text) if text.startswith("[") else [json.loads(x) for x in text.split("\n")]
)
check(len(rows) == 1, "expected one container for " + service)
return rows[0]["State"], rows[0].get("ExitCode")
def wait_exit(self, service, timeout=60):
def probe():
state, code = self.state(service)
return (code,) if state in ("exited", "dead") else None
return poll(probe, timeout, service + " exit")[0]
def sql(self, name, **params):
query = f"SELECT coalesce(json_agg(q), '[]'::json) FROM ({SQL[name]}) q;\n"
variables = [x for key, value in params.items() for x in ("-v", f"{key}={value}")]
out = self.dc(
"exec", "-T", self.args.postgres, "sh", "-c", PSQL, "sh", *variables, stdin=query
)
return json.loads(out)
def queues(self):
columns = "name messages_ready messages_unacknowledged --formatter json"
listing = f"exec -T {self.args.broker} rabbitmqctl -q list_queues -p messenger {columns}"
rows = json.loads(self.dc(*listing.split()))
return {q["name"]: (q["messages_ready"], q["messages_unacknowledged"]) for q in rows}
def rabbitmqadmin(self, *argv, stdin=None):
return self.dc(
"exec", "-T", self.args.broker, "sh", "-c", RABBITMQADMIN, "sh", *argv, stdin=stdin
)
def publish(self, payload):
properties = {"delivery_mode": 2, "content_type": "application/json", "type": ROUTING_KEY}
properties["message_id"] = payload["job_id"]
command = f"publish message -e {EXCHANGE} -k {ROUTING_KEY} -f - -p".split()
out = self.rabbitmqadmin(*command, json.dumps(properties), stdin=json.dumps(payload))
check("routed successfully" in out, "broker did not route the injected job")
def s3_json(self, *argv, missing=False):
done = subprocess.run(
[*self.base, "run", "--rm", "--no-deps", "-T", "s3-cli", *argv, "--json"],
capture_output=True,
text=True,
timeout=60,
check=False,
)
body = json.loads(done.stdout or done.stderr)
if missing and done.returncode == 5 and body.get("details", {}).get("type") == "not_found":
return None
check(done.returncode == 0, "S3 inventory command failed: exit " + str(done.returncode))
return body
def objects(self, ids):
bucket = self.dc("exec", "-T", self.args.relay, "printenv", "S3_BUCKET").strip()
keys = set()
for ident in ids:
for prefix in ("staging", "originals"):
body = self.s3_json(
"object", "stat", f"root/{bucket}/{prefix}/{ident}", missing=True
)
if body is not None:
keys.add(body["name"])
body = self.s3_json("object", "list", "--recursive", f"root/{bucket}/variants/{ident}/")
check(not body["truncated"], "S3 variant inventory is truncated")
keys.update(item["key"] for item in body["items"])
return keys
def admin(self, *argv):
command = shlex.split(self.args.admin) + list(argv)
out = self.dc("run", "--rm", "--no-deps", "-T", self.args.worker, *command)
return json.loads(out.strip().splitlines()[-1])
class Api:
def __init__(self, args):
self.base_url = args.base_url
self.client = Client(args.base_url, args.ca_file)
self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
self.accepted, self.traces, self.final = [], {}, {}
def upload(self, data, kind="image/png", name="sample.png", expected=202, final=("ready",)):
path = f"/api/v1/chats/{self.chat['id']}/attachments?filename={quote(name)}"
allowed = [expected] if isinstance(expected, int) else expected
headers = {"Content-Type": kind}
response = self.client.call(
"POST", path, allowed, actor=self.alice, raw=data, headers=headers, schema="Attachment"
)
if response["status"] != 202:
return None
body = response["json"]
snapshot = (
body["status"] == "queued" and body["variants"] == [] and body["error_code"] is None
)
check(snapshot, "202 is not a queued snapshot")
same = body["sha256"] == hashlib.sha256(data).hexdigest() and body["size_bytes"] == len(
data
)
check(same, "202 checksum or size differs from the uploaded bytes")
self.accepted.append(body["id"])
self.final[body["id"]] = set(final)
self.traces[body["id"]] = ["queued"]
return body
def fixture(self, **kwargs):
return self.upload(FIXTURE.read_bytes(), **kwargs)
def get(self, ident):
path = "/api/v1/attachments/" + ident
body = self.client.call("GET", path, actor=self.bob, schema="Attachment")["json"]
self.traces.setdefault(ident, []).append(body["status"])
return body
def wait(self, ident, final=("ready",), timeout=90):
def probe():
body = self.get(ident)
failed = body["status"] == "failed" and "failed" not in final
check(not failed, f"attachment reached failed: {body['error_code']}")
return body if body["status"] in final else None
return poll(probe, timeout, f"{ident} {'/'.join(final)}", 0.3)
def content(self, ident, variant=None, expected=200):
path = f"/api/v1/attachments/{ident}/content" + (f"?variant={variant}" if variant else "")
return self.client.call("GET", path, expected, actor=self.bob)
def message(self, text, attachments=None, expected=201):
if expected == 201:
return self.client.message(self.alice, self.chat, text, attachments=attachments)["json"]
body = {"text": text, "attachment_ids": attachments or []}
headers = {"Idempotency-Key": os.urandom(8).hex()}
path = f"/api/v1/chats/{self.chat['id']}/messages"
return self.client.call(
"POST", path, expected, actor=self.alice, body=body, headers=headers
)
class Suite:
def __init__(self, args):
self.args = args
self.stack = Stack(args)
self.api = None
self.samples = {}
self.dead_letters = 0
def scenarios(self):
return [(name, level, window, getattr(self, name)) for name, level, window in SCENARIOS]
def drained(self, timeout=60):
poll(lambda: self.stack.queues()[QUEUE] == (0, 0), timeout, "main queue drained")
def queued(self, count, timeout=30):
poll(lambda: self.stack.queues()[QUEUE][0] >= count, timeout, f"{count} ready messages")
def record(self, ident):
row = self.stack.sql("attachment", id=ident)[0]
self.samples.setdefault(ident, []).append(row["published_version"])
self.api.traces.setdefault(ident, []).append(row["status"])
return row
def snapshot(self, ident):
body = self.api.get(ident)
public = sorted(body["variants"], key=lambda v: v["name"])
published = self.record(ident)["published_version"]
rows = self.stack.sql("variants", id=ident)
return {"status": body["status"], "public": public, "rows": rows, "published": published}
def payload(self, ident, version=1):
rows = self.stack.sql("outbox", id=ident)
found = [r["payload"] for r in rows if r["payload"]["pipeline_version"] == version]
check(bool(found), f"no outbox payload for v{version}")
return found[-1]
def verify(self, ready, expect=None):
names = sorted(v["name"] for v in ready["variants"])
check(len(names) == len(set(names)), "duplicate variant names in API")
missing = set(self.args.expect_variants.split(",")) - set(names)
check(not missing, f"missing variants {sorted(missing)}")
original = self.api.content(ready["id"])
data = original["bytes"]
same = (
hashlib.sha256(data).hexdigest() == ready["sha256"] and len(data) == ready["size_bytes"]
)
check(same, "original bytes differ from metadata")
check(
original["headers"].get_content_type() == ready["content_type"], "original type differs"
)
for variant in ready["variants"]:
name = variant["name"]
got = self.api.content(ready["id"], name)
data, kind = got["bytes"], got["headers"].get_content_type()
check(kind == variant["content_type"], name + " content type differs")
same = len(data) == variant["size_bytes"]
check(
same and hashlib.sha256(data).hexdigest() == variant["sha256"],
name + " bytes differ",
)
size = (variant["width"], variant["height"])
check(tuple(image_size(data, kind)) == size, name + " real size differs from metadata")
wanted = (expect or {}).get(name)
slack = 1 if name == "optimized" else 0
fits = not wanted or all(abs(a - b) <= slack for a, b in zip(wanted, size))
check(fits, f"{name} is {size}, expected {wanted}")
return names
def crash(self, service, point, observe):
self.drained()
self.stack.launch(service, point)
ident = self.api.fixture()["id"]
code = self.stack.wait_exit(service)
check(code == 137, f"{service} exited with {code}, expected 137 at {point}")
seen = observe(ident)
seen["containers"] = self.stack.dc(
"ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", service
).splitlines()
self.stack.launch(service)
ready = self.api.wait(ident, timeout=self.args.recovery_timeout)
poll(lambda: all(r["published"] for r in self.stack.sql("outbox", id=ident)), 30, "outbox")
self.drained()
self.verify(ready)
variants = self.stack.sql("variants", id=ident)
return {
"attachment_id": ident,
"exit_code": code,
"after_crash": seen,
"variants": variants,
}
def smoke_pipeline(self):
att = self.api.fixture()
ready = self.api.wait(att["id"])
return {
"attachment_id": att["id"],
"variants": self.verify(ready, {"thumbnail": [256, 160]}),
}
def worker_down_broker_restart(self):
steps = []
def mark(step, **extra):
steps.append({"at": now(), "step": step, **extra})
self.stack.stop(self.args.worker)
mark(
"worker stopped",
containers=self.stack.dc(
"ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", self.args.worker
).splitlines(),
)
att = self.api.fixture()
mark("upload 202 queued", attachment_id=att["id"])
self.api.content(att["id"], expected=409)
mark("content 409 while queued")
message = self.api.message("", [att["id"]])
self.api.message("text messages still work")
path = f"/api/v1/chats/{self.api.chat['id']}/messages?limit=5"
self.api.client.call("GET", path, actor=self.api.bob, schema="MessagePage")
mark("message with attachment 201, other messages work", message_id=message["id"])
self.queued(1)
mark("job is in the broker queue", queues=self.stack.queues())
self.stack.restart(*self.args.api, self.args.relay)
poll(lambda: http_ready(self.api.base_url), 90, "API ready")
mark("API replicas and relay restarted")
self.stack.restart(self.args.broker)
self.queued(1)
mark("broker restarted with its volume, job survived", queues=self.stack.queues())
self.stack.launch(self.args.worker)
mark("worker started")
ready = self.api.wait(att["id"])
mark("ready", variants=self.verify(ready))
return {"attachment_id": att["id"], "timeline": steps}
def duplicate_after_ready(self):
ident = self.api.fixture()["id"]
self.api.wait(ident)
self.drained()
before = self.snapshot(ident)
self.stack.publish(self.payload(ident))
self.drained()
time.sleep(1)
check(self.snapshot(ident) == before, "duplicate delivery changed the published result")
return {"attachment_id": ident, "events": self.stack.sql("events", id=ident)}
def poison_image(self):
data = make_images.png(320, 200, make_images.gradient(320, 200, 3))
broken = data[: data.index(b"IDAT") + 600]
att = self.api.upload(broken, expected=[202, 415], final=("failed",))
if att is None:
return {"rejected_by_api": 415}
failed = self.api.wait(att["id"], ("failed",))
check(
failed["error_code"] == "invalid_image", f"unexpected error_code {failed['error_code']}"
)
jobs = self.stack.sql("jobs", id=att["id"])
check(all(j["attempt"] == 1 for j in jobs), "permanent error was retried")
self.api.content(att["id"], expected=409)
self.api.message("", [att["id"]], expected=409)
return {"attachment_id": att["id"], "error_code": failed["error_code"], "jobs": jobs}
def s3_outage_recovers(self):
self.stack.stop(self.args.worker)
ident = self.api.fixture()["id"]
self.queued(1)
self.stack.stop(self.args.s3)
try:
self.stack.launch(self.args.worker)
poll(
lambda: any(j["attempt"] > 1 for j in self.stack.sql("jobs", id=ident)), 60, "retry"
)
status = self.record(ident)["status"]
check(status in ("queued", "processing"), "transient error produced " + status)
finally:
self.stack.up(self.args.s3)
poll(lambda: http_ready(self.api.base_url), 90, "API ready")
self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout))
return {"attachment_id": ident, "jobs": self.stack.sql("jobs", id=ident)}
def exhaust(self):
self.drained()
self.stack.launch(self.args.worker, "transient_error")
ident = self.api.fixture(final=("failed", "ready"))["id"]
failed = self.api.wait(ident, ("failed",), timeout=self.args.recovery_timeout)
self.stack.launch(self.args.worker)
jobs = self.stack.sql("jobs", id=ident)
check(failed["error_code"] == "retry_exhausted", f"unexpected {failed['error_code']}")
check(max(j["attempt"] for j in jobs) == self.args.max_attempts, "attempts != max")
return ident, jobs
def retry_exhausted(self):
ident, jobs = self.exhaust()
self.api.final[ident] = {"failed"}
self.api.content(ident, expected=409)
self.api.message("", [ident], expected=409)
return {"attachment_id": ident, "jobs": jobs}
def replay_failed(self):
ident, _ = self.exhaust()
self.api.final[ident] = {"ready"}
first = self.stack.admin("replay", "--attachment-id", ident)
self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout))
self.drained()
before = self.snapshot(ident)
second = self.stack.admin("replay", "--attachment-id", ident)
self.drained()
check(self.snapshot(ident) == before, "replay of a ready attachment changed the result")
events = self.stack.sql("events", id=ident)
check(any("replay" in e["kind"] for e in events), "replay left no audit event")
return {
"attachment_id": ident,
"replay": first,
"replay_of_ready": second,
"events": events,
}
def malformed_message_dead_lettered(self):
before = self.stack.queues()[DLQ][0]
self.stack.publish({"schema_version": 2, "job_id": str(uuid.uuid4())})
poll(lambda: self.stack.queues()[DLQ][0] > before, 30, "dead letter")
self.dead_letters += 1
return {"dead_letters": self.stack.queues()[DLQ][0] - before}
def limited(self, data, **overrides):
self.stack.launch(self.args.worker, **overrides)
att = self.api.upload(data, name=HEAVY, final=("failed",))
failed = self.api.wait(att["id"], ("failed",), self.args.recovery_timeout)
check(failed["error_code"] == "processing_failed", f"limit gave {failed['error_code']}")
jobs = self.stack.sql("jobs", id=att["id"])
check([j["attempt"] for j in jobs] == [1], "a limit breach was retried")
state, _ = self.stack.state(self.args.worker)
check(state == "running", "worker died on a limit breach: " + state)
return {"attachment_id": att["id"], "error_code": failed["error_code"], "worker": state}
def worker_limits(self):
folder = Path(self.args.images) / "limits"
make_images.build(folder, True, {HEAVY})
data = (folder / HEAVY).read_bytes()
self.drained()
timed = self.limited(data, FAULT_RENDER_TIMEOUT_SECONDS=str(self.args.tight_timeout))
memory = self.limited(data, FAULT_RENDER_MEMORY_MB=str(self.args.tight_memory_mb))
self.stack.launch(self.args.worker)
att = self.api.upload(data, name=HEAVY)
ready = self.api.wait(att["id"], timeout=self.args.recovery_timeout)
sizes = {"thumbnail": [256, 160], "crop": [128, 128], "optimized": [1280, 800]}
names = self.verify(ready, sizes)
peak = int(self.stack.cgroup(self.args.worker, "memory.peak"))
limit = int(self.stack.cgroup(self.args.worker, "memory.max"))
check(peak < limit, f"worker peak {peak} reached its limit {limit}")
return {
"image": {"file": HEAVY, "bytes": len(data), "pixels": 8000 * 5000},
"render_timeout": {"seconds": self.args.tight_timeout, **timed},
"render_memory": {"mib": self.args.tight_memory_mb, **memory},
"default_limits": {
"attachment_id": att["id"],
"variants": names,
"memory_peak_mib": round(peak / 2**20, 1),
"memory_max_mib": round(limit / 2**20, 1),
},
}
def retry_backoff_jitter(self):
ident, jobs = self.exhaust()
self.api.final[ident] = {"failed"}
rows = self.stack.sql("backoff", id=ident)
check([r["attempt"] for r in rows] == [2, 3, 4, 5], f"unexpected retries {rows}")
shares = []
for row in rows:
ceiling = min(60.0, self.args.backoff_base * 2 ** (row["attempt"] - 2))
share = row["delay"] / ceiling
check(0.5 <= share <= 1.0, f"delay {row['delay']} outside [{ceiling / 2}, {ceiling}]")
shares.append(round(share, 3))
check(len(set(shares)) > 1, f"no jitter: shares {shares}")
return {"attachment_id": ident, "delays": rows, "share_of_ceiling": shares, "jobs": jobs}
def failed_retention(self):
ident, _ = self.exhaust()
self.api.final[ident] = {"failed"}
kept = self.record(ident)["staging_key"]
check(kept is not None, "failed staging was removed before the retention period")
retention = int(
self.stack.dc("exec", "-T", self.args.relay, "printenv", "STAGING_RETENTION_SECONDS")
)
aged = self.stack.dc(
"exec",
"-T",
self.args.postgres,
"sh",
"-c",
PSQL,
"sh",
"-v",
"id=" + ident,
"-v",
"chat=" + self.api.chat["id"],
"-v",
"age=" + str(retention + 1),
stdin="UPDATE attachments SET created_at = created_at - :age * interval '1 second', "
"updated_at = updated_at - :age * interval '1 second' "
"WHERE id = :'id' AND chat_id = :'chat' AND status = 'failed' RETURNING id;\n",
)
check(aged.strip() == ident, "did not age exactly the suite's failed attachment")
poll(lambda: self.record(ident)["staging_key"] is None, 30, "failed staging cleanup")
replay = self.stack.admin("replay", "--attachment-id", ident)
check(replay["result"] == "expired", f"replay after retention gave {replay}")
check(self.api.get(ident)["status"] == "failed", "expired replay changed the status")
return {
"attachment_id": ident,
"staging_before": "kept",
"retention_seconds": retention,
"aged_only": ident,
"replay": replay,
}
def image_matrix(self):
folder = Path(self.args.images)
results = []
for item in make_images.build(folder, self.args.heavy):
expect, name = item["expect"], item["file"]
ready = expect.get("final") == "ready"
data = (folder / name).read_bytes()
final = ("ready",) if ready else ("failed", "ready")
att = self.api.upload(data, item["content_type"], name, expect["upload"], final)
row = {"file": name, "http": 202 if att else "rejected"}
if att:
done = self.api.wait(att["id"], ("ready", "failed"), self.args.recovery_timeout)
row.update(status=done["status"], error_code=done["error_code"])
if ready:
check(done["status"] == "ready", name + " did not become ready")
row["variants"] = self.verify(done, expect["variants"])
else:
self.api.final[att["id"]] = {"failed"}
code = expect.get("error_code")
check(done["error_code"] == code, f"{name} should fail with {code}")
results.append(row)
return {"images": results}
def crash_commit_publish(self):
def observe(ident):
outbox = self.stack.sql("outbox", id=ident)
check(outbox and not any(r["published"] for r in outbox), "outbox already published")
check(self.stack.queues()[QUEUE] == (0, 0), "job reached the broker before publish")
return {"outbox_published": False, "status": self.record(ident)["status"]}
return self.crash(self.args.relay, "after_commit_before_publish", observe)
def crash_publish_confirm(self):
def observe(ident):
outbox = [r["published"] for r in self.stack.sql("outbox", id=ident)]
check(outbox == [False], "outbox row was marked before the crash point")
return {"outbox_published": outbox, "queues": self.stack.queues()}
result = self.crash(self.args.relay, "after_publish_before_confirm", observe)
kinds = [e["kind"] for e in self.stack.sql("events", id=result["attachment_id"])]
check("duplicate_noop" in kinds, "the republished copy was not a no-op: " + str(kinds))
return {**result, "events": kinds}
def crash_after_s3_before_commit(self):
def observe(ident):
check(not self.stack.sql("variants", id=ident), "variants committed before the crash")
return {"status": self.record(ident)["status"], "variant_rows": 0}
return self.crash(self.args.worker, "after_s3_write_before_commit", observe)
def crash_after_commit_before_ack(self):
def observe(ident):
row = self.record(ident)
check(row["status"] == "ready", "result was not committed before the crash point")
variants = self.stack.sql("variants", id=ident)
return {"status": row["status"], "variants": variants, "queues": self.stack.queues()}
result = self.crash(self.args.worker, "after_commit_before_ack", observe)
same = result["variants"] == result["after_crash"]["variants"]
check(same, "redelivery after COMMIT changed variants")
return result
def crash_before_side_effect(self):
def observe(ident):
return {
"status": self.record(ident)["status"],
"jobs": self.stack.sql("jobs", id=ident),
}
return self.crash(self.args.worker, "after_claim_before_side_effect", observe)
def lease_expiry_fencing(self):
self.drained()
self.stack.launch(self.args.worker, "stall_after_claim")
ident = self.api.fixture()["id"]
self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout))
before = self.snapshot(ident)
time.sleep(self.args.stall_seconds + 3)
after = self.snapshot(ident)
self.stack.launch(self.args.worker)
check(before == after, "stalled worker overwrote the result after its lease expired")
jobs, events = self.stack.sql("jobs", id=ident), self.stack.sql("events", id=ident)
check(any(e["kind"] == "fenced" for e in events), "stalled holder was not fenced")
return {"attachment_id": ident, "jobs": jobs, "events": events}
def worker_kill_mid_task(self):
self.drained()
self.stack.launch(self.args.worker, "stall_after_claim")
ident = self.api.fixture()["id"]
poll(lambda: self.record(ident)["status"] == "processing", 30, "processing")
self.stack.kill(self.args.worker)
code = self.stack.wait_exit(self.args.worker)
containers = self.stack.dc(
"ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", self.args.worker
).splitlines()
self.stack.launch(self.args.worker)
ready = self.api.wait(ident, timeout=self.args.recovery_timeout)
self.drained()
self.verify(ready)
return {
"attachment_id": ident,
"exit_code": code,
"containers": containers,
"jobs": self.stack.sql("jobs", id=ident),
}
def duplicate_in_flight(self):
self.stack.stop(self.args.worker)
ident = self.api.fixture()["id"]
self.queued(1)
self.stack.publish(self.payload(ident))
self.queued(2)
self.stack.launch(self.args.worker)
ready = self.api.wait(ident)
self.drained()
self.verify(ready)
variants, events = self.stack.sql("variants", id=ident), self.stack.sql("events", id=ident)
return {"attachment_id": ident, "variants": variants, "events": events}
def reorder_attempts(self):
self.stack.stop(self.args.worker, self.args.relay)
ident = self.api.fixture()["id"]
original = self.payload(ident)
self.stack.publish({**original, "attempt": original["attempt"] + 1})
self.queued(1)
self.stack.launch(self.args.relay)
self.queued(2)
self.stack.launch(self.args.worker)
ready = self.api.wait(ident)
self.drained()
self.verify(ready)
jobs, events = self.stack.sql("jobs", id=ident), self.stack.sql("events", id=ident)
return {"attachment_id": ident, "jobs": jobs, "events": events}
def published(self, ident, version):
poll(
lambda: self.record(ident)["published_version"] >= version,
self.args.recovery_timeout,
f"published v{version}",
)
def reprocess(self, ident):
return self.stack.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
def reorder_versions(self):
self.stack.stop(self.args.worker, self.args.relay)
ident = self.api.fixture()["id"]
check(self.reprocess(ident)["result"] == "queued", "version 2 was not scheduled")
check(self.reprocess(ident)["result"] == "noop", "duplicate reprocess was not a noop")
self.stack.publish(self.payload(ident, 2))
self.queued(1)
self.stack.launch(self.args.worker)
self.published(ident, 2)
before = self.snapshot(ident)
self.stack.launch(self.args.relay)
poll(
lambda: (
{j["pipeline_version"]: j["status"] for j in self.stack.sql("jobs", id=ident)}
== {1: "superseded", 2: "done"}
),
self.args.recovery_timeout,
"old version superseded",
)
poll(lambda: all(r["published"] for r in self.stack.sql("outbox", id=ident)), 30, "outbox")
self.drained()
final = self.snapshot(ident)
check(final["published"] == 2, f"published_version rolled back to {final['published']}")
check(before["public"] == final["public"], "late v1 changed the published v2 variants")
served = {v["name"]: v["sha256"] for v in final["public"]}
stored = {r["name"]: r["sha256"] for r in final["rows"] if r["pipeline_version"] == 2}
check(served == stored, "API serves variants of a version that is not published")
jobs = self.stack.sql("jobs", id=ident)
check(
{j["pipeline_version"]: j["status"] for j in jobs} == {1: "superseded", 2: "done"},
"v1 did not finish after v2",
)
self.verify(self.api.get(ident))
return {"attachment_id": ident, "jobs": jobs, "v2_before_v1": before, "after_v1": final}
def stale_after_new_version(self):
ident = self.api.fixture()["id"]
self.api.wait(ident)
first = self.snapshot(ident)
check(self.reprocess(ident)["result"] == "queued", "version 2 was not scheduled")
check(self.api.get(ident)["status"] == "ready", "reprocess rolled back ready")
self.published(ident, 2)
self.drained()
before = self.snapshot(ident)
check(first["public"] != before["public"], "recipes did not produce distinct results")
self.stack.publish(self.payload(ident, 1))
self.drained()
time.sleep(1)
after = self.snapshot(ident)
check(before == after and after["published"] == 2, "stale v1 job changed the v2 result")
return {
"attachment_id": ident,
"published_version": after["published"],
"v1": first,
"v2_before_stale": before,
"v2_after_stale": after,
}
def invariants(self, dead_before):
ids = self.api.accepted
array = "{" + ",".join(ids) + "}"
if not ids:
return [{"id": "accepted_work", "result": "FAIL", "details": "no upload got 202"}]
def empty(name, message, **params):
rows = self.stack.sql(name, ids=array, **params)
check(not rows, f"{message}: {rows[:5]}")
return {"rows": 0}
def terminal():
empty("nonterminal", "non-terminal attachments left")
wrong = [i for i in ids if self.api.get(i)["status"] not in self.api.final[i]]
check(not wrong, "unexpected final status: " + ", ".join(wrong))
return {"accepted": len(ids)}
def no_rollback():
traces = self.api.traces.items()
broken = [
i for i, t in traces if "ready" in t and set(t[t.index("ready") :]) != {"ready"}
]
check(not broken, "ready rolled back: " + ", ".join(broken))
return {"traced": len(self.api.traces)}
def monotonic():
broken = [i for i, values in self.samples.items() if values != sorted(values)]
check(not broken, "published_version went backwards: " + ", ".join(broken))
empty("unpublished_version", "published_version differs from the newest done job")
return {"sampled": len(self.samples)}
def served():
bodies = [self.api.get(i) for i in ids]
ready = [b for b in bodies if b["status"] == "ready"]
for body in ready:
self.verify(body)
return {"verified": len(ready)}
def orphans():
empty("pending_outbox", "unpublished outbox rows")
empty("staging_left", "staging kept after ready")
queues = self.stack.queues()
check(queues[QUEUE] == (0, 0), "messages left in " + QUEUE)
expected = dead_before + self.dead_letters
check(queues[DLQ][0] == expected, f"unexpected dead letters: {queues[DLQ]}")
keys = {r["key"] for r in self.stack.sql("object_keys", ids=array)}
if self.args.s3_list:
listing = {
x.split()[-1]
for x in run(shlex.split(self.args.s3_list)).splitlines()
if x.strip()
}
listing = {key for key in listing if any("/" + ident in key for ident in ids)}
else:
listing = self.stack.objects(ids)
check(
listing == keys,
f"S3 mismatch: extra={sorted(listing - keys)}, missing={sorted(keys - listing)}",
)
return {"queues": queues, "s3_objects": len(listing), "scoped_attachments": len(ids)}
def no_loss():
known = {r["id"] for r in self.stack.sql("known", ids=array)}
check(known == set(ids), "accepted attachments are missing in PostgreSQL")
return {"accepted": len(ids)}
claims = [
("terminal", "every 202 reaches its expected terminal state", terminal),
(
"single_effect",
"one variant set per (attachment_id, pipeline_version, variant)",
lambda: empty("duplicate_effects", "duplicate effect rows"),
),
("no_ready_rollback", "ready never returns to queued or processing", no_rollback),
("monotonic_publication", "published_version only moves forward", monotonic),
("bytes_match_metadata", "sha256, size and real dimensions match served bytes", served),
(
"bounded_attempts",
"attempt <= JOB_MAX_ATTEMPTS",
lambda: empty("attempts", "attempt above limit", max=self.args.max_attempts),
),
("no_orphans", "no pending outbox, messages, dead letters, staging, objects", orphans),
("no_lost_202", "every 202 is present in PostgreSQL and the API", no_loss),
]
return [self.claim(name, statement, probe) for name, statement, probe in claims]
@staticmethod
def claim(name, statement, probe):
try:
return {"id": name, "statement": statement, "result": "PASS", "details": probe()}
except ERRORS as error:
return {"id": name, "statement": statement, "result": "FAIL", "details": str(error)}
def preflight(self):
def services():
self.detect_api()
present = set(self.stack.dc("config", "--services").split())
wanted = {*self.args.api, self.args.relay, self.args.worker, self.args.broker}
wanted |= {self.args.postgres, self.args.s3}
check(wanted <= present, f"missing services {sorted(wanted - present)}")
return sorted(wanted)
def schema():
columns = {}
for row in self.stack.sql("columns"):
columns.setdefault(row["table_name"], set()).add(row["column_name"])
needed = {t: set(c.split()) for t, c in REQUIRED_COLUMNS.items()}
missing = {t: sorted(c - columns.get(t, set())) for t, c in needed.items()}
check(not any(missing.values()), f"missing columns {missing}")
return sorted(needed)
def broker():
queues = self.stack.queues()
check({QUEUE, DLQ} <= set(queues), "queues are not declared")
check(QUEUE in self.stack.rabbitmqadmin("list", "queues"), "rabbitmqadmin failed")
return queues
def faults():
check("FAULT_POINT" in self.stack.dc("config", faults=True), "no FAULT_POINT in faults")
return self.args.compose_faults
def api():
check(http_ready(self.args.base_url), "GET /health/ready is not 200")
return self.args.base_url
steps = [
("compose services", services),
("api ready", api),
("postgres schema", schema),
("broker topology and ops CLI", broker),
("faults override", faults),
]
return [self.claim(name, name, probe) for name, probe in steps]
def main(self):
report = {
"suite": "lab5-pipeline-idempotency",
"started_at": now(),
"source_sha": run(["git", "rev-parse", "HEAD"]).strip(),
"config": self.config(),
}
if self.args.preflight:
report["preflight"] = self.preflight()
return self.finish(report, report["preflight"])
results = []
try:
self.detect_api()
self.api = Api(self.args)
dead_before = self.stack.queues()[DLQ][0]
self.stack.launch(self.args.relay)
self.stack.launch(self.args.worker)
for name, level, window, scenario in self.scenarios():
if self.selected(name, level):
results.append(self.attempt(name, level, window, scenario))
report["invariants"] = self.invariants(dead_before)
except ERRORS as error:
report["error"] = str(error)
finally:
self.stack.up(self.args.s3)
self.stack.restore()
report["accepted_ids"] = self.api.accepted if self.api else []
report["scenarios"] = results
return self.finish(report, results + report.get("invariants", []))
def attempt(self, name, level, window, scenario):
began = time.monotonic()
print("RUN " + name, flush=True)
try:
outcome = {"result": "PASS", "observed": scenario()}
except ERRORS as error:
outcome = {"result": "FAIL", "error": str(error)}
self.heal()
print(f"{outcome['result']} {name}", flush=True)
seconds = round(time.monotonic() - began, 1)
return {"name": name, "level": level, "window": window, "seconds": seconds, **outcome}
def heal(self):
try:
self.stack.up(self.args.s3, self.args.broker)
self.stack.launch(self.args.relay)
self.stack.launch(self.args.worker)
except ERRORS as error:
print("heal failed: " + str(error), flush=True)
def detect_api(self):
if not self.args.api:
names = self.stack.dc("config", "--services").split()
api = [x for x in names if x == "api" or (x[:4] == "api-" and x[4:].isdigit())]
self.args.api = api
check(bool(self.args.api), "no API service: expected api or api-N, or pass --api")
def selected(self, name, level):
return name in self.args.only.split(",") if self.args.only else level <= self.args.level
def config(self):
keys = (
"base_url level only expect_variants max_attempts stall_seconds compose compose_faults"
)
return {key: getattr(self.args, key) for key in keys.split()}
def finish(self, report, verdicts):
passed = bool(verdicts) and "error" not in report
report["result"] = (
"PASS" if passed and all(v["result"] == "PASS" for v in verdicts) else "FAIL"
)
report["finished_at"] = now()
text = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
if self.args.output:
Path(self.args.output).parent.mkdir(parents=True, exist_ok=True)
Path(self.args.output).write_text(text)
else:
print(text)
print(f"{report['result']}: lab5 pipeline suite, report {self.args.output or 'stdout'}")
return 0 if report["result"] == "PASS" else 1
def arguments():
env = os.environ.get
faults = "docker compose -f compose.yaml -f infra/compose.faults.yaml"
parser = argparse.ArgumentParser(description="Lab5 pipeline fault injection with invariants")
parser.add_argument("--base-url", default=env("BASE_URL", "http://localhost:8080"))
parser.add_argument("--ca-file", default=env("CA_FILE"))
parser.add_argument("--compose", default=env("COMPOSE", "docker compose"))
parser.add_argument("--compose-faults", default=env("COMPOSE_FAULTS", faults))
parser.add_argument("--admin", default=env("ADMIN_CLI", "python -m messenger.admin"))
parser.add_argument("--api", nargs="+", default=env("API_SERVICES", "").split())
parser.add_argument("--worker", default="worker")
parser.add_argument("--relay", default="relay")
parser.add_argument("--broker", default="broker")
parser.add_argument("--postgres", default="postgres")
parser.add_argument("--s3", default="s3")
parser.add_argument("--s3-list", default=env("S3_LIST_CMD", ""), help="prints object keys")
parser.add_argument("--level", type=int, choices=[3, 4, 5], default=5)
parser.add_argument("--only", default="", help="comma separated scenario names")
parser.add_argument("--list", action="store_true")
parser.add_argument("--preflight", action="store_true")
parser.add_argument("--expect-variants", default=None)
parser.add_argument("--max-attempts", type=int, default=int(env("JOB_MAX_ATTEMPTS", "5")))
parser.add_argument(
"--stall-seconds", type=float, default=float(env("FAULT_STALL_SECONDS", "15"))
)
parser.add_argument("--recovery-timeout", type=float, default=120)
parser.add_argument(
"--backoff-base", type=float, default=float(env("FAULT_BACKOFF_BASE_SECONDS", "1"))
)
parser.add_argument("--tight-timeout", type=float, default=0.5, help="render seconds")
parser.add_argument("--tight-memory-mb", type=int, default=200, help="render address space")
parser.add_argument("--images", default=str(ROOT / "build/lab5-images"))
parser.add_argument("--heavy", action="store_true")
parser.add_argument("--output", default="")
args = parser.parse_args()
if args.expect_variants is None:
args.expect_variants = "thumbnail" if args.level == 3 else "thumbnail,crop,optimized"
return args
def main():
suite = Suite(arguments())
if suite.args.list:
for name, level, window, _ in suite.scenarios():
print(f"{name:32} level {level} {window}")
return 0
return suite.main()
if __name__ == "__main__":
sys.exit(main())