Source: immutable lab5 snapshot f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071; forward the allowed diff from 64d5c67. Carry SQLAlchemy, Alembic, reliability fixes, versioned jobs, infrastructure and student tests without changing protected coursework files.
Validation: make check, validate, lint; 515 student tests passed with one Linux-only skip; public smoke with telemetry enabled and repeat migration passed. Stop the owned stack and retain volumes. Preserve the existing OTel work in progress; grade implementation remains a separate phase.
1139 lines
50 KiB
Python
1139 lines
50 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shlex
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
|
|
import make_images
|
|
from client import Client, Failure, check
|
|
from image_probe import jpeg_size
|
|
|
|
FIXTURE = ROOT / "tests/fixtures/sample.png"
|
|
HEAVY = "png-at-pixel-limit-8000x5000.png"
|
|
EXCHANGE, ROUTING_KEY = "attachments", "attachment.process.v1"
|
|
QUEUE, DLQ = "attachments.process", "attachments.process.dlq"
|
|
RABBITMQADMIN = (
|
|
'RABBITMQADMIN_USERNAME=messenger-ops RABBITMQADMIN_PASSWORD="$(cat /run/secrets/'
|
|
'rabbitmq_ops_password)" exec rabbitmqadmin --non-interactive -V messenger "$@"'
|
|
)
|
|
PSQL = 'exec psql -X -q -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"'
|
|
ERRORS = (Failure, OSError, ValueError, KeyError, subprocess.TimeoutExpired)
|
|
SCENARIOS = (
|
|
("smoke_pipeline", 3, "upload, ready, variants match metadata"),
|
|
("worker_down_broker_restart", 3, "worker down, API and broker restart"),
|
|
("duplicate_after_ready", 3, "redelivery of a finished job"),
|
|
("poison_image", 3, "permanent error without retries"),
|
|
("s3_outage_recovers", 3, "transient error with retry"),
|
|
("retry_exhausted", 3, "transient error until JOB_MAX_ATTEMPTS"),
|
|
("image_matrix", 4, "limits and variants"),
|
|
("crash_commit_publish", 4, "after COMMIT, before publish"),
|
|
("crash_publish_confirm", 4, "after publish, before confirm"),
|
|
("crash_after_s3_before_commit", 4, "after S3 write, before COMMIT"),
|
|
("crash_after_commit_before_ack", 4, "after COMMIT, before ack"),
|
|
("lease_expiry_fencing", 4, "stalled worker, lease expiry"),
|
|
("replay_failed", 4, "operator replay"),
|
|
("malformed_message_dead_lettered", 4, "message that is not a job goes to the DLQ"),
|
|
("worker_limits", 4, "render time and memory limits, 40 Mpx in the worker limit"),
|
|
("retry_backoff_jitter", 4, "backoff with equal jitter between attempts"),
|
|
("failed_retention", 4, "failed staging kept until retention, then replay expires"),
|
|
("crash_before_side_effect", 5, "after claim, before side effects"),
|
|
("worker_kill_mid_task", 5, "SIGKILL during processing"),
|
|
("duplicate_in_flight", 5, "two copies of one job in flight"),
|
|
("reorder_attempts", 5, "attempt 2 delivered before attempt 1"),
|
|
("reorder_versions", 5, "v2 delivered before v1"),
|
|
("stale_after_new_version", 5, "v1 job after v2 is published"),
|
|
)
|
|
QUERIES = """
|
|
attachment:
|
|
SELECT status, published_version, error_code, staging_key FROM attachments WHERE id = :'id'
|
|
|
|
jobs:
|
|
SELECT job_id, pipeline_version, status, attempt FROM jobs
|
|
WHERE attachment_id = :'id' ORDER BY pipeline_version
|
|
|
|
outbox:
|
|
SELECT o.payload, o.published_at IS NOT NULL AS published FROM outbox o JOIN jobs j USING (job_id)
|
|
WHERE j.attachment_id = :'id' ORDER BY o.id
|
|
|
|
variants:
|
|
SELECT pipeline_version, name, sha256, size_bytes, object_key FROM attachment_variants
|
|
WHERE attachment_id = :'id' ORDER BY pipeline_version, name
|
|
|
|
events:
|
|
SELECT kind, attempt FROM job_events WHERE attachment_id = :'id' ORDER BY id
|
|
|
|
backoff:
|
|
SELECT attempt, (detail->>'delay_seconds')::float8 AS delay FROM job_events
|
|
WHERE attachment_id = :'id' AND kind = 'retry_scheduled' ORDER BY id
|
|
|
|
nonterminal:
|
|
SELECT id, status FROM attachments
|
|
WHERE id = ANY(:'ids'::uuid[]) AND status NOT IN ('ready', 'failed')
|
|
|
|
known:
|
|
SELECT id FROM attachments WHERE id = ANY(:'ids'::uuid[])
|
|
|
|
duplicate_effects:
|
|
SELECT attachment_id, pipeline_version, name, count(*) AS copies FROM attachment_variants
|
|
WHERE attachment_id = ANY(:'ids'::uuid[]) GROUP BY 1, 2, 3 HAVING count(*) > 1
|
|
|
|
attempts:
|
|
SELECT job_id, attempt FROM jobs WHERE attachment_id = ANY(:'ids'::uuid[]) AND attempt > :max
|
|
|
|
pending_outbox:
|
|
SELECT o.job_id FROM outbox o JOIN jobs j USING (job_id)
|
|
WHERE j.attachment_id = ANY(:'ids'::uuid[]) AND o.published_at IS NULL
|
|
|
|
unpublished_version:
|
|
SELECT a.id, a.published_version, max(j.pipeline_version) AS done FROM attachments a
|
|
JOIN jobs j ON j.attachment_id = a.id AND j.status = 'done' WHERE a.id = ANY(:'ids'::uuid[])
|
|
GROUP BY a.id, a.published_version HAVING a.published_version <> max(j.pipeline_version)
|
|
|
|
staging_left:
|
|
SELECT id FROM attachments
|
|
WHERE id = ANY(:'ids'::uuid[]) AND status = 'ready' AND staging_key IS NOT NULL
|
|
|
|
object_keys:
|
|
SELECT object_key AS key FROM attachment_variants WHERE attachment_id = ANY(:'ids'::uuid[])
|
|
UNION ALL SELECT original_key FROM attachments
|
|
WHERE id = ANY(:'ids'::uuid[]) AND original_key IS NOT NULL
|
|
UNION ALL SELECT staging_key FROM attachments
|
|
WHERE id = ANY(:'ids'::uuid[]) AND staging_key IS NOT NULL
|
|
|
|
columns:
|
|
SELECT table_name, column_name FROM information_schema.columns
|
|
WHERE table_schema = current_schema()
|
|
"""
|
|
SQL = {
|
|
name.strip(): " ".join(body.split())
|
|
for name, body in (block.split(":\n", 1) for block in QUERIES.strip().split("\n\n"))
|
|
}
|
|
REQUIRED_COLUMNS = {
|
|
"attachments": "id status published_version error_code staging_key original_key",
|
|
"attachment_variants": "attachment_id pipeline_version name sha256 size_bytes object_key",
|
|
"jobs": "job_id attachment_id pipeline_version status attempt",
|
|
"outbox": "job_id payload published_at",
|
|
"job_events": "job_id attachment_id kind attempt",
|
|
}
|
|
|
|
|
|
def now():
|
|
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
|
|
|
|
|
def run(cmd, stdin=None, env=None, timeout=180):
|
|
done = subprocess.run(
|
|
cmd, input=stdin, capture_output=True, text=True, env=env, timeout=timeout, check=False
|
|
)
|
|
if done.returncode:
|
|
raise Failure(f"{' '.join(cmd[:7])}: exit {done.returncode}: {done.stderr.strip()[-400:]}")
|
|
return done.stdout
|
|
|
|
|
|
def poll(probe, timeout, what, interval=0.5):
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
value = probe()
|
|
if value:
|
|
return value
|
|
if time.monotonic() > deadline:
|
|
raise Failure(f"timeout {timeout}s waiting for {what}")
|
|
time.sleep(interval)
|
|
|
|
|
|
def http_ready(base_url):
|
|
try:
|
|
with urllib.request.urlopen(base_url.rstrip("/") + "/health/ready", timeout=3) as response:
|
|
return response.status == 200
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def image_size(data, content_type):
|
|
if content_type == "image/jpeg":
|
|
return jpeg_size(data)
|
|
if content_type == "image/png":
|
|
return struct.unpack(">II", data[16:24])
|
|
check(data[:4] == b"RIFF" and data[8:12] == b"WEBP", "optimized is not WebP")
|
|
if data[12:16] == b"VP8 ":
|
|
width, height = struct.unpack("<HH", data[26:30])
|
|
return width & 0x3FFF, height & 0x3FFF
|
|
if data[12:16] == b"VP8L":
|
|
bits = int.from_bytes(data[21:25], "little")
|
|
return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1
|
|
return int.from_bytes(data[24:27], "little") + 1, int.from_bytes(data[27:30], "little") + 1
|
|
|
|
|
|
class Stack:
|
|
def __init__(self, args):
|
|
self.args = args
|
|
self.base = shlex.split(args.compose)
|
|
self.faults = shlex.split(args.compose_faults)
|
|
|
|
def dc(self, *argv, faults=False, stdin=None, env=None, timeout=180):
|
|
command = [*(self.faults if faults else self.base), *argv]
|
|
return run(command, stdin, dict(os.environ, **(env or {})), timeout)
|
|
|
|
def up(self, *services, faults=False, env=None):
|
|
wait = ["up", "-d", "--no-deps", "--no-recreate", "--wait", "--wait-timeout", "90"]
|
|
if faults or env:
|
|
wait.remove("--no-recreate")
|
|
self.dc(*wait, *services, faults=faults, env=env)
|
|
|
|
def launch(self, service, point="", **overrides):
|
|
variable = "WORKER_FAULT_POINT" if service == self.args.worker else "RELAY_FAULT_POINT"
|
|
self.up(service, faults=True, env={variable: point, **overrides})
|
|
|
|
def cgroup(self, service, name):
|
|
return self.dc("exec", "-T", service, "cat", "/sys/fs/cgroup/" + name).strip()
|
|
|
|
def restore(self):
|
|
self.dc("up", "-d", "--no-deps", "--wait", self.args.relay, self.args.worker)
|
|
|
|
def stop(self, *services):
|
|
self.dc("stop", "-t", "10", *services)
|
|
|
|
def kill(self, service):
|
|
self.dc("kill", "-s", "KILL", service)
|
|
|
|
def restart(self, *services):
|
|
self.dc("restart", "-t", "20", *services)
|
|
self.up(*services)
|
|
|
|
def state(self, service):
|
|
text = self.dc("ps", "-a", "--format", "json", service).strip()
|
|
rows = (
|
|
json.loads(text) if text.startswith("[") else [json.loads(x) for x in text.split("\n")]
|
|
)
|
|
check(len(rows) == 1, "expected one container for " + service)
|
|
return rows[0]["State"], rows[0].get("ExitCode")
|
|
|
|
def wait_exit(self, service, timeout=60):
|
|
def probe():
|
|
state, code = self.state(service)
|
|
return (code,) if state in ("exited", "dead") else None
|
|
|
|
return poll(probe, timeout, service + " exit")[0]
|
|
|
|
def sql(self, name, **params):
|
|
query = f"SELECT coalesce(json_agg(q), '[]'::json) FROM ({SQL[name]}) q;\n"
|
|
variables = [x for key, value in params.items() for x in ("-v", f"{key}={value}")]
|
|
out = self.dc(
|
|
"exec", "-T", self.args.postgres, "sh", "-c", PSQL, "sh", *variables, stdin=query
|
|
)
|
|
return json.loads(out)
|
|
|
|
def queues(self):
|
|
columns = "name messages_ready messages_unacknowledged --formatter json"
|
|
listing = f"exec -T {self.args.broker} rabbitmqctl -q list_queues -p messenger {columns}"
|
|
rows = json.loads(self.dc(*listing.split()))
|
|
return {q["name"]: (q["messages_ready"], q["messages_unacknowledged"]) for q in rows}
|
|
|
|
def rabbitmqadmin(self, *argv, stdin=None):
|
|
return self.dc(
|
|
"exec", "-T", self.args.broker, "sh", "-c", RABBITMQADMIN, "sh", *argv, stdin=stdin
|
|
)
|
|
|
|
def publish(self, payload):
|
|
properties = {"delivery_mode": 2, "content_type": "application/json", "type": ROUTING_KEY}
|
|
properties["message_id"] = payload["job_id"]
|
|
command = f"publish message -e {EXCHANGE} -k {ROUTING_KEY} -f - -p".split()
|
|
out = self.rabbitmqadmin(*command, json.dumps(properties), stdin=json.dumps(payload))
|
|
check("routed successfully" in out, "broker did not route the injected job")
|
|
|
|
def s3_json(self, *argv, missing=False):
|
|
done = subprocess.run(
|
|
[*self.base, "run", "--rm", "--no-deps", "-T", "s3-cli", *argv, "--json"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=60,
|
|
check=False,
|
|
)
|
|
body = json.loads(done.stdout or done.stderr)
|
|
if missing and done.returncode == 5 and body.get("details", {}).get("type") == "not_found":
|
|
return None
|
|
check(done.returncode == 0, "S3 inventory command failed: exit " + str(done.returncode))
|
|
return body
|
|
|
|
def objects(self, ids):
|
|
bucket = self.dc("exec", "-T", self.args.relay, "printenv", "S3_BUCKET").strip()
|
|
keys = set()
|
|
for ident in ids:
|
|
for prefix in ("staging", "originals"):
|
|
body = self.s3_json(
|
|
"object", "stat", f"root/{bucket}/{prefix}/{ident}", missing=True
|
|
)
|
|
if body is not None:
|
|
keys.add(body["name"])
|
|
body = self.s3_json("object", "list", "--recursive", f"root/{bucket}/variants/{ident}/")
|
|
check(not body["truncated"], "S3 variant inventory is truncated")
|
|
keys.update(item["key"] for item in body["items"])
|
|
return keys
|
|
|
|
def admin(self, *argv):
|
|
command = shlex.split(self.args.admin) + list(argv)
|
|
out = self.dc("run", "--rm", "--no-deps", "-T", self.args.worker, *command)
|
|
return json.loads(out.strip().splitlines()[-1])
|
|
|
|
|
|
class Api:
|
|
def __init__(self, args):
|
|
self.base_url = args.base_url
|
|
self.client = Client(args.base_url, args.ca_file)
|
|
self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
|
|
self.accepted, self.traces, self.final = [], {}, {}
|
|
|
|
def upload(self, data, kind="image/png", name="sample.png", expected=202, final=("ready",)):
|
|
path = f"/api/v1/chats/{self.chat['id']}/attachments?filename={quote(name)}"
|
|
allowed = [expected] if isinstance(expected, int) else expected
|
|
headers = {"Content-Type": kind}
|
|
response = self.client.call(
|
|
"POST", path, allowed, actor=self.alice, raw=data, headers=headers, schema="Attachment"
|
|
)
|
|
if response["status"] != 202:
|
|
return None
|
|
body = response["json"]
|
|
snapshot = (
|
|
body["status"] == "queued" and body["variants"] == [] and body["error_code"] is None
|
|
)
|
|
check(snapshot, "202 is not a queued snapshot")
|
|
same = body["sha256"] == hashlib.sha256(data).hexdigest() and body["size_bytes"] == len(
|
|
data
|
|
)
|
|
check(same, "202 checksum or size differs from the uploaded bytes")
|
|
self.accepted.append(body["id"])
|
|
self.final[body["id"]] = set(final)
|
|
self.traces[body["id"]] = ["queued"]
|
|
return body
|
|
|
|
def fixture(self, **kwargs):
|
|
return self.upload(FIXTURE.read_bytes(), **kwargs)
|
|
|
|
def get(self, ident):
|
|
path = "/api/v1/attachments/" + ident
|
|
body = self.client.call("GET", path, actor=self.bob, schema="Attachment")["json"]
|
|
self.traces.setdefault(ident, []).append(body["status"])
|
|
return body
|
|
|
|
def wait(self, ident, final=("ready",), timeout=90):
|
|
def probe():
|
|
body = self.get(ident)
|
|
failed = body["status"] == "failed" and "failed" not in final
|
|
check(not failed, f"attachment reached failed: {body['error_code']}")
|
|
return body if body["status"] in final else None
|
|
|
|
return poll(probe, timeout, f"{ident} {'/'.join(final)}", 0.3)
|
|
|
|
def content(self, ident, variant=None, expected=200):
|
|
path = f"/api/v1/attachments/{ident}/content" + (f"?variant={variant}" if variant else "")
|
|
return self.client.call("GET", path, expected, actor=self.bob)
|
|
|
|
def message(self, text, attachments=None, expected=201):
|
|
if expected == 201:
|
|
return self.client.message(self.alice, self.chat, text, attachments=attachments)["json"]
|
|
body = {"text": text, "attachment_ids": attachments or []}
|
|
headers = {"Idempotency-Key": os.urandom(8).hex()}
|
|
path = f"/api/v1/chats/{self.chat['id']}/messages"
|
|
return self.client.call(
|
|
"POST", path, expected, actor=self.alice, body=body, headers=headers
|
|
)
|
|
|
|
|
|
class Suite:
|
|
def __init__(self, args):
|
|
self.args = args
|
|
self.stack = Stack(args)
|
|
self.api = None
|
|
self.samples = {}
|
|
self.dead_letters = 0
|
|
|
|
def scenarios(self):
|
|
return [(name, level, window, getattr(self, name)) for name, level, window in SCENARIOS]
|
|
|
|
def drained(self, timeout=60):
|
|
poll(lambda: self.stack.queues()[QUEUE] == (0, 0), timeout, "main queue drained")
|
|
|
|
def queued(self, count, timeout=30):
|
|
poll(lambda: self.stack.queues()[QUEUE][0] >= count, timeout, f"{count} ready messages")
|
|
|
|
def record(self, ident):
|
|
row = self.stack.sql("attachment", id=ident)[0]
|
|
self.samples.setdefault(ident, []).append(row["published_version"])
|
|
self.api.traces.setdefault(ident, []).append(row["status"])
|
|
return row
|
|
|
|
def snapshot(self, ident):
|
|
body = self.api.get(ident)
|
|
public = sorted(body["variants"], key=lambda v: v["name"])
|
|
published = self.record(ident)["published_version"]
|
|
rows = self.stack.sql("variants", id=ident)
|
|
return {"status": body["status"], "public": public, "rows": rows, "published": published}
|
|
|
|
def payload(self, ident, version=1):
|
|
rows = self.stack.sql("outbox", id=ident)
|
|
found = [r["payload"] for r in rows if r["payload"]["pipeline_version"] == version]
|
|
check(bool(found), f"no outbox payload for v{version}")
|
|
return found[-1]
|
|
|
|
def verify(self, ready, expect=None):
|
|
names = sorted(v["name"] for v in ready["variants"])
|
|
check(len(names) == len(set(names)), "duplicate variant names in API")
|
|
missing = set(self.args.expect_variants.split(",")) - set(names)
|
|
check(not missing, f"missing variants {sorted(missing)}")
|
|
original = self.api.content(ready["id"])
|
|
data = original["bytes"]
|
|
same = (
|
|
hashlib.sha256(data).hexdigest() == ready["sha256"] and len(data) == ready["size_bytes"]
|
|
)
|
|
check(same, "original bytes differ from metadata")
|
|
check(
|
|
original["headers"].get_content_type() == ready["content_type"], "original type differs"
|
|
)
|
|
for variant in ready["variants"]:
|
|
name = variant["name"]
|
|
got = self.api.content(ready["id"], name)
|
|
data, kind = got["bytes"], got["headers"].get_content_type()
|
|
check(kind == variant["content_type"], name + " content type differs")
|
|
same = len(data) == variant["size_bytes"]
|
|
check(
|
|
same and hashlib.sha256(data).hexdigest() == variant["sha256"],
|
|
name + " bytes differ",
|
|
)
|
|
size = (variant["width"], variant["height"])
|
|
check(tuple(image_size(data, kind)) == size, name + " real size differs from metadata")
|
|
wanted = (expect or {}).get(name)
|
|
slack = 1 if name == "optimized" else 0
|
|
fits = not wanted or all(abs(a - b) <= slack for a, b in zip(wanted, size))
|
|
check(fits, f"{name} is {size}, expected {wanted}")
|
|
return names
|
|
|
|
def crash(self, service, point, observe):
|
|
self.drained()
|
|
self.stack.launch(service, point)
|
|
ident = self.api.fixture()["id"]
|
|
code = self.stack.wait_exit(service)
|
|
check(code == 137, f"{service} exited with {code}, expected 137 at {point}")
|
|
seen = observe(ident)
|
|
seen["containers"] = self.stack.dc(
|
|
"ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", service
|
|
).splitlines()
|
|
self.stack.launch(service)
|
|
ready = self.api.wait(ident, timeout=self.args.recovery_timeout)
|
|
poll(lambda: all(r["published"] for r in self.stack.sql("outbox", id=ident)), 30, "outbox")
|
|
self.drained()
|
|
self.verify(ready)
|
|
variants = self.stack.sql("variants", id=ident)
|
|
return {
|
|
"attachment_id": ident,
|
|
"exit_code": code,
|
|
"after_crash": seen,
|
|
"variants": variants,
|
|
}
|
|
|
|
def smoke_pipeline(self):
|
|
att = self.api.fixture()
|
|
ready = self.api.wait(att["id"])
|
|
return {
|
|
"attachment_id": att["id"],
|
|
"variants": self.verify(ready, {"thumbnail": [256, 160]}),
|
|
}
|
|
|
|
def worker_down_broker_restart(self):
|
|
steps = []
|
|
|
|
def mark(step, **extra):
|
|
steps.append({"at": now(), "step": step, **extra})
|
|
|
|
self.stack.stop(self.args.worker)
|
|
mark(
|
|
"worker stopped",
|
|
containers=self.stack.dc(
|
|
"ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", self.args.worker
|
|
).splitlines(),
|
|
)
|
|
att = self.api.fixture()
|
|
mark("upload 202 queued", attachment_id=att["id"])
|
|
self.api.content(att["id"], expected=409)
|
|
mark("content 409 while queued")
|
|
message = self.api.message("", [att["id"]])
|
|
self.api.message("text messages still work")
|
|
path = f"/api/v1/chats/{self.api.chat['id']}/messages?limit=5"
|
|
self.api.client.call("GET", path, actor=self.api.bob, schema="MessagePage")
|
|
mark("message with attachment 201, other messages work", message_id=message["id"])
|
|
self.queued(1)
|
|
mark("job is in the broker queue", queues=self.stack.queues())
|
|
self.stack.restart(*self.args.api, self.args.relay)
|
|
poll(lambda: http_ready(self.api.base_url), 90, "API ready")
|
|
mark("API replicas and relay restarted")
|
|
self.stack.restart(self.args.broker)
|
|
self.queued(1)
|
|
mark("broker restarted with its volume, job survived", queues=self.stack.queues())
|
|
self.stack.launch(self.args.worker)
|
|
mark("worker started")
|
|
ready = self.api.wait(att["id"])
|
|
mark("ready", variants=self.verify(ready))
|
|
return {"attachment_id": att["id"], "timeline": steps}
|
|
|
|
def duplicate_after_ready(self):
|
|
ident = self.api.fixture()["id"]
|
|
self.api.wait(ident)
|
|
self.drained()
|
|
before = self.snapshot(ident)
|
|
self.stack.publish(self.payload(ident))
|
|
self.drained()
|
|
time.sleep(1)
|
|
check(self.snapshot(ident) == before, "duplicate delivery changed the published result")
|
|
return {"attachment_id": ident, "events": self.stack.sql("events", id=ident)}
|
|
|
|
def poison_image(self):
|
|
data = make_images.png(320, 200, make_images.gradient(320, 200, 3))
|
|
broken = data[: data.index(b"IDAT") + 600]
|
|
att = self.api.upload(broken, expected=[202, 415], final=("failed",))
|
|
if att is None:
|
|
return {"rejected_by_api": 415}
|
|
failed = self.api.wait(att["id"], ("failed",))
|
|
check(
|
|
failed["error_code"] == "invalid_image", f"unexpected error_code {failed['error_code']}"
|
|
)
|
|
jobs = self.stack.sql("jobs", id=att["id"])
|
|
check(all(j["attempt"] == 1 for j in jobs), "permanent error was retried")
|
|
self.api.content(att["id"], expected=409)
|
|
self.api.message("", [att["id"]], expected=409)
|
|
return {"attachment_id": att["id"], "error_code": failed["error_code"], "jobs": jobs}
|
|
|
|
def s3_outage_recovers(self):
|
|
self.stack.stop(self.args.worker)
|
|
ident = self.api.fixture()["id"]
|
|
self.queued(1)
|
|
self.stack.stop(self.args.s3)
|
|
try:
|
|
self.stack.launch(self.args.worker)
|
|
poll(
|
|
lambda: any(j["attempt"] > 1 for j in self.stack.sql("jobs", id=ident)), 60, "retry"
|
|
)
|
|
status = self.record(ident)["status"]
|
|
check(status in ("queued", "processing"), "transient error produced " + status)
|
|
finally:
|
|
self.stack.up(self.args.s3)
|
|
poll(lambda: http_ready(self.api.base_url), 90, "API ready")
|
|
self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout))
|
|
return {"attachment_id": ident, "jobs": self.stack.sql("jobs", id=ident)}
|
|
|
|
def exhaust(self):
|
|
self.drained()
|
|
self.stack.launch(self.args.worker, "transient_error")
|
|
ident = self.api.fixture(final=("failed", "ready"))["id"]
|
|
failed = self.api.wait(ident, ("failed",), timeout=self.args.recovery_timeout)
|
|
self.stack.launch(self.args.worker)
|
|
jobs = self.stack.sql("jobs", id=ident)
|
|
check(failed["error_code"] == "retry_exhausted", f"unexpected {failed['error_code']}")
|
|
check(max(j["attempt"] for j in jobs) == self.args.max_attempts, "attempts != max")
|
|
return ident, jobs
|
|
|
|
def retry_exhausted(self):
|
|
ident, jobs = self.exhaust()
|
|
self.api.final[ident] = {"failed"}
|
|
self.api.content(ident, expected=409)
|
|
self.api.message("", [ident], expected=409)
|
|
return {"attachment_id": ident, "jobs": jobs}
|
|
|
|
def replay_failed(self):
|
|
ident, _ = self.exhaust()
|
|
self.api.final[ident] = {"ready"}
|
|
first = self.stack.admin("replay", "--attachment-id", ident)
|
|
self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout))
|
|
self.drained()
|
|
before = self.snapshot(ident)
|
|
second = self.stack.admin("replay", "--attachment-id", ident)
|
|
self.drained()
|
|
check(self.snapshot(ident) == before, "replay of a ready attachment changed the result")
|
|
events = self.stack.sql("events", id=ident)
|
|
check(any("replay" in e["kind"] for e in events), "replay left no audit event")
|
|
return {
|
|
"attachment_id": ident,
|
|
"replay": first,
|
|
"replay_of_ready": second,
|
|
"events": events,
|
|
}
|
|
|
|
def malformed_message_dead_lettered(self):
|
|
before = self.stack.queues()[DLQ][0]
|
|
self.stack.publish({"schema_version": 2, "job_id": str(uuid.uuid4())})
|
|
poll(lambda: self.stack.queues()[DLQ][0] > before, 30, "dead letter")
|
|
self.dead_letters += 1
|
|
return {"dead_letters": self.stack.queues()[DLQ][0] - before}
|
|
|
|
def limited(self, data, **overrides):
|
|
self.stack.launch(self.args.worker, **overrides)
|
|
att = self.api.upload(data, name=HEAVY, final=("failed",))
|
|
failed = self.api.wait(att["id"], ("failed",), self.args.recovery_timeout)
|
|
check(failed["error_code"] == "processing_failed", f"limit gave {failed['error_code']}")
|
|
jobs = self.stack.sql("jobs", id=att["id"])
|
|
check([j["attempt"] for j in jobs] == [1], "a limit breach was retried")
|
|
state, _ = self.stack.state(self.args.worker)
|
|
check(state == "running", "worker died on a limit breach: " + state)
|
|
return {"attachment_id": att["id"], "error_code": failed["error_code"], "worker": state}
|
|
|
|
def worker_limits(self):
|
|
folder = Path(self.args.images) / "limits"
|
|
make_images.build(folder, True, {HEAVY})
|
|
data = (folder / HEAVY).read_bytes()
|
|
self.drained()
|
|
timed = self.limited(data, FAULT_RENDER_TIMEOUT_SECONDS=str(self.args.tight_timeout))
|
|
memory = self.limited(data, FAULT_RENDER_MEMORY_MB=str(self.args.tight_memory_mb))
|
|
self.stack.launch(self.args.worker)
|
|
att = self.api.upload(data, name=HEAVY)
|
|
ready = self.api.wait(att["id"], timeout=self.args.recovery_timeout)
|
|
sizes = {"thumbnail": [256, 160], "crop": [128, 128], "optimized": [1280, 800]}
|
|
names = self.verify(ready, sizes)
|
|
peak = int(self.stack.cgroup(self.args.worker, "memory.peak"))
|
|
limit = int(self.stack.cgroup(self.args.worker, "memory.max"))
|
|
check(peak < limit, f"worker peak {peak} reached its limit {limit}")
|
|
return {
|
|
"image": {"file": HEAVY, "bytes": len(data), "pixels": 8000 * 5000},
|
|
"render_timeout": {"seconds": self.args.tight_timeout, **timed},
|
|
"render_memory": {"mib": self.args.tight_memory_mb, **memory},
|
|
"default_limits": {
|
|
"attachment_id": att["id"],
|
|
"variants": names,
|
|
"memory_peak_mib": round(peak / 2**20, 1),
|
|
"memory_max_mib": round(limit / 2**20, 1),
|
|
},
|
|
}
|
|
|
|
def retry_backoff_jitter(self):
|
|
ident, jobs = self.exhaust()
|
|
self.api.final[ident] = {"failed"}
|
|
rows = self.stack.sql("backoff", id=ident)
|
|
check([r["attempt"] for r in rows] == [2, 3, 4, 5], f"unexpected retries {rows}")
|
|
shares = []
|
|
for row in rows:
|
|
ceiling = min(60.0, self.args.backoff_base * 2 ** (row["attempt"] - 2))
|
|
share = row["delay"] / ceiling
|
|
check(0.5 <= share <= 1.0, f"delay {row['delay']} outside [{ceiling / 2}, {ceiling}]")
|
|
shares.append(round(share, 3))
|
|
check(len(set(shares)) > 1, f"no jitter: shares {shares}")
|
|
return {"attachment_id": ident, "delays": rows, "share_of_ceiling": shares, "jobs": jobs}
|
|
|
|
def failed_retention(self):
|
|
ident, _ = self.exhaust()
|
|
self.api.final[ident] = {"failed"}
|
|
kept = self.record(ident)["staging_key"]
|
|
check(kept is not None, "failed staging was removed before the retention period")
|
|
retention = int(
|
|
self.stack.dc("exec", "-T", self.args.relay, "printenv", "STAGING_RETENTION_SECONDS")
|
|
)
|
|
aged = self.stack.dc(
|
|
"exec",
|
|
"-T",
|
|
self.args.postgres,
|
|
"sh",
|
|
"-c",
|
|
PSQL,
|
|
"sh",
|
|
"-v",
|
|
"id=" + ident,
|
|
"-v",
|
|
"chat=" + self.api.chat["id"],
|
|
"-v",
|
|
"age=" + str(retention + 1),
|
|
stdin="UPDATE attachments SET created_at = created_at - :age * interval '1 second', "
|
|
"updated_at = updated_at - :age * interval '1 second' "
|
|
"WHERE id = :'id' AND chat_id = :'chat' AND status = 'failed' RETURNING id;\n",
|
|
)
|
|
check(aged.strip() == ident, "did not age exactly the suite's failed attachment")
|
|
poll(lambda: self.record(ident)["staging_key"] is None, 30, "failed staging cleanup")
|
|
replay = self.stack.admin("replay", "--attachment-id", ident)
|
|
check(replay["result"] == "expired", f"replay after retention gave {replay}")
|
|
check(self.api.get(ident)["status"] == "failed", "expired replay changed the status")
|
|
return {
|
|
"attachment_id": ident,
|
|
"staging_before": "kept",
|
|
"retention_seconds": retention,
|
|
"aged_only": ident,
|
|
"replay": replay,
|
|
}
|
|
|
|
def image_matrix(self):
|
|
folder = Path(self.args.images)
|
|
results = []
|
|
for item in make_images.build(folder, self.args.heavy):
|
|
expect, name = item["expect"], item["file"]
|
|
ready = expect.get("final") == "ready"
|
|
data = (folder / name).read_bytes()
|
|
final = ("ready",) if ready else ("failed", "ready")
|
|
att = self.api.upload(data, item["content_type"], name, expect["upload"], final)
|
|
row = {"file": name, "http": 202 if att else "rejected"}
|
|
if att:
|
|
done = self.api.wait(att["id"], ("ready", "failed"), self.args.recovery_timeout)
|
|
row.update(status=done["status"], error_code=done["error_code"])
|
|
if ready:
|
|
check(done["status"] == "ready", name + " did not become ready")
|
|
row["variants"] = self.verify(done, expect["variants"])
|
|
else:
|
|
self.api.final[att["id"]] = {"failed"}
|
|
code = expect.get("error_code")
|
|
check(done["error_code"] == code, f"{name} should fail with {code}")
|
|
results.append(row)
|
|
return {"images": results}
|
|
|
|
def crash_commit_publish(self):
|
|
def observe(ident):
|
|
outbox = self.stack.sql("outbox", id=ident)
|
|
check(outbox and not any(r["published"] for r in outbox), "outbox already published")
|
|
check(self.stack.queues()[QUEUE] == (0, 0), "job reached the broker before publish")
|
|
return {"outbox_published": False, "status": self.record(ident)["status"]}
|
|
|
|
return self.crash(self.args.relay, "after_commit_before_publish", observe)
|
|
|
|
def crash_publish_confirm(self):
|
|
def observe(ident):
|
|
outbox = [r["published"] for r in self.stack.sql("outbox", id=ident)]
|
|
check(outbox == [False], "outbox row was marked before the crash point")
|
|
return {"outbox_published": outbox, "queues": self.stack.queues()}
|
|
|
|
result = self.crash(self.args.relay, "after_publish_before_confirm", observe)
|
|
kinds = [e["kind"] for e in self.stack.sql("events", id=result["attachment_id"])]
|
|
check("duplicate_noop" in kinds, "the republished copy was not a no-op: " + str(kinds))
|
|
return {**result, "events": kinds}
|
|
|
|
def crash_after_s3_before_commit(self):
|
|
def observe(ident):
|
|
check(not self.stack.sql("variants", id=ident), "variants committed before the crash")
|
|
return {"status": self.record(ident)["status"], "variant_rows": 0}
|
|
|
|
return self.crash(self.args.worker, "after_s3_write_before_commit", observe)
|
|
|
|
def crash_after_commit_before_ack(self):
|
|
def observe(ident):
|
|
row = self.record(ident)
|
|
check(row["status"] == "ready", "result was not committed before the crash point")
|
|
variants = self.stack.sql("variants", id=ident)
|
|
return {"status": row["status"], "variants": variants, "queues": self.stack.queues()}
|
|
|
|
result = self.crash(self.args.worker, "after_commit_before_ack", observe)
|
|
same = result["variants"] == result["after_crash"]["variants"]
|
|
check(same, "redelivery after COMMIT changed variants")
|
|
return result
|
|
|
|
def crash_before_side_effect(self):
|
|
def observe(ident):
|
|
return {
|
|
"status": self.record(ident)["status"],
|
|
"jobs": self.stack.sql("jobs", id=ident),
|
|
}
|
|
|
|
return self.crash(self.args.worker, "after_claim_before_side_effect", observe)
|
|
|
|
def lease_expiry_fencing(self):
|
|
self.drained()
|
|
self.stack.launch(self.args.worker, "stall_after_claim")
|
|
ident = self.api.fixture()["id"]
|
|
self.verify(self.api.wait(ident, timeout=self.args.recovery_timeout))
|
|
before = self.snapshot(ident)
|
|
time.sleep(self.args.stall_seconds + 3)
|
|
after = self.snapshot(ident)
|
|
self.stack.launch(self.args.worker)
|
|
check(before == after, "stalled worker overwrote the result after its lease expired")
|
|
jobs, events = self.stack.sql("jobs", id=ident), self.stack.sql("events", id=ident)
|
|
check(any(e["kind"] == "fenced" for e in events), "stalled holder was not fenced")
|
|
return {"attachment_id": ident, "jobs": jobs, "events": events}
|
|
|
|
def worker_kill_mid_task(self):
|
|
self.drained()
|
|
self.stack.launch(self.args.worker, "stall_after_claim")
|
|
ident = self.api.fixture()["id"]
|
|
poll(lambda: self.record(ident)["status"] == "processing", 30, "processing")
|
|
self.stack.kill(self.args.worker)
|
|
code = self.stack.wait_exit(self.args.worker)
|
|
containers = self.stack.dc(
|
|
"ps", "-a", "--format", "{{.Name}} {{.ID}} {{.State}} {{.Status}}", self.args.worker
|
|
).splitlines()
|
|
self.stack.launch(self.args.worker)
|
|
ready = self.api.wait(ident, timeout=self.args.recovery_timeout)
|
|
self.drained()
|
|
self.verify(ready)
|
|
return {
|
|
"attachment_id": ident,
|
|
"exit_code": code,
|
|
"containers": containers,
|
|
"jobs": self.stack.sql("jobs", id=ident),
|
|
}
|
|
|
|
def duplicate_in_flight(self):
|
|
self.stack.stop(self.args.worker)
|
|
ident = self.api.fixture()["id"]
|
|
self.queued(1)
|
|
self.stack.publish(self.payload(ident))
|
|
self.queued(2)
|
|
self.stack.launch(self.args.worker)
|
|
ready = self.api.wait(ident)
|
|
self.drained()
|
|
self.verify(ready)
|
|
variants, events = self.stack.sql("variants", id=ident), self.stack.sql("events", id=ident)
|
|
return {"attachment_id": ident, "variants": variants, "events": events}
|
|
|
|
def reorder_attempts(self):
|
|
self.stack.stop(self.args.worker, self.args.relay)
|
|
ident = self.api.fixture()["id"]
|
|
original = self.payload(ident)
|
|
self.stack.publish({**original, "attempt": original["attempt"] + 1})
|
|
self.queued(1)
|
|
self.stack.launch(self.args.relay)
|
|
self.queued(2)
|
|
self.stack.launch(self.args.worker)
|
|
ready = self.api.wait(ident)
|
|
self.drained()
|
|
self.verify(ready)
|
|
jobs, events = self.stack.sql("jobs", id=ident), self.stack.sql("events", id=ident)
|
|
return {"attachment_id": ident, "jobs": jobs, "events": events}
|
|
|
|
def published(self, ident, version):
|
|
poll(
|
|
lambda: self.record(ident)["published_version"] >= version,
|
|
self.args.recovery_timeout,
|
|
f"published v{version}",
|
|
)
|
|
|
|
def reprocess(self, ident):
|
|
return self.stack.admin("reprocess", "--attachment-id", ident, "--pipeline-version", "2")
|
|
|
|
def reorder_versions(self):
|
|
self.stack.stop(self.args.worker, self.args.relay)
|
|
ident = self.api.fixture()["id"]
|
|
check(self.reprocess(ident)["result"] == "queued", "version 2 was not scheduled")
|
|
check(self.reprocess(ident)["result"] == "noop", "duplicate reprocess was not a noop")
|
|
self.stack.publish(self.payload(ident, 2))
|
|
self.queued(1)
|
|
self.stack.launch(self.args.worker)
|
|
self.published(ident, 2)
|
|
before = self.snapshot(ident)
|
|
self.stack.launch(self.args.relay)
|
|
poll(
|
|
lambda: (
|
|
{j["pipeline_version"]: j["status"] for j in self.stack.sql("jobs", id=ident)}
|
|
== {1: "superseded", 2: "done"}
|
|
),
|
|
self.args.recovery_timeout,
|
|
"old version superseded",
|
|
)
|
|
poll(lambda: all(r["published"] for r in self.stack.sql("outbox", id=ident)), 30, "outbox")
|
|
self.drained()
|
|
final = self.snapshot(ident)
|
|
check(final["published"] == 2, f"published_version rolled back to {final['published']}")
|
|
check(before["public"] == final["public"], "late v1 changed the published v2 variants")
|
|
served = {v["name"]: v["sha256"] for v in final["public"]}
|
|
stored = {r["name"]: r["sha256"] for r in final["rows"] if r["pipeline_version"] == 2}
|
|
check(served == stored, "API serves variants of a version that is not published")
|
|
jobs = self.stack.sql("jobs", id=ident)
|
|
check(
|
|
{j["pipeline_version"]: j["status"] for j in jobs} == {1: "superseded", 2: "done"},
|
|
"v1 did not finish after v2",
|
|
)
|
|
self.verify(self.api.get(ident))
|
|
return {"attachment_id": ident, "jobs": jobs, "v2_before_v1": before, "after_v1": final}
|
|
|
|
def stale_after_new_version(self):
|
|
ident = self.api.fixture()["id"]
|
|
self.api.wait(ident)
|
|
first = self.snapshot(ident)
|
|
check(self.reprocess(ident)["result"] == "queued", "version 2 was not scheduled")
|
|
check(self.api.get(ident)["status"] == "ready", "reprocess rolled back ready")
|
|
self.published(ident, 2)
|
|
self.drained()
|
|
before = self.snapshot(ident)
|
|
check(first["public"] != before["public"], "recipes did not produce distinct results")
|
|
self.stack.publish(self.payload(ident, 1))
|
|
self.drained()
|
|
time.sleep(1)
|
|
after = self.snapshot(ident)
|
|
check(before == after and after["published"] == 2, "stale v1 job changed the v2 result")
|
|
return {
|
|
"attachment_id": ident,
|
|
"published_version": after["published"],
|
|
"v1": first,
|
|
"v2_before_stale": before,
|
|
"v2_after_stale": after,
|
|
}
|
|
|
|
def invariants(self, dead_before):
|
|
ids = self.api.accepted
|
|
array = "{" + ",".join(ids) + "}"
|
|
if not ids:
|
|
return [{"id": "accepted_work", "result": "FAIL", "details": "no upload got 202"}]
|
|
|
|
def empty(name, message, **params):
|
|
rows = self.stack.sql(name, ids=array, **params)
|
|
check(not rows, f"{message}: {rows[:5]}")
|
|
return {"rows": 0}
|
|
|
|
def terminal():
|
|
empty("nonterminal", "non-terminal attachments left")
|
|
wrong = [i for i in ids if self.api.get(i)["status"] not in self.api.final[i]]
|
|
check(not wrong, "unexpected final status: " + ", ".join(wrong))
|
|
return {"accepted": len(ids)}
|
|
|
|
def no_rollback():
|
|
traces = self.api.traces.items()
|
|
broken = [
|
|
i for i, t in traces if "ready" in t and set(t[t.index("ready") :]) != {"ready"}
|
|
]
|
|
check(not broken, "ready rolled back: " + ", ".join(broken))
|
|
return {"traced": len(self.api.traces)}
|
|
|
|
def monotonic():
|
|
broken = [i for i, values in self.samples.items() if values != sorted(values)]
|
|
check(not broken, "published_version went backwards: " + ", ".join(broken))
|
|
empty("unpublished_version", "published_version differs from the newest done job")
|
|
return {"sampled": len(self.samples)}
|
|
|
|
def served():
|
|
bodies = [self.api.get(i) for i in ids]
|
|
ready = [b for b in bodies if b["status"] == "ready"]
|
|
for body in ready:
|
|
self.verify(body)
|
|
return {"verified": len(ready)}
|
|
|
|
def orphans():
|
|
empty("pending_outbox", "unpublished outbox rows")
|
|
empty("staging_left", "staging kept after ready")
|
|
queues = self.stack.queues()
|
|
check(queues[QUEUE] == (0, 0), "messages left in " + QUEUE)
|
|
expected = dead_before + self.dead_letters
|
|
check(queues[DLQ][0] == expected, f"unexpected dead letters: {queues[DLQ]}")
|
|
keys = {r["key"] for r in self.stack.sql("object_keys", ids=array)}
|
|
if self.args.s3_list:
|
|
listing = {
|
|
x.split()[-1]
|
|
for x in run(shlex.split(self.args.s3_list)).splitlines()
|
|
if x.strip()
|
|
}
|
|
listing = {key for key in listing if any("/" + ident in key for ident in ids)}
|
|
else:
|
|
listing = self.stack.objects(ids)
|
|
check(
|
|
listing == keys,
|
|
f"S3 mismatch: extra={sorted(listing - keys)}, missing={sorted(keys - listing)}",
|
|
)
|
|
return {"queues": queues, "s3_objects": len(listing), "scoped_attachments": len(ids)}
|
|
|
|
def no_loss():
|
|
known = {r["id"] for r in self.stack.sql("known", ids=array)}
|
|
check(known == set(ids), "accepted attachments are missing in PostgreSQL")
|
|
return {"accepted": len(ids)}
|
|
|
|
claims = [
|
|
("terminal", "every 202 reaches its expected terminal state", terminal),
|
|
(
|
|
"single_effect",
|
|
"one variant set per (attachment_id, pipeline_version, variant)",
|
|
lambda: empty("duplicate_effects", "duplicate effect rows"),
|
|
),
|
|
("no_ready_rollback", "ready never returns to queued or processing", no_rollback),
|
|
("monotonic_publication", "published_version only moves forward", monotonic),
|
|
("bytes_match_metadata", "sha256, size and real dimensions match served bytes", served),
|
|
(
|
|
"bounded_attempts",
|
|
"attempt <= JOB_MAX_ATTEMPTS",
|
|
lambda: empty("attempts", "attempt above limit", max=self.args.max_attempts),
|
|
),
|
|
("no_orphans", "no pending outbox, messages, dead letters, staging, objects", orphans),
|
|
("no_lost_202", "every 202 is present in PostgreSQL and the API", no_loss),
|
|
]
|
|
return [self.claim(name, statement, probe) for name, statement, probe in claims]
|
|
|
|
@staticmethod
|
|
def claim(name, statement, probe):
|
|
try:
|
|
return {"id": name, "statement": statement, "result": "PASS", "details": probe()}
|
|
except ERRORS as error:
|
|
return {"id": name, "statement": statement, "result": "FAIL", "details": str(error)}
|
|
|
|
def preflight(self):
|
|
def services():
|
|
self.detect_api()
|
|
present = set(self.stack.dc("config", "--services").split())
|
|
wanted = {*self.args.api, self.args.relay, self.args.worker, self.args.broker}
|
|
wanted |= {self.args.postgres, self.args.s3}
|
|
check(wanted <= present, f"missing services {sorted(wanted - present)}")
|
|
return sorted(wanted)
|
|
|
|
def schema():
|
|
columns = {}
|
|
for row in self.stack.sql("columns"):
|
|
columns.setdefault(row["table_name"], set()).add(row["column_name"])
|
|
needed = {t: set(c.split()) for t, c in REQUIRED_COLUMNS.items()}
|
|
missing = {t: sorted(c - columns.get(t, set())) for t, c in needed.items()}
|
|
check(not any(missing.values()), f"missing columns {missing}")
|
|
return sorted(needed)
|
|
|
|
def broker():
|
|
queues = self.stack.queues()
|
|
check({QUEUE, DLQ} <= set(queues), "queues are not declared")
|
|
check(QUEUE in self.stack.rabbitmqadmin("list", "queues"), "rabbitmqadmin failed")
|
|
return queues
|
|
|
|
def faults():
|
|
check("FAULT_POINT" in self.stack.dc("config", faults=True), "no FAULT_POINT in faults")
|
|
return self.args.compose_faults
|
|
|
|
def api():
|
|
check(http_ready(self.args.base_url), "GET /health/ready is not 200")
|
|
return self.args.base_url
|
|
|
|
steps = [
|
|
("compose services", services),
|
|
("api ready", api),
|
|
("postgres schema", schema),
|
|
("broker topology and ops CLI", broker),
|
|
("faults override", faults),
|
|
]
|
|
return [self.claim(name, name, probe) for name, probe in steps]
|
|
|
|
def main(self):
|
|
report = {
|
|
"suite": "lab5-pipeline-idempotency",
|
|
"started_at": now(),
|
|
"source_sha": run(["git", "rev-parse", "HEAD"]).strip(),
|
|
"config": self.config(),
|
|
}
|
|
if self.args.preflight:
|
|
report["preflight"] = self.preflight()
|
|
return self.finish(report, report["preflight"])
|
|
results = []
|
|
try:
|
|
self.detect_api()
|
|
self.api = Api(self.args)
|
|
dead_before = self.stack.queues()[DLQ][0]
|
|
self.stack.launch(self.args.relay)
|
|
self.stack.launch(self.args.worker)
|
|
for name, level, window, scenario in self.scenarios():
|
|
if self.selected(name, level):
|
|
results.append(self.attempt(name, level, window, scenario))
|
|
report["invariants"] = self.invariants(dead_before)
|
|
except ERRORS as error:
|
|
report["error"] = str(error)
|
|
finally:
|
|
self.stack.up(self.args.s3)
|
|
self.stack.restore()
|
|
report["accepted_ids"] = self.api.accepted if self.api else []
|
|
report["scenarios"] = results
|
|
return self.finish(report, results + report.get("invariants", []))
|
|
|
|
def attempt(self, name, level, window, scenario):
|
|
began = time.monotonic()
|
|
print("RUN " + name, flush=True)
|
|
try:
|
|
outcome = {"result": "PASS", "observed": scenario()}
|
|
except ERRORS as error:
|
|
outcome = {"result": "FAIL", "error": str(error)}
|
|
self.heal()
|
|
print(f"{outcome['result']} {name}", flush=True)
|
|
seconds = round(time.monotonic() - began, 1)
|
|
return {"name": name, "level": level, "window": window, "seconds": seconds, **outcome}
|
|
|
|
def heal(self):
|
|
try:
|
|
self.stack.up(self.args.s3, self.args.broker)
|
|
self.stack.launch(self.args.relay)
|
|
self.stack.launch(self.args.worker)
|
|
except ERRORS as error:
|
|
print("heal failed: " + str(error), flush=True)
|
|
|
|
def detect_api(self):
|
|
if not self.args.api:
|
|
names = self.stack.dc("config", "--services").split()
|
|
api = [x for x in names if x == "api" or (x[:4] == "api-" and x[4:].isdigit())]
|
|
self.args.api = api
|
|
check(bool(self.args.api), "no API service: expected api or api-N, or pass --api")
|
|
|
|
def selected(self, name, level):
|
|
return name in self.args.only.split(",") if self.args.only else level <= self.args.level
|
|
|
|
def config(self):
|
|
keys = (
|
|
"base_url level only expect_variants max_attempts stall_seconds compose compose_faults"
|
|
)
|
|
return {key: getattr(self.args, key) for key in keys.split()}
|
|
|
|
def finish(self, report, verdicts):
|
|
passed = bool(verdicts) and "error" not in report
|
|
report["result"] = (
|
|
"PASS" if passed and all(v["result"] == "PASS" for v in verdicts) else "FAIL"
|
|
)
|
|
report["finished_at"] = now()
|
|
text = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
|
if self.args.output:
|
|
Path(self.args.output).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(self.args.output).write_text(text)
|
|
else:
|
|
print(text)
|
|
print(f"{report['result']}: lab5 pipeline suite, report {self.args.output or 'stdout'}")
|
|
return 0 if report["result"] == "PASS" else 1
|
|
|
|
|
|
def arguments():
|
|
env = os.environ.get
|
|
faults = "docker compose -f compose.yaml -f infra/compose.faults.yaml"
|
|
parser = argparse.ArgumentParser(description="Lab5 pipeline fault injection with invariants")
|
|
parser.add_argument("--base-url", default=env("BASE_URL", "http://localhost:8080"))
|
|
parser.add_argument("--ca-file", default=env("CA_FILE"))
|
|
parser.add_argument("--compose", default=env("COMPOSE", "docker compose"))
|
|
parser.add_argument("--compose-faults", default=env("COMPOSE_FAULTS", faults))
|
|
parser.add_argument("--admin", default=env("ADMIN_CLI", "python -m messenger.admin"))
|
|
parser.add_argument("--api", nargs="+", default=env("API_SERVICES", "").split())
|
|
parser.add_argument("--worker", default="worker")
|
|
parser.add_argument("--relay", default="relay")
|
|
parser.add_argument("--broker", default="broker")
|
|
parser.add_argument("--postgres", default="postgres")
|
|
parser.add_argument("--s3", default="s3")
|
|
parser.add_argument("--s3-list", default=env("S3_LIST_CMD", ""), help="prints object keys")
|
|
parser.add_argument("--level", type=int, choices=[3, 4, 5], default=5)
|
|
parser.add_argument("--only", default="", help="comma separated scenario names")
|
|
parser.add_argument("--list", action="store_true")
|
|
parser.add_argument("--preflight", action="store_true")
|
|
parser.add_argument("--expect-variants", default=None)
|
|
parser.add_argument("--max-attempts", type=int, default=int(env("JOB_MAX_ATTEMPTS", "5")))
|
|
parser.add_argument(
|
|
"--stall-seconds", type=float, default=float(env("FAULT_STALL_SECONDS", "15"))
|
|
)
|
|
parser.add_argument("--recovery-timeout", type=float, default=120)
|
|
parser.add_argument(
|
|
"--backoff-base", type=float, default=float(env("FAULT_BACKOFF_BASE_SECONDS", "1"))
|
|
)
|
|
parser.add_argument("--tight-timeout", type=float, default=0.5, help="render seconds")
|
|
parser.add_argument("--tight-memory-mb", type=int, default=200, help="render address space")
|
|
parser.add_argument("--images", default=str(ROOT / "build/lab5-images"))
|
|
parser.add_argument("--heavy", action="store_true")
|
|
parser.add_argument("--output", default="")
|
|
args = parser.parse_args()
|
|
if args.expect_variants is None:
|
|
args.expect_variants = "thumbnail" if args.level == 3 else "thumbnail,crop,optimized"
|
|
return args
|
|
|
|
|
|
def main():
|
|
suite = Suite(arguments())
|
|
if suite.args.list:
|
|
for name, level, window, _ in suite.scenarios():
|
|
print(f"{name:32} level {level} {window}")
|
|
return 0
|
|
return suite.main()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|