Files
mtusi2026lab5-fork/student-tests/pipeline_faults.py
T

321 lines
14 KiB
Python

"""Fault experiment on a dedicated lab5 Compose stand with synthetic data."""
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check
def run(*args, expected=0, timeout=120):
proc = subprocess.run(
args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=timeout,
)
if proc.returncode != expected:
raise Failure(f"{args[:5]} exit={proc.returncode}, expected={expected}")
return proc.stdout.strip()
def compose(*args, **kwargs):
return run("docker", "compose", "--ansi", "never", *args, **kwargs)
def sql(query):
return compose("exec", "-T", "db", "psql", "-XAt", "-v", "ON_ERROR_STOP=1",
"-U", "messenger", "-d", "messenger", "-c", query)
def broker(*args):
output = compose(
"exec", "-T", "broker", "/bin/sh", "-ec",
'export REDISCLI_AUTH="$(cat /run/secrets/broker_password)"; exec valkey-cli --json "$@"',
"sh", *args,
)
return json.loads(output)
def until(fn, timeout=90):
start = time.monotonic()
while time.monotonic() - start < timeout:
try:
result = fn()
except Failure:
result = None
if result:
return result
time.sleep(0.3)
raise Failure(f"condition timed out after {timeout}s")
def job(att, version=1):
return json.loads(sql(f"""
SELECT json_build_object('id',id,'state',state,'attempt',attempt,'error',error_code,
'published',published,'created',to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"'),
'updated',updated_at,'delay_ms',round(extract(epoch FROM (next_at-updated_at))*1000))
FROM image_jobs WHERE attachment_id='{att['id']}' AND version={version}
"""))
def carrier(att, version):
row = job(att, version)
return {
"schema_version": 1, "job_id": row["id"], "type": "attachment.process.v1",
"attachment_id": att["id"], "pipeline_version": version,
"attempt": row["attempt"], "created_at": row["created"],
}
def entries(att):
return [json.loads(fields[1]) for _, fields in broker("XRANGE", "attachment-jobs", "-", "+")
if json.loads(fields[1])["attachment_id"] == att["id"]]
def publish(payload):
return broker("XADD", "attachment-jobs", "*", "job", json.dumps(payload, separators=(",", ":")))
def s3stat(key):
setup = ('mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" '
'"$(cat /run/secrets/s3_root_password)" >/dev/null; '
'mc stat --json "local/messenger-attachments/$1"')
# mc reports an absent object in JSON on stdout.
args = ("docker", "compose", "--ansi", "never", "run", "--rm", "--no-deps",
"--entrypoint", "/bin/sh", "s3-init", "-ec", setup, "sh", key)
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=60)
result = json.loads(proc.stdout.strip().splitlines()[-1])
if proc.returncode:
check("does not exist" in json.dumps(result).lower(), "unexpected object-store error")
return None
return result
def fault(service, phase, att):
out = compose("run", "--rm", "--no-deps", "-e", f"LAB5_FAULT={phase}",
"-e", f"LAB5_FAULT_ATTACHMENT={att['id']}", service, expected=86)
check('"test crash"' in out and f'"phase":"{phase}"' in out, "fault marker missing")
def generations(*services):
ids = compose("ps", "-q", *services).splitlines()
return dict(line.split("|", 1) for line in run(
"docker", "inspect", "--format", "{{.Id}}|{{.State.StartedAt}}", *ids,
).splitlines())
class Experiment:
def __init__(self):
self.client = Client()
self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
self.results = []
def record(self, scenario, att, started, **data):
row = {"scenario": scenario, "attachment_id": att["id"],
"seconds": round(time.monotonic() - started, 3), **data}
self.results.append(row)
print(json.dumps(row, ensure_ascii=False), flush=True)
def metadata(self, att):
return self.client.call("GET", "/api/v1/attachments/" + att["id"],
actor=self.bob, schema="Attachment")["json"]
def content(self, att, variant, status=200, actor=None):
return self.client.call("GET", f"/api/v1/attachments/{att['id']}/content?variant={variant}",
status, actor=actor or self.bob)
def ready(self, att, raw):
result = self.client.wait_ready(self.bob, att, wait_seconds=155)
check(self.content(att, "original")["bytes"] == raw, "original changed")
variants = {v["name"]: v for v in result["variants"]}
check(len(result["variants"]) == 3 and set(variants) == {"thumbnail", "crop", "optimized"},
"expected exactly one of each variant")
for name, (width, height, mime) in {
"thumbnail": (256, 160, "image/jpeg"), "crop": (128, 128, "image/jpeg"),
"optimized": (320, 200, "image/webp"),
}.items():
variant = variants[name]
download = self.content(att, name)
body = download["bytes"]
check((variant["width"], variant["height"], download["headers"].get_content_type())
== (width, height, mime), "wrong dimensions or MIME")
check(variant["size_bytes"] == len(body) and variant["sha256"] == hashlib.sha256(body).hexdigest(),
"wrong variant size or checksum")
check(body.startswith(b"\xff\xd8") if mime == "image/jpeg" else
body[:4] == b"RIFF" and body[8:12] == b"WEBP", "wrong encoding")
return result
def restart(self):
compose("stop", "worker")
att, raw = self.client.attachment(self.alice, self.chat)
started = time.monotonic()
check(self.metadata(att)["status"] == "queued", "job ran while worker stopped")
self.content(att, "original", 409)
self.content(att, "original", 404, self.eve)
message = self.client.message(self.alice, self.chat, "", attachments=[att["id"]])["json"]
before = until(lambda: entries(att))
services = ("api", "broker")
old = generations(*services)
check(len(old) >= 3, "expected two API replicas and broker")
compose("restart", *services)
new = generations(*services)
check(old.keys() == new.keys() and all(old[key] != new[key] for key in old), "restart not observed")
check(until(lambda: entries(att)) == before, "durable broker entry changed")
until(lambda: self.client.call("GET", "/health/ready", schema="Health"))
compose("start", "worker")
self.ready(att, raw)
until(lambda: not entries(att))
self.record("api_broker_restart_with_worker_stopped", att, started,
message_id=message["id"], job_id=before[0]["job_id"], processes_restarted=len(old))
def publish_crash(self, phase):
compose("stop", "dispatcher", "worker")
att, raw = self.client.attachment(self.alice, self.chat)
started = time.monotonic()
fault("dispatcher", phase, att)
old = job(att)
check(old["state"] == "queued" and not old["published"], "outbox lost pending intent")
observed = entries(att)
check(len(observed) == (phase == "after_publish"), "incorrect broker state at crash")
# Keep the worker stopped until both XADDs are visible in the ambiguous case.
compose("start", "dispatcher")
count = 2 if phase == "after_publish" else 1
until(lambda: len(entries(att)) == count)
check(all(entry == carrier(att, 1) for entry in entries(att)), "unexpected carrier")
compose("start", "worker")
self.ready(att, raw)
until(lambda: not entries(att))
check(job(att)["attempt"] == 1, "duplicate delivery consumed an attempt")
self.record(phase, att, started, copies=count, attempt=1)
def worker_crash(self, phase):
compose("stop", "worker")
att, raw = self.client.attachment(self.alice, self.chat)
started = time.monotonic()
fault("worker", phase, att)
old = job(att)
path = f"original/{att['id']}/v1"
written = s3stat(path)
check((written is not None) == (phase != "before_effect"), "incorrect S3 state at crash")
check(old["state"] == ("ready" if phase == "after_commit" else "processing"), "incorrect job state at crash")
check(broker("XPENDING", "attachment-jobs", "image-workers")[0] > 0, "missing unacked delivery")
compose("start", "worker")
self.ready(att, raw)
until(lambda: not entries(att), timeout=155)
if written:
final = s3stat(path)
check(final["lastModified"] == written["lastModified"] and final["etag"] == written["etag"],
"retry rewrote an existing effect")
check(job(att)["attempt"] == (1 if phase == "after_commit" else 2), "wrong attempt after recovery")
until(lambda: s3stat(f"staging/{att['id']}") is None, timeout=50)
self.record(phase, att, started, state_at_crash=old["state"], object_at_crash=written is not None,
final_attempt=job(att)["attempt"], staging_removed=True)
def replay(self, phase):
compose("stop", "worker")
att, raw = self.client.attachment(self.alice, self.chat)
started = time.monotonic()
name = f"lab5-fault-{uuid.uuid4().hex[:12]}"
compose("run", "-d", "--no-deps", "--name", name, "-e", f"LAB5_FAULT={phase}",
"-e", f"LAB5_FAULT_ATTACHMENT={att['id']}", "worker")
delays = {}
try:
def failure():
row = job(att)
if row["attempt"] > 1 and row["delay_ms"] > 0:
delays.setdefault(row["attempt"], row["delay_ms"])
return row if row["state"] == "failed" else None
failed = until(failure, timeout=155)
finally:
run("docker", "stop", name)
run("docker", "rm", name)
code = "invalid_image" if phase == "invalid_image" else "retry_exhausted"
meta = self.metadata(att)
check(meta["status"] == "failed" and meta["error_code"] == code, "incorrect final failure")
if phase == "invalid_image":
check(failed["attempt"] == 1, "permanent error was retried")
else:
config = json.loads(compose("config", "--format", "json"))
max_attempts = int(config["services"]["worker"]["environment"]["JOB_MAX_ATTEMPTS"])
check(failed["attempt"] == max_attempts, "incorrect retry limit")
check(set(delays) == set(range(2, max_attempts + 1)), "missed retry schedule")
for attempt, delay in delays.items():
base = 1000 << min(attempt - 2, 5)
check(base - 5 <= delay < 2 * base, "retry delay outside jitter window")
self.content(att, "original", 409)
self.client.call("POST", f"/api/v1/chats/{self.chat['id']}/messages", 409, actor=self.alice,
headers={"Idempotency-Key": uuid.uuid4().hex}, body={"text": "", "attachment_ids": [att["id"]]})
def operator(*args, **kwargs):
return compose("run", "--rm", "--no-deps", "dispatcher", "jobs", *args, **kwargs)
check(failed["id"] in operator("failed"), "failed registry did not contain job")
operator("replay", failed["id"])
check(self.metadata(att)["status"] == "queued", "replay failed to queue job")
compose("start", "worker")
self.ready(att, raw)
operator("replay", failed["id"], expected=1)
audit = sql(f"SELECT count(*) FROM image_job_replays WHERE job_id='{failed['id']}'")
check(audit == "1" and job(att)["attempt"] == failed["attempt"] + 1, "replay audit/attempt mismatch")
self.record(phase + "_replay", att, started, failed_attempt=failed["attempt"],
error_code=code, retry_delays_ms=delays, replay_audit=int(audit))
def version_order(self):
att, raw = self.client.attachment(self.alice, self.chat)
started = time.monotonic()
v1 = self.ready(att, raw)
old_carrier = carrier(att, 1)
old_object = s3stat(f"optimized/{att['id']}/v1")
until(lambda: not entries(att))
compose("stop", "worker")
compose("run", "--rm", "--no-deps", "dispatcher", "jobs", "upgrade", att["id"])
until(lambda: job(att, 2)["published"])
new_carrier = carrier(att, 2)
publish(old_carrier)
publish(new_carrier)
check([entry["pipeline_version"] for entry in entries(att)] == [2, 1, 2], "reorder not injected")
check(self.metadata(att) == v1, "upgrade hid the previous result")
compose("start", "worker")
until(lambda: sql(f"SELECT result_version FROM attachments WHERE id='{att['id']}'") == "2")
v2 = self.ready(att, raw)
until(lambda: not entries(att))
check(v1["variants"] != v2["variants"], "v2 was not distinct")
new_object = s3stat(f"optimized/{att['id']}/v2")
publish(old_carrier)
publish(new_carrier)
until(lambda: not entries(att))
check(self.ready(att, raw) == v2 and job(att, 2)["attempt"] == 1, "stale job changed v2")
for version, obj in ((1, old_object), (2, new_object)):
current = s3stat(f"optimized/{att['id']}/v{version}")
check(current["lastModified"] == obj["lastModified"] and current["etag"] == obj["etag"],
"duplicate or reordered job rewrote an object")
self.record("reorder_and_stale_version", att, started, delivery_versions=[2, 1, 2, 1, 2],
result_version=2, attempts=[job(att)["attempt"], job(att, 2)["attempt"]])
def main():
experiment = Experiment()
try:
experiment.restart()
for phase in ("before_publish", "after_publish"):
experiment.publish_crash(phase)
for phase in ("before_effect", "after_effect", "after_commit"):
experiment.worker_crash(phase)
for phase in ("invalid_image", "temporary"):
experiment.replay(phase)
experiment.version_order()
finally:
compose("start", "dispatcher", "worker")
(ROOT / "evidence" / "pipeline.json").write_text(
json.dumps(experiment.results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8",
)
print("PASS pipeline fault experiment")
if __name__ == "__main__":
main()