#!/usr/bin/env python3 """Run only on the disposable lab5 fault-overlay Compose stand. Retains volumes.""" import argparse from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone import hashlib import json from pathlib import Path import struct import subprocess import sys import time from urllib.parse import urlsplit import uuid import zlib ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, check # noqa: E402 MARKERS = "/tmp/lab5-faults/" VARIANTS = {"thumbnail", "crop", "optimized"} def digest(data): return hashlib.sha256(data).hexdigest() def fixture(seed): """A deterministic textured PNG produces different WebP bytes at the two qualities.""" size = 160 rows = bytearray() for y in range(size): rows.append(0) for x in range(size): rows.extend(((x * 3 + y * 7 + seed) % 256, (x * y + seed) % 256, (x ^ y ^ seed) % 256)) def chunk(kind, data): return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0)) + chunk(b"IDAT", zlib.compress(rows)) + chunk(b"IEND", b"")) def command(args, *, allow_failure=False, binary=False, timeout=60): proc = subprocess.run(args, cwd=ROOT, capture_output=True, timeout=timeout) if proc.returncode and not allow_failure: raise Failure(f"{args[:4]} exited {proc.returncode}: " + proc.stderr.decode(errors="replace")[-600:]) if allow_failure: return proc return proc.stdout if binary else proc.stdout.decode().strip() def wait(test, label, timeout=90): end = time.monotonic() + timeout while time.monotonic() < end: value = test() if value: return value time.sleep(0.16) raise Failure("timed out waiting for " + label) class Experiment: def __init__(self, base_url, project): self.compose = ["docker", "compose", "-p", project, "-f", "compose.yaml", "-f", "compose.faults.yaml"] address = urlsplit(base_url) port = self.dc("port", "lb", "8080").splitlines()[0].rsplit(":", 1)[-1] check(address.hostname in {"127.0.0.1", "localhost"} and str(address.port) == port, "base URL must match the explicit local Compose stand") self.worker = self.ids("worker")[0] self.s3 = self.ids("s3")[0] self.bucket = self.exe(self.worker, "printenv", "S3_BUCKET") self.apis = self.ids("api") check(len(self.apis) >= 2, "two API replicas required") check("--barriers=" + MARKERS.rstrip("/") in command(["docker", "inspect", "-f", "{{json .Config.Cmd}}", self.worker]), "worker must run with compose.faults.yaml") for api in self.apis: check(self.exe(api, "printenv", "LAB5_BARRIERS") == MARKERS.rstrip("/"), "API barrier overlay required") self.client = Client(base_url) self.actor, self.reader, _, self.chat = self.client.bootstrap() self.content = fixture(41) self.stopped = set() self.armed = set() def dc(self, *args, **kwargs): return command(self.compose + list(args), **kwargs) def ids(self, service): ids = self.dc("ps", "-q", service).splitlines() check(bool(ids), service + " must be running") return ids def exe(self, container, *args, **kwargs): return command(["docker", "exec", container, *args], **kwargs) def sql(self, statement): return self.dc("exec", "-T", "postgres", "psql", "-XqAt", "-v", "ON_ERROR_STOP=1", "-U", "messenger", "-d", "messenger", "-c", statement) def marker(self, container, stem, operation): path = MARKERS + stem if operation == "arm": self.exe(container, "mkdir", "-p", MARKERS) self.armed.add((container, stem)) self.exe(container, "touch", path + ".arm") elif operation == "hit": return self.exe(container, "test", "-f", path + ".hit", allow_failure=True).returncode == 0 elif operation == "release": self.exe(container, "rm", "-f", path + ".arm", path + ".hit", path + ".id") self.armed.discard((container, stem)) def arm(self, stem, containers): for container in containers: self.marker(container, stem, "arm") def hit(self, stem, containers): return wait(lambda: next((c for c in containers if self.marker(c, stem, "hit")), None), stem) def stop(self, service): self.stopped.add(service) self.dc("stop", "-t", "3", service) def start(self, service): self.dc("start", service) self.stopped.discard(service) def crash(self, container): command(["docker", "kill", "--signal", "KILL", container]) command(["docker", "start", container]) def cli(self, *args, **kwargs): return self.exe(self.worker, "/usr/local/bin/worker", *args, **kwargs) def object_hash(self,key): # Read through the storage process's loopback-only filer endpoint. This # can observe a worker's PUT before HTTP publication has committed. url = "http://127.0.0.1:8888/buckets/" + self.bucket + "/" + key fetched = self.exe(self.s3,"wget","-SO-",url,allow_failure=True) if fetched.returncode: check(b"404 Not Found" in fetched.stderr, "cannot inspect filer object " + key) return None return digest(fetched.stdout) def upload(self, data=None): data = self.content if data is None else data att = self.client.call("POST", "/api/v1/chats/" + self.chat["id"] + "/attachments?filename=track2.png", 202, actor=self.actor, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment")["json"] check(att["status"] == "queued" and att["sha256"] == digest(data), "invalid accepted upload") return str(uuid.UUID(att["id"])) def state(self, aid): return json.loads(self.sql(f""" SELECT json_build_object('target',a.target_version,'published',a.published_version, 'jobs',(SELECT json_agg(json_build_object('version',j.pipeline_version,'id',j.job_id, 'attempt',j.attempt,'state',j.state,'error',j.error_code,'confirmed',j.published_at IS NOT NULL) ORDER BY j.pipeline_version) FROM attachment_jobs j WHERE j.attachment_id=a.id), 'variant_count',(SELECT count(*) FROM attachment_variants WHERE attachment_id=a.id)) FROM attachments a WHERE a.id='{aid}'""")) def result(self, aid, data=None): data = self.content if data is None else data att = self.client.wait_ready(self.reader, {"id": aid}, 120) base = "/api/v1/attachments/" + aid + "/content" original = self.client.call("GET", base, actor=self.reader)["bytes"] check(original == data, "accepted original changed") hashes = {} for variant in att["variants"]: name = variant["name"] obj = self.client.call("GET", base + "?variant=" + name, actor=self.reader) h = digest(obj["bytes"]) check(h == variant["sha256"] and len(obj["bytes"]) == variant["size_bytes"], "variant bytes differ from metadata") check(name not in hashes, "duplicate public variant") hashes[name] = h check(set(hashes) == VARIANTS, "missing public variant") state = self.state(aid) rows = self.sql(f"""SELECT variant||' '||encode(sha256,'hex') FROM attachment_variants WHERE attachment_id='{aid}' AND pipeline_version={int(state['published'])}""") check(dict(row.split() for row in rows.splitlines()) == hashes, "SQL and downloaded bytes differ") return {"attachment": aid, "original_sha256": digest(original), "variant_sha256": hashes, **state} def queued(self): self.stop("broker") aid = self.upload() state = self.state(aid) check(state["jobs"][0]["state"] == "queued" and not state["jobs"][0]["confirmed"], "expected pending outbox") self.client.call("GET", "/api/v1/attachments/" + aid + "/content", 409, actor=self.reader) return aid def staged_crash(self): data = fixture(uuid.uuid4().int % 255) marker = digest(data) + "-staged" self.arm(marker, self.apis) with ThreadPoolExecutor(max_workers=1) as pool: future = pool.submit(self.upload, data) owner = self.hit(marker, self.apis) path = MARKERS + marker + ".id" def staged_id(): p = self.exe(owner, "cat", path, allow_failure=True) return p.stdout.decode().strip() if p.returncode == 0 else None aid = str(uuid.UUID(wait(staged_id, "uncommitted staging UUID"))) self.crash(owner) try: future.result() except Failure: pass else: raise Failure("interrupted staging unexpectedly returned 202") for api in self.apis: self.marker(api, marker, "release") rows = int(self.sql(f"SELECT count(*) FROM attachments WHERE id='{aid}'")) check(rows == 0, "staged-only upload acquired a logical attachment") check(self.object_hash("staging/" + aid) == digest(data), "crashed upload did not leave the expected staged bytes") return {"unaccepted": aid, "source_sha256": digest(data), "attachment_rows": rows} def outbox_restart(self): aid = self.queued() message = self.client.message(self.actor, self.chat, "Принято без брокера", attachments=[aid])["json"] before = self.state(aid) self.dc("restart", "api") def api_ready(): try: return self.client.call("GET", "/health/ready", schema="Health")["status"] == 200 except Failure: return False wait(api_ready, "API readiness without broker") check(self.state(aid) == before, "durable intent changed during API restart") self.start("broker") result = self.result(aid) check(result["jobs"][0]["id"] == before["jobs"][0]["id"], "restart recreated job") path = "/api/v1/chats/" + self.chat["id"] + "/messages" history = self.client.call("GET", path, actor=self.reader, schema="MessagePage")["json"]["items"] check(message in history, "confirmed message disappeared") return {"before": before, "message": message["id"], "after": result} def published_crash(self): aid = self.queued() self.stop("worker") marker = aid + "-v1-published" self.arm(marker, self.apis) self.start("broker") owner = self.hit(marker, self.apis) for api in self.apis: if api != owner: self.marker(api, marker, "release") before = self.state(aid) check(not before["jobs"][0]["confirmed"], "SQL confirmation already committed") self.crash(owner) self.marker(owner, marker, "release") wait(lambda: self.state(aid)["jobs"][0]["confirmed"], "republished envelope") def queued_copies(): rows = json.loads(self.dc("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "messages_ready", "--formatter=json")) copies = next((row["messages_ready"] for row in rows if row["name"] == "attachment.jobs"), 0) return copies if copies >= 2 else 0 copies = wait(queued_copies, "two confirmed broker copies while consumers are stopped") confirmed = self.state(aid) self.dc("restart", "broker") def after_restart(): try: return queued_copies() except Failure: return 0 restored = wait(after_restart, "durable queue after broker restart", 120) check(copies == restored and self.state(aid) == confirmed, "confirmed queue or SQL job changed across broker restart") self.start("worker") result = self.result(aid) check(result["variant_count"] == 3 and result["jobs"][0]["attempt"] == 1, "publication crash duplicated the effect") return {"broker_confirm_before_sql_commit": before, "confirmed_job_before_broker_restart": confirmed, "broker_messages_before_restart":copies,"broker_messages_after_restart":restored,"after": result} def worker_crash(self, phase, reference): aid = self.queued() marker = aid + "-v1-" + phase self.arm(marker, [self.worker]) self.start("broker") self.hit(marker, [self.worker]) before = self.state(aid) check(before["variant_count"] == (3 if phase == "committed" else 0), "crash occurred on the wrong side of SQL commit") observed = {name: self.object_hash(f"variants/{aid}/1/{name}/{value}") for name,value in reference.items()} expected = {name:None for name in VARIANTS} if phase == "claimed" else reference check(observed == expected, "crash occurred on the wrong side of storage writes") if phase == "committed": duplicate = aid + "-v1-duplicate" self.arm(duplicate, [self.worker]) self.crash(self.worker) self.marker(self.worker, marker, "release") if phase == "committed": self.hit(duplicate, [self.worker]) self.marker(self.worker, duplicate, "release") result = self.result(aid) want = 1 if phase == "committed" else 2 check(result["jobs"][0]["attempt"] == want and result["variant_count"] == 3, "incorrect crash recovery") return {"phase": phase, "before": before, "storage_sha256_at_barrier": observed, "after": result} def duplicate_ack(self, aid, version): prefix = f"{aid}-v{version}-" for phase in ["duplicate", "acked"]: self.arm(prefix + phase, [self.worker]) self.cli("duplicate", aid, str(version), "1") for phase in ["duplicate", "acked"]: self.hit(prefix + phase, [self.worker]) self.marker(self.worker, prefix + phase, "release") def lease_takeover(self): aid = self.queued() objects = aid + "-v1-objects" committed = aid + "-v1-committed" acked = aid + "-v1-acked" self.arm(objects, [self.worker]) self.arm(committed, [self.worker]) self.start("broker") self.hit(objects, [self.worker]) first = self.state(aid) self.hit(committed, [self.worker]) winner = self.result(aid) check(first["jobs"][0]["attempt"] == 1 and winner["jobs"][0]["attempt"] == 2 and first["jobs"][0]["id"] == winner["jobs"][0]["id"], "lease takeover changed logical work") for barrier in [committed, objects]: self.arm(acked, [self.worker]) self.marker(self.worker, barrier, "release") self.hit(acked, [self.worker]) self.marker(self.worker, acked, "release") after = self.result(aid) check(after == winner and after["variant_count"] == 3, "expired worker modified published result") return {"first_held": first, "second_committed": winner, "after_first_ack": after} def reordered(self, v1_hashes): aid = self.queued() marker = aid + "-v1-objects" self.arm(marker, [self.worker]) self.start("broker") self.hit(marker, [self.worker]) self.duplicate_ack(aid, 1) self.cli("reprocess", aid, "2") before = self.result(aid) check(before["published"] == 2 and before["variant_count"] == 3 and before["jobs"][0]["state"] == "superseded" and before["jobs"][0]["attempt"] == 1, "new version did not finish while first worker held its result") check(v1_hashes["optimized"] != before["variant_sha256"]["optimized"], "fixture must distinguish the profiles") acked = aid + "-v1-acked" self.arm(acked, [self.worker]) self.marker(self.worker, marker, "release") self.hit(acked, [self.worker]) self.marker(self.worker, acked, "release") for version in [2, 1]: self.duplicate_ack(aid, version) after = self.result(aid) check(after == before, "stale commit or delivery changed new result") return {"reference_v1_webp": v1_hashes["optimized"], "before_releasing_v1": before, "after_acks_in_order": [2, 1], "after": after} def retry_replay(self): aid = self.queued() real = "staging/" + aid self.sql(f"UPDATE attachment_jobs SET source_key='staging/{uuid.uuid4()}' WHERE attachment_id='{aid}'") self.start("broker") wait(lambda: self.state(aid)["jobs"][0]["state"] == "failed", "retry exhaustion", 180) failed = self.state(aid) limit = int(self.exe(self.worker, "printenv", "JOB_MAX_ATTEMPTS")) job = failed["jobs"][0] check(job["attempt"] == limit and job["error"] == "source_missing", "wrong failure budget") entries = json.loads(self.cli("failed")) check(any(row["job_id"] == job["id"] for row in entries), "missing failed job") self.sql(f"UPDATE attachment_jobs SET source_key='{real}' WHERE attachment_id='{aid}'") self.cli("replay", aid, "track2") result = self.result(aid) check(result["jobs"][0]["attempt"] == limit + 1, "replay reused a stale attempt") rejected = self.cli("replay", aid, "track2", allow_failure=True) check(rejected.returncode != 0, "accepted replay of successful job") count = int(self.sql(f"SELECT count(*) FROM attachment_replays WHERE job_id='{job['id']}' AND actor='track2'")) check(count == 1, "incorrect audit count") return {"injection": "absent source key assigned only to synthetic job", "exhausted": failed, "after": result, "audit_rows": count} def permanent_failure(self, other_sha): self.stop("broker") aid = self.upload(fixture(19)) self.sql(f"UPDATE attachment_jobs SET source_key='blobs/{other_sha}' WHERE attachment_id='{aid}'") self.start("broker") wait(lambda: self.state(aid)["jobs"][0]["state"] == "failed", "permanent source mismatch") failed = self.state(aid) j = failed["jobs"][0] check(j["attempt"] == 1 and j["error"] == "source_mismatch" and failed["variant_count"] == 0, "permanent failure consumed additional attempts") self.sql(f"UPDATE attachment_jobs SET source_key='staging/{aid}' WHERE attachment_id='{aid}'") return {"injection": "different valid original assigned only to synthetic job", "failed": failed} def cleanup_stage(self, aid, created): # Run normal GC after its real minimum grace. No object timestamps are forged. remaining = created + 125 - time.monotonic() if remaining > 0: time.sleep(remaining) def run_gc(apply): args = ["run", "--rm", "--no-deps", "-T", "reconcile", "--grace=2m"] if apply: args.append("--apply") return [json.loads(line) for line in self.dc(*args, timeout=180).splitlines()] def matches(rows): return [row for row in rows if row.get("key") == "staging/" + aid] before = matches(run_gc(False)) check(len(before) == 1 and before[0]["kind"] == "stale_staging", "unaccepted stage not reported") applied = matches(run_gc(True)) check(len(applied) == 1 and applied[0]["applied"], "unaccepted stage not removed") check(not matches(run_gc(False)), "removed stage still listed") return {"before": before[0], "applied": applied[0], "after_findings": 0} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-url", required=True) parser.add_argument("--project", choices=["lab5"], required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() exp = Experiment(args.base_url, args.project) started = time.monotonic() report = {"project": args.project, "complete": False, "scenarios": {}, "seconds": {}, "started_at":datetime.now(timezone.utc).isoformat()} args.output.parent.mkdir(parents=True, exist_ok=True) def save(): args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") def record(name, callback): print(name, flush=True) beginning = time.monotonic() report["scenarios"][name] = callback() report["seconds"][name] = round(time.monotonic() - beginning, 3) save() return report["scenarios"][name] save() try: stage = record("staging_before_commit", exp.staged_crash) created = time.monotonic() accepted = record("commit_before_publish", exp.outbox_restart)["after"] record("broker_confirm_before_sql", exp.published_crash) for phase in ["claimed", "objects", "committed"]: record("worker_crash_" + phase, lambda: exp.worker_crash(phase, accepted["variant_sha256"])) record("same_version_lease_takeover", exp.lease_takeover) record("parallel_duplicate_and_reorder", lambda: exp.reordered(accepted["variant_sha256"])) record("retry_exhaustion_replay", exp.retry_replay) record("permanent_failure", lambda: exp.permanent_failure(accepted["original_sha256"])) record("unaccepted_staging_cleanup", lambda: exp.cleanup_stage(stage["unaccepted"], created)) report["complete"] = True report["finished_at"] = datetime.now(timezone.utc).isoformat() report["elapsed_seconds"] = round(time.monotonic() - started, 3) save() finally: for service in sorted(exp.stopped): exp.start(service) for container, stem in list(exp.armed): exp.marker(container, stem, "release") if __name__ == "__main__": main()