#!/usr/bin/env python3 """Real-process fault matrix for the versioned pipeline; test Compose override required.""" import argparse import hashlib import io import json import subprocess import sys import time from pathlib import Path from uuid import uuid4 from botocore.exceptions import ClientError from cluster import ROOT, compose, inspect, instances, wait_lb from PIL import Image from pipeline_lab import settings from messenger.broker import STREAM, Broker, Job from messenger.pipeline import Pipeline from messenger.repository import PostgresRepository, stamp from messenger.storage import ObjectStore sys.path.insert(0, str(ROOT / "tests")) from client import Client, check # noqa: E402 def until(fn, timeout=60): deadline = time.monotonic() + timeout while time.monotonic() < deadline: result = fn() if result: return result time.sleep(0.2) raise AssertionError("Timed out waiting for a durable pipeline condition") def main(): parser = argparse.ArgumentParser(__doc__) parser.add_argument("--base-url", default="http://127.0.0.1:8085") parser.add_argument("--output", type=Path, default=ROOT / "evidence/pipeline.json") args = parser.parse_args() if args.output.exists(): parser.error("Output already exists; choose a new evidence file") config = settings() db = PostgresRepository(config.database_url, pool_size=4) db.open() broker, store = Broker(config), ObjectStore(config) c = Client(args.base_url) alice, bob, eve, room = c.bootstrap() records, accepted = [], [] data = (ROOT / "tests/fixtures/sample.png").read_bytes() def query(sql, params=()): with db.connection() as conn: return conn.execute(sql, params).fetchall() def job(a, version=1): return query( "SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s", (a["id"], version), )[0] def audit(j): return query( "SELECT event,details,created_at FROM pipeline_audit WHERE job_id=%s ORDER BY sequence", (j["job_id"],), ) def injected(j, phase): return any( e["event"] == "fault_injected" and e["details"]["phase"] == phase for e in audit(j) ) def fault(j, phase, action, count=1): with db.connection(write=True) as conn: conn.execute( "INSERT INTO pipeline_faults(job_id,phase,action,remaining) VALUES (%s,%s,%s,%s)", (j["job_id"], phase, action, count), ) def upload(): a, original = c.attachment(alice, room) check(original == data, "Unexpected fixture") accepted.append(a["id"]) return a def ready(a): value = c.wait_ready(bob, a) path = "/api/v1/attachments/" + a["id"] check(c.call("GET", path + "/content", actor=bob)["bytes"] == data, "Original changed") for v in value["variants"]: content = c.call("GET", path + "/content?variant=" + v["name"], actor=bob)["bytes"] check(hashlib.sha256(content).hexdigest() == v["sha256"], "Variant checksum mismatch") with Image.open(io.BytesIO(content)) as im: check(im.size == (v["width"], v["height"]), "Variant dimensions mismatch") check(len(value["variants"]) == 3, "Missing variants") check( len(query("SELECT 1 FROM message_attachments WHERE attachment_id=%s", (a["id"],))) <= 1, "Unexpected duplicate link", ) return value def saved(name, a, start, **details): j = job(a, details.get("version", 1)) current = query( "SELECT storage_state,pipeline_version FROM attachments WHERE id=%s", (a["id"],) )[0] keys = [ o["Key"] for o in store.client.list_objects_v2( Bucket=store.bucket, Prefix="objects/" + a["id"] + "/v" + str(current["pipeline_version"]) + "/", ).get("Contents", []) ] check(len(keys) == 4, "Repeated deliveries multiplied the published variant set") record = { "scenario": name, "attachment_id": a["id"], "job_id": str(j["job_id"]), "elapsed_seconds": time.monotonic() - start, "final_state": current["storage_state"], "attempts": j["attempt"], "final_version": current["pipeline_version"], "published_objects": len(keys), "audit": audit(j), **details, } records.append(record) print("PASS: " + name, flush=True) def envelope(j, attempt=None): return Job( schema_version=1, type="attachment.process.v1", job_id=str(j["job_id"]), attachment_id=str(j["attachment_id"]), pipeline_version=j["pipeline_version"], attempt=attempt or max(1, j["attempt"]), created_at=stamp(j["created_at"]), ) try: # Actual COMMIT/publish gap: no relay exists while the acknowledged upload is stored. compose("stop", "--timeout", "2", "worker", "relay") start = time.monotonic() a = upload() j = job(a) check( query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][ "published_at" ] is None, "Gap was not established", ) message = c.message(alice, room, "", attachments=[a["id"]])["json"] c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", 409, actor=bob) before = instances() compose("restart", "--timeout", "3", "api") wait_lb(args.base_url) after = instances() check( {r["instance"] for r in before}.isdisjoint({r["instance"] for r in after}), "API did not restart", ) compose("start", "relay") until( lambda: query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][ "published_at" ] ) entry_ids = [ id for id, fields in broker.client.xrange(STREAM) if json.loads(fields["job"])["job_id"] == str(j["job_id"]) ] check(bool(entry_ids), "Broker has no accepted work") broker_id = compose("ps", "-q", "broker").strip() broker_before = inspect(broker_id) subprocess.run( ["docker", "kill", "--signal", "KILL", broker_id], check=True, capture_output=True ) subprocess.run(["docker", "start", broker_id], check=True, capture_output=True) def broker_restored(): try: return broker.client.ping() except Exception: return False until(broker_restored) check( set(entry_ids) <= {id for id, _ in broker.client.xrange(STREAM)}, "Durable stream lost confirmed work", ) compose("start", "worker") ready(a) history = c.call( "GET", "/api/v1/chats/" + room["id"] + "/messages", actor=bob, schema="MessagePage" )["json"] check(message in history["items"], "Queued attachment message lost") saved( "commit_publish_gap_api_restart_broker_sigkill", a, start, api_before=before, api_after=after, broker_before=broker_before, broker_after=inspect(broker_id), broker_entry_ids=entry_ids, message_id=message["id"], ) # A publisher can lose its COMMIT after XADD; same logical ID is sent twice. compose("stop", "--timeout", "2", "worker", "relay") start = time.monotonic() a = upload() j = job(a) fault(j, "after_publish", "crash") compose("start", "relay") until(lambda: injected(j, "after_publish")) until( lambda: ( sum( json.loads(f["job"])["job_id"] == str(j["job_id"]) for _, f in broker.client.xrange(STREAM) ) >= 2 ) ) compose("start", "worker") ready(a) until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j))) saved("publish_before_outbox_commit_duplicate", a, start) for phase in ("before_effect", "after_effect", "after_commit"): compose("stop", "--timeout", "2", "worker") start = time.monotonic() a = upload() j = job(a) fault(j, phase, "crash") compose("start", "worker") ready(a) until(lambda j=j, phase=phase: injected(j, phase)) if phase == "after_commit": until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j))) check(job(a)["attempt"] == 1, "Committed job was processed twice") else: check(job(a)["attempt"] == 2, "Expired lease did not recover exactly once") saved("worker_crash_" + phase, a, start) for action, count, expected in ( ("transient", 2, "ready"), ("hang", 1, "ready"), ("permanent", 1, "failed"), ("transient", 5, "failed"), ): compose("stop", "--timeout", "2", "worker") start = time.monotonic() a = upload() j = job(a) fault(j, "before_effect", action, count) compose("start", "worker") path = "/api/v1/attachments/" + a["id"] if expected == "failed": until( lambda path=path: ( c.call("GET", path, actor=bob, schema="Attachment")["json"]["status"] == "failed" ) ) terminal = job(a) check(terminal["attempt"] == count, "Wrong retry budget") c.call("GET", path + "/content", 409, actor=bob) c.call( "POST", "/api/v1/chats/" + room["id"] + "/messages", 409, actor=alice, body={"text": "", "attachment_ids": [a["id"]]}, headers={"Idempotency-Key": uuid4().hex}, ) # Explicit CLI replay, same logical job; no public administrator endpoint. replay = json.loads( compose( "exec", "-T", "worker", "python", "-m", "messenger.operator", "replay", a["id"], ) ) check(replay["job_id"] == str(j["job_id"]), "Replay changed logical job") else: terminal = None ready(a) events = audit(j) delays = [e["details"]["delay_seconds"] for e in events if e["event"] == "queued"] check(all(0.25 <= delay <= 30 for delay in delays), "Invalid bounded jitter") rejected = subprocess.run( [ "docker", "compose", "--env-file", ".env", "exec", "-T", "worker", "python", "-m", "messenger.operator", "replay", a["id"], ], cwd=ROOT, capture_output=True, ) check(rejected.returncode != 0, "Ready replay was accepted") saved( action + "_" + str(count) + ("_failed_then_explicit_replay" if terminal else ""), a, start, terminal_before_replay={k: terminal[k] for k in ("state", "attempt", "error_code")} if terminal else None, backoff_delays=delays, ) # Reorder and stale v1 delivered after v2; exercise the DB publication fence directly too. compose("stop", "--timeout", "2", "worker", "relay") start = time.monotonic() a = upload() old = job(a) old_envelope = envelope(old, 1) outcome, old_claim = Pipeline(db, config).claim(old_envelope) check(outcome == "claimed", "Cannot establish old writer") Pipeline(db, config).replay(a["id"], new_version=True) new = job(a, 2) # Publish v2 first, then v1, plus duplicates through both worker slots. for value in (envelope(new, 1), old_envelope, envelope(new, 1), old_envelope): broker.publish(value) compose("start", "worker", "relay") current = ready(a) row = db.attachment(a["id"]) check( not Pipeline(db, config).publish(old_claim, "objects/old-writer", []), "Old writer overwrote v2", ) for _ in range(20): broker.publish(old_envelope) broker.publish(envelope(new, 1)) until(lambda: sum(e["event"] == "stale_delivery" for e in audit(old)) >= 21) until(lambda: sum(e["event"] == "duplicate_ready" for e in audit(new)) >= 20) check( db.attachment(a["id"])["object_key"] == row["object_key"], "Stale delivery changed current pointer", ) check( c.call("GET", "/api/v1/attachments/" + a["id"], actor=bob, schema="Attachment")["json"] == current, "Old version mutated ready metadata", ) saved( "reorder_stale_v1_after_v2_and_publication_fence", a, start, version=2, old_audit=audit(old), duplicates=44, ) # ACL remains enforced through both API replicas; S3 is private and role-scoped. a = {"id": accepted[0]} for suffix in ("", "/content", "/content?variant=crop"): c.call("GET", "/api/v1/attachments/" + a["id"] + suffix, 404, actor=eve) # Direct bucket access and write/delete privilege checks use no public credentials. import boto3 from botocore import UNSIGNED from botocore.config import Config first = db.attachment(accepted[0]) unsigned = boto3.client( "s3", endpoint_url=config.s3_endpoint, config=Config(signature_version=UNSIGNED) ) try: unsigned.get_object(Bucket=store.bucket, Key=first["object_key"]) raise AssertionError("Anonymous S3 read succeeded") except ClientError as exc: check( exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Wrong anonymous status" ) finally: unsigned.close() denied = [] for role, action in ( ("api", "put_object"), ("api", "delete_object"), ("worker", "delete_object"), ("reconciler", "put_object"), ): scoped = ObjectStore(settings(role)) try: kw = {"Bucket": store.bucket, "Key": "objects/" + uuid4().hex + "/forbidden"} if action == "put_object": kw["Body"] = b"forbidden" try: getattr(scoped.client, action)(**kw) raise AssertionError("IAM action unexpectedly allowed") except ClientError as exc: check( exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Wrong IAM status", ) denied.append(role + ":" + action) finally: scoped.close() # Stop the actual uploader, then read original and thumbnail through another replica. response = c.call( "POST", "/api/v1/chats/" + room["id"] + "/attachments?filename=cross.png", 202, actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment", ) cross = response["json"] accepted.append(cross["id"]) ready(cross) uploader = response["headers"]["X-Instance-Id"] process = next(v for v in instances() if v["instance"] == uploader) subprocess.run( ["docker", "stop", "--time", "3", process["id"]], check=True, capture_output=True ) try: wait_lb(args.base_url, count=1) download = c.call("GET", "/api/v1/attachments/" + cross["id"] + "/content", actor=bob) check( download["bytes"] == data and download["headers"]["X-Instance-Id"] != uploader, "Other replica could not serve the original", ) c.call( "GET", "/api/v1/attachments/" + cross["id"] + "/content?variant=thumbnail", actor=bob, ) cross_proof = { "uploader": uploader, "stopped": inspect(process["id"]), "reader": download["headers"]["X-Instance-Id"], } finally: subprocess.run(["docker", "start", process["id"]], check=True, capture_output=True) wait_lb(args.base_url) logs = compose("logs", "--no-color", "api", "worker", "relay", "lb") check( not any( actor["token"] in logs or actor["registration"]["password"] in logs for actor in (alice, bob, eve) ), "Credential in logs", ) states = query( "SELECT id,storage_state FROM attachments WHERE id=ANY(%s::uuid[])", (accepted,) ) check( len(states) == len(accepted) and all(s["storage_state"] == "ready" for s in states), "Confirmed work lost", ) stats = compose( "exec", "-T", "worker", "python", "-c", "import json;from pathlib import Path;print(json.dumps({" "n:Path('/sys/fs/cgroup/'+n).read_text().strip() " "for n in ('memory.current','memory.peak','memory.events','cpu.stat')}))", ) result = { "anonymous_s3_status": 403, "denied_iam_actions": denied, "cross_replica": cross_proof, "track": 2, "delivery": "at-least-once", "effect_key": ["attachment_id", "pipeline_version", "variant"], "worker_concurrency": config.worker_concurrency, "prefetch_per_slot": 1, "job_timeout_seconds": config.job_timeout, "lease_seconds": config.lease_seconds, "max_attempts": config.job_max_attempts, "scenarios": records, "accepted": len(accepted), "lost_confirmed": 0, "stale_overwrites": 0, "worker_cgroup": json.loads(stats), "logs_credentials_absent": True, } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2, default=str) + "\n") print("PASS: all acknowledged work ready; no stale overwrites", flush=True) finally: compose("start", "worker", "relay", "broker", "api") broker.close() store.close() db.close() if __name__ == "__main__": main()