#!/usr/bin/env python3 """Track 2: versioned pipeline under duplicate, reorder, crash and a stale job. Prints statuses and attachment ids only. Tokens stay in memory. """ import base64 import hashlib import json import os import subprocess import time import urllib.error import urllib.request import uuid from pathlib import Path ROOT = Path(__file__).resolve().parents[1] OUT = ROOT / "evidence" / "track2.json" BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") FIXTURE = ROOT / "tests" / "fixtures" / "sample.png" def compose(*args: str) -> None: command = ["docker", "compose", "--env-file", ".env"] extra = os.environ.get("LAB5_COMPOSE_FILES", "") for name in extra.split(os.pathsep): if name: command.extend(["-f", name]) subprocess.run([*command, *args], cwd=ROOT, check=True) class Api: def __init__(self) -> None: self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None): hdrs = dict(headers or {}) data = raw if body is not None: data = json.dumps(body).encode("utf-8") hdrs["Content-Type"] = "application/json" if content_type: hdrs["Content-Type"] = content_type if token: hdrs["Authorization"] = "Bearer " + token request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method) try: response = self.opener.open(request, timeout=30) except urllib.error.HTTPError as error: response = error with response: payload = response.read() parsed = json.loads(payload) if payload and response.headers.get_content_type() == "application/json" else None return response.status, parsed, payload def expect(status, got, label): if got != status: raise SystemExit("FAIL %s expected %s got %s" % (label, status, got)) def wait_applied(api: Api, attachment_id: str, token: str, version: int, seconds: int = 150, allow_failed: bool = False): deadline = time.monotonic() + seconds while time.monotonic() < deadline: status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token) expect(200, status, "poll") state = jobs(api, attachment_id) if body["status"] == "failed" and allow_failed: return body if ( body["status"] == "ready" and state["applied_pipeline_version"] == version and any(item["name"] == "thumbnail" for item in body["variants"]) ): return body time.sleep(0.4) raise SystemExit("FAIL timeout waiting for pipeline %s" % version) deadline = time.monotonic() + seconds last = None while time.monotonic() < deadline: status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token) expect(200, status, "poll") last = body if body["status"] == "ready" and any(item["name"] == "thumbnail" for item in body["variants"]): return body if body["status"] == "failed": return body time.sleep(0.4) raise SystemExit("FAIL timeout last=%s" % (last or {}).get("status")) def jobs(api: Api, attachment_id: str): status, body, _ = api.call("GET", "/api/v1/_lab/attachments/%s/jobs" % attachment_id) expect(200, status, "jobs") return body def thumb(body): found = [item for item in body["variants"] if item["name"] == "thumbnail"] if len(found) != 1: raise SystemExit("FAIL thumbnail count %s" % len(found)) return found[0]["sha256"] def user(api: Api): username = "t2_" + uuid.uuid4().hex[:10] password = "T3st-" + uuid.uuid4().hex status, body, _ = api.call( "POST", "/api/v1/users", {"username": username, "display_name": "Track2", "password": password}, ) expect(201, status, "register") basic = base64.b64encode((username + ":" + password).encode()).decode() status, session, _ = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}) expect(201, status, "login") return body["id"], session["token"] def upload(api: Api, chat_id: str, token: str, filename: str): status, body, _ = api.call( "POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat_id, filename), token=token, raw=FIXTURE.read_bytes(), content_type="image/png", ) expect(202, status, "upload " + filename) return body def main() -> None: api = Api() content = FIXTURE.read_bytes() original = hashlib.sha256(content).hexdigest() alice, token = user(api) status, chat, _ = api.call( "POST", "/api/v1/chats", {"title": "track2", "member_ids": [alice]}, token=token, ) expect(201, status, "chat") chat_id = chat["id"] report = {"hypothesis": "at-least-once plus a versioned effect key keeps the newest committed pipeline", "cases": []} first = upload(api, chat_id, token, "v1.png") ready = wait_applied(api, first["id"], token, 1) sha_v1 = thumb(ready) state = jobs(api, first["id"]) v1_job = state["jobs"][0]["job_id"] expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "duplicate 1") expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "duplicate 2") time.sleep(2) again = wait_applied(api, first["id"], token, 1) state = jobs(api, first["id"]) v1_rows = [item for item in state["variants"] if item["pipeline_version"] == 1] if thumb(again) != sha_v1 or len(v1_rows) != 3: raise SystemExit("FAIL duplicate changed the effect") report["cases"].append({"name": "duplicate", "attachment_id": first["id"], "thumbnail_sha256": sha_v1, "variant_rows": len(v1_rows)}) expect(202, api.call("POST", "/api/v1/_lab/attachments/%s/pipelines" % first["id"])[0], "bump") bumped = wait_applied(api, first["id"], token, 2) sha_v2 = thumb(bumped) state = jobs(api, first["id"]) if sha_v2 == sha_v1 or state["applied_pipeline_version"] != 2: raise SystemExit("FAIL version 2 did not replace the served bytes") v1_job = [item["job_id"] for item in state["jobs"] if item["pipeline_version"] == 1][0] expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % v1_job)[0], "stale redelivery") time.sleep(2) after_stale = wait_applied(api, first["id"], token, 2) state = jobs(api, first["id"]) if thumb(after_stale) != sha_v2 or state["applied_pipeline_version"] != 2: raise SystemExit("FAIL stale job overwrote version 2") report["cases"].append( { "name": "reorder_and_stale_version", "attachment_id": first["id"], "version1_sha256": sha_v1, "version2_sha256": sha_v2, "applied_pipeline_version": state["applied_pipeline_version"], "variant_rows": len(state["variants"]), } ) def crashed(mode: str): expect(200, api.call("POST", "/api/v1/_lab/faults", {"mode": mode, "remaining": 1})[0], mode) created = upload(api, chat_id, token, mode + ".png") # The worker process exits. Docker restarts it; the lease must expire before retry. body = wait_applied(api, created["id"], token, 1, seconds=180) if body["status"] != "ready": raise SystemExit("FAIL %s ended %s" % (mode, body["status"])) status_code, _, payload = api.call("GET", "/api/v1/attachments/%s/content" % created["id"], token=token) expect(200, status_code, mode + " original") if hashlib.sha256(payload).hexdigest() != original: raise SystemExit("FAIL %s lost the original" % mode) state_now = jobs(api, created["id"]) rows = [item for item in state_now["variants"] if item["pipeline_version"] == 1] if len(rows) != 3: raise SystemExit("FAIL %s variant rows %s" % (mode, len(rows))) report["cases"].append( { "name": mode, "attachment_id": created["id"], "original_sha256": original, "variant_rows": len(rows), "thumbnail_sha256": thumb(body), } ) crashed("crash_before_s3") crashed("crash_after_s3") expect(200, api.call("POST", "/api/v1/_lab/faults", {"mode": "fail_once", "remaining": 1})[0], "fail_once") failed = upload(api, chat_id, token, "fail.png") failed_body = wait_applied(api, failed["id"], token, 1, allow_failed=True) if failed_body["status"] != "failed" or failed_body["error_code"] != "processing_failed": raise SystemExit("FAIL expected a terminal processing_failed") expect(200, api.call("POST", "/api/v1/_lab/attachments/%s/replay" % failed["id"])[0], "replay") restored = wait_applied(api, failed["id"], token, 1) if restored["status"] != "ready": raise SystemExit("FAIL replay did not reach ready") state = jobs(api, failed["id"]) replay_job = state["jobs"][0]["job_id"] expect(202, api.call("POST", "/api/v1/_lab/jobs/%s/redeliver" % replay_job)[0], "replay duplicate") time.sleep(2) state = jobs(api, failed["id"]) if len(state["variants"]) != 3 or state["jobs"][0]["status"] != "ready": raise SystemExit("FAIL replay produced another effect") report["cases"].append( { "name": "replay_then_duplicate", "attachment_id": failed["id"], "variant_rows": len(state["variants"]), "job_status": state["jobs"][0]["status"], } ) report["not_exactly_once"] = ( "The broker delivers a message more than once. A crash after the object PUT and before ack " "runs the S3 write again. The database commit of a pipeline version happens once, and an older " "version cannot replace the applied one. That is idempotent effect, not exactly-once delivery." ) OUT.parent.mkdir(exist_ok=True) OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print("PASS track2", OUT) if __name__ == "__main__": main()