From e453ee23125ef1d489bdc67e71fd0ded3a0c1c87 Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:17:16 +0300 Subject: [PATCH] =?UTF-8?q?=D1=82=D1=80=D0=B5=D0=BA=202:=20read-only=20?= =?UTF-8?q?=D0=B0=D1=83=D0=B4=D0=B8=D1=82=20=D0=B8=D0=BD=D0=B2=D0=B0=D1=80?= =?UTF-8?q?=D0=B8=D0=B0=D0=BD=D1=82=D0=BE=D0=B2=20I1-I7=20=D0=B8=20=D0=B0?= =?UTF-8?q?=D0=B2=D1=82=D0=BE=D0=BC=D0=B0=D1=82=D0=B8=D1=87=D0=B5=D1=81?= =?UTF-8?q?=D0=BA=D0=B0=D1=8F=20=D0=BC=D0=B0=D1=82=D1=80=D0=B8=D1=86=D0=B0?= =?UTF-8?q?=20=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 audit: БД только READ ONLY-транзакциями, S3 ключом auditor (Read+List): потерянная работа, один эффект на версию, опубликованные объекты и SHA-256, монотонная версия, источник у активной работы, посторонние объекты, попытки. pipeline_faults: дубль, reorder, crash relay после confirm, crash worker до/после побочного эффекта и после COMMIT, временные ошибки, два worker, устаревшая версия (v2 медленная, v3 быстрая; дубль v1), reorder версий в очереди, зависший worker (docker pause). --- bench/pipeline_faults.py | 309 +++++++++++++++++++++++++++++++++++++++ src/messenger/audit.py | 190 ++++++++++++++++++++++++ 2 files changed, 499 insertions(+) create mode 100644 bench/pipeline_faults.py create mode 100644 src/messenger/audit.py diff --git a/bench/pipeline_faults.py b/bench/pipeline_faults.py new file mode 100644 index 0000000..ea225a7 --- /dev/null +++ b/bench/pipeline_faults.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +"""Track 2 experiment: proven idempotency of the versioned pipeline under injected faults. + +Hypothesis. With at-least-once delivery, duplicated and reordered messages, crashes of the worker +before/after its side effects and stale jobs of an older pipeline version: + H1 no accepted (202) work is lost - every upload ends ready (or failed with a code, never stuck); + H2 every effect (attachment, pipeline_version, variant) is published exactly once - one set of + variant rows written by ONE successful attempt, one set of objects in the bucket; + H3 the published version only grows - an older job never overwrites a newer result, `ready` + never goes back to queued/processing; + H4 objects of interrupted attempts are garbage that the cleanup removes; nothing published is + ever deleted. + +Method. For every scenario the affected service (relay/worker) is recreated with FAULT_POINTS of +the scenario (the real code paths, real crashes = os._exit of the process, real `docker pause`), +N uploads go through the LB, then the scenario-specific facts are read from PostgreSQL and S3 and +the read-only invariant audit (messenger.audit, I1-I7, with SHA-256 of every published object) +runs over the WHOLE stand. The normal configuration is restored after each scenario. + + python bench/pipeline_faults.py --output evidence/track2 [--uploads 5] +""" +from __future__ import annotations + +import argparse +import datetime as dt +import json +import platform +import statistics +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +import imagegen # noqa: E402 +from conftest import (Api, attachment_row, compose, events_of, jobctl, psql, s3_keys, service_with_env, # noqa: E402 + wait_healthy) + +FAST = {"JOB_LEASE_SECONDS": "6", "ORPHAN_GRACE_SECONDS": "3", "CLEANUP_INTERVAL_SECONDS": "1"} + + +def now() -> str: + return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def wait(condition, timeout: float, what: str): + deadline = time.monotonic() + timeout + while True: + value = condition() + if value: + return value + if time.monotonic() > deadline: + raise AssertionError(what) + time.sleep(0.5) + + +def facts(attachment_id: str, objects: set[str]) -> dict: + row = attachment_row(attachment_id) + out = psql("SELECT json_build_object(" + "'claims', (SELECT count(*) FROM job_attempts a JOIN jobs j ON j.id = a.job_id WHERE j.attachment_id = '{0}'), " + "'succeeded_attempts', (SELECT count(*) FROM job_attempts a JOIN jobs j ON j.id = a.job_id " + " WHERE j.attachment_id = '{0}' AND a.outcome = 'succeeded'), " + "'variant_rows', (SELECT count(*) FROM attachment_variants WHERE attachment_id = '{0}'), " + "'versions', (SELECT coalesce(json_agg(DISTINCT pipeline_version), '[]') FROM attachment_variants " + " WHERE attachment_id = '{0}'), " + "'job_states', (SELECT coalesce(json_agg(state || ':v' || pipeline_version ORDER BY created_at), '[]') " + " FROM jobs WHERE attachment_id = '{0}'), " + "'uncleaned_failed_attempts', (SELECT count(*) FROM job_attempts a JOIN jobs j ON j.id = a.job_id " + " WHERE j.attachment_id = '{0}' AND a.outcome <> 'succeeded' AND a.objects_cleaned_at IS NULL))" + .format(attachment_id)) + f = json.loads(out) + f.update(status=row["status"], published_version=row["published_version"], staging_state=row["staging_state"], + objects=len([k for k in objects if k.startswith(f"a/{attachment_id}/")])) + f["events"] = [e["event"] for e in events_of(attachment_id)] + return f + + +def audit() -> dict: + r = compose("run", "--rm", "--no-deps", "-T", "auditor", "--verify-checksums", check=False, timeout=600) + report = json.loads(r.stdout) + return {"ok": report["ok"], "violations": report["violations"], "notes": len(report["notes"]), + "objects": report["objects"]} + + +class Run: + def __init__(self, api: Api, uploads: int): + self.api = api + self.n = uploads + self.alice, self.bob = api.user("t2a"), api.user("t2b") + self.chat = api.chat(self.alice, self.bob) + + def upload_many(self, n: int | None = None) -> list[dict]: + return [self.api.upload(self.alice, self.chat, imagegen.png(60 + i, 40, noise=True)).json() + for i in range(n or self.n)] + + def finish(self, atts: list[dict], timeout: float = 180) -> tuple[list[dict], list[float]]: + started = time.monotonic() + took = [] + for a in atts: + self.api.wait_status(self.bob, a["id"], timeout=timeout) + took.append(round(time.monotonic() - started, 2)) + return atts, took + + +def expect_one_effect(f: dict, version: int = 1) -> list[str]: + """H1 + H2 for a plain upload of pipeline `version`.""" + problems = [] + if f["status"] != "ready": + problems.append(f"status {f['status']}") + if f["succeeded_attempts"] != 1 or f["variant_rows"] != 3 or f["published_version"] != version: + problems.append(f"effect {f['succeeded_attempts']} attempts/{f['variant_rows']} rows/v{f['published_version']}") + if f["objects"] != 4: # original + thumbnail + crop + optimized, cleanup done + problems.append(f"{f['objects']} objects in the bucket") + return problems + + +def plain_scenario(run: Run, name: str, service: str, env: dict, *, scale: int | None = None, + cleanup_wait: bool = True) -> dict: + with service_with_env(service, scale=scale, **env): + atts, took = run.finish(run.upload_many()) + if cleanup_wait: # garbage of interrupted attempts: cleaned after ORPHAN_GRACE_SECONDS + try: + wait(lambda: all(json.loads(psql( + "SELECT json_build_object('n', count(*)) FROM job_attempts a JOIN jobs j ON j.id = a.job_id " + f"WHERE j.attachment_id = '{a['id']}' AND a.outcome <> 'succeeded' " + "AND a.objects_cleaned_at IS NULL"))["n"] == 0 for a in atts), 60, "cleanup") + except AssertionError: + pass # reported below as uncleaned attempts + objects = s3_keys("a/") + per = [facts(a["id"], objects) for a in atts] + problems = {a["id"]: p for a, f in zip(atts, per) if (p := expect_one_effect(f))} + return {"scenario": name, "faults": {service: env}, "uploads": len(atts), + "ready": sum(f["status"] == "ready" for f in per), "claims_total": sum(f["claims"] for f in per), + "max_claims": max(f["claims"] for f in per), "time_to_ready_s": {"p50": statistics.median(took), + "max": max(took)}, + "event_sequences": sorted({" > ".join(f["events"]) for f in per}), "problems": problems} + + +def versions_scenario(run: Run) -> dict: + """Stale job after a newer version: v2 (slow) and v3 run at the same time; v3 publishes first, + v2 finishes later and must end superseded without touching the published v3 result.""" + att = run.upload_many(1)[0] + run.api.wait_status(run.bob, att["id"], "ready") + observed = set() + with service_with_env("worker", WORKER_FAULT_POINTS="slow=10,slow_pipeline=2"): + jobctl("reprocess", att["id"], "--pipeline", "2", "--operator", "track2", "--reason", "new encoder") + wait(lambda: "processing:v2" in facts(att["id"], set())["job_states"], 60, "v2 not claimed") + jobctl("reprocess", att["id"], "--pipeline", "3", "--operator", "track2", "--reason", "newer encoder") + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + meta = run.api.attachment(run.bob, att["id"]) + observed.add(meta["status"]) + states = facts(att["id"], set())["job_states"] + if not any(s.startswith(("queued", "processing")) for s in states): + break + time.sleep(0.3) + final = run.api.attachment(run.bob, att["id"]) + f = facts(att["id"], s3_keys("a/")) + thumb_v3 = psql(f"SELECT encode(sha256, 'hex') FROM attachment_variants WHERE attachment_id = '{att['id']}' " + "AND pipeline_version = 3 AND name = 'thumbnail'") + problems = [] + if f["job_states"] != ["succeeded:v1", "superseded:v2", "succeeded:v3"]: + problems.append(f"job states {f['job_states']}") + if f["published_version"] != 3 or observed != {"ready"}: + problems.append(f"published v{f['published_version']}, statuses seen {sorted(observed)}") + if next(v["sha256"] for v in final["variants"] if v["name"] == "thumbnail") != thumb_v3: + problems.append("published thumbnail is not the v3 one") + # Stale duplicate of the FIRST job, published by hand with the relay's broker user: a no-op. + job_v1 = psql(f"SELECT id FROM jobs WHERE attachment_id = '{att['id']}' AND pipeline_version = 1") + script = ("import asyncio, aio_pika, datetime as d\n" + "from messenger import broker\n" + "async def m():\n" + " c = await aio_pika.connect(open('/run/secrets/broker_url').read().strip())\n" + " ch = await c.channel(publisher_confirms=True)\n" + " ex = await ch.get_exchange(broker.EXCHANGE, ensure=False)\n" + f" body = broker.encode('{job_v1}', '{att['id']}', 1, 1, d.datetime.now(d.timezone.utc))\n" + " await ex.publish(aio_pika.Message(body), routing_key=broker.ROUTING_KEY)\n" + " await c.close()\n" + "asyncio.run(m())\n") + compose("exec", "-T", "relay", "python", "-c", script) + time.sleep(3) + after = run.api.attachment(run.bob, att["id"]) + if after != final: + problems.append("stale v1 message changed the attachment") + return {"scenario": "stale_version_and_reorder", "uploads": 1, "statuses_seen_during_reprocess": sorted(observed), + "job_states": f["job_states"], "published_version": f["published_version"], + "variant_rows_by_version": f["variant_rows"], "objects": f["objects"], + "events": f["events"], "problems": problems} + + +def queued_reorder_scenario(run: Run) -> dict: + """Both newer versions wait in the outbox; the relay publishes them in REVERSE order and one worker + takes them one by one: v3 is published first, v2 is then superseded at its claim (no work).""" + att = run.upload_many(1)[0] + run.api.wait_status(run.bob, att["id"], "ready") + # The worker stays stopped until it is recreated with concurrency 1 (no normal start in between). + # Worker and relay stay stopped until each is recreated with the scenario's settings: no normal + # start in between that would consume/publish the jobs in the ordinary order. + compose("stop", "worker", "relay") + try: + jobctl("reprocess", att["id"], "--pipeline", "2", "--operator", "track2", "--reason", "v2") + jobctl("reprocess", att["id"], "--pipeline", "3", "--operator", "track2", "--reason", "v3") + with service_with_env("relay", RELAY_FAULT_POINTS="reverse_order"): + wait(lambda: psql("SELECT count(*) FROM outbox o JOIN jobs j ON j.id = o.job_id WHERE j.attachment_id = " + f"'{att['id']}' AND j.pipeline_version > 1 AND o.published_at IS NOT NULL") == "2", + 30, "both reprocess jobs not published") # in one relay batch: v3 first, then v2 + with service_with_env("worker", WORKER_CONCURRENCY="1"): + wait(lambda: not any(s.startswith(("queued", "processing")) + for s in facts(att["id"], set())["job_states"]), 120, "reprocess jobs did not finish") + finally: + compose("start", "worker", "relay") + wait_healthy("worker", "relay") + f = facts(att["id"], s3_keys("a/")) + problems = [] + if f["job_states"] != ["succeeded:v1", "superseded:v2", "succeeded:v3"] or f["published_version"] != 3: + problems.append(f"{f['job_states']} published v{f['published_version']}") + if f["claims"] != 2: # v1 and v3 only: v2 was superseded before any work + problems.append(f"{f['claims']} claims") + return {"scenario": "reordered_versions_in_queue", "job_states": f["job_states"], "claims": f["claims"], + "published_version": f["published_version"], "events": f["events"], "problems": problems} + + +def hung_worker_scenario(run: Run) -> dict: + with service_with_env("worker", scale=2, WORKER_FAULT_POINTS="slow=8", **FAST): + att = run.upload_many(1)[0] + owner = wait(lambda: psql(f"SELECT lease_owner FROM jobs WHERE attachment_id = '{att['id']}' " + "AND state = 'processing'"), 30, "not claimed") + ids = compose("ps", "-q", "worker").stdout.split() + names = dict(line.split() for line in subprocess.run( + ["docker", "inspect", "-f", "{{.Config.Hostname}} {{.Id}}", *ids], capture_output=True, text=True, + check=True).stdout.splitlines()) + subprocess.run(["docker", "pause", names[owner]], check=True) + try: + run.api.wait_status(run.bob, att["id"], "ready", timeout=150) + published = psql(f"SELECT string_agg(object_key, ',' ORDER BY object_key) FROM attachment_variants " + f"WHERE attachment_id = '{att['id']}'") + finally: + subprocess.run(["docker", "unpause", names[owner]], check=True) + time.sleep(15) + again = psql(f"SELECT string_agg(object_key, ',' ORDER BY object_key) FROM attachment_variants " + f"WHERE attachment_id = '{att['id']}'") + wait(lambda: json.loads(psql("SELECT json_build_object('n', count(*)) FROM job_attempts a JOIN jobs j " + f"ON j.id = a.job_id WHERE j.attachment_id = '{att['id']}' AND a.outcome <> " + "'succeeded' AND a.objects_cleaned_at IS NULL"))["n"] == 0, 60, "cleanup") + f = facts(att["id"], s3_keys("a/")) + problems = expect_one_effect(f) + if published != again: + problems.append("the resumed (fenced) attempt changed the published result") + return {"scenario": "hung_worker_docker_pause", "frozen": owner, "claims": f["claims"], "events": f["events"], + "problems": problems} + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--output", default="evidence/track2") + p.add_argument("--uploads", type=int, default=5) + args = p.parse_args() + out = ROOT / args.output + out.mkdir(parents=True, exist_ok=True) + api = Api() + run = Run(api, args.uploads) + started = now() + results = [] + plan = [ + ("baseline_no_faults", "worker", {}), + ("duplicate_publish", "relay", {"RELAY_FAULT_POINTS": "duplicate_publish"}), + ("duplicate_and_reverse_order", "relay", {"RELAY_FAULT_POINTS": "duplicate_publish,reverse_order"}), + ("relay_crash_after_confirm_before_mark", "relay", {"RELAY_FAULT_POINTS": "crash_after_publish"}), + ("worker_crash_before_side_effect", "worker", {"WORKER_FAULT_POINTS": "crash_after_claim", **FAST}), + ("worker_crash_after_s3_before_commit", "worker", {"WORKER_FAULT_POINTS": "crash_after_s3_write", **FAST}), + ("worker_crash_after_commit_before_ack", "worker", {"WORKER_FAULT_POINTS": "crash_after_commit", **FAST}), + ("transient_errors_then_success", "worker", {"WORKER_FAULT_POINTS": "transient_error=2", + "JOB_RETRY_BASE_SECONDS": "0.5", **FAST}), + ] + for name, service, env in plan: + print(f"[{now()}] {name}", flush=True) + results.append(plain_scenario(run, name, service, env)) + print(f"[{now()}] two_workers_duplicate_publish", flush=True) + with service_with_env("relay", RELAY_FAULT_POINTS="duplicate_publish"): + results.append(plain_scenario(run, "two_workers_duplicate_publish", "worker", {}, scale=2)) + for scenario in (versions_scenario, queued_reorder_scenario, hung_worker_scenario): + print(f"[{now()}] {scenario.__name__}", flush=True) + results.append(scenario(run)) + wait_healthy("worker", "relay") + report = audit() + summary = { + "experiment": "track 2: idempotency of the versioned pipeline under injected faults", + "started": started, "finished": now(), + "host": {"os": platform.platform(), "python": platform.python_version()}, + "uploads_per_plain_scenario": args.uploads, + "scenarios": results, + "audit_after_all": report, + "violations": sum(len(r["problems"]) for r in results) + len(report["violations"]), + } + (out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=1), encoding="utf-8") + lines = [f"{'scenario':45} {'uploads':>7} {'ready':>5} {'claims':>6} problems"] + for r in results: + lines.append(f"{r['scenario']:45} {r.get('uploads', 1):>7} {r.get('ready', '-'):>5} " + f"{r.get('claims_total', r.get('claims', '-')):>6} {len(r['problems'])}") + lines.append(f"audit after all scenarios: ok={report['ok']} violations={len(report['violations'])} " + f"objects={report['objects']}") + (out / "summary.txt").write_text("\n".join(lines) + "\n", encoding="utf-8") + print("\n".join(lines)) + return 0 if summary["violations"] == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/audit.py b/src/messenger/audit.py new file mode 100644 index 0000000..a555817 --- /dev/null +++ b/src/messenger/audit.py @@ -0,0 +1,190 @@ +"""Track 2: read-only checker of the pipeline invariants: `python -m messenger.audit` +(compose service `auditor`, profile tools; `make audit`). + +Read-only by construction: every database transaction is `READ ONLY` (PostgreSQL refuses any +write), and the S3 key pair of the auditor has Read + List only. Output: one JSON report; exit 1 +if any invariant is violated. + +Invariants (docs/PIPELINE.md): + I1 no lost work - a queued/processing attachment has an active job; an active job has an + outbox row (published or due) - nothing waits for a message that nobody sends. + I2 one effect - per (attachment, pipeline_version) either no variant rows or exactly the + three variants, all written by ONE successful attempt of a job of that version. + I3 published data - every published object (original, variants) exists with the stored size + (--verify-checksums: and the stored SHA-256). + I4 monotonic - published_version = the highest version with a succeeded job; a ready + attachment is never queued/processing again (published_version stays set). + I5 source kept - an active job whose original is not published still has its staging copy + (state present AND the object exists in the staging bucket). + I6 no stray data - every object in the buckets is referenced: published, staging of a live row, + or planned by an attempt whose objects were not cleaned yet. An object of an + already cleaned attempt is reported as `leaked_object` (written by a stalled + attempt after its cleanup: garbage, never a loss). + I7 attempts - a succeeded job has exactly one succeeded attempt; at most one running attempt + per job, and it is the lease holder. +""" +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import sys + +from .config import ConfigError, Settings +from .db import Database +from .errors import Unavailable +from .pipeline import VARIANT_NAMES +from .storage import ObjectMissing, SENTINEL_KEY, Storage + + +async def rows(db: Database, sql: str, params=()) -> list[tuple]: + async def work(conn): + await conn.execute("SET TRANSACTION READ ONLY") + cur = await conn.execute(sql, params) + return await cur.fetchall() + return await db.run(work, deadline=db.deadline) + + +async def list_all(storage: Storage, prefix: str) -> dict[str, int]: + found, after = {}, None + while True: + page, more = await storage.list_page(prefix, after, 1000) + for item in page: + found[item.key] = item.size + if not more or not page: + return found + after = page[-1].key + + +async def audit(settings: Settings, verify: bool) -> dict: + db = Database(settings, name="messenger-audit") + final = Storage(settings) + staging = Storage(settings, bucket=settings.s3_staging_bucket) + await db.open() + await final.open() + await staging.open() + violations: list[dict] = [] + notes: list[dict] = [] + + def bad(kind: str, **detail) -> None: + violations.append({"invariant": kind, **{k: str(v) for k, v in detail.items()}}) + + try: + # I1: queued/processing attachment without an active job; active job without outbox row. + for (aid,) in await rows(db, "SELECT a.id FROM attachments a WHERE a.status IN ('queued', 'processing') " + "AND NOT EXISTS (SELECT 1 FROM jobs j WHERE j.attachment_id = a.id " + "AND j.state IN ('queued', 'processing'))"): + bad("I1_no_active_job", attachment_id=aid) + for (jid,) in await rows(db, "SELECT j.id FROM jobs j WHERE j.state = 'queued' AND NOT EXISTS " + "(SELECT 1 FROM outbox o WHERE o.job_id = j.id)"): + bad("I1_queued_without_outbox", job_id=jid) + + # I2: exactly three variants per published version, one successful attempt of that version. + for aid, version, names, tokens in await rows( + db, "SELECT attachment_id, pipeline_version, array_agg(name ORDER BY name), " + "array_agg(DISTINCT attempt_token) FROM attachment_variants GROUP BY 1, 2"): + if sorted(names) != sorted(VARIANT_NAMES): + bad("I2_variant_set", attachment_id=aid, version=version, names=names) + if len(tokens) != 1: + bad("I2_mixed_attempts", attachment_id=aid, version=version, attempts=len(tokens)) + for aid, version, outcome, jversion in await rows( + db, "SELECT v.attachment_id, v.pipeline_version, ja.outcome, j.pipeline_version " + "FROM attachment_variants v JOIN job_attempts ja ON ja.token = v.attempt_token " + "JOIN jobs j ON j.id = ja.job_id WHERE ja.outcome <> 'succeeded' OR j.pipeline_version " + "<> v.pipeline_version OR j.attachment_id <> v.attachment_id"): + bad("I2_variant_from_unsuccessful_attempt", attachment_id=aid, version=version, outcome=outcome) + + # I4: published_version = highest succeeded version; ready <=> published. + for aid, status, published, best in await rows( + db, "SELECT a.id, a.status, a.published_version, (SELECT max(j.pipeline_version) FROM jobs j " + "WHERE j.attachment_id = a.id AND j.state = 'succeeded') FROM attachments a " + "WHERE a.published_version IS DISTINCT FROM (SELECT max(j.pipeline_version) FROM jobs j " + "WHERE j.attachment_id = a.id AND j.state = 'succeeded') " + "OR (a.published_version IS NOT NULL AND a.status <> 'ready')"): + bad("I4_published_version", attachment_id=aid, status=status, published=published, best_succeeded=best) + + # I7: attempts. + for (jid,) in await rows(db, "SELECT j.id FROM jobs j WHERE j.state = 'succeeded' AND (SELECT count(*) " + "FROM job_attempts a WHERE a.job_id = j.id AND a.outcome = 'succeeded') <> 1"): + bad("I7_succeeded_attempts", job_id=jid) + for jid, token in await rows(db, "SELECT a.job_id, a.token FROM job_attempts a JOIN jobs j ON j.id = a.job_id " + "WHERE a.outcome = 'running' AND j.lease_token IS DISTINCT FROM a.token"): + bad("I7_running_not_holder", job_id=jid, token=token) + + # I3 + I5 + I6: storage. + published = {k: (s, h) for k, s, h in await rows( + db, "SELECT object_key, size_bytes, sha256 FROM attachment_variants UNION ALL " + "SELECT object_key, size_bytes, sha256 FROM attachments WHERE object_key IS NOT NULL")} + staging_live = {k: (s, st, status) for k, s, st, status in await rows( + db, "SELECT staging_key, size_bytes, staging_state, status FROM attachments WHERE staging_key IS NOT NULL")} + planned_open, planned_cleaned = set(), set() + for keys, cleaned in await rows(db, "SELECT planned_keys, objects_cleaned_at IS NOT NULL FROM job_attempts"): + (planned_cleaned if cleaned else planned_open).update(keys) + final_objects = await list_all(final, "") + staging_objects = await list_all(staging, "") + for key, (size, sha) in published.items(): + if key not in final_objects: + bad("I3_published_missing", key=key) + elif final_objects[key] != size: + bad("I3_published_size", key=key, stored=size, actual=final_objects[key]) + elif verify: + data = await final.read_all(key, limit=settings.max_upload_bytes) + if hashlib.sha256(data).digest() != bytes(sha): + bad("I3_published_checksum", key=key) + for key, (size, state, status) in staging_live.items(): + # 'pending' may lack its object (the PUT failed: the janitor removes the row); accepted + # work (queued/processing) must have it. + active = status in ("queued", "processing") + if state == "present" and key not in staging_objects and active: + bad("I5_staging_missing", key=key, status=status) + for (aid,) in await rows(db, "SELECT a.id FROM attachments a WHERE a.status IN ('queued', 'processing') " + "AND a.object_key IS NULL AND a.staging_state <> 'present'"): + bad("I5_active_without_source", attachment_id=aid) + for key in final_objects: + if key == SENTINEL_KEY or key in published or key in planned_open: + continue + if key in planned_cleaned: + notes.append({"kind": "leaked_object", "key": key}) + else: + bad("I6_unknown_final_object", key=key) + for key in staging_objects: + if key == SENTINEL_KEY: + continue + row = staging_live.get(key) + if row is None: + bad("I6_unknown_staging_object", key=key) + elif row[1] == "deleted": + notes.append({"kind": "staging_marked_deleted_still_present", "key": key}) + counts = dict(await rows(db, "SELECT status, count(*) FROM attachments GROUP BY status")) + jobs = dict(await rows(db, "SELECT state, count(*) FROM jobs GROUP BY state")) + return {"ok": not violations, "violations": violations, "notes": notes, + "attachments": counts, "jobs": jobs, + "objects": {"final": len(final_objects), "staging": len(staging_objects), + "published": len(published), "checksums_verified": verify}} + finally: + await staging.close() + await final.close() + await db.close() + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(prog="audit", description="Read-only check of the pipeline invariants") + p.add_argument("--verify-checksums", action="store_true", help="read every published object and compare SHA-256") + args = p.parse_args(argv) + try: + settings = Settings.from_env() + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + try: + report = asyncio.run(audit(settings, args.verify_checksums)) + except (Unavailable, ObjectMissing) as exc: + print(json.dumps({"ok": False, "error": "store_unavailable", "detail": type(exc).__name__})) + return 1 + print(json.dumps(report, ensure_ascii=False, indent=1, default=str)) + return 0 if report["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main())