#!/usr/bin/env python3 """Two-phase real-age GC check: seed, run other checks, then verify after >=300 seconds.""" import argparse import hashlib import io import json import sys import time from datetime import UTC, datetime from pathlib import Path from uuid import uuid4 from cluster import ROOT, compose from pipeline_lab import settings from messenger.cleanup import collect from messenger.pipeline import Pipeline, intent from messenger.repository import PostgresRepository from messenger.storage import ObjectStore sys.path.insert(0, str(ROOT / "tests")) from client import Client, check # noqa: E402 def main(): parser = argparse.ArgumentParser(__doc__) parser.add_argument("action", choices=("seed", "verify")) parser.add_argument("--seed", type=Path, required=True) parser.add_argument("--output", type=Path) parser.add_argument("--base-url", default="http://127.0.0.1:8085") args = parser.parse_args() config = settings("reconciler") db = PostgresRepository(config.database_url, pool_size=3) db.open() store = ObjectStore(config) try: if args.action == "seed": check(not args.seed.exists(), "Seed already exists") compose("stop", "--timeout", "2", "worker", "relay") try: c = Client(args.base_url) alice, _, _, room = c.bootstrap() active, data = c.attachment(alice, room) failed, _ = c.attachment(alice, room) orphan = str(uuid4()) Pipeline(db, config).reserve(orphan, "staging/" + orphan) uploader = ObjectStore(settings("api")) try: uploader.put( "staging/" + orphan, io.BytesIO(data), len(data), "image/png", hashlib.sha256(data).hexdigest(), ) finally: uploader.close() with db.connection(write=True) as conn: j = conn.execute( "SELECT * FROM attachment_jobs WHERE attachment_id=%s", (active["id"],) ).fetchone() conn.execute( "UPDATE attachment_jobs SET " "next_attempt_at=clock_timestamp()+interval '1 hour' " "WHERE job_id=%s", (j["job_id"],), ) conn.execute( "UPDATE job_outbox SET due_at=clock_timestamp()+interval '1 " "hour' WHERE job_id=%s", (j["job_id"],), ) f = conn.execute( "SELECT job_id FROM attachment_jobs WHERE attachment_id=%s", (failed["id"],) ).fetchone() conn.execute( "INSERT INTO pipeline_faults(job_id,phase,action) VALUES " "(%s,'before_effect','permanent')", (f["job_id"],), ) seed = { "created_at": datetime.now(UTC).isoformat(), "active_id": active["id"], "failed_id": failed["id"], "orphan_id": orphan, } args.seed.write_text(json.dumps(seed, indent=2) + "\n") print( "PASS: active job, failed candidate and unaccepted orphan seeded; no " "credentials saved" ) finally: compose("start", "worker", "relay") else: check(args.output is not None and not args.output.exists(), "Choose a new output file") seed = json.loads(args.seed.read_text()) # Real S3 timestamps and failed retention; no modified clocks. while True: with db.connection() as conn: f = conn.execute( "SELECT * FROM attachment_jobs WHERE attachment_id=%s", (seed["failed_id"],) ).fetchone() check(f["state"] == "failed", "Failed seed did not reach terminal state") age = (datetime.now(UTC) - f["updated_at"]).total_seconds() if age >= 301: break check(age >= 0, "Clock mismatch") time.sleep(min(1, 301 - age)) with db.connection() as conn: ready = conn.execute( "SELECT * FROM attachments WHERE storage_state='ready' LIMIT 1" ).fetchone() check(ready is not None, "Need a published attachment to protect") keys = [ready["object_key"], *[v["object_key"] for v in ready["variants"]]] before = {key: store.head(key)["ETag"] for key in keys} preview = collect(db, store, 300, False) result = collect(db, store, 300, True) check( store.head("staging/" + seed["active_id"]) is not None, "Active staging was deleted" ) check(store.head("staging/" + seed["orphan_id"]) is None, "Orphan not deleted") check( store.head("staging/" + seed["failed_id"]) is None, "Expired failed source not deleted", ) check( all(store.head(k)["ETag"] == tag for k, tag in before.items()), "Published result changed", ) with db.connection(write=True) as conn: expired = conn.execute( "SELECT source_expired FROM attachment_jobs WHERE attachment_id=%s", (seed["failed_id"],), ).fetchone() check(expired["source_expired"], "Expired source not registered") j = conn.execute( "UPDATE attachment_jobs SET next_attempt_at=clock_timestamp() WHERE " "attachment_id=%s RETURNING job_id", (seed["active_id"],), ).fetchone() intent(conn, j["job_id"]) deadline = time.monotonic() + 60 while time.monotonic() < deadline: if db.attachment(seed["active_id"])["storage_state"] == "ready": break time.sleep(0.2) check( db.attachment(seed["active_id"])["storage_state"] == "ready", "Protected work failed to complete", ) proof = { "retention_seconds": 300, "failed_real_age_seconds": age, "seed": seed, "dry_run": preview, "apply": result, "active_protected_then_ready": True, "orphan_deleted": True, "failed_source_expired": True, "published_objects_unchanged": len(keys), } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(proof, indent=2) + "\n") print("PASS: real-age cleanup preserves active work and published results") finally: store.close() db.close() if __name__ == "__main__": main()