"""Real S3 retention test: seed is separate from verification output.""" import argparse import json import time from datetime import datetime, timezone from pathlib import Path from uuid import uuid4 import httpx from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready from scripts.verify_pipeline import inside, operator, wait def seed(path): if path.exists(): raise ValueError("Choose a new seed file") ready() compose("stop", "--timeout", "1", "reconciler") raw = (ROOT / "tests/fixtures/sample.png").read_bytes() with httpx.Client(base_url=BASE, trust_env=False, timeout=20) as c: user, _, session = actor(c) headers = auth(session) result = c.post( "/api/v1/chats", headers=headers, json={ "title": "Cleanup synthetic seed", "member_ids": [user["id"]], }, ) assert result.status_code == 201 chat = result.json()["id"] r = c.post( f"/api/v1/chats/{chat}/attachments", headers={**headers, "Content-Type": "image/png"}, params={"filename": "retained.png"}, content=raw, ) assert r.status_code == 202 attachment = r.json()["id"] wait( lambda: ( c.get( "/api/v1/attachments/" + attachment, headers=headers ).json()["status"] == "ready" ) ) job = inside( f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({attachment!r},)).fetchone()\nprint(json.dumps(str(r['id'])))" ) data = { "seed_id": uuid4().hex, "source_commit": command(["git", "rev-parse", "HEAD"]), "date": datetime.now(timezone.utc).isoformat(), "ready_attachment": attachment, "ready_job": job, "chat": chat, "owner": user["id"], } # Real old v1 objects will become obsolete after v2 is ready. second = operator(job, version=2) wait( lambda: inside( f"with db.connection() as c: r=c.execute('SELECT state FROM jobs WHERE id=%s',({second!r},)).fetchone()\nprint(json.dumps(r['state']=='ready'))" ) ) data["version2_job"] = second data["orphan"] = f"objects/cleanup-{data['seed_id']}/old-orphan" data["failed_id"] = str(uuid4()) data["active_id"] = str(uuid4()) data["reservation_id"] = str(uuid4()) data["expired_reservation_id"] = str(uuid4()) data["failed_source"] = f"objects/staging/{data['failed_id']}/original" data["active_source"] = f"objects/staging/{data['active_id']}/original" data["reservation_key"] = ( f"objects/staging/{data['reservation_id']}/original" ) data["expired_reservation_key"] = ( f"objects/staging/{data['expired_reservation_id']}/original" ) inside(f"""from pathlib import Path raw=Path('/app/src/__init__.py').read_bytes() or b'fixture' keys={json.dumps([data[k] for k in ["orphan", "failed_source", "active_source", "reservation_key", "expired_reservation_key"]])} for key in keys: store.put(key=key,stream=io.BytesIO(raw),size=len(raw),content_type='image/png',sha256=hashlib.sha256(raw).hexdigest()) with db.connection() as c, c.transaction(): for identity,state in [({data["failed_id"]!r},'failed'),({data["active_id"]!r},'processing')]: key='objects/staging/'+identity+'/original' c.execute(\"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,error_code,uploader_instance,expires_at) VALUES(%s,%s,%s,'synthetic.png','image/png',%s,%s,%s,%s,%s,'cleanup-fixture',clock_timestamp())\",(identity,{chat!r},{user["id"]!r},len(raw),hashlib.sha256(raw).hexdigest(),key,state,'invalid_image' if state=='failed' else None)) c.execute(\"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state,attempt,lease_until,updated_at,error_code) VALUES(%s,%s,1,%s,%s,1,clock_timestamp()+interval '1 hour',clock_timestamp()-interval '2 days',%s)\",(uuid4(),identity,key,state,'invalid_image' if state=='failed' else None)) c.execute(\"INSERT INTO upload_reservations(id,object_key,expires_at) VALUES(%s,%s,clock_timestamp()+interval '1 hour'),(%s,%s,clock_timestamp()-interval '1 second')\",({data["reservation_id"]!r},{data["reservation_key"]!r},{data["expired_reservation_id"]!r},{data["expired_reservation_key"]!r})) print('true')""") data["seeded_unix"] = time.time() path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n") print( "PASS: retention seed created; real S3 age must reach300s", flush=True ) def verify(seed_path, output): if output.exists(): raise ValueError("Choose a new output path") data = json.loads(seed_path.read_text()) report = { "source_commit": command(["git", "rev-parse", "HEAD"]), "seed": seed_path.name, "complete": False, } try: compose("stop", "--timeout", "1", "reconciler") def age(): return inside( f"h=store.head(key={data['orphan']!r})\nassert h is not None\nwith db.connection() as c: now=c.execute('SELECT clock_timestamp() now').fetchone()['now']\nprint(json.dumps((now-h.last_modified).total_seconds()))" ) while age() < 302: time.sleep(min(5, 303 - age())) report["real_s3_age_seconds"] = age() fresh = f"objects/cleanup-{data['seed_id']}/fresh-orphan" inside( f"b=b'fresh'\nstore.put(key={fresh!r},stream=io.BytesIO(b),size=len(b),content_type='image/png',sha256=hashlib.sha256(b).hexdigest())\nprint('true')" ) oldkeys = [ data["orphan"], data["failed_source"], data["expired_reservation_key"], ] + [ f"objects/{data['ready_attachment']}/v1/{v}" for v in ["original", "thumbnail", "crop", "optimized"] ] protected = [data["active_source"], data["reservation_key"], fresh] + [ f"objects/{data['ready_attachment']}/v2/{v}" for v in ["original", "thumbnail", "crop", "optimized"] ] before = inside( f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))" ) assert all(before.values()) dry = inside( "print(json.dumps(reconcile(db,store,apply=False,grace=300)))" ) candidates = {a["key"] for a in dry["actions"]} assert set(oldkeys) <= candidates and not set(protected) & candidates afterdry = inside( f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))" ) assert afterdry == before applied = inside( "print(json.dumps(reconcile(db,store,apply=True,grace=300)))" ) repeat = inside( "print(json.dumps(reconcile(db,store,apply=True,grace=300)))" ) assert not repeat["actions"] after = inside( f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))" ) assert all(not after[k] for k in oldkeys) and all( after[k] for k in protected ) failed = inside( f"with db.connection() as c: j=c.execute('SELECT id,error_code FROM jobs WHERE attachment_id=%s',({data['failed_id']!r},)).fetchone()\nprint(json.dumps(j,default=str))" ) assert failed["error_code"] == "source_expired" try: operator(failed["id"]) except RuntimeError: replay_refused = True else: raise AssertionError("Expired source replay must fail") # Expired reservation was removed, so a delayed API cannot commit its handoff. fenced = inside( f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp()',({data['expired_reservation_id']!r},)).fetchone()\nprint(json.dumps(r['n']==0))" ) assert fenced report.update( dry_run=dry, apply=applied, repeat=repeat, before=before, after=after, dry_run_unchanged=True, active_current_and_fresh_protected=True, expired_source_replay_refused=replay_refused, expired_writer_fenced=fenced, complete=True, ) finally: # Remove only this fixture's artificial active reservation/job; never real jobs. inside( f"with db.connection() as c, c.transaction():\n c.execute(\"UPDATE jobs SET state='failed',error_code='fixture_complete',lease_until=NULL WHERE attachment_id=%s\",({data['active_id']!r},))\n c.execute(\"UPDATE attachments SET state='failed',error_code='fixture_complete' WHERE id=%s\",({data['active_id']!r},))\n c.execute('DELETE FROM upload_reservations WHERE id=%s',({data['reservation_id']!r},))\nprint('true')" ) compose("up", "-d", "--no-deps", "--no-recreate", "reconciler") output.parent.mkdir(parents=True, exist_ok=True) output.write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n" ) print( "PASS: real retention, dry-run/apply/repeat, current/active/fresh protection", flush=True, ) def main(): p = argparse.ArgumentParser() p.add_argument("action", choices=["seed", "verify"]) p.add_argument("--seed", required=True, type=Path) p.add_argument("--output", type=Path) a = p.parse_args() if a.action == "seed": seed(a.seed) else: if a.output is None: p.error("output required") verify(a.seed, a.output) if __name__ == "__main__": main()