Files
mtusi2026lab5/app/sweeper.py

163 lines
7.5 KiB
Python

"""Staging/orphan cleanup with documented retention. Never touches work that is still in progress.
Rules (every step idempotent, repeated runs are safe):
1. staging/<id> of a READY attachment -> delete (the worker deletes it right after COMMIT;
this repairs a crash/failed delete in between)
2. staging/<id> of a FAILED attachment older than STAGING_RETENTION_SECONDS -> delete, mark staging_deleted_at
(until then an operator replay can still use it; afterwards replay ends as failed staging_missing)
3. staging/<id> without any attachment row, older than ORPHAN_GRACE_SECONDS -> delete
(API crashed/failed between the staging PUT and the COMMIT of attachment+job+outbox; no 202 was sent)
4. attachments/<id>/... without any attachment row, or not referenced by any committed row (superseded
older pipeline version, fenced-out attempt) while no job of that attachment is active, older than grace
5. published outbox rows older than 7 days -> delete
Queued/processing attachments are never touched: their staging object is the only copy of accepted work.
python -m app.sweeper --once --dry-run # report only
python -m app.sweeper --once # one real pass
python -m app.sweeper # loop every SWEEP_INTERVAL_SECONDS
"""
from __future__ import annotations
import argparse
import asyncio
import json
import logging
import time
import uuid
from datetime import datetime, timedelta, timezone
import asyncpg
from starlette.concurrency import run_in_threadpool
from app import config
from app.objects import FINAL_PREFIX, STAGING_PREFIX, ObjectStore, S3Unavailable
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
logger = logging.getLogger("sweeper")
LOCK_ID = 2026_0005
HEARTBEAT_FILE = "/tmp/sweeper-heartbeat"
def _id_from_key(key: str, prefix: str):
part = key[len(prefix):].split("/", 1)[0]
try:
return uuid.UUID(hex=part)
except ValueError:
return None
async def sweep(conn: asyncpg.Connection, objects: ObjectStore, grace: int, retention: int, dry_run: bool) -> dict:
started = time.monotonic()
report = {"dry_run": dry_run, "grace_seconds": grace, "retention_seconds": retention,
"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_orphans": 0, "final_orphans": 0,
"outbox_pruned": 0, "objects_scanned": 0}
now = datetime.now(timezone.utc)
cutoff = now - timedelta(seconds=grace)
# 1 + 2: staging of finished attachments
rows = await conn.fetch(
"""
SELECT id, staging_key, status FROM attachments
WHERE staging_deleted_at IS NULL AND staging_key IS NOT NULL
AND (status = 'ready'
OR (status = 'failed' AND updated_at < clock_timestamp() - make_interval(secs => $1)))
""", float(retention))
for row in rows:
report["staging_ready_deleted" if row["status"] == "ready" else "staging_failed_expired"] += 1
if dry_run:
continue
await run_in_threadpool(objects.delete_sync, row["staging_key"])
await conn.execute(
"UPDATE attachments SET staging_deleted_at = clock_timestamp() WHERE id = $1 AND status IN ('ready', 'failed')",
row["id"])
# 3 + 4: objects without any row (no 202 was ever sent for them)
for prefix, counter in ((STAGING_PREFIX, "staging_orphans"), (FINAL_PREFIX, "final_orphans")):
def old_keys():
out = []
for key, modified in objects.list_prefix(prefix):
report["objects_scanned"] += 1
if modified < cutoff:
out.append(key)
return out
keys = await run_in_threadpool(old_keys)
by_id = {}
for key in keys:
ident = _id_from_key(key, prefix)
if ident is not None:
by_id.setdefault(ident, []).append(key)
if not by_id:
continue
known = {r["id"] for r in await conn.fetch("SELECT id FROM attachments WHERE id = ANY($1::uuid[])", list(by_id))}
orphans = [k for ident, ks in by_id.items() if ident not in known for k in ks]
if prefix == FINAL_PREFIX:
# objects of a known attachment that no committed row references: a superseded older pipeline
# version or a fenced-out attempt. Only after the grace period (longer than any live attempt).
candidates = [k for ident, ks in by_id.items() if ident in known for k in ks]
if candidates:
referenced = {r["k"] for r in await conn.fetch(
"""
SELECT object_key AS k FROM attachments WHERE object_key = ANY($1::text[]) AND status = 'ready'
UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY($1::text[])
UNION SELECT a.object_key FROM attachments a WHERE a.object_key = ANY($1::text[])
AND a.status IN ('queued', 'processing')
""", candidates)}
live = {r["id"] for r in await conn.fetch(
"SELECT DISTINCT attachment_id AS id FROM jobs WHERE status IN ('queued', 'processing')")}
orphans += [k for k in candidates if k not in referenced and _id_from_key(k, prefix) not in live]
report[counter] += len(orphans)
if orphans and not dry_run:
await run_in_threadpool(objects.delete_many, orphans)
# 5: outbox history
if not dry_run:
res = await conn.execute(
"DELETE FROM outbox WHERE published_at < clock_timestamp() - interval '7 days'")
report["outbox_pruned"] = int(res.split()[-1])
report["duration_ms"] = round((time.monotonic() - started) * 1000, 1)
return report
async def run_once(grace: int, retention: int, dry_run: bool) -> dict:
conn = await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT)
try:
if not await conn.fetchval("SELECT pg_try_advisory_lock($1)", LOCK_ID):
return {"skipped": "another sweeper holds the lock"}
try:
return await sweep(conn, ObjectStore(), grace, retention, dry_run)
finally:
await conn.execute("SELECT pg_advisory_unlock($1)", LOCK_ID)
finally:
await conn.close()
async def main() -> int:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("--once", action="store_true")
p.add_argument("--dry-run", action="store_true")
p.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS)
p.add_argument("--retention", type=int, default=config.STAGING_RETENTION_SECONDS)
p.add_argument("--interval", type=int, default=config.SWEEP_INTERVAL_SECONDS)
args = p.parse_args()
if args.grace < 60:
p.error("grace below 60 s could remove an upload that is still between staging PUT and COMMIT")
while True:
try:
report = await run_once(args.grace, args.retention, args.dry_run)
print(json.dumps(report, ensure_ascii=False), flush=True)
with open(HEARTBEAT_FILE, "w", encoding="ascii") as fh:
fh.write(str(time.time()))
except (OSError, asyncpg.PostgresError, asyncpg.InterfaceError, S3Unavailable) as exc:
logger.warning("sweep failed (%s); will retry", type(exc).__name__)
if args.once:
return 1
if args.once:
return 0
await asyncio.sleep(args.interval)
if __name__ == "__main__":
raise SystemExit(asyncio.run(main()))