forked from mtusi2026/mtusi2026lab5
163 lines
7.5 KiB
Python
163 lines
7.5 KiB
Python
"""Staging/orphan cleanup with documented retention. Never touches work that is still in progress.
|
|
|
|
Rules (every step idempotent, repeated runs are safe):
|
|
1. staging/<id> of a READY attachment -> delete (the worker deletes it right after COMMIT;
|
|
this repairs a crash/failed delete in between)
|
|
2. staging/<id> of a FAILED attachment older than STAGING_RETENTION_SECONDS -> delete, mark staging_deleted_at
|
|
(until then an operator replay can still use it; afterwards replay ends as failed staging_missing)
|
|
3. staging/<id> without any attachment row, older than ORPHAN_GRACE_SECONDS -> delete
|
|
(API crashed/failed between the staging PUT and the COMMIT of attachment+job+outbox; no 202 was sent)
|
|
4. attachments/<id>/... without any attachment row, or not referenced by any committed row (superseded
|
|
older pipeline version, fenced-out attempt) while no job of that attachment is active, older than grace
|
|
5. published outbox rows older than 7 days -> delete
|
|
Queued/processing attachments are never touched: their staging object is the only copy of accepted work.
|
|
|
|
python -m app.sweeper --once --dry-run # report only
|
|
python -m app.sweeper --once # one real pass
|
|
python -m app.sweeper # loop every SWEEP_INTERVAL_SECONDS
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import asyncpg
|
|
from starlette.concurrency import run_in_threadpool
|
|
|
|
from app import config
|
|
from app.objects import FINAL_PREFIX, STAGING_PREFIX, ObjectStore, S3Unavailable
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
|
|
logger = logging.getLogger("sweeper")
|
|
|
|
LOCK_ID = 2026_0005
|
|
HEARTBEAT_FILE = "/tmp/sweeper-heartbeat"
|
|
|
|
|
|
def _id_from_key(key: str, prefix: str):
|
|
part = key[len(prefix):].split("/", 1)[0]
|
|
try:
|
|
return uuid.UUID(hex=part)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
async def sweep(conn: asyncpg.Connection, objects: ObjectStore, grace: int, retention: int, dry_run: bool) -> dict:
|
|
started = time.monotonic()
|
|
report = {"dry_run": dry_run, "grace_seconds": grace, "retention_seconds": retention,
|
|
"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_orphans": 0, "final_orphans": 0,
|
|
"outbox_pruned": 0, "objects_scanned": 0}
|
|
now = datetime.now(timezone.utc)
|
|
cutoff = now - timedelta(seconds=grace)
|
|
|
|
# 1 + 2: staging of finished attachments
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT id, staging_key, status FROM attachments
|
|
WHERE staging_deleted_at IS NULL AND staging_key IS NOT NULL
|
|
AND (status = 'ready'
|
|
OR (status = 'failed' AND updated_at < clock_timestamp() - make_interval(secs => $1)))
|
|
""", float(retention))
|
|
for row in rows:
|
|
report["staging_ready_deleted" if row["status"] == "ready" else "staging_failed_expired"] += 1
|
|
if dry_run:
|
|
continue
|
|
await run_in_threadpool(objects.delete_sync, row["staging_key"])
|
|
await conn.execute(
|
|
"UPDATE attachments SET staging_deleted_at = clock_timestamp() WHERE id = $1 AND status IN ('ready', 'failed')",
|
|
row["id"])
|
|
|
|
# 3 + 4: objects without any row (no 202 was ever sent for them)
|
|
for prefix, counter in ((STAGING_PREFIX, "staging_orphans"), (FINAL_PREFIX, "final_orphans")):
|
|
def old_keys():
|
|
out = []
|
|
for key, modified in objects.list_prefix(prefix):
|
|
report["objects_scanned"] += 1
|
|
if modified < cutoff:
|
|
out.append(key)
|
|
return out
|
|
keys = await run_in_threadpool(old_keys)
|
|
by_id = {}
|
|
for key in keys:
|
|
ident = _id_from_key(key, prefix)
|
|
if ident is not None:
|
|
by_id.setdefault(ident, []).append(key)
|
|
if not by_id:
|
|
continue
|
|
known = {r["id"] for r in await conn.fetch("SELECT id FROM attachments WHERE id = ANY($1::uuid[])", list(by_id))}
|
|
orphans = [k for ident, ks in by_id.items() if ident not in known for k in ks]
|
|
if prefix == FINAL_PREFIX:
|
|
# objects of a known attachment that no committed row references: a superseded older pipeline
|
|
# version or a fenced-out attempt. Only after the grace period (longer than any live attempt).
|
|
candidates = [k for ident, ks in by_id.items() if ident in known for k in ks]
|
|
if candidates:
|
|
referenced = {r["k"] for r in await conn.fetch(
|
|
"""
|
|
SELECT object_key AS k FROM attachments WHERE object_key = ANY($1::text[]) AND status = 'ready'
|
|
UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY($1::text[])
|
|
UNION SELECT a.object_key FROM attachments a WHERE a.object_key = ANY($1::text[])
|
|
AND a.status IN ('queued', 'processing')
|
|
""", candidates)}
|
|
live = {r["id"] for r in await conn.fetch(
|
|
"SELECT DISTINCT attachment_id AS id FROM jobs WHERE status IN ('queued', 'processing')")}
|
|
orphans += [k for k in candidates if k not in referenced and _id_from_key(k, prefix) not in live]
|
|
report[counter] += len(orphans)
|
|
if orphans and not dry_run:
|
|
await run_in_threadpool(objects.delete_many, orphans)
|
|
|
|
# 5: outbox history
|
|
if not dry_run:
|
|
res = await conn.execute(
|
|
"DELETE FROM outbox WHERE published_at < clock_timestamp() - interval '7 days'")
|
|
report["outbox_pruned"] = int(res.split()[-1])
|
|
|
|
report["duration_ms"] = round((time.monotonic() - started) * 1000, 1)
|
|
return report
|
|
|
|
|
|
async def run_once(grace: int, retention: int, dry_run: bool) -> dict:
|
|
conn = await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT)
|
|
try:
|
|
if not await conn.fetchval("SELECT pg_try_advisory_lock($1)", LOCK_ID):
|
|
return {"skipped": "another sweeper holds the lock"}
|
|
try:
|
|
return await sweep(conn, ObjectStore(), grace, retention, dry_run)
|
|
finally:
|
|
await conn.execute("SELECT pg_advisory_unlock($1)", LOCK_ID)
|
|
finally:
|
|
await conn.close()
|
|
|
|
|
|
async def main() -> int:
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument("--once", action="store_true")
|
|
p.add_argument("--dry-run", action="store_true")
|
|
p.add_argument("--grace", type=int, default=config.ORPHAN_GRACE_SECONDS)
|
|
p.add_argument("--retention", type=int, default=config.STAGING_RETENTION_SECONDS)
|
|
p.add_argument("--interval", type=int, default=config.SWEEP_INTERVAL_SECONDS)
|
|
args = p.parse_args()
|
|
if args.grace < 60:
|
|
p.error("grace below 60 s could remove an upload that is still between staging PUT and COMMIT")
|
|
while True:
|
|
try:
|
|
report = await run_once(args.grace, args.retention, args.dry_run)
|
|
print(json.dumps(report, ensure_ascii=False), flush=True)
|
|
with open(HEARTBEAT_FILE, "w", encoding="ascii") as fh:
|
|
fh.write(str(time.time()))
|
|
except (OSError, asyncpg.PostgresError, asyncpg.InterfaceError, S3Unavailable) as exc:
|
|
logger.warning("sweep failed (%s); will retry", type(exc).__name__)
|
|
if args.once:
|
|
return 1
|
|
if args.once:
|
|
return 0
|
|
await asyncio.sleep(args.interval)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(asyncio.run(main()))
|