75 lines
4.5 KiB
Python
75 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Orphan sweeper (lab5): staging/ and attachments/ cleanup, retention of failed staging (see run_once).
|
|
|
|
Safety: only objects older than --grace-seconds are candidates (default 3600, far above the few seconds between
|
|
PUT and COMMIT), and the DB is re-checked immediately before each delete. Default mode is --dry-run.
|
|
"""
|
|
import argparse, os, sys, time
|
|
from datetime import datetime, timezone
|
|
import boto3, psycopg
|
|
from botocore.config import Config
|
|
|
|
def run_once(s3, conn, bucket, grace, apply, retention):
|
|
"""Lab5 cleanup rules (documented in REPORT.md):
|
|
1) staging/<k> without any attachments.staging_key row older than grace = orphan of a crash between PUT and COMMIT -> delete
|
|
(staging of ready attachments is deleted by the worker and its staging_key set to NULL, so a leftover is also an orphan);
|
|
2) attachments/<id>/... whose attachment row no longer exists -> delete;
|
|
3) staging of FAILED attachments older than retention -> delete object and NULL staging_key (replay is then impossible).
|
|
Current work (queued/processing, or failed within retention) is never touched; the DB is re-checked right before each delete."""
|
|
now = datetime.now(timezone.utc)
|
|
seen = old = orphans = deleted = 0
|
|
for prefix in ("staging/", "attachments/"):
|
|
for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix=prefix):
|
|
for obj in page.get("Contents", []):
|
|
seen += 1
|
|
key = obj["Key"]
|
|
if key == "attachments/.ready" or (now - obj["LastModified"]).total_seconds() < grace:
|
|
continue # fresh: may still be between PUT and COMMIT
|
|
old += 1
|
|
if prefix == "staging/":
|
|
referenced = lambda: conn.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone()
|
|
else:
|
|
att = key.split("/")[1] if key.count("/") >= 2 else ""
|
|
referenced = lambda: conn.execute("SELECT 1 FROM attachments WHERE id::text=%s", (att,)).fetchone()
|
|
if referenced():
|
|
continue
|
|
orphans += 1
|
|
print(("DELETE " if apply else "WOULD-DELETE ") + key, flush=True)
|
|
if apply and not referenced():
|
|
s3.delete_object(Bucket=bucket, Key=key); deleted += 1
|
|
expired = conn.execute("SELECT id, staging_key FROM attachments WHERE status='failed' AND staging_key IS NOT NULL "
|
|
"AND updated_at < now() - make_interval(secs=>%s)", (retention,)).fetchall()
|
|
for att_id, skey in expired:
|
|
print(("EXPIRE-STAGING " if apply else "WOULD-EXPIRE-STAGING ") + skey, flush=True)
|
|
if apply:
|
|
s3.delete_object(Bucket=bucket, Key=skey)
|
|
conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s", (att_id,))
|
|
deleted += 1
|
|
print(f"sweep: seen={seen} older_than_grace={old} orphans={orphans} failed_expired={len(expired)} deleted={deleted} apply={apply}", flush=True)
|
|
return orphans, deleted
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--apply", action="store_true", help="really delete (default: dry-run)")
|
|
p.add_argument("--grace-seconds", type=int, default=int(os.getenv("SWEEP_GRACE_SECONDS", "3600")))
|
|
p.add_argument("--loop", action="store_true"); p.add_argument("--interval", type=int, default=int(os.getenv("SWEEP_INTERVAL_SECONDS", "600")))
|
|
p.add_argument("--retention-seconds", type=int, default=int(os.getenv("STAGING_RETENTION_SECONDS", "86400")))
|
|
a = p.parse_args()
|
|
if a.grace_seconds < 0: sys.exit("grace must be >= 0")
|
|
s3 = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.getenv("S3_REGION", "us-east-1"),
|
|
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"], aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
|
|
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=3, read_timeout=15, retries={"max_attempts": 2}))
|
|
bucket = os.environ["S3_BUCKET"]
|
|
while True:
|
|
try:
|
|
with psycopg.connect(connect_timeout=5, autocommit=True) as conn:
|
|
run_once(s3, conn, bucket, a.grace_seconds, a.apply, a.retention_seconds)
|
|
except Exception as exc: # keep looping through outages
|
|
print("sweep failed:", type(exc).__name__, flush=True)
|
|
if not a.loop: raise
|
|
if not a.loop: return
|
|
time.sleep(a.interval)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|