Files
mtusi2026lab4/scripts/s3_sweep.py
T
2026-11-01 01:18:40 +03:00

58 lines
2.9 KiB
Python

#!/usr/bin/env python3
"""Orphan sweeper: deletes S3 objects under attachments/ that have no row in `attachments`.
Safety: only objects older than --grace-seconds are candidates (default 3600, far above the few seconds between
PUT and COMMIT), and the DB is re-checked immediately before each delete. Default mode is --dry-run.
"""
import argparse, os, sys, time
from datetime import datetime, timezone
import boto3, psycopg
from botocore.config import Config
def run_once(s3, conn, bucket, grace, apply):
now = datetime.now(timezone.utc)
seen = old = orphans = deleted = 0
for page in s3.get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix="attachments/"):
for obj in page.get("Contents", []):
seen += 1
if obj["Key"] == "attachments/.ready":
continue
if (now - obj["LastModified"]).total_seconds() < grace:
continue # fresh: may still be between PUT and COMMIT
old += 1
if conn.execute("SELECT 1 FROM attachments WHERE s3_key=%s", (obj["Key"],)).fetchone():
continue
orphans += 1
print(("DELETE " if apply else "WOULD-DELETE ") + obj["Key"], flush=True)
if apply:
# re-check right before delete (closes the race with a concurrent COMMIT)
if not conn.execute("SELECT 1 FROM attachments WHERE s3_key=%s", (obj["Key"],)).fetchone():
s3.delete_object(Bucket=bucket, Key=obj["Key"]); deleted += 1
print(f"sweep: seen={seen} older_than_grace={old} orphans={orphans} deleted={deleted} apply={apply}", flush=True)
return orphans, deleted
def main():
p = argparse.ArgumentParser()
p.add_argument("--apply", action="store_true", help="really delete (default: dry-run)")
p.add_argument("--grace-seconds", type=int, default=int(os.getenv("SWEEP_GRACE_SECONDS", "3600")))
p.add_argument("--loop", action="store_true"); p.add_argument("--interval", type=int, default=int(os.getenv("SWEEP_INTERVAL_SECONDS", "600")))
a = p.parse_args()
if a.grace_seconds < 0: sys.exit("grace must be >= 0")
s3 = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name=os.getenv("S3_REGION", "us-east-1"),
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"], aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=3, read_timeout=15, retries={"max_attempts": 2}))
bucket = os.environ["S3_BUCKET"]
while True:
try:
with psycopg.connect(connect_timeout=5, autocommit=True) as conn:
run_once(s3, conn, bucket, a.grace_seconds, a.apply)
except Exception as exc: # keep looping through outages
print("sweep failed:", type(exc).__name__, flush=True)
if not a.loop: raise
if not a.loop: return
time.sleep(a.interval)
if __name__ == "__main__":
main()