Files
mtusi2026lab6/scripts/cleanup_orphans.py
T

85 lines
2.5 KiB
Python

from __future__ import annotations
import logging
import os
import sys
from datetime import datetime, timedelta, timezone
import boto3
import psycopg
from botocore.client import Config
from botocore.exceptions import ClientError
from psycopg.rows import dict_row
logging.basicConfig(
level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
)
log = logging.getLogger("cleanup")
def _env(name: str, default: str | None = None) -> str:
value = os.environ.get(name, default)
if value is None:
raise RuntimeError(f"Environment variable {name} is required")
return value
def main() -> None:
grace_hours = int(os.environ.get("CLEANUP_GRACE_HOURS", "1"))
grace = timedelta(hours=grace_hours)
bucket = _env("S3_BUCKET", "messenger-attachments")
s3 = boto3.client(
"s3",
endpoint_url=_env("S3_ENDPOINT"),
region_name=_env("S3_REGION", "us-east-1"),
aws_access_key_id=_env("S3_ACCESS_KEY_ID"),
aws_secret_access_key=_env("S3_SECRET_ACCESS_KEY"),
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
)
s3_objects: dict[str, datetime] = {}
paginator = s3.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=bucket):
for obj in page.get("Contents", []):
s3_objects[obj["Key"]] = obj["LastModified"]
log.info("S3 objects: %d", len(s3_objects))
db_keys: set[str] = set()
with psycopg.connect(_env("DATABASE_URL"), row_factory=dict_row) as conn:
with conn.cursor() as cur:
cur.execute("SELECT s3_key FROM attachments")
for row in cur.fetchall():
db_keys.add(row["s3_key"])
log.info("DB attachments: %d", len(db_keys))
orphans = set(s3_objects) - db_keys
log.info("Orphans: %d", len(orphans))
now = datetime.now(timezone.utc)
deleted = 0
skipped = 0
for key in orphans:
age = now - s3_objects[key]
if age < grace:
log.info("skip %s (age %s < grace %s)", key, age, grace)
skipped += 1
continue
try:
s3.delete_object(Bucket=bucket, Key=key)
log.info("deleted orphan: %s (age %s)", key, age)
deleted += 1
except ClientError as exc:
log.error("failed to delete %s: %s", key, exc)
log.info("done: deleted=%d, skipped=%d", deleted, skipped)
if __name__ == "__main__":
try:
main()
except Exception as exc:
log.exception("cleanup failed")
sys.exit(1)