85 lines
2.5 KiB
Python
85 lines
2.5 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import boto3
|
|
import psycopg
|
|
from botocore.client import Config
|
|
from botocore.exceptions import ClientError
|
|
from psycopg.rows import dict_row
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
|
|
)
|
|
log = logging.getLogger("cleanup")
|
|
|
|
|
|
def _env(name: str, default: str | None = None) -> str:
|
|
value = os.environ.get(name, default)
|
|
if value is None:
|
|
raise RuntimeError(f"Environment variable {name} is required")
|
|
return value
|
|
|
|
|
|
def main() -> None:
|
|
grace_hours = int(os.environ.get("CLEANUP_GRACE_HOURS", "1"))
|
|
grace = timedelta(hours=grace_hours)
|
|
bucket = _env("S3_BUCKET", "messenger-attachments")
|
|
|
|
s3 = boto3.client(
|
|
"s3",
|
|
endpoint_url=_env("S3_ENDPOINT"),
|
|
region_name=_env("S3_REGION", "us-east-1"),
|
|
aws_access_key_id=_env("S3_ACCESS_KEY_ID"),
|
|
aws_secret_access_key=_env("S3_SECRET_ACCESS_KEY"),
|
|
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
|
)
|
|
|
|
s3_objects: dict[str, datetime] = {}
|
|
paginator = s3.get_paginator("list_objects_v2")
|
|
for page in paginator.paginate(Bucket=bucket):
|
|
for obj in page.get("Contents", []):
|
|
s3_objects[obj["Key"]] = obj["LastModified"]
|
|
|
|
log.info("S3 objects: %d", len(s3_objects))
|
|
|
|
db_keys: set[str] = set()
|
|
with psycopg.connect(_env("DATABASE_URL"), row_factory=dict_row) as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute("SELECT s3_key FROM attachments")
|
|
for row in cur.fetchall():
|
|
db_keys.add(row["s3_key"])
|
|
|
|
log.info("DB attachments: %d", len(db_keys))
|
|
|
|
orphans = set(s3_objects) - db_keys
|
|
log.info("Orphans: %d", len(orphans))
|
|
|
|
now = datetime.now(timezone.utc)
|
|
deleted = 0
|
|
skipped = 0
|
|
for key in orphans:
|
|
age = now - s3_objects[key]
|
|
if age < grace:
|
|
log.info("skip %s (age %s < grace %s)", key, age, grace)
|
|
skipped += 1
|
|
continue
|
|
try:
|
|
s3.delete_object(Bucket=bucket, Key=key)
|
|
log.info("deleted orphan: %s (age %s)", key, age)
|
|
deleted += 1
|
|
except ClientError as exc:
|
|
log.error("failed to delete %s: %s", key, exc)
|
|
|
|
log.info("done: deleted=%d, skipped=%d", deleted, skipped)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except Exception as exc:
|
|
log.exception("cleanup failed")
|
|
sys.exit(1) |