"""Remove old S3 objects without committed metadata. Never delete on a DB error.""" import argparse import logging import time from datetime import datetime, timedelta, timezone from sqlalchemy import select from sqlalchemy.orm import sessionmaker from .attachments import lock_object from .database import create_db_engine from .models import Attachment from .object_storage import BUCKET, PREFIX, s3_client GRACE = timedelta(hours=1) def cleanup(sessions, client, now=None): cutoff = (now or datetime.now(timezone.utc)) - GRACE deleted = 0 for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX): for obj in page.get('Contents', []): if obj['LastModified'] >= cutoff: continue key = obj['Key'] with sessions.begin() as session: lock_object(session, key) if session.scalar(select(Attachment.id).where(Attachment.object_key == key)): continue # Recheck age after taking the same lock as the uploader. if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff: continue client.delete_object(Bucket=BUCKET, Key=key) deleted += 1 return deleted def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--once', action='store_true') args = parser.parse_args() logging.basicConfig(level=logging.INFO) engine = create_db_engine() try: while True: try: count = cleanup(sessionmaker(engine), s3_client()) logging.info('Cleanup completed: deleted=%d', count) except Exception as exc: logging.error('Cleanup deferred: %s', type(exc).__name__) if args.once: raise SystemExit(1) from None if args.once: break time.sleep(3600) finally: engine.dispose() if __name__ == '__main__': main()