Files
mtusi2026lab5/src/cleanup.py
T
2026-11-06 17:08:07 +03:00

62 lines
2.0 KiB
Python

"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
import argparse
import logging
import time
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from .attachments import lock_object
from .database import create_db_engine
from .models import Attachment
from .object_storage import BUCKET, PREFIX, s3_client
GRACE = timedelta(hours=1)
def cleanup(sessions, client, now=None):
cutoff = (now or datetime.now(timezone.utc)) - GRACE
deleted = 0
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX):
for obj in page.get('Contents', []):
if obj['LastModified'] >= cutoff:
continue
key = obj['Key']
with sessions.begin() as session:
lock_object(session, key)
if session.scalar(select(Attachment.id).where(Attachment.object_key == key)):
continue
# Recheck age after taking the same lock as the uploader.
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
continue
client.delete_object(Bucket=BUCKET, Key=key)
deleted += 1
return deleted
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--once', action='store_true')
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
engine = create_db_engine()
try:
while True:
try:
count = cleanup(sessionmaker(engine), s3_client())
logging.info('Cleanup completed: deleted=%d', count)
except Exception as exc:
logging.error('Cleanup deferred: %s', type(exc).__name__)
if args.once:
raise SystemExit(1) from None
if args.once:
break
time.sleep(3600)
finally:
engine.dispose()
if __name__ == '__main__':
main()