forked from mtusi2026/mtusi2026lab5
62 lines
2.0 KiB
Python
62 lines
2.0 KiB
Python
"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
|
|
import argparse
|
|
import logging
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from .attachments import lock_object
|
|
from .database import create_db_engine
|
|
from .models import Attachment
|
|
from .object_storage import BUCKET, PREFIX, s3_client
|
|
|
|
GRACE = timedelta(hours=1)
|
|
|
|
|
|
def cleanup(sessions, client, now=None):
|
|
cutoff = (now or datetime.now(timezone.utc)) - GRACE
|
|
deleted = 0
|
|
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX):
|
|
for obj in page.get('Contents', []):
|
|
if obj['LastModified'] >= cutoff:
|
|
continue
|
|
key = obj['Key']
|
|
with sessions.begin() as session:
|
|
lock_object(session, key)
|
|
if session.scalar(select(Attachment.id).where(Attachment.object_key == key)):
|
|
continue
|
|
# Recheck age after taking the same lock as the uploader.
|
|
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
|
|
continue
|
|
client.delete_object(Bucket=BUCKET, Key=key)
|
|
deleted += 1
|
|
return deleted
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--once', action='store_true')
|
|
args = parser.parse_args()
|
|
logging.basicConfig(level=logging.INFO)
|
|
engine = create_db_engine()
|
|
try:
|
|
while True:
|
|
try:
|
|
count = cleanup(sessionmaker(engine), s3_client())
|
|
logging.info('Cleanup completed: deleted=%d', count)
|
|
except Exception as exc:
|
|
logging.error('Cleanup deferred: %s', type(exc).__name__)
|
|
if args.once:
|
|
raise SystemExit(1) from None
|
|
if args.once:
|
|
break
|
|
time.sleep(3600)
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|