forked from mtusi2026/mtusi2026lab5
Add 4 lab code
This commit is contained in:
@@ -0,0 +1,61 @@
|
||||
"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from .attachments import lock_object
|
||||
from .database import create_db_engine
|
||||
from .models import Attachment
|
||||
from .object_storage import BUCKET, PREFIX, s3_client
|
||||
|
||||
GRACE = timedelta(hours=1)
|
||||
|
||||
|
||||
def cleanup(sessions, client, now=None):
|
||||
cutoff = (now or datetime.now(timezone.utc)) - GRACE
|
||||
deleted = 0
|
||||
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX):
|
||||
for obj in page.get('Contents', []):
|
||||
if obj['LastModified'] >= cutoff:
|
||||
continue
|
||||
key = obj['Key']
|
||||
with sessions.begin() as session:
|
||||
lock_object(session, key)
|
||||
if session.scalar(select(Attachment.id).where(Attachment.object_key == key)):
|
||||
continue
|
||||
# Recheck age after taking the same lock as the uploader.
|
||||
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
|
||||
continue
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
deleted += 1
|
||||
return deleted
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--once', action='store_true')
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
engine = create_db_engine()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
count = cleanup(sessionmaker(engine), s3_client())
|
||||
logging.info('Cleanup completed: deleted=%d', count)
|
||||
except Exception as exc:
|
||||
logging.error('Cleanup deferred: %s', type(exc).__name__)
|
||||
if args.once:
|
||||
raise SystemExit(1) from None
|
||||
if args.once:
|
||||
break
|
||||
time.sleep(3600)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user