Files

49 lines
2.4 KiB
Python

"""Conservative staging cleanup; all current queued/processing jobs are retained."""
import argparse
import json
import os
import signal
import threading
from datetime import datetime,timezone
from psycopg.rows import dict_row
from app import store,object_store
stop=threading.Event()
def sweep(apply=False,retention=None):
retention=max(60,int(retention or os.getenv('STAGING_RETENTION_SECONDS','86400')))
deleted=[]
pages=object_store.client.get_paginator('list_objects_v2').paginate(Bucket=object_store.bucket,Prefix='staging/')
for page in pages:
for item in page.get('Contents',[]):
if (datetime.now(timezone.utc)-item['LastModified']).total_seconds()<retention:continue
# Admission holds this same lock from S3 put through DB COMMIT.
with store.connect() as conn:
conn.execute('SELECT pg_advisory_xact_lock(202605)')
rows=conn.cursor(row_factory=dict_row).execute('SELECT id,state,staging_key FROM attachments WHERE staging_key=%s FOR UPDATE',(item['Key'],)).fetchall()
if any(row['state'] in ('queued','processing','pending') for row in rows):continue
head=object_store.client.head_object(Bucket=object_store.bucket,Key=item['Key'])
if (datetime.now(timezone.utc)-head['LastModified']).total_seconds()<retention:continue
if apply:
object_store.client.delete_object(Bucket=object_store.bucket,Key=item['Key'])
conn.execute('UPDATE attachments SET staging_key=NULL WHERE staging_key=%s',(item['Key'],))
deleted.append(item['Key'])
return {'apply':apply,'retention_seconds':retention,'staging_candidates':deleted}
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('--once',action='store_true');p.add_argument('--apply',action='store_true');args=p.parse_args()
store.pool.open(wait=True,timeout=30)
for sig in (signal.SIGINT,signal.SIGTERM):signal.signal(sig,lambda *_:stop.set())
try:
while not stop.is_set():
try:print(json.dumps(sweep(args.apply or not args.once)),flush=True)
except Exception as exc:
print(json.dumps({'cleanup_error':type(exc).__name__}),flush=True)
if args.once:raise
if args.once:break
stop.wait(60)
finally:
store.pool.close();object_store.close()