forked from mtusi2026/mtusi2026lab6
53 lines
2.7 KiB
Python
53 lines
2.7 KiB
Python
"""Version-aware staging retention. Never deletes active jobs or referenced final effects."""
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import signal
|
|
import threading
|
|
|
|
from .jobs import database
|
|
from .storage import HEALTH_KEY, Storage
|
|
|
|
stop=threading.Event()
|
|
|
|
|
|
def run():
|
|
storage=Storage(); deleted=0; scanned=0
|
|
try:
|
|
with database() as db:
|
|
for prefix in ('staging/','originals/','variants/'):
|
|
for page in storage.client.get_paginator('list_objects_v2').paginate(Bucket=storage.bucket,Prefix=prefix):
|
|
for obj in page.get('Contents',[]):
|
|
key=obj['Key']; scanned+=1
|
|
if key==HEALTH_KEY: continue
|
|
age=datetime.now(timezone.utc).timestamp()-obj['LastModified'].timestamp()
|
|
if age<600: continue
|
|
with db.transaction():
|
|
if not db.execute('SELECT pg_try_advisory_xact_lock(hashtextextended(%s,0)) AS ok',(key,)).fetchone()['ok']: continue
|
|
if prefix=='staging/':
|
|
att=db.execute('SELECT * FROM attachments WHERE staging_key=%s FOR UPDATE',(key,)).fetchone()
|
|
if att and (att['status'] in ('queued','processing') or (att['status']=='failed' and age<7*86400)):
|
|
continue
|
|
# Ready staging is obsolete. Failed staging retained 7 days;
|
|
# explicit operator replay afterwards requires a restored source.
|
|
storage.delete(key); deleted+=1
|
|
if att:
|
|
db.execute('UPDATE attachments SET staging_key=NULL WHERE id=%s AND staging_key=%s',(att['id'],key))
|
|
else:
|
|
# Keep every version for a known attachment; old versions
|
|
# are bounded by the two supported pipelines and aid audit.
|
|
part=key.split('/')[1]
|
|
row=db.execute('SELECT 1 FROM attachments WHERE id::text=%s',(part,)).fetchone()
|
|
if row: continue
|
|
storage.delete(key); deleted+=1
|
|
return {'scanned':scanned,'deleted':deleted}
|
|
finally: storage.client.close()
|
|
|
|
|
|
if __name__=='__main__':
|
|
signal.signal(signal.SIGTERM,lambda *_:stop.set())
|
|
signal.signal(signal.SIGINT,lambda *_:stop.set())
|
|
while not stop.is_set():
|
|
try: print(json.dumps({'event':'cleanup',**run()}),flush=True)
|
|
except Exception as exc: print(json.dumps({'event':'cleanup_retry','type':type(exc).__name__}),flush=True)
|
|
stop.wait(60)
|