Files
mtusi2026lab5/src/cleanup.py
T

88 lines
3.5 KiB
Python

"""Очистка orphan PUT и staging только ready/просроченных failed; текущую работу не трогает."""
import argparse
import json
import logging
import os
import time
from src import db, object_store
from src.attachments import lock_upload
GRACE_SECONDS = int(os.environ.get("UPLOAD_CLEANUP_GRACE_SECONDS", "3600"))
if GRACE_SECONDS < 60:
raise ValueError("Безопасный grace period должен быть не меньше 60 секунд")
def cleanup_orphans():
result = {"removed": 0, "kept_ready": 0, "skipped_active": 0}
with db.connection() as conn:
candidates = conn.execute("""SELECT id, object_key FROM attachment_uploads
WHERE created_at < clock_timestamp() - %s * interval '1 second'
ORDER BY created_at LIMIT 100""", (GRACE_SECONDS,)).fetchall()
for row in candidates:
with db.connection() as conn:
if not lock_upload(conn, row["object_key"], blocking=False):
result["skipped_active"] += 1
continue
pending = conn.execute("""SELECT 1 FROM attachment_uploads WHERE id=%s
AND created_at < clock_timestamp() - %s * interval '1 second' FOR UPDATE""",
(row["id"], GRACE_SECONDS)).fetchone()
if not pending:
continue
ready = conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (row["object_key"],)).fetchone()
if ready:
result["kept_ready"] += 1
else:
object_store.remove(row["object_key"])
result["removed"] += 1
conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (row["id"],))
return result
RETENTION = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400"))
if RETENTION < 60:
raise ValueError("STAGING_RETENTION_SECONDS должен быть >=60")
def cleanup_staging():
removed = 0
with db.connection() as conn:
rows = conn.execute("""SELECT j.*,a.status,a.object_key FROM attachment_jobs j
JOIN attachments a ON a.id=j.attachment_id WHERE j.staging_deleted_at IS NULL
AND (a.status='ready' OR (a.status='failed' AND a.updated_at<clock_timestamp()-%s*interval '1 second'))
LIMIT 20 FOR UPDATE OF j,a SKIP LOCKED""", (RETENTION,)).fetchall()
for job in rows:
object_store.remove(job['staging_key'])
if job['status'] == 'failed':
prefix = job['object_key'].rsplit('/', 1)[0] + '/'
for variant in ('original', 'thumbnail', 'crop', 'optimized'):
object_store.remove(prefix + variant)
conn.execute('UPDATE attachment_jobs SET staging_deleted_at=clock_timestamp() WHERE id=%s', (job['id'],))
removed += 1
return removed
def cleanup_once():
return {**cleanup_orphans(), "staging_removed": cleanup_staging()}
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--once", action="store_true")
args = parser.parse_args()
db.pool.open(wait=True, timeout=15)
try:
while True:
try:
print(json.dumps(cleanup_once()), flush=True)
except Exception as error:
if args.once:
raise
logging.error("Cleanup postponed: %s", type(error).__name__)
if args.once:
break
time.sleep(30)
finally:
db.pool.close()