forked from mtusi2026/mtusi2026lab5
88 lines
3.5 KiB
Python
88 lines
3.5 KiB
Python
"""Очистка orphan PUT и staging только ready/просроченных failed; текущую работу не трогает."""
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
|
|
from src import db, object_store
|
|
from src.attachments import lock_upload
|
|
|
|
|
|
GRACE_SECONDS = int(os.environ.get("UPLOAD_CLEANUP_GRACE_SECONDS", "3600"))
|
|
if GRACE_SECONDS < 60:
|
|
raise ValueError("Безопасный grace period должен быть не меньше 60 секунд")
|
|
|
|
|
|
def cleanup_orphans():
|
|
result = {"removed": 0, "kept_ready": 0, "skipped_active": 0}
|
|
with db.connection() as conn:
|
|
candidates = conn.execute("""SELECT id, object_key FROM attachment_uploads
|
|
WHERE created_at < clock_timestamp() - %s * interval '1 second'
|
|
ORDER BY created_at LIMIT 100""", (GRACE_SECONDS,)).fetchall()
|
|
for row in candidates:
|
|
with db.connection() as conn:
|
|
if not lock_upload(conn, row["object_key"], blocking=False):
|
|
result["skipped_active"] += 1
|
|
continue
|
|
pending = conn.execute("""SELECT 1 FROM attachment_uploads WHERE id=%s
|
|
AND created_at < clock_timestamp() - %s * interval '1 second' FOR UPDATE""",
|
|
(row["id"], GRACE_SECONDS)).fetchone()
|
|
if not pending:
|
|
continue
|
|
ready = conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (row["object_key"],)).fetchone()
|
|
if ready:
|
|
result["kept_ready"] += 1
|
|
else:
|
|
object_store.remove(row["object_key"])
|
|
result["removed"] += 1
|
|
conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (row["id"],))
|
|
return result
|
|
|
|
|
|
RETENTION = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400"))
|
|
if RETENTION < 60:
|
|
raise ValueError("STAGING_RETENTION_SECONDS должен быть >=60")
|
|
|
|
|
|
def cleanup_staging():
|
|
removed = 0
|
|
with db.connection() as conn:
|
|
rows = conn.execute("""SELECT j.*,a.status,a.object_key FROM attachment_jobs j
|
|
JOIN attachments a ON a.id=j.attachment_id WHERE j.staging_deleted_at IS NULL
|
|
AND (a.status='ready' OR (a.status='failed' AND a.updated_at<clock_timestamp()-%s*interval '1 second'))
|
|
LIMIT 20 FOR UPDATE OF j,a SKIP LOCKED""", (RETENTION,)).fetchall()
|
|
for job in rows:
|
|
object_store.remove(job['staging_key'])
|
|
if job['status'] == 'failed':
|
|
prefix = job['object_key'].rsplit('/', 1)[0] + '/'
|
|
for variant in ('original', 'thumbnail', 'crop', 'optimized'):
|
|
object_store.remove(prefix + variant)
|
|
conn.execute('UPDATE attachment_jobs SET staging_deleted_at=clock_timestamp() WHERE id=%s', (job['id'],))
|
|
removed += 1
|
|
return removed
|
|
|
|
|
|
def cleanup_once():
|
|
return {**cleanup_orphans(), "staging_removed": cleanup_staging()}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--once", action="store_true")
|
|
args = parser.parse_args()
|
|
db.pool.open(wait=True, timeout=15)
|
|
try:
|
|
while True:
|
|
try:
|
|
print(json.dumps(cleanup_once()), flush=True)
|
|
except Exception as error:
|
|
if args.once:
|
|
raise
|
|
logging.error("Cleanup postponed: %s", type(error).__name__)
|
|
if args.once:
|
|
break
|
|
time.sleep(30)
|
|
finally:
|
|
db.pool.close()
|