forked from mtusi2026/mtusi2026lab5
перенос 4 лабы и реализация 5
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
"""Очистка orphan PUT и staging только ready/просроченных failed; текущую работу не трогает."""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from src import db, object_store
|
||||
from src.attachments import lock_upload
|
||||
|
||||
|
||||
GRACE_SECONDS = int(os.environ.get("UPLOAD_CLEANUP_GRACE_SECONDS", "3600"))
|
||||
if GRACE_SECONDS < 60:
|
||||
raise ValueError("Безопасный grace period должен быть не меньше 60 секунд")
|
||||
|
||||
|
||||
def cleanup_orphans():
|
||||
result = {"removed": 0, "kept_ready": 0, "skipped_active": 0}
|
||||
with db.connection() as conn:
|
||||
candidates = conn.execute("""SELECT id, object_key FROM attachment_uploads
|
||||
WHERE created_at < clock_timestamp() - %s * interval '1 second'
|
||||
ORDER BY created_at LIMIT 100""", (GRACE_SECONDS,)).fetchall()
|
||||
for row in candidates:
|
||||
with db.connection() as conn:
|
||||
if not lock_upload(conn, row["object_key"], blocking=False):
|
||||
result["skipped_active"] += 1
|
||||
continue
|
||||
pending = conn.execute("""SELECT 1 FROM attachment_uploads WHERE id=%s
|
||||
AND created_at < clock_timestamp() - %s * interval '1 second' FOR UPDATE""",
|
||||
(row["id"], GRACE_SECONDS)).fetchone()
|
||||
if not pending:
|
||||
continue
|
||||
ready = conn.execute("SELECT 1 FROM attachments WHERE object_key=%s", (row["object_key"],)).fetchone()
|
||||
if ready:
|
||||
result["kept_ready"] += 1
|
||||
else:
|
||||
object_store.remove(row["object_key"])
|
||||
result["removed"] += 1
|
||||
conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (row["id"],))
|
||||
return result
|
||||
|
||||
|
||||
RETENTION = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400"))
|
||||
if RETENTION < 60:
|
||||
raise ValueError("STAGING_RETENTION_SECONDS должен быть >=60")
|
||||
|
||||
|
||||
def cleanup_staging():
|
||||
removed = 0
|
||||
with db.connection() as conn:
|
||||
rows = conn.execute("""SELECT j.*,a.status,a.object_key FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id=j.attachment_id WHERE j.staging_deleted_at IS NULL
|
||||
AND (a.status='ready' OR (a.status='failed' AND a.updated_at<clock_timestamp()-%s*interval '1 second'))
|
||||
LIMIT 20 FOR UPDATE OF j,a SKIP LOCKED""", (RETENTION,)).fetchall()
|
||||
for job in rows:
|
||||
object_store.remove(job['staging_key'])
|
||||
if job['status'] == 'failed':
|
||||
prefix = job['object_key'].rsplit('/', 1)[0] + '/'
|
||||
for variant in ('original', 'thumbnail', 'crop', 'optimized'):
|
||||
object_store.remove(prefix + variant)
|
||||
conn.execute('UPDATE attachment_jobs SET staging_deleted_at=clock_timestamp() WHERE id=%s', (job['id'],))
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
|
||||
def cleanup_once():
|
||||
return {**cleanup_orphans(), "staging_removed": cleanup_staging()}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--once", action="store_true")
|
||||
args = parser.parse_args()
|
||||
db.pool.open(wait=True, timeout=15)
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
print(json.dumps(cleanup_once()), flush=True)
|
||||
except Exception as error:
|
||||
if args.once:
|
||||
raise
|
||||
logging.error("Cleanup postponed: %s", type(error).__name__)
|
||||
if args.once:
|
||||
break
|
||||
time.sleep(30)
|
||||
finally:
|
||||
db.pool.close()
|
||||
Reference in New Issue
Block a user