forked from mtusi2026/mtusi2026lab4
163 lines
7.7 KiB
Python
163 lines
7.7 KiB
Python
"""Очистка незавершённых загрузок и сверка PostgreSQL <-> S3.
|
|
|
|
Запускается отдельным сервисом `janitor` (раз в JANITOR_INTERVAL_SECONDS) или вручную:
|
|
|
|
python janitor.py --once [--dry-run] [--grace-seconds N]
|
|
|
|
Проход состоит из двух шагов. Оба трогают только то, что старше grace period
|
|
(ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч, минимум 2 x UPLOAD_TIMEOUT_SECONDS): загрузка,
|
|
которая ещё идёт, столько не живёт, поэтому свежие и готовые вложения не удаляются.
|
|
|
|
1. Брошенные намерения: строки state=uploading старше grace. Для каждой (под FOR UPDATE SKIP LOCKED)
|
|
удалить объект в S3 (идемпотентно) и затем строку. Если S3 недоступен, строка остаётся
|
|
до следующего прохода — указатель на объект не теряется.
|
|
2. Сирота в S3: объект под originals/ старше grace, для которого в БД нет строки ни в каком
|
|
состоянии. API кладёт объект только после COMMIT строки-намерения, поэтому у живой загрузки
|
|
строка всегда есть. Удаляется.
|
|
Заодно отчёт: ready-строки, чьего объекта нет в bucket (missing). Их не трогаем — это потеря
|
|
данных, её разбирает человек (восстановление из резервной копии), а не автоматическая очистка.
|
|
|
|
Одновременно работает только один janitor: pg_try_advisory_lock.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import signal
|
|
import threading
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import psycopg
|
|
from psycopg.rows import dict_row
|
|
|
|
import settings
|
|
import storage
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
log = logging.getLogger("messenger.janitor")
|
|
|
|
JANITOR_LOCK = 20260404
|
|
BATCH = 100
|
|
STOP = threading.Event()
|
|
|
|
|
|
def connect():
|
|
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
|
|
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
|
|
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
|
|
|
|
|
|
def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]:
|
|
"""Шаг 1. Возвращает (найдено, удалено)."""
|
|
found = deleted = 0
|
|
while not STOP.is_set():
|
|
with conn.transaction():
|
|
rows = conn.execute(
|
|
"""SELECT id, object_key FROM attachments
|
|
WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s)
|
|
ORDER BY created_at LIMIT %s
|
|
FOR UPDATE SKIP LOCKED""",
|
|
(grace, BATCH),
|
|
).fetchall()
|
|
found += len(rows)
|
|
if dry_run or not rows:
|
|
return found, deleted
|
|
for row in rows:
|
|
storage.delete(row["object_key"]) # недоступен S3 -> исключение, транзакция откатится
|
|
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],))
|
|
deleted += 1
|
|
log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"])
|
|
if len(rows) < BATCH:
|
|
return found, deleted
|
|
return found, deleted
|
|
|
|
|
|
def sweep_objects(conn, grace: int, dry_run: bool) -> dict:
|
|
"""Шаг 2. Обход bucket постранично; сверка ключей с БД пачками по 1000."""
|
|
started = datetime.now(timezone.utc)
|
|
cutoff = started - timedelta(seconds=grace)
|
|
scanned = orphans = orphans_deleted = 0
|
|
listed = set()
|
|
page: list[tuple[str, datetime]] = []
|
|
|
|
def check(page_items):
|
|
nonlocal orphans, orphans_deleted
|
|
keys = [key for key, _ in page_items]
|
|
known = {r["object_key"] for r in conn.execute(
|
|
"SELECT object_key FROM attachments WHERE object_key = ANY(%s)", (keys,)).fetchall()}
|
|
for key, modified in page_items:
|
|
if key in known or modified >= cutoff:
|
|
continue
|
|
orphans += 1
|
|
if not dry_run:
|
|
# Повторная проверка прямо перед удалением: строка могла появиться между запросами.
|
|
if conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (key,)).fetchone() is None:
|
|
storage.delete(key)
|
|
orphans_deleted += 1
|
|
log.info("deleted orphan object %s (no row in PostgreSQL)", key)
|
|
|
|
for key, modified, _size in storage.list_objects():
|
|
scanned += 1
|
|
listed.add(key)
|
|
page.append((key, modified))
|
|
if len(page) >= 1000:
|
|
check(page)
|
|
page = []
|
|
if page:
|
|
check(page)
|
|
|
|
missing = [str(r["id"]) for r in conn.execute(
|
|
"SELECT id, object_key FROM attachments WHERE state = 'ready' AND updated_at < %s", (started,)).fetchall()
|
|
if r["object_key"] not in listed]
|
|
if missing:
|
|
log.error("ready attachments without object in S3 (not deleted, needs restore): %s", ", ".join(missing[:20]))
|
|
return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted,
|
|
"missing_objects": len(missing)}
|
|
|
|
|
|
def run_once(grace: int, dry_run: bool) -> dict | None:
|
|
started = time.monotonic()
|
|
with connect() as conn:
|
|
if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]:
|
|
log.info("another janitor is running, skip")
|
|
return None
|
|
try:
|
|
found, deleted = reclaim_stale_intents(conn, grace, dry_run)
|
|
report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found,
|
|
"stale_uploads_deleted": deleted, **sweep_objects(conn, grace, dry_run)}
|
|
finally:
|
|
conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,))
|
|
report["seconds"] = round(time.monotonic() - started, 3)
|
|
return report
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Reclaim abandoned uploads and orphan S3 objects")
|
|
parser.add_argument("--once", action="store_true", help="one pass and exit")
|
|
parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing")
|
|
parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS)
|
|
args = parser.parse_args()
|
|
if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS:
|
|
parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} "
|
|
"(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted")
|
|
|
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
signal.signal(sig, lambda *_: STOP.set())
|
|
while True:
|
|
try:
|
|
report = run_once(args.grace_seconds, args.dry_run)
|
|
if report is not None:
|
|
print(json.dumps(report), flush=True)
|
|
failed = False
|
|
except (psycopg.Error, storage.StorageUnavailable) as exc:
|
|
log.warning("pass failed, will retry: %s", type(exc).__name__)
|
|
failed = True
|
|
if args.once:
|
|
raise SystemExit(1 if failed else 0)
|
|
if STOP.wait(settings.JANITOR_INTERVAL_SECONDS):
|
|
return
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|