"""Очистка и сверка PostgreSQL <-> S3 (сервис janitor). python janitor.py --once [--dry-run] [--grace-seconds N] Проход (раз в JANITOR_INTERVAL_SECONDS, один janitor за счёт pg_try_advisory_lock): 1. Брошенные намерения (как в lab4): state=uploading старше grace — исходник так и не был поставлен в очередь. Удалить объект staging (идемпотентно), затем строку. 2. Staging после безопасной фиксации: - ready: исходник уже опубликован в attachments//v/original -> удалить staging-объект (обычно это делает worker сразу после ack; здесь — если он не успел); - failed: исходник хранится STAGING_RETENTION_SECONDS после ошибки (для replay), потом удаляется. queued/processing не трогаются никогда: это текущая работа. 3. Outbox: опубликованные строки старше OUTBOX_RETENTION_SECONDS удаляются. 4. Обход bucket по всем prefix (объекты старше grace): - staging/, originals/: сирота, если ни одна строка не ссылается на ключ как source_key; - attachments//...: сирота, только если вложение в КОНЕЧНОМ состоянии и ключ ему не нужен: вложения нет; оно failed и replay уже невозможен (source_key IS NULL — исходник удалён по retention); оно ready, но ключ не входит в опубликованный набор (оригинал + варианты). У queued/processing и у failed с исходником в staging объекты не трогаются: replay/следующая попытка перезапишет их теми же ключами, и удаление устаревшим проходом сломало бы новый результат. Повторная проверка и DELETE выполняются в одной транзакции под FOR SHARE на строке вложения: replay (FOR UPDATE) и захват worker'ом (UPDATE) ждут конца DELETE, а не идут параллельно с ним. Отчёт о ready-вложениях без объекта в bucket (missing) — только отчёт, это потеря данных. Grace (ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч) больше любой живой загрузки/попытки, поэтому janitor не удаляет то, что ещё пишется. """ import argparse import json import logging import re import signal import threading import time from datetime import datetime, timedelta, timezone from pathlib import Path from uuid import UUID import psycopg from psycopg.rows import dict_row import pipeline import settings import storage logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") log = logging.getLogger("messenger.janitor") JANITOR_LOCK = 20260404 BATCH = 100 STOP = threading.Event() ACTOR = "janitor" FINAL_KEY_RE = re.compile(r"^attachments/([0-9a-f-]{36})/") def touch_alive() -> None: try: Path(settings.ALIVE_FILE).touch() except OSError: pass def idle(seconds: float) -> bool: """Пауза между проходами с обновлением файла живости (healthcheck). True — пришёл сигнал остановки.""" deadline = time.monotonic() + seconds while (left := deadline - time.monotonic()) > 0: touch_alive() if STOP.wait(min(left, 10)): return True return False def connect(): return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]: """Шаг 1. Возвращает (найдено, удалено).""" found = deleted = 0 while not STOP.is_set(): with conn.transaction(): rows = conn.execute( """SELECT id, source_key FROM attachments WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s) ORDER BY created_at LIMIT %s FOR UPDATE SKIP LOCKED""", (grace, BATCH), ).fetchall() found += len(rows) if dry_run or not rows: return found, deleted for row in rows: storage.delete(row["source_key"]) # недоступен S3 -> исключение, транзакция откатится conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],)) deleted += 1 log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"]) if len(rows) < BATCH: return found, deleted return found, deleted def cleanup_staging(conn, dry_run: bool) -> dict: """Шаг 2. Исходники ready (сразу) и failed (после retention).""" report = {"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0} while not STOP.is_set(): with conn.transaction(): rows = conn.execute( """SELECT a.id, a.state, a.source_key, j.job_id, j.attempt FROM attachments a LEFT JOIN attachment_jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version WHERE a.source_key IS NOT NULL AND ((a.state = 'ready' AND a.updated_at < now() - interval '60 seconds') OR (a.state = 'failed' AND a.updated_at < now() - make_interval(secs => %s))) ORDER BY a.updated_at LIMIT %s FOR UPDATE OF a SKIP LOCKED""", (settings.STAGING_RETENTION_SECONDS, BATCH), ).fetchall() if dry_run or not rows: if dry_run: report["staging_ready_deleted"] = sum(r["state"] == "ready" for r in rows) report["staging_failed_expired"] = sum(r["state"] == "failed" for r in rows) break for row in rows: storage.delete(row["source_key"]) conn.execute("UPDATE attachments SET source_key = NULL WHERE id = %s", (row["id"],)) if row["state"] == "ready": report["staging_ready_deleted"] += 1 else: report["staging_failed_expired"] += 1 if row["job_id"] is not None: pipeline.event(conn, row["job_id"], row["attempt"], "staging_expired", ACTOR, "replay no longer possible") if len(rows) < BATCH: break report["staging_kept_for_replay"] = conn.execute( "SELECT count(*) AS n FROM attachments WHERE state = 'failed' AND source_key IS NOT NULL").fetchone()["n"] return report def cleanup_outbox(conn, dry_run: bool) -> int: if dry_run: return conn.execute( """SELECT count(*) AS n FROM outbox WHERE published_at < now() - make_interval(secs => %s)""", (settings.OUTBOX_RETENTION_SECONDS,)).fetchone()["n"] return conn.execute( """DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)""", (settings.OUTBOX_RETENTION_SECONDS,)).rowcount def classify_final(conn, keys: list[str]) -> set[str]: """Какие ключи attachments//... не принадлежат ни одному живому вложению (кандидаты в сироты).""" ids = {} for key in keys: match = FINAL_KEY_RE.match(key) if match: try: ids[key] = UUID(match.group(1)) except ValueError: pass unique = list(set(ids.values())) rows = {r["id"]: r for r in conn.execute( """SELECT a.id, a.state, a.object_key, a.source_key, ARRAY(SELECT v.object_key FROM attachment_variants v WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys FROM attachments a WHERE a.id = ANY(%s)""", (unique,)).fetchall()} orphans = set() for key in keys: attachment_id = ids.get(key) row = rows.get(attachment_id) if attachment_id else None if row is None: orphans.add(key) elif row["state"] == "failed" and row["source_key"] is None: # failed без исходника — конечное состояние: replay отказывает («source file is no longer in # staging»), других переходов из failed нет, поэтому никто больше не запишет эти ключи. orphans.add(key) elif row["state"] == "ready" and key != row["object_key"] and key not in row["variant_keys"]: orphans.add(key) return orphans def sweep_objects(conn, grace: int, dry_run: bool) -> dict: """Шаг 4. Обход bucket постранично; сверка ключей с БД пачками по 1000.""" started = datetime.now(timezone.utc) cutoff = started - timedelta(seconds=grace) scanned = orphans = orphans_deleted = 0 listed = set() def orphan_keys(prefix, items): keys = [key for key, _ in items] if prefix == storage.FINAL_PREFIX: return classify_final(conn, keys) known = {r["source_key"] for r in conn.execute( "SELECT source_key FROM attachments WHERE source_key = ANY(%s)", (keys,)).fetchall()} return {key for key in keys if key not in known} def delete_if_orphan(prefix, key) -> bool: """Повторная проверка прямо перед удалением и DELETE — атомарно относительно replay/worker.""" if prefix != storage.FINAL_PREFIX: # staging/originals: строка-указатель создаётся ДО PUT, ключ случайный и не переиспользуется, # поэтому «нет строки сейчас» означает «не будет никогда». if conn.execute("SELECT 1 FROM attachments WHERE source_key = %s", (key,)).fetchone() is not None: return False storage.delete(key) return True match = FINAL_KEY_RE.match(key) try: attachment_id = UUID(match.group(1)) if match else None except ValueError: attachment_id = None with conn.transaction(): if attachment_id is not None: # FOR SHARE совместим с FOR KEY SHARE (вставка сообщения со ссылкой на вложение), но # конфликтует с FOR UPDATE replay и с UPDATE worker'а: состояние не изменится до COMMIT. conn.execute("SELECT 1 FROM attachments WHERE id = %s FOR SHARE", (attachment_id,)) if key not in classify_final(conn, [key]): return False storage.delete(key) # S3 недоступен -> исключение, транзакция откатится, повтор на след. проходе return True def check(prefix, items): nonlocal orphans, orphans_deleted candidates = orphan_keys(prefix, items) for key, modified in items: if key not in candidates or modified >= cutoff: continue orphans += 1 if not dry_run and delete_if_orphan(prefix, key): orphans_deleted += 1 log.info("deleted orphan object %s", key) for prefix in storage.ALL_PREFIXES: page = [] for key, modified, _size in storage.list_objects(prefix): scanned += 1 listed.add(key) page.append((key, modified)) if len(page) >= 1000: check(prefix, page) page = [] if page: check(prefix, page) missing = [] for r in conn.execute( """SELECT a.id, a.object_key, ARRAY(SELECT v.object_key FROM attachment_variants v WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys FROM attachments a WHERE a.state = 'ready' AND a.updated_at < %s""", (started,)).fetchall(): if r["object_key"] not in listed or any(k not in listed for k in r["variant_keys"]): missing.append(str(r["id"])) if missing: log.error("ready attachments with objects missing in S3 (not deleted, needs restore): %s", ", ".join(missing[:20])) return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted, "missing_objects": len(missing)} def run_once(grace: int, dry_run: bool) -> dict | None: started = time.monotonic() with connect() as conn: if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]: log.info("another janitor is running, skip") return None try: found, deleted = reclaim_stale_intents(conn, grace, dry_run) report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found, "stale_uploads_deleted": deleted, **cleanup_staging(conn, dry_run), "outbox_rows_deleted": cleanup_outbox(conn, dry_run), **sweep_objects(conn, grace, dry_run)} finally: conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,)) report["seconds"] = round(time.monotonic() - started, 3) return report def main(): parser = argparse.ArgumentParser(description="Reclaim abandoned uploads, clean staging, sweep orphan objects") parser.add_argument("--once", action="store_true", help="one pass and exit") parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing") parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS) args = parser.parse_args() if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS: parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} " "(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted") for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, lambda *_: STOP.set()) while True: touch_alive() try: report = run_once(args.grace_seconds, args.dry_run) if report is not None: print(json.dumps(report), flush=True) failed = False except (psycopg.Error, storage.StorageUnavailable) as exc: log.warning("pass failed, will retry: %s", type(exc).__name__) failed = True if args.once: raise SystemExit(1 if failed else 0) if idle(settings.JANITOR_INTERVAL_SECONDS): return if __name__ == "__main__": main()