Files
mtusi2026lab5/janitor.py
T

279 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Очистка и сверка PostgreSQL <-> S3 (сервис janitor).
python janitor.py --once [--dry-run] [--grace-seconds N]
Проход (раз в JANITOR_INTERVAL_SECONDS, один janitor за счёт pg_try_advisory_lock):
1. Брошенные намерения (как в lab4): state=uploading старше grace — исходник так и не был поставлен
в очередь. Удалить объект staging (идемпотентно), затем строку.
2. Staging после безопасной фиксации:
- ready: исходник уже опубликован в attachments/<id>/v<N>/original -> удалить staging-объект
(обычно это делает worker сразу после ack; здесь — если он не успел);
- failed: исходник хранится STAGING_RETENTION_SECONDS после ошибки (для replay), потом удаляется.
queued/processing не трогаются никогда: это текущая работа.
3. Outbox: опубликованные строки старше OUTBOX_RETENTION_SECONDS удаляются.
4. Обход bucket по всем prefix (объекты старше grace):
- staging/, originals/: сирота, если ни одна строка не ссылается на ключ как source_key;
- attachments/<id>/...: сирота, если вложения нет, оно failed, или оно ready, но ключ не входит
в его опубликованный набор (оригинал + варианты). У queued/processing объекты не трогаются:
их перезапишет следующая попытка теми же ключами.
Отчёт о ready-вложениях без объекта в bucket (missing) — только отчёт, это потеря данных.
Grace (ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч) больше любой живой загрузки/попытки, поэтому
janitor не удаляет то, что ещё пишется.
"""
import argparse
import json
import logging
import re
import signal
import threading
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from uuid import UUID
import psycopg
from psycopg.rows import dict_row
import pipeline
import settings
import storage
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
log = logging.getLogger("messenger.janitor")
JANITOR_LOCK = 20260404
BATCH = 100
STOP = threading.Event()
ACTOR = "janitor"
FINAL_KEY_RE = re.compile(r"^attachments/([0-9a-f-]{36})/")
def touch_alive() -> None:
try:
Path(settings.ALIVE_FILE).touch()
except OSError:
pass
def idle(seconds: float) -> bool:
"""Пауза между проходами с обновлением файла живости (healthcheck). True — пришёл сигнал остановки."""
deadline = time.monotonic() + seconds
while (left := deadline - time.monotonic()) > 0:
touch_alive()
if STOP.wait(min(left, 10)):
return True
return False
def connect():
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]:
"""Шаг 1. Возвращает (найдено, удалено)."""
found = deleted = 0
while not STOP.is_set():
with conn.transaction():
rows = conn.execute(
"""SELECT id, source_key FROM attachments
WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s)
ORDER BY created_at LIMIT %s
FOR UPDATE SKIP LOCKED""",
(grace, BATCH),
).fetchall()
found += len(rows)
if dry_run or not rows:
return found, deleted
for row in rows:
storage.delete(row["source_key"]) # недоступен S3 -> исключение, транзакция откатится
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],))
deleted += 1
log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"])
if len(rows) < BATCH:
return found, deleted
return found, deleted
def cleanup_staging(conn, dry_run: bool) -> dict:
"""Шаг 2. Исходники ready (сразу) и failed (после retention)."""
report = {"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0}
while not STOP.is_set():
with conn.transaction():
rows = conn.execute(
"""SELECT a.id, a.state, a.source_key, j.job_id, j.attempt FROM attachments a
LEFT JOIN attachment_jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version
WHERE a.source_key IS NOT NULL
AND ((a.state = 'ready' AND a.updated_at < now() - interval '60 seconds')
OR (a.state = 'failed' AND a.updated_at < now() - make_interval(secs => %s)))
ORDER BY a.updated_at LIMIT %s FOR UPDATE OF a SKIP LOCKED""",
(settings.STAGING_RETENTION_SECONDS, BATCH),
).fetchall()
if dry_run or not rows:
if dry_run:
report["staging_ready_deleted"] = sum(r["state"] == "ready" for r in rows)
report["staging_failed_expired"] = sum(r["state"] == "failed" for r in rows)
break
for row in rows:
storage.delete(row["source_key"])
conn.execute("UPDATE attachments SET source_key = NULL WHERE id = %s", (row["id"],))
if row["state"] == "ready":
report["staging_ready_deleted"] += 1
else:
report["staging_failed_expired"] += 1
if row["job_id"] is not None:
pipeline.event(conn, row["job_id"], row["attempt"], "staging_expired", ACTOR,
"replay no longer possible")
if len(rows) < BATCH:
break
report["staging_kept_for_replay"] = conn.execute(
"SELECT count(*) AS n FROM attachments WHERE state = 'failed' AND source_key IS NOT NULL").fetchone()["n"]
return report
def cleanup_outbox(conn, dry_run: bool) -> int:
if dry_run:
return conn.execute(
"""SELECT count(*) AS n FROM outbox WHERE published_at < now() - make_interval(secs => %s)""",
(settings.OUTBOX_RETENTION_SECONDS,)).fetchone()["n"]
return conn.execute(
"""DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)""",
(settings.OUTBOX_RETENTION_SECONDS,)).rowcount
def classify_final(conn, keys: list[str]) -> set[str]:
"""Какие ключи attachments/<id>/... не принадлежат ни одному живому вложению (кандидаты в сироты)."""
ids = {}
for key in keys:
match = FINAL_KEY_RE.match(key)
if match:
try:
ids[key] = UUID(match.group(1))
except ValueError:
pass
unique = list(set(ids.values()))
rows = {r["id"]: r for r in conn.execute(
"""SELECT a.id, a.state, a.object_key,
ARRAY(SELECT v.object_key FROM attachment_variants v
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys
FROM attachments a WHERE a.id = ANY(%s)""", (unique,)).fetchall()}
orphans = set()
for key in keys:
attachment_id = ids.get(key)
row = rows.get(attachment_id) if attachment_id else None
if row is None or row["state"] == "failed":
orphans.add(key)
elif row["state"] == "ready" and key != row["object_key"] and key not in row["variant_keys"]:
orphans.add(key)
return orphans
def sweep_objects(conn, grace: int, dry_run: bool) -> dict:
"""Шаг 4. Обход bucket постранично; сверка ключей с БД пачками по 1000."""
started = datetime.now(timezone.utc)
cutoff = started - timedelta(seconds=grace)
scanned = orphans = orphans_deleted = 0
listed = set()
def orphan_keys(prefix, items):
keys = [key for key, _ in items]
if prefix == storage.FINAL_PREFIX:
return classify_final(conn, keys)
known = {r["source_key"] for r in conn.execute(
"SELECT source_key FROM attachments WHERE source_key = ANY(%s)", (keys,)).fetchall()}
return {key for key in keys if key not in known}
def still_orphan(prefix, key):
if prefix == storage.FINAL_PREFIX:
return key in classify_final(conn, [key])
return conn.execute("SELECT 1 FROM attachments WHERE source_key = %s", (key,)).fetchone() is None
def check(prefix, items):
nonlocal orphans, orphans_deleted
candidates = orphan_keys(prefix, items)
for key, modified in items:
if key not in candidates or modified >= cutoff:
continue
orphans += 1
if not dry_run and still_orphan(prefix, key): # повторная проверка прямо перед удалением
storage.delete(key)
orphans_deleted += 1
log.info("deleted orphan object %s", key)
for prefix in storage.ALL_PREFIXES:
page = []
for key, modified, _size in storage.list_objects(prefix):
scanned += 1
listed.add(key)
page.append((key, modified))
if len(page) >= 1000:
check(prefix, page)
page = []
if page:
check(prefix, page)
missing = []
for r in conn.execute(
"""SELECT a.id, a.object_key, ARRAY(SELECT v.object_key FROM attachment_variants v
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys
FROM attachments a WHERE a.state = 'ready' AND a.updated_at < %s""", (started,)).fetchall():
if r["object_key"] not in listed or any(k not in listed for k in r["variant_keys"]):
missing.append(str(r["id"]))
if missing:
log.error("ready attachments with objects missing in S3 (not deleted, needs restore): %s",
", ".join(missing[:20]))
return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted,
"missing_objects": len(missing)}
def run_once(grace: int, dry_run: bool) -> dict | None:
started = time.monotonic()
with connect() as conn:
if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]:
log.info("another janitor is running, skip")
return None
try:
found, deleted = reclaim_stale_intents(conn, grace, dry_run)
report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found,
"stale_uploads_deleted": deleted, **cleanup_staging(conn, dry_run),
"outbox_rows_deleted": cleanup_outbox(conn, dry_run), **sweep_objects(conn, grace, dry_run)}
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,))
report["seconds"] = round(time.monotonic() - started, 3)
return report
def main():
parser = argparse.ArgumentParser(description="Reclaim abandoned uploads, clean staging, sweep orphan objects")
parser.add_argument("--once", action="store_true", help="one pass and exit")
parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing")
parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS)
args = parser.parse_args()
if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS:
parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} "
"(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted")
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, lambda *_: STOP.set())
while True:
touch_alive()
try:
report = run_once(args.grace_seconds, args.dry_run)
if report is not None:
print(json.dumps(report), flush=True)
failed = False
except (psycopg.Error, storage.StorageUnavailable) as exc:
log.warning("pass failed, will retry: %s", type(exc).__name__)
failed = True
if args.once:
raise SystemExit(1 if failed else 0)
if idle(settings.JANITOR_INTERVAL_SECONDS):
return
if __name__ == "__main__":
main()