forked from mtusi2026/mtusi2026lab6
323 lines
16 KiB
Python
323 lines
16 KiB
Python
"""Очистка и сверка PostgreSQL <-> S3 (сервис janitor).
|
||
|
||
python janitor.py --once [--dry-run] [--grace-seconds N]
|
||
|
||
Проход (раз в JANITOR_INTERVAL_SECONDS, один janitor за счёт pg_try_advisory_lock):
|
||
|
||
1. Брошенные намерения (как в lab4): state=uploading старше grace — исходник так и не был поставлен
|
||
в очередь. Удалить объект staging (идемпотентно), затем строку.
|
||
2. Staging после безопасной фиксации:
|
||
- ready: исходник уже опубликован в attachments/<id>/v<N>/original -> удалить staging-объект
|
||
(обычно это делает worker сразу после ack; здесь — если он не успел);
|
||
- failed: исходник хранится STAGING_RETENTION_SECONDS после ошибки (для replay), потом удаляется.
|
||
queued/processing не трогаются никогда: это текущая работа.
|
||
3. Outbox: опубликованные строки старше OUTBOX_RETENTION_SECONDS удаляются.
|
||
4. Обход bucket по всем prefix (объекты старше grace):
|
||
- staging/, originals/: сирота, если ни одна строка не ссылается на ключ как source_key;
|
||
- attachments/<id>/...: сирота, только если вложение в КОНЕЧНОМ состоянии и ключ ему не нужен:
|
||
вложения нет; оно failed и replay уже невозможен (source_key IS NULL — исходник удалён по retention);
|
||
оно ready, но ключ не входит в опубликованный набор (оригинал + варианты). У queued/processing
|
||
и у failed с исходником в staging объекты не трогаются: replay/следующая попытка перезапишет их
|
||
теми же ключами, и удаление устаревшим проходом сломало бы новый результат.
|
||
Повторная проверка и DELETE выполняются в одной транзакции под FOR SHARE на строке вложения:
|
||
replay (FOR UPDATE) и захват worker'ом (UPDATE) ждут конца DELETE, а не идут параллельно с ним.
|
||
Отчёт о ready-вложениях без объекта в bucket (missing) — только отчёт, это потеря данных.
|
||
|
||
Grace (ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч) больше любой живой загрузки/попытки, поэтому
|
||
janitor не удаляет то, что ещё пишется.
|
||
"""
|
||
import argparse
|
||
import json
|
||
import logging
|
||
import re
|
||
import signal
|
||
import threading
|
||
import time
|
||
from datetime import datetime, timedelta, timezone
|
||
from pathlib import Path
|
||
from uuid import UUID
|
||
|
||
import psycopg
|
||
from psycopg.rows import dict_row
|
||
|
||
import settings
|
||
import telemetry
|
||
|
||
telemetry.setup("messenger-janitor")
|
||
|
||
import pipeline # noqa: E402
|
||
import storage # noqa: E402
|
||
|
||
log = logging.getLogger("messenger.janitor")
|
||
|
||
JANITOR_LOCK = 20260404
|
||
BATCH = 100
|
||
STOP = threading.Event()
|
||
ACTOR = "janitor"
|
||
FINAL_KEY_RE = re.compile(r"^attachments/([0-9a-f-]{36})/")
|
||
|
||
|
||
def touch_alive() -> None:
|
||
try:
|
||
Path(settings.ALIVE_FILE).touch()
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def idle(seconds: float) -> bool:
|
||
"""Пауза между проходами с обновлением файла живости (healthcheck). True — пришёл сигнал остановки."""
|
||
deadline = time.monotonic() + seconds
|
||
while (left := deadline - time.monotonic()) > 0:
|
||
touch_alive()
|
||
if STOP.wait(min(left, 10)):
|
||
return True
|
||
return False
|
||
|
||
|
||
def connect():
|
||
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
|
||
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
|
||
|
||
|
||
def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]:
|
||
"""Шаг 1. Возвращает (найдено, удалено)."""
|
||
found = deleted = 0
|
||
while not STOP.is_set():
|
||
with conn.transaction():
|
||
rows = conn.execute(
|
||
"""SELECT id, source_key FROM attachments
|
||
WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s)
|
||
ORDER BY created_at LIMIT %s
|
||
FOR UPDATE SKIP LOCKED""",
|
||
(grace, BATCH),
|
||
).fetchall()
|
||
found += len(rows)
|
||
if dry_run or not rows:
|
||
return found, deleted
|
||
for row in rows:
|
||
storage.delete(row["source_key"]) # недоступен S3 -> исключение, транзакция откатится
|
||
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],))
|
||
deleted += 1
|
||
log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"])
|
||
if len(rows) < BATCH:
|
||
return found, deleted
|
||
return found, deleted
|
||
|
||
|
||
def cleanup_staging(conn, dry_run: bool) -> dict:
|
||
"""Шаг 2. Исходники ready (сразу) и failed (после retention)."""
|
||
report = {"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0}
|
||
while not STOP.is_set():
|
||
with conn.transaction():
|
||
rows = conn.execute(
|
||
"""SELECT a.id, a.state, a.source_key, j.job_id, j.attempt FROM attachments a
|
||
LEFT JOIN attachment_jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version
|
||
WHERE a.source_key IS NOT NULL
|
||
AND ((a.state = 'ready' AND a.updated_at < now() - interval '60 seconds')
|
||
OR (a.state = 'failed' AND a.updated_at < now() - make_interval(secs => %s)))
|
||
ORDER BY a.updated_at LIMIT %s FOR UPDATE OF a SKIP LOCKED""",
|
||
(settings.STAGING_RETENTION_SECONDS, BATCH),
|
||
).fetchall()
|
||
if dry_run or not rows:
|
||
if dry_run:
|
||
report["staging_ready_deleted"] = sum(r["state"] == "ready" for r in rows)
|
||
report["staging_failed_expired"] = sum(r["state"] == "failed" for r in rows)
|
||
break
|
||
for row in rows:
|
||
storage.delete(row["source_key"])
|
||
conn.execute("UPDATE attachments SET source_key = NULL WHERE id = %s", (row["id"],))
|
||
if row["state"] == "ready":
|
||
report["staging_ready_deleted"] += 1
|
||
else:
|
||
report["staging_failed_expired"] += 1
|
||
if row["job_id"] is not None:
|
||
pipeline.event(conn, row["job_id"], row["attempt"], "staging_expired", ACTOR,
|
||
"replay no longer possible")
|
||
if len(rows) < BATCH:
|
||
break
|
||
report["staging_kept_for_replay"] = conn.execute(
|
||
"SELECT count(*) AS n FROM attachments WHERE state = 'failed' AND source_key IS NOT NULL").fetchone()["n"]
|
||
return report
|
||
|
||
|
||
def cleanup_outbox(conn, dry_run: bool) -> int:
|
||
if dry_run:
|
||
return conn.execute(
|
||
"""SELECT count(*) AS n FROM outbox WHERE published_at < now() - make_interval(secs => %s)""",
|
||
(settings.OUTBOX_RETENTION_SECONDS,)).fetchone()["n"]
|
||
return conn.execute(
|
||
"""DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)""",
|
||
(settings.OUTBOX_RETENTION_SECONDS,)).rowcount
|
||
|
||
|
||
def classify_final(conn, keys: list[str]) -> set[str]:
|
||
"""Какие ключи attachments/<id>/... не принадлежат ни одному живому вложению (кандидаты в сироты)."""
|
||
ids = {}
|
||
for key in keys:
|
||
match = FINAL_KEY_RE.match(key)
|
||
if match:
|
||
try:
|
||
ids[key] = UUID(match.group(1))
|
||
except ValueError:
|
||
pass
|
||
unique = list(set(ids.values()))
|
||
rows = {r["id"]: r for r in conn.execute(
|
||
"""SELECT a.id, a.state, a.object_key, a.source_key,
|
||
ARRAY(SELECT v.object_key FROM attachment_variants v
|
||
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys
|
||
FROM attachments a WHERE a.id = ANY(%s)""", (unique,)).fetchall()}
|
||
orphans = set()
|
||
for key in keys:
|
||
attachment_id = ids.get(key)
|
||
row = rows.get(attachment_id) if attachment_id else None
|
||
if row is None:
|
||
orphans.add(key)
|
||
elif row["state"] == "failed" and row["source_key"] is None:
|
||
# failed без исходника — конечное состояние: replay отказывает («source file is no longer in
|
||
# staging»), других переходов из failed нет, поэтому никто больше не запишет эти ключи.
|
||
orphans.add(key)
|
||
elif row["state"] == "ready" and key != row["object_key"] and key not in row["variant_keys"]:
|
||
orphans.add(key)
|
||
return orphans
|
||
|
||
|
||
def sweep_objects(conn, grace: int, dry_run: bool) -> dict:
|
||
"""Шаг 4. Обход bucket постранично; сверка ключей с БД пачками по 1000."""
|
||
started = datetime.now(timezone.utc)
|
||
cutoff = started - timedelta(seconds=grace)
|
||
scanned = orphans = orphans_deleted = 0
|
||
listed = set()
|
||
|
||
def orphan_keys(prefix, items):
|
||
keys = [key for key, _ in items]
|
||
if prefix == storage.FINAL_PREFIX:
|
||
return classify_final(conn, keys)
|
||
known = {r["source_key"] for r in conn.execute(
|
||
"SELECT source_key FROM attachments WHERE source_key = ANY(%s)", (keys,)).fetchall()}
|
||
return {key for key in keys if key not in known}
|
||
|
||
def delete_if_orphan(prefix, key) -> bool:
|
||
"""Повторная проверка прямо перед удалением и DELETE — атомарно относительно replay/worker."""
|
||
if prefix != storage.FINAL_PREFIX:
|
||
# staging/originals: строка-указатель создаётся ДО PUT, ключ случайный и не переиспользуется,
|
||
# поэтому «нет строки сейчас» означает «не будет никогда».
|
||
if conn.execute("SELECT 1 FROM attachments WHERE source_key = %s", (key,)).fetchone() is not None:
|
||
return False
|
||
storage.delete(key)
|
||
return True
|
||
match = FINAL_KEY_RE.match(key)
|
||
try:
|
||
attachment_id = UUID(match.group(1)) if match else None
|
||
except ValueError:
|
||
attachment_id = None
|
||
with conn.transaction():
|
||
if attachment_id is not None:
|
||
# FOR SHARE совместим с FOR KEY SHARE (вставка сообщения со ссылкой на вложение), но
|
||
# конфликтует с FOR UPDATE replay и с UPDATE worker'а: состояние не изменится до COMMIT.
|
||
conn.execute("SELECT 1 FROM attachments WHERE id = %s FOR SHARE", (attachment_id,))
|
||
if key not in classify_final(conn, [key]):
|
||
return False
|
||
storage.delete(key) # S3 недоступен -> исключение, транзакция откатится, повтор на след. проходе
|
||
return True
|
||
|
||
def check(prefix, items):
|
||
nonlocal orphans, orphans_deleted
|
||
candidates = orphan_keys(prefix, items)
|
||
for key, modified in items:
|
||
if key not in candidates or modified >= cutoff:
|
||
continue
|
||
orphans += 1
|
||
if not dry_run and delete_if_orphan(prefix, key):
|
||
orphans_deleted += 1
|
||
log.info("deleted orphan object %s", key)
|
||
|
||
for prefix in storage.ALL_PREFIXES:
|
||
page = []
|
||
for key, modified, _size in storage.list_objects(prefix):
|
||
scanned += 1
|
||
listed.add(key)
|
||
page.append((key, modified))
|
||
if len(page) >= 1000:
|
||
check(prefix, page)
|
||
page = []
|
||
if page:
|
||
check(prefix, page)
|
||
|
||
missing = []
|
||
for r in conn.execute(
|
||
"""SELECT a.id, a.object_key, ARRAY(SELECT v.object_key FROM attachment_variants v
|
||
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys
|
||
FROM attachments a WHERE a.state = 'ready' AND a.updated_at < %s""", (started,)).fetchall():
|
||
if r["object_key"] not in listed or any(k not in listed for k in r["variant_keys"]):
|
||
missing.append(str(r["id"]))
|
||
if missing:
|
||
log.error("ready attachments with objects missing in S3 (not deleted, needs restore): %s",
|
||
", ".join(missing[:20]))
|
||
return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted,
|
||
"missing_objects": len(missing)}
|
||
|
||
|
||
def run_once(grace: int, dry_run: bool) -> dict | None:
|
||
# Один проход = одна трасса (раз в JANITOR_INTERVAL_SECONDS): запросы к БД и S3 — её дети.
|
||
with telemetry.internal_span("janitor pass", {"messenger.janitor.dry_run": dry_run}) as span:
|
||
report = _run_once(grace, dry_run)
|
||
if report:
|
||
span.set_attributes({f"messenger.janitor.{k}": v for k, v in report.items()
|
||
if isinstance(v, (int, float, bool))})
|
||
return report
|
||
|
||
|
||
def _run_once(grace: int, dry_run: bool) -> dict | None:
|
||
started = time.monotonic()
|
||
with connect() as conn:
|
||
if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]:
|
||
log.info("another janitor is running, skip")
|
||
return None
|
||
try:
|
||
found, deleted = reclaim_stale_intents(conn, grace, dry_run)
|
||
report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found,
|
||
"stale_uploads_deleted": deleted, **cleanup_staging(conn, dry_run),
|
||
"outbox_rows_deleted": cleanup_outbox(conn, dry_run), **sweep_objects(conn, grace, dry_run)}
|
||
finally:
|
||
conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,))
|
||
report["seconds"] = round(time.monotonic() - started, 3)
|
||
return report
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="Reclaim abandoned uploads, clean staging, sweep orphan objects")
|
||
parser.add_argument("--once", action="store_true", help="one pass and exit")
|
||
parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing")
|
||
parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS)
|
||
args = parser.parse_args()
|
||
if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS:
|
||
parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} "
|
||
"(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted")
|
||
|
||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||
signal.signal(sig, lambda *_: STOP.set())
|
||
while True:
|
||
touch_alive()
|
||
try:
|
||
report = run_once(args.grace_seconds, args.dry_run)
|
||
if report is not None:
|
||
print(json.dumps(report), flush=True)
|
||
log.info("janitor pass done", extra={"event": "janitor.pass", **{
|
||
k: v for k, v in report.items() if isinstance(v, (int, float, bool))}})
|
||
failed = False
|
||
except (psycopg.Error, storage.StorageUnavailable) as exc:
|
||
log.warning("pass failed, will retry: %s", type(exc).__name__,
|
||
extra={"event": "janitor.pass_failed", "error.type": type(exc).__name__})
|
||
failed = True
|
||
if args.once:
|
||
telemetry.shutdown()
|
||
raise SystemExit(1 if failed else 0)
|
||
if idle(settings.JANITOR_INTERVAL_SECONDS):
|
||
telemetry.shutdown()
|
||
return
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|