Files
mtusi2026lab6/janitor.py
T

323 lines
16 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Очистка и сверка PostgreSQL <-> S3 (сервис janitor).
python janitor.py --once [--dry-run] [--grace-seconds N]
Проход (раз в JANITOR_INTERVAL_SECONDS, один janitor за счёт pg_try_advisory_lock):
1. Брошенные намерения (как в lab4): state=uploading старше grace — исходник так и не был поставлен
в очередь. Удалить объект staging (идемпотентно), затем строку.
2. Staging после безопасной фиксации:
- ready: исходник уже опубликован в attachments/<id>/v<N>/original -> удалить staging-объект
(обычно это делает worker сразу после ack; здесь — если он не успел);
- failed: исходник хранится STAGING_RETENTION_SECONDS после ошибки (для replay), потом удаляется.
queued/processing не трогаются никогда: это текущая работа.
3. Outbox: опубликованные строки старше OUTBOX_RETENTION_SECONDS удаляются.
4. Обход bucket по всем prefix (объекты старше grace):
- staging/, originals/: сирота, если ни одна строка не ссылается на ключ как source_key;
- attachments/<id>/...: сирота, только если вложение в КОНЕЧНОМ состоянии и ключ ему не нужен:
вложения нет; оно failed и replay уже невозможен (source_key IS NULL — исходник удалён по retention);
оно ready, но ключ не входит в опубликованный набор (оригинал + варианты). У queued/processing
и у failed с исходником в staging объекты не трогаются: replay/следующая попытка перезапишет их
теми же ключами, и удаление устаревшим проходом сломало бы новый результат.
Повторная проверка и DELETE выполняются в одной транзакции под FOR SHARE на строке вложения:
replay (FOR UPDATE) и захват worker'ом (UPDATE) ждут конца DELETE, а не идут параллельно с ним.
Отчёт о ready-вложениях без объекта в bucket (missing) — только отчёт, это потеря данных.
Grace (ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч) больше любой живой загрузки/попытки, поэтому
janitor не удаляет то, что ещё пишется.
"""
import argparse
import json
import logging
import re
import signal
import threading
import time
from datetime import datetime, timedelta, timezone
from pathlib import Path
from uuid import UUID
import psycopg
from psycopg.rows import dict_row
import settings
import telemetry
telemetry.setup("messenger-janitor")
import pipeline # noqa: E402
import storage # noqa: E402
log = logging.getLogger("messenger.janitor")
JANITOR_LOCK = 20260404
BATCH = 100
STOP = threading.Event()
ACTOR = "janitor"
FINAL_KEY_RE = re.compile(r"^attachments/([0-9a-f-]{36})/")
def touch_alive() -> None:
try:
Path(settings.ALIVE_FILE).touch()
except OSError:
pass
def idle(seconds: float) -> bool:
"""Пауза между проходами с обновлением файла живости (healthcheck). True — пришёл сигнал остановки."""
deadline = time.monotonic() + seconds
while (left := deadline - time.monotonic()) > 0:
touch_alive()
if STOP.wait(min(left, 10)):
return True
return False
def connect():
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]:
"""Шаг 1. Возвращает (найдено, удалено)."""
found = deleted = 0
while not STOP.is_set():
with conn.transaction():
rows = conn.execute(
"""SELECT id, source_key FROM attachments
WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s)
ORDER BY created_at LIMIT %s
FOR UPDATE SKIP LOCKED""",
(grace, BATCH),
).fetchall()
found += len(rows)
if dry_run or not rows:
return found, deleted
for row in rows:
storage.delete(row["source_key"]) # недоступен S3 -> исключение, транзакция откатится
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],))
deleted += 1
log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"])
if len(rows) < BATCH:
return found, deleted
return found, deleted
def cleanup_staging(conn, dry_run: bool) -> dict:
"""Шаг 2. Исходники ready (сразу) и failed (после retention)."""
report = {"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0}
while not STOP.is_set():
with conn.transaction():
rows = conn.execute(
"""SELECT a.id, a.state, a.source_key, j.job_id, j.attempt FROM attachments a
LEFT JOIN attachment_jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version
WHERE a.source_key IS NOT NULL
AND ((a.state = 'ready' AND a.updated_at < now() - interval '60 seconds')
OR (a.state = 'failed' AND a.updated_at < now() - make_interval(secs => %s)))
ORDER BY a.updated_at LIMIT %s FOR UPDATE OF a SKIP LOCKED""",
(settings.STAGING_RETENTION_SECONDS, BATCH),
).fetchall()
if dry_run or not rows:
if dry_run:
report["staging_ready_deleted"] = sum(r["state"] == "ready" for r in rows)
report["staging_failed_expired"] = sum(r["state"] == "failed" for r in rows)
break
for row in rows:
storage.delete(row["source_key"])
conn.execute("UPDATE attachments SET source_key = NULL WHERE id = %s", (row["id"],))
if row["state"] == "ready":
report["staging_ready_deleted"] += 1
else:
report["staging_failed_expired"] += 1
if row["job_id"] is not None:
pipeline.event(conn, row["job_id"], row["attempt"], "staging_expired", ACTOR,
"replay no longer possible")
if len(rows) < BATCH:
break
report["staging_kept_for_replay"] = conn.execute(
"SELECT count(*) AS n FROM attachments WHERE state = 'failed' AND source_key IS NOT NULL").fetchone()["n"]
return report
def cleanup_outbox(conn, dry_run: bool) -> int:
if dry_run:
return conn.execute(
"""SELECT count(*) AS n FROM outbox WHERE published_at < now() - make_interval(secs => %s)""",
(settings.OUTBOX_RETENTION_SECONDS,)).fetchone()["n"]
return conn.execute(
"""DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)""",
(settings.OUTBOX_RETENTION_SECONDS,)).rowcount
def classify_final(conn, keys: list[str]) -> set[str]:
"""Какие ключи attachments/<id>/... не принадлежат ни одному живому вложению (кандидаты в сироты)."""
ids = {}
for key in keys:
match = FINAL_KEY_RE.match(key)
if match:
try:
ids[key] = UUID(match.group(1))
except ValueError:
pass
unique = list(set(ids.values()))
rows = {r["id"]: r for r in conn.execute(
"""SELECT a.id, a.state, a.object_key, a.source_key,
ARRAY(SELECT v.object_key FROM attachment_variants v
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys
FROM attachments a WHERE a.id = ANY(%s)""", (unique,)).fetchall()}
orphans = set()
for key in keys:
attachment_id = ids.get(key)
row = rows.get(attachment_id) if attachment_id else None
if row is None:
orphans.add(key)
elif row["state"] == "failed" and row["source_key"] is None:
# failed без исходника — конечное состояние: replay отказывает («source file is no longer in
# staging»), других переходов из failed нет, поэтому никто больше не запишет эти ключи.
orphans.add(key)
elif row["state"] == "ready" and key != row["object_key"] and key not in row["variant_keys"]:
orphans.add(key)
return orphans
def sweep_objects(conn, grace: int, dry_run: bool) -> dict:
"""Шаг 4. Обход bucket постранично; сверка ключей с БД пачками по 1000."""
started = datetime.now(timezone.utc)
cutoff = started - timedelta(seconds=grace)
scanned = orphans = orphans_deleted = 0
listed = set()
def orphan_keys(prefix, items):
keys = [key for key, _ in items]
if prefix == storage.FINAL_PREFIX:
return classify_final(conn, keys)
known = {r["source_key"] for r in conn.execute(
"SELECT source_key FROM attachments WHERE source_key = ANY(%s)", (keys,)).fetchall()}
return {key for key in keys if key not in known}
def delete_if_orphan(prefix, key) -> bool:
"""Повторная проверка прямо перед удалением и DELETE — атомарно относительно replay/worker."""
if prefix != storage.FINAL_PREFIX:
# staging/originals: строка-указатель создаётся ДО PUT, ключ случайный и не переиспользуется,
# поэтому «нет строки сейчас» означает «не будет никогда».
if conn.execute("SELECT 1 FROM attachments WHERE source_key = %s", (key,)).fetchone() is not None:
return False
storage.delete(key)
return True
match = FINAL_KEY_RE.match(key)
try:
attachment_id = UUID(match.group(1)) if match else None
except ValueError:
attachment_id = None
with conn.transaction():
if attachment_id is not None:
# FOR SHARE совместим с FOR KEY SHARE (вставка сообщения со ссылкой на вложение), но
# конфликтует с FOR UPDATE replay и с UPDATE worker'а: состояние не изменится до COMMIT.
conn.execute("SELECT 1 FROM attachments WHERE id = %s FOR SHARE", (attachment_id,))
if key not in classify_final(conn, [key]):
return False
storage.delete(key) # S3 недоступен -> исключение, транзакция откатится, повтор на след. проходе
return True
def check(prefix, items):
nonlocal orphans, orphans_deleted
candidates = orphan_keys(prefix, items)
for key, modified in items:
if key not in candidates or modified >= cutoff:
continue
orphans += 1
if not dry_run and delete_if_orphan(prefix, key):
orphans_deleted += 1
log.info("deleted orphan object %s", key)
for prefix in storage.ALL_PREFIXES:
page = []
for key, modified, _size in storage.list_objects(prefix):
scanned += 1
listed.add(key)
page.append((key, modified))
if len(page) >= 1000:
check(prefix, page)
page = []
if page:
check(prefix, page)
missing = []
for r in conn.execute(
"""SELECT a.id, a.object_key, ARRAY(SELECT v.object_key FROM attachment_variants v
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys
FROM attachments a WHERE a.state = 'ready' AND a.updated_at < %s""", (started,)).fetchall():
if r["object_key"] not in listed or any(k not in listed for k in r["variant_keys"]):
missing.append(str(r["id"]))
if missing:
log.error("ready attachments with objects missing in S3 (not deleted, needs restore): %s",
", ".join(missing[:20]))
return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted,
"missing_objects": len(missing)}
def run_once(grace: int, dry_run: bool) -> dict | None:
# Один проход = одна трасса (раз в JANITOR_INTERVAL_SECONDS): запросы к БД и S3 — её дети.
with telemetry.internal_span("janitor pass", {"messenger.janitor.dry_run": dry_run}) as span:
report = _run_once(grace, dry_run)
if report:
span.set_attributes({f"messenger.janitor.{k}": v for k, v in report.items()
if isinstance(v, (int, float, bool))})
return report
def _run_once(grace: int, dry_run: bool) -> dict | None:
started = time.monotonic()
with connect() as conn:
if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]:
log.info("another janitor is running, skip")
return None
try:
found, deleted = reclaim_stale_intents(conn, grace, dry_run)
report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found,
"stale_uploads_deleted": deleted, **cleanup_staging(conn, dry_run),
"outbox_rows_deleted": cleanup_outbox(conn, dry_run), **sweep_objects(conn, grace, dry_run)}
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,))
report["seconds"] = round(time.monotonic() - started, 3)
return report
def main():
parser = argparse.ArgumentParser(description="Reclaim abandoned uploads, clean staging, sweep orphan objects")
parser.add_argument("--once", action="store_true", help="one pass and exit")
parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing")
parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS)
args = parser.parse_args()
if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS:
parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} "
"(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted")
for sig in (signal.SIGTERM, signal.SIGINT):
signal.signal(sig, lambda *_: STOP.set())
while True:
touch_alive()
try:
report = run_once(args.grace_seconds, args.dry_run)
if report is not None:
print(json.dumps(report), flush=True)
log.info("janitor pass done", extra={"event": "janitor.pass", **{
k: v for k, v in report.items() if isinstance(v, (int, float, bool))}})
failed = False
except (psycopg.Error, storage.StorageUnavailable) as exc:
log.warning("pass failed, will retry: %s", type(exc).__name__,
extra={"event": "janitor.pass_failed", "error.type": type(exc).__name__})
failed = True
if args.once:
telemetry.shutdown()
raise SystemExit(1 if failed else 0)
if idle(settings.JANITOR_INTERVAL_SECONDS):
telemetry.shutdown()
return
if __name__ == "__main__":
main()