585 lines
33 KiB
Python
585 lines
33 KiB
Python
"""
|
||
Worker фоновой обработки вложений (lab5). Отдельный процесс/контейнер: `python -m app.worker`.
|
||
|
||
Поток одной задачи (сообщение брокера = только ID, см. app/broker.py):
|
||
|
||
XREADGROUP ─► claim в БД (FOR UPDATE): queued → processing, lease_token, lease_until
|
||
─► скачать оригинал из staging, сверить sha256 с БД (доверенные метаданные)
|
||
─► дочерний процесс: thumbnail / crop / optimized (лимиты памяти и времени)
|
||
─► S3: copy staging → originals/<id>, PUT variants/<id>/v1/* (детерминированные ключи)
|
||
─► COMMIT результата (только владелец аренды): variants ON CONFLICT DO NOTHING,
|
||
attachment и job → ready
|
||
─► XACK (+XDEL) ─► удалить staging (после безопасной фиксации)
|
||
|
||
Ошибки:
|
||
постоянные (битое изображение, sha256 не совпал, исходника нет) → failed + error_code;
|
||
временные (S3/БД недоступны, таймаут обработки) → queued, attempt+1, backoff с jitter
|
||
через outbox (available_at в будущем); попытки кончились → failed retry_exhausted.
|
||
|
||
Отказы worker:
|
||
умер до XACK → сообщение остаётся в PEL брокера; после рестарта worker читает свой PEL,
|
||
чужой брошенный PEL забирает XAUTOCLAIM (CLAIM_IDLE_SECONDS);
|
||
завис/умер с арендой → аренду никто не продлевает, после истечения задачу
|
||
забирает другой (lease takeover) или reaper переводит её в queued с новой попыткой.
|
||
Устаревший worker после пробуждения ничего не перезапишет: COMMIT проверяет lease_token.
|
||
|
||
Поток maintenance (в каждом worker, безопасно параллельно — FOR UPDATE SKIP LOCKED):
|
||
outbox relay, reaper просроченных аренд, очистка staging, heartbeat-файл для healthcheck.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import multiprocessing
|
||
import os
|
||
import random
|
||
import shutil
|
||
import signal
|
||
import tempfile
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
|
||
import psycopg
|
||
import redis
|
||
from psycopg_pool import ConnectionPool
|
||
|
||
from . import broker, imaging, storage
|
||
from .config import settings
|
||
|
||
log = logging.getLogger("messenger.worker")
|
||
|
||
|
||
class Transient(Exception):
|
||
def __init__(self, code: str, detail: str = ""):
|
||
super().__init__(code)
|
||
self.code, self.detail = code, detail
|
||
|
||
|
||
class Permanent(Exception):
|
||
def __init__(self, code: str, detail: str = ""):
|
||
super().__init__(code)
|
||
self.code, self.detail = code, detail
|
||
|
||
|
||
class LeaseLost(Exception):
|
||
"""Аренду забрали (истекла и задачу взял другой) — результат фиксировать нельзя."""
|
||
|
||
|
||
@dataclass
|
||
class Job:
|
||
job_id: str
|
||
attachment_id: str
|
||
pipeline_version: int
|
||
attempt: int
|
||
max_attempts: int
|
||
token: str
|
||
staging_key: str | None
|
||
object_key: str
|
||
sha256: str
|
||
size_bytes: int
|
||
content_type: str
|
||
|
||
|
||
def backoff_seconds(attempt: int) -> float:
|
||
"""Экспоненциальный backoff с jitter («equal jitter»): половина задержки
|
||
фиксирована, половина случайна — повторы разных задач не идут синхронной волной."""
|
||
d = min(settings.job_backoff_max_seconds, settings.job_backoff_base_seconds * 2 ** (attempt - 1))
|
||
return d / 2 + random.uniform(0, d / 2)
|
||
|
||
|
||
class Worker:
|
||
def __init__(self) -> None:
|
||
self.stop = threading.Event()
|
||
self.name = settings.instance_id
|
||
self.pool = ConnectionPool(
|
||
conninfo=settings.database_url, min_size=1,
|
||
max_size=settings.worker_concurrency + 3, timeout=settings.db_pool_timeout,
|
||
kwargs={"connect_timeout": settings.db_connect_timeout,
|
||
"options": f"-c statement_timeout={settings.db_statement_timeout_ms}",
|
||
"application_name": "messenger-worker"},
|
||
check=ConnectionPool.check_connection, open=False)
|
||
self.r = broker.sync_client(read_timeout=settings.broker_timeout + 5)
|
||
self.mp = multiprocessing.get_context("forkserver")
|
||
self.mp.set_forkserver_preload(["app.imaging"])
|
||
self.work_root = Path(tempfile.gettempdir()) / "worker-jobs"
|
||
|
||
# ================================================================ запуск
|
||
def run(self) -> None:
|
||
signal.signal(signal.SIGTERM, lambda *_: self.stop.set())
|
||
signal.signal(signal.SIGINT, lambda *_: self.stop.set())
|
||
self.pool.open(wait=False)
|
||
storage.init()
|
||
shutil.rmtree(self.work_root, ignore_errors=True) # мусор прошлой жизни контейнера
|
||
self.work_root.mkdir(parents=True, exist_ok=True)
|
||
threads = [threading.Thread(target=self.maintenance_loop, name="maintenance", daemon=True)]
|
||
for i in range(settings.worker_concurrency):
|
||
threads.append(threading.Thread(target=self.consume_loop, args=(f"{self.name}-{i}",),
|
||
name=f"consumer-{i}", daemon=True))
|
||
for t in threads:
|
||
t.start()
|
||
log.info("worker %s started: concurrency=%d max_attempts=%d timeout=%ss lease=%ss",
|
||
self.name, settings.worker_concurrency, settings.job_max_attempts,
|
||
settings.job_timeout_seconds, settings.job_lease_seconds)
|
||
while not self.stop.is_set():
|
||
self.stop.wait(1)
|
||
log.info("worker %s stopping: finishing current jobs", self.name)
|
||
for t in threads[1:]:
|
||
t.join(timeout=settings.job_timeout_seconds + 10)
|
||
self.pool.close()
|
||
|
||
# ================================================================ потребитель
|
||
def consume_loop(self, consumer: str) -> None:
|
||
"""Ограниченная конкуренция: каждый поток берёт по ОДНОМУ сообщению (COUNT 1),
|
||
потоков WORKER_CONCURRENCY → не больше N задач на процесс (аналог prefetch)."""
|
||
last_scan = 0.0
|
||
while not self.stop.is_set():
|
||
try:
|
||
self._ensure_group()
|
||
batch = []
|
||
if time.monotonic() - last_scan > 10:
|
||
last_scan = time.monotonic()
|
||
# 1) свои неподтверждённые (worker перезапустился в том же контейнере);
|
||
resp = self.r.xreadgroup(settings.jobs_group, consumer,
|
||
{settings.jobs_stream: "0"}, count=10)
|
||
batch += [m for _, msgs in resp for m in msgs]
|
||
# 2) брошенные другими (их процесс умер до XACK).
|
||
claimed = self.r.xautoclaim(settings.jobs_stream, settings.jobs_group, consumer,
|
||
min_idle_time=int(settings.claim_idle_seconds * 1000),
|
||
start_id="0-0", count=5)
|
||
batch += [m for m in claimed[1] if m]
|
||
if not batch:
|
||
resp = self.r.xreadgroup(settings.jobs_group, consumer,
|
||
{settings.jobs_stream: ">"}, count=1, block=2000)
|
||
batch = [m for _, msgs in resp for m in msgs]
|
||
for msg_id, fields in batch:
|
||
if self.stop.is_set():
|
||
break
|
||
if self.handle(consumer, msg_id, fields) == "ack":
|
||
self._ack(msg_id)
|
||
except redis.ResponseError as exc:
|
||
if "NOGROUP" in str(exc): # брокер пересоздан без данных — создаём группу снова
|
||
self._group_ok = False
|
||
log.warning("broker error (%s), retry in 1s", str(exc)[:80])
|
||
self.stop.wait(1)
|
||
except redis.RedisError as exc:
|
||
log.warning("broker unavailable (%s), retry in 1s", type(exc).__name__)
|
||
self.stop.wait(1)
|
||
except Exception: # noqa: BLE001 — поток не должен умирать
|
||
log.exception("consumer loop error")
|
||
self.stop.wait(1)
|
||
|
||
def _ensure_group(self) -> None:
|
||
if getattr(self, "_group_ok", False):
|
||
return
|
||
broker.ensure_group(self.r)
|
||
self._group_ok = True
|
||
|
||
def _ack(self, msg_id: str) -> None:
|
||
# XACK = consumer ack: результат уже устойчиво зафиксирован в БД и S3.
|
||
self.r.xack(settings.jobs_stream, settings.jobs_group, msg_id)
|
||
self.r.xdel(settings.jobs_stream, msg_id)
|
||
|
||
# ================================================================ одна задача
|
||
def handle(self, consumer: str, msg_id: str, fields: dict | None) -> str:
|
||
raw = (fields or {}).get("job")
|
||
if raw is None: # запись удалена из stream (XDEL) — подтверждаем
|
||
return "ack"
|
||
try:
|
||
payload = broker.parse(raw)
|
||
except broker.BadMessage as exc:
|
||
log.error("bad message %s → DLQ: %s", msg_id, exc)
|
||
broker.to_dlq(self.r, raw, f"bad_message: {exc}", msg_id)
|
||
return "ack"
|
||
try:
|
||
decision, job = self.claim(payload, consumer)
|
||
except Exception as exc: # noqa: BLE001 — БД недоступна: не ack, повтор позже
|
||
log.warning("claim failed for job %s: %s", payload["job_id"], type(exc).__name__)
|
||
self.stop.wait(1)
|
||
return "leave"
|
||
if decision != "process":
|
||
log.info("job=%s attempt=%s msg=%s → %s", payload["job_id"], payload["attempt"],
|
||
msg_id, decision)
|
||
if decision == "poison":
|
||
broker.to_dlq(self.r, raw, "unknown job", msg_id)
|
||
return "leave" if decision == "busy" else "ack"
|
||
return self.process(job, msg_id)
|
||
|
||
def claim(self, p: dict, consumer: str) -> tuple[str, Job | None]:
|
||
"""Таблица решений (сообщение несёт attempt):
|
||
job нет / не то вложение → poison (DLQ, ack)
|
||
ready → already_ready (no-op, ack): повторная доставка
|
||
failed → failed (ack); вернуть может только replay
|
||
attempt сообщения ≠ attempt job → stale (ack): устаревшая попытка
|
||
queued → берём в работу
|
||
processing, аренда истекла → takeover той же попытки
|
||
processing, аренда жива → busy: не ack, проверим позже (другой работает)"""
|
||
with self.pool.connection() as conn, conn.transaction():
|
||
row = conn.execute(
|
||
"SELECT j.status, j.attempt, j.max_attempts, j.lease_until < now(), "
|
||
"j.pipeline_version, j.attachment_id, a.staging_key, a.object_key, a.sha256, "
|
||
"a.size_bytes, a.content_type "
|
||
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id "
|
||
"WHERE j.job_id = %s FOR UPDATE OF j", (p["job_id"],)).fetchone()
|
||
if row is None or str(row[5]) != p["attachment_id"] or row[4] != p["pipeline_version"]:
|
||
return "poison", None
|
||
status, attempt, max_attempts, expired = row[0], row[1], row[2], row[3]
|
||
if status == "ready":
|
||
return "already_ready", None
|
||
if status == "failed":
|
||
return "failed", None
|
||
if p["attempt"] != attempt:
|
||
return "stale", None
|
||
if status == "processing" and not expired:
|
||
return "busy", None
|
||
event = "lease_takeover" if status == "processing" else "claimed"
|
||
token = str(uuid.uuid4())
|
||
conn.execute(
|
||
"UPDATE attachment_jobs SET status = 'processing', lease_token = %s, lease_owner = %s, "
|
||
"lease_until = now() + make_interval(secs => %s), updated_at = now() WHERE job_id = %s",
|
||
(token, consumer, settings.job_lease_seconds, p["job_id"]))
|
||
conn.execute("UPDATE attachments SET status = 'processing', updated_at = now() "
|
||
"WHERE id = %s AND status <> 'ready'", (p["attachment_id"],))
|
||
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor) VALUES (%s, %s, %s, %s)",
|
||
(p["job_id"], event, attempt, consumer))
|
||
return "process", Job(p["job_id"], p["attachment_id"], row[4], attempt, max_attempts, token,
|
||
row[6], row[7], row[8], row[9], row[10])
|
||
|
||
def process(self, job: Job, msg_id: str) -> str:
|
||
log.info("job=%s attachment=%s attempt=%d/%d msg=%s: processing", job.job_id,
|
||
job.attachment_id, job.attempt, job.max_attempts, msg_id)
|
||
lost = threading.Event()
|
||
frozen = threading.Event()
|
||
done = threading.Event()
|
||
hb = threading.Thread(target=self._heartbeat, args=(job, lost, frozen, done), daemon=True)
|
||
hb.start()
|
||
workdir = Path(tempfile.mkdtemp(prefix=f"{job.attachment_id}-", dir=self.work_root))
|
||
try:
|
||
if self._fault("freeze-after-claim"):
|
||
# Имитация зависшего worker: аренду не продлеваем, «спим» дольше аренды.
|
||
frozen.set()
|
||
log.warning("FAULT INJECTION: freeze-after-claim, sleeping %ss", 3 * settings.job_lease_seconds)
|
||
time.sleep(3 * settings.job_lease_seconds)
|
||
self._check_lease(job)
|
||
variants = self._build(job, workdir, lost)
|
||
self._check_lease(job) # не пишем в S3, если задачу уже забрали
|
||
self._publish_objects(job, variants)
|
||
if self._fault("crash-after-s3-write"):
|
||
log.warning("FAULT INJECTION: crash-after-s3-write (before COMMIT and ack)")
|
||
os._exit(1)
|
||
self._finalize(job, variants)
|
||
if self._fault("crash-after-commit-before-ack"):
|
||
log.warning("FAULT INJECTION: crash-after-commit-before-ack")
|
||
os._exit(1)
|
||
log.info("job=%s attachment=%s attempt=%d: ready", job.job_id, job.attachment_id, job.attempt)
|
||
done.set()
|
||
self._ack(msg_id)
|
||
self._drop_staging(job)
|
||
return "acked"
|
||
except LeaseLost:
|
||
log.warning("job=%s attempt=%d: lease lost, result discarded (another worker owns it)",
|
||
job.job_id, job.attempt)
|
||
return "ack"
|
||
except (Transient, Permanent) as exc:
|
||
return self._failure(job, exc, msg_id)
|
||
except storage.StorageUnavailable as exc:
|
||
return self._failure(job, Transient("storage_unavailable", type(exc).__name__), msg_id)
|
||
except storage.ObjectMissing:
|
||
return self._failure(job, Permanent("processing_failed", "staging object is missing"), msg_id)
|
||
except psycopg.Error as exc:
|
||
return self._failure(job, Transient("database_unavailable", type(exc).__name__), msg_id)
|
||
finally:
|
||
done.set()
|
||
hb.join(timeout=2)
|
||
shutil.rmtree(workdir, ignore_errors=True)
|
||
|
||
def _heartbeat(self, job: Job, lost: threading.Event, frozen: threading.Event,
|
||
done: threading.Event) -> None:
|
||
"""Живой worker продлевает аренду каждые lease/3 с — так «долгая» задача
|
||
отличается от «зависшей»: у долгой аренда свежая, у мёртвой/замёрзшей — истекает."""
|
||
interval = settings.job_lease_seconds / 3
|
||
while not done.wait(interval):
|
||
if frozen.is_set():
|
||
continue
|
||
try:
|
||
with self.pool.connection() as conn:
|
||
ok = conn.execute(
|
||
"UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), "
|
||
"updated_at = now() WHERE job_id = %s AND lease_token = %s "
|
||
"AND status = 'processing' RETURNING 1",
|
||
(settings.job_lease_seconds, job.job_id, job.token)).fetchone()
|
||
if ok is None:
|
||
lost.set()
|
||
return
|
||
except psycopg.Error as exc:
|
||
log.warning("lease renew failed: %s", type(exc).__name__)
|
||
|
||
# ---------------------------------------------------------------- шаги
|
||
def _check_lease(self, job: Job) -> None:
|
||
with self.pool.connection() as conn:
|
||
row = conn.execute("SELECT status, lease_token FROM attachment_jobs WHERE job_id = %s",
|
||
(job.job_id,)).fetchone()
|
||
if row is None or row[0] != "processing" or str(row[1]) != job.token:
|
||
raise LeaseLost()
|
||
|
||
def _build(self, job: Job, workdir: Path, lost: threading.Event) -> dict:
|
||
if not job.staging_key:
|
||
raise Permanent("processing_failed", "staging source is gone")
|
||
if self._fault("transient-error", counter=True):
|
||
raise Transient("injected_transient", "fault injection")
|
||
src = workdir / "source"
|
||
with open(src, "wb") as f:
|
||
try:
|
||
storage.download_sync(job.staging_key, f, settings.max_upload_bytes)
|
||
except storage.ObjectMissing:
|
||
raise Permanent("processing_failed", "staging object is missing")
|
||
except ValueError:
|
||
raise Permanent("invalid_image", "source larger than limit")
|
||
h = hashlib.sha256()
|
||
with open(src, "rb") as f:
|
||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||
h.update(chunk)
|
||
if h.hexdigest() != job.sha256:
|
||
raise Permanent("processing_failed", "source checksum mismatch")
|
||
|
||
out = workdir / "out"
|
||
out.mkdir()
|
||
result_path = workdir / "result.json"
|
||
slow = float(self.r.get("fault:slow-seconds") or 0) if settings.fault_injection else 0.0
|
||
proc = self.mp.Process(target=imaging.child_main, args=(
|
||
str(src), str(out), job.content_type, settings.max_image_pixels,
|
||
settings.image_memory_mb, int(settings.job_timeout_seconds) + 5, str(result_path), slow))
|
||
proc.start()
|
||
deadline = time.monotonic() + settings.job_timeout_seconds
|
||
while proc.is_alive() and time.monotonic() < deadline and not lost.is_set():
|
||
proc.join(timeout=0.5)
|
||
if proc.is_alive():
|
||
proc.kill()
|
||
proc.join()
|
||
if lost.is_set():
|
||
raise LeaseLost()
|
||
raise Transient("processing_timeout", f"> {settings.job_timeout_seconds}s")
|
||
if lost.is_set():
|
||
raise LeaseLost()
|
||
if not result_path.exists():
|
||
raise Transient("processing_crashed", f"exit code {proc.exitcode}")
|
||
result = json.loads(result_path.read_text(encoding="utf-8"))
|
||
if not result["ok"]:
|
||
if result.get("trace"):
|
||
log.warning("job=%s child error: %s", job.job_id, result["trace"])
|
||
exc = Permanent if result["permanent"] else Transient
|
||
raise exc(result["code"], result.get("detail", ""))
|
||
variants = {}
|
||
for name in imaging.VARIANTS:
|
||
v = result["variants"][name]
|
||
data = Path(v["path"]).read_bytes()
|
||
v["size_bytes"] = len(data)
|
||
v["sha256"] = hashlib.sha256(data).hexdigest()
|
||
v["object_key"] = storage.variant_key(job.attachment_id, job.pipeline_version, name,
|
||
imaging.VARIANT_EXT[name])
|
||
variants[name] = v
|
||
return variants
|
||
|
||
def _publish_objects(self, job: Job, variants: dict) -> None:
|
||
"""S3-эффекты идемпотентны: те же ключи и те же байты при повторе.
|
||
Оригинал копируется на стороне S3 (staging → originals), без скачивания."""
|
||
storage.copy_sync(job.staging_key, job.object_key)
|
||
for name, v in variants.items():
|
||
storage.put_file_sync(v["object_key"], v["path"], v["content_type"], v["sha256"])
|
||
|
||
def _finalize(self, job: Job, variants: dict) -> None:
|
||
"""Атомарная публикация результата. Только владелец аренды (lease_token);
|
||
ready не откатывается; ключ эффекта (attachment, pipeline_version, variant)."""
|
||
with self.pool.connection() as conn, conn.transaction():
|
||
row = conn.execute("SELECT status, lease_token FROM attachment_jobs WHERE job_id = %s "
|
||
"FOR UPDATE", (job.job_id,)).fetchone()
|
||
if row is None or row[0] != "processing" or str(row[1]) != job.token:
|
||
raise LeaseLost()
|
||
for name, v in variants.items():
|
||
conn.execute(
|
||
"INSERT INTO attachment_variants (attachment_id, pipeline_version, variant, "
|
||
"object_key, content_type, width, height, size_bytes, sha256, created_at) "
|
||
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, now()) "
|
||
"ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING",
|
||
(job.attachment_id, job.pipeline_version, name, v["object_key"],
|
||
v["content_type"], v["width"], v["height"], v["size_bytes"], v["sha256"]))
|
||
conn.execute(
|
||
"UPDATE attachment_jobs SET status = 'ready', lease_token = NULL, lease_owner = NULL, "
|
||
"lease_until = NULL, error_code = NULL, finished_at = now(), updated_at = now() "
|
||
"WHERE job_id = %s", (job.job_id,))
|
||
conn.execute("UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = now() "
|
||
"WHERE id = %s", (job.attachment_id,))
|
||
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor) "
|
||
"VALUES (%s, 'ready', %s, %s)", (job.job_id, job.attempt, self.name))
|
||
|
||
def _drop_staging(self, job: Job) -> None:
|
||
"""Staging удаляется только ПОСЛЕ COMMIT ready и ack. Не вышло — подчистит maintenance."""
|
||
try:
|
||
storage.delete_sync(job.staging_key)
|
||
with self.pool.connection() as conn:
|
||
conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND status = 'ready'",
|
||
(job.attachment_id,))
|
||
except Exception as exc: # noqa: BLE001
|
||
log.info("staging cleanup deferred: %s", type(exc).__name__)
|
||
|
||
def _failure(self, job: Job, exc: Transient | Permanent, msg_id: str) -> str:
|
||
permanent = isinstance(exc, Permanent)
|
||
log.warning("job=%s attempt=%d/%d %s error: %s %s", job.job_id, job.attempt, job.max_attempts,
|
||
"permanent" if permanent else "transient", exc.code, exc.detail)
|
||
try:
|
||
outcome = schedule_retry_or_fail(self.pool, job.job_id, job.token, exc.code, exc.detail,
|
||
permanent, self.name)
|
||
except psycopg.Error as db_exc:
|
||
log.warning("cannot record failure (%s): leave message pending", type(db_exc).__name__)
|
||
return "leave"
|
||
if outcome == "failed":
|
||
try:
|
||
broker.to_dlq(self.r, broker.dumps({"job_id": job.job_id, "attachment_id": job.attachment_id}),
|
||
exc.code if permanent else f"retry_exhausted ({exc.code})", msg_id)
|
||
except redis.RedisError:
|
||
pass # реестр failed — в БД; DLQ-stream лишь копия для оператора
|
||
return "ack"
|
||
|
||
# ---------------------------------------------------------------- fault injection
|
||
def _fault(self, point: str, counter: bool = False) -> bool:
|
||
"""Только при FAULT_INJECTION=true (сервис worker-fault). Флаг ставит тест:
|
||
redis-cli SET fault:<point> 1 (одноразовый), для counter — число срабатываний."""
|
||
if not settings.fault_injection:
|
||
return False
|
||
key = f"fault:{point}"
|
||
try:
|
||
if counter:
|
||
left = self.r.decr(key)
|
||
if left < 0:
|
||
self.r.delete(key)
|
||
return False
|
||
return True
|
||
return self.r.getdel(key) is not None
|
||
except redis.RedisError:
|
||
return False
|
||
|
||
# ================================================================ maintenance
|
||
def maintenance_loop(self) -> None:
|
||
while not self.stop.is_set():
|
||
Path(settings.heartbeat_file).touch()
|
||
for step in (self.relay_outbox, self.reap_leases, self.sweep_staging):
|
||
try:
|
||
step()
|
||
except Exception as exc: # noqa: BLE001
|
||
log.warning("maintenance %s failed: %s", step.__name__, type(exc).__name__)
|
||
self.stop.wait(settings.maintenance_interval_seconds)
|
||
|
||
def relay_outbox(self) -> None:
|
||
"""Transactional outbox → брокер. Публикует строки, которые API не отметил
|
||
(упал между COMMIT и publish, брокер был недоступен) и отложенные retry.
|
||
SKIP LOCKED: несколько worker не публикуют одну строку одновременно.
|
||
Упали после XADD, до COMMIT → строка опубликуется ещё раз: дубль безопасен."""
|
||
with self.pool.connection() as conn, conn.transaction():
|
||
rows = conn.execute(
|
||
"SELECT id, payload FROM job_outbox WHERE published_at IS NULL "
|
||
"AND available_at <= now() - make_interval(secs => %s) "
|
||
"ORDER BY available_at, id LIMIT 100 FOR UPDATE SKIP LOCKED",
|
||
(settings.outbox_relay_delay_seconds,)).fetchall()
|
||
for oid, payload in rows:
|
||
try:
|
||
stream_id = broker.publish_sync(self.r, payload)
|
||
except broker.BrokerUnavailable as exc:
|
||
conn.execute("UPDATE job_outbox SET publish_attempts = publish_attempts + 1, "
|
||
"last_error = %s WHERE id = %s", (str(exc)[:100], oid))
|
||
break
|
||
conn.execute("UPDATE job_outbox SET published_at = now(), stream_id = %s, "
|
||
"publish_attempts = publish_attempts + 1 WHERE id = %s", (stream_id, oid))
|
||
log.info("relay: published outbox=%s job=%s attempt=%s", oid, payload["job_id"],
|
||
payload["attempt"])
|
||
|
||
def reap_leases(self) -> None:
|
||
"""Аренда истекла и за ещё одну аренду никто не перехватил (redelivery) →
|
||
worker мёртв: новая управляемая попытка через outbox (или failed, если попыток нет)."""
|
||
with self.pool.connection() as conn:
|
||
ids = [r[0] for r in conn.execute(
|
||
"SELECT job_id FROM attachment_jobs WHERE status = 'processing' "
|
||
"AND lease_until < now() - make_interval(secs => %s) LIMIT 20",
|
||
(settings.job_lease_seconds,)).fetchall()]
|
||
for job_id in ids:
|
||
outcome = schedule_retry_or_fail(self.pool, job_id, None, "lease_expired",
|
||
"worker stopped renewing lease", False, "reaper")
|
||
log.warning("reaper: job=%s lease expired → %s", job_id, outcome)
|
||
|
||
def sweep_staging(self) -> None:
|
||
"""Очистка staging: ready — сразу после фиксации (страховка, если worker не успел);
|
||
failed — после STAGING_RETENTION_SECONDS (до этого исходник нужен для replay)."""
|
||
with self.pool.connection() as conn:
|
||
rows = conn.execute(
|
||
"SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND ("
|
||
" (status = 'ready' AND updated_at < now() - interval '30 seconds') OR"
|
||
" (status = 'failed' AND updated_at < now() - make_interval(secs => %s))) LIMIT 50",
|
||
(settings.staging_retention_seconds,)).fetchall()
|
||
for aid, key in rows:
|
||
storage.delete_sync(key)
|
||
with self.pool.connection() as conn:
|
||
conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s "
|
||
"AND status IN ('ready', 'failed')", (aid,))
|
||
log.info("staging cleaned for attachment %s", aid)
|
||
|
||
|
||
def schedule_retry_or_fail(pool: ConnectionPool, job_id, token: str | None, code: str, detail: str,
|
||
permanent: bool, actor: str) -> str:
|
||
"""processing → queued (новая попытка через outbox с backoff) или → failed.
|
||
token задан — проверяем владельца аренды; None — вызывает reaper (аренда уже истекла)."""
|
||
with pool.connection() as conn, conn.transaction():
|
||
row = conn.execute(
|
||
"SELECT status, attempt, max_attempts, lease_token, attachment_id, pipeline_version, "
|
||
"lease_until < now() - make_interval(secs => %s) FROM attachment_jobs "
|
||
"WHERE job_id = %s FOR UPDATE SKIP LOCKED", (settings.job_lease_seconds, job_id)).fetchone()
|
||
if row is None or row[0] != "processing":
|
||
return "skipped"
|
||
status, attempt, max_attempts, lease_token, attachment_id, pv, long_expired = row
|
||
if token is not None and str(lease_token) != token:
|
||
return "skipped" # аренду уже забрали
|
||
if token is None and not long_expired:
|
||
return "skipped"
|
||
if permanent or attempt >= max_attempts:
|
||
final = code if permanent else "retry_exhausted"
|
||
conn.execute(
|
||
"UPDATE attachment_jobs SET status = 'failed', error_code = %s, last_error = %s, "
|
||
"lease_token = NULL, lease_owner = NULL, lease_until = NULL, finished_at = now(), "
|
||
"updated_at = now() WHERE job_id = %s", (final, f"{code}: {detail}"[:300], job_id))
|
||
conn.execute("UPDATE attachments SET status = 'failed', error_code = %s, updated_at = now() "
|
||
"WHERE id = %s", (final, attachment_id))
|
||
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) "
|
||
"VALUES (%s, 'failed', %s, %s, %s)", (job_id, attempt, actor, f"{code}: {detail}"[:300]))
|
||
return "failed"
|
||
delay = backoff_seconds(attempt)
|
||
nxt = attempt + 1
|
||
conn.execute(
|
||
"UPDATE attachment_jobs SET status = 'queued', attempt = %s, last_error = %s, "
|
||
"next_attempt_at = now() + make_interval(secs => %s), lease_token = NULL, "
|
||
"lease_owner = NULL, lease_until = NULL, updated_at = now() WHERE job_id = %s",
|
||
(nxt, f"{code}: {detail}"[:300], delay, job_id))
|
||
conn.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = %s",
|
||
(attachment_id,))
|
||
created = conn.execute("SELECT now()").fetchone()[0]
|
||
payload = broker.make_payload(job_id, attachment_id, pv, nxt, created)
|
||
conn.execute(
|
||
"INSERT INTO job_outbox (job_id, attempt, payload, available_at, created_at) "
|
||
"VALUES (%s, %s, %s, now() + make_interval(secs => %s), now())",
|
||
(job_id, nxt, json.dumps(payload), delay))
|
||
conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) "
|
||
"VALUES (%s, 'retry_scheduled', %s, %s, %s)",
|
||
(job_id, attempt, actor, f"{code}; next attempt {nxt} in {delay:.1f}s"))
|
||
return f"retry #{nxt} in {delay:.1f}s"
|
||
|
||
|
||
def main() -> None:
|
||
logging.basicConfig(level=settings.log_level.upper(),
|
||
format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||
logging.getLogger("botocore").setLevel(logging.WARNING)
|
||
Worker().run()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|