forked from mtusi2026/mtusi2026lab5
75 lines
3.3 KiB
Python
75 lines
3.3 KiB
Python
"""Transactional outbox: relay публикует задания из PostgreSQL в брокер.
|
|
|
|
Гарантия: задание, зафиксированное в БД (COMMIT), будет опубликовано даже если процесс упал между COMMIT и publish.
|
|
Relay запускается в каждой реплике API и в worker; строки берутся `FOR UPDATE SKIP LOCKED`, поэтому одну строку
|
|
публикует один relay. Если процесс упадёт после подтверждения брокера, но до отметки published_at, задание будет
|
|
опубликовано повторно — доставка at-least-once, дубликаты безвредны (обработка идемпотентна).
|
|
"""
|
|
import asyncio
|
|
from datetime import timezone
|
|
|
|
from . import config
|
|
from .web import format_dt, log
|
|
|
|
JOB_TYPE = "attachment.process.v1"
|
|
|
|
|
|
def job_message(row):
|
|
return {"schema_version": 1, "job_id": str(row["job_id"]), "type": JOB_TYPE, "attachment_id": str(row["attachment_id"]),
|
|
"pipeline_version": row["pipeline_version"], "attempt": row["attempt"],
|
|
"created_at": format_dt(row["created_at"].astimezone(timezone.utc))}
|
|
|
|
|
|
class Relay:
|
|
def __init__(self, db, broker):
|
|
self.db, self.broker = db, broker
|
|
self._kick = asyncio.Event()
|
|
self._stop = False
|
|
self.last_error = None
|
|
|
|
def kick(self):
|
|
self._kick.set()
|
|
|
|
def stop(self):
|
|
self._stop = True
|
|
self._kick.set()
|
|
|
|
async def relay_once(self):
|
|
published = 0
|
|
async with self.db.conn() as c:
|
|
async with c.transaction():
|
|
rows = await c.fetch(
|
|
"SELECT id, job_id, attachment_id, pipeline_version, attempt, created_at FROM outbox "
|
|
"WHERE published_at IS NULL AND available_at <= now() ORDER BY id LIMIT $1 FOR UPDATE SKIP LOCKED",
|
|
config.RELAY_BATCH)
|
|
for row in rows:
|
|
await self.broker.publish(job_message(row), headers=self.headers_for(row))
|
|
await c.execute("UPDATE outbox SET published_at = now() WHERE id = $1", row["id"])
|
|
published += 1
|
|
return published
|
|
|
|
def headers_for(self, row):
|
|
return {}
|
|
|
|
async def run(self):
|
|
delay = config.RELAY_INTERVAL
|
|
while not self._stop:
|
|
try:
|
|
n = await self.relay_once()
|
|
self.last_error = None
|
|
delay = config.RELAY_INTERVAL
|
|
if n >= config.RELAY_BATCH:
|
|
continue # возможно, есть ещё
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
if self.last_error != type(exc).__name__:
|
|
log("warning", "outbox_relay_failed", error_type=type(exc).__name__)
|
|
self.last_error = type(exc).__name__
|
|
delay = min(delay * 2, 10.0) # брокер/БД недоступны: пауза с ростом до 10 с, задания остаются в outbox
|
|
try:
|
|
await asyncio.wait_for(self._kick.wait(), delay)
|
|
except asyncio.TimeoutError:
|
|
pass
|
|
self._kick.clear()
|