Files
mtusi2026lab5/app/relay.py

121 lines
4.9 KiB
Python

"""Transactional-outbox relay: PostgreSQL outbox -> RabbitMQ with publisher confirms.
SELECT ... FOR UPDATE SKIP LOCKED -> publish + wait for confirm -> UPDATE published_at -> COMMIT
A crash after the confirm but before COMMIT republishes the row later: a duplicate message, which the worker
treats as a no-op. A row is never marked published without a confirm, so a message accepted by the API is never
lost between COMMIT and publish. Several relays may run concurrently (SKIP LOCKED).
"""
from __future__ import annotations
import asyncio
import json
import logging
import secrets
import signal
import time
import uuid
import asyncpg
from app import config, faults
from app.broker import Broker
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
logger = logging.getLogger("relay")
HEARTBEAT_FILE = "/tmp/relay-heartbeat"
class Relay:
def __init__(self):
self.id = "relay-" + secrets.token_hex(4)
self.pool = None
self.broker = Broker()
self.wakeup = asyncio.Event()
self.stopping = asyncio.Event()
async def publish_batch(self) -> int:
async with self.pool.acquire() as conn, conn.transaction():
rows = await conn.fetch(
"""
SELECT id, job_id, payload FROM outbox
WHERE published_at IS NULL AND available_at <= clock_timestamp()
ORDER BY available_at, id LIMIT $1
FOR UPDATE SKIP LOCKED
""",
config.RELAY_BATCH,
)
for row in rows:
payload = json.loads(row["payload"])
await self.broker.publish(payload) # returns after the publisher confirm
await faults.hit(self.pool, "relay.after_confirm_before_mark", uuid.UUID(payload["attachment_id"]), self.id)
if rows:
await conn.execute(
"""
UPDATE outbox SET published_at = clock_timestamp(), publish_count = publish_count + 1
WHERE id = ANY($1::bigint[])
""",
[r["id"] for r in rows],
)
return len(rows)
async def listen(self) -> asyncpg.Connection:
conn = await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT)
await conn.add_listener("outbox", lambda *_: self.wakeup.set())
return conn
async def run(self) -> None:
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
loop.add_signal_handler(sig, self.stopping.set)
listener = None
failures = 0
while not self.stopping.is_set():
try:
if self.pool is None:
self.pool = await asyncpg.create_pool(config.DATABASE_URL, min_size=1, max_size=3,
timeout=config.DB_CONNECT_TIMEOUT,
command_timeout=config.DB_COMMAND_TIMEOUT)
if self.broker.exchange is None:
await self.broker.connect()
logger.info("%s connected; fault_injection=%s", self.id, config.FAULT_INJECTION)
if listener is None or listener.is_closed():
listener = await self.listen()
self.wakeup.clear()
published = await self.publish_batch()
failures = 0
with open(HEARTBEAT_FILE, "w", encoding="ascii") as fh:
fh.write(str(time.time()))
if published:
logger.info("published %d outbox message(s)", published)
if published == config.RELAY_BATCH:
continue
try:
# NOTIFY wakes us immediately; the poll interval covers delayed (backoff) rows.
await asyncio.wait_for(self.wakeup.wait(), config.RELAY_POLL_SECONDS)
except asyncio.TimeoutError:
pass
except Exception as exc:
failures += 1
delay = min(10.0, 0.5 * 2 ** min(failures, 5))
logger.warning("relay cycle failed (%s); retry in %.1fs; rows stay unpublished", type(exc).__name__, delay)
if self.broker.exchange is None or self.broker.connection.is_closed:
try:
await self.broker.close()
except Exception:
pass
self.broker = Broker()
if listener is not None and not listener.is_closed():
try:
await listener.close()
except Exception:
pass
listener = None
await asyncio.sleep(delay)
await self.broker.close()
if __name__ == "__main__":
asyncio.run(Relay().run())