forked from mtusi2026/mtusi2026lab5
121 lines
4.9 KiB
Python
121 lines
4.9 KiB
Python
"""Transactional-outbox relay: PostgreSQL outbox -> RabbitMQ with publisher confirms.
|
|
|
|
SELECT ... FOR UPDATE SKIP LOCKED -> publish + wait for confirm -> UPDATE published_at -> COMMIT
|
|
|
|
A crash after the confirm but before COMMIT republishes the row later: a duplicate message, which the worker
|
|
treats as a no-op. A row is never marked published without a confirm, so a message accepted by the API is never
|
|
lost between COMMIT and publish. Several relays may run concurrently (SKIP LOCKED).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import secrets
|
|
import signal
|
|
import time
|
|
import uuid
|
|
|
|
import asyncpg
|
|
|
|
from app import config, faults
|
|
from app.broker import Broker
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
logger = logging.getLogger("relay")
|
|
|
|
HEARTBEAT_FILE = "/tmp/relay-heartbeat"
|
|
|
|
|
|
class Relay:
|
|
def __init__(self):
|
|
self.id = "relay-" + secrets.token_hex(4)
|
|
self.pool = None
|
|
self.broker = Broker()
|
|
self.wakeup = asyncio.Event()
|
|
self.stopping = asyncio.Event()
|
|
|
|
async def publish_batch(self) -> int:
|
|
async with self.pool.acquire() as conn, conn.transaction():
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT id, job_id, payload FROM outbox
|
|
WHERE published_at IS NULL AND available_at <= clock_timestamp()
|
|
ORDER BY available_at, id LIMIT $1
|
|
FOR UPDATE SKIP LOCKED
|
|
""",
|
|
config.RELAY_BATCH,
|
|
)
|
|
for row in rows:
|
|
payload = json.loads(row["payload"])
|
|
await self.broker.publish(payload) # returns after the publisher confirm
|
|
await faults.hit(self.pool, "relay.after_confirm_before_mark", uuid.UUID(payload["attachment_id"]), self.id)
|
|
if rows:
|
|
await conn.execute(
|
|
"""
|
|
UPDATE outbox SET published_at = clock_timestamp(), publish_count = publish_count + 1
|
|
WHERE id = ANY($1::bigint[])
|
|
""",
|
|
[r["id"] for r in rows],
|
|
)
|
|
return len(rows)
|
|
|
|
async def listen(self) -> asyncpg.Connection:
|
|
conn = await asyncpg.connect(config.DATABASE_URL, timeout=config.DB_CONNECT_TIMEOUT)
|
|
await conn.add_listener("outbox", lambda *_: self.wakeup.set())
|
|
return conn
|
|
|
|
async def run(self) -> None:
|
|
loop = asyncio.get_running_loop()
|
|
for sig in (signal.SIGTERM, signal.SIGINT):
|
|
loop.add_signal_handler(sig, self.stopping.set)
|
|
listener = None
|
|
failures = 0
|
|
while not self.stopping.is_set():
|
|
try:
|
|
if self.pool is None:
|
|
self.pool = await asyncpg.create_pool(config.DATABASE_URL, min_size=1, max_size=3,
|
|
timeout=config.DB_CONNECT_TIMEOUT,
|
|
command_timeout=config.DB_COMMAND_TIMEOUT)
|
|
if self.broker.exchange is None:
|
|
await self.broker.connect()
|
|
logger.info("%s connected; fault_injection=%s", self.id, config.FAULT_INJECTION)
|
|
if listener is None or listener.is_closed():
|
|
listener = await self.listen()
|
|
self.wakeup.clear()
|
|
published = await self.publish_batch()
|
|
failures = 0
|
|
with open(HEARTBEAT_FILE, "w", encoding="ascii") as fh:
|
|
fh.write(str(time.time()))
|
|
if published:
|
|
logger.info("published %d outbox message(s)", published)
|
|
if published == config.RELAY_BATCH:
|
|
continue
|
|
try:
|
|
# NOTIFY wakes us immediately; the poll interval covers delayed (backoff) rows.
|
|
await asyncio.wait_for(self.wakeup.wait(), config.RELAY_POLL_SECONDS)
|
|
except asyncio.TimeoutError:
|
|
pass
|
|
except Exception as exc:
|
|
failures += 1
|
|
delay = min(10.0, 0.5 * 2 ** min(failures, 5))
|
|
logger.warning("relay cycle failed (%s); retry in %.1fs; rows stay unpublished", type(exc).__name__, delay)
|
|
if self.broker.exchange is None or self.broker.connection.is_closed:
|
|
try:
|
|
await self.broker.close()
|
|
except Exception:
|
|
pass
|
|
self.broker = Broker()
|
|
if listener is not None and not listener.is_closed():
|
|
try:
|
|
await listener.close()
|
|
except Exception:
|
|
pass
|
|
listener = None
|
|
await asyncio.sleep(delay)
|
|
await self.broker.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(Relay().run())
|