50 lines
2.2 KiB
Python
50 lines
2.2 KiB
Python
"""Transactional-outbox dispatcher; safe to run as one or more replicas."""
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import json
|
|
|
|
from .broker import publish
|
|
from .config import load_settings
|
|
from .storage.postgres import PostgresStore
|
|
from . import telemetry as tel
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s outbox %(message)s")
|
|
|
|
|
|
async def run() -> None:
|
|
tel.setup("messenger-outbox")
|
|
settings = load_settings()
|
|
store = PostgresStore(settings.database_url)
|
|
await store.connect()
|
|
interval = max(0.1, float(os.environ.get("OUTBOX_DISPATCH_INTERVAL_SECONDS", "1")))
|
|
try:
|
|
while True:
|
|
try:
|
|
await store.schedule_due_jobs(int(os.environ.get('JOB_MAX_ATTEMPTS', '5')))
|
|
for event in await store.due_outbox_events():
|
|
context = event['trace_context']
|
|
await publish(event["payload"], json.loads(context) if isinstance(context, str) else context)
|
|
await store.mark_outbox_published(event["id"])
|
|
try:
|
|
queue = await store.pool.fetchrow("SELECT count(*) AS depth, COALESCE(EXTRACT(EPOCH FROM now()-min(created_at)), 0)::float8 AS age FROM attachment_jobs WHERE status='queued'")
|
|
slo = await store.pool.fetchrow("""
|
|
SELECT count(*) FILTER (WHERE created_at <= now() - interval '30 seconds') AS eligible,
|
|
count(*) FILTER (WHERE created_at <= now() - interval '30 seconds'
|
|
AND (status <> 'ready' OR updated_at > created_at + interval '30 seconds')) AS bad
|
|
FROM attachment_jobs
|
|
""")
|
|
tel.state.update(queue_depth=queue['depth'], queue_age=queue['age'],
|
|
slo_ready_eligible=slo['eligible'], slo_ready_bad=slo['bad'])
|
|
except Exception:
|
|
logging.warning("queue telemetry unavailable")
|
|
except Exception:
|
|
logging.exception("outbox dispatch failed; durable event remains pending")
|
|
await asyncio.sleep(interval)
|
|
finally:
|
|
await store.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(run())
|