"""Transactional outbox: bridges the PostgreSQL COMMIT and the broker publish. enqueue() writes the job row in the SAME transaction as the attachments INSERT, so either both happen or neither does. The caller then makes a best-effort immediate publish; if the process crashes between COMMIT and that publish, the row is still there with published_at IS NULL, and the background relay (any API replica) picks it up and publishes it later -- no job is silently lost, and none is invented without a committed row. """ import asyncio import datetime as dt import logging import uuid import asyncpg from . import broker logger = logging.getLogger("app.outbox") RELAY_INTERVAL_SECONDS = 2 RELAY_GRACE_SECONDS = 1 # don't race the enqueuing request's own immediate publish async def enqueue(conn: asyncpg.Connection, attachment_id: uuid.UUID) -> dict: job_id = uuid.uuid4() await conn.execute( "INSERT INTO job_outbox (id, attachment_id, job_id, pipeline_version, attempt) " "VALUES (gen_random_uuid(), $1, $2, 1, 1)", attachment_id, job_id, ) return { "schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1", "attachment_id": str(attachment_id), "pipeline_version": 1, "attempt": 1, "created_at": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"), } async def publish_best_effort(pool: asyncpg.Pool, job: dict) -> None: """Try to publish right away; on failure, leave it for the relay. Never raises -- a broker hiccup here must not fail the already-committed upload.""" try: await broker.publish(job) async with pool.acquire() as conn: await conn.execute( "UPDATE job_outbox SET published_at=now() WHERE job_id=$1", uuid.UUID(job["job_id"]) ) except Exception: # noqa: BLE001 logger.warning("immediate publish failed for job %s; relay will retry", job["job_id"]) async def relay_once(pool: asyncpg.Pool) -> int: async with pool.acquire() as conn: rows = await conn.fetch( """ SELECT id, attachment_id, job_id, pipeline_version, attempt, created_at FROM job_outbox WHERE published_at IS NULL AND created_at < now() - ($1 * interval '1 second') ORDER BY created_at LIMIT 50 """, RELAY_GRACE_SECONDS, ) for row in rows: job = { "schema_version": 1, "job_id": str(row["job_id"]), "type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]), "pipeline_version": row["pipeline_version"], "attempt": row["attempt"], "created_at": row["created_at"].astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z"), } await broker.publish(job) await conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=$1", row["id"]) return len(rows) async def relay_loop(pool: asyncpg.Pool) -> None: while True: try: await relay_once(pool) except asyncio.CancelledError: raise except Exception: # noqa: BLE001 - the relay must survive transient DB/broker hiccups logger.exception("outbox relay tick failed") await asyncio.sleep(RELAY_INTERVAL_SECONDS)