Carry forward the lab4 API and storage baseline, stage verified originals in MinIO, and atomically commit attachment, job, and outbox records before 202. Publish identifier-only jobs with broker confirms and reclaimable outbox leases. Clean abandoned and completed staging objects after database checks.
71 lines
2.3 KiB
Python
71 lines
2.3 KiB
Python
"""Publish PostgreSQL outbox jobs with RabbitMQ confirms."""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pika
|
|
import psycopg
|
|
from psycopg_pool import PoolTimeout
|
|
|
|
from .db import pool
|
|
from .pipeline_jobs import claim_outbox, confirm_outbox, job_payload
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
|
|
|
|
|
|
def publish_batch(channel):
|
|
with pool().connection() as conn:
|
|
token, rows = claim_outbox(conn, limit=10, lease_seconds=30)
|
|
for row in rows:
|
|
body = json.dumps(job_payload(row), separators=(",", ":")).encode("utf-8")
|
|
channel.basic_publish(
|
|
exchange="", routing_key=QUEUE, body=body, mandatory=True,
|
|
properties=pika.BasicProperties(
|
|
content_type="application/json", delivery_mode=2,
|
|
message_id=str(row["job_id"]), type="attachment.process.v1",
|
|
),
|
|
)
|
|
# basic_publish returns only after a broker confirm in confirm mode.
|
|
with pool().connection() as conn:
|
|
if not confirm_outbox(conn, row["id"], token):
|
|
logger.warning("Outbox claim expired before confirm for row %s", row["id"])
|
|
return len(rows)
|
|
|
|
|
|
def main():
|
|
logging.basicConfig(level=logging.INFO)
|
|
logging.getLogger("pika").setLevel(logging.ERROR)
|
|
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
|
|
params = pika.URLParameters(url)
|
|
params.heartbeat = 30
|
|
params.socket_timeout = 5
|
|
params.blocked_connection_timeout = 10
|
|
while True:
|
|
broker = None
|
|
try:
|
|
broker = pika.BlockingConnection(params)
|
|
channel = broker.channel()
|
|
channel.queue_declare(queue=QUEUE, passive=True)
|
|
channel.confirm_delivery()
|
|
while broker.is_open:
|
|
if not publish_batch(channel):
|
|
broker.process_data_events(time_limit=1)
|
|
except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc:
|
|
logger.warning("Publisher reconnecting after %s", type(exc).__name__)
|
|
time.sleep(2)
|
|
finally:
|
|
if broker is not None and broker.is_open:
|
|
try:
|
|
broker.close()
|
|
except (pika.exceptions.AMQPError, OSError):
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|