Files
mtusi2026lab5/src/publisher.py
T
Самохвалов Андрей Ремович b23ecc111f feat: Accept durable queued uploads and publish outbox jobs (U2)
Carry forward the lab4 API and storage baseline, stage verified originals in
MinIO, and atomically commit attachment, job, and outbox records before 202.
Publish identifier-only jobs with broker confirms and reclaimable outbox leases.
Clean abandoned and completed staging objects after database checks.
2026-10-02 14:35:35 +03:00

71 lines
2.3 KiB
Python

"""Publish PostgreSQL outbox jobs with RabbitMQ confirms."""
import json
import logging
import os
import time
from pathlib import Path
import pika
import psycopg
from psycopg_pool import PoolTimeout
from .db import pool
from .pipeline_jobs import claim_outbox, confirm_outbox, job_payload
logger = logging.getLogger(__name__)
QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
def publish_batch(channel):
with pool().connection() as conn:
token, rows = claim_outbox(conn, limit=10, lease_seconds=30)
for row in rows:
body = json.dumps(job_payload(row), separators=(",", ":")).encode("utf-8")
channel.basic_publish(
exchange="", routing_key=QUEUE, body=body, mandatory=True,
properties=pika.BasicProperties(
content_type="application/json", delivery_mode=2,
message_id=str(row["job_id"]), type="attachment.process.v1",
),
)
# basic_publish returns only after a broker confirm in confirm mode.
with pool().connection() as conn:
if not confirm_outbox(conn, row["id"], token):
logger.warning("Outbox claim expired before confirm for row %s", row["id"])
return len(rows)
def main():
logging.basicConfig(level=logging.INFO)
logging.getLogger("pika").setLevel(logging.ERROR)
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
params = pika.URLParameters(url)
params.heartbeat = 30
params.socket_timeout = 5
params.blocked_connection_timeout = 10
while True:
broker = None
try:
broker = pika.BlockingConnection(params)
channel = broker.channel()
channel.queue_declare(queue=QUEUE, passive=True)
channel.confirm_delivery()
while broker.is_open:
if not publish_batch(channel):
broker.process_data_events(time_limit=1)
except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc:
logger.warning("Publisher reconnecting after %s", type(exc).__name__)
time.sleep(2)
finally:
if broker is not None and broker.is_open:
try:
broker.close()
except (pika.exceptions.AMQPError, OSError):
pass
if __name__ == "__main__":
main()