Import committed lab5 baseline 03899d7 via git archive: API replicas, PostgreSQL, private S3, RabbitMQ, publisher, worker, migrations, and hooks. Preserve lab6 contracts and the separate telemetry stack. Static checks pass; Docker runtime integration remains pending.
71 lines
2.3 KiB
Python
71 lines
2.3 KiB
Python
"""Publish PostgreSQL outbox jobs with RabbitMQ confirms."""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pika
|
|
import psycopg
|
|
from psycopg_pool import PoolTimeout
|
|
|
|
from .db import pool
|
|
from .pipeline_jobs import claim_outbox, confirm_outbox, job_payload
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
|
|
|
|
|
|
def publish_batch(channel):
|
|
with pool().connection() as conn:
|
|
token, rows = claim_outbox(conn, limit=10, lease_seconds=30)
|
|
for row in rows:
|
|
body = json.dumps(job_payload(row), separators=(",", ":")).encode("utf-8")
|
|
channel.basic_publish(
|
|
exchange="", routing_key=QUEUE, body=body, mandatory=True,
|
|
properties=pika.BasicProperties(
|
|
content_type="application/json", delivery_mode=2,
|
|
message_id=str(row["job_id"]), type="attachment.process.v1",
|
|
),
|
|
)
|
|
# basic_publish returns only after a broker confirm in confirm mode.
|
|
with pool().connection() as conn:
|
|
if not confirm_outbox(conn, row["id"], token):
|
|
logger.warning("Outbox claim expired before confirm for row %s", row["id"])
|
|
return len(rows)
|
|
|
|
|
|
def main():
|
|
logging.basicConfig(level=logging.INFO)
|
|
logging.getLogger("pika").setLevel(logging.ERROR)
|
|
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
|
|
params = pika.URLParameters(url)
|
|
params.heartbeat = 30
|
|
params.socket_timeout = 5
|
|
params.blocked_connection_timeout = 10
|
|
while True:
|
|
broker = None
|
|
try:
|
|
broker = pika.BlockingConnection(params)
|
|
channel = broker.channel()
|
|
channel.queue_declare(queue=QUEUE, passive=True)
|
|
channel.confirm_delivery()
|
|
while broker.is_open:
|
|
if not publish_batch(channel):
|
|
broker.process_data_events(time_limit=1)
|
|
except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc:
|
|
logger.warning("Publisher reconnecting after %s", type(exc).__name__)
|
|
time.sleep(2)
|
|
finally:
|
|
if broker is not None and broker.is_open:
|
|
try:
|
|
broker.close()
|
|
except (pika.exceptions.AMQPError, OSError):
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|