import base64 import json import uuid from urllib.parse import quote, urlparse, unquote from urllib.request import Request, urlopen import pika from app import telemetry from app.db import jsonb from app.settings import settings QUEUE = "attachment.process.v1" def _params(): parsed = urlparse(settings.broker_url) return pika.ConnectionParameters( host=parsed.hostname or "rabbitmq", port=parsed.port or 5672, virtual_host=unquote(parsed.path[1:] or "/"), credentials=pika.PlainCredentials(unquote(parsed.username or ""), unquote(parsed.password or "")), heartbeat=60, blocked_connection_timeout=2, socket_timeout=2, connection_attempts=1, retry_delay=0, ) def _credentials(): parsed = urlparse(settings.broker_url) return unquote(parsed.username or ""), unquote(parsed.password or ""), unquote(parsed.path[1:] or "/") def init(): connection = pika.BlockingConnection(_params()) try: channel = connection.channel() channel.queue_declare(queue=QUEUE, durable=True) finally: connection.close() def ready(): try: connection = pika.BlockingConnection(_params()) try: channel = connection.channel() channel.queue_declare(queue=QUEUE, durable=True, passive=True) finally: connection.close() return True except Exception: return False def insert_outbox(conn, payload, created_at): conn.execute( """ INSERT INTO outbox (job_id, attachment_id, payload, created_at) VALUES (%s, %s, %s, %s) """, (uuid.UUID(str(payload["job_id"])), uuid.UUID(str(payload["attachment_id"])), jsonb(payload), created_at), ) def publish_pending(conn, job_id=None): query = """ SELECT job_id, payload FROM outbox WHERE published_at IS NULL """ args = [] if job_id is not None: query += " AND job_id = %s" args.append(job_id) query += " ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 20" rows = conn.execute(query, args).fetchall() for row in rows: payload = row["payload"] if isinstance(payload, str): payload = json.loads(payload) _publish(payload) conn.execute("UPDATE outbox SET published_at = NOW() WHERE job_id = %s", (row["job_id"],)) return len(rows) def _publish(payload): carrier = {} if payload.get("traceparent"): carrier["traceparent"] = payload["traceparent"] if payload.get("tracestate"): carrier["tracestate"] = payload["tracestate"] parent = telemetry.extract_context(carrier) with telemetry.tracer().start_as_current_span( "attachment.process publish", context=parent, kind=telemetry.trace.SpanKind.PRODUCER, attributes={ "messaging.system": "rabbitmq", "messaging.destination.name": QUEUE, "messaging.operation.type": "publish", }, ): outgoing = telemetry.inject_carrier() body = dict(payload) if outgoing.get("traceparent"): body["traceparent"] = outgoing["traceparent"] if outgoing.get("tracestate"): body["tracestate"] = outgoing["tracestate"] _send(body, outgoing) telemetry.log_event( "job.published", job_id=str(payload.get("job_id")), attempt=int(payload.get("attempt") or 1), ) def _send(payload, headers): connection = pika.BlockingConnection(_params()) try: channel = connection.channel() channel.confirm_delivery() channel.basic_publish( exchange="", routing_key=QUEUE, body=json.dumps(payload, separators=(",", ":")).encode("utf-8"), properties=pika.BasicProperties( delivery_mode=2, content_type="application/json", headers=headers, ), mandatory=True, ) finally: connection.close() def consumer_count(): user, password, vhost = _credentials() url = ( settings.rabbit_management_url.rstrip("/") + "/api/queues/" + quote(vhost, safe="") + "/" + quote(QUEUE, safe="") ) token = base64.b64encode(f"{user}:{password}".encode("utf-8")).decode("ascii") request = Request(url, headers={"Authorization": "Basic " + token}) with urlopen(request, timeout=2) as response: data = json.loads(response.read().decode("utf-8")) return int(data.get("consumers") or 0)