983 lines
24 KiB
Python
983 lines
24 KiB
Python
|
|
"""RabbitMQ consumer process for durable image processing."""
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import random
|
|
import socket
|
|
import threading
|
|
import uuid
|
|
import time
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import aio_pika
|
|
from aio_pika import DeliveryMode, Message
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
from sqlalchemy import select, update
|
|
from sqlalchemy.exc import SQLAlchemyError
|
|
|
|
from app.core.config import settings
|
|
from app.db.models import Attachment, AttachmentVariant, ProcessingJob
|
|
from app.db.session import SessionLocal
|
|
from app.services.image_processor import (
|
|
ImageProcessingTimeout,
|
|
create_variants_with_timeout,
|
|
)
|
|
from app.services.object_storage import client as s3_client
|
|
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
|
)
|
|
|
|
logger = logging.getLogger("attachment-worker")
|
|
|
|
WORKER_ID = (
|
|
f"{socket.gethostname()[:40]}-{uuid.uuid4().hex[:16]}"
|
|
)
|
|
|
|
DEAD_ROUTING_KEY = f"{settings.rabbitmq_queue}.dead"
|
|
|
|
PROCESSING_SEMAPHORE = None
|
|
|
|
|
|
def utc_now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def _safe_uuid(value) -> uuid.UUID:
|
|
return uuid.UUID(str(value))
|
|
|
|
|
|
def _claim_job(job_id, attachment_id, pipeline_version) -> dict:
|
|
"""Atomically claim a job or return the reason it cannot be claimed."""
|
|
|
|
job_id = _safe_uuid(job_id)
|
|
attachment_id = _safe_uuid(attachment_id)
|
|
pipeline_version = int(pipeline_version)
|
|
|
|
db = SessionLocal()
|
|
|
|
try:
|
|
job = db.scalar(
|
|
select(ProcessingJob)
|
|
.where(
|
|
ProcessingJob.id == job_id,
|
|
ProcessingJob.attachment_id == attachment_id,
|
|
ProcessingJob.pipeline_version == pipeline_version,
|
|
)
|
|
.with_for_update()
|
|
)
|
|
|
|
if job is None:
|
|
db.rollback()
|
|
return {"state": "missing"}
|
|
|
|
attachment = db.get(Attachment, attachment_id)
|
|
|
|
if attachment is None:
|
|
db.rollback()
|
|
return {"state": "missing"}
|
|
|
|
now = utc_now()
|
|
|
|
if attachment.status == "ready":
|
|
db.commit()
|
|
return {"state": "ready"}
|
|
|
|
if job.status == "failed":
|
|
db.commit()
|
|
return {"state": "failed"}
|
|
|
|
if (
|
|
job.status == "processing"
|
|
and job.lease_until is not None
|
|
and job.lease_until > now
|
|
):
|
|
delay = (job.lease_until - now).total_seconds() + 1
|
|
|
|
db.commit()
|
|
|
|
return {
|
|
"state": "busy",
|
|
"delay": max(1.0, delay),
|
|
}
|
|
|
|
if job.available_at > now:
|
|
delay = (job.available_at - now).total_seconds() + 0.5
|
|
|
|
db.commit()
|
|
|
|
return {
|
|
"state": "deferred",
|
|
"delay": max(1.0, delay),
|
|
}
|
|
|
|
if job.attempt >= settings.worker_max_attempts:
|
|
job.status = "failed"
|
|
job.worker_id = None
|
|
job.lease_until = None
|
|
job.last_error_code = "processing_attempts_exhausted"
|
|
|
|
attachment.status = "failed"
|
|
attachment.error_code = "processing_attempts_exhausted"
|
|
|
|
db.commit()
|
|
|
|
return {"state": "failed"}
|
|
|
|
claim_worker_id = f"{WORKER_ID}:{uuid.uuid4().hex[:16]}"
|
|
|
|
job.status = "processing"
|
|
job.worker_id = claim_worker_id
|
|
job.lease_until = now + timedelta(
|
|
seconds=settings.worker_lease_seconds
|
|
)
|
|
job.attempt += 1
|
|
|
|
attachment.status = "processing"
|
|
attachment.error_code = None
|
|
|
|
claimed_attempt = job.attempt
|
|
|
|
queue_lag_ms = max(
|
|
0.0,
|
|
(now - job.available_at).total_seconds() * 1000,
|
|
)
|
|
|
|
db.commit()
|
|
|
|
logger.info(
|
|
"Claimed job=%s attachment=%s attempt=%s queue_lag_ms=%.2f",
|
|
job_id,
|
|
attachment_id,
|
|
claimed_attempt,
|
|
queue_lag_ms,
|
|
)
|
|
|
|
return {
|
|
"state": "claimed",
|
|
"worker_id": claim_worker_id,
|
|
"attempt": claimed_attempt,
|
|
"queue_lag_ms": queue_lag_ms,
|
|
}
|
|
|
|
except Exception:
|
|
db.rollback()
|
|
raise
|
|
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def _renew_lease(
|
|
job_id: uuid.UUID,
|
|
worker_id: str,
|
|
attempt: int,
|
|
stop_event: threading.Event,
|
|
lost_event: threading.Event,
|
|
) -> None:
|
|
"""Renew the lease while image processing is running."""
|
|
|
|
interval = max(
|
|
5,
|
|
settings.worker_lease_seconds // 3,
|
|
)
|
|
|
|
while not stop_event.wait(interval):
|
|
db = SessionLocal()
|
|
|
|
try:
|
|
result = db.execute(
|
|
update(ProcessingJob)
|
|
.where(
|
|
ProcessingJob.id == job_id,
|
|
ProcessingJob.worker_id == worker_id,
|
|
ProcessingJob.attempt == attempt,
|
|
ProcessingJob.status == "processing",
|
|
)
|
|
.values(
|
|
lease_until=utc_now()
|
|
+ timedelta(seconds=settings.worker_lease_seconds)
|
|
)
|
|
)
|
|
|
|
db.commit()
|
|
|
|
if result.rowcount != 1:
|
|
lost_event.set()
|
|
|
|
logger.warning(
|
|
"Lease ownership lost: job=%s worker=%s",
|
|
job_id,
|
|
worker_id,
|
|
)
|
|
|
|
return
|
|
|
|
except Exception:
|
|
db.rollback()
|
|
lost_event.set()
|
|
|
|
logger.exception(
|
|
"Could not renew lease for job=%s",
|
|
job_id,
|
|
)
|
|
|
|
return
|
|
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def _fail_job(
|
|
job_id,
|
|
attachment_id,
|
|
worker_id,
|
|
attempt,
|
|
error_code,
|
|
permanent=False,
|
|
) -> dict:
|
|
"""Record a failure only if this worker still owns the job."""
|
|
|
|
db = SessionLocal()
|
|
|
|
try:
|
|
job = db.scalar(
|
|
select(ProcessingJob)
|
|
.where(ProcessingJob.id == job_id)
|
|
.with_for_update()
|
|
)
|
|
|
|
attachment = db.scalar(
|
|
select(Attachment)
|
|
.where(Attachment.id == attachment_id)
|
|
.with_for_update()
|
|
)
|
|
|
|
if job is None or attachment is None:
|
|
db.rollback()
|
|
return {"state": "missing"}
|
|
|
|
if (
|
|
job.status != "processing"
|
|
or job.worker_id != worker_id
|
|
or job.attempt != attempt
|
|
):
|
|
db.rollback()
|
|
|
|
logger.warning(
|
|
"Ignoring failure from stale worker: job=%s worker=%s",
|
|
job_id,
|
|
worker_id,
|
|
)
|
|
|
|
return {
|
|
"state": "stale",
|
|
"delay": float(settings.worker_lease_seconds),
|
|
}
|
|
|
|
exhausted = job.attempt >= settings.worker_max_attempts
|
|
|
|
job.last_error_code = error_code
|
|
|
|
if permanent or exhausted:
|
|
job.status = "failed"
|
|
job.worker_id = None
|
|
job.lease_until = None
|
|
|
|
attachment.status = "failed"
|
|
attachment.error_code = error_code
|
|
|
|
db.commit()
|
|
|
|
logger.error(
|
|
"Job permanently failed: job=%s error=%s",
|
|
job_id,
|
|
error_code,
|
|
)
|
|
|
|
return {"state": "failed"}
|
|
|
|
delay = min(
|
|
settings.worker_retry_base_seconds
|
|
* (2 ** max(job.attempt - 1, 0)),
|
|
settings.worker_retry_max_seconds,
|
|
)
|
|
|
|
delay += random.uniform(0, delay * 0.25)
|
|
|
|
job.status = "queued"
|
|
job.available_at = utc_now() + timedelta(seconds=delay)
|
|
job.lease_until = None
|
|
job.worker_id = None
|
|
|
|
attachment.status = "queued"
|
|
attachment.error_code = error_code
|
|
|
|
db.commit()
|
|
|
|
logger.warning(
|
|
"Job scheduled for retry: job=%s delay=%.2fs error=%s",
|
|
job_id,
|
|
delay,
|
|
error_code,
|
|
)
|
|
|
|
return {
|
|
"state": "retry",
|
|
"delay": delay,
|
|
}
|
|
|
|
except Exception:
|
|
db.rollback()
|
|
raise
|
|
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def _process(payload: dict) -> dict:
|
|
"""Process one image using a fenced, renewable job lease."""
|
|
|
|
job_id = _safe_uuid(payload["job_id"])
|
|
attachment_id = _safe_uuid(payload["attachment_id"])
|
|
pipeline_version = int(payload["pipeline_version"])
|
|
|
|
claim = _claim_job(
|
|
job_id,
|
|
attachment_id,
|
|
pipeline_version,
|
|
)
|
|
|
|
if claim["state"] != "claimed":
|
|
return claim
|
|
|
|
worker_id = claim["worker_id"]
|
|
attempt = claim["attempt"]
|
|
queue_lag_ms = claim["queue_lag_ms"]
|
|
processing_started_at = time.perf_counter()
|
|
|
|
stop_event = threading.Event()
|
|
lost_event = threading.Event()
|
|
|
|
heartbeat = threading.Thread(
|
|
target=_renew_lease,
|
|
args=(
|
|
job_id,
|
|
worker_id,
|
|
attempt,
|
|
stop_event,
|
|
lost_event,
|
|
),
|
|
daemon=True,
|
|
)
|
|
|
|
heartbeat.start()
|
|
|
|
staging_key = None
|
|
db = None
|
|
|
|
try:
|
|
db = SessionLocal()
|
|
|
|
attachment = db.get(Attachment, attachment_id)
|
|
|
|
if attachment is None:
|
|
raise ValueError("attachment_missing")
|
|
|
|
staging_key = attachment.staging_key
|
|
object_key = attachment.object_key
|
|
content_type = attachment.content_type
|
|
|
|
if not staging_key:
|
|
raise ValueError("staging_missing")
|
|
|
|
db.close()
|
|
db = None
|
|
|
|
if lost_event.is_set():
|
|
return {
|
|
"state": "retry",
|
|
"delay": float(settings.worker_lease_seconds),
|
|
}
|
|
|
|
s3 = s3_client()
|
|
|
|
response = s3.get_object(
|
|
Bucket=settings.s3_bucket,
|
|
Key=staging_key,
|
|
)
|
|
|
|
body = response["Body"]
|
|
|
|
try:
|
|
attachment_data = body.read(
|
|
settings.max_upload_bytes + 1
|
|
)
|
|
finally:
|
|
body.close()
|
|
|
|
if len(attachment_data) > settings.max_upload_bytes:
|
|
raise ValueError("image_too_large")
|
|
|
|
if lost_event.is_set():
|
|
return {
|
|
"state": "retry",
|
|
"delay": float(settings.worker_lease_seconds),
|
|
}
|
|
|
|
variants = create_variants_with_timeout(
|
|
attachment_data,
|
|
settings.image_processing_timeout_seconds,
|
|
)
|
|
|
|
# The original object uses a deterministic key.
|
|
s3.put_object(
|
|
Bucket=settings.s3_bucket,
|
|
Key=object_key,
|
|
Body=attachment_data,
|
|
ContentType=content_type,
|
|
)
|
|
|
|
variant_records = []
|
|
|
|
for variant in variants:
|
|
key = (
|
|
f"variants/{attachment_id}/"
|
|
f"v{pipeline_version}/"
|
|
f"{variant.name}.{variant.extension}"
|
|
)
|
|
|
|
s3.put_object(
|
|
Bucket=settings.s3_bucket,
|
|
Key=key,
|
|
Body=variant.data,
|
|
ContentType=variant.content_type,
|
|
)
|
|
|
|
variant_records.append(
|
|
{
|
|
"name": variant.name,
|
|
"key": key,
|
|
"content_type": variant.content_type,
|
|
"width": variant.width,
|
|
"height": variant.height,
|
|
"size_bytes": len(variant.data),
|
|
"sha256": variant.sha256,
|
|
}
|
|
)
|
|
|
|
if lost_event.is_set():
|
|
logger.warning(
|
|
"Worker lost lease before final commit: job=%s",
|
|
job_id,
|
|
)
|
|
|
|
return {
|
|
"state": "retry",
|
|
"delay": float(settings.worker_lease_seconds),
|
|
}
|
|
|
|
# Use a fresh transaction and verify the fencing token.
|
|
db = SessionLocal()
|
|
|
|
job = db.scalar(
|
|
select(ProcessingJob)
|
|
.where(ProcessingJob.id == job_id)
|
|
.with_for_update()
|
|
)
|
|
|
|
attachment = db.scalar(
|
|
select(Attachment)
|
|
.where(Attachment.id == attachment_id)
|
|
.with_for_update()
|
|
)
|
|
|
|
if job is None or attachment is None:
|
|
db.rollback()
|
|
return {"state": "missing"}
|
|
|
|
if (
|
|
job.status != "processing"
|
|
or job.worker_id != worker_id
|
|
or job.attempt != attempt
|
|
or job.lease_until is None
|
|
or job.lease_until <= utc_now()
|
|
or lost_event.is_set()
|
|
):
|
|
db.rollback()
|
|
|
|
logger.warning(
|
|
"Fencing check rejected stale worker: job=%s",
|
|
job_id,
|
|
)
|
|
|
|
return {
|
|
"state": "retry",
|
|
"delay": float(settings.worker_lease_seconds),
|
|
}
|
|
|
|
for item in variant_records:
|
|
existing = db.scalar(
|
|
select(AttachmentVariant)
|
|
.where(
|
|
AttachmentVariant.attachment_id == attachment_id,
|
|
AttachmentVariant.variant == item["name"],
|
|
)
|
|
)
|
|
|
|
if existing is None:
|
|
existing = AttachmentVariant(
|
|
attachment_id=attachment_id,
|
|
variant=item["name"],
|
|
object_key=item["key"],
|
|
content_type=item["content_type"],
|
|
width=item["width"],
|
|
height=item["height"],
|
|
size_bytes=item["size_bytes"],
|
|
sha256=item["sha256"],
|
|
)
|
|
|
|
db.add(existing)
|
|
|
|
else:
|
|
existing.object_key = item["key"]
|
|
existing.content_type = item["content_type"]
|
|
existing.width = item["width"]
|
|
existing.height = item["height"]
|
|
existing.size_bytes = item["size_bytes"]
|
|
existing.sha256 = item["sha256"]
|
|
|
|
job.status = "ready"
|
|
job.lease_until = None
|
|
job.worker_id = None
|
|
job.last_error_code = None
|
|
|
|
attachment.status = "ready"
|
|
attachment.error_code = None
|
|
attachment.updated_at = utc_now()
|
|
|
|
db.commit()
|
|
|
|
processing_elapsed_ms = (
|
|
time.perf_counter() - processing_started_at
|
|
) * 1000
|
|
|
|
logger.info(
|
|
(
|
|
"Image processing completed: attachment=%s "
|
|
"attempt=%s queue_lag_ms=%.2f processing_elapsed_ms=%.2f"
|
|
),
|
|
attachment_id,
|
|
attempt,
|
|
queue_lag_ms,
|
|
processing_elapsed_ms,
|
|
)
|
|
|
|
db.close()
|
|
db = None
|
|
|
|
# Staging cleanup is performed only after the final DB commit.
|
|
try:
|
|
s3.delete_object(
|
|
Bucket=settings.s3_bucket,
|
|
Key=staging_key,
|
|
)
|
|
|
|
cleanup_db = SessionLocal()
|
|
|
|
try:
|
|
cleanup_attachment = cleanup_db.scalar(
|
|
select(Attachment)
|
|
.where(Attachment.id == attachment_id)
|
|
.with_for_update()
|
|
)
|
|
|
|
if (
|
|
cleanup_attachment is not None
|
|
and cleanup_attachment.status == "ready"
|
|
and cleanup_attachment.staging_key == staging_key
|
|
):
|
|
cleanup_attachment.staging_key = None
|
|
cleanup_db.commit()
|
|
else:
|
|
cleanup_db.rollback()
|
|
|
|
finally:
|
|
cleanup_db.close()
|
|
|
|
except Exception:
|
|
logger.warning(
|
|
"Could not delete staging object for attachment=%s",
|
|
attachment_id,
|
|
exc_info=True,
|
|
)
|
|
|
|
return {"state": "ready"}
|
|
|
|
except ImageProcessingTimeout:
|
|
if db is not None:
|
|
db.rollback()
|
|
|
|
stop_event.set()
|
|
heartbeat.join(timeout=2)
|
|
|
|
logger.warning(
|
|
"Image processing timed out: job=%s attachment=%s timeout=%ss",
|
|
job_id,
|
|
attachment_id,
|
|
settings.image_processing_timeout_seconds,
|
|
)
|
|
|
|
return _fail_job(
|
|
job_id,
|
|
attachment_id,
|
|
worker_id,
|
|
attempt,
|
|
"image_processing_timeout",
|
|
permanent=False,
|
|
)
|
|
|
|
except ValueError as exc:
|
|
if db is not None:
|
|
db.rollback()
|
|
|
|
stop_event.set()
|
|
heartbeat.join(timeout=2)
|
|
|
|
code = str(exc)[:64] or "invalid_image"
|
|
|
|
return _fail_job(
|
|
job_id,
|
|
attachment_id,
|
|
worker_id,
|
|
attempt,
|
|
code,
|
|
permanent=True,
|
|
)
|
|
|
|
except ClientError as exc:
|
|
if db is not None:
|
|
db.rollback()
|
|
|
|
stop_event.set()
|
|
heartbeat.join(timeout=2)
|
|
|
|
error = exc.response.get("Error", {})
|
|
error_code = error.get("Code", "")
|
|
|
|
if error_code in {
|
|
"NoSuchKey",
|
|
"NoSuchBucket",
|
|
"404",
|
|
"NotFound",
|
|
}:
|
|
return _fail_job(
|
|
job_id,
|
|
attachment_id,
|
|
worker_id,
|
|
attempt,
|
|
"staging_missing",
|
|
permanent=True,
|
|
)
|
|
|
|
return _fail_job(
|
|
job_id,
|
|
attachment_id,
|
|
worker_id,
|
|
attempt,
|
|
"storage_unavailable",
|
|
permanent=False,
|
|
)
|
|
|
|
except (BotoCoreError, SQLAlchemyError):
|
|
if db is not None:
|
|
db.rollback()
|
|
|
|
stop_event.set()
|
|
heartbeat.join(timeout=2)
|
|
|
|
return _fail_job(
|
|
job_id,
|
|
attachment_id,
|
|
worker_id,
|
|
attempt,
|
|
"processing_unavailable",
|
|
permanent=False,
|
|
)
|
|
|
|
except Exception:
|
|
if db is not None:
|
|
db.rollback()
|
|
|
|
logger.exception(
|
|
"Unexpected processing error: job=%s",
|
|
job_id,
|
|
)
|
|
|
|
stop_event.set()
|
|
heartbeat.join(timeout=2)
|
|
|
|
return _fail_job(
|
|
job_id,
|
|
attachment_id,
|
|
worker_id,
|
|
attempt,
|
|
"processing_failed",
|
|
permanent=False,
|
|
)
|
|
|
|
finally:
|
|
stop_event.set()
|
|
heartbeat.join(timeout=2)
|
|
|
|
if db is not None:
|
|
db.close()
|
|
|
|
|
|
async def _publish_delayed(
|
|
exchange,
|
|
payload: dict,
|
|
delay_seconds: float,
|
|
) -> None:
|
|
"""Publish a persistent message to the TTL retry queue."""
|
|
|
|
delay_ms = max(
|
|
1000,
|
|
int(delay_seconds * 1000),
|
|
)
|
|
|
|
message = Message(
|
|
body=json.dumps(
|
|
payload,
|
|
separators=(",", ":"),
|
|
).encode("utf-8"),
|
|
content_type="application/json",
|
|
delivery_mode=DeliveryMode.PERSISTENT,
|
|
message_id=str(uuid.uuid4()),
|
|
type="attachment.process.retry",
|
|
expiration=timedelta(milliseconds=delay_ms),
|
|
)
|
|
|
|
await exchange.publish(
|
|
message,
|
|
routing_key=f"{settings.rabbitmq_queue}.retry",
|
|
mandatory=True,
|
|
)
|
|
|
|
logger.info(
|
|
"Scheduled delayed delivery in %.2fs",
|
|
delay_ms / 1000,
|
|
)
|
|
|
|
|
|
async def _quarantine_message(exchange, message) -> None:
|
|
"""Move an invalid message to the durable dead-letter queue."""
|
|
|
|
dead_message = Message(
|
|
body=message.body,
|
|
content_type=message.content_type or "application/octet-stream",
|
|
delivery_mode=DeliveryMode.PERSISTENT,
|
|
message_id=message.message_id or str(uuid.uuid4()),
|
|
type="attachment.process.invalid",
|
|
)
|
|
|
|
await exchange.publish(
|
|
dead_message,
|
|
routing_key=DEAD_ROUTING_KEY,
|
|
mandatory=True,
|
|
)
|
|
|
|
await message.ack()
|
|
|
|
logger.error(
|
|
"Invalid message moved to dead-letter queue: message_id=%s",
|
|
message.message_id,
|
|
)
|
|
|
|
|
|
async def handle_message(
|
|
message: aio_pika.IncomingMessage,
|
|
retry_exchange,
|
|
dead_exchange,
|
|
semaphore: asyncio.Semaphore,
|
|
) -> None:
|
|
"""Consume, process, retry or quarantine one RabbitMQ message."""
|
|
|
|
async with semaphore:
|
|
try:
|
|
payload = json.loads(message.body.decode("utf-8"))
|
|
|
|
if not isinstance(payload, dict):
|
|
raise ValueError("invalid_payload")
|
|
|
|
for field in (
|
|
"job_id",
|
|
"attachment_id",
|
|
"pipeline_version",
|
|
):
|
|
if field not in payload:
|
|
raise ValueError("missing_payload_field")
|
|
|
|
_safe_uuid(payload["job_id"])
|
|
_safe_uuid(payload["attachment_id"])
|
|
int(payload["pipeline_version"])
|
|
|
|
except (ValueError, TypeError, KeyError, UnicodeDecodeError):
|
|
try:
|
|
await _quarantine_message(
|
|
dead_exchange,
|
|
message,
|
|
)
|
|
except Exception:
|
|
logger.exception(
|
|
"Could not quarantine invalid message"
|
|
)
|
|
|
|
await message.nack(requeue=True)
|
|
|
|
return
|
|
|
|
try:
|
|
result = await asyncio.to_thread(
|
|
_process,
|
|
payload,
|
|
)
|
|
|
|
state = result["state"]
|
|
|
|
if state in {
|
|
"busy",
|
|
"deferred",
|
|
"retry",
|
|
}:
|
|
await _publish_delayed(
|
|
retry_exchange,
|
|
payload,
|
|
result.get("delay", 1.0),
|
|
)
|
|
|
|
await message.ack()
|
|
|
|
logger.info(
|
|
"Message handled: job=%s state=%s",
|
|
payload["job_id"],
|
|
state,
|
|
)
|
|
|
|
except asyncio.CancelledError:
|
|
raise
|
|
|
|
except Exception:
|
|
logger.exception(
|
|
"Message processing failed: message_id=%s",
|
|
message.message_id,
|
|
)
|
|
|
|
# If delayed publication or DB processing fails,
|
|
# preserve the original message.
|
|
await message.nack(requeue=True)
|
|
|
|
|
|
async def main() -> None:
|
|
"""Start the durable RabbitMQ consumer."""
|
|
|
|
connection = await aio_pika.connect_robust(
|
|
settings.rabbitmq_url
|
|
)
|
|
|
|
async with connection:
|
|
channel = await connection.channel(
|
|
publisher_confirms=True
|
|
)
|
|
|
|
await channel.set_qos(
|
|
prefetch_count=settings.rabbitmq_prefetch
|
|
)
|
|
|
|
exchange = await channel.declare_exchange(
|
|
settings.rabbitmq_exchange,
|
|
aio_pika.ExchangeType.DIRECT,
|
|
durable=True,
|
|
)
|
|
|
|
queue = await channel.declare_queue(
|
|
settings.rabbitmq_queue,
|
|
durable=True,
|
|
)
|
|
|
|
await queue.bind(
|
|
exchange,
|
|
routing_key=settings.rabbitmq_queue,
|
|
)
|
|
|
|
# Delayed retry exchange and queue.
|
|
retry_exchange = await channel.declare_exchange(
|
|
f"{settings.rabbitmq_exchange}.retry",
|
|
aio_pika.ExchangeType.DIRECT,
|
|
durable=True,
|
|
)
|
|
|
|
retry_queue = await channel.declare_queue(
|
|
f"{settings.rabbitmq_queue}.retry",
|
|
durable=True,
|
|
arguments={
|
|
"x-dead-letter-exchange": settings.rabbitmq_exchange,
|
|
"x-dead-letter-routing-key": settings.rabbitmq_queue,
|
|
},
|
|
)
|
|
|
|
await retry_queue.bind(
|
|
retry_exchange,
|
|
routing_key=f"{settings.rabbitmq_queue}.retry",
|
|
)
|
|
|
|
# Durable quarantine queue for invalid messages.
|
|
dead_exchange = await channel.declare_exchange(
|
|
f"{settings.rabbitmq_exchange}.dead",
|
|
aio_pika.ExchangeType.DIRECT,
|
|
durable=True,
|
|
)
|
|
|
|
dead_queue = await channel.declare_queue(
|
|
f"{settings.rabbitmq_queue}.dead",
|
|
durable=True,
|
|
)
|
|
|
|
await dead_queue.bind(
|
|
dead_exchange,
|
|
routing_key=DEAD_ROUTING_KEY,
|
|
)
|
|
|
|
semaphore = asyncio.Semaphore(
|
|
settings.worker_concurrency
|
|
)
|
|
|
|
async def consumer(message):
|
|
await handle_message(
|
|
message,
|
|
retry_exchange,
|
|
dead_exchange,
|
|
semaphore,
|
|
)
|
|
|
|
logger.info(
|
|
"Worker %s consuming queue=%s concurrency=%s prefetch=%s",
|
|
WORKER_ID,
|
|
settings.rabbitmq_queue,
|
|
settings.worker_concurrency,
|
|
settings.rabbitmq_prefetch,
|
|
)
|
|
|
|
await queue.consume(consumer)
|
|
|
|
await asyncio.Future()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|