Files
mtusi2026lab6/app/worker.py

996 lines
24 KiB
Python

"""RabbitMQ consumer process for durable image processing."""
import asyncio
import json
import logging
import random
import socket
import threading
import uuid
import time
from datetime import datetime, timedelta, timezone
import aio_pika
from aio_pika import DeliveryMode, Message
from botocore.exceptions import BotoCoreError, ClientError
from sqlalchemy import select, update
from sqlalchemy.exc import SQLAlchemyError
from app.core.config import settings
from app.db.models import Attachment, AttachmentVariant, ProcessingJob
from app.db.session import SessionLocal
from app.services.image_processor import (
ImageProcessingTimeout,
create_variants_with_timeout,
)
from app.services.object_storage import client as s3_client
from app.core.worker_telemetry import (
process_job_with_telemetry,
setup_worker_telemetry,
)
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
logger = logging.getLogger("attachment-worker")
WORKER_ID = (
f"{socket.gethostname()[:40]}-{uuid.uuid4().hex[:16]}"
)
DEAD_ROUTING_KEY = f"{settings.rabbitmq_queue}.dead"
PROCESSING_SEMAPHORE = None
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def _safe_uuid(value) -> uuid.UUID:
return uuid.UUID(str(value))
def _claim_job(job_id, attachment_id, pipeline_version) -> dict:
"""Atomically claim a job or return the reason it cannot be claimed."""
job_id = _safe_uuid(job_id)
attachment_id = _safe_uuid(attachment_id)
pipeline_version = int(pipeline_version)
db = SessionLocal()
try:
job = db.scalar(
select(ProcessingJob)
.where(
ProcessingJob.id == job_id,
ProcessingJob.attachment_id == attachment_id,
ProcessingJob.pipeline_version == pipeline_version,
)
.with_for_update()
)
if job is None:
db.rollback()
return {"state": "missing"}
attachment = db.get(Attachment, attachment_id)
if attachment is None:
db.rollback()
return {"state": "missing"}
now = utc_now()
if attachment.status == "ready":
db.commit()
return {"state": "ready"}
if job.status == "failed":
db.commit()
return {"state": "failed"}
if (
job.status == "processing"
and job.lease_until is not None
and job.lease_until > now
):
delay = (job.lease_until - now).total_seconds() + 1
db.commit()
return {
"state": "busy",
"delay": max(1.0, delay),
}
if job.available_at > now:
delay = (job.available_at - now).total_seconds() + 0.5
db.commit()
return {
"state": "deferred",
"delay": max(1.0, delay),
}
if job.attempt >= settings.worker_max_attempts:
job.status = "failed"
job.worker_id = None
job.lease_until = None
job.last_error_code = "processing_attempts_exhausted"
attachment.status = "failed"
attachment.error_code = "processing_attempts_exhausted"
db.commit()
return {"state": "failed"}
claim_worker_id = f"{WORKER_ID}:{uuid.uuid4().hex[:16]}"
job.status = "processing"
job.worker_id = claim_worker_id
job.lease_until = now + timedelta(
seconds=settings.worker_lease_seconds
)
job.attempt += 1
attachment.status = "processing"
attachment.error_code = None
claimed_attempt = job.attempt
queue_lag_ms = max(
0.0,
(now - job.available_at).total_seconds() * 1000,
)
db.commit()
logger.info(
"Claimed job=%s attachment=%s attempt=%s queue_lag_ms=%.2f",
job_id,
attachment_id,
claimed_attempt,
queue_lag_ms,
)
return {
"state": "claimed",
"worker_id": claim_worker_id,
"attempt": claimed_attempt,
"queue_lag_ms": queue_lag_ms,
}
except Exception:
db.rollback()
raise
finally:
db.close()
def _renew_lease(
job_id: uuid.UUID,
worker_id: str,
attempt: int,
stop_event: threading.Event,
lost_event: threading.Event,
) -> None:
"""Renew the lease while image processing is running."""
interval = max(
5,
settings.worker_lease_seconds // 3,
)
while not stop_event.wait(interval):
db = SessionLocal()
try:
result = db.execute(
update(ProcessingJob)
.where(
ProcessingJob.id == job_id,
ProcessingJob.worker_id == worker_id,
ProcessingJob.attempt == attempt,
ProcessingJob.status == "processing",
)
.values(
lease_until=utc_now()
+ timedelta(seconds=settings.worker_lease_seconds)
)
)
db.commit()
if result.rowcount != 1:
lost_event.set()
logger.warning(
"Lease ownership lost: job=%s worker=%s",
job_id,
worker_id,
)
return
except Exception:
db.rollback()
lost_event.set()
logger.exception(
"Could not renew lease for job=%s",
job_id,
)
return
finally:
db.close()
def _fail_job(
job_id,
attachment_id,
worker_id,
attempt,
error_code,
permanent=False,
) -> dict:
"""Record a failure only if this worker still owns the job."""
db = SessionLocal()
try:
job = db.scalar(
select(ProcessingJob)
.where(ProcessingJob.id == job_id)
.with_for_update()
)
attachment = db.scalar(
select(Attachment)
.where(Attachment.id == attachment_id)
.with_for_update()
)
if job is None or attachment is None:
db.rollback()
return {"state": "missing"}
if (
job.status != "processing"
or job.worker_id != worker_id
or job.attempt != attempt
):
db.rollback()
logger.warning(
"Ignoring failure from stale worker: job=%s worker=%s",
job_id,
worker_id,
)
return {
"state": "stale",
"delay": float(settings.worker_lease_seconds),
}
exhausted = job.attempt >= settings.worker_max_attempts
job.last_error_code = error_code
if permanent or exhausted:
job.status = "failed"
job.worker_id = None
job.lease_until = None
attachment.status = "failed"
attachment.error_code = error_code
db.commit()
logger.error(
"Job permanently failed: job=%s error=%s",
job_id,
error_code,
)
return {"state": "failed"}
delay = min(
settings.worker_retry_base_seconds
* (2 ** max(job.attempt - 1, 0)),
settings.worker_retry_max_seconds,
)
delay += random.uniform(0, delay * 0.25)
job.status = "queued"
job.available_at = utc_now() + timedelta(seconds=delay)
job.lease_until = None
job.worker_id = None
attachment.status = "queued"
attachment.error_code = error_code
db.commit()
logger.warning(
"Job scheduled for retry: job=%s delay=%.2fs error=%s",
job_id,
delay,
error_code,
)
return {
"state": "retry",
"delay": delay,
}
except Exception:
db.rollback()
raise
finally:
db.close()
def _process_impl(payload: dict) -> dict:
"""Process one image using a fenced, renewable job lease."""
job_id = _safe_uuid(payload["job_id"])
attachment_id = _safe_uuid(payload["attachment_id"])
pipeline_version = int(payload["pipeline_version"])
claim = _claim_job(
job_id,
attachment_id,
pipeline_version,
)
if claim["state"] != "claimed":
return claim
worker_id = claim["worker_id"]
attempt = claim["attempt"]
queue_lag_ms = claim["queue_lag_ms"]
processing_started_at = time.perf_counter()
stop_event = threading.Event()
lost_event = threading.Event()
heartbeat = threading.Thread(
target=_renew_lease,
args=(
job_id,
worker_id,
attempt,
stop_event,
lost_event,
),
daemon=True,
)
heartbeat.start()
staging_key = None
db = None
try:
db = SessionLocal()
attachment = db.get(Attachment, attachment_id)
if attachment is None:
raise ValueError("attachment_missing")
staging_key = attachment.staging_key
object_key = attachment.object_key
content_type = attachment.content_type
if not staging_key:
raise ValueError("staging_missing")
db.close()
db = None
if lost_event.is_set():
return {
"state": "retry",
"delay": float(settings.worker_lease_seconds),
}
s3 = s3_client()
response = s3.get_object(
Bucket=settings.s3_bucket,
Key=staging_key,
)
body = response["Body"]
try:
attachment_data = body.read(
settings.max_upload_bytes + 1
)
finally:
body.close()
if len(attachment_data) > settings.max_upload_bytes:
raise ValueError("image_too_large")
if lost_event.is_set():
return {
"state": "retry",
"delay": float(settings.worker_lease_seconds),
}
variants = create_variants_with_timeout(
attachment_data,
settings.image_processing_timeout_seconds,
)
# The original object uses a deterministic key.
s3.put_object(
Bucket=settings.s3_bucket,
Key=object_key,
Body=attachment_data,
ContentType=content_type,
)
variant_records = []
for variant in variants:
key = (
f"variants/{attachment_id}/"
f"v{pipeline_version}/"
f"{variant.name}.{variant.extension}"
)
s3.put_object(
Bucket=settings.s3_bucket,
Key=key,
Body=variant.data,
ContentType=variant.content_type,
)
variant_records.append(
{
"name": variant.name,
"key": key,
"content_type": variant.content_type,
"width": variant.width,
"height": variant.height,
"size_bytes": len(variant.data),
"sha256": variant.sha256,
}
)
if lost_event.is_set():
logger.warning(
"Worker lost lease before final commit: job=%s",
job_id,
)
return {
"state": "retry",
"delay": float(settings.worker_lease_seconds),
}
# Use a fresh transaction and verify the fencing token.
db = SessionLocal()
job = db.scalar(
select(ProcessingJob)
.where(ProcessingJob.id == job_id)
.with_for_update()
)
attachment = db.scalar(
select(Attachment)
.where(Attachment.id == attachment_id)
.with_for_update()
)
if job is None or attachment is None:
db.rollback()
return {"state": "missing"}
if (
job.status != "processing"
or job.worker_id != worker_id
or job.attempt != attempt
or job.lease_until is None
or job.lease_until <= utc_now()
or lost_event.is_set()
):
db.rollback()
logger.warning(
"Fencing check rejected stale worker: job=%s",
job_id,
)
return {
"state": "retry",
"delay": float(settings.worker_lease_seconds),
}
for item in variant_records:
existing = db.scalar(
select(AttachmentVariant)
.where(
AttachmentVariant.attachment_id == attachment_id,
AttachmentVariant.variant == item["name"],
)
)
if existing is None:
existing = AttachmentVariant(
attachment_id=attachment_id,
variant=item["name"],
object_key=item["key"],
content_type=item["content_type"],
width=item["width"],
height=item["height"],
size_bytes=item["size_bytes"],
sha256=item["sha256"],
)
db.add(existing)
else:
existing.object_key = item["key"]
existing.content_type = item["content_type"]
existing.width = item["width"]
existing.height = item["height"]
existing.size_bytes = item["size_bytes"]
existing.sha256 = item["sha256"]
job.status = "ready"
job.lease_until = None
job.worker_id = None
job.last_error_code = None
attachment.status = "ready"
attachment.error_code = None
attachment.updated_at = utc_now()
db.commit()
processing_elapsed_ms = (
time.perf_counter() - processing_started_at
) * 1000
logger.info(
(
"Image processing completed: attachment=%s "
"attempt=%s queue_lag_ms=%.2f processing_elapsed_ms=%.2f"
),
attachment_id,
attempt,
queue_lag_ms,
processing_elapsed_ms,
)
db.close()
db = None
# Staging cleanup is performed only after the final DB commit.
try:
s3.delete_object(
Bucket=settings.s3_bucket,
Key=staging_key,
)
cleanup_db = SessionLocal()
try:
cleanup_attachment = cleanup_db.scalar(
select(Attachment)
.where(Attachment.id == attachment_id)
.with_for_update()
)
if (
cleanup_attachment is not None
and cleanup_attachment.status == "ready"
and cleanup_attachment.staging_key == staging_key
):
cleanup_attachment.staging_key = None
cleanup_db.commit()
else:
cleanup_db.rollback()
finally:
cleanup_db.close()
except Exception:
logger.warning(
"Could not delete staging object for attachment=%s",
attachment_id,
exc_info=True,
)
return {"state": "ready"}
except ImageProcessingTimeout:
if db is not None:
db.rollback()
stop_event.set()
heartbeat.join(timeout=2)
logger.warning(
"Image processing timed out: job=%s attachment=%s timeout=%ss",
job_id,
attachment_id,
settings.image_processing_timeout_seconds,
)
return _fail_job(
job_id,
attachment_id,
worker_id,
attempt,
"image_processing_timeout",
permanent=False,
)
except ValueError as exc:
if db is not None:
db.rollback()
stop_event.set()
heartbeat.join(timeout=2)
code = str(exc)[:64] or "invalid_image"
return _fail_job(
job_id,
attachment_id,
worker_id,
attempt,
code,
permanent=True,
)
except ClientError as exc:
if db is not None:
db.rollback()
stop_event.set()
heartbeat.join(timeout=2)
error = exc.response.get("Error", {})
error_code = error.get("Code", "")
if error_code in {
"NoSuchKey",
"NoSuchBucket",
"404",
"NotFound",
}:
return _fail_job(
job_id,
attachment_id,
worker_id,
attempt,
"staging_missing",
permanent=True,
)
return _fail_job(
job_id,
attachment_id,
worker_id,
attempt,
"storage_unavailable",
permanent=False,
)
except (BotoCoreError, SQLAlchemyError):
if db is not None:
db.rollback()
stop_event.set()
heartbeat.join(timeout=2)
return _fail_job(
job_id,
attachment_id,
worker_id,
attempt,
"processing_unavailable",
permanent=False,
)
except Exception:
if db is not None:
db.rollback()
logger.exception(
"Unexpected processing error: job=%s",
job_id,
)
stop_event.set()
heartbeat.join(timeout=2)
return _fail_job(
job_id,
attachment_id,
worker_id,
attempt,
"processing_failed",
permanent=False,
)
finally:
stop_event.set()
heartbeat.join(timeout=2)
if db is not None:
db.close()
def _process(payload: dict) -> dict:
return process_job_with_telemetry(
_process_impl,
payload,
)
async def _publish_delayed(
exchange,
payload: dict,
delay_seconds: float,
) -> None:
"""Publish a persistent message to the TTL retry queue."""
delay_ms = max(
1000,
int(delay_seconds * 1000),
)
message = Message(
body=json.dumps(
payload,
separators=(",", ":"),
).encode("utf-8"),
content_type="application/json",
delivery_mode=DeliveryMode.PERSISTENT,
message_id=str(uuid.uuid4()),
type="attachment.process.retry",
expiration=timedelta(milliseconds=delay_ms),
)
await exchange.publish(
message,
routing_key=f"{settings.rabbitmq_queue}.retry",
mandatory=True,
)
logger.info(
"Scheduled delayed delivery in %.2fs",
delay_ms / 1000,
)
async def _quarantine_message(exchange, message) -> None:
"""Move an invalid message to the durable dead-letter queue."""
dead_message = Message(
body=message.body,
content_type=message.content_type or "application/octet-stream",
delivery_mode=DeliveryMode.PERSISTENT,
message_id=message.message_id or str(uuid.uuid4()),
type="attachment.process.invalid",
)
await exchange.publish(
dead_message,
routing_key=DEAD_ROUTING_KEY,
mandatory=True,
)
await message.ack()
logger.error(
"Invalid message moved to dead-letter queue: message_id=%s",
message.message_id,
)
async def handle_message(
message: aio_pika.IncomingMessage,
retry_exchange,
dead_exchange,
semaphore: asyncio.Semaphore,
) -> None:
"""Consume, process, retry or quarantine one RabbitMQ message."""
async with semaphore:
try:
payload = json.loads(message.body.decode("utf-8"))
if not isinstance(payload, dict):
raise ValueError("invalid_payload")
for field in (
"job_id",
"attachment_id",
"pipeline_version",
):
if field not in payload:
raise ValueError("missing_payload_field")
_safe_uuid(payload["job_id"])
_safe_uuid(payload["attachment_id"])
int(payload["pipeline_version"])
except (ValueError, TypeError, KeyError, UnicodeDecodeError):
try:
await _quarantine_message(
dead_exchange,
message,
)
except Exception:
logger.exception(
"Could not quarantine invalid message"
)
await message.nack(requeue=True)
return
try:
result = await asyncio.to_thread(
_process,
payload,
)
state = result["state"]
if state in {
"busy",
"deferred",
"retry",
}:
await _publish_delayed(
retry_exchange,
payload,
result.get("delay", 1.0),
)
await message.ack()
logger.info(
"Message handled: job=%s state=%s",
payload["job_id"],
state,
)
except asyncio.CancelledError:
raise
except Exception:
logger.exception(
"Message processing failed: message_id=%s",
message.message_id,
)
# If delayed publication or DB processing fails,
# preserve the original message.
await message.nack(requeue=True)
async def main() -> None:
"""Start the durable RabbitMQ consumer."""
setup_worker_telemetry()
connection = await aio_pika.connect_robust(
settings.rabbitmq_url
)
async with connection:
channel = await connection.channel(
publisher_confirms=True
)
await channel.set_qos(
prefetch_count=settings.rabbitmq_prefetch
)
exchange = await channel.declare_exchange(
settings.rabbitmq_exchange,
aio_pika.ExchangeType.DIRECT,
durable=True,
)
queue = await channel.declare_queue(
settings.rabbitmq_queue,
durable=True,
)
await queue.bind(
exchange,
routing_key=settings.rabbitmq_queue,
)
# Delayed retry exchange and queue.
retry_exchange = await channel.declare_exchange(
f"{settings.rabbitmq_exchange}.retry",
aio_pika.ExchangeType.DIRECT,
durable=True,
)
retry_queue = await channel.declare_queue(
f"{settings.rabbitmq_queue}.retry",
durable=True,
arguments={
"x-dead-letter-exchange": settings.rabbitmq_exchange,
"x-dead-letter-routing-key": settings.rabbitmq_queue,
},
)
await retry_queue.bind(
retry_exchange,
routing_key=f"{settings.rabbitmq_queue}.retry",
)
# Durable quarantine queue for invalid messages.
dead_exchange = await channel.declare_exchange(
f"{settings.rabbitmq_exchange}.dead",
aio_pika.ExchangeType.DIRECT,
durable=True,
)
dead_queue = await channel.declare_queue(
f"{settings.rabbitmq_queue}.dead",
durable=True,
)
await dead_queue.bind(
dead_exchange,
routing_key=DEAD_ROUTING_KEY,
)
semaphore = asyncio.Semaphore(
settings.worker_concurrency
)
async def consumer(message):
await handle_message(
message,
retry_exchange,
dead_exchange,
semaphore,
)
logger.info(
"Worker %s consuming queue=%s concurrency=%s prefetch=%s",
WORKER_ID,
settings.rabbitmq_queue,
settings.worker_concurrency,
settings.rabbitmq_prefetch,
)
await queue.consume(consumer)
await asyncio.Future()
if __name__ == "__main__":
asyncio.run(main())