diff --git a/app/broker.py b/app/broker.py index 93e34a27..cd878f39 100644 --- a/app/broker.py +++ b/app/broker.py @@ -1,5 +1,6 @@ import os import json +import random import logging import asyncio from datetime import datetime, timezone @@ -12,45 +13,125 @@ logger = logging.getLogger(__name__) BROKER_URL = os.getenv("BROKER_URL", "amqp://messenger:mq_secret_password@rabbitmq:5672/") +MAIN_QUEUE = "attachment.process.v1" +DLX_NAME = "attachments.dlx" +DLQ_NAME = "attachment.process.dlq" +RETRY_EXCHANGE = "attachments.retry" + +# Лестница отложенных очередей: фиксированный TTL на очередь, +# поэтому нет head-of-line blocking (в отличие от per-message TTL). +RETRY_TIERS = [1, 2, 4, 8, 16, 32, 60] +BASE_DELAY = 1 +MAX_DELAY = 60 + _connection = None _channel = None -async def get_channel(): + +def retry_queue_name(tier: int) -> str: + return f"attachment.process.retry.{tier}s" + + +def choose_retry_tier(attempt: int) -> int: + """Экспоненциальный backoff с equal jitter, квантованный по лестнице TTL.""" + exp = min(BASE_DELAY * (2 ** (attempt - 1)), MAX_DELAY) + delay = random.uniform(exp / 2, exp) # equal jitter + for tier in RETRY_TIERS: + if tier >= delay: + return tier + return RETRY_TIERS[-1] + + +async def declare_topology(channel: aio_pika.Channel): + """Идемпотентное объявление обменов и очередей. Аргументы неизменяемы!""" + dlx = await channel.declare_exchange(DLX_NAME, aio_pika.ExchangeType.DIRECT, durable=True) + dlq = await channel.declare_queue(DLQ_NAME, durable=True) + await dlq.bind(dlx, routing_key=DLQ_NAME) + + retry_ex = await channel.declare_exchange(RETRY_EXCHANGE, aio_pika.ExchangeType.DIRECT, durable=True) + for tier in RETRY_TIERS: + name = retry_queue_name(tier) + q = await channel.declare_queue( + name, + durable=True, + arguments={ + "x-message-ttl": tier * 1000, + "x-dead-letter-exchange": "", # default exchange + "x-dead-letter-routing-key": MAIN_QUEUE, # после TTL -> в основную очередь + }, + ) + await q.bind(retry_ex, routing_key=name) + + await channel.declare_queue( + MAIN_QUEUE, + durable=True, + arguments={ + "x-dead-letter-exchange": DLX_NAME, + "x-dead-letter-routing-key": DLQ_NAME, + }, + ) + logger.info("RabbitMQ topology ready: main=%s, retry tiers=%s, dlq=%s", MAIN_QUEUE, RETRY_TIERS, DLQ_NAME) + + +async def get_channel() -> aio_pika.Channel: + """Канал с publisher confirms (aio-pika по умолчанию publisher_confirms=True).""" global _connection, _channel - if _connection is None or _connection.is_closed: + if _connection is None or _connection.is_closed or _channel is None or _channel.is_closed: _connection = await aio_pika.connect_robust(BROKER_URL) - _channel = await _connection.channel() - # Гарантируем, что очередь существует - await _channel.declare_queue("attachment.process.v1", durable=True) + _channel = await _connection.channel() # publisher_confirms=True + await declare_topology(_channel) return _channel + +async def publish_retry(payload: dict, tier: int): + """Отложенная повторная доставка: сообщение полежит tier секунд и вернётся в main.""" + channel = await get_channel() + retry_ex = await channel.get_exchange(RETRY_EXCHANGE) + await retry_ex.publish( + aio_pika.Message( + body=json.dumps(payload).encode(), + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + ), + routing_key=retry_queue_name(tier), + ) + logger.info("Scheduled retry in %ss for job %s (attempt %s)", tier, payload.get("job_id"), payload.get("attempt")) + + +async def publish_main(payload: dict): + channel = await get_channel() + await channel.default_exchange.publish( + aio_pika.Message( + body=json.dumps(payload).encode(), + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + ), + routing_key=MAIN_QUEUE, + ) + + async def relay_outbox(db: AsyncSession): - """Забирает непубликованные задачи из outbox и отправляет их в RabbitMQ""" - global _channel - if _channel is None: - await get_channel() - - # Берём до 10 непубликованных записей + """Transactional outbox: publish + confirm, только потом published_at.""" stmt = select(JobOutbox).where(JobOutbox.published_at.is_(None)).limit(10) result = await db.execute(stmt) - outbox_records = result.scalars().all() - - for record in outbox_records: + for record in result.scalars().all(): try: - payload = json.dumps(record.payload) - message = aio_pika.Message( - body=payload.encode(), - delivery_mode=aio_pika.DeliveryMode.PERSISTENT - ) - await _channel.default_exchange.publish( - message, - routing_key="attachment.process.v1" - ) - - # Помечаем как опубликованное + await publish_main(record.payload) # ждём publisher confirm record.published_at = datetime.now(timezone.utc) await db.commit() - logger.info(f"Relayed job {record.job_id} to broker") + logger.info("Relayed job %s to broker", record.job_id) except Exception as e: - logger.error(f"Failed to relay job {record.job_id}: {e}") - await db.rollback() \ No newline at end of file + logger.error("Failed to relay job %s: %s", record.job_id, e) + await db.rollback() + + +async def run_relay(): + from app.database import AsyncSessionLocal + while True: + try: + async with AsyncSessionLocal() as db: + await relay_outbox(db) + await asyncio.sleep(1) + except asyncio.CancelledError: + raise + except Exception as e: + logger.error("Relay loop error: %s", e) + await asyncio.sleep(5) \ No newline at end of file diff --git a/app/main.py b/app/main.py index 87e0d6ad..8b111fe3 100644 --- a/app/main.py +++ b/app/main.py @@ -9,8 +9,7 @@ from fastapi.responses import JSONResponse from fastapi.exceptions import RequestValidationError from starlette.exceptions import HTTPException as StarletteHTTPException from app.routers import auth, chats, attachments -from app.database import AsyncSessionLocal -from app.broker import relay_outbox +from app.broker import run_relay logger = logging.getLogger(__name__) @@ -22,16 +21,6 @@ async def lifespan(app: FastAPI): # Останавливаем при завершении task.cancel() -async def run_relay(): - while True: - try: - async with AsyncSessionLocal() as db: - await relay_outbox(db) - await asyncio.sleep(1) # Проверяем outbox каждую секунду - except Exception as e: - logger.error(f"Relay loop crashed: {e}") - await asyncio.sleep(5) - app = FastAPI(lifespan=lifespan) app.include_router(auth.router) diff --git a/app/worker.py b/app/worker.py index 8806e005..09d99f90 100644 --- a/app/worker.py +++ b/app/worker.py @@ -1,218 +1,347 @@ import os -import sys +import io +import json import uuid import hashlib import asyncio import logging import tempfile -import json from datetime import datetime, timezone, timedelta -from sqlalchemy import select, update -from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession -from sqlalchemy.orm import sessionmaker + import aio_pika from minio import Minio from PIL import Image -import io +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +from sqlalchemy.orm import sessionmaker +from sqlalchemy.dialects.postgresql import insert as pg_insert -logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') +from app.models import Attachment, AttachmentJob, AttachmentVariant +from app.broker import ( + get_channel, declare_topology, publish_retry, choose_retry_tier, MAIN_QUEUE, +) + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger(__name__) -DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+asyncpg://messenger:secure_db_password_123@postgres:5432/messenger_db") -BROKER_URL = os.getenv("BROKER_URL", "amqp://messenger:mq_secret_password@rabbitmq:5672/") +DATABASE_URL = os.getenv("DATABASE_URL") +BROKER_URL = os.getenv("BROKER_URL") S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333").replace("http://", "").replace("https://", "") S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "admin") S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "admin") S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "messenger-attachments") + WORKER_ID = str(uuid.uuid4())[:8] JOB_LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "30")) JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) +WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "2")) +MAX_PIXELS = 40_000_000 +MAX_BYTES = 10 * 1024 * 1024 engine = create_async_engine(DATABASE_URL, echo=False) AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - s3_client = Minio(S3_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=False) -async def process_image(body: bytes, att_id: str, chat_id: str): - img = Image.open(io.BytesIO(body)).convert("RGB") - width, height = img.size - variants_to_save = [] - - # 1. Thumbnail (вписан в 256x256) + +class PermanentError(Exception): + """Повтор бессмысленен: битый файл, слишком большое изображение, нет staging.""" + def __init__(self, code, message=""): + self.code = code + super().__init__(f"{code}: {message}") + + +class TemporaryError(Exception): + """Повтор имеет смысл: S3 недоступен, таймаут, сеть.""" + def __init__(self, code, message=""): + self.code = code + super().__init__(f"{code}: {message}") + + +def s3_put(key: str, data: bytes, content_type: str): + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmp.write(data) + path = tmp.name + try: + s3_client.fput_object(S3_BUCKET_NAME, key, path, content_type=content_type) + except Exception as e: + raise TemporaryError("storage_unavailable", str(e)) + finally: + os.unlink(path) + + +def build_variants(body: bytes, att_id: str, chat_id: str): + """CPU-bound часть. Только worker, никогда API.""" + if len(body) > MAX_BYTES: + raise PermanentError("payload_too_large") + try: + img = Image.open(io.BytesIO(body)) + img.load() + img = img.convert("RGB") # EXIF/ICC не переносим — энкодер получает только пиксели + except Exception as e: + raise PermanentError("invalid_image", str(e)) + + w, h = img.size + if w * h > MAX_PIXELS: + raise PermanentError("image_too_large") + + out = [] + thumb = img.copy() thumb.thumbnail((256, 256), Image.Resampling.LANCZOS) - thumb_bytes = io.BytesIO() - thumb.save(thumb_bytes, format="JPEG", quality=85) - thumb_data = thumb_bytes.getvalue() - thumb_key = f"variants/{chat_id}/{att_id}/thumbnail.jpg" - with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp: - tmp.write(thumb_data) - s3_client.fput_object(S3_BUCKET_NAME, thumb_key, tmp.name, content_type="image/jpeg") - os.unlink(tmp.name) - variants_to_save.append({ - "variant_name": "thumbnail", "s3_key": thumb_key, "content_type": "image/jpeg", - "size_bytes": len(thumb_data), "width": thumb.width, "height": thumb.height, - "sha256": hashlib.sha256(thumb_data).hexdigest() - }) + buf = io.BytesIO(); thumb.save(buf, format="JPEG", quality=85) + data = buf.getvalue() + key = f"variants/{chat_id}/{att_id}/thumbnail.jpg" + s3_put(key, data, "image/jpeg") + out.append(dict(variant_name="thumbnail", s3_key=key, content_type="image/jpeg", + size_bytes=len(data), width=thumb.width, height=thumb.height, + sha256=hashlib.sha256(data).hexdigest())) - # 2. Crop (центральный квадрат 128x128) - min_dim = min(width, height) - left = (width - min_dim) // 2 - top = (height - min_dim) // 2 - crop = img.crop((left, top, left + min_dim, top + min_dim)).resize((128, 128), Image.Resampling.LANCZOS) - crop_bytes = io.BytesIO() - crop.save(crop_bytes, format="JPEG", quality=85) - crop_data = crop_bytes.getvalue() - crop_key = f"variants/{chat_id}/{att_id}/crop.jpg" - with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp: - tmp.write(crop_data) - s3_client.fput_object(S3_BUCKET_NAME, crop_key, tmp.name, content_type="image/jpeg") - os.unlink(tmp.name) - variants_to_save.append({ - "variant_name": "crop", "s3_key": crop_key, "content_type": "image/jpeg", - "size_bytes": len(crop_data), "width": 128, "height": 128, - "sha256": hashlib.sha256(crop_data).hexdigest() - }) + side = min(w, h) + left, top = (w - side) // 2, (h - side) // 2 + crop = img.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS) + buf = io.BytesIO(); crop.save(buf, format="JPEG", quality=85) + data = buf.getvalue() + key = f"variants/{chat_id}/{att_id}/crop.jpg" + s3_put(key, data, "image/jpeg") + out.append(dict(variant_name="crop", s3_key=key, content_type="image/jpeg", + size_bytes=len(data), width=128, height=128, + sha256=hashlib.sha256(data).hexdigest())) - # 3. Optimized WebP (вписан в 1280x1280) opt = img.copy() opt.thumbnail((1280, 1280), Image.Resampling.LANCZOS) - opt_bytes = io.BytesIO() - opt.save(opt_bytes, format="WEBP", quality=80) - opt_data = opt_bytes.getvalue() - opt_key = f"variants/{chat_id}/{att_id}/optimized.webp" - with tempfile.NamedTemporaryFile(delete=False, suffix=".webp") as tmp: - tmp.write(opt_data) - s3_client.fput_object(S3_BUCKET_NAME, opt_key, tmp.name, content_type="image/webp") - os.unlink(tmp.name) - variants_to_save.append({ - "variant_name": "optimized", "s3_key": opt_key, "content_type": "image/webp", - "size_bytes": len(opt_data), "width": opt.width, "height": opt.height, - "sha256": hashlib.sha256(opt_data).hexdigest() - }) + buf = io.BytesIO(); opt.save(buf, format="WEBP", quality=80, method=4) + data = buf.getvalue() + key = f"variants/{chat_id}/{att_id}/optimized.webp" + s3_put(key, data, "image/webp") + out.append(dict(variant_name="optimized", s3_key=key, content_type="image/webp", + size_bytes=len(data), width=opt.width, height=opt.height, + sha256=hashlib.sha256(data).hexdigest())) - # 4. Сохраняем оригинал в конечную папку - ext = "png" if b"png" in body[:8] else "jpeg" + ext = "png" if body[:8].startswith(b"\x89PNG") else "jpeg" final_key = f"attachments/{chat_id}/{att_id}.{ext}" - content_type = "image/png" if ext == "png" else "image/jpeg" - with tempfile.NamedTemporaryFile(delete=False, suffix=f".{ext}") as tmp: - tmp.write(body) - s3_client.fput_object(S3_BUCKET_NAME, final_key, tmp.name, content_type=content_type) - os.unlink(tmp.name) + s3_put(final_key, body, "image/png" if ext == "png" else "image/jpeg") + return final_key, out + + +async def handle_job(job_id: str, att_id: str, attempt: int): + """Возвращает (outcome, error_code). outcome: done|noop|permanent|temporary""" + async with AsyncSessionLocal() as db: + job = (await db.execute( + select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update() + )).scalar_one_or_none() + + if job is None: + return "noop", None + if job.status in ("ready", "failed"): + logger.info("Job %s already %s -> no-op ack", job_id, job.status) + return "noop", None + if attempt < job.attempt: + logger.info("Job %s stale delivery (msg attempt=%s < db attempt=%s) -> no-op", job_id, attempt, job.attempt) + return "noop", None + + now = datetime.now(timezone.utc) + if job.status == "processing" and job.lease_until and job.lease_until > now and not job.lease_owner == WORKER_ID: + logger.info("Job %s leased by %s until %s -> no-op", job_id, job.lease_owner, job.lease_until) + return "noop", None + + job.status = "processing" + job.attempt = attempt + job.lease_until = now + timedelta(seconds=JOB_LEASE_SECONDS) + job.lease_owner = WORKER_ID + + att = (await db.execute(select(Attachment).where(Attachment.id == att_id))).scalar_one_or_none() + if att is None: + return "noop", None + att.status = "processing" + await db.commit() + + staging_key = att.staging_key + if not staging_key: + return "permanent", "missing_staging" + + try: + resp = s3_client.get_object(S3_BUCKET_NAME, staging_key) + try: + body = resp.read() + finally: + resp.close(); resp.release_conn() + except Exception as e: + return "temporary", "storage_unavailable" + + if hashlib.sha256(body).hexdigest() != att.sha256: + return "permanent", "checksum_mismatch" + + try: + final_key, variants = await asyncio.wait_for( + asyncio.to_thread(build_variants, body, str(att.id), str(att.chat_id)), + timeout=int(os.getenv("JOB_TIMEOUT_SECONDS", "60")), + ) + except asyncio.TimeoutError: + return "temporary", "processing_timeout" + except PermanentError as e: + return "permanent", e.code + except TemporaryError as e: + return "temporary", e.code + except Exception as e: + logger.exception("Unexpected processing failure") + return "temporary", "processing_error" + + # Идемпотентная фиксация: устаревшая попытка не перезапишет результат + res = await db.execute( + update(AttachmentJob) + .where(AttachmentJob.job_id == job_id, + AttachmentJob.attempt == attempt, + AttachmentJob.status == "processing") + .values(status="ready", lease_until=None) + ) + if res.rowcount == 0: + logger.info("Job %s attempt %s lost ownership -> discard result", job_id, attempt) + return "noop", None + + for v in variants: + await db.execute( + pg_insert(AttachmentVariant) + .values(attachment_id=att.id, **v) + .on_conflict_do_nothing(index_elements=["attachment_id", "variant_name"]) + ) + + await db.execute( + update(Attachment).where(Attachment.id == att.id) + .values(status="ready", s3_key=final_key, staging_key=None, error_code=None) + ) + await db.commit() + + # Очистка staging — только после безопасной фиксации + try: + s3_client.remove_object(S3_BUCKET_NAME, staging_key) + except Exception as e: + logger.warning("Staging cleanup failed for %s: %s", staging_key, e) + + logger.info("Job %s completed successfully (attempt %s)", job_id, attempt) + return "done", None + + +async def mark_failed(job_id: str, code: str): + async with AsyncSessionLocal() as db: + job = (await db.execute(select(AttachmentJob).where(AttachmentJob.job_id == job_id))).scalar_one_or_none() + if job is None or job.status == "ready": + return + await db.execute(update(AttachmentJob).where(AttachmentJob.job_id == job_id) + .values(status="failed", lease_until=None)) + await db.execute(update(Attachment).where(Attachment.id == job.attachment_id) + .values(status="failed", error_code=code)) + await db.commit() + logger.warning("Job %s -> failed (%s)", job_id, code) + + +async def schedule_retry(job_id: str, payload: dict) -> int: + """Возвращает выбранный tier; -1 если бюджет попыток исчерпан.""" + async with AsyncSessionLocal() as db: + job = (await db.execute( + select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update() + )).scalar_one_or_none() + if job is None or job.status == "ready": + return -1 + if job.attempt >= JOB_MAX_ATTEMPTS: + return -1 + next_attempt = job.attempt + 1 + tier = choose_retry_tier(job.attempt) + job.status = "queued" + job.attempt = next_attempt + job.lease_until = None + job.lease_owner = None + await db.execute(update(Attachment).where(Attachment.id == job.attachment_id).values(status="queued")) + await db.commit() + + await publish_retry({**payload, "attempt": next_attempt}, tier) + return tier - return final_key, variants_to_save async def process_message(message: aio_pika.IncomingMessage): - async with message.process(requeue=False): + # Ручные подтверждения: без message.process(), чтобы не было двойного ack + try: + data = json.loads(message.body) + job_id, att_id, attempt = data["job_id"], data["attachment_id"], int(data["attempt"]) + uuid.UUID(job_id); uuid.UUID(att_id) + except Exception: + logger.error("Malformed message -> DLQ") + await message.nack(requeue=False) + return + + try: + outcome, code = await handle_job(job_id, att_id, attempt) + except Exception: + logger.exception("Handler crashed for job %s", job_id) + outcome, code = "temporary", "internal_error" + + try: + if outcome in ("done", "noop"): + await message.ack() + return + + if outcome == "permanent": + await mark_failed(job_id, code) + await message.nack(requeue=False) # -> DLX -> DLQ + return + + # temporary + tier = await schedule_retry(job_id, data) + if tier < 0: + await mark_failed(job_id, "retry_exhausted") + await message.nack(requeue=False) # -> DLQ + else: + logger.info("Job %s temporary error (%s), retry in ~%ss", job_id, code, tier) + await message.ack() # копия уже в отложенной очереди + except Exception: + logger.exception("Failed to finalize delivery for job %s -> requeue", job_id) + await message.nack(requeue=True) + + +async def sweeper(): + """Зависшие processing (lease истёк) возвращаем в работу.""" + while True: try: - payload = message.body.decode() - data = json.loads(payload) - job_id = data["job_id"] - att_id = data["attachment_id"] - attempt = data["attempt"] - - logger.info(f"Processing job {job_id} for attachment {att_id}, attempt {attempt}") - + await asyncio.sleep(5) async with AsyncSessionLocal() as db: - from app.models import AttachmentJob, Attachment, AttachmentVariant - - # 1. Блокируем задачу для обработки (Claim) - stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update() - result = await db.execute(stmt) - job = result.scalar_one_or_none() - - if not job: - logger.warning(f"Job {job_id} not found") - return - - if job.status in ("ready", "failed"): - logger.info(f"Job {job_id} already {job.status}, skipping") - return - - if job.status == "processing" and job.lease_until and job.lease_until > datetime.now(timezone.utc): - logger.info(f"Job {job_id} is already being processed by another worker") - return - now = datetime.now(timezone.utc) - job.status = "processing" - job.attempt = attempt - job.lease_until = now + timedelta(seconds=JOB_LEASE_SECONDS) - job.lease_owner = WORKER_ID - await db.commit() - - # 2. Получаем метаданные вложения - att_stmt = select(Attachment).where(Attachment.id == att_id) - att_result = await db.execute(att_stmt) - attachment = att_result.scalar_one_or_none() - - if not attachment or not attachment.staging_key: - job.status = "failed" - job.error_code = "missing_staging" + rows = (await db.execute( + select(AttachmentJob).where( + AttachmentJob.status == "processing", + AttachmentJob.lease_until.isnot(None), + AttachmentJob.lease_until < now, + ).limit(20) + )).scalars().all() + for job in rows: + if job.attempt >= JOB_MAX_ATTEMPTS: + continue + payload = {"job_id": str(job.job_id), "attachment_id": str(job.attachment_id), + "attempt": job.attempt + 1} + job.status = "queued" + job.attempt += 1 + job.lease_until = None + job.lease_owner = None + await db.execute(update(Attachment).where(Attachment.id == job.attachment_id) + .values(status="queued")) await db.commit() - return - - staging_key_to_delete = attachment.staging_key # Сохраняем ключ до обнуления - - # 3. Скачиваем из staging - response = s3_client.get_object(S3_BUCKET_NAME, staging_key_to_delete) - body = response.read() - response.close() - response.release_conn() - - # 4. Обрабатываем изображение - final_key, variants_data = await process_image(body, str(attachment.id), str(attachment.chat_id)) - - # 5. Фиксируем результат в БД - attachment.status = "ready" - attachment.s3_key = final_key - attachment.staging_key = None # Очищаем staging - - job.status = "ready" - - for v_data in variants_data: - variant = AttachmentVariant( - attachment_id=attachment.id, - **v_data - ) - db.add(variant) - - await db.commit() - logger.info(f"Job {job_id} completed successfully") - - # 6. Удаляем staging из S3 - s3_client.remove_object(S3_BUCKET_NAME, staging_key_to_delete) - + await publish_retry(payload, choose_retry_tier(1)) + logger.warning("Sweeper: lease expired for job %s -> retry attempt %s", job.job_id, job.attempt) + except asyncio.CancelledError: + raise except Exception as e: - logger.error(f"Error processing message: {e}") - try: - async with AsyncSessionLocal() as db: - from app.models import AttachmentJob - stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update() - result = await db.execute(stmt) - job = result.scalar_one_or_none() - if job: - if job.attempt >= JOB_MAX_ATTEMPTS: - job.status = "failed" - job.error_code = "retry_exhausted" - else: - job.status = "queued" - job.attempt += 1 - job.lease_until = None - await db.commit() - except Exception as db_err: - logger.error(f"Failed to update job status after error: {db_err}") + logger.error("Sweeper error: %s", e) + async def main(): - logger.info("Starting worker...") - connection = await aio_pika.connect_robust(BROKER_URL) - channel = await connection.channel() - - queue = await channel.declare_queue("attachment.process.v1", durable=True) - - logger.info("Waiting for messages...") + logger.info("Starting worker %s (concurrency/prefetch=%s)", WORKER_ID, WORKER_CONCURRENCY) + channel = await get_channel() + await declare_topology(channel) + await channel.set_qos(prefetch_count=WORKER_CONCURRENCY) + queue = await channel.get_queue(MAIN_QUEUE) await queue.consume(process_message) - + asyncio.create_task(sweeper()) + logger.info("Waiting for messages on %s ...", MAIN_QUEUE) await asyncio.Future() + if __name__ == "__main__": asyncio.run(main()) \ No newline at end of file