исправленная версия с dlq

This commit is contained in:
2026-11-14 01:57:24 +03:00
parent 651d838778
commit b5b7ab204b
3 changed files with 409 additions and 210 deletions
+109 -28
View File
@@ -1,5 +1,6 @@
import os
import json
import random
import logging
import asyncio
from datetime import datetime, timezone
@@ -12,45 +13,125 @@ logger = logging.getLogger(__name__)
BROKER_URL = os.getenv("BROKER_URL", "amqp://messenger:mq_secret_password@rabbitmq:5672/")
MAIN_QUEUE = "attachment.process.v1"
DLX_NAME = "attachments.dlx"
DLQ_NAME = "attachment.process.dlq"
RETRY_EXCHANGE = "attachments.retry"
# Лестница отложенных очередей: фиксированный TTL на очередь,
# поэтому нет head-of-line blocking (в отличие от per-message TTL).
RETRY_TIERS = [1, 2, 4, 8, 16, 32, 60]
BASE_DELAY = 1
MAX_DELAY = 60
_connection = None
_channel = None
async def get_channel():
def retry_queue_name(tier: int) -> str:
return f"attachment.process.retry.{tier}s"
def choose_retry_tier(attempt: int) -> int:
"""Экспоненциальный backoff с equal jitter, квантованный по лестнице TTL."""
exp = min(BASE_DELAY * (2 ** (attempt - 1)), MAX_DELAY)
delay = random.uniform(exp / 2, exp) # equal jitter
for tier in RETRY_TIERS:
if tier >= delay:
return tier
return RETRY_TIERS[-1]
async def declare_topology(channel: aio_pika.Channel):
"""Идемпотентное объявление обменов и очередей. Аргументы неизменяемы!"""
dlx = await channel.declare_exchange(DLX_NAME, aio_pika.ExchangeType.DIRECT, durable=True)
dlq = await channel.declare_queue(DLQ_NAME, durable=True)
await dlq.bind(dlx, routing_key=DLQ_NAME)
retry_ex = await channel.declare_exchange(RETRY_EXCHANGE, aio_pika.ExchangeType.DIRECT, durable=True)
for tier in RETRY_TIERS:
name = retry_queue_name(tier)
q = await channel.declare_queue(
name,
durable=True,
arguments={
"x-message-ttl": tier * 1000,
"x-dead-letter-exchange": "", # default exchange
"x-dead-letter-routing-key": MAIN_QUEUE, # после TTL -> в основную очередь
},
)
await q.bind(retry_ex, routing_key=name)
await channel.declare_queue(
MAIN_QUEUE,
durable=True,
arguments={
"x-dead-letter-exchange": DLX_NAME,
"x-dead-letter-routing-key": DLQ_NAME,
},
)
logger.info("RabbitMQ topology ready: main=%s, retry tiers=%s, dlq=%s", MAIN_QUEUE, RETRY_TIERS, DLQ_NAME)
async def get_channel() -> aio_pika.Channel:
"""Канал с publisher confirms (aio-pika по умолчанию publisher_confirms=True)."""
global _connection, _channel
if _connection is None or _connection.is_closed:
if _connection is None or _connection.is_closed or _channel is None or _channel.is_closed:
_connection = await aio_pika.connect_robust(BROKER_URL)
_channel = await _connection.channel()
# Гарантируем, что очередь существует
await _channel.declare_queue("attachment.process.v1", durable=True)
_channel = await _connection.channel() # publisher_confirms=True
await declare_topology(_channel)
return _channel
async def publish_retry(payload: dict, tier: int):
"""Отложенная повторная доставка: сообщение полежит tier секунд и вернётся в main."""
channel = await get_channel()
retry_ex = await channel.get_exchange(RETRY_EXCHANGE)
await retry_ex.publish(
aio_pika.Message(
body=json.dumps(payload).encode(),
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
),
routing_key=retry_queue_name(tier),
)
logger.info("Scheduled retry in %ss for job %s (attempt %s)", tier, payload.get("job_id"), payload.get("attempt"))
async def publish_main(payload: dict):
channel = await get_channel()
await channel.default_exchange.publish(
aio_pika.Message(
body=json.dumps(payload).encode(),
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
),
routing_key=MAIN_QUEUE,
)
async def relay_outbox(db: AsyncSession):
"""Забирает непубликованные задачи из outbox и отправляет их в RabbitMQ"""
global _channel
if _channel is None:
await get_channel()
# Берём до 10 непубликованных записей
"""Transactional outbox: publish + confirm, только потом published_at."""
stmt = select(JobOutbox).where(JobOutbox.published_at.is_(None)).limit(10)
result = await db.execute(stmt)
outbox_records = result.scalars().all()
for record in outbox_records:
for record in result.scalars().all():
try:
payload = json.dumps(record.payload)
message = aio_pika.Message(
body=payload.encode(),
delivery_mode=aio_pika.DeliveryMode.PERSISTENT
)
await _channel.default_exchange.publish(
message,
routing_key="attachment.process.v1"
)
# Помечаем как опубликованное
await publish_main(record.payload) # ждём publisher confirm
record.published_at = datetime.now(timezone.utc)
await db.commit()
logger.info(f"Relayed job {record.job_id} to broker")
logger.info("Relayed job %s to broker", record.job_id)
except Exception as e:
logger.error(f"Failed to relay job {record.job_id}: {e}")
await db.rollback()
logger.error("Failed to relay job %s: %s", record.job_id, e)
await db.rollback()
async def run_relay():
from app.database import AsyncSessionLocal
while True:
try:
async with AsyncSessionLocal() as db:
await relay_outbox(db)
await asyncio.sleep(1)
except asyncio.CancelledError:
raise
except Exception as e:
logger.error("Relay loop error: %s", e)
await asyncio.sleep(5)
+1 -12
View File
@@ -9,8 +9,7 @@ from fastapi.responses import JSONResponse
from fastapi.exceptions import RequestValidationError
from starlette.exceptions import HTTPException as StarletteHTTPException
from app.routers import auth, chats, attachments
from app.database import AsyncSessionLocal
from app.broker import relay_outbox
from app.broker import run_relay
logger = logging.getLogger(__name__)
@@ -22,16 +21,6 @@ async def lifespan(app: FastAPI):
# Останавливаем при завершении
task.cancel()
async def run_relay():
while True:
try:
async with AsyncSessionLocal() as db:
await relay_outbox(db)
await asyncio.sleep(1) # Проверяем outbox каждую секунду
except Exception as e:
logger.error(f"Relay loop crashed: {e}")
await asyncio.sleep(5)
app = FastAPI(lifespan=lifespan)
app.include_router(auth.router)
+299 -170
View File
@@ -1,218 +1,347 @@
import os
import sys
import io
import json
import uuid
import hashlib
import asyncio
import logging
import tempfile
import json
from datetime import datetime, timezone, timedelta
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.orm import sessionmaker
import aio_pika
from minio import Minio
from PIL import Image
import io
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.orm import sessionmaker
from sqlalchemy.dialects.postgresql import insert as pg_insert
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
from app.models import Attachment, AttachmentJob, AttachmentVariant
from app.broker import (
get_channel, declare_topology, publish_retry, choose_retry_tier, MAIN_QUEUE,
)
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
logger = logging.getLogger(__name__)
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+asyncpg://messenger:secure_db_password_123@postgres:5432/messenger_db")
BROKER_URL = os.getenv("BROKER_URL", "amqp://messenger:mq_secret_password@rabbitmq:5672/")
DATABASE_URL = os.getenv("DATABASE_URL")
BROKER_URL = os.getenv("BROKER_URL")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333").replace("http://", "").replace("https://", "")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "admin")
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "admin")
S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "messenger-attachments")
WORKER_ID = str(uuid.uuid4())[:8]
JOB_LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "30"))
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "2"))
MAX_PIXELS = 40_000_000
MAX_BYTES = 10 * 1024 * 1024
engine = create_async_engine(DATABASE_URL, echo=False)
AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
s3_client = Minio(S3_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=False)
async def process_image(body: bytes, att_id: str, chat_id: str):
img = Image.open(io.BytesIO(body)).convert("RGB")
width, height = img.size
variants_to_save = []
# 1. Thumbnail (вписан в 256x256)
class PermanentError(Exception):
"""Повтор бессмысленен: битый файл, слишком большое изображение, нет staging."""
def __init__(self, code, message=""):
self.code = code
super().__init__(f"{code}: {message}")
class TemporaryError(Exception):
"""Повтор имеет смысл: S3 недоступен, таймаут, сеть."""
def __init__(self, code, message=""):
self.code = code
super().__init__(f"{code}: {message}")
def s3_put(key: str, data: bytes, content_type: str):
with tempfile.NamedTemporaryFile(delete=False) as tmp:
tmp.write(data)
path = tmp.name
try:
s3_client.fput_object(S3_BUCKET_NAME, key, path, content_type=content_type)
except Exception as e:
raise TemporaryError("storage_unavailable", str(e))
finally:
os.unlink(path)
def build_variants(body: bytes, att_id: str, chat_id: str):
"""CPU-bound часть. Только worker, никогда API."""
if len(body) > MAX_BYTES:
raise PermanentError("payload_too_large")
try:
img = Image.open(io.BytesIO(body))
img.load()
img = img.convert("RGB") # EXIF/ICC не переносим — энкодер получает только пиксели
except Exception as e:
raise PermanentError("invalid_image", str(e))
w, h = img.size
if w * h > MAX_PIXELS:
raise PermanentError("image_too_large")
out = []
thumb = img.copy()
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
thumb_bytes = io.BytesIO()
thumb.save(thumb_bytes, format="JPEG", quality=85)
thumb_data = thumb_bytes.getvalue()
thumb_key = f"variants/{chat_id}/{att_id}/thumbnail.jpg"
with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp:
tmp.write(thumb_data)
s3_client.fput_object(S3_BUCKET_NAME, thumb_key, tmp.name, content_type="image/jpeg")
os.unlink(tmp.name)
variants_to_save.append({
"variant_name": "thumbnail", "s3_key": thumb_key, "content_type": "image/jpeg",
"size_bytes": len(thumb_data), "width": thumb.width, "height": thumb.height,
"sha256": hashlib.sha256(thumb_data).hexdigest()
})
buf = io.BytesIO(); thumb.save(buf, format="JPEG", quality=85)
data = buf.getvalue()
key = f"variants/{chat_id}/{att_id}/thumbnail.jpg"
s3_put(key, data, "image/jpeg")
out.append(dict(variant_name="thumbnail", s3_key=key, content_type="image/jpeg",
size_bytes=len(data), width=thumb.width, height=thumb.height,
sha256=hashlib.sha256(data).hexdigest()))
# 2. Crop (центральный квадрат 128x128)
min_dim = min(width, height)
left = (width - min_dim) // 2
top = (height - min_dim) // 2
crop = img.crop((left, top, left + min_dim, top + min_dim)).resize((128, 128), Image.Resampling.LANCZOS)
crop_bytes = io.BytesIO()
crop.save(crop_bytes, format="JPEG", quality=85)
crop_data = crop_bytes.getvalue()
crop_key = f"variants/{chat_id}/{att_id}/crop.jpg"
with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp:
tmp.write(crop_data)
s3_client.fput_object(S3_BUCKET_NAME, crop_key, tmp.name, content_type="image/jpeg")
os.unlink(tmp.name)
variants_to_save.append({
"variant_name": "crop", "s3_key": crop_key, "content_type": "image/jpeg",
"size_bytes": len(crop_data), "width": 128, "height": 128,
"sha256": hashlib.sha256(crop_data).hexdigest()
})
side = min(w, h)
left, top = (w - side) // 2, (h - side) // 2
crop = img.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS)
buf = io.BytesIO(); crop.save(buf, format="JPEG", quality=85)
data = buf.getvalue()
key = f"variants/{chat_id}/{att_id}/crop.jpg"
s3_put(key, data, "image/jpeg")
out.append(dict(variant_name="crop", s3_key=key, content_type="image/jpeg",
size_bytes=len(data), width=128, height=128,
sha256=hashlib.sha256(data).hexdigest()))
# 3. Optimized WebP (вписан в 1280x1280)
opt = img.copy()
opt.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
opt_bytes = io.BytesIO()
opt.save(opt_bytes, format="WEBP", quality=80)
opt_data = opt_bytes.getvalue()
opt_key = f"variants/{chat_id}/{att_id}/optimized.webp"
with tempfile.NamedTemporaryFile(delete=False, suffix=".webp") as tmp:
tmp.write(opt_data)
s3_client.fput_object(S3_BUCKET_NAME, opt_key, tmp.name, content_type="image/webp")
os.unlink(tmp.name)
variants_to_save.append({
"variant_name": "optimized", "s3_key": opt_key, "content_type": "image/webp",
"size_bytes": len(opt_data), "width": opt.width, "height": opt.height,
"sha256": hashlib.sha256(opt_data).hexdigest()
})
buf = io.BytesIO(); opt.save(buf, format="WEBP", quality=80, method=4)
data = buf.getvalue()
key = f"variants/{chat_id}/{att_id}/optimized.webp"
s3_put(key, data, "image/webp")
out.append(dict(variant_name="optimized", s3_key=key, content_type="image/webp",
size_bytes=len(data), width=opt.width, height=opt.height,
sha256=hashlib.sha256(data).hexdigest()))
# 4. Сохраняем оригинал в конечную папку
ext = "png" if b"png" in body[:8] else "jpeg"
ext = "png" if body[:8].startswith(b"\x89PNG") else "jpeg"
final_key = f"attachments/{chat_id}/{att_id}.{ext}"
content_type = "image/png" if ext == "png" else "image/jpeg"
with tempfile.NamedTemporaryFile(delete=False, suffix=f".{ext}") as tmp:
tmp.write(body)
s3_client.fput_object(S3_BUCKET_NAME, final_key, tmp.name, content_type=content_type)
os.unlink(tmp.name)
s3_put(final_key, body, "image/png" if ext == "png" else "image/jpeg")
return final_key, out
async def handle_job(job_id: str, att_id: str, attempt: int):
"""Возвращает (outcome, error_code). outcome: done|noop|permanent|temporary"""
async with AsyncSessionLocal() as db:
job = (await db.execute(
select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
)).scalar_one_or_none()
if job is None:
return "noop", None
if job.status in ("ready", "failed"):
logger.info("Job %s already %s -> no-op ack", job_id, job.status)
return "noop", None
if attempt < job.attempt:
logger.info("Job %s stale delivery (msg attempt=%s < db attempt=%s) -> no-op", job_id, attempt, job.attempt)
return "noop", None
now = datetime.now(timezone.utc)
if job.status == "processing" and job.lease_until and job.lease_until > now and not job.lease_owner == WORKER_ID:
logger.info("Job %s leased by %s until %s -> no-op", job_id, job.lease_owner, job.lease_until)
return "noop", None
job.status = "processing"
job.attempt = attempt
job.lease_until = now + timedelta(seconds=JOB_LEASE_SECONDS)
job.lease_owner = WORKER_ID
att = (await db.execute(select(Attachment).where(Attachment.id == att_id))).scalar_one_or_none()
if att is None:
return "noop", None
att.status = "processing"
await db.commit()
staging_key = att.staging_key
if not staging_key:
return "permanent", "missing_staging"
try:
resp = s3_client.get_object(S3_BUCKET_NAME, staging_key)
try:
body = resp.read()
finally:
resp.close(); resp.release_conn()
except Exception as e:
return "temporary", "storage_unavailable"
if hashlib.sha256(body).hexdigest() != att.sha256:
return "permanent", "checksum_mismatch"
try:
final_key, variants = await asyncio.wait_for(
asyncio.to_thread(build_variants, body, str(att.id), str(att.chat_id)),
timeout=int(os.getenv("JOB_TIMEOUT_SECONDS", "60")),
)
except asyncio.TimeoutError:
return "temporary", "processing_timeout"
except PermanentError as e:
return "permanent", e.code
except TemporaryError as e:
return "temporary", e.code
except Exception as e:
logger.exception("Unexpected processing failure")
return "temporary", "processing_error"
# Идемпотентная фиксация: устаревшая попытка не перезапишет результат
res = await db.execute(
update(AttachmentJob)
.where(AttachmentJob.job_id == job_id,
AttachmentJob.attempt == attempt,
AttachmentJob.status == "processing")
.values(status="ready", lease_until=None)
)
if res.rowcount == 0:
logger.info("Job %s attempt %s lost ownership -> discard result", job_id, attempt)
return "noop", None
for v in variants:
await db.execute(
pg_insert(AttachmentVariant)
.values(attachment_id=att.id, **v)
.on_conflict_do_nothing(index_elements=["attachment_id", "variant_name"])
)
await db.execute(
update(Attachment).where(Attachment.id == att.id)
.values(status="ready", s3_key=final_key, staging_key=None, error_code=None)
)
await db.commit()
# Очистка staging — только после безопасной фиксации
try:
s3_client.remove_object(S3_BUCKET_NAME, staging_key)
except Exception as e:
logger.warning("Staging cleanup failed for %s: %s", staging_key, e)
logger.info("Job %s completed successfully (attempt %s)", job_id, attempt)
return "done", None
async def mark_failed(job_id: str, code: str):
async with AsyncSessionLocal() as db:
job = (await db.execute(select(AttachmentJob).where(AttachmentJob.job_id == job_id))).scalar_one_or_none()
if job is None or job.status == "ready":
return
await db.execute(update(AttachmentJob).where(AttachmentJob.job_id == job_id)
.values(status="failed", lease_until=None))
await db.execute(update(Attachment).where(Attachment.id == job.attachment_id)
.values(status="failed", error_code=code))
await db.commit()
logger.warning("Job %s -> failed (%s)", job_id, code)
async def schedule_retry(job_id: str, payload: dict) -> int:
"""Возвращает выбранный tier; -1 если бюджет попыток исчерпан."""
async with AsyncSessionLocal() as db:
job = (await db.execute(
select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
)).scalar_one_or_none()
if job is None or job.status == "ready":
return -1
if job.attempt >= JOB_MAX_ATTEMPTS:
return -1
next_attempt = job.attempt + 1
tier = choose_retry_tier(job.attempt)
job.status = "queued"
job.attempt = next_attempt
job.lease_until = None
job.lease_owner = None
await db.execute(update(Attachment).where(Attachment.id == job.attachment_id).values(status="queued"))
await db.commit()
await publish_retry({**payload, "attempt": next_attempt}, tier)
return tier
return final_key, variants_to_save
async def process_message(message: aio_pika.IncomingMessage):
async with message.process(requeue=False):
# Ручные подтверждения: без message.process(), чтобы не было двойного ack
try:
data = json.loads(message.body)
job_id, att_id, attempt = data["job_id"], data["attachment_id"], int(data["attempt"])
uuid.UUID(job_id); uuid.UUID(att_id)
except Exception:
logger.error("Malformed message -> DLQ")
await message.nack(requeue=False)
return
try:
outcome, code = await handle_job(job_id, att_id, attempt)
except Exception:
logger.exception("Handler crashed for job %s", job_id)
outcome, code = "temporary", "internal_error"
try:
if outcome in ("done", "noop"):
await message.ack()
return
if outcome == "permanent":
await mark_failed(job_id, code)
await message.nack(requeue=False) # -> DLX -> DLQ
return
# temporary
tier = await schedule_retry(job_id, data)
if tier < 0:
await mark_failed(job_id, "retry_exhausted")
await message.nack(requeue=False) # -> DLQ
else:
logger.info("Job %s temporary error (%s), retry in ~%ss", job_id, code, tier)
await message.ack() # копия уже в отложенной очереди
except Exception:
logger.exception("Failed to finalize delivery for job %s -> requeue", job_id)
await message.nack(requeue=True)
async def sweeper():
"""Зависшие processing (lease истёк) возвращаем в работу."""
while True:
try:
payload = message.body.decode()
data = json.loads(payload)
job_id = data["job_id"]
att_id = data["attachment_id"]
attempt = data["attempt"]
logger.info(f"Processing job {job_id} for attachment {att_id}, attempt {attempt}")
await asyncio.sleep(5)
async with AsyncSessionLocal() as db:
from app.models import AttachmentJob, Attachment, AttachmentVariant
# 1. Блокируем задачу для обработки (Claim)
stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
result = await db.execute(stmt)
job = result.scalar_one_or_none()
if not job:
logger.warning(f"Job {job_id} not found")
return
if job.status in ("ready", "failed"):
logger.info(f"Job {job_id} already {job.status}, skipping")
return
if job.status == "processing" and job.lease_until and job.lease_until > datetime.now(timezone.utc):
logger.info(f"Job {job_id} is already being processed by another worker")
return
now = datetime.now(timezone.utc)
job.status = "processing"
job.attempt = attempt
job.lease_until = now + timedelta(seconds=JOB_LEASE_SECONDS)
job.lease_owner = WORKER_ID
await db.commit()
# 2. Получаем метаданные вложения
att_stmt = select(Attachment).where(Attachment.id == att_id)
att_result = await db.execute(att_stmt)
attachment = att_result.scalar_one_or_none()
if not attachment or not attachment.staging_key:
job.status = "failed"
job.error_code = "missing_staging"
rows = (await db.execute(
select(AttachmentJob).where(
AttachmentJob.status == "processing",
AttachmentJob.lease_until.isnot(None),
AttachmentJob.lease_until < now,
).limit(20)
)).scalars().all()
for job in rows:
if job.attempt >= JOB_MAX_ATTEMPTS:
continue
payload = {"job_id": str(job.job_id), "attachment_id": str(job.attachment_id),
"attempt": job.attempt + 1}
job.status = "queued"
job.attempt += 1
job.lease_until = None
job.lease_owner = None
await db.execute(update(Attachment).where(Attachment.id == job.attachment_id)
.values(status="queued"))
await db.commit()
return
staging_key_to_delete = attachment.staging_key # Сохраняем ключ до обнуления
# 3. Скачиваем из staging
response = s3_client.get_object(S3_BUCKET_NAME, staging_key_to_delete)
body = response.read()
response.close()
response.release_conn()
# 4. Обрабатываем изображение
final_key, variants_data = await process_image(body, str(attachment.id), str(attachment.chat_id))
# 5. Фиксируем результат в БД
attachment.status = "ready"
attachment.s3_key = final_key
attachment.staging_key = None # Очищаем staging
job.status = "ready"
for v_data in variants_data:
variant = AttachmentVariant(
attachment_id=attachment.id,
**v_data
)
db.add(variant)
await db.commit()
logger.info(f"Job {job_id} completed successfully")
# 6. Удаляем staging из S3
s3_client.remove_object(S3_BUCKET_NAME, staging_key_to_delete)
await publish_retry(payload, choose_retry_tier(1))
logger.warning("Sweeper: lease expired for job %s -> retry attempt %s", job.job_id, job.attempt)
except asyncio.CancelledError:
raise
except Exception as e:
logger.error(f"Error processing message: {e}")
try:
async with AsyncSessionLocal() as db:
from app.models import AttachmentJob
stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
result = await db.execute(stmt)
job = result.scalar_one_or_none()
if job:
if job.attempt >= JOB_MAX_ATTEMPTS:
job.status = "failed"
job.error_code = "retry_exhausted"
else:
job.status = "queued"
job.attempt += 1
job.lease_until = None
await db.commit()
except Exception as db_err:
logger.error(f"Failed to update job status after error: {db_err}")
logger.error("Sweeper error: %s", e)
async def main():
logger.info("Starting worker...")
connection = await aio_pika.connect_robust(BROKER_URL)
channel = await connection.channel()
queue = await channel.declare_queue("attachment.process.v1", durable=True)
logger.info("Waiting for messages...")
logger.info("Starting worker %s (concurrency/prefetch=%s)", WORKER_ID, WORKER_CONCURRENCY)
channel = await get_channel()
await declare_topology(channel)
await channel.set_qos(prefetch_count=WORKER_CONCURRENCY)
queue = await channel.get_queue(MAIN_QUEUE)
await queue.consume(process_message)
asyncio.create_task(sweeper())
logger.info("Waiting for messages on %s ...", MAIN_QUEUE)
await asyncio.Future()
if __name__ == "__main__":
asyncio.run(main())