исправленная версия с dlq
This commit is contained in:
+109
-28
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import json
|
||||
import random
|
||||
import logging
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
@@ -12,45 +13,125 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
BROKER_URL = os.getenv("BROKER_URL", "amqp://messenger:mq_secret_password@rabbitmq:5672/")
|
||||
|
||||
MAIN_QUEUE = "attachment.process.v1"
|
||||
DLX_NAME = "attachments.dlx"
|
||||
DLQ_NAME = "attachment.process.dlq"
|
||||
RETRY_EXCHANGE = "attachments.retry"
|
||||
|
||||
# Лестница отложенных очередей: фиксированный TTL на очередь,
|
||||
# поэтому нет head-of-line blocking (в отличие от per-message TTL).
|
||||
RETRY_TIERS = [1, 2, 4, 8, 16, 32, 60]
|
||||
BASE_DELAY = 1
|
||||
MAX_DELAY = 60
|
||||
|
||||
_connection = None
|
||||
_channel = None
|
||||
|
||||
async def get_channel():
|
||||
|
||||
def retry_queue_name(tier: int) -> str:
|
||||
return f"attachment.process.retry.{tier}s"
|
||||
|
||||
|
||||
def choose_retry_tier(attempt: int) -> int:
|
||||
"""Экспоненциальный backoff с equal jitter, квантованный по лестнице TTL."""
|
||||
exp = min(BASE_DELAY * (2 ** (attempt - 1)), MAX_DELAY)
|
||||
delay = random.uniform(exp / 2, exp) # equal jitter
|
||||
for tier in RETRY_TIERS:
|
||||
if tier >= delay:
|
||||
return tier
|
||||
return RETRY_TIERS[-1]
|
||||
|
||||
|
||||
async def declare_topology(channel: aio_pika.Channel):
|
||||
"""Идемпотентное объявление обменов и очередей. Аргументы неизменяемы!"""
|
||||
dlx = await channel.declare_exchange(DLX_NAME, aio_pika.ExchangeType.DIRECT, durable=True)
|
||||
dlq = await channel.declare_queue(DLQ_NAME, durable=True)
|
||||
await dlq.bind(dlx, routing_key=DLQ_NAME)
|
||||
|
||||
retry_ex = await channel.declare_exchange(RETRY_EXCHANGE, aio_pika.ExchangeType.DIRECT, durable=True)
|
||||
for tier in RETRY_TIERS:
|
||||
name = retry_queue_name(tier)
|
||||
q = await channel.declare_queue(
|
||||
name,
|
||||
durable=True,
|
||||
arguments={
|
||||
"x-message-ttl": tier * 1000,
|
||||
"x-dead-letter-exchange": "", # default exchange
|
||||
"x-dead-letter-routing-key": MAIN_QUEUE, # после TTL -> в основную очередь
|
||||
},
|
||||
)
|
||||
await q.bind(retry_ex, routing_key=name)
|
||||
|
||||
await channel.declare_queue(
|
||||
MAIN_QUEUE,
|
||||
durable=True,
|
||||
arguments={
|
||||
"x-dead-letter-exchange": DLX_NAME,
|
||||
"x-dead-letter-routing-key": DLQ_NAME,
|
||||
},
|
||||
)
|
||||
logger.info("RabbitMQ topology ready: main=%s, retry tiers=%s, dlq=%s", MAIN_QUEUE, RETRY_TIERS, DLQ_NAME)
|
||||
|
||||
|
||||
async def get_channel() -> aio_pika.Channel:
|
||||
"""Канал с publisher confirms (aio-pika по умолчанию publisher_confirms=True)."""
|
||||
global _connection, _channel
|
||||
if _connection is None or _connection.is_closed:
|
||||
if _connection is None or _connection.is_closed or _channel is None or _channel.is_closed:
|
||||
_connection = await aio_pika.connect_robust(BROKER_URL)
|
||||
_channel = await _connection.channel()
|
||||
# Гарантируем, что очередь существует
|
||||
await _channel.declare_queue("attachment.process.v1", durable=True)
|
||||
_channel = await _connection.channel() # publisher_confirms=True
|
||||
await declare_topology(_channel)
|
||||
return _channel
|
||||
|
||||
|
||||
async def publish_retry(payload: dict, tier: int):
|
||||
"""Отложенная повторная доставка: сообщение полежит tier секунд и вернётся в main."""
|
||||
channel = await get_channel()
|
||||
retry_ex = await channel.get_exchange(RETRY_EXCHANGE)
|
||||
await retry_ex.publish(
|
||||
aio_pika.Message(
|
||||
body=json.dumps(payload).encode(),
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
),
|
||||
routing_key=retry_queue_name(tier),
|
||||
)
|
||||
logger.info("Scheduled retry in %ss for job %s (attempt %s)", tier, payload.get("job_id"), payload.get("attempt"))
|
||||
|
||||
|
||||
async def publish_main(payload: dict):
|
||||
channel = await get_channel()
|
||||
await channel.default_exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=json.dumps(payload).encode(),
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
),
|
||||
routing_key=MAIN_QUEUE,
|
||||
)
|
||||
|
||||
|
||||
async def relay_outbox(db: AsyncSession):
|
||||
"""Забирает непубликованные задачи из outbox и отправляет их в RabbitMQ"""
|
||||
global _channel
|
||||
if _channel is None:
|
||||
await get_channel()
|
||||
|
||||
# Берём до 10 непубликованных записей
|
||||
"""Transactional outbox: publish + confirm, только потом published_at."""
|
||||
stmt = select(JobOutbox).where(JobOutbox.published_at.is_(None)).limit(10)
|
||||
result = await db.execute(stmt)
|
||||
outbox_records = result.scalars().all()
|
||||
|
||||
for record in outbox_records:
|
||||
for record in result.scalars().all():
|
||||
try:
|
||||
payload = json.dumps(record.payload)
|
||||
message = aio_pika.Message(
|
||||
body=payload.encode(),
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT
|
||||
)
|
||||
await _channel.default_exchange.publish(
|
||||
message,
|
||||
routing_key="attachment.process.v1"
|
||||
)
|
||||
|
||||
# Помечаем как опубликованное
|
||||
await publish_main(record.payload) # ждём publisher confirm
|
||||
record.published_at = datetime.now(timezone.utc)
|
||||
await db.commit()
|
||||
logger.info(f"Relayed job {record.job_id} to broker")
|
||||
logger.info("Relayed job %s to broker", record.job_id)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to relay job {record.job_id}: {e}")
|
||||
await db.rollback()
|
||||
logger.error("Failed to relay job %s: %s", record.job_id, e)
|
||||
await db.rollback()
|
||||
|
||||
|
||||
async def run_relay():
|
||||
from app.database import AsyncSessionLocal
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
await relay_outbox(db)
|
||||
await asyncio.sleep(1)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Relay loop error: %s", e)
|
||||
await asyncio.sleep(5)
|
||||
+1
-12
@@ -9,8 +9,7 @@ from fastapi.responses import JSONResponse
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
from app.routers import auth, chats, attachments
|
||||
from app.database import AsyncSessionLocal
|
||||
from app.broker import relay_outbox
|
||||
from app.broker import run_relay
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -22,16 +21,6 @@ async def lifespan(app: FastAPI):
|
||||
# Останавливаем при завершении
|
||||
task.cancel()
|
||||
|
||||
async def run_relay():
|
||||
while True:
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
await relay_outbox(db)
|
||||
await asyncio.sleep(1) # Проверяем outbox каждую секунду
|
||||
except Exception as e:
|
||||
logger.error(f"Relay loop crashed: {e}")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
|
||||
app.include_router(auth.router)
|
||||
|
||||
+299
-170
@@ -1,218 +1,347 @@
|
||||
import os
|
||||
import sys
|
||||
import io
|
||||
import json
|
||||
import uuid
|
||||
import hashlib
|
||||
import asyncio
|
||||
import logging
|
||||
import tempfile
|
||||
import json
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
import aio_pika
|
||||
from minio import Minio
|
||||
from PIL import Image
|
||||
import io
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
|
||||
from app.models import Attachment, AttachmentJob, AttachmentVariant
|
||||
from app.broker import (
|
||||
get_channel, declare_topology, publish_retry, choose_retry_tier, MAIN_QUEUE,
|
||||
)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql+asyncpg://messenger:secure_db_password_123@postgres:5432/messenger_db")
|
||||
BROKER_URL = os.getenv("BROKER_URL", "amqp://messenger:mq_secret_password@rabbitmq:5672/")
|
||||
DATABASE_URL = os.getenv("DATABASE_URL")
|
||||
BROKER_URL = os.getenv("BROKER_URL")
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333").replace("http://", "").replace("https://", "")
|
||||
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "admin")
|
||||
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "admin")
|
||||
S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "messenger-attachments")
|
||||
|
||||
WORKER_ID = str(uuid.uuid4())[:8]
|
||||
JOB_LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "30"))
|
||||
JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "2"))
|
||||
MAX_PIXELS = 40_000_000
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
|
||||
engine = create_async_engine(DATABASE_URL, echo=False)
|
||||
AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
|
||||
s3_client = Minio(S3_ENDPOINT, access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=False)
|
||||
|
||||
async def process_image(body: bytes, att_id: str, chat_id: str):
|
||||
img = Image.open(io.BytesIO(body)).convert("RGB")
|
||||
width, height = img.size
|
||||
variants_to_save = []
|
||||
|
||||
# 1. Thumbnail (вписан в 256x256)
|
||||
|
||||
class PermanentError(Exception):
|
||||
"""Повтор бессмысленен: битый файл, слишком большое изображение, нет staging."""
|
||||
def __init__(self, code, message=""):
|
||||
self.code = code
|
||||
super().__init__(f"{code}: {message}")
|
||||
|
||||
|
||||
class TemporaryError(Exception):
|
||||
"""Повтор имеет смысл: S3 недоступен, таймаут, сеть."""
|
||||
def __init__(self, code, message=""):
|
||||
self.code = code
|
||||
super().__init__(f"{code}: {message}")
|
||||
|
||||
|
||||
def s3_put(key: str, data: bytes, content_type: str):
|
||||
with tempfile.NamedTemporaryFile(delete=False) as tmp:
|
||||
tmp.write(data)
|
||||
path = tmp.name
|
||||
try:
|
||||
s3_client.fput_object(S3_BUCKET_NAME, key, path, content_type=content_type)
|
||||
except Exception as e:
|
||||
raise TemporaryError("storage_unavailable", str(e))
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
def build_variants(body: bytes, att_id: str, chat_id: str):
|
||||
"""CPU-bound часть. Только worker, никогда API."""
|
||||
if len(body) > MAX_BYTES:
|
||||
raise PermanentError("payload_too_large")
|
||||
try:
|
||||
img = Image.open(io.BytesIO(body))
|
||||
img.load()
|
||||
img = img.convert("RGB") # EXIF/ICC не переносим — энкодер получает только пиксели
|
||||
except Exception as e:
|
||||
raise PermanentError("invalid_image", str(e))
|
||||
|
||||
w, h = img.size
|
||||
if w * h > MAX_PIXELS:
|
||||
raise PermanentError("image_too_large")
|
||||
|
||||
out = []
|
||||
|
||||
thumb = img.copy()
|
||||
thumb.thumbnail((256, 256), Image.Resampling.LANCZOS)
|
||||
thumb_bytes = io.BytesIO()
|
||||
thumb.save(thumb_bytes, format="JPEG", quality=85)
|
||||
thumb_data = thumb_bytes.getvalue()
|
||||
thumb_key = f"variants/{chat_id}/{att_id}/thumbnail.jpg"
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp:
|
||||
tmp.write(thumb_data)
|
||||
s3_client.fput_object(S3_BUCKET_NAME, thumb_key, tmp.name, content_type="image/jpeg")
|
||||
os.unlink(tmp.name)
|
||||
variants_to_save.append({
|
||||
"variant_name": "thumbnail", "s3_key": thumb_key, "content_type": "image/jpeg",
|
||||
"size_bytes": len(thumb_data), "width": thumb.width, "height": thumb.height,
|
||||
"sha256": hashlib.sha256(thumb_data).hexdigest()
|
||||
})
|
||||
buf = io.BytesIO(); thumb.save(buf, format="JPEG", quality=85)
|
||||
data = buf.getvalue()
|
||||
key = f"variants/{chat_id}/{att_id}/thumbnail.jpg"
|
||||
s3_put(key, data, "image/jpeg")
|
||||
out.append(dict(variant_name="thumbnail", s3_key=key, content_type="image/jpeg",
|
||||
size_bytes=len(data), width=thumb.width, height=thumb.height,
|
||||
sha256=hashlib.sha256(data).hexdigest()))
|
||||
|
||||
# 2. Crop (центральный квадрат 128x128)
|
||||
min_dim = min(width, height)
|
||||
left = (width - min_dim) // 2
|
||||
top = (height - min_dim) // 2
|
||||
crop = img.crop((left, top, left + min_dim, top + min_dim)).resize((128, 128), Image.Resampling.LANCZOS)
|
||||
crop_bytes = io.BytesIO()
|
||||
crop.save(crop_bytes, format="JPEG", quality=85)
|
||||
crop_data = crop_bytes.getvalue()
|
||||
crop_key = f"variants/{chat_id}/{att_id}/crop.jpg"
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as tmp:
|
||||
tmp.write(crop_data)
|
||||
s3_client.fput_object(S3_BUCKET_NAME, crop_key, tmp.name, content_type="image/jpeg")
|
||||
os.unlink(tmp.name)
|
||||
variants_to_save.append({
|
||||
"variant_name": "crop", "s3_key": crop_key, "content_type": "image/jpeg",
|
||||
"size_bytes": len(crop_data), "width": 128, "height": 128,
|
||||
"sha256": hashlib.sha256(crop_data).hexdigest()
|
||||
})
|
||||
side = min(w, h)
|
||||
left, top = (w - side) // 2, (h - side) // 2
|
||||
crop = img.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS)
|
||||
buf = io.BytesIO(); crop.save(buf, format="JPEG", quality=85)
|
||||
data = buf.getvalue()
|
||||
key = f"variants/{chat_id}/{att_id}/crop.jpg"
|
||||
s3_put(key, data, "image/jpeg")
|
||||
out.append(dict(variant_name="crop", s3_key=key, content_type="image/jpeg",
|
||||
size_bytes=len(data), width=128, height=128,
|
||||
sha256=hashlib.sha256(data).hexdigest()))
|
||||
|
||||
# 3. Optimized WebP (вписан в 1280x1280)
|
||||
opt = img.copy()
|
||||
opt.thumbnail((1280, 1280), Image.Resampling.LANCZOS)
|
||||
opt_bytes = io.BytesIO()
|
||||
opt.save(opt_bytes, format="WEBP", quality=80)
|
||||
opt_data = opt_bytes.getvalue()
|
||||
opt_key = f"variants/{chat_id}/{att_id}/optimized.webp"
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".webp") as tmp:
|
||||
tmp.write(opt_data)
|
||||
s3_client.fput_object(S3_BUCKET_NAME, opt_key, tmp.name, content_type="image/webp")
|
||||
os.unlink(tmp.name)
|
||||
variants_to_save.append({
|
||||
"variant_name": "optimized", "s3_key": opt_key, "content_type": "image/webp",
|
||||
"size_bytes": len(opt_data), "width": opt.width, "height": opt.height,
|
||||
"sha256": hashlib.sha256(opt_data).hexdigest()
|
||||
})
|
||||
buf = io.BytesIO(); opt.save(buf, format="WEBP", quality=80, method=4)
|
||||
data = buf.getvalue()
|
||||
key = f"variants/{chat_id}/{att_id}/optimized.webp"
|
||||
s3_put(key, data, "image/webp")
|
||||
out.append(dict(variant_name="optimized", s3_key=key, content_type="image/webp",
|
||||
size_bytes=len(data), width=opt.width, height=opt.height,
|
||||
sha256=hashlib.sha256(data).hexdigest()))
|
||||
|
||||
# 4. Сохраняем оригинал в конечную папку
|
||||
ext = "png" if b"png" in body[:8] else "jpeg"
|
||||
ext = "png" if body[:8].startswith(b"\x89PNG") else "jpeg"
|
||||
final_key = f"attachments/{chat_id}/{att_id}.{ext}"
|
||||
content_type = "image/png" if ext == "png" else "image/jpeg"
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=f".{ext}") as tmp:
|
||||
tmp.write(body)
|
||||
s3_client.fput_object(S3_BUCKET_NAME, final_key, tmp.name, content_type=content_type)
|
||||
os.unlink(tmp.name)
|
||||
s3_put(final_key, body, "image/png" if ext == "png" else "image/jpeg")
|
||||
return final_key, out
|
||||
|
||||
|
||||
async def handle_job(job_id: str, att_id: str, attempt: int):
|
||||
"""Возвращает (outcome, error_code). outcome: done|noop|permanent|temporary"""
|
||||
async with AsyncSessionLocal() as db:
|
||||
job = (await db.execute(
|
||||
select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
|
||||
)).scalar_one_or_none()
|
||||
|
||||
if job is None:
|
||||
return "noop", None
|
||||
if job.status in ("ready", "failed"):
|
||||
logger.info("Job %s already %s -> no-op ack", job_id, job.status)
|
||||
return "noop", None
|
||||
if attempt < job.attempt:
|
||||
logger.info("Job %s stale delivery (msg attempt=%s < db attempt=%s) -> no-op", job_id, attempt, job.attempt)
|
||||
return "noop", None
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
if job.status == "processing" and job.lease_until and job.lease_until > now and not job.lease_owner == WORKER_ID:
|
||||
logger.info("Job %s leased by %s until %s -> no-op", job_id, job.lease_owner, job.lease_until)
|
||||
return "noop", None
|
||||
|
||||
job.status = "processing"
|
||||
job.attempt = attempt
|
||||
job.lease_until = now + timedelta(seconds=JOB_LEASE_SECONDS)
|
||||
job.lease_owner = WORKER_ID
|
||||
|
||||
att = (await db.execute(select(Attachment).where(Attachment.id == att_id))).scalar_one_or_none()
|
||||
if att is None:
|
||||
return "noop", None
|
||||
att.status = "processing"
|
||||
await db.commit()
|
||||
|
||||
staging_key = att.staging_key
|
||||
if not staging_key:
|
||||
return "permanent", "missing_staging"
|
||||
|
||||
try:
|
||||
resp = s3_client.get_object(S3_BUCKET_NAME, staging_key)
|
||||
try:
|
||||
body = resp.read()
|
||||
finally:
|
||||
resp.close(); resp.release_conn()
|
||||
except Exception as e:
|
||||
return "temporary", "storage_unavailable"
|
||||
|
||||
if hashlib.sha256(body).hexdigest() != att.sha256:
|
||||
return "permanent", "checksum_mismatch"
|
||||
|
||||
try:
|
||||
final_key, variants = await asyncio.wait_for(
|
||||
asyncio.to_thread(build_variants, body, str(att.id), str(att.chat_id)),
|
||||
timeout=int(os.getenv("JOB_TIMEOUT_SECONDS", "60")),
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
return "temporary", "processing_timeout"
|
||||
except PermanentError as e:
|
||||
return "permanent", e.code
|
||||
except TemporaryError as e:
|
||||
return "temporary", e.code
|
||||
except Exception as e:
|
||||
logger.exception("Unexpected processing failure")
|
||||
return "temporary", "processing_error"
|
||||
|
||||
# Идемпотентная фиксация: устаревшая попытка не перезапишет результат
|
||||
res = await db.execute(
|
||||
update(AttachmentJob)
|
||||
.where(AttachmentJob.job_id == job_id,
|
||||
AttachmentJob.attempt == attempt,
|
||||
AttachmentJob.status == "processing")
|
||||
.values(status="ready", lease_until=None)
|
||||
)
|
||||
if res.rowcount == 0:
|
||||
logger.info("Job %s attempt %s lost ownership -> discard result", job_id, attempt)
|
||||
return "noop", None
|
||||
|
||||
for v in variants:
|
||||
await db.execute(
|
||||
pg_insert(AttachmentVariant)
|
||||
.values(attachment_id=att.id, **v)
|
||||
.on_conflict_do_nothing(index_elements=["attachment_id", "variant_name"])
|
||||
)
|
||||
|
||||
await db.execute(
|
||||
update(Attachment).where(Attachment.id == att.id)
|
||||
.values(status="ready", s3_key=final_key, staging_key=None, error_code=None)
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
# Очистка staging — только после безопасной фиксации
|
||||
try:
|
||||
s3_client.remove_object(S3_BUCKET_NAME, staging_key)
|
||||
except Exception as e:
|
||||
logger.warning("Staging cleanup failed for %s: %s", staging_key, e)
|
||||
|
||||
logger.info("Job %s completed successfully (attempt %s)", job_id, attempt)
|
||||
return "done", None
|
||||
|
||||
|
||||
async def mark_failed(job_id: str, code: str):
|
||||
async with AsyncSessionLocal() as db:
|
||||
job = (await db.execute(select(AttachmentJob).where(AttachmentJob.job_id == job_id))).scalar_one_or_none()
|
||||
if job is None or job.status == "ready":
|
||||
return
|
||||
await db.execute(update(AttachmentJob).where(AttachmentJob.job_id == job_id)
|
||||
.values(status="failed", lease_until=None))
|
||||
await db.execute(update(Attachment).where(Attachment.id == job.attachment_id)
|
||||
.values(status="failed", error_code=code))
|
||||
await db.commit()
|
||||
logger.warning("Job %s -> failed (%s)", job_id, code)
|
||||
|
||||
|
||||
async def schedule_retry(job_id: str, payload: dict) -> int:
|
||||
"""Возвращает выбранный tier; -1 если бюджет попыток исчерпан."""
|
||||
async with AsyncSessionLocal() as db:
|
||||
job = (await db.execute(
|
||||
select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
|
||||
)).scalar_one_or_none()
|
||||
if job is None or job.status == "ready":
|
||||
return -1
|
||||
if job.attempt >= JOB_MAX_ATTEMPTS:
|
||||
return -1
|
||||
next_attempt = job.attempt + 1
|
||||
tier = choose_retry_tier(job.attempt)
|
||||
job.status = "queued"
|
||||
job.attempt = next_attempt
|
||||
job.lease_until = None
|
||||
job.lease_owner = None
|
||||
await db.execute(update(Attachment).where(Attachment.id == job.attachment_id).values(status="queued"))
|
||||
await db.commit()
|
||||
|
||||
await publish_retry({**payload, "attempt": next_attempt}, tier)
|
||||
return tier
|
||||
|
||||
return final_key, variants_to_save
|
||||
|
||||
async def process_message(message: aio_pika.IncomingMessage):
|
||||
async with message.process(requeue=False):
|
||||
# Ручные подтверждения: без message.process(), чтобы не было двойного ack
|
||||
try:
|
||||
data = json.loads(message.body)
|
||||
job_id, att_id, attempt = data["job_id"], data["attachment_id"], int(data["attempt"])
|
||||
uuid.UUID(job_id); uuid.UUID(att_id)
|
||||
except Exception:
|
||||
logger.error("Malformed message -> DLQ")
|
||||
await message.nack(requeue=False)
|
||||
return
|
||||
|
||||
try:
|
||||
outcome, code = await handle_job(job_id, att_id, attempt)
|
||||
except Exception:
|
||||
logger.exception("Handler crashed for job %s", job_id)
|
||||
outcome, code = "temporary", "internal_error"
|
||||
|
||||
try:
|
||||
if outcome in ("done", "noop"):
|
||||
await message.ack()
|
||||
return
|
||||
|
||||
if outcome == "permanent":
|
||||
await mark_failed(job_id, code)
|
||||
await message.nack(requeue=False) # -> DLX -> DLQ
|
||||
return
|
||||
|
||||
# temporary
|
||||
tier = await schedule_retry(job_id, data)
|
||||
if tier < 0:
|
||||
await mark_failed(job_id, "retry_exhausted")
|
||||
await message.nack(requeue=False) # -> DLQ
|
||||
else:
|
||||
logger.info("Job %s temporary error (%s), retry in ~%ss", job_id, code, tier)
|
||||
await message.ack() # копия уже в отложенной очереди
|
||||
except Exception:
|
||||
logger.exception("Failed to finalize delivery for job %s -> requeue", job_id)
|
||||
await message.nack(requeue=True)
|
||||
|
||||
|
||||
async def sweeper():
|
||||
"""Зависшие processing (lease истёк) возвращаем в работу."""
|
||||
while True:
|
||||
try:
|
||||
payload = message.body.decode()
|
||||
data = json.loads(payload)
|
||||
job_id = data["job_id"]
|
||||
att_id = data["attachment_id"]
|
||||
attempt = data["attempt"]
|
||||
|
||||
logger.info(f"Processing job {job_id} for attachment {att_id}, attempt {attempt}")
|
||||
|
||||
await asyncio.sleep(5)
|
||||
async with AsyncSessionLocal() as db:
|
||||
from app.models import AttachmentJob, Attachment, AttachmentVariant
|
||||
|
||||
# 1. Блокируем задачу для обработки (Claim)
|
||||
stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
|
||||
result = await db.execute(stmt)
|
||||
job = result.scalar_one_or_none()
|
||||
|
||||
if not job:
|
||||
logger.warning(f"Job {job_id} not found")
|
||||
return
|
||||
|
||||
if job.status in ("ready", "failed"):
|
||||
logger.info(f"Job {job_id} already {job.status}, skipping")
|
||||
return
|
||||
|
||||
if job.status == "processing" and job.lease_until and job.lease_until > datetime.now(timezone.utc):
|
||||
logger.info(f"Job {job_id} is already being processed by another worker")
|
||||
return
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
job.status = "processing"
|
||||
job.attempt = attempt
|
||||
job.lease_until = now + timedelta(seconds=JOB_LEASE_SECONDS)
|
||||
job.lease_owner = WORKER_ID
|
||||
await db.commit()
|
||||
|
||||
# 2. Получаем метаданные вложения
|
||||
att_stmt = select(Attachment).where(Attachment.id == att_id)
|
||||
att_result = await db.execute(att_stmt)
|
||||
attachment = att_result.scalar_one_or_none()
|
||||
|
||||
if not attachment or not attachment.staging_key:
|
||||
job.status = "failed"
|
||||
job.error_code = "missing_staging"
|
||||
rows = (await db.execute(
|
||||
select(AttachmentJob).where(
|
||||
AttachmentJob.status == "processing",
|
||||
AttachmentJob.lease_until.isnot(None),
|
||||
AttachmentJob.lease_until < now,
|
||||
).limit(20)
|
||||
)).scalars().all()
|
||||
for job in rows:
|
||||
if job.attempt >= JOB_MAX_ATTEMPTS:
|
||||
continue
|
||||
payload = {"job_id": str(job.job_id), "attachment_id": str(job.attachment_id),
|
||||
"attempt": job.attempt + 1}
|
||||
job.status = "queued"
|
||||
job.attempt += 1
|
||||
job.lease_until = None
|
||||
job.lease_owner = None
|
||||
await db.execute(update(Attachment).where(Attachment.id == job.attachment_id)
|
||||
.values(status="queued"))
|
||||
await db.commit()
|
||||
return
|
||||
|
||||
staging_key_to_delete = attachment.staging_key # Сохраняем ключ до обнуления
|
||||
|
||||
# 3. Скачиваем из staging
|
||||
response = s3_client.get_object(S3_BUCKET_NAME, staging_key_to_delete)
|
||||
body = response.read()
|
||||
response.close()
|
||||
response.release_conn()
|
||||
|
||||
# 4. Обрабатываем изображение
|
||||
final_key, variants_data = await process_image(body, str(attachment.id), str(attachment.chat_id))
|
||||
|
||||
# 5. Фиксируем результат в БД
|
||||
attachment.status = "ready"
|
||||
attachment.s3_key = final_key
|
||||
attachment.staging_key = None # Очищаем staging
|
||||
|
||||
job.status = "ready"
|
||||
|
||||
for v_data in variants_data:
|
||||
variant = AttachmentVariant(
|
||||
attachment_id=attachment.id,
|
||||
**v_data
|
||||
)
|
||||
db.add(variant)
|
||||
|
||||
await db.commit()
|
||||
logger.info(f"Job {job_id} completed successfully")
|
||||
|
||||
# 6. Удаляем staging из S3
|
||||
s3_client.remove_object(S3_BUCKET_NAME, staging_key_to_delete)
|
||||
|
||||
await publish_retry(payload, choose_retry_tier(1))
|
||||
logger.warning("Sweeper: lease expired for job %s -> retry attempt %s", job.job_id, job.attempt)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing message: {e}")
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
from app.models import AttachmentJob
|
||||
stmt = select(AttachmentJob).where(AttachmentJob.job_id == job_id).with_for_update()
|
||||
result = await db.execute(stmt)
|
||||
job = result.scalar_one_or_none()
|
||||
if job:
|
||||
if job.attempt >= JOB_MAX_ATTEMPTS:
|
||||
job.status = "failed"
|
||||
job.error_code = "retry_exhausted"
|
||||
else:
|
||||
job.status = "queued"
|
||||
job.attempt += 1
|
||||
job.lease_until = None
|
||||
await db.commit()
|
||||
except Exception as db_err:
|
||||
logger.error(f"Failed to update job status after error: {db_err}")
|
||||
logger.error("Sweeper error: %s", e)
|
||||
|
||||
|
||||
async def main():
|
||||
logger.info("Starting worker...")
|
||||
connection = await aio_pika.connect_robust(BROKER_URL)
|
||||
channel = await connection.channel()
|
||||
|
||||
queue = await channel.declare_queue("attachment.process.v1", durable=True)
|
||||
|
||||
logger.info("Waiting for messages...")
|
||||
logger.info("Starting worker %s (concurrency/prefetch=%s)", WORKER_ID, WORKER_CONCURRENCY)
|
||||
channel = await get_channel()
|
||||
await declare_topology(channel)
|
||||
await channel.set_qos(prefetch_count=WORKER_CONCURRENCY)
|
||||
queue = await channel.get_queue(MAIN_QUEUE)
|
||||
await queue.consume(process_message)
|
||||
|
||||
asyncio.create_task(sweeper())
|
||||
logger.info("Waiting for messages on %s ...", MAIN_QUEUE)
|
||||
await asyncio.Future()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user