Files
mtusi2026lab5/app/worker.py
T

437 lines
18 KiB
Python

from __future__ import annotations
import asyncio
import io
import json
import logging
import random
import signal
import uuid
from typing import Any
import aio_pika
import asyncpg
import boto3
from botocore.config import Config
from PIL import UnidentifiedImageError
from app.broker import (
DL_ROUTING_KEY,
DLX_NAME,
EXCHANGE_NAME,
QUEUE_NAME,
ROUTING_KEY,
setup_broker_topology,
)
from app.config import (
BROKER_URL,
DATABASE_URL,
MAX_JOB_RETRIES,
S3_ACCESS_KEY_ID,
S3_BUCKET,
S3_ENDPOINT,
S3_REGION,
S3_SECRET_ACCESS_KEY,
WORKER_CONCURRENCY,
WORKER_LEASE_SECONDS,
)
from app.imaging import process_image_variants
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("messenger.worker")
class AttachmentWorker:
def __init__(self) -> None:
self.worker_id = f"worker-{uuid.uuid4().hex[:8]}"
self.pool: asyncpg.Pool | None = None
self.s3_client: Any = None
self._connection: aio_pika.abc.AbstractRobustConnection | None = None
self._channel: aio_pika.abc.AbstractChannel | None = None
self._running = False
self._watchdog_task: asyncio.Task[None] | None = None
async def init_resources(self) -> None:
logger.info("Initializing worker resources (%s)...", self.worker_id)
# 1. PostgreSQL pool
self.pool = await asyncpg.create_pool(
dsn=DATABASE_URL,
min_size=2,
max_size=WORKER_CONCURRENCY + 2,
command_timeout=10.0,
)
# 2. S3 client
s3_cfg = Config(
signature_version="s3v4",
connect_timeout=3,
read_timeout=10,
retries={"max_attempts": 3, "mode": "standard"},
)
self.s3_client = boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
region_name=S3_REGION,
aws_access_key_id=S3_ACCESS_KEY_ID,
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
config=s3_cfg,
)
async def _lease_watchdog(self) -> None:
while self._running:
try:
await asyncio.sleep(15.0)
if not self.pool:
continue
async with self.pool.acquire() as conn:
rows = await conn.fetch(
"""
UPDATE attachments
SET status = 'queued',
processing_started_at = NULL,
processing_worker_id = NULL,
updated_at = NOW()
WHERE status = 'processing'
AND processing_started_at < NOW() - ($1 || ' seconds')::interval
RETURNING id
""",
str(WORKER_LEASE_SECONDS),
)
for r in rows:
logger.warning(
"Lease expired for attachment %s, reverting to queued and republishing", r["id"]
)
if self._channel and not self._channel.is_closed:
try:
exchange = await self._channel.get_exchange(EXCHANGE_NAME)
payload = {
"schema_version": 1,
"job_id": str(uuid.uuid4()),
"type": "attachment.process.v1",
"attachment_id": str(r["id"]),
"pipeline_version": 1,
"attempt": 1,
}
msg = aio_pika.Message(
body=json.dumps(payload).encode("utf-8"),
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
content_type="application/json",
)
await exchange.publish(msg, routing_key=ROUTING_KEY)
except Exception as pub_err:
logger.error("Failed republish on lease expiration for %s: %s", r["id"], pub_err)
except asyncio.CancelledError:
break
except Exception as exc:
logger.error("Error in lease watchdog: %s", exc)
async def process_job_message(self, message: aio_pika.abc.AbstractIncomingMessage) -> None:
"""Обрабатывает одно задание на генерацию вариантов изображения."""
async with message.process(requeue=False):
try:
payload = json.loads(message.body.decode("utf-8"))
except Exception as parse_err:
logger.error("Failed to parse job JSON: %s. Acking.", parse_err)
return
att_id_str = payload.get("attachment_id")
if not att_id_str:
logger.error("Job payload missing attachment_id: %s", payload)
return
try:
att_id = uuid.UUID(att_id_str)
except ValueError:
logger.error("Invalid attachment_id UUID: %s", att_id_str)
return
attempt = int(payload.get("attempt", 1))
if not self.pool:
logger.error("DB pool not initialized")
return
async with self.pool.acquire() as conn:
row = await conn.fetchrow(
"""
UPDATE attachments
SET status = 'processing',
processing_started_at = NOW(),
processing_worker_id = $1,
updated_at = NOW()
WHERE id = $2 AND status IN ('queued', 'failed')
RETURNING id, chat_id, filename, content_type, size_bytes, sha256, s3_key, status
""",
self.worker_id,
att_id,
)
if not row:
current_status = await conn.fetchval(
"SELECT status FROM attachments WHERE id = $1", att_id
)
logger.info(
"Attachment %s is in state '%s', skipping processing (idempotent no-op).",
att_id,
current_status,
)
return
try:
async with asyncio.timeout(30.0):
staging_key = row["s3_key"]
try:
s3_resp = await asyncio.to_thread(
self.s3_client.get_object,
Bucket=S3_BUCKET,
Key=staging_key,
)
raw_bytes = s3_resp["Body"].read()
except Exception as s3_err:
logger.error("Failed to read staging S3 key %s: %s", staging_key, s3_err)
await self._handle_retryable_error(payload, attempt, str(s3_err))
return
# 3. Генерируем варианты через Pillow
try:
variants = await asyncio.to_thread(process_image_variants, raw_bytes)
except (UnidentifiedImageError, ValueError) as img_err:
logger.warning(
"Permanent image processing error for %s: %s", att_id, img_err
)
await self._fail_attachment(att_id, "invalid_image")
await self._send_to_dlq(payload, f"permanent_image_error: {img_err}")
return
except Exception as unk_err:
logger.error(
"Unexpected processing error for %s: %s", att_id, unk_err
)
await self._handle_retryable_error(payload, attempt, str(unk_err))
return
# 4. Сохраняем оригинал и варианты в постоянные S3-ключи
chat_id = str(row["chat_id"])
final_original_key = f"chats/{chat_id}/{att_id}/original"
try:
# Оригинал
await asyncio.to_thread(
self.s3_client.put_object,
Bucket=S3_BUCKET,
Key=final_original_key,
Body=raw_bytes,
ContentType=row["content_type"],
)
# Варианты
for var in variants:
var_key = f"chats/{chat_id}/{att_id}/{var['name']}"
await asyncio.to_thread(
self.s3_client.put_object,
Bucket=S3_BUCKET,
Key=var_key,
Body=var["data"],
ContentType=var["content_type"],
)
except Exception as upload_err:
logger.error("Failed uploading variants to S3 for %s: %s", att_id, upload_err)
await self._handle_retryable_error(payload, attempt, str(upload_err))
return
# 5. Атомарно обновляем PostgreSQL в 'ready'
variants_metadata = [
{
"name": v["name"],
"content_type": v["content_type"],
"size_bytes": v["size_bytes"],
"width": v["width"],
"height": v["height"],
"sha256": v["sha256"],
}
for v in variants
]
async with self.pool.acquire() as conn:
await conn.execute(
"""
UPDATE attachments
SET status = 'ready',
s3_key = $1,
variants = $2::jsonb,
error_code = NULL,
processing_started_at = NULL,
processing_worker_id = NULL,
updated_at = NOW()
WHERE id = $3
""",
final_original_key,
json.dumps(variants_metadata),
att_id,
)
# 6. Очистка staging-файла в MinIO
if staging_key != final_original_key:
try:
await asyncio.to_thread(
self.s3_client.delete_object,
Bucket=S3_BUCKET,
Key=staging_key,
)
except Exception as del_err:
logger.warning("Failed to delete staging key %s: %s", staging_key, del_err)
logger.info("Attachment %s successfully processed into ready with 3 variants.", att_id)
except TimeoutError:
logger.error("Job processing timed out (>30s) for %s. Aborting hung worker task.", att_id)
await self._handle_retryable_error(payload, attempt, "job_deadline_exceeded_30s")
return
async def _fail_attachment(self, att_id: uuid.UUID, error_code: str) -> None:
"""Переводит вложение в статус failed с безопасным машинным кодом."""
if not self.pool:
return
async with self.pool.acquire() as conn:
await conn.execute(
"""
UPDATE attachments
SET status = 'failed',
error_code = $1,
processing_started_at = NULL,
processing_worker_id = NULL,
updated_at = NOW()
WHERE id = $2
""",
error_code,
att_id,
)
async def _send_to_dlq(self, payload: dict[str, Any], reason: str) -> None:
try:
if self._channel and not self._channel.is_closed:
dlx = await self._channel.get_exchange(DLX_NAME)
dlq_payload = dict(payload, error="permanent_or_exhausted", last_reason=reason)
dlq_msg = aio_pika.Message(
body=json.dumps(dlq_payload).encode("utf-8"),
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
content_type="application/json",
)
await dlx.publish(dlq_msg, routing_key=DL_ROUTING_KEY)
except Exception as dlx_err:
logger.error("Failed publishing to DLQ: %s", dlx_err)
async def _handle_retryable_error(self, payload: dict[str, Any], attempt: int, reason: str) -> None:
att_id_str = payload.get("attachment_id")
att_id = uuid.UUID(att_id_str)
if attempt < MAX_JOB_RETRIES:
next_attempt = attempt + 1
delay = 0.5 * (2 ** attempt) + random.uniform(0.1, 0.5)
logger.warning(
"Retryable error for %s (attempt %d/%d). Retrying in %.2fs: %s",
att_id, attempt, MAX_JOB_RETRIES, delay, reason
)
if self.pool:
async with self.pool.acquire() as conn:
await conn.execute(
"""
UPDATE attachments
SET status = 'queued',
processing_started_at = NULL,
processing_worker_id = NULL,
updated_at = NOW()
WHERE id = $1
""",
att_id,
)
async def _delayed_republish() -> None:
await asyncio.sleep(delay)
try:
if self._channel and not self._channel.is_closed:
exchange = await self._channel.get_exchange(EXCHANGE_NAME)
new_payload = dict(payload, attempt=next_attempt)
msg = aio_pika.Message(
body=json.dumps(new_payload).encode("utf-8"),
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
content_type="application/json",
)
await exchange.publish(msg, routing_key=ROUTING_KEY)
except Exception as pub_err:
logger.error("Failed delayed republish for %s: %s", att_id, pub_err)
asyncio.create_task(_delayed_republish())
else:
logger.error(
"Max retries (%d) exhausted for attachment %s. Moving to DLQ.",
MAX_JOB_RETRIES, att_id
)
await self._fail_attachment(att_id, "retry_exhausted")
await self._send_to_dlq(payload, f"retry_exhausted: {reason}")
async def start(self) -> None:
self._running = True
await self.init_resources()
# Подключение к RabbitMQ с автоматическим reconnect
logger.info("Connecting to RabbitMQ at %s...", BROKER_URL.split("@")[-1])
while self._running:
try:
self._connection = await aio_pika.connect_robust(BROKER_URL)
break
except Exception as conn_err:
logger.warning("RabbitMQ not ready yet (%s). Retrying in 2s...", conn_err)
await asyncio.sleep(2.0)
assert self._connection is not None
self._channel = await self._connection.channel()
await self._channel.set_qos(prefetch_count=WORKER_CONCURRENCY)
_, queue = await setup_broker_topology(self._channel)
self._watchdog_task = asyncio.create_task(self._lease_watchdog())
logger.info(
"Worker %s started consuming queue '%s' (prefetch=%d)...",
self.worker_id, QUEUE_NAME, WORKER_CONCURRENCY
)
await queue.consume(self.process_job_message)
async def stop(self) -> None:
logger.info("Stopping worker %s...", self.worker_id)
self._running = False
if self._watchdog_task:
self._watchdog_task.cancel()
if self._channel and not self._channel.is_closed:
await self._channel.close()
if self._connection and not self._connection.is_closed:
await self._connection.close()
if self.pool:
await self.pool.close()
logger.info("Worker stopped.")
async def main() -> None:
worker = AttachmentWorker()
loop = asyncio.get_running_loop()
stop_event = asyncio.Event()
def _sig_handler() -> None:
logger.info("Received termination signal")
stop_event.set()
for sig in (signal.SIGINT, signal.SIGTERM):
try:
loop.add_signal_handler(sig, _sig_handler)
except NotImplementedError:
pass
await worker.start()
await stop_event.wait()
await worker.stop()
if __name__ == "__main__":
asyncio.run(main())