forked from mtusi2026/mtusi2026lab5
437 lines
18 KiB
Python
437 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import io
|
|
import json
|
|
import logging
|
|
import random
|
|
import signal
|
|
import uuid
|
|
from typing import Any
|
|
|
|
import aio_pika
|
|
import asyncpg
|
|
import boto3
|
|
from botocore.config import Config
|
|
from PIL import UnidentifiedImageError
|
|
|
|
from app.broker import (
|
|
DL_ROUTING_KEY,
|
|
DLX_NAME,
|
|
EXCHANGE_NAME,
|
|
QUEUE_NAME,
|
|
ROUTING_KEY,
|
|
setup_broker_topology,
|
|
)
|
|
from app.config import (
|
|
BROKER_URL,
|
|
DATABASE_URL,
|
|
MAX_JOB_RETRIES,
|
|
S3_ACCESS_KEY_ID,
|
|
S3_BUCKET,
|
|
S3_ENDPOINT,
|
|
S3_REGION,
|
|
S3_SECRET_ACCESS_KEY,
|
|
WORKER_CONCURRENCY,
|
|
WORKER_LEASE_SECONDS,
|
|
)
|
|
from app.imaging import process_image_variants
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
|
)
|
|
logger = logging.getLogger("messenger.worker")
|
|
|
|
|
|
class AttachmentWorker:
|
|
def __init__(self) -> None:
|
|
self.worker_id = f"worker-{uuid.uuid4().hex[:8]}"
|
|
self.pool: asyncpg.Pool | None = None
|
|
self.s3_client: Any = None
|
|
self._connection: aio_pika.abc.AbstractRobustConnection | None = None
|
|
self._channel: aio_pika.abc.AbstractChannel | None = None
|
|
self._running = False
|
|
self._watchdog_task: asyncio.Task[None] | None = None
|
|
|
|
async def init_resources(self) -> None:
|
|
logger.info("Initializing worker resources (%s)...", self.worker_id)
|
|
# 1. PostgreSQL pool
|
|
self.pool = await asyncpg.create_pool(
|
|
dsn=DATABASE_URL,
|
|
min_size=2,
|
|
max_size=WORKER_CONCURRENCY + 2,
|
|
command_timeout=10.0,
|
|
)
|
|
|
|
# 2. S3 client
|
|
s3_cfg = Config(
|
|
signature_version="s3v4",
|
|
connect_timeout=3,
|
|
read_timeout=10,
|
|
retries={"max_attempts": 3, "mode": "standard"},
|
|
)
|
|
self.s3_client = boto3.client(
|
|
"s3",
|
|
endpoint_url=S3_ENDPOINT,
|
|
region_name=S3_REGION,
|
|
aws_access_key_id=S3_ACCESS_KEY_ID,
|
|
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
|
config=s3_cfg,
|
|
)
|
|
|
|
async def _lease_watchdog(self) -> None:
|
|
while self._running:
|
|
try:
|
|
await asyncio.sleep(15.0)
|
|
if not self.pool:
|
|
continue
|
|
async with self.pool.acquire() as conn:
|
|
rows = await conn.fetch(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'queued',
|
|
processing_started_at = NULL,
|
|
processing_worker_id = NULL,
|
|
updated_at = NOW()
|
|
WHERE status = 'processing'
|
|
AND processing_started_at < NOW() - ($1 || ' seconds')::interval
|
|
RETURNING id
|
|
""",
|
|
str(WORKER_LEASE_SECONDS),
|
|
)
|
|
for r in rows:
|
|
logger.warning(
|
|
"Lease expired for attachment %s, reverting to queued and republishing", r["id"]
|
|
)
|
|
if self._channel and not self._channel.is_closed:
|
|
try:
|
|
exchange = await self._channel.get_exchange(EXCHANGE_NAME)
|
|
payload = {
|
|
"schema_version": 1,
|
|
"job_id": str(uuid.uuid4()),
|
|
"type": "attachment.process.v1",
|
|
"attachment_id": str(r["id"]),
|
|
"pipeline_version": 1,
|
|
"attempt": 1,
|
|
}
|
|
msg = aio_pika.Message(
|
|
body=json.dumps(payload).encode("utf-8"),
|
|
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
|
content_type="application/json",
|
|
)
|
|
await exchange.publish(msg, routing_key=ROUTING_KEY)
|
|
except Exception as pub_err:
|
|
logger.error("Failed republish on lease expiration for %s: %s", r["id"], pub_err)
|
|
except asyncio.CancelledError:
|
|
break
|
|
except Exception as exc:
|
|
logger.error("Error in lease watchdog: %s", exc)
|
|
|
|
async def process_job_message(self, message: aio_pika.abc.AbstractIncomingMessage) -> None:
|
|
"""Обрабатывает одно задание на генерацию вариантов изображения."""
|
|
async with message.process(requeue=False):
|
|
try:
|
|
payload = json.loads(message.body.decode("utf-8"))
|
|
except Exception as parse_err:
|
|
logger.error("Failed to parse job JSON: %s. Acking.", parse_err)
|
|
return
|
|
|
|
att_id_str = payload.get("attachment_id")
|
|
if not att_id_str:
|
|
logger.error("Job payload missing attachment_id: %s", payload)
|
|
return
|
|
|
|
try:
|
|
att_id = uuid.UUID(att_id_str)
|
|
except ValueError:
|
|
logger.error("Invalid attachment_id UUID: %s", att_id_str)
|
|
return
|
|
|
|
attempt = int(payload.get("attempt", 1))
|
|
|
|
if not self.pool:
|
|
logger.error("DB pool not initialized")
|
|
return
|
|
|
|
async with self.pool.acquire() as conn:
|
|
row = await conn.fetchrow(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'processing',
|
|
processing_started_at = NOW(),
|
|
processing_worker_id = $1,
|
|
updated_at = NOW()
|
|
WHERE id = $2 AND status IN ('queued', 'failed')
|
|
RETURNING id, chat_id, filename, content_type, size_bytes, sha256, s3_key, status
|
|
""",
|
|
self.worker_id,
|
|
att_id,
|
|
)
|
|
if not row:
|
|
current_status = await conn.fetchval(
|
|
"SELECT status FROM attachments WHERE id = $1", att_id
|
|
)
|
|
logger.info(
|
|
"Attachment %s is in state '%s', skipping processing (idempotent no-op).",
|
|
att_id,
|
|
current_status,
|
|
)
|
|
return
|
|
|
|
try:
|
|
async with asyncio.timeout(30.0):
|
|
staging_key = row["s3_key"]
|
|
try:
|
|
s3_resp = await asyncio.to_thread(
|
|
self.s3_client.get_object,
|
|
Bucket=S3_BUCKET,
|
|
Key=staging_key,
|
|
)
|
|
raw_bytes = s3_resp["Body"].read()
|
|
except Exception as s3_err:
|
|
logger.error("Failed to read staging S3 key %s: %s", staging_key, s3_err)
|
|
await self._handle_retryable_error(payload, attempt, str(s3_err))
|
|
return
|
|
|
|
# 3. Генерируем варианты через Pillow
|
|
try:
|
|
variants = await asyncio.to_thread(process_image_variants, raw_bytes)
|
|
except (UnidentifiedImageError, ValueError) as img_err:
|
|
logger.warning(
|
|
"Permanent image processing error for %s: %s", att_id, img_err
|
|
)
|
|
await self._fail_attachment(att_id, "invalid_image")
|
|
await self._send_to_dlq(payload, f"permanent_image_error: {img_err}")
|
|
return
|
|
except Exception as unk_err:
|
|
logger.error(
|
|
"Unexpected processing error for %s: %s", att_id, unk_err
|
|
)
|
|
await self._handle_retryable_error(payload, attempt, str(unk_err))
|
|
return
|
|
|
|
# 4. Сохраняем оригинал и варианты в постоянные S3-ключи
|
|
chat_id = str(row["chat_id"])
|
|
final_original_key = f"chats/{chat_id}/{att_id}/original"
|
|
|
|
try:
|
|
# Оригинал
|
|
await asyncio.to_thread(
|
|
self.s3_client.put_object,
|
|
Bucket=S3_BUCKET,
|
|
Key=final_original_key,
|
|
Body=raw_bytes,
|
|
ContentType=row["content_type"],
|
|
)
|
|
# Варианты
|
|
for var in variants:
|
|
var_key = f"chats/{chat_id}/{att_id}/{var['name']}"
|
|
await asyncio.to_thread(
|
|
self.s3_client.put_object,
|
|
Bucket=S3_BUCKET,
|
|
Key=var_key,
|
|
Body=var["data"],
|
|
ContentType=var["content_type"],
|
|
)
|
|
except Exception as upload_err:
|
|
logger.error("Failed uploading variants to S3 for %s: %s", att_id, upload_err)
|
|
await self._handle_retryable_error(payload, attempt, str(upload_err))
|
|
return
|
|
|
|
# 5. Атомарно обновляем PostgreSQL в 'ready'
|
|
variants_metadata = [
|
|
{
|
|
"name": v["name"],
|
|
"content_type": v["content_type"],
|
|
"size_bytes": v["size_bytes"],
|
|
"width": v["width"],
|
|
"height": v["height"],
|
|
"sha256": v["sha256"],
|
|
}
|
|
for v in variants
|
|
]
|
|
|
|
async with self.pool.acquire() as conn:
|
|
await conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'ready',
|
|
s3_key = $1,
|
|
variants = $2::jsonb,
|
|
error_code = NULL,
|
|
processing_started_at = NULL,
|
|
processing_worker_id = NULL,
|
|
updated_at = NOW()
|
|
WHERE id = $3
|
|
""",
|
|
final_original_key,
|
|
json.dumps(variants_metadata),
|
|
att_id,
|
|
)
|
|
|
|
# 6. Очистка staging-файла в MinIO
|
|
if staging_key != final_original_key:
|
|
try:
|
|
await asyncio.to_thread(
|
|
self.s3_client.delete_object,
|
|
Bucket=S3_BUCKET,
|
|
Key=staging_key,
|
|
)
|
|
except Exception as del_err:
|
|
logger.warning("Failed to delete staging key %s: %s", staging_key, del_err)
|
|
|
|
logger.info("Attachment %s successfully processed into ready with 3 variants.", att_id)
|
|
except TimeoutError:
|
|
logger.error("Job processing timed out (>30s) for %s. Aborting hung worker task.", att_id)
|
|
await self._handle_retryable_error(payload, attempt, "job_deadline_exceeded_30s")
|
|
return
|
|
|
|
async def _fail_attachment(self, att_id: uuid.UUID, error_code: str) -> None:
|
|
"""Переводит вложение в статус failed с безопасным машинным кодом."""
|
|
if not self.pool:
|
|
return
|
|
async with self.pool.acquire() as conn:
|
|
await conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'failed',
|
|
error_code = $1,
|
|
processing_started_at = NULL,
|
|
processing_worker_id = NULL,
|
|
updated_at = NOW()
|
|
WHERE id = $2
|
|
""",
|
|
error_code,
|
|
att_id,
|
|
)
|
|
|
|
async def _send_to_dlq(self, payload: dict[str, Any], reason: str) -> None:
|
|
try:
|
|
if self._channel and not self._channel.is_closed:
|
|
dlx = await self._channel.get_exchange(DLX_NAME)
|
|
dlq_payload = dict(payload, error="permanent_or_exhausted", last_reason=reason)
|
|
dlq_msg = aio_pika.Message(
|
|
body=json.dumps(dlq_payload).encode("utf-8"),
|
|
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
|
content_type="application/json",
|
|
)
|
|
await dlx.publish(dlq_msg, routing_key=DL_ROUTING_KEY)
|
|
except Exception as dlx_err:
|
|
logger.error("Failed publishing to DLQ: %s", dlx_err)
|
|
|
|
async def _handle_retryable_error(self, payload: dict[str, Any], attempt: int, reason: str) -> None:
|
|
att_id_str = payload.get("attachment_id")
|
|
att_id = uuid.UUID(att_id_str)
|
|
|
|
if attempt < MAX_JOB_RETRIES:
|
|
next_attempt = attempt + 1
|
|
delay = 0.5 * (2 ** attempt) + random.uniform(0.1, 0.5)
|
|
logger.warning(
|
|
"Retryable error for %s (attempt %d/%d). Retrying in %.2fs: %s",
|
|
att_id, attempt, MAX_JOB_RETRIES, delay, reason
|
|
)
|
|
if self.pool:
|
|
async with self.pool.acquire() as conn:
|
|
await conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'queued',
|
|
processing_started_at = NULL,
|
|
processing_worker_id = NULL,
|
|
updated_at = NOW()
|
|
WHERE id = $1
|
|
""",
|
|
att_id,
|
|
)
|
|
|
|
async def _delayed_republish() -> None:
|
|
await asyncio.sleep(delay)
|
|
try:
|
|
if self._channel and not self._channel.is_closed:
|
|
exchange = await self._channel.get_exchange(EXCHANGE_NAME)
|
|
new_payload = dict(payload, attempt=next_attempt)
|
|
msg = aio_pika.Message(
|
|
body=json.dumps(new_payload).encode("utf-8"),
|
|
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
|
content_type="application/json",
|
|
)
|
|
await exchange.publish(msg, routing_key=ROUTING_KEY)
|
|
except Exception as pub_err:
|
|
logger.error("Failed delayed republish for %s: %s", att_id, pub_err)
|
|
|
|
asyncio.create_task(_delayed_republish())
|
|
else:
|
|
logger.error(
|
|
"Max retries (%d) exhausted for attachment %s. Moving to DLQ.",
|
|
MAX_JOB_RETRIES, att_id
|
|
)
|
|
await self._fail_attachment(att_id, "retry_exhausted")
|
|
await self._send_to_dlq(payload, f"retry_exhausted: {reason}")
|
|
|
|
async def start(self) -> None:
|
|
self._running = True
|
|
await self.init_resources()
|
|
|
|
# Подключение к RabbitMQ с автоматическим reconnect
|
|
logger.info("Connecting to RabbitMQ at %s...", BROKER_URL.split("@")[-1])
|
|
while self._running:
|
|
try:
|
|
self._connection = await aio_pika.connect_robust(BROKER_URL)
|
|
break
|
|
except Exception as conn_err:
|
|
logger.warning("RabbitMQ not ready yet (%s). Retrying in 2s...", conn_err)
|
|
await asyncio.sleep(2.0)
|
|
|
|
assert self._connection is not None
|
|
self._channel = await self._connection.channel()
|
|
await self._channel.set_qos(prefetch_count=WORKER_CONCURRENCY)
|
|
|
|
_, queue = await setup_broker_topology(self._channel)
|
|
|
|
self._watchdog_task = asyncio.create_task(self._lease_watchdog())
|
|
logger.info(
|
|
"Worker %s started consuming queue '%s' (prefetch=%d)...",
|
|
self.worker_id, QUEUE_NAME, WORKER_CONCURRENCY
|
|
)
|
|
|
|
await queue.consume(self.process_job_message)
|
|
|
|
async def stop(self) -> None:
|
|
logger.info("Stopping worker %s...", self.worker_id)
|
|
self._running = False
|
|
if self._watchdog_task:
|
|
self._watchdog_task.cancel()
|
|
if self._channel and not self._channel.is_closed:
|
|
await self._channel.close()
|
|
if self._connection and not self._connection.is_closed:
|
|
await self._connection.close()
|
|
if self.pool:
|
|
await self.pool.close()
|
|
logger.info("Worker stopped.")
|
|
|
|
|
|
async def main() -> None:
|
|
worker = AttachmentWorker()
|
|
loop = asyncio.get_running_loop()
|
|
|
|
stop_event = asyncio.Event()
|
|
|
|
def _sig_handler() -> None:
|
|
logger.info("Received termination signal")
|
|
stop_event.set()
|
|
|
|
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
try:
|
|
loop.add_signal_handler(sig, _sig_handler)
|
|
except NotImplementedError:
|
|
pass
|
|
|
|
await worker.start()
|
|
await stop_event.wait()
|
|
await worker.stop()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|