forked from mtusi2026/mtusi2026lab5
639 lines
17 KiB
Python
639 lines
17 KiB
Python
import asyncio
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import random
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
import aio_pika
|
|
from minio import Minio
|
|
from PIL import Image
|
|
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
|
|
|
|
from app.models.attachment import Attachment
|
|
from app.models.user import User
|
|
from app.models.chat import Chat, ChatMember
|
|
from app.config import (
|
|
DATABASE_URL,
|
|
S3_ENDPOINT,
|
|
S3_ACCESS_KEY_ID,
|
|
S3_SECRET_ACCESS_KEY,
|
|
S3_BUCKET,
|
|
)
|
|
|
|
RABBITMQ_URL = os.getenv(
|
|
"RABBITMQ_URL",
|
|
"amqp://guest:guest@rabbitmq:5672/",
|
|
)
|
|
|
|
QUEUE_NAME = "image_tasks"
|
|
|
|
MAX_ATTEMPTS = max(
|
|
1, int(os.getenv("WORKER_MAX_ATTEMPTS", "5"))
|
|
)
|
|
|
|
CONCURRENCY = max(
|
|
1, int(os.getenv("WORKER_CONCURRENCY", "1"))
|
|
)
|
|
|
|
PREFETCH = max(
|
|
1, int(os.getenv("WORKER_PREFETCH", "1"))
|
|
)
|
|
|
|
MAX_IMAGE_BYTES = 10 * 1024 * 1024
|
|
PROCESS_TIMEOUT_SECONDS = 30
|
|
LEASE_SECONDS = 300
|
|
|
|
endpoint = S3_ENDPOINT.replace(
|
|
"http://", ""
|
|
).replace(
|
|
"https://", ""
|
|
)
|
|
|
|
minio_client = Minio(
|
|
endpoint,
|
|
access_key=S3_ACCESS_KEY_ID,
|
|
secret_key=S3_SECRET_ACCESS_KEY,
|
|
secure=False,
|
|
)
|
|
|
|
engine = create_async_engine(DATABASE_URL)
|
|
|
|
async_session_maker = async_sessionmaker(
|
|
engine,
|
|
expire_on_commit=False,
|
|
)
|
|
|
|
|
|
class RetryableProcessingError(Exception):
|
|
def __init__(self, error_code: str):
|
|
self.error_code = error_code
|
|
super().__init__(error_code)
|
|
|
|
|
|
class PermanentProcessingError(Exception):
|
|
def __init__(self, error_code: str):
|
|
self.error_code = error_code
|
|
super().__init__(error_code)
|
|
|
|
|
|
class LeaseBusyError(Exception):
|
|
pass
|
|
|
|
|
|
class LeaseLostError(Exception):
|
|
pass
|
|
|
|
|
|
def get_rss_mb():
|
|
try:
|
|
with open("/proc/self/status") as file:
|
|
for line in file:
|
|
if line.startswith("VmRSS:"):
|
|
kb = int(line.split()[1])
|
|
return round(kb / 1024, 2)
|
|
except (OSError, ValueError, IndexError):
|
|
pass
|
|
|
|
return None
|
|
|
|
|
|
async def claim_attachment(attachment_id: str):
|
|
async with async_session_maker() as session:
|
|
async with session.begin():
|
|
att = await session.get(
|
|
Attachment,
|
|
uuid.UUID(attachment_id),
|
|
with_for_update=True,
|
|
)
|
|
|
|
if att is None:
|
|
print(
|
|
f"[Worker] Файл {attachment_id} не найден",
|
|
flush=True,
|
|
)
|
|
return None
|
|
|
|
if att.status == "ready" and att.thumbnail_key:
|
|
print(
|
|
f"[Worker] Файл {attachment_id} уже обработан",
|
|
flush=True,
|
|
)
|
|
return None
|
|
|
|
if att.status == "failed":
|
|
print(
|
|
f"[Worker] Файл {attachment_id} уже failed",
|
|
flush=True,
|
|
)
|
|
return None
|
|
|
|
now = datetime.now(timezone.utc)
|
|
|
|
if (
|
|
att.status == "processing"
|
|
and att.lease_expires_at is not None
|
|
and att.lease_expires_at > now
|
|
):
|
|
raise LeaseBusyError()
|
|
|
|
lease_expires_at = now + timedelta(
|
|
seconds=LEASE_SECONDS
|
|
)
|
|
|
|
att.status = "processing"
|
|
att.error_code = None
|
|
att.lease_expires_at = lease_expires_at
|
|
|
|
object_key = att.object_key
|
|
|
|
return object_key, lease_expires_at
|
|
|
|
|
|
async def release_lease(
|
|
attachment_id: str,
|
|
lease_expires_at: datetime,
|
|
):
|
|
async with async_session_maker() as session:
|
|
async with session.begin():
|
|
att = await session.get(
|
|
Attachment,
|
|
uuid.UUID(attachment_id),
|
|
with_for_update=True,
|
|
)
|
|
|
|
if (
|
|
att is not None
|
|
and att.status == "processing"
|
|
and att.lease_expires_at == lease_expires_at
|
|
):
|
|
att.status = "queued"
|
|
att.lease_expires_at = None
|
|
|
|
|
|
async def mark_failed(
|
|
attachment_id: str,
|
|
error_code: str,
|
|
):
|
|
async with async_session_maker() as session:
|
|
async with session.begin():
|
|
att = await session.get(
|
|
Attachment,
|
|
uuid.UUID(attachment_id),
|
|
with_for_update=True,
|
|
)
|
|
|
|
if att is None or att.status == "ready":
|
|
return
|
|
|
|
if att.status == "processing":
|
|
raise LeaseBusyError()
|
|
|
|
att.status = "failed"
|
|
att.error_code = error_code
|
|
att.lease_expires_at = None
|
|
|
|
print(
|
|
f"[Worker] Файл {attachment_id}: "
|
|
f"failed, error_code={error_code}",
|
|
flush=True,
|
|
)
|
|
|
|
|
|
async def process_image(image_data: bytes):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
directory = Path(temp_dir)
|
|
|
|
input_path = directory / "original"
|
|
input_path.write_bytes(image_data)
|
|
|
|
try:
|
|
process = await asyncio.create_subprocess_exec(
|
|
sys.executable,
|
|
"-m",
|
|
"app.image_processor",
|
|
str(input_path),
|
|
str(directory),
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
except OSError as exc:
|
|
raise RetryableProcessingError(
|
|
"image_processor_start_error"
|
|
) from exc
|
|
|
|
try:
|
|
await asyncio.wait_for(
|
|
process.communicate(),
|
|
timeout=PROCESS_TIMEOUT_SECONDS,
|
|
)
|
|
except asyncio.TimeoutError as exc:
|
|
if process.returncode is None:
|
|
try:
|
|
process.kill()
|
|
except ProcessLookupError:
|
|
pass
|
|
|
|
await process.communicate()
|
|
|
|
raise RetryableProcessingError(
|
|
"image_processing_timeout"
|
|
) from exc
|
|
|
|
if process.returncode == 20:
|
|
raise PermanentProcessingError(
|
|
"image_too_many_pixels"
|
|
)
|
|
|
|
if process.returncode == 21:
|
|
raise PermanentProcessingError(
|
|
"image_processing_error"
|
|
)
|
|
|
|
if process.returncode != 0:
|
|
raise RetryableProcessingError(
|
|
"image_processor_error"
|
|
)
|
|
|
|
try:
|
|
thumb_bytes = (
|
|
directory / "thumbnail.jpg"
|
|
).read_bytes()
|
|
|
|
crop_bytes = (
|
|
directory / "crop.jpg"
|
|
).read_bytes()
|
|
|
|
webp_bytes = (
|
|
directory / "optimized.webp"
|
|
).read_bytes()
|
|
|
|
with Image.open(
|
|
io.BytesIO(thumb_bytes)
|
|
) as thumbnail:
|
|
thumb_width, thumb_height = thumbnail.size
|
|
|
|
except OSError as exc:
|
|
raise RetryableProcessingError(
|
|
"image_processor_output_error"
|
|
) from exc
|
|
|
|
return (
|
|
thumb_bytes,
|
|
crop_bytes,
|
|
webp_bytes,
|
|
thumb_width,
|
|
thumb_height,
|
|
)
|
|
|
|
|
|
async def process_attachment(attachment_id: str):
|
|
print(
|
|
f"[Worker] Обработка файла: {attachment_id}",
|
|
flush=True,
|
|
)
|
|
|
|
claim = await claim_attachment(attachment_id)
|
|
|
|
if claim is None:
|
|
return
|
|
|
|
object_key, lease_expires_at = claim
|
|
|
|
try:
|
|
try:
|
|
response = minio_client.get_object(
|
|
S3_BUCKET,
|
|
object_key,
|
|
)
|
|
|
|
try:
|
|
image_data = response.read(
|
|
MAX_IMAGE_BYTES + 1
|
|
)
|
|
finally:
|
|
response.close()
|
|
response.release_conn()
|
|
|
|
except Exception as exc:
|
|
raise RetryableProcessingError(
|
|
"s3_download_error"
|
|
) from exc
|
|
|
|
if len(image_data) > MAX_IMAGE_BYTES:
|
|
raise PermanentProcessingError(
|
|
"image_too_large"
|
|
)
|
|
|
|
(
|
|
thumb_bytes,
|
|
crop_bytes,
|
|
webp_bytes,
|
|
thumb_width,
|
|
thumb_height,
|
|
) = await process_image(image_data)
|
|
|
|
thumb_key = f"thumbnails/{object_key}"
|
|
crop_key = f"crops/{attachment_id}.jpg"
|
|
webp_key = f"webp/{attachment_id}.webp"
|
|
|
|
try:
|
|
minio_client.put_object(
|
|
S3_BUCKET,
|
|
thumb_key,
|
|
io.BytesIO(thumb_bytes),
|
|
len(thumb_bytes),
|
|
content_type="image/jpeg",
|
|
)
|
|
|
|
minio_client.put_object(
|
|
S3_BUCKET,
|
|
crop_key,
|
|
io.BytesIO(crop_bytes),
|
|
len(crop_bytes),
|
|
content_type="image/jpeg",
|
|
)
|
|
|
|
minio_client.put_object(
|
|
S3_BUCKET,
|
|
webp_key,
|
|
io.BytesIO(webp_bytes),
|
|
len(webp_bytes),
|
|
content_type="image/webp",
|
|
)
|
|
|
|
except Exception as exc:
|
|
raise RetryableProcessingError(
|
|
"s3_upload_error"
|
|
) from exc
|
|
|
|
async with async_session_maker() as session:
|
|
async with session.begin():
|
|
att = await session.get(
|
|
Attachment,
|
|
uuid.UUID(attachment_id),
|
|
with_for_update=True,
|
|
)
|
|
|
|
if (
|
|
att is None
|
|
or att.status != "processing"
|
|
or att.lease_expires_at != lease_expires_at
|
|
):
|
|
raise LeaseLostError()
|
|
|
|
att.thumbnail_key = thumb_key
|
|
att.thumbnail_sha256 = hashlib.sha256(
|
|
thumb_bytes
|
|
).hexdigest()
|
|
att.thumbnail_size_bytes = len(thumb_bytes)
|
|
att.thumbnail_width = thumb_width
|
|
att.thumbnail_height = thumb_height
|
|
att.crop_key = crop_key
|
|
att.webp_key = webp_key
|
|
att.status = "ready"
|
|
att.error_code = None
|
|
att.lease_expires_at = None
|
|
|
|
print(
|
|
f"[Worker] Успешно обработан {attachment_id}",
|
|
flush=True,
|
|
)
|
|
|
|
except Exception:
|
|
await release_lease(
|
|
attachment_id,
|
|
lease_expires_at,
|
|
)
|
|
raise
|
|
|
|
|
|
async def main():
|
|
connection = await aio_pika.connect_robust(
|
|
RABBITMQ_URL
|
|
)
|
|
|
|
async with connection:
|
|
channel = await connection.channel(
|
|
publisher_confirms=True
|
|
)
|
|
|
|
await channel.set_qos(
|
|
prefetch_count=PREFETCH
|
|
)
|
|
|
|
queue = await channel.declare_queue(
|
|
QUEUE_NAME,
|
|
durable=True,
|
|
)
|
|
|
|
semaphore = asyncio.Semaphore(CONCURRENCY)
|
|
|
|
print(
|
|
f"[Worker] CONCURRENCY={CONCURRENCY}, "
|
|
f"PREFETCH={PREFETCH}",
|
|
flush=True,
|
|
)
|
|
|
|
async def on_message(
|
|
message: aio_pika.IncomingMessage
|
|
):
|
|
async with semaphore:
|
|
started = time.perf_counter()
|
|
|
|
try:
|
|
data = json.loads(
|
|
message.body.decode()
|
|
)
|
|
|
|
attachment_id = str(
|
|
uuid.UUID(data["attachment_id"])
|
|
)
|
|
|
|
except (
|
|
ValueError,
|
|
KeyError,
|
|
TypeError,
|
|
):
|
|
print(
|
|
"[Worker] Некорректное сообщение",
|
|
flush=True,
|
|
)
|
|
|
|
await message.reject(
|
|
requeue=False
|
|
)
|
|
return
|
|
|
|
headers = dict(message.headers or {})
|
|
|
|
try:
|
|
attempt = max(
|
|
1,
|
|
int(headers.get("x-attempt", 1)),
|
|
)
|
|
except (ValueError, TypeError):
|
|
attempt = 1
|
|
|
|
print(
|
|
f"[Worker] Попытка "
|
|
f"{attempt}/{MAX_ATTEMPTS} "
|
|
f"для {attachment_id}",
|
|
flush=True,
|
|
)
|
|
|
|
try:
|
|
await process_attachment(
|
|
attachment_id
|
|
)
|
|
|
|
except LeaseBusyError:
|
|
print(
|
|
"[Worker] Задание уже "
|
|
"обрабатывается, ждём Lease",
|
|
flush=True,
|
|
)
|
|
|
|
await asyncio.sleep(5)
|
|
|
|
await message.nack(
|
|
requeue=True
|
|
)
|
|
return
|
|
|
|
except LeaseLostError:
|
|
print(
|
|
"[Worker] Lease перешёл "
|
|
"другому Worker",
|
|
flush=True,
|
|
)
|
|
|
|
await message.ack()
|
|
return
|
|
|
|
except PermanentProcessingError as exc:
|
|
try:
|
|
await mark_failed(
|
|
attachment_id,
|
|
exc.error_code,
|
|
)
|
|
except Exception:
|
|
await message.nack(
|
|
requeue=True
|
|
)
|
|
return
|
|
|
|
await message.ack()
|
|
return
|
|
|
|
except Exception as exc:
|
|
error_code = (
|
|
exc.error_code
|
|
if isinstance(
|
|
exc,
|
|
RetryableProcessingError,
|
|
)
|
|
else "worker_error"
|
|
)
|
|
|
|
if attempt < MAX_ATTEMPTS:
|
|
next_attempt = attempt + 1
|
|
|
|
print(
|
|
f"[Worker] Ошибка {error_code}. "
|
|
f"Повтор "
|
|
f"{next_attempt}/{MAX_ATTEMPTS}",
|
|
flush=True,
|
|
)
|
|
|
|
headers["x-attempt"] = next_attempt
|
|
|
|
base_delay = min(
|
|
30,
|
|
2 ** (attempt - 1),
|
|
)
|
|
|
|
delay = base_delay + random.uniform(
|
|
0,
|
|
base_delay / 2,
|
|
)
|
|
|
|
print(
|
|
f"[Worker] Повтор через "
|
|
f"{delay:.1f} сек.",
|
|
flush=True,
|
|
)
|
|
|
|
await asyncio.sleep(delay)
|
|
|
|
try:
|
|
await channel.default_exchange.publish(
|
|
aio_pika.Message(
|
|
body=message.body,
|
|
headers=headers,
|
|
delivery_mode=(
|
|
aio_pika.DeliveryMode.PERSISTENT
|
|
),
|
|
content_type="application/json",
|
|
),
|
|
routing_key=QUEUE_NAME,
|
|
)
|
|
|
|
except Exception:
|
|
await message.nack(
|
|
requeue=True
|
|
)
|
|
return
|
|
|
|
await message.ack()
|
|
return
|
|
|
|
print(
|
|
f"[Worker] Попытки исчерпаны: "
|
|
f"{attachment_id}",
|
|
flush=True,
|
|
)
|
|
|
|
try:
|
|
await mark_failed(
|
|
attachment_id,
|
|
error_code,
|
|
)
|
|
except Exception:
|
|
await message.nack(
|
|
requeue=True
|
|
)
|
|
return
|
|
|
|
await message.ack()
|
|
return
|
|
|
|
await message.ack()
|
|
|
|
duration = time.perf_counter() - started
|
|
rss_mb = get_rss_mb()
|
|
|
|
print(
|
|
f"[Metrics] attachment_id={attachment_id} "
|
|
f"duration_s={duration:.3f} "
|
|
f"rss_mb={rss_mb}",
|
|
flush=True,
|
|
)
|
|
|
|
print(
|
|
"[Worker] Ждем сообщения из RabbitMQ...",
|
|
flush=True,
|
|
)
|
|
|
|
await queue.consume(on_message)
|
|
await asyncio.Future()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main()) |