From 429eaef5e2f75063c42542ce5ba318302ed7edfa Mon Sep 17 00:00:00 2001 From: vasyukova Date: Fri, 23 Oct 2026 00:07:41 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB?= =?UTF-8?q?=D0=B0=20RabbitMQ?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 3 +- src/app/outbox_worker.py | 75 ++++++++++ src/app/storage/rabbit.py | 156 ++++++++++++++++++++ src/app/worker.py | 296 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 529 insertions(+), 1 deletion(-) create mode 100644 src/app/outbox_worker.py create mode 100644 src/app/storage/rabbit.py create mode 100644 src/app/worker.py diff --git a/requirements.txt b/requirements.txt index 9e7336e1..12d8b8ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ psycopg_pool==3.2.3 redis==5.2.0 openapi-spec-validator==0.7.2 boto3==1.35.74 -Pillow==11.0.0 \ No newline at end of file +Pillow==11.0.0 +pika==1.3.2 \ No newline at end of file diff --git a/src/app/outbox_worker.py b/src/app/outbox_worker.py new file mode 100644 index 00000000..ed777f0b --- /dev/null +++ b/src/app/outbox_worker.py @@ -0,0 +1,75 @@ +"""Outbox worker lab5 level 4: publish неопубликованных job в RabbitMQ. + +Закрывает разрыв COMMIT → publish: если API упал после COMMIT, outbox +осталась в БД; этот процесс опубликует её после рестарта. Publisher +confirm гарантирует приём брокером перед mark published. +""" +import logging +import signal +import sys +import time +import traceback + +from .storage.db import init_pool, close_pool +from .storage.postgres import storage +from .storage.rabbit import publish_job + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, +) +log = logging.getLogger("outbox") + +_stop = False + + +def _handle_sigterm(signum, frame): + global _stop + log.info("event=sigterm_received signal=%s", signum) + _stop = True + + +def main() -> None: + signal.signal(signal.SIGTERM, _handle_sigterm) + init_pool() + log.info("event=outbox_start") + + while not _stop: + try: + rows = storage.list_unpublished_outbox(limit=50) + except Exception as exc: # noqa: BLE001 + log.error("event=outbox_select_failed type=%s msg=%s", + type(exc).__name__, str(exc)[:300]) + time.sleep(2) + continue + + for outbox_id, payload in rows: + if _stop: + break + try: + publish_job(payload) + storage.mark_outbox_published(outbox_id) + log.info("event=outbox_published id=%s job_id=%s", + outbox_id, payload.get("job_id")) + except Exception as exc: # noqa: BLE001 + log.warning( + "event=outbox_publish_failed id=%s type=%s msg=%s", + outbox_id, type(exc).__name__, str(exc)[:300], + ) + # не выходим, попробуем в следующем цикле + time.sleep(2) + break + + time.sleep(2) + + close_pool() + log.info("event=outbox_stop") + + +if __name__ == "__main__": + try: + main() + except Exception: + log.error("event=outbox_fatal\n%s", traceback.format_exc()) + sys.exit(1) \ No newline at end of file diff --git a/src/app/storage/rabbit.py b/src/app/storage/rabbit.py new file mode 100644 index 00000000..bbf064dc --- /dev/null +++ b/src/app/storage/rabbit.py @@ -0,0 +1,156 @@ +"""RabbitMQ: topology, publisher with confirms, consumer с manual ack. + +- Основная очередь: durable quorum queue. +- Retry: TTL-очереди с dead-letter обратно в основную; x-retry-count. +- DLQ: финальные ошибки и исчерпанные retry. +- Publisher confirms: publish() ждёт подтверждения брокера. + +ВАЖНО: setup_topology вызывается в worker при старте и в outbox +однократно при первом publish. Не декларируем топологию на каждый +publish — иначе гонка с параллельными декларациями. +""" +import json +import logging +import os +import threading +from typing import Optional + +import pika + +log = logging.getLogger("rabbit") + +EXCHANGE = "messenger.attachments" +QUEUE = "attachment.process.v1" +RETRY_EXCHANGE = "messenger.attachments.retry" +DLX = "messenger.attachments.dlx" +DLQ = "attachment.process.v1.dlq" + +RETRY_DELAYS = [5, 15, 60] +MAX_ATTEMPTS = len(RETRY_DELAYS) + 1 + +_topology_ready = False +_topology_lock = threading.Lock() + + +def _url() -> str: + return os.environ.get("RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/%2F") + + +def connection_params() -> pika.ConnectionParameters: + return pika.URLParameters(_url()) + + +def connect() -> pika.BlockingConnection: + return pika.BlockingConnection(connection_params()) + + +def setup_topology(channel: pika.adapters.blocking_connection.BlockingChannel) -> None: + channel.exchange_declare(EXCHANGE, exchange_type="direct", durable=True) + channel.exchange_declare(RETRY_EXCHANGE, exchange_type="direct", durable=True) + channel.exchange_declare(DLX, exchange_type="direct", durable=True) + + channel.queue_declare(DLQ, durable=True, arguments={"x-queue-type": "quorum"}) + channel.queue_bind(DLQ, DLX, routing_key=DLQ) + + channel.queue_declare( + QUEUE, + durable=True, + arguments={ + "x-queue-type": "quorum", + "x-dead-letter-exchange": DLX, + "x-dead-letter-routing-key": DLQ, + }, + ) + channel.queue_bind(QUEUE, EXCHANGE, routing_key=QUEUE) + + for delay in RETRY_DELAYS: + qname = f"attachment.retry.{delay}s" + channel.queue_declare( + qname, + durable=True, + arguments={ + "x-message-ttl": delay * 1000, + "x-dead-letter-exchange": EXCHANGE, + "x-dead-letter-routing-key": QUEUE, + }, + ) + channel.queue_bind(qname, RETRY_EXCHANGE, routing_key=qname) + + +def _ensure_topology(ch: pika.adapters.blocking_connection.BlockingChannel) -> None: + global _topology_ready + if _topology_ready: + return + with _topology_lock: + if _topology_ready: + return + setup_topology(ch) + _topology_ready = True + + +def publish_job(payload: dict) -> None: + """Публикация job с publisher confirm. Бросает при неудаче.""" + conn = connect() + try: + ch = conn.channel() + ch.confirm_delivery() + _ensure_topology(ch) + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + ch.basic_publish( + exchange=EXCHANGE, + routing_key=QUEUE, + body=body.encode("utf-8"), + properties=pika.BasicProperties( + delivery_mode=2, + content_type="application/json", + ), + mandatory=True, + ) + finally: + conn.close() + + +def publish_retry(payload: dict, retry_count: int) -> None: + idx = min(retry_count - 1, len(RETRY_DELAYS) - 1) + delay = RETRY_DELAYS[idx] + qname = f"attachment.retry.{delay}s" + conn = connect() + try: + ch = conn.channel() + ch.confirm_delivery() + _ensure_topology(ch) + headers = dict(payload.get("_headers") or {}) + headers["x-retry-count"] = retry_count + clean = {k: v for k, v in payload.items() if k != "_headers"} + body = json.dumps(clean, ensure_ascii=False, separators=(",", ":")) + ch.basic_publish( + exchange=RETRY_EXCHANGE, + routing_key=qname, + body=body.encode("utf-8"), + properties=pika.BasicProperties( + delivery_mode=2, + content_type="application/json", + headers=headers, + ), + mandatory=True, + ) + finally: + conn.close() + + +def publish_dlq(payload: dict) -> None: + conn = connect() + try: + ch = conn.channel() + ch.confirm_delivery() + _ensure_topology(ch) + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + ch.basic_publish( + exchange=DLX, + routing_key=DLQ, + body=body.encode("utf-8"), + properties=pika.BasicProperties(delivery_mode=2, content_type="application/json"), + mandatory=True, + ) + finally: + conn.close() \ No newline at end of file diff --git a/src/app/worker.py b/src/app/worker.py new file mode 100644 index 00000000..d82df84e --- /dev/null +++ b/src/app/worker.py @@ -0,0 +1,296 @@ +"""Worker lab5: обработка вложений из RabbitMQ. + +- at-least-once: manual ack после durable эффекта. +- идемпотентность: ключ (attachment_id, pipeline_version, variant). +- retry с backoff через TTL retry-очереди, затем DLQ. +- lease: processing_until, зависшие возвращаются в queued. +- thumbnail 256×256 (JPEG), crop 128×128 (JPEG), optimized WebP 1280×1280. +""" +import io +import json +import logging +import os +import signal +import sys +import time +from datetime import datetime, timedelta, timezone + +import pika +from PIL import Image, ImageOps + +from .storage import s3 +from .storage.db import init_pool, close_pool +from .storage.postgres import storage +from .storage.rabbit import ( + QUEUE, EXCHANGE, MAX_ATTEMPTS, connect, setup_topology, + publish_retry, publish_dlq, +) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, +) +log = logging.getLogger("worker") + +PIPELINE_VERSION = 1 +LEASE_SECONDS = 60 +THUMB_SIZE = (256, 256) +CROP_SIZE = (128, 128) +OPT_SIZE = (1280, 1280) +OPT_QUALITY = 82 + +_stop = False + + +def _handle_sigterm(signum, frame): + global _stop + log.info("event=sigterm_received signal=%s", signum) + _stop = True + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _to_rgb_white(img: Image.Image) -> Image.Image: + """Alpha компонуется на белый; EXIF orientation применяется.""" + img = ImageOps.exif_transpose(img) + if img.mode in ("RGBA", "LA", "P"): + img = img.convert("RGBA") + bg = Image.new("RGB", img.size, (255, 255, 255)) + bg.paste(img, mask=img.split()[-1]) + return bg + if img.mode != "RGB": + return img.convert("RGB") + return img + + +def _make_thumbnail(img: Image.Image) -> tuple[Image.Image, bytes]: + """Вписать в 256×256 без увеличения. Округление вниз, минимум 1 px.""" + w, h = img.size + scale = min(THUMB_SIZE[0] / w, THUMB_SIZE[1] / h, 1.0) + nw = max(1, int(w * scale)) + nh = max(1, int(h * scale)) + thumb = img.resize((nw, nh), Image.LANCZOS) + buf = io.BytesIO() + thumb.save(buf, format="JPEG", quality=88, optimize=True) + return thumb, buf.getvalue() + + +def _make_crop(img: Image.Image) -> tuple[Image.Image, bytes]: + """Центрированный квадрат 128×128 (маленькие допускают увеличение).""" + w, h = img.size + side = min(w, h) + left = (w - side) // 2 + top = (h - side) // 2 + sq = img.crop((left, top, left + side, top + side)) + crop = sq.resize(CROP_SIZE, Image.LANCZOS) + buf = io.BytesIO() + crop.save(buf, format="JPEG", quality=88, optimize=True) + return crop, buf.getvalue() + + +def _make_optimized(img: Image.Image) -> tuple[Image.Image, bytes]: + """WebP, вписан в 1280×1280 без увеличения, EXIF удалён.""" + w, h = img.size + scale = min(OPT_SIZE[0] / w, OPT_SIZE[1] / h, 1.0) + nw = max(1, int(w * scale)) + nh = max(1, int(h * scale)) + opt = img.resize((nw, nh), Image.LANCZOS) + buf = io.BytesIO() + opt.save(buf, format="WEBP", quality=OPT_QUALITY, method=4) + return opt, buf.getvalue() + + +def _sha256(data: bytes) -> str: + import hashlib + return hashlib.sha256(data).hexdigest() + + +def process_job(payload: dict) -> None: + """Обработка одного job. Идемпотентна: finalize_ready использует ON CONFLICT.""" + attachment_id = payload["attachment_id"] + pipeline_version = payload.get("pipeline_version", 1) + job_id = payload["job_id"] + + log.info("event=job_received job_id=%s attachment_id=%s attempt=%s", + job_id, attachment_id, payload.get("attempt", 1)) + + # 1) Захватить lease. Если уже ready/failed — no-op (дубль). + if not storage.claim_processing(attachment_id, LEASE_SECONDS): + log.info("event=job_skipped_already_done attachment_id=%s", attachment_id) + return + + # 2) Получить staging key и метаданные. + pair = storage.get_attachment_with_key(attachment_id) + if pair is None: + log.warning("event=attachment_missing attachment_id=%s", attachment_id) + return + att, staging_key = pair + if att.status == "ready": + log.info("event=already_ready attachment_id=%s", attachment_id) + return + + # 3) Скачать staging из S3. + try: + raw, ctype = s3.get_object(staging_key) + except Exception as exc: # noqa: BLE001 + # Временная ошибка S3 — retry. + raise RuntimeError("staging_unavailable") from exc + + # 4) Проверить, что файл всё ещё валиден (worker defense-in-depth). + try: + img = Image.open(io.BytesIO(raw)) + img.load() + except Exception as exc: # noqa: BLE001 + storage.finalize_failed(attachment_id, "invalid_image") + log.warning("event=invalid_image attachment_id=%s type=%s", + attachment_id, type(exc).__name__) + return + + img_rgb = _to_rgb_white(img) + + # 5) Записать финальный оригинал. + final_key = s3.original_key(att.chat_id, att.id) + s3.put_bytes(final_key, raw, att.content_type) + + variants = [{ + "name": "original", + "object_key": final_key, + "content_type": att.content_type, + "size_bytes": len(raw), + "width": img.width, + "height": img.height, + "sha256": _sha256(raw), + }] + + # 6) Thumbnail (обязателен). + _, thumb_bytes = _make_thumbnail(img_rgb) + thumb_key = s3.variant_key(att.chat_id, att.id, "thumbnail") + s3.put_bytes(thumb_key, thumb_bytes, "image/jpeg") + tw, th = Image.open(io.BytesIO(thumb_bytes)).size + variants.append({ + "name": "thumbnail", + "object_key": thumb_key, + "content_type": "image/jpeg", + "size_bytes": len(thumb_bytes), + "width": tw, + "height": th, + "sha256": _sha256(thumb_bytes), + }) + + # 7) Crop (level 4). + _, crop_bytes = _make_crop(img_rgb) + crop_key = s3.variant_key(att.chat_id, att.id, "crop") + s3.put_bytes(crop_key, crop_bytes, "image/jpeg") + cw, ch = Image.open(io.BytesIO(crop_bytes)).size + variants.append({ + "name": "crop", + "object_key": crop_key, + "content_type": "image/jpeg", + "size_bytes": len(crop_bytes), + "width": cw, + "height": ch, + "sha256": _sha256(crop_bytes), + }) + + # 8) Optimized WebP (level 4). + _, opt_bytes = _make_optimized(img_rgb) + opt_key = s3.variant_key(att.chat_id, att.id, "optimized") + s3.put_bytes(opt_key, opt_bytes, "image/webp") + ow, oh = Image.open(io.BytesIO(opt_bytes)).size + variants.append({ + "name": "optimized", + "object_key": opt_key, + "content_type": "image/webp", + "size_bytes": len(opt_bytes), + "width": ow, + "height": oh, + "sha256": _sha256(opt_bytes), + }) + + # 9) Атомарно: variants + status=ready. + storage.finalize_ready(attachment_id, pipeline_version, variants) + + # 10) Удалить staging после безопасной фиксации. + try: + s3.delete_object(staging_key) + except Exception: + pass + + log.info("event=job_ready attachment_id=%s job_id=%s variants=%d", + attachment_id, job_id, len(variants)) + + +def _on_message(ch, method, properties, body): + """Callback consumer. ack/nack — явно.""" + try: + payload = json.loads(body) + except Exception: + # Битый JSON — постоянная ошибка. + log.error("event=invalid_job_json") + ch.basic_nack(method.delivery_tag, requeue=False) + return + + headers = dict(properties.headers or {}) + retry_count = int(headers.get("x-retry-count", 0)) + payload["attempt"] = retry_count + 1 + payload["_headers"] = headers + + try: + process_job(payload) + ch.basic_ack(method.delivery_tag) + except Exception as exc: # noqa: BLE001 + log.warning("event=job_retry attachment_id=%s attempt=%d type=%s", + payload.get("attachment_id"), retry_count + 1, type(exc).__name__) + if retry_count + 1 >= MAX_ATTEMPTS: + # Исчерпаны попытки — failed + DLQ. + try: + storage.finalize_failed(payload["attachment_id"], "retry_exhausted") + except Exception: + pass + try: + publish_dlq({k: v for k, v in payload.items() if k != "_headers"}) + except Exception: + pass + ch.basic_ack(method.delivery_tag) + return + # Retry через TTL-очередь, затем ack текущей. + try: + publish_retry(payload, retry_count + 1) + ch.basic_ack(method.delivery_tag) + except Exception: + # Не смогли опубликовать retry — вернуть в очередь. + ch.basic_nack(method.delivery_tag, requeue=True) + + +def main() -> None: + signal.signal(signal.SIGTERM, _handle_sigterm) + init_pool() + log.info("event=worker_start queue=%s max_attempts=%s", QUEUE, MAX_ATTEMPTS) + + while not _stop: + try: + conn = connect() + ch = conn.channel() + setup_topology(ch) + ch.basic_qos(prefetch_count=1) + ch.basic_consume(queue=QUEUE, on_message_callback=_on_message) + log.info("event=consumer_started") + + while not _stop and conn.is_open: + conn.process_data_events(time_limit=1) + + ch.stop_consuming() + conn.close() + except Exception as exc: # noqa: BLE001 + log.error("event=worker_loop_error type=%s", type(exc).__name__) + time.sleep(3) + + close_pool() + log.info("event=worker_stop") + + +if __name__ == "__main__": + main() \ No newline at end of file