Добавила RabbitMQ

This commit is contained in:
vasyukova
2026-10-23 00:07:41 +03:00
parent 8030c8821a
commit 429eaef5e2
4 changed files with 529 additions and 1 deletions
+2 -1
View File
@@ -5,4 +5,5 @@ psycopg_pool==3.2.3
redis==5.2.0
openapi-spec-validator==0.7.2
boto3==1.35.74
Pillow==11.0.0
Pillow==11.0.0
pika==1.3.2
+75
View File
@@ -0,0 +1,75 @@
"""Outbox worker lab5 level 4: publish неопубликованных job в RabbitMQ.
Закрывает разрыв COMMIT → publish: если API упал после COMMIT, outbox
осталась в БД; этот процесс опубликует её после рестарта. Publisher
confirm гарантирует приём брокером перед mark published.
"""
import logging
import signal
import sys
import time
import traceback
from .storage.db import init_pool, close_pool
from .storage.postgres import storage
from .storage.rabbit import publish_job
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
stream=sys.stdout,
)
log = logging.getLogger("outbox")
_stop = False
def _handle_sigterm(signum, frame):
global _stop
log.info("event=sigterm_received signal=%s", signum)
_stop = True
def main() -> None:
signal.signal(signal.SIGTERM, _handle_sigterm)
init_pool()
log.info("event=outbox_start")
while not _stop:
try:
rows = storage.list_unpublished_outbox(limit=50)
except Exception as exc: # noqa: BLE001
log.error("event=outbox_select_failed type=%s msg=%s",
type(exc).__name__, str(exc)[:300])
time.sleep(2)
continue
for outbox_id, payload in rows:
if _stop:
break
try:
publish_job(payload)
storage.mark_outbox_published(outbox_id)
log.info("event=outbox_published id=%s job_id=%s",
outbox_id, payload.get("job_id"))
except Exception as exc: # noqa: BLE001
log.warning(
"event=outbox_publish_failed id=%s type=%s msg=%s",
outbox_id, type(exc).__name__, str(exc)[:300],
)
# не выходим, попробуем в следующем цикле
time.sleep(2)
break
time.sleep(2)
close_pool()
log.info("event=outbox_stop")
if __name__ == "__main__":
try:
main()
except Exception:
log.error("event=outbox_fatal\n%s", traceback.format_exc())
sys.exit(1)
+156
View File
@@ -0,0 +1,156 @@
"""RabbitMQ: topology, publisher with confirms, consumer с manual ack.
- Основная очередь: durable quorum queue.
- Retry: TTL-очереди с dead-letter обратно в основную; x-retry-count.
- DLQ: финальные ошибки и исчерпанные retry.
- Publisher confirms: publish() ждёт подтверждения брокера.
ВАЖНО: setup_topology вызывается в worker при старте и в outbox
однократно при первом publish. Не декларируем топологию на каждый
publish — иначе гонка с параллельными декларациями.
"""
import json
import logging
import os
import threading
from typing import Optional
import pika
log = logging.getLogger("rabbit")
EXCHANGE = "messenger.attachments"
QUEUE = "attachment.process.v1"
RETRY_EXCHANGE = "messenger.attachments.retry"
DLX = "messenger.attachments.dlx"
DLQ = "attachment.process.v1.dlq"
RETRY_DELAYS = [5, 15, 60]
MAX_ATTEMPTS = len(RETRY_DELAYS) + 1
_topology_ready = False
_topology_lock = threading.Lock()
def _url() -> str:
return os.environ.get("RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/%2F")
def connection_params() -> pika.ConnectionParameters:
return pika.URLParameters(_url())
def connect() -> pika.BlockingConnection:
return pika.BlockingConnection(connection_params())
def setup_topology(channel: pika.adapters.blocking_connection.BlockingChannel) -> None:
channel.exchange_declare(EXCHANGE, exchange_type="direct", durable=True)
channel.exchange_declare(RETRY_EXCHANGE, exchange_type="direct", durable=True)
channel.exchange_declare(DLX, exchange_type="direct", durable=True)
channel.queue_declare(DLQ, durable=True, arguments={"x-queue-type": "quorum"})
channel.queue_bind(DLQ, DLX, routing_key=DLQ)
channel.queue_declare(
QUEUE,
durable=True,
arguments={
"x-queue-type": "quorum",
"x-dead-letter-exchange": DLX,
"x-dead-letter-routing-key": DLQ,
},
)
channel.queue_bind(QUEUE, EXCHANGE, routing_key=QUEUE)
for delay in RETRY_DELAYS:
qname = f"attachment.retry.{delay}s"
channel.queue_declare(
qname,
durable=True,
arguments={
"x-message-ttl": delay * 1000,
"x-dead-letter-exchange": EXCHANGE,
"x-dead-letter-routing-key": QUEUE,
},
)
channel.queue_bind(qname, RETRY_EXCHANGE, routing_key=qname)
def _ensure_topology(ch: pika.adapters.blocking_connection.BlockingChannel) -> None:
global _topology_ready
if _topology_ready:
return
with _topology_lock:
if _topology_ready:
return
setup_topology(ch)
_topology_ready = True
def publish_job(payload: dict) -> None:
"""Публикация job с publisher confirm. Бросает при неудаче."""
conn = connect()
try:
ch = conn.channel()
ch.confirm_delivery()
_ensure_topology(ch)
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
ch.basic_publish(
exchange=EXCHANGE,
routing_key=QUEUE,
body=body.encode("utf-8"),
properties=pika.BasicProperties(
delivery_mode=2,
content_type="application/json",
),
mandatory=True,
)
finally:
conn.close()
def publish_retry(payload: dict, retry_count: int) -> None:
idx = min(retry_count - 1, len(RETRY_DELAYS) - 1)
delay = RETRY_DELAYS[idx]
qname = f"attachment.retry.{delay}s"
conn = connect()
try:
ch = conn.channel()
ch.confirm_delivery()
_ensure_topology(ch)
headers = dict(payload.get("_headers") or {})
headers["x-retry-count"] = retry_count
clean = {k: v for k, v in payload.items() if k != "_headers"}
body = json.dumps(clean, ensure_ascii=False, separators=(",", ":"))
ch.basic_publish(
exchange=RETRY_EXCHANGE,
routing_key=qname,
body=body.encode("utf-8"),
properties=pika.BasicProperties(
delivery_mode=2,
content_type="application/json",
headers=headers,
),
mandatory=True,
)
finally:
conn.close()
def publish_dlq(payload: dict) -> None:
conn = connect()
try:
ch = conn.channel()
ch.confirm_delivery()
_ensure_topology(ch)
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
ch.basic_publish(
exchange=DLX,
routing_key=DLQ,
body=body.encode("utf-8"),
properties=pika.BasicProperties(delivery_mode=2, content_type="application/json"),
mandatory=True,
)
finally:
conn.close()
+296
View File
@@ -0,0 +1,296 @@
"""Worker lab5: обработка вложений из RabbitMQ.
- at-least-once: manual ack после durable эффекта.
- идемпотентность: ключ (attachment_id, pipeline_version, variant).
- retry с backoff через TTL retry-очереди, затем DLQ.
- lease: processing_until, зависшие возвращаются в queued.
- thumbnail 256×256 (JPEG), crop 128×128 (JPEG), optimized WebP 1280×1280.
"""
import io
import json
import logging
import os
import signal
import sys
import time
from datetime import datetime, timedelta, timezone
import pika
from PIL import Image, ImageOps
from .storage import s3
from .storage.db import init_pool, close_pool
from .storage.postgres import storage
from .storage.rabbit import (
QUEUE, EXCHANGE, MAX_ATTEMPTS, connect, setup_topology,
publish_retry, publish_dlq,
)
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
stream=sys.stdout,
)
log = logging.getLogger("worker")
PIPELINE_VERSION = 1
LEASE_SECONDS = 60
THUMB_SIZE = (256, 256)
CROP_SIZE = (128, 128)
OPT_SIZE = (1280, 1280)
OPT_QUALITY = 82
_stop = False
def _handle_sigterm(signum, frame):
global _stop
log.info("event=sigterm_received signal=%s", signum)
_stop = True
def _now_iso() -> str:
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
def _to_rgb_white(img: Image.Image) -> Image.Image:
"""Alpha компонуется на белый; EXIF orientation применяется."""
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA", "P"):
img = img.convert("RGBA")
bg = Image.new("RGB", img.size, (255, 255, 255))
bg.paste(img, mask=img.split()[-1])
return bg
if img.mode != "RGB":
return img.convert("RGB")
return img
def _make_thumbnail(img: Image.Image) -> tuple[Image.Image, bytes]:
"""Вписать в 256×256 без увеличения. Округление вниз, минимум 1 px."""
w, h = img.size
scale = min(THUMB_SIZE[0] / w, THUMB_SIZE[1] / h, 1.0)
nw = max(1, int(w * scale))
nh = max(1, int(h * scale))
thumb = img.resize((nw, nh), Image.LANCZOS)
buf = io.BytesIO()
thumb.save(buf, format="JPEG", quality=88, optimize=True)
return thumb, buf.getvalue()
def _make_crop(img: Image.Image) -> tuple[Image.Image, bytes]:
"""Центрированный квадрат 128×128 (маленькие допускают увеличение)."""
w, h = img.size
side = min(w, h)
left = (w - side) // 2
top = (h - side) // 2
sq = img.crop((left, top, left + side, top + side))
crop = sq.resize(CROP_SIZE, Image.LANCZOS)
buf = io.BytesIO()
crop.save(buf, format="JPEG", quality=88, optimize=True)
return crop, buf.getvalue()
def _make_optimized(img: Image.Image) -> tuple[Image.Image, bytes]:
"""WebP, вписан в 1280×1280 без увеличения, EXIF удалён."""
w, h = img.size
scale = min(OPT_SIZE[0] / w, OPT_SIZE[1] / h, 1.0)
nw = max(1, int(w * scale))
nh = max(1, int(h * scale))
opt = img.resize((nw, nh), Image.LANCZOS)
buf = io.BytesIO()
opt.save(buf, format="WEBP", quality=OPT_QUALITY, method=4)
return opt, buf.getvalue()
def _sha256(data: bytes) -> str:
import hashlib
return hashlib.sha256(data).hexdigest()
def process_job(payload: dict) -> None:
"""Обработка одного job. Идемпотентна: finalize_ready использует ON CONFLICT."""
attachment_id = payload["attachment_id"]
pipeline_version = payload.get("pipeline_version", 1)
job_id = payload["job_id"]
log.info("event=job_received job_id=%s attachment_id=%s attempt=%s",
job_id, attachment_id, payload.get("attempt", 1))
# 1) Захватить lease. Если уже ready/failed — no-op (дубль).
if not storage.claim_processing(attachment_id, LEASE_SECONDS):
log.info("event=job_skipped_already_done attachment_id=%s", attachment_id)
return
# 2) Получить staging key и метаданные.
pair = storage.get_attachment_with_key(attachment_id)
if pair is None:
log.warning("event=attachment_missing attachment_id=%s", attachment_id)
return
att, staging_key = pair
if att.status == "ready":
log.info("event=already_ready attachment_id=%s", attachment_id)
return
# 3) Скачать staging из S3.
try:
raw, ctype = s3.get_object(staging_key)
except Exception as exc: # noqa: BLE001
# Временная ошибка S3 — retry.
raise RuntimeError("staging_unavailable") from exc
# 4) Проверить, что файл всё ещё валиден (worker defense-in-depth).
try:
img = Image.open(io.BytesIO(raw))
img.load()
except Exception as exc: # noqa: BLE001
storage.finalize_failed(attachment_id, "invalid_image")
log.warning("event=invalid_image attachment_id=%s type=%s",
attachment_id, type(exc).__name__)
return
img_rgb = _to_rgb_white(img)
# 5) Записать финальный оригинал.
final_key = s3.original_key(att.chat_id, att.id)
s3.put_bytes(final_key, raw, att.content_type)
variants = [{
"name": "original",
"object_key": final_key,
"content_type": att.content_type,
"size_bytes": len(raw),
"width": img.width,
"height": img.height,
"sha256": _sha256(raw),
}]
# 6) Thumbnail (обязателен).
_, thumb_bytes = _make_thumbnail(img_rgb)
thumb_key = s3.variant_key(att.chat_id, att.id, "thumbnail")
s3.put_bytes(thumb_key, thumb_bytes, "image/jpeg")
tw, th = Image.open(io.BytesIO(thumb_bytes)).size
variants.append({
"name": "thumbnail",
"object_key": thumb_key,
"content_type": "image/jpeg",
"size_bytes": len(thumb_bytes),
"width": tw,
"height": th,
"sha256": _sha256(thumb_bytes),
})
# 7) Crop (level 4).
_, crop_bytes = _make_crop(img_rgb)
crop_key = s3.variant_key(att.chat_id, att.id, "crop")
s3.put_bytes(crop_key, crop_bytes, "image/jpeg")
cw, ch = Image.open(io.BytesIO(crop_bytes)).size
variants.append({
"name": "crop",
"object_key": crop_key,
"content_type": "image/jpeg",
"size_bytes": len(crop_bytes),
"width": cw,
"height": ch,
"sha256": _sha256(crop_bytes),
})
# 8) Optimized WebP (level 4).
_, opt_bytes = _make_optimized(img_rgb)
opt_key = s3.variant_key(att.chat_id, att.id, "optimized")
s3.put_bytes(opt_key, opt_bytes, "image/webp")
ow, oh = Image.open(io.BytesIO(opt_bytes)).size
variants.append({
"name": "optimized",
"object_key": opt_key,
"content_type": "image/webp",
"size_bytes": len(opt_bytes),
"width": ow,
"height": oh,
"sha256": _sha256(opt_bytes),
})
# 9) Атомарно: variants + status=ready.
storage.finalize_ready(attachment_id, pipeline_version, variants)
# 10) Удалить staging после безопасной фиксации.
try:
s3.delete_object(staging_key)
except Exception:
pass
log.info("event=job_ready attachment_id=%s job_id=%s variants=%d",
attachment_id, job_id, len(variants))
def _on_message(ch, method, properties, body):
"""Callback consumer. ack/nack — явно."""
try:
payload = json.loads(body)
except Exception:
# Битый JSON — постоянная ошибка.
log.error("event=invalid_job_json")
ch.basic_nack(method.delivery_tag, requeue=False)
return
headers = dict(properties.headers or {})
retry_count = int(headers.get("x-retry-count", 0))
payload["attempt"] = retry_count + 1
payload["_headers"] = headers
try:
process_job(payload)
ch.basic_ack(method.delivery_tag)
except Exception as exc: # noqa: BLE001
log.warning("event=job_retry attachment_id=%s attempt=%d type=%s",
payload.get("attachment_id"), retry_count + 1, type(exc).__name__)
if retry_count + 1 >= MAX_ATTEMPTS:
# Исчерпаны попытки — failed + DLQ.
try:
storage.finalize_failed(payload["attachment_id"], "retry_exhausted")
except Exception:
pass
try:
publish_dlq({k: v for k, v in payload.items() if k != "_headers"})
except Exception:
pass
ch.basic_ack(method.delivery_tag)
return
# Retry через TTL-очередь, затем ack текущей.
try:
publish_retry(payload, retry_count + 1)
ch.basic_ack(method.delivery_tag)
except Exception:
# Не смогли опубликовать retry — вернуть в очередь.
ch.basic_nack(method.delivery_tag, requeue=True)
def main() -> None:
signal.signal(signal.SIGTERM, _handle_sigterm)
init_pool()
log.info("event=worker_start queue=%s max_attempts=%s", QUEUE, MAX_ATTEMPTS)
while not _stop:
try:
conn = connect()
ch = conn.channel()
setup_topology(ch)
ch.basic_qos(prefetch_count=1)
ch.basic_consume(queue=QUEUE, on_message_callback=_on_message)
log.info("event=consumer_started")
while not _stop and conn.is_open:
conn.process_data_events(time_limit=1)
ch.stop_consuming()
conn.close()
except Exception as exc: # noqa: BLE001
log.error("event=worker_loop_error type=%s", type(exc).__name__)
time.sleep(3)
close_pool()
log.info("event=worker_stop")
if __name__ == "__main__":
main()