Добавила RabbitMQ
This commit is contained in:
+2
-1
@@ -5,4 +5,5 @@ psycopg_pool==3.2.3
|
||||
redis==5.2.0
|
||||
openapi-spec-validator==0.7.2
|
||||
boto3==1.35.74
|
||||
Pillow==11.0.0
|
||||
Pillow==11.0.0
|
||||
pika==1.3.2
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Outbox worker lab5 level 4: publish неопубликованных job в RabbitMQ.
|
||||
|
||||
Закрывает разрыв COMMIT → publish: если API упал после COMMIT, outbox
|
||||
осталась в БД; этот процесс опубликует её после рестарта. Publisher
|
||||
confirm гарантирует приём брокером перед mark published.
|
||||
"""
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
|
||||
from .storage.db import init_pool, close_pool
|
||||
from .storage.postgres import storage
|
||||
from .storage.rabbit import publish_job
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
log = logging.getLogger("outbox")
|
||||
|
||||
_stop = False
|
||||
|
||||
|
||||
def _handle_sigterm(signum, frame):
|
||||
global _stop
|
||||
log.info("event=sigterm_received signal=%s", signum)
|
||||
_stop = True
|
||||
|
||||
|
||||
def main() -> None:
|
||||
signal.signal(signal.SIGTERM, _handle_sigterm)
|
||||
init_pool()
|
||||
log.info("event=outbox_start")
|
||||
|
||||
while not _stop:
|
||||
try:
|
||||
rows = storage.list_unpublished_outbox(limit=50)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.error("event=outbox_select_failed type=%s msg=%s",
|
||||
type(exc).__name__, str(exc)[:300])
|
||||
time.sleep(2)
|
||||
continue
|
||||
|
||||
for outbox_id, payload in rows:
|
||||
if _stop:
|
||||
break
|
||||
try:
|
||||
publish_job(payload)
|
||||
storage.mark_outbox_published(outbox_id)
|
||||
log.info("event=outbox_published id=%s job_id=%s",
|
||||
outbox_id, payload.get("job_id"))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning(
|
||||
"event=outbox_publish_failed id=%s type=%s msg=%s",
|
||||
outbox_id, type(exc).__name__, str(exc)[:300],
|
||||
)
|
||||
# не выходим, попробуем в следующем цикле
|
||||
time.sleep(2)
|
||||
break
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
close_pool()
|
||||
log.info("event=outbox_stop")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except Exception:
|
||||
log.error("event=outbox_fatal\n%s", traceback.format_exc())
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,156 @@
|
||||
"""RabbitMQ: topology, publisher with confirms, consumer с manual ack.
|
||||
|
||||
- Основная очередь: durable quorum queue.
|
||||
- Retry: TTL-очереди с dead-letter обратно в основную; x-retry-count.
|
||||
- DLQ: финальные ошибки и исчерпанные retry.
|
||||
- Publisher confirms: publish() ждёт подтверждения брокера.
|
||||
|
||||
ВАЖНО: setup_topology вызывается в worker при старте и в outbox
|
||||
однократно при первом publish. Не декларируем топологию на каждый
|
||||
publish — иначе гонка с параллельными декларациями.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
import pika
|
||||
|
||||
log = logging.getLogger("rabbit")
|
||||
|
||||
EXCHANGE = "messenger.attachments"
|
||||
QUEUE = "attachment.process.v1"
|
||||
RETRY_EXCHANGE = "messenger.attachments.retry"
|
||||
DLX = "messenger.attachments.dlx"
|
||||
DLQ = "attachment.process.v1.dlq"
|
||||
|
||||
RETRY_DELAYS = [5, 15, 60]
|
||||
MAX_ATTEMPTS = len(RETRY_DELAYS) + 1
|
||||
|
||||
_topology_ready = False
|
||||
_topology_lock = threading.Lock()
|
||||
|
||||
|
||||
def _url() -> str:
|
||||
return os.environ.get("RABBITMQ_URL", "amqp://guest:guest@rabbitmq:5672/%2F")
|
||||
|
||||
|
||||
def connection_params() -> pika.ConnectionParameters:
|
||||
return pika.URLParameters(_url())
|
||||
|
||||
|
||||
def connect() -> pika.BlockingConnection:
|
||||
return pika.BlockingConnection(connection_params())
|
||||
|
||||
|
||||
def setup_topology(channel: pika.adapters.blocking_connection.BlockingChannel) -> None:
|
||||
channel.exchange_declare(EXCHANGE, exchange_type="direct", durable=True)
|
||||
channel.exchange_declare(RETRY_EXCHANGE, exchange_type="direct", durable=True)
|
||||
channel.exchange_declare(DLX, exchange_type="direct", durable=True)
|
||||
|
||||
channel.queue_declare(DLQ, durable=True, arguments={"x-queue-type": "quorum"})
|
||||
channel.queue_bind(DLQ, DLX, routing_key=DLQ)
|
||||
|
||||
channel.queue_declare(
|
||||
QUEUE,
|
||||
durable=True,
|
||||
arguments={
|
||||
"x-queue-type": "quorum",
|
||||
"x-dead-letter-exchange": DLX,
|
||||
"x-dead-letter-routing-key": DLQ,
|
||||
},
|
||||
)
|
||||
channel.queue_bind(QUEUE, EXCHANGE, routing_key=QUEUE)
|
||||
|
||||
for delay in RETRY_DELAYS:
|
||||
qname = f"attachment.retry.{delay}s"
|
||||
channel.queue_declare(
|
||||
qname,
|
||||
durable=True,
|
||||
arguments={
|
||||
"x-message-ttl": delay * 1000,
|
||||
"x-dead-letter-exchange": EXCHANGE,
|
||||
"x-dead-letter-routing-key": QUEUE,
|
||||
},
|
||||
)
|
||||
channel.queue_bind(qname, RETRY_EXCHANGE, routing_key=qname)
|
||||
|
||||
|
||||
def _ensure_topology(ch: pika.adapters.blocking_connection.BlockingChannel) -> None:
|
||||
global _topology_ready
|
||||
if _topology_ready:
|
||||
return
|
||||
with _topology_lock:
|
||||
if _topology_ready:
|
||||
return
|
||||
setup_topology(ch)
|
||||
_topology_ready = True
|
||||
|
||||
|
||||
def publish_job(payload: dict) -> None:
|
||||
"""Публикация job с publisher confirm. Бросает при неудаче."""
|
||||
conn = connect()
|
||||
try:
|
||||
ch = conn.channel()
|
||||
ch.confirm_delivery()
|
||||
_ensure_topology(ch)
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
||||
ch.basic_publish(
|
||||
exchange=EXCHANGE,
|
||||
routing_key=QUEUE,
|
||||
body=body.encode("utf-8"),
|
||||
properties=pika.BasicProperties(
|
||||
delivery_mode=2,
|
||||
content_type="application/json",
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def publish_retry(payload: dict, retry_count: int) -> None:
|
||||
idx = min(retry_count - 1, len(RETRY_DELAYS) - 1)
|
||||
delay = RETRY_DELAYS[idx]
|
||||
qname = f"attachment.retry.{delay}s"
|
||||
conn = connect()
|
||||
try:
|
||||
ch = conn.channel()
|
||||
ch.confirm_delivery()
|
||||
_ensure_topology(ch)
|
||||
headers = dict(payload.get("_headers") or {})
|
||||
headers["x-retry-count"] = retry_count
|
||||
clean = {k: v for k, v in payload.items() if k != "_headers"}
|
||||
body = json.dumps(clean, ensure_ascii=False, separators=(",", ":"))
|
||||
ch.basic_publish(
|
||||
exchange=RETRY_EXCHANGE,
|
||||
routing_key=qname,
|
||||
body=body.encode("utf-8"),
|
||||
properties=pika.BasicProperties(
|
||||
delivery_mode=2,
|
||||
content_type="application/json",
|
||||
headers=headers,
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def publish_dlq(payload: dict) -> None:
|
||||
conn = connect()
|
||||
try:
|
||||
ch = conn.channel()
|
||||
ch.confirm_delivery()
|
||||
_ensure_topology(ch)
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
||||
ch.basic_publish(
|
||||
exchange=DLX,
|
||||
routing_key=DLQ,
|
||||
body=body.encode("utf-8"),
|
||||
properties=pika.BasicProperties(delivery_mode=2, content_type="application/json"),
|
||||
mandatory=True,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -0,0 +1,296 @@
|
||||
"""Worker lab5: обработка вложений из RabbitMQ.
|
||||
|
||||
- at-least-once: manual ack после durable эффекта.
|
||||
- идемпотентность: ключ (attachment_id, pipeline_version, variant).
|
||||
- retry с backoff через TTL retry-очереди, затем DLQ.
|
||||
- lease: processing_until, зависшие возвращаются в queued.
|
||||
- thumbnail 256×256 (JPEG), crop 128×128 (JPEG), optimized WebP 1280×1280.
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pika
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
from .storage import s3
|
||||
from .storage.db import init_pool, close_pool
|
||||
from .storage.postgres import storage
|
||||
from .storage.rabbit import (
|
||||
QUEUE, EXCHANGE, MAX_ATTEMPTS, connect, setup_topology,
|
||||
publish_retry, publish_dlq,
|
||||
)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
log = logging.getLogger("worker")
|
||||
|
||||
PIPELINE_VERSION = 1
|
||||
LEASE_SECONDS = 60
|
||||
THUMB_SIZE = (256, 256)
|
||||
CROP_SIZE = (128, 128)
|
||||
OPT_SIZE = (1280, 1280)
|
||||
OPT_QUALITY = 82
|
||||
|
||||
_stop = False
|
||||
|
||||
|
||||
def _handle_sigterm(signum, frame):
|
||||
global _stop
|
||||
log.info("event=sigterm_received signal=%s", signum)
|
||||
_stop = True
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _to_rgb_white(img: Image.Image) -> Image.Image:
|
||||
"""Alpha компонуется на белый; EXIF orientation применяется."""
|
||||
img = ImageOps.exif_transpose(img)
|
||||
if img.mode in ("RGBA", "LA", "P"):
|
||||
img = img.convert("RGBA")
|
||||
bg = Image.new("RGB", img.size, (255, 255, 255))
|
||||
bg.paste(img, mask=img.split()[-1])
|
||||
return bg
|
||||
if img.mode != "RGB":
|
||||
return img.convert("RGB")
|
||||
return img
|
||||
|
||||
|
||||
def _make_thumbnail(img: Image.Image) -> tuple[Image.Image, bytes]:
|
||||
"""Вписать в 256×256 без увеличения. Округление вниз, минимум 1 px."""
|
||||
w, h = img.size
|
||||
scale = min(THUMB_SIZE[0] / w, THUMB_SIZE[1] / h, 1.0)
|
||||
nw = max(1, int(w * scale))
|
||||
nh = max(1, int(h * scale))
|
||||
thumb = img.resize((nw, nh), Image.LANCZOS)
|
||||
buf = io.BytesIO()
|
||||
thumb.save(buf, format="JPEG", quality=88, optimize=True)
|
||||
return thumb, buf.getvalue()
|
||||
|
||||
|
||||
def _make_crop(img: Image.Image) -> tuple[Image.Image, bytes]:
|
||||
"""Центрированный квадрат 128×128 (маленькие допускают увеличение)."""
|
||||
w, h = img.size
|
||||
side = min(w, h)
|
||||
left = (w - side) // 2
|
||||
top = (h - side) // 2
|
||||
sq = img.crop((left, top, left + side, top + side))
|
||||
crop = sq.resize(CROP_SIZE, Image.LANCZOS)
|
||||
buf = io.BytesIO()
|
||||
crop.save(buf, format="JPEG", quality=88, optimize=True)
|
||||
return crop, buf.getvalue()
|
||||
|
||||
|
||||
def _make_optimized(img: Image.Image) -> tuple[Image.Image, bytes]:
|
||||
"""WebP, вписан в 1280×1280 без увеличения, EXIF удалён."""
|
||||
w, h = img.size
|
||||
scale = min(OPT_SIZE[0] / w, OPT_SIZE[1] / h, 1.0)
|
||||
nw = max(1, int(w * scale))
|
||||
nh = max(1, int(h * scale))
|
||||
opt = img.resize((nw, nh), Image.LANCZOS)
|
||||
buf = io.BytesIO()
|
||||
opt.save(buf, format="WEBP", quality=OPT_QUALITY, method=4)
|
||||
return opt, buf.getvalue()
|
||||
|
||||
|
||||
def _sha256(data: bytes) -> str:
|
||||
import hashlib
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def process_job(payload: dict) -> None:
|
||||
"""Обработка одного job. Идемпотентна: finalize_ready использует ON CONFLICT."""
|
||||
attachment_id = payload["attachment_id"]
|
||||
pipeline_version = payload.get("pipeline_version", 1)
|
||||
job_id = payload["job_id"]
|
||||
|
||||
log.info("event=job_received job_id=%s attachment_id=%s attempt=%s",
|
||||
job_id, attachment_id, payload.get("attempt", 1))
|
||||
|
||||
# 1) Захватить lease. Если уже ready/failed — no-op (дубль).
|
||||
if not storage.claim_processing(attachment_id, LEASE_SECONDS):
|
||||
log.info("event=job_skipped_already_done attachment_id=%s", attachment_id)
|
||||
return
|
||||
|
||||
# 2) Получить staging key и метаданные.
|
||||
pair = storage.get_attachment_with_key(attachment_id)
|
||||
if pair is None:
|
||||
log.warning("event=attachment_missing attachment_id=%s", attachment_id)
|
||||
return
|
||||
att, staging_key = pair
|
||||
if att.status == "ready":
|
||||
log.info("event=already_ready attachment_id=%s", attachment_id)
|
||||
return
|
||||
|
||||
# 3) Скачать staging из S3.
|
||||
try:
|
||||
raw, ctype = s3.get_object(staging_key)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# Временная ошибка S3 — retry.
|
||||
raise RuntimeError("staging_unavailable") from exc
|
||||
|
||||
# 4) Проверить, что файл всё ещё валиден (worker defense-in-depth).
|
||||
try:
|
||||
img = Image.open(io.BytesIO(raw))
|
||||
img.load()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
storage.finalize_failed(attachment_id, "invalid_image")
|
||||
log.warning("event=invalid_image attachment_id=%s type=%s",
|
||||
attachment_id, type(exc).__name__)
|
||||
return
|
||||
|
||||
img_rgb = _to_rgb_white(img)
|
||||
|
||||
# 5) Записать финальный оригинал.
|
||||
final_key = s3.original_key(att.chat_id, att.id)
|
||||
s3.put_bytes(final_key, raw, att.content_type)
|
||||
|
||||
variants = [{
|
||||
"name": "original",
|
||||
"object_key": final_key,
|
||||
"content_type": att.content_type,
|
||||
"size_bytes": len(raw),
|
||||
"width": img.width,
|
||||
"height": img.height,
|
||||
"sha256": _sha256(raw),
|
||||
}]
|
||||
|
||||
# 6) Thumbnail (обязателен).
|
||||
_, thumb_bytes = _make_thumbnail(img_rgb)
|
||||
thumb_key = s3.variant_key(att.chat_id, att.id, "thumbnail")
|
||||
s3.put_bytes(thumb_key, thumb_bytes, "image/jpeg")
|
||||
tw, th = Image.open(io.BytesIO(thumb_bytes)).size
|
||||
variants.append({
|
||||
"name": "thumbnail",
|
||||
"object_key": thumb_key,
|
||||
"content_type": "image/jpeg",
|
||||
"size_bytes": len(thumb_bytes),
|
||||
"width": tw,
|
||||
"height": th,
|
||||
"sha256": _sha256(thumb_bytes),
|
||||
})
|
||||
|
||||
# 7) Crop (level 4).
|
||||
_, crop_bytes = _make_crop(img_rgb)
|
||||
crop_key = s3.variant_key(att.chat_id, att.id, "crop")
|
||||
s3.put_bytes(crop_key, crop_bytes, "image/jpeg")
|
||||
cw, ch = Image.open(io.BytesIO(crop_bytes)).size
|
||||
variants.append({
|
||||
"name": "crop",
|
||||
"object_key": crop_key,
|
||||
"content_type": "image/jpeg",
|
||||
"size_bytes": len(crop_bytes),
|
||||
"width": cw,
|
||||
"height": ch,
|
||||
"sha256": _sha256(crop_bytes),
|
||||
})
|
||||
|
||||
# 8) Optimized WebP (level 4).
|
||||
_, opt_bytes = _make_optimized(img_rgb)
|
||||
opt_key = s3.variant_key(att.chat_id, att.id, "optimized")
|
||||
s3.put_bytes(opt_key, opt_bytes, "image/webp")
|
||||
ow, oh = Image.open(io.BytesIO(opt_bytes)).size
|
||||
variants.append({
|
||||
"name": "optimized",
|
||||
"object_key": opt_key,
|
||||
"content_type": "image/webp",
|
||||
"size_bytes": len(opt_bytes),
|
||||
"width": ow,
|
||||
"height": oh,
|
||||
"sha256": _sha256(opt_bytes),
|
||||
})
|
||||
|
||||
# 9) Атомарно: variants + status=ready.
|
||||
storage.finalize_ready(attachment_id, pipeline_version, variants)
|
||||
|
||||
# 10) Удалить staging после безопасной фиксации.
|
||||
try:
|
||||
s3.delete_object(staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
log.info("event=job_ready attachment_id=%s job_id=%s variants=%d",
|
||||
attachment_id, job_id, len(variants))
|
||||
|
||||
|
||||
def _on_message(ch, method, properties, body):
|
||||
"""Callback consumer. ack/nack — явно."""
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except Exception:
|
||||
# Битый JSON — постоянная ошибка.
|
||||
log.error("event=invalid_job_json")
|
||||
ch.basic_nack(method.delivery_tag, requeue=False)
|
||||
return
|
||||
|
||||
headers = dict(properties.headers or {})
|
||||
retry_count = int(headers.get("x-retry-count", 0))
|
||||
payload["attempt"] = retry_count + 1
|
||||
payload["_headers"] = headers
|
||||
|
||||
try:
|
||||
process_job(payload)
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("event=job_retry attachment_id=%s attempt=%d type=%s",
|
||||
payload.get("attachment_id"), retry_count + 1, type(exc).__name__)
|
||||
if retry_count + 1 >= MAX_ATTEMPTS:
|
||||
# Исчерпаны попытки — failed + DLQ.
|
||||
try:
|
||||
storage.finalize_failed(payload["attachment_id"], "retry_exhausted")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
publish_dlq({k: v for k, v in payload.items() if k != "_headers"})
|
||||
except Exception:
|
||||
pass
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
return
|
||||
# Retry через TTL-очередь, затем ack текущей.
|
||||
try:
|
||||
publish_retry(payload, retry_count + 1)
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
except Exception:
|
||||
# Не смогли опубликовать retry — вернуть в очередь.
|
||||
ch.basic_nack(method.delivery_tag, requeue=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
signal.signal(signal.SIGTERM, _handle_sigterm)
|
||||
init_pool()
|
||||
log.info("event=worker_start queue=%s max_attempts=%s", QUEUE, MAX_ATTEMPTS)
|
||||
|
||||
while not _stop:
|
||||
try:
|
||||
conn = connect()
|
||||
ch = conn.channel()
|
||||
setup_topology(ch)
|
||||
ch.basic_qos(prefetch_count=1)
|
||||
ch.basic_consume(queue=QUEUE, on_message_callback=_on_message)
|
||||
log.info("event=consumer_started")
|
||||
|
||||
while not _stop and conn.is_open:
|
||||
conn.process_data_events(time_limit=1)
|
||||
|
||||
ch.stop_consuming()
|
||||
conn.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.error("event=worker_loop_error type=%s", type(exc).__name__)
|
||||
time.sleep(3)
|
||||
|
||||
close_pool()
|
||||
log.info("event=worker_stop")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user