317 lines
11 KiB
Python
317 lines
11 KiB
Python
"""Worker lab5/lab6: обработка вложений из RabbitMQ с трейсингом.
|
|
|
|
- at-least-once, идемпотентно, retry через TTL-очереди, DLQ.
|
|
- OTel: consumer span на каждый job, извлечение trace context из заголовков
|
|
сообщения (W3C traceparent/tracestate).
|
|
- Метрики: jobs_processed_total{outcome}, job_processing_duration_seconds.
|
|
"""
|
|
import io
|
|
import json
|
|
import logging
|
|
import os
|
|
import signal
|
|
import sys
|
|
import time
|
|
|
|
import pika
|
|
from PIL import Image, ImageOps
|
|
from opentelemetry import context as otel_context, trace
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
|
|
|
from .storage import s3
|
|
from .storage.db import init_pool, close_pool
|
|
from .storage.postgres import storage
|
|
from .storage.rabbit import (
|
|
QUEUE, EXCHANGE, MAX_ATTEMPTS, connect, setup_topology,
|
|
publish_retry, publish_dlq,
|
|
)
|
|
from .telemetry import init_telemetry, meter
|
|
|
|
# Базовый логгер настраивается в telemetry.install_log_format() при старте
|
|
# init_telemetry: trace_id/span_id подставляются фильтром.
|
|
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
|
|
log = logging.getLogger("worker")
|
|
|
|
PIPELINE_VERSION = 1
|
|
LEASE_SECONDS = 60
|
|
THUMB_SIZE = (256, 256)
|
|
CROP_SIZE = (128, 128)
|
|
OPT_SIZE = (1280, 1280)
|
|
OPT_QUALITY = 82
|
|
|
|
_stop = False
|
|
_propagator = TraceContextTextMapPropagator()
|
|
|
|
|
|
# OTel инициализация — до первого сообщения.
|
|
init_telemetry("messenger-worker", "1.6.0")
|
|
_tracer = trace.get_tracer("worker")
|
|
_meter = meter("messenger-worker")
|
|
_jobs_counter = _meter.create_counter(
|
|
"jobs_processed_total", description="Jobs processed by outcome", unit="1",
|
|
)
|
|
_jobs_hist = _meter.create_histogram(
|
|
"job_processing_duration_seconds", description="Job processing duration", unit="s",
|
|
)
|
|
|
|
|
|
def _handle_sigterm(signum, frame):
|
|
global _stop
|
|
log.info("event=sigterm_received signal=%s", signum)
|
|
_stop = True
|
|
|
|
|
|
def _now_iso() -> str:
|
|
from datetime import datetime, timezone
|
|
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def _to_rgb_white(img: Image.Image) -> Image.Image:
|
|
img = ImageOps.exif_transpose(img)
|
|
if img.mode in ("RGBA", "LA", "P"):
|
|
img = img.convert("RGBA")
|
|
bg = Image.new("RGB", img.size, (255, 255, 255))
|
|
bg.paste(img, mask=img.split()[-1])
|
|
return bg
|
|
if img.mode != "RGB":
|
|
return img.convert("RGB")
|
|
return img
|
|
|
|
|
|
def _make_thumbnail(img: Image.Image) -> tuple[Image.Image, bytes]:
|
|
w, h = img.size
|
|
scale = min(THUMB_SIZE[0] / w, THUMB_SIZE[1] / h, 1.0)
|
|
nw = max(1, int(w * scale))
|
|
nh = max(1, int(h * scale))
|
|
thumb = img.resize((nw, nh), Image.LANCZOS)
|
|
buf = io.BytesIO()
|
|
thumb.save(buf, format="JPEG", quality=88, optimize=True)
|
|
return thumb, buf.getvalue()
|
|
|
|
|
|
def _make_crop(img: Image.Image) -> tuple[Image.Image, bytes]:
|
|
w, h = img.size
|
|
side = min(w, h)
|
|
left = (w - side) // 2
|
|
top = (h - side) // 2
|
|
sq = img.crop((left, top, left + side, top + side))
|
|
crop = sq.resize(CROP_SIZE, Image.LANCZOS)
|
|
buf = io.BytesIO()
|
|
crop.save(buf, format="JPEG", quality=88, optimize=True)
|
|
return crop, buf.getvalue()
|
|
|
|
|
|
def _make_optimized(img: Image.Image) -> tuple[Image.Image, bytes]:
|
|
w, h = img.size
|
|
scale = min(OPT_SIZE[0] / w, OPT_SIZE[1] / h, 1.0)
|
|
nw = max(1, int(w * scale))
|
|
nh = max(1, int(h * scale))
|
|
opt = img.resize((nw, nh), Image.LANCZOS)
|
|
buf = io.BytesIO()
|
|
opt.save(buf, format="WEBP", quality=OPT_QUALITY, method=4)
|
|
return opt, buf.getvalue()
|
|
|
|
|
|
def _sha256(data: bytes) -> str:
|
|
import hashlib
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def process_job(payload: dict) -> None:
|
|
"""Обработка одного job. Идемпотентна через finalize_ready."""
|
|
attachment_id = payload["attachment_id"]
|
|
pipeline_version = payload.get("pipeline_version", 1)
|
|
job_id = payload["job_id"]
|
|
|
|
log.info("event=job_received job_id=%s attachment_id=%s attempt=%s",
|
|
job_id, attachment_id, payload.get("attempt", 1))
|
|
|
|
if not storage.claim_processing(attachment_id, LEASE_SECONDS):
|
|
log.info("event=job_skipped_already_done attachment_id=%s", attachment_id)
|
|
return
|
|
|
|
pair = storage.get_attachment_with_key(attachment_id)
|
|
if pair is None:
|
|
log.warning("event=attachment_missing attachment_id=%s", attachment_id)
|
|
return
|
|
att, staging_key = pair
|
|
if att.status == "ready":
|
|
log.info("event=already_ready attachment_id=%s", attachment_id)
|
|
return
|
|
|
|
try:
|
|
raw, ctype = s3.get_object(staging_key)
|
|
except Exception as exc: # noqa: BLE001
|
|
raise RuntimeError("staging_unavailable") from exc
|
|
|
|
try:
|
|
img = Image.open(io.BytesIO(raw))
|
|
img.load()
|
|
except Exception as exc: # noqa: BLE001
|
|
storage.finalize_failed(attachment_id, "invalid_image")
|
|
log.warning("event=invalid_image attachment_id=%s type=%s",
|
|
attachment_id, type(exc).__name__)
|
|
return
|
|
|
|
img_rgb = _to_rgb_white(img)
|
|
final_key = s3.original_key(att.chat_id, att.id)
|
|
s3.put_bytes(final_key, raw, att.content_type)
|
|
|
|
variants = [{
|
|
"name": "original",
|
|
"object_key": final_key,
|
|
"content_type": att.content_type,
|
|
"size_bytes": len(raw),
|
|
"width": img.width,
|
|
"height": img.height,
|
|
"sha256": _sha256(raw),
|
|
}]
|
|
|
|
_, thumb_bytes = _make_thumbnail(img_rgb)
|
|
thumb_key = s3.variant_key(att.chat_id, att.id, "thumbnail")
|
|
s3.put_bytes(thumb_key, thumb_bytes, "image/jpeg")
|
|
tw, th = Image.open(io.BytesIO(thumb_bytes)).size
|
|
variants.append({
|
|
"name": "thumbnail", "object_key": thumb_key,
|
|
"content_type": "image/jpeg", "size_bytes": len(thumb_bytes),
|
|
"width": tw, "height": th, "sha256": _sha256(thumb_bytes),
|
|
})
|
|
|
|
_, crop_bytes = _make_crop(img_rgb)
|
|
crop_key = s3.variant_key(att.chat_id, att.id, "crop")
|
|
s3.put_bytes(crop_key, crop_bytes, "image/jpeg")
|
|
cw, ch = Image.open(io.BytesIO(crop_bytes)).size
|
|
variants.append({
|
|
"name": "crop", "object_key": crop_key,
|
|
"content_type": "image/jpeg", "size_bytes": len(crop_bytes),
|
|
"width": cw, "height": ch, "sha256": _sha256(crop_bytes),
|
|
})
|
|
|
|
_, opt_bytes = _make_optimized(img_rgb)
|
|
opt_key = s3.variant_key(att.chat_id, att.id, "optimized")
|
|
s3.put_bytes(opt_key, opt_bytes, "image/webp")
|
|
ow, oh = Image.open(io.BytesIO(opt_bytes)).size
|
|
variants.append({
|
|
"name": "optimized", "object_key": opt_key,
|
|
"content_type": "image/webp", "size_bytes": len(opt_bytes),
|
|
"width": ow, "height": oh, "sha256": _sha256(opt_bytes),
|
|
})
|
|
|
|
storage.finalize_ready(attachment_id, pipeline_version, variants)
|
|
|
|
try:
|
|
s3.delete_object(staging_key)
|
|
except Exception:
|
|
pass
|
|
|
|
log.info("event=job_ready attachment_id=%s job_id=%s variants=%d",
|
|
attachment_id, job_id, len(variants))
|
|
|
|
|
|
def _extract_trace_context(properties) -> otel_context.Context:
|
|
"""Извлекаем W3C traceparent/tracestate из headers. Ошибки → пустой контекст."""
|
|
try:
|
|
headers = dict(properties.headers or {})
|
|
carrier = {
|
|
k.lower(): str(v)
|
|
for k, v in headers.items()
|
|
if k.lower() in ("traceparent", "tracestate")
|
|
}
|
|
if not carrier:
|
|
return otel_context.Context()
|
|
return _propagator.extract(carrier=carrier)
|
|
except Exception as exc: # noqa: BLE001
|
|
log.warning("event=trace_extract_failed type=%s", type(exc).__name__)
|
|
return otel_context.Context()
|
|
|
|
|
|
def _on_message(ch, method, properties, body):
|
|
"""Callback consumer. ack/nack — явно. Обёрнуто в consumer span."""
|
|
parent_ctx = _extract_trace_context(properties)
|
|
started = time.monotonic()
|
|
with _tracer.start_as_current_span(
|
|
"job.consume", context=parent_ctx, kind=SpanKind.CONSUMER,
|
|
attributes={"messaging.system": "rabbitmq",
|
|
"messaging.destination.name": QUEUE},
|
|
) as span:
|
|
try:
|
|
payload = json.loads(body)
|
|
except Exception:
|
|
log.error("event=invalid_job_json")
|
|
_jobs_counter.add(1, {"job.type": "unknown", "outcome": "invalid"})
|
|
ch.basic_nack(method.delivery_tag, requeue=False)
|
|
return
|
|
|
|
headers = dict(properties.headers or {})
|
|
retry_count = int(headers.get("x-retry-count", 0))
|
|
payload["attempt"] = retry_count + 1
|
|
payload["_headers"] = headers
|
|
span.set_attribute("job.id", str(payload.get("job_id", "")))
|
|
span.set_attribute("job.type", str(payload.get("type", "unknown")))
|
|
span.set_attribute("job.attempt", retry_count + 1)
|
|
|
|
try:
|
|
process_job(payload)
|
|
duration = time.monotonic() - started
|
|
_jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "success"})
|
|
_jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "success"})
|
|
span.set_status(Status(StatusCode.OK))
|
|
ch.basic_ack(method.delivery_tag)
|
|
except Exception as exc: # noqa: BLE001
|
|
duration = time.monotonic() - started
|
|
span.record_exception(exc)
|
|
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
|
|
log.warning("event=job_retry attachment_id=%s attempt=%d type=%s",
|
|
payload.get("attachment_id"), retry_count + 1, type(exc).__name__)
|
|
if retry_count + 1 >= MAX_ATTEMPTS:
|
|
try:
|
|
storage.finalize_failed(payload["attachment_id"], "retry_exhausted")
|
|
except Exception:
|
|
pass
|
|
try:
|
|
publish_dlq({k: v for k, v in payload.items() if k != "_headers"})
|
|
except Exception:
|
|
pass
|
|
_jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "failed"})
|
|
_jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "failed"})
|
|
ch.basic_ack(method.delivery_tag)
|
|
return
|
|
_jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "retry"})
|
|
_jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "retry"})
|
|
try:
|
|
publish_retry(payload, retry_count + 1)
|
|
ch.basic_ack(method.delivery_tag)
|
|
except Exception:
|
|
ch.basic_nack(method.delivery_tag, requeue=True)
|
|
|
|
|
|
def main() -> None:
|
|
signal.signal(signal.SIGTERM, _handle_sigterm)
|
|
init_pool()
|
|
log.info("event=worker_start queue=%s max_attempts=%s", QUEUE, MAX_ATTEMPTS)
|
|
|
|
while not _stop:
|
|
try:
|
|
conn = connect()
|
|
ch = conn.channel()
|
|
setup_topology(ch)
|
|
ch.basic_qos(prefetch_count=1)
|
|
ch.basic_consume(queue=QUEUE, on_message_callback=_on_message)
|
|
log.info("event=consumer_started")
|
|
|
|
while not _stop and conn.is_open:
|
|
conn.process_data_events(time_limit=1)
|
|
|
|
ch.stop_consuming()
|
|
conn.close()
|
|
except Exception as exc: # noqa: BLE001
|
|
log.error("event=worker_loop_error type=%s", type(exc).__name__)
|
|
time.sleep(3)
|
|
|
|
close_pool()
|
|
log.info("event=worker_stop")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |