Files
mtusi2026lab6/src/app/worker.py
T

317 lines
11 KiB
Python

"""Worker lab5/lab6: обработка вложений из RabbitMQ с трейсингом.
- at-least-once, идемпотентно, retry через TTL-очереди, DLQ.
- OTel: consumer span на каждый job, извлечение trace context из заголовков
сообщения (W3C traceparent/tracestate).
- Метрики: jobs_processed_total{outcome}, job_processing_duration_seconds.
"""
import io
import json
import logging
import os
import signal
import sys
import time
import pika
from PIL import Image, ImageOps
from opentelemetry import context as otel_context, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from .storage import s3
from .storage.db import init_pool, close_pool
from .storage.postgres import storage
from .storage.rabbit import (
QUEUE, EXCHANGE, MAX_ATTEMPTS, connect, setup_topology,
publish_retry, publish_dlq,
)
from .telemetry import init_telemetry, meter
# Базовый логгер настраивается в telemetry.install_log_format() при старте
# init_telemetry: trace_id/span_id подставляются фильтром.
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
log = logging.getLogger("worker")
PIPELINE_VERSION = 1
LEASE_SECONDS = 60
THUMB_SIZE = (256, 256)
CROP_SIZE = (128, 128)
OPT_SIZE = (1280, 1280)
OPT_QUALITY = 82
_stop = False
_propagator = TraceContextTextMapPropagator()
# OTel инициализация — до первого сообщения.
init_telemetry("messenger-worker", "1.6.0")
_tracer = trace.get_tracer("worker")
_meter = meter("messenger-worker")
_jobs_counter = _meter.create_counter(
"jobs_processed_total", description="Jobs processed by outcome", unit="1",
)
_jobs_hist = _meter.create_histogram(
"job_processing_duration_seconds", description="Job processing duration", unit="s",
)
def _handle_sigterm(signum, frame):
global _stop
log.info("event=sigterm_received signal=%s", signum)
_stop = True
def _now_iso() -> str:
from datetime import datetime, timezone
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
def _to_rgb_white(img: Image.Image) -> Image.Image:
img = ImageOps.exif_transpose(img)
if img.mode in ("RGBA", "LA", "P"):
img = img.convert("RGBA")
bg = Image.new("RGB", img.size, (255, 255, 255))
bg.paste(img, mask=img.split()[-1])
return bg
if img.mode != "RGB":
return img.convert("RGB")
return img
def _make_thumbnail(img: Image.Image) -> tuple[Image.Image, bytes]:
w, h = img.size
scale = min(THUMB_SIZE[0] / w, THUMB_SIZE[1] / h, 1.0)
nw = max(1, int(w * scale))
nh = max(1, int(h * scale))
thumb = img.resize((nw, nh), Image.LANCZOS)
buf = io.BytesIO()
thumb.save(buf, format="JPEG", quality=88, optimize=True)
return thumb, buf.getvalue()
def _make_crop(img: Image.Image) -> tuple[Image.Image, bytes]:
w, h = img.size
side = min(w, h)
left = (w - side) // 2
top = (h - side) // 2
sq = img.crop((left, top, left + side, top + side))
crop = sq.resize(CROP_SIZE, Image.LANCZOS)
buf = io.BytesIO()
crop.save(buf, format="JPEG", quality=88, optimize=True)
return crop, buf.getvalue()
def _make_optimized(img: Image.Image) -> tuple[Image.Image, bytes]:
w, h = img.size
scale = min(OPT_SIZE[0] / w, OPT_SIZE[1] / h, 1.0)
nw = max(1, int(w * scale))
nh = max(1, int(h * scale))
opt = img.resize((nw, nh), Image.LANCZOS)
buf = io.BytesIO()
opt.save(buf, format="WEBP", quality=OPT_QUALITY, method=4)
return opt, buf.getvalue()
def _sha256(data: bytes) -> str:
import hashlib
return hashlib.sha256(data).hexdigest()
def process_job(payload: dict) -> None:
"""Обработка одного job. Идемпотентна через finalize_ready."""
attachment_id = payload["attachment_id"]
pipeline_version = payload.get("pipeline_version", 1)
job_id = payload["job_id"]
log.info("event=job_received job_id=%s attachment_id=%s attempt=%s",
job_id, attachment_id, payload.get("attempt", 1))
if not storage.claim_processing(attachment_id, LEASE_SECONDS):
log.info("event=job_skipped_already_done attachment_id=%s", attachment_id)
return
pair = storage.get_attachment_with_key(attachment_id)
if pair is None:
log.warning("event=attachment_missing attachment_id=%s", attachment_id)
return
att, staging_key = pair
if att.status == "ready":
log.info("event=already_ready attachment_id=%s", attachment_id)
return
try:
raw, ctype = s3.get_object(staging_key)
except Exception as exc: # noqa: BLE001
raise RuntimeError("staging_unavailable") from exc
try:
img = Image.open(io.BytesIO(raw))
img.load()
except Exception as exc: # noqa: BLE001
storage.finalize_failed(attachment_id, "invalid_image")
log.warning("event=invalid_image attachment_id=%s type=%s",
attachment_id, type(exc).__name__)
return
img_rgb = _to_rgb_white(img)
final_key = s3.original_key(att.chat_id, att.id)
s3.put_bytes(final_key, raw, att.content_type)
variants = [{
"name": "original",
"object_key": final_key,
"content_type": att.content_type,
"size_bytes": len(raw),
"width": img.width,
"height": img.height,
"sha256": _sha256(raw),
}]
_, thumb_bytes = _make_thumbnail(img_rgb)
thumb_key = s3.variant_key(att.chat_id, att.id, "thumbnail")
s3.put_bytes(thumb_key, thumb_bytes, "image/jpeg")
tw, th = Image.open(io.BytesIO(thumb_bytes)).size
variants.append({
"name": "thumbnail", "object_key": thumb_key,
"content_type": "image/jpeg", "size_bytes": len(thumb_bytes),
"width": tw, "height": th, "sha256": _sha256(thumb_bytes),
})
_, crop_bytes = _make_crop(img_rgb)
crop_key = s3.variant_key(att.chat_id, att.id, "crop")
s3.put_bytes(crop_key, crop_bytes, "image/jpeg")
cw, ch = Image.open(io.BytesIO(crop_bytes)).size
variants.append({
"name": "crop", "object_key": crop_key,
"content_type": "image/jpeg", "size_bytes": len(crop_bytes),
"width": cw, "height": ch, "sha256": _sha256(crop_bytes),
})
_, opt_bytes = _make_optimized(img_rgb)
opt_key = s3.variant_key(att.chat_id, att.id, "optimized")
s3.put_bytes(opt_key, opt_bytes, "image/webp")
ow, oh = Image.open(io.BytesIO(opt_bytes)).size
variants.append({
"name": "optimized", "object_key": opt_key,
"content_type": "image/webp", "size_bytes": len(opt_bytes),
"width": ow, "height": oh, "sha256": _sha256(opt_bytes),
})
storage.finalize_ready(attachment_id, pipeline_version, variants)
try:
s3.delete_object(staging_key)
except Exception:
pass
log.info("event=job_ready attachment_id=%s job_id=%s variants=%d",
attachment_id, job_id, len(variants))
def _extract_trace_context(properties) -> otel_context.Context:
"""Извлекаем W3C traceparent/tracestate из headers. Ошибки → пустой контекст."""
try:
headers = dict(properties.headers or {})
carrier = {
k.lower(): str(v)
for k, v in headers.items()
if k.lower() in ("traceparent", "tracestate")
}
if not carrier:
return otel_context.Context()
return _propagator.extract(carrier=carrier)
except Exception as exc: # noqa: BLE001
log.warning("event=trace_extract_failed type=%s", type(exc).__name__)
return otel_context.Context()
def _on_message(ch, method, properties, body):
"""Callback consumer. ack/nack — явно. Обёрнуто в consumer span."""
parent_ctx = _extract_trace_context(properties)
started = time.monotonic()
with _tracer.start_as_current_span(
"job.consume", context=parent_ctx, kind=SpanKind.CONSUMER,
attributes={"messaging.system": "rabbitmq",
"messaging.destination.name": QUEUE},
) as span:
try:
payload = json.loads(body)
except Exception:
log.error("event=invalid_job_json")
_jobs_counter.add(1, {"job.type": "unknown", "outcome": "invalid"})
ch.basic_nack(method.delivery_tag, requeue=False)
return
headers = dict(properties.headers or {})
retry_count = int(headers.get("x-retry-count", 0))
payload["attempt"] = retry_count + 1
payload["_headers"] = headers
span.set_attribute("job.id", str(payload.get("job_id", "")))
span.set_attribute("job.type", str(payload.get("type", "unknown")))
span.set_attribute("job.attempt", retry_count + 1)
try:
process_job(payload)
duration = time.monotonic() - started
_jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "success"})
_jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "success"})
span.set_status(Status(StatusCode.OK))
ch.basic_ack(method.delivery_tag)
except Exception as exc: # noqa: BLE001
duration = time.monotonic() - started
span.record_exception(exc)
span.set_status(Status(StatusCode.ERROR, type(exc).__name__))
log.warning("event=job_retry attachment_id=%s attempt=%d type=%s",
payload.get("attachment_id"), retry_count + 1, type(exc).__name__)
if retry_count + 1 >= MAX_ATTEMPTS:
try:
storage.finalize_failed(payload["attachment_id"], "retry_exhausted")
except Exception:
pass
try:
publish_dlq({k: v for k, v in payload.items() if k != "_headers"})
except Exception:
pass
_jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "failed"})
_jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "failed"})
ch.basic_ack(method.delivery_tag)
return
_jobs_counter.add(1, {"job.type": "attachment.process.v1", "outcome": "retry"})
_jobs_hist.record(duration, {"job.type": "attachment.process.v1", "outcome": "retry"})
try:
publish_retry(payload, retry_count + 1)
ch.basic_ack(method.delivery_tag)
except Exception:
ch.basic_nack(method.delivery_tag, requeue=True)
def main() -> None:
signal.signal(signal.SIGTERM, _handle_sigterm)
init_pool()
log.info("event=worker_start queue=%s max_attempts=%s", QUEUE, MAX_ATTEMPTS)
while not _stop:
try:
conn = connect()
ch = conn.channel()
setup_topology(ch)
ch.basic_qos(prefetch_count=1)
ch.basic_consume(queue=QUEUE, on_message_callback=_on_message)
log.info("event=consumer_started")
while not _stop and conn.is_open:
conn.process_data_events(time_limit=1)
ch.stop_consuming()
conn.close()
except Exception as exc: # noqa: BLE001
log.error("event=worker_loop_error type=%s", type(exc).__name__)
time.sleep(3)
close_pool()
log.info("event=worker_stop")
if __name__ == "__main__":
main()