forked from mtusi2026/mtusi2026lab6
377 lines
12 KiB
Python
377 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import io
|
|
import logging
|
|
import os
|
|
import time
|
|
import random
|
|
|
|
from PIL import Image, ImageOps
|
|
|
|
from . import broker, storage
|
|
from .db import init_pool, close_pool, transaction
|
|
from .store import store, iso
|
|
|
|
from opentelemetry import trace, metrics
|
|
from opentelemetry.trace import SpanKind
|
|
|
|
from . import otel as otel_module
|
|
from .logging_config import configure_logging
|
|
from prometheus_client import (
|
|
Counter as _PromCounter,
|
|
Histogram as _PromHistogram,
|
|
Gauge as _PromGauge,
|
|
CollectorRegistry as _PromRegistry,
|
|
generate_latest as _prom_generate,
|
|
CONTENT_TYPE_LATEST as _PROM_CONTENT_TYPE,
|
|
)
|
|
|
|
configure_logging("messenger-worker")
|
|
log = logging.getLogger("messenger.worker")
|
|
|
|
THUMB_SIZE = (256, 256)
|
|
MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
|
|
_PROM_REGISTRY = _PromRegistry()
|
|
_JOBS_PROCESSED = _PromCounter(
|
|
"jobs_processed_total",
|
|
"Jobs processed by outcome",
|
|
["job_type", "outcome"],
|
|
registry=_PROM_REGISTRY,
|
|
)
|
|
_JOB_DURATION = _PromHistogram(
|
|
"job_processing_duration_seconds",
|
|
"Job processing duration in seconds",
|
|
["job_type", "outcome"],
|
|
buckets=(0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0),
|
|
registry=_PROM_REGISTRY,
|
|
)
|
|
_WORKER_IN_FLIGHT = _PromGauge(
|
|
"worker_in_flight",
|
|
"Jobs currently being processed by this worker",
|
|
registry=_PROM_REGISTRY,
|
|
)
|
|
_WORKER_CAPACITY = _PromGauge(
|
|
"worker_capacity",
|
|
"Worker prefetch capacity (WORKER_CONCURRENCY)",
|
|
registry=_PROM_REGISTRY,
|
|
)
|
|
|
|
_tracer = trace.get_tracer("messenger.worker")
|
|
|
|
|
|
def _load_original(staging_key: str) -> bytes:
|
|
return storage.get_object(staging_key)
|
|
|
|
|
|
def _make_thumbnail(data: bytes) -> tuple[bytes, int, int]:
|
|
img = Image.open(io.BytesIO(data))
|
|
img = ImageOps.exif_transpose(img)
|
|
if img.mode in ("RGBA", "LA", "P"):
|
|
background = Image.new("RGB", img.size, (255, 255, 255))
|
|
rgba = img.convert("RGBA")
|
|
background.paste(rgba, mask=rgba.split()[-1])
|
|
img = background
|
|
else:
|
|
img = img.convert("RGB")
|
|
img.thumbnail(THUMB_SIZE, Image.LANCZOS)
|
|
w, h = img.size
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="JPEG", quality=85, optimize=True)
|
|
return buf.getvalue(), w, h
|
|
|
|
def _make_crop(data: bytes, size: int = 128) -> tuple[bytes, int, int]:
|
|
img = Image.open(io.BytesIO(data))
|
|
img = ImageOps.exif_transpose(img)
|
|
if img.mode in ("RGBA", "LA", "P"):
|
|
background = Image.new("RGB", img.size, (255, 255, 255))
|
|
rgba = img.convert("RGBA")
|
|
background.paste(rgba, mask=rgba.split()[-1])
|
|
img = background
|
|
else:
|
|
img = img.convert("RGB")
|
|
w, h = img.size
|
|
side = min(w, h)
|
|
left = (w - side) // 2
|
|
top = (h - side) // 2
|
|
img = img.crop((left, top, left + side, top + side))
|
|
img = img.resize((size, size), Image.LANCZOS)
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="JPEG", quality=85, optimize=True)
|
|
return buf.getvalue(), size, size
|
|
|
|
|
|
def _make_optimized(data: bytes, max_side: int = 1280) -> tuple[bytes, int, int]:
|
|
img = Image.open(io.BytesIO(data))
|
|
img = ImageOps.exif_transpose(img)
|
|
if img.mode in ("RGBA", "LA", "P"):
|
|
background = Image.new("RGB", img.size, (255, 255, 255))
|
|
rgba = img.convert("RGBA")
|
|
background.paste(rgba, mask=rgba.split()[-1])
|
|
img = background
|
|
else:
|
|
img = img.convert("RGB")
|
|
img.thumbnail((max_side, max_side), Image.LANCZOS)
|
|
w, h = img.size
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="WEBP", quality=80, method=4)
|
|
return buf.getvalue(), w, h
|
|
|
|
def _handle(job: dict, parent_ctx=None) -> None:
|
|
attachment_id = job["attachment_id"]
|
|
job_type = job.get("type", "attachment.process.v1")
|
|
attempt = int(job.get("attempt", 1))
|
|
|
|
with _tracer.start_as_current_span(
|
|
"job.process",
|
|
context=parent_ctx,
|
|
kind=SpanKind.CONSUMER,
|
|
attributes={
|
|
"messaging.system": "rabbitmq",
|
|
"messaging.destination.name": "attachment.process.v1",
|
|
"messaging.operation": "process",
|
|
"messaging.message.id": job["job_id"],
|
|
"job.type": job_type,
|
|
"job.attempt": attempt,
|
|
"job.attachment_id": attachment_id,
|
|
},
|
|
) as span:
|
|
_WORKER_IN_FLIGHT.inc()
|
|
import time as _t
|
|
_start = _t.monotonic()
|
|
outcome = "success"
|
|
try:
|
|
_handle_inner(job)
|
|
except Exception as exc:
|
|
outcome = "failure"
|
|
span.record_exception(exc)
|
|
span.set_status(trace.Status(trace.StatusCode.ERROR, str(exc)))
|
|
raise
|
|
finally:
|
|
elapsed = _t.monotonic() - _start
|
|
_JOB_DURATION.labels(job_type=job_type, outcome=outcome).observe(elapsed)
|
|
_JOBS_PROCESSED.labels(job_type=job_type, outcome=outcome).inc()
|
|
_WORKER_IN_FLIGHT.dec()
|
|
|
|
|
|
def _handle_inner(job: dict) -> None:
|
|
attachment_id = job["attachment_id"]
|
|
attempt = int(job.get("attempt", 1))
|
|
job_id = job["job_id"]
|
|
log.info("processing attachment %s attempt %d (job %s)", attachment_id, attempt, job_id)
|
|
|
|
with transaction() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
UPDATE jobs
|
|
SET status = 'processing',
|
|
lease_until = NOW() + INTERVAL '60 seconds'
|
|
WHERE job_id = %s AND status IN ('published', 'pending')
|
|
RETURNING job_id
|
|
""",
|
|
(job_id,),
|
|
)
|
|
acquired = cur.fetchone()
|
|
if acquired is None:
|
|
log.info("job %s already acquired or done, skip", job_id)
|
|
return
|
|
|
|
with transaction() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"UPDATE attachments SET status='processing', updated_at=NOW() "
|
|
"WHERE id=%s AND status='queued'",
|
|
(attachment_id,),
|
|
)
|
|
cur.execute(
|
|
"SELECT id, chat_id, owner_id, filename, content_type, "
|
|
"size_bytes, sha256, s3_key, staging_key, status "
|
|
"FROM attachments WHERE id = %s",
|
|
(attachment_id,),
|
|
)
|
|
row = cur.fetchone()
|
|
if row is None:
|
|
log.error("attachment %s not found, skip", attachment_id)
|
|
return
|
|
if row["status"] == "ready":
|
|
log.info("attachment %s already ready, no-op", attachment_id)
|
|
return
|
|
|
|
staging_key = row.get("staging_key") or row["s3_key"]
|
|
|
|
data = _load_original(staging_key)
|
|
|
|
thumb_bytes, tw, th = _make_thumbnail(data)
|
|
crop_bytes, cw, ch = _make_crop(data, 128)
|
|
opt_bytes, ow, oh = _make_optimized(data, 1280)
|
|
|
|
ext = row["content_type"].split("/")[-1]
|
|
final_key = f"originals/{attachment_id}.{ext}"
|
|
if staging_key != final_key:
|
|
storage.put_object(final_key, data, row["content_type"])
|
|
|
|
thumb_key = f"thumbnails/{attachment_id}.jpg"
|
|
thumb_sha = storage.put_object(thumb_key, thumb_bytes, "image/jpeg")
|
|
|
|
crop_key = f"crops/{attachment_id}.jpg"
|
|
crop_sha = storage.put_object(crop_key, crop_bytes, "image/jpeg")
|
|
|
|
opt_key = f"optimized/{attachment_id}.webp"
|
|
opt_sha = storage.put_object(opt_key, opt_bytes, "image/webp")
|
|
|
|
import json
|
|
variants = [
|
|
{
|
|
"name": "thumbnail",
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": len(thumb_bytes),
|
|
"width": tw,
|
|
"height": th,
|
|
"sha256": thumb_sha,
|
|
},
|
|
{
|
|
"name": "crop",
|
|
"content_type": "image/jpeg",
|
|
"size_bytes": len(crop_bytes),
|
|
"width": cw,
|
|
"height": ch,
|
|
"sha256": crop_sha,
|
|
},
|
|
{
|
|
"name": "optimized",
|
|
"content_type": "image/webp",
|
|
"size_bytes": len(opt_bytes),
|
|
"width": ow,
|
|
"height": oh,
|
|
"sha256": opt_sha,
|
|
},
|
|
]
|
|
with transaction() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'ready',
|
|
s3_key = %s,
|
|
variants = %s::jsonb,
|
|
error_code = NULL,
|
|
updated_at = NOW()
|
|
WHERE id = %s
|
|
""",
|
|
(final_key, json.dumps(variants), attachment_id),
|
|
)
|
|
cur.execute(
|
|
"UPDATE jobs SET status='published', lease_until=NULL, published_at=NOW() "
|
|
"WHERE job_id=%s",
|
|
(job_id,),
|
|
)
|
|
|
|
if staging_key and staging_key != final_key:
|
|
try:
|
|
storage.delete_object(staging_key)
|
|
except Exception as exc:
|
|
log.warning("staging cleanup failed: %s", exc)
|
|
|
|
log.info(
|
|
"attachment %s ready: thumbnail %dx%d, crop %dx%d, optimized %dx%d",
|
|
attachment_id, tw, th, cw, ch, ow, oh,
|
|
)
|
|
|
|
|
|
PERMANENT_ERRORS = ("invalid_image", "unsupported_media_type", "payload_too_large")
|
|
|
|
|
|
def _is_permanent(exc: Exception) -> bool:
|
|
name = type(exc).__name__.lower()
|
|
msg = str(exc).lower()
|
|
return any(p in name or p in msg for p in PERMANENT_ERRORS)
|
|
|
|
|
|
def _handler_with_retry(job: dict, parent_ctx=None) -> None:
|
|
attempt = int(job.get("attempt", 1))
|
|
try:
|
|
_handle(job, parent_ctx=parent_ctx)
|
|
except Exception as exc:
|
|
log.exception("job %s attempt %d failed: %s", job.get("job_id"), attempt, exc)
|
|
permanent = _is_permanent(exc)
|
|
if permanent or attempt >= MAX_ATTEMPTS:
|
|
try:
|
|
broker.publish_dlq(job, str(exc))
|
|
except Exception as dlq_exc:
|
|
log.warning("DLQ publish failed: %s", dlq_exc)
|
|
with transaction() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"UPDATE attachments SET status='failed', "
|
|
"error_code=%s, updated_at=NOW() WHERE id=%s",
|
|
("invalid_image" if permanent else "processing_failed",
|
|
job["attachment_id"]),
|
|
)
|
|
cur.execute(
|
|
"UPDATE jobs SET status='failed', lease_until=NULL "
|
|
"WHERE job_id=%s",
|
|
(job["job_id"],),
|
|
)
|
|
log.error(
|
|
"attachment %s failed (permanent=%s) after %d attempts",
|
|
job["attachment_id"], permanent, attempt,
|
|
)
|
|
return
|
|
delay = min(60, 2 ** attempt) + random.uniform(0, 1)
|
|
log.info("retry in %.1fs (attempt %d -> %d)", delay, attempt, attempt + 1)
|
|
with transaction() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"UPDATE jobs SET status='pending', lease_until=NULL, attempt=%s "
|
|
"WHERE job_id=%s",
|
|
(attempt + 1, job["job_id"]),
|
|
)
|
|
|
|
|
|
def main() -> None:
|
|
otel_module.setup_otel("messenger-worker")
|
|
otel_module.instrument_libraries(psycopg=True, botocore=True, redis=False)
|
|
|
|
init_pool()
|
|
storage.init_storage()
|
|
_WORKER_CAPACITY.set(int(os.environ.get("WORKER_CONCURRENCY", "2")))
|
|
|
|
import threading
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
|
class MetricsHandler(BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
if self.path == "/metrics":
|
|
data = _prom_generate(_PROM_REGISTRY)
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", _PROM_CONTENT_TYPE)
|
|
self.send_header("Content-Length", str(len(data)))
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
elif self.path == "/health/live":
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.end_headers()
|
|
self.wfile.write(b'{"status":"ok"}')
|
|
else:
|
|
self.send_response(404)
|
|
self.end_headers()
|
|
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
metrics_port = int(os.environ.get("WORKER_METRICS_PORT", "9100"))
|
|
httpd = ThreadingHTTPServer(("0.0.0.0", metrics_port), MetricsHandler)
|
|
threading.Thread(target=httpd.serve_forever, daemon=True, name="metrics-http").start()
|
|
log.info("worker metrics on :%d/metrics", metrics_port)
|
|
|
|
log.info("worker starting, max_attempts=%d", MAX_ATTEMPTS)
|
|
try:
|
|
broker.consume(_handler_with_retry, prefetch=int(os.environ.get("WORKER_CONCURRENCY", "2")))
|
|
finally:
|
|
otel_module.shutdown_otel()
|
|
close_pool()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |