import os import io import json import time import signal import random import hashlib import logging from datetime import datetime, timezone from concurrent.futures import ThreadPoolExecutor import psycopg2 import psycopg2.extras import redis import boto3 import botocore.exceptions from botocore.client import Config from PIL import Image, ImageOps, UnidentifiedImageError from opentelemetry import metrics, trace from opentelemetry._logs import set_logger_provider from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry.propagate import extract class JsonLogFormatter(logging.Formatter): def format(self, record: logging.LogRecord) -> str: span = trace.get_current_span() ctx = span.get_span_context() if span else None trace_id = format(ctx.trace_id, "032x") if ctx and ctx.is_valid else None span_id = format(ctx.span_id, "016x") if ctx and ctx.is_valid else None payload = { "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), "level": record.levelname, "logger": record.name, "service": os.getenv("OTEL_SERVICE_NAME", "messenger-worker"), "msg": record.getMessage(), } if trace_id and span_id: payload["trace_id"] = trace_id payload["span_id"] = span_id for attr in ("method", "path", "status_code", "job_id", "attachment_id", "job_status", "attempt", "error_code"): if hasattr(record, attr): payload[attr] = getattr(record, attr) if record.exc_info: payload["exception"] = record.exc_info[0].__name__ if record.exc_info[0] else "Exception" return json.dumps(payload, ensure_ascii=False) def setup_worker_telemetry(): service_name = os.getenv("OTEL_SERVICE_NAME", "messenger-worker") endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/") env_attrs = os.getenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=lab") attrs = {"service.name": service_name} for part in env_attrs.split(","): if "=" in part: k, v = part.split("=", 1) attrs[k.strip()] = v.strip() resource = Resource.create(attrs) tracer_provider = TracerProvider(resource=resource) tracer_provider.add_span_processor( BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")) ) trace.set_tracer_provider(tracer_provider) metric_reader = PeriodicExportingMetricReader( OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics"), export_interval_millis=5000, ) meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader]) metrics.set_meter_provider(meter_provider) logger_provider = LoggerProvider(resource=resource) logger_provider.add_log_record_processor( BatchLogRecordProcessor(OTLPLogExporter(endpoint=f"{endpoint}/v1/logs")) ) set_logger_provider(logger_provider) root_logger = logging.getLogger() root_logger.setLevel(logging.INFO) root_logger.handlers.clear() stream_handler = logging.StreamHandler() stream_handler.setFormatter(JsonLogFormatter()) root_logger.addHandler(stream_handler) otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) otel_handler.setFormatter(JsonLogFormatter()) root_logger.addHandler(otel_handler) return trace.get_tracer(service_name), metrics.get_meter(service_name) DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser") S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword") STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs") DLQ_STREAM_NAME = f"{STREAM_NAME}_dlq" GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers") CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1") JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30")) WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4")) CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000 Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True) s3_client = boto3.client( "s3", endpoint_url=S3_ENDPOINT, aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY, config=Config(signature_version="s3v4"), region_name="us-east-1" ) def observe_queue_metrics(options): try: q_len = redis_client.xlen(STREAM_NAME) entries = redis_client.xrange(STREAM_NAME, count=1) oldest_age = 0.0 if entries: msg_id, fields = entries[0] try: payload = json.loads(fields.get("payload", "{}")) created_str = payload.get("created_at") if created_str: created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00")) oldest_age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds()) except Exception: ms = int(msg_id.split("-")[0]) oldest_age = max(0.0, time.time() - (ms / 1000.0)) yield metrics.Observation(value=q_len, attributes={"stream": STREAM_NAME}) except Exception: yield metrics.Observation(value=0, attributes={"stream": STREAM_NAME}) def observe_oldest_job_age(options): try: entries = redis_client.xrange(STREAM_NAME, count=1) if not entries: yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME}) return msg_id, fields = entries[0] try: payload = json.loads(fields.get("payload", "{}")) created_str = payload.get("created_at") if created_str: created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00")) age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds()) yield metrics.Observation(value=age, attributes={"stream": STREAM_NAME}) return except Exception: pass ms = int(msg_id.split("-")[0]) yield metrics.Observation(value=max(0.0, time.time() - (ms / 1000.0)), attributes={"stream": STREAM_NAME}) except Exception: yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME}) tracer, meter = setup_worker_telemetry() logger = logging.getLogger("attachment-worker") worker_jobs_counter = meter.create_counter( name="worker_jobs_total", description="Completed and failed attachment jobs", unit="1", ) worker_retries_counter = meter.create_counter( name="worker_job_retries_total", description="Transient retry attempts scheduled for attachment jobs", unit="1", ) worker_duration_histogram = meter.create_histogram( name="worker_job_duration_seconds", description="Attachment job processing duration in seconds", unit="s", ) worker_jobs_in_progress = meter.create_up_down_counter( name="worker_jobs_in_progress", description="Jobs currently being processed", unit="1", ) meter.create_observable_gauge( name="worker_queue_depth", callbacks=[observe_queue_metrics], description="Number of messages currently in attachment jobs stream", unit="1", ) meter.create_observable_gauge( name="worker_oldest_job_age_seconds", callbacks=[observe_oldest_job_age], description="Age of the oldest message in attachment jobs stream in seconds", unit="s", ) running = True def handle_sigterm(signum, frame): global running logger.info("Received shutdown signal, finishing in-flight jobs...") running = False signal.signal(signal.SIGTERM, handle_sigterm) signal.signal(signal.SIGINT, handle_sigterm) def get_db_connection(): return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) def ensure_consumer_groups(): while running: try: redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True) redis_client.xgroup_create(DLQ_STREAM_NAME, GROUP_NAME, id="0", mkstream=True) logger.info("Consumer groups ensured on '%s' and '%s'", STREAM_NAME, DLQ_STREAM_NAME) return except redis.exceptions.ResponseError as e: if "BUSYGROUP" in str(e): return raise except Exception as e: logger.warning("Waiting for Redis: %s", type(e).__name__) time.sleep(1) def prepare_base_image(raw_bytes): img = Image.open(io.BytesIO(raw_bytes)) img = ImageOps.exif_transpose(img) if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info): rgba = img.convert("RGBA") bg = Image.new("RGB", rgba.size, (255, 255, 255)) bg.paste(rgba, mask=rgba.split()[3]) img = bg elif img.mode != "RGB": img = img.convert("RGB") return img def generate_grade4_variants(raw_bytes): base_img = prepare_base_image(raw_bytes) orig_w, orig_h = base_img.size meta = [] blobs = {} thumb = base_img.copy() if orig_w <= 256 and orig_h <= 256: t_w, t_h = orig_w, orig_h else: s = min(256.0 / orig_w, 256.0 / orig_h) t_w, t_h = max(1, int(orig_w * s)), max(1, int(orig_h * s)) if (t_w, t_h) != (orig_w, orig_h): thumb = thumb.resize((t_w, t_h), Image.Resampling.LANCZOS) buf = io.BytesIO() thumb.save(buf, format="JPEG", quality=85, optimize=True) tb = buf.getvalue() blobs["thumbnail"] = tb meta.append({ "name": "thumbnail", "content_type": "image/jpeg", "size_bytes": len(tb), "width": t_w, "height": t_h, "sha256": hashlib.sha256(tb).hexdigest() }) opt = base_img.copy() if orig_w <= 1280 and orig_h <= 1280: o_w, o_h = orig_w, orig_h else: s = min(1280.0 / orig_w, 1280.0 / orig_h) o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s)) if (o_w, o_h) != (orig_w, orig_h): opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS) buf = io.BytesIO() opt.save(buf, format="WEBP", quality=85) ob = buf.getvalue() blobs["optimized"] = ob meta.append({ "name": "optimized", "content_type": "image/webp", "size_bytes": len(ob), "width": o_w, "height": o_h, "sha256": hashlib.sha256(ob).hexdigest() }) c = min(orig_w, orig_h) left, top = (orig_w - c) // 2, (orig_h - c) // 2 crop = base_img.crop((left, top, left + c, top + c)) if c != 128: crop = crop.resize((128, 128), Image.Resampling.LANCZOS) buf = io.BytesIO() crop.save(buf, format="JPEG", quality=85, optimize=True) cb = buf.getvalue() blobs["crop"] = cb meta.append({ "name": "crop", "content_type": "image/jpeg", "size_bytes": len(cb), "width": 128, "height": 128, "sha256": hashlib.sha256(cb).hexdigest() }) return meta, blobs def mark_failed(attachment_id, error_code): now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") conn = get_db_connection() try: with conn.cursor() as cur: cur.execute( "UPDATE attachments SET status='failed', error_code=%s, updated_at=%s " "WHERE id=%s AND status != 'ready'", (error_code, now, attachment_id) ) conn.commit() finally: conn.close() TRANSIENT_ERRORS = ( TimeoutError, botocore.exceptions.EndpointConnectionError, botocore.exceptions.ReadTimeoutError, psycopg2.OperationalError, redis.exceptions.ConnectionError, ) def _record_job_outcome(span, status: str, elapsed: float): span.set_attribute("job.outcome", status) worker_jobs_counter.add(1, {"status": status}) worker_duration_histogram.record(elapsed, {"status": status}) def process_job(message_id, fields): job_start = time.monotonic() try: job = json.loads(fields.get("payload", "{}")) except Exception: logger.error("Malformed job %s, ack to drop", message_id) redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) return ctx = extract(job) with tracer.start_as_current_span("worker.process_attachment", context=ctx) as span: worker_jobs_in_progress.add(1) try: attachment_id = job.get("attachment_id") job_id = job.get("job_id") pipeline_version = int(job.get("pipeline_version", 1)) attempt = int(job.get("attempt", 1)) span.set_attribute("job.pipeline_version", pipeline_version) span.set_attribute("job.attempt", attempt) if not attachment_id: redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) _record_job_outcome(span, "failed", time.monotonic() - job_start) return conn = get_db_connection() try: with conn.cursor() as cur: cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)) att = cur.fetchone() if not att: conn.commit() redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) _record_job_outcome(span, "failed", time.monotonic() - job_start) return if att["status"] == "ready" or att["pipeline_version"] > pipeline_version: conn.commit() redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) _record_job_outcome(span, "success", time.monotonic() - job_start) return if att["status"] == "failed": conn.commit() redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) _record_job_outcome(span, "failed", time.monotonic() - job_start) return if att["status"] == "processing": updated = att["updated_at"] if isinstance(updated, str): updated = datetime.fromisoformat(updated.replace("Z", "+00:00")) age_seconds = (datetime.now(timezone.utc) - updated).total_seconds() if age_seconds < (JOB_TIMEOUT_SECONDS + 10): conn.commit() logger.info( "Attachment already processing, ack duplicate", extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt}, ) redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) return now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") cur.execute( "UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s", (attempt, now, attachment_id) ) conn.commit() finally: conn.close() chat_id = str(att["chat_id"]) staging_key = f"staging/{chat_id}/{attachment_id}/original" final_original_key = f"attachments/{chat_id}/{attachment_id}/original" start_time = time.monotonic() def check_timeout(): if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS: raise TimeoutError("Job exceeded time limit") try: try: obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key) except Exception: obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key) raw_bytes = obj["Body"].read() check_timeout() if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]: raise ValueError("Staging checksum mismatch") variants_meta, variants_bytes = generate_grade4_variants(raw_bytes) check_timeout() s3_client.put_object( Bucket=S3_BUCKET, Key=final_original_key, Body=raw_bytes, ContentType=att["content_type"] ) for var_name, var_bytes in variants_bytes.items(): ct = "image/webp" if var_name == "optimized" else "image/jpeg" s3_client.put_object( Bucket=S3_BUCKET, Key=f"attachments/{chat_id}/{attachment_id}/{var_name}", Body=var_bytes, ContentType=ct ) check_timeout() ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") conn = get_db_connection() try: with conn.cursor() as cur: cur.execute(""" UPDATE attachments SET status='ready', variants=%s::jsonb, error_code=NULL, updated_at=%s WHERE id=%s AND pipeline_version <= %s """, (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version)) conn.commit() finally: conn.close() redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) _record_job_outcome(span, "success", time.monotonic() - job_start) logger.info( "Job completed: attachment ready", extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt}, ) try: s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key) except Exception: pass except (UnidentifiedImageError, Image.DecompressionBombError): logger.error( "Permanent image error", extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"}, ) mark_failed(attachment_id, "invalid_image") redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) _record_job_outcome(span, "failed", time.monotonic() - job_start) except ValueError as e: logger.error( "Checksum error", extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"}, ) mark_failed(attachment_id, "invalid_checksum") redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) _record_job_outcome(span, "failed", time.monotonic() - job_start) except psycopg2.errors.DataError: logger.error( "DB data error", extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"}, ) mark_failed(attachment_id, "data_error") redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) _record_job_outcome(span, "failed", time.monotonic() - job_start) except TRANSIENT_ERRORS as e: logger.warning( "Transient error during job processing", extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt}, ) if attempt >= JOB_MAX_ATTEMPTS: mark_failed(attachment_id, "retry_exhausted") try: dlq_job = dict(job) dlq_job["attempt"] = 1 redis_client.xadd( DLQ_STREAM_NAME, {"payload": json.dumps(dlq_job), "error": type(e).__name__} ) except Exception: pass redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) _record_job_outcome(span, "failed", time.monotonic() - job_start) else: worker_retries_counter.add(1, {"reason": type(e).__name__}) delay = min(2 ** attempt, 30) + random.uniform(0, 1) time.sleep(delay) next_job = dict(job) next_job["attempt"] = attempt + 1 redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)}) redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) except Exception: logger.error( "Unknown error during job processing", extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"}, ) mark_failed(attachment_id, "processing_failed") redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) _record_job_outcome(span, "failed", time.monotonic() - job_start) finally: worker_jobs_in_progress.add(-1) def run_staging_cleanup(): try: cutoff = datetime.now(timezone.utc).timestamp() - 3600 paginator = s3_client.get_paginator("list_objects_v2") for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="staging/"): for obj in page.get("Contents", []): if obj["LastModified"].timestamp() < cutoff: try: s3_client.delete_object(Bucket=S3_BUCKET, Key=obj["Key"]) except Exception: pass except Exception as e: logger.debug("Staging cleanup: %s", type(e).__name__) def safe_process(msg_id, fields): try: process_job(msg_id, fields) except Exception as e: logger.error("Unhandled in process_job %s: %s", msg_id, type(e).__name__) def main(): ensure_consumer_groups() logger.info("Worker '%s' started, concurrency=%d", CONSUMER_NAME, WORKER_CONCURRENCY) executor = ThreadPoolExecutor(max_workers=WORKER_CONCURRENCY) last_claim = 0 last_cleanup = 0 try: while running: try: now = time.time() if now - last_claim > 15: last_claim = now try: claimed = redis_client.xautoclaim( STREAM_NAME, GROUP_NAME, CONSUMER_NAME, min_idle_time=CLAIM_MIN_IDLE_MS, start_id="0-0", count=WORKER_CONCURRENCY ) if claimed and len(claimed) > 1 and claimed[1]: for msg_id, fields in claimed[1]: if not running: break executor.submit(safe_process, msg_id, fields) except Exception: pass if now - last_cleanup > 600: last_cleanup = now executor.submit(run_staging_cleanup) resp = redis_client.xreadgroup( GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"}, count=WORKER_CONCURRENCY, block=2000 ) if not resp: continue for _, messages in resp: for msg_id, fields in messages: if not running: break executor.submit(safe_process, msg_id, fields) except Exception as e: logger.warning("Worker loop: %s", type(e).__name__) time.sleep(1) finally: logger.info("Shutting down, waiting for in-flight jobs...") executor.shutdown(wait=True) logger.info("Worker stopped cleanly") if __name__ == "__main__": main()