import os import io import json import time import signal import random import hashlib import logging from datetime import datetime, timezone from concurrent.futures import ThreadPoolExecutor import psycopg2 import psycopg2.extras import redis import boto3 import botocore.exceptions from botocore.client import Config from PIL import Image, ImageOps, UnidentifiedImageError logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") logger = logging.getLogger("attachment-worker") DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser") S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword") STREAM_NAME = os.getenv("BROKER_STREAM", "attachment_jobs") DLQ_STREAM_NAME = f"{STREAM_NAME}_dlq" GROUP_NAME = os.getenv("BROKER_GROUP", "attachment_workers") CONSUMER_NAME = os.getenv("HOSTNAME", "worker-1") JOB_MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30")) WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4")) CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000 Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True) s3_client = boto3.client( "s3", endpoint_url=S3_ENDPOINT, aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY, config=Config(signature_version="s3v4"), region_name="us-east-1" ) running = True def handle_sigterm(signum, frame): global running logger.info("Received shutdown signal, finishing in-flight jobs...") running = False signal.signal(signal.SIGTERM, handle_sigterm) signal.signal(signal.SIGINT, handle_sigterm) def get_db_connection(): return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) def ensure_consumer_groups(): while running: try: redis_client.xgroup_create(STREAM_NAME, GROUP_NAME, id="0", mkstream=True) redis_client.xgroup_create(DLQ_STREAM_NAME, GROUP_NAME, id="0", mkstream=True) logger.info("Consumer groups ensured on '%s' and '%s'", STREAM_NAME, DLQ_STREAM_NAME) return except redis.exceptions.ResponseError as e: if "BUSYGROUP" in str(e): return raise except Exception as e: logger.warning("Waiting for Redis: %s", type(e).__name__) time.sleep(1) def prepare_base_image(raw_bytes): img = Image.open(io.BytesIO(raw_bytes)) img = ImageOps.exif_transpose(img) if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info): rgba = img.convert("RGBA") bg = Image.new("RGB", rgba.size, (255, 255, 255)) bg.paste(rgba, mask=rgba.split()[3]) img = bg elif img.mode != "RGB": img = img.convert("RGB") return img def generate_grade4_variants(raw_bytes): base_img = prepare_base_image(raw_bytes) orig_w, orig_h = base_img.size meta = [] blobs = {} thumb = base_img.copy() if orig_w <= 256 and orig_h <= 256: t_w, t_h = orig_w, orig_h else: s = min(256.0 / orig_w, 256.0 / orig_h) t_w, t_h = max(1, int(orig_w * s)), max(1, int(orig_h * s)) if (t_w, t_h) != (orig_w, orig_h): thumb = thumb.resize((t_w, t_h), Image.Resampling.LANCZOS) buf = io.BytesIO(); thumb.save(buf, format="JPEG", quality=85, optimize=True) tb = buf.getvalue(); blobs["thumbnail"] = tb meta.append({"name": "thumbnail", "content_type": "image/jpeg", "size_bytes": len(tb), "width": t_w, "height": t_h, "sha256": hashlib.sha256(tb).hexdigest()}) opt = base_img.copy() if orig_w <= 1280 and orig_h <= 1280: o_w, o_h = orig_w, orig_h else: s = min(1280.0 / orig_w, 1280.0 / orig_h) o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s)) if (o_w, o_h) != (orig_w, orig_h): opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS) buf = io.BytesIO(); opt.save(buf, format="WEBP", quality=85) ob = buf.getvalue(); blobs["optimized"] = ob meta.append({"name": "optimized", "content_type": "image/webp", "size_bytes": len(ob), "width": o_w, "height": o_h, "sha256": hashlib.sha256(ob).hexdigest()}) c = min(orig_w, orig_h) left, top = (orig_w - c) // 2, (orig_h - c) // 2 crop = base_img.crop((left, top, left + c, top + c)) if c != 128: crop = crop.resize((128, 128), Image.Resampling.LANCZOS) buf = io.BytesIO(); crop.save(buf, format="JPEG", quality=85, optimize=True) cb = buf.getvalue(); blobs["crop"] = cb meta.append({"name": "crop", "content_type": "image/jpeg", "size_bytes": len(cb), "width": 128, "height": 128, "sha256": hashlib.sha256(cb).hexdigest()}) return meta, blobs def mark_failed(attachment_id, error_code): now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") conn = get_db_connection() try: with conn.cursor() as cur: cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=%s " "WHERE id=%s AND status != 'ready'", (error_code, now, attachment_id)) conn.commit() finally: conn.close() PERMANENT_ERRORS = ( UnidentifiedImageError, Image.DecompressionBombError, ValueError, psycopg2.errors.DataError, ) TRANSIENT_ERRORS = ( TimeoutError, botocore.exceptions.EndpointConnectionError, botocore.exceptions.ReadTimeoutError, psycopg2.OperationalError, redis.exceptions.ConnectionError, ) def process_job(message_id, fields): try: job = json.loads(fields.get("payload", "{}")) except Exception: logger.error("Malformed job %s, ack to drop", message_id) redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) return attachment_id = job.get("attachment_id") job_id = job.get("job_id") pipeline_version = int(job.get("pipeline_version", 1)) attempt = int(job.get("attempt", 1)) if not attachment_id: redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) return conn = get_db_connection() try: with conn.cursor() as cur: cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)) att = cur.fetchone() if not att: conn.commit() redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) return if att["status"] == "ready" or att["pipeline_version"] > pipeline_version: conn.commit() redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) return if att["status"] == "failed": conn.commit() redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) return if att["status"] == "processing": updated = att["updated_at"] if isinstance(updated, str): updated = datetime.fromisoformat(updated.replace("Z", "+00:00")) age_seconds = (datetime.now(timezone.utc) - updated).total_seconds() if age_seconds < (JOB_TIMEOUT_SECONDS + 10): conn.commit() logger.info("Attachment %s already processing (age=%.1fs), ack duplicate", attachment_id, age_seconds) redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) return now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") cur.execute( "UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s", (attempt, now, attachment_id) ) conn.commit() finally: conn.close() chat_id = str(att["chat_id"]) staging_key = f"staging/{chat_id}/{attachment_id}/original" final_original_key = f"attachments/{chat_id}/{attachment_id}/original" start_time = time.monotonic() def check_timeout(): if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS: raise TimeoutError("Job exceeded time limit") try: try: obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key) except Exception: obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key) raw_bytes = obj["Body"].read() check_timeout() if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]: raise ValueError("Staging checksum mismatch") variants_meta, variants_bytes = generate_grade4_variants(raw_bytes) check_timeout() s3_client.put_object( Bucket=S3_BUCKET, Key=final_original_key, Body=raw_bytes, ContentType=att["content_type"] ) for var_name, var_bytes in variants_bytes.items(): ct = "image/webp" if var_name == "optimized" else "image/jpeg" s3_client.put_object( Bucket=S3_BUCKET, Key=f"attachments/{chat_id}/{attachment_id}/{var_name}", Body=var_bytes, ContentType=ct ) check_timeout() ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") conn = get_db_connection() try: with conn.cursor() as cur: cur.execute(""" UPDATE attachments SET status='ready', variants=%s::jsonb, error_code=NULL, updated_at=%s WHERE id=%s AND pipeline_version <= %s """, (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version)) conn.commit() finally: conn.close() redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) logger.info("Job %s completed: attachment %s ready", job_id, attachment_id) try: s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key) except Exception: pass except (UnidentifiedImageError, Image.DecompressionBombError) as e: logger.error("Permanent image error for %s: %s", attachment_id, type(e).__name__) mark_failed(attachment_id, "invalid_image") redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) except ValueError as e: logger.error("Checksum error for %s: %s", attachment_id, e) mark_failed(attachment_id, "invalid_checksum") redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) except psycopg2.errors.DataError as e: logger.error("DB data error for %s: %s", attachment_id, type(e).__name__) mark_failed(attachment_id, "data_error") redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) except TRANSIENT_ERRORS as e: logger.warning( "Transient error for %s (attempt %d/%d): %s", attachment_id, attempt, JOB_MAX_ATTEMPTS, type(e).__name__ ) if attempt >= JOB_MAX_ATTEMPTS: mark_failed(attachment_id, "retry_exhausted") try: dlq_job = dict(job) dlq_job["attempt"] = 1 redis_client.xadd( DLQ_STREAM_NAME, {"payload": json.dumps(dlq_job), "error": str(e)} ) except Exception: pass redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) else: delay = min(2 ** attempt, 30) + random.uniform(0, 1) time.sleep(delay) next_job = dict(job) next_job["attempt"] = attempt + 1 redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)}) redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) except Exception as e: logger.error("Unknown error for %s: %s, marking failed", attachment_id, type(e).__name__) mark_failed(attachment_id, "processing_failed") redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) def run_staging_cleanup(): try: cutoff = datetime.now(timezone.utc).timestamp() - 3600 paginator = s3_client.get_paginator("list_objects_v2") for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="staging/"): for obj in page.get("Contents", []): if obj["LastModified"].timestamp() < cutoff: try: s3_client.delete_object(Bucket=S3_BUCKET, Key=obj["Key"]) except Exception: pass except Exception as e: logger.debug("Staging cleanup: %s", e) def safe_process(msg_id, fields): """Wrapper - exceptions в threads иначе исчезают молча.""" try: process_job(msg_id, fields) except Exception as e: logger.error("Unhandled in process_job %s: %s", msg_id, type(e).__name__) def main(): ensure_consumer_groups() logger.info("Worker '%s' started, concurrency=%d", CONSUMER_NAME, WORKER_CONCURRENCY) executor = ThreadPoolExecutor(max_workers=WORKER_CONCURRENCY) last_claim = 0 last_cleanup = 0 try: while running: try: now = time.time() if now - last_claim > 15: last_claim = now try: claimed = redis_client.xautoclaim( STREAM_NAME, GROUP_NAME, CONSUMER_NAME, min_idle_time=CLAIM_MIN_IDLE_MS, start_id="0-0", count=WORKER_CONCURRENCY ) if claimed and len(claimed) > 1 and claimed[1]: for msg_id, fields in claimed[1]: if not running: break executor.submit(safe_process, msg_id, fields) except Exception: pass if now - last_cleanup > 600: last_cleanup = now executor.submit(run_staging_cleanup) resp = redis_client.xreadgroup( GROUP_NAME, CONSUMER_NAME, {STREAM_NAME: ">"}, count=WORKER_CONCURRENCY, block=2000 ) if not resp: continue for _, messages in resp: for msg_id, fields in messages: if not running: break executor.submit(safe_process, msg_id, fields) except Exception as e: logger.warning("Worker loop: %s", type(e).__name__) time.sleep(1) finally: logger.info("Shutting down, waiting for in-flight jobs...") executor.shutdown(wait=True) logger.info("Worker stopped cleanly") if __name__ == "__main__": main()