diff --git a/app.py b/app.py index fcf14b3..9c09d2c 100644 --- a/app.py +++ b/app.py @@ -9,12 +9,14 @@ import struct import hashlib import hmac import logging +import threading from datetime import datetime, timezone, timedelta from typing import List, Optional from uuid import UUID import psycopg2 import psycopg2.extras +from psycopg2.pool import ThreadedConnectionPool psycopg2.extras.register_uuid() import bcrypt from fastapi import FastAPI, Header, HTTPException, Query, Response, Request @@ -26,8 +28,6 @@ import redis import boto3 from botocore.client import Config -import threading - from opentelemetry import metrics, trace from opentelemetry._logs import set_logger_provider from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter @@ -41,6 +41,7 @@ from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.propagate import inject class JsonLogFormatter(logging.Formatter): @@ -134,6 +135,11 @@ api_errors_counter = meter.create_counter( description="Total HTTP 4xx/5xx responses served by API", unit="1", ) +api_dependency_errors_counter = meter.create_counter( + name="http_server_dependency_errors_total", + description="Dependency and internal errors in API (DB/Redis/S3)", + unit="1", +) api_duration_histogram = meter.create_histogram( name="http_server_request_duration_seconds", description="HTTP request duration in seconds", @@ -165,16 +171,66 @@ s3_client = boto3.client( region_name="us-east-1" ) +db_pool_lock = threading.Lock() +db_pool: Optional[ThreadedConnectionPool] = None +_active_db_conns = 0 -def get_db_connection(): - return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor) + +def init_db_pool(minconn: int = 1, maxconn: int = 15): + global db_pool + if db_pool is None: + db_pool = ThreadedConnectionPool( + minconn=minconn, + maxconn=maxconn, + dsn=DATABASE_URL, + cursor_factory=psycopg2.extras.RealDictCursor, + ) + + +def acquire_db_conn(): + global _active_db_conns + if db_pool is None: + init_db_pool() + conn = db_pool.getconn() + with db_pool_lock: + _active_db_conns += 1 + return conn + + +def release_db_conn(conn): + global _active_db_conns + if conn is None or db_pool is None: + return + try: + close_broken = bool(getattr(conn, "closed", 0)) + db_pool.putconn(conn, close=close_broken) + finally: + with db_pool_lock: + _active_db_conns = max(0, _active_db_conns - 1) + + +def observe_db_pool(options): + max_c = db_pool.maxconn if db_pool else 0 + with db_pool_lock: + used = min(_active_db_conns, max_c) + idle = max(0, max_c - used) + yield metrics.Observation(value=used, attributes={"state": "used", "instance": INSTANCE_ID}) + yield metrics.Observation(value=idle, attributes={"state": "idle", "instance": INSTANCE_ID}) + + +meter.create_observable_gauge( + name="db_pool_connections", + callbacks=[observe_db_pool], + description="PostgreSQL connection pool connections by state", + unit="1", +) def outbox_publisher(): while True: conn = None try: - conn = get_db_connection() + conn = acquire_db_conn() with conn.cursor() as cur: cur.execute(""" SELECT id, stream, payload FROM outbox @@ -188,18 +244,37 @@ def outbox_publisher(): cur.execute("UPDATE outbox SET published_at=NOW() WHERE id=%s", (row["id"],)) conn.commit() except Exception as e: + if conn and not conn.closed: + try: + conn.rollback() + except Exception: + pass + api_dependency_errors_counter.add(1, {"component": "outbox"}) api_logger.error("Outbox publisher error: %s", type(e).__name__) finally: - if conn is not None and not conn.closed: - conn.close() + release_db_conn(conn) time.sleep(1) @app.on_event("startup") -def _start_outbox_publisher(): +def _start_services(): + try: + init_db_pool(minconn=1, maxconn=15) + except Exception as e: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + api_logger.error("DB pool init error: %s", type(e).__name__) threading.Thread(target=outbox_publisher, daemon=True).start() +@app.on_event("shutdown") +def _shutdown_services(): + if db_pool is not None: + try: + db_pool.closeall() + except Exception: + pass + + @app.middleware("http") async def telemetry_middleware(request: Request, call_next): start = time.monotonic() @@ -342,6 +417,7 @@ async def http_exception_handler(request: Request, exc: HTTPException): @app.exception_handler(Exception) async def unhandled_exception_handler(request: Request, exc: Exception): req_id = request.headers.get("X-Request-Id") or generate_request_id() + api_dependency_errors_counter.add(1, {"component": "unhandled"}) return make_error_response( status_code=500, code="internal_error", @@ -400,13 +476,19 @@ def health_ready(response: Response): req_id = generate_request_id() response.headers["X-Request-Id"] = req_id response.headers["X-Instance-Id"] = INSTANCE_ID + conn = None try: - conn = get_db_connection() - conn.close() + conn = acquire_db_conn() + with conn.cursor() as cur: + cur.execute("SELECT 1") + cur.fetchone() redis_client.ping() s3_client.head_bucket(Bucket=S3_BUCKET) except Exception: + api_dependency_errors_counter.add(1, {"component": "readiness"}) raise HTTPException(status_code=503, detail="Database, Redis or S3 unavailable") + finally: + release_db_conn(conn) return {"status": "ok"} @@ -420,7 +502,7 @@ def register_user(body: CreateUser, response: Response): user_id = str(uuid.uuid4()) created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - conn = get_db_connection() + conn = acquire_db_conn() try: with conn.cursor() as cur: cur.execute( @@ -434,8 +516,12 @@ def register_user(body: CreateUser, response: Response): status_code=409, detail={"error": {"code": "conflict", "message": "Username already taken", "request_id": req_id}} ) + except Exception: + conn.rollback() + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") finally: - conn.close() + release_db_conn(conn) return { "id": user_id, @@ -469,11 +555,16 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} ) - conn = get_db_connection() - with conn.cursor() as cur: - cur.execute("SELECT * FROM users WHERE username = %s", (username,)) - user = cur.fetchone() - conn.close() + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute("SELECT * FROM users WHERE username = %s", (username,)) + user = cur.fetchone() + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) if not user or not bcrypt.checkpw(password.encode('utf-8'), user['password_hash'].encode('utf-8')): raise HTTPException( @@ -483,7 +574,12 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon ) token = secrets.token_urlsafe(48) - redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, str(user['id'])) + try: + redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, str(user['id'])) + except Exception: + api_dependency_errors_counter.add(1, {"component": "redis"}) + raise HTTPException(status_code=503, detail="Session store unavailable") + expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS) expires_at_str = expires_at.isoformat().replace("+00:00", "Z") @@ -512,6 +608,7 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> try: user_id = redis_client.get(f"session:{token}") except Exception: + api_dependency_errors_counter.add(1, {"component": "redis"}) raise HTTPException(status_code=503, detail="Session store unavailable") if not user_id: @@ -521,11 +618,16 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) -> headers={"WWW-Authenticate": 'Bearer realm="messenger"'} ) - conn = get_db_connection() - with conn.cursor() as cur: - cur.execute("SELECT * FROM users WHERE id = %s", (user_id,)) - row = cur.fetchone() - conn.close() + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute("SELECT * FROM users WHERE id = %s", (user_id,)) + row = cur.fetchone() + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) if not row: raise HTTPException( @@ -554,7 +656,11 @@ def revoke_session(authorization: Optional[str] = Header(None)): ) token = authorization.split(" ")[1] - deleted = redis_client.delete(f"session:{token}") + try: + deleted = redis_client.delete(f"session:{token}") + except Exception: + api_dependency_errors_counter.add(1, {"component": "redis"}) + raise HTTPException(status_code=503, detail="Session store unavailable") if not deleted: raise HTTPException( @@ -594,19 +700,17 @@ def create_chat(body: CreateChat, response: Response, authorization: Optional[st if len(member_ids_str) != len(set(member_ids_str)): raise HTTPException(status_code=400, detail="Duplicate member IDs") - conn = get_db_connection() - with conn.cursor() as cur: - for mid_str in member_ids_str: - cur.execute("SELECT id FROM users WHERE id = %s", (mid_str,)) - if not cur.fetchone(): - conn.close() - raise HTTPException(status_code=400, detail=f"User {mid_str} not found") - chat_id = str(uuid.uuid4()) created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + conn = acquire_db_conn() try: with conn.cursor() as cur: + for mid_str in member_ids_str: + cur.execute("SELECT id FROM users WHERE id = %s", (mid_str,)) + if not cur.fetchone(): + raise HTTPException(status_code=400, detail=f"User {mid_str} not found") + cur.execute( "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", (chat_id, body.title, created_at) @@ -617,11 +721,15 @@ def create_chat(body: CreateChat, response: Response, authorization: Optional[st (chat_id, mid_str) ) conn.commit() + except HTTPException: + conn.rollback() + raise except Exception: conn.rollback() - conn.close() + api_dependency_errors_counter.add(1, {"component": "postgres"}) raise HTTPException(status_code=500, detail="Internal error creating chat") - conn.close() + finally: + release_db_conn(conn) return { "id": chat_id, @@ -641,27 +749,32 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response if limit < 1 or limit > 100: raise HTTPException(status_code=400, detail="Invalid limit") - conn = get_db_connection() - with conn.cursor() as cur: - cur.execute(""" - SELECT c.id, c.title, c.created_at FROM chats c - JOIN chat_members cm ON c.id = cm.chat_id - WHERE cm.user_id = %s - ORDER BY c.created_at ASC, c.id ASC - """, (user["id"],)) - rows = cur.fetchall() + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT c.id, c.title, c.created_at FROM chats c + JOIN chat_members cm ON c.id = cm.chat_id + WHERE cm.user_id = %s + ORDER BY c.created_at ASC, c.id ASC + """, (user["id"],)) + rows = cur.fetchall() - user_chats = [] - for r in rows: - cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (r['id'],)) - m_rows = cur.fetchall() - user_chats.append({ - "id": str(r['id']), - "title": r['title'], - "member_ids": [str(m['user_id']) for m in m_rows], - "created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at'] - }) - conn.close() + user_chats = [] + for r in rows: + cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (r['id'],)) + m_rows = cur.fetchall() + user_chats.append({ + "id": str(r['id']), + "title": r['title'], + "member_ids": [str(m['user_id']) for m in m_rows], + "created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at'] + }) + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) start_idx = 0 if cursor: @@ -696,18 +809,24 @@ def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = H user = get_current_user_from_token(authorization) chat_id_str = str(chat_id) - conn = get_db_connection() - with conn.cursor() as cur: - cur.execute(""" - SELECT c.* FROM chats c - JOIN chat_members cm ON c.id = cm.chat_id - WHERE c.id = %s AND cm.user_id = %s - """, (chat_id_str, user["id"])) - chat = cur.fetchone() - if chat: - cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (chat_id_str,)) - members = cur.fetchall() - conn.close() + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT c.* FROM chats c + JOIN chat_members cm ON c.id = cm.chat_id + WHERE c.id = %s AND cm.user_id = %s + """, (chat_id_str, user["id"])) + chat = cur.fetchone() + members = [] + if chat: + cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (chat_id_str,)) + members = cur.fetchall() + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) if not chat: raise HTTPException(status_code=404, detail="Chat not found") @@ -780,16 +899,22 @@ async def upload_attachment( user = get_current_user_from_token(authorization) chat_id_str = str(chat_id) - conn = get_db_connection() - with conn.cursor() as cur: - cur.execute( - "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", - (chat_id_str, user["id"]) - ) - if not cur.fetchone(): - conn.close() - raise HTTPException(status_code=404, detail="Chat not found or inaccessible") - conn.close() + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id_str, user["id"]) + ) + if not cur.fetchone(): + raise HTTPException(status_code=404, detail="Chat not found or inaccessible") + except HTTPException: + raise + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) if any(ord(c) < 0x20 or ord(c) == 0x7f for c in filename) or '/' in filename or '\\' in filename: raise HTTPException(status_code=400, detail="Invalid filename") @@ -833,6 +958,7 @@ async def upload_attachment( ContentType=content_type ) except Exception: + api_dependency_errors_counter.add(1, {"component": "s3"}) raise HTTPException(status_code=503, detail="S3 storage unavailable") job_payload = { @@ -845,7 +971,9 @@ async def upload_attachment( "created_at": now } - conn = get_db_connection() + inject(job_payload) + + conn = acquire_db_conn() try: with conn.cursor() as cur: cur.execute(""" @@ -866,10 +994,10 @@ async def upload_attachment( conn.commit() except Exception: conn.rollback() + api_dependency_errors_counter.add(1, {"component": "postgres"}) raise HTTPException(status_code=500, detail="Internal error saving attachment") finally: - if not conn.closed: - conn.close() + release_db_conn(conn) return { "id": attachment_id, @@ -895,15 +1023,20 @@ def get_attachment(attachment_id: UUID, response: Response, authorization: Optio user = get_current_user_from_token(authorization) att_id_str = str(attachment_id) - conn = get_db_connection() - with conn.cursor() as cur: - cur.execute(""" - SELECT a.* FROM attachments a - JOIN chat_members cm ON a.chat_id = cm.chat_id - WHERE a.id = %s AND cm.user_id = %s - """, (att_id_str, user["id"])) - row = cur.fetchone() - conn.close() + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT a.* FROM attachments a + JOIN chat_members cm ON a.chat_id = cm.chat_id + WHERE a.id = %s AND cm.user_id = %s + """, (att_id_str, user["id"])) + row = cur.fetchone() + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) if not row: raise HTTPException(status_code=404, detail="Attachment not found or inaccessible") @@ -937,15 +1070,20 @@ def download_attachment( user = get_current_user_from_token(authorization) att_id_str = str(attachment_id) - conn = get_db_connection() - with conn.cursor() as cur: - cur.execute(""" - SELECT a.* FROM attachments a - JOIN chat_members cm ON a.chat_id = cm.chat_id - WHERE a.id = %s AND cm.user_id = %s - """, (att_id_str, user["id"])) - row = cur.fetchone() - conn.close() + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT a.* FROM attachments a + JOIN chat_members cm ON a.chat_id = cm.chat_id + WHERE a.id = %s AND cm.user_id = %s + """, (att_id_str, user["id"])) + row = cur.fetchone() + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) if not row: raise HTTPException(status_code=404, detail="Attachment not found or inaccessible") @@ -968,6 +1106,7 @@ def download_attachment( obj = s3_client.get_object(Bucket=S3_BUCKET, Key=s3_key) data = obj['Body'].read() except Exception: + api_dependency_errors_counter.add(1, {"component": "s3"}) raise HTTPException(status_code=503, detail="S3 storage unavailable or object missing") return Response( @@ -1003,7 +1142,7 @@ def create_message( body_json_str = json.dumps(body.model_dump(mode='json'), sort_keys=True) request_hash = hashlib.sha256(body_json_str.encode("utf-8")).hexdigest() - conn = get_db_connection() + conn = acquire_db_conn() try: with conn.cursor() as cur: cur.execute( @@ -1011,7 +1150,6 @@ def create_message( (chat_id_str, user["id"]) ) if not cur.fetchone(): - conn.close() raise HTTPException(status_code=404, detail="Chat not found") attachment_ids_str = [str(aid) for aid in body.attachment_ids] @@ -1022,11 +1160,9 @@ def create_message( """, (attachment_ids_str, chat_id_str)) found_atts = cur.fetchall() if len(found_atts) != len(attachment_ids_str): - conn.close() raise HTTPException(status_code=404, detail="One or more attachments not found or in another chat") for att in found_atts: if att['status'] == 'failed': - conn.close() raise HTTPException(status_code=409, detail="Cannot attach failed attachment") msg_id = str(uuid.uuid4()) @@ -1055,7 +1191,6 @@ def create_message( """, (user["id"], method, path, idempotency_key)) row = cur.fetchone() conn.commit() - conn.close() if row["request_hash"] != request_hash: raise HTTPException(status_code=409, detail="Idempotency key conflict with different payload") @@ -1068,14 +1203,14 @@ def create_message( ) conn.commit() except HTTPException: + conn.rollback() raise except Exception: conn.rollback() - conn.close() + api_dependency_errors_counter.add(1, {"component": "postgres"}) raise HTTPException(status_code=500, detail="Internal error processing message") finally: - if not conn.closed: - conn.close() + release_db_conn(conn) return msg_response @@ -1089,31 +1224,39 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None, user = get_current_user_from_token(authorization) chat_id_str = str(chat_id) - conn = get_db_connection() - with conn.cursor() as cur: - cur.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", - (chat_id_str, user["id"])) - if not cur.fetchone(): - conn.close() - raise HTTPException(status_code=404, detail="Chat not found") - - if limit < 1 or limit > 100: - conn.close() - raise HTTPException(status_code=400, detail="Invalid limit") - - query = "SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id=%s" - params = [chat_id_str] if cursor: c_time, c_id = decode_cursor(cursor, chat_id_str) - query += " AND (created_at, id) > (%s, %s)" - params.extend([c_time, c_id]) - query += " ORDER BY created_at ASC, id ASC LIMIT %s" - params.append(limit + 1) + else: + c_time, c_id = None, None - with conn.cursor() as cur: - cur.execute(query, tuple(params)) - rows = cur.fetchall() - conn.close() + if limit < 1 or limit > 100: + raise HTTPException(status_code=400, detail="Invalid limit") + + conn = acquire_db_conn() + try: + with conn.cursor() as cur: + cur.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", + (chat_id_str, user["id"])) + if not cur.fetchone(): + raise HTTPException(status_code=404, detail="Chat not found") + + query = "SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id=%s" + params = [chat_id_str] + if cursor: + query += " AND (created_at, id) > (%s, %s)" + params.extend([c_time, c_id]) + query += " ORDER BY created_at ASC, id ASC LIMIT %s" + params.append(limit + 1) + + cur.execute(query, tuple(params)) + rows = cur.fetchall() + except HTTPException: + raise + except Exception: + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") + finally: + release_db_conn(conn) has_next = len(rows) > limit page_items = rows[:limit] @@ -1147,7 +1290,7 @@ def replay_attachment(attachment_id: UUID, response: Response, user = get_current_user_from_token(authorization) att_id = str(attachment_id) - conn = get_db_connection() + conn = acquire_db_conn() try: with conn.cursor() as cur: cur.execute(""" @@ -1157,10 +1300,8 @@ def replay_attachment(attachment_id: UUID, response: Response, """, (att_id, user["id"])) row = cur.fetchone() if not row: - conn.close() raise HTTPException(status_code=404, detail="Attachment not found") if row["status"] != "failed": - conn.close() raise HTTPException(status_code=409, detail="Only failed attachments can be replayed") now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") @@ -1172,12 +1313,19 @@ def replay_attachment(attachment_id: UUID, response: Response, job = {"schema_version": 1, "job_id": str(uuid.uuid4()), "type": "attachment.process.v1", "attachment_id": att_id, "pipeline_version": 1, "attempt": 1, "created_at": now} + inject(job) cur.execute("INSERT INTO outbox (id, stream, payload, created_at) " "VALUES (%s, %s, %s::jsonb, %s)", (str(uuid.uuid4()), STREAM_NAME, json.dumps(job), now)) conn.commit() + except HTTPException: + conn.rollback() + raise + except Exception: + conn.rollback() + api_dependency_errors_counter.add(1, {"component": "postgres"}) + raise HTTPException(status_code=503, detail="Database unavailable") finally: - if not conn.closed: - conn.close() + release_db_conn(conn) return {"id": att_id, "status": "queued"} \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 456ab94..32958ee 100644 --- a/compose.yaml +++ b/compose.yaml @@ -97,8 +97,9 @@ services: - "--storage.tsdb.retention.time=7d" - "--web.enable-remote-write-receiver" volumes: - - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro - - promdata:/prometheus + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro + - promdata:/prometheus healthcheck: test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9090/-/healthy || exit 1"] interval: 5s diff --git a/infra/grafana/dashboards/messenger-grade3.json b/infra/grafana/dashboards/messenger-grade3.json index 6b16499..ef8b211 100644 --- a/infra/grafana/dashboards/messenger-grade3.json +++ b/infra/grafana/dashboards/messenger-grade3.json @@ -78,7 +78,7 @@ "targets": [ { "datasource": { "type": "loki", "uid": "loki" }, - "expr": "{service_name=~\"messenger-api|messenger-worker\"}" + "expr": "{service_name=~\".+\"}" } ] } diff --git a/infra/grafana/dashboards/messenger-grade4.json b/infra/grafana/dashboards/messenger-grade4.json new file mode 100644 index 0000000..e979547 --- /dev/null +++ b/infra/grafana/dashboards/messenger-grade4.json @@ -0,0 +1,147 @@ +{ + "title": "MTUSI Messenger — Lab 6 (Grade 4)", + "uid": "mtusi-messenger-lab6-g4", + "schemaVersion": 39, + "version": 3, + "refresh": "5s", + "panels": [ + { + "id": 1, + "title": "API RPS (req/s)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "reqps" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(http_server_requests_total[1m]))", + "legendFormat": "req/s" + } + ] + }, + { + "id": 2, + "title": "API Error Ratio (%)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "percentunit" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(http_server_errors_total[1m])) / clamp_min(sum(rate(http_server_requests_total[1m])), 0.0001)", + "legendFormat": "error_ratio" + } + ] + }, + { + "id": 3, + "title": "API Latency p50 / p95 / p99 (s)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 6, "x": 12, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.50, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p50" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p95" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p99" + } + ] + }, + { + "id": 4, + "title": "DB Connection Pool (used / idle)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 6, "x": 18, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(db_pool_connections) by (state)", + "legendFormat": "{{state}}" + } + ] + }, + { + "id": 5, + "title": "Queue Depth & Oldest Job Age", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 7 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(worker_queue_depth)", + "legendFormat": "queue_depth (jobs)" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "max(worker_oldest_job_age_seconds)", + "legendFormat": "oldest_job_age (s)" + } + ] + }, + { + "id": 6, + "title": "Worker Saturation & Retries", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 7 }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(worker_jobs_in_progress)", + "legendFormat": "in_progress (jobs)" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "sum(rate(worker_job_retries_total[1m])) by (reason)", + "legendFormat": "retries/s: {{reason}}" + } + ] + }, + { + "id": 7, + "title": "Worker Job Duration p50 / p95 / p99 (s)", + "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 7 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.50, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p50" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.95, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p95" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "histogram_quantile(0.99, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))", + "legendFormat": "p99" + } + ] + }, + { + "id": 8, + "title": "Logs with TraceID (Loki)", + "type": "logs", + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 14 }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "loki" }, + "expr": "{service_name=~\".+\"}" + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml index e817167..09c1046 100644 --- a/infra/otel/collector.yaml +++ b/infra/otel/collector.yaml @@ -24,18 +24,25 @@ exporters: endpoint: http://prometheus:9090/api/v1/write tls: insecure: true + sending_queue: + enabled: true + queue_size: 1000 otlphttp/tempo: endpoint: http://tempo:4318 tls: insecure: true + sending_queue: + enabled: true + queue_size: 1000 otlphttp/loki: endpoint: http://loki:3100/otlp headers: X-Scope-OrgID: fake tls: insecure: true - debug: - verbosity: detailed + sending_queue: + enabled: true + queue_size: 1000 service: extensions: [health_check] diff --git a/infra/prometheus/alerts.yml b/infra/prometheus/alerts.yml new file mode 100644 index 0000000..cb4d25b --- /dev/null +++ b/infra/prometheus/alerts.yml @@ -0,0 +1,62 @@ +groups: + - name: messenger_api_alerts + interval: 15s + rules: + - alert: MessengerApiHighErrorRate + expr: | + ( + sum(rate(http_server_errors_total[1m])) + / + clamp_min(sum(rate(http_server_requests_total[1m])), 0.001) + ) > 0.05 + and sum(rate(http_server_requests_total[1m])) > 0.2 + for: 30s + labels: + severity: warning + service: messenger-api + annotations: + summary: "High HTTP error ratio (>5%) in messenger-api" + description: "Error ratio exceeded 5% over 30s; inspect Loki logs and Tempo traces." + + - alert: MessengerApiDependencyErrors + expr: sum(rate(http_server_dependency_errors_total[1m])) > 0 + for: 30s + labels: + severity: critical + service: messenger-api + annotations: + summary: "Dependency errors in messenger-api" + description: "PostgreSQL, Redis, S3, or outbox errors occurred." + + - alert: MessengerApiHighLatencyP95 + expr: histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le)) > 1.0 + for: 1m + labels: + severity: warning + service: messenger-api + annotations: + summary: "High API p95 latency (>1.0s)" + description: "API p95 duration is above 1.0s." + + - name: messenger_worker_alerts + interval: 15s + rules: + - alert: MessengerWorkerJobFailures + expr: sum(rate(worker_jobs_total{status="failed"}[1m])) > 0 + for: 30s + labels: + severity: warning + service: messenger-worker + annotations: + summary: "Attachment worker jobs failing" + description: "Failed worker jobs detected; check error_code and trace_id in Loki/Tempo." + + - alert: MessengerWorkerQueueLag + expr: max(worker_oldest_job_age_seconds) > 30 or sum(worker_queue_depth) > 20 + for: 30s + labels: + severity: critical + service: messenger-worker + annotations: + summary: "Growing attachment queue lag or backlog" + description: "Oldest job age > 30s or queue depth > 20; check worker saturation and dependencies." \ No newline at end of file diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml index 22240f6..cf87d7b 100644 --- a/infra/prometheus/prometheus.yml +++ b/infra/prometheus/prometheus.yml @@ -2,6 +2,9 @@ global: scrape_interval: 15s evaluation_interval: 15s +rule_files: + - /etc/prometheus/alerts.yml + scrape_configs: - job_name: prometheus static_configs: diff --git a/worker.py b/worker.py index 5f6a9fe..90da41b 100644 --- a/worker.py +++ b/worker.py @@ -29,6 +29,7 @@ from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.propagate import extract class JsonLogFormatter(logging.Formatter): @@ -106,20 +107,6 @@ def setup_worker_telemetry(): return trace.get_tracer(service_name), metrics.get_meter(service_name) -tracer, meter = setup_worker_telemetry() -logger = logging.getLogger("attachment-worker") - -worker_jobs_counter = meter.create_counter( - name="worker_jobs_total", - description="Completed and failed attachment jobs", - unit="1", -) -worker_duration_histogram = meter.create_histogram( - name="worker_job_duration_seconds", - description="Attachment job processing duration in seconds", - unit="s", -) - DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger") REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333") @@ -137,7 +124,6 @@ JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30")) WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4")) CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000 - Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True) @@ -147,6 +133,87 @@ s3_client = boto3.client( config=Config(signature_version="s3v4"), region_name="us-east-1" ) + +def observe_queue_metrics(options): + try: + q_len = redis_client.xlen(STREAM_NAME) + entries = redis_client.xrange(STREAM_NAME, count=1) + oldest_age = 0.0 + if entries: + msg_id, fields = entries[0] + try: + payload = json.loads(fields.get("payload", "{}")) + created_str = payload.get("created_at") + if created_str: + created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00")) + oldest_age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds()) + except Exception: + ms = int(msg_id.split("-")[0]) + oldest_age = max(0.0, time.time() - (ms / 1000.0)) + yield metrics.Observation(value=q_len, attributes={"stream": STREAM_NAME}) + except Exception: + yield metrics.Observation(value=0, attributes={"stream": STREAM_NAME}) + + +def observe_oldest_job_age(options): + try: + entries = redis_client.xrange(STREAM_NAME, count=1) + if not entries: + yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME}) + return + msg_id, fields = entries[0] + try: + payload = json.loads(fields.get("payload", "{}")) + created_str = payload.get("created_at") + if created_str: + created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00")) + age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds()) + yield metrics.Observation(value=age, attributes={"stream": STREAM_NAME}) + return + except Exception: + pass + ms = int(msg_id.split("-")[0]) + yield metrics.Observation(value=max(0.0, time.time() - (ms / 1000.0)), attributes={"stream": STREAM_NAME}) + except Exception: + yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME}) + + +tracer, meter = setup_worker_telemetry() +logger = logging.getLogger("attachment-worker") + +worker_jobs_counter = meter.create_counter( + name="worker_jobs_total", + description="Completed and failed attachment jobs", + unit="1", +) +worker_retries_counter = meter.create_counter( + name="worker_job_retries_total", + description="Transient retry attempts scheduled for attachment jobs", + unit="1", +) +worker_duration_histogram = meter.create_histogram( + name="worker_job_duration_seconds", + description="Attachment job processing duration in seconds", + unit="s", +) +worker_jobs_in_progress = meter.create_up_down_counter( + name="worker_jobs_in_progress", + description="Jobs currently being processed", + unit="1", +) +meter.create_observable_gauge( + name="worker_queue_depth", + callbacks=[observe_queue_metrics], + description="Number of messages currently in attachment jobs stream", + unit="1", +) +meter.create_observable_gauge( + name="worker_oldest_job_age_seconds", + callbacks=[observe_oldest_job_age], + description="Age of the oldest message in attachment jobs stream in seconds", + unit="s", +) + running = True @@ -193,7 +260,7 @@ def prepare_base_image(raw_bytes): return img -def generate_grade3_variants(raw_bytes): +def generate_grade4_variants(raw_bytes): base_img = prepare_base_image(raw_bytes) orig_w, orig_h = base_img.size meta = [] @@ -212,12 +279,42 @@ def generate_grade3_variants(raw_bytes): tb = buf.getvalue() blobs["thumbnail"] = tb meta.append({ - "name": "thumbnail", - "content_type": "image/jpeg", - "size_bytes": len(tb), - "width": t_w, - "height": t_h, - "sha256": hashlib.sha256(tb).hexdigest(), + "name": "thumbnail", "content_type": "image/jpeg", + "size_bytes": len(tb), "width": t_w, "height": t_h, + "sha256": hashlib.sha256(tb).hexdigest() + }) + + opt = base_img.copy() + if orig_w <= 1280 and orig_h <= 1280: + o_w, o_h = orig_w, orig_h + else: + s = min(1280.0 / orig_w, 1280.0 / orig_h) + o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s)) + if (o_w, o_h) != (orig_w, orig_h): + opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS) + buf = io.BytesIO() + opt.save(buf, format="WEBP", quality=85) + ob = buf.getvalue() + blobs["optimized"] = ob + meta.append({ + "name": "optimized", "content_type": "image/webp", + "size_bytes": len(ob), "width": o_w, "height": o_h, + "sha256": hashlib.sha256(ob).hexdigest() + }) + + c = min(orig_w, orig_h) + left, top = (orig_w - c) // 2, (orig_h - c) // 2 + crop = base_img.crop((left, top, left + c, top + c)) + if c != 128: + crop = crop.resize((128, 128), Image.Resampling.LANCZOS) + buf = io.BytesIO() + crop.save(buf, format="JPEG", quality=85, optimize=True) + cb = buf.getvalue() + blobs["crop"] = cb + meta.append({ + "name": "crop", "content_type": "image/jpeg", + "size_bytes": len(cb), "width": 128, "height": 128, + "sha256": hashlib.sha256(cb).hexdigest() }) return meta, blobs @@ -247,202 +344,209 @@ TRANSIENT_ERRORS = ( ) -def _record_job_outcome(status: str, elapsed: float): +def _record_job_outcome(span, status: str, elapsed: float): + span.set_attribute("job.outcome", status) worker_jobs_counter.add(1, {"status": status}) worker_duration_histogram.record(elapsed, {"status": status}) def process_job(message_id, fields): job_start = time.monotonic() - with tracer.start_as_current_span("worker.process_attachment") as span: + try: + job = json.loads(fields.get("payload", "{}")) + except Exception: + logger.error("Malformed job %s, ack to drop", message_id) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + return + + ctx = extract(job) + + with tracer.start_as_current_span("worker.process_attachment", context=ctx) as span: + worker_jobs_in_progress.add(1) try: - job = json.loads(fields.get("payload", "{}")) - except Exception: - logger.error("Malformed job %s, ack to drop", message_id) - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - _record_job_outcome("failed", time.monotonic() - job_start) - return + attachment_id = job.get("attachment_id") + job_id = job.get("job_id") + pipeline_version = int(job.get("pipeline_version", 1)) + attempt = int(job.get("attempt", 1)) - attachment_id = job.get("attachment_id") - job_id = job.get("job_id") - pipeline_version = int(job.get("pipeline_version", 1)) - attempt = int(job.get("attempt", 1)) + span.set_attribute("job.pipeline_version", pipeline_version) + span.set_attribute("job.attempt", attempt) - span.set_attribute("job.pipeline_version", pipeline_version) - span.set_attribute("job.attempt", attempt) + if not attachment_id: + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + return - if not attachment_id: - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - _record_job_outcome("failed", time.monotonic() - job_start) - return - - conn = get_db_connection() - try: - with conn.cursor() as cur: - cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)) - att = cur.fetchone() - if not att: - conn.commit() - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - _record_job_outcome("failed", time.monotonic() - job_start) - return - if att["status"] == "ready" or att["pipeline_version"] > pipeline_version: - conn.commit() - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - _record_job_outcome("success", time.monotonic() - job_start) - return - if att["status"] == "failed": - conn.commit() - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - _record_job_outcome("failed", time.monotonic() - job_start) - return - - if att["status"] == "processing": - updated = att["updated_at"] - if isinstance(updated, str): - updated = datetime.fromisoformat(updated.replace("Z", "+00:00")) - age_seconds = (datetime.now(timezone.utc) - updated).total_seconds() - if age_seconds < (JOB_TIMEOUT_SECONDS + 10): - conn.commit() - logger.info( - "Attachment already processing, ack duplicate", - extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt}, - ) - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - return - - now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - cur.execute( - "UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s", - (attempt, now, attachment_id) - ) - conn.commit() - finally: - conn.close() - - chat_id = str(att["chat_id"]) - staging_key = f"staging/{chat_id}/{attachment_id}/original" - final_original_key = f"attachments/{chat_id}/{attachment_id}/original" - - start_time = time.monotonic() - - def check_timeout(): - if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS: - raise TimeoutError("Job exceeded time limit") - - try: - try: - obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key) - except Exception: - obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key) - raw_bytes = obj["Body"].read() - check_timeout() - - if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]: - raise ValueError("Staging checksum mismatch") - - variants_meta, variants_bytes = generate_grade3_variants(raw_bytes) - check_timeout() - - s3_client.put_object( - Bucket=S3_BUCKET, Key=final_original_key, - Body=raw_bytes, ContentType=att["content_type"] - ) - for var_name, var_bytes in variants_bytes.items(): - ct = "image/webp" if var_name == "optimized" else "image/jpeg" - s3_client.put_object( - Bucket=S3_BUCKET, - Key=f"attachments/{chat_id}/{attachment_id}/{var_name}", - Body=var_bytes, ContentType=ct - ) - check_timeout() - - ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") conn = get_db_connection() try: with conn.cursor() as cur: - cur.execute(""" - UPDATE attachments SET status='ready', variants=%s::jsonb, - error_code=NULL, updated_at=%s - WHERE id=%s AND pipeline_version <= %s - """, (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version)) + cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,)) + att = cur.fetchone() + if not att: + conn.commit() + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + return + if att["status"] == "ready" or att["pipeline_version"] > pipeline_version: + conn.commit() + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "success", time.monotonic() - job_start) + return + if att["status"] == "failed": + conn.commit() + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + return + + if att["status"] == "processing": + updated = att["updated_at"] + if isinstance(updated, str): + updated = datetime.fromisoformat(updated.replace("Z", "+00:00")) + age_seconds = (datetime.now(timezone.utc) - updated).total_seconds() + if age_seconds < (JOB_TIMEOUT_SECONDS + 10): + conn.commit() + logger.info( + "Attachment already processing, ack duplicate", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt}, + ) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + return + + now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + cur.execute( + "UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s", + (attempt, now, attachment_id) + ) conn.commit() finally: conn.close() - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - _record_job_outcome("success", time.monotonic() - job_start) - logger.info( - "Job completed: attachment ready", - extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt}, - ) + chat_id = str(att["chat_id"]) + staging_key = f"staging/{chat_id}/{attachment_id}/original" + final_original_key = f"attachments/{chat_id}/{attachment_id}/original" + + start_time = time.monotonic() + + def check_timeout(): + if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS: + raise TimeoutError("Job exceeded time limit") try: - s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key) - except Exception: - pass - - except (UnidentifiedImageError, Image.DecompressionBombError) as e: - logger.error( - "Permanent image error", - extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"}, - ) - mark_failed(attachment_id, "invalid_image") - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - _record_job_outcome("failed", time.monotonic() - job_start) - - except ValueError as e: - logger.error( - "Checksum error", - extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"}, - ) - mark_failed(attachment_id, "invalid_checksum") - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - _record_job_outcome("failed", time.monotonic() - job_start) - - except psycopg2.errors.DataError as e: - logger.error( - "DB data error", - extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"}, - ) - mark_failed(attachment_id, "data_error") - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - _record_job_outcome("failed", time.monotonic() - job_start) - - except TRANSIENT_ERRORS as e: - logger.warning( - "Transient error during job processing", - extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt}, - ) - if attempt >= JOB_MAX_ATTEMPTS: - mark_failed(attachment_id, "retry_exhausted") try: - dlq_job = dict(job) - dlq_job["attempt"] = 1 - redis_client.xadd( - DLQ_STREAM_NAME, - {"payload": json.dumps(dlq_job), "error": type(e).__name__} + obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key) + except Exception: + obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key) + raw_bytes = obj["Body"].read() + check_timeout() + + if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]: + raise ValueError("Staging checksum mismatch") + + variants_meta, variants_bytes = generate_grade4_variants(raw_bytes) + check_timeout() + + s3_client.put_object( + Bucket=S3_BUCKET, Key=final_original_key, + Body=raw_bytes, ContentType=att["content_type"] + ) + for var_name, var_bytes in variants_bytes.items(): + ct = "image/webp" if var_name == "optimized" else "image/jpeg" + s3_client.put_object( + Bucket=S3_BUCKET, + Key=f"attachments/{chat_id}/{attachment_id}/{var_name}", + Body=var_bytes, ContentType=ct ) + check_timeout() + + ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + conn = get_db_connection() + try: + with conn.cursor() as cur: + cur.execute(""" + UPDATE attachments SET status='ready', variants=%s::jsonb, + error_code=NULL, updated_at=%s + WHERE id=%s AND pipeline_version <= %s + """, (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version)) + conn.commit() + finally: + conn.close() + + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "success", time.monotonic() - job_start) + logger.info( + "Job completed: attachment ready", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt}, + ) + + try: + s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key) except Exception: pass - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - _record_job_outcome("failed", time.monotonic() - job_start) - else: - delay = min(2 ** attempt, 30) + random.uniform(0, 1) - time.sleep(delay) - next_job = dict(job) - next_job["attempt"] = attempt + 1 - redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)}) - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - except Exception as e: - logger.error( - "Unknown error during job processing", - extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"}, - ) - mark_failed(attachment_id, "processing_failed") - redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) - _record_job_outcome("failed", time.monotonic() - job_start) + except (UnidentifiedImageError, Image.DecompressionBombError): + logger.error( + "Permanent image error", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"}, + ) + mark_failed(attachment_id, "invalid_image") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + + except ValueError as e: + logger.error( + "Checksum error", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"}, + ) + mark_failed(attachment_id, "invalid_checksum") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + + except psycopg2.errors.DataError: + logger.error( + "DB data error", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"}, + ) + mark_failed(attachment_id, "data_error") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + + except TRANSIENT_ERRORS as e: + logger.warning( + "Transient error during job processing", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt}, + ) + if attempt >= JOB_MAX_ATTEMPTS: + mark_failed(attachment_id, "retry_exhausted") + try: + dlq_job = dict(job) + dlq_job["attempt"] = 1 + redis_client.xadd( + DLQ_STREAM_NAME, + {"payload": json.dumps(dlq_job), "error": type(e).__name__} + ) + except Exception: + pass + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + else: + worker_retries_counter.add(1, {"reason": type(e).__name__}) + delay = min(2 ** attempt, 30) + random.uniform(0, 1) + time.sleep(delay) + next_job = dict(job) + next_job["attempt"] = attempt + 1 + redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)}) + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + + except Exception: + logger.error( + "Unknown error during job processing", + extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"}, + ) + mark_failed(attachment_id, "processing_failed") + redis_client.xack(STREAM_NAME, GROUP_NAME, message_id) + _record_job_outcome(span, "failed", time.monotonic() - job_start) + finally: + worker_jobs_in_progress.add(-1) def run_staging_cleanup():