This commit is contained in:
Getlles
2026-11-19 18:05:43 +03:00
parent 042d5f83f2
commit 33f2448e20
8 changed files with 803 additions and 331 deletions
+280 -132
View File
@@ -9,12 +9,14 @@ import struct
import hashlib
import hmac
import logging
import threading
from datetime import datetime, timezone, timedelta
from typing import List, Optional
from uuid import UUID
import psycopg2
import psycopg2.extras
from psycopg2.pool import ThreadedConnectionPool
psycopg2.extras.register_uuid()
import bcrypt
from fastapi import FastAPI, Header, HTTPException, Query, Response, Request
@@ -26,8 +28,6 @@ import redis
import boto3
from botocore.client import Config
import threading
from opentelemetry import metrics, trace
from opentelemetry._logs import set_logger_provider
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
@@ -41,6 +41,7 @@ from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.propagate import inject
class JsonLogFormatter(logging.Formatter):
@@ -134,6 +135,11 @@ api_errors_counter = meter.create_counter(
description="Total HTTP 4xx/5xx responses served by API",
unit="1",
)
api_dependency_errors_counter = meter.create_counter(
name="http_server_dependency_errors_total",
description="Dependency and internal errors in API (DB/Redis/S3)",
unit="1",
)
api_duration_histogram = meter.create_histogram(
name="http_server_request_duration_seconds",
description="HTTP request duration in seconds",
@@ -165,16 +171,66 @@ s3_client = boto3.client(
region_name="us-east-1"
)
db_pool_lock = threading.Lock()
db_pool: Optional[ThreadedConnectionPool] = None
_active_db_conns = 0
def get_db_connection():
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
def init_db_pool(minconn: int = 1, maxconn: int = 15):
global db_pool
if db_pool is None:
db_pool = ThreadedConnectionPool(
minconn=minconn,
maxconn=maxconn,
dsn=DATABASE_URL,
cursor_factory=psycopg2.extras.RealDictCursor,
)
def acquire_db_conn():
global _active_db_conns
if db_pool is None:
init_db_pool()
conn = db_pool.getconn()
with db_pool_lock:
_active_db_conns += 1
return conn
def release_db_conn(conn):
global _active_db_conns
if conn is None or db_pool is None:
return
try:
close_broken = bool(getattr(conn, "closed", 0))
db_pool.putconn(conn, close=close_broken)
finally:
with db_pool_lock:
_active_db_conns = max(0, _active_db_conns - 1)
def observe_db_pool(options):
max_c = db_pool.maxconn if db_pool else 0
with db_pool_lock:
used = min(_active_db_conns, max_c)
idle = max(0, max_c - used)
yield metrics.Observation(value=used, attributes={"state": "used", "instance": INSTANCE_ID})
yield metrics.Observation(value=idle, attributes={"state": "idle", "instance": INSTANCE_ID})
meter.create_observable_gauge(
name="db_pool_connections",
callbacks=[observe_db_pool],
description="PostgreSQL connection pool connections by state",
unit="1",
)
def outbox_publisher():
while True:
conn = None
try:
conn = get_db_connection()
conn = acquire_db_conn()
with conn.cursor() as cur:
cur.execute("""
SELECT id, stream, payload FROM outbox
@@ -188,18 +244,37 @@ def outbox_publisher():
cur.execute("UPDATE outbox SET published_at=NOW() WHERE id=%s", (row["id"],))
conn.commit()
except Exception as e:
if conn and not conn.closed:
try:
conn.rollback()
except Exception:
pass
api_dependency_errors_counter.add(1, {"component": "outbox"})
api_logger.error("Outbox publisher error: %s", type(e).__name__)
finally:
if conn is not None and not conn.closed:
conn.close()
release_db_conn(conn)
time.sleep(1)
@app.on_event("startup")
def _start_outbox_publisher():
def _start_services():
try:
init_db_pool(minconn=1, maxconn=15)
except Exception as e:
api_dependency_errors_counter.add(1, {"component": "postgres"})
api_logger.error("DB pool init error: %s", type(e).__name__)
threading.Thread(target=outbox_publisher, daemon=True).start()
@app.on_event("shutdown")
def _shutdown_services():
if db_pool is not None:
try:
db_pool.closeall()
except Exception:
pass
@app.middleware("http")
async def telemetry_middleware(request: Request, call_next):
start = time.monotonic()
@@ -342,6 +417,7 @@ async def http_exception_handler(request: Request, exc: HTTPException):
@app.exception_handler(Exception)
async def unhandled_exception_handler(request: Request, exc: Exception):
req_id = request.headers.get("X-Request-Id") or generate_request_id()
api_dependency_errors_counter.add(1, {"component": "unhandled"})
return make_error_response(
status_code=500,
code="internal_error",
@@ -400,13 +476,19 @@ def health_ready(response: Response):
req_id = generate_request_id()
response.headers["X-Request-Id"] = req_id
response.headers["X-Instance-Id"] = INSTANCE_ID
conn = None
try:
conn = get_db_connection()
conn.close()
conn = acquire_db_conn()
with conn.cursor() as cur:
cur.execute("SELECT 1")
cur.fetchone()
redis_client.ping()
s3_client.head_bucket(Bucket=S3_BUCKET)
except Exception:
api_dependency_errors_counter.add(1, {"component": "readiness"})
raise HTTPException(status_code=503, detail="Database, Redis or S3 unavailable")
finally:
release_db_conn(conn)
return {"status": "ok"}
@@ -420,7 +502,7 @@ def register_user(body: CreateUser, response: Response):
user_id = str(uuid.uuid4())
created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
cur.execute(
@@ -434,8 +516,12 @@ def register_user(body: CreateUser, response: Response):
status_code=409,
detail={"error": {"code": "conflict", "message": "Username already taken", "request_id": req_id}}
)
except Exception:
conn.rollback()
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=503, detail="Database unavailable")
finally:
conn.close()
release_db_conn(conn)
return {
"id": user_id,
@@ -469,11 +555,16 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
)
conn = get_db_connection()
with conn.cursor() as cur:
cur.execute("SELECT * FROM users WHERE username = %s", (username,))
user = cur.fetchone()
conn.close()
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
cur.execute("SELECT * FROM users WHERE username = %s", (username,))
user = cur.fetchone()
except Exception:
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=503, detail="Database unavailable")
finally:
release_db_conn(conn)
if not user or not bcrypt.checkpw(password.encode('utf-8'), user['password_hash'].encode('utf-8')):
raise HTTPException(
@@ -483,7 +574,12 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
)
token = secrets.token_urlsafe(48)
redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, str(user['id']))
try:
redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, str(user['id']))
except Exception:
api_dependency_errors_counter.add(1, {"component": "redis"})
raise HTTPException(status_code=503, detail="Session store unavailable")
expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS)
expires_at_str = expires_at.isoformat().replace("+00:00", "Z")
@@ -512,6 +608,7 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) ->
try:
user_id = redis_client.get(f"session:{token}")
except Exception:
api_dependency_errors_counter.add(1, {"component": "redis"})
raise HTTPException(status_code=503, detail="Session store unavailable")
if not user_id:
@@ -521,11 +618,16 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) ->
headers={"WWW-Authenticate": 'Bearer realm="messenger"'}
)
conn = get_db_connection()
with conn.cursor() as cur:
cur.execute("SELECT * FROM users WHERE id = %s", (user_id,))
row = cur.fetchone()
conn.close()
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
cur.execute("SELECT * FROM users WHERE id = %s", (user_id,))
row = cur.fetchone()
except Exception:
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=503, detail="Database unavailable")
finally:
release_db_conn(conn)
if not row:
raise HTTPException(
@@ -554,7 +656,11 @@ def revoke_session(authorization: Optional[str] = Header(None)):
)
token = authorization.split(" ")[1]
deleted = redis_client.delete(f"session:{token}")
try:
deleted = redis_client.delete(f"session:{token}")
except Exception:
api_dependency_errors_counter.add(1, {"component": "redis"})
raise HTTPException(status_code=503, detail="Session store unavailable")
if not deleted:
raise HTTPException(
@@ -594,19 +700,17 @@ def create_chat(body: CreateChat, response: Response, authorization: Optional[st
if len(member_ids_str) != len(set(member_ids_str)):
raise HTTPException(status_code=400, detail="Duplicate member IDs")
conn = get_db_connection()
with conn.cursor() as cur:
for mid_str in member_ids_str:
cur.execute("SELECT id FROM users WHERE id = %s", (mid_str,))
if not cur.fetchone():
conn.close()
raise HTTPException(status_code=400, detail=f"User {mid_str} not found")
chat_id = str(uuid.uuid4())
created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
for mid_str in member_ids_str:
cur.execute("SELECT id FROM users WHERE id = %s", (mid_str,))
if not cur.fetchone():
raise HTTPException(status_code=400, detail=f"User {mid_str} not found")
cur.execute(
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)",
(chat_id, body.title, created_at)
@@ -617,11 +721,15 @@ def create_chat(body: CreateChat, response: Response, authorization: Optional[st
(chat_id, mid_str)
)
conn.commit()
except HTTPException:
conn.rollback()
raise
except Exception:
conn.rollback()
conn.close()
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=500, detail="Internal error creating chat")
conn.close()
finally:
release_db_conn(conn)
return {
"id": chat_id,
@@ -641,27 +749,32 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response
if limit < 1 or limit > 100:
raise HTTPException(status_code=400, detail="Invalid limit")
conn = get_db_connection()
with conn.cursor() as cur:
cur.execute("""
SELECT c.id, c.title, c.created_at FROM chats c
JOIN chat_members cm ON c.id = cm.chat_id
WHERE cm.user_id = %s
ORDER BY c.created_at ASC, c.id ASC
""", (user["id"],))
rows = cur.fetchall()
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
cur.execute("""
SELECT c.id, c.title, c.created_at FROM chats c
JOIN chat_members cm ON c.id = cm.chat_id
WHERE cm.user_id = %s
ORDER BY c.created_at ASC, c.id ASC
""", (user["id"],))
rows = cur.fetchall()
user_chats = []
for r in rows:
cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (r['id'],))
m_rows = cur.fetchall()
user_chats.append({
"id": str(r['id']),
"title": r['title'],
"member_ids": [str(m['user_id']) for m in m_rows],
"created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at']
})
conn.close()
user_chats = []
for r in rows:
cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (r['id'],))
m_rows = cur.fetchall()
user_chats.append({
"id": str(r['id']),
"title": r['title'],
"member_ids": [str(m['user_id']) for m in m_rows],
"created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at']
})
except Exception:
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=503, detail="Database unavailable")
finally:
release_db_conn(conn)
start_idx = 0
if cursor:
@@ -696,18 +809,24 @@ def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = H
user = get_current_user_from_token(authorization)
chat_id_str = str(chat_id)
conn = get_db_connection()
with conn.cursor() as cur:
cur.execute("""
SELECT c.* FROM chats c
JOIN chat_members cm ON c.id = cm.chat_id
WHERE c.id = %s AND cm.user_id = %s
""", (chat_id_str, user["id"]))
chat = cur.fetchone()
if chat:
cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (chat_id_str,))
members = cur.fetchall()
conn.close()
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
cur.execute("""
SELECT c.* FROM chats c
JOIN chat_members cm ON c.id = cm.chat_id
WHERE c.id = %s AND cm.user_id = %s
""", (chat_id_str, user["id"]))
chat = cur.fetchone()
members = []
if chat:
cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (chat_id_str,))
members = cur.fetchall()
except Exception:
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=503, detail="Database unavailable")
finally:
release_db_conn(conn)
if not chat:
raise HTTPException(status_code=404, detail="Chat not found")
@@ -780,16 +899,22 @@ async def upload_attachment(
user = get_current_user_from_token(authorization)
chat_id_str = str(chat_id)
conn = get_db_connection()
with conn.cursor() as cur:
cur.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id_str, user["id"])
)
if not cur.fetchone():
conn.close()
raise HTTPException(status_code=404, detail="Chat not found or inaccessible")
conn.close()
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
cur.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id_str, user["id"])
)
if not cur.fetchone():
raise HTTPException(status_code=404, detail="Chat not found or inaccessible")
except HTTPException:
raise
except Exception:
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=503, detail="Database unavailable")
finally:
release_db_conn(conn)
if any(ord(c) < 0x20 or ord(c) == 0x7f for c in filename) or '/' in filename or '\\' in filename:
raise HTTPException(status_code=400, detail="Invalid filename")
@@ -833,6 +958,7 @@ async def upload_attachment(
ContentType=content_type
)
except Exception:
api_dependency_errors_counter.add(1, {"component": "s3"})
raise HTTPException(status_code=503, detail="S3 storage unavailable")
job_payload = {
@@ -845,7 +971,9 @@ async def upload_attachment(
"created_at": now
}
conn = get_db_connection()
inject(job_payload)
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
cur.execute("""
@@ -866,10 +994,10 @@ async def upload_attachment(
conn.commit()
except Exception:
conn.rollback()
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=500, detail="Internal error saving attachment")
finally:
if not conn.closed:
conn.close()
release_db_conn(conn)
return {
"id": attachment_id,
@@ -895,15 +1023,20 @@ def get_attachment(attachment_id: UUID, response: Response, authorization: Optio
user = get_current_user_from_token(authorization)
att_id_str = str(attachment_id)
conn = get_db_connection()
with conn.cursor() as cur:
cur.execute("""
SELECT a.* FROM attachments a
JOIN chat_members cm ON a.chat_id = cm.chat_id
WHERE a.id = %s AND cm.user_id = %s
""", (att_id_str, user["id"]))
row = cur.fetchone()
conn.close()
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
cur.execute("""
SELECT a.* FROM attachments a
JOIN chat_members cm ON a.chat_id = cm.chat_id
WHERE a.id = %s AND cm.user_id = %s
""", (att_id_str, user["id"]))
row = cur.fetchone()
except Exception:
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=503, detail="Database unavailable")
finally:
release_db_conn(conn)
if not row:
raise HTTPException(status_code=404, detail="Attachment not found or inaccessible")
@@ -937,15 +1070,20 @@ def download_attachment(
user = get_current_user_from_token(authorization)
att_id_str = str(attachment_id)
conn = get_db_connection()
with conn.cursor() as cur:
cur.execute("""
SELECT a.* FROM attachments a
JOIN chat_members cm ON a.chat_id = cm.chat_id
WHERE a.id = %s AND cm.user_id = %s
""", (att_id_str, user["id"]))
row = cur.fetchone()
conn.close()
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
cur.execute("""
SELECT a.* FROM attachments a
JOIN chat_members cm ON a.chat_id = cm.chat_id
WHERE a.id = %s AND cm.user_id = %s
""", (att_id_str, user["id"]))
row = cur.fetchone()
except Exception:
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=503, detail="Database unavailable")
finally:
release_db_conn(conn)
if not row:
raise HTTPException(status_code=404, detail="Attachment not found or inaccessible")
@@ -968,6 +1106,7 @@ def download_attachment(
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=s3_key)
data = obj['Body'].read()
except Exception:
api_dependency_errors_counter.add(1, {"component": "s3"})
raise HTTPException(status_code=503, detail="S3 storage unavailable or object missing")
return Response(
@@ -1003,7 +1142,7 @@ def create_message(
body_json_str = json.dumps(body.model_dump(mode='json'), sort_keys=True)
request_hash = hashlib.sha256(body_json_str.encode("utf-8")).hexdigest()
conn = get_db_connection()
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
cur.execute(
@@ -1011,7 +1150,6 @@ def create_message(
(chat_id_str, user["id"])
)
if not cur.fetchone():
conn.close()
raise HTTPException(status_code=404, detail="Chat not found")
attachment_ids_str = [str(aid) for aid in body.attachment_ids]
@@ -1022,11 +1160,9 @@ def create_message(
""", (attachment_ids_str, chat_id_str))
found_atts = cur.fetchall()
if len(found_atts) != len(attachment_ids_str):
conn.close()
raise HTTPException(status_code=404, detail="One or more attachments not found or in another chat")
for att in found_atts:
if att['status'] == 'failed':
conn.close()
raise HTTPException(status_code=409, detail="Cannot attach failed attachment")
msg_id = str(uuid.uuid4())
@@ -1055,7 +1191,6 @@ def create_message(
""", (user["id"], method, path, idempotency_key))
row = cur.fetchone()
conn.commit()
conn.close()
if row["request_hash"] != request_hash:
raise HTTPException(status_code=409, detail="Idempotency key conflict with different payload")
@@ -1068,14 +1203,14 @@ def create_message(
)
conn.commit()
except HTTPException:
conn.rollback()
raise
except Exception:
conn.rollback()
conn.close()
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=500, detail="Internal error processing message")
finally:
if not conn.closed:
conn.close()
release_db_conn(conn)
return msg_response
@@ -1089,31 +1224,39 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None,
user = get_current_user_from_token(authorization)
chat_id_str = str(chat_id)
conn = get_db_connection()
with conn.cursor() as cur:
cur.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",
(chat_id_str, user["id"]))
if not cur.fetchone():
conn.close()
raise HTTPException(status_code=404, detail="Chat not found")
if limit < 1 or limit > 100:
conn.close()
raise HTTPException(status_code=400, detail="Invalid limit")
query = "SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id=%s"
params = [chat_id_str]
if cursor:
c_time, c_id = decode_cursor(cursor, chat_id_str)
query += " AND (created_at, id) > (%s, %s)"
params.extend([c_time, c_id])
query += " ORDER BY created_at ASC, id ASC LIMIT %s"
params.append(limit + 1)
else:
c_time, c_id = None, None
with conn.cursor() as cur:
cur.execute(query, tuple(params))
rows = cur.fetchall()
conn.close()
if limit < 1 or limit > 100:
raise HTTPException(status_code=400, detail="Invalid limit")
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
cur.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",
(chat_id_str, user["id"]))
if not cur.fetchone():
raise HTTPException(status_code=404, detail="Chat not found")
query = "SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id=%s"
params = [chat_id_str]
if cursor:
query += " AND (created_at, id) > (%s, %s)"
params.extend([c_time, c_id])
query += " ORDER BY created_at ASC, id ASC LIMIT %s"
params.append(limit + 1)
cur.execute(query, tuple(params))
rows = cur.fetchall()
except HTTPException:
raise
except Exception:
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=503, detail="Database unavailable")
finally:
release_db_conn(conn)
has_next = len(rows) > limit
page_items = rows[:limit]
@@ -1147,7 +1290,7 @@ def replay_attachment(attachment_id: UUID, response: Response,
user = get_current_user_from_token(authorization)
att_id = str(attachment_id)
conn = get_db_connection()
conn = acquire_db_conn()
try:
with conn.cursor() as cur:
cur.execute("""
@@ -1157,10 +1300,8 @@ def replay_attachment(attachment_id: UUID, response: Response,
""", (att_id, user["id"]))
row = cur.fetchone()
if not row:
conn.close()
raise HTTPException(status_code=404, detail="Attachment not found")
if row["status"] != "failed":
conn.close()
raise HTTPException(status_code=409, detail="Only failed attachments can be replayed")
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
@@ -1172,12 +1313,19 @@ def replay_attachment(attachment_id: UUID, response: Response,
job = {"schema_version": 1, "job_id": str(uuid.uuid4()),
"type": "attachment.process.v1", "attachment_id": att_id,
"pipeline_version": 1, "attempt": 1, "created_at": now}
inject(job)
cur.execute("INSERT INTO outbox (id, stream, payload, created_at) "
"VALUES (%s, %s, %s::jsonb, %s)",
(str(uuid.uuid4()), STREAM_NAME, json.dumps(job), now))
conn.commit()
except HTTPException:
conn.rollback()
raise
except Exception:
conn.rollback()
api_dependency_errors_counter.add(1, {"component": "postgres"})
raise HTTPException(status_code=503, detail="Database unavailable")
finally:
if not conn.closed:
conn.close()
release_db_conn(conn)
return {"id": att_id, "status": "queued"}
+3 -2
View File
@@ -97,8 +97,9 @@ services:
- "--storage.tsdb.retention.time=7d"
- "--web.enable-remote-write-receiver"
volumes:
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- promdata:/prometheus
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- ./infra/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
- promdata:/prometheus
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9090/-/healthy || exit 1"]
interval: 5s
@@ -78,7 +78,7 @@
"targets": [
{
"datasource": { "type": "loki", "uid": "loki" },
"expr": "{service_name=~\"messenger-api|messenger-worker\"}"
"expr": "{service_name=~\".+\"}"
}
]
}
@@ -0,0 +1,147 @@
{
"title": "MTUSI Messenger — Lab 6 (Grade 4)",
"uid": "mtusi-messenger-lab6-g4",
"schemaVersion": 39,
"version": 3,
"refresh": "5s",
"panels": [
{
"id": 1,
"title": "API RPS (req/s)",
"type": "timeseries",
"gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 },
"fieldConfig": { "defaults": { "unit": "reqps" } },
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(rate(http_server_requests_total[1m]))",
"legendFormat": "req/s"
}
]
},
{
"id": 2,
"title": "API Error Ratio (%)",
"type": "timeseries",
"gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 },
"fieldConfig": { "defaults": { "unit": "percentunit" } },
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(rate(http_server_errors_total[1m])) / clamp_min(sum(rate(http_server_requests_total[1m])), 0.0001)",
"legendFormat": "error_ratio"
}
]
},
{
"id": 3,
"title": "API Latency p50 / p95 / p99 (s)",
"type": "timeseries",
"gridPos": { "h": 7, "w": 6, "x": 12, "y": 0 },
"fieldConfig": { "defaults": { "unit": "s" } },
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.50, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))",
"legendFormat": "p50"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))",
"legendFormat": "p95"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.99, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))",
"legendFormat": "p99"
}
]
},
{
"id": 4,
"title": "DB Connection Pool (used / idle)",
"type": "timeseries",
"gridPos": { "h": 7, "w": 6, "x": 18, "y": 0 },
"fieldConfig": { "defaults": { "unit": "short" } },
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(db_pool_connections) by (state)",
"legendFormat": "{{state}}"
}
]
},
{
"id": 5,
"title": "Queue Depth & Oldest Job Age",
"type": "timeseries",
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 7 },
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(worker_queue_depth)",
"legendFormat": "queue_depth (jobs)"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "max(worker_oldest_job_age_seconds)",
"legendFormat": "oldest_job_age (s)"
}
]
},
{
"id": 6,
"title": "Worker Saturation & Retries",
"type": "timeseries",
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 7 },
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(worker_jobs_in_progress)",
"legendFormat": "in_progress (jobs)"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "sum(rate(worker_job_retries_total[1m])) by (reason)",
"legendFormat": "retries/s: {{reason}}"
}
]
},
{
"id": 7,
"title": "Worker Job Duration p50 / p95 / p99 (s)",
"type": "timeseries",
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 7 },
"fieldConfig": { "defaults": { "unit": "s" } },
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.50, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))",
"legendFormat": "p50"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.95, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))",
"legendFormat": "p95"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(0.99, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))",
"legendFormat": "p99"
}
]
},
{
"id": 8,
"title": "Logs with TraceID (Loki)",
"type": "logs",
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 14 },
"targets": [
{
"datasource": { "type": "loki", "uid": "loki" },
"expr": "{service_name=~\".+\"}"
}
]
}
]
}
+9 -2
View File
@@ -24,18 +24,25 @@ exporters:
endpoint: http://prometheus:9090/api/v1/write
tls:
insecure: true
sending_queue:
enabled: true
queue_size: 1000
otlphttp/tempo:
endpoint: http://tempo:4318
tls:
insecure: true
sending_queue:
enabled: true
queue_size: 1000
otlphttp/loki:
endpoint: http://loki:3100/otlp
headers:
X-Scope-OrgID: fake
tls:
insecure: true
debug:
verbosity: detailed
sending_queue:
enabled: true
queue_size: 1000
service:
extensions: [health_check]
+62
View File
@@ -0,0 +1,62 @@
groups:
- name: messenger_api_alerts
interval: 15s
rules:
- alert: MessengerApiHighErrorRate
expr: |
(
sum(rate(http_server_errors_total[1m]))
/
clamp_min(sum(rate(http_server_requests_total[1m])), 0.001)
) > 0.05
and sum(rate(http_server_requests_total[1m])) > 0.2
for: 30s
labels:
severity: warning
service: messenger-api
annotations:
summary: "High HTTP error ratio (>5%) in messenger-api"
description: "Error ratio exceeded 5% over 30s; inspect Loki logs and Tempo traces."
- alert: MessengerApiDependencyErrors
expr: sum(rate(http_server_dependency_errors_total[1m])) > 0
for: 30s
labels:
severity: critical
service: messenger-api
annotations:
summary: "Dependency errors in messenger-api"
description: "PostgreSQL, Redis, S3, or outbox errors occurred."
- alert: MessengerApiHighLatencyP95
expr: histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le)) > 1.0
for: 1m
labels:
severity: warning
service: messenger-api
annotations:
summary: "High API p95 latency (>1.0s)"
description: "API p95 duration is above 1.0s."
- name: messenger_worker_alerts
interval: 15s
rules:
- alert: MessengerWorkerJobFailures
expr: sum(rate(worker_jobs_total{status="failed"}[1m])) > 0
for: 30s
labels:
severity: warning
service: messenger-worker
annotations:
summary: "Attachment worker jobs failing"
description: "Failed worker jobs detected; check error_code and trace_id in Loki/Tempo."
- alert: MessengerWorkerQueueLag
expr: max(worker_oldest_job_age_seconds) > 30 or sum(worker_queue_depth) > 20
for: 30s
labels:
severity: critical
service: messenger-worker
annotations:
summary: "Growing attachment queue lag or backlog"
description: "Oldest job age > 30s or queue depth > 20; check worker saturation and dependencies."
+3
View File
@@ -2,6 +2,9 @@ global:
scrape_interval: 15s
evaluation_interval: 15s
rule_files:
- /etc/prometheus/alerts.yml
scrape_configs:
- job_name: prometheus
static_configs:
+298 -194
View File
@@ -29,6 +29,7 @@ from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.propagate import extract
class JsonLogFormatter(logging.Formatter):
@@ -106,20 +107,6 @@ def setup_worker_telemetry():
return trace.get_tracer(service_name), metrics.get_meter(service_name)
tracer, meter = setup_worker_telemetry()
logger = logging.getLogger("attachment-worker")
worker_jobs_counter = meter.create_counter(
name="worker_jobs_total",
description="Completed and failed attachment jobs",
unit="1",
)
worker_duration_histogram = meter.create_histogram(
name="worker_job_duration_seconds",
description="Attachment job processing duration in seconds",
unit="s",
)
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
@@ -137,7 +124,6 @@ JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30"))
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4"))
CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
@@ -147,6 +133,87 @@ s3_client = boto3.client(
config=Config(signature_version="s3v4"), region_name="us-east-1"
)
def observe_queue_metrics(options):
try:
q_len = redis_client.xlen(STREAM_NAME)
entries = redis_client.xrange(STREAM_NAME, count=1)
oldest_age = 0.0
if entries:
msg_id, fields = entries[0]
try:
payload = json.loads(fields.get("payload", "{}"))
created_str = payload.get("created_at")
if created_str:
created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00"))
oldest_age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds())
except Exception:
ms = int(msg_id.split("-")[0])
oldest_age = max(0.0, time.time() - (ms / 1000.0))
yield metrics.Observation(value=q_len, attributes={"stream": STREAM_NAME})
except Exception:
yield metrics.Observation(value=0, attributes={"stream": STREAM_NAME})
def observe_oldest_job_age(options):
try:
entries = redis_client.xrange(STREAM_NAME, count=1)
if not entries:
yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME})
return
msg_id, fields = entries[0]
try:
payload = json.loads(fields.get("payload", "{}"))
created_str = payload.get("created_at")
if created_str:
created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00"))
age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds())
yield metrics.Observation(value=age, attributes={"stream": STREAM_NAME})
return
except Exception:
pass
ms = int(msg_id.split("-")[0])
yield metrics.Observation(value=max(0.0, time.time() - (ms / 1000.0)), attributes={"stream": STREAM_NAME})
except Exception:
yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME})
tracer, meter = setup_worker_telemetry()
logger = logging.getLogger("attachment-worker")
worker_jobs_counter = meter.create_counter(
name="worker_jobs_total",
description="Completed and failed attachment jobs",
unit="1",
)
worker_retries_counter = meter.create_counter(
name="worker_job_retries_total",
description="Transient retry attempts scheduled for attachment jobs",
unit="1",
)
worker_duration_histogram = meter.create_histogram(
name="worker_job_duration_seconds",
description="Attachment job processing duration in seconds",
unit="s",
)
worker_jobs_in_progress = meter.create_up_down_counter(
name="worker_jobs_in_progress",
description="Jobs currently being processed",
unit="1",
)
meter.create_observable_gauge(
name="worker_queue_depth",
callbacks=[observe_queue_metrics],
description="Number of messages currently in attachment jobs stream",
unit="1",
)
meter.create_observable_gauge(
name="worker_oldest_job_age_seconds",
callbacks=[observe_oldest_job_age],
description="Age of the oldest message in attachment jobs stream in seconds",
unit="s",
)
running = True
@@ -193,7 +260,7 @@ def prepare_base_image(raw_bytes):
return img
def generate_grade3_variants(raw_bytes):
def generate_grade4_variants(raw_bytes):
base_img = prepare_base_image(raw_bytes)
orig_w, orig_h = base_img.size
meta = []
@@ -212,12 +279,42 @@ def generate_grade3_variants(raw_bytes):
tb = buf.getvalue()
blobs["thumbnail"] = tb
meta.append({
"name": "thumbnail",
"content_type": "image/jpeg",
"size_bytes": len(tb),
"width": t_w,
"height": t_h,
"sha256": hashlib.sha256(tb).hexdigest(),
"name": "thumbnail", "content_type": "image/jpeg",
"size_bytes": len(tb), "width": t_w, "height": t_h,
"sha256": hashlib.sha256(tb).hexdigest()
})
opt = base_img.copy()
if orig_w <= 1280 and orig_h <= 1280:
o_w, o_h = orig_w, orig_h
else:
s = min(1280.0 / orig_w, 1280.0 / orig_h)
o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
if (o_w, o_h) != (orig_w, orig_h):
opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS)
buf = io.BytesIO()
opt.save(buf, format="WEBP", quality=85)
ob = buf.getvalue()
blobs["optimized"] = ob
meta.append({
"name": "optimized", "content_type": "image/webp",
"size_bytes": len(ob), "width": o_w, "height": o_h,
"sha256": hashlib.sha256(ob).hexdigest()
})
c = min(orig_w, orig_h)
left, top = (orig_w - c) // 2, (orig_h - c) // 2
crop = base_img.crop((left, top, left + c, top + c))
if c != 128:
crop = crop.resize((128, 128), Image.Resampling.LANCZOS)
buf = io.BytesIO()
crop.save(buf, format="JPEG", quality=85, optimize=True)
cb = buf.getvalue()
blobs["crop"] = cb
meta.append({
"name": "crop", "content_type": "image/jpeg",
"size_bytes": len(cb), "width": 128, "height": 128,
"sha256": hashlib.sha256(cb).hexdigest()
})
return meta, blobs
@@ -247,202 +344,209 @@ TRANSIENT_ERRORS = (
)
def _record_job_outcome(status: str, elapsed: float):
def _record_job_outcome(span, status: str, elapsed: float):
span.set_attribute("job.outcome", status)
worker_jobs_counter.add(1, {"status": status})
worker_duration_histogram.record(elapsed, {"status": status})
def process_job(message_id, fields):
job_start = time.monotonic()
with tracer.start_as_current_span("worker.process_attachment") as span:
try:
job = json.loads(fields.get("payload", "{}"))
except Exception:
logger.error("Malformed job %s, ack to drop", message_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
ctx = extract(job)
with tracer.start_as_current_span("worker.process_attachment", context=ctx) as span:
worker_jobs_in_progress.add(1)
try:
job = json.loads(fields.get("payload", "{}"))
except Exception:
logger.error("Malformed job %s, ack to drop", message_id)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome("failed", time.monotonic() - job_start)
return
attachment_id = job.get("attachment_id")
job_id = job.get("job_id")
pipeline_version = int(job.get("pipeline_version", 1))
attempt = int(job.get("attempt", 1))
attachment_id = job.get("attachment_id")
job_id = job.get("job_id")
pipeline_version = int(job.get("pipeline_version", 1))
attempt = int(job.get("attempt", 1))
span.set_attribute("job.pipeline_version", pipeline_version)
span.set_attribute("job.attempt", attempt)
span.set_attribute("job.pipeline_version", pipeline_version)
span.set_attribute("job.attempt", attempt)
if not attachment_id:
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
return
if not attachment_id:
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome("failed", time.monotonic() - job_start)
return
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,))
att = cur.fetchone()
if not att:
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome("failed", time.monotonic() - job_start)
return
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome("success", time.monotonic() - job_start)
return
if att["status"] == "failed":
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome("failed", time.monotonic() - job_start)
return
if att["status"] == "processing":
updated = att["updated_at"]
if isinstance(updated, str):
updated = datetime.fromisoformat(updated.replace("Z", "+00:00"))
age_seconds = (datetime.now(timezone.utc) - updated).total_seconds()
if age_seconds < (JOB_TIMEOUT_SECONDS + 10):
conn.commit()
logger.info(
"Attachment already processing, ack duplicate",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
cur.execute(
"UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s",
(attempt, now, attachment_id)
)
conn.commit()
finally:
conn.close()
chat_id = str(att["chat_id"])
staging_key = f"staging/{chat_id}/{attachment_id}/original"
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
start_time = time.monotonic()
def check_timeout():
if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS:
raise TimeoutError("Job exceeded time limit")
try:
try:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
raw_bytes = obj["Body"].read()
check_timeout()
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
raise ValueError("Staging checksum mismatch")
variants_meta, variants_bytes = generate_grade3_variants(raw_bytes)
check_timeout()
s3_client.put_object(
Bucket=S3_BUCKET, Key=final_original_key,
Body=raw_bytes, ContentType=att["content_type"]
)
for var_name, var_bytes in variants_bytes.items():
ct = "image/webp" if var_name == "optimized" else "image/jpeg"
s3_client.put_object(
Bucket=S3_BUCKET,
Key=f"attachments/{chat_id}/{attachment_id}/{var_name}",
Body=var_bytes, ContentType=ct
)
check_timeout()
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("""
UPDATE attachments SET status='ready', variants=%s::jsonb,
error_code=NULL, updated_at=%s
WHERE id=%s AND pipeline_version <= %s
""", (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version))
cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,))
att = cur.fetchone()
if not att:
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
return
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "success", time.monotonic() - job_start)
return
if att["status"] == "failed":
conn.commit()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
return
if att["status"] == "processing":
updated = att["updated_at"]
if isinstance(updated, str):
updated = datetime.fromisoformat(updated.replace("Z", "+00:00"))
age_seconds = (datetime.now(timezone.utc) - updated).total_seconds()
if age_seconds < (JOB_TIMEOUT_SECONDS + 10):
conn.commit()
logger.info(
"Attachment already processing, ack duplicate",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
)
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
return
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
cur.execute(
"UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s",
(attempt, now, attachment_id)
)
conn.commit()
finally:
conn.close()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome("success", time.monotonic() - job_start)
logger.info(
"Job completed: attachment ready",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt},
)
chat_id = str(att["chat_id"])
staging_key = f"staging/{chat_id}/{attachment_id}/original"
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
start_time = time.monotonic()
def check_timeout():
if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS:
raise TimeoutError("Job exceeded time limit")
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
pass
except (UnidentifiedImageError, Image.DecompressionBombError) as e:
logger.error(
"Permanent image error",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"},
)
mark_failed(attachment_id, "invalid_image")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome("failed", time.monotonic() - job_start)
except ValueError as e:
logger.error(
"Checksum error",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"},
)
mark_failed(attachment_id, "invalid_checksum")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome("failed", time.monotonic() - job_start)
except psycopg2.errors.DataError as e:
logger.error(
"DB data error",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"},
)
mark_failed(attachment_id, "data_error")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome("failed", time.monotonic() - job_start)
except TRANSIENT_ERRORS as e:
logger.warning(
"Transient error during job processing",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
)
if attempt >= JOB_MAX_ATTEMPTS:
mark_failed(attachment_id, "retry_exhausted")
try:
dlq_job = dict(job)
dlq_job["attempt"] = 1
redis_client.xadd(
DLQ_STREAM_NAME,
{"payload": json.dumps(dlq_job), "error": type(e).__name__}
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
raw_bytes = obj["Body"].read()
check_timeout()
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
raise ValueError("Staging checksum mismatch")
variants_meta, variants_bytes = generate_grade4_variants(raw_bytes)
check_timeout()
s3_client.put_object(
Bucket=S3_BUCKET, Key=final_original_key,
Body=raw_bytes, ContentType=att["content_type"]
)
for var_name, var_bytes in variants_bytes.items():
ct = "image/webp" if var_name == "optimized" else "image/jpeg"
s3_client.put_object(
Bucket=S3_BUCKET,
Key=f"attachments/{chat_id}/{attachment_id}/{var_name}",
Body=var_bytes, ContentType=ct
)
check_timeout()
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
conn = get_db_connection()
try:
with conn.cursor() as cur:
cur.execute("""
UPDATE attachments SET status='ready', variants=%s::jsonb,
error_code=NULL, updated_at=%s
WHERE id=%s AND pipeline_version <= %s
""", (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version))
conn.commit()
finally:
conn.close()
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "success", time.monotonic() - job_start)
logger.info(
"Job completed: attachment ready",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt},
)
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
except Exception:
pass
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome("failed", time.monotonic() - job_start)
else:
delay = min(2 ** attempt, 30) + random.uniform(0, 1)
time.sleep(delay)
next_job = dict(job)
next_job["attempt"] = attempt + 1
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except Exception as e:
logger.error(
"Unknown error during job processing",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"},
)
mark_failed(attachment_id, "processing_failed")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome("failed", time.monotonic() - job_start)
except (UnidentifiedImageError, Image.DecompressionBombError):
logger.error(
"Permanent image error",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"},
)
mark_failed(attachment_id, "invalid_image")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
except ValueError as e:
logger.error(
"Checksum error",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"},
)
mark_failed(attachment_id, "invalid_checksum")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
except psycopg2.errors.DataError:
logger.error(
"DB data error",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"},
)
mark_failed(attachment_id, "data_error")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
except TRANSIENT_ERRORS as e:
logger.warning(
"Transient error during job processing",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
)
if attempt >= JOB_MAX_ATTEMPTS:
mark_failed(attachment_id, "retry_exhausted")
try:
dlq_job = dict(job)
dlq_job["attempt"] = 1
redis_client.xadd(
DLQ_STREAM_NAME,
{"payload": json.dumps(dlq_job), "error": type(e).__name__}
)
except Exception:
pass
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
else:
worker_retries_counter.add(1, {"reason": type(e).__name__})
delay = min(2 ** attempt, 30) + random.uniform(0, 1)
time.sleep(delay)
next_job = dict(job)
next_job["attempt"] = attempt + 1
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
except Exception:
logger.error(
"Unknown error during job processing",
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"},
)
mark_failed(attachment_id, "processing_failed")
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
_record_job_outcome(span, "failed", time.monotonic() - job_start)
finally:
worker_jobs_in_progress.add(-1)
def run_staging_cleanup():