forked from mtusi2026/mtusi2026lab6
grade4
This commit is contained in:
@@ -9,12 +9,14 @@ import struct
|
||||
import hashlib
|
||||
import hmac
|
||||
import logging
|
||||
import threading
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import List, Optional
|
||||
from uuid import UUID
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
from psycopg2.pool import ThreadedConnectionPool
|
||||
psycopg2.extras.register_uuid()
|
||||
import bcrypt
|
||||
from fastapi import FastAPI, Header, HTTPException, Query, Response, Request
|
||||
@@ -26,8 +28,6 @@ import redis
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
|
||||
import threading
|
||||
|
||||
from opentelemetry import metrics, trace
|
||||
from opentelemetry._logs import set_logger_provider
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
@@ -41,6 +41,7 @@ from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.propagate import inject
|
||||
|
||||
|
||||
class JsonLogFormatter(logging.Formatter):
|
||||
@@ -134,6 +135,11 @@ api_errors_counter = meter.create_counter(
|
||||
description="Total HTTP 4xx/5xx responses served by API",
|
||||
unit="1",
|
||||
)
|
||||
api_dependency_errors_counter = meter.create_counter(
|
||||
name="http_server_dependency_errors_total",
|
||||
description="Dependency and internal errors in API (DB/Redis/S3)",
|
||||
unit="1",
|
||||
)
|
||||
api_duration_histogram = meter.create_histogram(
|
||||
name="http_server_request_duration_seconds",
|
||||
description="HTTP request duration in seconds",
|
||||
@@ -165,16 +171,66 @@ s3_client = boto3.client(
|
||||
region_name="us-east-1"
|
||||
)
|
||||
|
||||
db_pool_lock = threading.Lock()
|
||||
db_pool: Optional[ThreadedConnectionPool] = None
|
||||
_active_db_conns = 0
|
||||
|
||||
def get_db_connection():
|
||||
return psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
||||
|
||||
def init_db_pool(minconn: int = 1, maxconn: int = 15):
|
||||
global db_pool
|
||||
if db_pool is None:
|
||||
db_pool = ThreadedConnectionPool(
|
||||
minconn=minconn,
|
||||
maxconn=maxconn,
|
||||
dsn=DATABASE_URL,
|
||||
cursor_factory=psycopg2.extras.RealDictCursor,
|
||||
)
|
||||
|
||||
|
||||
def acquire_db_conn():
|
||||
global _active_db_conns
|
||||
if db_pool is None:
|
||||
init_db_pool()
|
||||
conn = db_pool.getconn()
|
||||
with db_pool_lock:
|
||||
_active_db_conns += 1
|
||||
return conn
|
||||
|
||||
|
||||
def release_db_conn(conn):
|
||||
global _active_db_conns
|
||||
if conn is None or db_pool is None:
|
||||
return
|
||||
try:
|
||||
close_broken = bool(getattr(conn, "closed", 0))
|
||||
db_pool.putconn(conn, close=close_broken)
|
||||
finally:
|
||||
with db_pool_lock:
|
||||
_active_db_conns = max(0, _active_db_conns - 1)
|
||||
|
||||
|
||||
def observe_db_pool(options):
|
||||
max_c = db_pool.maxconn if db_pool else 0
|
||||
with db_pool_lock:
|
||||
used = min(_active_db_conns, max_c)
|
||||
idle = max(0, max_c - used)
|
||||
yield metrics.Observation(value=used, attributes={"state": "used", "instance": INSTANCE_ID})
|
||||
yield metrics.Observation(value=idle, attributes={"state": "idle", "instance": INSTANCE_ID})
|
||||
|
||||
|
||||
meter.create_observable_gauge(
|
||||
name="db_pool_connections",
|
||||
callbacks=[observe_db_pool],
|
||||
description="PostgreSQL connection pool connections by state",
|
||||
unit="1",
|
||||
)
|
||||
|
||||
|
||||
def outbox_publisher():
|
||||
while True:
|
||||
conn = None
|
||||
try:
|
||||
conn = get_db_connection()
|
||||
conn = acquire_db_conn()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT id, stream, payload FROM outbox
|
||||
@@ -188,18 +244,37 @@ def outbox_publisher():
|
||||
cur.execute("UPDATE outbox SET published_at=NOW() WHERE id=%s", (row["id"],))
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
if conn and not conn.closed:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
api_dependency_errors_counter.add(1, {"component": "outbox"})
|
||||
api_logger.error("Outbox publisher error: %s", type(e).__name__)
|
||||
finally:
|
||||
if conn is not None and not conn.closed:
|
||||
conn.close()
|
||||
release_db_conn(conn)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
def _start_outbox_publisher():
|
||||
def _start_services():
|
||||
try:
|
||||
init_db_pool(minconn=1, maxconn=15)
|
||||
except Exception as e:
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
api_logger.error("DB pool init error: %s", type(e).__name__)
|
||||
threading.Thread(target=outbox_publisher, daemon=True).start()
|
||||
|
||||
|
||||
@app.on_event("shutdown")
|
||||
def _shutdown_services():
|
||||
if db_pool is not None:
|
||||
try:
|
||||
db_pool.closeall()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def telemetry_middleware(request: Request, call_next):
|
||||
start = time.monotonic()
|
||||
@@ -342,6 +417,7 @@ async def http_exception_handler(request: Request, exc: HTTPException):
|
||||
@app.exception_handler(Exception)
|
||||
async def unhandled_exception_handler(request: Request, exc: Exception):
|
||||
req_id = request.headers.get("X-Request-Id") or generate_request_id()
|
||||
api_dependency_errors_counter.add(1, {"component": "unhandled"})
|
||||
return make_error_response(
|
||||
status_code=500,
|
||||
code="internal_error",
|
||||
@@ -400,13 +476,19 @@ def health_ready(response: Response):
|
||||
req_id = generate_request_id()
|
||||
response.headers["X-Request-Id"] = req_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
conn = None
|
||||
try:
|
||||
conn = get_db_connection()
|
||||
conn.close()
|
||||
conn = acquire_db_conn()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT 1")
|
||||
cur.fetchone()
|
||||
redis_client.ping()
|
||||
s3_client.head_bucket(Bucket=S3_BUCKET)
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "readiness"})
|
||||
raise HTTPException(status_code=503, detail="Database, Redis or S3 unavailable")
|
||||
finally:
|
||||
release_db_conn(conn)
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@@ -420,7 +502,7 @@ def register_user(body: CreateUser, response: Response):
|
||||
user_id = str(uuid.uuid4())
|
||||
created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
conn = get_db_connection()
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
@@ -434,8 +516,12 @@ def register_user(body: CreateUser, response: Response):
|
||||
status_code=409,
|
||||
detail={"error": {"code": "conflict", "message": "Username already taken", "request_id": req_id}}
|
||||
)
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=503, detail="Database unavailable")
|
||||
finally:
|
||||
conn.close()
|
||||
release_db_conn(conn)
|
||||
|
||||
return {
|
||||
"id": user_id,
|
||||
@@ -469,11 +555,16 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
|
||||
headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
)
|
||||
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT * FROM users WHERE username = %s", (username,))
|
||||
user = cur.fetchone()
|
||||
conn.close()
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT * FROM users WHERE username = %s", (username,))
|
||||
user = cur.fetchone()
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=503, detail="Database unavailable")
|
||||
finally:
|
||||
release_db_conn(conn)
|
||||
|
||||
if not user or not bcrypt.checkpw(password.encode('utf-8'), user['password_hash'].encode('utf-8')):
|
||||
raise HTTPException(
|
||||
@@ -483,7 +574,12 @@ def create_session(authorization: Optional[str] = Header(None), response: Respon
|
||||
)
|
||||
|
||||
token = secrets.token_urlsafe(48)
|
||||
redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, str(user['id']))
|
||||
try:
|
||||
redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, str(user['id']))
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "redis"})
|
||||
raise HTTPException(status_code=503, detail="Session store unavailable")
|
||||
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS)
|
||||
expires_at_str = expires_at.isoformat().replace("+00:00", "Z")
|
||||
|
||||
@@ -512,6 +608,7 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) ->
|
||||
try:
|
||||
user_id = redis_client.get(f"session:{token}")
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "redis"})
|
||||
raise HTTPException(status_code=503, detail="Session store unavailable")
|
||||
|
||||
if not user_id:
|
||||
@@ -521,11 +618,16 @@ def get_current_user_from_token(authorization: Optional[str] = Header(None)) ->
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
)
|
||||
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT * FROM users WHERE id = %s", (user_id,))
|
||||
row = cur.fetchone()
|
||||
conn.close()
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT * FROM users WHERE id = %s", (user_id,))
|
||||
row = cur.fetchone()
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=503, detail="Database unavailable")
|
||||
finally:
|
||||
release_db_conn(conn)
|
||||
|
||||
if not row:
|
||||
raise HTTPException(
|
||||
@@ -554,7 +656,11 @@ def revoke_session(authorization: Optional[str] = Header(None)):
|
||||
)
|
||||
|
||||
token = authorization.split(" ")[1]
|
||||
deleted = redis_client.delete(f"session:{token}")
|
||||
try:
|
||||
deleted = redis_client.delete(f"session:{token}")
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "redis"})
|
||||
raise HTTPException(status_code=503, detail="Session store unavailable")
|
||||
|
||||
if not deleted:
|
||||
raise HTTPException(
|
||||
@@ -594,19 +700,17 @@ def create_chat(body: CreateChat, response: Response, authorization: Optional[st
|
||||
if len(member_ids_str) != len(set(member_ids_str)):
|
||||
raise HTTPException(status_code=400, detail="Duplicate member IDs")
|
||||
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
for mid_str in member_ids_str:
|
||||
cur.execute("SELECT id FROM users WHERE id = %s", (mid_str,))
|
||||
if not cur.fetchone():
|
||||
conn.close()
|
||||
raise HTTPException(status_code=400, detail=f"User {mid_str} not found")
|
||||
|
||||
chat_id = str(uuid.uuid4())
|
||||
created_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
for mid_str in member_ids_str:
|
||||
cur.execute("SELECT id FROM users WHERE id = %s", (mid_str,))
|
||||
if not cur.fetchone():
|
||||
raise HTTPException(status_code=400, detail=f"User {mid_str} not found")
|
||||
|
||||
cur.execute(
|
||||
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)",
|
||||
(chat_id, body.title, created_at)
|
||||
@@ -617,11 +721,15 @@ def create_chat(body: CreateChat, response: Response, authorization: Optional[st
|
||||
(chat_id, mid_str)
|
||||
)
|
||||
conn.commit()
|
||||
except HTTPException:
|
||||
conn.rollback()
|
||||
raise
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
conn.close()
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=500, detail="Internal error creating chat")
|
||||
conn.close()
|
||||
finally:
|
||||
release_db_conn(conn)
|
||||
|
||||
return {
|
||||
"id": chat_id,
|
||||
@@ -641,27 +749,32 @@ def list_chats(limit: int = 50, cursor: Optional[str] = None, response: Response
|
||||
if limit < 1 or limit > 100:
|
||||
raise HTTPException(status_code=400, detail="Invalid limit")
|
||||
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT c.id, c.title, c.created_at FROM chats c
|
||||
JOIN chat_members cm ON c.id = cm.chat_id
|
||||
WHERE cm.user_id = %s
|
||||
ORDER BY c.created_at ASC, c.id ASC
|
||||
""", (user["id"],))
|
||||
rows = cur.fetchall()
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT c.id, c.title, c.created_at FROM chats c
|
||||
JOIN chat_members cm ON c.id = cm.chat_id
|
||||
WHERE cm.user_id = %s
|
||||
ORDER BY c.created_at ASC, c.id ASC
|
||||
""", (user["id"],))
|
||||
rows = cur.fetchall()
|
||||
|
||||
user_chats = []
|
||||
for r in rows:
|
||||
cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (r['id'],))
|
||||
m_rows = cur.fetchall()
|
||||
user_chats.append({
|
||||
"id": str(r['id']),
|
||||
"title": r['title'],
|
||||
"member_ids": [str(m['user_id']) for m in m_rows],
|
||||
"created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at']
|
||||
})
|
||||
conn.close()
|
||||
user_chats = []
|
||||
for r in rows:
|
||||
cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (r['id'],))
|
||||
m_rows = cur.fetchall()
|
||||
user_chats.append({
|
||||
"id": str(r['id']),
|
||||
"title": r['title'],
|
||||
"member_ids": [str(m['user_id']) for m in m_rows],
|
||||
"created_at": r['created_at'].isoformat().replace("+00:00", "Z") if isinstance(r['created_at'], datetime) else r['created_at']
|
||||
})
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=503, detail="Database unavailable")
|
||||
finally:
|
||||
release_db_conn(conn)
|
||||
|
||||
start_idx = 0
|
||||
if cursor:
|
||||
@@ -696,18 +809,24 @@ def get_chat(chat_id: UUID, response: Response, authorization: Optional[str] = H
|
||||
user = get_current_user_from_token(authorization)
|
||||
|
||||
chat_id_str = str(chat_id)
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT c.* FROM chats c
|
||||
JOIN chat_members cm ON c.id = cm.chat_id
|
||||
WHERE c.id = %s AND cm.user_id = %s
|
||||
""", (chat_id_str, user["id"]))
|
||||
chat = cur.fetchone()
|
||||
if chat:
|
||||
cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (chat_id_str,))
|
||||
members = cur.fetchall()
|
||||
conn.close()
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT c.* FROM chats c
|
||||
JOIN chat_members cm ON c.id = cm.chat_id
|
||||
WHERE c.id = %s AND cm.user_id = %s
|
||||
""", (chat_id_str, user["id"]))
|
||||
chat = cur.fetchone()
|
||||
members = []
|
||||
if chat:
|
||||
cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s", (chat_id_str,))
|
||||
members = cur.fetchall()
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=503, detail="Database unavailable")
|
||||
finally:
|
||||
release_db_conn(conn)
|
||||
|
||||
if not chat:
|
||||
raise HTTPException(status_code=404, detail="Chat not found")
|
||||
@@ -780,16 +899,22 @@ async def upload_attachment(
|
||||
user = get_current_user_from_token(authorization)
|
||||
chat_id_str = str(chat_id)
|
||||
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id_str, user["id"])
|
||||
)
|
||||
if not cur.fetchone():
|
||||
conn.close()
|
||||
raise HTTPException(status_code=404, detail="Chat not found or inaccessible")
|
||||
conn.close()
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id_str, user["id"])
|
||||
)
|
||||
if not cur.fetchone():
|
||||
raise HTTPException(status_code=404, detail="Chat not found or inaccessible")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=503, detail="Database unavailable")
|
||||
finally:
|
||||
release_db_conn(conn)
|
||||
|
||||
if any(ord(c) < 0x20 or ord(c) == 0x7f for c in filename) or '/' in filename or '\\' in filename:
|
||||
raise HTTPException(status_code=400, detail="Invalid filename")
|
||||
@@ -833,6 +958,7 @@ async def upload_attachment(
|
||||
ContentType=content_type
|
||||
)
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "s3"})
|
||||
raise HTTPException(status_code=503, detail="S3 storage unavailable")
|
||||
|
||||
job_payload = {
|
||||
@@ -845,7 +971,9 @@ async def upload_attachment(
|
||||
"created_at": now
|
||||
}
|
||||
|
||||
conn = get_db_connection()
|
||||
inject(job_payload)
|
||||
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
@@ -866,10 +994,10 @@ async def upload_attachment(
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=500, detail="Internal error saving attachment")
|
||||
finally:
|
||||
if not conn.closed:
|
||||
conn.close()
|
||||
release_db_conn(conn)
|
||||
|
||||
return {
|
||||
"id": attachment_id,
|
||||
@@ -895,15 +1023,20 @@ def get_attachment(attachment_id: UUID, response: Response, authorization: Optio
|
||||
user = get_current_user_from_token(authorization)
|
||||
att_id_str = str(attachment_id)
|
||||
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT a.* FROM attachments a
|
||||
JOIN chat_members cm ON a.chat_id = cm.chat_id
|
||||
WHERE a.id = %s AND cm.user_id = %s
|
||||
""", (att_id_str, user["id"]))
|
||||
row = cur.fetchone()
|
||||
conn.close()
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT a.* FROM attachments a
|
||||
JOIN chat_members cm ON a.chat_id = cm.chat_id
|
||||
WHERE a.id = %s AND cm.user_id = %s
|
||||
""", (att_id_str, user["id"]))
|
||||
row = cur.fetchone()
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=503, detail="Database unavailable")
|
||||
finally:
|
||||
release_db_conn(conn)
|
||||
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="Attachment not found or inaccessible")
|
||||
@@ -937,15 +1070,20 @@ def download_attachment(
|
||||
user = get_current_user_from_token(authorization)
|
||||
att_id_str = str(attachment_id)
|
||||
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT a.* FROM attachments a
|
||||
JOIN chat_members cm ON a.chat_id = cm.chat_id
|
||||
WHERE a.id = %s AND cm.user_id = %s
|
||||
""", (att_id_str, user["id"]))
|
||||
row = cur.fetchone()
|
||||
conn.close()
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT a.* FROM attachments a
|
||||
JOIN chat_members cm ON a.chat_id = cm.chat_id
|
||||
WHERE a.id = %s AND cm.user_id = %s
|
||||
""", (att_id_str, user["id"]))
|
||||
row = cur.fetchone()
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=503, detail="Database unavailable")
|
||||
finally:
|
||||
release_db_conn(conn)
|
||||
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="Attachment not found or inaccessible")
|
||||
@@ -968,6 +1106,7 @@ def download_attachment(
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=s3_key)
|
||||
data = obj['Body'].read()
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "s3"})
|
||||
raise HTTPException(status_code=503, detail="S3 storage unavailable or object missing")
|
||||
|
||||
return Response(
|
||||
@@ -1003,7 +1142,7 @@ def create_message(
|
||||
body_json_str = json.dumps(body.model_dump(mode='json'), sort_keys=True)
|
||||
request_hash = hashlib.sha256(body_json_str.encode("utf-8")).hexdigest()
|
||||
|
||||
conn = get_db_connection()
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
@@ -1011,7 +1150,6 @@ def create_message(
|
||||
(chat_id_str, user["id"])
|
||||
)
|
||||
if not cur.fetchone():
|
||||
conn.close()
|
||||
raise HTTPException(status_code=404, detail="Chat not found")
|
||||
|
||||
attachment_ids_str = [str(aid) for aid in body.attachment_ids]
|
||||
@@ -1022,11 +1160,9 @@ def create_message(
|
||||
""", (attachment_ids_str, chat_id_str))
|
||||
found_atts = cur.fetchall()
|
||||
if len(found_atts) != len(attachment_ids_str):
|
||||
conn.close()
|
||||
raise HTTPException(status_code=404, detail="One or more attachments not found or in another chat")
|
||||
for att in found_atts:
|
||||
if att['status'] == 'failed':
|
||||
conn.close()
|
||||
raise HTTPException(status_code=409, detail="Cannot attach failed attachment")
|
||||
|
||||
msg_id = str(uuid.uuid4())
|
||||
@@ -1055,7 +1191,6 @@ def create_message(
|
||||
""", (user["id"], method, path, idempotency_key))
|
||||
row = cur.fetchone()
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
if row["request_hash"] != request_hash:
|
||||
raise HTTPException(status_code=409, detail="Idempotency key conflict with different payload")
|
||||
@@ -1068,14 +1203,14 @@ def create_message(
|
||||
)
|
||||
conn.commit()
|
||||
except HTTPException:
|
||||
conn.rollback()
|
||||
raise
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
conn.close()
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=500, detail="Internal error processing message")
|
||||
finally:
|
||||
if not conn.closed:
|
||||
conn.close()
|
||||
release_db_conn(conn)
|
||||
|
||||
return msg_response
|
||||
|
||||
@@ -1089,31 +1224,39 @@ def list_messages(chat_id: UUID, limit: int = 50, cursor: Optional[str] = None,
|
||||
user = get_current_user_from_token(authorization)
|
||||
chat_id_str = str(chat_id)
|
||||
|
||||
conn = get_db_connection()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",
|
||||
(chat_id_str, user["id"]))
|
||||
if not cur.fetchone():
|
||||
conn.close()
|
||||
raise HTTPException(status_code=404, detail="Chat not found")
|
||||
|
||||
if limit < 1 or limit > 100:
|
||||
conn.close()
|
||||
raise HTTPException(status_code=400, detail="Invalid limit")
|
||||
|
||||
query = "SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id=%s"
|
||||
params = [chat_id_str]
|
||||
if cursor:
|
||||
c_time, c_id = decode_cursor(cursor, chat_id_str)
|
||||
query += " AND (created_at, id) > (%s, %s)"
|
||||
params.extend([c_time, c_id])
|
||||
query += " ORDER BY created_at ASC, id ASC LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
else:
|
||||
c_time, c_id = None, None
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(query, tuple(params))
|
||||
rows = cur.fetchall()
|
||||
conn.close()
|
||||
if limit < 1 or limit > 100:
|
||||
raise HTTPException(status_code=400, detail="Invalid limit")
|
||||
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",
|
||||
(chat_id_str, user["id"]))
|
||||
if not cur.fetchone():
|
||||
raise HTTPException(status_code=404, detail="Chat not found")
|
||||
|
||||
query = "SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id=%s"
|
||||
params = [chat_id_str]
|
||||
if cursor:
|
||||
query += " AND (created_at, id) > (%s, %s)"
|
||||
params.extend([c_time, c_id])
|
||||
query += " ORDER BY created_at ASC, id ASC LIMIT %s"
|
||||
params.append(limit + 1)
|
||||
|
||||
cur.execute(query, tuple(params))
|
||||
rows = cur.fetchall()
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception:
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=503, detail="Database unavailable")
|
||||
finally:
|
||||
release_db_conn(conn)
|
||||
|
||||
has_next = len(rows) > limit
|
||||
page_items = rows[:limit]
|
||||
@@ -1147,7 +1290,7 @@ def replay_attachment(attachment_id: UUID, response: Response,
|
||||
user = get_current_user_from_token(authorization)
|
||||
att_id = str(attachment_id)
|
||||
|
||||
conn = get_db_connection()
|
||||
conn = acquire_db_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
@@ -1157,10 +1300,8 @@ def replay_attachment(attachment_id: UUID, response: Response,
|
||||
""", (att_id, user["id"]))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
conn.close()
|
||||
raise HTTPException(status_code=404, detail="Attachment not found")
|
||||
if row["status"] != "failed":
|
||||
conn.close()
|
||||
raise HTTPException(status_code=409, detail="Only failed attachments can be replayed")
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
@@ -1172,12 +1313,19 @@ def replay_attachment(attachment_id: UUID, response: Response,
|
||||
job = {"schema_version": 1, "job_id": str(uuid.uuid4()),
|
||||
"type": "attachment.process.v1", "attachment_id": att_id,
|
||||
"pipeline_version": 1, "attempt": 1, "created_at": now}
|
||||
inject(job)
|
||||
cur.execute("INSERT INTO outbox (id, stream, payload, created_at) "
|
||||
"VALUES (%s, %s, %s::jsonb, %s)",
|
||||
(str(uuid.uuid4()), STREAM_NAME, json.dumps(job), now))
|
||||
conn.commit()
|
||||
except HTTPException:
|
||||
conn.rollback()
|
||||
raise
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
api_dependency_errors_counter.add(1, {"component": "postgres"})
|
||||
raise HTTPException(status_code=503, detail="Database unavailable")
|
||||
finally:
|
||||
if not conn.closed:
|
||||
conn.close()
|
||||
release_db_conn(conn)
|
||||
|
||||
return {"id": att_id, "status": "queued"}
|
||||
+3
-2
@@ -97,8 +97,9 @@ services:
|
||||
- "--storage.tsdb.retention.time=7d"
|
||||
- "--web.enable-remote-write-receiver"
|
||||
volumes:
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- promdata:/prometheus
|
||||
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
|
||||
- promdata:/prometheus
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9090/-/healthy || exit 1"]
|
||||
interval: 5s
|
||||
|
||||
@@ -78,7 +78,7 @@
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "{service_name=~\"messenger-api|messenger-worker\"}"
|
||||
"expr": "{service_name=~\".+\"}"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
{
|
||||
"title": "MTUSI Messenger — Lab 6 (Grade 4)",
|
||||
"uid": "mtusi-messenger-lab6-g4",
|
||||
"schemaVersion": 39,
|
||||
"version": 3,
|
||||
"refresh": "5s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API RPS (req/s)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 },
|
||||
"fieldConfig": { "defaults": { "unit": "reqps" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(http_server_requests_total[1m]))",
|
||||
"legendFormat": "req/s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API Error Ratio (%)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 },
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(http_server_errors_total[1m])) / clamp_min(sum(rate(http_server_requests_total[1m])), 0.0001)",
|
||||
"legendFormat": "error_ratio"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API Latency p50 / p95 / p99 (s)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 12, "y": 0 },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.50, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.99, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "DB Connection Pool (used / idle)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 18, "y": 0 },
|
||||
"fieldConfig": { "defaults": { "unit": "short" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(db_pool_connections) by (state)",
|
||||
"legendFormat": "{{state}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Queue Depth & Oldest Job Age",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 7 },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(worker_queue_depth)",
|
||||
"legendFormat": "queue_depth (jobs)"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "max(worker_oldest_job_age_seconds)",
|
||||
"legendFormat": "oldest_job_age (s)"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Worker Saturation & Retries",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 7 },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(worker_jobs_in_progress)",
|
||||
"legendFormat": "in_progress (jobs)"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "sum(rate(worker_job_retries_total[1m])) by (reason)",
|
||||
"legendFormat": "retries/s: {{reason}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Worker Job Duration p50 / p95 / p99 (s)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 7 },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.50, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.95, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"expr": "histogram_quantile(0.99, sum(rate(worker_job_duration_seconds_bucket[1m])) by (le))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Logs with TraceID (Loki)",
|
||||
"type": "logs",
|
||||
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 14 },
|
||||
"targets": [
|
||||
{
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"expr": "{service_name=~\".+\"}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -24,18 +24,25 @@ exporters:
|
||||
endpoint: http://prometheus:9090/api/v1/write
|
||||
tls:
|
||||
insecure: true
|
||||
sending_queue:
|
||||
enabled: true
|
||||
queue_size: 1000
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
tls:
|
||||
insecure: true
|
||||
sending_queue:
|
||||
enabled: true
|
||||
queue_size: 1000
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
headers:
|
||||
X-Scope-OrgID: fake
|
||||
tls:
|
||||
insecure: true
|
||||
debug:
|
||||
verbosity: detailed
|
||||
sending_queue:
|
||||
enabled: true
|
||||
queue_size: 1000
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
groups:
|
||||
- name: messenger_api_alerts
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: MessengerApiHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(http_server_errors_total[1m]))
|
||||
/
|
||||
clamp_min(sum(rate(http_server_requests_total[1m])), 0.001)
|
||||
) > 0.05
|
||||
and sum(rate(http_server_requests_total[1m])) > 0.2
|
||||
for: 30s
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-api
|
||||
annotations:
|
||||
summary: "High HTTP error ratio (>5%) in messenger-api"
|
||||
description: "Error ratio exceeded 5% over 30s; inspect Loki logs and Tempo traces."
|
||||
|
||||
- alert: MessengerApiDependencyErrors
|
||||
expr: sum(rate(http_server_dependency_errors_total[1m])) > 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
service: messenger-api
|
||||
annotations:
|
||||
summary: "Dependency errors in messenger-api"
|
||||
description: "PostgreSQL, Redis, S3, or outbox errors occurred."
|
||||
|
||||
- alert: MessengerApiHighLatencyP95
|
||||
expr: histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket[1m])) by (le)) > 1.0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-api
|
||||
annotations:
|
||||
summary: "High API p95 latency (>1.0s)"
|
||||
description: "API p95 duration is above 1.0s."
|
||||
|
||||
- name: messenger_worker_alerts
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: MessengerWorkerJobFailures
|
||||
expr: sum(rate(worker_jobs_total{status="failed"}[1m])) > 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: warning
|
||||
service: messenger-worker
|
||||
annotations:
|
||||
summary: "Attachment worker jobs failing"
|
||||
description: "Failed worker jobs detected; check error_code and trace_id in Loki/Tempo."
|
||||
|
||||
- alert: MessengerWorkerQueueLag
|
||||
expr: max(worker_oldest_job_age_seconds) > 30 or sum(worker_queue_depth) > 20
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
service: messenger-worker
|
||||
annotations:
|
||||
summary: "Growing attachment queue lag or backlog"
|
||||
description: "Oldest job age > 30s or queue depth > 20; check worker saturation and dependencies."
|
||||
@@ -2,6 +2,9 @@ global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
|
||||
@@ -29,6 +29,7 @@ from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.propagate import extract
|
||||
|
||||
|
||||
class JsonLogFormatter(logging.Formatter):
|
||||
@@ -106,20 +107,6 @@ def setup_worker_telemetry():
|
||||
return trace.get_tracer(service_name), metrics.get_meter(service_name)
|
||||
|
||||
|
||||
tracer, meter = setup_worker_telemetry()
|
||||
logger = logging.getLogger("attachment-worker")
|
||||
|
||||
worker_jobs_counter = meter.create_counter(
|
||||
name="worker_jobs_total",
|
||||
description="Completed and failed attachment jobs",
|
||||
unit="1",
|
||||
)
|
||||
worker_duration_histogram = meter.create_histogram(
|
||||
name="worker_job_duration_seconds",
|
||||
description="Attachment job processing duration in seconds",
|
||||
unit="s",
|
||||
)
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
||||
@@ -137,7 +124,6 @@ JOB_TIMEOUT_SECONDS = int(os.getenv("JOB_TIMEOUT_SECONDS", "30"))
|
||||
WORKER_CONCURRENCY = int(os.getenv("WORKER_CONCURRENCY", "4"))
|
||||
|
||||
CLAIM_MIN_IDLE_MS = (JOB_TIMEOUT_SECONDS + 20) * 1000
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
|
||||
|
||||
redis_client = redis.Redis.from_url(REDIS_URL, decode_responses=True)
|
||||
@@ -147,6 +133,87 @@ s3_client = boto3.client(
|
||||
config=Config(signature_version="s3v4"), region_name="us-east-1"
|
||||
)
|
||||
|
||||
|
||||
def observe_queue_metrics(options):
|
||||
try:
|
||||
q_len = redis_client.xlen(STREAM_NAME)
|
||||
entries = redis_client.xrange(STREAM_NAME, count=1)
|
||||
oldest_age = 0.0
|
||||
if entries:
|
||||
msg_id, fields = entries[0]
|
||||
try:
|
||||
payload = json.loads(fields.get("payload", "{}"))
|
||||
created_str = payload.get("created_at")
|
||||
if created_str:
|
||||
created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00"))
|
||||
oldest_age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds())
|
||||
except Exception:
|
||||
ms = int(msg_id.split("-")[0])
|
||||
oldest_age = max(0.0, time.time() - (ms / 1000.0))
|
||||
yield metrics.Observation(value=q_len, attributes={"stream": STREAM_NAME})
|
||||
except Exception:
|
||||
yield metrics.Observation(value=0, attributes={"stream": STREAM_NAME})
|
||||
|
||||
|
||||
def observe_oldest_job_age(options):
|
||||
try:
|
||||
entries = redis_client.xrange(STREAM_NAME, count=1)
|
||||
if not entries:
|
||||
yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME})
|
||||
return
|
||||
msg_id, fields = entries[0]
|
||||
try:
|
||||
payload = json.loads(fields.get("payload", "{}"))
|
||||
created_str = payload.get("created_at")
|
||||
if created_str:
|
||||
created_dt = datetime.fromisoformat(created_str.replace("Z", "+00:00"))
|
||||
age = max(0.0, (datetime.now(timezone.utc) - created_dt).total_seconds())
|
||||
yield metrics.Observation(value=age, attributes={"stream": STREAM_NAME})
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
ms = int(msg_id.split("-")[0])
|
||||
yield metrics.Observation(value=max(0.0, time.time() - (ms / 1000.0)), attributes={"stream": STREAM_NAME})
|
||||
except Exception:
|
||||
yield metrics.Observation(value=0.0, attributes={"stream": STREAM_NAME})
|
||||
|
||||
|
||||
tracer, meter = setup_worker_telemetry()
|
||||
logger = logging.getLogger("attachment-worker")
|
||||
|
||||
worker_jobs_counter = meter.create_counter(
|
||||
name="worker_jobs_total",
|
||||
description="Completed and failed attachment jobs",
|
||||
unit="1",
|
||||
)
|
||||
worker_retries_counter = meter.create_counter(
|
||||
name="worker_job_retries_total",
|
||||
description="Transient retry attempts scheduled for attachment jobs",
|
||||
unit="1",
|
||||
)
|
||||
worker_duration_histogram = meter.create_histogram(
|
||||
name="worker_job_duration_seconds",
|
||||
description="Attachment job processing duration in seconds",
|
||||
unit="s",
|
||||
)
|
||||
worker_jobs_in_progress = meter.create_up_down_counter(
|
||||
name="worker_jobs_in_progress",
|
||||
description="Jobs currently being processed",
|
||||
unit="1",
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
name="worker_queue_depth",
|
||||
callbacks=[observe_queue_metrics],
|
||||
description="Number of messages currently in attachment jobs stream",
|
||||
unit="1",
|
||||
)
|
||||
meter.create_observable_gauge(
|
||||
name="worker_oldest_job_age_seconds",
|
||||
callbacks=[observe_oldest_job_age],
|
||||
description="Age of the oldest message in attachment jobs stream in seconds",
|
||||
unit="s",
|
||||
)
|
||||
|
||||
running = True
|
||||
|
||||
|
||||
@@ -193,7 +260,7 @@ def prepare_base_image(raw_bytes):
|
||||
return img
|
||||
|
||||
|
||||
def generate_grade3_variants(raw_bytes):
|
||||
def generate_grade4_variants(raw_bytes):
|
||||
base_img = prepare_base_image(raw_bytes)
|
||||
orig_w, orig_h = base_img.size
|
||||
meta = []
|
||||
@@ -212,12 +279,42 @@ def generate_grade3_variants(raw_bytes):
|
||||
tb = buf.getvalue()
|
||||
blobs["thumbnail"] = tb
|
||||
meta.append({
|
||||
"name": "thumbnail",
|
||||
"content_type": "image/jpeg",
|
||||
"size_bytes": len(tb),
|
||||
"width": t_w,
|
||||
"height": t_h,
|
||||
"sha256": hashlib.sha256(tb).hexdigest(),
|
||||
"name": "thumbnail", "content_type": "image/jpeg",
|
||||
"size_bytes": len(tb), "width": t_w, "height": t_h,
|
||||
"sha256": hashlib.sha256(tb).hexdigest()
|
||||
})
|
||||
|
||||
opt = base_img.copy()
|
||||
if orig_w <= 1280 and orig_h <= 1280:
|
||||
o_w, o_h = orig_w, orig_h
|
||||
else:
|
||||
s = min(1280.0 / orig_w, 1280.0 / orig_h)
|
||||
o_w, o_h = max(1, int(orig_w * s)), max(1, int(orig_h * s))
|
||||
if (o_w, o_h) != (orig_w, orig_h):
|
||||
opt = opt.resize((o_w, o_h), Image.Resampling.LANCZOS)
|
||||
buf = io.BytesIO()
|
||||
opt.save(buf, format="WEBP", quality=85)
|
||||
ob = buf.getvalue()
|
||||
blobs["optimized"] = ob
|
||||
meta.append({
|
||||
"name": "optimized", "content_type": "image/webp",
|
||||
"size_bytes": len(ob), "width": o_w, "height": o_h,
|
||||
"sha256": hashlib.sha256(ob).hexdigest()
|
||||
})
|
||||
|
||||
c = min(orig_w, orig_h)
|
||||
left, top = (orig_w - c) // 2, (orig_h - c) // 2
|
||||
crop = base_img.crop((left, top, left + c, top + c))
|
||||
if c != 128:
|
||||
crop = crop.resize((128, 128), Image.Resampling.LANCZOS)
|
||||
buf = io.BytesIO()
|
||||
crop.save(buf, format="JPEG", quality=85, optimize=True)
|
||||
cb = buf.getvalue()
|
||||
blobs["crop"] = cb
|
||||
meta.append({
|
||||
"name": "crop", "content_type": "image/jpeg",
|
||||
"size_bytes": len(cb), "width": 128, "height": 128,
|
||||
"sha256": hashlib.sha256(cb).hexdigest()
|
||||
})
|
||||
|
||||
return meta, blobs
|
||||
@@ -247,202 +344,209 @@ TRANSIENT_ERRORS = (
|
||||
)
|
||||
|
||||
|
||||
def _record_job_outcome(status: str, elapsed: float):
|
||||
def _record_job_outcome(span, status: str, elapsed: float):
|
||||
span.set_attribute("job.outcome", status)
|
||||
worker_jobs_counter.add(1, {"status": status})
|
||||
worker_duration_histogram.record(elapsed, {"status": status})
|
||||
|
||||
|
||||
def process_job(message_id, fields):
|
||||
job_start = time.monotonic()
|
||||
with tracer.start_as_current_span("worker.process_attachment") as span:
|
||||
try:
|
||||
job = json.loads(fields.get("payload", "{}"))
|
||||
except Exception:
|
||||
logger.error("Malformed job %s, ack to drop", message_id)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
ctx = extract(job)
|
||||
|
||||
with tracer.start_as_current_span("worker.process_attachment", context=ctx) as span:
|
||||
worker_jobs_in_progress.add(1)
|
||||
try:
|
||||
job = json.loads(fields.get("payload", "{}"))
|
||||
except Exception:
|
||||
logger.error("Malformed job %s, ack to drop", message_id)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
return
|
||||
attachment_id = job.get("attachment_id")
|
||||
job_id = job.get("job_id")
|
||||
pipeline_version = int(job.get("pipeline_version", 1))
|
||||
attempt = int(job.get("attempt", 1))
|
||||
|
||||
attachment_id = job.get("attachment_id")
|
||||
job_id = job.get("job_id")
|
||||
pipeline_version = int(job.get("pipeline_version", 1))
|
||||
attempt = int(job.get("attempt", 1))
|
||||
span.set_attribute("job.pipeline_version", pipeline_version)
|
||||
span.set_attribute("job.attempt", attempt)
|
||||
|
||||
span.set_attribute("job.pipeline_version", pipeline_version)
|
||||
span.set_attribute("job.attempt", attempt)
|
||||
if not attachment_id:
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||
return
|
||||
|
||||
if not attachment_id:
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
return
|
||||
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,))
|
||||
att = cur.fetchone()
|
||||
if not att:
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
return
|
||||
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("success", time.monotonic() - job_start)
|
||||
return
|
||||
if att["status"] == "failed":
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
return
|
||||
|
||||
if att["status"] == "processing":
|
||||
updated = att["updated_at"]
|
||||
if isinstance(updated, str):
|
||||
updated = datetime.fromisoformat(updated.replace("Z", "+00:00"))
|
||||
age_seconds = (datetime.now(timezone.utc) - updated).total_seconds()
|
||||
if age_seconds < (JOB_TIMEOUT_SECONDS + 10):
|
||||
conn.commit()
|
||||
logger.info(
|
||||
"Attachment already processing, ack duplicate",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
|
||||
)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
cur.execute(
|
||||
"UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s",
|
||||
(attempt, now, attachment_id)
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
chat_id = str(att["chat_id"])
|
||||
staging_key = f"staging/{chat_id}/{attachment_id}/original"
|
||||
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
def check_timeout():
|
||||
if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS:
|
||||
raise TimeoutError("Job exceeded time limit")
|
||||
|
||||
try:
|
||||
try:
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
|
||||
raw_bytes = obj["Body"].read()
|
||||
check_timeout()
|
||||
|
||||
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
|
||||
raise ValueError("Staging checksum mismatch")
|
||||
|
||||
variants_meta, variants_bytes = generate_grade3_variants(raw_bytes)
|
||||
check_timeout()
|
||||
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET, Key=final_original_key,
|
||||
Body=raw_bytes, ContentType=att["content_type"]
|
||||
)
|
||||
for var_name, var_bytes in variants_bytes.items():
|
||||
ct = "image/webp" if var_name == "optimized" else "image/jpeg"
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=f"attachments/{chat_id}/{attachment_id}/{var_name}",
|
||||
Body=var_bytes, ContentType=ct
|
||||
)
|
||||
check_timeout()
|
||||
|
||||
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
UPDATE attachments SET status='ready', variants=%s::jsonb,
|
||||
error_code=NULL, updated_at=%s
|
||||
WHERE id=%s AND pipeline_version <= %s
|
||||
""", (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version))
|
||||
cur.execute("SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,))
|
||||
att = cur.fetchone()
|
||||
if not att:
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||
return
|
||||
if att["status"] == "ready" or att["pipeline_version"] > pipeline_version:
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome(span, "success", time.monotonic() - job_start)
|
||||
return
|
||||
if att["status"] == "failed":
|
||||
conn.commit()
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||
return
|
||||
|
||||
if att["status"] == "processing":
|
||||
updated = att["updated_at"]
|
||||
if isinstance(updated, str):
|
||||
updated = datetime.fromisoformat(updated.replace("Z", "+00:00"))
|
||||
age_seconds = (datetime.now(timezone.utc) - updated).total_seconds()
|
||||
if age_seconds < (JOB_TIMEOUT_SECONDS + 10):
|
||||
conn.commit()
|
||||
logger.info(
|
||||
"Attachment already processing, ack duplicate",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
|
||||
)
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
return
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
cur.execute(
|
||||
"UPDATE attachments SET status='processing', attempt=%s, updated_at=%s WHERE id=%s",
|
||||
(attempt, now, attachment_id)
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("success", time.monotonic() - job_start)
|
||||
logger.info(
|
||||
"Job completed: attachment ready",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt},
|
||||
)
|
||||
chat_id = str(att["chat_id"])
|
||||
staging_key = f"staging/{chat_id}/{attachment_id}/original"
|
||||
final_original_key = f"attachments/{chat_id}/{attachment_id}/original"
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
def check_timeout():
|
||||
if time.monotonic() - start_time > JOB_TIMEOUT_SECONDS:
|
||||
raise TimeoutError("Job exceeded time limit")
|
||||
|
||||
try:
|
||||
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except (UnidentifiedImageError, Image.DecompressionBombError) as e:
|
||||
logger.error(
|
||||
"Permanent image error",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"},
|
||||
)
|
||||
mark_failed(attachment_id, "invalid_image")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
|
||||
except ValueError as e:
|
||||
logger.error(
|
||||
"Checksum error",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"},
|
||||
)
|
||||
mark_failed(attachment_id, "invalid_checksum")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
|
||||
except psycopg2.errors.DataError as e:
|
||||
logger.error(
|
||||
"DB data error",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"},
|
||||
)
|
||||
mark_failed(attachment_id, "data_error")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
|
||||
except TRANSIENT_ERRORS as e:
|
||||
logger.warning(
|
||||
"Transient error during job processing",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
|
||||
)
|
||||
if attempt >= JOB_MAX_ATTEMPTS:
|
||||
mark_failed(attachment_id, "retry_exhausted")
|
||||
try:
|
||||
dlq_job = dict(job)
|
||||
dlq_job["attempt"] = 1
|
||||
redis_client.xadd(
|
||||
DLQ_STREAM_NAME,
|
||||
{"payload": json.dumps(dlq_job), "error": type(e).__name__}
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
obj = s3_client.get_object(Bucket=S3_BUCKET, Key=final_original_key)
|
||||
raw_bytes = obj["Body"].read()
|
||||
check_timeout()
|
||||
|
||||
if hashlib.sha256(raw_bytes).hexdigest() != att["sha256"] or len(raw_bytes) != att["size_bytes"]:
|
||||
raise ValueError("Staging checksum mismatch")
|
||||
|
||||
variants_meta, variants_bytes = generate_grade4_variants(raw_bytes)
|
||||
check_timeout()
|
||||
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET, Key=final_original_key,
|
||||
Body=raw_bytes, ContentType=att["content_type"]
|
||||
)
|
||||
for var_name, var_bytes in variants_bytes.items():
|
||||
ct = "image/webp" if var_name == "optimized" else "image/jpeg"
|
||||
s3_client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=f"attachments/{chat_id}/{attachment_id}/{var_name}",
|
||||
Body=var_bytes, ContentType=ct
|
||||
)
|
||||
check_timeout()
|
||||
|
||||
ready_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
conn = get_db_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
UPDATE attachments SET status='ready', variants=%s::jsonb,
|
||||
error_code=NULL, updated_at=%s
|
||||
WHERE id=%s AND pipeline_version <= %s
|
||||
""", (json.dumps(variants_meta), ready_at, attachment_id, pipeline_version))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome(span, "success", time.monotonic() - job_start)
|
||||
logger.info(
|
||||
"Job completed: attachment ready",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "ready", "attempt": attempt},
|
||||
)
|
||||
|
||||
try:
|
||||
s3_client.delete_object(Bucket=S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
else:
|
||||
delay = min(2 ** attempt, 30) + random.uniform(0, 1)
|
||||
time.sleep(delay)
|
||||
next_job = dict(job)
|
||||
next_job["attempt"] = attempt + 1
|
||||
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Unknown error during job processing",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"},
|
||||
)
|
||||
mark_failed(attachment_id, "processing_failed")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome("failed", time.monotonic() - job_start)
|
||||
except (UnidentifiedImageError, Image.DecompressionBombError):
|
||||
logger.error(
|
||||
"Permanent image error",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_image"},
|
||||
)
|
||||
mark_failed(attachment_id, "invalid_image")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||
|
||||
except ValueError as e:
|
||||
logger.error(
|
||||
"Checksum error",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "invalid_checksum"},
|
||||
)
|
||||
mark_failed(attachment_id, "invalid_checksum")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||
|
||||
except psycopg2.errors.DataError:
|
||||
logger.error(
|
||||
"DB data error",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "data_error"},
|
||||
)
|
||||
mark_failed(attachment_id, "data_error")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||
|
||||
except TRANSIENT_ERRORS as e:
|
||||
logger.warning(
|
||||
"Transient error during job processing",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "processing", "attempt": attempt},
|
||||
)
|
||||
if attempt >= JOB_MAX_ATTEMPTS:
|
||||
mark_failed(attachment_id, "retry_exhausted")
|
||||
try:
|
||||
dlq_job = dict(job)
|
||||
dlq_job["attempt"] = 1
|
||||
redis_client.xadd(
|
||||
DLQ_STREAM_NAME,
|
||||
{"payload": json.dumps(dlq_job), "error": type(e).__name__}
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||
else:
|
||||
worker_retries_counter.add(1, {"reason": type(e).__name__})
|
||||
delay = min(2 ** attempt, 30) + random.uniform(0, 1)
|
||||
time.sleep(delay)
|
||||
next_job = dict(job)
|
||||
next_job["attempt"] = attempt + 1
|
||||
redis_client.xadd(STREAM_NAME, {"payload": json.dumps(next_job)})
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
|
||||
except Exception:
|
||||
logger.error(
|
||||
"Unknown error during job processing",
|
||||
extra={"job_id": job_id, "attachment_id": attachment_id, "job_status": "failed", "attempt": attempt, "error_code": "processing_failed"},
|
||||
)
|
||||
mark_failed(attachment_id, "processing_failed")
|
||||
redis_client.xack(STREAM_NAME, GROUP_NAME, message_id)
|
||||
_record_job_outcome(span, "failed", time.monotonic() - job_start)
|
||||
finally:
|
||||
worker_jobs_in_progress.add(-1)
|
||||
|
||||
|
||||
def run_staging_cleanup():
|
||||
|
||||
Reference in New Issue
Block a user