482 lines
19 KiB
Python
482 lines
19 KiB
Python
"""Мессенджер, лабораторная 6: lab5 + OpenTelemetry-инструментация."""
|
||
import hashlib
|
||
import logging
|
||
import re
|
||
import tempfile
|
||
import threading
|
||
import uuid
|
||
from contextlib import asynccontextmanager
|
||
from typing import NamedTuple
|
||
from urllib.parse import quote
|
||
|
||
import anyio.to_thread
|
||
import psycopg
|
||
from fastapi import Depends, FastAPI, Request
|
||
from fastapi.exceptions import RequestValidationError
|
||
from fastapi.responses import JSONResponse, Response
|
||
from opentelemetry import metrics as metrics_module
|
||
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
|
||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||
from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
|
||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||
|
||
from app import (attachments, broker, config, cursor, db, idempotency, jobs,
|
||
repo, security, sessions, s3, telemetry, validation)
|
||
from app.errors import (BASIC_CHALLENGE, BEARER_CHALLENGE, ApiError,
|
||
bad_request, not_found, unauthorized)
|
||
|
||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||
for _noisy in ("pika", "aiohttp", "urllib3", "botocore", "opentelemetry"):
|
||
_l = logging.getLogger(_noisy)
|
||
_l.setLevel(logging.WARNING)
|
||
_l.propagate = False
|
||
log = logging.getLogger("api")
|
||
|
||
_REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
|
||
INSTANCE_ID = config.INSTANCE_ID
|
||
_stop = threading.Event()
|
||
|
||
|
||
def _outbox_loop():
|
||
"""API тоже публикует outbox — снижает latency до broker."""
|
||
while not _stop.wait(2):
|
||
try:
|
||
with db.connection() as conn:
|
||
jobs.publish_pending_outbox(conn, broker.publish)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _register_metrics():
|
||
"""Observable gauges: SDK сам вызывает callbacks при каждом экспорте.
|
||
|
||
Никакого отдельного потока не нужно — MeterProvider с PeriodicExportingMetricReader
|
||
сам опрашивает при экспорте.
|
||
"""
|
||
meter = telemetry.get_meter("messenger-api")
|
||
|
||
def _queue_depth(_options):
|
||
try:
|
||
with db.connection() as conn:
|
||
row = conn.execute(
|
||
"SELECT count(*) AS c FROM jobs WHERE status='queued'").fetchone()
|
||
yield metrics_module.Observation(
|
||
int(row["c"] or 0), {"queue.name": broker.QUEUE})
|
||
except Exception:
|
||
return
|
||
|
||
def _queue_age(_options):
|
||
try:
|
||
with db.connection() as conn:
|
||
row = conn.execute(
|
||
"""SELECT EXTRACT(EPOCH FROM
|
||
COALESCE(now() - min(created_at), '0'::interval))::float AS age
|
||
FROM jobs WHERE status='queued'""").fetchone()
|
||
yield metrics_module.Observation(
|
||
float(row["age"] or 0.0), {"queue.name": broker.QUEUE})
|
||
except Exception:
|
||
return
|
||
|
||
def _db_pool(_options):
|
||
try:
|
||
stats = db.get_pool().get_stats()
|
||
yield metrics_module.Observation(
|
||
int(stats.get("pool_available", 0)), {"state": "available"})
|
||
yield metrics_module.Observation(
|
||
int(stats.get("requests_waiting", 0)), {"state": "waiting"})
|
||
except Exception:
|
||
return
|
||
|
||
meter.create_observable_gauge(
|
||
"messenger_queue_depth", callbacks=[_queue_depth],
|
||
description="Jobs in queued status")
|
||
meter.create_observable_gauge(
|
||
"messenger_queue_oldest_job_age_seconds", callbacks=[_queue_age],
|
||
description="Age of oldest queued job (seconds)")
|
||
meter.create_observable_gauge(
|
||
"messenger_db_pool", callbacks=[_db_pool],
|
||
description="DB pool state (available/waiting)")
|
||
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(_app):
|
||
anyio.to_thread.current_default_thread_limiter().total_tokens = config.THREAD_LIMIT
|
||
|
||
# Psycopg/Botocore можно инструментировать и здесь — они не привязаны к ASGI-приложению.
|
||
PsycopgInstrumentor().instrument()
|
||
BotocoreInstrumentor().instrument()
|
||
_register_metrics()
|
||
|
||
threading.Thread(target=_outbox_loop, daemon=True).start()
|
||
|
||
try:
|
||
broker.declare_only()
|
||
except Exception as exc:
|
||
log.warning("broker topology init failed: %s", type(exc).__name__)
|
||
|
||
yield
|
||
_stop.set()
|
||
db.close()
|
||
|
||
|
||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||
# lab6: OTel инициализируется до старта приложения.
|
||
# FastAPIInstrumentor.instrument_app() обязан вызываться ДО первого запроса,
|
||
# иначе Starlette не даёт добавить middleware.
|
||
telemetry.init("messenger-api")
|
||
FastAPIInstrumentor.instrument_app(app, excluded_urls="health")
|
||
|
||
|
||
def make_error(code, message, request_id):
|
||
return {"error": {"code": code, "message": message, "request_id": request_id}}
|
||
|
||
|
||
def _rid(request):
|
||
return getattr(request.state, "request_id", None) or uuid.uuid4().hex
|
||
|
||
|
||
def error_response(request, status, code, message, headers=None):
|
||
rid = _rid(request)
|
||
out = {"X-Request-Id": rid, "X-Instance-Id": INSTANCE_ID, "Cache-Control": "no-store"}
|
||
out.update(headers or {})
|
||
return JSONResponse(status_code=status, content=make_error(code, message, rid), headers=out)
|
||
|
||
|
||
@app.middleware("http")
|
||
async def add_request_id(request: Request, call_next):
|
||
incoming = request.headers.get("X-Request-Id", "")
|
||
rid = incoming if _REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
|
||
request.state.request_id = rid
|
||
response = await call_next(request)
|
||
response.headers["X-Request-Id"] = rid
|
||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||
response.headers["Cache-Control"] = "no-store"
|
||
return response
|
||
|
||
|
||
@app.exception_handler(ApiError)
|
||
async def api_error_handler(request: Request, exc: ApiError):
|
||
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
|
||
|
||
|
||
@app.exception_handler(RequestValidationError)
|
||
async def validation_handler(request: Request, exc):
|
||
return error_response(request, 400, "invalid_request", "Некорректный запрос")
|
||
|
||
|
||
@app.exception_handler(StarletteHTTPException)
|
||
async def http_exc_handler(request: Request, exc: StarletteHTTPException):
|
||
mapping = {404: ("not_found", "Объект не найден"), 405: ("invalid_request", "Метод не поддерживается")}
|
||
code, message = mapping.get(exc.status_code, ("invalid_request", "Ошибка запроса"))
|
||
return error_response(request, exc.status_code, code, message)
|
||
|
||
|
||
@app.exception_handler(Exception)
|
||
async def generic_handler(request: Request, exc: Exception):
|
||
if isinstance(exc, db.TRANSIENT):
|
||
log.warning("dependency unavailable [%s]: %s", _rid(request), type(exc).__name__)
|
||
return error_response(request, 503, "unavailable", "Сервис временно недоступен")
|
||
log.error("unhandled error [%s]", _rid(request), exc_info=exc)
|
||
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
|
||
|
||
|
||
class Auth(NamedTuple):
|
||
user: dict
|
||
user_id: uuid.UUID
|
||
digest: bytes
|
||
|
||
|
||
def authenticated(request: Request) -> Auth:
|
||
token = security.parse_bearer(request.headers.get("Authorization"))
|
||
if token is None:
|
||
raise unauthorized(BEARER_CHALLENGE)
|
||
found = sessions.lookup(token)
|
||
if found is None:
|
||
raise unauthorized(BEARER_CHALLENGE)
|
||
return Auth(found["user"], uuid.UUID(found["user"]["id"]), found["digest"])
|
||
|
||
|
||
class RawBody(NamedTuple):
|
||
content_type: str
|
||
data: bytes
|
||
|
||
|
||
async def raw_body(request: Request) -> RawBody:
|
||
declared = request.headers.get("content-length")
|
||
if declared is not None and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES:
|
||
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое")
|
||
chunks, size = [], 0
|
||
async for chunk in request.stream():
|
||
size += len(chunk)
|
||
if size > config.MAX_BODY_BYTES:
|
||
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое")
|
||
chunks.append(chunk)
|
||
return RawBody(request.headers.get("content-type", ""), b"".join(chunks))
|
||
|
||
|
||
@app.get("/health/live")
|
||
def live():
|
||
return {"status": "ok"}
|
||
|
||
|
||
@app.get("/health/ready")
|
||
def ready():
|
||
try:
|
||
db.ping()
|
||
except Exception:
|
||
raise ApiError(503, "unavailable", "PostgreSQL недоступен")
|
||
try:
|
||
s3.ping()
|
||
except Exception:
|
||
raise ApiError(503, "unavailable", "S3 недоступен")
|
||
try:
|
||
if not broker.wait_ready(timeout=2):
|
||
raise ApiError(503, "unavailable", "Broker недоступен")
|
||
except ApiError:
|
||
raise
|
||
except Exception:
|
||
raise ApiError(503, "unavailable", "Broker недоступен")
|
||
return {"status": "ok"}
|
||
|
||
|
||
# ---------- пользователи / сессии / чаты ----------
|
||
|
||
@app.post("/api/v1/users", status_code=201)
|
||
def create_user(body: RawBody = Depends(raw_body)):
|
||
data = validation.json_body(body.content_type, body.data,
|
||
{"username", "display_name", "password"},
|
||
{"username", "display_name", "password"})
|
||
username = validation.username_field(data["username"])
|
||
display_name = validation.text_field(data["display_name"], "display_name", 1, 100)
|
||
password = validation.text_field(data["password"], "password", 12, 128)
|
||
with db.connection() as conn:
|
||
if repo.username_exists(conn, username):
|
||
raise ApiError(409, "conflict", "Имя пользователя занято")
|
||
password_hash = security.hash_password(password)
|
||
try:
|
||
with db.connection() as conn:
|
||
row = repo.insert_user(conn, username, display_name, password_hash)
|
||
except psycopg.errors.UniqueViolation:
|
||
raise ApiError(409, "conflict", "Имя пользователя занято")
|
||
return sessions.user_json(row)
|
||
|
||
|
||
@app.get("/api/v1/users/me")
|
||
def get_me(auth: Auth = Depends(authenticated)):
|
||
return auth.user
|
||
|
||
|
||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||
def login(request: Request):
|
||
creds = security.parse_basic(request.headers.get("Authorization"))
|
||
if creds is None:
|
||
raise unauthorized(BASIC_CHALLENGE)
|
||
username, password = creds
|
||
with db.connection() as conn:
|
||
user = repo.user_for_login(conn, username) if validation._USERNAME_RE.fullmatch(username) else None
|
||
if user is None:
|
||
security.burn_verify(password)
|
||
raise unauthorized(BASIC_CHALLENGE)
|
||
ok, new_hash = security.verify_password(user["password_hash"], password)
|
||
if not ok:
|
||
raise unauthorized(BASIC_CHALLENGE)
|
||
with db.connection() as conn:
|
||
if new_hash:
|
||
repo.update_password_hash(conn, user["id"], new_hash)
|
||
token, expires_at = sessions.create(conn, user)
|
||
user_json = sessions.user_json(user)
|
||
sessions.warm(token, user_json, expires_at)
|
||
return {"token": token, "token_type": "Bearer",
|
||
"expires_at": sessions.fmt_ts(expires_at), "user": user_json}
|
||
|
||
|
||
@app.delete("/api/v1/auth/sessions/current")
|
||
def logout(auth: Auth = Depends(authenticated)):
|
||
sessions.revoke(auth.digest)
|
||
return Response(status_code=204)
|
||
|
||
|
||
@app.post("/api/v1/chats", status_code=201)
|
||
def create_chat(auth: Auth = Depends(authenticated), body: RawBody = Depends(raw_body)):
|
||
data = validation.json_body(body.content_type, body.data,
|
||
{"title", "member_ids"}, {"title", "member_ids"})
|
||
title = validation.text_field(data["title"], "title", 1, 100)
|
||
raw = data["member_ids"]
|
||
if not isinstance(raw, list) or not 1 <= len(raw) <= 100:
|
||
raise bad_request("member_ids: от 1 до 100 элементов")
|
||
members = [validation.require_uuid(x, "member_id") for x in raw]
|
||
if len(set(members)) != len(members) or auth.user_id not in members:
|
||
raise bad_request("member_ids должны быть уникальны и включать вызывающего")
|
||
with db.connection() as conn:
|
||
chat = repo.create_chat(conn, auth.user_id, title, members)
|
||
return repo.chat_json(chat)
|
||
|
||
|
||
@app.get("/api/v1/chats")
|
||
def list_chats(request: Request, auth: Auth = Depends(authenticated)):
|
||
limit = validation.limit_param(request.query_params)
|
||
raw = request.query_params.get("cursor")
|
||
after = cursor.decode(raw, "chats", auth.user_id) if raw is not None else None
|
||
with db.connection() as conn:
|
||
items, nxt = repo.list_chats(conn, auth.user_id, after, limit)
|
||
return {"items": items, "next_cursor": nxt}
|
||
|
||
|
||
@app.get("/api/v1/chats/{chat_id}")
|
||
def get_chat(chat_id: str, auth: Auth = Depends(authenticated)):
|
||
cid = validation.require_uuid(chat_id, "chat_id")
|
||
with db.connection() as conn:
|
||
chat = repo.get_chat(conn, cid, auth.user_id)
|
||
if chat is None:
|
||
raise not_found()
|
||
return repo.chat_json(chat)
|
||
|
||
|
||
# ---------- сообщения ----------
|
||
|
||
def _parse_attachment_ids(data):
|
||
raw = data.get("attachment_ids", [])
|
||
if not isinstance(raw, list) or len(raw) > 10:
|
||
raise bad_request("attachment_ids: до 10 элементов")
|
||
ids = [validation.require_uuid(x, "attachment_id") for x in raw]
|
||
if len(set(ids)) != len(ids):
|
||
raise bad_request("attachment_ids должны быть уникальны")
|
||
return ids
|
||
|
||
|
||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||
def create_message(chat_id: str, request: Request, auth: Auth = Depends(authenticated),
|
||
body: RawBody = Depends(raw_body)):
|
||
cid = validation.require_uuid(chat_id, "chat_id")
|
||
with db.connection() as conn:
|
||
if not repo.is_member(conn, cid, auth.user_id):
|
||
raise not_found()
|
||
|
||
data = validation.json_body(body.content_type, body.data,
|
||
{"text", "attachment_ids"}, {"text"})
|
||
text = validation.text_field(data["text"], "text", 0, 4000, allow_blank=True)
|
||
attachment_ids = _parse_attachment_ids(data)
|
||
if not text.strip() and not attachment_ids:
|
||
raise bad_request("Пустое сообщение без вложений")
|
||
|
||
idem_key = request.headers.get("Idempotency-Key")
|
||
path = "/api/v1/chats/%s/messages" % chat_id
|
||
req_hash = idempotency.get_request_hash(data)
|
||
|
||
with db.connection() as conn:
|
||
idempotency.cleanup(conn, config.IDEMPOTENCY_TTL_SECONDS)
|
||
need_exec, cached = idempotency.check_and_lock(
|
||
conn, str(auth.user_id), "POST", path, idem_key, req_hash)
|
||
if not need_exec:
|
||
return JSONResponse(status_code=201, content=cached)
|
||
msg = repo.insert_message(conn, cid, auth.user_id, text, attachment_ids)
|
||
msg_json = repo.message_json(msg)
|
||
idempotency.save_result(conn, str(auth.user_id), "POST", path, idem_key,
|
||
req_hash, 201, msg_json)
|
||
return msg_json
|
||
|
||
|
||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||
def list_messages(chat_id: str, request: Request, auth: Auth = Depends(authenticated)):
|
||
cid = validation.require_uuid(chat_id, "chat_id")
|
||
limit = validation.limit_param(request.query_params)
|
||
raw = request.query_params.get("cursor")
|
||
after = cursor.decode(raw, "messages", cid) if raw is not None else None
|
||
with db.connection() as conn:
|
||
if not repo.is_member(conn, cid, auth.user_id):
|
||
raise not_found()
|
||
items, nxt = repo.list_messages(conn, cid, after, limit)
|
||
return {"items": items, "next_cursor": nxt}
|
||
|
||
|
||
# ---------- вложения (lab5: 202 queued) ----------
|
||
|
||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||
async def upload_attachment(chat_id: str, request: Request, auth: Auth = Depends(authenticated)):
|
||
cid = validation.require_uuid(chat_id, "chat_id")
|
||
filename = request.query_params.get("filename")
|
||
if filename is None:
|
||
raise bad_request("filename обязателен")
|
||
|
||
declared = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||
if declared not in ("image/png", "image/jpeg"):
|
||
raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg")
|
||
|
||
def _check_member():
|
||
with db.connection() as conn:
|
||
return repo.is_member(conn, cid, auth.user_id)
|
||
|
||
if not await anyio.to_thread.run_sync(_check_member):
|
||
raise not_found()
|
||
|
||
declared_len = request.headers.get("content-length")
|
||
if declared_len is not None and declared_len.isdigit() and int(declared_len) > config.MAX_UPLOAD_BYTES:
|
||
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
|
||
|
||
spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024)
|
||
hasher = hashlib.sha256()
|
||
total = 0
|
||
try:
|
||
async for chunk in request.stream():
|
||
total += len(chunk)
|
||
if total > config.MAX_UPLOAD_BYTES:
|
||
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
|
||
hasher.update(chunk)
|
||
spool.write(chunk)
|
||
if total == 0:
|
||
raise bad_request("Пустой файл")
|
||
spool.seek(0)
|
||
digest = hasher.hexdigest()
|
||
|
||
def _process():
|
||
actual = attachments.inspect_image_header(spool, total)
|
||
if actual != declared:
|
||
raise ApiError(415, "unsupported_media_type", "MIME не соответствует содержимому")
|
||
return attachments.upload_to_staging(
|
||
cid, auth.user_id, filename, spool, total, digest, actual)
|
||
|
||
row = await anyio.to_thread.run_sync(_process)
|
||
return repo.attachment_json(row)
|
||
finally:
|
||
spool.close()
|
||
|
||
|
||
@app.get("/api/v1/attachments/{attachment_id}")
|
||
def get_attachment(attachment_id: str, auth: Auth = Depends(authenticated)):
|
||
aid = validation.require_uuid(attachment_id, "attachment_id")
|
||
with db.connection() as conn:
|
||
row = repo.get_attachment_for_member(conn, aid, auth.user_id)
|
||
if row is None:
|
||
raise not_found()
|
||
variants = repo.list_variants(conn, aid)
|
||
return repo.attachment_json(row, variants)
|
||
|
||
|
||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||
async def download_attachment(attachment_id: str, request: Request,
|
||
auth: Auth = Depends(authenticated)):
|
||
aid = validation.require_uuid(attachment_id, "attachment_id")
|
||
variant_name = request.query_params.get("variant", "original")
|
||
if variant_name not in ("original", "thumbnail", "crop", "optimized"):
|
||
raise bad_request("variant: original|thumbnail|crop|optimized")
|
||
|
||
def _read_all():
|
||
if variant_name == "original":
|
||
row, body = attachments.download_original(aid, auth.user_id)
|
||
try:
|
||
data = body.read()
|
||
finally:
|
||
body.close()
|
||
return row, data
|
||
v, body = attachments.download_variant(aid, auth.user_id, variant_name)
|
||
try:
|
||
data = body.read()
|
||
finally:
|
||
body.close()
|
||
return v, data
|
||
|
||
row, data = await anyio.to_thread.run_sync(_read_all)
|
||
if variant_name == "original":
|
||
headers = {"Content-Disposition": "attachment; filename*=UTF-8''%s" % quote(row["filename"], safe="")}
|
||
return Response(content=data, media_type=row["content_type"], headers=headers)
|
||
return Response(content=data, media_type=row["content_type"]) |