Files
mtusi2026lab6/main.py
T

482 lines
19 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Мессенджер, лабораторная 6: lab5 + OpenTelemetry-инструментация."""
import hashlib
import logging
import re
import tempfile
import threading
import uuid
from contextlib import asynccontextmanager
from typing import NamedTuple
from urllib.parse import quote
import anyio.to_thread
import psycopg
from fastapi import Depends, FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response
from opentelemetry import metrics as metrics_module
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
from starlette.exceptions import HTTPException as StarletteHTTPException
from app import (attachments, broker, config, cursor, db, idempotency, jobs,
repo, security, sessions, s3, telemetry, validation)
from app.errors import (BASIC_CHALLENGE, BEARER_CHALLENGE, ApiError,
bad_request, not_found, unauthorized)
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
for _noisy in ("pika", "aiohttp", "urllib3", "botocore", "opentelemetry"):
_l = logging.getLogger(_noisy)
_l.setLevel(logging.WARNING)
_l.propagate = False
log = logging.getLogger("api")
_REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
INSTANCE_ID = config.INSTANCE_ID
_stop = threading.Event()
def _outbox_loop():
"""API тоже публикует outbox — снижает latency до broker."""
while not _stop.wait(2):
try:
with db.connection() as conn:
jobs.publish_pending_outbox(conn, broker.publish)
except Exception:
pass
def _register_metrics():
"""Observable gauges: SDK сам вызывает callbacks при каждом экспорте.
Никакого отдельного потока не нужно — MeterProvider с PeriodicExportingMetricReader
сам опрашивает при экспорте.
"""
meter = telemetry.get_meter("messenger-api")
def _queue_depth(_options):
try:
with db.connection() as conn:
row = conn.execute(
"SELECT count(*) AS c FROM jobs WHERE status='queued'").fetchone()
yield metrics_module.Observation(
int(row["c"] or 0), {"queue.name": broker.QUEUE})
except Exception:
return
def _queue_age(_options):
try:
with db.connection() as conn:
row = conn.execute(
"""SELECT EXTRACT(EPOCH FROM
COALESCE(now() - min(created_at), '0'::interval))::float AS age
FROM jobs WHERE status='queued'""").fetchone()
yield metrics_module.Observation(
float(row["age"] or 0.0), {"queue.name": broker.QUEUE})
except Exception:
return
def _db_pool(_options):
try:
stats = db.get_pool().get_stats()
yield metrics_module.Observation(
int(stats.get("pool_available", 0)), {"state": "available"})
yield metrics_module.Observation(
int(stats.get("requests_waiting", 0)), {"state": "waiting"})
except Exception:
return
meter.create_observable_gauge(
"messenger_queue_depth", callbacks=[_queue_depth],
description="Jobs in queued status")
meter.create_observable_gauge(
"messenger_queue_oldest_job_age_seconds", callbacks=[_queue_age],
description="Age of oldest queued job (seconds)")
meter.create_observable_gauge(
"messenger_db_pool", callbacks=[_db_pool],
description="DB pool state (available/waiting)")
@asynccontextmanager
async def lifespan(_app):
anyio.to_thread.current_default_thread_limiter().total_tokens = config.THREAD_LIMIT
# Psycopg/Botocore можно инструментировать и здесь — они не привязаны к ASGI-приложению.
PsycopgInstrumentor().instrument()
BotocoreInstrumentor().instrument()
_register_metrics()
threading.Thread(target=_outbox_loop, daemon=True).start()
try:
broker.declare_only()
except Exception as exc:
log.warning("broker topology init failed: %s", type(exc).__name__)
yield
_stop.set()
db.close()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
# lab6: OTel инициализируется до старта приложения.
# FastAPIInstrumentor.instrument_app() обязан вызываться ДО первого запроса,
# иначе Starlette не даёт добавить middleware.
telemetry.init("messenger-api")
FastAPIInstrumentor.instrument_app(app, excluded_urls="health")
def make_error(code, message, request_id):
return {"error": {"code": code, "message": message, "request_id": request_id}}
def _rid(request):
return getattr(request.state, "request_id", None) or uuid.uuid4().hex
def error_response(request, status, code, message, headers=None):
rid = _rid(request)
out = {"X-Request-Id": rid, "X-Instance-Id": INSTANCE_ID, "Cache-Control": "no-store"}
out.update(headers or {})
return JSONResponse(status_code=status, content=make_error(code, message, rid), headers=out)
@app.middleware("http")
async def add_request_id(request: Request, call_next):
incoming = request.headers.get("X-Request-Id", "")
rid = incoming if _REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
request.state.request_id = rid
response = await call_next(request)
response.headers["X-Request-Id"] = rid
response.headers["X-Instance-Id"] = INSTANCE_ID
response.headers["Cache-Control"] = "no-store"
return response
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
@app.exception_handler(RequestValidationError)
async def validation_handler(request: Request, exc):
return error_response(request, 400, "invalid_request", "Некорректный запрос")
@app.exception_handler(StarletteHTTPException)
async def http_exc_handler(request: Request, exc: StarletteHTTPException):
mapping = {404: ("not_found", "Объект не найден"), 405: ("invalid_request", "Метод не поддерживается")}
code, message = mapping.get(exc.status_code, ("invalid_request", "Ошибка запроса"))
return error_response(request, exc.status_code, code, message)
@app.exception_handler(Exception)
async def generic_handler(request: Request, exc: Exception):
if isinstance(exc, db.TRANSIENT):
log.warning("dependency unavailable [%s]: %s", _rid(request), type(exc).__name__)
return error_response(request, 503, "unavailable", "Сервис временно недоступен")
log.error("unhandled error [%s]", _rid(request), exc_info=exc)
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
class Auth(NamedTuple):
user: dict
user_id: uuid.UUID
digest: bytes
def authenticated(request: Request) -> Auth:
token = security.parse_bearer(request.headers.get("Authorization"))
if token is None:
raise unauthorized(BEARER_CHALLENGE)
found = sessions.lookup(token)
if found is None:
raise unauthorized(BEARER_CHALLENGE)
return Auth(found["user"], uuid.UUID(found["user"]["id"]), found["digest"])
class RawBody(NamedTuple):
content_type: str
data: bytes
async def raw_body(request: Request) -> RawBody:
declared = request.headers.get("content-length")
if declared is not None and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое")
chunks, size = [], 0
async for chunk in request.stream():
size += len(chunk)
if size > config.MAX_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Тело запроса слишком большое")
chunks.append(chunk)
return RawBody(request.headers.get("content-type", ""), b"".join(chunks))
@app.get("/health/live")
def live():
return {"status": "ok"}
@app.get("/health/ready")
def ready():
try:
db.ping()
except Exception:
raise ApiError(503, "unavailable", "PostgreSQL недоступен")
try:
s3.ping()
except Exception:
raise ApiError(503, "unavailable", "S3 недоступен")
try:
if not broker.wait_ready(timeout=2):
raise ApiError(503, "unavailable", "Broker недоступен")
except ApiError:
raise
except Exception:
raise ApiError(503, "unavailable", "Broker недоступен")
return {"status": "ok"}
# ---------- пользователи / сессии / чаты ----------
@app.post("/api/v1/users", status_code=201)
def create_user(body: RawBody = Depends(raw_body)):
data = validation.json_body(body.content_type, body.data,
{"username", "display_name", "password"},
{"username", "display_name", "password"})
username = validation.username_field(data["username"])
display_name = validation.text_field(data["display_name"], "display_name", 1, 100)
password = validation.text_field(data["password"], "password", 12, 128)
with db.connection() as conn:
if repo.username_exists(conn, username):
raise ApiError(409, "conflict", "Имя пользователя занято")
password_hash = security.hash_password(password)
try:
with db.connection() as conn:
row = repo.insert_user(conn, username, display_name, password_hash)
except psycopg.errors.UniqueViolation:
raise ApiError(409, "conflict", "Имя пользователя занято")
return sessions.user_json(row)
@app.get("/api/v1/users/me")
def get_me(auth: Auth = Depends(authenticated)):
return auth.user
@app.post("/api/v1/auth/sessions", status_code=201)
def login(request: Request):
creds = security.parse_basic(request.headers.get("Authorization"))
if creds is None:
raise unauthorized(BASIC_CHALLENGE)
username, password = creds
with db.connection() as conn:
user = repo.user_for_login(conn, username) if validation._USERNAME_RE.fullmatch(username) else None
if user is None:
security.burn_verify(password)
raise unauthorized(BASIC_CHALLENGE)
ok, new_hash = security.verify_password(user["password_hash"], password)
if not ok:
raise unauthorized(BASIC_CHALLENGE)
with db.connection() as conn:
if new_hash:
repo.update_password_hash(conn, user["id"], new_hash)
token, expires_at = sessions.create(conn, user)
user_json = sessions.user_json(user)
sessions.warm(token, user_json, expires_at)
return {"token": token, "token_type": "Bearer",
"expires_at": sessions.fmt_ts(expires_at), "user": user_json}
@app.delete("/api/v1/auth/sessions/current")
def logout(auth: Auth = Depends(authenticated)):
sessions.revoke(auth.digest)
return Response(status_code=204)
@app.post("/api/v1/chats", status_code=201)
def create_chat(auth: Auth = Depends(authenticated), body: RawBody = Depends(raw_body)):
data = validation.json_body(body.content_type, body.data,
{"title", "member_ids"}, {"title", "member_ids"})
title = validation.text_field(data["title"], "title", 1, 100)
raw = data["member_ids"]
if not isinstance(raw, list) or not 1 <= len(raw) <= 100:
raise bad_request("member_ids: от 1 до 100 элементов")
members = [validation.require_uuid(x, "member_id") for x in raw]
if len(set(members)) != len(members) or auth.user_id not in members:
raise bad_request("member_ids должны быть уникальны и включать вызывающего")
with db.connection() as conn:
chat = repo.create_chat(conn, auth.user_id, title, members)
return repo.chat_json(chat)
@app.get("/api/v1/chats")
def list_chats(request: Request, auth: Auth = Depends(authenticated)):
limit = validation.limit_param(request.query_params)
raw = request.query_params.get("cursor")
after = cursor.decode(raw, "chats", auth.user_id) if raw is not None else None
with db.connection() as conn:
items, nxt = repo.list_chats(conn, auth.user_id, after, limit)
return {"items": items, "next_cursor": nxt}
@app.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: str, auth: Auth = Depends(authenticated)):
cid = validation.require_uuid(chat_id, "chat_id")
with db.connection() as conn:
chat = repo.get_chat(conn, cid, auth.user_id)
if chat is None:
raise not_found()
return repo.chat_json(chat)
# ---------- сообщения ----------
def _parse_attachment_ids(data):
raw = data.get("attachment_ids", [])
if not isinstance(raw, list) or len(raw) > 10:
raise bad_request("attachment_ids: до 10 элементов")
ids = [validation.require_uuid(x, "attachment_id") for x in raw]
if len(set(ids)) != len(ids):
raise bad_request("attachment_ids должны быть уникальны")
return ids
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def create_message(chat_id: str, request: Request, auth: Auth = Depends(authenticated),
body: RawBody = Depends(raw_body)):
cid = validation.require_uuid(chat_id, "chat_id")
with db.connection() as conn:
if not repo.is_member(conn, cid, auth.user_id):
raise not_found()
data = validation.json_body(body.content_type, body.data,
{"text", "attachment_ids"}, {"text"})
text = validation.text_field(data["text"], "text", 0, 4000, allow_blank=True)
attachment_ids = _parse_attachment_ids(data)
if not text.strip() and not attachment_ids:
raise bad_request("Пустое сообщение без вложений")
idem_key = request.headers.get("Idempotency-Key")
path = "/api/v1/chats/%s/messages" % chat_id
req_hash = idempotency.get_request_hash(data)
with db.connection() as conn:
idempotency.cleanup(conn, config.IDEMPOTENCY_TTL_SECONDS)
need_exec, cached = idempotency.check_and_lock(
conn, str(auth.user_id), "POST", path, idem_key, req_hash)
if not need_exec:
return JSONResponse(status_code=201, content=cached)
msg = repo.insert_message(conn, cid, auth.user_id, text, attachment_ids)
msg_json = repo.message_json(msg)
idempotency.save_result(conn, str(auth.user_id), "POST", path, idem_key,
req_hash, 201, msg_json)
return msg_json
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(chat_id: str, request: Request, auth: Auth = Depends(authenticated)):
cid = validation.require_uuid(chat_id, "chat_id")
limit = validation.limit_param(request.query_params)
raw = request.query_params.get("cursor")
after = cursor.decode(raw, "messages", cid) if raw is not None else None
with db.connection() as conn:
if not repo.is_member(conn, cid, auth.user_id):
raise not_found()
items, nxt = repo.list_messages(conn, cid, after, limit)
return {"items": items, "next_cursor": nxt}
# ---------- вложения (lab5: 202 queued) ----------
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
async def upload_attachment(chat_id: str, request: Request, auth: Auth = Depends(authenticated)):
cid = validation.require_uuid(chat_id, "chat_id")
filename = request.query_params.get("filename")
if filename is None:
raise bad_request("filename обязателен")
declared = request.headers.get("content-type", "").split(";")[0].strip().lower()
if declared not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg")
def _check_member():
with db.connection() as conn:
return repo.is_member(conn, cid, auth.user_id)
if not await anyio.to_thread.run_sync(_check_member):
raise not_found()
declared_len = request.headers.get("content-length")
if declared_len is not None and declared_len.isdigit() and int(declared_len) > config.MAX_UPLOAD_BYTES:
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024)
hasher = hashlib.sha256()
total = 0
try:
async for chunk in request.stream():
total += len(chunk)
if total > config.MAX_UPLOAD_BYTES:
raise ApiError(413, "payload_too_large", "Файл больше 10 MiB")
hasher.update(chunk)
spool.write(chunk)
if total == 0:
raise bad_request("Пустой файл")
spool.seek(0)
digest = hasher.hexdigest()
def _process():
actual = attachments.inspect_image_header(spool, total)
if actual != declared:
raise ApiError(415, "unsupported_media_type", "MIME не соответствует содержимому")
return attachments.upload_to_staging(
cid, auth.user_id, filename, spool, total, digest, actual)
row = await anyio.to_thread.run_sync(_process)
return repo.attachment_json(row)
finally:
spool.close()
@app.get("/api/v1/attachments/{attachment_id}")
def get_attachment(attachment_id: str, auth: Auth = Depends(authenticated)):
aid = validation.require_uuid(attachment_id, "attachment_id")
with db.connection() as conn:
row = repo.get_attachment_for_member(conn, aid, auth.user_id)
if row is None:
raise not_found()
variants = repo.list_variants(conn, aid)
return repo.attachment_json(row, variants)
@app.get("/api/v1/attachments/{attachment_id}/content")
async def download_attachment(attachment_id: str, request: Request,
auth: Auth = Depends(authenticated)):
aid = validation.require_uuid(attachment_id, "attachment_id")
variant_name = request.query_params.get("variant", "original")
if variant_name not in ("original", "thumbnail", "crop", "optimized"):
raise bad_request("variant: original|thumbnail|crop|optimized")
def _read_all():
if variant_name == "original":
row, body = attachments.download_original(aid, auth.user_id)
try:
data = body.read()
finally:
body.close()
return row, data
v, body = attachments.download_variant(aid, auth.user_id, variant_name)
try:
data = body.read()
finally:
body.close()
return v, data
row, data = await anyio.to_thread.run_sync(_read_all)
if variant_name == "original":
headers = {"Content-Disposition": "attachment; filename*=UTF-8''%s" % quote(row["filename"], safe="")}
return Response(content=data, media_type=row["content_type"], headers=headers)
return Response(content=data, media_type=row["content_type"])