"""Мессенджер, лабораторная 6: lab5 + OpenTelemetry-инструментация.""" import hashlib import logging import re import tempfile import threading import uuid from contextlib import asynccontextmanager from typing import NamedTuple from urllib.parse import quote import anyio.to_thread import psycopg from fastapi import Depends, FastAPI, Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse, Response from opentelemetry import metrics as metrics_module from opentelemetry.instrumentation.botocore import BotocoreInstrumentor from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor from starlette.exceptions import HTTPException as StarletteHTTPException from app import (attachments, broker, config, cursor, db, idempotency, jobs, repo, security, sessions, s3, telemetry, validation) from app.errors import (BASIC_CHALLENGE, BEARER_CHALLENGE, ApiError, bad_request, not_found, unauthorized) logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") for _noisy in ("pika", "aiohttp", "urllib3", "botocore", "opentelemetry"): _l = logging.getLogger(_noisy) _l.setLevel(logging.WARNING) _l.propagate = False log = logging.getLogger("api") _REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}") INSTANCE_ID = config.INSTANCE_ID _stop = threading.Event() def _outbox_loop(): """API тоже публикует outbox — снижает latency до broker.""" while not _stop.wait(2): try: with db.connection() as conn: jobs.publish_pending_outbox(conn, broker.publish) except Exception: pass def _register_metrics(): """Observable gauges: SDK сам вызывает callbacks при каждом экспорте. Никакого отдельного потока не нужно — MeterProvider с PeriodicExportingMetricReader сам опрашивает при экспорте. """ meter = telemetry.get_meter("messenger-api") def _queue_depth(_options): try: with db.connection() as conn: row = conn.execute( "SELECT count(*) AS c FROM jobs WHERE status='queued'").fetchone() yield metrics_module.Observation( int(row["c"] or 0), {"queue.name": broker.QUEUE}) except Exception: return def _queue_age(_options): try: with db.connection() as conn: row = conn.execute( """SELECT EXTRACT(EPOCH FROM COALESCE(now() - min(created_at), '0'::interval))::float AS age FROM jobs WHERE status='queued'""").fetchone() yield metrics_module.Observation( float(row["age"] or 0.0), {"queue.name": broker.QUEUE}) except Exception: return def _db_pool(_options): try: stats = db.get_pool().get_stats() yield metrics_module.Observation( int(stats.get("pool_available", 0)), {"state": "available"}) yield metrics_module.Observation( int(stats.get("requests_waiting", 0)), {"state": "waiting"}) except Exception: return meter.create_observable_gauge( "messenger_queue_depth", callbacks=[_queue_depth], description="Jobs in queued status") meter.create_observable_gauge( "messenger_queue_oldest_job_age_seconds", callbacks=[_queue_age], description="Age of oldest queued job (seconds)") meter.create_observable_gauge( "messenger_db_pool", callbacks=[_db_pool], description="DB pool state (available/waiting)") @asynccontextmanager async def lifespan(_app): anyio.to_thread.current_default_thread_limiter().total_tokens = config.THREAD_LIMIT # Psycopg/Botocore можно инструментировать и здесь — они не привязаны к ASGI-приложению. PsycopgInstrumentor().instrument() BotocoreInstrumentor().instrument() _register_metrics() threading.Thread(target=_outbox_loop, daemon=True).start() try: broker.declare_only() except Exception as exc: log.warning("broker topology init failed: %s", type(exc).__name__) yield _stop.set() db.close() app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) # lab6: OTel инициализируется до старта приложения. # FastAPIInstrumentor.instrument_app() обязан вызываться ДО первого запроса, # иначе Starlette не даёт добавить middleware. telemetry.init("messenger-api") FastAPIInstrumentor.instrument_app(app, excluded_urls="health") def make_error(code, message, request_id): return {"error": {"code": code, "message": message, "request_id": request_id}} def _rid(request): return getattr(request.state, "request_id", None) or uuid.uuid4().hex def error_response(request, status, code, message, headers=None): rid = _rid(request) out = {"X-Request-Id": rid, "X-Instance-Id": INSTANCE_ID, "Cache-Control": "no-store"} out.update(headers or {}) return JSONResponse(status_code=status, content=make_error(code, message, rid), headers=out) @app.middleware("http") async def add_request_id(request: Request, call_next): incoming = request.headers.get("X-Request-Id", "") rid = incoming if _REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex request.state.request_id = rid response = await call_next(request) response.headers["X-Request-Id"] = rid response.headers["X-Instance-Id"] = INSTANCE_ID response.headers["Cache-Control"] = "no-store" return response @app.exception_handler(ApiError) async def api_error_handler(request: Request, exc: ApiError): return error_response(request, exc.status, exc.code, exc.message, exc.headers) @app.exception_handler(RequestValidationError) async def validation_handler(request: Request, exc): return error_response(request, 400, "invalid_request", "Некорректный запрос") @app.exception_handler(StarletteHTTPException) async def http_exc_handler(request: Request, exc: StarletteHTTPException): mapping = {404: ("not_found", "Объект не найден"), 405: ("invalid_request", "Метод не поддерживается")} code, message = mapping.get(exc.status_code, ("invalid_request", "Ошибка запроса")) return error_response(request, exc.status_code, code, message) @app.exception_handler(Exception) async def generic_handler(request: Request, exc: Exception): if isinstance(exc, db.TRANSIENT): log.warning("dependency unavailable [%s]: %s", _rid(request), type(exc).__name__) return error_response(request, 503, "unavailable", "Сервис временно недоступен") log.error("unhandled error [%s]", _rid(request), exc_info=exc) return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера") class Auth(NamedTuple): user: dict user_id: uuid.UUID digest: bytes def authenticated(request: Request) -> Auth: token = security.parse_bearer(request.headers.get("Authorization")) if token is None: raise unauthorized(BEARER_CHALLENGE) found = sessions.lookup(token) if found is None: raise unauthorized(BEARER_CHALLENGE) return Auth(found["user"], uuid.UUID(found["user"]["id"]), found["digest"]) class RawBody(NamedTuple): content_type: str data: bytes async def raw_body(request: Request) -> RawBody: declared = request.headers.get("content-length") if declared is not None and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES: raise ApiError(413, "payload_too_large", "Тело запроса слишком большое") chunks, size = [], 0 async for chunk in request.stream(): size += len(chunk) if size > config.MAX_BODY_BYTES: raise ApiError(413, "payload_too_large", "Тело запроса слишком большое") chunks.append(chunk) return RawBody(request.headers.get("content-type", ""), b"".join(chunks)) @app.get("/health/live") def live(): return {"status": "ok"} @app.get("/health/ready") def ready(): try: db.ping() except Exception: raise ApiError(503, "unavailable", "PostgreSQL недоступен") try: s3.ping() except Exception: raise ApiError(503, "unavailable", "S3 недоступен") try: if not broker.wait_ready(timeout=2): raise ApiError(503, "unavailable", "Broker недоступен") except ApiError: raise except Exception: raise ApiError(503, "unavailable", "Broker недоступен") return {"status": "ok"} # ---------- пользователи / сессии / чаты ---------- @app.post("/api/v1/users", status_code=201) def create_user(body: RawBody = Depends(raw_body)): data = validation.json_body(body.content_type, body.data, {"username", "display_name", "password"}, {"username", "display_name", "password"}) username = validation.username_field(data["username"]) display_name = validation.text_field(data["display_name"], "display_name", 1, 100) password = validation.text_field(data["password"], "password", 12, 128) with db.connection() as conn: if repo.username_exists(conn, username): raise ApiError(409, "conflict", "Имя пользователя занято") password_hash = security.hash_password(password) try: with db.connection() as conn: row = repo.insert_user(conn, username, display_name, password_hash) except psycopg.errors.UniqueViolation: raise ApiError(409, "conflict", "Имя пользователя занято") return sessions.user_json(row) @app.get("/api/v1/users/me") def get_me(auth: Auth = Depends(authenticated)): return auth.user @app.post("/api/v1/auth/sessions", status_code=201) def login(request: Request): creds = security.parse_basic(request.headers.get("Authorization")) if creds is None: raise unauthorized(BASIC_CHALLENGE) username, password = creds with db.connection() as conn: user = repo.user_for_login(conn, username) if validation._USERNAME_RE.fullmatch(username) else None if user is None: security.burn_verify(password) raise unauthorized(BASIC_CHALLENGE) ok, new_hash = security.verify_password(user["password_hash"], password) if not ok: raise unauthorized(BASIC_CHALLENGE) with db.connection() as conn: if new_hash: repo.update_password_hash(conn, user["id"], new_hash) token, expires_at = sessions.create(conn, user) user_json = sessions.user_json(user) sessions.warm(token, user_json, expires_at) return {"token": token, "token_type": "Bearer", "expires_at": sessions.fmt_ts(expires_at), "user": user_json} @app.delete("/api/v1/auth/sessions/current") def logout(auth: Auth = Depends(authenticated)): sessions.revoke(auth.digest) return Response(status_code=204) @app.post("/api/v1/chats", status_code=201) def create_chat(auth: Auth = Depends(authenticated), body: RawBody = Depends(raw_body)): data = validation.json_body(body.content_type, body.data, {"title", "member_ids"}, {"title", "member_ids"}) title = validation.text_field(data["title"], "title", 1, 100) raw = data["member_ids"] if not isinstance(raw, list) or not 1 <= len(raw) <= 100: raise bad_request("member_ids: от 1 до 100 элементов") members = [validation.require_uuid(x, "member_id") for x in raw] if len(set(members)) != len(members) or auth.user_id not in members: raise bad_request("member_ids должны быть уникальны и включать вызывающего") with db.connection() as conn: chat = repo.create_chat(conn, auth.user_id, title, members) return repo.chat_json(chat) @app.get("/api/v1/chats") def list_chats(request: Request, auth: Auth = Depends(authenticated)): limit = validation.limit_param(request.query_params) raw = request.query_params.get("cursor") after = cursor.decode(raw, "chats", auth.user_id) if raw is not None else None with db.connection() as conn: items, nxt = repo.list_chats(conn, auth.user_id, after, limit) return {"items": items, "next_cursor": nxt} @app.get("/api/v1/chats/{chat_id}") def get_chat(chat_id: str, auth: Auth = Depends(authenticated)): cid = validation.require_uuid(chat_id, "chat_id") with db.connection() as conn: chat = repo.get_chat(conn, cid, auth.user_id) if chat is None: raise not_found() return repo.chat_json(chat) # ---------- сообщения ---------- def _parse_attachment_ids(data): raw = data.get("attachment_ids", []) if not isinstance(raw, list) or len(raw) > 10: raise bad_request("attachment_ids: до 10 элементов") ids = [validation.require_uuid(x, "attachment_id") for x in raw] if len(set(ids)) != len(ids): raise bad_request("attachment_ids должны быть уникальны") return ids @app.post("/api/v1/chats/{chat_id}/messages", status_code=201) def create_message(chat_id: str, request: Request, auth: Auth = Depends(authenticated), body: RawBody = Depends(raw_body)): cid = validation.require_uuid(chat_id, "chat_id") with db.connection() as conn: if not repo.is_member(conn, cid, auth.user_id): raise not_found() data = validation.json_body(body.content_type, body.data, {"text", "attachment_ids"}, {"text"}) text = validation.text_field(data["text"], "text", 0, 4000, allow_blank=True) attachment_ids = _parse_attachment_ids(data) if not text.strip() and not attachment_ids: raise bad_request("Пустое сообщение без вложений") idem_key = request.headers.get("Idempotency-Key") path = "/api/v1/chats/%s/messages" % chat_id req_hash = idempotency.get_request_hash(data) with db.connection() as conn: idempotency.cleanup(conn, config.IDEMPOTENCY_TTL_SECONDS) need_exec, cached = idempotency.check_and_lock( conn, str(auth.user_id), "POST", path, idem_key, req_hash) if not need_exec: return JSONResponse(status_code=201, content=cached) msg = repo.insert_message(conn, cid, auth.user_id, text, attachment_ids) msg_json = repo.message_json(msg) idempotency.save_result(conn, str(auth.user_id), "POST", path, idem_key, req_hash, 201, msg_json) return msg_json @app.get("/api/v1/chats/{chat_id}/messages") def list_messages(chat_id: str, request: Request, auth: Auth = Depends(authenticated)): cid = validation.require_uuid(chat_id, "chat_id") limit = validation.limit_param(request.query_params) raw = request.query_params.get("cursor") after = cursor.decode(raw, "messages", cid) if raw is not None else None with db.connection() as conn: if not repo.is_member(conn, cid, auth.user_id): raise not_found() items, nxt = repo.list_messages(conn, cid, after, limit) return {"items": items, "next_cursor": nxt} # ---------- вложения (lab5: 202 queued) ---------- @app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) async def upload_attachment(chat_id: str, request: Request, auth: Auth = Depends(authenticated)): cid = validation.require_uuid(chat_id, "chat_id") filename = request.query_params.get("filename") if filename is None: raise bad_request("filename обязателен") declared = request.headers.get("content-type", "").split(";")[0].strip().lower() if declared not in ("image/png", "image/jpeg"): raise ApiError(415, "unsupported_media_type", "Ожидается image/png или image/jpeg") def _check_member(): with db.connection() as conn: return repo.is_member(conn, cid, auth.user_id) if not await anyio.to_thread.run_sync(_check_member): raise not_found() declared_len = request.headers.get("content-length") if declared_len is not None and declared_len.isdigit() and int(declared_len) > config.MAX_UPLOAD_BYTES: raise ApiError(413, "payload_too_large", "Файл больше 10 MiB") spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024) hasher = hashlib.sha256() total = 0 try: async for chunk in request.stream(): total += len(chunk) if total > config.MAX_UPLOAD_BYTES: raise ApiError(413, "payload_too_large", "Файл больше 10 MiB") hasher.update(chunk) spool.write(chunk) if total == 0: raise bad_request("Пустой файл") spool.seek(0) digest = hasher.hexdigest() def _process(): actual = attachments.inspect_image_header(spool, total) if actual != declared: raise ApiError(415, "unsupported_media_type", "MIME не соответствует содержимому") return attachments.upload_to_staging( cid, auth.user_id, filename, spool, total, digest, actual) row = await anyio.to_thread.run_sync(_process) return repo.attachment_json(row) finally: spool.close() @app.get("/api/v1/attachments/{attachment_id}") def get_attachment(attachment_id: str, auth: Auth = Depends(authenticated)): aid = validation.require_uuid(attachment_id, "attachment_id") with db.connection() as conn: row = repo.get_attachment_for_member(conn, aid, auth.user_id) if row is None: raise not_found() variants = repo.list_variants(conn, aid) return repo.attachment_json(row, variants) @app.get("/api/v1/attachments/{attachment_id}/content") async def download_attachment(attachment_id: str, request: Request, auth: Auth = Depends(authenticated)): aid = validation.require_uuid(attachment_id, "attachment_id") variant_name = request.query_params.get("variant", "original") if variant_name not in ("original", "thumbnail", "crop", "optimized"): raise bad_request("variant: original|thumbnail|crop|optimized") def _read_all(): if variant_name == "original": row, body = attachments.download_original(aid, auth.user_id) try: data = body.read() finally: body.close() return row, data v, body = attachments.download_variant(aid, auth.user_id, variant_name) try: data = body.read() finally: body.close() return v, data row, data = await anyio.to_thread.run_sync(_read_all) if variant_name == "original": headers = {"Content-Disposition": "attachment; filename*=UTF-8''%s" % quote(row["filename"], safe="")} return Response(content=data, media_type=row["content_type"], headers=headers) return Response(content=data, media_type=row["content_type"])