feat: add OpenTelemetry SDK, collector and telemetry stores

Collector (contrib), Prometheus, Loki, Tempo and Grafana with provisioning
from git; SDK in API and worker with level 3 metrics, traces and logs
carrying trace_id/span_id; telemetry off by default outside the stand;
broker healthcheck runs as the broker user so a fresh stand starts once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-11 10:19:50 +03:00
co-authored by Claude Opus 5.5
parent 9b35c256b7
commit a4497e3f32
62 changed files with 6175 additions and 214 deletions
+31 -12
View File
@@ -1,7 +1,9 @@
"""Request id, instance id и итоговая запись о запросе — чистое ASGI-middleware.
Без `BaseHTTPMiddleware`: тот на каждый запрос создаёт отдельную задачу и
поток памяти для тела ответа (design, решение 7).
поток памяти для тела ответа (design, решение 7). С лабы 6 здесь же server span
и histogram `http.server.request.duration` (design change `otel-telemetry`,
решение 10): шаблон пути решён здесь, health исключён тем же списком.
"""
import time
@@ -12,6 +14,7 @@ from fastapi.routing import iter_route_contexts
from starlette.types import ASGIApp, Message, Receive, Scope, Send
from src.config import settings
from src.infra import telemetry
from src.infra.logger import AbstractLogger, get_logger, http_request_id
from src.utils.constants import (
HEALTH_PATHS,
@@ -84,23 +87,39 @@ class RequestContextMiddleware:
message = {**message, "headers": headers}
await send(message)
try:
if scope["path"] in HEALTH_PATHS:
# ни записи, ни span, ни метрики: проверки LB раз в секунду
await self.app(scope, receive, send_with_ids)
finally:
if scope["path"] not in HEALTH_PATHS:
return
# server span — корень трассы (входящий traceparent не продолжается),
# итоговая запись пишется внутри него и получает trace_id
with telemetry.server_span() as span:
error: BaseException | None = None
try:
await self.app(scope, receive, send_with_ids)
except BaseException as e:
error = e
raise
finally:
# шаблон пути, а не фактический: id в пути не попадают в лог
if self._route_templates is None:
self._route_templates = _route_templates(scope["app"])
route = self._route_templates.get(id(scope.get("route")))
duration = time.perf_counter() - started
telemetry.finish_request(
span,
method=scope["method"],
route=route,
status_code=status_code,
request_id=request_id,
duration_seconds=duration,
error=error,
)
logger.info(
"HTTP request completed",
method=scope["method"],
path=self._route_templates.get(
id(scope.get("route")),
_UNMATCHED_PATH,
),
path=route if route is not None else _UNMATCHED_PATH,
status=status_code,
duration_ms=round(
(time.perf_counter() - started) * 1000,
3,
),
duration_ms=round(duration * 1000, 3),
)