Files
mtusi2026lab6/app/metrics.py
T
Artu-RitoandClaude Sonnet 5 6c50a63743 Лаба 6: OpenTelemetry + Grafana наблюдаемость (оценка 4)
Переиспользован стек lab5. API/worker шлют OTLP (traces/metrics/logs) в
otel-collector -> Prometheus/Loki/Tempo; Grafana с provisioning датасорсов,
дашборда и alert rules в Git. W3C trace context переносится через поле
traceparent в задании очереди (API producer span -> worker consumer span
в одном trace, подтверждено прямым запросом к Tempo). Три контролируемых
инцидента прогнаны и задокументированы (worker down -> queue lag + alert,
DB down отличим от медленного handler по error-span, telemetry backend down
не роняет API/Collector). В процессе найдены и исправлены 4 реальных бага
(middleware после старта приложения, detached span context, замерзающая
метрика очереди при падении её единственного источника, неатомарный
XACK+XDEL) — см. REPORT.md и evidence/.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-30 01:59:33 +03:00

83 lines
3.7 KiB
Python

"""Custom business/HTTP metrics per contracts/observability.md.
Every label here is a fixed, low-cardinality string (route template, job
type, outcome, queue name) -- never a UUID (user_id/chat_id/message_id/
attachment_id/request_id never become a label value).
"""
from . import telemetry
_http_requests_total = None
_http_request_duration_seconds = None
_jobs_processed_total = None
_job_processing_duration_seconds = None
_time_to_ready_seconds = None
def init() -> None:
global _http_requests_total, _http_request_duration_seconds
global _jobs_processed_total, _job_processing_duration_seconds, _time_to_ready_seconds
m = telemetry.meter()
_http_requests_total = m.create_counter(
"http_requests_total", description="HTTP requests by method/route template/status class"
)
_http_request_duration_seconds = m.create_histogram(
"http_request_duration_seconds", unit="s", description="HTTP request duration"
)
_jobs_processed_total = m.create_counter(
"jobs_processed_total", description="Attachment jobs by job type and outcome"
)
_job_processing_duration_seconds = m.create_histogram(
"job_processing_duration_seconds", unit="s", description="Worker job processing duration"
)
_time_to_ready_seconds = m.create_histogram(
"time_to_ready_seconds", unit="s",
description="End-to-end time from durable accept (202) to ready",
)
def record_http_request(method: str, route: str, status_code: int, duration_seconds: float) -> None:
status_class = f"{status_code // 100}xx"
attrs = {"method": method, "route": route, "status_class": status_class}
_http_requests_total.add(1, attrs)
_http_request_duration_seconds.record(duration_seconds, attrs)
def record_job(job_type: str, outcome: str, duration_seconds: float) -> None:
attrs = {"job_type": job_type, "outcome": outcome}
_jobs_processed_total.add(1, attrs)
_job_processing_duration_seconds.record(duration_seconds, attrs)
def record_time_to_ready(seconds: float) -> None:
_time_to_ready_seconds.record(seconds, {"job_type": "attachment.process.v1"})
def observable_gauges(db_pool_getter=None, queue_depth_getter=None, worker_slots_getter=None) -> None:
"""Register async-callback gauges; called once at startup with closures
that read live state (DB pool, broker queue depth, worker concurrency)."""
m = telemetry.meter()
if db_pool_getter:
def _db_pool_cb(options):
used, idle = db_pool_getter()
from opentelemetry.metrics import Observation
yield Observation(used, {"state": "used"})
yield Observation(idle, {"state": "idle"})
m.create_observable_gauge("db_pool_connections", callbacks=[_db_pool_cb], description="asyncpg pool connections")
if queue_depth_getter:
def _queue_cb(options):
from opentelemetry.metrics import Observation
depth, oldest_age = queue_depth_getter()
yield Observation(depth, {"queue": "attachment_jobs", "metric": "depth"})
if oldest_age is not None:
yield Observation(oldest_age, {"queue": "attachment_jobs", "metric": "oldest_age_seconds"})
m.create_observable_gauge("queue_backlog", callbacks=[_queue_cb], description="Pending job count and oldest pending age")
if worker_slots_getter:
def _worker_cb(options):
from opentelemetry.metrics import Observation
busy, capacity = worker_slots_getter()
yield Observation(busy, {"state": "busy"})
yield Observation(capacity, {"state": "capacity"})
m.create_observable_gauge("worker_concurrency_slots", callbacks=[_worker_cb], description="Worker busy vs capacity slots")