forked from mtusi2026/mtusi2026lab6
Переиспользован стек lab5. API/worker шлют OTLP (traces/metrics/logs) в otel-collector -> Prometheus/Loki/Tempo; Grafana с provisioning датасорсов, дашборда и alert rules в Git. W3C trace context переносится через поле traceparent в задании очереди (API producer span -> worker consumer span в одном trace, подтверждено прямым запросом к Tempo). Три контролируемых инцидента прогнаны и задокументированы (worker down -> queue lag + alert, DB down отличим от медленного handler по error-span, telemetry backend down не роняет API/Collector). В процессе найдены и исправлены 4 реальных бага (middleware после старта приложения, detached span context, замерзающая метрика очереди при падении её единственного источника, неатомарный XACK+XDEL) — см. REPORT.md и evidence/. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
83 lines
3.7 KiB
Python
83 lines
3.7 KiB
Python
"""Custom business/HTTP metrics per contracts/observability.md.
|
|
|
|
Every label here is a fixed, low-cardinality string (route template, job
|
|
type, outcome, queue name) -- never a UUID (user_id/chat_id/message_id/
|
|
attachment_id/request_id never become a label value).
|
|
"""
|
|
from . import telemetry
|
|
|
|
_http_requests_total = None
|
|
_http_request_duration_seconds = None
|
|
_jobs_processed_total = None
|
|
_job_processing_duration_seconds = None
|
|
_time_to_ready_seconds = None
|
|
|
|
|
|
def init() -> None:
|
|
global _http_requests_total, _http_request_duration_seconds
|
|
global _jobs_processed_total, _job_processing_duration_seconds, _time_to_ready_seconds
|
|
m = telemetry.meter()
|
|
_http_requests_total = m.create_counter(
|
|
"http_requests_total", description="HTTP requests by method/route template/status class"
|
|
)
|
|
_http_request_duration_seconds = m.create_histogram(
|
|
"http_request_duration_seconds", unit="s", description="HTTP request duration"
|
|
)
|
|
_jobs_processed_total = m.create_counter(
|
|
"jobs_processed_total", description="Attachment jobs by job type and outcome"
|
|
)
|
|
_job_processing_duration_seconds = m.create_histogram(
|
|
"job_processing_duration_seconds", unit="s", description="Worker job processing duration"
|
|
)
|
|
_time_to_ready_seconds = m.create_histogram(
|
|
"time_to_ready_seconds", unit="s",
|
|
description="End-to-end time from durable accept (202) to ready",
|
|
)
|
|
|
|
|
|
def record_http_request(method: str, route: str, status_code: int, duration_seconds: float) -> None:
|
|
status_class = f"{status_code // 100}xx"
|
|
attrs = {"method": method, "route": route, "status_class": status_class}
|
|
_http_requests_total.add(1, attrs)
|
|
_http_request_duration_seconds.record(duration_seconds, attrs)
|
|
|
|
|
|
def record_job(job_type: str, outcome: str, duration_seconds: float) -> None:
|
|
attrs = {"job_type": job_type, "outcome": outcome}
|
|
_jobs_processed_total.add(1, attrs)
|
|
_job_processing_duration_seconds.record(duration_seconds, attrs)
|
|
|
|
|
|
def record_time_to_ready(seconds: float) -> None:
|
|
_time_to_ready_seconds.record(seconds, {"job_type": "attachment.process.v1"})
|
|
|
|
|
|
def observable_gauges(db_pool_getter=None, queue_depth_getter=None, worker_slots_getter=None) -> None:
|
|
"""Register async-callback gauges; called once at startup with closures
|
|
that read live state (DB pool, broker queue depth, worker concurrency)."""
|
|
m = telemetry.meter()
|
|
if db_pool_getter:
|
|
def _db_pool_cb(options):
|
|
used, idle = db_pool_getter()
|
|
from opentelemetry.metrics import Observation
|
|
yield Observation(used, {"state": "used"})
|
|
yield Observation(idle, {"state": "idle"})
|
|
m.create_observable_gauge("db_pool_connections", callbacks=[_db_pool_cb], description="asyncpg pool connections")
|
|
|
|
if queue_depth_getter:
|
|
def _queue_cb(options):
|
|
from opentelemetry.metrics import Observation
|
|
depth, oldest_age = queue_depth_getter()
|
|
yield Observation(depth, {"queue": "attachment_jobs", "metric": "depth"})
|
|
if oldest_age is not None:
|
|
yield Observation(oldest_age, {"queue": "attachment_jobs", "metric": "oldest_age_seconds"})
|
|
m.create_observable_gauge("queue_backlog", callbacks=[_queue_cb], description="Pending job count and oldest pending age")
|
|
|
|
if worker_slots_getter:
|
|
def _worker_cb(options):
|
|
from opentelemetry.metrics import Observation
|
|
busy, capacity = worker_slots_getter()
|
|
yield Observation(busy, {"state": "busy"})
|
|
yield Observation(capacity, {"state": "capacity"})
|
|
m.create_observable_gauge("worker_concurrency_slots", callbacks=[_worker_cb], description="Worker busy vs capacity slots")
|