Files
mtusi2026lab6/app/telemetry.py
T
dawuduk baee48072c Подключить OpenTelemetry, Collector и Grafana для оценки 5.
API и worker отдают метрики, логи и трассы. Правила SLO записаны, прогон инцидентов ещё впереди.
2026-11-17 15:23:33 +03:00

431 lines
14 KiB
Python

"""OTLP telemetry for the API and the worker.
Metric labels stay low-cardinality: method, route template, status class,
job type, outcome, queue name. User, chat, message and attachment ids are
never metric labels. Trace and log records may carry a job id so one upload
can be found; passwords, tokens, message text and image bytes are not exported.
"""
from __future__ import annotations
import asyncio
import json
import logging
import re
import sys
import time
from contextlib import contextmanager
import psutil
from opentelemetry import metrics, propagate, trace
from opentelemetry._logs import set_logger_provider
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.trace import SpanKind, Status, StatusCode
from app import config
logger = logging.getLogger("messenger.telemetry")
_HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
_JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60)
_READY_BUCKETS = (0.5, 1, 2, 5, 10, 30, 60, 120, 300)
_JOB_TYPE = "attachment.process.v1"
_QUEUE = "attachment.jobs"
_ROUTES = (
(re.compile(r"^/api/v1/chats/[^/]+/messages$"), "/api/v1/chats/{chat_id}/messages"),
(re.compile(r"^/api/v1/chats/[^/]+/attachments$"), "/api/v1/chats/{chat_id}/attachments"),
(re.compile(r"^/api/v1/chats/[^/]+/uploads$"), "/api/v1/chats/{chat_id}/uploads"),
(re.compile(r"^/api/v1/chats/[^/]+$"), "/api/v1/chats/{chat_id}"),
(re.compile(r"^/api/v1/uploads/[^/]+/complete$"), "/api/v1/uploads/{upload_id}/complete"),
(re.compile(r"^/api/v1/attachments/[^/]+/content$"), "/api/v1/attachments/{attachment_id}/content"),
(re.compile(r"^/api/v1/attachments/[^/]+$"), "/api/v1/attachments/{attachment_id}"),
(re.compile(r"^/api/v1/_lab/attachments/[^/]+/jobs$"), "/api/v1/_lab/attachments/{attachment_id}/jobs"),
(re.compile(r"^/api/v1/_lab/attachments/[^/]+/pipelines$"), "/api/v1/_lab/attachments/{attachment_id}/pipelines"),
(re.compile(r"^/api/v1/_lab/attachments/[^/]+/replay$"), "/api/v1/_lab/attachments/{attachment_id}/replay"),
(re.compile(r"^/api/v1/_lab/jobs/[^/]+/redeliver$"), "/api/v1/_lab/jobs/{job_id}/redeliver"),
)
_SECRET = re.compile(r"(?i)(authorization|password|token|secret|session)([\"']?\s*[:=]\s*[\"']?)(\S+)")
_ready = False
_tracer = None
_http_requests = None
_http_duration = None
_jobs = None
_job_duration = None
_time_to_ready = None
_log_handler = None
_providers = []
_process = psutil.Process()
_process.cpu_percent(interval=None)
_inflight = 0
_queue_depth = 0
_queue_oldest = 0.0
_probe_task: asyncio.Task | None = None
_KIND = {
"internal": SpanKind.INTERNAL,
"server": SpanKind.SERVER,
"client": SpanKind.CLIENT,
"producer": SpanKind.PRODUCER,
"consumer": SpanKind.CONSUMER,
}
class JsonFormatter(logging.Formatter):
def __init__(self, service_name: str) -> None:
super().__init__()
self.service_name = service_name
def format(self, record: logging.LogRecord) -> str:
span = trace.get_current_span()
ctx = span.get_span_context()
payload = {
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z", time.gmtime(record.created)),
"severity": record.levelname,
"service.name": self.service_name,
"event": _SECRET.sub(r"\1\2[redacted]", record.getMessage()),
"logger": record.name,
}
if ctx.is_valid:
payload["trace_id"] = format(ctx.trace_id, "032x")
payload["span_id"] = format(ctx.span_id, "016x")
if record.exc_info:
payload["exception"] = _SECRET.sub(r"\1\2[redacted]", self.formatException(record.exc_info))
return json.dumps(payload, ensure_ascii=False)
def route_template(path: str) -> str:
for pattern, template in _ROUTES:
if pattern.match(path):
return template
return path
def observe_http(path: str) -> bool:
return path not in ("/health/live", "/health/ready", "/docs", "/openapi.json")
def status_class(code: int) -> str:
return "%sxx" % (int(code) // 100)
def configure_logging() -> None:
setup()
root = logging.getLogger()
root.handlers.clear()
root.setLevel(getattr(logging, config.LOG_LEVEL.upper(), logging.INFO))
stream = logging.StreamHandler(sys.stdout)
stream.setFormatter(JsonFormatter(config.OTEL_SERVICE_NAME))
root.addHandler(stream)
if _log_handler is not None:
root.addHandler(_log_handler)
logging.getLogger("opentelemetry").setLevel(logging.ERROR)
def setup() -> None:
global _ready, _tracer, _http_requests, _http_duration, _jobs, _job_duration, _time_to_ready, _log_handler
if _ready:
return
_ready = True
if not config.TELEMETRY_ENABLED:
logger.info("telemetry disabled")
return
resource = Resource.create(
{
"service.name": config.OTEL_SERVICE_NAME,
"service.version": config.SERVICE_VERSION,
"service.instance.id": config.INSTANCE_ID,
"deployment.environment.name": "lab",
}
)
ratio = min(1.0, max(0.0, config.TRACE_SAMPLE_RATIO))
tracer_provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(ratio)))
tracer_provider.add_span_processor(
BatchSpanProcessor(
OTLPSpanExporter(endpoint=_signal_endpoint("traces"), timeout=5),
max_queue_size=2048,
schedule_delay_millis=5000,
max_export_batch_size=512,
export_timeout_millis=5000,
)
)
trace.set_tracer_provider(tracer_provider)
_tracer = trace.get_tracer("messenger", config.SERVICE_VERSION)
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=_signal_endpoint("metrics"), timeout=5),
export_interval_millis=10000,
export_timeout_millis=5000,
)
meter_provider = MeterProvider(
resource=resource,
metric_readers=[reader],
views=[
View(
instrument_name="messenger.http.request.duration",
aggregation=ExplicitBucketHistogramAggregation(_HTTP_BUCKETS),
),
View(
instrument_name="messenger.job.duration",
aggregation=ExplicitBucketHistogramAggregation(_JOB_BUCKETS),
),
View(
instrument_name="messenger.time_to_ready",
aggregation=ExplicitBucketHistogramAggregation(_READY_BUCKETS),
),
],
)
metrics.set_meter_provider(meter_provider)
meter = metrics.get_meter("messenger", config.SERVICE_VERSION)
_http_requests = meter.create_counter("messenger.http.requests", unit="1")
_http_duration = meter.create_histogram("messenger.http.request.duration", unit="s")
_jobs = meter.create_counter("messenger.jobs", unit="1")
_job_duration = meter.create_histogram("messenger.job.duration", unit="s")
_time_to_ready = meter.create_histogram("messenger.time_to_ready", unit="s")
meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used])
meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting])
meter.create_observable_gauge("process.cpu.percent", callbacks=[_cpu])
meter.create_observable_gauge("process.memory.rss", unit="By", callbacks=[_rss])
if config.OTEL_SERVICE_NAME == "messenger-api":
meter.create_observable_gauge("messenger.queue.depth", callbacks=[_depth])
meter.create_observable_gauge("messenger.queue.oldest_age", unit="s", callbacks=[_oldest])
if config.OTEL_SERVICE_NAME == "messenger-worker":
meter.create_observable_gauge("messenger.worker.busy", callbacks=[_busy])
meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_capacity])
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(
OTLPLogExporter(endpoint=_signal_endpoint("logs"), timeout=5),
max_queue_size=2048,
schedule_delay_millis=5000,
max_export_batch_size=512,
export_timeout_millis=5000,
)
)
set_logger_provider(logger_provider)
_log_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
_providers.extend([tracer_provider, meter_provider, logger_provider])
def shutdown() -> None:
for provider in _providers:
try:
provider.shutdown()
except Exception:
logger.warning("telemetry shutdown failed")
_providers.clear()
def _signal_endpoint(signal: str) -> str:
base = config.OTEL_EXPORTER_OTLP_ENDPOINT.rstrip("/")
return "%s/v1/%s" % (base, signal)
def current_carrier() -> dict:
if _tracer is None:
return {}
carrier: dict = {}
propagate.inject(carrier)
return carrier
def remote_context(traceparent: str | None, tracestate: str | None = None):
carrier = {}
if traceparent:
carrier["traceparent"] = traceparent
if tracestate:
carrier["tracestate"] = tracestate
if not carrier:
return None
return propagate.extract(carrier)
def carrier_from_headers(headers) -> dict:
carrier = {}
if not headers:
return carrier
for key in ("traceparent", "tracestate"):
value = headers.get(key)
if value is None:
continue
if isinstance(value, bytes):
value = value.decode("latin1")
carrier[key] = str(value)
return carrier
@contextmanager
def span(name: str, context=None, kind: str = "internal", attributes: dict | None = None):
if _tracer is None:
yield None
return
kwargs = {"kind": _KIND.get(kind, SpanKind.INTERNAL)}
if context is not None:
kwargs["context"] = context
with _tracer.start_as_current_span(name, **kwargs) as current:
if attributes:
for key, value in attributes.items():
if value is not None:
current.set_attribute(key, value)
yield current
def finish_http_span(current, status_code: int) -> None:
if current is None:
return
current.set_attribute("http.response.status_code", int(status_code))
if status_code >= 500:
current.set_status(Status(StatusCode.ERROR))
else:
current.set_status(Status(StatusCode.OK))
def record_http(method: str, route: str, status_code: int, duration_seconds: float) -> None:
if _http_requests is None:
return
attributes = {
"method": method,
"route": route,
"status_class": status_class(status_code),
}
_http_requests.add(1, attributes)
_http_duration.record(max(0.0, duration_seconds), attributes)
def record_job(outcome: str, duration_seconds: float) -> None:
if _jobs is None:
return
attributes = {"job_type": _JOB_TYPE, "outcome": outcome}
_jobs.add(1, attributes)
_job_duration.record(max(0.0, duration_seconds), attributes)
def record_time_to_ready(duration_seconds: float) -> None:
if _time_to_ready is None:
return
_time_to_ready.record(max(0.0, duration_seconds), {"job_type": _JOB_TYPE})
def worker_enter() -> None:
global _inflight
_inflight += 1
def worker_leave() -> None:
global _inflight
_inflight = max(0, _inflight - 1)
def start_probes() -> None:
global _probe_task
if _tracer is None or config.OTEL_SERVICE_NAME != "messenger-api":
return
if _probe_task is None or _probe_task.done():
_probe_task = asyncio.create_task(_probe_loop())
async def stop_probes() -> None:
global _probe_task
task = _probe_task
_probe_task = None
if task is None:
return
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
async def _probe_loop() -> None:
from app import db
global _queue_depth, _queue_oldest
while True:
try:
async with db.require().acquire() as conn:
row = await conn.fetchrow(
"""
SELECT
count(*) FILTER (WHERE status IN ('queued', 'processing')) AS depth,
COALESCE(
EXTRACT(EPOCH FROM (now() - min(created_at) FILTER (WHERE status = 'queued'))),
0
) AS oldest
FROM attachment_jobs
"""
)
_queue_depth = int(row["depth"] or 0)
_queue_oldest = float(row["oldest"] or 0)
except asyncio.CancelledError:
raise
except Exception:
logger.warning("queue probe failed")
await asyncio.sleep(5)
def _observation(value, attributes=None):
from opentelemetry.metrics import Observation
return [Observation(value, attributes or {})]
def _pool_used(_options):
from app import db
pool = db.pool
if pool is None:
return _observation(0)
return _observation(max(0, pool.get_size() - pool.get_idle_size()))
def _pool_waiting(_options):
from app import db
pool = db.pool
if pool is None:
return _observation(0)
waiters = getattr(pool, "_waiters", None)
if waiters is not None:
return _observation(len(waiters))
queue = getattr(pool, "_queue", None)
getters = getattr(queue, "_getters", None)
if getters is not None:
return _observation(len(getters))
return _observation(0)
def _depth(_options):
return _observation(_queue_depth, {"queue": _QUEUE})
def _oldest(_options):
return _observation(_queue_oldest, {"queue": _QUEUE})
def _busy(_options):
return _observation(_inflight)
def _capacity(_options):
return _observation(config.WORKER_CONCURRENCY)
def _cpu(_options):
return _observation(_process.cpu_percent(interval=None))
def _rss(_options):
return _observation(_process.memory_info().rss)