Files
mtusi2026lab6/app/telemetry.py
T
Artu-RitoandClaude Sonnet 5 6c50a63743 Лаба 6: OpenTelemetry + Grafana наблюдаемость (оценка 4)
Переиспользован стек lab5. API/worker шлют OTLP (traces/metrics/logs) в
otel-collector -> Prometheus/Loki/Tempo; Grafana с provisioning датасорсов,
дашборда и alert rules в Git. W3C trace context переносится через поле
traceparent в задании очереди (API producer span -> worker consumer span
в одном trace, подтверждено прямым запросом к Tempo). Три контролируемых
инцидента прогнаны и задокументированы (worker down -> queue lag + alert,
DB down отличим от медленного handler по error-span, telemetry backend down
не роняет API/Collector). В процессе найдены и исправлены 4 реальных бага
(middleware после старта приложения, detached span context, замерзающая
метрика очереди при падении её единственного источника, неатомарный
XACK+XDEL) — см. REPORT.md и evidence/.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-30 01:59:33 +03:00

134 lines
5.3 KiB
Python

"""OpenTelemetry wiring: traces/metrics via OTLP to the Collector, JSON logs
on stdout correlated with the active trace/span. Shared by the API and the
worker -- only service.name differs, set via OTEL_SERVICE_NAME per process.
No user_id/chat_id/message_id/attachment_id/request_id ever becomes a metric
label (contracts/observability.md): job/route identifiers used as labels are
fixed strings (job type, route template), never raw UUIDs.
"""
import json
import logging
import os
import sys
import time
from opentelemetry import metrics, trace
from opentelemetry._logs import set_logger_provider
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.redis import RedisInstrumentor
from opentelemetry.propagate import extract, inject
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.semconv.resource import ResourceAttributes
_meter = None
_initialized = False
class JsonLogFormatter(logging.Formatter):
"""timestamp, severity, service.name, event, trace_id/span_id when a span is active."""
def __init__(self, service_name: str):
super().__init__()
self.service_name = service_name
def format(self, record: logging.LogRecord) -> str:
payload = {
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + "Z",
"severity": record.levelname,
"service.name": self.service_name,
"event": record.getMessage(),
}
span = trace.get_current_span()
ctx = span.get_span_context() if span else None
if ctx and ctx.is_valid:
payload["trace_id"] = format(ctx.trace_id, "032x")
payload["span_id"] = format(ctx.span_id, "016x")
if record.exc_info:
payload["exception"] = self.formatException(record.exc_info)
return json.dumps(payload, ensure_ascii=False)
def init(service_name: str, app=None) -> None:
"""Call once per process (API or worker). `app` is the FastAPI instance, if any."""
global _meter, _initialized
if _initialized:
return
_initialized = True
endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318")
resource = Resource.create(
{
ResourceAttributes.SERVICE_NAME: service_name,
ResourceAttributes.SERVICE_VERSION: os.environ.get("SERVICE_VERSION", "0.1.0"),
ResourceAttributes.SERVICE_INSTANCE_ID: os.environ.get("INSTANCE_ID", "unknown"),
"deployment.environment.name": os.environ.get("DEPLOYMENT_ENVIRONMENT", "lab"),
}
)
tracer_provider = TracerProvider(resource=resource)
tracer_provider.add_span_processor(
BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint + "/v1/traces"))
)
trace.set_tracer_provider(tracer_provider)
meter_provider = MeterProvider(
resource=resource,
metric_readers=[
PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint + "/v1/metrics"), export_interval_millis=5000
)
],
)
metrics.set_meter_provider(meter_provider)
_meter = metrics.get_meter(service_name)
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint + "/v1/logs"))
)
set_logger_provider(logger_provider)
stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.setFormatter(JsonLogFormatter(service_name))
otlp_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
root = logging.getLogger()
root.handlers = [stdout_handler, otlp_handler]
root.setLevel(logging.INFO)
if app is not None:
FastAPIInstrumentor.instrument_app(app, tracer_provider=tracer_provider, meter_provider=meter_provider)
# Dedicated DB/Redis client spans: lets a slow or failing query show up as
# its own child span with its own duration/error, distinct from a slow
# application handler that spends its time elsewhere (contracts/
# observability.md: distinguish DB failure from a slow HTTP handler).
AsyncPGInstrumentor().instrument(tracer_provider=tracer_provider)
RedisInstrumentor().instrument(tracer_provider=tracer_provider)
def meter():
if _meter is None:
raise RuntimeError("telemetry.init() was not called")
return _meter
def inject_context(carrier: dict) -> dict:
"""W3C traceparent/tracestate into a job payload before publishing to the broker."""
inject(carrier)
return carrier
def extract_context(carrier: dict):
"""Build a Context from a job payload's traceparent/tracestate for the consumer span."""
return extract(carrier)