forked from mtusi2026/mtusi2026lab6
Переиспользован стек lab5. API/worker шлют OTLP (traces/metrics/logs) в otel-collector -> Prometheus/Loki/Tempo; Grafana с provisioning датасорсов, дашборда и alert rules в Git. W3C trace context переносится через поле traceparent в задании очереди (API producer span -> worker consumer span в одном trace, подтверждено прямым запросом к Tempo). Три контролируемых инцидента прогнаны и задокументированы (worker down -> queue lag + alert, DB down отличим от медленного handler по error-span, telemetry backend down не роняет API/Collector). В процессе найдены и исправлены 4 реальных бага (middleware после старта приложения, detached span context, замерзающая метрика очереди при падении её единственного источника, неатомарный XACK+XDEL) — см. REPORT.md и evidence/. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
134 lines
5.3 KiB
Python
134 lines
5.3 KiB
Python
"""OpenTelemetry wiring: traces/metrics via OTLP to the Collector, JSON logs
|
|
on stdout correlated with the active trace/span. Shared by the API and the
|
|
worker -- only service.name differs, set via OTEL_SERVICE_NAME per process.
|
|
|
|
No user_id/chat_id/message_id/attachment_id/request_id ever becomes a metric
|
|
label (contracts/observability.md): job/route identifiers used as labels are
|
|
fixed strings (job type, route template), never raw UUIDs.
|
|
"""
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
from opentelemetry import metrics, trace
|
|
from opentelemetry._logs import set_logger_provider
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor
|
|
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
|
from opentelemetry.instrumentation.redis import RedisInstrumentor
|
|
from opentelemetry.propagate import extract, inject
|
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.semconv.resource import ResourceAttributes
|
|
|
|
_meter = None
|
|
_initialized = False
|
|
|
|
|
|
class JsonLogFormatter(logging.Formatter):
|
|
"""timestamp, severity, service.name, event, trace_id/span_id when a span is active."""
|
|
|
|
def __init__(self, service_name: str):
|
|
super().__init__()
|
|
self.service_name = service_name
|
|
|
|
def format(self, record: logging.LogRecord) -> str:
|
|
payload = {
|
|
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + "Z",
|
|
"severity": record.levelname,
|
|
"service.name": self.service_name,
|
|
"event": record.getMessage(),
|
|
}
|
|
span = trace.get_current_span()
|
|
ctx = span.get_span_context() if span else None
|
|
if ctx and ctx.is_valid:
|
|
payload["trace_id"] = format(ctx.trace_id, "032x")
|
|
payload["span_id"] = format(ctx.span_id, "016x")
|
|
if record.exc_info:
|
|
payload["exception"] = self.formatException(record.exc_info)
|
|
return json.dumps(payload, ensure_ascii=False)
|
|
|
|
|
|
def init(service_name: str, app=None) -> None:
|
|
"""Call once per process (API or worker). `app` is the FastAPI instance, if any."""
|
|
global _meter, _initialized
|
|
if _initialized:
|
|
return
|
|
_initialized = True
|
|
|
|
endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318")
|
|
resource = Resource.create(
|
|
{
|
|
ResourceAttributes.SERVICE_NAME: service_name,
|
|
ResourceAttributes.SERVICE_VERSION: os.environ.get("SERVICE_VERSION", "0.1.0"),
|
|
ResourceAttributes.SERVICE_INSTANCE_ID: os.environ.get("INSTANCE_ID", "unknown"),
|
|
"deployment.environment.name": os.environ.get("DEPLOYMENT_ENVIRONMENT", "lab"),
|
|
}
|
|
)
|
|
|
|
tracer_provider = TracerProvider(resource=resource)
|
|
tracer_provider.add_span_processor(
|
|
BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint + "/v1/traces"))
|
|
)
|
|
trace.set_tracer_provider(tracer_provider)
|
|
|
|
meter_provider = MeterProvider(
|
|
resource=resource,
|
|
metric_readers=[
|
|
PeriodicExportingMetricReader(
|
|
OTLPMetricExporter(endpoint=endpoint + "/v1/metrics"), export_interval_millis=5000
|
|
)
|
|
],
|
|
)
|
|
metrics.set_meter_provider(meter_provider)
|
|
_meter = metrics.get_meter(service_name)
|
|
|
|
logger_provider = LoggerProvider(resource=resource)
|
|
logger_provider.add_log_record_processor(
|
|
BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint + "/v1/logs"))
|
|
)
|
|
set_logger_provider(logger_provider)
|
|
|
|
stdout_handler = logging.StreamHandler(sys.stdout)
|
|
stdout_handler.setFormatter(JsonLogFormatter(service_name))
|
|
otlp_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
|
root = logging.getLogger()
|
|
root.handlers = [stdout_handler, otlp_handler]
|
|
root.setLevel(logging.INFO)
|
|
|
|
if app is not None:
|
|
FastAPIInstrumentor.instrument_app(app, tracer_provider=tracer_provider, meter_provider=meter_provider)
|
|
|
|
# Dedicated DB/Redis client spans: lets a slow or failing query show up as
|
|
# its own child span with its own duration/error, distinct from a slow
|
|
# application handler that spends its time elsewhere (contracts/
|
|
# observability.md: distinguish DB failure from a slow HTTP handler).
|
|
AsyncPGInstrumentor().instrument(tracer_provider=tracer_provider)
|
|
RedisInstrumentor().instrument(tracer_provider=tracer_provider)
|
|
|
|
|
|
def meter():
|
|
if _meter is None:
|
|
raise RuntimeError("telemetry.init() was not called")
|
|
return _meter
|
|
|
|
|
|
def inject_context(carrier: dict) -> dict:
|
|
"""W3C traceparent/tracestate into a job payload before publishing to the broker."""
|
|
inject(carrier)
|
|
return carrier
|
|
|
|
|
|
def extract_context(carrier: dict):
|
|
"""Build a Context from a job payload's traceparent/tracestate for the consumer span."""
|
|
return extract(carrier)
|