Files
mtusi2026labs/src/messenger/shared/infrastructure/telemetry.py
T
ArtyomandClaude Opus 5.5 e60b4eac98 Start the OpenTelemetry SDK in every process
Add the OpenTelemetry API and SDK, the OTLP over HTTP exporter and the
FastAPI and asyncpg instrumentations to the lock, and a platform module
that builds one resource per process (service.name, service.version,
service.instance.id as hostname and pid, deployment.environment.name
from OTEL_RESOURCE_ATTRIBUTES), batch processors for traces and logs,
a periodic metric reader, PostgreSQL client spans and a bounded
shutdown in a daemon thread instead of the 30 s flush of the SDK. The
SDK stays off without a Collector address or with OTEL_SDK_DISABLED.

The sampler lets only server, consumer and producer spans open a
trace, so readiness probes, outbox polls and sweeps do not become
one-span traces. Log records carry trace_id and span_id in stdout and
go to OTLP through a handler that keeps only the allowlisted fields,
and deferred event subscribers now run in the context the event was
emitted in, so audit records stay linked to their request or job.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 04:45:14 +03:00

143 lines
5.4 KiB
Python

import logging
import os
import threading
from collections.abc import Sequence
from importlib.metadata import version
from typing import override
from opentelemetry import _logs, metrics, trace
from opentelemetry._logs import SeverityNumber
from opentelemetry.context import Context
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import DropAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import Decision, ParentBased, Sampler, SamplingResult
from opentelemetry.trace import Link, SpanKind, TraceState
from opentelemetry.util.types import Attributes
from messenger.shared.config import PlatformSettings
SERVICE_VERSION = version("mtusi-messenger")
REQUEST_ID = "messenger.request_id"
ROOT_KINDS = frozenset({SpanKind.SERVER, SpanKind.CONSUMER, SpanKind.PRODUCER})
TRACE_FIELDS = frozenset({"trace_id", "span_id"})
FOREIGN_METER = "faststream.opentelemetry.middleware"
SEVERITY = {
logging.DEBUG: SeverityNumber.DEBUG,
logging.INFO: SeverityNumber.INFO,
logging.WARNING: SeverityNumber.WARN,
logging.ERROR: SeverityNumber.ERROR,
logging.CRITICAL: SeverityNumber.FATAL,
}
type Provider = TracerProvider | MeterProvider | LoggerProvider
class ServerRoots(Sampler):
@override
def should_sample(
self,
parent_context: Context | None,
trace_id: int,
name: str,
kind: SpanKind | None = None,
attributes: Attributes = None,
links: Sequence[Link] | None = None,
trace_state: TraceState | None = None,
) -> SamplingResult:
if kind in ROOT_KINDS:
return SamplingResult(Decision.RECORD_AND_SAMPLE, attributes, trace_state)
return SamplingResult(Decision.DROP, None, trace_state)
def get_description(self) -> str:
return "ServerRoots"
class OtlpLogHandler(logging.Handler):
def __init__(self, logger: _logs.Logger, allowed_fields: frozenset[str]) -> None:
super().__init__()
self._logger = logger
self._allowed = allowed_fields - TRACE_FIELDS
def emit(self, record: logging.LogRecord) -> None:
if record.name.startswith("opentelemetry"):
return
fields = record.msg if isinstance(record.msg, dict) else {"event": record.getMessage()}
event = str(fields.get("event", ""))
attributes = {
key: value
for key, value in fields.items()
if key in self._allowed and isinstance(value, str | bool | int | float)
}
self._logger.emit(
timestamp=int(record.created * 1e9),
severity_number=SEVERITY.get(record.levelno, SeverityNumber.UNSPECIFIED),
severity_text=record.levelname,
body=event,
attributes={**attributes, "event": event},
)
class Telemetry:
__slots__ = ("_handler", "_providers")
def __init__(self, providers: tuple[Provider, ...], handler: logging.Handler) -> None:
self._providers = providers
self._handler = handler
def shutdown(self, timeout: float) -> None:
logging.getLogger().removeHandler(self._handler)
closer = threading.Thread(target=self._close, name="telemetry-shutdown", daemon=True)
closer.start()
closer.join(timeout)
def _close(self) -> None:
for provider in self._providers:
provider.shutdown()
def start_telemetry(
settings: PlatformSettings, allowed_fields: frozenset[str], service: str
) -> Telemetry | None:
if not settings.telemetry_enabled:
return None
resource = Resource.create(
{
"service.name": settings.otel_service_name or service,
"service.version": SERVICE_VERSION,
"service.instance.id": f"{settings.instance_id}-{os.getpid()}",
}
)
tracer = TracerProvider(
resource=resource, sampler=ParentBased(ServerRoots()), shutdown_on_exit=False
)
tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
meter = MeterProvider(
resource=resource,
metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())],
views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())],
shutdown_on_exit=False,
)
logs = LoggerProvider(resource=resource, shutdown_on_exit=False)
logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
trace.set_tracer_provider(tracer)
metrics.set_meter_provider(meter)
_logs.set_logger_provider(logs)
AsyncPGInstrumentor().instrument(tracer_provider=tracer)
handler = OtlpLogHandler(logs.get_logger("messenger", SERVICE_VERSION), allowed_fields)
logging.getLogger().addHandler(handler)
return Telemetry((tracer, meter, logs), handler)
def tag_request(request_id: str) -> None:
trace.get_current_span().set_attribute(REQUEST_ID, request_id)