Start the OpenTelemetry SDK in every process
Add the OpenTelemetry API and SDK, the OTLP over HTTP exporter and the FastAPI and asyncpg instrumentations to the lock, and a platform module that builds one resource per process (service.name, service.version, service.instance.id as hostname and pid, deployment.environment.name from OTEL_RESOURCE_ATTRIBUTES), batch processors for traces and logs, a periodic metric reader, PostgreSQL client spans and a bounded shutdown in a daemon thread instead of the 30 s flush of the SDK. The SDK stays off without a Collector address or with OTEL_SDK_DISABLED. The sampler lets only server, consumer and producer spans open a trace, so readiness probes, outbox polls and sweeps do not become one-span traces. Log records carry trace_id and span_id in stdout and go to OTLP through a handler that keeps only the allowlisted fields, and deferred event subscribers now run in the context the event was emitted in, so audit records stay linked to their request or job. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a41d6d39e8
commit
e60b4eac98
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
import contextvars
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from contextlib import suppress
|
||||
from typing import Any
|
||||
@@ -41,7 +42,9 @@ class EventBus:
|
||||
|
||||
def __init__(self, logger: Logger, *, max_queue: int = 10_000) -> None:
|
||||
self._logger = logger
|
||||
self._queue: asyncio.Queue[DomainEvent] = asyncio.Queue(maxsize=max_queue)
|
||||
self._queue: asyncio.Queue[tuple[DomainEvent, contextvars.Context]] = asyncio.Queue(
|
||||
maxsize=max_queue
|
||||
)
|
||||
self._projections: dict[type[DomainEvent], list[Any]] = {}
|
||||
self._deferred: dict[type[DomainEvent], list[Any]] = {}
|
||||
self._worker: asyncio.Task[None] | None = None
|
||||
@@ -62,7 +65,7 @@ class EventBus:
|
||||
if not self._deferred.get(type(event)):
|
||||
return True
|
||||
try:
|
||||
self._queue.put_nowait(event)
|
||||
self._queue.put_nowait((event, contextvars.copy_context()))
|
||||
except asyncio.QueueFull:
|
||||
self._dropped += 1
|
||||
self._logger.warning(
|
||||
@@ -94,9 +97,10 @@ class EventBus:
|
||||
|
||||
async def _drain(self) -> None:
|
||||
while True:
|
||||
event = await self._queue.get()
|
||||
event, context = await self._queue.get()
|
||||
handlers = self._deferred.get(type(event), [])
|
||||
try:
|
||||
await self._run(self._deferred.get(type(event), []), event, "subscriber")
|
||||
await asyncio.create_task(self._run(handlers, event, "subscriber"), context=context)
|
||||
finally:
|
||||
self._queue.task_done()
|
||||
|
||||
|
||||
@@ -47,3 +47,11 @@ class PlatformSettings(Settings):
|
||||
db_connect_timeout_seconds: float = Field(default=1.0, gt=0, le=30)
|
||||
db_acquire_timeout_seconds: float = Field(default=1.0, gt=0, le=30)
|
||||
db_command_timeout_seconds: float = Field(default=2.0, gt=0, le=60)
|
||||
otel_sdk_disabled: bool = False
|
||||
otel_exporter_otlp_endpoint: str = ""
|
||||
otel_service_name: str = ""
|
||||
telemetry_shutdown_seconds: float = Field(default=2.0, gt=0, le=10)
|
||||
|
||||
@property
|
||||
def telemetry_enabled(self) -> bool:
|
||||
return not self.otel_sdk_disabled and bool(self.otel_exporter_otlp_endpoint)
|
||||
|
||||
@@ -4,6 +4,7 @@ from typing import Any
|
||||
|
||||
import orjson
|
||||
import structlog
|
||||
from opentelemetry import trace
|
||||
from structlog.typing import EventDict, Processor, WrappedLogger
|
||||
|
||||
PLATFORM_FIELDS = frozenset(
|
||||
@@ -17,6 +18,8 @@ PLATFORM_FIELDS = frozenset(
|
||||
"error",
|
||||
"reason",
|
||||
"limit",
|
||||
"trace_id",
|
||||
"span_id",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -51,11 +54,20 @@ def field_filter(allowed: frozenset[str]) -> Processor:
|
||||
return drop_unlisted_fields
|
||||
|
||||
|
||||
def add_trace_context(_logger: WrappedLogger, _method: str, event_dict: EventDict) -> EventDict:
|
||||
context = trace.get_current_span().get_span_context()
|
||||
if context.is_valid:
|
||||
event_dict["trace_id"] = trace.format_trace_id(context.trace_id)
|
||||
event_dict["span_id"] = trace.format_span_id(context.span_id)
|
||||
return event_dict
|
||||
|
||||
|
||||
def configure(level: str = "INFO", allowed_fields: frozenset[str] = PLATFORM_FIELDS) -> None:
|
||||
shared = [
|
||||
structlog.contextvars.merge_contextvars,
|
||||
structlog.processors.add_log_level,
|
||||
structlog.processors.TimeStamper(fmt="iso", utc=True, key="ts"),
|
||||
add_trace_context,
|
||||
]
|
||||
structlog.configure(
|
||||
processors=[*shared, structlog.stdlib.ProcessorFormatter.wrap_for_formatter],
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Sequence
|
||||
from importlib.metadata import version
|
||||
from typing import override
|
||||
|
||||
from opentelemetry import _logs, metrics, trace
|
||||
from opentelemetry._logs import SeverityNumber
|
||||
from opentelemetry.context import Context
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor
|
||||
from opentelemetry.sdk._logs import LoggerProvider
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import DropAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.trace.sampling import Decision, ParentBased, Sampler, SamplingResult
|
||||
from opentelemetry.trace import Link, SpanKind, TraceState
|
||||
from opentelemetry.util.types import Attributes
|
||||
|
||||
from messenger.shared.config import PlatformSettings
|
||||
|
||||
SERVICE_VERSION = version("mtusi-messenger")
|
||||
REQUEST_ID = "messenger.request_id"
|
||||
ROOT_KINDS = frozenset({SpanKind.SERVER, SpanKind.CONSUMER, SpanKind.PRODUCER})
|
||||
TRACE_FIELDS = frozenset({"trace_id", "span_id"})
|
||||
FOREIGN_METER = "faststream.opentelemetry.middleware"
|
||||
SEVERITY = {
|
||||
logging.DEBUG: SeverityNumber.DEBUG,
|
||||
logging.INFO: SeverityNumber.INFO,
|
||||
logging.WARNING: SeverityNumber.WARN,
|
||||
logging.ERROR: SeverityNumber.ERROR,
|
||||
logging.CRITICAL: SeverityNumber.FATAL,
|
||||
}
|
||||
|
||||
type Provider = TracerProvider | MeterProvider | LoggerProvider
|
||||
|
||||
|
||||
class ServerRoots(Sampler):
|
||||
@override
|
||||
def should_sample(
|
||||
self,
|
||||
parent_context: Context | None,
|
||||
trace_id: int,
|
||||
name: str,
|
||||
kind: SpanKind | None = None,
|
||||
attributes: Attributes = None,
|
||||
links: Sequence[Link] | None = None,
|
||||
trace_state: TraceState | None = None,
|
||||
) -> SamplingResult:
|
||||
if kind in ROOT_KINDS:
|
||||
return SamplingResult(Decision.RECORD_AND_SAMPLE, attributes, trace_state)
|
||||
return SamplingResult(Decision.DROP, None, trace_state)
|
||||
|
||||
def get_description(self) -> str:
|
||||
return "ServerRoots"
|
||||
|
||||
|
||||
class OtlpLogHandler(logging.Handler):
|
||||
def __init__(self, logger: _logs.Logger, allowed_fields: frozenset[str]) -> None:
|
||||
super().__init__()
|
||||
self._logger = logger
|
||||
self._allowed = allowed_fields - TRACE_FIELDS
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
if record.name.startswith("opentelemetry"):
|
||||
return
|
||||
fields = record.msg if isinstance(record.msg, dict) else {"event": record.getMessage()}
|
||||
event = str(fields.get("event", ""))
|
||||
attributes = {
|
||||
key: value
|
||||
for key, value in fields.items()
|
||||
if key in self._allowed and isinstance(value, str | bool | int | float)
|
||||
}
|
||||
self._logger.emit(
|
||||
timestamp=int(record.created * 1e9),
|
||||
severity_number=SEVERITY.get(record.levelno, SeverityNumber.UNSPECIFIED),
|
||||
severity_text=record.levelname,
|
||||
body=event,
|
||||
attributes={**attributes, "event": event},
|
||||
)
|
||||
|
||||
|
||||
class Telemetry:
|
||||
__slots__ = ("_handler", "_providers")
|
||||
|
||||
def __init__(self, providers: tuple[Provider, ...], handler: logging.Handler) -> None:
|
||||
self._providers = providers
|
||||
self._handler = handler
|
||||
|
||||
def shutdown(self, timeout: float) -> None:
|
||||
logging.getLogger().removeHandler(self._handler)
|
||||
closer = threading.Thread(target=self._close, name="telemetry-shutdown", daemon=True)
|
||||
closer.start()
|
||||
closer.join(timeout)
|
||||
|
||||
def _close(self) -> None:
|
||||
for provider in self._providers:
|
||||
provider.shutdown()
|
||||
|
||||
|
||||
def start_telemetry(
|
||||
settings: PlatformSettings, allowed_fields: frozenset[str], service: str
|
||||
) -> Telemetry | None:
|
||||
if not settings.telemetry_enabled:
|
||||
return None
|
||||
resource = Resource.create(
|
||||
{
|
||||
"service.name": settings.otel_service_name or service,
|
||||
"service.version": SERVICE_VERSION,
|
||||
"service.instance.id": f"{settings.instance_id}-{os.getpid()}",
|
||||
}
|
||||
)
|
||||
tracer = TracerProvider(
|
||||
resource=resource, sampler=ParentBased(ServerRoots()), shutdown_on_exit=False
|
||||
)
|
||||
tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
|
||||
meter = MeterProvider(
|
||||
resource=resource,
|
||||
metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())],
|
||||
views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())],
|
||||
shutdown_on_exit=False,
|
||||
)
|
||||
logs = LoggerProvider(resource=resource, shutdown_on_exit=False)
|
||||
logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
|
||||
trace.set_tracer_provider(tracer)
|
||||
metrics.set_meter_provider(meter)
|
||||
_logs.set_logger_provider(logs)
|
||||
AsyncPGInstrumentor().instrument(tracer_provider=tracer)
|
||||
handler = OtlpLogHandler(logs.get_logger("messenger", SERVICE_VERSION), allowed_fields)
|
||||
logging.getLogger().addHandler(handler)
|
||||
return Telemetry((tracer, meter, logs), handler)
|
||||
|
||||
|
||||
def tag_request(request_id: str) -> None:
|
||||
trace.get_current_span().set_attribute(REQUEST_ID, request_id)
|
||||
Reference in New Issue
Block a user