forked from mtusi2026/mtusi2026lab2
Add the OpenTelemetry API and SDK, the OTLP over HTTP exporter and the FastAPI and asyncpg instrumentations to the lock, and a platform module that builds one resource per process (service.name, service.version, service.instance.id as hostname and pid, deployment.environment.name from OTEL_RESOURCE_ATTRIBUTES), batch processors for traces and logs, a periodic metric reader, PostgreSQL client spans and a bounded shutdown in a daemon thread instead of the 30 s flush of the SDK. The SDK stays off without a Collector address or with OTEL_SDK_DISABLED. The sampler lets only server, consumer and producer spans open a trace, so readiness probes, outbox polls and sweeps do not become one-span traces. Log records carry trace_id and span_id in stdout and go to OTLP through a handler that keeps only the allowlisted fields, and deferred event subscribers now run in the context the event was emitted in, so audit records stay linked to their request or job. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
143 lines
5.4 KiB
Python
143 lines
5.4 KiB
Python
import logging
|
|
import os
|
|
import threading
|
|
from collections.abc import Sequence
|
|
from importlib.metadata import version
|
|
from typing import override
|
|
|
|
from opentelemetry import _logs, metrics, trace
|
|
from opentelemetry._logs import SeverityNumber
|
|
from opentelemetry.context import Context
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor
|
|
from opentelemetry.sdk._logs import LoggerProvider
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.metrics.view import DropAggregation, View
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.sdk.trace.sampling import Decision, ParentBased, Sampler, SamplingResult
|
|
from opentelemetry.trace import Link, SpanKind, TraceState
|
|
from opentelemetry.util.types import Attributes
|
|
|
|
from messenger.shared.config import PlatformSettings
|
|
|
|
SERVICE_VERSION = version("mtusi-messenger")
|
|
REQUEST_ID = "messenger.request_id"
|
|
ROOT_KINDS = frozenset({SpanKind.SERVER, SpanKind.CONSUMER, SpanKind.PRODUCER})
|
|
TRACE_FIELDS = frozenset({"trace_id", "span_id"})
|
|
FOREIGN_METER = "faststream.opentelemetry.middleware"
|
|
SEVERITY = {
|
|
logging.DEBUG: SeverityNumber.DEBUG,
|
|
logging.INFO: SeverityNumber.INFO,
|
|
logging.WARNING: SeverityNumber.WARN,
|
|
logging.ERROR: SeverityNumber.ERROR,
|
|
logging.CRITICAL: SeverityNumber.FATAL,
|
|
}
|
|
|
|
type Provider = TracerProvider | MeterProvider | LoggerProvider
|
|
|
|
|
|
class ServerRoots(Sampler):
|
|
@override
|
|
def should_sample(
|
|
self,
|
|
parent_context: Context | None,
|
|
trace_id: int,
|
|
name: str,
|
|
kind: SpanKind | None = None,
|
|
attributes: Attributes = None,
|
|
links: Sequence[Link] | None = None,
|
|
trace_state: TraceState | None = None,
|
|
) -> SamplingResult:
|
|
if kind in ROOT_KINDS:
|
|
return SamplingResult(Decision.RECORD_AND_SAMPLE, attributes, trace_state)
|
|
return SamplingResult(Decision.DROP, None, trace_state)
|
|
|
|
def get_description(self) -> str:
|
|
return "ServerRoots"
|
|
|
|
|
|
class OtlpLogHandler(logging.Handler):
|
|
def __init__(self, logger: _logs.Logger, allowed_fields: frozenset[str]) -> None:
|
|
super().__init__()
|
|
self._logger = logger
|
|
self._allowed = allowed_fields - TRACE_FIELDS
|
|
|
|
def emit(self, record: logging.LogRecord) -> None:
|
|
if record.name.startswith("opentelemetry"):
|
|
return
|
|
fields = record.msg if isinstance(record.msg, dict) else {"event": record.getMessage()}
|
|
event = str(fields.get("event", ""))
|
|
attributes = {
|
|
key: value
|
|
for key, value in fields.items()
|
|
if key in self._allowed and isinstance(value, str | bool | int | float)
|
|
}
|
|
self._logger.emit(
|
|
timestamp=int(record.created * 1e9),
|
|
severity_number=SEVERITY.get(record.levelno, SeverityNumber.UNSPECIFIED),
|
|
severity_text=record.levelname,
|
|
body=event,
|
|
attributes={**attributes, "event": event},
|
|
)
|
|
|
|
|
|
class Telemetry:
|
|
__slots__ = ("_handler", "_providers")
|
|
|
|
def __init__(self, providers: tuple[Provider, ...], handler: logging.Handler) -> None:
|
|
self._providers = providers
|
|
self._handler = handler
|
|
|
|
def shutdown(self, timeout: float) -> None:
|
|
logging.getLogger().removeHandler(self._handler)
|
|
closer = threading.Thread(target=self._close, name="telemetry-shutdown", daemon=True)
|
|
closer.start()
|
|
closer.join(timeout)
|
|
|
|
def _close(self) -> None:
|
|
for provider in self._providers:
|
|
provider.shutdown()
|
|
|
|
|
|
def start_telemetry(
|
|
settings: PlatformSettings, allowed_fields: frozenset[str], service: str
|
|
) -> Telemetry | None:
|
|
if not settings.telemetry_enabled:
|
|
return None
|
|
resource = Resource.create(
|
|
{
|
|
"service.name": settings.otel_service_name or service,
|
|
"service.version": SERVICE_VERSION,
|
|
"service.instance.id": f"{settings.instance_id}-{os.getpid()}",
|
|
}
|
|
)
|
|
tracer = TracerProvider(
|
|
resource=resource, sampler=ParentBased(ServerRoots()), shutdown_on_exit=False
|
|
)
|
|
tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
|
|
meter = MeterProvider(
|
|
resource=resource,
|
|
metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())],
|
|
views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())],
|
|
shutdown_on_exit=False,
|
|
)
|
|
logs = LoggerProvider(resource=resource, shutdown_on_exit=False)
|
|
logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
|
|
trace.set_tracer_provider(tracer)
|
|
metrics.set_meter_provider(meter)
|
|
_logs.set_logger_provider(logs)
|
|
AsyncPGInstrumentor().instrument(tracer_provider=tracer)
|
|
handler = OtlpLogHandler(logs.get_logger("messenger", SERVICE_VERSION), allowed_fields)
|
|
logging.getLogger().addHandler(handler)
|
|
return Telemetry((tracer, meter, logs), handler)
|
|
|
|
|
|
def tag_request(request_id: str) -> None:
|
|
trace.get_current_span().set_attribute(REQUEST_ID, request_id)
|