forked from mtusi2026/mtusi2026lab2
169 lines
6.2 KiB
Python
169 lines
6.2 KiB
Python
import logging
|
|
import os
|
|
import threading
|
|
from collections.abc import Sequence
|
|
from importlib.metadata import version
|
|
from typing import override
|
|
|
|
from opentelemetry import _logs, metrics, trace
|
|
from opentelemetry._logs import SeverityNumber
|
|
from opentelemetry.context import Context
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor
|
|
from opentelemetry.sdk._logs import LoggerProvider
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.metrics.view import DropAggregation, View
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.sdk.trace.sampling import (
|
|
Decision,
|
|
ParentBased,
|
|
Sampler,
|
|
SamplingResult,
|
|
TraceIdRatioBased,
|
|
)
|
|
from opentelemetry.trace import Link, SpanKind, TraceState
|
|
from opentelemetry.util.types import Attributes
|
|
|
|
from messenger.shared.config import PlatformSettings
|
|
from messenger.shared.infrastructure.runtime_metrics import observe_process
|
|
|
|
SERVICE_VERSION = version("mtusi-messenger")
|
|
REQUEST_ID = "messenger.request_id"
|
|
ROOT_KINDS = frozenset({SpanKind.SERVER, SpanKind.CONSUMER, SpanKind.PRODUCER})
|
|
TRACE_FIELDS = frozenset({"trace_id", "span_id"})
|
|
FOREIGN_METER = "faststream.opentelemetry.middleware"
|
|
SEVERITY = {
|
|
logging.DEBUG: SeverityNumber.DEBUG,
|
|
logging.INFO: SeverityNumber.INFO,
|
|
logging.WARNING: SeverityNumber.WARN,
|
|
logging.ERROR: SeverityNumber.ERROR,
|
|
logging.CRITICAL: SeverityNumber.FATAL,
|
|
}
|
|
|
|
type Provider = TracerProvider | MeterProvider | LoggerProvider
|
|
|
|
|
|
class ServerRoots(Sampler):
|
|
def __init__(self, ratio: float = 1.0) -> None:
|
|
self._root = TraceIdRatioBased(ratio)
|
|
|
|
@override
|
|
def should_sample(
|
|
self,
|
|
parent_context: Context | None,
|
|
trace_id: int,
|
|
name: str,
|
|
kind: SpanKind | None = None,
|
|
attributes: Attributes = None,
|
|
links: Sequence[Link] | None = None,
|
|
trace_state: TraceState | None = None,
|
|
) -> SamplingResult:
|
|
if kind in ROOT_KINDS:
|
|
return self._root.should_sample(
|
|
parent_context, trace_id, name, kind, attributes, links, trace_state
|
|
)
|
|
return SamplingResult(Decision.DROP, None, trace_state)
|
|
|
|
def get_description(self) -> str:
|
|
return "ServerRoots"
|
|
|
|
|
|
class OtlpLogHandler(logging.Handler):
|
|
def __init__(self, logger: _logs.Logger, allowed_fields: frozenset[str]) -> None:
|
|
super().__init__()
|
|
self._logger = logger
|
|
self._allowed = allowed_fields - TRACE_FIELDS
|
|
|
|
def emit(self, record: logging.LogRecord) -> None:
|
|
if record.name.startswith("opentelemetry"):
|
|
return
|
|
fields = record.msg if isinstance(record.msg, dict) else {"event": record.getMessage()}
|
|
event = str(fields.get("event", ""))
|
|
attributes = {
|
|
key: value
|
|
for key, value in fields.items()
|
|
if key in self._allowed and isinstance(value, str | bool | int | float)
|
|
}
|
|
self._logger.emit(
|
|
timestamp=int(record.created * 1e9),
|
|
severity_number=SEVERITY.get(record.levelno, SeverityNumber.UNSPECIFIED),
|
|
severity_text=record.levelname,
|
|
body=event,
|
|
attributes={**attributes, "event": event},
|
|
)
|
|
|
|
|
|
class SpanAccounting(SpanProcessor):
|
|
def __init__(self, meter: MeterProvider) -> None:
|
|
self._ended = meter.get_meter("messenger.telemetry").create_counter(
|
|
"messenger.telemetry.spans.ended", unit="{span}"
|
|
)
|
|
|
|
def on_end(self, span: ReadableSpan) -> None:
|
|
self._ended.add(1, {"span.kind": span.kind.name.lower()})
|
|
|
|
|
|
class Telemetry:
|
|
__slots__ = ("_handler", "_providers")
|
|
|
|
def __init__(self, providers: tuple[Provider, ...], handler: logging.Handler) -> None:
|
|
self._providers = providers
|
|
self._handler = handler
|
|
|
|
def shutdown(self, timeout: float) -> None:
|
|
logging.getLogger().removeHandler(self._handler)
|
|
closer = threading.Thread(target=self._close, name="telemetry-shutdown", daemon=True)
|
|
closer.start()
|
|
closer.join(timeout)
|
|
|
|
def _close(self) -> None:
|
|
for provider in self._providers:
|
|
provider.shutdown()
|
|
|
|
|
|
def start_telemetry(
|
|
settings: PlatformSettings, allowed_fields: frozenset[str], service: str
|
|
) -> Telemetry | None:
|
|
if not settings.telemetry_enabled:
|
|
return None
|
|
resource = Resource.create(
|
|
{
|
|
"service.name": settings.otel_service_name or service,
|
|
"service.version": SERVICE_VERSION,
|
|
"service.instance.id": f"{settings.instance_id}-{os.getpid()}",
|
|
}
|
|
)
|
|
tracer = TracerProvider(
|
|
resource=resource,
|
|
sampler=ParentBased(ServerRoots(settings.otel_trace_ratio)),
|
|
shutdown_on_exit=False,
|
|
)
|
|
tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
|
|
meter = MeterProvider(
|
|
resource=resource,
|
|
metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())],
|
|
views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())],
|
|
shutdown_on_exit=False,
|
|
)
|
|
tracer.add_span_processor(SpanAccounting(meter))
|
|
observe_process(meter)
|
|
logs = LoggerProvider(resource=resource, shutdown_on_exit=False)
|
|
logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
|
|
trace.set_tracer_provider(tracer)
|
|
metrics.set_meter_provider(meter)
|
|
_logs.set_logger_provider(logs)
|
|
AsyncPGInstrumentor().instrument(tracer_provider=tracer)
|
|
handler = OtlpLogHandler(logs.get_logger("messenger", SERVICE_VERSION), allowed_fields)
|
|
logging.getLogger().addHandler(handler)
|
|
return Telemetry((tracer, meter, logs), handler)
|
|
|
|
|
|
def tag_request(request_id: str) -> None:
|
|
trace.get_current_span().set_attribute(REQUEST_ID, request_id)
|