246 lines
11 KiB
Python
246 lines
11 KiB
Python
"""Explicit, bounded OTLP instrumentation; no body/headers/SQL auto-capture.
|
|
|
|
Only this module's safe event logger is exported. Existing application and
|
|
third-party loggers are deliberately not attached to the OTLP handler.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from contextlib import contextmanager
|
|
import json
|
|
import logging
|
|
import os
|
|
import signal
|
|
import time
|
|
from uuid import uuid4
|
|
|
|
from opentelemetry import metrics, trace
|
|
from opentelemetry._logs import set_logger_provider
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.propagate import extract
|
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, LogExportResult
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import MetricExportResult, PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
|
|
meter = metrics.get_meter("messenger", "1.6.0")
|
|
tracer = trace.get_tracer("messenger", "1.6.0")
|
|
http_requests = meter.create_counter("messenger.http.requests")
|
|
http_duration = meter.create_histogram("messenger.http.duration", unit="s")
|
|
jobs_processed = meter.create_counter("messenger.jobs.processed")
|
|
jobs_duration = meter.create_histogram("messenger.jobs.duration", unit="s")
|
|
export_failures = meter.create_counter("messenger.telemetry.export.failures")
|
|
dropped_records = meter.create_counter("messenger.telemetry.dropped.records")
|
|
|
|
_providers: list = []
|
|
_service = "messenger"
|
|
_event_logger = logging.getLogger("messenger.telemetry.events")
|
|
_event_logger.propagate = False
|
|
_event_logger.setLevel(logging.INFO)
|
|
_ALLOWED_FIELDS = {
|
|
"event", "request_id", "trace_id", "span_id", "method", "route", "status_code",
|
|
"outcome", "duration_seconds", "job_type", "job_id", "attachment_id", "attempt",
|
|
"exception_type", "operation", "queue", "count",
|
|
}
|
|
|
|
|
|
class _NoHttpRetry:
|
|
# Verified against pinned OTel 1.28.2. Retry/backpressure is the Collector's
|
|
# responsibility; a failing SDK export gets one HTTP attempt, <=2 s socket
|
|
# timeout, then discards its bounded batch. No request waits on export.
|
|
@staticmethod
|
|
def _retryable(_response):
|
|
return False
|
|
|
|
|
|
class _SpanExporter(_NoHttpRetry, OTLPSpanExporter):
|
|
def export(self, spans):
|
|
try:
|
|
result = super().export(spans)
|
|
except Exception:
|
|
result = SpanExportResult.FAILURE
|
|
if result != SpanExportResult.SUCCESS:
|
|
export_failures.add(1, {"signal": "traces"})
|
|
dropped_records.add(len(spans), {"signal": "traces"})
|
|
return result
|
|
|
|
|
|
class _LogExporter(_NoHttpRetry, OTLPLogExporter):
|
|
def export(self, batch):
|
|
try:
|
|
result = super().export(batch)
|
|
except Exception:
|
|
result = LogExportResult.FAILURE
|
|
if result != LogExportResult.SUCCESS:
|
|
export_failures.add(1, {"signal": "logs"})
|
|
dropped_records.add(len(batch), {"signal": "logs"})
|
|
return result
|
|
|
|
|
|
class _MetricExporter(_NoHttpRetry, OTLPMetricExporter):
|
|
def export(self, metrics_data, timeout_millis=10000, **kwargs):
|
|
try:
|
|
result = super().export(metrics_data, timeout_millis, **kwargs)
|
|
except Exception:
|
|
result = MetricExportResult.FAILURE
|
|
if result != MetricExportResult.SUCCESS:
|
|
export_failures.add(1, {"signal": "metrics"})
|
|
return result
|
|
|
|
|
|
class _JsonFormatter(logging.Formatter):
|
|
def format(self, record):
|
|
value = {
|
|
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(record.created)),
|
|
"severity": record.levelname, "service.name": _service,
|
|
}
|
|
value.update({key: getattr(record, key) for key in _ALLOWED_FIELDS
|
|
if hasattr(record, key)})
|
|
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
|
|
|
|
def configure(service: str, instance_id: str | None = None) -> None:
|
|
"""One resource/provider set per process. OTEL_SDK_DISABLED supports off runs."""
|
|
global _service
|
|
if _providers:
|
|
return
|
|
_service = service
|
|
console = logging.StreamHandler()
|
|
console.setFormatter(_JsonFormatter())
|
|
_event_logger.addHandler(console)
|
|
if os.getenv("OTEL_SDK_DISABLED", "false").lower() == "true":
|
|
return
|
|
resource = Resource({
|
|
"service.name": service,
|
|
"service.version": os.getenv("APP_VERSION", "1.6.0"),
|
|
"service.instance.id": instance_id or uuid4().hex,
|
|
"deployment.environment.name": "lab",
|
|
})
|
|
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
|
trace_provider = TracerProvider(resource=resource)
|
|
trace_provider.add_span_processor(BatchSpanProcessor(
|
|
_SpanExporter(endpoint=endpoint + "/v1/traces", timeout=2),
|
|
max_queue_size=512, max_export_batch_size=64,
|
|
schedule_delay_millis=1000, export_timeout_millis=2000,
|
|
))
|
|
trace.set_tracer_provider(trace_provider)
|
|
log_provider = LoggerProvider(resource=resource)
|
|
log_provider.add_log_record_processor(BatchLogRecordProcessor(
|
|
_LogExporter(endpoint=endpoint + "/v1/logs", timeout=2),
|
|
max_queue_size=512, max_export_batch_size=64,
|
|
schedule_delay_millis=1000, export_timeout_millis=2000,
|
|
))
|
|
set_logger_provider(log_provider)
|
|
_event_logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=log_provider))
|
|
reader = PeriodicExportingMetricReader(
|
|
_MetricExporter(endpoint=endpoint + "/v1/metrics", timeout=2),
|
|
export_interval_millis=5000, export_timeout_millis=2500,
|
|
)
|
|
# Boundaries are seconds. Quantiles are computed over merged replica buckets.
|
|
boundaries = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120)
|
|
metric_provider = MeterProvider(resource=resource, metric_readers=[reader], views=[
|
|
View(instrument_type=metrics.Histogram,
|
|
aggregation=ExplicitBucketHistogramAggregation(boundaries)),
|
|
])
|
|
metrics.set_meter_provider(metric_provider)
|
|
_providers.extend((metric_provider, log_provider, trace_provider))
|
|
|
|
|
|
def shutdown() -> None:
|
|
"""Flush bounded background buffers on normal termination (not SIGKILL)."""
|
|
for provider in _providers:
|
|
provider.shutdown()
|
|
_providers.clear()
|
|
|
|
|
|
def install_termination_handler() -> None:
|
|
"""Turn worker/publisher SIGTERM into cancellation so their finally runs."""
|
|
import asyncio
|
|
loop = asyncio.get_running_loop()
|
|
task = asyncio.current_task()
|
|
if task is not None:
|
|
try:
|
|
loop.add_signal_handler(signal.SIGTERM, task.cancel)
|
|
except NotImplementedError: # Windows host; Compose containers use POSIX.
|
|
signal.signal(signal.SIGTERM, lambda *_: loop.call_soon_threadsafe(task.cancel))
|
|
|
|
|
|
def emit(event: str, *, severity: int = logging.INFO, **fields) -> None:
|
|
"""Call with a source-defined event and technical fields, never free text."""
|
|
current = trace.get_current_span().get_span_context()
|
|
attrs = {key: value for key, value in fields.items()
|
|
if key in _ALLOWED_FIELDS and value is not None}
|
|
attrs.update(event=event, trace_id=f"{current.trace_id:032x}", span_id=f"{current.span_id:016x}")
|
|
_event_logger.log(severity, event, extra=attrs)
|
|
|
|
|
|
@contextmanager
|
|
def span(name: str, attributes=None, kind=SpanKind.INTERNAL, parent=None):
|
|
"""Safe span helper: never export exception messages or stack traces."""
|
|
with tracer.start_as_current_span(
|
|
name, attributes=attributes, kind=kind, context=parent,
|
|
record_exception=False, set_status_on_exception=False,
|
|
) as active:
|
|
try:
|
|
yield active
|
|
except BaseException as exc:
|
|
active.set_attribute("exception_type", type(exc).__name__)
|
|
active.set_status(Status(StatusCode.ERROR))
|
|
raise
|
|
|
|
|
|
class HttpTelemetryMiddleware:
|
|
"""Outer ASGI wrapper sees final status, streaming duration and 500 responses."""
|
|
def __init__(self, app):
|
|
self.app = app
|
|
|
|
async def __call__(self, scope, receive, send):
|
|
if scope["type"] != "http":
|
|
return await self.app(scope, receive, send)
|
|
from starlette.requests import Request
|
|
from app.errors import common_headers, incoming_request_id
|
|
|
|
request = Request(scope)
|
|
request.state.request_id = incoming_request_id(request)
|
|
method = scope.get("method", "OTHER")
|
|
if method not in {"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}:
|
|
method = "OTHER"
|
|
status = 500
|
|
started = time.perf_counter()
|
|
# Only W3C propagation fields are consumed; headers are never exported.
|
|
parent = extract(dict(request.headers))
|
|
with span("HTTP request", kind=SpanKind.SERVER, parent=parent) as active:
|
|
async def observed_send(message):
|
|
nonlocal status
|
|
if message["type"] == "http.response.start":
|
|
status = message["status"]
|
|
headers = dict(message.get("headers", []))
|
|
for key, value in common_headers(request).items():
|
|
headers[key.lower().encode()] = value.encode()
|
|
headers[b"x-trace-id"] = f"{active.get_span_context().trace_id:032x}".encode()
|
|
message = {**message, "headers": list(headers.items())}
|
|
await send(message)
|
|
|
|
try:
|
|
await self.app(scope, receive, observed_send)
|
|
finally:
|
|
route = getattr(scope.get("route"), "path", "unmatched")
|
|
duration = time.perf_counter() - started
|
|
attrs = {"method": method, "route": route, "status_class": f"{status // 100}xx"}
|
|
active.update_name(f"{method} {route}")
|
|
active.set_attributes({**attrs, "http.response.status_code": status})
|
|
if status >= 500:
|
|
active.set_status(Status(StatusCode.ERROR))
|
|
http_requests.add(1, attrs)
|
|
http_duration.record(duration, attrs)
|
|
emit("http.request.complete", severity=logging.ERROR if status >= 500 else logging.INFO,
|
|
request_id=request.state.request_id, method=method, route=route,
|
|
status_code=status, outcome="error" if status >= 500 else "success",
|
|
duration_seconds=duration)
|