Files
mtusi2026lab6/app/telemetry.py
T

246 lines
11 KiB
Python

"""Explicit, bounded OTLP instrumentation; no body/headers/SQL auto-capture.
Only this module's safe event logger is exported. Existing application and
third-party loggers are deliberately not attached to the OTLP handler.
"""
from __future__ import annotations
from contextlib import contextmanager
import json
import logging
import os
import signal
import time
from uuid import uuid4
from opentelemetry import metrics, trace
from opentelemetry._logs import set_logger_provider
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.propagate import extract
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, LogExportResult
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import MetricExportResult, PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
from opentelemetry.trace import SpanKind, Status, StatusCode
meter = metrics.get_meter("messenger", "1.6.0")
tracer = trace.get_tracer("messenger", "1.6.0")
http_requests = meter.create_counter("messenger.http.requests")
http_duration = meter.create_histogram("messenger.http.duration", unit="s")
jobs_processed = meter.create_counter("messenger.jobs.processed")
jobs_duration = meter.create_histogram("messenger.jobs.duration", unit="s")
export_failures = meter.create_counter("messenger.telemetry.export.failures")
dropped_records = meter.create_counter("messenger.telemetry.dropped.records")
_providers: list = []
_service = "messenger"
_event_logger = logging.getLogger("messenger.telemetry.events")
_event_logger.propagate = False
_event_logger.setLevel(logging.INFO)
_ALLOWED_FIELDS = {
"event", "request_id", "trace_id", "span_id", "method", "route", "status_code",
"outcome", "duration_seconds", "job_type", "job_id", "attachment_id", "attempt",
"exception_type", "operation", "queue", "count",
}
class _NoHttpRetry:
# Verified against pinned OTel 1.28.2. Retry/backpressure is the Collector's
# responsibility; a failing SDK export gets one HTTP attempt, <=2 s socket
# timeout, then discards its bounded batch. No request waits on export.
@staticmethod
def _retryable(_response):
return False
class _SpanExporter(_NoHttpRetry, OTLPSpanExporter):
def export(self, spans):
try:
result = super().export(spans)
except Exception:
result = SpanExportResult.FAILURE
if result != SpanExportResult.SUCCESS:
export_failures.add(1, {"signal": "traces"})
dropped_records.add(len(spans), {"signal": "traces"})
return result
class _LogExporter(_NoHttpRetry, OTLPLogExporter):
def export(self, batch):
try:
result = super().export(batch)
except Exception:
result = LogExportResult.FAILURE
if result != LogExportResult.SUCCESS:
export_failures.add(1, {"signal": "logs"})
dropped_records.add(len(batch), {"signal": "logs"})
return result
class _MetricExporter(_NoHttpRetry, OTLPMetricExporter):
def export(self, metrics_data, timeout_millis=10000, **kwargs):
try:
result = super().export(metrics_data, timeout_millis, **kwargs)
except Exception:
result = MetricExportResult.FAILURE
if result != MetricExportResult.SUCCESS:
export_failures.add(1, {"signal": "metrics"})
return result
class _JsonFormatter(logging.Formatter):
def format(self, record):
value = {
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(record.created)),
"severity": record.levelname, "service.name": _service,
}
value.update({key: getattr(record, key) for key in _ALLOWED_FIELDS
if hasattr(record, key)})
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
def configure(service: str, instance_id: str | None = None) -> None:
"""One resource/provider set per process. OTEL_SDK_DISABLED supports off runs."""
global _service
if _providers:
return
_service = service
console = logging.StreamHandler()
console.setFormatter(_JsonFormatter())
_event_logger.addHandler(console)
if os.getenv("OTEL_SDK_DISABLED", "false").lower() == "true":
return
resource = Resource({
"service.name": service,
"service.version": os.getenv("APP_VERSION", "1.6.0"),
"service.instance.id": instance_id or uuid4().hex,
"deployment.environment.name": "lab",
})
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
trace_provider = TracerProvider(resource=resource)
trace_provider.add_span_processor(BatchSpanProcessor(
_SpanExporter(endpoint=endpoint + "/v1/traces", timeout=2),
max_queue_size=512, max_export_batch_size=64,
schedule_delay_millis=1000, export_timeout_millis=2000,
))
trace.set_tracer_provider(trace_provider)
log_provider = LoggerProvider(resource=resource)
log_provider.add_log_record_processor(BatchLogRecordProcessor(
_LogExporter(endpoint=endpoint + "/v1/logs", timeout=2),
max_queue_size=512, max_export_batch_size=64,
schedule_delay_millis=1000, export_timeout_millis=2000,
))
set_logger_provider(log_provider)
_event_logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=log_provider))
reader = PeriodicExportingMetricReader(
_MetricExporter(endpoint=endpoint + "/v1/metrics", timeout=2),
export_interval_millis=5000, export_timeout_millis=2500,
)
# Boundaries are seconds. Quantiles are computed over merged replica buckets.
boundaries = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120)
metric_provider = MeterProvider(resource=resource, metric_readers=[reader], views=[
View(instrument_type=metrics.Histogram,
aggregation=ExplicitBucketHistogramAggregation(boundaries)),
])
metrics.set_meter_provider(metric_provider)
_providers.extend((metric_provider, log_provider, trace_provider))
def shutdown() -> None:
"""Flush bounded background buffers on normal termination (not SIGKILL)."""
for provider in _providers:
provider.shutdown()
_providers.clear()
def install_termination_handler() -> None:
"""Turn worker/publisher SIGTERM into cancellation so their finally runs."""
import asyncio
loop = asyncio.get_running_loop()
task = asyncio.current_task()
if task is not None:
try:
loop.add_signal_handler(signal.SIGTERM, task.cancel)
except NotImplementedError: # Windows host; Compose containers use POSIX.
signal.signal(signal.SIGTERM, lambda *_: loop.call_soon_threadsafe(task.cancel))
def emit(event: str, *, severity: int = logging.INFO, **fields) -> None:
"""Call with a source-defined event and technical fields, never free text."""
current = trace.get_current_span().get_span_context()
attrs = {key: value for key, value in fields.items()
if key in _ALLOWED_FIELDS and value is not None}
attrs.update(event=event, trace_id=f"{current.trace_id:032x}", span_id=f"{current.span_id:016x}")
_event_logger.log(severity, event, extra=attrs)
@contextmanager
def span(name: str, attributes=None, kind=SpanKind.INTERNAL, parent=None):
"""Safe span helper: never export exception messages or stack traces."""
with tracer.start_as_current_span(
name, attributes=attributes, kind=kind, context=parent,
record_exception=False, set_status_on_exception=False,
) as active:
try:
yield active
except BaseException as exc:
active.set_attribute("exception_type", type(exc).__name__)
active.set_status(Status(StatusCode.ERROR))
raise
class HttpTelemetryMiddleware:
"""Outer ASGI wrapper sees final status, streaming duration and 500 responses."""
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
from starlette.requests import Request
from app.errors import common_headers, incoming_request_id
request = Request(scope)
request.state.request_id = incoming_request_id(request)
method = scope.get("method", "OTHER")
if method not in {"GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"}:
method = "OTHER"
status = 500
started = time.perf_counter()
# Only W3C propagation fields are consumed; headers are never exported.
parent = extract(dict(request.headers))
with span("HTTP request", kind=SpanKind.SERVER, parent=parent) as active:
async def observed_send(message):
nonlocal status
if message["type"] == "http.response.start":
status = message["status"]
headers = dict(message.get("headers", []))
for key, value in common_headers(request).items():
headers[key.lower().encode()] = value.encode()
headers[b"x-trace-id"] = f"{active.get_span_context().trace_id:032x}".encode()
message = {**message, "headers": list(headers.items())}
await send(message)
try:
await self.app(scope, receive, observed_send)
finally:
route = getattr(scope.get("route"), "path", "unmatched")
duration = time.perf_counter() - started
attrs = {"method": method, "route": route, "status_class": f"{status // 100}xx"}
active.update_name(f"{method} {route}")
active.set_attributes({**attrs, "http.response.status_code": status})
if status >= 500:
active.set_status(Status(StatusCode.ERROR))
http_requests.add(1, attrs)
http_duration.record(duration, attrs)
emit("http.request.complete", severity=logging.ERROR if status >= 500 else logging.INFO,
request_id=request.state.request_id, method=method, route=route,
status_code=status, outcome="error" if status >= 500 else "success",
duration_seconds=duration)