"""Explicit OTLP signals; no HTTP bodies, credentials, SQL or raw URLs.""" import json import os import re import threading import time import weakref from contextlib import contextmanager from pathlib import Path from types import SimpleNamespace from uuid import uuid4 import psutil from opentelemetry import trace from opentelemetry._logs import SeverityNumber from opentelemetry.exporter.otlp.proto.http._log_exporter import ( OTLPLogExporter, ) from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( OTLPMetricExporter, ) from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( OTLPSpanExporter, ) from opentelemetry.metrics import Observation from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.sdk.metrics import Histogram, MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.metrics.view import ( ExplicitBucketHistogramAggregation, View, ) from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased from opentelemetry.trace import SpanKind, Status, StatusCode from opentelemetry.trace.propagation.tracecontext import ( TraceContextTextMapPropagator, ) _state = None _lock = threading.Lock() _pools = weakref.WeakSet() _instruments = {} _observer = {} _propagator = TraceContextTextMapPropagator() BUCKETS = ( 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 60, 120, ) LOG_FIELDS = { "request_id", "job_id", "attachment_id", "method", "route", "status", "duration_seconds", "attempt", "version", "fence", "outcome", "error_code", "operation", "cycle", } TRACE_FIELDS = { "request_id", "job.id", "attachment.id", "pipeline.version", "job.attempt", "db.system.name", "operation", "messaging.system", "messaging.destination.name", "http.route", "http.request.method", "http.response.status_code", "error.type", } def enabled(): return os.getenv("OTEL_ENABLED", "0") == "1" def register_pool(pool): _pools.add(pool) def observe(values): _observer.update(values) def _pool_stat(key): stats = [p.get_stats() for p in list(_pools)] if key == "used": return sum( max(0, s.get("pool_size", 0) - s.get("pool_available", 0)) for s in stats ) return sum(s.get("requests_waiting", 0) for s in stats) def configure(service, *, instance=None, with_metrics=True): global _state if not enabled(): return with _lock: if _state is not None: return instance = instance or uuid4().hex resource = Resource.create( { "service.name": service, "service.version": os.getenv("APP_VERSION", "development"), "service.instance.id": instance, "deployment.environment.name": "lab", } ) endpoint = os.getenv( "OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel:4318" ).rstrip("/") options = {"timeout": 0.75} # Certificate options are also honored by the exporters in the TLS lab. mp = None if with_metrics: reader = PeriodicExportingMetricReader( OTLPMetricExporter( endpoint=endpoint + "/v1/metrics", **options ), export_interval_millis=2000, export_timeout_millis=1000, ) mp = MeterProvider( resource=resource, metric_readers=[reader], views=[ View( instrument_type=Histogram, aggregation=ExplicitBucketHistogramAggregation( BUCKETS ), ) ], ) meter = mp.get_meter("messenger", "1.0") for name in ["http_requests", "job_attempts"]: _instruments[name] = meter.create_counter("messenger_" + name) _instruments["worker_busy"] = meter.create_up_down_counter( "messenger_worker_busy" ) for name in ["http_duration", "job_duration", "time_to_ready"]: _instruments[name] = meter.create_histogram( "messenger_" + name, unit="s" ) for key in ["used", "waiting"]: meter.create_observable_gauge( "messenger_db_pool_" + key, callbacks=[lambda _, k=key: [Observation(_pool_stat(k))]], ) process = psutil.Process() meter.create_observable_counter( "messenger_process_cpu", unit="s", callbacks=[ lambda _: [Observation(sum(process.cpu_times()[:2]))] ], ) meter.create_observable_gauge( "messenger_process_memory", unit="By", callbacks=[lambda _: [Observation(process.memory_info().rss)]], ) def container_memory(_): p = Path("/sys/fs/cgroup/memory.current") return [Observation(int(p.read_text()))] if p.exists() else [] meter.create_observable_gauge( "messenger_container_memory", unit="By", callbacks=[container_memory], ) if service == "messenger-worker": meter.create_observable_gauge( "messenger_worker_capacity", callbacks=[ lambda _: [ Observation( int(os.getenv("WORKER_CONCURRENCY", "2")) ) ] ], ) if service == "messenger-observer": for key in ["queue_depth", "oldest_job_age"]: meter.create_observable_gauge( "messenger_" + key, unit="s" if key.endswith("age") else "", callbacks=[ lambda _, k=key: [ Observation( _observer.get(k, 0), {"queue": "attachments"}, ) ] ], ) for name, prefix in [ ("slo_decisions", "slo_"), ("jobs_completed", "completed_"), ]: meter.create_observable_counter( "messenger_" + name, callbacks=[ lambda _, p=prefix: [ Observation( _observer.get(p + outcome, 0), { "outcome": outcome, "job_type": "attachment.process.v1", }, ) for outcome in ( ["good", "bad"] if p == "slo_" else ["ready", "failed"] ) ] ], ) tp = TracerProvider( resource=resource, sampler=ParentBased( TraceIdRatioBased(float(os.getenv("OTEL_TRACE_RATIO", "1"))) ), ) tp.add_span_processor( BatchSpanProcessor( OTLPSpanExporter(endpoint=endpoint + "/v1/traces", **options), max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=200, export_timeout_millis=1000, ) ) lp = LoggerProvider(resource=resource) lp.add_log_record_processor( BatchLogRecordProcessor( OTLPLogExporter(endpoint=endpoint + "/v1/logs", **options), max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=200, export_timeout_millis=1000, ) ) _state = SimpleNamespace( service=service, instance=instance, traces=tp, metrics=mp, logs=lp, tracer=tp.get_tracer("messenger", "1.0"), logger=lp.get_logger("messenger", "1.0"), ) def stop(): if _state is None: return for provider in [_state.traces, _state.logs, _state.metrics]: if provider is not None: provider.force_flush(timeout_millis=1000) provider.shutdown() def count(name, value=1, attributes=None): instrument = _instruments.get(name) if instrument is not None: instrument.add(value, attributes or {}) def duration(name, seconds, attributes=None): instrument = _instruments.get(name) if instrument is not None: instrument.record(max(0, seconds), attributes or {}) def carrier(): result = {} _propagator.inject(result) return result @contextmanager def span(name, *, kind=SpanKind.INTERNAL, parent=None, attributes=None): tracer = ( _state.tracer if _state is not None else trace.get_tracer("messenger") ) attrs = {k: v for k, v in (attributes or {}).items() if k in TRACE_FIELDS} context = _propagator.extract(parent) if parent is not None else None with tracer.start_as_current_span( name, kind=kind, context=context, attributes=attrs, record_exception=False, set_status_on_exception=False, ) as current: try: yield current except BaseException as exc: current.set_attribute("error.type", type(exc).__name__) current.set_status(Status(StatusCode.ERROR, "operation failed")) raise def log(event, *, error=False, **fields): event = ( event if re.fullmatch(r"[a-z][a-z0-9_.-]{0,79}", event) else "application.event" ) attrs = { k: v[:256] if isinstance(v, str) else v for k, v in fields.items() if k in LOG_FIELDS and isinstance(v, (str, int, float, bool)) } current = trace.get_current_span().get_span_context() record = { "event": event, "timestamp": time.time(), "severity": "ERROR" if error else "INFO", "service.name": _state.service if _state else "messenger", **attrs, } if current.is_valid: record.update( trace_id=f"{current.trace_id:032x}", span_id=f"{current.span_id:016x}", ) print(json.dumps(record, ensure_ascii=False), flush=True) if _state is not None: _state.logger.emit( body=event, attributes=attrs, severity_text=record["severity"], severity_number=SeverityNumber.ERROR if error else SeverityNumber.INFO, ) class TelemetryMiddleware: def __init__(self, app): self.app = app async def __call__(self, scope, receive, send): if scope["type"] != "http": return await self.app(scope, receive, send) # Health polling does not enter business SLIs or spans. if scope.get("path", "").startswith("/health/"): return await self.app(scope, receive, send) headers = { k.decode("latin1"): v.decode("latin1") for k, v in scope.get("headers", []) if k.lower() in {b"traceparent", b"tracestate"} } status = 500 started = time.monotonic() async def measured_send(message): nonlocal status if message["type"] == "http.response.start": status = message["status"] await send(message) with span("HTTP", kind=SpanKind.SERVER, parent=headers) as current: try: await self.app(scope, receive, measured_send) finally: route = getattr(scope.get("route"), "path", "unmatched") method = scope.get("method", "UNKNOWN") if method not in { "GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", }: method = "OTHER" current.update_name(method + " " + route) current.set_attributes( { "http.route": route, "http.request.method": method, "http.response.status_code": status, } ) if status >= 500: current.set_status( Status(StatusCode.ERROR, "HTTP server error") ) elapsed = time.monotonic() - started labels = { "route": route, "method": method, "status_class": str(status // 100) + "xx", } count("http_requests", attributes=labels) duration("http_duration", elapsed, labels) log( "http.request", request_id=scope.get("state", {}).get("request_id", ""), method=method, route=route, status=status, duration_seconds=round(elapsed, 6), error=status >= 500, )