"""Explicit, allowlisted OTLP instrumentation; no automatic headers, SQL or bodies. SDK exporters run asynchronously with bounded queues; telemetry is best effort. """ import logging import os import socket import time from contextlib import contextmanager, nullcontext from pathlib import Path from opentelemetry import metrics, trace from opentelemetry.trace import SpanKind, Status, StatusCode from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter PROP = TraceContextTextMapPropagator() tracer = trace.get_tracer('messenger.lab6') log = logging.getLogger('messenger.telemetry') log.setLevel(logging.INFO) log.propagate = False providers = [] started = False m = None instruments = {} worker_busy = 0 backlog_depth = None backlog_age = None broker_ready = None broker_consumers = None QUEUE = 'attachment-processing.v1' BUCKETS = (0.005, .01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10, 15, 30, 60, 120, 300, 600) def init(role, *, child=False): global started, tracer, m if started or os.getenv('OTEL_ENABLED', 'no') != 'yes': return started = True resource = Resource.create({'service.name': 'messenger-' + role, 'service.version': '1.6.0', 'service.instance.id': os.getenv('API_INSTANCE_ID', socket.gethostname()), 'deployment.environment.name': 'lab'}) base = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/') tp = TracerProvider(resource=resource) tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=base+'/v1/traces', timeout=2), max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000)) trace.set_tracer_provider(tp) tracer = tp.get_tracer('messenger.lab6') lp = LoggerProvider(resource=resource) lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=base+'/v1/logs', timeout=2), max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000)) log.addHandler(LoggingHandler(logger_provider=lp)) providers.extend([tp, lp]) if child: return reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=base+'/v1/metrics', timeout=2), export_interval_millis=3000, export_timeout_millis=2000) mp = MeterProvider(resource=resource, metric_readers=[reader], views=[View(instrument_name=n, aggregation=ExplicitBucketHistogramAggregation(boundaries=BUCKETS)) for n in ('lab_http_duration','lab_job_duration','lab_ready_duration')]) metrics.set_meter_provider(mp) providers.append(mp) m = mp.get_meter('messenger.lab6') for name in ('lab_http_requests','lab_jobs','lab_job_retries'): instruments[name] = m.create_counter(name) for name in ('lab_http_duration','lab_job_duration','lab_ready_duration'): instruments[name] = m.create_histogram(name, unit='s') if role == 'api': for name, key in [('lab_db_pool_used','used'),('lab_db_pool_waiting','requests_waiting'),('lab_db_pool_capacity','pool_max')]: m.create_observable_gauge(name, callbacks=[lambda _, key=key: pool_stats(key)]) if role == 'worker': m.create_observable_gauge('lab_worker_busy', callbacks=[lambda _: [metrics.Observation(worker_busy)]]) m.create_observable_gauge('lab_worker_capacity', callbacks=[lambda _: [metrics.Observation(1)]]) if role == 'dispatcher': m.create_observable_gauge('lab_broker_ready', callbacks=[lambda _: queue_observation(broker_ready)]) m.create_observable_gauge('lab_broker_consumers', callbacks=[lambda _: queue_observation(broker_consumers)]) m.create_observable_gauge('lab_queue_depth', callbacks=[lambda _: queue_observation(backlog_depth)]) m.create_observable_gauge('lab_queue_oldest_age', unit='s', callbacks=[lambda _: queue_observation(backlog_age)]) m.create_observable_gauge('lab_container_memory', unit='By', callbacks=[lambda _: container_observation('memory')]) m.create_observable_gauge('lab_container_cpu_cores', callbacks=[lambda _: container_observation('cpu')]) def pool_stats(key): from . import storage if storage.pool is None: return [] stats = storage.pool.get_stats() val = stats.get('pool_size',0)-stats.get('pool_available',0) if key=='used' else stats.get(key,0) return [metrics.Observation(max(0,val))] def queue_observation(value): return [] if value is None else [metrics.Observation(value, {'queue': QUEUE})] _cpu_previous = None def container_observation(kind): global _cpu_previous try: if kind == 'memory': return [metrics.Observation(int(Path('/sys/fs/cgroup/memory.current').read_text()))] cpu = int(dict(line.split() for line in Path('/sys/fs/cgroup/cpu.stat').read_text().splitlines())['usage_usec']) / 1e6 now = time.monotonic() old = _cpu_previous _cpu_previous = (now,cpu) return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))] except (OSError, KeyError, ValueError): # cgroup v1 fallback, process tree (not full container accounting). import psutil try: p = psutil.Process() processes = [p]+p.children(recursive=True) if kind == 'memory': value = 0 for proc in processes: try: value += proc.memory_info().rss except psutil.Error: pass return [metrics.Observation(value)] # cgroup v1 (Docker Linux) if available, CPU usage in nanoseconds. for path in ('/sys/fs/cgroup/cpuacct/cpuacct.usage','/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage'): if Path(path).is_file(): cpu = int(Path(path).read_text()) / 1e9 now = time.monotonic() old = _cpu_previous _cpu_previous = (now,cpu) return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))] except (psutil.Error,OSError,ValueError): pass return [] def count(name, attrs=None): if name in instruments: instruments[name].add(1, attrs or {}) def observe(name, value, attrs=None): if name in instruments: instruments[name].record(max(0,value), attrs or {}) def event(name, *, level=logging.INFO, **attrs): # Callers pass fixed event names and technical ID/status only, never payloads. if started: log.log(level, name, extra={'event':name, **attrs}) @contextmanager def span(name, *, kind=SpanKind.INTERNAL, context=None, attrs=None): with tracer.start_as_current_span(name, kind=kind, context=context, attributes=attrs, record_exception=False, set_status_on_exception=False) as s: try: yield s except BaseException as exc: s.set_status(Status(StatusCode.ERROR)) s.set_attribute('error.type', type(exc).__name__) event('operation.error', level=logging.ERROR, operation=name, error_type=type(exc).__name__) raise def inject(): result = {} PROP.inject(result) return result def extract(carrier): # Ignore baggage and all unrelated fields. return PROP.extract({k:v for k,v in carrier.items() if k in ('traceparent','tracestate') and isinstance(v,str)}) def dependency_span(name, **kwargs): return span(name, **kwargs) if trace_id() else nullcontext() def trace_id(): ctx = trace.get_current_span().get_span_context() return f'{ctx.trace_id:032x}' if ctx.is_valid else '' def flush(): for p in providers: p.force_flush(timeout_millis=2500) class S3Tracing: """Only operation name is exported; bucket/key/bytes/signed URLs never exported.""" def __init__(self, client): self.client = client def __getattr__(self, name): f = getattr(self.client,name) if name not in ('upload_fileobj','get_object','put_object','copy_object','delete_object','head_object','head_bucket'): return f def call(*a, **kw): with dependency_span('s3.'+name, kind=SpanKind.CLIENT, attrs={'storage.system':'s3'}): return f(*a, **kw) return call