Files

205 lines
8.8 KiB
Python

"""Explicit, allowlisted OTLP instrumentation; no automatic headers, SQL or bodies.
SDK exporters run asynchronously with bounded queues; telemetry is best effort.
"""
import logging
import os
import socket
import time
from contextlib import contextmanager, nullcontext
from pathlib import Path
from opentelemetry import metrics, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
PROP = TraceContextTextMapPropagator()
tracer = trace.get_tracer('messenger.lab6')
log = logging.getLogger('messenger.telemetry')
log.setLevel(logging.INFO)
log.propagate = False
providers = []
started = False
m = None
instruments = {}
worker_busy = 0
backlog_depth = None
backlog_age = None
broker_ready = None
broker_consumers = None
QUEUE = 'attachment-processing.v1'
BUCKETS = (0.005, .01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10, 15, 30, 60, 120, 300, 600)
def init(role, *, child=False):
global started, tracer, m
if started or os.getenv('OTEL_ENABLED', 'no') != 'yes':
return
started = True
resource = Resource.create({'service.name': 'messenger-' + role,
'service.version': '1.6.0', 'service.instance.id': os.getenv('API_INSTANCE_ID', socket.gethostname()),
'deployment.environment.name': 'lab'})
base = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/')
tp = TracerProvider(resource=resource)
tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=base+'/v1/traces', timeout=2),
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000))
trace.set_tracer_provider(tp)
tracer = tp.get_tracer('messenger.lab6')
lp = LoggerProvider(resource=resource)
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=base+'/v1/logs', timeout=2),
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000))
log.addHandler(LoggingHandler(logger_provider=lp))
providers.extend([tp, lp])
if child:
return
reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=base+'/v1/metrics', timeout=2),
export_interval_millis=3000, export_timeout_millis=2000)
mp = MeterProvider(resource=resource, metric_readers=[reader], views=[View(instrument_name=n,
aggregation=ExplicitBucketHistogramAggregation(boundaries=BUCKETS)) for n in
('lab_http_duration','lab_job_duration','lab_ready_duration')])
metrics.set_meter_provider(mp)
providers.append(mp)
m = mp.get_meter('messenger.lab6')
for name in ('lab_http_requests','lab_jobs','lab_job_retries'):
instruments[name] = m.create_counter(name)
for name in ('lab_http_duration','lab_job_duration','lab_ready_duration'):
instruments[name] = m.create_histogram(name, unit='s')
if role == 'api':
for name, key in [('lab_db_pool_used','used'),('lab_db_pool_waiting','requests_waiting'),('lab_db_pool_capacity','pool_max')]:
m.create_observable_gauge(name, callbacks=[lambda _, key=key: pool_stats(key)])
if role == 'worker':
m.create_observable_gauge('lab_worker_busy', callbacks=[lambda _: [metrics.Observation(worker_busy)]])
m.create_observable_gauge('lab_worker_capacity', callbacks=[lambda _: [metrics.Observation(1)]])
if role == 'dispatcher':
m.create_observable_gauge('lab_broker_ready', callbacks=[lambda _: queue_observation(broker_ready)])
m.create_observable_gauge('lab_broker_consumers', callbacks=[lambda _: queue_observation(broker_consumers)])
m.create_observable_gauge('lab_queue_depth', callbacks=[lambda _: queue_observation(backlog_depth)])
m.create_observable_gauge('lab_queue_oldest_age', unit='s', callbacks=[lambda _: queue_observation(backlog_age)])
m.create_observable_gauge('lab_container_memory', unit='By', callbacks=[lambda _: container_observation('memory')])
m.create_observable_gauge('lab_container_cpu_cores', callbacks=[lambda _: container_observation('cpu')])
def pool_stats(key):
from . import storage
if storage.pool is None:
return []
stats = storage.pool.get_stats()
val = stats.get('pool_size',0)-stats.get('pool_available',0) if key=='used' else stats.get(key,0)
return [metrics.Observation(max(0,val))]
def queue_observation(value):
return [] if value is None else [metrics.Observation(value, {'queue': QUEUE})]
_cpu_previous = None
def container_observation(kind):
global _cpu_previous
try:
if kind == 'memory':
return [metrics.Observation(int(Path('/sys/fs/cgroup/memory.current').read_text()))]
cpu = int(dict(line.split() for line in Path('/sys/fs/cgroup/cpu.stat').read_text().splitlines())['usage_usec']) / 1e6
now = time.monotonic()
old = _cpu_previous
_cpu_previous = (now,cpu)
return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))]
except (OSError, KeyError, ValueError):
# cgroup v1 fallback, process tree (not full container accounting).
import psutil
try:
p = psutil.Process()
processes = [p]+p.children(recursive=True)
if kind == 'memory':
value = 0
for proc in processes:
try: value += proc.memory_info().rss
except psutil.Error: pass
return [metrics.Observation(value)]
# cgroup v1 (Docker Linux) if available, CPU usage in nanoseconds.
for path in ('/sys/fs/cgroup/cpuacct/cpuacct.usage','/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage'):
if Path(path).is_file():
cpu = int(Path(path).read_text()) / 1e9
now = time.monotonic()
old = _cpu_previous
_cpu_previous = (now,cpu)
return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))]
except (psutil.Error,OSError,ValueError):
pass
return []
def count(name, attrs=None):
if name in instruments: instruments[name].add(1, attrs or {})
def observe(name, value, attrs=None):
if name in instruments: instruments[name].record(max(0,value), attrs or {})
def event(name, *, level=logging.INFO, **attrs):
# Callers pass fixed event names and technical ID/status only, never payloads.
if started:
log.log(level, name, extra={'event':name, **attrs})
@contextmanager
def span(name, *, kind=SpanKind.INTERNAL, context=None, attrs=None):
with tracer.start_as_current_span(name, kind=kind, context=context, attributes=attrs,
record_exception=False, set_status_on_exception=False) as s:
try:
yield s
except BaseException as exc:
s.set_status(Status(StatusCode.ERROR))
s.set_attribute('error.type', type(exc).__name__)
event('operation.error', level=logging.ERROR, operation=name, error_type=type(exc).__name__)
raise
def inject():
result = {}
PROP.inject(result)
return result
def extract(carrier):
# Ignore baggage and all unrelated fields.
return PROP.extract({k:v for k,v in carrier.items() if k in ('traceparent','tracestate') and isinstance(v,str)})
def dependency_span(name, **kwargs):
return span(name, **kwargs) if trace_id() else nullcontext()
def trace_id():
ctx = trace.get_current_span().get_span_context()
return f'{ctx.trace_id:032x}' if ctx.is_valid else ''
def flush():
for p in providers:
p.force_flush(timeout_millis=2500)
class S3Tracing:
"""Only operation name is exported; bucket/key/bytes/signed URLs never exported."""
def __init__(self, client): self.client = client
def __getattr__(self, name):
f = getattr(self.client,name)
if name not in ('upload_fileobj','get_object','put_object','copy_object','delete_object','head_object','head_bucket'):
return f
def call(*a, **kw):
with dependency_span('s3.'+name, kind=SpanKind.CLIENT, attrs={'storage.system':'s3'}):
return f(*a, **kw)
return call