forked from mtusi2026/mtusi2026lab6
205 lines
8.8 KiB
Python
205 lines
8.8 KiB
Python
"""Explicit, allowlisted OTLP instrumentation; no automatic headers, SQL or bodies.
|
|
SDK exporters run asynchronously with bounded queues; telemetry is best effort.
|
|
"""
|
|
import logging
|
|
import os
|
|
import socket
|
|
import time
|
|
from contextlib import contextmanager, nullcontext
|
|
from pathlib import Path
|
|
|
|
from opentelemetry import metrics, trace
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
|
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
|
|
PROP = TraceContextTextMapPropagator()
|
|
tracer = trace.get_tracer('messenger.lab6')
|
|
log = logging.getLogger('messenger.telemetry')
|
|
log.setLevel(logging.INFO)
|
|
log.propagate = False
|
|
providers = []
|
|
started = False
|
|
m = None
|
|
instruments = {}
|
|
worker_busy = 0
|
|
backlog_depth = None
|
|
backlog_age = None
|
|
broker_ready = None
|
|
broker_consumers = None
|
|
QUEUE = 'attachment-processing.v1'
|
|
BUCKETS = (0.005, .01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10, 15, 30, 60, 120, 300, 600)
|
|
|
|
|
|
def init(role, *, child=False):
|
|
global started, tracer, m
|
|
if started or os.getenv('OTEL_ENABLED', 'no') != 'yes':
|
|
return
|
|
started = True
|
|
resource = Resource.create({'service.name': 'messenger-' + role,
|
|
'service.version': '1.6.0', 'service.instance.id': os.getenv('API_INSTANCE_ID', socket.gethostname()),
|
|
'deployment.environment.name': 'lab'})
|
|
base = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/')
|
|
tp = TracerProvider(resource=resource)
|
|
tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=base+'/v1/traces', timeout=2),
|
|
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000))
|
|
trace.set_tracer_provider(tp)
|
|
tracer = tp.get_tracer('messenger.lab6')
|
|
lp = LoggerProvider(resource=resource)
|
|
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=base+'/v1/logs', timeout=2),
|
|
max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=500, export_timeout_millis=2000))
|
|
log.addHandler(LoggingHandler(logger_provider=lp))
|
|
providers.extend([tp, lp])
|
|
if child:
|
|
return
|
|
reader = PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=base+'/v1/metrics', timeout=2),
|
|
export_interval_millis=3000, export_timeout_millis=2000)
|
|
mp = MeterProvider(resource=resource, metric_readers=[reader], views=[View(instrument_name=n,
|
|
aggregation=ExplicitBucketHistogramAggregation(boundaries=BUCKETS)) for n in
|
|
('lab_http_duration','lab_job_duration','lab_ready_duration')])
|
|
metrics.set_meter_provider(mp)
|
|
providers.append(mp)
|
|
m = mp.get_meter('messenger.lab6')
|
|
for name in ('lab_http_requests','lab_jobs','lab_job_retries'):
|
|
instruments[name] = m.create_counter(name)
|
|
for name in ('lab_http_duration','lab_job_duration','lab_ready_duration'):
|
|
instruments[name] = m.create_histogram(name, unit='s')
|
|
if role == 'api':
|
|
for name, key in [('lab_db_pool_used','used'),('lab_db_pool_waiting','requests_waiting'),('lab_db_pool_capacity','pool_max')]:
|
|
m.create_observable_gauge(name, callbacks=[lambda _, key=key: pool_stats(key)])
|
|
if role == 'worker':
|
|
m.create_observable_gauge('lab_worker_busy', callbacks=[lambda _: [metrics.Observation(worker_busy)]])
|
|
m.create_observable_gauge('lab_worker_capacity', callbacks=[lambda _: [metrics.Observation(1)]])
|
|
if role == 'dispatcher':
|
|
m.create_observable_gauge('lab_broker_ready', callbacks=[lambda _: queue_observation(broker_ready)])
|
|
m.create_observable_gauge('lab_broker_consumers', callbacks=[lambda _: queue_observation(broker_consumers)])
|
|
m.create_observable_gauge('lab_queue_depth', callbacks=[lambda _: queue_observation(backlog_depth)])
|
|
m.create_observable_gauge('lab_queue_oldest_age', unit='s', callbacks=[lambda _: queue_observation(backlog_age)])
|
|
m.create_observable_gauge('lab_container_memory', unit='By', callbacks=[lambda _: container_observation('memory')])
|
|
m.create_observable_gauge('lab_container_cpu_cores', callbacks=[lambda _: container_observation('cpu')])
|
|
|
|
|
|
def pool_stats(key):
|
|
from . import storage
|
|
if storage.pool is None:
|
|
return []
|
|
stats = storage.pool.get_stats()
|
|
val = stats.get('pool_size',0)-stats.get('pool_available',0) if key=='used' else stats.get(key,0)
|
|
return [metrics.Observation(max(0,val))]
|
|
|
|
|
|
def queue_observation(value):
|
|
return [] if value is None else [metrics.Observation(value, {'queue': QUEUE})]
|
|
|
|
|
|
_cpu_previous = None
|
|
|
|
def container_observation(kind):
|
|
global _cpu_previous
|
|
try:
|
|
if kind == 'memory':
|
|
return [metrics.Observation(int(Path('/sys/fs/cgroup/memory.current').read_text()))]
|
|
cpu = int(dict(line.split() for line in Path('/sys/fs/cgroup/cpu.stat').read_text().splitlines())['usage_usec']) / 1e6
|
|
now = time.monotonic()
|
|
old = _cpu_previous
|
|
_cpu_previous = (now,cpu)
|
|
return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))]
|
|
except (OSError, KeyError, ValueError):
|
|
# cgroup v1 fallback, process tree (not full container accounting).
|
|
import psutil
|
|
try:
|
|
p = psutil.Process()
|
|
processes = [p]+p.children(recursive=True)
|
|
if kind == 'memory':
|
|
value = 0
|
|
for proc in processes:
|
|
try: value += proc.memory_info().rss
|
|
except psutil.Error: pass
|
|
return [metrics.Observation(value)]
|
|
# cgroup v1 (Docker Linux) if available, CPU usage in nanoseconds.
|
|
for path in ('/sys/fs/cgroup/cpuacct/cpuacct.usage','/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage'):
|
|
if Path(path).is_file():
|
|
cpu = int(Path(path).read_text()) / 1e9
|
|
now = time.monotonic()
|
|
old = _cpu_previous
|
|
_cpu_previous = (now,cpu)
|
|
return [] if old is None else [metrics.Observation(max(0,(cpu-old[1])/(now-old[0])))]
|
|
except (psutil.Error,OSError,ValueError):
|
|
pass
|
|
return []
|
|
|
|
|
|
def count(name, attrs=None):
|
|
if name in instruments: instruments[name].add(1, attrs or {})
|
|
|
|
|
|
def observe(name, value, attrs=None):
|
|
if name in instruments: instruments[name].record(max(0,value), attrs or {})
|
|
|
|
|
|
def event(name, *, level=logging.INFO, **attrs):
|
|
# Callers pass fixed event names and technical ID/status only, never payloads.
|
|
if started:
|
|
log.log(level, name, extra={'event':name, **attrs})
|
|
|
|
|
|
@contextmanager
|
|
def span(name, *, kind=SpanKind.INTERNAL, context=None, attrs=None):
|
|
with tracer.start_as_current_span(name, kind=kind, context=context, attributes=attrs,
|
|
record_exception=False, set_status_on_exception=False) as s:
|
|
try:
|
|
yield s
|
|
except BaseException as exc:
|
|
s.set_status(Status(StatusCode.ERROR))
|
|
s.set_attribute('error.type', type(exc).__name__)
|
|
event('operation.error', level=logging.ERROR, operation=name, error_type=type(exc).__name__)
|
|
raise
|
|
|
|
|
|
def inject():
|
|
result = {}
|
|
PROP.inject(result)
|
|
return result
|
|
|
|
|
|
def extract(carrier):
|
|
# Ignore baggage and all unrelated fields.
|
|
return PROP.extract({k:v for k,v in carrier.items() if k in ('traceparent','tracestate') and isinstance(v,str)})
|
|
|
|
|
|
def dependency_span(name, **kwargs):
|
|
return span(name, **kwargs) if trace_id() else nullcontext()
|
|
|
|
|
|
def trace_id():
|
|
ctx = trace.get_current_span().get_span_context()
|
|
return f'{ctx.trace_id:032x}' if ctx.is_valid else ''
|
|
|
|
|
|
def flush():
|
|
for p in providers:
|
|
p.force_flush(timeout_millis=2500)
|
|
|
|
|
|
class S3Tracing:
|
|
"""Only operation name is exported; bucket/key/bytes/signed URLs never exported."""
|
|
def __init__(self, client): self.client = client
|
|
def __getattr__(self, name):
|
|
f = getattr(self.client,name)
|
|
if name not in ('upload_fileobj','get_object','put_object','copy_object','delete_object','head_object','head_bucket'):
|
|
return f
|
|
def call(*a, **kw):
|
|
with dependency_span('s3.'+name, kind=SpanKind.CLIENT, attrs={'storage.system':'s3'}):
|
|
return f(*a, **kw)
|
|
return call
|