Files
mtusi2026lab6/src/telemetry.py
T

111 lines
5.7 KiB
Python

"""Explicit low-cardinality OTLP signals. Business operation never waits for export."""
from contextlib import contextmanager
import json
import os
import resource
import time
from uuid import uuid4
from opentelemetry import metrics, trace
from opentelemetry._logs import SeverityNumber
from opentelemetry.propagate import extract, inject
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
configured=False
log_provider=None
service='messenger'
instance=uuid4().hex
def setup(name):
global configured,log_provider,service,http_count,http_duration,job_count,job_duration,ready_duration,pool_used,pool_waiting,worker_busy,worker_capacity,queue_depth,queue_age,process_rss,process_cpu,slo_total,slo_bad
if configured: return
configured=True; service=name
enabled=os.environ.get('TELEMETRY_ENABLED','1')=='1'
endpoint=os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT','http://otel-collector:4318').rstrip('/')
res=Resource.create({'service.name':name,'service.version':'lab6-v1','service.instance.id':instance,
'deployment.environment.name':'lab'})
if enabled:
tracer=TracerProvider(resource=res,sampler=ParentBased(TraceIdRatioBased(float(os.environ.get('TRACE_SAMPLE_RATIO','1')))))
tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint+'/v1/traces',timeout=2),
max_queue_size=256,max_export_batch_size=64,schedule_delay_millis=1000,export_timeout_millis=2000))
trace.set_tracer_provider(tracer)
log_provider=LoggerProvider(resource=res)
log_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint+'/v1/logs',timeout=2),
max_queue_size=256,max_export_batch_size=64,schedule_delay_millis=1000,export_timeout_millis=2000))
reader=PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint+'/v1/metrics',timeout=2),export_interval_millis=5000,export_timeout_millis=3000)
metrics.set_meter_provider(MeterProvider(resource=res,metric_readers=[reader],views=[
View(instrument_type=metrics.Histogram,aggregation=ExplicitBucketHistogramAggregation([.005,.01,.025,.05,.1,.25,.5,1,2.5,5,10,30,60,120]))]))
meter=metrics.get_meter('messenger')
http_count=meter.create_counter('http_server_requests')
http_duration=meter.create_histogram('http_server_duration',unit='s')
job_count=meter.create_counter('jobs_processed')
job_duration=meter.create_histogram('job_processing_duration',unit='s')
ready_duration=meter.create_histogram('attachment_time_to_ready',unit='s')
pool_used=meter.create_gauge('db_pool_used')
pool_waiting=meter.create_gauge('db_pool_waiting')
worker_busy=meter.create_gauge('worker_busy')
worker_capacity=meter.create_gauge('worker_capacity')
queue_depth=meter.create_gauge('queue_depth')
queue_age=meter.create_gauge('queue_oldest_age',unit='s')
process_rss=meter.create_gauge('process_rss',unit='By')
process_cpu=meter.create_gauge('process_cpu_time',unit='s')
slo_total=meter.create_gauge('attachment_slo_eligible')
slo_bad=meter.create_gauge('attachment_slo_bad')
def tracer(): return trace.get_tracer('messenger')
@contextmanager
def span(name,kind=trace.SpanKind.INTERNAL,carrier=None,attributes=None):
with tracer().start_as_current_span(name,kind=kind,context=extract(carrier) if carrier else None,
attributes=attributes or {},record_exception=False,set_status_on_exception=False) as current:
try: yield current
except BaseException:
current.set_status(trace.Status(trace.StatusCode.ERROR))
raise
def carrier():
result={}; inject(result); return result
def event(name,**fields):
# Callers pass controlled fields only. Never capture arbitrary exceptions,
# HTTP headers/bodies/raw URLs, passwords, tokens, filenames or S3 URLs.
context=trace.get_current_span().get_span_context()
row={'timestamp':time.time(),'severity':'INFO','service.name':service,'event':name,
'trace_id':format(context.trace_id,'032x'),'span_id':format(context.span_id,'016x'),**fields}
print(json.dumps(row),flush=True)
if log_provider:
log_provider.get_logger('messenger').emit(body=json.dumps(row),severity_number=SeverityNumber.INFO,
severity_text='INFO',attributes={'event':name})
def process_metrics():
# Linux ru_maxrss is KiB; use actual current RSS from proc for memory trend.
with open('/proc/self/statm') as file:
rss=int(file.read().split()[1])*os.sysconf('SC_PAGE_SIZE')
usage=resource.getrusage(resource.RUSAGE_SELF)
attrs={'service':service,'process_instance':instance}
process_rss.set(rss,attrs); process_cpu.set(usage.ru_utime+usage.ru_stime,attrs)
def shutdown():
for provider in (trace.get_tracer_provider(),metrics.get_meter_provider(),log_provider):
if provider and hasattr(provider,'shutdown'):
try: provider.shutdown()
except Exception: pass