111 lines
5.7 KiB
Python
111 lines
5.7 KiB
Python
"""Explicit low-cardinality OTLP signals. Business operation never waits for export."""
|
|
from contextlib import contextmanager
|
|
import json
|
|
import os
|
|
import resource
|
|
import time
|
|
from uuid import uuid4
|
|
|
|
from opentelemetry import metrics, trace
|
|
from opentelemetry._logs import SeverityNumber
|
|
from opentelemetry.propagate import extract, inject
|
|
from opentelemetry.sdk._logs import LoggerProvider
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
|
|
configured=False
|
|
log_provider=None
|
|
service='messenger'
|
|
instance=uuid4().hex
|
|
|
|
|
|
def setup(name):
|
|
global configured,log_provider,service,http_count,http_duration,job_count,job_duration,ready_duration,pool_used,pool_waiting,worker_busy,worker_capacity,queue_depth,queue_age,process_rss,process_cpu,slo_total,slo_bad
|
|
if configured: return
|
|
configured=True; service=name
|
|
enabled=os.environ.get('TELEMETRY_ENABLED','1')=='1'
|
|
endpoint=os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT','http://otel-collector:4318').rstrip('/')
|
|
res=Resource.create({'service.name':name,'service.version':'lab6-v1','service.instance.id':instance,
|
|
'deployment.environment.name':'lab'})
|
|
if enabled:
|
|
tracer=TracerProvider(resource=res,sampler=ParentBased(TraceIdRatioBased(float(os.environ.get('TRACE_SAMPLE_RATIO','1')))))
|
|
tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint+'/v1/traces',timeout=2),
|
|
max_queue_size=256,max_export_batch_size=64,schedule_delay_millis=1000,export_timeout_millis=2000))
|
|
trace.set_tracer_provider(tracer)
|
|
log_provider=LoggerProvider(resource=res)
|
|
log_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(endpoint=endpoint+'/v1/logs',timeout=2),
|
|
max_queue_size=256,max_export_batch_size=64,schedule_delay_millis=1000,export_timeout_millis=2000))
|
|
reader=PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=endpoint+'/v1/metrics',timeout=2),export_interval_millis=5000,export_timeout_millis=3000)
|
|
metrics.set_meter_provider(MeterProvider(resource=res,metric_readers=[reader],views=[
|
|
View(instrument_type=metrics.Histogram,aggregation=ExplicitBucketHistogramAggregation([.005,.01,.025,.05,.1,.25,.5,1,2.5,5,10,30,60,120]))]))
|
|
meter=metrics.get_meter('messenger')
|
|
http_count=meter.create_counter('http_server_requests')
|
|
http_duration=meter.create_histogram('http_server_duration',unit='s')
|
|
job_count=meter.create_counter('jobs_processed')
|
|
job_duration=meter.create_histogram('job_processing_duration',unit='s')
|
|
ready_duration=meter.create_histogram('attachment_time_to_ready',unit='s')
|
|
pool_used=meter.create_gauge('db_pool_used')
|
|
pool_waiting=meter.create_gauge('db_pool_waiting')
|
|
worker_busy=meter.create_gauge('worker_busy')
|
|
worker_capacity=meter.create_gauge('worker_capacity')
|
|
queue_depth=meter.create_gauge('queue_depth')
|
|
queue_age=meter.create_gauge('queue_oldest_age',unit='s')
|
|
process_rss=meter.create_gauge('process_rss',unit='By')
|
|
process_cpu=meter.create_gauge('process_cpu_time',unit='s')
|
|
slo_total=meter.create_gauge('attachment_slo_eligible')
|
|
slo_bad=meter.create_gauge('attachment_slo_bad')
|
|
|
|
|
|
def tracer(): return trace.get_tracer('messenger')
|
|
|
|
|
|
@contextmanager
|
|
def span(name,kind=trace.SpanKind.INTERNAL,carrier=None,attributes=None):
|
|
with tracer().start_as_current_span(name,kind=kind,context=extract(carrier) if carrier else None,
|
|
attributes=attributes or {},record_exception=False,set_status_on_exception=False) as current:
|
|
try: yield current
|
|
except BaseException:
|
|
current.set_status(trace.Status(trace.StatusCode.ERROR))
|
|
raise
|
|
|
|
|
|
def carrier():
|
|
result={}; inject(result); return result
|
|
|
|
|
|
def event(name,**fields):
|
|
# Callers pass controlled fields only. Never capture arbitrary exceptions,
|
|
# HTTP headers/bodies/raw URLs, passwords, tokens, filenames or S3 URLs.
|
|
context=trace.get_current_span().get_span_context()
|
|
row={'timestamp':time.time(),'severity':'INFO','service.name':service,'event':name,
|
|
'trace_id':format(context.trace_id,'032x'),'span_id':format(context.span_id,'016x'),**fields}
|
|
print(json.dumps(row),flush=True)
|
|
if log_provider:
|
|
log_provider.get_logger('messenger').emit(body=json.dumps(row),severity_number=SeverityNumber.INFO,
|
|
severity_text='INFO',attributes={'event':name})
|
|
|
|
|
|
def process_metrics():
|
|
# Linux ru_maxrss is KiB; use actual current RSS from proc for memory trend.
|
|
with open('/proc/self/statm') as file:
|
|
rss=int(file.read().split()[1])*os.sysconf('SC_PAGE_SIZE')
|
|
usage=resource.getrusage(resource.RUSAGE_SELF)
|
|
attrs={'service':service,'process_instance':instance}
|
|
process_rss.set(rss,attrs); process_cpu.set(usage.ru_utime+usage.ru_stime,attrs)
|
|
|
|
|
|
def shutdown():
|
|
for provider in (trace.get_tracer_provider(),metrics.get_meter_provider(),log_provider):
|
|
if provider and hasattr(provider,'shutdown'):
|
|
try: provider.shutdown()
|
|
except Exception: pass
|