Files
mtusi2026lab6/worker/telemetry.py
T

147 lines
7.5 KiB
Python

"""Bounded OTLP exporters; explicit technical data only, never exception text."""
import atexit
import json
import resource
from contextlib import contextmanager
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
from opentelemetry.metrics import Observation
import logging
import os
import socket
import time
from opentelemetry import trace, metrics
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
logger = logging.getLogger('jobs')
tracer = trace.get_tracer('messenger-worker')
counter = histogram = ready_duration = None
busy = 0
queue = {"depth":0,"oldest":None,"up":0}
propagator = TraceContextTextMapPropagator()
def init(service="messenger-worker"):
global tracer, counter, histogram, ready_duration
if not os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT') or os.getenv('OTEL_SDK_DISABLED') == 'true':
return
res = Resource.create({'service.name': service, 'service.version': 'lab6-1',
'service.instance.id': socket.gethostname(),
'deployment.environment.name': 'lab'})
tp = TracerProvider(resource=res, sampler=ParentBased(TraceIdRatioBased(float(os.getenv('OTEL_TRACES_SAMPLER_ARG','1')))))
tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(timeout=3), max_queue_size=1024, max_export_batch_size=128))
trace.set_tracer_provider(tp)
mp = MeterProvider(resource=res, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter(timeout=3), export_interval_millis=5000)])
metrics.set_meter_provider(mp)
lp = LoggerProvider(resource=res)
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(timeout=3), max_queue_size=1024, max_export_batch_size=128))
logger.addHandler(LoggingHandler(logger_provider=lp)); logger.setLevel(logging.INFO); logger.propagate = False
tracer = trace.get_tracer('messenger-worker')
meter = metrics.get_meter('messenger-worker')
counter = meter.create_counter('worker_jobs')
histogram = meter.create_histogram('worker_duration', unit='s', explicit_bucket_boundaries_advisory=(.01,.05,.1,.25,.5,1,2.5,5,10,30,60))
ready_duration = meter.create_histogram('attachment_ready_duration', unit='s', explicit_bucket_boundaries_advisory=(.1,.5,1,2,5,10,30,60,120))
meter.create_observable_counter('process_cpu', unit='s', callbacks=[lambda _: [Observation(resource.getrusage(resource.RUSAGE_SELF).ru_utime + resource.getrusage(resource.RUSAGE_SELF).ru_stime)]])
meter.create_observable_gauge('process_memory', unit='By', callbacks=[lambda _: [Observation(int(open('/proc/self/statm').read().split()[1])*os.sysconf('SC_PAGE_SIZE'))]])
if service == 'messenger-worker':
meter.create_observable_gauge('worker_busy', callbacks=[lambda _: [Observation(busy)]])
meter.create_observable_gauge('worker_capacity', callbacks=[lambda _: [Observation(1)]])
else:
attrs = {'queue_name':'attachment-jobs-v1'}
meter.create_observable_gauge('queue_depth', callbacks=[lambda _: [Observation(queue['depth'],attrs)]])
meter.create_observable_gauge('queue_oldest_age', unit='s', callbacks=[lambda _: [Observation(max(0,time.time()-queue['oldest']) if queue['oldest'] else 0,attrs)]])
meter.create_observable_gauge('queue_monitor_up', callbacks=[lambda _: [Observation(queue['up'],attrs)]])
atexit.register(lp.shutdown); atexit.register(mp.shutdown); atexit.register(tp.shutdown)
def event(name):
span = trace.get_current_span()
sc = span.get_span_context()
logger.info(name, extra={'event': name, 'trace_id': format(sc.trace_id,'032x'), 'span_id': format(sc.span_id,'016x')})
def run_job(fn, *args):
global busy
parent = propagator.extract(args[-1])
payload = json.loads(args[-1]['job'])
with tracer.start_as_current_span('attachment.process', context=parent, kind=trace.SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False) as span:
span.set_attribute('messaging.message.id', payload['job_id'])
span.set_attribute('job.attempt', payload['attempt'])
busy += 1
start = time.monotonic()
outcome = 'error'
try:
result = fn(*args)
outcome = result or 'skipped'
return result
finally:
busy -= 1
attrs = {'job_type': 'attachment.process.v1', 'outcome': outcome}
span.set_attributes(attrs)
if outcome in ('error','failed','retry'):
span.set_status(trace.Status(trace.StatusCode.ERROR, outcome))
event('job.' + outcome)
if counter:
counter.add(1, attrs); histogram.record(time.monotonic()-start, attrs)
def call(name, fn, **kwargs):
with tracer.start_as_current_span(name, kind=trace.SpanKind.CLIENT,
record_exception=False, set_status_on_exception=False) as span:
try:
return fn(**kwargs)
except Exception as exc:
span.set_attribute('error.type', type(exc).__name__)
span.set_status(trace.Status(trace.StatusCode.ERROR, 'S3 operation failed'))
event('s3.error')
raise
@contextmanager
def publishing(row):
parent = propagator.extract({'traceparent':row['traceparent'], 'tracestate':row['tracestate']})
with tracer.start_as_current_span('attachment.publish', context=parent, kind=trace.SpanKind.PRODUCER,
record_exception=False, set_status_on_exception=False):
carrier = {}
propagator.inject(carrier)
yield carrier
def carrier():
result = {}
propagator.inject(result)
return result
class Database:
"""Instrument only queries inside an active request/job, omit SQL and parameters."""
def __init__(self, connection):
self.connection = connection
def __enter__(self):
self.connection.__enter__()
return self
def __exit__(self, *args):
return self.connection.__exit__(*args)
def __getattr__(self, name):
return getattr(self.connection, name)
def execute(self, sql, *args, **kwargs):
if not trace.get_current_span().get_span_context().is_valid:
return self.connection.execute(sql,*args,**kwargs)
op = sql.strip().split()[0].upper()
if op not in ('SELECT','UPDATE','INSERT','DELETE'):
op = 'query'
with tracer.start_as_current_span('db.'+op, kind=trace.SpanKind.CLIENT,
attributes={'db.system.name':'postgresql'},
record_exception=False, set_status_on_exception=False) as span:
try:
return self.connection.execute(sql,*args,**kwargs)
except Exception as exc:
span.set_attribute('error.type', type(exc).__name__)
span.set_status(trace.Status(trace.StatusCode.ERROR,'database operation failed'))
event('db.error')
raise