"""Bounded OTLP exporters; explicit technical data only, never exception text.""" import atexit import json import resource from contextlib import contextmanager from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from opentelemetry.metrics import Observation import logging import os import socket import time from opentelemetry import trace, metrics from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter logger = logging.getLogger('jobs') tracer = trace.get_tracer('messenger-worker') counter = histogram = ready_duration = None busy = 0 queue = {"depth":0,"oldest":None,"up":0} propagator = TraceContextTextMapPropagator() def init(service="messenger-worker"): global tracer, counter, histogram, ready_duration if not os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT') or os.getenv('OTEL_SDK_DISABLED') == 'true': return res = Resource.create({'service.name': service, 'service.version': 'lab6-1', 'service.instance.id': socket.gethostname(), 'deployment.environment.name': 'lab'}) tp = TracerProvider(resource=res, sampler=ParentBased(TraceIdRatioBased(float(os.getenv('OTEL_TRACES_SAMPLER_ARG','1'))))) tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(timeout=3), max_queue_size=1024, max_export_batch_size=128)) trace.set_tracer_provider(tp) mp = MeterProvider(resource=res, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter(timeout=3), export_interval_millis=5000)]) metrics.set_meter_provider(mp) lp = LoggerProvider(resource=res) lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(timeout=3), max_queue_size=1024, max_export_batch_size=128)) logger.addHandler(LoggingHandler(logger_provider=lp)); logger.setLevel(logging.INFO); logger.propagate = False tracer = trace.get_tracer('messenger-worker') meter = metrics.get_meter('messenger-worker') counter = meter.create_counter('worker_jobs') histogram = meter.create_histogram('worker_duration', unit='s', explicit_bucket_boundaries_advisory=(.01,.05,.1,.25,.5,1,2.5,5,10,30,60)) ready_duration = meter.create_histogram('attachment_ready_duration', unit='s', explicit_bucket_boundaries_advisory=(.1,.5,1,2,5,10,30,60,120)) meter.create_observable_counter('process_cpu', unit='s', callbacks=[lambda _: [Observation(resource.getrusage(resource.RUSAGE_SELF).ru_utime + resource.getrusage(resource.RUSAGE_SELF).ru_stime)]]) meter.create_observable_gauge('process_memory', unit='By', callbacks=[lambda _: [Observation(int(open('/proc/self/statm').read().split()[1])*os.sysconf('SC_PAGE_SIZE'))]]) if service == 'messenger-worker': meter.create_observable_gauge('worker_busy', callbacks=[lambda _: [Observation(busy)]]) meter.create_observable_gauge('worker_capacity', callbacks=[lambda _: [Observation(1)]]) else: attrs = {'queue_name':'attachment-jobs-v1'} meter.create_observable_gauge('queue_depth', callbacks=[lambda _: [Observation(queue['depth'],attrs)]]) meter.create_observable_gauge('queue_oldest_age', unit='s', callbacks=[lambda _: [Observation(max(0,time.time()-queue['oldest']) if queue['oldest'] else 0,attrs)]]) meter.create_observable_gauge('queue_monitor_up', callbacks=[lambda _: [Observation(queue['up'],attrs)]]) atexit.register(lp.shutdown); atexit.register(mp.shutdown); atexit.register(tp.shutdown) def event(name): span = trace.get_current_span() sc = span.get_span_context() logger.info(name, extra={'event': name, 'trace_id': format(sc.trace_id,'032x'), 'span_id': format(sc.span_id,'016x')}) def run_job(fn, *args): global busy parent = propagator.extract(args[-1]) payload = json.loads(args[-1]['job']) with tracer.start_as_current_span('attachment.process', context=parent, kind=trace.SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False) as span: span.set_attribute('messaging.message.id', payload['job_id']) span.set_attribute('job.attempt', payload['attempt']) busy += 1 start = time.monotonic() outcome = 'error' try: result = fn(*args) outcome = result or 'skipped' return result finally: busy -= 1 attrs = {'job_type': 'attachment.process.v1', 'outcome': outcome} span.set_attributes(attrs) if outcome in ('error','failed','retry'): span.set_status(trace.Status(trace.StatusCode.ERROR, outcome)) event('job.' + outcome) if counter: counter.add(1, attrs); histogram.record(time.monotonic()-start, attrs) def call(name, fn, **kwargs): with tracer.start_as_current_span(name, kind=trace.SpanKind.CLIENT, record_exception=False, set_status_on_exception=False) as span: try: return fn(**kwargs) except Exception as exc: span.set_attribute('error.type', type(exc).__name__) span.set_status(trace.Status(trace.StatusCode.ERROR, 'S3 operation failed')) event('s3.error') raise @contextmanager def publishing(row): parent = propagator.extract({'traceparent':row['traceparent'], 'tracestate':row['tracestate']}) with tracer.start_as_current_span('attachment.publish', context=parent, kind=trace.SpanKind.PRODUCER, record_exception=False, set_status_on_exception=False): carrier = {} propagator.inject(carrier) yield carrier def carrier(): result = {} propagator.inject(result) return result class Database: """Instrument only queries inside an active request/job, omit SQL and parameters.""" def __init__(self, connection): self.connection = connection def __enter__(self): self.connection.__enter__() return self def __exit__(self, *args): return self.connection.__exit__(*args) def __getattr__(self, name): return getattr(self.connection, name) def execute(self, sql, *args, **kwargs): if not trace.get_current_span().get_span_context().is_valid: return self.connection.execute(sql,*args,**kwargs) op = sql.strip().split()[0].upper() if op not in ('SELECT','UPDATE','INSERT','DELETE'): op = 'query' with tracer.start_as_current_span('db.'+op, kind=trace.SpanKind.CLIENT, attributes={'db.system.name':'postgresql'}, record_exception=False, set_status_on_exception=False) as span: try: return self.connection.execute(sql,*args,**kwargs) except Exception as exc: span.set_attribute('error.type', type(exc).__name__) span.set_status(trace.Status(trace.StatusCode.ERROR,'database operation failed')) event('db.error') raise