forked from mtusi2026/mtusi2026lab6
147 lines
7.5 KiB
Python
147 lines
7.5 KiB
Python
"""Bounded OTLP exporters; explicit technical data only, never exception text."""
|
|
import atexit
|
|
import json
|
|
import resource
|
|
from contextlib import contextmanager
|
|
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
|
from opentelemetry.metrics import Observation
|
|
import logging
|
|
import os
|
|
import socket
|
|
import time
|
|
from opentelemetry import trace, metrics
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
|
|
logger = logging.getLogger('jobs')
|
|
tracer = trace.get_tracer('messenger-worker')
|
|
counter = histogram = ready_duration = None
|
|
busy = 0
|
|
queue = {"depth":0,"oldest":None,"up":0}
|
|
propagator = TraceContextTextMapPropagator()
|
|
|
|
def init(service="messenger-worker"):
|
|
global tracer, counter, histogram, ready_duration
|
|
if not os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT') or os.getenv('OTEL_SDK_DISABLED') == 'true':
|
|
return
|
|
res = Resource.create({'service.name': service, 'service.version': 'lab6-1',
|
|
'service.instance.id': socket.gethostname(),
|
|
'deployment.environment.name': 'lab'})
|
|
tp = TracerProvider(resource=res, sampler=ParentBased(TraceIdRatioBased(float(os.getenv('OTEL_TRACES_SAMPLER_ARG','1')))))
|
|
tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(timeout=3), max_queue_size=1024, max_export_batch_size=128))
|
|
trace.set_tracer_provider(tp)
|
|
mp = MeterProvider(resource=res, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter(timeout=3), export_interval_millis=5000)])
|
|
metrics.set_meter_provider(mp)
|
|
lp = LoggerProvider(resource=res)
|
|
lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(timeout=3), max_queue_size=1024, max_export_batch_size=128))
|
|
logger.addHandler(LoggingHandler(logger_provider=lp)); logger.setLevel(logging.INFO); logger.propagate = False
|
|
tracer = trace.get_tracer('messenger-worker')
|
|
meter = metrics.get_meter('messenger-worker')
|
|
counter = meter.create_counter('worker_jobs')
|
|
histogram = meter.create_histogram('worker_duration', unit='s', explicit_bucket_boundaries_advisory=(.01,.05,.1,.25,.5,1,2.5,5,10,30,60))
|
|
ready_duration = meter.create_histogram('attachment_ready_duration', unit='s', explicit_bucket_boundaries_advisory=(.1,.5,1,2,5,10,30,60,120))
|
|
meter.create_observable_counter('process_cpu', unit='s', callbacks=[lambda _: [Observation(resource.getrusage(resource.RUSAGE_SELF).ru_utime + resource.getrusage(resource.RUSAGE_SELF).ru_stime)]])
|
|
meter.create_observable_gauge('process_memory', unit='By', callbacks=[lambda _: [Observation(int(open('/proc/self/statm').read().split()[1])*os.sysconf('SC_PAGE_SIZE'))]])
|
|
if service == 'messenger-worker':
|
|
meter.create_observable_gauge('worker_busy', callbacks=[lambda _: [Observation(busy)]])
|
|
meter.create_observable_gauge('worker_capacity', callbacks=[lambda _: [Observation(1)]])
|
|
else:
|
|
attrs = {'queue_name':'attachment-jobs-v1'}
|
|
meter.create_observable_gauge('queue_depth', callbacks=[lambda _: [Observation(queue['depth'],attrs)]])
|
|
meter.create_observable_gauge('queue_oldest_age', unit='s', callbacks=[lambda _: [Observation(max(0,time.time()-queue['oldest']) if queue['oldest'] else 0,attrs)]])
|
|
meter.create_observable_gauge('queue_monitor_up', callbacks=[lambda _: [Observation(queue['up'],attrs)]])
|
|
atexit.register(lp.shutdown); atexit.register(mp.shutdown); atexit.register(tp.shutdown)
|
|
|
|
def event(name):
|
|
span = trace.get_current_span()
|
|
sc = span.get_span_context()
|
|
logger.info(name, extra={'event': name, 'trace_id': format(sc.trace_id,'032x'), 'span_id': format(sc.span_id,'016x')})
|
|
|
|
def run_job(fn, *args):
|
|
global busy
|
|
parent = propagator.extract(args[-1])
|
|
payload = json.loads(args[-1]['job'])
|
|
with tracer.start_as_current_span('attachment.process', context=parent, kind=trace.SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False) as span:
|
|
span.set_attribute('messaging.message.id', payload['job_id'])
|
|
span.set_attribute('job.attempt', payload['attempt'])
|
|
busy += 1
|
|
start = time.monotonic()
|
|
outcome = 'error'
|
|
try:
|
|
result = fn(*args)
|
|
outcome = result or 'skipped'
|
|
return result
|
|
finally:
|
|
busy -= 1
|
|
attrs = {'job_type': 'attachment.process.v1', 'outcome': outcome}
|
|
span.set_attributes(attrs)
|
|
if outcome in ('error','failed','retry'):
|
|
span.set_status(trace.Status(trace.StatusCode.ERROR, outcome))
|
|
event('job.' + outcome)
|
|
if counter:
|
|
counter.add(1, attrs); histogram.record(time.monotonic()-start, attrs)
|
|
|
|
|
|
def call(name, fn, **kwargs):
|
|
with tracer.start_as_current_span(name, kind=trace.SpanKind.CLIENT,
|
|
record_exception=False, set_status_on_exception=False) as span:
|
|
try:
|
|
return fn(**kwargs)
|
|
except Exception as exc:
|
|
span.set_attribute('error.type', type(exc).__name__)
|
|
span.set_status(trace.Status(trace.StatusCode.ERROR, 'S3 operation failed'))
|
|
event('s3.error')
|
|
raise
|
|
|
|
|
|
@contextmanager
|
|
def publishing(row):
|
|
parent = propagator.extract({'traceparent':row['traceparent'], 'tracestate':row['tracestate']})
|
|
with tracer.start_as_current_span('attachment.publish', context=parent, kind=trace.SpanKind.PRODUCER,
|
|
record_exception=False, set_status_on_exception=False):
|
|
carrier = {}
|
|
propagator.inject(carrier)
|
|
yield carrier
|
|
|
|
def carrier():
|
|
result = {}
|
|
propagator.inject(result)
|
|
return result
|
|
|
|
class Database:
|
|
"""Instrument only queries inside an active request/job, omit SQL and parameters."""
|
|
def __init__(self, connection):
|
|
self.connection = connection
|
|
def __enter__(self):
|
|
self.connection.__enter__()
|
|
return self
|
|
def __exit__(self, *args):
|
|
return self.connection.__exit__(*args)
|
|
def __getattr__(self, name):
|
|
return getattr(self.connection, name)
|
|
def execute(self, sql, *args, **kwargs):
|
|
if not trace.get_current_span().get_span_context().is_valid:
|
|
return self.connection.execute(sql,*args,**kwargs)
|
|
op = sql.strip().split()[0].upper()
|
|
if op not in ('SELECT','UPDATE','INSERT','DELETE'):
|
|
op = 'query'
|
|
with tracer.start_as_current_span('db.'+op, kind=trace.SpanKind.CLIENT,
|
|
attributes={'db.system.name':'postgresql'},
|
|
record_exception=False, set_status_on_exception=False) as span:
|
|
try:
|
|
return self.connection.execute(sql,*args,**kwargs)
|
|
except Exception as exc:
|
|
span.set_attribute('error.type', type(exc).__name__)
|
|
span.set_status(trace.Status(trace.StatusCode.ERROR,'database operation failed'))
|
|
event('db.error')
|
|
raise
|