Files
mtusi2026lab6/src/messenger/telemetry.py

242 lines
8.3 KiB
Python

"""Application telemetry with bounded export and an explicit attribute allowlist."""
import json
import logging
import os
import resource
import time
from contextlib import contextmanager
from importlib.metadata import version
from uuid import uuid4
from opentelemetry import trace
from opentelemetry._logs import LogRecord, SeverityNumber
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.metrics import Observation
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
logger = logging.getLogger("messenger")
PROPAGATOR = TraceContextTextMapPropagator()
BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120)
HTTP_METHODS = frozenset(("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"))
JOB_TYPE = "attachment.process.v1"
runtime = None
class Telemetry:
def __init__(self, role, instance, metrics):
resource_attrs = Resource(
{
"service.name": f"messenger-{role}",
"service.version": version("mtusi-messenger"),
"service.instance.id": instance,
"deployment.environment.name": "lab",
}
)
ratio = float(os.getenv("OTEL_SAMPLE_RATIO", "1"))
self.traces = TracerProvider(
resource=resource_attrs, sampler=ParentBased(TraceIdRatioBased(ratio))
)
self.traces.add_span_processor(
BatchSpanProcessor(
OTLPSpanExporter(timeout=1),
max_queue_size=512,
max_export_batch_size=64,
schedule_delay_millis=500,
)
)
self.tracer = self.traces.get_tracer("messenger")
self.logs = LoggerProvider(resource=resource_attrs)
self.logs.add_log_record_processor(
BatchLogRecordProcessor(
OTLPLogExporter(timeout=1),
max_queue_size=512,
max_export_batch_size=64,
schedule_delay_millis=500,
)
)
self.log = self.logs.get_logger("messenger")
self.meters = None
self.instruments = {}
if metrics:
self.meters = MeterProvider(
resource=resource_attrs,
metric_readers=[
PeriodicExportingMetricReader(
OTLPMetricExporter(timeout=1),
export_interval_millis=2000,
export_timeout_millis=1000,
)
],
views=[
View(
instrument_name="messenger.*.duration",
aggregation=ExplicitBucketHistogramAggregation(BUCKETS),
)
],
)
self.meter = self.meters.get_meter("messenger")
self.meter.create_observable_counter(
"messenger.process.cpu", unit="s", callbacks=[self.cpu_seconds]
)
self.meter.create_observable_gauge(
"messenger.process.rss", unit="By", callbacks=[self.rss_bytes]
)
@staticmethod
def cpu_seconds(_):
usage = resource.getrusage(resource.RUSAGE_CHILDREN)
yield Observation(time.process_time() + usage.ru_utime + usage.ru_stime)
@staticmethod
def rss_bytes(_):
try:
with open("/proc/self/statm", encoding="ascii") as stats:
yield Observation(int(stats.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"))
except OSError:
return
def shutdown(self):
if self.meters:
self.meters.shutdown(timeout_millis=1500)
self.logs.shutdown()
self.traces.shutdown()
@contextmanager
def active(role, instance=None, *, metrics=True):
global runtime
if os.getenv("TELEMETRY_ENABLED", "0") != "1":
yield
return
runtime = Telemetry(role, instance or uuid4().hex, metrics)
try:
yield
finally:
state, runtime = runtime, None
state.shutdown()
@contextmanager
def span(name, *, context=None, kind=SpanKind.INTERNAL, **attrs):
if runtime is None:
yield trace.INVALID_SPAN
return
with runtime.tracer.start_as_current_span(
name,
context=context,
kind=kind,
attributes=attrs,
record_exception=False,
set_status_on_exception=False,
) as current:
try:
yield current
except Exception as exc:
current.set_attribute("error.type", type(exc).__name__)
current.set_status(Status(StatusCode.ERROR))
raise
def carrier():
result = {}
if runtime is not None:
PROPAGATOR.inject(result)
return result
def context(headers):
return PROPAGATOR.extract(
{k: headers[k] for k in ("traceparent", "tracestate") if headers.get(k)}
)
def record(name, value, *, kind="counter", unit="1", **attrs):
if runtime is None or runtime.meters is None:
return
instrument = runtime.instruments.get(name)
if instrument is None:
create = {
"counter": runtime.meter.create_counter,
"histogram": runtime.meter.create_histogram,
"gauge": runtime.meter.create_gauge,
}[kind]
instrument = runtime.instruments.setdefault(name, create(name, unit=unit))
if kind == "counter":
instrument.add(value, attrs)
elif kind == "histogram":
instrument.record(value, attrs)
else:
instrument.set(value, attrs)
def duration(name, seconds, **attrs):
record(name, seconds, kind="histogram", unit="s", **attrs)
def gauge(name, value, *, unit="", **attrs):
record(name, value, kind="gauge", unit=unit, **attrs)
def pool(pool):
if runtime is not None and runtime.meters is not None:
stats = pool.get_stats()
gauge(
"messenger.db.pool.used",
max(0, stats["pool_size"] - stats["pool_available"]),
)
gauge("messenger.db.pool.waiting", stats["requests_waiting"])
def event(event_name, *, error=False, **attrs):
fields = {"event": event_name, **attrs}
current = trace.get_current_span().get_span_context()
if current.is_valid:
fields["trace_id"] = format(current.trace_id, "032x")
fields["span_id"] = format(current.span_id, "016x")
logger.log(logging.ERROR if error else logging.INFO, json.dumps(fields))
if runtime is not None:
runtime.log.emit(
LogRecord(
timestamp=time.time_ns(),
observed_timestamp=time.time_ns(),
severity_text="ERROR" if error else "INFO",
severity_number=SeverityNumber.ERROR if error else SeverityNumber.INFO,
body=event_name,
attributes=fields,
)
)
def http(current, method, route, status, elapsed, request_id):
method = method if method in HTTP_METHODS else "_OTHER"
attrs = {"method": method, "route": route, "status_class": f"{status // 100}xx"}
current.update_name(f"{method} {route}")
current.set_attributes(
{"http.request.method": method, "http.route": route, "http.response.status_code": status}
)
if status >= 500:
current.set_status(Status(StatusCode.ERROR))
record("messenger.http.requests", 1, **attrs)
duration("messenger.http.duration", elapsed, **attrs)
event(
"http_request",
error=status >= 500,
request_id=request_id,
status=status,
duration_ms=round(elapsed * 1000, 3),
**attrs,
)