forked from mtusi2026/mtusi2026lab6
242 lines
8.3 KiB
Python
242 lines
8.3 KiB
Python
"""Application telemetry with bounded export and an explicit attribute allowlist."""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import resource
|
|
import time
|
|
from contextlib import contextmanager
|
|
from importlib.metadata import version
|
|
from uuid import uuid4
|
|
|
|
from opentelemetry import trace
|
|
from opentelemetry._logs import LogRecord, SeverityNumber
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.metrics import Observation
|
|
from opentelemetry.sdk._logs import LoggerProvider
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
|
|
|
logger = logging.getLogger("messenger")
|
|
PROPAGATOR = TraceContextTextMapPropagator()
|
|
BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120)
|
|
HTTP_METHODS = frozenset(("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"))
|
|
JOB_TYPE = "attachment.process.v1"
|
|
runtime = None
|
|
|
|
|
|
class Telemetry:
|
|
def __init__(self, role, instance, metrics):
|
|
resource_attrs = Resource(
|
|
{
|
|
"service.name": f"messenger-{role}",
|
|
"service.version": version("mtusi-messenger"),
|
|
"service.instance.id": instance,
|
|
"deployment.environment.name": "lab",
|
|
}
|
|
)
|
|
ratio = float(os.getenv("OTEL_SAMPLE_RATIO", "1"))
|
|
self.traces = TracerProvider(
|
|
resource=resource_attrs, sampler=ParentBased(TraceIdRatioBased(ratio))
|
|
)
|
|
self.traces.add_span_processor(
|
|
BatchSpanProcessor(
|
|
OTLPSpanExporter(timeout=1),
|
|
max_queue_size=512,
|
|
max_export_batch_size=64,
|
|
schedule_delay_millis=500,
|
|
)
|
|
)
|
|
self.tracer = self.traces.get_tracer("messenger")
|
|
self.logs = LoggerProvider(resource=resource_attrs)
|
|
self.logs.add_log_record_processor(
|
|
BatchLogRecordProcessor(
|
|
OTLPLogExporter(timeout=1),
|
|
max_queue_size=512,
|
|
max_export_batch_size=64,
|
|
schedule_delay_millis=500,
|
|
)
|
|
)
|
|
self.log = self.logs.get_logger("messenger")
|
|
self.meters = None
|
|
self.instruments = {}
|
|
if metrics:
|
|
self.meters = MeterProvider(
|
|
resource=resource_attrs,
|
|
metric_readers=[
|
|
PeriodicExportingMetricReader(
|
|
OTLPMetricExporter(timeout=1),
|
|
export_interval_millis=2000,
|
|
export_timeout_millis=1000,
|
|
)
|
|
],
|
|
views=[
|
|
View(
|
|
instrument_name="messenger.*.duration",
|
|
aggregation=ExplicitBucketHistogramAggregation(BUCKETS),
|
|
)
|
|
],
|
|
)
|
|
self.meter = self.meters.get_meter("messenger")
|
|
self.meter.create_observable_counter(
|
|
"messenger.process.cpu", unit="s", callbacks=[self.cpu_seconds]
|
|
)
|
|
self.meter.create_observable_gauge(
|
|
"messenger.process.rss", unit="By", callbacks=[self.rss_bytes]
|
|
)
|
|
|
|
@staticmethod
|
|
def cpu_seconds(_):
|
|
usage = resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
yield Observation(time.process_time() + usage.ru_utime + usage.ru_stime)
|
|
|
|
@staticmethod
|
|
def rss_bytes(_):
|
|
try:
|
|
with open("/proc/self/statm", encoding="ascii") as stats:
|
|
yield Observation(int(stats.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"))
|
|
except OSError:
|
|
return
|
|
|
|
def shutdown(self):
|
|
if self.meters:
|
|
self.meters.shutdown(timeout_millis=1500)
|
|
self.logs.shutdown()
|
|
self.traces.shutdown()
|
|
|
|
|
|
@contextmanager
|
|
def active(role, instance=None, *, metrics=True):
|
|
global runtime
|
|
if os.getenv("TELEMETRY_ENABLED", "0") != "1":
|
|
yield
|
|
return
|
|
runtime = Telemetry(role, instance or uuid4().hex, metrics)
|
|
try:
|
|
yield
|
|
finally:
|
|
state, runtime = runtime, None
|
|
state.shutdown()
|
|
|
|
|
|
@contextmanager
|
|
def span(name, *, context=None, kind=SpanKind.INTERNAL, **attrs):
|
|
if runtime is None:
|
|
yield trace.INVALID_SPAN
|
|
return
|
|
with runtime.tracer.start_as_current_span(
|
|
name,
|
|
context=context,
|
|
kind=kind,
|
|
attributes=attrs,
|
|
record_exception=False,
|
|
set_status_on_exception=False,
|
|
) as current:
|
|
try:
|
|
yield current
|
|
except Exception as exc:
|
|
current.set_attribute("error.type", type(exc).__name__)
|
|
current.set_status(Status(StatusCode.ERROR))
|
|
raise
|
|
|
|
|
|
def carrier():
|
|
result = {}
|
|
if runtime is not None:
|
|
PROPAGATOR.inject(result)
|
|
return result
|
|
|
|
|
|
def context(headers):
|
|
return PROPAGATOR.extract(
|
|
{k: headers[k] for k in ("traceparent", "tracestate") if headers.get(k)}
|
|
)
|
|
|
|
|
|
def record(name, value, *, kind="counter", unit="1", **attrs):
|
|
if runtime is None or runtime.meters is None:
|
|
return
|
|
instrument = runtime.instruments.get(name)
|
|
if instrument is None:
|
|
create = {
|
|
"counter": runtime.meter.create_counter,
|
|
"histogram": runtime.meter.create_histogram,
|
|
"gauge": runtime.meter.create_gauge,
|
|
}[kind]
|
|
instrument = runtime.instruments.setdefault(name, create(name, unit=unit))
|
|
if kind == "counter":
|
|
instrument.add(value, attrs)
|
|
elif kind == "histogram":
|
|
instrument.record(value, attrs)
|
|
else:
|
|
instrument.set(value, attrs)
|
|
|
|
|
|
def duration(name, seconds, **attrs):
|
|
record(name, seconds, kind="histogram", unit="s", **attrs)
|
|
|
|
|
|
def gauge(name, value, *, unit="", **attrs):
|
|
record(name, value, kind="gauge", unit=unit, **attrs)
|
|
|
|
|
|
def pool(pool):
|
|
if runtime is not None and runtime.meters is not None:
|
|
stats = pool.get_stats()
|
|
gauge(
|
|
"messenger.db.pool.used",
|
|
max(0, stats["pool_size"] - stats["pool_available"]),
|
|
)
|
|
gauge("messenger.db.pool.waiting", stats["requests_waiting"])
|
|
|
|
|
|
def event(event_name, *, error=False, **attrs):
|
|
fields = {"event": event_name, **attrs}
|
|
current = trace.get_current_span().get_span_context()
|
|
if current.is_valid:
|
|
fields["trace_id"] = format(current.trace_id, "032x")
|
|
fields["span_id"] = format(current.span_id, "016x")
|
|
logger.log(logging.ERROR if error else logging.INFO, json.dumps(fields))
|
|
if runtime is not None:
|
|
runtime.log.emit(
|
|
LogRecord(
|
|
timestamp=time.time_ns(),
|
|
observed_timestamp=time.time_ns(),
|
|
severity_text="ERROR" if error else "INFO",
|
|
severity_number=SeverityNumber.ERROR if error else SeverityNumber.INFO,
|
|
body=event_name,
|
|
attributes=fields,
|
|
)
|
|
)
|
|
|
|
|
|
def http(current, method, route, status, elapsed, request_id):
|
|
method = method if method in HTTP_METHODS else "_OTHER"
|
|
attrs = {"method": method, "route": route, "status_class": f"{status // 100}xx"}
|
|
current.update_name(f"{method} {route}")
|
|
current.set_attributes(
|
|
{"http.request.method": method, "http.route": route, "http.response.status_code": status}
|
|
)
|
|
if status >= 500:
|
|
current.set_status(Status(StatusCode.ERROR))
|
|
record("messenger.http.requests", 1, **attrs)
|
|
duration("messenger.http.duration", elapsed, **attrs)
|
|
event(
|
|
"http_request",
|
|
error=status >= 500,
|
|
request_id=request_id,
|
|
status=status,
|
|
duration_ms=round(elapsed * 1000, 3),
|
|
**attrs,
|
|
)
|