Files
mtusi2026lab6/src/telemetry.py
T

440 lines
14 KiB
Python

"""Explicit OTLP signals; no HTTP bodies, credentials, SQL or raw URLs."""
import json
import os
import re
import threading
import time
import weakref
from contextlib import contextmanager
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
import psutil
from opentelemetry import trace
from opentelemetry._logs import SeverityNumber
from opentelemetry.exporter.otlp.proto.http._log_exporter import (
OTLPLogExporter,
)
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
OTLPMetricExporter,
)
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter,
)
from opentelemetry.metrics import Observation
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import Histogram, MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import (
ExplicitBucketHistogramAggregation,
View,
)
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.trace.propagation.tracecontext import (
TraceContextTextMapPropagator,
)
_state = None
_lock = threading.Lock()
_pools = weakref.WeakSet()
_instruments = {}
_observer = {}
_propagator = TraceContextTextMapPropagator()
BUCKETS = (
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1,
2.5,
5,
10,
20,
60,
120,
)
LOG_FIELDS = {
"request_id",
"job_id",
"attachment_id",
"method",
"route",
"status",
"duration_seconds",
"attempt",
"version",
"fence",
"outcome",
"error_code",
"operation",
"cycle",
}
TRACE_FIELDS = {
"request_id",
"job.id",
"attachment.id",
"pipeline.version",
"job.attempt",
"db.system.name",
"operation",
"messaging.system",
"messaging.destination.name",
"http.route",
"http.request.method",
"http.response.status_code",
"error.type",
}
def enabled():
return os.getenv("OTEL_ENABLED", "0") == "1"
def register_pool(pool):
_pools.add(pool)
def observe(values):
_observer.update(values)
def _pool_stat(key):
stats = [p.get_stats() for p in list(_pools)]
if key == "used":
return sum(
max(0, s.get("pool_size", 0) - s.get("pool_available", 0))
for s in stats
)
return sum(s.get("requests_waiting", 0) for s in stats)
def configure(service, *, instance=None, with_metrics=True):
global _state
if not enabled():
return
with _lock:
if _state is not None:
return
instance = instance or uuid4().hex
resource = Resource.create(
{
"service.name": service,
"service.version": os.getenv("APP_VERSION", "development"),
"service.instance.id": instance,
"deployment.environment.name": "lab",
}
)
endpoint = os.getenv(
"OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel:4318"
).rstrip("/")
options = {"timeout": 0.75}
# Certificate options are also honored by the exporters in the TLS lab.
mp = None
if with_metrics:
reader = PeriodicExportingMetricReader(
OTLPMetricExporter(
endpoint=endpoint + "/v1/metrics", **options
),
export_interval_millis=2000,
export_timeout_millis=1000,
)
mp = MeterProvider(
resource=resource,
metric_readers=[reader],
views=[
View(
instrument_type=Histogram,
aggregation=ExplicitBucketHistogramAggregation(
BUCKETS
),
)
],
)
meter = mp.get_meter("messenger", "1.0")
for name in ["http_requests", "job_attempts"]:
_instruments[name] = meter.create_counter("messenger_" + name)
_instruments["worker_busy"] = meter.create_up_down_counter(
"messenger_worker_busy"
)
for name in ["http_duration", "job_duration", "time_to_ready"]:
_instruments[name] = meter.create_histogram(
"messenger_" + name, unit="s"
)
for key in ["used", "waiting"]:
meter.create_observable_gauge(
"messenger_db_pool_" + key,
callbacks=[lambda _, k=key: [Observation(_pool_stat(k))]],
)
process = psutil.Process()
meter.create_observable_counter(
"messenger_process_cpu",
unit="s",
callbacks=[
lambda _: [Observation(sum(process.cpu_times()[:2]))]
],
)
meter.create_observable_gauge(
"messenger_process_memory",
unit="By",
callbacks=[lambda _: [Observation(process.memory_info().rss)]],
)
def container_memory(_):
p = Path("/sys/fs/cgroup/memory.current")
return [Observation(int(p.read_text()))] if p.exists() else []
meter.create_observable_gauge(
"messenger_container_memory",
unit="By",
callbacks=[container_memory],
)
if service == "messenger-worker":
meter.create_observable_gauge(
"messenger_worker_capacity",
callbacks=[
lambda _: [
Observation(
int(os.getenv("WORKER_CONCURRENCY", "2"))
)
]
],
)
if service == "messenger-observer":
for key in ["queue_depth", "oldest_job_age"]:
meter.create_observable_gauge(
"messenger_" + key,
unit="s" if key.endswith("age") else "",
callbacks=[
lambda _, k=key: [
Observation(
_observer.get(k, 0),
{"queue": "attachments"},
)
]
],
)
for name, prefix in [
("slo_decisions", "slo_"),
("jobs_completed", "completed_"),
]:
meter.create_observable_counter(
"messenger_" + name,
callbacks=[
lambda _, p=prefix: [
Observation(
_observer.get(p + outcome, 0),
{
"outcome": outcome,
"job_type": "attachment.process.v1",
},
)
for outcome in (
["good", "bad"]
if p == "slo_"
else ["ready", "failed"]
)
]
],
)
tp = TracerProvider(
resource=resource,
sampler=ParentBased(
TraceIdRatioBased(float(os.getenv("OTEL_TRACE_RATIO", "1")))
),
)
tp.add_span_processor(
BatchSpanProcessor(
OTLPSpanExporter(endpoint=endpoint + "/v1/traces", **options),
max_queue_size=512,
max_export_batch_size=64,
schedule_delay_millis=200,
export_timeout_millis=1000,
)
)
lp = LoggerProvider(resource=resource)
lp.add_log_record_processor(
BatchLogRecordProcessor(
OTLPLogExporter(endpoint=endpoint + "/v1/logs", **options),
max_queue_size=512,
max_export_batch_size=64,
schedule_delay_millis=200,
export_timeout_millis=1000,
)
)
_state = SimpleNamespace(
service=service,
instance=instance,
traces=tp,
metrics=mp,
logs=lp,
tracer=tp.get_tracer("messenger", "1.0"),
logger=lp.get_logger("messenger", "1.0"),
)
def stop():
if _state is None:
return
for provider in [_state.traces, _state.logs, _state.metrics]:
if provider is not None:
provider.force_flush(timeout_millis=1000)
provider.shutdown()
def count(name, value=1, attributes=None):
instrument = _instruments.get(name)
if instrument is not None:
instrument.add(value, attributes or {})
def duration(name, seconds, attributes=None):
instrument = _instruments.get(name)
if instrument is not None:
instrument.record(max(0, seconds), attributes or {})
def carrier():
result = {}
_propagator.inject(result)
return result
@contextmanager
def span(name, *, kind=SpanKind.INTERNAL, parent=None, attributes=None):
tracer = (
_state.tracer if _state is not None else trace.get_tracer("messenger")
)
attrs = {k: v for k, v in (attributes or {}).items() if k in TRACE_FIELDS}
context = _propagator.extract(parent) if parent is not None else None
with tracer.start_as_current_span(
name,
kind=kind,
context=context,
attributes=attrs,
record_exception=False,
set_status_on_exception=False,
) as current:
try:
yield current
except BaseException as exc:
current.set_attribute("error.type", type(exc).__name__)
current.set_status(Status(StatusCode.ERROR, "operation failed"))
raise
def log(event, *, error=False, **fields):
event = (
event
if re.fullmatch(r"[a-z][a-z0-9_.-]{0,79}", event)
else "application.event"
)
attrs = {
k: v[:256] if isinstance(v, str) else v
for k, v in fields.items()
if k in LOG_FIELDS and isinstance(v, (str, int, float, bool))
}
current = trace.get_current_span().get_span_context()
record = {
"event": event,
"timestamp": time.time(),
"severity": "ERROR" if error else "INFO",
"service.name": _state.service if _state else "messenger",
**attrs,
}
if current.is_valid:
record.update(
trace_id=f"{current.trace_id:032x}",
span_id=f"{current.span_id:016x}",
)
print(json.dumps(record, ensure_ascii=False), flush=True)
if _state is not None:
_state.logger.emit(
body=event,
attributes=attrs,
severity_text=record["severity"],
severity_number=SeverityNumber.ERROR
if error
else SeverityNumber.INFO,
)
class TelemetryMiddleware:
def __init__(self, app):
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
# Health polling does not enter business SLIs or spans.
if scope.get("path", "").startswith("/health/"):
return await self.app(scope, receive, send)
headers = {
k.decode("latin1"): v.decode("latin1")
for k, v in scope.get("headers", [])
if k.lower() in {b"traceparent", b"tracestate"}
}
status = 500
started = time.monotonic()
async def measured_send(message):
nonlocal status
if message["type"] == "http.response.start":
status = message["status"]
await send(message)
with span("HTTP", kind=SpanKind.SERVER, parent=headers) as current:
try:
await self.app(scope, receive, measured_send)
finally:
route = getattr(scope.get("route"), "path", "unmatched")
method = scope.get("method", "UNKNOWN")
if method not in {
"GET",
"POST",
"PUT",
"PATCH",
"DELETE",
"HEAD",
"OPTIONS",
}:
method = "OTHER"
current.update_name(method + " " + route)
current.set_attributes(
{
"http.route": route,
"http.request.method": method,
"http.response.status_code": status,
}
)
if status >= 500:
current.set_status(
Status(StatusCode.ERROR, "HTTP server error")
)
elapsed = time.monotonic() - started
labels = {
"route": route,
"method": method,
"status_class": str(status // 100) + "xx",
}
count("http_requests", attributes=labels)
duration("http_duration", elapsed, labels)
log(
"http.request",
request_id=scope.get("state", {}).get("request_id", ""),
method=method,
route=route,
status=status,
duration_seconds=round(elapsed, 6),
error=status >= 500,
)