440 lines
14 KiB
Python
440 lines
14 KiB
Python
"""Explicit OTLP signals; no HTTP bodies, credentials, SQL or raw URLs."""
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import threading
|
|
import time
|
|
import weakref
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from uuid import uuid4
|
|
|
|
import psutil
|
|
from opentelemetry import trace
|
|
from opentelemetry._logs import SeverityNumber
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import (
|
|
OTLPLogExporter,
|
|
)
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
|
|
OTLPMetricExporter,
|
|
)
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
|
|
OTLPSpanExporter,
|
|
)
|
|
from opentelemetry.metrics import Observation
|
|
from opentelemetry.sdk._logs import LoggerProvider
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.sdk.metrics import Histogram, MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.metrics.view import (
|
|
ExplicitBucketHistogramAggregation,
|
|
View,
|
|
)
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
from opentelemetry.trace.propagation.tracecontext import (
|
|
TraceContextTextMapPropagator,
|
|
)
|
|
|
|
_state = None
|
|
_lock = threading.Lock()
|
|
_pools = weakref.WeakSet()
|
|
_instruments = {}
|
|
_observer = {}
|
|
_propagator = TraceContextTextMapPropagator()
|
|
BUCKETS = (
|
|
0.005,
|
|
0.01,
|
|
0.025,
|
|
0.05,
|
|
0.1,
|
|
0.25,
|
|
0.5,
|
|
1,
|
|
2.5,
|
|
5,
|
|
10,
|
|
20,
|
|
60,
|
|
120,
|
|
)
|
|
LOG_FIELDS = {
|
|
"request_id",
|
|
"job_id",
|
|
"attachment_id",
|
|
"method",
|
|
"route",
|
|
"status",
|
|
"duration_seconds",
|
|
"attempt",
|
|
"version",
|
|
"fence",
|
|
"outcome",
|
|
"error_code",
|
|
"operation",
|
|
"cycle",
|
|
}
|
|
TRACE_FIELDS = {
|
|
"request_id",
|
|
"job.id",
|
|
"attachment.id",
|
|
"pipeline.version",
|
|
"job.attempt",
|
|
"db.system.name",
|
|
"operation",
|
|
"messaging.system",
|
|
"messaging.destination.name",
|
|
"http.route",
|
|
"http.request.method",
|
|
"http.response.status_code",
|
|
"error.type",
|
|
}
|
|
|
|
|
|
def enabled():
|
|
return os.getenv("OTEL_ENABLED", "0") == "1"
|
|
|
|
|
|
def register_pool(pool):
|
|
_pools.add(pool)
|
|
|
|
|
|
def observe(values):
|
|
_observer.update(values)
|
|
|
|
|
|
def _pool_stat(key):
|
|
stats = [p.get_stats() for p in list(_pools)]
|
|
if key == "used":
|
|
return sum(
|
|
max(0, s.get("pool_size", 0) - s.get("pool_available", 0))
|
|
for s in stats
|
|
)
|
|
return sum(s.get("requests_waiting", 0) for s in stats)
|
|
|
|
|
|
def configure(service, *, instance=None, with_metrics=True):
|
|
global _state
|
|
if not enabled():
|
|
return
|
|
with _lock:
|
|
if _state is not None:
|
|
return
|
|
instance = instance or uuid4().hex
|
|
resource = Resource.create(
|
|
{
|
|
"service.name": service,
|
|
"service.version": os.getenv("APP_VERSION", "development"),
|
|
"service.instance.id": instance,
|
|
"deployment.environment.name": "lab",
|
|
}
|
|
)
|
|
endpoint = os.getenv(
|
|
"OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel:4318"
|
|
).rstrip("/")
|
|
options = {"timeout": 0.75}
|
|
# Certificate options are also honored by the exporters in the TLS lab.
|
|
mp = None
|
|
if with_metrics:
|
|
reader = PeriodicExportingMetricReader(
|
|
OTLPMetricExporter(
|
|
endpoint=endpoint + "/v1/metrics", **options
|
|
),
|
|
export_interval_millis=2000,
|
|
export_timeout_millis=1000,
|
|
)
|
|
mp = MeterProvider(
|
|
resource=resource,
|
|
metric_readers=[reader],
|
|
views=[
|
|
View(
|
|
instrument_type=Histogram,
|
|
aggregation=ExplicitBucketHistogramAggregation(
|
|
BUCKETS
|
|
),
|
|
)
|
|
],
|
|
)
|
|
meter = mp.get_meter("messenger", "1.0")
|
|
for name in ["http_requests", "job_attempts"]:
|
|
_instruments[name] = meter.create_counter("messenger_" + name)
|
|
_instruments["worker_busy"] = meter.create_up_down_counter(
|
|
"messenger_worker_busy"
|
|
)
|
|
for name in ["http_duration", "job_duration", "time_to_ready"]:
|
|
_instruments[name] = meter.create_histogram(
|
|
"messenger_" + name, unit="s"
|
|
)
|
|
for key in ["used", "waiting"]:
|
|
meter.create_observable_gauge(
|
|
"messenger_db_pool_" + key,
|
|
callbacks=[lambda _, k=key: [Observation(_pool_stat(k))]],
|
|
)
|
|
process = psutil.Process()
|
|
meter.create_observable_counter(
|
|
"messenger_process_cpu",
|
|
unit="s",
|
|
callbacks=[
|
|
lambda _: [Observation(sum(process.cpu_times()[:2]))]
|
|
],
|
|
)
|
|
meter.create_observable_gauge(
|
|
"messenger_process_memory",
|
|
unit="By",
|
|
callbacks=[lambda _: [Observation(process.memory_info().rss)]],
|
|
)
|
|
|
|
def container_memory(_):
|
|
p = Path("/sys/fs/cgroup/memory.current")
|
|
return [Observation(int(p.read_text()))] if p.exists() else []
|
|
|
|
meter.create_observable_gauge(
|
|
"messenger_container_memory",
|
|
unit="By",
|
|
callbacks=[container_memory],
|
|
)
|
|
if service == "messenger-worker":
|
|
meter.create_observable_gauge(
|
|
"messenger_worker_capacity",
|
|
callbacks=[
|
|
lambda _: [
|
|
Observation(
|
|
int(os.getenv("WORKER_CONCURRENCY", "2"))
|
|
)
|
|
]
|
|
],
|
|
)
|
|
if service == "messenger-observer":
|
|
for key in ["queue_depth", "oldest_job_age"]:
|
|
meter.create_observable_gauge(
|
|
"messenger_" + key,
|
|
unit="s" if key.endswith("age") else "",
|
|
callbacks=[
|
|
lambda _, k=key: [
|
|
Observation(
|
|
_observer.get(k, 0),
|
|
{"queue": "attachments"},
|
|
)
|
|
]
|
|
],
|
|
)
|
|
for name, prefix in [
|
|
("slo_decisions", "slo_"),
|
|
("jobs_completed", "completed_"),
|
|
]:
|
|
meter.create_observable_counter(
|
|
"messenger_" + name,
|
|
callbacks=[
|
|
lambda _, p=prefix: [
|
|
Observation(
|
|
_observer.get(p + outcome, 0),
|
|
{
|
|
"outcome": outcome,
|
|
"job_type": "attachment.process.v1",
|
|
},
|
|
)
|
|
for outcome in (
|
|
["good", "bad"]
|
|
if p == "slo_"
|
|
else ["ready", "failed"]
|
|
)
|
|
]
|
|
],
|
|
)
|
|
tp = TracerProvider(
|
|
resource=resource,
|
|
sampler=ParentBased(
|
|
TraceIdRatioBased(float(os.getenv("OTEL_TRACE_RATIO", "1")))
|
|
),
|
|
)
|
|
tp.add_span_processor(
|
|
BatchSpanProcessor(
|
|
OTLPSpanExporter(endpoint=endpoint + "/v1/traces", **options),
|
|
max_queue_size=512,
|
|
max_export_batch_size=64,
|
|
schedule_delay_millis=200,
|
|
export_timeout_millis=1000,
|
|
)
|
|
)
|
|
lp = LoggerProvider(resource=resource)
|
|
lp.add_log_record_processor(
|
|
BatchLogRecordProcessor(
|
|
OTLPLogExporter(endpoint=endpoint + "/v1/logs", **options),
|
|
max_queue_size=512,
|
|
max_export_batch_size=64,
|
|
schedule_delay_millis=200,
|
|
export_timeout_millis=1000,
|
|
)
|
|
)
|
|
_state = SimpleNamespace(
|
|
service=service,
|
|
instance=instance,
|
|
traces=tp,
|
|
metrics=mp,
|
|
logs=lp,
|
|
tracer=tp.get_tracer("messenger", "1.0"),
|
|
logger=lp.get_logger("messenger", "1.0"),
|
|
)
|
|
|
|
|
|
def stop():
|
|
if _state is None:
|
|
return
|
|
for provider in [_state.traces, _state.logs, _state.metrics]:
|
|
if provider is not None:
|
|
provider.force_flush(timeout_millis=1000)
|
|
provider.shutdown()
|
|
|
|
|
|
def count(name, value=1, attributes=None):
|
|
instrument = _instruments.get(name)
|
|
if instrument is not None:
|
|
instrument.add(value, attributes or {})
|
|
|
|
|
|
def duration(name, seconds, attributes=None):
|
|
instrument = _instruments.get(name)
|
|
if instrument is not None:
|
|
instrument.record(max(0, seconds), attributes or {})
|
|
|
|
|
|
def carrier():
|
|
result = {}
|
|
_propagator.inject(result)
|
|
return result
|
|
|
|
|
|
@contextmanager
|
|
def span(name, *, kind=SpanKind.INTERNAL, parent=None, attributes=None):
|
|
tracer = (
|
|
_state.tracer if _state is not None else trace.get_tracer("messenger")
|
|
)
|
|
attrs = {k: v for k, v in (attributes or {}).items() if k in TRACE_FIELDS}
|
|
context = _propagator.extract(parent) if parent is not None else None
|
|
with tracer.start_as_current_span(
|
|
name,
|
|
kind=kind,
|
|
context=context,
|
|
attributes=attrs,
|
|
record_exception=False,
|
|
set_status_on_exception=False,
|
|
) as current:
|
|
try:
|
|
yield current
|
|
except BaseException as exc:
|
|
current.set_attribute("error.type", type(exc).__name__)
|
|
current.set_status(Status(StatusCode.ERROR, "operation failed"))
|
|
raise
|
|
|
|
|
|
def log(event, *, error=False, **fields):
|
|
event = (
|
|
event
|
|
if re.fullmatch(r"[a-z][a-z0-9_.-]{0,79}", event)
|
|
else "application.event"
|
|
)
|
|
attrs = {
|
|
k: v[:256] if isinstance(v, str) else v
|
|
for k, v in fields.items()
|
|
if k in LOG_FIELDS and isinstance(v, (str, int, float, bool))
|
|
}
|
|
current = trace.get_current_span().get_span_context()
|
|
record = {
|
|
"event": event,
|
|
"timestamp": time.time(),
|
|
"severity": "ERROR" if error else "INFO",
|
|
"service.name": _state.service if _state else "messenger",
|
|
**attrs,
|
|
}
|
|
if current.is_valid:
|
|
record.update(
|
|
trace_id=f"{current.trace_id:032x}",
|
|
span_id=f"{current.span_id:016x}",
|
|
)
|
|
print(json.dumps(record, ensure_ascii=False), flush=True)
|
|
if _state is not None:
|
|
_state.logger.emit(
|
|
body=event,
|
|
attributes=attrs,
|
|
severity_text=record["severity"],
|
|
severity_number=SeverityNumber.ERROR
|
|
if error
|
|
else SeverityNumber.INFO,
|
|
)
|
|
|
|
|
|
class TelemetryMiddleware:
|
|
def __init__(self, app):
|
|
self.app = app
|
|
|
|
async def __call__(self, scope, receive, send):
|
|
if scope["type"] != "http":
|
|
return await self.app(scope, receive, send)
|
|
# Health polling does not enter business SLIs or spans.
|
|
if scope.get("path", "").startswith("/health/"):
|
|
return await self.app(scope, receive, send)
|
|
headers = {
|
|
k.decode("latin1"): v.decode("latin1")
|
|
for k, v in scope.get("headers", [])
|
|
if k.lower() in {b"traceparent", b"tracestate"}
|
|
}
|
|
status = 500
|
|
started = time.monotonic()
|
|
|
|
async def measured_send(message):
|
|
nonlocal status
|
|
if message["type"] == "http.response.start":
|
|
status = message["status"]
|
|
await send(message)
|
|
|
|
with span("HTTP", kind=SpanKind.SERVER, parent=headers) as current:
|
|
try:
|
|
await self.app(scope, receive, measured_send)
|
|
finally:
|
|
route = getattr(scope.get("route"), "path", "unmatched")
|
|
method = scope.get("method", "UNKNOWN")
|
|
if method not in {
|
|
"GET",
|
|
"POST",
|
|
"PUT",
|
|
"PATCH",
|
|
"DELETE",
|
|
"HEAD",
|
|
"OPTIONS",
|
|
}:
|
|
method = "OTHER"
|
|
current.update_name(method + " " + route)
|
|
current.set_attributes(
|
|
{
|
|
"http.route": route,
|
|
"http.request.method": method,
|
|
"http.response.status_code": status,
|
|
}
|
|
)
|
|
if status >= 500:
|
|
current.set_status(
|
|
Status(StatusCode.ERROR, "HTTP server error")
|
|
)
|
|
elapsed = time.monotonic() - started
|
|
labels = {
|
|
"route": route,
|
|
"method": method,
|
|
"status_class": str(status // 100) + "xx",
|
|
}
|
|
count("http_requests", attributes=labels)
|
|
duration("http_duration", elapsed, labels)
|
|
log(
|
|
"http.request",
|
|
request_id=scope.get("state", {}).get("request_id", ""),
|
|
method=method,
|
|
route=route,
|
|
status=status,
|
|
duration_seconds=round(elapsed, 6),
|
|
error=status >= 500,
|
|
)
|