Files
mtusi2026lab7-fork/app/core/telemetry.py

198 lines
9.5 KiB
Python

"""Явная OTLP-инструментация: экспортируются только выбранные технические поля."""
from contextlib import contextmanager
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import socket
import time
from opentelemetry import trace
from opentelemetry.context import Context
from opentelemetry._logs import SeverityNumber
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.metrics import Observation
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.trace import SpanKind, Status, StatusCode
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
JOB_BUCKETS = (0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60)
class Telemetry:
def __init__(self, service: str, instance: str | None = None, *, enabled=None):
self.enabled = (os.getenv("OTEL_ENABLED", "false").lower() == "true"
if enabled is None else enabled)
self.service = service
self.busy = 0
if not self.enabled:
self.tracer = trace.NoOpTracerProvider().get_tracer("messenger")
return
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
resource = Resource({
"service.name": service,
"service.version": "1.6.0",
"service.instance.id": instance or socket.gethostname(),
"deployment.environment.name": os.getenv("APP_ENV", "development"),
})
self.traces = TracerProvider(resource=resource)
self.traces.add_span_processor(BatchSpanProcessor(
OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=2),
max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=1000,
))
self.tracer = self.traces.get_tracer("messenger", "1.6.0")
self.logs = LoggerProvider(resource=resource)
self.logs.add_log_record_processor(BatchLogRecordProcessor(
OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=2),
max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=1000,
))
self.logger = self.logs.get_logger("messenger", "1.6.0")
self.metrics = MeterProvider(resource=resource, metric_readers=[
PeriodicExportingMetricReader(
OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=2),
export_interval_millis=5000, export_timeout_millis=3000,
)
], views=[
View(instrument_name="messenger.http.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)),
View(instrument_name="messenger.job.time_to_ready", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
])
meter = self.meter = self.metrics.get_meter("messenger", "1.6.0")
self.http_requests = meter.create_counter("messenger.http.requests", unit="1")
self.http_duration = meter.create_histogram("messenger.http.duration", unit="s")
self.jobs = meter.create_counter("messenger.job.attempts", unit="1")
self.job_duration = meter.create_histogram("messenger.job.duration", unit="s")
self.time_to_ready = meter.create_histogram("messenger.job.time_to_ready", unit="s")
self.gauge("messenger.process.cpu", time.process_time, unit="s", counter=True)
self.gauge("messenger.process.rss", resident_bytes, unit="By")
def gauge(self, name, sample, *, unit="1", attributes=None, counter=False):
if self.enabled:
create = (self.meter.create_observable_counter if counter
else self.meter.create_observable_gauge)
create(name, callbacks=[lambda _: [Observation(sample(), attributes)]], unit=unit)
def observe_pool(self, pool):
if self.enabled:
pool.telemetry = self
self.gauge("messenger.db.pool.used", pool.checkedout)
self.gauge("messenger.db.pool.waiting", lambda: pool.waiting)
@contextmanager
def span(self, name, **kwargs):
with self.tracer.start_as_current_span(
name, record_exception=False, set_status_on_exception=False, **kwargs,
) as span:
try:
yield span
except BaseException:
span.set_status(Status(StatusCode.ERROR))
raise
def event(self, event: str, *, error=False, **fields):
if not self.enabled:
return
context = trace.get_current_span().get_span_context()
body = {
"timestamp": datetime.now(timezone.utc).isoformat(),
"severity": "ERROR" if error else "INFO",
"service_name": self.service, "event": event,
"trace_id": f"{context.trace_id:032x}", "span_id": f"{context.span_id:016x}",
**fields,
}
text = json.dumps(body, ensure_ascii=False)
# Не подключаем root logging: SDK/SQL-исключения могут содержать секреты.
self.logger.emit(body=text, severity_text=body["severity"],
severity_number=SeverityNumber.ERROR if error else SeverityNumber.INFO,
attributes={"event": event})
print(text, flush=True)
@contextmanager
def job_attempt(self):
self.busy += 1
started = time.monotonic()
result = {"outcome": "success"}
try:
yield result
except BaseException:
result["outcome"] = "error"
raise
finally:
self.busy -= 1
if self.enabled:
attrs = {"job.type": "attachment.process.v1", "outcome": result["outcome"]}
elapsed = time.monotonic() - started
self.jobs.add(1, attrs)
self.job_duration.record(elapsed, attrs)
span = trace.get_current_span()
span.set_attribute("outcome", result["outcome"])
span.set_status(Status(StatusCode.ERROR if result["outcome"] == "error" else StatusCode.OK))
self.event("job.processed", error=result["outcome"] == "error",
job_type="attachment.process.v1", duration_seconds=elapsed, **result)
def shutdown(self):
if self.enabled:
self.metrics.shutdown(timeout_millis=5000)
self.logs.shutdown()
self.traces.shutdown()
def resident_bytes():
return int(Path("/proc/self/statm").read_text().split()[1]) * os.sysconf("SC_PAGE_SIZE")
class HttpTelemetryMiddleware:
def __init__(self, app, telemetry: Telemetry):
self.app = app
self.telemetry = telemetry
async def __call__(self, scope, receive, send):
if scope["type"] != "http" or not self.telemetry.enabled:
return await self.app(scope, receive, send)
telemetry = self.telemetry
method = scope["method"]
if method not in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}:
method = "OTHER"
started = time.monotonic()
status = 500
request_id = ""
with telemetry.tracer.start_as_current_span(
"HTTP", context=Context(), kind=SpanKind.SERVER,
record_exception=False, set_status_on_exception=False,
) as span:
async def observed_send(message):
nonlocal status, request_id
if message["type"] == "http.response.start":
status = message["status"]
headers = list(message.get("headers", []))
request_id = next((v.decode() for k, v in headers if k.lower() == b"x-request-id"), "")
headers.append((b"x-trace-id", f"{span.get_span_context().trace_id:032x}".encode()))
message = {**message, "headers": headers}
await send(message)
try:
await self.app(scope, receive, observed_send)
finally:
request_id = request_id or scope.get("state", {}).get("request_id", "")
route = getattr(scope.get("route"), "path", "unmatched")
attrs = {"http.request.method": method, "http.route": route,
"http.response.status_class": f"{status // 100}xx"}
elapsed = time.monotonic() - started
telemetry.http_requests.add(1, attrs)
telemetry.http_duration.record(elapsed, attrs)
span.update_name(f"{method} {route}")
span.set_attributes({**attrs, "http.response.status_code": status})
span.set_status(Status(StatusCode.ERROR if status >= 500 else StatusCode.OK))
telemetry.event("http.request", error=status >= 500, method=method, route=route,
status_code=status, request_id=request_id, duration_seconds=elapsed)