forked from mtusi2026/mtusi2026lab7
198 lines
9.5 KiB
Python
198 lines
9.5 KiB
Python
"""Явная OTLP-инструментация: экспортируются только выбранные технические поля."""
|
|
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import time
|
|
|
|
from opentelemetry import trace
|
|
from opentelemetry.context import Context
|
|
from opentelemetry._logs import SeverityNumber
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.sdk._logs import LoggerProvider
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.metrics import Observation
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
|
|
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
|
|
JOB_BUCKETS = (0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60)
|
|
|
|
|
|
class Telemetry:
|
|
def __init__(self, service: str, instance: str | None = None, *, enabled=None):
|
|
self.enabled = (os.getenv("OTEL_ENABLED", "false").lower() == "true"
|
|
if enabled is None else enabled)
|
|
self.service = service
|
|
self.busy = 0
|
|
if not self.enabled:
|
|
self.tracer = trace.NoOpTracerProvider().get_tracer("messenger")
|
|
return
|
|
endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318").rstrip("/")
|
|
resource = Resource({
|
|
"service.name": service,
|
|
"service.version": "1.6.0",
|
|
"service.instance.id": instance or socket.gethostname(),
|
|
"deployment.environment.name": os.getenv("APP_ENV", "development"),
|
|
})
|
|
self.traces = TracerProvider(resource=resource)
|
|
self.traces.add_span_processor(BatchSpanProcessor(
|
|
OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=2),
|
|
max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=1000,
|
|
))
|
|
self.tracer = self.traces.get_tracer("messenger", "1.6.0")
|
|
self.logs = LoggerProvider(resource=resource)
|
|
self.logs.add_log_record_processor(BatchLogRecordProcessor(
|
|
OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=2),
|
|
max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=1000,
|
|
))
|
|
self.logger = self.logs.get_logger("messenger", "1.6.0")
|
|
self.metrics = MeterProvider(resource=resource, metric_readers=[
|
|
PeriodicExportingMetricReader(
|
|
OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=2),
|
|
export_interval_millis=5000, export_timeout_millis=3000,
|
|
)
|
|
], views=[
|
|
View(instrument_name="messenger.http.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)),
|
|
View(instrument_name="messenger.job.time_to_ready", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
|
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
|
])
|
|
meter = self.meter = self.metrics.get_meter("messenger", "1.6.0")
|
|
self.http_requests = meter.create_counter("messenger.http.requests", unit="1")
|
|
self.http_duration = meter.create_histogram("messenger.http.duration", unit="s")
|
|
self.jobs = meter.create_counter("messenger.job.attempts", unit="1")
|
|
self.job_duration = meter.create_histogram("messenger.job.duration", unit="s")
|
|
|
|
self.time_to_ready = meter.create_histogram("messenger.job.time_to_ready", unit="s")
|
|
self.gauge("messenger.process.cpu", time.process_time, unit="s", counter=True)
|
|
self.gauge("messenger.process.rss", resident_bytes, unit="By")
|
|
|
|
def gauge(self, name, sample, *, unit="1", attributes=None, counter=False):
|
|
if self.enabled:
|
|
create = (self.meter.create_observable_counter if counter
|
|
else self.meter.create_observable_gauge)
|
|
create(name, callbacks=[lambda _: [Observation(sample(), attributes)]], unit=unit)
|
|
|
|
def observe_pool(self, pool):
|
|
if self.enabled:
|
|
pool.telemetry = self
|
|
self.gauge("messenger.db.pool.used", pool.checkedout)
|
|
self.gauge("messenger.db.pool.waiting", lambda: pool.waiting)
|
|
|
|
@contextmanager
|
|
def span(self, name, **kwargs):
|
|
with self.tracer.start_as_current_span(
|
|
name, record_exception=False, set_status_on_exception=False, **kwargs,
|
|
) as span:
|
|
try:
|
|
yield span
|
|
except BaseException:
|
|
span.set_status(Status(StatusCode.ERROR))
|
|
raise
|
|
|
|
def event(self, event: str, *, error=False, **fields):
|
|
if not self.enabled:
|
|
return
|
|
context = trace.get_current_span().get_span_context()
|
|
body = {
|
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
"severity": "ERROR" if error else "INFO",
|
|
"service_name": self.service, "event": event,
|
|
"trace_id": f"{context.trace_id:032x}", "span_id": f"{context.span_id:016x}",
|
|
**fields,
|
|
}
|
|
text = json.dumps(body, ensure_ascii=False)
|
|
# Не подключаем root logging: SDK/SQL-исключения могут содержать секреты.
|
|
self.logger.emit(body=text, severity_text=body["severity"],
|
|
severity_number=SeverityNumber.ERROR if error else SeverityNumber.INFO,
|
|
attributes={"event": event})
|
|
print(text, flush=True)
|
|
|
|
@contextmanager
|
|
def job_attempt(self):
|
|
self.busy += 1
|
|
started = time.monotonic()
|
|
result = {"outcome": "success"}
|
|
try:
|
|
yield result
|
|
except BaseException:
|
|
result["outcome"] = "error"
|
|
raise
|
|
finally:
|
|
self.busy -= 1
|
|
if self.enabled:
|
|
attrs = {"job.type": "attachment.process.v1", "outcome": result["outcome"]}
|
|
elapsed = time.monotonic() - started
|
|
self.jobs.add(1, attrs)
|
|
self.job_duration.record(elapsed, attrs)
|
|
span = trace.get_current_span()
|
|
span.set_attribute("outcome", result["outcome"])
|
|
span.set_status(Status(StatusCode.ERROR if result["outcome"] == "error" else StatusCode.OK))
|
|
self.event("job.processed", error=result["outcome"] == "error",
|
|
job_type="attachment.process.v1", duration_seconds=elapsed, **result)
|
|
|
|
def shutdown(self):
|
|
if self.enabled:
|
|
self.metrics.shutdown(timeout_millis=5000)
|
|
self.logs.shutdown()
|
|
self.traces.shutdown()
|
|
|
|
|
|
def resident_bytes():
|
|
return int(Path("/proc/self/statm").read_text().split()[1]) * os.sysconf("SC_PAGE_SIZE")
|
|
|
|
|
|
class HttpTelemetryMiddleware:
|
|
def __init__(self, app, telemetry: Telemetry):
|
|
self.app = app
|
|
self.telemetry = telemetry
|
|
|
|
async def __call__(self, scope, receive, send):
|
|
if scope["type"] != "http" or not self.telemetry.enabled:
|
|
return await self.app(scope, receive, send)
|
|
telemetry = self.telemetry
|
|
method = scope["method"]
|
|
if method not in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}:
|
|
method = "OTHER"
|
|
started = time.monotonic()
|
|
status = 500
|
|
request_id = ""
|
|
with telemetry.tracer.start_as_current_span(
|
|
"HTTP", context=Context(), kind=SpanKind.SERVER,
|
|
record_exception=False, set_status_on_exception=False,
|
|
) as span:
|
|
async def observed_send(message):
|
|
nonlocal status, request_id
|
|
if message["type"] == "http.response.start":
|
|
status = message["status"]
|
|
headers = list(message.get("headers", []))
|
|
request_id = next((v.decode() for k, v in headers if k.lower() == b"x-request-id"), "")
|
|
headers.append((b"x-trace-id", f"{span.get_span_context().trace_id:032x}".encode()))
|
|
message = {**message, "headers": headers}
|
|
await send(message)
|
|
try:
|
|
await self.app(scope, receive, observed_send)
|
|
finally:
|
|
request_id = request_id or scope.get("state", {}).get("request_id", "")
|
|
route = getattr(scope.get("route"), "path", "unmatched")
|
|
attrs = {"http.request.method": method, "http.route": route,
|
|
"http.response.status_class": f"{status // 100}xx"}
|
|
elapsed = time.monotonic() - started
|
|
telemetry.http_requests.add(1, attrs)
|
|
telemetry.http_duration.record(elapsed, attrs)
|
|
span.update_name(f"{method} {route}")
|
|
span.set_attributes({**attrs, "http.response.status_code": status})
|
|
span.set_status(Status(StatusCode.ERROR if status >= 500 else StatusCode.OK))
|
|
telemetry.event("http.request", error=status >= 500, method=method, route=route,
|
|
status_code=status, request_id=request_id, duration_seconds=elapsed)
|