forked from mtusi2026/mtusi2026lab6
API и worker отдают метрики, логи и трассы. Правила SLO записаны, прогон инцидентов ещё впереди.
431 lines
14 KiB
Python
431 lines
14 KiB
Python
"""OTLP telemetry for the API and the worker.
|
|
|
|
Metric labels stay low-cardinality: method, route template, status class,
|
|
job type, outcome, queue name. User, chat, message and attachment ids are
|
|
never metric labels. Trace and log records may carry a job id so one upload
|
|
can be found; passwords, tokens, message text and image bytes are not exported.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import re
|
|
import sys
|
|
import time
|
|
from contextlib import contextmanager
|
|
|
|
import psutil
|
|
from opentelemetry import metrics, propagate, trace
|
|
from opentelemetry._logs import set_logger_provider
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
|
|
from app import config
|
|
|
|
logger = logging.getLogger("messenger.telemetry")
|
|
|
|
_HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
|
|
_JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60)
|
|
_READY_BUCKETS = (0.5, 1, 2, 5, 10, 30, 60, 120, 300)
|
|
_JOB_TYPE = "attachment.process.v1"
|
|
_QUEUE = "attachment.jobs"
|
|
|
|
_ROUTES = (
|
|
(re.compile(r"^/api/v1/chats/[^/]+/messages$"), "/api/v1/chats/{chat_id}/messages"),
|
|
(re.compile(r"^/api/v1/chats/[^/]+/attachments$"), "/api/v1/chats/{chat_id}/attachments"),
|
|
(re.compile(r"^/api/v1/chats/[^/]+/uploads$"), "/api/v1/chats/{chat_id}/uploads"),
|
|
(re.compile(r"^/api/v1/chats/[^/]+$"), "/api/v1/chats/{chat_id}"),
|
|
(re.compile(r"^/api/v1/uploads/[^/]+/complete$"), "/api/v1/uploads/{upload_id}/complete"),
|
|
(re.compile(r"^/api/v1/attachments/[^/]+/content$"), "/api/v1/attachments/{attachment_id}/content"),
|
|
(re.compile(r"^/api/v1/attachments/[^/]+$"), "/api/v1/attachments/{attachment_id}"),
|
|
(re.compile(r"^/api/v1/_lab/attachments/[^/]+/jobs$"), "/api/v1/_lab/attachments/{attachment_id}/jobs"),
|
|
(re.compile(r"^/api/v1/_lab/attachments/[^/]+/pipelines$"), "/api/v1/_lab/attachments/{attachment_id}/pipelines"),
|
|
(re.compile(r"^/api/v1/_lab/attachments/[^/]+/replay$"), "/api/v1/_lab/attachments/{attachment_id}/replay"),
|
|
(re.compile(r"^/api/v1/_lab/jobs/[^/]+/redeliver$"), "/api/v1/_lab/jobs/{job_id}/redeliver"),
|
|
)
|
|
_SECRET = re.compile(r"(?i)(authorization|password|token|secret|session)([\"']?\s*[:=]\s*[\"']?)(\S+)")
|
|
|
|
_ready = False
|
|
_tracer = None
|
|
_http_requests = None
|
|
_http_duration = None
|
|
_jobs = None
|
|
_job_duration = None
|
|
_time_to_ready = None
|
|
_log_handler = None
|
|
_providers = []
|
|
_process = psutil.Process()
|
|
_process.cpu_percent(interval=None)
|
|
_inflight = 0
|
|
_queue_depth = 0
|
|
_queue_oldest = 0.0
|
|
_probe_task: asyncio.Task | None = None
|
|
|
|
_KIND = {
|
|
"internal": SpanKind.INTERNAL,
|
|
"server": SpanKind.SERVER,
|
|
"client": SpanKind.CLIENT,
|
|
"producer": SpanKind.PRODUCER,
|
|
"consumer": SpanKind.CONSUMER,
|
|
}
|
|
|
|
|
|
class JsonFormatter(logging.Formatter):
|
|
def __init__(self, service_name: str) -> None:
|
|
super().__init__()
|
|
self.service_name = service_name
|
|
|
|
def format(self, record: logging.LogRecord) -> str:
|
|
span = trace.get_current_span()
|
|
ctx = span.get_span_context()
|
|
payload = {
|
|
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z", time.gmtime(record.created)),
|
|
"severity": record.levelname,
|
|
"service.name": self.service_name,
|
|
"event": _SECRET.sub(r"\1\2[redacted]", record.getMessage()),
|
|
"logger": record.name,
|
|
}
|
|
if ctx.is_valid:
|
|
payload["trace_id"] = format(ctx.trace_id, "032x")
|
|
payload["span_id"] = format(ctx.span_id, "016x")
|
|
if record.exc_info:
|
|
payload["exception"] = _SECRET.sub(r"\1\2[redacted]", self.formatException(record.exc_info))
|
|
return json.dumps(payload, ensure_ascii=False)
|
|
|
|
|
|
def route_template(path: str) -> str:
|
|
for pattern, template in _ROUTES:
|
|
if pattern.match(path):
|
|
return template
|
|
return path
|
|
|
|
|
|
def observe_http(path: str) -> bool:
|
|
return path not in ("/health/live", "/health/ready", "/docs", "/openapi.json")
|
|
|
|
|
|
def status_class(code: int) -> str:
|
|
return "%sxx" % (int(code) // 100)
|
|
|
|
|
|
def configure_logging() -> None:
|
|
setup()
|
|
root = logging.getLogger()
|
|
root.handlers.clear()
|
|
root.setLevel(getattr(logging, config.LOG_LEVEL.upper(), logging.INFO))
|
|
stream = logging.StreamHandler(sys.stdout)
|
|
stream.setFormatter(JsonFormatter(config.OTEL_SERVICE_NAME))
|
|
root.addHandler(stream)
|
|
if _log_handler is not None:
|
|
root.addHandler(_log_handler)
|
|
logging.getLogger("opentelemetry").setLevel(logging.ERROR)
|
|
|
|
|
|
def setup() -> None:
|
|
global _ready, _tracer, _http_requests, _http_duration, _jobs, _job_duration, _time_to_ready, _log_handler
|
|
if _ready:
|
|
return
|
|
_ready = True
|
|
if not config.TELEMETRY_ENABLED:
|
|
logger.info("telemetry disabled")
|
|
return
|
|
resource = Resource.create(
|
|
{
|
|
"service.name": config.OTEL_SERVICE_NAME,
|
|
"service.version": config.SERVICE_VERSION,
|
|
"service.instance.id": config.INSTANCE_ID,
|
|
"deployment.environment.name": "lab",
|
|
}
|
|
)
|
|
ratio = min(1.0, max(0.0, config.TRACE_SAMPLE_RATIO))
|
|
tracer_provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(ratio)))
|
|
tracer_provider.add_span_processor(
|
|
BatchSpanProcessor(
|
|
OTLPSpanExporter(endpoint=_signal_endpoint("traces"), timeout=5),
|
|
max_queue_size=2048,
|
|
schedule_delay_millis=5000,
|
|
max_export_batch_size=512,
|
|
export_timeout_millis=5000,
|
|
)
|
|
)
|
|
trace.set_tracer_provider(tracer_provider)
|
|
_tracer = trace.get_tracer("messenger", config.SERVICE_VERSION)
|
|
reader = PeriodicExportingMetricReader(
|
|
OTLPMetricExporter(endpoint=_signal_endpoint("metrics"), timeout=5),
|
|
export_interval_millis=10000,
|
|
export_timeout_millis=5000,
|
|
)
|
|
meter_provider = MeterProvider(
|
|
resource=resource,
|
|
metric_readers=[reader],
|
|
views=[
|
|
View(
|
|
instrument_name="messenger.http.request.duration",
|
|
aggregation=ExplicitBucketHistogramAggregation(_HTTP_BUCKETS),
|
|
),
|
|
View(
|
|
instrument_name="messenger.job.duration",
|
|
aggregation=ExplicitBucketHistogramAggregation(_JOB_BUCKETS),
|
|
),
|
|
View(
|
|
instrument_name="messenger.time_to_ready",
|
|
aggregation=ExplicitBucketHistogramAggregation(_READY_BUCKETS),
|
|
),
|
|
],
|
|
)
|
|
metrics.set_meter_provider(meter_provider)
|
|
meter = metrics.get_meter("messenger", config.SERVICE_VERSION)
|
|
_http_requests = meter.create_counter("messenger.http.requests", unit="1")
|
|
_http_duration = meter.create_histogram("messenger.http.request.duration", unit="s")
|
|
_jobs = meter.create_counter("messenger.jobs", unit="1")
|
|
_job_duration = meter.create_histogram("messenger.job.duration", unit="s")
|
|
_time_to_ready = meter.create_histogram("messenger.time_to_ready", unit="s")
|
|
meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used])
|
|
meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting])
|
|
meter.create_observable_gauge("process.cpu.percent", callbacks=[_cpu])
|
|
meter.create_observable_gauge("process.memory.rss", unit="By", callbacks=[_rss])
|
|
if config.OTEL_SERVICE_NAME == "messenger-api":
|
|
meter.create_observable_gauge("messenger.queue.depth", callbacks=[_depth])
|
|
meter.create_observable_gauge("messenger.queue.oldest_age", unit="s", callbacks=[_oldest])
|
|
if config.OTEL_SERVICE_NAME == "messenger-worker":
|
|
meter.create_observable_gauge("messenger.worker.busy", callbacks=[_busy])
|
|
meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_capacity])
|
|
logger_provider = LoggerProvider(resource=resource)
|
|
logger_provider.add_log_record_processor(
|
|
BatchLogRecordProcessor(
|
|
OTLPLogExporter(endpoint=_signal_endpoint("logs"), timeout=5),
|
|
max_queue_size=2048,
|
|
schedule_delay_millis=5000,
|
|
max_export_batch_size=512,
|
|
export_timeout_millis=5000,
|
|
)
|
|
)
|
|
set_logger_provider(logger_provider)
|
|
_log_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
|
_providers.extend([tracer_provider, meter_provider, logger_provider])
|
|
|
|
|
|
def shutdown() -> None:
|
|
for provider in _providers:
|
|
try:
|
|
provider.shutdown()
|
|
except Exception:
|
|
logger.warning("telemetry shutdown failed")
|
|
_providers.clear()
|
|
|
|
|
|
def _signal_endpoint(signal: str) -> str:
|
|
base = config.OTEL_EXPORTER_OTLP_ENDPOINT.rstrip("/")
|
|
return "%s/v1/%s" % (base, signal)
|
|
|
|
|
|
def current_carrier() -> dict:
|
|
if _tracer is None:
|
|
return {}
|
|
carrier: dict = {}
|
|
propagate.inject(carrier)
|
|
return carrier
|
|
|
|
|
|
def remote_context(traceparent: str | None, tracestate: str | None = None):
|
|
carrier = {}
|
|
if traceparent:
|
|
carrier["traceparent"] = traceparent
|
|
if tracestate:
|
|
carrier["tracestate"] = tracestate
|
|
if not carrier:
|
|
return None
|
|
return propagate.extract(carrier)
|
|
|
|
|
|
def carrier_from_headers(headers) -> dict:
|
|
carrier = {}
|
|
if not headers:
|
|
return carrier
|
|
for key in ("traceparent", "tracestate"):
|
|
value = headers.get(key)
|
|
if value is None:
|
|
continue
|
|
if isinstance(value, bytes):
|
|
value = value.decode("latin1")
|
|
carrier[key] = str(value)
|
|
return carrier
|
|
|
|
|
|
@contextmanager
|
|
def span(name: str, context=None, kind: str = "internal", attributes: dict | None = None):
|
|
if _tracer is None:
|
|
yield None
|
|
return
|
|
kwargs = {"kind": _KIND.get(kind, SpanKind.INTERNAL)}
|
|
if context is not None:
|
|
kwargs["context"] = context
|
|
with _tracer.start_as_current_span(name, **kwargs) as current:
|
|
if attributes:
|
|
for key, value in attributes.items():
|
|
if value is not None:
|
|
current.set_attribute(key, value)
|
|
yield current
|
|
|
|
|
|
def finish_http_span(current, status_code: int) -> None:
|
|
if current is None:
|
|
return
|
|
current.set_attribute("http.response.status_code", int(status_code))
|
|
if status_code >= 500:
|
|
current.set_status(Status(StatusCode.ERROR))
|
|
else:
|
|
current.set_status(Status(StatusCode.OK))
|
|
|
|
|
|
def record_http(method: str, route: str, status_code: int, duration_seconds: float) -> None:
|
|
if _http_requests is None:
|
|
return
|
|
attributes = {
|
|
"method": method,
|
|
"route": route,
|
|
"status_class": status_class(status_code),
|
|
}
|
|
_http_requests.add(1, attributes)
|
|
_http_duration.record(max(0.0, duration_seconds), attributes)
|
|
|
|
|
|
def record_job(outcome: str, duration_seconds: float) -> None:
|
|
if _jobs is None:
|
|
return
|
|
attributes = {"job_type": _JOB_TYPE, "outcome": outcome}
|
|
_jobs.add(1, attributes)
|
|
_job_duration.record(max(0.0, duration_seconds), attributes)
|
|
|
|
|
|
def record_time_to_ready(duration_seconds: float) -> None:
|
|
if _time_to_ready is None:
|
|
return
|
|
_time_to_ready.record(max(0.0, duration_seconds), {"job_type": _JOB_TYPE})
|
|
|
|
|
|
def worker_enter() -> None:
|
|
global _inflight
|
|
_inflight += 1
|
|
|
|
|
|
def worker_leave() -> None:
|
|
global _inflight
|
|
_inflight = max(0, _inflight - 1)
|
|
|
|
|
|
def start_probes() -> None:
|
|
global _probe_task
|
|
if _tracer is None or config.OTEL_SERVICE_NAME != "messenger-api":
|
|
return
|
|
if _probe_task is None or _probe_task.done():
|
|
_probe_task = asyncio.create_task(_probe_loop())
|
|
|
|
|
|
async def stop_probes() -> None:
|
|
global _probe_task
|
|
task = _probe_task
|
|
_probe_task = None
|
|
if task is None:
|
|
return
|
|
task.cancel()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
|
|
|
|
async def _probe_loop() -> None:
|
|
from app import db
|
|
|
|
global _queue_depth, _queue_oldest
|
|
while True:
|
|
try:
|
|
async with db.require().acquire() as conn:
|
|
row = await conn.fetchrow(
|
|
"""
|
|
SELECT
|
|
count(*) FILTER (WHERE status IN ('queued', 'processing')) AS depth,
|
|
COALESCE(
|
|
EXTRACT(EPOCH FROM (now() - min(created_at) FILTER (WHERE status = 'queued'))),
|
|
0
|
|
) AS oldest
|
|
FROM attachment_jobs
|
|
"""
|
|
)
|
|
_queue_depth = int(row["depth"] or 0)
|
|
_queue_oldest = float(row["oldest"] or 0)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception:
|
|
logger.warning("queue probe failed")
|
|
await asyncio.sleep(5)
|
|
|
|
|
|
def _observation(value, attributes=None):
|
|
from opentelemetry.metrics import Observation
|
|
|
|
return [Observation(value, attributes or {})]
|
|
|
|
|
|
def _pool_used(_options):
|
|
from app import db
|
|
|
|
pool = db.pool
|
|
if pool is None:
|
|
return _observation(0)
|
|
return _observation(max(0, pool.get_size() - pool.get_idle_size()))
|
|
|
|
|
|
def _pool_waiting(_options):
|
|
from app import db
|
|
|
|
pool = db.pool
|
|
if pool is None:
|
|
return _observation(0)
|
|
waiters = getattr(pool, "_waiters", None)
|
|
if waiters is not None:
|
|
return _observation(len(waiters))
|
|
queue = getattr(pool, "_queue", None)
|
|
getters = getattr(queue, "_getters", None)
|
|
if getters is not None:
|
|
return _observation(len(getters))
|
|
return _observation(0)
|
|
|
|
|
|
def _depth(_options):
|
|
return _observation(_queue_depth, {"queue": _QUEUE})
|
|
|
|
|
|
def _oldest(_options):
|
|
return _observation(_queue_oldest, {"queue": _QUEUE})
|
|
|
|
|
|
def _busy(_options):
|
|
return _observation(_inflight)
|
|
|
|
|
|
def _capacity(_options):
|
|
return _observation(config.WORKER_CONCURRENCY)
|
|
|
|
|
|
def _cpu(_options):
|
|
return _observation(_process.cpu_percent(interval=None))
|
|
|
|
|
|
def _rss(_options):
|
|
return _observation(_process.memory_info().rss)
|