forked from mtusi2026/mtusi2026lab6
451 lines
22 KiB
Python
451 lines
22 KiB
Python
"""Наблюдаемость (lab6): OpenTelemetry SDK для api, worker, dispatcher, janitor и ops.
|
||
|
||
Все три сигнала уходят по OTLP/HTTP (protobuf) в OTel Collector (OTEL_EXPORTER_OTLP_ENDPOINT):
|
||
traces -> Collector -> Tempo
|
||
metrics -> Collector -> Prometheus (встроенный OTLP receiver)
|
||
logs -> Collector -> Loki (OTLP endpoint); те же записи пишутся в stderr одной JSON-строкой
|
||
(docker compose logs работает и без стека наблюдаемости).
|
||
|
||
Ресурс (одинаковый у всех сигналов процесса): service.name (messenger-api / messenger-worker /
|
||
messenger-dispatcher / messenger-janitor), service.version (APP_VERSION), service.instance.id
|
||
(hostname контейнера = X-Instance-Id), deployment.environment.name=lab (OTEL_RESOURCE_ATTRIBUTES).
|
||
|
||
Память при недоступном Collector или хранилище ограничена (экспорт идёт в фоновых потоках SDK,
|
||
запрос пользователя его не ждёт):
|
||
- спаны: BatchSpanProcessor, очередь OTEL_BSP_MAX_QUEUE_SIZE (2048); лишние отбрасываются и считаются
|
||
в otel.sdk.processor.span.processed{error.type="queue_full"} (OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED);
|
||
- логи: BatchLogRecordProcessor, очередь OTEL_BLRP_MAX_QUEUE_SIZE (2048), так же;
|
||
- метрики: агрегаты фиксированного размера (метки низкой кардинальности), экспорт раз в
|
||
OTEL_METRIC_EXPORT_INTERVAL; неудачный экспорт не накапливается, cumulative-счётчики догонят;
|
||
- каждый экспорт ограничен OTEL_EXPORTER_OTLP_TIMEOUT (5 с).
|
||
|
||
Контекст между процессами — только W3C Trace Context (traceparent/tracestate); baggage не
|
||
распространяется, чтобы через заголовки не утекали произвольные данные.
|
||
|
||
Чего НЕТ в телеметрии: паролей, Authorization, токенов сессий и их хешей, текстов сообщений, имён
|
||
файлов, байтов изображений, ключей и подписей S3. Метки метрик — только низкокардинальные: метод,
|
||
шаблон маршрута, класс/код статуса, тип и исход задания, имя очереди, состояние соединения.
|
||
В спанах и логах есть технические ID синтетических данных (job_id, attachment_id, request_id):
|
||
без них нельзя пройти от загрузки к результату worker; ID пользователя/чата туда не пишутся.
|
||
"""
|
||
import json
|
||
import logging
|
||
import os
|
||
import sys
|
||
import threading
|
||
import time
|
||
from contextlib import contextmanager
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
from opentelemetry import context as otel_context
|
||
from opentelemetry import metrics, propagate, trace
|
||
from opentelemetry.trace import Link, SpanKind, Status, StatusCode
|
||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||
|
||
import settings
|
||
|
||
SERVICE = "messenger"
|
||
VERSION = settings.APP_VERSION
|
||
_propagator = TraceContextTextMapPropagator()
|
||
propagate.set_global_textmap(_propagator)
|
||
|
||
tracer = trace.get_tracer("messenger", VERSION)
|
||
meter = metrics.get_meter("messenger", VERSION)
|
||
|
||
_state = {"configured": False, "providers": []}
|
||
|
||
# Границы buckets (секунды) под ожидаемые задержки: p50/p95/p99 считаются из buckets в Prometheus.
|
||
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10)
|
||
JOB_BUCKETS = (0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120)
|
||
READY_BUCKETS = (0.5, 1, 2, 5, 10, 15, 30, 60, 120, 300, 600)
|
||
POOL_WAIT_BUCKETS = (0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5)
|
||
|
||
|
||
def enabled() -> bool:
|
||
return os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() != "true"
|
||
|
||
|
||
# ---------- логи: JSON в stderr + OTLP ----------
|
||
|
||
_RESERVED = set(vars(logging.LogRecord("", 0, "", 0, "", (), None))) | {"message", "asctime", "taskName"}
|
||
# Ключи, которые никогда не уходят ни в stderr, ни в OTLP (защита от случайного extra={...}).
|
||
_FORBIDDEN_KEYS = {"password", "authorization", "token", "secret", "text", "body", "filename"}
|
||
|
||
|
||
def _extra(record: logging.LogRecord) -> dict:
|
||
out = {}
|
||
for key, value in vars(record).items():
|
||
if key in _RESERVED or key.startswith("_") or key.lower() in _FORBIDDEN_KEYS:
|
||
continue
|
||
if isinstance(value, (str, bool, int, float)):
|
||
out[key] = value
|
||
elif value is not None:
|
||
out[key] = str(value)
|
||
return out
|
||
|
||
|
||
class _EventFilter(logging.Filter):
|
||
"""У каждой записи есть поле event (контракт): явное extra={"event": ...} или имя логгера."""
|
||
|
||
def filter(self, record):
|
||
if not getattr(record, "event", None):
|
||
record.event = record.name
|
||
return True
|
||
|
||
|
||
class _RateLimit(logging.Filter):
|
||
"""Ошибки экспорта самого SDK (Collector недоступен) — не чаще раза в 30 с на источник."""
|
||
|
||
def __init__(self, seconds=30.0):
|
||
super().__init__()
|
||
self.seconds = seconds
|
||
self.last = {}
|
||
self.lock = threading.Lock()
|
||
|
||
def filter(self, record):
|
||
if not record.name.startswith("opentelemetry"):
|
||
return True
|
||
key = (record.name, record.levelno)
|
||
now = time.monotonic()
|
||
with self.lock:
|
||
if now - self.last.get(key, -1e9) < self.seconds:
|
||
return False
|
||
self.last[key] = now
|
||
return True
|
||
|
||
|
||
class JsonFormatter(logging.Formatter):
|
||
"""Одна JSON-строка: ts, severity, service.name, event, message, trace_id/span_id (если есть спан)."""
|
||
|
||
def __init__(self, service_name: str):
|
||
super().__init__()
|
||
self.service_name = service_name
|
||
|
||
def format(self, record):
|
||
data = {
|
||
"ts": datetime.fromtimestamp(record.created, timezone.utc).isoformat(timespec="milliseconds"),
|
||
"severity": record.levelname,
|
||
"service.name": self.service_name,
|
||
"event": getattr(record, "event", record.name),
|
||
"logger": record.name,
|
||
"message": record.getMessage(),
|
||
}
|
||
span = trace.get_current_span().get_span_context()
|
||
if span.is_valid:
|
||
data["trace_id"] = format(span.trace_id, "032x")
|
||
data["span_id"] = format(span.span_id, "016x")
|
||
data.update({k: v for k, v in _extra(record).items() if k != "event"})
|
||
if record.exc_info and record.exc_info[0] is not None:
|
||
data["exception.type"] = record.exc_info[0].__name__
|
||
data["exception.stacktrace"] = self.formatException(record.exc_info)
|
||
return json.dumps(data, ensure_ascii=False, default=str)
|
||
|
||
|
||
def _severity_map():
|
||
from opentelemetry._logs import SeverityNumber
|
||
return {"DEBUG": SeverityNumber.DEBUG, "INFO": SeverityNumber.INFO, "WARNING": SeverityNumber.WARN,
|
||
"ERROR": SeverityNumber.ERROR, "CRITICAL": SeverityNumber.FATAL}
|
||
|
||
|
||
_SEVERITY = _severity_map()
|
||
|
||
|
||
class OtlpLogHandler(logging.Handler):
|
||
"""logging -> OpenTelemetry Logs API. trace_id/span_id берутся из текущего контекста автоматически."""
|
||
|
||
def __init__(self, logger_provider):
|
||
super().__init__(level=logging.INFO)
|
||
self.provider = logger_provider
|
||
self.loggers = {}
|
||
|
||
def emit(self, record):
|
||
if record.name.startswith("opentelemetry"):
|
||
return # ошибки экспорта не отправляем тем же экспортом (иначе петля при недоступном Collector)
|
||
try:
|
||
from opentelemetry._logs import LogRecord, SeverityNumber
|
||
|
||
logger = self.loggers.get(record.name)
|
||
if logger is None:
|
||
logger = self.loggers[record.name] = self.provider.get_logger(record.name)
|
||
attributes = _extra(record)
|
||
attributes["code.function.name"] = record.funcName
|
||
attributes["code.line.number"] = record.lineno
|
||
if record.exc_info and record.exc_info[0] is not None:
|
||
attributes["exception.type"] = record.exc_info[0].__name__
|
||
attributes["exception.stacktrace"] = logging.Formatter().formatException(record.exc_info)
|
||
level = {"WARNING": "WARN", "CRITICAL": "FATAL"}.get(record.levelname, record.levelname)
|
||
logger.emit(LogRecord(
|
||
timestamp=int(record.created * 1e9),
|
||
severity_text=level,
|
||
severity_number=_SEVERITY.get(record.levelname, SeverityNumber.INFO),
|
||
body=record.getMessage(),
|
||
attributes=attributes,
|
||
event_name=attributes.get("event"),
|
||
))
|
||
except Exception: # noqa: BLE001 — сбой телеметрии не должен ломать логирование и запрос
|
||
self.handleError(record)
|
||
|
||
|
||
def _setup_logging(service_name: str, logger_provider=None) -> None:
|
||
root = logging.getLogger()
|
||
for handler in list(root.handlers):
|
||
root.removeHandler(handler)
|
||
root.setLevel(logging.INFO)
|
||
# stderr, как logging.basicConfig в lab5: stdout CLI (janitor --once, ops.py) остаётся чистым JSON-отчётом.
|
||
console = logging.StreamHandler(sys.stderr)
|
||
console.setFormatter(JsonFormatter(service_name))
|
||
console.addFilter(_EventFilter())
|
||
console.addFilter(_RateLimit())
|
||
root.addHandler(console)
|
||
if logger_provider is not None:
|
||
otlp = OtlpLogHandler(logger_provider)
|
||
otlp.addFilter(_EventFilter())
|
||
root.addHandler(otlp)
|
||
logging.getLogger("pika").setLevel(logging.WARNING)
|
||
logging.getLogger("botocore").setLevel(logging.WARNING)
|
||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||
logging.getLogger("opentelemetry").setLevel(logging.WARNING)
|
||
|
||
|
||
# ---------- сэмплирование ----------
|
||
|
||
def _sampler():
|
||
"""ParentBased(AlwaysOn), но клиентский спан БЕЗ родителя не записывается.
|
||
|
||
Такие спаны — фоновые опросы (dispatcher раз в секунду читает outbox, продление аренды, janitor):
|
||
отдельная трасса на каждый SELECT только забивала бы Tempo. Всё, что относится к запросу или
|
||
заданию, имеет родителя (server/consumer/producer span) и сохраняется полностью.
|
||
"""
|
||
from opentelemetry.sdk.trace.sampling import (ALWAYS_ON, Decision, ParentBased, Sampler,
|
||
SamplingResult)
|
||
|
||
class DropOrphanClientSpans(Sampler):
|
||
def __init__(self):
|
||
self.inner = ParentBased(ALWAYS_ON)
|
||
|
||
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
|
||
trace_state=None):
|
||
parent = trace.get_current_span(parent_context).get_span_context()
|
||
if not parent.is_valid and kind == SpanKind.CLIENT:
|
||
return SamplingResult(Decision.DROP)
|
||
return self.inner.should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state)
|
||
|
||
def get_description(self):
|
||
return "ParentBased(AlwaysOn) without orphan client spans"
|
||
|
||
return DropOrphanClientSpans()
|
||
|
||
|
||
# ---------- process / container ----------
|
||
|
||
def _read(path: str) -> str | None:
|
||
try:
|
||
return Path(path).read_text(encoding="ascii").strip()
|
||
except OSError:
|
||
return None
|
||
|
||
|
||
def _register_process_metrics() -> None:
|
||
from opentelemetry.metrics import CallbackOptions, Observation
|
||
|
||
page = os.sysconf("SC_PAGE_SIZE")
|
||
|
||
def cpu(_options: CallbackOptions):
|
||
times = os.times()
|
||
return [Observation(times.user, {"cpu.mode": "user"}), Observation(times.system, {"cpu.mode": "system"})]
|
||
|
||
def memory(_options: CallbackOptions):
|
||
statm = _read("/proc/self/statm")
|
||
return [Observation(int(statm.split()[1]) * page)] if statm else []
|
||
|
||
meter.create_observable_counter("process.cpu.time", [cpu], unit="s", description="CPU time of the process")
|
||
meter.create_observable_up_down_counter("process.memory.usage", [memory], unit="By",
|
||
description="Resident set size of the process")
|
||
|
||
# cgroup v2 (Docker Desktop, современный Linux): контейнер целиком, включая дочерние процессы
|
||
# (у worker это imaging.py). Вне контейнера/на cgroup v1 этих метрик просто нет.
|
||
if _read("/sys/fs/cgroup/memory.current") is None:
|
||
return
|
||
|
||
def container_memory(_options: CallbackOptions):
|
||
value = _read("/sys/fs/cgroup/memory.current")
|
||
return [Observation(int(value))] if value else []
|
||
|
||
def container_memory_limit(_options: CallbackOptions):
|
||
value = _read("/sys/fs/cgroup/memory.max")
|
||
return [Observation(int(value))] if value and value != "max" else []
|
||
|
||
def container_cpu(_options: CallbackOptions):
|
||
stat = _read("/sys/fs/cgroup/cpu.stat") or ""
|
||
for line in stat.splitlines():
|
||
name, _, value = line.partition(" ")
|
||
if name == "usage_usec":
|
||
return [Observation(int(value) / 1e6)]
|
||
return []
|
||
|
||
def container_cpu_limit(_options: CallbackOptions):
|
||
value = (_read("/sys/fs/cgroup/cpu.max") or "").split()
|
||
if len(value) == 2 and value[0] != "max":
|
||
return [Observation(int(value[0]) / int(value[1]))]
|
||
return []
|
||
|
||
meter.create_observable_up_down_counter("container.memory.usage", [container_memory], unit="By",
|
||
description="Memory of the container cgroup")
|
||
meter.create_observable_up_down_counter("container.memory.limit", [container_memory_limit], unit="By",
|
||
description="Memory limit of the container cgroup")
|
||
meter.create_observable_counter("container.cpu.time", [container_cpu], unit="s",
|
||
description="CPU time of the container cgroup")
|
||
meter.create_observable_up_down_counter("container.cpu.limit", [container_cpu_limit], unit="{cpu}",
|
||
description="CPU limit of the container (cores)")
|
||
|
||
|
||
# ---------- настройка ----------
|
||
|
||
def setup(service_name: str, *, instrument_db: bool = True) -> None:
|
||
"""Вызывается один раз при старте процесса, ДО открытия соединений с БД."""
|
||
if _state["configured"]:
|
||
return
|
||
_state["configured"] = True
|
||
if not enabled():
|
||
_setup_logging(service_name)
|
||
logging.getLogger("messenger.telemetry").info("OpenTelemetry disabled (OTEL_SDK_DISABLED=true)",
|
||
extra={"event": "telemetry.disabled"})
|
||
return
|
||
|
||
from opentelemetry._logs import set_logger_provider
|
||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||
from opentelemetry.sdk._logs import LoggerProvider
|
||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||
from opentelemetry.sdk.metrics import MeterProvider
|
||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
||
from opentelemetry.sdk.resources import Resource
|
||
from opentelemetry.sdk.trace import TracerProvider
|
||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||
|
||
# OTEL_RESOURCE_ATTRIBUTES (deployment.environment.name=lab) читает Resource.create; явные атрибуты
|
||
# важнее переменных окружения: janitor и ops.py с общим .env не станут «messenger-api».
|
||
resource = Resource.create({
|
||
"service.name": service_name,
|
||
"service.version": VERSION,
|
||
"service.instance.id": settings.INSTANCE_ID,
|
||
})
|
||
|
||
views = [
|
||
View(instrument_name="http.server.request.duration",
|
||
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)),
|
||
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
||
View(instrument_name="messenger.attachment.time_to_ready",
|
||
aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)),
|
||
View(instrument_name="db.client.connection.wait_time",
|
||
aggregation=ExplicitBucketHistogramAggregation(POOL_WAIT_BUCKETS)),
|
||
]
|
||
interval = int(os.environ.get("OTEL_METRIC_EXPORT_INTERVAL", "10000"))
|
||
meter_provider = MeterProvider(
|
||
resource=resource, views=views,
|
||
metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter(), export_interval_millis=interval)],
|
||
)
|
||
metrics.set_meter_provider(meter_provider)
|
||
|
||
tracer_provider = TracerProvider(resource=resource, sampler=_sampler())
|
||
tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(), meter_provider=meter_provider))
|
||
trace.set_tracer_provider(tracer_provider)
|
||
|
||
logger_provider = LoggerProvider(resource=resource)
|
||
logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter(),
|
||
meter_provider=meter_provider))
|
||
set_logger_provider(logger_provider)
|
||
_state["providers"] = [tracer_provider, logger_provider, meter_provider]
|
||
|
||
_setup_logging(service_name, logger_provider)
|
||
_register_process_metrics()
|
||
if instrument_db:
|
||
# Спаны PostgreSQL: один на запрос, текст SQL с плейсхолдерами; значения параметров НЕ пишутся
|
||
# (capture_parameters=False), sqlcommenter выключен.
|
||
from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
|
||
PsycopgInstrumentor().instrument(tracer_provider=tracer_provider, enable_commenter=False,
|
||
capture_parameters=False, skip_dep_check=True)
|
||
logging.getLogger("messenger.telemetry").info(
|
||
"OpenTelemetry enabled: OTLP %s, service %s", os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "default"),
|
||
service_name, extra={"event": "telemetry.enabled"})
|
||
|
||
|
||
def shutdown() -> None:
|
||
"""Дослать буферы при штатной остановке (не дольше таймаута экспорта)."""
|
||
for provider in _state["providers"]:
|
||
try:
|
||
provider.shutdown()
|
||
except Exception: # noqa: BLE001
|
||
pass
|
||
_state["providers"] = []
|
||
|
||
|
||
# ---------- контекст W3C ----------
|
||
|
||
def inject(carrier: dict | None = None) -> dict:
|
||
"""traceparent/tracestate текущего спана в carrier (заголовки AMQP, поле сообщения)."""
|
||
carrier = {} if carrier is None else carrier
|
||
_propagator.inject(carrier)
|
||
return carrier
|
||
|
||
|
||
def extract(carrier) -> otel_context.Context:
|
||
"""Контекст из входящих заголовков (HTTP, AMQP headers, поле traceparent сообщения)."""
|
||
return _propagator.extract(carrier or {})
|
||
|
||
|
||
def current_traceparent() -> str | None:
|
||
return inject().get("traceparent")
|
||
|
||
|
||
def context_from(traceparent: str | None, tracestate: str | None = None) -> otel_context.Context:
|
||
carrier = {"traceparent": traceparent} if traceparent else {}
|
||
if traceparent and tracestate:
|
||
carrier["tracestate"] = tracestate
|
||
return extract(carrier)
|
||
|
||
|
||
def link_to(traceparent: str | None) -> list:
|
||
"""Link на другой контекст (например, исходная загрузка при replay оператора)."""
|
||
span_context = trace.get_current_span(context_from(traceparent)).get_span_context()
|
||
return [Link(span_context)] if span_context.is_valid else []
|
||
|
||
|
||
def trace_id_hex() -> str | None:
|
||
span = trace.get_current_span().get_span_context()
|
||
return format(span.trace_id, "032x") if span.is_valid else None
|
||
|
||
|
||
# ---------- помощники для спанов ----------
|
||
|
||
def mark_error(span, error_type: str, message: str | None = None) -> None:
|
||
"""Статус ERROR + error.type (низкокардинальный код). Текст — только безопасный код/класс."""
|
||
span.set_attribute("error.type", error_type)
|
||
span.set_status(Status(StatusCode.ERROR, message or error_type))
|
||
|
||
|
||
@contextmanager
|
||
def client_span(name: str, attributes: dict | None = None):
|
||
"""Клиентский спан (S3, Redis). Исключение -> ERROR с типом исключения; само исключение не глотается."""
|
||
with tracer.start_as_current_span(name, kind=SpanKind.CLIENT, attributes=attributes or {},
|
||
record_exception=False, set_status_on_exception=False) as span:
|
||
try:
|
||
yield span
|
||
except Exception as exc:
|
||
mark_error(span, type(exc).__name__)
|
||
raise
|
||
|
||
|
||
@contextmanager
|
||
def internal_span(name: str, attributes: dict | None = None):
|
||
with tracer.start_as_current_span(name, kind=SpanKind.INTERNAL, attributes=attributes or {},
|
||
record_exception=False, set_status_on_exception=False) as span:
|
||
try:
|
||
yield span
|
||
except Exception as exc:
|
||
mark_error(span, type(exc).__name__)
|
||
raise
|