Files
mtusi2026lab7/app/telemetry.py
T
alenchha 861e7722b5 ЛР 7 — БВТ2301 — Курлова — на 4
Перенесён минимум lab6 (API без изменений), добавлена защита на границах доверия:
- внешний HTTPS; обязательный mTLS LB -> API с allowlist identity и отзывом по serial;
- TLS на всех связях: PostgreSQL verify-full, MinIO HTTPS, RabbitMQ AMQPS mTLS (EXTERNAL),
  Valkey TLS + ACL, Collector и хранилища телеметрии через mTLS с отдельными CA;
- OpenBao в server mode: cert auth, токены 15 мин, политики только на свой путь;
- роли БД, S3-политики и права брокера по принципу минимальных прав;
- Argon2id с замером, rehash, общий лимит входа, бюджет защиты от login DoS;
- hardening всех контейнеров, scan образов с разбором, модель угроз, матрица доверия;
- отрицательные тесты: TLS-аудит (135 проверок), ACL, ротации, sealed OpenBao.
2026-11-16 10:49:13 +03:00

394 lines
18 KiB
Python

"""OpenTelemetry for the API and the worker (lab6): traces, metrics and logs over OTLP/HTTP to the Collector.
Everything is configured from the standard OTEL_* environment variables plus a few of our own:
OTEL_SDK_DISABLED=true telemetry off: no providers, no instrumentation, no exporters (track 1 baseline)
OTEL_TRACES_SAMPLER_ARG=<0..1> head sampling ratio for ROOT spans (children follow the parent decision)
OTEL_EXPORTER_OTLP_ENDPOINT http://otel-collector:4318 (the SDK appends /v1/traces, /v1/metrics, /v1/logs)
OTEL_METRIC_EXPORT_INTERVAL metric push period, ms
OTEL_BSP_* / OTEL_BLRP_* bounded SDK queues (spans / log records); overflow is dropped, never blocks a request
Privacy rules (contracts/observability.md): no message bodies, passwords, Authorization, tokens, images or signed
URLs in any signal. Metric attributes are a fixed low-cardinality set (method, route template, status code, job
type, outcome, queue, state). IDs of synthetic objects appear only in logs/span attributes, never in metric labels.
"""
import json
import logging
import os
import re
import resource
import sys
import time
from contextlib import contextmanager
from typing import Optional
from opentelemetry import context as otel_context
from opentelemetry import metrics, propagate, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from app import config
ENABLED = os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() != "true"
SERVICE_VERSION = os.environ.get("SERVICE_VERSION", "1.6.0")
TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
QUEUE = config.BROKER_QUEUE
JOB_TYPE = "attachment.process.v1"
# Bucket boundaries (seconds) match the expected latencies of each operation, see REPORT.md.
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.35, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 30)
JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120)
READY_BUCKETS = (0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600)
tracer = trace.get_tracer("messenger")
meter = metrics.get_meter("messenger")
_state = {"service": "messenger", "providers": []}
# ---------------------------------------------------------------------------------------------- logging
class EventFilter(logging.Filter):
"""Every record gets a machine-readable `event`: explicit extra={"event": ...} or the first snake_case word."""
EVENT_RE = re.compile(r"^([a-z][a-z0-9_]*)(?=[\s:]|$)")
def filter(self, record: logging.LogRecord) -> bool:
if not getattr(record, "event", None):
match = self.EVENT_RE.match(str(record.msg))
record.event = match.group(1) if match else record.name.rsplit(".", 1)[-1] + "_log"
return True
class JsonFormatter(logging.Formatter):
"""One JSON object per line on stdout, with trace_id/span_id of the active span (container logs)."""
def format(self, record: logging.LogRecord) -> str:
out = {
"ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + ".%03dZ" % record.msecs,
"level": record.levelname,
"service": _state["service"],
"event": getattr(record, "event", ""),
"logger": record.name,
"msg": record.getMessage(),
}
ctx = trace.get_current_span().get_span_context()
if ctx.is_valid:
out["trace_id"] = format(ctx.trace_id, "032x")
out["span_id"] = format(ctx.span_id, "016x")
if record.exc_info:
out["exc"] = self.formatException(record.exc_info)
return json.dumps(out, ensure_ascii=False)
def _setup_logging(logger_provider=None) -> None:
root = logging.getLogger()
for handler in list(root.handlers):
root.removeHandler(handler)
root.setLevel(logging.INFO)
stdout = logging.StreamHandler(sys.stdout)
stdout.setFormatter(JsonFormatter())
stdout.addFilter(EventFilter())
root.addHandler(stdout)
# The exporter's own warnings (backend down) must not be fed back into the exporter.
sdk_log = logging.getLogger("opentelemetry")
sdk_log.propagate = False
sdk_log.addHandler(stdout)
for noisy in ("botocore", "boto3", "urllib3", "aiormq", "aio_pika"):
logging.getLogger(noisy).setLevel(logging.WARNING)
if logger_provider is not None:
from opentelemetry.sdk._logs import LoggingHandler
otlp = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
otlp.addFilter(EventFilter())
root.addHandler(otlp)
# ---------------------------------------------------------------------------------------------- sampling
def _build_sampler():
"""ParentBased(root=KindFilter(TraceIdRatio)).
Only SERVER / CONSUMER / PRODUCER spans may start a trace. Root spans of background loops (outbox polling,
lease sweeper, readiness probe: plain DB client spans) would otherwise create thousands of one-span traces.
Children always follow the parent's decision, so an upload and its worker job are sampled together.
"""
from opentelemetry.sdk.trace.sampling import (Decision, ParentBased, Sampler, SamplingResult,
TraceIdRatioBased)
ratio = float(os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0"))
decisions = meter.create_counter(
"messenger.telemetry.sampler.decisions", unit="{span}",
description="Head sampler decisions for spans (sample/drop) and the rule that decided")
class KindFilter(Sampler):
def __init__(self, delegate):
self.delegate = delegate
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
trace_state=None):
if kind not in (SpanKind.SERVER, SpanKind.CONSUMER, SpanKind.PRODUCER):
decisions.add(1, {"decision": "drop", "rule": "root_kind"})
return SamplingResult(Decision.DROP)
result = self.delegate.should_sample(parent_context, trace_id, name, kind, attributes, links,
trace_state)
decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop", "rule": "ratio"})
return result
def get_description(self):
return "KindFilter{%s}" % self.delegate.get_description()
class CountingParentBased(ParentBased):
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
trace_state=None):
result = super().should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state)
parent = trace.get_current_span(parent_context).get_span_context()
if parent.is_valid:
decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop",
"rule": "parent"})
return result
return CountingParentBased(root=KindFilter(TraceIdRatioBased(ratio)))
class _CountingExporter:
"""Wraps the OTLP span exporter to count what really left the process (track 1: span loss on the SDK side)."""
def __init__(self, delegate):
from opentelemetry.sdk.trace.export import SpanExportResult
self.delegate = delegate
self.ok = SpanExportResult.SUCCESS
self.counter = meter.create_counter(
"messenger.telemetry.spans.exported", unit="{span}",
description="Spans handed to the OTLP exporter by outcome (success/failure)")
def export(self, spans):
result = self.delegate.export(spans)
self.counter.add(len(spans), {"outcome": "success" if result == self.ok else "failure"})
return result
def shutdown(self):
return self.delegate.shutdown()
def force_flush(self, timeout_millis: int = 30000):
return self.delegate.force_flush(timeout_millis)
# ---------------------------------------------------------------------------------------------- setup
def setup(service_name: str, fastapi_app=None) -> None:
_state["service"] = service_name
if not ENABLED:
_setup_logging(None)
logging.getLogger("messenger.telemetry").info("telemetry_disabled OTEL_SDK_DISABLED=true")
return
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk._logs import LoggerProvider
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
# Resource.create also merges OTEL_RESOURCE_ATTRIBUTES (deployment.environment.name=lab).
res = Resource.create({
"service.name": service_name,
"service.namespace": "messenger",
"service.version": SERVICE_VERSION,
"service.instance.id": config.INSTANCE_ID,
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
})
views = [
# Only these attributes survive on the HTTP histogram: no server.address, url.*, user ids, etc.
View(instrument_name="http.server.request.duration",
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS),
attribute_keys={"http.request.method", "http.route", "http.response.status_code"}),
View(instrument_name="http.server.active_requests", attribute_keys={"http.request.method"}),
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
View(instrument_name="messenger.attachment.time_to_ready",
aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)),
# No catch-all View("*"): an instrument matched by two views would be exported twice.
# Instruments without a matching view get the SDK default aggregation.
]
meter_provider = MeterProvider(resource=res, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())],
views=views)
metrics.set_meter_provider(meter_provider)
tracer_provider = TracerProvider(resource=res, sampler=_build_sampler())
tracer_provider.add_span_processor(BatchSpanProcessor(_CountingExporter(OTLPSpanExporter())))
trace.set_tracer_provider(tracer_provider)
logger_provider = LoggerProvider(resource=res)
logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
_setup_logging(logger_provider)
_state["providers"] = [tracer_provider, meter_provider, logger_provider]
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
# PostgreSQL spans are created in app/db.py (accurate async timing); SQL text is parametrised (%s
# placeholders), so values, message bodies and hashes never reach db.query.text.
BotocoreInstrumentor().instrument()
if fastapi_app is not None:
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
# Health probes (LB every second) are excluded from spans and from the HTTP SLI.
FastAPIInstrumentor.instrument_app(fastapi_app, excluded_urls="/health/live,/health/ready",
exclude_spans=["receive", "send"])
_process_metrics()
logging.getLogger("messenger.telemetry").info(
"telemetry_started service=%s endpoint=%s sampler_ratio=%s", service_name,
os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", ""), os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0"))
def shutdown() -> None:
for provider in _state["providers"]:
try:
provider.shutdown()
except Exception:
pass
# ---------------------------------------------------------------------------------------------- context carrier
def inject() -> dict:
"""W3C trace context of the active span as job-message fields (traceparent/tracestate), or {}."""
carrier: dict = {}
if trace.get_current_span().get_span_context().is_valid:
propagate.inject(carrier)
return {k: v for k, v in carrier.items() if k in ("traceparent", "tracestate")}
def extract(carrier: Optional[dict]):
"""Context from a job message/AMQP headers; an empty or malformed carrier gives a fresh root context."""
if not carrier or not TRACEPARENT_RE.match(str(carrier.get("traceparent", ""))):
return otel_context.Context()
clean = {"traceparent": str(carrier["traceparent"])}
if isinstance(carrier.get("tracestate"), str) and len(carrier["tracestate"]) <= 512:
clean["tracestate"] = carrier["tracestate"]
return propagate.extract(clean, context=otel_context.Context())
def set_attributes(**attrs) -> None:
span = trace.get_current_span()
for key, value in attrs.items():
if value is not None:
span.set_attribute(key.replace("__", "."), value)
@contextmanager
def span(name: str, kind=SpanKind.INTERNAL, context=None, attributes=None):
with tracer.start_as_current_span(name, context=context, kind=kind, attributes=attributes or {},
record_exception=True, set_status_on_exception=True) as current:
yield current
def record_exception(exc: BaseException) -> None:
"""Exception event on the active span (type + stack, never request data) for errors handled by our code."""
current = trace.get_current_span()
current.record_exception(exc)
current.set_status(Status(StatusCode.ERROR, type(exc).__name__))
def mark_error(current, description: str) -> None:
current.set_status(Status(StatusCode.ERROR, description))
# ---------------------------------------------------------------------------------------------- instruments
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
def _rss_bytes() -> int:
try:
with open("/proc/self/statm") as handle:
return int(handle.read().split()[1]) * PAGE_SIZE
except OSError:
return 0
def _process_metrics() -> None:
from opentelemetry.metrics import Observation
def cpu(_options):
own = resource.getrusage(resource.RUSAGE_SELF)
kids = resource.getrusage(resource.RUSAGE_CHILDREN) # worker image children (after they exit)
yield Observation(own.ru_utime + own.ru_stime, {"process.cpu.owner": "self"})
yield Observation(kids.ru_utime + kids.ru_stime, {"process.cpu.owner": "children"})
meter.create_observable_counter("process.cpu.time", [cpu], unit="s", description="CPU seconds (user+system)")
meter.create_observable_gauge("process.memory.usage", [lambda _o: [Observation(_rss_bytes())]], unit="By",
description="Resident set size of the process")
def register_pool(db) -> None:
"""DB pool saturation: used / idle connections, waiting requests, max size (psycopg_pool stats)."""
if not ENABLED:
return
from opentelemetry.metrics import Observation
def count(_options):
stats = db.pool.get_stats()
size, idle = stats.get("pool_size", 0), stats.get("pool_available", 0)
yield Observation(max(0, size - idle), {"db.client.connection.state": "used"})
yield Observation(idle, {"db.client.connection.state": "idle"})
meter.create_observable_up_down_counter("db.client.connection.count", [count], unit="{connection}")
meter.create_observable_up_down_counter(
"db.client.connection.pending_requests",
[lambda _o: [Observation(db.pool.get_stats().get("requests_waiting", 0))]], unit="{request}")
meter.create_observable_up_down_counter(
"db.client.connection.max", [lambda _o: [Observation(db.pool.max_size)]], unit="{connection}")
def register_queue(state: dict) -> None:
"""Backlog of the attachment pipeline from PostgreSQL (the source of truth), refreshed by the API every second.
Reported by every API replica (identical values): dashboards and alerts use max() across replicas.
"""
if not ENABLED:
return
from opentelemetry.metrics import Observation
def depth(_options):
queue = state.get("queue")
if queue is None:
return []
return [Observation(queue["queued"], {"queue": QUEUE, "state": "queued"}),
Observation(queue["processing"], {"queue": QUEUE, "state": "processing"}),
Observation(queue["outbox"], {"queue": "outbox", "state": "unpublished"})]
def oldest(_options):
queue = state.get("queue")
return [] if queue is None else [Observation(queue["oldest_age"], {"queue": QUEUE})]
meter.create_observable_gauge("messenger.queue.depth", [depth], unit="{job}",
description="Jobs accepted but not finished, by state")
meter.create_observable_gauge("messenger.queue.oldest_age", [oldest], unit="s",
description="Age of the oldest queued/processing job since durable acceptance")
def register_worker(worker) -> None:
if not ENABLED:
return
from opentelemetry.metrics import Observation
meter.create_observable_gauge("messenger.worker.busy", [lambda _o: [Observation(worker.inflight)]],
unit="{job}", description="Jobs being processed right now")
meter.create_observable_gauge("messenger.worker.capacity",
[lambda _o: [Observation(config.WORKER_CONCURRENCY)]], unit="{job}",
description="Concurrent job slots of this worker")
jobs_processed = meter.create_counter(
"messenger.jobs.processed", unit="{job}", description="Job deliveries handled by the worker, by outcome")
job_duration = meter.create_histogram(
"messenger.job.duration", unit="s", description="Worker time per delivery (claim to ack), by outcome")
time_to_ready = meter.create_histogram(
"messenger.attachment.time_to_ready", unit="s",
description="From durable acceptance (job COMMIT, 202) to the ready COMMIT")