forked from mtusi2026/mtusi2026lab7
Перенесён минимум lab6 (API без изменений), добавлена защита на границах доверия: - внешний HTTPS; обязательный mTLS LB -> API с allowlist identity и отзывом по serial; - TLS на всех связях: PostgreSQL verify-full, MinIO HTTPS, RabbitMQ AMQPS mTLS (EXTERNAL), Valkey TLS + ACL, Collector и хранилища телеметрии через mTLS с отдельными CA; - OpenBao в server mode: cert auth, токены 15 мин, политики только на свой путь; - роли БД, S3-политики и права брокера по принципу минимальных прав; - Argon2id с замером, rehash, общий лимит входа, бюджет защиты от login DoS; - hardening всех контейнеров, scan образов с разбором, модель угроз, матрица доверия; - отрицательные тесты: TLS-аудит (135 проверок), ACL, ротации, sealed OpenBao.
394 lines
18 KiB
Python
394 lines
18 KiB
Python
"""OpenTelemetry for the API and the worker (lab6): traces, metrics and logs over OTLP/HTTP to the Collector.
|
|
|
|
Everything is configured from the standard OTEL_* environment variables plus a few of our own:
|
|
|
|
OTEL_SDK_DISABLED=true telemetry off: no providers, no instrumentation, no exporters (track 1 baseline)
|
|
OTEL_TRACES_SAMPLER_ARG=<0..1> head sampling ratio for ROOT spans (children follow the parent decision)
|
|
OTEL_EXPORTER_OTLP_ENDPOINT http://otel-collector:4318 (the SDK appends /v1/traces, /v1/metrics, /v1/logs)
|
|
OTEL_METRIC_EXPORT_INTERVAL metric push period, ms
|
|
OTEL_BSP_* / OTEL_BLRP_* bounded SDK queues (spans / log records); overflow is dropped, never blocks a request
|
|
|
|
Privacy rules (contracts/observability.md): no message bodies, passwords, Authorization, tokens, images or signed
|
|
URLs in any signal. Metric attributes are a fixed low-cardinality set (method, route template, status code, job
|
|
type, outcome, queue, state). IDs of synthetic objects appear only in logs/span attributes, never in metric labels.
|
|
"""
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import resource
|
|
import sys
|
|
import time
|
|
from contextlib import contextmanager
|
|
from typing import Optional
|
|
|
|
from opentelemetry import context as otel_context
|
|
from opentelemetry import metrics, propagate, trace
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
|
|
from app import config
|
|
|
|
ENABLED = os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() != "true"
|
|
SERVICE_VERSION = os.environ.get("SERVICE_VERSION", "1.6.0")
|
|
TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
|
|
QUEUE = config.BROKER_QUEUE
|
|
JOB_TYPE = "attachment.process.v1"
|
|
|
|
# Bucket boundaries (seconds) match the expected latencies of each operation, see REPORT.md.
|
|
HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.25, 0.35, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 30)
|
|
JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120)
|
|
READY_BUCKETS = (0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600)
|
|
|
|
tracer = trace.get_tracer("messenger")
|
|
meter = metrics.get_meter("messenger")
|
|
_state = {"service": "messenger", "providers": []}
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- logging
|
|
|
|
class EventFilter(logging.Filter):
|
|
"""Every record gets a machine-readable `event`: explicit extra={"event": ...} or the first snake_case word."""
|
|
|
|
EVENT_RE = re.compile(r"^([a-z][a-z0-9_]*)(?=[\s:]|$)")
|
|
|
|
def filter(self, record: logging.LogRecord) -> bool:
|
|
if not getattr(record, "event", None):
|
|
match = self.EVENT_RE.match(str(record.msg))
|
|
record.event = match.group(1) if match else record.name.rsplit(".", 1)[-1] + "_log"
|
|
return True
|
|
|
|
|
|
class JsonFormatter(logging.Formatter):
|
|
"""One JSON object per line on stdout, with trace_id/span_id of the active span (container logs)."""
|
|
|
|
def format(self, record: logging.LogRecord) -> str:
|
|
out = {
|
|
"ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + ".%03dZ" % record.msecs,
|
|
"level": record.levelname,
|
|
"service": _state["service"],
|
|
"event": getattr(record, "event", ""),
|
|
"logger": record.name,
|
|
"msg": record.getMessage(),
|
|
}
|
|
ctx = trace.get_current_span().get_span_context()
|
|
if ctx.is_valid:
|
|
out["trace_id"] = format(ctx.trace_id, "032x")
|
|
out["span_id"] = format(ctx.span_id, "016x")
|
|
if record.exc_info:
|
|
out["exc"] = self.formatException(record.exc_info)
|
|
return json.dumps(out, ensure_ascii=False)
|
|
|
|
|
|
def _setup_logging(logger_provider=None) -> None:
|
|
root = logging.getLogger()
|
|
for handler in list(root.handlers):
|
|
root.removeHandler(handler)
|
|
root.setLevel(logging.INFO)
|
|
stdout = logging.StreamHandler(sys.stdout)
|
|
stdout.setFormatter(JsonFormatter())
|
|
stdout.addFilter(EventFilter())
|
|
root.addHandler(stdout)
|
|
# The exporter's own warnings (backend down) must not be fed back into the exporter.
|
|
sdk_log = logging.getLogger("opentelemetry")
|
|
sdk_log.propagate = False
|
|
sdk_log.addHandler(stdout)
|
|
for noisy in ("botocore", "boto3", "urllib3", "aiormq", "aio_pika"):
|
|
logging.getLogger(noisy).setLevel(logging.WARNING)
|
|
if logger_provider is not None:
|
|
from opentelemetry.sdk._logs import LoggingHandler
|
|
|
|
otlp = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
|
otlp.addFilter(EventFilter())
|
|
root.addHandler(otlp)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- sampling
|
|
|
|
def _build_sampler():
|
|
"""ParentBased(root=KindFilter(TraceIdRatio)).
|
|
|
|
Only SERVER / CONSUMER / PRODUCER spans may start a trace. Root spans of background loops (outbox polling,
|
|
lease sweeper, readiness probe: plain DB client spans) would otherwise create thousands of one-span traces.
|
|
Children always follow the parent's decision, so an upload and its worker job are sampled together.
|
|
"""
|
|
from opentelemetry.sdk.trace.sampling import (Decision, ParentBased, Sampler, SamplingResult,
|
|
TraceIdRatioBased)
|
|
|
|
ratio = float(os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0"))
|
|
decisions = meter.create_counter(
|
|
"messenger.telemetry.sampler.decisions", unit="{span}",
|
|
description="Head sampler decisions for spans (sample/drop) and the rule that decided")
|
|
|
|
class KindFilter(Sampler):
|
|
def __init__(self, delegate):
|
|
self.delegate = delegate
|
|
|
|
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
|
|
trace_state=None):
|
|
if kind not in (SpanKind.SERVER, SpanKind.CONSUMER, SpanKind.PRODUCER):
|
|
decisions.add(1, {"decision": "drop", "rule": "root_kind"})
|
|
return SamplingResult(Decision.DROP)
|
|
result = self.delegate.should_sample(parent_context, trace_id, name, kind, attributes, links,
|
|
trace_state)
|
|
decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop", "rule": "ratio"})
|
|
return result
|
|
|
|
def get_description(self):
|
|
return "KindFilter{%s}" % self.delegate.get_description()
|
|
|
|
class CountingParentBased(ParentBased):
|
|
def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None,
|
|
trace_state=None):
|
|
result = super().should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state)
|
|
parent = trace.get_current_span(parent_context).get_span_context()
|
|
if parent.is_valid:
|
|
decisions.add(1, {"decision": "sample" if result.decision.is_sampled() else "drop",
|
|
"rule": "parent"})
|
|
return result
|
|
|
|
return CountingParentBased(root=KindFilter(TraceIdRatioBased(ratio)))
|
|
|
|
|
|
class _CountingExporter:
|
|
"""Wraps the OTLP span exporter to count what really left the process (track 1: span loss on the SDK side)."""
|
|
|
|
def __init__(self, delegate):
|
|
from opentelemetry.sdk.trace.export import SpanExportResult
|
|
|
|
self.delegate = delegate
|
|
self.ok = SpanExportResult.SUCCESS
|
|
self.counter = meter.create_counter(
|
|
"messenger.telemetry.spans.exported", unit="{span}",
|
|
description="Spans handed to the OTLP exporter by outcome (success/failure)")
|
|
|
|
def export(self, spans):
|
|
result = self.delegate.export(spans)
|
|
self.counter.add(len(spans), {"outcome": "success" if result == self.ok else "failure"})
|
|
return result
|
|
|
|
def shutdown(self):
|
|
return self.delegate.shutdown()
|
|
|
|
def force_flush(self, timeout_millis: int = 30000):
|
|
return self.delegate.force_flush(timeout_millis)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- setup
|
|
|
|
def setup(service_name: str, fastapi_app=None) -> None:
|
|
_state["service"] = service_name
|
|
if not ENABLED:
|
|
_setup_logging(None)
|
|
logging.getLogger("messenger.telemetry").info("telemetry_disabled OTEL_SDK_DISABLED=true")
|
|
return
|
|
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.sdk._logs import LoggerProvider
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
|
|
# Resource.create also merges OTEL_RESOURCE_ATTRIBUTES (deployment.environment.name=lab).
|
|
res = Resource.create({
|
|
"service.name": service_name,
|
|
"service.namespace": "messenger",
|
|
"service.version": SERVICE_VERSION,
|
|
"service.instance.id": config.INSTANCE_ID,
|
|
"deployment.environment.name": os.environ.get("APP_ENV", "lab"),
|
|
})
|
|
|
|
views = [
|
|
# Only these attributes survive on the HTTP histogram: no server.address, url.*, user ids, etc.
|
|
View(instrument_name="http.server.request.duration",
|
|
aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS),
|
|
attribute_keys={"http.request.method", "http.route", "http.response.status_code"}),
|
|
View(instrument_name="http.server.active_requests", attribute_keys={"http.request.method"}),
|
|
View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)),
|
|
View(instrument_name="messenger.attachment.time_to_ready",
|
|
aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)),
|
|
# No catch-all View("*"): an instrument matched by two views would be exported twice.
|
|
# Instruments without a matching view get the SDK default aggregation.
|
|
]
|
|
meter_provider = MeterProvider(resource=res, metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())],
|
|
views=views)
|
|
metrics.set_meter_provider(meter_provider)
|
|
|
|
tracer_provider = TracerProvider(resource=res, sampler=_build_sampler())
|
|
tracer_provider.add_span_processor(BatchSpanProcessor(_CountingExporter(OTLPSpanExporter())))
|
|
trace.set_tracer_provider(tracer_provider)
|
|
|
|
logger_provider = LoggerProvider(resource=res)
|
|
logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
|
|
_setup_logging(logger_provider)
|
|
_state["providers"] = [tracer_provider, meter_provider, logger_provider]
|
|
|
|
from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
|
|
|
|
# PostgreSQL spans are created in app/db.py (accurate async timing); SQL text is parametrised (%s
|
|
# placeholders), so values, message bodies and hashes never reach db.query.text.
|
|
BotocoreInstrumentor().instrument()
|
|
if fastapi_app is not None:
|
|
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
|
|
|
# Health probes (LB every second) are excluded from spans and from the HTTP SLI.
|
|
FastAPIInstrumentor.instrument_app(fastapi_app, excluded_urls="/health/live,/health/ready",
|
|
exclude_spans=["receive", "send"])
|
|
_process_metrics()
|
|
logging.getLogger("messenger.telemetry").info(
|
|
"telemetry_started service=%s endpoint=%s sampler_ratio=%s", service_name,
|
|
os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", ""), os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0"))
|
|
|
|
|
|
def shutdown() -> None:
|
|
for provider in _state["providers"]:
|
|
try:
|
|
provider.shutdown()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- context carrier
|
|
|
|
def inject() -> dict:
|
|
"""W3C trace context of the active span as job-message fields (traceparent/tracestate), or {}."""
|
|
carrier: dict = {}
|
|
if trace.get_current_span().get_span_context().is_valid:
|
|
propagate.inject(carrier)
|
|
return {k: v for k, v in carrier.items() if k in ("traceparent", "tracestate")}
|
|
|
|
|
|
def extract(carrier: Optional[dict]):
|
|
"""Context from a job message/AMQP headers; an empty or malformed carrier gives a fresh root context."""
|
|
if not carrier or not TRACEPARENT_RE.match(str(carrier.get("traceparent", ""))):
|
|
return otel_context.Context()
|
|
clean = {"traceparent": str(carrier["traceparent"])}
|
|
if isinstance(carrier.get("tracestate"), str) and len(carrier["tracestate"]) <= 512:
|
|
clean["tracestate"] = carrier["tracestate"]
|
|
return propagate.extract(clean, context=otel_context.Context())
|
|
|
|
|
|
def set_attributes(**attrs) -> None:
|
|
span = trace.get_current_span()
|
|
for key, value in attrs.items():
|
|
if value is not None:
|
|
span.set_attribute(key.replace("__", "."), value)
|
|
|
|
|
|
@contextmanager
|
|
def span(name: str, kind=SpanKind.INTERNAL, context=None, attributes=None):
|
|
with tracer.start_as_current_span(name, context=context, kind=kind, attributes=attributes or {},
|
|
record_exception=True, set_status_on_exception=True) as current:
|
|
yield current
|
|
|
|
|
|
def record_exception(exc: BaseException) -> None:
|
|
"""Exception event on the active span (type + stack, never request data) for errors handled by our code."""
|
|
current = trace.get_current_span()
|
|
current.record_exception(exc)
|
|
current.set_status(Status(StatusCode.ERROR, type(exc).__name__))
|
|
|
|
|
|
def mark_error(current, description: str) -> None:
|
|
current.set_status(Status(StatusCode.ERROR, description))
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------- instruments
|
|
|
|
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
|
|
|
|
|
|
def _rss_bytes() -> int:
|
|
try:
|
|
with open("/proc/self/statm") as handle:
|
|
return int(handle.read().split()[1]) * PAGE_SIZE
|
|
except OSError:
|
|
return 0
|
|
|
|
|
|
def _process_metrics() -> None:
|
|
from opentelemetry.metrics import Observation
|
|
|
|
def cpu(_options):
|
|
own = resource.getrusage(resource.RUSAGE_SELF)
|
|
kids = resource.getrusage(resource.RUSAGE_CHILDREN) # worker image children (after they exit)
|
|
yield Observation(own.ru_utime + own.ru_stime, {"process.cpu.owner": "self"})
|
|
yield Observation(kids.ru_utime + kids.ru_stime, {"process.cpu.owner": "children"})
|
|
|
|
meter.create_observable_counter("process.cpu.time", [cpu], unit="s", description="CPU seconds (user+system)")
|
|
meter.create_observable_gauge("process.memory.usage", [lambda _o: [Observation(_rss_bytes())]], unit="By",
|
|
description="Resident set size of the process")
|
|
|
|
|
|
def register_pool(db) -> None:
|
|
"""DB pool saturation: used / idle connections, waiting requests, max size (psycopg_pool stats)."""
|
|
if not ENABLED:
|
|
return
|
|
from opentelemetry.metrics import Observation
|
|
|
|
def count(_options):
|
|
stats = db.pool.get_stats()
|
|
size, idle = stats.get("pool_size", 0), stats.get("pool_available", 0)
|
|
yield Observation(max(0, size - idle), {"db.client.connection.state": "used"})
|
|
yield Observation(idle, {"db.client.connection.state": "idle"})
|
|
|
|
meter.create_observable_up_down_counter("db.client.connection.count", [count], unit="{connection}")
|
|
meter.create_observable_up_down_counter(
|
|
"db.client.connection.pending_requests",
|
|
[lambda _o: [Observation(db.pool.get_stats().get("requests_waiting", 0))]], unit="{request}")
|
|
meter.create_observable_up_down_counter(
|
|
"db.client.connection.max", [lambda _o: [Observation(db.pool.max_size)]], unit="{connection}")
|
|
|
|
|
|
def register_queue(state: dict) -> None:
|
|
"""Backlog of the attachment pipeline from PostgreSQL (the source of truth), refreshed by the API every second.
|
|
|
|
Reported by every API replica (identical values): dashboards and alerts use max() across replicas.
|
|
"""
|
|
if not ENABLED:
|
|
return
|
|
from opentelemetry.metrics import Observation
|
|
|
|
def depth(_options):
|
|
queue = state.get("queue")
|
|
if queue is None:
|
|
return []
|
|
return [Observation(queue["queued"], {"queue": QUEUE, "state": "queued"}),
|
|
Observation(queue["processing"], {"queue": QUEUE, "state": "processing"}),
|
|
Observation(queue["outbox"], {"queue": "outbox", "state": "unpublished"})]
|
|
|
|
def oldest(_options):
|
|
queue = state.get("queue")
|
|
return [] if queue is None else [Observation(queue["oldest_age"], {"queue": QUEUE})]
|
|
|
|
meter.create_observable_gauge("messenger.queue.depth", [depth], unit="{job}",
|
|
description="Jobs accepted but not finished, by state")
|
|
meter.create_observable_gauge("messenger.queue.oldest_age", [oldest], unit="s",
|
|
description="Age of the oldest queued/processing job since durable acceptance")
|
|
|
|
|
|
def register_worker(worker) -> None:
|
|
if not ENABLED:
|
|
return
|
|
from opentelemetry.metrics import Observation
|
|
|
|
meter.create_observable_gauge("messenger.worker.busy", [lambda _o: [Observation(worker.inflight)]],
|
|
unit="{job}", description="Jobs being processed right now")
|
|
meter.create_observable_gauge("messenger.worker.capacity",
|
|
[lambda _o: [Observation(config.WORKER_CONCURRENCY)]], unit="{job}",
|
|
description="Concurrent job slots of this worker")
|
|
|
|
|
|
jobs_processed = meter.create_counter(
|
|
"messenger.jobs.processed", unit="{job}", description="Job deliveries handled by the worker, by outcome")
|
|
job_duration = meter.create_histogram(
|
|
"messenger.job.duration", unit="s", description="Worker time per delivery (claim to ack), by outcome")
|
|
time_to_ready = meter.create_histogram(
|
|
"messenger.attachment.time_to_ready", unit="s",
|
|
description="From durable acceptance (job COMMIT, 202) to the ready COMMIT")
|