Files
mtusi2026lab7/app/telemetry.py
T
vika baafe2a3ae Добавить стенд лабораторной 7 с mTLS, OpenBao и least privilege worker.
Закрывает обязательный минимум и трек 1: ротация leaf и секрета, sealed OpenBao и сохранность queued-задачи.
2026-11-07 20:46:02 +03:00

390 lines
12 KiB
Python

import json
import logging
import threading
import time
from opentelemetry import context as otel_context
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.propagate import extract, inject
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk._logs.export import LogExportResult
from opentelemetry.sdk.metrics.export import MetricExportResult
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
from opentelemetry.trace import Status, StatusCode
from app.settings import settings
BANNED_FIELDS = {
"password",
"token",
"authorization",
"secret",
"filename",
"text",
"body",
"database_url",
"broker_url",
"image",
}
_dropped = None
_http_requests = None
_http_duration = None
_jobs = None
_job_duration = None
_ready_duration = None
_db_duration = None
_state = {
"queue_depth": 0,
"queue_age": 0.0,
"db_up": 0,
"pool_used": 0,
"pool_waiting": 0,
"worker_busy": 0,
"worker_capacity": 0,
"worker_consumers": 0,
"cpu": 0.0,
"memory": 0,
}
_cpu_last = None
_lock = threading.Lock()
class _GuardedExporter:
def __init__(self, inner, signal):
self.inner = inner
self.signal = signal
def __getattr__(self, name):
return getattr(self.inner, name)
def export(self, items, **kwargs):
try:
result = self.inner.export(items)
except Exception:
_count_drop(self.signal, len(items))
return self._failure()
if getattr(result, "name", None) != "SUCCESS":
_count_drop(self.signal, len(items))
return result
def _failure(self):
if self.signal == "metrics":
return MetricExportResult.FAILURE
if self.signal == "logs":
return LogExportResult.FAILURE
return SpanExportResult.FAILURE
def shutdown(self, **kwargs):
try:
return self.inner.shutdown(**kwargs)
except Exception:
return None
def force_flush(self, timeout_millis=3000):
try:
return self.inner.force_flush(timeout_millis)
except Exception:
return False
class _RawFormatter(logging.Formatter):
def format(self, record):
return record.getMessage()
def _count_drop(signal, count):
if _dropped is not None and count:
_dropped.add(count, {"signal": signal})
def _resource():
return Resource.create(
{
"service.name": settings.service_name,
"service.version": settings.service_version,
"service.instance.id": settings.instance_id,
"deployment.environment.name": settings.environment,
}
)
def _histogram(name, buckets):
return View(instrument_name=name, aggregation=ExplicitBucketHistogramAggregation(buckets))
def _otlp(endpoint):
return dict(
endpoint=endpoint,
timeout=5,
certificate_file=settings.tls_ca,
client_certificate_file=settings.client_cert,
client_key_file=settings.client_key,
)
def init():
global _dropped, _http_requests, _http_duration, _jobs, _job_duration, _ready_duration, _db_duration
resource = _resource()
endpoint = settings.otlp_endpoint
tracer_provider = TracerProvider(resource=resource)
tracer_provider.add_span_processor(
BatchSpanProcessor(
_GuardedExporter(OTLPSpanExporter(**_otlp(endpoint + "/v1/traces")), "spans"),
max_queue_size=256,
schedule_delay_millis=2000,
max_export_batch_size=64,
export_timeout_millis=5000,
)
)
trace.set_tracer_provider(tracer_provider)
reader = PeriodicExportingMetricReader(
_GuardedExporter(OTLPMetricExporter(**_otlp(endpoint + "/v1/metrics")), "metrics"),
export_interval_millis=5000,
export_timeout_millis=4000,
)
meter_provider = MeterProvider(
resource=resource,
metric_readers=[reader],
views=[
_histogram("http.server.duration", [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]),
_histogram("messenger.job.duration", [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]),
_histogram("messenger.attachment.time_to_ready", [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]),
_histogram("messenger.db.duration", [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]),
],
)
from opentelemetry import metrics
metrics.set_meter_provider(meter_provider)
meter = metrics.get_meter("messenger", settings.service_version)
_dropped = meter.create_counter("messenger.telemetry.dropped", unit="1")
_http_requests = meter.create_counter("http.server.requests", unit="1")
_http_duration = meter.create_histogram("http.server.duration", unit="s")
_jobs = meter.create_counter("messenger.jobs", unit="1")
_job_duration = meter.create_histogram("messenger.job.duration", unit="s")
_ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s")
_db_duration = meter.create_histogram("messenger.db.duration", unit="s")
meter.create_observable_gauge("messenger.queue.depth", callbacks=[_queue_depth])
meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[_queue_age], unit="s")
meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used])
meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting])
meter.create_observable_gauge("messenger.worker.busy", callbacks=[_worker_busy])
meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_worker_capacity])
meter.create_observable_gauge("messenger.worker.consumers", callbacks=[_worker_consumers])
meter.create_observable_gauge("messenger.db.up", callbacks=[_db_up])
meter.create_observable_gauge("messenger.process.cpu.utilization", callbacks=[_cpu])
meter.create_observable_gauge("messenger.process.memory", callbacks=[_memory], unit="By")
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(
_GuardedExporter(OTLPLogExporter(**_otlp(endpoint + "/v1/logs")), "logs"),
max_queue_size=256,
schedule_delay_millis=2000,
max_export_batch_size=64,
export_timeout_millis=5000,
)
)
root = logging.getLogger()
root.handlers.clear()
root.setLevel(logging.INFO)
stream = logging.StreamHandler()
stream.setFormatter(_RawFormatter())
root.addHandler(stream)
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
root.addHandler(otel_handler)
for name in ("pika", "botocore", "boto3", "urllib3", "uvicorn", "uvicorn.error", "uvicorn.access", "redis"):
logging.getLogger(name).setLevel(logging.WARNING)
quiet = logging.getLogger("opentelemetry")
quiet.handlers.clear()
quiet.propagate = False
quiet.setLevel(logging.CRITICAL)
log_event("service.started", role=settings.role)
def tracer():
return trace.get_tracer("messenger", settings.service_version)
def current_ids():
span = trace.get_current_span()
ctx = span.get_span_context()
if not ctx or not ctx.is_valid:
return None, None
return format(ctx.trace_id, "032x"), format(ctx.span_id, "016x")
def log_event(event, severity="INFO", **fields):
trace_id, span_id = current_ids()
payload = {
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + f".{int(time.time() * 1000000) % 1000000:06d}Z",
"severity": severity,
"service.name": settings.service_name,
"event": event,
"trace_id": trace_id,
"span_id": span_id,
}
for key, value in fields.items():
if key in BANNED_FIELDS or value is None:
continue
if isinstance(value, (str, int, float, bool)):
payload[key] = value
level = getattr(logging, severity, logging.INFO)
logging.getLogger("messenger").log(level, json.dumps(payload, ensure_ascii=False))
def inject_carrier(carrier=None):
target = {} if carrier is None else carrier
inject(target)
return target
def extract_context(carrier):
clean = {}
for key, value in (carrier or {}).items():
if isinstance(value, bytes):
value = value.decode("utf-8", "replace")
if isinstance(value, str):
clean[str(key).lower()] = value
return extract(clean)
def record_http(method, route, status, duration):
if _http_requests is None:
return
klass = f"{int(status) // 100}xx"
attributes = {
"http.request.method": method,
"http.route": route,
"http.response.status_class": klass,
}
_http_requests.add(1, attributes)
_http_duration.record(duration, attributes)
def record_job(outcome, duration):
attributes = {"job_type": "attachment.process.v1", "outcome": outcome}
if _jobs is not None:
_jobs.add(1, attributes)
_job_duration.record(max(duration, 0.0), attributes)
def record_ready(duration):
if _ready_duration is not None:
_ready_duration.record(max(duration, 0.0), {"job_type": "attachment.process.v1"})
def record_db(duration):
if _db_duration is not None:
_db_duration.record(max(duration, 0.0), {"db.system": "postgresql"})
def update_runtime(**values):
with _lock:
_state.update(values)
def note_worker(busy, capacity):
update_runtime(worker_busy=busy, worker_capacity=capacity)
def sample_process():
global _cpu_last
now_cpu = time.process_time()
now_wall = time.monotonic()
ratio = 0.0
if _cpu_last is not None:
dc = now_cpu - _cpu_last[0]
dw = now_wall - _cpu_last[1]
if dw > 0:
ratio = dc / dw
_cpu_last = (now_cpu, now_wall)
memory = 0
try:
with open("/proc/self/statm", encoding="ascii") as handle:
resident = int(handle.read().split()[1])
memory = resident * os_page()
except (OSError, IndexError, ValueError):
memory = 0
update_runtime(cpu=ratio, memory=memory)
_page = None
def os_page():
global _page
if _page is None:
import os
_page = os.sysconf("SC_PAGE_SIZE")
return _page
def _obs(value, attributes=None):
from opentelemetry.metrics import Observation
return [Observation(value, attributes or {})]
def _queue_depth(_options):
return _obs(_state["queue_depth"], {"queue": "attachment.process.v1"})
def _queue_age(_options):
return _obs(_state["queue_age"], {"queue": "attachment.process.v1"})
def _pool_used(_options):
return _obs(_state["pool_used"], {"pool": "postgres"})
def _pool_waiting(_options):
return _obs(_state["pool_waiting"], {"pool": "postgres"})
def _worker_busy(_options):
if settings.role != "worker":
return []
return _obs(_state["worker_busy"], {"queue": "attachment.process.v1"})
def _worker_capacity(_options):
if settings.role != "worker":
return []
return _obs(_state["worker_capacity"], {"queue": "attachment.process.v1"})
def _worker_consumers(_options):
return _obs(_state["worker_consumers"], {"queue": "attachment.process.v1"})
def _db_up(_options):
return _obs(_state["db_up"])
def _cpu(_options):
return _obs(_state["cpu"])
def _memory(_options):
return _obs(_state["memory"])
def mark_error(span):
span.set_status(Status(StatusCode.ERROR))
def attach(ctx):
return otel_context.attach(ctx)