forked from mtusi2026/mtusi2026lab7
Закрывает обязательный минимум и трек 1: ротация leaf и секрета, sealed OpenBao и сохранность queued-задачи.
390 lines
12 KiB
Python
390 lines
12 KiB
Python
import json
|
|
import logging
|
|
import threading
|
|
import time
|
|
|
|
from opentelemetry import context as otel_context
|
|
from opentelemetry import trace
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
|
from opentelemetry.propagate import extract, inject
|
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk._logs.export import LogExportResult
|
|
from opentelemetry.sdk.metrics.export import MetricExportResult
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult
|
|
from opentelemetry.trace import Status, StatusCode
|
|
|
|
from app.settings import settings
|
|
|
|
BANNED_FIELDS = {
|
|
"password",
|
|
"token",
|
|
"authorization",
|
|
"secret",
|
|
"filename",
|
|
"text",
|
|
"body",
|
|
"database_url",
|
|
"broker_url",
|
|
"image",
|
|
}
|
|
|
|
_dropped = None
|
|
_http_requests = None
|
|
_http_duration = None
|
|
_jobs = None
|
|
_job_duration = None
|
|
_ready_duration = None
|
|
_db_duration = None
|
|
_state = {
|
|
"queue_depth": 0,
|
|
"queue_age": 0.0,
|
|
"db_up": 0,
|
|
"pool_used": 0,
|
|
"pool_waiting": 0,
|
|
"worker_busy": 0,
|
|
"worker_capacity": 0,
|
|
"worker_consumers": 0,
|
|
"cpu": 0.0,
|
|
"memory": 0,
|
|
}
|
|
_cpu_last = None
|
|
_lock = threading.Lock()
|
|
|
|
|
|
class _GuardedExporter:
|
|
def __init__(self, inner, signal):
|
|
self.inner = inner
|
|
self.signal = signal
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(self.inner, name)
|
|
|
|
def export(self, items, **kwargs):
|
|
try:
|
|
result = self.inner.export(items)
|
|
except Exception:
|
|
_count_drop(self.signal, len(items))
|
|
return self._failure()
|
|
if getattr(result, "name", None) != "SUCCESS":
|
|
_count_drop(self.signal, len(items))
|
|
return result
|
|
|
|
def _failure(self):
|
|
if self.signal == "metrics":
|
|
return MetricExportResult.FAILURE
|
|
if self.signal == "logs":
|
|
return LogExportResult.FAILURE
|
|
return SpanExportResult.FAILURE
|
|
|
|
def shutdown(self, **kwargs):
|
|
try:
|
|
return self.inner.shutdown(**kwargs)
|
|
except Exception:
|
|
return None
|
|
|
|
def force_flush(self, timeout_millis=3000):
|
|
try:
|
|
return self.inner.force_flush(timeout_millis)
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
class _RawFormatter(logging.Formatter):
|
|
def format(self, record):
|
|
return record.getMessage()
|
|
|
|
|
|
def _count_drop(signal, count):
|
|
if _dropped is not None and count:
|
|
_dropped.add(count, {"signal": signal})
|
|
|
|
|
|
def _resource():
|
|
return Resource.create(
|
|
{
|
|
"service.name": settings.service_name,
|
|
"service.version": settings.service_version,
|
|
"service.instance.id": settings.instance_id,
|
|
"deployment.environment.name": settings.environment,
|
|
}
|
|
)
|
|
|
|
|
|
def _histogram(name, buckets):
|
|
return View(instrument_name=name, aggregation=ExplicitBucketHistogramAggregation(buckets))
|
|
|
|
|
|
def _otlp(endpoint):
|
|
return dict(
|
|
endpoint=endpoint,
|
|
timeout=5,
|
|
certificate_file=settings.tls_ca,
|
|
client_certificate_file=settings.client_cert,
|
|
client_key_file=settings.client_key,
|
|
)
|
|
|
|
|
|
def init():
|
|
global _dropped, _http_requests, _http_duration, _jobs, _job_duration, _ready_duration, _db_duration
|
|
resource = _resource()
|
|
endpoint = settings.otlp_endpoint
|
|
tracer_provider = TracerProvider(resource=resource)
|
|
tracer_provider.add_span_processor(
|
|
BatchSpanProcessor(
|
|
_GuardedExporter(OTLPSpanExporter(**_otlp(endpoint + "/v1/traces")), "spans"),
|
|
max_queue_size=256,
|
|
schedule_delay_millis=2000,
|
|
max_export_batch_size=64,
|
|
export_timeout_millis=5000,
|
|
)
|
|
)
|
|
trace.set_tracer_provider(tracer_provider)
|
|
|
|
reader = PeriodicExportingMetricReader(
|
|
_GuardedExporter(OTLPMetricExporter(**_otlp(endpoint + "/v1/metrics")), "metrics"),
|
|
export_interval_millis=5000,
|
|
export_timeout_millis=4000,
|
|
)
|
|
meter_provider = MeterProvider(
|
|
resource=resource,
|
|
metric_readers=[reader],
|
|
views=[
|
|
_histogram("http.server.duration", [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]),
|
|
_histogram("messenger.job.duration", [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]),
|
|
_histogram("messenger.attachment.time_to_ready", [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]),
|
|
_histogram("messenger.db.duration", [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]),
|
|
],
|
|
)
|
|
from opentelemetry import metrics
|
|
|
|
metrics.set_meter_provider(meter_provider)
|
|
meter = metrics.get_meter("messenger", settings.service_version)
|
|
_dropped = meter.create_counter("messenger.telemetry.dropped", unit="1")
|
|
_http_requests = meter.create_counter("http.server.requests", unit="1")
|
|
_http_duration = meter.create_histogram("http.server.duration", unit="s")
|
|
_jobs = meter.create_counter("messenger.jobs", unit="1")
|
|
_job_duration = meter.create_histogram("messenger.job.duration", unit="s")
|
|
_ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s")
|
|
_db_duration = meter.create_histogram("messenger.db.duration", unit="s")
|
|
meter.create_observable_gauge("messenger.queue.depth", callbacks=[_queue_depth])
|
|
meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[_queue_age], unit="s")
|
|
meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used])
|
|
meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting])
|
|
meter.create_observable_gauge("messenger.worker.busy", callbacks=[_worker_busy])
|
|
meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_worker_capacity])
|
|
meter.create_observable_gauge("messenger.worker.consumers", callbacks=[_worker_consumers])
|
|
meter.create_observable_gauge("messenger.db.up", callbacks=[_db_up])
|
|
meter.create_observable_gauge("messenger.process.cpu.utilization", callbacks=[_cpu])
|
|
meter.create_observable_gauge("messenger.process.memory", callbacks=[_memory], unit="By")
|
|
|
|
logger_provider = LoggerProvider(resource=resource)
|
|
logger_provider.add_log_record_processor(
|
|
BatchLogRecordProcessor(
|
|
_GuardedExporter(OTLPLogExporter(**_otlp(endpoint + "/v1/logs")), "logs"),
|
|
max_queue_size=256,
|
|
schedule_delay_millis=2000,
|
|
max_export_batch_size=64,
|
|
export_timeout_millis=5000,
|
|
)
|
|
)
|
|
root = logging.getLogger()
|
|
root.handlers.clear()
|
|
root.setLevel(logging.INFO)
|
|
stream = logging.StreamHandler()
|
|
stream.setFormatter(_RawFormatter())
|
|
root.addHandler(stream)
|
|
otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider)
|
|
root.addHandler(otel_handler)
|
|
for name in ("pika", "botocore", "boto3", "urllib3", "uvicorn", "uvicorn.error", "uvicorn.access", "redis"):
|
|
logging.getLogger(name).setLevel(logging.WARNING)
|
|
quiet = logging.getLogger("opentelemetry")
|
|
quiet.handlers.clear()
|
|
quiet.propagate = False
|
|
quiet.setLevel(logging.CRITICAL)
|
|
log_event("service.started", role=settings.role)
|
|
|
|
|
|
def tracer():
|
|
return trace.get_tracer("messenger", settings.service_version)
|
|
|
|
|
|
def current_ids():
|
|
span = trace.get_current_span()
|
|
ctx = span.get_span_context()
|
|
if not ctx or not ctx.is_valid:
|
|
return None, None
|
|
return format(ctx.trace_id, "032x"), format(ctx.span_id, "016x")
|
|
|
|
|
|
def log_event(event, severity="INFO", **fields):
|
|
trace_id, span_id = current_ids()
|
|
payload = {
|
|
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + f".{int(time.time() * 1000000) % 1000000:06d}Z",
|
|
"severity": severity,
|
|
"service.name": settings.service_name,
|
|
"event": event,
|
|
"trace_id": trace_id,
|
|
"span_id": span_id,
|
|
}
|
|
for key, value in fields.items():
|
|
if key in BANNED_FIELDS or value is None:
|
|
continue
|
|
if isinstance(value, (str, int, float, bool)):
|
|
payload[key] = value
|
|
level = getattr(logging, severity, logging.INFO)
|
|
logging.getLogger("messenger").log(level, json.dumps(payload, ensure_ascii=False))
|
|
|
|
|
|
def inject_carrier(carrier=None):
|
|
target = {} if carrier is None else carrier
|
|
inject(target)
|
|
return target
|
|
|
|
|
|
def extract_context(carrier):
|
|
clean = {}
|
|
for key, value in (carrier or {}).items():
|
|
if isinstance(value, bytes):
|
|
value = value.decode("utf-8", "replace")
|
|
if isinstance(value, str):
|
|
clean[str(key).lower()] = value
|
|
return extract(clean)
|
|
|
|
|
|
def record_http(method, route, status, duration):
|
|
if _http_requests is None:
|
|
return
|
|
klass = f"{int(status) // 100}xx"
|
|
attributes = {
|
|
"http.request.method": method,
|
|
"http.route": route,
|
|
"http.response.status_class": klass,
|
|
}
|
|
_http_requests.add(1, attributes)
|
|
_http_duration.record(duration, attributes)
|
|
|
|
|
|
def record_job(outcome, duration):
|
|
attributes = {"job_type": "attachment.process.v1", "outcome": outcome}
|
|
if _jobs is not None:
|
|
_jobs.add(1, attributes)
|
|
_job_duration.record(max(duration, 0.0), attributes)
|
|
|
|
|
|
def record_ready(duration):
|
|
if _ready_duration is not None:
|
|
_ready_duration.record(max(duration, 0.0), {"job_type": "attachment.process.v1"})
|
|
|
|
|
|
def record_db(duration):
|
|
if _db_duration is not None:
|
|
_db_duration.record(max(duration, 0.0), {"db.system": "postgresql"})
|
|
|
|
|
|
def update_runtime(**values):
|
|
with _lock:
|
|
_state.update(values)
|
|
|
|
|
|
def note_worker(busy, capacity):
|
|
update_runtime(worker_busy=busy, worker_capacity=capacity)
|
|
|
|
|
|
def sample_process():
|
|
global _cpu_last
|
|
now_cpu = time.process_time()
|
|
now_wall = time.monotonic()
|
|
ratio = 0.0
|
|
if _cpu_last is not None:
|
|
dc = now_cpu - _cpu_last[0]
|
|
dw = now_wall - _cpu_last[1]
|
|
if dw > 0:
|
|
ratio = dc / dw
|
|
_cpu_last = (now_cpu, now_wall)
|
|
memory = 0
|
|
try:
|
|
with open("/proc/self/statm", encoding="ascii") as handle:
|
|
resident = int(handle.read().split()[1])
|
|
memory = resident * os_page()
|
|
except (OSError, IndexError, ValueError):
|
|
memory = 0
|
|
update_runtime(cpu=ratio, memory=memory)
|
|
|
|
|
|
_page = None
|
|
|
|
|
|
def os_page():
|
|
global _page
|
|
if _page is None:
|
|
import os
|
|
|
|
_page = os.sysconf("SC_PAGE_SIZE")
|
|
return _page
|
|
|
|
|
|
def _obs(value, attributes=None):
|
|
from opentelemetry.metrics import Observation
|
|
|
|
return [Observation(value, attributes or {})]
|
|
|
|
|
|
def _queue_depth(_options):
|
|
return _obs(_state["queue_depth"], {"queue": "attachment.process.v1"})
|
|
|
|
|
|
def _queue_age(_options):
|
|
return _obs(_state["queue_age"], {"queue": "attachment.process.v1"})
|
|
|
|
|
|
def _pool_used(_options):
|
|
return _obs(_state["pool_used"], {"pool": "postgres"})
|
|
|
|
|
|
def _pool_waiting(_options):
|
|
return _obs(_state["pool_waiting"], {"pool": "postgres"})
|
|
|
|
|
|
def _worker_busy(_options):
|
|
if settings.role != "worker":
|
|
return []
|
|
return _obs(_state["worker_busy"], {"queue": "attachment.process.v1"})
|
|
|
|
|
|
def _worker_capacity(_options):
|
|
if settings.role != "worker":
|
|
return []
|
|
return _obs(_state["worker_capacity"], {"queue": "attachment.process.v1"})
|
|
|
|
|
|
def _worker_consumers(_options):
|
|
return _obs(_state["worker_consumers"], {"queue": "attachment.process.v1"})
|
|
|
|
|
|
def _db_up(_options):
|
|
return _obs(_state["db_up"])
|
|
|
|
|
|
def _cpu(_options):
|
|
return _obs(_state["cpu"])
|
|
|
|
|
|
def _memory(_options):
|
|
return _obs(_state["memory"])
|
|
|
|
|
|
def mark_error(span):
|
|
span.set_status(Status(StatusCode.ERROR))
|
|
|
|
|
|
def attach(ctx):
|
|
return otel_context.attach(ctx)
|