import json import logging import threading import time from opentelemetry import context as otel_context from opentelemetry import trace from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.propagate import extract, inject from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk._logs.export import LogExportResult from opentelemetry.sdk.metrics.export import MetricExportResult from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExportResult from opentelemetry.trace import Status, StatusCode from app.settings import settings BANNED_FIELDS = { "password", "token", "authorization", "secret", "filename", "text", "body", "database_url", "broker_url", "image", } _dropped = None _http_requests = None _http_duration = None _jobs = None _job_duration = None _ready_duration = None _db_duration = None _state = { "queue_depth": 0, "queue_age": 0.0, "db_up": 0, "pool_used": 0, "pool_waiting": 0, "worker_busy": 0, "worker_capacity": 0, "worker_consumers": 0, "cpu": 0.0, "memory": 0, } _cpu_last = None _lock = threading.Lock() class _GuardedExporter: def __init__(self, inner, signal): self.inner = inner self.signal = signal def __getattr__(self, name): return getattr(self.inner, name) def export(self, items, **kwargs): try: result = self.inner.export(items) except Exception: _count_drop(self.signal, len(items)) return self._failure() if getattr(result, "name", None) != "SUCCESS": _count_drop(self.signal, len(items)) return result def _failure(self): if self.signal == "metrics": return MetricExportResult.FAILURE if self.signal == "logs": return LogExportResult.FAILURE return SpanExportResult.FAILURE def shutdown(self, **kwargs): try: return self.inner.shutdown(**kwargs) except Exception: return None def force_flush(self, timeout_millis=3000): try: return self.inner.force_flush(timeout_millis) except Exception: return False class _RawFormatter(logging.Formatter): def format(self, record): return record.getMessage() def _count_drop(signal, count): if _dropped is not None and count: _dropped.add(count, {"signal": signal}) def _resource(): return Resource.create( { "service.name": settings.service_name, "service.version": settings.service_version, "service.instance.id": settings.instance_id, "deployment.environment.name": settings.environment, } ) def _histogram(name, buckets): return View(instrument_name=name, aggregation=ExplicitBucketHistogramAggregation(buckets)) def init(): global _dropped, _http_requests, _http_duration, _jobs, _job_duration, _ready_duration, _db_duration resource = _resource() endpoint = settings.otlp_endpoint tracer_provider = TracerProvider(resource=resource) tracer_provider.add_span_processor( BatchSpanProcessor( _GuardedExporter(OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=5), "spans"), max_queue_size=256, schedule_delay_millis=2000, max_export_batch_size=64, export_timeout_millis=5000, ) ) trace.set_tracer_provider(tracer_provider) reader = PeriodicExportingMetricReader( _GuardedExporter(OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=5), "metrics"), export_interval_millis=5000, export_timeout_millis=4000, ) meter_provider = MeterProvider( resource=resource, metric_readers=[reader], views=[ _histogram("http.server.duration", [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]), _histogram("messenger.job.duration", [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]), _histogram("messenger.attachment.time_to_ready", [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60]), _histogram("messenger.db.duration", [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]), ], ) from opentelemetry import metrics metrics.set_meter_provider(meter_provider) meter = metrics.get_meter("messenger", settings.service_version) _dropped = meter.create_counter("messenger.telemetry.dropped", unit="1") _http_requests = meter.create_counter("http.server.requests", unit="1") _http_duration = meter.create_histogram("http.server.duration", unit="s") _jobs = meter.create_counter("messenger.jobs", unit="1") _job_duration = meter.create_histogram("messenger.job.duration", unit="s") _ready_duration = meter.create_histogram("messenger.attachment.time_to_ready", unit="s") _db_duration = meter.create_histogram("messenger.db.duration", unit="s") meter.create_observable_gauge("messenger.queue.depth", callbacks=[_queue_depth]) meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[_queue_age], unit="s") meter.create_observable_gauge("messenger.db.pool.used", callbacks=[_pool_used]) meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[_pool_waiting]) meter.create_observable_gauge("messenger.worker.busy", callbacks=[_worker_busy]) meter.create_observable_gauge("messenger.worker.capacity", callbacks=[_worker_capacity]) meter.create_observable_gauge("messenger.worker.consumers", callbacks=[_worker_consumers]) meter.create_observable_gauge("messenger.db.up", callbacks=[_db_up]) meter.create_observable_gauge("messenger.process.cpu.utilization", callbacks=[_cpu]) meter.create_observable_gauge("messenger.process.memory", callbacks=[_memory], unit="By") logger_provider = LoggerProvider(resource=resource) logger_provider.add_log_record_processor( BatchLogRecordProcessor( _GuardedExporter(OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=5), "logs"), max_queue_size=256, schedule_delay_millis=2000, max_export_batch_size=64, export_timeout_millis=5000, ) ) root = logging.getLogger() root.handlers.clear() root.setLevel(logging.INFO) stream = logging.StreamHandler() stream.setFormatter(_RawFormatter()) root.addHandler(stream) otel_handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) root.addHandler(otel_handler) for name in ("pika", "botocore", "boto3", "urllib3", "uvicorn", "uvicorn.error", "uvicorn.access"): logging.getLogger(name).setLevel(logging.WARNING) quiet = logging.getLogger("opentelemetry") quiet.handlers.clear() quiet.propagate = False quiet.setLevel(logging.CRITICAL) log_event("service.started", role=settings.role) def tracer(): return trace.get_tracer("messenger", settings.service_version) def current_ids(): span = trace.get_current_span() ctx = span.get_span_context() if not ctx or not ctx.is_valid: return None, None return format(ctx.trace_id, "032x"), format(ctx.span_id, "016x") def log_event(event, severity="INFO", **fields): trace_id, span_id = current_ids() payload = { "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()) + f".{int(time.time() * 1000000) % 1000000:06d}Z", "severity": severity, "service.name": settings.service_name, "event": event, "trace_id": trace_id, "span_id": span_id, } for key, value in fields.items(): if key in BANNED_FIELDS or value is None: continue if isinstance(value, (str, int, float, bool)): payload[key] = value level = getattr(logging, severity, logging.INFO) logging.getLogger("messenger").log(level, json.dumps(payload, ensure_ascii=False)) def inject_carrier(carrier=None): target = {} if carrier is None else carrier inject(target) return target def extract_context(carrier): clean = {} for key, value in (carrier or {}).items(): if isinstance(value, bytes): value = value.decode("utf-8", "replace") if isinstance(value, str): clean[str(key).lower()] = value return extract(clean) def record_http(method, route, status, duration): if _http_requests is None: return klass = f"{int(status) // 100}xx" attributes = { "http.request.method": method, "http.route": route, "http.response.status_class": klass, } _http_requests.add(1, attributes) _http_duration.record(duration, attributes) def record_job(outcome, duration): attributes = {"job_type": "attachment.process.v1", "outcome": outcome} if _jobs is not None: _jobs.add(1, attributes) _job_duration.record(max(duration, 0.0), attributes) def record_ready(duration): if _ready_duration is not None: _ready_duration.record(max(duration, 0.0), {"job_type": "attachment.process.v1"}) def record_db(duration): if _db_duration is not None: _db_duration.record(max(duration, 0.0), {"db.system": "postgresql"}) def update_runtime(**values): with _lock: _state.update(values) def note_worker(busy, capacity): update_runtime(worker_busy=busy, worker_capacity=capacity) def sample_process(): global _cpu_last now_cpu = time.process_time() now_wall = time.monotonic() ratio = 0.0 if _cpu_last is not None: dc = now_cpu - _cpu_last[0] dw = now_wall - _cpu_last[1] if dw > 0: ratio = dc / dw _cpu_last = (now_cpu, now_wall) memory = 0 try: with open("/proc/self/statm", encoding="ascii") as handle: resident = int(handle.read().split()[1]) memory = resident * os_page() except (OSError, IndexError, ValueError): memory = 0 update_runtime(cpu=ratio, memory=memory) _page = None def os_page(): global _page if _page is None: import os _page = os.sysconf("SC_PAGE_SIZE") return _page def _obs(value, attributes=None): from opentelemetry.metrics import Observation return [Observation(value, attributes or {})] def _queue_depth(_options): return _obs(_state["queue_depth"], {"queue": "attachment.process.v1"}) def _queue_age(_options): return _obs(_state["queue_age"], {"queue": "attachment.process.v1"}) def _pool_used(_options): return _obs(_state["pool_used"], {"pool": "postgres"}) def _pool_waiting(_options): return _obs(_state["pool_waiting"], {"pool": "postgres"}) def _worker_busy(_options): if settings.role != "worker": return [] return _obs(_state["worker_busy"], {"queue": "attachment.process.v1"}) def _worker_capacity(_options): if settings.role != "worker": return [] return _obs(_state["worker_capacity"], {"queue": "attachment.process.v1"}) def _worker_consumers(_options): return _obs(_state["worker_consumers"], {"queue": "attachment.process.v1"}) def _db_up(_options): return _obs(_state["db_up"]) def _cpu(_options): return _obs(_state["cpu"]) def _memory(_options): return _obs(_state["memory"]) def mark_error(span): span.set_status(Status(StatusCode.ERROR)) def attach(ctx): return otel_context.attach(ctx)