Add reproducible observability cost modes and external measurements

This commit is contained in:
Artyom
2026-11-06 21:22:38 +03:00
parent 20e7759f87
commit 202b985dea
16 changed files with 624 additions and 8 deletions
+1
View File
@@ -147,6 +147,7 @@ def create_app(
)
app.add_middleware(
RequestPipeline,
cost_experiment=transport.cost_experiment,
logger=logger,
max_bytes=transport.max_body_bytes,
timeout=transport.request_timeout_seconds,
+2 -1
View File
@@ -52,7 +52,8 @@ async def serve() -> None:
logger,
)
queue_metrics = QueueMetrics(await container.get(Database), logger)
queue_metrics.start()
if telemetry is not None:
queue_metrics.start()
relay.start()
heartbeat.start()
logger.info("relay.started")
+1
View File
@@ -47,6 +47,7 @@ class PlatformSettings(Settings):
db_connect_timeout_seconds: float = Field(default=1.0, gt=0, le=30)
db_acquire_timeout_seconds: float = Field(default=1.0, gt=0, le=30)
db_command_timeout_seconds: float = Field(default=2.0, gt=0, le=60)
otel_trace_ratio: float = Field(default=1.0, ge=0, le=1)
otel_sdk_disabled: bool = False
otel_exporter_otlp_endpoint: str = ""
otel_service_name: str = ""
@@ -5,6 +5,7 @@ from messenger.shared.config import Settings
class TransportSettings(Settings):
docs: bool = False
cost_experiment: bool = False
max_body_bytes: int = Field(default=1_048_576, ge=4_096)
request_timeout_seconds: float = Field(default=8.0, gt=0, le=300)
min_transfer_rate_bytes: int = Field(default=131_072, ge=1_024)
+12 -1
View File
@@ -89,6 +89,7 @@ class Pace:
class RequestPipeline:
__slots__ = (
"_app",
"_cost_experiment",
"_guard",
"_instance",
"_limits",
@@ -108,8 +109,10 @@ class RequestPipeline:
instance: str,
limits: BodyLimits = (),
min_rate: float = 131_072,
cost_experiment: bool = False,
) -> None:
self._app = app
self._cost_experiment = cost_experiment
self._logger = logger
self._max_bytes = max_bytes
self._limits = limits
@@ -155,11 +158,11 @@ class RequestPipeline:
raise HTTPException(413)
return message
rejection: Rejection | None = None
try:
async with asyncio.timeout(self._timeout) as deadline:
rejection = await self._guard(scope) or self._oversized(scope, limit)
if rejection is None:
await self._experiment(scope)
pace = Pace(deadline, self._timeout, self._min_rate)
await self._app(scope, *pace.around(limited_receive, wrapped_send))
except Unavailable as error:
@@ -201,6 +204,14 @@ class RequestPipeline:
)
self._log(scope, status_holder[0], request_id, started)
async def _experiment(self, scope: Scope) -> None:
if self._cost_experiment and scope["path"] == "/api/v1/users/me":
fault = header(scope, b"x-lab-cost")
if fault == "error":
raise Unavailable("controlled cost experiment")
if fault == "slow":
await asyncio.sleep(0.35)
def _limit(self, path: str) -> int:
for pattern, limit in self._limits:
if pattern.fullmatch(path):
@@ -18,9 +18,15 @@ from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import DropAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import Decision, ParentBased, Sampler, SamplingResult
from opentelemetry.sdk.trace.sampling import (
Decision,
ParentBased,
Sampler,
SamplingResult,
TraceIdRatioBased,
)
from opentelemetry.trace import Link, SpanKind, TraceState
from opentelemetry.util.types import Attributes
@@ -44,6 +50,9 @@ type Provider = TracerProvider | MeterProvider | LoggerProvider
class ServerRoots(Sampler):
def __init__(self, ratio: float = 1.0) -> None:
self._root = TraceIdRatioBased(ratio)
@override
def should_sample(
self,
@@ -56,7 +65,9 @@ class ServerRoots(Sampler):
trace_state: TraceState | None = None,
) -> SamplingResult:
if kind in ROOT_KINDS:
return SamplingResult(Decision.RECORD_AND_SAMPLE, attributes, trace_state)
return self._root.should_sample(
parent_context, trace_id, name, kind, attributes, links, trace_state
)
return SamplingResult(Decision.DROP, None, trace_state)
def get_description(self) -> str:
@@ -88,6 +99,16 @@ class OtlpLogHandler(logging.Handler):
)
class SpanAccounting(SpanProcessor):
def __init__(self, meter: MeterProvider) -> None:
self._ended = meter.get_meter("messenger.telemetry").create_counter(
"messenger.telemetry.spans.ended", unit="{span}"
)
def on_end(self, span: ReadableSpan) -> None:
self._ended.add(1, {"span.kind": span.kind.name.lower()})
class Telemetry:
__slots__ = ("_handler", "_providers")
@@ -119,7 +140,9 @@ def start_telemetry(
}
)
tracer = TracerProvider(
resource=resource, sampler=ParentBased(ServerRoots()), shutdown_on_exit=False
resource=resource,
sampler=ParentBased(ServerRoots(settings.otel_trace_ratio)),
shutdown_on_exit=False,
)
tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
meter = MeterProvider(
@@ -128,6 +151,7 @@ def start_telemetry(
views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())],
shutdown_on_exit=False,
)
tracer.add_span_processor(SpanAccounting(meter))
observe_process(meter)
logs = LoggerProvider(resource=resource, shutdown_on_exit=False)
logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))