Add reproducible observability cost modes and external measurements
This commit is contained in:
@@ -47,6 +47,7 @@ class PlatformSettings(Settings):
|
||||
db_connect_timeout_seconds: float = Field(default=1.0, gt=0, le=30)
|
||||
db_acquire_timeout_seconds: float = Field(default=1.0, gt=0, le=30)
|
||||
db_command_timeout_seconds: float = Field(default=2.0, gt=0, le=60)
|
||||
otel_trace_ratio: float = Field(default=1.0, ge=0, le=1)
|
||||
otel_sdk_disabled: bool = False
|
||||
otel_exporter_otlp_endpoint: str = ""
|
||||
otel_service_name: str = ""
|
||||
|
||||
@@ -5,6 +5,7 @@ from messenger.shared.config import Settings
|
||||
|
||||
class TransportSettings(Settings):
|
||||
docs: bool = False
|
||||
cost_experiment: bool = False
|
||||
max_body_bytes: int = Field(default=1_048_576, ge=4_096)
|
||||
request_timeout_seconds: float = Field(default=8.0, gt=0, le=300)
|
||||
min_transfer_rate_bytes: int = Field(default=131_072, ge=1_024)
|
||||
|
||||
@@ -89,6 +89,7 @@ class Pace:
|
||||
class RequestPipeline:
|
||||
__slots__ = (
|
||||
"_app",
|
||||
"_cost_experiment",
|
||||
"_guard",
|
||||
"_instance",
|
||||
"_limits",
|
||||
@@ -108,8 +109,10 @@ class RequestPipeline:
|
||||
instance: str,
|
||||
limits: BodyLimits = (),
|
||||
min_rate: float = 131_072,
|
||||
cost_experiment: bool = False,
|
||||
) -> None:
|
||||
self._app = app
|
||||
self._cost_experiment = cost_experiment
|
||||
self._logger = logger
|
||||
self._max_bytes = max_bytes
|
||||
self._limits = limits
|
||||
@@ -155,11 +158,11 @@ class RequestPipeline:
|
||||
raise HTTPException(413)
|
||||
return message
|
||||
|
||||
rejection: Rejection | None = None
|
||||
try:
|
||||
async with asyncio.timeout(self._timeout) as deadline:
|
||||
rejection = await self._guard(scope) or self._oversized(scope, limit)
|
||||
if rejection is None:
|
||||
await self._experiment(scope)
|
||||
pace = Pace(deadline, self._timeout, self._min_rate)
|
||||
await self._app(scope, *pace.around(limited_receive, wrapped_send))
|
||||
except Unavailable as error:
|
||||
@@ -201,6 +204,14 @@ class RequestPipeline:
|
||||
)
|
||||
self._log(scope, status_holder[0], request_id, started)
|
||||
|
||||
async def _experiment(self, scope: Scope) -> None:
|
||||
if self._cost_experiment and scope["path"] == "/api/v1/users/me":
|
||||
fault = header(scope, b"x-lab-cost")
|
||||
if fault == "error":
|
||||
raise Unavailable("controlled cost experiment")
|
||||
if fault == "slow":
|
||||
await asyncio.sleep(0.35)
|
||||
|
||||
def _limit(self, path: str) -> int:
|
||||
for pattern, limit in self._limits:
|
||||
if pattern.fullmatch(path):
|
||||
|
||||
@@ -18,9 +18,15 @@ from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import DropAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.trace.sampling import Decision, ParentBased, Sampler, SamplingResult
|
||||
from opentelemetry.sdk.trace.sampling import (
|
||||
Decision,
|
||||
ParentBased,
|
||||
Sampler,
|
||||
SamplingResult,
|
||||
TraceIdRatioBased,
|
||||
)
|
||||
from opentelemetry.trace import Link, SpanKind, TraceState
|
||||
from opentelemetry.util.types import Attributes
|
||||
|
||||
@@ -44,6 +50,9 @@ type Provider = TracerProvider | MeterProvider | LoggerProvider
|
||||
|
||||
|
||||
class ServerRoots(Sampler):
|
||||
def __init__(self, ratio: float = 1.0) -> None:
|
||||
self._root = TraceIdRatioBased(ratio)
|
||||
|
||||
@override
|
||||
def should_sample(
|
||||
self,
|
||||
@@ -56,7 +65,9 @@ class ServerRoots(Sampler):
|
||||
trace_state: TraceState | None = None,
|
||||
) -> SamplingResult:
|
||||
if kind in ROOT_KINDS:
|
||||
return SamplingResult(Decision.RECORD_AND_SAMPLE, attributes, trace_state)
|
||||
return self._root.should_sample(
|
||||
parent_context, trace_id, name, kind, attributes, links, trace_state
|
||||
)
|
||||
return SamplingResult(Decision.DROP, None, trace_state)
|
||||
|
||||
def get_description(self) -> str:
|
||||
@@ -88,6 +99,16 @@ class OtlpLogHandler(logging.Handler):
|
||||
)
|
||||
|
||||
|
||||
class SpanAccounting(SpanProcessor):
|
||||
def __init__(self, meter: MeterProvider) -> None:
|
||||
self._ended = meter.get_meter("messenger.telemetry").create_counter(
|
||||
"messenger.telemetry.spans.ended", unit="{span}"
|
||||
)
|
||||
|
||||
def on_end(self, span: ReadableSpan) -> None:
|
||||
self._ended.add(1, {"span.kind": span.kind.name.lower()})
|
||||
|
||||
|
||||
class Telemetry:
|
||||
__slots__ = ("_handler", "_providers")
|
||||
|
||||
@@ -119,7 +140,9 @@ def start_telemetry(
|
||||
}
|
||||
)
|
||||
tracer = TracerProvider(
|
||||
resource=resource, sampler=ParentBased(ServerRoots()), shutdown_on_exit=False
|
||||
resource=resource,
|
||||
sampler=ParentBased(ServerRoots(settings.otel_trace_ratio)),
|
||||
shutdown_on_exit=False,
|
||||
)
|
||||
tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
|
||||
meter = MeterProvider(
|
||||
@@ -128,6 +151,7 @@ def start_telemetry(
|
||||
views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())],
|
||||
shutdown_on_exit=False,
|
||||
)
|
||||
tracer.add_span_processor(SpanAccounting(meter))
|
||||
observe_process(meter)
|
||||
logs = LoggerProvider(resource=resource, shutdown_on_exit=False)
|
||||
logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
|
||||
|
||||
Reference in New Issue
Block a user