Add reproducible observability cost modes and external measurements

This commit is contained in:
Artyom
2026-11-06 21:22:38 +03:00
parent 20e7759f87
commit 202b985dea
16 changed files with 624 additions and 8 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ COMPOSE ?= docker compose
RATE ?= 100
FROM_REV ?=
SHIFT ?= 0
SCRIPTS_PY = scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
SCRIPTS_PY = scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \
scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
SCENARIO ?= failover-upload
+3 -1
View File
@@ -45,6 +45,7 @@ x-pipeline: &pipeline
condition: service_healthy
x-otel-env: &otel-env
OTEL_TRACE_RATIO: ${OTEL_TRACE_RATIO:-1}
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
@@ -148,6 +149,7 @@ services:
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
COST_EXPERIMENT: ${COST_EXPERIMENT:-false}
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8}
@@ -438,7 +440,7 @@ services:
otel-collector:
<<: *obs-hardening
image: otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849
command: ["--config=/etc/otelcol/collector.yaml"]
command: ["--config=/etc/otelcol/${OTEL_COLLECTOR_CONFIG:-collector.yaml}"]
networks: [backend, edge]
environment:
GOMEMLIMIT: 200MiB
+122
View File
@@ -0,0 +1,122 @@
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
max_request_body_size: 8388608
processors:
tail_sampling:
decision_wait: 10s
num_traces: 2000
expected_new_traces_per_sec: 30
decision_cache:
sampled_cache_size: 10000
non_sampled_cache_size: 10000
policies:
- name: errors
type: status_code
status_code: {status_codes: [ERROR]}
- name: slow
type: latency
latency: {threshold_ms: 300}
- name: baseline
type: probabilistic
probabilistic: {sampling_percentage: 5}
memory_limiter:
check_interval: 1s
limit_percentage: 75
spike_limit_percentage: 20
redaction:
allow_all_keys: true
summary: silent
blocked_key_patterns:
- "(?i)^(.*[._-])?(password|passwd|secret|token|authorization|cookie|api[._-]?key|private[._-]?key|credentials?)$"
- "^http\\.(request|response)\\.header\\..+"
- "^url\\.(query|full)$"
- "^http\\.(url|target)$"
- "^(messenger\\.)?(message\\.)?(text|body)$"
- "^(file\\.)?filename$"
blocked_values:
- "(?i)bearer\\s+[A-Za-z0-9._~+/=-]{8,}"
- "(?i)basic\\s+[A-Za-z0-9+/=]{8,}"
- "(?i)x-amz-(signature|credential)=[^&\\s]+"
- "-----BEGIN [A-Z ]*PRIVATE KEY-----"
batch:
send_batch_size: 1024
send_batch_max_size: 2048
timeout: 2s
exporters:
otlp_http/tempo:
traces_endpoint: http://tempo:4318/v1/traces
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 20000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlp_http/prometheus:
metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 50000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlp_http/loki:
logs_endpoint: http://loki:3100/otlp/v1/logs
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 20000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
service:
extensions: [health_check]
telemetry:
logs:
level: info
metrics:
level: normal
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
without_type_suffix: false
without_units: false
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, redaction, tail_sampling, batch]
exporters: [otlp_http/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, redaction, batch]
exporters: [otlp_http/prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, redaction, batch]
exporters: [otlp_http/loki]
+122
View File
@@ -0,0 +1,122 @@
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
max_request_body_size: 8388608
processors:
tail_sampling:
decision_wait: 10s
num_traces: 2000
expected_new_traces_per_sec: 30
decision_cache:
sampled_cache_size: 10000
non_sampled_cache_size: 10000
policies:
- name: errors
type: status_code
status_code: {status_codes: [ERROR]}
- name: slow
type: latency
latency: {threshold_ms: 300}
- name: baseline
type: probabilistic
probabilistic: {sampling_percentage: 10}
memory_limiter:
check_interval: 1s
limit_percentage: 75
spike_limit_percentage: 20
redaction:
allow_all_keys: true
summary: silent
blocked_key_patterns:
- "(?i)^(.*[._-])?(password|passwd|secret|token|authorization|cookie|api[._-]?key|private[._-]?key|credentials?)$"
- "^http\\.(request|response)\\.header\\..+"
- "^url\\.(query|full)$"
- "^http\\.(url|target)$"
- "^(messenger\\.)?(message\\.)?(text|body)$"
- "^(file\\.)?filename$"
blocked_values:
- "(?i)bearer\\s+[A-Za-z0-9._~+/=-]{8,}"
- "(?i)basic\\s+[A-Za-z0-9+/=]{8,}"
- "(?i)x-amz-(signature|credential)=[^&\\s]+"
- "-----BEGIN [A-Z ]*PRIVATE KEY-----"
batch:
send_batch_size: 1024
send_batch_max_size: 2048
timeout: 2s
exporters:
otlp_http/tempo:
traces_endpoint: http://tempo:4318/v1/traces
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 20000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlp_http/prometheus:
metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 50000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlp_http/loki:
logs_endpoint: http://loki:3100/otlp/v1/logs
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 20000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
service:
extensions: [health_check]
telemetry:
logs:
level: info
metrics:
level: normal
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
without_type_suffix: false
without_units: false
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, redaction, tail_sampling, batch]
exporters: [otlp_http/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, redaction, batch]
exporters: [otlp_http/prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, redaction, batch]
exporters: [otlp_http/loki]
@@ -0,0 +1,13 @@
## Context
SDK/Collector и реальные отказы уровня 4 проверены. Измерения проводятся последовательно на одном Docker project, с сохранением volumes.
## Goals / Non-Goals
Измерить цену наблюдаемости и сохранность редких событий. Не искать максимальную пропускную способность и не публиковать решение.
## Decisions
Гипотеза: head 10% уменьшит ingest и CPU приложения, но потеряет большую часть редких ошибок/задержек; tail сохранит ошибки и slow, снижая хранение, но оставит цену SDK. Balanced использует tail с 5% normal и metric interval 30 s.
Режимы: off (SDK disabled), on (100%), head (10% ParentBased), tail (100% SDK, Collector errors OR latency>=300ms OR 10%), balanced (5% baseline tail, interval30s). Tail decision_wait10s, num_traces2000, bounded decision cache. Все остальные параметры, образ, ресурсы и нагрузка одинаковые.
Driver: 40 s, 20 arrivals/s, 64 client threads, 3 повтора режима; 2% error, 2% slow, остальные authenticated GET users/me. Это контролируемый HTTP/DB workload, не тест насыщения изображений; pipeline отдельно проверен уровнями3/4. Ошибки/slow задаются opt-in заголовком только на users/me после auth guard. Секреты существуют только в памяти клиента. Raw latency включает delay планировщика отдельно. После прогрева и перед/после серии — выдержка экспорта; fault hooks отключаются после опыта.
CPU: delta Docker cgroup total_usage / elapsed, включая дочерние процессы. RAM: working set usage-inactive_file, samples раз в 2s. Это внешний сбор, независимый от SDK. Ingest: Tempo/Loki bytes и Prometheus appended samples отдельно, включая постоянный фон self-scrape. Spans: SDK ended, Collector accepted/sent/failed, найденные server roots normal/error/slow; намеренный sampling отдельно от export failure. Search limit превышает число запросов. Нагрузка не параллелится с тестами/другими сериями; Docker background inventory сохраняется.
## Risks / Trade-offs
Один локальный Docker Desktop host, короткие steady windows, клиент на том же host; не production capacity. Ротация режимов по повтору уменьшает временной дрейф. Три повтора описывают наблюдаемый диапазон, не доверительный интервал. Tail сохраняет только события внутри decision window; поздние spans/retry обсуждаются отдельно. Результат гипотезы не предрешён.
## Validation
Unit tests sampler/fault opt-in, make check/lint/student-test, Collector validation, smoke. Реальные серии, raw requests/resources/counters/trace IDs, summary и ограничения. Архив только после завершения всех повторов.
@@ -0,0 +1,11 @@
## Why
Уровни 3/4 доказаны; цена SDK и sampling ещё не измерена. Трек 1 требует сравнения latency, CPU/RAM, ingest и потерь spans при одинаковой нагрузке.
## What Changes
- Настраиваемый root sampling ratio и счётчик завершённых sampled spans.
- Изолированный opt-in cost fault hook: 503 и задержка 350 ms только для контрольных GET users/me.
- Collector tail и balanced configurations; воспроизводимый open-loop driver, внешние Docker counters, raw results и заметки.
## Capabilities
### New Capabilities
- `lab6-observability-cost`: воспроизводимое сравнение пяти режимов наблюдаемости.
## Impact
Только собственный код, scripts, compose, infra и evidence. API-контракт не меняется; fault hook выключен по умолчанию. REPORT готовит следующий агент.
@@ -0,0 +1,11 @@
## ADDED Requirements
### Requirement: Reproducible cost comparison
The system SHALL support SDK off/on, head and tail sampling and a verified balanced configuration, measured under the same open-loop workload with raw latency, external CPU/RAM, ingest and span-loss evidence.
#### Scenario: Three repeat rounds
- **WHEN** the cost driver runs all five modes in three sequential rounds
- **THEN** each run records actual requests, resources, ingest counters and retained normal/error/slow traces without fabricating missing observations
### Requirement: Disabled experimental faults
Controlled error and slow hooks SHALL be opt-in and SHALL NOT affect default requests.
#### Scenario: Ordinary deployment
- **WHEN** the experiment setting is false
- **THEN** experiment headers cannot inject failure or delay
@@ -0,0 +1,6 @@
- [ ] 1. Sampling, span accounting and opt-in fault hooks with meaningful tests
- [ ] 2. Tail/balanced Collector configs and sequential external measurement driver
- [ ] 3. make check/lint/student-test, config validation and smoke
- [ ] 4. Three real rounds of off/on/head/tail/balanced; raw evidence
- [ ] 5. Compare observations, recommend verified balance, record limitations
- [ ] 6. Restore default configuration, final checks, sync/archive and stop own stand
+264
View File
@@ -0,0 +1,264 @@
import argparse
import concurrent.futures
import json
import os
import socket
import subprocess
import threading
import time
import uuid
from contextlib import closing
from http.client import HTTPConnection
from pathlib import Path
from obs_check import Api, http, now
from obs_incidents import collector_metrics
from obs_recovery import backends, compose
MODES = {
"off": ("true", "1", "collector.yaml", "10000"),
"on": ("false", "1", "collector.yaml", "10000"),
"head": ("false", "0.1", "collector.yaml", "10000"),
"tail": ("false", "1", "collector-tail.yaml", "10000"),
"balanced": ("false", "1", "collector-balanced.yaml", "30000"),
}
def configure(mode):
disabled, ratio, config, interval = MODES[mode]
os.environ.update(
OTEL_SDK_DISABLED=disabled,
OTEL_TRACE_RATIO=ratio,
OTEL_COLLECTOR_CONFIG=config,
OTEL_METRIC_EXPORT_INTERVAL=interval,
COST_EXPERIMENT="true",
)
compose(
"up",
"-d",
"--no-deps",
"--force-recreate",
"--wait",
"otel-collector",
"api",
"worker",
"relay",
)
subprocess.run(["scripts/lb-wait.sh"], check=True, capture_output=True)
class Engine(HTTPConnection):
def __init__(self, path):
super().__init__("localhost", timeout=10)
self.path = path
def connect(self):
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.sock.settimeout(self.timeout)
self.sock.connect(self.path)
def engine_read(path, route):
with closing(Engine(path)) as connection:
connection.request("GET", route)
response = connection.getresponse()
if response.status != 200:
raise RuntimeError("Docker stats unavailable")
return json.load(response)
class Resources:
def __init__(self):
host = subprocess.check_output(
["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], text=True
).strip()
if not host.startswith("unix://"):
raise RuntimeError("cost measurement requires a local Docker socket")
self.path = host[7:]
self.ids = compose("ps", "-q").splitlines()
self.samples = []
self.stop = threading.Event()
self.thread = threading.Thread(target=self.poll)
def sample(self):
rows = []
for identity in self.ids:
data = engine_read(
self.path, f"/containers/{identity}/stats?stream=false&one-shot=true"
)
memory = data["memory_stats"]
rows.append(
{
"id": identity[:12],
"name": data["name"],
"cpu_ns": data["cpu_stats"]["cpu_usage"]["total_usage"],
"working_set_bytes": memory["usage"]
- memory.get("stats", {}).get("inactive_file", 0),
"memory_limit": memory["limit"],
}
)
return {"at": time.time(), "containers": rows}
def poll(self):
while not self.stop.wait(2):
self.samples.append(self.sample())
def begin(self):
self.samples.append(self.sample())
self.thread.start()
def finish(self):
self.stop.set()
self.thread.join()
self.samples.append(self.sample())
return self.samples
def counters(url, names):
raw = http("GET", url + "/metrics", expect=(200,))[3].decode()
result = {}
for line in raw.splitlines():
if not line or line.startswith("#"):
continue
name = line.split("{", 1)[0].split(" ", 1)[0]
if name in names:
result[name] = result.get(name, 0) + float(line.rsplit(" ", 1)[1])
return result
def accounting(b, hosts):
metric = b.promql('sum(messenger_telemetry_spans_ended_total{instance=~"' + hosts + '"})')
return {
"collector": collector_metrics(),
"sdk_ended_spans": float(metric[0]["value"][1]) if metric else 0,
"tempo": counters(b.tempo, {"tempo_distributor_bytes_received_total"}),
"loki": counters(b.loki, {"loki_distributor_bytes_received_total"}),
"prometheus": counters(b.prometheus, {"prometheus_tsdb_head_samples_appended_total"}),
}
def run(mode, repeat, duration, rate, destination):
configure(mode)
b = backends()
api = Api(os.environ["BASE_URL"])
_, username, password, _ = api.register()
auth, _ = api.login(username, password)
for _ in range(20):
api.call("GET", "/api/v1/users/me", headers=auth)
time.sleep(35)
hosts = "|".join(
subprocess.check_output(
["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True
).strip()
+ "-.*"
for cid in compose("ps", "-q", "api", "worker", "relay").splitlines()
)
before = accounting(b, hosts)
background = subprocess.check_output(
["docker", "ps", "--format", '{{.Names}} {{.Label "com.docker.compose.project"}}'],
text=True,
).splitlines()
resources = Resources()
resources.begin()
started = time.time()
origin = time.perf_counter()
prefix = "cost-" + uuid.uuid4().hex[:12]
def request(index):
category = "error" if index % 50 == 0 else "slow" if index % 50 == 1 else "normal"
rid = prefix + "-" + category + "-" + str(index)
began = time.perf_counter()
headers = dict(auth, **{"X-Request-Id": rid, "X-Lab-Cost": category})
try:
status = http("GET", api.base + "/api/v1/users/me", headers=headers)[0]
except OSError:
status = 0
return {
"id": rid,
"category": category,
"status": status,
"latency_ms": (time.perf_counter() - began) * 1000,
"schedule_lag_ms": (began - origin - index / rate) * 1000,
}
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool:
futures = []
for index in range(int(duration * rate)):
time.sleep(max(0, origin + index / rate - time.perf_counter()))
futures.append(pool.submit(request, index))
requests = [future.result() for future in futures]
finished = time.time()
samples = resources.finish()
time.sleep(35)
after = accounting(b, hosts)
retained = {}
for category in ("normal", "error", "slow"):
found = b.traceql(
'{ span.messenger.request_id =~ "' + prefix + "-" + category + '-.*" }',
started - 1,
time.time() + 1,
limit=5000,
)
retained[category] = sorted({t["traceID"] for t in found})
api.logout(auth)
unexpected = [r for r in requests if r["status"] != (503 if r["category"] == "error" else 200)]
report = {
"utc": now(),
"sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
"mode": mode,
"repeat": repeat,
"duration": duration,
"rate": rate,
"started": started,
"finished": finished,
"background": background,
"requests": requests,
"resources": samples,
"before": before,
"after": after,
"retained_trace_ids": retained,
"unexpected_responses": len(unexpected),
"result": "PASS" if not unexpected else "FAIL",
}
destination.write_text(json.dumps(report, indent=2) + "\n")
print(
json.dumps(
{
"mode": mode,
"repeat": repeat,
"result": report["result"],
"retained": {k: len(v) for k, v in retained.items()},
}
),
flush=True,
)
if unexpected:
raise RuntimeError("unexpected benchmark responses; raw series saved")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--mode", choices=MODES)
parser.add_argument("--repeat", type=int, default=1)
parser.add_argument("--duration", type=int, default=40)
parser.add_argument("--rate", type=int, default=20)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
if args.output.exists():
raise RuntimeError("refusing to overwrite evidence")
args.output.parent.mkdir(parents=True, exist_ok=True)
try:
run(args.mode, args.repeat, args.duration, args.rate, args.output)
finally:
os.environ.update(
OTEL_SDK_DISABLED="false",
OTEL_TRACE_RATIO="1",
OTEL_COLLECTOR_CONFIG="collector.yaml",
OTEL_METRIC_EXPORT_INTERVAL="10000",
COST_EXPERIMENT="false",
)
compose("up", "-d", "--no-deps", "--wait", "otel-collector", "api", "worker", "relay")
if __name__ == "__main__":
main()
+1
View File
@@ -147,6 +147,7 @@ def create_app(
)
app.add_middleware(
RequestPipeline,
cost_experiment=transport.cost_experiment,
logger=logger,
max_bytes=transport.max_body_bytes,
timeout=transport.request_timeout_seconds,
+2 -1
View File
@@ -52,7 +52,8 @@ async def serve() -> None:
logger,
)
queue_metrics = QueueMetrics(await container.get(Database), logger)
queue_metrics.start()
if telemetry is not None:
queue_metrics.start()
relay.start()
heartbeat.start()
logger.info("relay.started")
+1
View File
@@ -47,6 +47,7 @@ class PlatformSettings(Settings):
db_connect_timeout_seconds: float = Field(default=1.0, gt=0, le=30)
db_acquire_timeout_seconds: float = Field(default=1.0, gt=0, le=30)
db_command_timeout_seconds: float = Field(default=2.0, gt=0, le=60)
otel_trace_ratio: float = Field(default=1.0, ge=0, le=1)
otel_sdk_disabled: bool = False
otel_exporter_otlp_endpoint: str = ""
otel_service_name: str = ""
@@ -5,6 +5,7 @@ from messenger.shared.config import Settings
class TransportSettings(Settings):
docs: bool = False
cost_experiment: bool = False
max_body_bytes: int = Field(default=1_048_576, ge=4_096)
request_timeout_seconds: float = Field(default=8.0, gt=0, le=300)
min_transfer_rate_bytes: int = Field(default=131_072, ge=1_024)
+12 -1
View File
@@ -89,6 +89,7 @@ class Pace:
class RequestPipeline:
__slots__ = (
"_app",
"_cost_experiment",
"_guard",
"_instance",
"_limits",
@@ -108,8 +109,10 @@ class RequestPipeline:
instance: str,
limits: BodyLimits = (),
min_rate: float = 131_072,
cost_experiment: bool = False,
) -> None:
self._app = app
self._cost_experiment = cost_experiment
self._logger = logger
self._max_bytes = max_bytes
self._limits = limits
@@ -155,11 +158,11 @@ class RequestPipeline:
raise HTTPException(413)
return message
rejection: Rejection | None = None
try:
async with asyncio.timeout(self._timeout) as deadline:
rejection = await self._guard(scope) or self._oversized(scope, limit)
if rejection is None:
await self._experiment(scope)
pace = Pace(deadline, self._timeout, self._min_rate)
await self._app(scope, *pace.around(limited_receive, wrapped_send))
except Unavailable as error:
@@ -201,6 +204,14 @@ class RequestPipeline:
)
self._log(scope, status_holder[0], request_id, started)
async def _experiment(self, scope: Scope) -> None:
if self._cost_experiment and scope["path"] == "/api/v1/users/me":
fault = header(scope, b"x-lab-cost")
if fault == "error":
raise Unavailable("controlled cost experiment")
if fault == "slow":
await asyncio.sleep(0.35)
def _limit(self, path: str) -> int:
for pattern, limit in self._limits:
if pattern.fullmatch(path):
@@ -18,9 +18,15 @@ from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.metrics.view import DropAggregation, View
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.sampling import Decision, ParentBased, Sampler, SamplingResult
from opentelemetry.sdk.trace.sampling import (
Decision,
ParentBased,
Sampler,
SamplingResult,
TraceIdRatioBased,
)
from opentelemetry.trace import Link, SpanKind, TraceState
from opentelemetry.util.types import Attributes
@@ -44,6 +50,9 @@ type Provider = TracerProvider | MeterProvider | LoggerProvider
class ServerRoots(Sampler):
def __init__(self, ratio: float = 1.0) -> None:
self._root = TraceIdRatioBased(ratio)
@override
def should_sample(
self,
@@ -56,7 +65,9 @@ class ServerRoots(Sampler):
trace_state: TraceState | None = None,
) -> SamplingResult:
if kind in ROOT_KINDS:
return SamplingResult(Decision.RECORD_AND_SAMPLE, attributes, trace_state)
return self._root.should_sample(
parent_context, trace_id, name, kind, attributes, links, trace_state
)
return SamplingResult(Decision.DROP, None, trace_state)
def get_description(self) -> str:
@@ -88,6 +99,16 @@ class OtlpLogHandler(logging.Handler):
)
class SpanAccounting(SpanProcessor):
def __init__(self, meter: MeterProvider) -> None:
self._ended = meter.get_meter("messenger.telemetry").create_counter(
"messenger.telemetry.spans.ended", unit="{span}"
)
def on_end(self, span: ReadableSpan) -> None:
self._ended.add(1, {"span.kind": span.kind.name.lower()})
class Telemetry:
__slots__ = ("_handler", "_providers")
@@ -119,7 +140,9 @@ def start_telemetry(
}
)
tracer = TracerProvider(
resource=resource, sampler=ParentBased(ServerRoots()), shutdown_on_exit=False
resource=resource,
sampler=ParentBased(ServerRoots(settings.otel_trace_ratio)),
shutdown_on_exit=False,
)
tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
meter = MeterProvider(
@@ -128,6 +151,7 @@ def start_telemetry(
views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())],
shutdown_on_exit=False,
)
tracer.add_span_processor(SpanAccounting(meter))
observe_process(meter)
logs = LoggerProvider(resource=resource, shutdown_on_exit=False)
logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
+26
View File
@@ -0,0 +1,26 @@
import pytest
from opentelemetry.sdk.trace.sampling import Decision
from opentelemetry.trace import SpanKind
from conftest import actor, register, running_app
from messenger.shared.infrastructure.telemetry import ServerRoots
@pytest.mark.parametrize("enabled", [False, True])
async def test_controlled_errors_are_opt_in(enabled: bool) -> None:
async with running_app(cost_experiment=enabled) as client:
user = await register(client, "cost")
result = await client.get(
"/api/v1/users/me", headers={**actor(user), "X-Lab-Cost": "error"}
)
assert result.status_code == (503 if enabled else 200)
assert (await client.get("/api/v1/users/me", headers=actor(user))).status_code == 200
def test_head_zero_drops_roots_and_full_keeps_them() -> None:
for kind in (SpanKind.SERVER, SpanKind.PRODUCER, SpanKind.CONSUMER):
assert ServerRoots(0).should_sample(None, 42, "root", kind).decision == Decision.DROP
assert (
ServerRoots(1).should_sample(None, 42, "root", kind).decision
== Decision.RECORD_AND_SAMPLE
)