forked from mtusi2026/mtusi2026lab6
Add reproducible observability cost modes and external measurements
This commit is contained in:
@@ -10,7 +10,7 @@ COMPOSE ?= docker compose
|
||||
RATE ?= 100
|
||||
FROM_REV ?=
|
||||
SHIFT ?= 0
|
||||
SCRIPTS_PY = scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
|
||||
SCRIPTS_PY = scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
|
||||
scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \
|
||||
scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
|
||||
SCENARIO ?= failover-upload
|
||||
|
||||
+3
-1
@@ -45,6 +45,7 @@ x-pipeline: &pipeline
|
||||
condition: service_healthy
|
||||
|
||||
x-otel-env: &otel-env
|
||||
OTEL_TRACE_RATIO: ${OTEL_TRACE_RATIO:-1}
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf}
|
||||
@@ -148,6 +149,7 @@ services:
|
||||
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
COST_EXPERIMENT: ${COST_EXPERIMENT:-false}
|
||||
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
|
||||
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
|
||||
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8}
|
||||
@@ -438,7 +440,7 @@ services:
|
||||
otel-collector:
|
||||
<<: *obs-hardening
|
||||
image: otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849
|
||||
command: ["--config=/etc/otelcol/collector.yaml"]
|
||||
command: ["--config=/etc/otelcol/${OTEL_COLLECTOR_CONFIG:-collector.yaml}"]
|
||||
networks: [backend, edge]
|
||||
environment:
|
||||
GOMEMLIMIT: 200MiB
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
max_request_body_size: 8388608
|
||||
|
||||
processors:
|
||||
tail_sampling:
|
||||
decision_wait: 10s
|
||||
num_traces: 2000
|
||||
expected_new_traces_per_sec: 30
|
||||
decision_cache:
|
||||
sampled_cache_size: 10000
|
||||
non_sampled_cache_size: 10000
|
||||
policies:
|
||||
- name: errors
|
||||
type: status_code
|
||||
status_code: {status_codes: [ERROR]}
|
||||
- name: slow
|
||||
type: latency
|
||||
latency: {threshold_ms: 300}
|
||||
- name: baseline
|
||||
type: probabilistic
|
||||
probabilistic: {sampling_percentage: 5}
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_percentage: 75
|
||||
spike_limit_percentage: 20
|
||||
redaction:
|
||||
allow_all_keys: true
|
||||
summary: silent
|
||||
blocked_key_patterns:
|
||||
- "(?i)^(.*[._-])?(password|passwd|secret|token|authorization|cookie|api[._-]?key|private[._-]?key|credentials?)$"
|
||||
- "^http\\.(request|response)\\.header\\..+"
|
||||
- "^url\\.(query|full)$"
|
||||
- "^http\\.(url|target)$"
|
||||
- "^(messenger\\.)?(message\\.)?(text|body)$"
|
||||
- "^(file\\.)?filename$"
|
||||
blocked_values:
|
||||
- "(?i)bearer\\s+[A-Za-z0-9._~+/=-]{8,}"
|
||||
- "(?i)basic\\s+[A-Za-z0-9+/=]{8,}"
|
||||
- "(?i)x-amz-(signature|credential)=[^&\\s]+"
|
||||
- "-----BEGIN [A-Z ]*PRIVATE KEY-----"
|
||||
batch:
|
||||
send_batch_size: 1024
|
||||
send_batch_max_size: 2048
|
||||
timeout: 2s
|
||||
|
||||
exporters:
|
||||
otlp_http/tempo:
|
||||
traces_endpoint: http://tempo:4318/v1/traces
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
sizer: items
|
||||
queue_size: 20000
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
otlp_http/prometheus:
|
||||
metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
sizer: items
|
||||
queue_size: 50000
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
otlp_http/loki:
|
||||
logs_endpoint: http://loki:3100/otlp/v1/logs
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
sizer: items
|
||||
queue_size: 20000
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
logs:
|
||||
level: info
|
||||
metrics:
|
||||
level: normal
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
without_type_suffix: false
|
||||
without_units: false
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, redaction, tail_sampling, batch]
|
||||
exporters: [otlp_http/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, redaction, batch]
|
||||
exporters: [otlp_http/prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, redaction, batch]
|
||||
exporters: [otlp_http/loki]
|
||||
@@ -0,0 +1,122 @@
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
max_request_body_size: 8388608
|
||||
|
||||
processors:
|
||||
tail_sampling:
|
||||
decision_wait: 10s
|
||||
num_traces: 2000
|
||||
expected_new_traces_per_sec: 30
|
||||
decision_cache:
|
||||
sampled_cache_size: 10000
|
||||
non_sampled_cache_size: 10000
|
||||
policies:
|
||||
- name: errors
|
||||
type: status_code
|
||||
status_code: {status_codes: [ERROR]}
|
||||
- name: slow
|
||||
type: latency
|
||||
latency: {threshold_ms: 300}
|
||||
- name: baseline
|
||||
type: probabilistic
|
||||
probabilistic: {sampling_percentage: 10}
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_percentage: 75
|
||||
spike_limit_percentage: 20
|
||||
redaction:
|
||||
allow_all_keys: true
|
||||
summary: silent
|
||||
blocked_key_patterns:
|
||||
- "(?i)^(.*[._-])?(password|passwd|secret|token|authorization|cookie|api[._-]?key|private[._-]?key|credentials?)$"
|
||||
- "^http\\.(request|response)\\.header\\..+"
|
||||
- "^url\\.(query|full)$"
|
||||
- "^http\\.(url|target)$"
|
||||
- "^(messenger\\.)?(message\\.)?(text|body)$"
|
||||
- "^(file\\.)?filename$"
|
||||
blocked_values:
|
||||
- "(?i)bearer\\s+[A-Za-z0-9._~+/=-]{8,}"
|
||||
- "(?i)basic\\s+[A-Za-z0-9+/=]{8,}"
|
||||
- "(?i)x-amz-(signature|credential)=[^&\\s]+"
|
||||
- "-----BEGIN [A-Z ]*PRIVATE KEY-----"
|
||||
batch:
|
||||
send_batch_size: 1024
|
||||
send_batch_max_size: 2048
|
||||
timeout: 2s
|
||||
|
||||
exporters:
|
||||
otlp_http/tempo:
|
||||
traces_endpoint: http://tempo:4318/v1/traces
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
sizer: items
|
||||
queue_size: 20000
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
otlp_http/prometheus:
|
||||
metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
sizer: items
|
||||
queue_size: 50000
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
otlp_http/loki:
|
||||
logs_endpoint: http://loki:3100/otlp/v1/logs
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
sizer: items
|
||||
queue_size: 20000
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
logs:
|
||||
level: info
|
||||
metrics:
|
||||
level: normal
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
without_type_suffix: false
|
||||
without_units: false
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, redaction, tail_sampling, batch]
|
||||
exporters: [otlp_http/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, redaction, batch]
|
||||
exporters: [otlp_http/prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, redaction, batch]
|
||||
exporters: [otlp_http/loki]
|
||||
@@ -0,0 +1,13 @@
|
||||
## Context
|
||||
SDK/Collector и реальные отказы уровня 4 проверены. Измерения проводятся последовательно на одном Docker project, с сохранением volumes.
|
||||
## Goals / Non-Goals
|
||||
Измерить цену наблюдаемости и сохранность редких событий. Не искать максимальную пропускную способность и не публиковать решение.
|
||||
## Decisions
|
||||
Гипотеза: head 10% уменьшит ingest и CPU приложения, но потеряет большую часть редких ошибок/задержек; tail сохранит ошибки и slow, снижая хранение, но оставит цену SDK. Balanced использует tail с 5% normal и metric interval 30 s.
|
||||
Режимы: off (SDK disabled), on (100%), head (10% ParentBased), tail (100% SDK, Collector errors OR latency>=300ms OR 10%), balanced (5% baseline tail, interval30s). Tail decision_wait10s, num_traces2000, bounded decision cache. Все остальные параметры, образ, ресурсы и нагрузка одинаковые.
|
||||
Driver: 40 s, 20 arrivals/s, 64 client threads, 3 повтора режима; 2% error, 2% slow, остальные authenticated GET users/me. Это контролируемый HTTP/DB workload, не тест насыщения изображений; pipeline отдельно проверен уровнями3/4. Ошибки/slow задаются opt-in заголовком только на users/me после auth guard. Секреты существуют только в памяти клиента. Raw latency включает delay планировщика отдельно. После прогрева и перед/после серии — выдержка экспорта; fault hooks отключаются после опыта.
|
||||
CPU: delta Docker cgroup total_usage / elapsed, включая дочерние процессы. RAM: working set usage-inactive_file, samples раз в 2s. Это внешний сбор, независимый от SDK. Ingest: Tempo/Loki bytes и Prometheus appended samples отдельно, включая постоянный фон self-scrape. Spans: SDK ended, Collector accepted/sent/failed, найденные server roots normal/error/slow; намеренный sampling отдельно от export failure. Search limit превышает число запросов. Нагрузка не параллелится с тестами/другими сериями; Docker background inventory сохраняется.
|
||||
## Risks / Trade-offs
|
||||
Один локальный Docker Desktop host, короткие steady windows, клиент на том же host; не production capacity. Ротация режимов по повтору уменьшает временной дрейф. Три повтора описывают наблюдаемый диапазон, не доверительный интервал. Tail сохраняет только события внутри decision window; поздние spans/retry обсуждаются отдельно. Результат гипотезы не предрешён.
|
||||
## Validation
|
||||
Unit tests sampler/fault opt-in, make check/lint/student-test, Collector validation, smoke. Реальные серии, raw requests/resources/counters/trace IDs, summary и ограничения. Архив только после завершения всех повторов.
|
||||
@@ -0,0 +1,11 @@
|
||||
## Why
|
||||
Уровни 3/4 доказаны; цена SDK и sampling ещё не измерена. Трек 1 требует сравнения latency, CPU/RAM, ingest и потерь spans при одинаковой нагрузке.
|
||||
## What Changes
|
||||
- Настраиваемый root sampling ratio и счётчик завершённых sampled spans.
|
||||
- Изолированный opt-in cost fault hook: 503 и задержка 350 ms только для контрольных GET users/me.
|
||||
- Collector tail и balanced configurations; воспроизводимый open-loop driver, внешние Docker counters, raw results и заметки.
|
||||
## Capabilities
|
||||
### New Capabilities
|
||||
- `lab6-observability-cost`: воспроизводимое сравнение пяти режимов наблюдаемости.
|
||||
## Impact
|
||||
Только собственный код, scripts, compose, infra и evidence. API-контракт не меняется; fault hook выключен по умолчанию. REPORT готовит следующий агент.
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
## ADDED Requirements
|
||||
### Requirement: Reproducible cost comparison
|
||||
The system SHALL support SDK off/on, head and tail sampling and a verified balanced configuration, measured under the same open-loop workload with raw latency, external CPU/RAM, ingest and span-loss evidence.
|
||||
#### Scenario: Three repeat rounds
|
||||
- **WHEN** the cost driver runs all five modes in three sequential rounds
|
||||
- **THEN** each run records actual requests, resources, ingest counters and retained normal/error/slow traces without fabricating missing observations
|
||||
### Requirement: Disabled experimental faults
|
||||
Controlled error and slow hooks SHALL be opt-in and SHALL NOT affect default requests.
|
||||
#### Scenario: Ordinary deployment
|
||||
- **WHEN** the experiment setting is false
|
||||
- **THEN** experiment headers cannot inject failure or delay
|
||||
@@ -0,0 +1,6 @@
|
||||
- [ ] 1. Sampling, span accounting and opt-in fault hooks with meaningful tests
|
||||
- [ ] 2. Tail/balanced Collector configs and sequential external measurement driver
|
||||
- [ ] 3. make check/lint/student-test, config validation and smoke
|
||||
- [ ] 4. Three real rounds of off/on/head/tail/balanced; raw evidence
|
||||
- [ ] 5. Compare observations, recommend verified balance, record limitations
|
||||
- [ ] 6. Restore default configuration, final checks, sync/archive and stop own stand
|
||||
@@ -0,0 +1,264 @@
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import closing
|
||||
from http.client import HTTPConnection
|
||||
from pathlib import Path
|
||||
|
||||
from obs_check import Api, http, now
|
||||
from obs_incidents import collector_metrics
|
||||
from obs_recovery import backends, compose
|
||||
|
||||
MODES = {
|
||||
"off": ("true", "1", "collector.yaml", "10000"),
|
||||
"on": ("false", "1", "collector.yaml", "10000"),
|
||||
"head": ("false", "0.1", "collector.yaml", "10000"),
|
||||
"tail": ("false", "1", "collector-tail.yaml", "10000"),
|
||||
"balanced": ("false", "1", "collector-balanced.yaml", "30000"),
|
||||
}
|
||||
|
||||
|
||||
def configure(mode):
|
||||
disabled, ratio, config, interval = MODES[mode]
|
||||
os.environ.update(
|
||||
OTEL_SDK_DISABLED=disabled,
|
||||
OTEL_TRACE_RATIO=ratio,
|
||||
OTEL_COLLECTOR_CONFIG=config,
|
||||
OTEL_METRIC_EXPORT_INTERVAL=interval,
|
||||
COST_EXPERIMENT="true",
|
||||
)
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--force-recreate",
|
||||
"--wait",
|
||||
"otel-collector",
|
||||
"api",
|
||||
"worker",
|
||||
"relay",
|
||||
)
|
||||
subprocess.run(["scripts/lb-wait.sh"], check=True, capture_output=True)
|
||||
|
||||
|
||||
class Engine(HTTPConnection):
|
||||
def __init__(self, path):
|
||||
super().__init__("localhost", timeout=10)
|
||||
self.path = path
|
||||
|
||||
def connect(self):
|
||||
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
self.sock.settimeout(self.timeout)
|
||||
self.sock.connect(self.path)
|
||||
|
||||
|
||||
def engine_read(path, route):
|
||||
with closing(Engine(path)) as connection:
|
||||
connection.request("GET", route)
|
||||
response = connection.getresponse()
|
||||
if response.status != 200:
|
||||
raise RuntimeError("Docker stats unavailable")
|
||||
return json.load(response)
|
||||
|
||||
|
||||
class Resources:
|
||||
def __init__(self):
|
||||
host = subprocess.check_output(
|
||||
["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], text=True
|
||||
).strip()
|
||||
if not host.startswith("unix://"):
|
||||
raise RuntimeError("cost measurement requires a local Docker socket")
|
||||
self.path = host[7:]
|
||||
self.ids = compose("ps", "-q").splitlines()
|
||||
self.samples = []
|
||||
self.stop = threading.Event()
|
||||
self.thread = threading.Thread(target=self.poll)
|
||||
|
||||
def sample(self):
|
||||
rows = []
|
||||
for identity in self.ids:
|
||||
data = engine_read(
|
||||
self.path, f"/containers/{identity}/stats?stream=false&one-shot=true"
|
||||
)
|
||||
memory = data["memory_stats"]
|
||||
rows.append(
|
||||
{
|
||||
"id": identity[:12],
|
||||
"name": data["name"],
|
||||
"cpu_ns": data["cpu_stats"]["cpu_usage"]["total_usage"],
|
||||
"working_set_bytes": memory["usage"]
|
||||
- memory.get("stats", {}).get("inactive_file", 0),
|
||||
"memory_limit": memory["limit"],
|
||||
}
|
||||
)
|
||||
return {"at": time.time(), "containers": rows}
|
||||
|
||||
def poll(self):
|
||||
while not self.stop.wait(2):
|
||||
self.samples.append(self.sample())
|
||||
|
||||
def begin(self):
|
||||
self.samples.append(self.sample())
|
||||
self.thread.start()
|
||||
|
||||
def finish(self):
|
||||
self.stop.set()
|
||||
self.thread.join()
|
||||
self.samples.append(self.sample())
|
||||
return self.samples
|
||||
|
||||
|
||||
def counters(url, names):
|
||||
raw = http("GET", url + "/metrics", expect=(200,))[3].decode()
|
||||
result = {}
|
||||
for line in raw.splitlines():
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
name = line.split("{", 1)[0].split(" ", 1)[0]
|
||||
if name in names:
|
||||
result[name] = result.get(name, 0) + float(line.rsplit(" ", 1)[1])
|
||||
return result
|
||||
|
||||
|
||||
def accounting(b, hosts):
|
||||
metric = b.promql('sum(messenger_telemetry_spans_ended_total{instance=~"' + hosts + '"})')
|
||||
return {
|
||||
"collector": collector_metrics(),
|
||||
"sdk_ended_spans": float(metric[0]["value"][1]) if metric else 0,
|
||||
"tempo": counters(b.tempo, {"tempo_distributor_bytes_received_total"}),
|
||||
"loki": counters(b.loki, {"loki_distributor_bytes_received_total"}),
|
||||
"prometheus": counters(b.prometheus, {"prometheus_tsdb_head_samples_appended_total"}),
|
||||
}
|
||||
|
||||
|
||||
def run(mode, repeat, duration, rate, destination):
|
||||
configure(mode)
|
||||
b = backends()
|
||||
api = Api(os.environ["BASE_URL"])
|
||||
_, username, password, _ = api.register()
|
||||
auth, _ = api.login(username, password)
|
||||
for _ in range(20):
|
||||
api.call("GET", "/api/v1/users/me", headers=auth)
|
||||
time.sleep(35)
|
||||
hosts = "|".join(
|
||||
subprocess.check_output(
|
||||
["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True
|
||||
).strip()
|
||||
+ "-.*"
|
||||
for cid in compose("ps", "-q", "api", "worker", "relay").splitlines()
|
||||
)
|
||||
before = accounting(b, hosts)
|
||||
background = subprocess.check_output(
|
||||
["docker", "ps", "--format", '{{.Names}} {{.Label "com.docker.compose.project"}}'],
|
||||
text=True,
|
||||
).splitlines()
|
||||
resources = Resources()
|
||||
resources.begin()
|
||||
started = time.time()
|
||||
origin = time.perf_counter()
|
||||
prefix = "cost-" + uuid.uuid4().hex[:12]
|
||||
|
||||
def request(index):
|
||||
category = "error" if index % 50 == 0 else "slow" if index % 50 == 1 else "normal"
|
||||
rid = prefix + "-" + category + "-" + str(index)
|
||||
began = time.perf_counter()
|
||||
headers = dict(auth, **{"X-Request-Id": rid, "X-Lab-Cost": category})
|
||||
try:
|
||||
status = http("GET", api.base + "/api/v1/users/me", headers=headers)[0]
|
||||
except OSError:
|
||||
status = 0
|
||||
return {
|
||||
"id": rid,
|
||||
"category": category,
|
||||
"status": status,
|
||||
"latency_ms": (time.perf_counter() - began) * 1000,
|
||||
"schedule_lag_ms": (began - origin - index / rate) * 1000,
|
||||
}
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool:
|
||||
futures = []
|
||||
for index in range(int(duration * rate)):
|
||||
time.sleep(max(0, origin + index / rate - time.perf_counter()))
|
||||
futures.append(pool.submit(request, index))
|
||||
requests = [future.result() for future in futures]
|
||||
finished = time.time()
|
||||
samples = resources.finish()
|
||||
time.sleep(35)
|
||||
after = accounting(b, hosts)
|
||||
retained = {}
|
||||
for category in ("normal", "error", "slow"):
|
||||
found = b.traceql(
|
||||
'{ span.messenger.request_id =~ "' + prefix + "-" + category + '-.*" }',
|
||||
started - 1,
|
||||
time.time() + 1,
|
||||
limit=5000,
|
||||
)
|
||||
retained[category] = sorted({t["traceID"] for t in found})
|
||||
api.logout(auth)
|
||||
unexpected = [r for r in requests if r["status"] != (503 if r["category"] == "error" else 200)]
|
||||
report = {
|
||||
"utc": now(),
|
||||
"sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(),
|
||||
"mode": mode,
|
||||
"repeat": repeat,
|
||||
"duration": duration,
|
||||
"rate": rate,
|
||||
"started": started,
|
||||
"finished": finished,
|
||||
"background": background,
|
||||
"requests": requests,
|
||||
"resources": samples,
|
||||
"before": before,
|
||||
"after": after,
|
||||
"retained_trace_ids": retained,
|
||||
"unexpected_responses": len(unexpected),
|
||||
"result": "PASS" if not unexpected else "FAIL",
|
||||
}
|
||||
destination.write_text(json.dumps(report, indent=2) + "\n")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"mode": mode,
|
||||
"repeat": repeat,
|
||||
"result": report["result"],
|
||||
"retained": {k: len(v) for k, v in retained.items()},
|
||||
}
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
if unexpected:
|
||||
raise RuntimeError("unexpected benchmark responses; raw series saved")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--mode", choices=MODES)
|
||||
parser.add_argument("--repeat", type=int, default=1)
|
||||
parser.add_argument("--duration", type=int, default=40)
|
||||
parser.add_argument("--rate", type=int, default=20)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
raise RuntimeError("refusing to overwrite evidence")
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
run(args.mode, args.repeat, args.duration, args.rate, args.output)
|
||||
finally:
|
||||
os.environ.update(
|
||||
OTEL_SDK_DISABLED="false",
|
||||
OTEL_TRACE_RATIO="1",
|
||||
OTEL_COLLECTOR_CONFIG="collector.yaml",
|
||||
OTEL_METRIC_EXPORT_INTERVAL="10000",
|
||||
COST_EXPERIMENT="false",
|
||||
)
|
||||
compose("up", "-d", "--no-deps", "--wait", "otel-collector", "api", "worker", "relay")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -147,6 +147,7 @@ def create_app(
|
||||
)
|
||||
app.add_middleware(
|
||||
RequestPipeline,
|
||||
cost_experiment=transport.cost_experiment,
|
||||
logger=logger,
|
||||
max_bytes=transport.max_body_bytes,
|
||||
timeout=transport.request_timeout_seconds,
|
||||
|
||||
@@ -52,7 +52,8 @@ async def serve() -> None:
|
||||
logger,
|
||||
)
|
||||
queue_metrics = QueueMetrics(await container.get(Database), logger)
|
||||
queue_metrics.start()
|
||||
if telemetry is not None:
|
||||
queue_metrics.start()
|
||||
relay.start()
|
||||
heartbeat.start()
|
||||
logger.info("relay.started")
|
||||
|
||||
@@ -47,6 +47,7 @@ class PlatformSettings(Settings):
|
||||
db_connect_timeout_seconds: float = Field(default=1.0, gt=0, le=30)
|
||||
db_acquire_timeout_seconds: float = Field(default=1.0, gt=0, le=30)
|
||||
db_command_timeout_seconds: float = Field(default=2.0, gt=0, le=60)
|
||||
otel_trace_ratio: float = Field(default=1.0, ge=0, le=1)
|
||||
otel_sdk_disabled: bool = False
|
||||
otel_exporter_otlp_endpoint: str = ""
|
||||
otel_service_name: str = ""
|
||||
|
||||
@@ -5,6 +5,7 @@ from messenger.shared.config import Settings
|
||||
|
||||
class TransportSettings(Settings):
|
||||
docs: bool = False
|
||||
cost_experiment: bool = False
|
||||
max_body_bytes: int = Field(default=1_048_576, ge=4_096)
|
||||
request_timeout_seconds: float = Field(default=8.0, gt=0, le=300)
|
||||
min_transfer_rate_bytes: int = Field(default=131_072, ge=1_024)
|
||||
|
||||
@@ -89,6 +89,7 @@ class Pace:
|
||||
class RequestPipeline:
|
||||
__slots__ = (
|
||||
"_app",
|
||||
"_cost_experiment",
|
||||
"_guard",
|
||||
"_instance",
|
||||
"_limits",
|
||||
@@ -108,8 +109,10 @@ class RequestPipeline:
|
||||
instance: str,
|
||||
limits: BodyLimits = (),
|
||||
min_rate: float = 131_072,
|
||||
cost_experiment: bool = False,
|
||||
) -> None:
|
||||
self._app = app
|
||||
self._cost_experiment = cost_experiment
|
||||
self._logger = logger
|
||||
self._max_bytes = max_bytes
|
||||
self._limits = limits
|
||||
@@ -155,11 +158,11 @@ class RequestPipeline:
|
||||
raise HTTPException(413)
|
||||
return message
|
||||
|
||||
rejection: Rejection | None = None
|
||||
try:
|
||||
async with asyncio.timeout(self._timeout) as deadline:
|
||||
rejection = await self._guard(scope) or self._oversized(scope, limit)
|
||||
if rejection is None:
|
||||
await self._experiment(scope)
|
||||
pace = Pace(deadline, self._timeout, self._min_rate)
|
||||
await self._app(scope, *pace.around(limited_receive, wrapped_send))
|
||||
except Unavailable as error:
|
||||
@@ -201,6 +204,14 @@ class RequestPipeline:
|
||||
)
|
||||
self._log(scope, status_holder[0], request_id, started)
|
||||
|
||||
async def _experiment(self, scope: Scope) -> None:
|
||||
if self._cost_experiment and scope["path"] == "/api/v1/users/me":
|
||||
fault = header(scope, b"x-lab-cost")
|
||||
if fault == "error":
|
||||
raise Unavailable("controlled cost experiment")
|
||||
if fault == "slow":
|
||||
await asyncio.sleep(0.35)
|
||||
|
||||
def _limit(self, path: str) -> int:
|
||||
for pattern, limit in self._limits:
|
||||
if pattern.fullmatch(path):
|
||||
|
||||
@@ -18,9 +18,15 @@ from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import DropAggregation, View
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.sdk.trace.sampling import Decision, ParentBased, Sampler, SamplingResult
|
||||
from opentelemetry.sdk.trace.sampling import (
|
||||
Decision,
|
||||
ParentBased,
|
||||
Sampler,
|
||||
SamplingResult,
|
||||
TraceIdRatioBased,
|
||||
)
|
||||
from opentelemetry.trace import Link, SpanKind, TraceState
|
||||
from opentelemetry.util.types import Attributes
|
||||
|
||||
@@ -44,6 +50,9 @@ type Provider = TracerProvider | MeterProvider | LoggerProvider
|
||||
|
||||
|
||||
class ServerRoots(Sampler):
|
||||
def __init__(self, ratio: float = 1.0) -> None:
|
||||
self._root = TraceIdRatioBased(ratio)
|
||||
|
||||
@override
|
||||
def should_sample(
|
||||
self,
|
||||
@@ -56,7 +65,9 @@ class ServerRoots(Sampler):
|
||||
trace_state: TraceState | None = None,
|
||||
) -> SamplingResult:
|
||||
if kind in ROOT_KINDS:
|
||||
return SamplingResult(Decision.RECORD_AND_SAMPLE, attributes, trace_state)
|
||||
return self._root.should_sample(
|
||||
parent_context, trace_id, name, kind, attributes, links, trace_state
|
||||
)
|
||||
return SamplingResult(Decision.DROP, None, trace_state)
|
||||
|
||||
def get_description(self) -> str:
|
||||
@@ -88,6 +99,16 @@ class OtlpLogHandler(logging.Handler):
|
||||
)
|
||||
|
||||
|
||||
class SpanAccounting(SpanProcessor):
|
||||
def __init__(self, meter: MeterProvider) -> None:
|
||||
self._ended = meter.get_meter("messenger.telemetry").create_counter(
|
||||
"messenger.telemetry.spans.ended", unit="{span}"
|
||||
)
|
||||
|
||||
def on_end(self, span: ReadableSpan) -> None:
|
||||
self._ended.add(1, {"span.kind": span.kind.name.lower()})
|
||||
|
||||
|
||||
class Telemetry:
|
||||
__slots__ = ("_handler", "_providers")
|
||||
|
||||
@@ -119,7 +140,9 @@ def start_telemetry(
|
||||
}
|
||||
)
|
||||
tracer = TracerProvider(
|
||||
resource=resource, sampler=ParentBased(ServerRoots()), shutdown_on_exit=False
|
||||
resource=resource,
|
||||
sampler=ParentBased(ServerRoots(settings.otel_trace_ratio)),
|
||||
shutdown_on_exit=False,
|
||||
)
|
||||
tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
|
||||
meter = MeterProvider(
|
||||
@@ -128,6 +151,7 @@ def start_telemetry(
|
||||
views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())],
|
||||
shutdown_on_exit=False,
|
||||
)
|
||||
tracer.add_span_processor(SpanAccounting(meter))
|
||||
observe_process(meter)
|
||||
logs = LoggerProvider(resource=resource, shutdown_on_exit=False)
|
||||
logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import pytest
|
||||
from opentelemetry.sdk.trace.sampling import Decision
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from conftest import actor, register, running_app
|
||||
from messenger.shared.infrastructure.telemetry import ServerRoots
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enabled", [False, True])
|
||||
async def test_controlled_errors_are_opt_in(enabled: bool) -> None:
|
||||
async with running_app(cost_experiment=enabled) as client:
|
||||
user = await register(client, "cost")
|
||||
result = await client.get(
|
||||
"/api/v1/users/me", headers={**actor(user), "X-Lab-Cost": "error"}
|
||||
)
|
||||
assert result.status_code == (503 if enabled else 200)
|
||||
assert (await client.get("/api/v1/users/me", headers=actor(user))).status_code == 200
|
||||
|
||||
|
||||
def test_head_zero_drops_roots_and_full_keeps_them() -> None:
|
||||
for kind in (SpanKind.SERVER, SpanKind.PRODUCER, SpanKind.CONSUMER):
|
||||
assert ServerRoots(0).should_sample(None, 42, "root", kind).decision == Decision.DROP
|
||||
assert (
|
||||
ServerRoots(1).should_sample(None, 42, "root", kind).decision
|
||||
== Decision.RECORD_AND_SAMPLE
|
||||
)
|
||||
Reference in New Issue
Block a user