From 202b985deacda79b6d8df65dcefe8a1af43f1a8e Mon Sep 17 00:00:00 2001 From: Artyom Date: Fri, 6 Nov 2026 21:22:38 +0300 Subject: [PATCH] Add reproducible observability cost modes and external measurements --- Makefile | 2 +- compose.yaml | 4 +- infra/otel/collector-balanced.yaml | 122 ++++++++ infra/otel/collector-tail.yaml | 122 ++++++++ .../measure-lab6-observability-cost/design.md | 13 + .../proposal.md | 11 + .../specs/lab6-observability-cost/spec.md | 11 + .../measure-lab6-observability-cost/tasks.md | 6 + scripts/obs_cost.py | 264 ++++++++++++++++++ src/messenger/main.py | 1 + src/messenger/relay.py | 3 +- src/messenger/shared/config.py | 1 + src/messenger/shared/entrypoints/config.py | 1 + src/messenger/shared/entrypoints/pipeline.py | 13 +- .../shared/infrastructure/telemetry.py | 32 ++- student-tests/api/test_cost_experiment.py | 26 ++ 16 files changed, 624 insertions(+), 8 deletions(-) create mode 100644 infra/otel/collector-balanced.yaml create mode 100644 infra/otel/collector-tail.yaml create mode 100644 openspec/changes/measure-lab6-observability-cost/design.md create mode 100644 openspec/changes/measure-lab6-observability-cost/proposal.md create mode 100644 openspec/changes/measure-lab6-observability-cost/specs/lab6-observability-cost/spec.md create mode 100644 openspec/changes/measure-lab6-observability-cost/tasks.md create mode 100644 scripts/obs_cost.py create mode 100644 student-tests/api/test_cost_experiment.py diff --git a/Makefile b/Makefile index 3efc28f0..0a9963b3 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ COMPOSE ?= docker compose RATE ?= 100 FROM_REV ?= SHIFT ?= 0 -SCRIPTS_PY = scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ +SCRIPTS_PY = scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \ scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \ scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py SCENARIO ?= failover-upload diff --git a/compose.yaml b/compose.yaml index 35e1fe5e..d6ddd70b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -45,6 +45,7 @@ x-pipeline: &pipeline condition: service_healthy x-otel-env: &otel-env + OTEL_TRACE_RATIO: ${OTEL_TRACE_RATIO:-1} OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318} OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-http/protobuf} @@ -148,6 +149,7 @@ services: CURSOR_SECRET_FILE: /run/secrets/cursor_secret SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + COST_EXPERIMENT: ${COST_EXPERIMENT:-false} WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2} DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8} REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8} @@ -438,7 +440,7 @@ services: otel-collector: <<: *obs-hardening image: otel/opentelemetry-collector-contrib:0.163.2@sha256:7f22d2a6748110ffb6a59bc0df448bccbf501cdb667d610ede4c61d3863d5849 - command: ["--config=/etc/otelcol/collector.yaml"] + command: ["--config=/etc/otelcol/${OTEL_COLLECTOR_CONFIG:-collector.yaml}"] networks: [backend, edge] environment: GOMEMLIMIT: 200MiB diff --git a/infra/otel/collector-balanced.yaml b/infra/otel/collector-balanced.yaml new file mode 100644 index 00000000..a455cb73 --- /dev/null +++ b/infra/otel/collector-balanced.yaml @@ -0,0 +1,122 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + max_request_body_size: 8388608 + +processors: + tail_sampling: + decision_wait: 10s + num_traces: 2000 + expected_new_traces_per_sec: 30 + decision_cache: + sampled_cache_size: 10000 + non_sampled_cache_size: 10000 + policies: + - name: errors + type: status_code + status_code: {status_codes: [ERROR]} + - name: slow + type: latency + latency: {threshold_ms: 300} + - name: baseline + type: probabilistic + probabilistic: {sampling_percentage: 5} + memory_limiter: + check_interval: 1s + limit_percentage: 75 + spike_limit_percentage: 20 + redaction: + allow_all_keys: true + summary: silent + blocked_key_patterns: + - "(?i)^(.*[._-])?(password|passwd|secret|token|authorization|cookie|api[._-]?key|private[._-]?key|credentials?)$" + - "^http\\.(request|response)\\.header\\..+" + - "^url\\.(query|full)$" + - "^http\\.(url|target)$" + - "^(messenger\\.)?(message\\.)?(text|body)$" + - "^(file\\.)?filename$" + blocked_values: + - "(?i)bearer\\s+[A-Za-z0-9._~+/=-]{8,}" + - "(?i)basic\\s+[A-Za-z0-9+/=]{8,}" + - "(?i)x-amz-(signature|credential)=[^&\\s]+" + - "-----BEGIN [A-Z ]*PRIVATE KEY-----" + batch: + send_batch_size: 1024 + send_batch_max_size: 2048 + timeout: 2s + +exporters: + otlp_http/tempo: + traces_endpoint: http://tempo:4318/v1/traces + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + sizer: items + queue_size: 20000 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + otlp_http/prometheus: + metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + sizer: items + queue_size: 50000 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + otlp_http/loki: + logs_endpoint: http://loki:3100/otlp/v1/logs + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + sizer: items + queue_size: 20000 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + +service: + extensions: [health_check] + telemetry: + logs: + level: info + metrics: + level: normal + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + without_type_suffix: false + without_units: false + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, redaction, tail_sampling, batch] + exporters: [otlp_http/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, redaction, batch] + exporters: [otlp_http/prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, redaction, batch] + exporters: [otlp_http/loki] diff --git a/infra/otel/collector-tail.yaml b/infra/otel/collector-tail.yaml new file mode 100644 index 00000000..1c7fe06e --- /dev/null +++ b/infra/otel/collector-tail.yaml @@ -0,0 +1,122 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + max_request_body_size: 8388608 + +processors: + tail_sampling: + decision_wait: 10s + num_traces: 2000 + expected_new_traces_per_sec: 30 + decision_cache: + sampled_cache_size: 10000 + non_sampled_cache_size: 10000 + policies: + - name: errors + type: status_code + status_code: {status_codes: [ERROR]} + - name: slow + type: latency + latency: {threshold_ms: 300} + - name: baseline + type: probabilistic + probabilistic: {sampling_percentage: 10} + memory_limiter: + check_interval: 1s + limit_percentage: 75 + spike_limit_percentage: 20 + redaction: + allow_all_keys: true + summary: silent + blocked_key_patterns: + - "(?i)^(.*[._-])?(password|passwd|secret|token|authorization|cookie|api[._-]?key|private[._-]?key|credentials?)$" + - "^http\\.(request|response)\\.header\\..+" + - "^url\\.(query|full)$" + - "^http\\.(url|target)$" + - "^(messenger\\.)?(message\\.)?(text|body)$" + - "^(file\\.)?filename$" + blocked_values: + - "(?i)bearer\\s+[A-Za-z0-9._~+/=-]{8,}" + - "(?i)basic\\s+[A-Za-z0-9+/=]{8,}" + - "(?i)x-amz-(signature|credential)=[^&\\s]+" + - "-----BEGIN [A-Z ]*PRIVATE KEY-----" + batch: + send_batch_size: 1024 + send_batch_max_size: 2048 + timeout: 2s + +exporters: + otlp_http/tempo: + traces_endpoint: http://tempo:4318/v1/traces + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + sizer: items + queue_size: 20000 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + otlp_http/prometheus: + metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + sizer: items + queue_size: 50000 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + otlp_http/loki: + logs_endpoint: http://loki:3100/otlp/v1/logs + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + sizer: items + queue_size: 20000 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + +service: + extensions: [health_check] + telemetry: + logs: + level: info + metrics: + level: normal + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + without_type_suffix: false + without_units: false + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, redaction, tail_sampling, batch] + exporters: [otlp_http/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, redaction, batch] + exporters: [otlp_http/prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, redaction, batch] + exporters: [otlp_http/loki] diff --git a/openspec/changes/measure-lab6-observability-cost/design.md b/openspec/changes/measure-lab6-observability-cost/design.md new file mode 100644 index 00000000..559e1ad8 --- /dev/null +++ b/openspec/changes/measure-lab6-observability-cost/design.md @@ -0,0 +1,13 @@ +## Context +SDK/Collector и реальные отказы уровня 4 проверены. Измерения проводятся последовательно на одном Docker project, с сохранением volumes. +## Goals / Non-Goals +Измерить цену наблюдаемости и сохранность редких событий. Не искать максимальную пропускную способность и не публиковать решение. +## Decisions +Гипотеза: head 10% уменьшит ingest и CPU приложения, но потеряет большую часть редких ошибок/задержек; tail сохранит ошибки и slow, снижая хранение, но оставит цену SDK. Balanced использует tail с 5% normal и metric interval 30 s. +Режимы: off (SDK disabled), on (100%), head (10% ParentBased), tail (100% SDK, Collector errors OR latency>=300ms OR 10%), balanced (5% baseline tail, interval30s). Tail decision_wait10s, num_traces2000, bounded decision cache. Все остальные параметры, образ, ресурсы и нагрузка одинаковые. +Driver: 40 s, 20 arrivals/s, 64 client threads, 3 повтора режима; 2% error, 2% slow, остальные authenticated GET users/me. Это контролируемый HTTP/DB workload, не тест насыщения изображений; pipeline отдельно проверен уровнями3/4. Ошибки/slow задаются opt-in заголовком только на users/me после auth guard. Секреты существуют только в памяти клиента. Raw latency включает delay планировщика отдельно. После прогрева и перед/после серии — выдержка экспорта; fault hooks отключаются после опыта. +CPU: delta Docker cgroup total_usage / elapsed, включая дочерние процессы. RAM: working set usage-inactive_file, samples раз в 2s. Это внешний сбор, независимый от SDK. Ingest: Tempo/Loki bytes и Prometheus appended samples отдельно, включая постоянный фон self-scrape. Spans: SDK ended, Collector accepted/sent/failed, найденные server roots normal/error/slow; намеренный sampling отдельно от export failure. Search limit превышает число запросов. Нагрузка не параллелится с тестами/другими сериями; Docker background inventory сохраняется. +## Risks / Trade-offs +Один локальный Docker Desktop host, короткие steady windows, клиент на том же host; не production capacity. Ротация режимов по повтору уменьшает временной дрейф. Три повтора описывают наблюдаемый диапазон, не доверительный интервал. Tail сохраняет только события внутри decision window; поздние spans/retry обсуждаются отдельно. Результат гипотезы не предрешён. +## Validation +Unit tests sampler/fault opt-in, make check/lint/student-test, Collector validation, smoke. Реальные серии, raw requests/resources/counters/trace IDs, summary и ограничения. Архив только после завершения всех повторов. diff --git a/openspec/changes/measure-lab6-observability-cost/proposal.md b/openspec/changes/measure-lab6-observability-cost/proposal.md new file mode 100644 index 00000000..b1915cd7 --- /dev/null +++ b/openspec/changes/measure-lab6-observability-cost/proposal.md @@ -0,0 +1,11 @@ +## Why +Уровни 3/4 доказаны; цена SDK и sampling ещё не измерена. Трек 1 требует сравнения latency, CPU/RAM, ingest и потерь spans при одинаковой нагрузке. +## What Changes +- Настраиваемый root sampling ratio и счётчик завершённых sampled spans. +- Изолированный opt-in cost fault hook: 503 и задержка 350 ms только для контрольных GET users/me. +- Collector tail и balanced configurations; воспроизводимый open-loop driver, внешние Docker counters, raw results и заметки. +## Capabilities +### New Capabilities +- `lab6-observability-cost`: воспроизводимое сравнение пяти режимов наблюдаемости. +## Impact +Только собственный код, scripts, compose, infra и evidence. API-контракт не меняется; fault hook выключен по умолчанию. REPORT готовит следующий агент. diff --git a/openspec/changes/measure-lab6-observability-cost/specs/lab6-observability-cost/spec.md b/openspec/changes/measure-lab6-observability-cost/specs/lab6-observability-cost/spec.md new file mode 100644 index 00000000..b58ae97f --- /dev/null +++ b/openspec/changes/measure-lab6-observability-cost/specs/lab6-observability-cost/spec.md @@ -0,0 +1,11 @@ +## ADDED Requirements +### Requirement: Reproducible cost comparison +The system SHALL support SDK off/on, head and tail sampling and a verified balanced configuration, measured under the same open-loop workload with raw latency, external CPU/RAM, ingest and span-loss evidence. +#### Scenario: Three repeat rounds +- **WHEN** the cost driver runs all five modes in three sequential rounds +- **THEN** each run records actual requests, resources, ingest counters and retained normal/error/slow traces without fabricating missing observations +### Requirement: Disabled experimental faults +Controlled error and slow hooks SHALL be opt-in and SHALL NOT affect default requests. +#### Scenario: Ordinary deployment +- **WHEN** the experiment setting is false +- **THEN** experiment headers cannot inject failure or delay diff --git a/openspec/changes/measure-lab6-observability-cost/tasks.md b/openspec/changes/measure-lab6-observability-cost/tasks.md new file mode 100644 index 00000000..c48cce2d --- /dev/null +++ b/openspec/changes/measure-lab6-observability-cost/tasks.md @@ -0,0 +1,6 @@ +- [ ] 1. Sampling, span accounting and opt-in fault hooks with meaningful tests +- [ ] 2. Tail/balanced Collector configs and sequential external measurement driver +- [ ] 3. make check/lint/student-test, config validation and smoke +- [ ] 4. Three real rounds of off/on/head/tail/balanced; raw evidence +- [ ] 5. Compare observations, recommend verified balance, record limitations +- [ ] 6. Restore default configuration, final checks, sync/archive and stop own stand diff --git a/scripts/obs_cost.py b/scripts/obs_cost.py new file mode 100644 index 00000000..c620e3ad --- /dev/null +++ b/scripts/obs_cost.py @@ -0,0 +1,264 @@ +import argparse +import concurrent.futures +import json +import os +import socket +import subprocess +import threading +import time +import uuid +from contextlib import closing +from http.client import HTTPConnection +from pathlib import Path + +from obs_check import Api, http, now +from obs_incidents import collector_metrics +from obs_recovery import backends, compose + +MODES = { + "off": ("true", "1", "collector.yaml", "10000"), + "on": ("false", "1", "collector.yaml", "10000"), + "head": ("false", "0.1", "collector.yaml", "10000"), + "tail": ("false", "1", "collector-tail.yaml", "10000"), + "balanced": ("false", "1", "collector-balanced.yaml", "30000"), +} + + +def configure(mode): + disabled, ratio, config, interval = MODES[mode] + os.environ.update( + OTEL_SDK_DISABLED=disabled, + OTEL_TRACE_RATIO=ratio, + OTEL_COLLECTOR_CONFIG=config, + OTEL_METRIC_EXPORT_INTERVAL=interval, + COST_EXPERIMENT="true", + ) + compose( + "up", + "-d", + "--no-deps", + "--force-recreate", + "--wait", + "otel-collector", + "api", + "worker", + "relay", + ) + subprocess.run(["scripts/lb-wait.sh"], check=True, capture_output=True) + + +class Engine(HTTPConnection): + def __init__(self, path): + super().__init__("localhost", timeout=10) + self.path = path + + def connect(self): + self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.sock.settimeout(self.timeout) + self.sock.connect(self.path) + + +def engine_read(path, route): + with closing(Engine(path)) as connection: + connection.request("GET", route) + response = connection.getresponse() + if response.status != 200: + raise RuntimeError("Docker stats unavailable") + return json.load(response) + + +class Resources: + def __init__(self): + host = subprocess.check_output( + ["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], text=True + ).strip() + if not host.startswith("unix://"): + raise RuntimeError("cost measurement requires a local Docker socket") + self.path = host[7:] + self.ids = compose("ps", "-q").splitlines() + self.samples = [] + self.stop = threading.Event() + self.thread = threading.Thread(target=self.poll) + + def sample(self): + rows = [] + for identity in self.ids: + data = engine_read( + self.path, f"/containers/{identity}/stats?stream=false&one-shot=true" + ) + memory = data["memory_stats"] + rows.append( + { + "id": identity[:12], + "name": data["name"], + "cpu_ns": data["cpu_stats"]["cpu_usage"]["total_usage"], + "working_set_bytes": memory["usage"] + - memory.get("stats", {}).get("inactive_file", 0), + "memory_limit": memory["limit"], + } + ) + return {"at": time.time(), "containers": rows} + + def poll(self): + while not self.stop.wait(2): + self.samples.append(self.sample()) + + def begin(self): + self.samples.append(self.sample()) + self.thread.start() + + def finish(self): + self.stop.set() + self.thread.join() + self.samples.append(self.sample()) + return self.samples + + +def counters(url, names): + raw = http("GET", url + "/metrics", expect=(200,))[3].decode() + result = {} + for line in raw.splitlines(): + if not line or line.startswith("#"): + continue + name = line.split("{", 1)[0].split(" ", 1)[0] + if name in names: + result[name] = result.get(name, 0) + float(line.rsplit(" ", 1)[1]) + return result + + +def accounting(b, hosts): + metric = b.promql('sum(messenger_telemetry_spans_ended_total{instance=~"' + hosts + '"})') + return { + "collector": collector_metrics(), + "sdk_ended_spans": float(metric[0]["value"][1]) if metric else 0, + "tempo": counters(b.tempo, {"tempo_distributor_bytes_received_total"}), + "loki": counters(b.loki, {"loki_distributor_bytes_received_total"}), + "prometheus": counters(b.prometheus, {"prometheus_tsdb_head_samples_appended_total"}), + } + + +def run(mode, repeat, duration, rate, destination): + configure(mode) + b = backends() + api = Api(os.environ["BASE_URL"]) + _, username, password, _ = api.register() + auth, _ = api.login(username, password) + for _ in range(20): + api.call("GET", "/api/v1/users/me", headers=auth) + time.sleep(35) + hosts = "|".join( + subprocess.check_output( + ["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True + ).strip() + + "-.*" + for cid in compose("ps", "-q", "api", "worker", "relay").splitlines() + ) + before = accounting(b, hosts) + background = subprocess.check_output( + ["docker", "ps", "--format", '{{.Names}} {{.Label "com.docker.compose.project"}}'], + text=True, + ).splitlines() + resources = Resources() + resources.begin() + started = time.time() + origin = time.perf_counter() + prefix = "cost-" + uuid.uuid4().hex[:12] + + def request(index): + category = "error" if index % 50 == 0 else "slow" if index % 50 == 1 else "normal" + rid = prefix + "-" + category + "-" + str(index) + began = time.perf_counter() + headers = dict(auth, **{"X-Request-Id": rid, "X-Lab-Cost": category}) + try: + status = http("GET", api.base + "/api/v1/users/me", headers=headers)[0] + except OSError: + status = 0 + return { + "id": rid, + "category": category, + "status": status, + "latency_ms": (time.perf_counter() - began) * 1000, + "schedule_lag_ms": (began - origin - index / rate) * 1000, + } + + with concurrent.futures.ThreadPoolExecutor(max_workers=64) as pool: + futures = [] + for index in range(int(duration * rate)): + time.sleep(max(0, origin + index / rate - time.perf_counter())) + futures.append(pool.submit(request, index)) + requests = [future.result() for future in futures] + finished = time.time() + samples = resources.finish() + time.sleep(35) + after = accounting(b, hosts) + retained = {} + for category in ("normal", "error", "slow"): + found = b.traceql( + '{ span.messenger.request_id =~ "' + prefix + "-" + category + '-.*" }', + started - 1, + time.time() + 1, + limit=5000, + ) + retained[category] = sorted({t["traceID"] for t in found}) + api.logout(auth) + unexpected = [r for r in requests if r["status"] != (503 if r["category"] == "error" else 200)] + report = { + "utc": now(), + "sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + "mode": mode, + "repeat": repeat, + "duration": duration, + "rate": rate, + "started": started, + "finished": finished, + "background": background, + "requests": requests, + "resources": samples, + "before": before, + "after": after, + "retained_trace_ids": retained, + "unexpected_responses": len(unexpected), + "result": "PASS" if not unexpected else "FAIL", + } + destination.write_text(json.dumps(report, indent=2) + "\n") + print( + json.dumps( + { + "mode": mode, + "repeat": repeat, + "result": report["result"], + "retained": {k: len(v) for k, v in retained.items()}, + } + ), + flush=True, + ) + if unexpected: + raise RuntimeError("unexpected benchmark responses; raw series saved") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--mode", choices=MODES) + parser.add_argument("--repeat", type=int, default=1) + parser.add_argument("--duration", type=int, default=40) + parser.add_argument("--rate", type=int, default=20) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.output.exists(): + raise RuntimeError("refusing to overwrite evidence") + args.output.parent.mkdir(parents=True, exist_ok=True) + try: + run(args.mode, args.repeat, args.duration, args.rate, args.output) + finally: + os.environ.update( + OTEL_SDK_DISABLED="false", + OTEL_TRACE_RATIO="1", + OTEL_COLLECTOR_CONFIG="collector.yaml", + OTEL_METRIC_EXPORT_INTERVAL="10000", + COST_EXPERIMENT="false", + ) + compose("up", "-d", "--no-deps", "--wait", "otel-collector", "api", "worker", "relay") + + +if __name__ == "__main__": + main() diff --git a/src/messenger/main.py b/src/messenger/main.py index 21866ec4..8314db69 100644 --- a/src/messenger/main.py +++ b/src/messenger/main.py @@ -147,6 +147,7 @@ def create_app( ) app.add_middleware( RequestPipeline, + cost_experiment=transport.cost_experiment, logger=logger, max_bytes=transport.max_body_bytes, timeout=transport.request_timeout_seconds, diff --git a/src/messenger/relay.py b/src/messenger/relay.py index 4b6b1f66..daf2689e 100644 --- a/src/messenger/relay.py +++ b/src/messenger/relay.py @@ -52,7 +52,8 @@ async def serve() -> None: logger, ) queue_metrics = QueueMetrics(await container.get(Database), logger) - queue_metrics.start() + if telemetry is not None: + queue_metrics.start() relay.start() heartbeat.start() logger.info("relay.started") diff --git a/src/messenger/shared/config.py b/src/messenger/shared/config.py index f6b212ab..f4add11a 100644 --- a/src/messenger/shared/config.py +++ b/src/messenger/shared/config.py @@ -47,6 +47,7 @@ class PlatformSettings(Settings): db_connect_timeout_seconds: float = Field(default=1.0, gt=0, le=30) db_acquire_timeout_seconds: float = Field(default=1.0, gt=0, le=30) db_command_timeout_seconds: float = Field(default=2.0, gt=0, le=60) + otel_trace_ratio: float = Field(default=1.0, ge=0, le=1) otel_sdk_disabled: bool = False otel_exporter_otlp_endpoint: str = "" otel_service_name: str = "" diff --git a/src/messenger/shared/entrypoints/config.py b/src/messenger/shared/entrypoints/config.py index 05f3cb40..ca9857f4 100644 --- a/src/messenger/shared/entrypoints/config.py +++ b/src/messenger/shared/entrypoints/config.py @@ -5,6 +5,7 @@ from messenger.shared.config import Settings class TransportSettings(Settings): docs: bool = False + cost_experiment: bool = False max_body_bytes: int = Field(default=1_048_576, ge=4_096) request_timeout_seconds: float = Field(default=8.0, gt=0, le=300) min_transfer_rate_bytes: int = Field(default=131_072, ge=1_024) diff --git a/src/messenger/shared/entrypoints/pipeline.py b/src/messenger/shared/entrypoints/pipeline.py index 628cfe12..61d83592 100644 --- a/src/messenger/shared/entrypoints/pipeline.py +++ b/src/messenger/shared/entrypoints/pipeline.py @@ -89,6 +89,7 @@ class Pace: class RequestPipeline: __slots__ = ( "_app", + "_cost_experiment", "_guard", "_instance", "_limits", @@ -108,8 +109,10 @@ class RequestPipeline: instance: str, limits: BodyLimits = (), min_rate: float = 131_072, + cost_experiment: bool = False, ) -> None: self._app = app + self._cost_experiment = cost_experiment self._logger = logger self._max_bytes = max_bytes self._limits = limits @@ -155,11 +158,11 @@ class RequestPipeline: raise HTTPException(413) return message - rejection: Rejection | None = None try: async with asyncio.timeout(self._timeout) as deadline: rejection = await self._guard(scope) or self._oversized(scope, limit) if rejection is None: + await self._experiment(scope) pace = Pace(deadline, self._timeout, self._min_rate) await self._app(scope, *pace.around(limited_receive, wrapped_send)) except Unavailable as error: @@ -201,6 +204,14 @@ class RequestPipeline: ) self._log(scope, status_holder[0], request_id, started) + async def _experiment(self, scope: Scope) -> None: + if self._cost_experiment and scope["path"] == "/api/v1/users/me": + fault = header(scope, b"x-lab-cost") + if fault == "error": + raise Unavailable("controlled cost experiment") + if fault == "slow": + await asyncio.sleep(0.35) + def _limit(self, path: str) -> int: for pattern, limit in self._limits: if pattern.fullmatch(path): diff --git a/src/messenger/shared/infrastructure/telemetry.py b/src/messenger/shared/infrastructure/telemetry.py index cd2e9d3b..35614cb3 100644 --- a/src/messenger/shared/infrastructure/telemetry.py +++ b/src/messenger/shared/infrastructure/telemetry.py @@ -18,9 +18,15 @@ from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.metrics.view import DropAggregation, View from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor, TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.sdk.trace.sampling import Decision, ParentBased, Sampler, SamplingResult +from opentelemetry.sdk.trace.sampling import ( + Decision, + ParentBased, + Sampler, + SamplingResult, + TraceIdRatioBased, +) from opentelemetry.trace import Link, SpanKind, TraceState from opentelemetry.util.types import Attributes @@ -44,6 +50,9 @@ type Provider = TracerProvider | MeterProvider | LoggerProvider class ServerRoots(Sampler): + def __init__(self, ratio: float = 1.0) -> None: + self._root = TraceIdRatioBased(ratio) + @override def should_sample( self, @@ -56,7 +65,9 @@ class ServerRoots(Sampler): trace_state: TraceState | None = None, ) -> SamplingResult: if kind in ROOT_KINDS: - return SamplingResult(Decision.RECORD_AND_SAMPLE, attributes, trace_state) + return self._root.should_sample( + parent_context, trace_id, name, kind, attributes, links, trace_state + ) return SamplingResult(Decision.DROP, None, trace_state) def get_description(self) -> str: @@ -88,6 +99,16 @@ class OtlpLogHandler(logging.Handler): ) +class SpanAccounting(SpanProcessor): + def __init__(self, meter: MeterProvider) -> None: + self._ended = meter.get_meter("messenger.telemetry").create_counter( + "messenger.telemetry.spans.ended", unit="{span}" + ) + + def on_end(self, span: ReadableSpan) -> None: + self._ended.add(1, {"span.kind": span.kind.name.lower()}) + + class Telemetry: __slots__ = ("_handler", "_providers") @@ -119,7 +140,9 @@ def start_telemetry( } ) tracer = TracerProvider( - resource=resource, sampler=ParentBased(ServerRoots()), shutdown_on_exit=False + resource=resource, + sampler=ParentBased(ServerRoots(settings.otel_trace_ratio)), + shutdown_on_exit=False, ) tracer.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) meter = MeterProvider( @@ -128,6 +151,7 @@ def start_telemetry( views=[View(meter_name=FOREIGN_METER, aggregation=DropAggregation())], shutdown_on_exit=False, ) + tracer.add_span_processor(SpanAccounting(meter)) observe_process(meter) logs = LoggerProvider(resource=resource, shutdown_on_exit=False) logs.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter())) diff --git a/student-tests/api/test_cost_experiment.py b/student-tests/api/test_cost_experiment.py new file mode 100644 index 00000000..6db346b0 --- /dev/null +++ b/student-tests/api/test_cost_experiment.py @@ -0,0 +1,26 @@ +import pytest +from opentelemetry.sdk.trace.sampling import Decision +from opentelemetry.trace import SpanKind + +from conftest import actor, register, running_app +from messenger.shared.infrastructure.telemetry import ServerRoots + + +@pytest.mark.parametrize("enabled", [False, True]) +async def test_controlled_errors_are_opt_in(enabled: bool) -> None: + async with running_app(cost_experiment=enabled) as client: + user = await register(client, "cost") + result = await client.get( + "/api/v1/users/me", headers={**actor(user), "X-Lab-Cost": "error"} + ) + assert result.status_code == (503 if enabled else 200) + assert (await client.get("/api/v1/users/me", headers=actor(user))).status_code == 200 + + +def test_head_zero_drops_roots_and_full_keeps_them() -> None: + for kind in (SpanKind.SERVER, SpanKind.PRODUCER, SpanKind.CONSUMER): + assert ServerRoots(0).should_sample(None, 42, "root", kind).decision == Decision.DROP + assert ( + ServerRoots(1).should_sample(None, 42, "root", kind).decision + == Decision.RECORD_AND_SAMPLE + )