diff --git a/docs/runbooks.md b/docs/runbooks.md index 36de540..baed5a1 100644 --- a/docs/runbooks.md +++ b/docs/runbooks.md @@ -24,6 +24,18 @@ Alert rules: [infra/prometheus/rules/messenger.yml](../infra/prometheus/rules/me **Resolved:** доля 5xx за 5 минут снова ниже 5%. +## MessengerApiNotReady + +**Условие:** больше 10 ответов 5xx на `/health/core` (проверка HAProxy раз в секунду, `SELECT 1` в PostgreSQL) за минуту, держится 30 с. Штатная остановка реплики даёт 3–4 таких ответа (3 с drain) и alert не вызывает; недоступная БД — около 140 в минуту на две реплики. + +**Почему отдельный alert:** когда все реплики не готовы, HAProxy сам отвечает клиентам `503 no API replica available`; эти ответы не проходят через API и не входят в `http_server_requests_total`, поэтому доля 5xx бизнес-запросов может оставаться нулевой. + +**Проверить:** Loki `{service_name="messenger-api"} | event="http.error"` — `route=/health/core`, `message` (`database unavailable` или `shutting down`); `docker compose ps postgres`; панель «Пул БД» (рост `waiting`). + +**Действия:** восстановить PostgreSQL (`docker compose start postgres` / `docker compose unpause postgres`). После первых успешных проверок (`rise 2`) HAProxy возвращает реплики. + +**Resolved:** проверки готовности снова 200 в течение минуты. + ## MessengerJobQueueLag **Условие:** старейшее задание в состоянии `queued` с наступившим `next_attempt_at` ждёт дольше 60 с, держится 1 минуту. Источник — таблица `attachment_jobs` (метрика `jobs_queue_oldest_age_seconds` от relay), а не брокер: она показывает и задания, сообщения которых ещё не опубликованы. diff --git a/infra/prometheus/rules/messenger.yml b/infra/prometheus/rules/messenger.yml index 6025dfb..a8254f4 100644 --- a/infra/prometheus/rules/messenger.yml +++ b/infra/prometheus/rules/messenger.yml @@ -36,6 +36,15 @@ groups: summary: "Доля 5xx бизнес-запросов API {{ $value | humanizePercentage }} за 5 минут (порог 5%)" description: "SLO доступности 99% за 1 ч: при 5% ошибок бюджет сгорает в 5 раз быстрее допустимого. Health-запросы не учитываются." runbook: "docs/runbooks.md#messengerhigherrorrate" + - alert: MessengerApiNotReady + expr: sum(increase(http_server_requests_total{job="messenger-api", http_route="/health/core", http_response_status_class="5xx"}[1m])) > 10 + for: 30s + labels: + severity: critical + annotations: + summary: "Реплики API отвечают 503 на /health/core — HAProxy выводит их из балансировки" + description: "Обычно недоступна БД или реплика останавливается. Пока все реплики не готовы, клиенты получают 503 от LB, и такие ответы не попадают в метрики API (MessengerHighErrorRate их не видит)." + runbook: "docs/runbooks.md#messengerapinotready" - alert: MessengerJobQueueLag expr: messenger:jobs_queue_oldest_age:max > 60 for: 1m diff --git a/scripts/collect-evidence.sh b/scripts/collect-evidence.sh index 20da164..daacfd0 100755 --- a/scripts/collect-evidence.sh +++ b/scripts/collect-evidence.sh @@ -37,8 +37,13 @@ echo 'container.txt' run docker compose ps -a --format 'table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)' run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -aq)" + echo '## Опубликованные порты: только LB и Grafana на 127.0.0.1; OTLP, Prometheus, Loki, Tempo наружу не открыты' + run sh -c "docker compose ps --format '{{.Service}} {{.Ports}}' | grep -v '^\\S* *$' || true" + echo '## Сети телеметрии: хранилища только во внутренней сети telemetry' + run sh -c "for s in otel-collector prometheus loki tempo grafana api worker; do printf '%s: ' \$s; docker inspect -f '{{range \$k, \$v := .NetworkSettings.Networks}}{{\$k}} {{end}}' \$(docker compose ps -q \$s | head -1); done" + run docker network inspect -f '{{.Name}} internal={{.Internal}}' messenger-lab6_telemetry echo '## Секреты: каждый сервис видит только свои файлы (содержимое не выводится)' - for service in postgres redis s3 api reconciler relay worker; do + for service in postgres redis s3 api reconciler relay worker grafana; do run docker compose exec -T "$service" ls /run/secrets done run docker compose exec -T rabbitmq ls /etc/rabbitmq/conf.d @@ -49,6 +54,7 @@ echo 'container.txt' run docker compose exec -T redis redis-server --version run docker compose exec -T s3 minio --version run docker compose exec -T rabbitmq rabbitmqctl -q version + run sh -c "docker compose images otel-collector prometheus loki tempo grafana" echo '## Топология брокера: durable quorum-очереди, delivery-limit и DLQ' run docker compose exec -T rabbitmq rabbitmqctl -q list_queues name type durable arguments messages run docker compose exec -T rabbitmq rabbitmqctl -q list_consumers queue_name prefetch_count ack_required @@ -62,10 +68,12 @@ echo 'container.txt' echo 'tests.txt' { - header 'Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml)' + header 'Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml); основной стенд на время тестов остановлен ради памяти Docker' + docker compose stop lb api worker relay reconciler otel-collector prometheus loki tempo grafana rabbitmq s3 redis postgres >/dev/null 2>&1 docker compose -f compose.test.yaml build -q tests run docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose docker compose -f compose.test.yaml down -v >/dev/null 2>&1 + docker compose up -d --wait >/dev/null 2>&1 } > evidence/tests.txt echo 'smoke.txt' @@ -120,4 +128,32 @@ echo 'jobs.txt' docker compose logs --no-log-prefix --tail 1 reconciler 2>&1 } > evidence/jobs.txt +echo 'telemetry.txt' +{ + header 'Наблюдаемость: контрольный запрос (trace + логи), dashboard, перцентили, приватность, правила' + echo '## Prometheus: правила загружены (promtool и /api/v1/rules)' + run docker compose exec -T prometheus promtool check rules /etc/prometheus/rules/messenger.yml + run sh -c "wget -qO- http://grafana:3000/api/datasources/proxy/uid/prometheus/api/v1/rules | python3 -c 'import json,sys; [print(r[\"type\"], r[\"name\"], r[\"health\"]) for g in json.load(sys.stdin)[\"data\"][\"groups\"] for r in g[\"rules\"]]'" + echo '## Grafana: datasources с фиксированными UID, dashboard из provisioning' + run sh -c "for uid in prometheus loki tempo; do wget -qO- http://grafana:3000/api/datasources/uid/\$uid | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d[\"uid\"], d[\"type\"], d[\"url\"])'; done" + echo '## Контрольный upload с клиентским traceparent: trace в Tempo и логи в Loki' + run python3 scripts/trace-check.py + echo '## Секреты и содержимое не попадают в telemetry; в labels метрик нет ID' + run python3 scripts/telemetry-privacy.py + echo '## Перцентили dashboard против клиентского замера tests/load.py' + run python3 scripts/latency-compare.py --duration 30 + echo '## Все запросы dashboard (пустые панели alerts/WARN-логов допустимы без инцидентов)' + run python3 scripts/dashboard-check.py + echo '## Пример структурного лога (stdout API)' + docker compose logs --no-log-prefix --tail 200 api 2>&1 | grep '"event":"http.request"' | tail -2 +} > evidence/telemetry.txt + +for scenario in retry-trace worker-stop db telemetry; do + echo "incident-$scenario.txt" + { + header "Инцидент: scripts/incidents.py $scenario" + run python3 scripts/incidents.py "$scenario" + } > "evidence/incident-$scenario.txt" +done + echo 'done' diff --git a/scripts/incidents.py b/scripts/incidents.py new file mode 100755 index 0000000..7106dd0 --- /dev/null +++ b/scripts/incidents.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python3 +"""Controlled incidents for lab6 observability; run inside the tools container. + + docker compose run --rm tools python3 scripts/incidents.py worker-stop|db|telemetry|retry-trace + +Every scenario injects a fault, keeps synthetic traffic going, samples Prometheus/Tempo/Loki through Grafana +and waits for the related alert to become firing and then resolved after recovery. +""" +import base64 +import importlib.util +import json +import os +import statistics +import struct +import subprocess +import sys +import threading +import time +import urllib.parse +import urllib.request +import uuid +import zlib + +spec = importlib.util.spec_from_file_location('job_scenario', os.path.join(os.path.dirname(__file__), 'job-scenario.py')) +js = importlib.util.module_from_spec(spec) +spec.loader.exec_module(js) +log = js.log + +GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000') +PROM = GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/' +TEMPO = GRAFANA + '/api/datasources/proxy/uid/tempo/api/' +LOKI = GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/' + + +def get(url, **params): + with urllib.request.urlopen(url + ('?' + urllib.parse.urlencode(params) if params else ''), timeout=20) as r: + return json.loads(r.read()) + + +def prom(expr): + try: + result = get(PROM + 'query', query=expr)['data']['result'] + except OSError: + return None + return float(result[0]['value'][1]) if result else None + + +def alert_state(name): + try: + alerts = get(PROM + 'alerts')['data']['alerts'] + except OSError: + return 'unknown' + states = [a['state'] for a in alerts if a['labels'].get('alertname') == name] + return 'firing' if 'firing' in states else ('pending' if 'pending' in states else 'inactive') + + +def fmt(value, unit=''): + if value is None: + return '-' + if unit == '%': + return '%.1f%%' % (value * 100) + if unit == 'ms': + return '%.0fms' % (value * 1000) + if unit == 'MiB': + return '%.0fMiB' % (value / 2 ** 20) + return ('%.1f' % value).rstrip('0').rstrip('.') + + +def sample(columns, alerts): + parts = ['%s=%s' % (label, fmt(prom(expr), unit)) for label, expr, unit in columns] + parts += ['%s=%s' % (name, alert_state(name)) for name in alerts] + log(' '.join(parts)) + + +def watch(columns, alerts, until, timeout, every=10): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + sample(columns, alerts) + if until(): + return True + time.sleep(every) + sample(columns, alerts) + return False + + +class Traffic: + """Closed-loop synthetic traffic through the LB: POST message + GET history, counts statuses per phase.""" + + def __init__(self, auth, chat, threads=4, pause=0.2): + self.auth, self.chat, self.threads, self.pause = auth, chat, threads, pause + self.stop = threading.Event() + self.lock = threading.Lock() + self.reset() + + def reset(self): + with self.lock: + self.codes, self.latencies = {}, [] + + def snapshot(self, phase): + with self.lock: + codes, latencies = dict(self.codes), sorted(self.latencies) + p = lambda q: latencies[min(len(latencies) - 1, int(q * len(latencies)))] * 1000 if latencies else 0 + log('client [%s]: %s; p50=%.0fms p95=%.0fms p99=%.0fms' % ( + phase, ' '.join('%s:%d' % kv for kv in sorted(codes.items())), p(.5), p(.95), p(.99))) + self.reset() + + def run(self): + def loop(n): + i = 0 + while not self.stop.is_set(): + started = time.monotonic() + if i % 2 == 0: + code = js.call('POST', '/api/v1/chats/%s/messages' % self.chat, {'text': 'incident %d-%d' % (n, i)}, + {**self.auth, 'Idempotency-Key': uuid.uuid4().hex})[0] + else: + code = js.call('GET', '/api/v1/chats/%s/messages?limit=20' % self.chat, headers=self.auth)[0] + with self.lock: + self.codes[code] = self.codes.get(code, 0) + 1 + if 200 <= code < 300: + self.latencies.append(time.monotonic() - started) + i += 1 + time.sleep(self.pause) + self.workers = [threading.Thread(target=loop, args=(n,), daemon=True) for n in range(self.threads)] + for worker in self.workers: + worker.start() + return self + + def close(self): + self.stop.set() + for worker in self.workers: + worker.join(20) + + +def b64hex(value): + return base64.b64decode(value).hex() if value else '' + + +def trace_spans(trace_id): + body = get(TEMPO + 'traces/' + trace_id) + spans = [] + for batch in body.get('batches', []): + service = {a['key']: a['value'] for a in batch['resource']['attributes']}['service.name']['stringValue'] + for scope in batch.get('scopeSpans', []): + for span in scope.get('spans', []): + span['service'] = service + span['ms'] = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6 + span['attrs'] = {a['key']: list(a['value'].values())[0] for a in span.get('attributes', [])} + spans.append(span) + return spans + + +def find_traces(query, since, limit=5): + try: + return get(TEMPO + 'search', q=query, limit=limit, start=int(since), end=int(time.time()) + 60).get('traces', []) + except OSError: + return [] + + +def breakdown(trace_id, title): + spans = trace_spans(trace_id) + server = next((s for s in spans if s.get('kind') == 'SPAN_KIND_SERVER'), None) + if not server: + log('trace %s: no server span' % trace_id) + return + + names = {s['spanId']: s['name'] for s in spans} + + def total(prefix, top_level=False): + return sum(s['ms'] for s in spans if s['name'].startswith(prefix) + and not (top_level and names.get(s.get('parentSpanId'), '').startswith('prisma:'))) + db, s3 = total('prisma:', top_level=True), total('S3.') + log('%s: trace %s %s -> %s, %.0f ms' % (title, trace_id, server['name'], server['attrs'].get('http.status_code'), server['ms'])) + log(' DB calls %.0f ms (pool wait %.0f ms, SQL %.0f ms); S3 %.0f ms; handler itself %.0f ms' % ( + db, total('prisma:engine:connection'), total('prisma:engine:db_query'), s3, max(0, server['ms'] - db - s3))) + errors = sorted({s['name'] for s in spans if s.get('status', {}).get('code') == 'STATUS_CODE_ERROR'}) + if errors: + log(' spans with error status:', ', '.join(errors)) + + +def noise_png(width, height): + raw = b''.join(b'\0' + os.urandom(width * 3) for _ in range(height)) + + def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'') + + +def docker(*args): + log('$ docker', ' '.join(args)) + subprocess.run(['docker', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def container(service): + return subprocess.run(['docker', 'compose', 'ps', '-q', service], check=True, capture_output=True, text=True).stdout.split() + + +def scenario_worker_stop(): + """Incident 1: worker stopped -> jobs age, MessengerJobQueueLag fires -> worker started -> backlog drains, resolved.""" + js.wait_ready() + auth, chat = js.session() + columns = [ + ('due', 'max(jobs_queue_depth{state="due"})', ''), + ('processing', 'max(jobs_queue_depth{state="processing"})', ''), + ('oldest_age_s', 'messenger:jobs_queue_oldest_age:max', ''), + ('worker_busy', 'sum(worker_jobs_busy)', ''), + ('worker_up', 'count(timestamp(worker_jobs_capacity) > time() - 30)', ''), + ('ready_per_s', 'sum(rate(jobs_processed_total{outcome="success"}[1m]))', ''), + ] + alerts = ['MessengerJobQueueLag'] + js.compose('stop', 'worker') + stopped = time.time() + uploads = [] + + def upload_more(): + if len(uploads) < 12 and time.time() - stopped < 150: + uploads.append(js.upload(auth, chat)) + return alert_state('MessengerJobQueueLag') == 'firing' + fired = watch(columns, alerts, upload_more, timeout=300) + log('alert MessengerJobQueueLag firing after %.0f s of worker downtime: %s' % (time.time() - stopped, fired)) + js.start_only('worker') + started = time.time() + resolved = watch(columns, alerts, lambda: alert_state('MessengerJobQueueLag') == 'inactive', timeout=240) + log('backlog drained and alert resolved %.0f s after worker start: %s' % (time.time() - started, resolved)) + ready = sum(1 for a in uploads if js.metadata(auth, a)[1]['status'] == 'ready') + log('uploads during downtime: %d, ready now: %d' % (len(uploads), ready)) + time.sleep(15) + log('time to ready, histogram over 5m: p50=%s s, p99=%s s' % tuple(fmt(prom( + 'histogram_quantile(%s, sum by (le) (increase(attachments_time_to_ready_seconds_bucket[5m])))' % q)) for q in (0.5, 0.99))) + log('time to ready from the database (finished_at - created_at), s:', js.psql( + "SELECT round(min(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) || ' .. ' || " + "round(max(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) FROM attachments a " + "JOIN attachment_jobs j ON j.attachment_id = a.id WHERE a.id IN (%s)" % ','.join("'%s'" % a for a in uploads))) + if not (fired and resolved and ready == len(uploads)): + sys.exit('FAIL') + log('PASS') + + +def scenario_db(): + """Incident 2: slow DB (CPU-throttled postgres) vs slow handler (large upload); then DB unavailable (paused).""" + js.wait_ready() + auth, chat = js.session() + columns = [ + ('p95_1m', 'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api",http_route!~"/health.*"}[1m])))', 'ms'), + ('5xx_1m', '(sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*",http_response_status_class="5xx"}[1m])) or vector(0)) / sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*"}[1m]))', '%'), + ('pool_wait', 'sum(db_client_connection_pending_requests{job="messenger-api"})', ''), + ('pool_used', 'sum(db_client_connection_count{job="messenger-api",db_client_connection_state="used"})', ''), + ('core_5xx_1m', 'sum(increase(http_server_requests_total{http_route="/health/core",http_response_status_class="5xx"}[1m]))', ''), + ('api_cpu', 'sum(rate(process_cpu_time_seconds_total{job="messenger-api"}[1m]))', ''), + ] + alerts = ['MessengerSlowApi', 'MessengerHighErrorRate', 'MessengerApiNotReady'] + traffic = Traffic(auth, chat).run() + try: + log('== baseline: 60 s of traffic') + watch(columns, alerts, lambda: False, timeout=60, every=20) + traffic.snapshot('baseline') + + log('== slow handler (not DB): 1700x1700 noise PNG upload — decoding and S3 dominate') + big = noise_png(1700, 1700) + since = time.time() + code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat, + headers={**auth, 'Content-Type': 'image/png'}, raw=big) + log('POST %.1f MiB attachment -> %s' % (len(big) / 2 ** 20, code)) + time.sleep(15) + for t in find_traces('{resource.service.name="messenger-api" && kind=server && span.http.route="/api/v1/chats/{chat_id}/attachments"}', since, 1): + breakdown(t['traceID'], 'slow handler') + + log('== slow DB: postgres limited to 0.05 CPU for 150 s') + postgres = container('postgres') + since = time.time() + docker('update', '--cpus', '0.05', *postgres) + watch(columns, alerts, lambda: alert_state('MessengerSlowApi') == 'firing', timeout=150, every=15) + traffic.snapshot('slow DB') + slow = find_traces('{resource.service.name="messenger-api" && kind=server && duration > 500ms}', since, 3) + for t in slow[:2]: + breakdown(t['traceID'], 'slow DB') + docker('update', '--cpus', '1.0', *postgres) + watch(columns, alerts, lambda: alert_state('MessengerSlowApi') != 'firing', timeout=240, every=15) + traffic.snapshot('after slow DB') + + log('== DB unavailable: postgres paused for 120 s') + since = time.time() + docker('pause', *postgres) + not_ready = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'firing', timeout=120, every=15) + traffic.snapshot('DB paused') + for t in find_traces('{resource.service.name="messenger-api" && kind=server && status=error}', since, 2): + breakdown(t['traceID'], 'DB unavailable') + logs = get(LOKI + 'query_range', query='{service_name="messenger-api"} | event="http.error"', limit=3, since='3m') + for stream in logs['data']['result'][:3]: + log(' loki:', stream['values'][0][1][:200]) + docker('unpause', *postgres) + recovered = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'inactive', timeout=240, every=15) + traffic.snapshot('after DB unpaused') + finally: + traffic.close() + subprocess.run(['docker', 'unpause', *container('postgres')], capture_output=True) + subprocess.run(['docker', 'update', '--cpus', '1.0', *container('postgres')], capture_output=True) + log('MessengerApiNotReady fired: %s, resolved after recovery: %s' % (not_ready, recovered)) + log('PASS' if not_ready and recovered else 'FAIL') + + +def scenario_telemetry(): + """Incident 3: Loki and Tempo stopped, then the Collector stopped; API keeps serving, memory stays bounded.""" + js.wait_ready() + auth, chat = js.session() + columns = [ + ('spans_failed_1m', 'sum(increase(otelcol_exporter_send_failed_spans[1m]))', ''), + ('logs_failed_1m', 'sum(increase(otelcol_exporter_send_failed_log_records[1m]))', ''), + ('queue', 'max(otelcol_exporter_queue_size)', ''), + ('queue_cap', 'max(otelcol_exporter_queue_capacity)', ''), + ('refused_1m', 'sum(increase(otelcol_receiver_refused_spans[1m])) + sum(increase(otelcol_receiver_refused_log_records[1m]))', ''), + ('collector_rss', 'max(otelcol_process_memory_rss)', 'MiB'), + ('api_rss', 'max(process_memory_usage_bytes{job="messenger-api"})', 'MiB'), + ] + alerts = ['TelemetryExportFailing', 'MessengerHighErrorRate'] + traffic = Traffic(auth, chat, threads=6, pause=0.05).run() + try: + watch(columns, alerts, lambda: False, timeout=30, every=15) + traffic.snapshot('baseline') + log('== Loki and Tempo stopped until TelemetryExportFailing fires (max 240 s)') + js.compose('stop', 'loki', 'tempo') + fired = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'firing', timeout=240, every=15) + traffic.snapshot('Loki+Tempo down') + js.compose('start', 'loki', 'tempo') + resolved = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'inactive', timeout=240, every=15) + traffic.snapshot('after start') + + log('== Collector stopped for 60 s: SDK export fails fast (5 s timeout, bounded queues)') + js.compose('stop', 'otel-collector') + for _ in range(4): + time.sleep(15) + log('docker stats:', subprocess.run( + ['docker', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', *container('api'), *container('worker')], + capture_output=True, text=True).stdout.replace('\n', '; ')) + traffic.snapshot('collector down') + js.compose('start', 'otel-collector') + time.sleep(30) + traffic.snapshot('collector back') + finally: + traffic.close() + subprocess.run(['docker', 'compose', 'start', 'loki', 'tempo', 'otel-collector'], capture_output=True) + log('TelemetryExportFailing fired: %s, resolved: %s' % (fired, resolved)) + log('PASS' if fired and resolved else 'FAIL') + + +def scenario_retry_trace(): + """Level 4 trace: one upload whose first attempt fails on S3, then succeeds; all attempts in one trace.""" + js.wait_ready() + auth, chat = js.session() + trace_id = os.urandom(16).hex() + js.compose('stop', 'worker') + code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=sample.png' % chat, + headers={**auth, 'Content-Type': 'image/png', 'traceparent': '00-%s-%s-01' % (trace_id, os.urandom(8).hex())}, + raw=js.SAMPLE) + attachment = js.json_of(raw)['id'] + log('upload ->', code, attachment, 'trace', trace_id) + js.compose('stop', 's3') + js.start_only('worker') + time.sleep(5) + js.start_only('s3') + js.wait_status(auth, attachment, ('ready', 'failed')) + js.job_state(attachment) + time.sleep(15) + spans = trace_spans(trace_id) + for span in sorted(spans, key=lambda s: int(s['startTimeUnixNano'])): + if span.get('kind') in ('SPAN_KIND_SERVER', 'SPAN_KIND_PRODUCER', 'SPAN_KIND_CONSUMER') or span['name'].startswith('S3.'): + log(' %-17s %-20s %-36s attempt=%-2s %8.1f ms %s' % ( + span['service'], span.get('kind', '')[10:], span['name'], span['attrs'].get('messaging.job.attempt', ''), + span['ms'], span.get('status', {}).get('code', '')[12:])) + consumers = [s for s in spans if s.get('kind') == 'SPAN_KIND_CONSUMER'] + log('consumer spans in one trace: %d (attempts %s)' % ( + len(consumers), sorted({s['attrs'].get('messaging.job.attempt') for s in consumers}))) + log('PASS' if len(consumers) >= 2 else 'FAIL') + + +SCENARIOS = { + 'worker-stop': scenario_worker_stop, + 'db': scenario_db, + 'telemetry': scenario_telemetry, + 'retry-trace': scenario_retry_trace, +} + +if __name__ == '__main__': + if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS: + sys.exit('usage: incidents.py %s' % '|'.join(SCENARIOS)) + SCENARIOS[sys.argv[1]]() diff --git a/scripts/telemetry-privacy.py b/scripts/telemetry-privacy.py new file mode 100755 index 0000000..8b061d6 --- /dev/null +++ b/scripts/telemetry-privacy.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Checks that passwords, tokens, message text and filenames do not reach Loki/Tempo and that metric labels hold no IDs. + +Run inside the tools container: docker compose run --rm tools python3 scripts/telemetry-privacy.py +""" +import base64 +import json +import os +import re +import sys +import time +import urllib.parse +import urllib.request +import uuid + +BASE = os.environ.get('BASE_URL', 'http://lb:8080') +GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000') +SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read() +UUID = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}') + + +def call(method, path, body=None, headers=None, raw=None): + data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None) + headers = dict(headers or {}) + if body is not None: + headers['Content-Type'] = 'application/json' + with urllib.request.urlopen(urllib.request.Request(BASE + path, data=data, method=method, headers=headers), timeout=15) as r: + return json.loads(r.read() or b'null') + + +def grafana(path, **params): + with urllib.request.urlopen(GRAFANA + path + '?' + urllib.parse.urlencode(params), timeout=30) as r: + return r.read().decode() + + +def main(): + marker = uuid.uuid4().hex[:12] + secrets = { + 'password': 'Privacy-password-' + marker, + 'message text': 'privacy message body ' + marker, + 'filename': 'privacy-file-%s.png' % marker, + } + trace_ids = [] + + def traced(): + trace_id = os.urandom(16).hex() + trace_ids.append(trace_id) + return {'traceparent': '00-%s-%s-01' % (trace_id, os.urandom(8).hex())} + + username = 'privacy_' + marker + user = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Privacy', 'password': secrets['password']}, traced()) + basic = base64.b64encode(('%s:%s' % (username, secrets['password'])).encode()).decode() + token = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic, **traced()})['token'] + secrets['session token'] = token + secrets['basic credentials'] = basic + auth = {'Authorization': 'Bearer ' + token} + chat = call('POST', '/api/v1/chats', {'title': 'Privacy', 'member_ids': [user['id']]}, {**auth, **traced()})['id'] + call('POST', '/api/v1/chats/%s/messages' % chat, {'text': secrets['message text']}, + {**auth, 'Idempotency-Key': marker, **traced()}) + call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat, secrets['filename']), headers={**auth, 'Content-Type': 'image/png', **traced()}, + raw=SAMPLE) + call('GET', '/api/v1/chats/%s/messages?limit=5' % chat, headers={**auth, **traced()}) + print('sent %d traced requests with known password, token, message text and filename; waiting 25 s for export' % len(trace_ids)) + time.sleep(25) + + traces = '' + found_traces = 0 + for trace_id in trace_ids: + try: + traces += grafana('/api/datasources/proxy/uid/tempo/api/traces/' + trace_id) + found_traces += 1 + except OSError: + pass + def loki_lines(query): + body = grafana('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=query, limit=100, since='5m') + return sum(len(s['values']) for s in json.loads(body)['data']['result']) + + own = loki_lines('{service_name=~"messenger-.+"} |~ "%s"' % '|'.join(trace_ids)) + print('Tempo: %d/%d traces fetched (%d bytes); Loki: %d log lines carry these trace_ids (control)' % ( + found_traces, len(trace_ids), len(traces), own)) + leaks = 0 + for name, value in list(secrets.items()) + [('Authorization hdr', 'authorization')]: + in_traces = value.lower() in traces.lower() + in_logs = loki_lines('{service_name=~"messenger-.+"} |~ "(?i)%s"' % re.escape(value).replace('\\', '\\\\')) + leaks += in_traces + in_logs + print(' %-18s in traces: %-5s log lines in Loki (5 min): %d' % (name, in_traces, in_logs)) + + series = json.loads(grafana('/api/datasources/proxy/uid/prometheus/api/v1/series', **{'match[]': '{job=~"messenger-.+"}'}))['data'] + names = {s['__name__'] for s in series} + with_ids = [s for s in series if any(UUID.search(v) for k, v in s.items() if k != 'instance')] + labels = sorted({k for s in series for k in s} - {'__name__'}) + print('Prometheus: %d series, %d metric names from messenger services; label keys: %s' % (len(series), len(names), ', '.join(labels))) + print(' series with a UUID in any label value: %d' % len(with_ids)) + routes = sorted({s.get('http_route') for s in series if s.get('http_route')}) + print(' http_route values: %s' % ', '.join(routes)) + ok = found_traces == len(trace_ids) and own > 0 and leaks == 0 and not with_ids + print('%s: no secrets/content in logs and traces, no IDs in metric labels' % ('PASS' if ok else 'FAIL')) + sys.exit(0 if ok else 1) + + +if __name__ == '__main__': + main() diff --git a/src/jobs/processor.ts b/src/jobs/processor.ts index 79576da..c24f2a1 100644 --- a/src/jobs/processor.ts +++ b/src/jobs/processor.ts @@ -56,7 +56,9 @@ const timeToReady = metrics.getMeter('messenger-worker').createHistogram('attach description: 'From durable upload acceptance to the committed ready state', unit: 's', advice: { - explicitBucketBoundaries: [0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600], + explicitBucketBoundaries: [ + 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 45, 60, 90, 120, 150, 180, 240, 300, 450, 600, + ], }, }); diff --git a/src/telemetry/sdk.ts b/src/telemetry/sdk.ts index 46a2e32..1ef30e9 100644 --- a/src/telemetry/sdk.ts +++ b/src/telemetry/sdk.ts @@ -31,7 +31,6 @@ const NOISY_LEAF_SPANS = new Set([ 'prisma:client:serialize', 'prisma:engine:serialize', 'prisma:engine:response_json_serialization', - 'prisma:engine:connection', ]); const NOT_RECORDED: SamplingResult = { decision: SamplingDecision.NOT_RECORD };