feat: add incident scenarios, privacy check and lab6 evidence collection
This commit is contained in:
@@ -24,6 +24,18 @@ Alert rules: [infra/prometheus/rules/messenger.yml](../infra/prometheus/rules/me
|
||||
|
||||
**Resolved:** доля 5xx за 5 минут снова ниже 5%.
|
||||
|
||||
## MessengerApiNotReady
|
||||
|
||||
**Условие:** больше 10 ответов 5xx на `/health/core` (проверка HAProxy раз в секунду, `SELECT 1` в PostgreSQL) за минуту, держится 30 с. Штатная остановка реплики даёт 3–4 таких ответа (3 с drain) и alert не вызывает; недоступная БД — около 140 в минуту на две реплики.
|
||||
|
||||
**Почему отдельный alert:** когда все реплики не готовы, HAProxy сам отвечает клиентам `503 no API replica available`; эти ответы не проходят через API и не входят в `http_server_requests_total`, поэтому доля 5xx бизнес-запросов может оставаться нулевой.
|
||||
|
||||
**Проверить:** Loki `{service_name="messenger-api"} | event="http.error"` — `route=/health/core`, `message` (`database unavailable` или `shutting down`); `docker compose ps postgres`; панель «Пул БД» (рост `waiting`).
|
||||
|
||||
**Действия:** восстановить PostgreSQL (`docker compose start postgres` / `docker compose unpause postgres`). После первых успешных проверок (`rise 2`) HAProxy возвращает реплики.
|
||||
|
||||
**Resolved:** проверки готовности снова 200 в течение минуты.
|
||||
|
||||
## MessengerJobQueueLag
|
||||
|
||||
**Условие:** старейшее задание в состоянии `queued` с наступившим `next_attempt_at` ждёт дольше 60 с, держится 1 минуту. Источник — таблица `attachment_jobs` (метрика `jobs_queue_oldest_age_seconds` от relay), а не брокер: она показывает и задания, сообщения которых ещё не опубликованы.
|
||||
|
||||
@@ -36,6 +36,15 @@ groups:
|
||||
summary: "Доля 5xx бизнес-запросов API {{ $value | humanizePercentage }} за 5 минут (порог 5%)"
|
||||
description: "SLO доступности 99% за 1 ч: при 5% ошибок бюджет сгорает в 5 раз быстрее допустимого. Health-запросы не учитываются."
|
||||
runbook: "docs/runbooks.md#messengerhigherrorrate"
|
||||
- alert: MessengerApiNotReady
|
||||
expr: sum(increase(http_server_requests_total{job="messenger-api", http_route="/health/core", http_response_status_class="5xx"}[1m])) > 10
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Реплики API отвечают 503 на /health/core — HAProxy выводит их из балансировки"
|
||||
description: "Обычно недоступна БД или реплика останавливается. Пока все реплики не готовы, клиенты получают 503 от LB, и такие ответы не попадают в метрики API (MessengerHighErrorRate их не видит)."
|
||||
runbook: "docs/runbooks.md#messengerapinotready"
|
||||
- alert: MessengerJobQueueLag
|
||||
expr: messenger:jobs_queue_oldest_age:max > 60
|
||||
for: 1m
|
||||
|
||||
@@ -37,8 +37,13 @@ echo 'container.txt'
|
||||
run docker compose ps -a --format 'table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'
|
||||
echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)'
|
||||
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -aq)"
|
||||
echo '## Опубликованные порты: только LB и Grafana на 127.0.0.1; OTLP, Prometheus, Loki, Tempo наружу не открыты'
|
||||
run sh -c "docker compose ps --format '{{.Service}} {{.Ports}}' | grep -v '^\\S* *$' || true"
|
||||
echo '## Сети телеметрии: хранилища только во внутренней сети telemetry'
|
||||
run sh -c "for s in otel-collector prometheus loki tempo grafana api worker; do printf '%s: ' \$s; docker inspect -f '{{range \$k, \$v := .NetworkSettings.Networks}}{{\$k}} {{end}}' \$(docker compose ps -q \$s | head -1); done"
|
||||
run docker network inspect -f '{{.Name}} internal={{.Internal}}' messenger-lab6_telemetry
|
||||
echo '## Секреты: каждый сервис видит только свои файлы (содержимое не выводится)'
|
||||
for service in postgres redis s3 api reconciler relay worker; do
|
||||
for service in postgres redis s3 api reconciler relay worker grafana; do
|
||||
run docker compose exec -T "$service" ls /run/secrets
|
||||
done
|
||||
run docker compose exec -T rabbitmq ls /etc/rabbitmq/conf.d
|
||||
@@ -49,6 +54,7 @@ echo 'container.txt'
|
||||
run docker compose exec -T redis redis-server --version
|
||||
run docker compose exec -T s3 minio --version
|
||||
run docker compose exec -T rabbitmq rabbitmqctl -q version
|
||||
run sh -c "docker compose images otel-collector prometheus loki tempo grafana"
|
||||
echo '## Топология брокера: durable quorum-очереди, delivery-limit и DLQ'
|
||||
run docker compose exec -T rabbitmq rabbitmqctl -q list_queues name type durable arguments messages
|
||||
run docker compose exec -T rabbitmq rabbitmqctl -q list_consumers queue_name prefetch_count ack_required
|
||||
@@ -62,10 +68,12 @@ echo 'container.txt'
|
||||
|
||||
echo 'tests.txt'
|
||||
{
|
||||
header 'Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml)'
|
||||
header 'Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml); основной стенд на время тестов остановлен ради памяти Docker'
|
||||
docker compose stop lb api worker relay reconciler otel-collector prometheus loki tempo grafana rabbitmq s3 redis postgres >/dev/null 2>&1
|
||||
docker compose -f compose.test.yaml build -q tests
|
||||
run docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose
|
||||
docker compose -f compose.test.yaml down -v >/dev/null 2>&1
|
||||
docker compose up -d --wait >/dev/null 2>&1
|
||||
} > evidence/tests.txt
|
||||
|
||||
echo 'smoke.txt'
|
||||
@@ -120,4 +128,32 @@ echo 'jobs.txt'
|
||||
docker compose logs --no-log-prefix --tail 1 reconciler 2>&1
|
||||
} > evidence/jobs.txt
|
||||
|
||||
echo 'telemetry.txt'
|
||||
{
|
||||
header 'Наблюдаемость: контрольный запрос (trace + логи), dashboard, перцентили, приватность, правила'
|
||||
echo '## Prometheus: правила загружены (promtool и /api/v1/rules)'
|
||||
run docker compose exec -T prometheus promtool check rules /etc/prometheus/rules/messenger.yml
|
||||
run sh -c "wget -qO- http://grafana:3000/api/datasources/proxy/uid/prometheus/api/v1/rules | python3 -c 'import json,sys; [print(r[\"type\"], r[\"name\"], r[\"health\"]) for g in json.load(sys.stdin)[\"data\"][\"groups\"] for r in g[\"rules\"]]'"
|
||||
echo '## Grafana: datasources с фиксированными UID, dashboard из provisioning'
|
||||
run sh -c "for uid in prometheus loki tempo; do wget -qO- http://grafana:3000/api/datasources/uid/\$uid | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d[\"uid\"], d[\"type\"], d[\"url\"])'; done"
|
||||
echo '## Контрольный upload с клиентским traceparent: trace в Tempo и логи в Loki'
|
||||
run python3 scripts/trace-check.py
|
||||
echo '## Секреты и содержимое не попадают в telemetry; в labels метрик нет ID'
|
||||
run python3 scripts/telemetry-privacy.py
|
||||
echo '## Перцентили dashboard против клиентского замера tests/load.py'
|
||||
run python3 scripts/latency-compare.py --duration 30
|
||||
echo '## Все запросы dashboard (пустые панели alerts/WARN-логов допустимы без инцидентов)'
|
||||
run python3 scripts/dashboard-check.py
|
||||
echo '## Пример структурного лога (stdout API)'
|
||||
docker compose logs --no-log-prefix --tail 200 api 2>&1 | grep '"event":"http.request"' | tail -2
|
||||
} > evidence/telemetry.txt
|
||||
|
||||
for scenario in retry-trace worker-stop db telemetry; do
|
||||
echo "incident-$scenario.txt"
|
||||
{
|
||||
header "Инцидент: scripts/incidents.py $scenario"
|
||||
run python3 scripts/incidents.py "$scenario"
|
||||
} > "evidence/incident-$scenario.txt"
|
||||
done
|
||||
|
||||
echo 'done'
|
||||
|
||||
Executable
+386
@@ -0,0 +1,386 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Controlled incidents for lab6 observability; run inside the tools container.
|
||||
|
||||
docker compose run --rm tools python3 scripts/incidents.py worker-stop|db|telemetry|retry-trace
|
||||
|
||||
Every scenario injects a fault, keeps synthetic traffic going, samples Prometheus/Tempo/Loki through Grafana
|
||||
and waits for the related alert to become firing and then resolved after recovery.
|
||||
"""
|
||||
import base64
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
spec = importlib.util.spec_from_file_location('job_scenario', os.path.join(os.path.dirname(__file__), 'job-scenario.py'))
|
||||
js = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(js)
|
||||
log = js.log
|
||||
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
PROM = GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/'
|
||||
TEMPO = GRAFANA + '/api/datasources/proxy/uid/tempo/api/'
|
||||
LOKI = GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/'
|
||||
|
||||
|
||||
def get(url, **params):
|
||||
with urllib.request.urlopen(url + ('?' + urllib.parse.urlencode(params) if params else ''), timeout=20) as r:
|
||||
return json.loads(r.read())
|
||||
|
||||
|
||||
def prom(expr):
|
||||
try:
|
||||
result = get(PROM + 'query', query=expr)['data']['result']
|
||||
except OSError:
|
||||
return None
|
||||
return float(result[0]['value'][1]) if result else None
|
||||
|
||||
|
||||
def alert_state(name):
|
||||
try:
|
||||
alerts = get(PROM + 'alerts')['data']['alerts']
|
||||
except OSError:
|
||||
return 'unknown'
|
||||
states = [a['state'] for a in alerts if a['labels'].get('alertname') == name]
|
||||
return 'firing' if 'firing' in states else ('pending' if 'pending' in states else 'inactive')
|
||||
|
||||
|
||||
def fmt(value, unit=''):
|
||||
if value is None:
|
||||
return '-'
|
||||
if unit == '%':
|
||||
return '%.1f%%' % (value * 100)
|
||||
if unit == 'ms':
|
||||
return '%.0fms' % (value * 1000)
|
||||
if unit == 'MiB':
|
||||
return '%.0fMiB' % (value / 2 ** 20)
|
||||
return ('%.1f' % value).rstrip('0').rstrip('.')
|
||||
|
||||
|
||||
def sample(columns, alerts):
|
||||
parts = ['%s=%s' % (label, fmt(prom(expr), unit)) for label, expr, unit in columns]
|
||||
parts += ['%s=%s' % (name, alert_state(name)) for name in alerts]
|
||||
log(' '.join(parts))
|
||||
|
||||
|
||||
def watch(columns, alerts, until, timeout, every=10):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
sample(columns, alerts)
|
||||
if until():
|
||||
return True
|
||||
time.sleep(every)
|
||||
sample(columns, alerts)
|
||||
return False
|
||||
|
||||
|
||||
class Traffic:
|
||||
"""Closed-loop synthetic traffic through the LB: POST message + GET history, counts statuses per phase."""
|
||||
|
||||
def __init__(self, auth, chat, threads=4, pause=0.2):
|
||||
self.auth, self.chat, self.threads, self.pause = auth, chat, threads, pause
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
with self.lock:
|
||||
self.codes, self.latencies = {}, []
|
||||
|
||||
def snapshot(self, phase):
|
||||
with self.lock:
|
||||
codes, latencies = dict(self.codes), sorted(self.latencies)
|
||||
p = lambda q: latencies[min(len(latencies) - 1, int(q * len(latencies)))] * 1000 if latencies else 0
|
||||
log('client [%s]: %s; p50=%.0fms p95=%.0fms p99=%.0fms' % (
|
||||
phase, ' '.join('%s:%d' % kv for kv in sorted(codes.items())), p(.5), p(.95), p(.99)))
|
||||
self.reset()
|
||||
|
||||
def run(self):
|
||||
def loop(n):
|
||||
i = 0
|
||||
while not self.stop.is_set():
|
||||
started = time.monotonic()
|
||||
if i % 2 == 0:
|
||||
code = js.call('POST', '/api/v1/chats/%s/messages' % self.chat, {'text': 'incident %d-%d' % (n, i)},
|
||||
{**self.auth, 'Idempotency-Key': uuid.uuid4().hex})[0]
|
||||
else:
|
||||
code = js.call('GET', '/api/v1/chats/%s/messages?limit=20' % self.chat, headers=self.auth)[0]
|
||||
with self.lock:
|
||||
self.codes[code] = self.codes.get(code, 0) + 1
|
||||
if 200 <= code < 300:
|
||||
self.latencies.append(time.monotonic() - started)
|
||||
i += 1
|
||||
time.sleep(self.pause)
|
||||
self.workers = [threading.Thread(target=loop, args=(n,), daemon=True) for n in range(self.threads)]
|
||||
for worker in self.workers:
|
||||
worker.start()
|
||||
return self
|
||||
|
||||
def close(self):
|
||||
self.stop.set()
|
||||
for worker in self.workers:
|
||||
worker.join(20)
|
||||
|
||||
|
||||
def b64hex(value):
|
||||
return base64.b64decode(value).hex() if value else ''
|
||||
|
||||
|
||||
def trace_spans(trace_id):
|
||||
body = get(TEMPO + 'traces/' + trace_id)
|
||||
spans = []
|
||||
for batch in body.get('batches', []):
|
||||
service = {a['key']: a['value'] for a in batch['resource']['attributes']}['service.name']['stringValue']
|
||||
for scope in batch.get('scopeSpans', []):
|
||||
for span in scope.get('spans', []):
|
||||
span['service'] = service
|
||||
span['ms'] = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
|
||||
span['attrs'] = {a['key']: list(a['value'].values())[0] for a in span.get('attributes', [])}
|
||||
spans.append(span)
|
||||
return spans
|
||||
|
||||
|
||||
def find_traces(query, since, limit=5):
|
||||
try:
|
||||
return get(TEMPO + 'search', q=query, limit=limit, start=int(since), end=int(time.time()) + 60).get('traces', [])
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def breakdown(trace_id, title):
|
||||
spans = trace_spans(trace_id)
|
||||
server = next((s for s in spans if s.get('kind') == 'SPAN_KIND_SERVER'), None)
|
||||
if not server:
|
||||
log('trace %s: no server span' % trace_id)
|
||||
return
|
||||
|
||||
names = {s['spanId']: s['name'] for s in spans}
|
||||
|
||||
def total(prefix, top_level=False):
|
||||
return sum(s['ms'] for s in spans if s['name'].startswith(prefix)
|
||||
and not (top_level and names.get(s.get('parentSpanId'), '').startswith('prisma:')))
|
||||
db, s3 = total('prisma:', top_level=True), total('S3.')
|
||||
log('%s: trace %s %s -> %s, %.0f ms' % (title, trace_id, server['name'], server['attrs'].get('http.status_code'), server['ms']))
|
||||
log(' DB calls %.0f ms (pool wait %.0f ms, SQL %.0f ms); S3 %.0f ms; handler itself %.0f ms' % (
|
||||
db, total('prisma:engine:connection'), total('prisma:engine:db_query'), s3, max(0, server['ms'] - db - s3)))
|
||||
errors = sorted({s['name'] for s in spans if s.get('status', {}).get('code') == 'STATUS_CODE_ERROR'})
|
||||
if errors:
|
||||
log(' spans with error status:', ', '.join(errors))
|
||||
|
||||
|
||||
def noise_png(width, height):
|
||||
raw = b''.join(b'\0' + os.urandom(width * 3) for _ in range(height))
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
||||
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
|
||||
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'')
|
||||
|
||||
|
||||
def docker(*args):
|
||||
log('$ docker', ' '.join(args))
|
||||
subprocess.run(['docker', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def container(service):
|
||||
return subprocess.run(['docker', 'compose', 'ps', '-q', service], check=True, capture_output=True, text=True).stdout.split()
|
||||
|
||||
|
||||
def scenario_worker_stop():
|
||||
"""Incident 1: worker stopped -> jobs age, MessengerJobQueueLag fires -> worker started -> backlog drains, resolved."""
|
||||
js.wait_ready()
|
||||
auth, chat = js.session()
|
||||
columns = [
|
||||
('due', 'max(jobs_queue_depth{state="due"})', ''),
|
||||
('processing', 'max(jobs_queue_depth{state="processing"})', ''),
|
||||
('oldest_age_s', 'messenger:jobs_queue_oldest_age:max', ''),
|
||||
('worker_busy', 'sum(worker_jobs_busy)', ''),
|
||||
('worker_up', 'count(timestamp(worker_jobs_capacity) > time() - 30)', ''),
|
||||
('ready_per_s', 'sum(rate(jobs_processed_total{outcome="success"}[1m]))', ''),
|
||||
]
|
||||
alerts = ['MessengerJobQueueLag']
|
||||
js.compose('stop', 'worker')
|
||||
stopped = time.time()
|
||||
uploads = []
|
||||
|
||||
def upload_more():
|
||||
if len(uploads) < 12 and time.time() - stopped < 150:
|
||||
uploads.append(js.upload(auth, chat))
|
||||
return alert_state('MessengerJobQueueLag') == 'firing'
|
||||
fired = watch(columns, alerts, upload_more, timeout=300)
|
||||
log('alert MessengerJobQueueLag firing after %.0f s of worker downtime: %s' % (time.time() - stopped, fired))
|
||||
js.start_only('worker')
|
||||
started = time.time()
|
||||
resolved = watch(columns, alerts, lambda: alert_state('MessengerJobQueueLag') == 'inactive', timeout=240)
|
||||
log('backlog drained and alert resolved %.0f s after worker start: %s' % (time.time() - started, resolved))
|
||||
ready = sum(1 for a in uploads if js.metadata(auth, a)[1]['status'] == 'ready')
|
||||
log('uploads during downtime: %d, ready now: %d' % (len(uploads), ready))
|
||||
time.sleep(15)
|
||||
log('time to ready, histogram over 5m: p50=%s s, p99=%s s' % tuple(fmt(prom(
|
||||
'histogram_quantile(%s, sum by (le) (increase(attachments_time_to_ready_seconds_bucket[5m])))' % q)) for q in (0.5, 0.99)))
|
||||
log('time to ready from the database (finished_at - created_at), s:', js.psql(
|
||||
"SELECT round(min(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) || ' .. ' || "
|
||||
"round(max(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) FROM attachments a "
|
||||
"JOIN attachment_jobs j ON j.attachment_id = a.id WHERE a.id IN (%s)" % ','.join("'%s'" % a for a in uploads)))
|
||||
if not (fired and resolved and ready == len(uploads)):
|
||||
sys.exit('FAIL')
|
||||
log('PASS')
|
||||
|
||||
|
||||
def scenario_db():
|
||||
"""Incident 2: slow DB (CPU-throttled postgres) vs slow handler (large upload); then DB unavailable (paused)."""
|
||||
js.wait_ready()
|
||||
auth, chat = js.session()
|
||||
columns = [
|
||||
('p95_1m', 'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api",http_route!~"/health.*"}[1m])))', 'ms'),
|
||||
('5xx_1m', '(sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*",http_response_status_class="5xx"}[1m])) or vector(0)) / sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*"}[1m]))', '%'),
|
||||
('pool_wait', 'sum(db_client_connection_pending_requests{job="messenger-api"})', ''),
|
||||
('pool_used', 'sum(db_client_connection_count{job="messenger-api",db_client_connection_state="used"})', ''),
|
||||
('core_5xx_1m', 'sum(increase(http_server_requests_total{http_route="/health/core",http_response_status_class="5xx"}[1m]))', ''),
|
||||
('api_cpu', 'sum(rate(process_cpu_time_seconds_total{job="messenger-api"}[1m]))', ''),
|
||||
]
|
||||
alerts = ['MessengerSlowApi', 'MessengerHighErrorRate', 'MessengerApiNotReady']
|
||||
traffic = Traffic(auth, chat).run()
|
||||
try:
|
||||
log('== baseline: 60 s of traffic')
|
||||
watch(columns, alerts, lambda: False, timeout=60, every=20)
|
||||
traffic.snapshot('baseline')
|
||||
|
||||
log('== slow handler (not DB): 1700x1700 noise PNG upload — decoding and S3 dominate')
|
||||
big = noise_png(1700, 1700)
|
||||
since = time.time()
|
||||
code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat,
|
||||
headers={**auth, 'Content-Type': 'image/png'}, raw=big)
|
||||
log('POST %.1f MiB attachment -> %s' % (len(big) / 2 ** 20, code))
|
||||
time.sleep(15)
|
||||
for t in find_traces('{resource.service.name="messenger-api" && kind=server && span.http.route="/api/v1/chats/{chat_id}/attachments"}', since, 1):
|
||||
breakdown(t['traceID'], 'slow handler')
|
||||
|
||||
log('== slow DB: postgres limited to 0.05 CPU for 150 s')
|
||||
postgres = container('postgres')
|
||||
since = time.time()
|
||||
docker('update', '--cpus', '0.05', *postgres)
|
||||
watch(columns, alerts, lambda: alert_state('MessengerSlowApi') == 'firing', timeout=150, every=15)
|
||||
traffic.snapshot('slow DB')
|
||||
slow = find_traces('{resource.service.name="messenger-api" && kind=server && duration > 500ms}', since, 3)
|
||||
for t in slow[:2]:
|
||||
breakdown(t['traceID'], 'slow DB')
|
||||
docker('update', '--cpus', '1.0', *postgres)
|
||||
watch(columns, alerts, lambda: alert_state('MessengerSlowApi') != 'firing', timeout=240, every=15)
|
||||
traffic.snapshot('after slow DB')
|
||||
|
||||
log('== DB unavailable: postgres paused for 120 s')
|
||||
since = time.time()
|
||||
docker('pause', *postgres)
|
||||
not_ready = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'firing', timeout=120, every=15)
|
||||
traffic.snapshot('DB paused')
|
||||
for t in find_traces('{resource.service.name="messenger-api" && kind=server && status=error}', since, 2):
|
||||
breakdown(t['traceID'], 'DB unavailable')
|
||||
logs = get(LOKI + 'query_range', query='{service_name="messenger-api"} | event="http.error"', limit=3, since='3m')
|
||||
for stream in logs['data']['result'][:3]:
|
||||
log(' loki:', stream['values'][0][1][:200])
|
||||
docker('unpause', *postgres)
|
||||
recovered = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'inactive', timeout=240, every=15)
|
||||
traffic.snapshot('after DB unpaused')
|
||||
finally:
|
||||
traffic.close()
|
||||
subprocess.run(['docker', 'unpause', *container('postgres')], capture_output=True)
|
||||
subprocess.run(['docker', 'update', '--cpus', '1.0', *container('postgres')], capture_output=True)
|
||||
log('MessengerApiNotReady fired: %s, resolved after recovery: %s' % (not_ready, recovered))
|
||||
log('PASS' if not_ready and recovered else 'FAIL')
|
||||
|
||||
|
||||
def scenario_telemetry():
|
||||
"""Incident 3: Loki and Tempo stopped, then the Collector stopped; API keeps serving, memory stays bounded."""
|
||||
js.wait_ready()
|
||||
auth, chat = js.session()
|
||||
columns = [
|
||||
('spans_failed_1m', 'sum(increase(otelcol_exporter_send_failed_spans[1m]))', ''),
|
||||
('logs_failed_1m', 'sum(increase(otelcol_exporter_send_failed_log_records[1m]))', ''),
|
||||
('queue', 'max(otelcol_exporter_queue_size)', ''),
|
||||
('queue_cap', 'max(otelcol_exporter_queue_capacity)', ''),
|
||||
('refused_1m', 'sum(increase(otelcol_receiver_refused_spans[1m])) + sum(increase(otelcol_receiver_refused_log_records[1m]))', ''),
|
||||
('collector_rss', 'max(otelcol_process_memory_rss)', 'MiB'),
|
||||
('api_rss', 'max(process_memory_usage_bytes{job="messenger-api"})', 'MiB'),
|
||||
]
|
||||
alerts = ['TelemetryExportFailing', 'MessengerHighErrorRate']
|
||||
traffic = Traffic(auth, chat, threads=6, pause=0.05).run()
|
||||
try:
|
||||
watch(columns, alerts, lambda: False, timeout=30, every=15)
|
||||
traffic.snapshot('baseline')
|
||||
log('== Loki and Tempo stopped until TelemetryExportFailing fires (max 240 s)')
|
||||
js.compose('stop', 'loki', 'tempo')
|
||||
fired = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'firing', timeout=240, every=15)
|
||||
traffic.snapshot('Loki+Tempo down')
|
||||
js.compose('start', 'loki', 'tempo')
|
||||
resolved = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'inactive', timeout=240, every=15)
|
||||
traffic.snapshot('after start')
|
||||
|
||||
log('== Collector stopped for 60 s: SDK export fails fast (5 s timeout, bounded queues)')
|
||||
js.compose('stop', 'otel-collector')
|
||||
for _ in range(4):
|
||||
time.sleep(15)
|
||||
log('docker stats:', subprocess.run(
|
||||
['docker', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', *container('api'), *container('worker')],
|
||||
capture_output=True, text=True).stdout.replace('\n', '; '))
|
||||
traffic.snapshot('collector down')
|
||||
js.compose('start', 'otel-collector')
|
||||
time.sleep(30)
|
||||
traffic.snapshot('collector back')
|
||||
finally:
|
||||
traffic.close()
|
||||
subprocess.run(['docker', 'compose', 'start', 'loki', 'tempo', 'otel-collector'], capture_output=True)
|
||||
log('TelemetryExportFailing fired: %s, resolved: %s' % (fired, resolved))
|
||||
log('PASS' if fired and resolved else 'FAIL')
|
||||
|
||||
|
||||
def scenario_retry_trace():
|
||||
"""Level 4 trace: one upload whose first attempt fails on S3, then succeeds; all attempts in one trace."""
|
||||
js.wait_ready()
|
||||
auth, chat = js.session()
|
||||
trace_id = os.urandom(16).hex()
|
||||
js.compose('stop', 'worker')
|
||||
code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
|
||||
headers={**auth, 'Content-Type': 'image/png', 'traceparent': '00-%s-%s-01' % (trace_id, os.urandom(8).hex())},
|
||||
raw=js.SAMPLE)
|
||||
attachment = js.json_of(raw)['id']
|
||||
log('upload ->', code, attachment, 'trace', trace_id)
|
||||
js.compose('stop', 's3')
|
||||
js.start_only('worker')
|
||||
time.sleep(5)
|
||||
js.start_only('s3')
|
||||
js.wait_status(auth, attachment, ('ready', 'failed'))
|
||||
js.job_state(attachment)
|
||||
time.sleep(15)
|
||||
spans = trace_spans(trace_id)
|
||||
for span in sorted(spans, key=lambda s: int(s['startTimeUnixNano'])):
|
||||
if span.get('kind') in ('SPAN_KIND_SERVER', 'SPAN_KIND_PRODUCER', 'SPAN_KIND_CONSUMER') or span['name'].startswith('S3.'):
|
||||
log(' %-17s %-20s %-36s attempt=%-2s %8.1f ms %s' % (
|
||||
span['service'], span.get('kind', '')[10:], span['name'], span['attrs'].get('messaging.job.attempt', ''),
|
||||
span['ms'], span.get('status', {}).get('code', '')[12:]))
|
||||
consumers = [s for s in spans if s.get('kind') == 'SPAN_KIND_CONSUMER']
|
||||
log('consumer spans in one trace: %d (attempts %s)' % (
|
||||
len(consumers), sorted({s['attrs'].get('messaging.job.attempt') for s in consumers})))
|
||||
log('PASS' if len(consumers) >= 2 else 'FAIL')
|
||||
|
||||
|
||||
SCENARIOS = {
|
||||
'worker-stop': scenario_worker_stop,
|
||||
'db': scenario_db,
|
||||
'telemetry': scenario_telemetry,
|
||||
'retry-trace': scenario_retry_trace,
|
||||
}
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS:
|
||||
sys.exit('usage: incidents.py %s' % '|'.join(SCENARIOS))
|
||||
SCENARIOS[sys.argv[1]]()
|
||||
Executable
+102
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Checks that passwords, tokens, message text and filenames do not reach Loki/Tempo and that metric labels hold no IDs.
|
||||
|
||||
Run inside the tools container: docker compose run --rm tools python3 scripts/telemetry-privacy.py
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE = os.environ.get('BASE_URL', 'http://lb:8080')
|
||||
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
||||
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
|
||||
UUID = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}')
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None, raw=None):
|
||||
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
|
||||
headers = dict(headers or {})
|
||||
if body is not None:
|
||||
headers['Content-Type'] = 'application/json'
|
||||
with urllib.request.urlopen(urllib.request.Request(BASE + path, data=data, method=method, headers=headers), timeout=15) as r:
|
||||
return json.loads(r.read() or b'null')
|
||||
|
||||
|
||||
def grafana(path, **params):
|
||||
with urllib.request.urlopen(GRAFANA + path + '?' + urllib.parse.urlencode(params), timeout=30) as r:
|
||||
return r.read().decode()
|
||||
|
||||
|
||||
def main():
|
||||
marker = uuid.uuid4().hex[:12]
|
||||
secrets = {
|
||||
'password': 'Privacy-password-' + marker,
|
||||
'message text': 'privacy message body ' + marker,
|
||||
'filename': 'privacy-file-%s.png' % marker,
|
||||
}
|
||||
trace_ids = []
|
||||
|
||||
def traced():
|
||||
trace_id = os.urandom(16).hex()
|
||||
trace_ids.append(trace_id)
|
||||
return {'traceparent': '00-%s-%s-01' % (trace_id, os.urandom(8).hex())}
|
||||
|
||||
username = 'privacy_' + marker
|
||||
user = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Privacy', 'password': secrets['password']}, traced())
|
||||
basic = base64.b64encode(('%s:%s' % (username, secrets['password'])).encode()).decode()
|
||||
token = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic, **traced()})['token']
|
||||
secrets['session token'] = token
|
||||
secrets['basic credentials'] = basic
|
||||
auth = {'Authorization': 'Bearer ' + token}
|
||||
chat = call('POST', '/api/v1/chats', {'title': 'Privacy', 'member_ids': [user['id']]}, {**auth, **traced()})['id']
|
||||
call('POST', '/api/v1/chats/%s/messages' % chat, {'text': secrets['message text']},
|
||||
{**auth, 'Idempotency-Key': marker, **traced()})
|
||||
call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat, secrets['filename']), headers={**auth, 'Content-Type': 'image/png', **traced()},
|
||||
raw=SAMPLE)
|
||||
call('GET', '/api/v1/chats/%s/messages?limit=5' % chat, headers={**auth, **traced()})
|
||||
print('sent %d traced requests with known password, token, message text and filename; waiting 25 s for export' % len(trace_ids))
|
||||
time.sleep(25)
|
||||
|
||||
traces = ''
|
||||
found_traces = 0
|
||||
for trace_id in trace_ids:
|
||||
try:
|
||||
traces += grafana('/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
|
||||
found_traces += 1
|
||||
except OSError:
|
||||
pass
|
||||
def loki_lines(query):
|
||||
body = grafana('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=query, limit=100, since='5m')
|
||||
return sum(len(s['values']) for s in json.loads(body)['data']['result'])
|
||||
|
||||
own = loki_lines('{service_name=~"messenger-.+"} |~ "%s"' % '|'.join(trace_ids))
|
||||
print('Tempo: %d/%d traces fetched (%d bytes); Loki: %d log lines carry these trace_ids (control)' % (
|
||||
found_traces, len(trace_ids), len(traces), own))
|
||||
leaks = 0
|
||||
for name, value in list(secrets.items()) + [('Authorization hdr', 'authorization')]:
|
||||
in_traces = value.lower() in traces.lower()
|
||||
in_logs = loki_lines('{service_name=~"messenger-.+"} |~ "(?i)%s"' % re.escape(value).replace('\\', '\\\\'))
|
||||
leaks += in_traces + in_logs
|
||||
print(' %-18s in traces: %-5s log lines in Loki (5 min): %d' % (name, in_traces, in_logs))
|
||||
|
||||
series = json.loads(grafana('/api/datasources/proxy/uid/prometheus/api/v1/series', **{'match[]': '{job=~"messenger-.+"}'}))['data']
|
||||
names = {s['__name__'] for s in series}
|
||||
with_ids = [s for s in series if any(UUID.search(v) for k, v in s.items() if k != 'instance')]
|
||||
labels = sorted({k for s in series for k in s} - {'__name__'})
|
||||
print('Prometheus: %d series, %d metric names from messenger services; label keys: %s' % (len(series), len(names), ', '.join(labels)))
|
||||
print(' series with a UUID in any label value: %d' % len(with_ids))
|
||||
routes = sorted({s.get('http_route') for s in series if s.get('http_route')})
|
||||
print(' http_route values: %s' % ', '.join(routes))
|
||||
ok = found_traces == len(trace_ids) and own > 0 and leaks == 0 and not with_ids
|
||||
print('%s: no secrets/content in logs and traces, no IDs in metric labels' % ('PASS' if ok else 'FAIL'))
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -56,7 +56,9 @@ const timeToReady = metrics.getMeter('messenger-worker').createHistogram('attach
|
||||
description: 'From durable upload acceptance to the committed ready state',
|
||||
unit: 's',
|
||||
advice: {
|
||||
explicitBucketBoundaries: [0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300, 600],
|
||||
explicitBucketBoundaries: [
|
||||
0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 20, 30, 45, 60, 90, 120, 150, 180, 240, 300, 450, 600,
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -31,7 +31,6 @@ const NOISY_LEAF_SPANS = new Set([
|
||||
'prisma:client:serialize',
|
||||
'prisma:engine:serialize',
|
||||
'prisma:engine:response_json_serialization',
|
||||
'prisma:engine:connection',
|
||||
]);
|
||||
|
||||
const NOT_RECORDED: SamplingResult = { decision: SamplingDecision.NOT_RECORD };
|
||||
|
||||
Reference in New Issue
Block a user