feat: add incident scenarios, privacy check and lab6 evidence collection

This commit is contained in:
2026-11-20 16:39:45 +03:00
parent 3171835fff
commit 5f870e4fb0
7 changed files with 550 additions and 4 deletions
+38 -2
View File
@@ -37,8 +37,13 @@ echo 'container.txt'
run docker compose ps -a --format 'table {{.Service}}\t{{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'
echo '## CPU/RAM-лимиты (NanoCpus: 1e9 = 1 CPU; Memory в байтах)'
run sh -c "docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}}' \$(docker compose ps -aq)"
echo '## Опубликованные порты: только LB и Grafana на 127.0.0.1; OTLP, Prometheus, Loki, Tempo наружу не открыты'
run sh -c "docker compose ps --format '{{.Service}} {{.Ports}}' | grep -v '^\\S* *$' || true"
echo '## Сети телеметрии: хранилища только во внутренней сети telemetry'
run sh -c "for s in otel-collector prometheus loki tempo grafana api worker; do printf '%s: ' \$s; docker inspect -f '{{range \$k, \$v := .NetworkSettings.Networks}}{{\$k}} {{end}}' \$(docker compose ps -q \$s | head -1); done"
run docker network inspect -f '{{.Name}} internal={{.Internal}}' messenger-lab6_telemetry
echo '## Секреты: каждый сервис видит только свои файлы (содержимое не выводится)'
for service in postgres redis s3 api reconciler relay worker; do
for service in postgres redis s3 api reconciler relay worker grafana; do
run docker compose exec -T "$service" ls /run/secrets
done
run docker compose exec -T rabbitmq ls /etc/rabbitmq/conf.d
@@ -49,6 +54,7 @@ echo 'container.txt'
run docker compose exec -T redis redis-server --version
run docker compose exec -T s3 minio --version
run docker compose exec -T rabbitmq rabbitmqctl -q version
run sh -c "docker compose images otel-collector prometheus loki tempo grafana"
echo '## Топология брокера: durable quorum-очереди, delivery-limit и DLQ'
run docker compose exec -T rabbitmq rabbitmqctl -q list_queues name type durable arguments messages
run docker compose exec -T rabbitmq rabbitmqctl -q list_consumers queue_name prefetch_count ack_required
@@ -62,10 +68,12 @@ echo 'container.txt'
echo 'tests.txt'
{
header 'Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml)'
header 'Собственные e2e-тесты (PostgreSQL + Redis + MinIO из compose.test.yaml); основной стенд на время тестов остановлен ради памяти Docker'
docker compose stop lb api worker relay reconciler otel-collector prometheus loki tempo grafana rabbitmq s3 redis postgres >/dev/null 2>&1
docker compose -f compose.test.yaml build -q tests
run docker compose -f compose.test.yaml run --rm tests npx jest --config ./test/jest-e2e.json --verbose
docker compose -f compose.test.yaml down -v >/dev/null 2>&1
docker compose up -d --wait >/dev/null 2>&1
} > evidence/tests.txt
echo 'smoke.txt'
@@ -120,4 +128,32 @@ echo 'jobs.txt'
docker compose logs --no-log-prefix --tail 1 reconciler 2>&1
} > evidence/jobs.txt
echo 'telemetry.txt'
{
header 'Наблюдаемость: контрольный запрос (trace + логи), dashboard, перцентили, приватность, правила'
echo '## Prometheus: правила загружены (promtool и /api/v1/rules)'
run docker compose exec -T prometheus promtool check rules /etc/prometheus/rules/messenger.yml
run sh -c "wget -qO- http://grafana:3000/api/datasources/proxy/uid/prometheus/api/v1/rules | python3 -c 'import json,sys; [print(r[\"type\"], r[\"name\"], r[\"health\"]) for g in json.load(sys.stdin)[\"data\"][\"groups\"] for r in g[\"rules\"]]'"
echo '## Grafana: datasources с фиксированными UID, dashboard из provisioning'
run sh -c "for uid in prometheus loki tempo; do wget -qO- http://grafana:3000/api/datasources/uid/\$uid | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d[\"uid\"], d[\"type\"], d[\"url\"])'; done"
echo '## Контрольный upload с клиентским traceparent: trace в Tempo и логи в Loki'
run python3 scripts/trace-check.py
echo '## Секреты и содержимое не попадают в telemetry; в labels метрик нет ID'
run python3 scripts/telemetry-privacy.py
echo '## Перцентили dashboard против клиентского замера tests/load.py'
run python3 scripts/latency-compare.py --duration 30
echo '## Все запросы dashboard (пустые панели alerts/WARN-логов допустимы без инцидентов)'
run python3 scripts/dashboard-check.py
echo '## Пример структурного лога (stdout API)'
docker compose logs --no-log-prefix --tail 200 api 2>&1 | grep '"event":"http.request"' | tail -2
} > evidence/telemetry.txt
for scenario in retry-trace worker-stop db telemetry; do
echo "incident-$scenario.txt"
{
header "Инцидент: scripts/incidents.py $scenario"
run python3 scripts/incidents.py "$scenario"
} > "evidence/incident-$scenario.txt"
done
echo 'done'
+386
View File
@@ -0,0 +1,386 @@
#!/usr/bin/env python3
"""Controlled incidents for lab6 observability; run inside the tools container.
docker compose run --rm tools python3 scripts/incidents.py worker-stop|db|telemetry|retry-trace
Every scenario injects a fault, keeps synthetic traffic going, samples Prometheus/Tempo/Loki through Grafana
and waits for the related alert to become firing and then resolved after recovery.
"""
import base64
import importlib.util
import json
import os
import statistics
import struct
import subprocess
import sys
import threading
import time
import urllib.parse
import urllib.request
import uuid
import zlib
spec = importlib.util.spec_from_file_location('job_scenario', os.path.join(os.path.dirname(__file__), 'job-scenario.py'))
js = importlib.util.module_from_spec(spec)
spec.loader.exec_module(js)
log = js.log
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
PROM = GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/'
TEMPO = GRAFANA + '/api/datasources/proxy/uid/tempo/api/'
LOKI = GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/'
def get(url, **params):
with urllib.request.urlopen(url + ('?' + urllib.parse.urlencode(params) if params else ''), timeout=20) as r:
return json.loads(r.read())
def prom(expr):
try:
result = get(PROM + 'query', query=expr)['data']['result']
except OSError:
return None
return float(result[0]['value'][1]) if result else None
def alert_state(name):
try:
alerts = get(PROM + 'alerts')['data']['alerts']
except OSError:
return 'unknown'
states = [a['state'] for a in alerts if a['labels'].get('alertname') == name]
return 'firing' if 'firing' in states else ('pending' if 'pending' in states else 'inactive')
def fmt(value, unit=''):
if value is None:
return '-'
if unit == '%':
return '%.1f%%' % (value * 100)
if unit == 'ms':
return '%.0fms' % (value * 1000)
if unit == 'MiB':
return '%.0fMiB' % (value / 2 ** 20)
return ('%.1f' % value).rstrip('0').rstrip('.')
def sample(columns, alerts):
parts = ['%s=%s' % (label, fmt(prom(expr), unit)) for label, expr, unit in columns]
parts += ['%s=%s' % (name, alert_state(name)) for name in alerts]
log(' '.join(parts))
def watch(columns, alerts, until, timeout, every=10):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
sample(columns, alerts)
if until():
return True
time.sleep(every)
sample(columns, alerts)
return False
class Traffic:
"""Closed-loop synthetic traffic through the LB: POST message + GET history, counts statuses per phase."""
def __init__(self, auth, chat, threads=4, pause=0.2):
self.auth, self.chat, self.threads, self.pause = auth, chat, threads, pause
self.stop = threading.Event()
self.lock = threading.Lock()
self.reset()
def reset(self):
with self.lock:
self.codes, self.latencies = {}, []
def snapshot(self, phase):
with self.lock:
codes, latencies = dict(self.codes), sorted(self.latencies)
p = lambda q: latencies[min(len(latencies) - 1, int(q * len(latencies)))] * 1000 if latencies else 0
log('client [%s]: %s; p50=%.0fms p95=%.0fms p99=%.0fms' % (
phase, ' '.join('%s:%d' % kv for kv in sorted(codes.items())), p(.5), p(.95), p(.99)))
self.reset()
def run(self):
def loop(n):
i = 0
while not self.stop.is_set():
started = time.monotonic()
if i % 2 == 0:
code = js.call('POST', '/api/v1/chats/%s/messages' % self.chat, {'text': 'incident %d-%d' % (n, i)},
{**self.auth, 'Idempotency-Key': uuid.uuid4().hex})[0]
else:
code = js.call('GET', '/api/v1/chats/%s/messages?limit=20' % self.chat, headers=self.auth)[0]
with self.lock:
self.codes[code] = self.codes.get(code, 0) + 1
if 200 <= code < 300:
self.latencies.append(time.monotonic() - started)
i += 1
time.sleep(self.pause)
self.workers = [threading.Thread(target=loop, args=(n,), daemon=True) for n in range(self.threads)]
for worker in self.workers:
worker.start()
return self
def close(self):
self.stop.set()
for worker in self.workers:
worker.join(20)
def b64hex(value):
return base64.b64decode(value).hex() if value else ''
def trace_spans(trace_id):
body = get(TEMPO + 'traces/' + trace_id)
spans = []
for batch in body.get('batches', []):
service = {a['key']: a['value'] for a in batch['resource']['attributes']}['service.name']['stringValue']
for scope in batch.get('scopeSpans', []):
for span in scope.get('spans', []):
span['service'] = service
span['ms'] = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
span['attrs'] = {a['key']: list(a['value'].values())[0] for a in span.get('attributes', [])}
spans.append(span)
return spans
def find_traces(query, since, limit=5):
try:
return get(TEMPO + 'search', q=query, limit=limit, start=int(since), end=int(time.time()) + 60).get('traces', [])
except OSError:
return []
def breakdown(trace_id, title):
spans = trace_spans(trace_id)
server = next((s for s in spans if s.get('kind') == 'SPAN_KIND_SERVER'), None)
if not server:
log('trace %s: no server span' % trace_id)
return
names = {s['spanId']: s['name'] for s in spans}
def total(prefix, top_level=False):
return sum(s['ms'] for s in spans if s['name'].startswith(prefix)
and not (top_level and names.get(s.get('parentSpanId'), '').startswith('prisma:')))
db, s3 = total('prisma:', top_level=True), total('S3.')
log('%s: trace %s %s -> %s, %.0f ms' % (title, trace_id, server['name'], server['attrs'].get('http.status_code'), server['ms']))
log(' DB calls %.0f ms (pool wait %.0f ms, SQL %.0f ms); S3 %.0f ms; handler itself %.0f ms' % (
db, total('prisma:engine:connection'), total('prisma:engine:db_query'), s3, max(0, server['ms'] - db - s3)))
errors = sorted({s['name'] for s in spans if s.get('status', {}).get('code') == 'STATUS_CODE_ERROR'})
if errors:
log(' spans with error status:', ', '.join(errors))
def noise_png(width, height):
raw = b''.join(b'\0' + os.urandom(width * 3) for _ in range(height))
def chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'')
def docker(*args):
log('$ docker', ' '.join(args))
subprocess.run(['docker', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def container(service):
return subprocess.run(['docker', 'compose', 'ps', '-q', service], check=True, capture_output=True, text=True).stdout.split()
def scenario_worker_stop():
"""Incident 1: worker stopped -> jobs age, MessengerJobQueueLag fires -> worker started -> backlog drains, resolved."""
js.wait_ready()
auth, chat = js.session()
columns = [
('due', 'max(jobs_queue_depth{state="due"})', ''),
('processing', 'max(jobs_queue_depth{state="processing"})', ''),
('oldest_age_s', 'messenger:jobs_queue_oldest_age:max', ''),
('worker_busy', 'sum(worker_jobs_busy)', ''),
('worker_up', 'count(timestamp(worker_jobs_capacity) > time() - 30)', ''),
('ready_per_s', 'sum(rate(jobs_processed_total{outcome="success"}[1m]))', ''),
]
alerts = ['MessengerJobQueueLag']
js.compose('stop', 'worker')
stopped = time.time()
uploads = []
def upload_more():
if len(uploads) < 12 and time.time() - stopped < 150:
uploads.append(js.upload(auth, chat))
return alert_state('MessengerJobQueueLag') == 'firing'
fired = watch(columns, alerts, upload_more, timeout=300)
log('alert MessengerJobQueueLag firing after %.0f s of worker downtime: %s' % (time.time() - stopped, fired))
js.start_only('worker')
started = time.time()
resolved = watch(columns, alerts, lambda: alert_state('MessengerJobQueueLag') == 'inactive', timeout=240)
log('backlog drained and alert resolved %.0f s after worker start: %s' % (time.time() - started, resolved))
ready = sum(1 for a in uploads if js.metadata(auth, a)[1]['status'] == 'ready')
log('uploads during downtime: %d, ready now: %d' % (len(uploads), ready))
time.sleep(15)
log('time to ready, histogram over 5m: p50=%s s, p99=%s s' % tuple(fmt(prom(
'histogram_quantile(%s, sum by (le) (increase(attachments_time_to_ready_seconds_bucket[5m])))' % q)) for q in (0.5, 0.99)))
log('time to ready from the database (finished_at - created_at), s:', js.psql(
"SELECT round(min(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) || ' .. ' || "
"round(max(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) FROM attachments a "
"JOIN attachment_jobs j ON j.attachment_id = a.id WHERE a.id IN (%s)" % ','.join("'%s'" % a for a in uploads)))
if not (fired and resolved and ready == len(uploads)):
sys.exit('FAIL')
log('PASS')
def scenario_db():
"""Incident 2: slow DB (CPU-throttled postgres) vs slow handler (large upload); then DB unavailable (paused)."""
js.wait_ready()
auth, chat = js.session()
columns = [
('p95_1m', 'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api",http_route!~"/health.*"}[1m])))', 'ms'),
('5xx_1m', '(sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*",http_response_status_class="5xx"}[1m])) or vector(0)) / sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*"}[1m]))', '%'),
('pool_wait', 'sum(db_client_connection_pending_requests{job="messenger-api"})', ''),
('pool_used', 'sum(db_client_connection_count{job="messenger-api",db_client_connection_state="used"})', ''),
('core_5xx_1m', 'sum(increase(http_server_requests_total{http_route="/health/core",http_response_status_class="5xx"}[1m]))', ''),
('api_cpu', 'sum(rate(process_cpu_time_seconds_total{job="messenger-api"}[1m]))', ''),
]
alerts = ['MessengerSlowApi', 'MessengerHighErrorRate', 'MessengerApiNotReady']
traffic = Traffic(auth, chat).run()
try:
log('== baseline: 60 s of traffic')
watch(columns, alerts, lambda: False, timeout=60, every=20)
traffic.snapshot('baseline')
log('== slow handler (not DB): 1700x1700 noise PNG upload — decoding and S3 dominate')
big = noise_png(1700, 1700)
since = time.time()
code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat,
headers={**auth, 'Content-Type': 'image/png'}, raw=big)
log('POST %.1f MiB attachment -> %s' % (len(big) / 2 ** 20, code))
time.sleep(15)
for t in find_traces('{resource.service.name="messenger-api" && kind=server && span.http.route="/api/v1/chats/{chat_id}/attachments"}', since, 1):
breakdown(t['traceID'], 'slow handler')
log('== slow DB: postgres limited to 0.05 CPU for 150 s')
postgres = container('postgres')
since = time.time()
docker('update', '--cpus', '0.05', *postgres)
watch(columns, alerts, lambda: alert_state('MessengerSlowApi') == 'firing', timeout=150, every=15)
traffic.snapshot('slow DB')
slow = find_traces('{resource.service.name="messenger-api" && kind=server && duration > 500ms}', since, 3)
for t in slow[:2]:
breakdown(t['traceID'], 'slow DB')
docker('update', '--cpus', '1.0', *postgres)
watch(columns, alerts, lambda: alert_state('MessengerSlowApi') != 'firing', timeout=240, every=15)
traffic.snapshot('after slow DB')
log('== DB unavailable: postgres paused for 120 s')
since = time.time()
docker('pause', *postgres)
not_ready = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'firing', timeout=120, every=15)
traffic.snapshot('DB paused')
for t in find_traces('{resource.service.name="messenger-api" && kind=server && status=error}', since, 2):
breakdown(t['traceID'], 'DB unavailable')
logs = get(LOKI + 'query_range', query='{service_name="messenger-api"} | event="http.error"', limit=3, since='3m')
for stream in logs['data']['result'][:3]:
log(' loki:', stream['values'][0][1][:200])
docker('unpause', *postgres)
recovered = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'inactive', timeout=240, every=15)
traffic.snapshot('after DB unpaused')
finally:
traffic.close()
subprocess.run(['docker', 'unpause', *container('postgres')], capture_output=True)
subprocess.run(['docker', 'update', '--cpus', '1.0', *container('postgres')], capture_output=True)
log('MessengerApiNotReady fired: %s, resolved after recovery: %s' % (not_ready, recovered))
log('PASS' if not_ready and recovered else 'FAIL')
def scenario_telemetry():
"""Incident 3: Loki and Tempo stopped, then the Collector stopped; API keeps serving, memory stays bounded."""
js.wait_ready()
auth, chat = js.session()
columns = [
('spans_failed_1m', 'sum(increase(otelcol_exporter_send_failed_spans[1m]))', ''),
('logs_failed_1m', 'sum(increase(otelcol_exporter_send_failed_log_records[1m]))', ''),
('queue', 'max(otelcol_exporter_queue_size)', ''),
('queue_cap', 'max(otelcol_exporter_queue_capacity)', ''),
('refused_1m', 'sum(increase(otelcol_receiver_refused_spans[1m])) + sum(increase(otelcol_receiver_refused_log_records[1m]))', ''),
('collector_rss', 'max(otelcol_process_memory_rss)', 'MiB'),
('api_rss', 'max(process_memory_usage_bytes{job="messenger-api"})', 'MiB'),
]
alerts = ['TelemetryExportFailing', 'MessengerHighErrorRate']
traffic = Traffic(auth, chat, threads=6, pause=0.05).run()
try:
watch(columns, alerts, lambda: False, timeout=30, every=15)
traffic.snapshot('baseline')
log('== Loki and Tempo stopped until TelemetryExportFailing fires (max 240 s)')
js.compose('stop', 'loki', 'tempo')
fired = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'firing', timeout=240, every=15)
traffic.snapshot('Loki+Tempo down')
js.compose('start', 'loki', 'tempo')
resolved = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'inactive', timeout=240, every=15)
traffic.snapshot('after start')
log('== Collector stopped for 60 s: SDK export fails fast (5 s timeout, bounded queues)')
js.compose('stop', 'otel-collector')
for _ in range(4):
time.sleep(15)
log('docker stats:', subprocess.run(
['docker', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', *container('api'), *container('worker')],
capture_output=True, text=True).stdout.replace('\n', '; '))
traffic.snapshot('collector down')
js.compose('start', 'otel-collector')
time.sleep(30)
traffic.snapshot('collector back')
finally:
traffic.close()
subprocess.run(['docker', 'compose', 'start', 'loki', 'tempo', 'otel-collector'], capture_output=True)
log('TelemetryExportFailing fired: %s, resolved: %s' % (fired, resolved))
log('PASS' if fired and resolved else 'FAIL')
def scenario_retry_trace():
"""Level 4 trace: one upload whose first attempt fails on S3, then succeeds; all attempts in one trace."""
js.wait_ready()
auth, chat = js.session()
trace_id = os.urandom(16).hex()
js.compose('stop', 'worker')
code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
headers={**auth, 'Content-Type': 'image/png', 'traceparent': '00-%s-%s-01' % (trace_id, os.urandom(8).hex())},
raw=js.SAMPLE)
attachment = js.json_of(raw)['id']
log('upload ->', code, attachment, 'trace', trace_id)
js.compose('stop', 's3')
js.start_only('worker')
time.sleep(5)
js.start_only('s3')
js.wait_status(auth, attachment, ('ready', 'failed'))
js.job_state(attachment)
time.sleep(15)
spans = trace_spans(trace_id)
for span in sorted(spans, key=lambda s: int(s['startTimeUnixNano'])):
if span.get('kind') in ('SPAN_KIND_SERVER', 'SPAN_KIND_PRODUCER', 'SPAN_KIND_CONSUMER') or span['name'].startswith('S3.'):
log(' %-17s %-20s %-36s attempt=%-2s %8.1f ms %s' % (
span['service'], span.get('kind', '')[10:], span['name'], span['attrs'].get('messaging.job.attempt', ''),
span['ms'], span.get('status', {}).get('code', '')[12:]))
consumers = [s for s in spans if s.get('kind') == 'SPAN_KIND_CONSUMER']
log('consumer spans in one trace: %d (attempts %s)' % (
len(consumers), sorted({s['attrs'].get('messaging.job.attempt') for s in consumers})))
log('PASS' if len(consumers) >= 2 else 'FAIL')
SCENARIOS = {
'worker-stop': scenario_worker_stop,
'db': scenario_db,
'telemetry': scenario_telemetry,
'retry-trace': scenario_retry_trace,
}
if __name__ == '__main__':
if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS:
sys.exit('usage: incidents.py %s' % '|'.join(SCENARIOS))
SCENARIOS[sys.argv[1]]()
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""Checks that passwords, tokens, message text and filenames do not reach Loki/Tempo and that metric labels hold no IDs.
Run inside the tools container: docker compose run --rm tools python3 scripts/telemetry-privacy.py
"""
import base64
import json
import os
import re
import sys
import time
import urllib.parse
import urllib.request
import uuid
BASE = os.environ.get('BASE_URL', 'http://lb:8080')
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
SAMPLE = open(os.path.join(os.path.dirname(__file__), '..', 'tests', 'fixtures', 'sample.png'), 'rb').read()
UUID = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}')
def call(method, path, body=None, headers=None, raw=None):
data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
headers = dict(headers or {})
if body is not None:
headers['Content-Type'] = 'application/json'
with urllib.request.urlopen(urllib.request.Request(BASE + path, data=data, method=method, headers=headers), timeout=15) as r:
return json.loads(r.read() or b'null')
def grafana(path, **params):
with urllib.request.urlopen(GRAFANA + path + '?' + urllib.parse.urlencode(params), timeout=30) as r:
return r.read().decode()
def main():
marker = uuid.uuid4().hex[:12]
secrets = {
'password': 'Privacy-password-' + marker,
'message text': 'privacy message body ' + marker,
'filename': 'privacy-file-%s.png' % marker,
}
trace_ids = []
def traced():
trace_id = os.urandom(16).hex()
trace_ids.append(trace_id)
return {'traceparent': '00-%s-%s-01' % (trace_id, os.urandom(8).hex())}
username = 'privacy_' + marker
user = call('POST', '/api/v1/users', {'username': username, 'display_name': 'Privacy', 'password': secrets['password']}, traced())
basic = base64.b64encode(('%s:%s' % (username, secrets['password'])).encode()).decode()
token = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic, **traced()})['token']
secrets['session token'] = token
secrets['basic credentials'] = basic
auth = {'Authorization': 'Bearer ' + token}
chat = call('POST', '/api/v1/chats', {'title': 'Privacy', 'member_ids': [user['id']]}, {**auth, **traced()})['id']
call('POST', '/api/v1/chats/%s/messages' % chat, {'text': secrets['message text']},
{**auth, 'Idempotency-Key': marker, **traced()})
call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat, secrets['filename']), headers={**auth, 'Content-Type': 'image/png', **traced()},
raw=SAMPLE)
call('GET', '/api/v1/chats/%s/messages?limit=5' % chat, headers={**auth, **traced()})
print('sent %d traced requests with known password, token, message text and filename; waiting 25 s for export' % len(trace_ids))
time.sleep(25)
traces = ''
found_traces = 0
for trace_id in trace_ids:
try:
traces += grafana('/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
found_traces += 1
except OSError:
pass
def loki_lines(query):
body = grafana('/api/datasources/proxy/uid/loki/loki/api/v1/query_range', query=query, limit=100, since='5m')
return sum(len(s['values']) for s in json.loads(body)['data']['result'])
own = loki_lines('{service_name=~"messenger-.+"} |~ "%s"' % '|'.join(trace_ids))
print('Tempo: %d/%d traces fetched (%d bytes); Loki: %d log lines carry these trace_ids (control)' % (
found_traces, len(trace_ids), len(traces), own))
leaks = 0
for name, value in list(secrets.items()) + [('Authorization hdr', 'authorization')]:
in_traces = value.lower() in traces.lower()
in_logs = loki_lines('{service_name=~"messenger-.+"} |~ "(?i)%s"' % re.escape(value).replace('\\', '\\\\'))
leaks += in_traces + in_logs
print(' %-18s in traces: %-5s log lines in Loki (5 min): %d' % (name, in_traces, in_logs))
series = json.loads(grafana('/api/datasources/proxy/uid/prometheus/api/v1/series', **{'match[]': '{job=~"messenger-.+"}'}))['data']
names = {s['__name__'] for s in series}
with_ids = [s for s in series if any(UUID.search(v) for k, v in s.items() if k != 'instance')]
labels = sorted({k for s in series for k in s} - {'__name__'})
print('Prometheus: %d series, %d metric names from messenger services; label keys: %s' % (len(series), len(names), ', '.join(labels)))
print(' series with a UUID in any label value: %d' % len(with_ids))
routes = sorted({s.get('http_route') for s in series if s.get('http_route')})
print(' http_route values: %s' % ', '.join(routes))
ok = found_traces == len(trace_ids) and own > 0 and leaks == 0 and not with_ids
print('%s: no secrets/content in logs and traces, no IDs in metric labels' % ('PASS' if ok else 'FAIL'))
sys.exit(0 if ok else 1)
if __name__ == '__main__':
main()