134 lines
6.8 KiB
Python
134 lines
6.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Live lab6 checks. No tokens, message contents or backend dumps are printed."""
|
|
import argparse
|
|
import base64
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, check
|
|
|
|
|
|
def get(url, auth=None):
|
|
headers = {'Accept': 'application/json'}
|
|
if auth:
|
|
headers['Authorization'] = 'Basic ' + base64.b64encode(auth.encode()).decode()
|
|
with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=15) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def eventually(fn, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
result = fn()
|
|
if result:
|
|
return result
|
|
except (urllib.error.URLError, AssertionError, KeyError, ValueError):
|
|
pass
|
|
time.sleep(2)
|
|
raise AssertionError('Telemetry check timed out')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--base-url', default='http://localhost:8080')
|
|
parser.add_argument('--prometheus-url', default='http://localhost:9090')
|
|
parser.add_argument('--grafana-url', default='http://localhost:3000')
|
|
parser.add_argument('--output', default='evidence/observability.json')
|
|
args = parser.parse_args()
|
|
auth = os.environ.get('GRAFANA_ADMIN_USER', 'admin') + ':' + (ROOT / 'secrets/grafana_admin_password').read_text().strip()
|
|
proxy = args.grafana_url + '/api/datasources/proxy/uid/'
|
|
def prom(query):
|
|
return get(args.prometheus_url + '/api/v1/query?' + urllib.parse.urlencode({'query': query}))['data']['result']
|
|
c = Client(args.base_url)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
trace = uuid.uuid4().hex
|
|
parent = '00-' + trace + '-0123456789abcdef-01'
|
|
secret_marker = 'private-' + uuid.uuid4().hex
|
|
c.message(alice, chat, secret_marker)
|
|
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202,
|
|
actor=alice, raw=content, headers={'Content-Type': 'image/png', 'traceparent': parent}, schema='Attachment')['json']
|
|
c.wait_ready(bob, att)
|
|
def trace_complete():
|
|
data = get(proxy + 'tempo/api/traces/' + trace, auth)
|
|
spans = []
|
|
def walk(obj):
|
|
if isinstance(obj, dict):
|
|
if 'spanId' in obj and 'name' in obj:
|
|
spans.append(obj)
|
|
for value in obj.values():
|
|
walk(value)
|
|
elif isinstance(obj, list):
|
|
for value in obj:
|
|
walk(value)
|
|
walk(data)
|
|
names = {span['name'] for span in spans}
|
|
return (data, spans) if {'attachment.enqueue', 'jobs:attachments publish', 'attachment.process', 's3.put_object', 'postgresql.publish_ready'} <= names else None
|
|
trace_data, spans = eventually(trace_complete)
|
|
by_id = {span['spanId']: span for span in spans}
|
|
consumer = next(span for span in spans if span['name'] == 'attachment.process')
|
|
publish = by_id.get(consumer['parentSpanId'])
|
|
check(publish and publish['name'] == 'jobs:attachments publish', 'consumer parent must be producer')
|
|
enqueue = by_id.get(publish['parentSpanId'])
|
|
check(enqueue and enqueue['name'] == 'attachment.enqueue', 'publish must continue durable outbox context')
|
|
server = by_id.get(enqueue['parentSpanId'])
|
|
check(server and server['name'].startswith('POST '), 'enqueue parent must be HTTP server')
|
|
query = '{service_name=~"messenger-api|messenger-worker"} | trace_id="' + trace + '"'
|
|
def correlated_logs():
|
|
result = get(proxy + 'loki/loki/api/v1/query_range?' + urllib.parse.urlencode({'query': query, 'limit': 1000}), auth)['data']['result']
|
|
services = {stream['stream'].get('service_name') for stream in result}
|
|
return result if {'messenger-api', 'messenger-worker'} <= services else None
|
|
logs = eventually(correlated_logs)
|
|
required = ['http_server_requests_total', 'http_server_request_duration_seconds_bucket', 'worker_jobs_total',
|
|
'worker_job_duration_seconds_bucket', 'attachment_ready_duration_seconds_bucket', 'queue_depth',
|
|
'queue_oldest_age', 'db_pool_used', 'db_pool_waiting', 'worker_busy', 'worker_capacity',
|
|
'process_memory_usage', 'process_cpu_time']
|
|
for name in required:
|
|
eventually(lambda: prom(name))
|
|
panel_count = 0
|
|
dashboard = get(args.grafana_url + '/api/dashboards/uid/messenger-observability', auth)['dashboard']
|
|
check(len(dashboard['panels']) == len(json.loads((ROOT / 'infra/grafana/dashboards/messenger.json').read_text())['panels']), 'Provisioned dashboard differs')
|
|
for panel in dashboard['panels']:
|
|
if panel['datasource']['uid'] != 'prometheus':
|
|
continue
|
|
for target in panel['targets']:
|
|
expr = target['expr'].replace('$__rate_interval', '1m')
|
|
result = prom(expr)
|
|
# Diagnostics are allowed to be absent when no failure has occurred.
|
|
if 'otelcol_' not in expr:
|
|
result = eventually(lambda: prom(expr), timeout=45)
|
|
panel_count += 1
|
|
for item in prom('http_server_requests_total'):
|
|
labels = item['metric']
|
|
check(not {'user_id','chat_id','message_id','attachment_id','request_id','filename'} & labels.keys(), 'High cardinality metric label')
|
|
check(chat['id'] not in json.dumps(labels), 'Raw chat ID in labels')
|
|
# Inspect full recent log/trace data in memory, never save private test values.
|
|
all_logs = get(proxy + 'loki/loki/api/v1/query_range?' + urllib.parse.urlencode({
|
|
'query': '{service_name=~"messenger-api|messenger-worker"}', 'limit': 5000}), auth)
|
|
exported = json.dumps([all_logs, trace_data])
|
|
for forbidden in [secret_marker, alice['token'], alice['registration']['password'], base64.b64encode(content).decode()]:
|
|
check(forbidden not in exported, 'Private test data leaked to telemetry')
|
|
report = {'timestamp': datetime.now(timezone.utc).isoformat(), 'result': 'PASS', 'trace_id': trace,
|
|
'span_names': sorted({s['name'] for s in spans}), 'consumer_parent_is_producer': True,
|
|
'log_services': sorted({s['stream']['service_name'] for s in logs}),
|
|
'metric_names_checked': required, 'dashboard_queries_checked': panel_count,
|
|
'privacy_canaries': 'absent in recent logs and upload trace'}
|
|
(ROOT / args.output).write_text(json.dumps(report, indent=2) + '\n')
|
|
print('PASS: real API -> outbox -> Redis -> worker -> S3/DB trace, correlated logs, metrics and dashboard queries; privacy canaries absent.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|