Files
mtusi2026lab6/student-tests/observability.py
T
2026-11-16 18:47:37 +03:00

134 lines
6.8 KiB
Python

#!/usr/bin/env python3
"""Live lab6 checks. No tokens, message contents or backend dumps are printed."""
import argparse
import base64
from datetime import datetime, timezone
import json
import math
import os
from pathlib import Path
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def get(url, auth=None):
headers = {'Accept': 'application/json'}
if auth:
headers['Authorization'] = 'Basic ' + base64.b64encode(auth.encode()).decode()
with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=15) as response:
return json.load(response)
def eventually(fn, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
result = fn()
if result:
return result
except (urllib.error.URLError, AssertionError, KeyError, ValueError):
pass
time.sleep(2)
raise AssertionError('Telemetry check timed out')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--prometheus-url', default='http://localhost:9090')
parser.add_argument('--grafana-url', default='http://localhost:3000')
parser.add_argument('--output', default='evidence/observability.json')
args = parser.parse_args()
auth = os.environ.get('GRAFANA_ADMIN_USER', 'admin') + ':' + (ROOT / 'secrets/grafana_admin_password').read_text().strip()
proxy = args.grafana_url + '/api/datasources/proxy/uid/'
def prom(query):
return get(args.prometheus_url + '/api/v1/query?' + urllib.parse.urlencode({'query': query}))['data']['result']
c = Client(args.base_url)
alice, bob, _, chat = c.bootstrap()
trace = uuid.uuid4().hex
parent = '00-' + trace + '-0123456789abcdef-01'
secret_marker = 'private-' + uuid.uuid4().hex
c.message(alice, chat, secret_marker)
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202,
actor=alice, raw=content, headers={'Content-Type': 'image/png', 'traceparent': parent}, schema='Attachment')['json']
c.wait_ready(bob, att)
def trace_complete():
data = get(proxy + 'tempo/api/traces/' + trace, auth)
spans = []
def walk(obj):
if isinstance(obj, dict):
if 'spanId' in obj and 'name' in obj:
spans.append(obj)
for value in obj.values():
walk(value)
elif isinstance(obj, list):
for value in obj:
walk(value)
walk(data)
names = {span['name'] for span in spans}
return (data, spans) if {'attachment.enqueue', 'jobs:attachments publish', 'attachment.process', 's3.put_object', 'postgresql.publish_ready'} <= names else None
trace_data, spans = eventually(trace_complete)
by_id = {span['spanId']: span for span in spans}
consumer = next(span for span in spans if span['name'] == 'attachment.process')
publish = by_id.get(consumer['parentSpanId'])
check(publish and publish['name'] == 'jobs:attachments publish', 'consumer parent must be producer')
enqueue = by_id.get(publish['parentSpanId'])
check(enqueue and enqueue['name'] == 'attachment.enqueue', 'publish must continue durable outbox context')
server = by_id.get(enqueue['parentSpanId'])
check(server and server['name'].startswith('POST '), 'enqueue parent must be HTTP server')
query = '{service_name=~"messenger-api|messenger-worker"} | trace_id="' + trace + '"'
def correlated_logs():
result = get(proxy + 'loki/loki/api/v1/query_range?' + urllib.parse.urlencode({'query': query, 'limit': 1000}), auth)['data']['result']
services = {stream['stream'].get('service_name') for stream in result}
return result if {'messenger-api', 'messenger-worker'} <= services else None
logs = eventually(correlated_logs)
required = ['http_server_requests_total', 'http_server_request_duration_seconds_bucket', 'worker_jobs_total',
'worker_job_duration_seconds_bucket', 'attachment_ready_duration_seconds_bucket', 'queue_depth',
'queue_oldest_age', 'db_pool_used', 'db_pool_waiting', 'worker_busy', 'worker_capacity',
'process_memory_usage', 'process_cpu_time']
for name in required:
eventually(lambda: prom(name))
panel_count = 0
dashboard = get(args.grafana_url + '/api/dashboards/uid/messenger-observability', auth)['dashboard']
check(len(dashboard['panels']) == len(json.loads((ROOT / 'infra/grafana/dashboards/messenger.json').read_text())['panels']), 'Provisioned dashboard differs')
for panel in dashboard['panels']:
if panel['datasource']['uid'] != 'prometheus':
continue
for target in panel['targets']:
expr = target['expr'].replace('$__rate_interval', '1m')
result = prom(expr)
# Diagnostics are allowed to be absent when no failure has occurred.
if 'otelcol_' not in expr:
result = eventually(lambda: prom(expr), timeout=45)
panel_count += 1
for item in prom('http_server_requests_total'):
labels = item['metric']
check(not {'user_id','chat_id','message_id','attachment_id','request_id','filename'} & labels.keys(), 'High cardinality metric label')
check(chat['id'] not in json.dumps(labels), 'Raw chat ID in labels')
# Inspect full recent log/trace data in memory, never save private test values.
all_logs = get(proxy + 'loki/loki/api/v1/query_range?' + urllib.parse.urlencode({
'query': '{service_name=~"messenger-api|messenger-worker"}', 'limit': 5000}), auth)
exported = json.dumps([all_logs, trace_data])
for forbidden in [secret_marker, alice['token'], alice['registration']['password'], base64.b64encode(content).decode()]:
check(forbidden not in exported, 'Private test data leaked to telemetry')
report = {'timestamp': datetime.now(timezone.utc).isoformat(), 'result': 'PASS', 'trace_id': trace,
'span_names': sorted({s['name'] for s in spans}), 'consumer_parent_is_producer': True,
'log_services': sorted({s['stream']['service_name'] for s in logs}),
'metric_names_checked': required, 'dashboard_queries_checked': panel_count,
'privacy_canaries': 'absent in recent logs and upload trace'}
(ROOT / args.output).write_text(json.dumps(report, indent=2) + '\n')
print('PASS: real API -> outbox -> Redis -> worker -> S3/DB trace, correlated logs, metrics and dashboard queries; privacy canaries absent.')
if __name__ == '__main__':
main()