#!/usr/bin/env python3 """Strict lab6 observability gate. It creates a fresh message and attachment, waits for all three signals, checks route/template cardinality, Grafana provisioning and rules, and verifies that user content and credentials never reached Loki. A missing signal is a failure, not a warning. """ import argparse import os import base64 import json import re import sys import time import urllib.error import urllib.parse import urllib.request import uuid from pathlib import Path # The lab tools always target loopback; ignore ambient corporate proxies. for _proxy_key in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'): os.environ.pop(_proxy_key, None) os.environ['NO_PROXY'] = 'localhost,127.0.0.1,::1' os.environ['no_proxy'] = 'localhost,127.0.0.1,::1' urllib.request.getproxies = lambda: {} ROOT = Path(__file__).resolve().parents[1] UUID_RE = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', re.I) FORBIDDEN_LABELS = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'filename', 'request_id', 'object_key', 'path', 'query'} def request(url, auth=None, timeout=10): headers = {'Accept': 'application/json'} if auth: headers['Authorization'] = 'Basic ' + base64.b64encode(auth.encode()).decode() req = urllib.request.Request(url, headers=headers) with urllib.request.urlopen(req, timeout=timeout) as response: return response.status, response.read() def prom(base, expr): url = base.rstrip('/') + '/api/v1/query?' + urllib.parse.urlencode({'query': expr}) _, raw = request(url) return json.loads(raw).get('data', {}).get('result', []) def prom_series(base, match): url = base.rstrip('/') + '/api/v1/series?' + urllib.parse.urlencode({'match[]': match}) _, raw = request(url) return json.loads(raw).get('data', []) def loki_query(base, query, start_ns, end_ns, limit=20): params = urllib.parse.urlencode({'query': query, 'start': start_ns, 'end': end_ns, 'limit': limit}) _, raw = request(base.rstrip('/') + '/loki/api/v1/query_range?' + params) return json.loads(raw).get('data', {}).get('result', []) def grafana_password(): path = ROOT / 'secrets/grafana.env' if not path.exists(): return None for line in path.read_text().splitlines(): if '=' in line and line.split('=', 1)[0].strip() == 'GF_SECURITY_ADMIN_PASSWORD': return line.split('=', 1)[1].strip() return None def wait_for(fn, attempts=30, delay=2): last = None for _ in range(attempts): try: value = fn() if value: return value except Exception as exc: # noqa: BLE001 - retry the external endpoint last = exc time.sleep(delay) if last: raise last return None def main(): parser = argparse.ArgumentParser(description='Strict lab6 telemetry gate') parser.add_argument('--base-url', default='http://localhost:8080') parser.add_argument('--prometheus', default='http://localhost:9090') parser.add_argument('--loki', default='http://localhost:3100') parser.add_argument('--tempo', default='http://localhost:3200') parser.add_argument('--grafana', default='http://localhost:3000') parser.add_argument('--wait-seconds', type=int, default=60) args = parser.parse_args() failures = [] notes = [] # 1. Generate a controlled request, a message and a job. sys.path.insert(0, str(ROOT / 'tests')) from client import Client client = Client(args.base_url, timeout=15) alice, _, _, chat = client.bootstrap() request_id = 'lab6-verify-' + uuid.uuid4().hex sentinel = 'SENTINEL-TELEMETRY-' + uuid.uuid4().hex client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, actor=alice, body={'text': sentinel}, headers={'Idempotency-Key': uuid.uuid4().hex, 'X-Request-ID': request_id}) att, content = client.attachment(alice, chat) client.wait_ready(alice, att, wait_seconds=60) filename = 'sentinel-' + uuid.uuid4().hex + '.png' from urllib.parse import quote client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + quote(filename), 202, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment') token = alice.get('token', '') notes.append('created request_id=' + request_id) # 2. Required metric series. Counter/histogram series can take one export # and one scrape interval to appear. required = [ ('http_server_request_duration_seconds_count', 'HTTP request histogram count'), ('http_server_request_duration_seconds_bucket', 'HTTP duration histogram'), ('messenger_jobs_processed_total', 'job attempt counter'), ('messenger_jobs_accepted_total', 'accepted jobs'), ('messenger_jobs_ready_within_budget_total', 'ready-within-budget'), ('messenger_job_duration_seconds_bucket', 'job duration histogram'), ('messenger_job_time_to_ready_seconds_bucket', 'time-to-ready histogram'), ('messenger_jobs_overdue', 'overdue jobs'), ('messenger_worker_busy', 'worker saturation'), ('messenger_db_pool_used', 'DB pool'), ('messenger_queue_oldest_job_age_seconds', 'oldest job age'), ('messenger_outbox_oldest_unpublished_age_seconds', 'outbox age'), ('messenger_process_cpu_seconds_total', 'process CPU'), ('rabbitmq_queue_messages_ready', 'RabbitMQ queue depth'), ('otelcol_exporter_queue_size', 'collector queue self-metric'), ] try: if not wait_for(lambda: prom(args.prometheus, 'sum(rate(messenger_job_time_to_ready_seconds_bucket{le="30.0"}[5m]))'), attempts=20, delay=3): failures.append('TTR good-event selector le="30.0" is empty') except Exception as exc: failures.append('TTR selector: ' + str(exc)) for metric, label in required: try: found = wait_for(lambda m=metric: prom(args.prometheus, 'count(' + m + ')'), attempts=max(1, args.wait_seconds // 3), delay=3) if not found: failures.append(label + ': no samples (' + metric + ')') except Exception as exc: failures.append(label + ': ' + str(exc)) try: up = prom(args.prometheus, 'up{job=~"otel-collector|haproxy|rabbitmq"} == 1') jobs = {item.get('metric', {}).get('job') for item in up} for job in ('otel-collector', 'haproxy', 'rabbitmq'): if job not in jobs: failures.append('Prometheus target is not up: ' + job) except Exception as exc: failures.append('Prometheus targets: ' + str(exc)) try: series = prom_series(args.prometheus, 'http_server_request_duration_seconds_count') routes = [item.get('http_route', '') for item in series] if not series: failures.append('HTTP metric has no series') for route in routes: if not route: failures.append('HTTP metric has an empty http_route label') if UUID_RE.search(route): failures.append('HTTP route label contains a UUID: ' + route) labels = {key for item in series for key in item} forbidden = labels & FORBIDDEN_LABELS if forbidden: failures.append('HTTP metric has forbidden labels: ' + ', '.join(sorted(forbidden))) methods = {item.get('http_request_method', '') for item in series} if len(methods) > 7: failures.append('HTTP method label cardinality is too high: ' + str(len(methods))) for method in methods: if method not in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS', 'OTHER'}: failures.append('HTTP method is not normalized: ' + method) except Exception as exc: failures.append('HTTP label check: ' + str(exc)) # 3. Loki must contain the access log for the controlled request and must # not contain the message text, session token or file bytes. end = int(time.time() * 1e9) start = end - 15 * 60 * int(1e9) try: found = wait_for(lambda: loki_query(args.loki, '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | request_id="' + request_id + '"', start, end), attempts=20, delay=2) if not found: failures.append('Loki: no log with the generated request_id (trace/log correlation)') for needle, label in ((sentinel, 'message text'), (token, 'session token'), (filename, 'filename')): if needle and loki_query(args.loki, '{service_name=~"messenger-api|messenger-worker|messenger-relay"} |= "' + needle + '"', start, end): failures.append('Loki contains forbidden ' + label) except Exception as exc: failures.append('Loki query: ' + str(exc)) # 4. Tempo must contain real application traces, including the worker span. # Tempo's search `name=` matches root spans only. job.process is a child of # the controlled API trace, so resolve its trace_id from the worker log and # fetch that trace directly. try: def tempo_search(tag): q = urllib.parse.quote(tag) _, raw = request(args.tempo.rstrip('/') + '/api/search?tags=' + q + '&limit=20') return json.loads(raw).get('traces', []) def worker_job_trace(): result = loki_query( args.loki, '{service_name="messenger-worker"} | event="attachment ready"', start, end, limit=50, ) for stream in result: for _, line in stream.get('values', []): try: entry = json.loads(line) except json.JSONDecodeError: entry = {} trace_id = entry.get('trace_id') or stream.get('stream', {}).get('trace_id', '') if not re.fullmatch(r'[0-9a-f]{32}', trace_id): continue try: _, raw = request(args.tempo.rstrip('/') + '/api/traces/' + trace_id) except Exception: continue try: payload = json.loads(raw) except json.JSONDecodeError: continue spans = [] for batch in payload.get('batches', []): for scope in batch.get('scopeSpans', []): spans.extend(scope.get('spans', [])) publishes = [s for s in spans if s.get('name') == 'job.publish' and s.get('kind') == 'SPAN_KIND_PRODUCER'] processes = [s for s in spans if s.get('name') == 'job.process' and s.get('kind') == 'SPAN_KIND_CONSUMER'] if any(p.get('parentSpanId') == pub.get('spanId') for p in processes for pub in publishes): return trace_id return None if not wait_for(lambda: tempo_search('service.name=messenger-api'), attempts=15, delay=2): failures.append('Tempo: no messenger-api traces') retry_path = ROOT / 'evidence/lab6/trace-retry.json' if retry_path.exists(): retry = json.loads(retry_path.read_text()) spans = [] for batch in retry.get('batches', []): for scope in batch.get('scopeSpans', []): spans.extend(scope.get('spans', [])) processes = [s for s in spans if s.get('name') == 'job.process'] attempts = set() for span in processes: for attr in span.get('attributes', []): if attr.get('key') == 'job.attempt': value = attr.get('value', {}) if 'intValue' in value: attempts.add(int(value['intValue'])) outcomes = set() for span in processes: for attr in span.get('attributes', []): if attr.get('key') == 'job.outcome': value = attr.get('value', {}).get('stringValue') if value: outcomes.add(value) if (len(attempts) < 2 or 'retried' not in outcomes or not any(s.get('name') == 'job.requeue' for s in spans)): failures.append('Tempo: retry trace lacks two attempts, retried outcome or job.requeue') if not wait_for(worker_job_trace, attempts=20, delay=2): failures.append('Tempo: no worker job.process trace linked to worker log') except Exception as exc: failures.append('Tempo query: ' + str(exc)) # 5. Grafana provisioning and rules. try: password = grafana_password() if not password: failures.append('Grafana admin secret is missing (secrets/grafana.env)') else: _, raw = request(args.grafana.rstrip('/') + '/api/health') if json.loads(raw).get('database') != 'ok': failures.append('Grafana health is not ok') _, raw = request(args.grafana.rstrip('/') + '/api/datasources', auth='admin:' + password) names = {item.get('name') for item in json.loads(raw)} for required_name in ('Prometheus', 'Loki', 'Tempo'): if required_name not in names: failures.append('Grafana datasource missing: ' + required_name) for uid in ('lab6-overview', 'lab6-jobs', 'lab6-infra'): _, raw = request(args.grafana.rstrip('/') + '/api/dashboards/uid/' + uid, auth='admin:' + password) if json.loads(raw).get('dashboard', {}).get('uid') != uid: failures.append('Grafana dashboard missing: ' + uid) except Exception as exc: failures.append('Grafana: ' + str(exc)) try: _, raw = request(args.prometheus.rstrip('/') + '/api/v1/rules') rule_names = {r.get('name') for g in json.loads(raw).get('data', {}).get('groups', []) for r in g.get('rules', [])} for name in ('APIErrorBudgetFastBurn', 'APIErrorBudgetSlowBurn', 'QueueOldestJobAgeHigh', 'TTRErrorBudgetFastBurn'): if name not in rule_names: failures.append('Prometheus rule missing: ' + name) except Exception as exc: failures.append('Prometheus rules: ' + str(exc)) for note in notes: print('NOTE:', note) if failures: for item in failures: print('FAIL:', item) return 1 print('PASS: lab6 strict telemetry gate (signals, trace/log correlation, labels, Grafana, rules, privacy sentinels)') return 0 if __name__ == '__main__': raise SystemExit(main())