Files

310 lines
14 KiB
Python
Executable File

#!/usr/bin/env python3
"""Strict lab6 observability gate.
It creates a fresh message and attachment, waits for all three signals, checks
route/template cardinality, Grafana provisioning and rules, and verifies that
user content and credentials never reached Loki. A missing signal is a failure,
not a warning.
"""
import argparse
import os
import base64
import json
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from pathlib import Path
from lab7_http import BASE_URL, CA_FILE, open_url, grafana_password
# The lab tools always target loopback; ignore ambient corporate proxies.
for _proxy_key in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'):
os.environ.pop(_proxy_key, None)
os.environ['NO_PROXY'] = 'localhost,127.0.0.1,::1'
os.environ['no_proxy'] = 'localhost,127.0.0.1,::1'
urllib.request.getproxies = lambda: {}
ROOT = Path(__file__).resolve().parents[1]
UUID_RE = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', re.I)
FORBIDDEN_LABELS = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'filename', 'request_id', 'object_key', 'path', 'query'}
def request(url, auth=None, timeout=10):
headers = {'Accept': 'application/json'}
if auth:
headers['Authorization'] = 'Basic ' + base64.b64encode(auth.encode()).decode()
req = urllib.request.Request(url, headers=headers)
with open_url(req, timeout=timeout, operator=True) as response:
return response.status, response.read()
def prom(base, expr):
url = base.rstrip('/') + '/api/v1/query?' + urllib.parse.urlencode({'query': expr})
_, raw = request(url)
return json.loads(raw).get('data', {}).get('result', [])
def prom_series(base, match):
url = base.rstrip('/') + '/api/v1/series?' + urllib.parse.urlencode({'match[]': match})
_, raw = request(url)
return json.loads(raw).get('data', [])
def loki_query(base, query, start_ns, end_ns, limit=20):
params = urllib.parse.urlencode({'query': query, 'start': start_ns, 'end': end_ns, 'limit': limit})
_, raw = request(base.rstrip('/') + '/loki/api/v1/query_range?' + params)
return json.loads(raw).get('data', {}).get('result', [])
def wait_for(fn, attempts=30, delay=2):
last = None
for _ in range(attempts):
try:
value = fn()
if value:
return value
except Exception as exc: # noqa: BLE001 - retry the external endpoint
last = exc
time.sleep(delay)
if last:
raise last
return None
def main():
parser = argparse.ArgumentParser(description='Strict lab6 telemetry gate')
parser.add_argument('--base-url', default=BASE_URL)
parser.add_argument('--ca-file', default=CA_FILE)
parser.add_argument('--prometheus', default='https://localhost:9090')
parser.add_argument('--loki', default='https://localhost:3100')
parser.add_argument('--tempo', default='https://localhost:3200')
parser.add_argument('--grafana', default='https://localhost:3000')
parser.add_argument('--wait-seconds', type=int, default=60)
args = parser.parse_args()
failures = []
notes = []
# 1. Generate a controlled request, a message and a job.
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client
client = Client(args.base_url, args.ca_file, timeout=15)
alice, _, _, chat = client.bootstrap()
request_id = 'lab6-verify-' + uuid.uuid4().hex
sentinel = 'SENTINEL-TELEMETRY-' + uuid.uuid4().hex
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, actor=alice,
body={'text': sentinel}, headers={'Idempotency-Key': uuid.uuid4().hex, 'X-Request-ID': request_id})
att, content = client.attachment(alice, chat)
client.wait_ready(alice, att, wait_seconds=60)
filename = 'sentinel-' + uuid.uuid4().hex + '.png'
from urllib.parse import quote
client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + quote(filename), 202,
actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
token = alice.get('token', '')
notes.append('created request_id=' + request_id)
# 2. Required metric series. Counter/histogram series can take one export
# and one scrape interval to appear.
required = [
('http_server_request_duration_seconds_count', 'HTTP request histogram count'),
('http_server_request_duration_seconds_bucket', 'HTTP duration histogram'),
('messenger_jobs_processed_total', 'job attempt counter'),
('messenger_jobs_accepted_total', 'accepted jobs'),
('messenger_jobs_ready_within_budget_total', 'ready-within-budget'),
('messenger_job_duration_seconds_bucket', 'job duration histogram'),
('messenger_job_time_to_ready_seconds_bucket', 'time-to-ready histogram'),
('messenger_jobs_overdue', 'overdue jobs'),
('messenger_worker_busy', 'worker saturation'),
('messenger_db_pool_used', 'DB pool'),
('messenger_queue_oldest_job_age_seconds', 'oldest job age'),
('messenger_outbox_oldest_unpublished_age_seconds', 'outbox age'),
('messenger_process_cpu_seconds_total', 'process CPU'),
('rabbitmq_queue_messages_ready', 'RabbitMQ queue depth'),
('otelcol_exporter_queue_size', 'collector queue self-metric'),
]
try:
if not wait_for(lambda: prom(args.prometheus, 'sum(rate(messenger_job_time_to_ready_seconds_bucket{le="30.0"}[5m]))'), attempts=20, delay=3):
failures.append('TTR good-event selector le="30.0" is empty')
except Exception as exc:
failures.append('TTR selector: ' + str(exc))
for metric, label in required:
try:
found = wait_for(lambda m=metric: prom(args.prometheus, 'count(' + m + ')'), attempts=max(1, args.wait_seconds // 3), delay=3)
if not found:
failures.append(label + ': no samples (' + metric + ')')
except Exception as exc:
failures.append(label + ': ' + str(exc))
try:
up = prom(args.prometheus, 'up{job=~"otel-collector|haproxy|rabbitmq"} == 1')
jobs = {item.get('metric', {}).get('job') for item in up}
for job in ('otel-collector', 'haproxy', 'rabbitmq'):
if job not in jobs:
failures.append('Prometheus target is not up: ' + job)
except Exception as exc:
failures.append('Prometheus targets: ' + str(exc))
try:
series = prom_series(args.prometheus, 'http_server_request_duration_seconds_count')
routes = [item.get('http_route', '') for item in series]
if not series:
failures.append('HTTP metric has no series')
for route in routes:
if not route:
failures.append('HTTP metric has an empty http_route label')
if UUID_RE.search(route):
failures.append('HTTP route label contains a UUID: ' + route)
labels = {key for item in series for key in item}
forbidden = labels & FORBIDDEN_LABELS
if forbidden:
failures.append('HTTP metric has forbidden labels: ' + ', '.join(sorted(forbidden)))
methods = {item.get('http_request_method', '') for item in series}
if len(methods) > 7:
failures.append('HTTP method label cardinality is too high: ' + str(len(methods)))
for method in methods:
if method not in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS', 'OTHER'}:
failures.append('HTTP method is not normalized: ' + method)
except Exception as exc:
failures.append('HTTP label check: ' + str(exc))
# 3. Loki must contain the access log for the controlled request and must
# not contain the message text, session token or file bytes.
end = int(time.time() * 1e9)
start = end - 15 * 60 * int(1e9)
try:
found = wait_for(lambda: loki_query(args.loki, '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | request_id="' + request_id + '"', start, end), attempts=20, delay=2)
if not found:
failures.append('Loki: no log with the generated request_id (trace/log correlation)')
for needle, label in ((sentinel, 'message text'), (token, 'session token'), (filename, 'filename')):
if needle and loki_query(args.loki, '{service_name=~"messenger-api|messenger-worker|messenger-relay"} |= "' + needle + '"', start, end):
failures.append('Loki contains forbidden ' + label)
except Exception as exc:
failures.append('Loki query: ' + str(exc))
# 4. Tempo must contain real application traces, including the worker span.
# Tempo's search `name=` matches root spans only. job.process is a child of
# the controlled API trace, so resolve its trace_id from the worker log and
# fetch that trace directly.
try:
def tempo_search(tag):
q = urllib.parse.quote(tag)
_, raw = request(args.tempo.rstrip('/') + '/api/search?tags=' + q + '&limit=20')
return json.loads(raw).get('traces', [])
def worker_job_trace():
result = loki_query(
args.loki,
'{service_name="messenger-worker"} | event="attachment ready"',
start, end, limit=50,
)
for stream in result:
for _, line in stream.get('values', []):
try:
entry = json.loads(line)
except json.JSONDecodeError:
entry = {}
trace_id = entry.get('trace_id') or stream.get('stream', {}).get('trace_id', '')
if not re.fullmatch(r'[0-9a-f]{32}', trace_id):
continue
try:
_, raw = request(args.tempo.rstrip('/') + '/api/traces/' + trace_id)
except Exception:
continue
try:
payload = json.loads(raw)
except json.JSONDecodeError:
continue
spans = []
for batch in payload.get('batches', []):
for scope in batch.get('scopeSpans', []):
spans.extend(scope.get('spans', []))
publishes = [s for s in spans if s.get('name') == 'job.publish'
and s.get('kind') == 'SPAN_KIND_PRODUCER']
processes = [s for s in spans if s.get('name') == 'job.process'
and s.get('kind') == 'SPAN_KIND_CONSUMER']
if any(p.get('parentSpanId') == pub.get('spanId')
for p in processes for pub in publishes):
return trace_id
return None
if not wait_for(lambda: tempo_search('service.name=messenger-api'), attempts=15, delay=2):
failures.append('Tempo: no messenger-api traces')
retry_path = ROOT / 'evidence/lab6/trace-retry.json'
if retry_path.exists():
retry = json.loads(retry_path.read_text())
spans = []
for batch in retry.get('batches', []):
for scope in batch.get('scopeSpans', []):
spans.extend(scope.get('spans', []))
processes = [s for s in spans if s.get('name') == 'job.process']
attempts = set()
for span in processes:
for attr in span.get('attributes', []):
if attr.get('key') == 'job.attempt':
value = attr.get('value', {})
if 'intValue' in value:
attempts.add(int(value['intValue']))
outcomes = set()
for span in processes:
for attr in span.get('attributes', []):
if attr.get('key') == 'job.outcome':
value = attr.get('value', {}).get('stringValue')
if value:
outcomes.add(value)
if (len(attempts) < 2 or 'retried' not in outcomes
or not any(s.get('name') == 'job.requeue' for s in spans)):
failures.append('Tempo: retry trace lacks two attempts, retried outcome or job.requeue')
if not wait_for(worker_job_trace, attempts=20, delay=2):
failures.append('Tempo: no worker job.process trace linked to worker log')
except Exception as exc:
failures.append('Tempo query: ' + str(exc))
# 5. Grafana provisioning and rules.
try:
password = grafana_password()
if not password:
failures.append('Grafana operator secret is missing (secrets/lab7/backends/credentials.json)')
else:
_, raw = request(args.grafana.rstrip('/') + '/api/health')
if json.loads(raw).get('database') != 'ok':
failures.append('Grafana health is not ok')
_, raw = request(args.grafana.rstrip('/') + '/api/datasources', auth='admin:' + password)
names = {item.get('name') for item in json.loads(raw)}
for required_name in ('Prometheus', 'Loki', 'Tempo'):
if required_name not in names:
failures.append('Grafana datasource missing: ' + required_name)
for uid in ('lab6-overview', 'lab6-jobs', 'lab6-infra'):
_, raw = request(args.grafana.rstrip('/') + '/api/dashboards/uid/' + uid, auth='admin:' + password)
if json.loads(raw).get('dashboard', {}).get('uid') != uid:
failures.append('Grafana dashboard missing: ' + uid)
except Exception as exc:
failures.append('Grafana: ' + str(exc))
try:
_, raw = request(args.prometheus.rstrip('/') + '/api/v1/rules')
rule_names = {r.get('name') for g in json.loads(raw).get('data', {}).get('groups', []) for r in g.get('rules', [])}
for name in ('APIErrorBudgetFastBurn', 'APIErrorBudgetSlowBurn', 'QueueOldestJobAgeHigh', 'TTRErrorBudgetFastBurn'):
if name not in rule_names:
failures.append('Prometheus rule missing: ' + name)
except Exception as exc:
failures.append('Prometheus rules: ' + str(exc))
for note in notes:
print('NOTE:', note)
if failures:
for item in failures:
print('FAIL:', item)
return 1
print('PASS: lab6 strict telemetry gate (signals, trace/log correlation, labels, Grafana, rules, privacy sentinels)')
return 0
if __name__ == '__main__':
raise SystemExit(main())