forked from mtusi2026/mtusi2026lab6
315 lines
15 KiB
Python
Executable File
315 lines
15 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Strict lab6 observability gate.
|
|
|
|
It creates a fresh message and attachment, waits for all three signals, checks
|
|
route/template cardinality, Grafana provisioning and rules, and verifies that
|
|
user content and credentials never reached Loki. A missing signal is a failure,
|
|
not a warning.
|
|
"""
|
|
import argparse
|
|
import os
|
|
import base64
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
# The lab tools always target loopback; ignore ambient corporate proxies.
|
|
for _proxy_key in ('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'):
|
|
os.environ.pop(_proxy_key, None)
|
|
os.environ['NO_PROXY'] = 'localhost,127.0.0.1,::1'
|
|
os.environ['no_proxy'] = 'localhost,127.0.0.1,::1'
|
|
urllib.request.getproxies = lambda: {}
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
UUID_RE = re.compile(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', re.I)
|
|
FORBIDDEN_LABELS = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'filename', 'request_id', 'object_key', 'path', 'query'}
|
|
|
|
|
|
def request(url, auth=None, timeout=10):
|
|
headers = {'Accept': 'application/json'}
|
|
if auth:
|
|
headers['Authorization'] = 'Basic ' + base64.b64encode(auth.encode()).decode()
|
|
req = urllib.request.Request(url, headers=headers)
|
|
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
return response.status, response.read()
|
|
|
|
|
|
def prom(base, expr):
|
|
url = base.rstrip('/') + '/api/v1/query?' + urllib.parse.urlencode({'query': expr})
|
|
_, raw = request(url)
|
|
return json.loads(raw).get('data', {}).get('result', [])
|
|
|
|
|
|
def prom_series(base, match):
|
|
url = base.rstrip('/') + '/api/v1/series?' + urllib.parse.urlencode({'match[]': match})
|
|
_, raw = request(url)
|
|
return json.loads(raw).get('data', [])
|
|
|
|
|
|
def loki_query(base, query, start_ns, end_ns, limit=20):
|
|
params = urllib.parse.urlencode({'query': query, 'start': start_ns, 'end': end_ns, 'limit': limit})
|
|
_, raw = request(base.rstrip('/') + '/loki/api/v1/query_range?' + params)
|
|
return json.loads(raw).get('data', {}).get('result', [])
|
|
|
|
|
|
def grafana_password():
|
|
path = ROOT / 'secrets/grafana.env'
|
|
if not path.exists():
|
|
return None
|
|
for line in path.read_text().splitlines():
|
|
if '=' in line and line.split('=', 1)[0].strip() == 'GF_SECURITY_ADMIN_PASSWORD':
|
|
return line.split('=', 1)[1].strip()
|
|
return None
|
|
|
|
|
|
def wait_for(fn, attempts=30, delay=2):
|
|
last = None
|
|
for _ in range(attempts):
|
|
try:
|
|
value = fn()
|
|
if value:
|
|
return value
|
|
except Exception as exc: # noqa: BLE001 - retry the external endpoint
|
|
last = exc
|
|
time.sleep(delay)
|
|
if last:
|
|
raise last
|
|
return None
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Strict lab6 telemetry gate')
|
|
parser.add_argument('--base-url', default='http://localhost:8080')
|
|
parser.add_argument('--prometheus', default='http://localhost:9090')
|
|
parser.add_argument('--loki', default='http://localhost:3100')
|
|
parser.add_argument('--tempo', default='http://localhost:3200')
|
|
parser.add_argument('--grafana', default='http://localhost:3000')
|
|
parser.add_argument('--wait-seconds', type=int, default=60)
|
|
args = parser.parse_args()
|
|
failures = []
|
|
notes = []
|
|
|
|
# 1. Generate a controlled request, a message and a job.
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client
|
|
client = Client(args.base_url, timeout=15)
|
|
alice, _, _, chat = client.bootstrap()
|
|
request_id = 'lab6-verify-' + uuid.uuid4().hex
|
|
sentinel = 'SENTINEL-TELEMETRY-' + uuid.uuid4().hex
|
|
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, actor=alice,
|
|
body={'text': sentinel}, headers={'Idempotency-Key': uuid.uuid4().hex, 'X-Request-ID': request_id})
|
|
att, content = client.attachment(alice, chat)
|
|
client.wait_ready(alice, att, wait_seconds=60)
|
|
filename = 'sentinel-' + uuid.uuid4().hex + '.png'
|
|
from urllib.parse import quote
|
|
client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + quote(filename), 202,
|
|
actor=alice, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
token = alice.get('token', '')
|
|
notes.append('created request_id=' + request_id)
|
|
|
|
# 2. Required metric series. Counter/histogram series can take one export
|
|
# and one scrape interval to appear.
|
|
required = [
|
|
('http_server_request_duration_seconds_count', 'HTTP request histogram count'),
|
|
('http_server_request_duration_seconds_bucket', 'HTTP duration histogram'),
|
|
('messenger_jobs_processed_total', 'job attempt counter'),
|
|
('messenger_jobs_accepted_total', 'accepted jobs'),
|
|
('messenger_jobs_ready_within_budget_total', 'ready-within-budget'),
|
|
('messenger_job_duration_seconds_bucket', 'job duration histogram'),
|
|
('messenger_job_time_to_ready_seconds_bucket', 'time-to-ready histogram'),
|
|
('messenger_jobs_overdue', 'overdue jobs'),
|
|
('messenger_worker_busy', 'worker saturation'),
|
|
('messenger_db_pool_used', 'DB pool'),
|
|
('messenger_queue_oldest_job_age_seconds', 'oldest job age'),
|
|
('messenger_outbox_oldest_unpublished_age_seconds', 'outbox age'),
|
|
('messenger_process_cpu_seconds_total', 'process CPU'),
|
|
('rabbitmq_queue_messages_ready', 'RabbitMQ queue depth'),
|
|
('otelcol_exporter_queue_size', 'collector queue self-metric'),
|
|
]
|
|
try:
|
|
if not wait_for(lambda: prom(args.prometheus, 'sum(rate(messenger_job_time_to_ready_seconds_bucket{le="30.0"}[5m]))'), attempts=20, delay=3):
|
|
failures.append('TTR good-event selector le="30.0" is empty')
|
|
except Exception as exc:
|
|
failures.append('TTR selector: ' + str(exc))
|
|
|
|
for metric, label in required:
|
|
try:
|
|
found = wait_for(lambda m=metric: prom(args.prometheus, 'count(' + m + ')'), attempts=max(1, args.wait_seconds // 3), delay=3)
|
|
if not found:
|
|
failures.append(label + ': no samples (' + metric + ')')
|
|
except Exception as exc:
|
|
failures.append(label + ': ' + str(exc))
|
|
|
|
try:
|
|
up = prom(args.prometheus, 'up{job=~"otel-collector|haproxy|rabbitmq"} == 1')
|
|
jobs = {item.get('metric', {}).get('job') for item in up}
|
|
for job in ('otel-collector', 'haproxy', 'rabbitmq'):
|
|
if job not in jobs:
|
|
failures.append('Prometheus target is not up: ' + job)
|
|
except Exception as exc:
|
|
failures.append('Prometheus targets: ' + str(exc))
|
|
|
|
try:
|
|
series = prom_series(args.prometheus, 'http_server_request_duration_seconds_count')
|
|
routes = [item.get('http_route', '') for item in series]
|
|
if not series:
|
|
failures.append('HTTP metric has no series')
|
|
for route in routes:
|
|
if not route:
|
|
failures.append('HTTP metric has an empty http_route label')
|
|
if UUID_RE.search(route):
|
|
failures.append('HTTP route label contains a UUID: ' + route)
|
|
labels = {key for item in series for key in item}
|
|
forbidden = labels & FORBIDDEN_LABELS
|
|
if forbidden:
|
|
failures.append('HTTP metric has forbidden labels: ' + ', '.join(sorted(forbidden)))
|
|
methods = {item.get('http_request_method', '') for item in series}
|
|
if len(methods) > 7:
|
|
failures.append('HTTP method label cardinality is too high: ' + str(len(methods)))
|
|
for method in methods:
|
|
if method not in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS', 'OTHER'}:
|
|
failures.append('HTTP method is not normalized: ' + method)
|
|
except Exception as exc:
|
|
failures.append('HTTP label check: ' + str(exc))
|
|
|
|
# 3. Loki must contain the access log for the controlled request and must
|
|
# not contain the message text, session token or file bytes.
|
|
end = int(time.time() * 1e9)
|
|
start = end - 15 * 60 * int(1e9)
|
|
try:
|
|
found = wait_for(lambda: loki_query(args.loki, '{service_name=~"messenger-api|messenger-worker|messenger-relay"} | request_id="' + request_id + '"', start, end), attempts=20, delay=2)
|
|
if not found:
|
|
failures.append('Loki: no log with the generated request_id (trace/log correlation)')
|
|
for needle, label in ((sentinel, 'message text'), (token, 'session token'), (filename, 'filename')):
|
|
if needle and loki_query(args.loki, '{service_name=~"messenger-api|messenger-worker|messenger-relay"} |= "' + needle + '"', start, end):
|
|
failures.append('Loki contains forbidden ' + label)
|
|
except Exception as exc:
|
|
failures.append('Loki query: ' + str(exc))
|
|
|
|
# 4. Tempo must contain real application traces, including the worker span.
|
|
# Tempo's search `name=` matches root spans only. job.process is a child of
|
|
# the controlled API trace, so resolve its trace_id from the worker log and
|
|
# fetch that trace directly.
|
|
try:
|
|
def tempo_search(tag):
|
|
q = urllib.parse.quote(tag)
|
|
_, raw = request(args.tempo.rstrip('/') + '/api/search?tags=' + q + '&limit=20')
|
|
return json.loads(raw).get('traces', [])
|
|
|
|
def worker_job_trace():
|
|
result = loki_query(
|
|
args.loki,
|
|
'{service_name="messenger-worker"} | event="attachment ready"',
|
|
start, end, limit=50,
|
|
)
|
|
for stream in result:
|
|
for _, line in stream.get('values', []):
|
|
try:
|
|
entry = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
entry = {}
|
|
trace_id = entry.get('trace_id') or stream.get('stream', {}).get('trace_id', '')
|
|
if not re.fullmatch(r'[0-9a-f]{32}', trace_id):
|
|
continue
|
|
try:
|
|
_, raw = request(args.tempo.rstrip('/') + '/api/traces/' + trace_id)
|
|
except Exception:
|
|
continue
|
|
try:
|
|
payload = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
spans = []
|
|
for batch in payload.get('batches', []):
|
|
for scope in batch.get('scopeSpans', []):
|
|
spans.extend(scope.get('spans', []))
|
|
publishes = [s for s in spans if s.get('name') == 'job.publish'
|
|
and s.get('kind') == 'SPAN_KIND_PRODUCER']
|
|
processes = [s for s in spans if s.get('name') == 'job.process'
|
|
and s.get('kind') == 'SPAN_KIND_CONSUMER']
|
|
if any(p.get('parentSpanId') == pub.get('spanId')
|
|
for p in processes for pub in publishes):
|
|
return trace_id
|
|
return None
|
|
|
|
if not wait_for(lambda: tempo_search('service.name=messenger-api'), attempts=15, delay=2):
|
|
failures.append('Tempo: no messenger-api traces')
|
|
retry_path = ROOT / 'evidence/lab6/trace-retry.json'
|
|
if retry_path.exists():
|
|
retry = json.loads(retry_path.read_text())
|
|
spans = []
|
|
for batch in retry.get('batches', []):
|
|
for scope in batch.get('scopeSpans', []):
|
|
spans.extend(scope.get('spans', []))
|
|
processes = [s for s in spans if s.get('name') == 'job.process']
|
|
attempts = set()
|
|
for span in processes:
|
|
for attr in span.get('attributes', []):
|
|
if attr.get('key') == 'job.attempt':
|
|
value = attr.get('value', {})
|
|
if 'intValue' in value:
|
|
attempts.add(int(value['intValue']))
|
|
outcomes = set()
|
|
for span in processes:
|
|
for attr in span.get('attributes', []):
|
|
if attr.get('key') == 'job.outcome':
|
|
value = attr.get('value', {}).get('stringValue')
|
|
if value:
|
|
outcomes.add(value)
|
|
if (len(attempts) < 2 or 'retried' not in outcomes
|
|
or not any(s.get('name') == 'job.requeue' for s in spans)):
|
|
failures.append('Tempo: retry trace lacks two attempts, retried outcome or job.requeue')
|
|
if not wait_for(worker_job_trace, attempts=20, delay=2):
|
|
failures.append('Tempo: no worker job.process trace linked to worker log')
|
|
except Exception as exc:
|
|
failures.append('Tempo query: ' + str(exc))
|
|
|
|
# 5. Grafana provisioning and rules.
|
|
try:
|
|
password = grafana_password()
|
|
if not password:
|
|
failures.append('Grafana admin secret is missing (secrets/grafana.env)')
|
|
else:
|
|
_, raw = request(args.grafana.rstrip('/') + '/api/health')
|
|
if json.loads(raw).get('database') != 'ok':
|
|
failures.append('Grafana health is not ok')
|
|
_, raw = request(args.grafana.rstrip('/') + '/api/datasources', auth='admin:' + password)
|
|
names = {item.get('name') for item in json.loads(raw)}
|
|
for required_name in ('Prometheus', 'Loki', 'Tempo'):
|
|
if required_name not in names:
|
|
failures.append('Grafana datasource missing: ' + required_name)
|
|
for uid in ('lab6-overview', 'lab6-jobs', 'lab6-infra'):
|
|
_, raw = request(args.grafana.rstrip('/') + '/api/dashboards/uid/' + uid, auth='admin:' + password)
|
|
if json.loads(raw).get('dashboard', {}).get('uid') != uid:
|
|
failures.append('Grafana dashboard missing: ' + uid)
|
|
except Exception as exc:
|
|
failures.append('Grafana: ' + str(exc))
|
|
|
|
try:
|
|
_, raw = request(args.prometheus.rstrip('/') + '/api/v1/rules')
|
|
rule_names = {r.get('name') for g in json.loads(raw).get('data', {}).get('groups', []) for r in g.get('rules', [])}
|
|
for name in ('APIErrorBudgetFastBurn', 'APIErrorBudgetSlowBurn', 'QueueOldestJobAgeHigh', 'TTRErrorBudgetFastBurn'):
|
|
if name not in rule_names:
|
|
failures.append('Prometheus rule missing: ' + name)
|
|
except Exception as exc:
|
|
failures.append('Prometheus rules: ' + str(exc))
|
|
|
|
for note in notes:
|
|
print('NOTE:', note)
|
|
if failures:
|
|
for item in failures:
|
|
print('FAIL:', item)
|
|
return 1
|
|
print('PASS: lab6 strict telemetry gate (signals, trace/log correlation, labels, Grafana, rules, privacy sentinels)')
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|