Files
mtusi2026lab6-fork/student-tests/observability.py
T

298 lines
14 KiB
Python

"""Live lab6 checks. Fault modes restore every stopped service in finally."""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
import json
import re
from pathlib import Path
import subprocess
import sys
import time
import uuid
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
PROM = 'http://localhost:19090'
LOKI = 'http://localhost:13100'
TEMPO = 'http://localhost:13200'
BASE = 'http://localhost:18083'
def get(url, headers=None):
with urlopen(Request(url, headers=headers or {}), timeout=8) as response:
return json.load(response)
def prom(query):
return get(PROM + '/api/v1/query?' + urlencode({'query': query}))['data']['result']
def eventually(function, timeout=75):
deadline = time.monotonic() + timeout
last = None
while time.monotonic() < deadline:
try:
value = function()
if value:
return value
except OSError as exc:
last = type(exc).__name__
time.sleep(2)
raise AssertionError('condition timeout: ' + str(last or function.__name__))
def compose(*args):
subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
check=True, timeout=100, stdout=subprocess.DEVNULL)
def alert_firing(name):
alerts = get(PROM + '/api/v1/alerts')['data']['alerts']
return any(a['labels']['alertname'] == name and a['state'] == 'firing' for a in alerts)
def trace_spans(trace_id):
data = get(TEMPO + '/api/traces/' + trace_id, {'Accept': 'application/json'})
spans = []
def visit(item):
if isinstance(item, dict):
if 'spanId' in item and 'name' in item:
spans.append(item)
for child in item.values():
visit(child)
elif isinstance(item, list):
for child in item:
visit(child)
visit(data)
return spans
def upload(client, actor, chat):
original = (ROOT / 'tests/fixtures/sample.png').read_bytes()
incoming = uuid.uuid4().hex
reply = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
202, actor=actor, raw=original,
headers={'Content-Type': 'image/png',
'traceparent': '00-' + incoming + '-1234567890abcdef-01',
'tracestate': 'lab=lab6'}, schema='Attachment')
check(reply['headers']['X-Trace-Id'] == incoming, 'upload context lost')
return reply['json'], reply['headers']['X-Trace-Id']
def linked_trace(trace_id, attempts=1):
def ready():
spans = trace_spans(trace_id)
names = [s['name'] for s in spans]
if (any(n.startswith('POST /api/v1/chats/') for n in names)
and names.count('attachment.publish') >= attempts
and names.count('attachment.consume') >= attempts
and 's3.put' in names and 'postgresql.transaction' in names):
return spans
spans = eventually(ready)
ids = {s['spanId'] for s in spans}
for item in spans:
if item['name'] in ('attachment.publish', 'attachment.consume'):
check(item.get('parentSpanId') in ids, 'producer/consumer parent missing')
check(item.get('traceState') == 'lab=lab6', 'AMQP tracestate lost')
return spans
def correlated_logs(trace_id):
query = '{service_name=~"messenger-.*"} |= "' + trace_id + '"'
def available():
response = get(LOKI + '/loki/api/v1/query_range?' + urlencode({
'query': query, 'since': '1h', 'limit': 1000}))
rows = response['data']['result']
names = {r['stream'].get('service_name') for r in rows}
return rows if {'messenger-api', 'messenger-worker', 'messenger-publisher'} <= names else None
return eventually(available)
def smoke(client, alice, bob, chat):
att, tid = upload(client, alice, chat)
client.wait_ready(bob, att)
spans = linked_trace(tid)
logs = correlated_logs(tid)
required = ['messenger_http_requests_total', 'messenger_http_duration_seconds_bucket',
'messenger_queue_depth', 'messenger_queue_oldest_age_seconds',
'messenger_db_pool_used', 'messenger_worker_busy',
'messenger_job_duration_seconds_bucket', 'messenger_attachment_time_to_ready_seconds_bucket',
'messenger_process_memory_bytes', 'messenger_process_cpu_seconds_total']
for name in required:
eventually(lambda: prom(name))
# Validate propagation from an incoming W3C header, including header case normalization.
incoming_id = '1234567890abcdef1234567890abcdef'
reply = client.call('GET', '/api/v1/users/me', actor=alice,
headers={'traceparent': '00-' + incoming_id + '-1234567890abcdef-01',
'tracestate': 'lab=propagation'})
check(reply['headers']['X-Trace-Id'] == incoming_id, 'incoming trace context lost')
# Synthetic message and token must never appear in persisted telemetry.
canary = 'PRIVATE_BODY_' + tid
client.message(alice, chat, canary)
time.sleep(5)
for secret in (canary, alice['token']):
query = '{service_name=~"messenger-.*"} |= "' + secret + '"'
found = get(LOKI + '/loki/api/v1/query_range?' + urlencode({
'query': query, 'since': '1h', 'limit': 10}))['data']['result']
check(not found, 'private content found in telemetry')
password = (ROOT / 'secrets/grafana_password').read_text().strip()
auth = 'Basic ' + base64.b64encode(('admin:' + password).encode()).decode()
dashboard = get('http://localhost:13000/api/dashboards/uid/messenger', {'Authorization': auth})
check(len(dashboard['dashboard']['panels']) >= 12, 'dashboard not provisioned')
for panel in dashboard['dashboard']['panels']:
for target in panel.get('targets', []):
if 'expr' in target:
prom(target['expr'])
search = urlencode({'q': '{resource.service.name =~ "messenger-.*"}',
'start': int(time.time() - 900), 'end': int(time.time()), 'limit': 20})
traces = get('http://localhost:13000/api/datasources/proxy/uid/tempo/api/search?' + search,
{'Authorization': auth})
check(traces.get('traces'), 'Tempo search through Grafana failed')
return {'trace_id': tid, 'span_names': [s['name'] for s in spans],
'log_services': sorted({r['stream']['service_name'] for r in logs}),
'metric_names_checked': required, 'dashboard_panels': len(dashboard['dashboard']['panels']),
'dashboard_queries_valid': True, 'incoming_w3c_context': True, 'body_and_token_absent_from_logs': True}
def worker_stop(client, alice, bob, chat):
compose('stop', 'worker')
try:
att, tid = upload(client, alice, chat)
started = time.monotonic()
eventually(lambda: alert_firing('QueueLagGrowing'))
observation = {'seconds_to_firing': round(time.monotonic() - started, 2),
'oldest_job_age': prom('messenger_queue_oldest_age_seconds'),
'backlog': prom('messenger_queue_depth'),
'alerts_firing': get(PROM + '/api/v1/alerts')['data']['alerts']}
finally:
compose('start', 'worker')
client.wait_ready(bob, att)
eventually(lambda: not alert_firing('QueueLagGrowing'))
spans = linked_trace(tid)
observation.update(resolved=True, trace_id=tid, span_count=len(spans),
alerts_after=get(PROM + '/api/v1/alerts')['data']['alerts'])
return observation
def db_outage(client, alice):
# A held table lock makes actual PostgreSQL operations time out. Readiness SELECT 1 stays healthy.
locker = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'postgres', 'psql',
'-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-c',
'BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(35); COMMIT;'],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
failures = []
try:
time.sleep(2)
def request(_):
reply = client.call('GET', '/api/v1/users/me', 503, actor=alice)
return reply['headers']['X-Trace-Id']
with ThreadPoolExecutor(max_workers=2) as executor:
failures = list(executor.map(request, range(8)))
eventually(lambda: alert_firing('HighApiErrorRate'))
error_spans = eventually(lambda: [s for s in trace_spans(failures[0])
if s['name'] == 'postgresql.transaction' and
s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR')])
observation = {'firing': True, 'failed_requests': len(failures),
'trace_id': failures[0], 'db_error_spans': len(error_spans),
'alerts_firing': get(PROM + '/api/v1/alerts')['data']['alerts'],
'error_ratio': prom('sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / sum(rate(messenger_http_requests_total[1m]))')}
finally:
locker.wait(timeout=50)
eventually(lambda: client.call('GET', '/api/v1/users/me', actor=alice, schema='User')['json'])
eventually(lambda: not alert_firing('HighApiErrorRate'), timeout=100)
observation['resolved'] = True
return observation
def retry(client, alice, bob, chat):
compose('stop', 'worker')
try:
att, tid = upload(client, alice, chat)
compose('stop', 's3')
# Compose start also starts dependencies in this version. Keep S3 stopped here.
subprocess.run(['docker', 'start', 'mtusi-lab6-worker-1'], check=True,
stdout=subprocess.DEVNULL)
def retried():
spans = trace_spans(tid)
return any(s['name'] == 'attachment.consume' and
s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR') for s in spans)
eventually(retried, timeout=45)
finally:
compose('start', 's3', 'worker')
client.wait_ready(bob, att)
spans = linked_trace(tid, attempts=2)
return {'trace_id': tid, 'consumer_attempts': sum(s['name'] == 'attachment.consume' for s in spans),
'span_names': [s['name'] for s in spans], 'ready_after_retry': True}
def backend_outage(client, alice, bob, chat):
before = prom('sum(otelcol_exporter_send_failed_spans_total)')
tracked = ['mtusi-lab6-' + name + '-1' for name in
('api-a', 'api-b', 'worker', 'outbox-publisher', 'otel-collector')]
before_containers = json.loads(subprocess.check_output(['docker', 'inspect', *tracked], text=True))
outage_start = str(int(time.time()))
compose('stop', 'tempo')
count = 0
samples = []
try:
started = time.monotonic()
while time.monotonic() - started < 40:
client.message(alice, chat, 'telemetry-outage')
count += 1
if count % 15 == 0:
raw = subprocess.check_output(['docker', 'stats', '--no-stream', '--format',
'{{json .}}', *tracked], cwd=ROOT, text=True)
samples.extend(json.loads(line) for line in raw.splitlines())
time.sleep(.15)
att, _ = upload(client, alice, chat)
client.wait_ready(bob, att)
failures = prom('sum(otelcol_exporter_send_failed_spans_total)')
before_count = float(before[0]['value'][1]) if before else 0
check(failures and float(failures[0]['value'][1]) > before_count, 'no new exporter failures')
finally:
compose('start', 'tempo')
collector_logs = subprocess.check_output(['docker', 'compose', 'logs', '--since', outage_start,
'otel-collector'], cwd=ROOT, text=True)
dropped = sum(int(value) for value in re.findall(r'"dropped_items"\s*:\s*(\d+)', collector_logs))
after_containers = json.loads(subprocess.check_output(['docker', 'inspect', *tracked], text=True))
for before_item, after_item in zip(before_containers, after_containers):
check(after_item['State']['Running'] and not after_item['State']['OOMKilled'], 'container died')
check(before_item['RestartCount'] == after_item['RestartCount'], 'container restarted')
restored_att, restored_trace = upload(client, alice, chat)
client.wait_ready(bob, restored_att)
linked_trace(restored_trace)
correlated_logs(restored_trace)
return {'successful_messages': count, 'attachment_ready_during_outage': True,
'collector_reported_dropped_spans': dropped, 'export_failures_before': before, 'export_failures_after': failures,
'application_and_collector_stats': samples, 'no_oom_or_restarts': True, 'telemetry_restored_trace': restored_trace}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('mode', choices=['smoke', 'worker-stop', 'db-outage', 'retry', 'backend-outage'])
args = parser.parse_args()
client = Client(BASE, None, timeout=20)
alice, bob, _, chat = client.bootstrap()
if args.mode == 'smoke':
result = smoke(client, alice, bob, chat)
elif args.mode == 'worker-stop':
result = worker_stop(client, alice, bob, chat)
elif args.mode == 'db-outage':
result = db_outage(client, alice)
elif args.mode == 'retry':
result = retry(client, alice, bob, chat)
else:
result = backend_outage(client, alice, bob, chat)
path = ROOT / 'evidence' / ('observability-' + args.mode + '.json')
path.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
print('PASS', args.mode, path.name)
if __name__ == '__main__':
main()