forked from mtusi2026/mtusi2026lab6
298 lines
14 KiB
Python
298 lines
14 KiB
Python
"""Live lab6 checks. Fault modes restore every stopped service in finally."""
|
|
import argparse
|
|
import base64
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlencode
|
|
from urllib.request import Request, urlopen
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, check
|
|
|
|
PROM = 'http://localhost:19090'
|
|
LOKI = 'http://localhost:13100'
|
|
TEMPO = 'http://localhost:13200'
|
|
BASE = 'http://localhost:18083'
|
|
|
|
|
|
def get(url, headers=None):
|
|
with urlopen(Request(url, headers=headers or {}), timeout=8) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def prom(query):
|
|
return get(PROM + '/api/v1/query?' + urlencode({'query': query}))['data']['result']
|
|
|
|
|
|
def eventually(function, timeout=75):
|
|
deadline = time.monotonic() + timeout
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
value = function()
|
|
if value:
|
|
return value
|
|
except OSError as exc:
|
|
last = type(exc).__name__
|
|
time.sleep(2)
|
|
raise AssertionError('condition timeout: ' + str(last or function.__name__))
|
|
|
|
|
|
def compose(*args):
|
|
subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT,
|
|
check=True, timeout=100, stdout=subprocess.DEVNULL)
|
|
|
|
|
|
def alert_firing(name):
|
|
alerts = get(PROM + '/api/v1/alerts')['data']['alerts']
|
|
return any(a['labels']['alertname'] == name and a['state'] == 'firing' for a in alerts)
|
|
|
|
|
|
def trace_spans(trace_id):
|
|
data = get(TEMPO + '/api/traces/' + trace_id, {'Accept': 'application/json'})
|
|
spans = []
|
|
def visit(item):
|
|
if isinstance(item, dict):
|
|
if 'spanId' in item and 'name' in item:
|
|
spans.append(item)
|
|
for child in item.values():
|
|
visit(child)
|
|
elif isinstance(item, list):
|
|
for child in item:
|
|
visit(child)
|
|
visit(data)
|
|
return spans
|
|
|
|
|
|
def upload(client, actor, chat):
|
|
original = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
incoming = uuid.uuid4().hex
|
|
reply = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
|
|
202, actor=actor, raw=original,
|
|
headers={'Content-Type': 'image/png',
|
|
'traceparent': '00-' + incoming + '-1234567890abcdef-01',
|
|
'tracestate': 'lab=lab6'}, schema='Attachment')
|
|
check(reply['headers']['X-Trace-Id'] == incoming, 'upload context lost')
|
|
return reply['json'], reply['headers']['X-Trace-Id']
|
|
|
|
|
|
def linked_trace(trace_id, attempts=1):
|
|
def ready():
|
|
spans = trace_spans(trace_id)
|
|
names = [s['name'] for s in spans]
|
|
if (any(n.startswith('POST /api/v1/chats/') for n in names)
|
|
and names.count('attachment.publish') >= attempts
|
|
and names.count('attachment.consume') >= attempts
|
|
and 's3.put' in names and 'postgresql.transaction' in names):
|
|
return spans
|
|
spans = eventually(ready)
|
|
ids = {s['spanId'] for s in spans}
|
|
for item in spans:
|
|
if item['name'] in ('attachment.publish', 'attachment.consume'):
|
|
check(item.get('parentSpanId') in ids, 'producer/consumer parent missing')
|
|
check(item.get('traceState') == 'lab=lab6', 'AMQP tracestate lost')
|
|
return spans
|
|
|
|
|
|
def correlated_logs(trace_id):
|
|
query = '{service_name=~"messenger-.*"} |= "' + trace_id + '"'
|
|
def available():
|
|
response = get(LOKI + '/loki/api/v1/query_range?' + urlencode({
|
|
'query': query, 'since': '1h', 'limit': 1000}))
|
|
rows = response['data']['result']
|
|
names = {r['stream'].get('service_name') for r in rows}
|
|
return rows if {'messenger-api', 'messenger-worker', 'messenger-publisher'} <= names else None
|
|
return eventually(available)
|
|
|
|
|
|
def smoke(client, alice, bob, chat):
|
|
att, tid = upload(client, alice, chat)
|
|
client.wait_ready(bob, att)
|
|
spans = linked_trace(tid)
|
|
logs = correlated_logs(tid)
|
|
required = ['messenger_http_requests_total', 'messenger_http_duration_seconds_bucket',
|
|
'messenger_queue_depth', 'messenger_queue_oldest_age_seconds',
|
|
'messenger_db_pool_used', 'messenger_worker_busy',
|
|
'messenger_job_duration_seconds_bucket', 'messenger_attachment_time_to_ready_seconds_bucket',
|
|
'messenger_process_memory_bytes', 'messenger_process_cpu_seconds_total']
|
|
for name in required:
|
|
eventually(lambda: prom(name))
|
|
# Validate propagation from an incoming W3C header, including header case normalization.
|
|
incoming_id = '1234567890abcdef1234567890abcdef'
|
|
reply = client.call('GET', '/api/v1/users/me', actor=alice,
|
|
headers={'traceparent': '00-' + incoming_id + '-1234567890abcdef-01',
|
|
'tracestate': 'lab=propagation'})
|
|
check(reply['headers']['X-Trace-Id'] == incoming_id, 'incoming trace context lost')
|
|
# Synthetic message and token must never appear in persisted telemetry.
|
|
canary = 'PRIVATE_BODY_' + tid
|
|
client.message(alice, chat, canary)
|
|
time.sleep(5)
|
|
for secret in (canary, alice['token']):
|
|
query = '{service_name=~"messenger-.*"} |= "' + secret + '"'
|
|
found = get(LOKI + '/loki/api/v1/query_range?' + urlencode({
|
|
'query': query, 'since': '1h', 'limit': 10}))['data']['result']
|
|
check(not found, 'private content found in telemetry')
|
|
password = (ROOT / 'secrets/grafana_password').read_text().strip()
|
|
auth = 'Basic ' + base64.b64encode(('admin:' + password).encode()).decode()
|
|
dashboard = get('http://localhost:13000/api/dashboards/uid/messenger', {'Authorization': auth})
|
|
check(len(dashboard['dashboard']['panels']) >= 12, 'dashboard not provisioned')
|
|
for panel in dashboard['dashboard']['panels']:
|
|
for target in panel.get('targets', []):
|
|
if 'expr' in target:
|
|
prom(target['expr'])
|
|
search = urlencode({'q': '{resource.service.name =~ "messenger-.*"}',
|
|
'start': int(time.time() - 900), 'end': int(time.time()), 'limit': 20})
|
|
traces = get('http://localhost:13000/api/datasources/proxy/uid/tempo/api/search?' + search,
|
|
{'Authorization': auth})
|
|
check(traces.get('traces'), 'Tempo search through Grafana failed')
|
|
return {'trace_id': tid, 'span_names': [s['name'] for s in spans],
|
|
'log_services': sorted({r['stream']['service_name'] for r in logs}),
|
|
'metric_names_checked': required, 'dashboard_panels': len(dashboard['dashboard']['panels']),
|
|
'dashboard_queries_valid': True, 'incoming_w3c_context': True, 'body_and_token_absent_from_logs': True}
|
|
|
|
|
|
def worker_stop(client, alice, bob, chat):
|
|
compose('stop', 'worker')
|
|
try:
|
|
att, tid = upload(client, alice, chat)
|
|
started = time.monotonic()
|
|
eventually(lambda: alert_firing('QueueLagGrowing'))
|
|
observation = {'seconds_to_firing': round(time.monotonic() - started, 2),
|
|
'oldest_job_age': prom('messenger_queue_oldest_age_seconds'),
|
|
'backlog': prom('messenger_queue_depth'),
|
|
'alerts_firing': get(PROM + '/api/v1/alerts')['data']['alerts']}
|
|
finally:
|
|
compose('start', 'worker')
|
|
client.wait_ready(bob, att)
|
|
eventually(lambda: not alert_firing('QueueLagGrowing'))
|
|
spans = linked_trace(tid)
|
|
observation.update(resolved=True, trace_id=tid, span_count=len(spans),
|
|
alerts_after=get(PROM + '/api/v1/alerts')['data']['alerts'])
|
|
return observation
|
|
|
|
|
|
def db_outage(client, alice):
|
|
# A held table lock makes actual PostgreSQL operations time out. Readiness SELECT 1 stays healthy.
|
|
locker = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'postgres', 'psql',
|
|
'-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-c',
|
|
'BEGIN; LOCK TABLE users IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(35); COMMIT;'],
|
|
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
failures = []
|
|
try:
|
|
time.sleep(2)
|
|
def request(_):
|
|
reply = client.call('GET', '/api/v1/users/me', 503, actor=alice)
|
|
return reply['headers']['X-Trace-Id']
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
failures = list(executor.map(request, range(8)))
|
|
eventually(lambda: alert_firing('HighApiErrorRate'))
|
|
error_spans = eventually(lambda: [s for s in trace_spans(failures[0])
|
|
if s['name'] == 'postgresql.transaction' and
|
|
s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR')])
|
|
observation = {'firing': True, 'failed_requests': len(failures),
|
|
'trace_id': failures[0], 'db_error_spans': len(error_spans),
|
|
'alerts_firing': get(PROM + '/api/v1/alerts')['data']['alerts'],
|
|
'error_ratio': prom('sum(rate(messenger_http_requests_total{status_class="5xx"}[1m])) / sum(rate(messenger_http_requests_total[1m]))')}
|
|
finally:
|
|
locker.wait(timeout=50)
|
|
eventually(lambda: client.call('GET', '/api/v1/users/me', actor=alice, schema='User')['json'])
|
|
eventually(lambda: not alert_firing('HighApiErrorRate'), timeout=100)
|
|
observation['resolved'] = True
|
|
return observation
|
|
|
|
|
|
def retry(client, alice, bob, chat):
|
|
compose('stop', 'worker')
|
|
try:
|
|
att, tid = upload(client, alice, chat)
|
|
compose('stop', 's3')
|
|
# Compose start also starts dependencies in this version. Keep S3 stopped here.
|
|
subprocess.run(['docker', 'start', 'mtusi-lab6-worker-1'], check=True,
|
|
stdout=subprocess.DEVNULL)
|
|
def retried():
|
|
spans = trace_spans(tid)
|
|
return any(s['name'] == 'attachment.consume' and
|
|
s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR') for s in spans)
|
|
eventually(retried, timeout=45)
|
|
finally:
|
|
compose('start', 's3', 'worker')
|
|
client.wait_ready(bob, att)
|
|
spans = linked_trace(tid, attempts=2)
|
|
return {'trace_id': tid, 'consumer_attempts': sum(s['name'] == 'attachment.consume' for s in spans),
|
|
'span_names': [s['name'] for s in spans], 'ready_after_retry': True}
|
|
|
|
|
|
def backend_outage(client, alice, bob, chat):
|
|
before = prom('sum(otelcol_exporter_send_failed_spans_total)')
|
|
tracked = ['mtusi-lab6-' + name + '-1' for name in
|
|
('api-a', 'api-b', 'worker', 'outbox-publisher', 'otel-collector')]
|
|
before_containers = json.loads(subprocess.check_output(['docker', 'inspect', *tracked], text=True))
|
|
outage_start = str(int(time.time()))
|
|
compose('stop', 'tempo')
|
|
count = 0
|
|
samples = []
|
|
try:
|
|
started = time.monotonic()
|
|
while time.monotonic() - started < 40:
|
|
client.message(alice, chat, 'telemetry-outage')
|
|
count += 1
|
|
if count % 15 == 0:
|
|
raw = subprocess.check_output(['docker', 'stats', '--no-stream', '--format',
|
|
'{{json .}}', *tracked], cwd=ROOT, text=True)
|
|
samples.extend(json.loads(line) for line in raw.splitlines())
|
|
time.sleep(.15)
|
|
att, _ = upload(client, alice, chat)
|
|
client.wait_ready(bob, att)
|
|
failures = prom('sum(otelcol_exporter_send_failed_spans_total)')
|
|
before_count = float(before[0]['value'][1]) if before else 0
|
|
check(failures and float(failures[0]['value'][1]) > before_count, 'no new exporter failures')
|
|
finally:
|
|
compose('start', 'tempo')
|
|
collector_logs = subprocess.check_output(['docker', 'compose', 'logs', '--since', outage_start,
|
|
'otel-collector'], cwd=ROOT, text=True)
|
|
dropped = sum(int(value) for value in re.findall(r'"dropped_items"\s*:\s*(\d+)', collector_logs))
|
|
after_containers = json.loads(subprocess.check_output(['docker', 'inspect', *tracked], text=True))
|
|
for before_item, after_item in zip(before_containers, after_containers):
|
|
check(after_item['State']['Running'] and not after_item['State']['OOMKilled'], 'container died')
|
|
check(before_item['RestartCount'] == after_item['RestartCount'], 'container restarted')
|
|
restored_att, restored_trace = upload(client, alice, chat)
|
|
client.wait_ready(bob, restored_att)
|
|
linked_trace(restored_trace)
|
|
correlated_logs(restored_trace)
|
|
return {'successful_messages': count, 'attachment_ready_during_outage': True,
|
|
'collector_reported_dropped_spans': dropped, 'export_failures_before': before, 'export_failures_after': failures,
|
|
'application_and_collector_stats': samples, 'no_oom_or_restarts': True, 'telemetry_restored_trace': restored_trace}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('mode', choices=['smoke', 'worker-stop', 'db-outage', 'retry', 'backend-outage'])
|
|
args = parser.parse_args()
|
|
client = Client(BASE, None, timeout=20)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
if args.mode == 'smoke':
|
|
result = smoke(client, alice, bob, chat)
|
|
elif args.mode == 'worker-stop':
|
|
result = worker_stop(client, alice, bob, chat)
|
|
elif args.mode == 'db-outage':
|
|
result = db_outage(client, alice)
|
|
elif args.mode == 'retry':
|
|
result = retry(client, alice, bob, chat)
|
|
else:
|
|
result = backend_outage(client, alice, bob, chat)
|
|
path = ROOT / 'evidence' / ('observability-' + args.mode + '.json')
|
|
path.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
|
|
print('PASS', args.mode, path.name)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|