#!/usr/bin/env python3 """Repeatable worker/DB/backend faults on this lab's isolated Compose project.""" import argparse import datetime import json import subprocess import sys import time import urllib.parse import uuid from concurrent.futures import ThreadPoolExecutor from observability_common import ROOT, compose, get_json, prom, ready, save, start_services sys.path.insert(0, str(ROOT / 'tests')) from client import Client def alerts(): return get_json('http://127.0.0.1:19090/api/v1/alerts')['data']['alerts'] def collect_trace(trace_id): for _ in range(20): try: trace = get_json('http://127.0.0.1:13200/api/traces/' + trace_id) if trace.get('batches') or trace.get('resourceSpans'): save('evidence/trace-' + trace_id + '.json', trace) query = '{service_name=~"messenger-api|messenger-worker"} | trace_id = "' + trace_id + '"' logs = get_json('http://127.0.0.1:13100/loki/api/v1/query_range?' + urllib.parse.urlencode({'query': query, 'since': '1h', 'limit': 100})) if not logs['data']['result']: raise RuntimeError('correlated logs not yet available') save('evidence/logs-' + trace_id + '.json', logs) return {'trace_id': trace_id, 'trace_file': 'evidence/trace-' + trace_id + '.json', 'log_streams': len(logs['data']['result']), 'logs_file': 'evidence/logs-' + trace_id + '.json'} except Exception: pass time.sleep(2) raise RuntimeError('trace/log correlation not found in backend') def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--base-url', default='http://127.0.0.1:18086') args = p.parse_args() ready(args.base_url) c = Client(args.base_url) actor = c.user('incidents') chat = c.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': 'Synthetic incidents', 'member_ids': [actor['id']]}, schema='Chat')['json'] report = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'cases': {}} traces = [] try: print('Worker outage', flush=True) compose('stop', 'worker') attachments = [] started = time.monotonic() for _ in range(6): trace_id = uuid.uuid4().hex result = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=synthetic.png', 202, actor=actor, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), headers={'Content-Type': 'image/png', 'traceparent': '00-' + trace_id + '-1234567890123456-01'}, schema='Attachment') attachments.append(result['json']) traces.append(trace_id) timeline = [] for _ in range(6): time.sleep(5) timeline.append({'seconds': round(time.monotonic()-started, 2), 'alerts': alerts(), 'queue': prom('messenger_queue_depth'), 'age': prom('messenger_queue_oldest_age_seconds')}) # Keep S3 unavailable for the first attempt; the same durable trace must # cover the failed attempt, retry publication and eventual ready. compose('stop', 'minio') start_services('worker') time.sleep(10) report['cases']['s3_retry'] = {'alerts_during': alerts(), 'jobs': prom('messenger_jobs_total')} start_services('minio') ready(args.base_url) for att in attachments: c.wait_ready(actor, att) time.sleep(35) report['cases']['worker_stop'] = {'queued': len(attachments), 'ready_after_recovery': len(attachments), 'timeline': timeline, 'resolved': alerts(), 'queue_after': prom('messenger_queue_depth')} save('evidence/lab6-faults.json', report) print('DB lock', flush=True) lock = subprocess.Popen([str(ROOT / 'scripts/compose.sh'), 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-c', 'BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(45); COMMIT;'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) time.sleep(2) observations = [] def write(index): trace_id = uuid.uuid4().hex result = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', [201, 500, 503], actor=actor, body={'text': 'synthetic incident'}, headers={'Idempotency-Key': uuid.uuid4().hex, 'traceparent': '00-' + trace_id + '-1234567890123456-01'}) return {'status': result['status'], 'elapsed': result['elapsed'], 'trace_id': trace_id} with ThreadPoolExecutor(max_workers=2) as pool: for block in range(7): observations.extend(pool.map(write, range(4))) report['cases']['db_lock'] = {'requests': observations, 'alerts_during': alerts()} save('evidence/lab6-faults.json', report) lock.wait(timeout=60) c.message(actor, chat, 'synthetic recovery') recovery_timeline = [] for _ in range(24): time.sleep(5) state = alerts() recovery_timeline.append(state) if not any(a['state'] == 'firing' for a in state): break c.message(actor, chat, 'synthetic recovery probe') report['cases']['db_lock']['recovery_alerts'] = recovery_timeline report['cases']['db_lock']['alerts_after'] = alerts() failed = next((x for x in observations if x['status'] >= 500), None) if failed: traces.append(failed['trace_id']) report['correlation'] = [collect_trace(t) for t in (traces[0], traces[-1])] print('Telemetry backend outage', flush=True) compose('stop', 'tempo') successful = 0 for _ in range(30): c.message(actor, chat, 'synthetic telemetry outage') successful += 1 time.sleep(.5) report['cases']['tempo_stop'] = {'successful_writes': successful, 'exporter_failed': prom('otelcol_exporter_send_failed_spans_total'), 'queue_size': prom('otelcol_exporter_queue_size'), 'collector_memory': prom('otelcol_process_memory_rss_bytes')} start_services('tempo') time.sleep(20) report['cases']['tempo_stop']['after'] = {'queue_size': prom('otelcol_exporter_queue_size'), 'collector_memory': prom('otelcol_process_memory_rss_bytes')} assert any(a['labels']['alertname'] == 'QueueLagGrowing' and a['state'] == 'firing' for row in report['cases']['worker_stop']['timeline'] for a in row['alerts']), 'queue alert never fired' assert any(a['labels']['alertname'] == 'APIHighErrorRate' and a['state'] == 'firing' for a in report['cases']['db_lock']['alerts_during']), 'API alert never fired' assert not any(a['state'] == 'firing' for a in report['cases']['db_lock']['alerts_after']), 'alerts did not resolve' assert all(float(x['value'][1]) == 0 for x in report['cases']['worker_stop']['queue_after']), 'backlog not empty' report['status'] = 'PASS' finally: start_services('worker', 'tempo', 'minio') report['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat() save('evidence/lab6-faults.json', report) print(json.dumps({'status': report['status'], 'cases': list(report['cases'])}), flush=True) if __name__ == '__main__': main()