Files
mtusi2026lab6/scripts/observability_faults.py
2026-11-18 21:37:25 +03:00

150 lines
7.6 KiB
Python

#!/usr/bin/env python3
"""Repeatable worker/DB/backend faults on this lab's isolated Compose project."""
import argparse
import datetime
import json
import subprocess
import sys
import time
import urllib.parse
import uuid
from concurrent.futures import ThreadPoolExecutor
from observability_common import ROOT, compose, get_json, prom, ready, save, start_services
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client
def alerts():
return get_json('http://127.0.0.1:19090/api/v1/alerts')['data']['alerts']
def collect_trace(trace_id):
for _ in range(20):
try:
trace = get_json('http://127.0.0.1:13200/api/traces/' + trace_id)
if trace.get('batches') or trace.get('resourceSpans'):
save('evidence/trace-' + trace_id + '.json', trace)
query = '{service_name=~"messenger-api|messenger-worker"} | trace_id = "' + trace_id + '"'
logs = get_json('http://127.0.0.1:13100/loki/api/v1/query_range?' + urllib.parse.urlencode({'query': query, 'since': '1h', 'limit': 100}))
if not logs['data']['result']:
raise RuntimeError('correlated logs not yet available')
save('evidence/logs-' + trace_id + '.json', logs)
return {'trace_id': trace_id, 'trace_file': 'evidence/trace-' + trace_id + '.json',
'log_streams': len(logs['data']['result']), 'logs_file': 'evidence/logs-' + trace_id + '.json'}
except Exception:
pass
time.sleep(2)
raise RuntimeError('trace/log correlation not found in backend')
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--base-url', default='http://127.0.0.1:18086')
args = p.parse_args()
ready(args.base_url)
c = Client(args.base_url)
actor = c.user('incidents')
chat = c.call('POST', '/api/v1/chats', 201, actor=actor,
body={'title': 'Synthetic incidents', 'member_ids': [actor['id']]}, schema='Chat')['json']
report = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'cases': {}}
traces = []
try:
print('Worker outage', flush=True)
compose('stop', 'worker')
attachments = []
started = time.monotonic()
for _ in range(6):
trace_id = uuid.uuid4().hex
result = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=synthetic.png',
202, actor=actor, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(),
headers={'Content-Type': 'image/png', 'traceparent': '00-' + trace_id + '-1234567890123456-01'},
schema='Attachment')
attachments.append(result['json'])
traces.append(trace_id)
timeline = []
for _ in range(6):
time.sleep(5)
timeline.append({'seconds': round(time.monotonic()-started, 2), 'alerts': alerts(),
'queue': prom('messenger_queue_depth'), 'age': prom('messenger_queue_oldest_age_seconds')})
# Keep S3 unavailable for the first attempt; the same durable trace must
# cover the failed attempt, retry publication and eventual ready.
compose('stop', 'minio')
start_services('worker')
time.sleep(10)
report['cases']['s3_retry'] = {'alerts_during': alerts(), 'jobs': prom('messenger_jobs_total')}
start_services('minio')
ready(args.base_url)
for att in attachments:
c.wait_ready(actor, att)
time.sleep(35)
report['cases']['worker_stop'] = {'queued': len(attachments), 'ready_after_recovery': len(attachments),
'timeline': timeline, 'resolved': alerts(),
'queue_after': prom('messenger_queue_depth')}
save('evidence/lab6-faults.json', report)
print('DB lock', flush=True)
lock = subprocess.Popen([str(ROOT / 'scripts/compose.sh'), 'exec', '-T', 'postgres', 'psql', '-U', 'messenger',
'-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-c',
'BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(45); COMMIT;'],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
time.sleep(2)
observations = []
def write(index):
trace_id = uuid.uuid4().hex
result = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', [201, 500, 503], actor=actor,
body={'text': 'synthetic incident'}, headers={'Idempotency-Key': uuid.uuid4().hex,
'traceparent': '00-' + trace_id + '-1234567890123456-01'})
return {'status': result['status'], 'elapsed': result['elapsed'], 'trace_id': trace_id}
with ThreadPoolExecutor(max_workers=2) as pool:
for block in range(7):
observations.extend(pool.map(write, range(4)))
report['cases']['db_lock'] = {'requests': observations, 'alerts_during': alerts()}
save('evidence/lab6-faults.json', report)
lock.wait(timeout=60)
c.message(actor, chat, 'synthetic recovery')
recovery_timeline = []
for _ in range(24):
time.sleep(5)
state = alerts()
recovery_timeline.append(state)
if not any(a['state'] == 'firing' for a in state):
break
c.message(actor, chat, 'synthetic recovery probe')
report['cases']['db_lock']['recovery_alerts'] = recovery_timeline
report['cases']['db_lock']['alerts_after'] = alerts()
failed = next((x for x in observations if x['status'] >= 500), None)
if failed:
traces.append(failed['trace_id'])
report['correlation'] = [collect_trace(t) for t in (traces[0], traces[-1])]
print('Telemetry backend outage', flush=True)
compose('stop', 'tempo')
successful = 0
for _ in range(30):
c.message(actor, chat, 'synthetic telemetry outage')
successful += 1
time.sleep(.5)
report['cases']['tempo_stop'] = {'successful_writes': successful,
'exporter_failed': prom('otelcol_exporter_send_failed_spans_total'),
'queue_size': prom('otelcol_exporter_queue_size'),
'collector_memory': prom('otelcol_process_memory_rss_bytes')}
start_services('tempo')
time.sleep(20)
report['cases']['tempo_stop']['after'] = {'queue_size': prom('otelcol_exporter_queue_size'),
'collector_memory': prom('otelcol_process_memory_rss_bytes')}
assert any(a['labels']['alertname'] == 'QueueLagGrowing' and a['state'] == 'firing'
for row in report['cases']['worker_stop']['timeline'] for a in row['alerts']), 'queue alert never fired'
assert any(a['labels']['alertname'] == 'APIHighErrorRate' and a['state'] == 'firing'
for a in report['cases']['db_lock']['alerts_during']), 'API alert never fired'
assert not any(a['state'] == 'firing' for a in report['cases']['db_lock']['alerts_after']), 'alerts did not resolve'
assert all(float(x['value'][1]) == 0 for x in report['cases']['worker_stop']['queue_after']), 'backlog not empty'
report['status'] = 'PASS'
finally:
start_services('worker', 'tempo', 'minio')
report['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat()
save('evidence/lab6-faults.json', report)
print(json.dumps({'status': report['status'], 'cases': list(report['cases'])}), flush=True)
if __name__ == '__main__':
main()