forked from mtusi2026/mtusi2026lab6
150 lines
7.6 KiB
Python
150 lines
7.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeatable worker/DB/backend faults on this lab's isolated Compose project."""
|
|
import argparse
|
|
import datetime
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
from observability_common import ROOT, compose, get_json, prom, ready, save, start_services
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client
|
|
|
|
|
|
def alerts():
|
|
return get_json('http://127.0.0.1:19090/api/v1/alerts')['data']['alerts']
|
|
|
|
|
|
def collect_trace(trace_id):
|
|
for _ in range(20):
|
|
try:
|
|
trace = get_json('http://127.0.0.1:13200/api/traces/' + trace_id)
|
|
if trace.get('batches') or trace.get('resourceSpans'):
|
|
save('evidence/trace-' + trace_id + '.json', trace)
|
|
query = '{service_name=~"messenger-api|messenger-worker"} | trace_id = "' + trace_id + '"'
|
|
logs = get_json('http://127.0.0.1:13100/loki/api/v1/query_range?' + urllib.parse.urlencode({'query': query, 'since': '1h', 'limit': 100}))
|
|
if not logs['data']['result']:
|
|
raise RuntimeError('correlated logs not yet available')
|
|
save('evidence/logs-' + trace_id + '.json', logs)
|
|
return {'trace_id': trace_id, 'trace_file': 'evidence/trace-' + trace_id + '.json',
|
|
'log_streams': len(logs['data']['result']), 'logs_file': 'evidence/logs-' + trace_id + '.json'}
|
|
except Exception:
|
|
pass
|
|
time.sleep(2)
|
|
raise RuntimeError('trace/log correlation not found in backend')
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--base-url', default='http://127.0.0.1:18086')
|
|
args = p.parse_args()
|
|
ready(args.base_url)
|
|
c = Client(args.base_url)
|
|
actor = c.user('incidents')
|
|
chat = c.call('POST', '/api/v1/chats', 201, actor=actor,
|
|
body={'title': 'Synthetic incidents', 'member_ids': [actor['id']]}, schema='Chat')['json']
|
|
report = {'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'cases': {}}
|
|
traces = []
|
|
try:
|
|
print('Worker outage', flush=True)
|
|
compose('stop', 'worker')
|
|
attachments = []
|
|
started = time.monotonic()
|
|
for _ in range(6):
|
|
trace_id = uuid.uuid4().hex
|
|
result = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=synthetic.png',
|
|
202, actor=actor, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(),
|
|
headers={'Content-Type': 'image/png', 'traceparent': '00-' + trace_id + '-1234567890123456-01'},
|
|
schema='Attachment')
|
|
attachments.append(result['json'])
|
|
traces.append(trace_id)
|
|
timeline = []
|
|
for _ in range(6):
|
|
time.sleep(5)
|
|
timeline.append({'seconds': round(time.monotonic()-started, 2), 'alerts': alerts(),
|
|
'queue': prom('messenger_queue_depth'), 'age': prom('messenger_queue_oldest_age_seconds')})
|
|
# Keep S3 unavailable for the first attempt; the same durable trace must
|
|
# cover the failed attempt, retry publication and eventual ready.
|
|
compose('stop', 'minio')
|
|
start_services('worker')
|
|
time.sleep(10)
|
|
report['cases']['s3_retry'] = {'alerts_during': alerts(), 'jobs': prom('messenger_jobs_total')}
|
|
start_services('minio')
|
|
ready(args.base_url)
|
|
for att in attachments:
|
|
c.wait_ready(actor, att)
|
|
time.sleep(35)
|
|
report['cases']['worker_stop'] = {'queued': len(attachments), 'ready_after_recovery': len(attachments),
|
|
'timeline': timeline, 'resolved': alerts(),
|
|
'queue_after': prom('messenger_queue_depth')}
|
|
save('evidence/lab6-faults.json', report)
|
|
print('DB lock', flush=True)
|
|
lock = subprocess.Popen([str(ROOT / 'scripts/compose.sh'), 'exec', '-T', 'postgres', 'psql', '-U', 'messenger',
|
|
'-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-c',
|
|
'BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(45); COMMIT;'],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
time.sleep(2)
|
|
observations = []
|
|
def write(index):
|
|
trace_id = uuid.uuid4().hex
|
|
result = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', [201, 500, 503], actor=actor,
|
|
body={'text': 'synthetic incident'}, headers={'Idempotency-Key': uuid.uuid4().hex,
|
|
'traceparent': '00-' + trace_id + '-1234567890123456-01'})
|
|
return {'status': result['status'], 'elapsed': result['elapsed'], 'trace_id': trace_id}
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
for block in range(7):
|
|
observations.extend(pool.map(write, range(4)))
|
|
report['cases']['db_lock'] = {'requests': observations, 'alerts_during': alerts()}
|
|
save('evidence/lab6-faults.json', report)
|
|
lock.wait(timeout=60)
|
|
c.message(actor, chat, 'synthetic recovery')
|
|
recovery_timeline = []
|
|
for _ in range(24):
|
|
time.sleep(5)
|
|
state = alerts()
|
|
recovery_timeline.append(state)
|
|
if not any(a['state'] == 'firing' for a in state):
|
|
break
|
|
c.message(actor, chat, 'synthetic recovery probe')
|
|
report['cases']['db_lock']['recovery_alerts'] = recovery_timeline
|
|
report['cases']['db_lock']['alerts_after'] = alerts()
|
|
failed = next((x for x in observations if x['status'] >= 500), None)
|
|
if failed:
|
|
traces.append(failed['trace_id'])
|
|
report['correlation'] = [collect_trace(t) for t in (traces[0], traces[-1])]
|
|
print('Telemetry backend outage', flush=True)
|
|
compose('stop', 'tempo')
|
|
successful = 0
|
|
for _ in range(30):
|
|
c.message(actor, chat, 'synthetic telemetry outage')
|
|
successful += 1
|
|
time.sleep(.5)
|
|
report['cases']['tempo_stop'] = {'successful_writes': successful,
|
|
'exporter_failed': prom('otelcol_exporter_send_failed_spans_total'),
|
|
'queue_size': prom('otelcol_exporter_queue_size'),
|
|
'collector_memory': prom('otelcol_process_memory_rss_bytes')}
|
|
start_services('tempo')
|
|
time.sleep(20)
|
|
report['cases']['tempo_stop']['after'] = {'queue_size': prom('otelcol_exporter_queue_size'),
|
|
'collector_memory': prom('otelcol_process_memory_rss_bytes')}
|
|
assert any(a['labels']['alertname'] == 'QueueLagGrowing' and a['state'] == 'firing'
|
|
for row in report['cases']['worker_stop']['timeline'] for a in row['alerts']), 'queue alert never fired'
|
|
assert any(a['labels']['alertname'] == 'APIHighErrorRate' and a['state'] == 'firing'
|
|
for a in report['cases']['db_lock']['alerts_during']), 'API alert never fired'
|
|
assert not any(a['state'] == 'firing' for a in report['cases']['db_lock']['alerts_after']), 'alerts did not resolve'
|
|
assert all(float(x['value'][1]) == 0 for x in report['cases']['worker_stop']['queue_after']), 'backlog not empty'
|
|
report['status'] = 'PASS'
|
|
finally:
|
|
start_services('worker', 'tempo', 'minio')
|
|
report['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
save('evidence/lab6-faults.json', report)
|
|
print(json.dumps({'status': report['status'], 'cases': list(report['cases'])}), flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|