Files
mtusi2026lab6/scripts/incidents.py
T

387 lines
18 KiB
Python
Executable File

#!/usr/bin/env python3
"""Controlled incidents for lab6 observability; run inside the tools container.
docker compose run --rm tools python3 scripts/incidents.py worker-stop|db|telemetry|retry-trace
Every scenario injects a fault, keeps synthetic traffic going, samples Prometheus/Tempo/Loki through Grafana
and waits for the related alert to become firing and then resolved after recovery.
"""
import base64
import importlib.util
import json
import os
import statistics
import struct
import subprocess
import sys
import threading
import time
import urllib.parse
import urllib.request
import uuid
import zlib
spec = importlib.util.spec_from_file_location('job_scenario', os.path.join(os.path.dirname(__file__), 'job-scenario.py'))
js = importlib.util.module_from_spec(spec)
spec.loader.exec_module(js)
log = js.log
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
PROM = GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/'
TEMPO = GRAFANA + '/api/datasources/proxy/uid/tempo/api/'
LOKI = GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/'
def get(url, **params):
with urllib.request.urlopen(url + ('?' + urllib.parse.urlencode(params) if params else ''), timeout=20) as r:
return json.loads(r.read())
def prom(expr):
try:
result = get(PROM + 'query', query=expr)['data']['result']
except OSError:
return None
return float(result[0]['value'][1]) if result else None
def alert_state(name):
try:
alerts = get(PROM + 'alerts')['data']['alerts']
except OSError:
return 'unknown'
states = [a['state'] for a in alerts if a['labels'].get('alertname') == name]
return 'firing' if 'firing' in states else ('pending' if 'pending' in states else 'inactive')
def fmt(value, unit=''):
if value is None:
return '-'
if unit == '%':
return '%.1f%%' % (value * 100)
if unit == 'ms':
return '%.0fms' % (value * 1000)
if unit == 'MiB':
return '%.0fMiB' % (value / 2 ** 20)
return ('%.1f' % value).rstrip('0').rstrip('.')
def sample(columns, alerts):
parts = ['%s=%s' % (label, fmt(prom(expr), unit)) for label, expr, unit in columns]
parts += ['%s=%s' % (name, alert_state(name)) for name in alerts]
log(' '.join(parts))
def watch(columns, alerts, until, timeout, every=10):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
sample(columns, alerts)
if until():
return True
time.sleep(every)
sample(columns, alerts)
return False
class Traffic:
"""Closed-loop synthetic traffic through the LB: POST message + GET history, counts statuses per phase."""
def __init__(self, auth, chat, threads=4, pause=0.2):
self.auth, self.chat, self.threads, self.pause = auth, chat, threads, pause
self.stop = threading.Event()
self.lock = threading.Lock()
self.reset()
def reset(self):
with self.lock:
self.codes, self.latencies = {}, []
def snapshot(self, phase):
with self.lock:
codes, latencies = dict(self.codes), sorted(self.latencies)
p = lambda q: latencies[min(len(latencies) - 1, int(q * len(latencies)))] * 1000 if latencies else 0
log('client [%s]: %s; p50=%.0fms p95=%.0fms p99=%.0fms' % (
phase, ' '.join('%s:%d' % kv for kv in sorted(codes.items())), p(.5), p(.95), p(.99)))
self.reset()
def run(self):
def loop(n):
i = 0
while not self.stop.is_set():
started = time.monotonic()
if i % 2 == 0:
code = js.call('POST', '/api/v1/chats/%s/messages' % self.chat, {'text': 'incident %d-%d' % (n, i)},
{**self.auth, 'Idempotency-Key': uuid.uuid4().hex})[0]
else:
code = js.call('GET', '/api/v1/chats/%s/messages?limit=20' % self.chat, headers=self.auth)[0]
with self.lock:
self.codes[code] = self.codes.get(code, 0) + 1
if 200 <= code < 300:
self.latencies.append(time.monotonic() - started)
i += 1
time.sleep(self.pause)
self.workers = [threading.Thread(target=loop, args=(n,), daemon=True) for n in range(self.threads)]
for worker in self.workers:
worker.start()
return self
def close(self):
self.stop.set()
for worker in self.workers:
worker.join(20)
def b64hex(value):
return base64.b64decode(value).hex() if value else ''
def trace_spans(trace_id):
body = get(TEMPO + 'traces/' + trace_id)
spans = []
for batch in body.get('batches', []):
service = {a['key']: a['value'] for a in batch['resource']['attributes']}['service.name']['stringValue']
for scope in batch.get('scopeSpans', []):
for span in scope.get('spans', []):
span['service'] = service
span['ms'] = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
span['attrs'] = {a['key']: list(a['value'].values())[0] for a in span.get('attributes', [])}
spans.append(span)
return spans
def find_traces(query, since, limit=5):
try:
return get(TEMPO + 'search', q=query, limit=limit, start=int(since), end=int(time.time()) + 60).get('traces', [])
except OSError:
return []
def breakdown(trace_id, title):
spans = trace_spans(trace_id)
server = next((s for s in spans if s.get('kind') == 'SPAN_KIND_SERVER'), None)
if not server:
log('trace %s: no server span' % trace_id)
return
names = {s['spanId']: s['name'] for s in spans}
def total(prefix, top_level=False):
return sum(s['ms'] for s in spans if s['name'].startswith(prefix)
and not (top_level and names.get(s.get('parentSpanId'), '').startswith('prisma:')))
db, s3 = total('prisma:', top_level=True), total('S3.')
log('%s: trace %s %s -> %s, %.0f ms' % (title, trace_id, server['name'], server['attrs'].get('http.status_code'), server['ms']))
log(' DB calls %.0f ms (pool wait %.0f ms, SQL %.0f ms); S3 %.0f ms; handler itself %.0f ms' % (
db, total('prisma:engine:connection'), total('prisma:engine:db_query'), s3, max(0, server['ms'] - db - s3)))
errors = sorted({s['name'] for s in spans if s.get('status', {}).get('code') == 'STATUS_CODE_ERROR'})
if errors:
log(' spans with error status:', ', '.join(errors))
def noise_png(width, height):
raw = b''.join(b'\0' + os.urandom(width * 3) for _ in range(height))
def chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'')
def docker(*args):
log('$ docker', ' '.join(args))
subprocess.run(['docker', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def container(service):
return subprocess.run(['docker', 'compose', 'ps', '-q', service], check=True, capture_output=True, text=True).stdout.split()
def scenario_worker_stop():
"""Incident 1: worker stopped -> jobs age, MessengerJobQueueLag fires -> worker started -> backlog drains, resolved."""
js.wait_ready()
auth, chat = js.session()
columns = [
('due', 'max(jobs_queue_depth{state="due"})', ''),
('processing', 'max(jobs_queue_depth{state="processing"})', ''),
('oldest_age_s', 'messenger:jobs_queue_oldest_age:max', ''),
('worker_busy', 'sum(worker_jobs_busy)', ''),
('worker_up', 'count(timestamp(worker_jobs_capacity) > time() - 30)', ''),
('ready_per_s', 'sum(rate(jobs_processed_total{outcome="success"}[1m]))', ''),
]
alerts = ['MessengerJobQueueLag']
js.compose('stop', 'worker')
stopped = time.time()
uploads = []
def upload_more():
if len(uploads) < 12 and time.time() - stopped < 150:
uploads.append(js.upload(auth, chat))
return alert_state('MessengerJobQueueLag') == 'firing'
fired = watch(columns, alerts, upload_more, timeout=300)
log('alert MessengerJobQueueLag firing after %.0f s of worker downtime: %s' % (time.time() - stopped, fired))
js.start_only('worker')
started = time.time()
resolved = watch(columns, alerts, lambda: alert_state('MessengerJobQueueLag') == 'inactive', timeout=240)
log('backlog drained and alert resolved %.0f s after worker start: %s' % (time.time() - started, resolved))
ready = sum(1 for a in uploads if js.metadata(auth, a)[1]['status'] == 'ready')
log('uploads during downtime: %d, ready now: %d' % (len(uploads), ready))
time.sleep(15)
log('time to ready, histogram over 5m: p50=%s s, p99=%s s' % tuple(fmt(prom(
'histogram_quantile(%s, sum by (le) (increase(attachments_time_to_ready_seconds_bucket[5m])))' % q)) for q in (0.5, 0.99)))
log('time to ready from the database (finished_at - created_at), s:', js.psql(
"SELECT round(min(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) || ' .. ' || "
"round(max(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) FROM attachments a "
"JOIN attachment_jobs j ON j.attachment_id = a.id WHERE a.id IN (%s)" % ','.join("'%s'" % a for a in uploads)))
if not (fired and resolved and ready == len(uploads)):
sys.exit('FAIL')
log('PASS')
def scenario_db():
"""Incident 2: slow DB (CPU-throttled postgres) vs slow handler (large upload); then DB unavailable (paused)."""
js.wait_ready()
auth, chat = js.session()
columns = [
('p95_1m', 'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api",http_route!~"/health.*"}[1m])))', 'ms'),
('5xx_1m', '(sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*",http_response_status_class="5xx"}[1m])) or vector(0)) / sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*"}[1m]))', '%'),
('pool_wait', 'sum(db_client_connection_pending_requests{job="messenger-api"})', ''),
('pool_used', 'sum(db_client_connection_count{job="messenger-api",db_client_connection_state="used"})', ''),
('core_5xx_1m', 'sum(increase(http_server_requests_total{http_route="/health/core",http_response_status_class="5xx"}[1m]))', ''),
('api_cpu', 'sum(rate(process_cpu_time_seconds_total{job="messenger-api"}[1m]))', ''),
]
alerts = ['MessengerSlowApi', 'MessengerHighErrorRate', 'MessengerApiNotReady']
traffic = Traffic(auth, chat).run()
try:
log('== baseline: 60 s of traffic')
watch(columns, alerts, lambda: False, timeout=60, every=20)
traffic.snapshot('baseline')
log('== slow handler (not DB): 1700x1700 noise PNG upload — decoding and S3 dominate')
big = noise_png(1700, 1700)
since = time.time()
code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat,
headers={**auth, 'Content-Type': 'image/png'}, raw=big)
log('POST %.1f MiB attachment -> %s' % (len(big) / 2 ** 20, code))
time.sleep(15)
for t in find_traces('{resource.service.name="messenger-api" && kind=server && span.http.route="/api/v1/chats/{chat_id}/attachments"}', since, 1):
breakdown(t['traceID'], 'slow handler')
log('== slow DB: postgres limited to 0.05 CPU for 150 s')
postgres = container('postgres')
since = time.time()
docker('update', '--cpus', '0.05', *postgres)
watch(columns, alerts, lambda: alert_state('MessengerSlowApi') == 'firing', timeout=150, every=15)
traffic.snapshot('slow DB')
slow = find_traces('{resource.service.name="messenger-api" && kind=server && duration > 500ms}', since, 3)
for t in slow[:2]:
breakdown(t['traceID'], 'slow DB')
docker('update', '--cpus', '1.0', *postgres)
watch(columns, alerts, lambda: alert_state('MessengerSlowApi') != 'firing', timeout=240, every=15)
traffic.snapshot('after slow DB')
log('== DB unavailable: postgres paused for 120 s')
since = time.time()
docker('pause', *postgres)
not_ready = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'firing', timeout=120, every=15)
traffic.snapshot('DB paused')
for t in find_traces('{resource.service.name="messenger-api" && kind=server && status=error}', since, 2):
breakdown(t['traceID'], 'DB unavailable')
logs = get(LOKI + 'query_range', query='{service_name="messenger-api"} | event="http.error"', limit=3, since='3m')
for stream in logs['data']['result'][:3]:
log(' loki:', stream['values'][0][1][:200])
docker('unpause', *postgres)
recovered = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'inactive', timeout=240, every=15)
traffic.snapshot('after DB unpaused')
finally:
traffic.close()
subprocess.run(['docker', 'unpause', *container('postgres')], capture_output=True)
subprocess.run(['docker', 'update', '--cpus', '1.0', *container('postgres')], capture_output=True)
log('MessengerApiNotReady fired: %s, resolved after recovery: %s' % (not_ready, recovered))
log('PASS' if not_ready and recovered else 'FAIL')
def scenario_telemetry():
"""Incident 3: Loki and Tempo stopped, then the Collector stopped; API keeps serving, memory stays bounded."""
js.wait_ready()
auth, chat = js.session()
columns = [
('spans_failed_1m', 'sum(increase(otelcol_exporter_send_failed_spans[1m]))', ''),
('logs_failed_1m', 'sum(increase(otelcol_exporter_send_failed_log_records[1m]))', ''),
('queue', 'max(otelcol_exporter_queue_size)', ''),
('queue_cap', 'max(otelcol_exporter_queue_capacity)', ''),
('refused_1m', 'sum(increase(otelcol_receiver_refused_spans[1m])) + sum(increase(otelcol_receiver_refused_log_records[1m]))', ''),
('collector_rss', 'max(otelcol_process_memory_rss)', 'MiB'),
('api_rss', 'max(process_memory_usage_bytes{job="messenger-api"})', 'MiB'),
]
alerts = ['TelemetryExportFailing', 'MessengerHighErrorRate']
traffic = Traffic(auth, chat, threads=6, pause=0.05).run()
try:
watch(columns, alerts, lambda: False, timeout=30, every=15)
traffic.snapshot('baseline')
log('== Loki and Tempo stopped until TelemetryExportFailing fires (max 240 s)')
js.compose('stop', 'loki', 'tempo')
fired = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'firing', timeout=240, every=15)
traffic.snapshot('Loki+Tempo down')
js.compose('start', 'loki', 'tempo')
resolved = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'inactive', timeout=240, every=15)
traffic.snapshot('after start')
log('== Collector stopped for 60 s: SDK export fails fast (5 s timeout, bounded queues)')
js.compose('stop', 'otel-collector')
for _ in range(4):
time.sleep(15)
log('docker stats:', subprocess.run(
['docker', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', *container('api'), *container('worker')],
capture_output=True, text=True).stdout.replace('\n', '; '))
traffic.snapshot('collector down')
js.compose('start', 'otel-collector')
time.sleep(30)
traffic.snapshot('collector back')
finally:
traffic.close()
subprocess.run(['docker', 'compose', 'start', 'loki', 'tempo', 'otel-collector'], capture_output=True)
log('TelemetryExportFailing fired: %s, resolved: %s' % (fired, resolved))
log('PASS' if fired and resolved else 'FAIL')
def scenario_retry_trace():
"""Level 4 trace: one upload whose first attempt fails on S3, then succeeds; all attempts in one trace."""
js.wait_ready()
auth, chat = js.session()
trace_id = os.urandom(16).hex()
js.compose('stop', 'worker')
code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
headers={**auth, 'Content-Type': 'image/png', 'traceparent': '00-%s-%s-01' % (trace_id, os.urandom(8).hex())},
raw=js.SAMPLE)
attachment = js.json_of(raw)['id']
log('upload ->', code, attachment, 'trace', trace_id)
js.compose('stop', 's3')
js.start_only('worker')
time.sleep(5)
js.start_only('s3')
js.wait_status(auth, attachment, ('ready', 'failed'))
js.job_state(attachment)
time.sleep(15)
spans = trace_spans(trace_id)
for span in sorted(spans, key=lambda s: int(s['startTimeUnixNano'])):
if span.get('kind') in ('SPAN_KIND_SERVER', 'SPAN_KIND_PRODUCER', 'SPAN_KIND_CONSUMER') or span['name'].startswith('S3.'):
log(' %-17s %-20s %-36s attempt=%-2s %8.1f ms %s' % (
span['service'], span.get('kind', '')[10:], span['name'], span['attrs'].get('messaging.job.attempt', ''),
span['ms'], span.get('status', {}).get('code', '')[12:]))
consumers = [s for s in spans if s.get('kind') == 'SPAN_KIND_CONSUMER']
log('consumer spans in one trace: %d (attempts %s)' % (
len(consumers), sorted({s['attrs'].get('messaging.job.attempt') for s in consumers})))
log('PASS' if len(consumers) >= 2 else 'FAIL')
SCENARIOS = {
'worker-stop': scenario_worker_stop,
'db': scenario_db,
'telemetry': scenario_telemetry,
'retry-trace': scenario_retry_trace,
}
if __name__ == '__main__':
if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS:
sys.exit('usage: incidents.py %s' % '|'.join(SCENARIOS))
SCENARIOS[sys.argv[1]]()