387 lines
18 KiB
Python
Executable File
387 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Controlled incidents for lab6 observability; run inside the tools container.
|
|
|
|
docker compose run --rm tools python3 scripts/incidents.py worker-stop|db|telemetry|retry-trace
|
|
|
|
Every scenario injects a fault, keeps synthetic traffic going, samples Prometheus/Tempo/Loki through Grafana
|
|
and waits for the related alert to become firing and then resolved after recovery.
|
|
"""
|
|
import base64
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import statistics
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
import zlib
|
|
|
|
spec = importlib.util.spec_from_file_location('job_scenario', os.path.join(os.path.dirname(__file__), 'job-scenario.py'))
|
|
js = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(js)
|
|
log = js.log
|
|
|
|
GRAFANA = os.environ.get('GRAFANA_URL', 'http://grafana:3000')
|
|
PROM = GRAFANA + '/api/datasources/proxy/uid/prometheus/api/v1/'
|
|
TEMPO = GRAFANA + '/api/datasources/proxy/uid/tempo/api/'
|
|
LOKI = GRAFANA + '/api/datasources/proxy/uid/loki/loki/api/v1/'
|
|
|
|
|
|
def get(url, **params):
|
|
with urllib.request.urlopen(url + ('?' + urllib.parse.urlencode(params) if params else ''), timeout=20) as r:
|
|
return json.loads(r.read())
|
|
|
|
|
|
def prom(expr):
|
|
try:
|
|
result = get(PROM + 'query', query=expr)['data']['result']
|
|
except OSError:
|
|
return None
|
|
return float(result[0]['value'][1]) if result else None
|
|
|
|
|
|
def alert_state(name):
|
|
try:
|
|
alerts = get(PROM + 'alerts')['data']['alerts']
|
|
except OSError:
|
|
return 'unknown'
|
|
states = [a['state'] for a in alerts if a['labels'].get('alertname') == name]
|
|
return 'firing' if 'firing' in states else ('pending' if 'pending' in states else 'inactive')
|
|
|
|
|
|
def fmt(value, unit=''):
|
|
if value is None:
|
|
return '-'
|
|
if unit == '%':
|
|
return '%.1f%%' % (value * 100)
|
|
if unit == 'ms':
|
|
return '%.0fms' % (value * 1000)
|
|
if unit == 'MiB':
|
|
return '%.0fMiB' % (value / 2 ** 20)
|
|
return ('%.1f' % value).rstrip('0').rstrip('.')
|
|
|
|
|
|
def sample(columns, alerts):
|
|
parts = ['%s=%s' % (label, fmt(prom(expr), unit)) for label, expr, unit in columns]
|
|
parts += ['%s=%s' % (name, alert_state(name)) for name in alerts]
|
|
log(' '.join(parts))
|
|
|
|
|
|
def watch(columns, alerts, until, timeout, every=10):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
sample(columns, alerts)
|
|
if until():
|
|
return True
|
|
time.sleep(every)
|
|
sample(columns, alerts)
|
|
return False
|
|
|
|
|
|
class Traffic:
|
|
"""Closed-loop synthetic traffic through the LB: POST message + GET history, counts statuses per phase."""
|
|
|
|
def __init__(self, auth, chat, threads=4, pause=0.2):
|
|
self.auth, self.chat, self.threads, self.pause = auth, chat, threads, pause
|
|
self.stop = threading.Event()
|
|
self.lock = threading.Lock()
|
|
self.reset()
|
|
|
|
def reset(self):
|
|
with self.lock:
|
|
self.codes, self.latencies = {}, []
|
|
|
|
def snapshot(self, phase):
|
|
with self.lock:
|
|
codes, latencies = dict(self.codes), sorted(self.latencies)
|
|
p = lambda q: latencies[min(len(latencies) - 1, int(q * len(latencies)))] * 1000 if latencies else 0
|
|
log('client [%s]: %s; p50=%.0fms p95=%.0fms p99=%.0fms' % (
|
|
phase, ' '.join('%s:%d' % kv for kv in sorted(codes.items())), p(.5), p(.95), p(.99)))
|
|
self.reset()
|
|
|
|
def run(self):
|
|
def loop(n):
|
|
i = 0
|
|
while not self.stop.is_set():
|
|
started = time.monotonic()
|
|
if i % 2 == 0:
|
|
code = js.call('POST', '/api/v1/chats/%s/messages' % self.chat, {'text': 'incident %d-%d' % (n, i)},
|
|
{**self.auth, 'Idempotency-Key': uuid.uuid4().hex})[0]
|
|
else:
|
|
code = js.call('GET', '/api/v1/chats/%s/messages?limit=20' % self.chat, headers=self.auth)[0]
|
|
with self.lock:
|
|
self.codes[code] = self.codes.get(code, 0) + 1
|
|
if 200 <= code < 300:
|
|
self.latencies.append(time.monotonic() - started)
|
|
i += 1
|
|
time.sleep(self.pause)
|
|
self.workers = [threading.Thread(target=loop, args=(n,), daemon=True) for n in range(self.threads)]
|
|
for worker in self.workers:
|
|
worker.start()
|
|
return self
|
|
|
|
def close(self):
|
|
self.stop.set()
|
|
for worker in self.workers:
|
|
worker.join(20)
|
|
|
|
|
|
def b64hex(value):
|
|
return base64.b64decode(value).hex() if value else ''
|
|
|
|
|
|
def trace_spans(trace_id):
|
|
body = get(TEMPO + 'traces/' + trace_id)
|
|
spans = []
|
|
for batch in body.get('batches', []):
|
|
service = {a['key']: a['value'] for a in batch['resource']['attributes']}['service.name']['stringValue']
|
|
for scope in batch.get('scopeSpans', []):
|
|
for span in scope.get('spans', []):
|
|
span['service'] = service
|
|
span['ms'] = (int(span['endTimeUnixNano']) - int(span['startTimeUnixNano'])) / 1e6
|
|
span['attrs'] = {a['key']: list(a['value'].values())[0] for a in span.get('attributes', [])}
|
|
spans.append(span)
|
|
return spans
|
|
|
|
|
|
def find_traces(query, since, limit=5):
|
|
try:
|
|
return get(TEMPO + 'search', q=query, limit=limit, start=int(since), end=int(time.time()) + 60).get('traces', [])
|
|
except OSError:
|
|
return []
|
|
|
|
|
|
def breakdown(trace_id, title):
|
|
spans = trace_spans(trace_id)
|
|
server = next((s for s in spans if s.get('kind') == 'SPAN_KIND_SERVER'), None)
|
|
if not server:
|
|
log('trace %s: no server span' % trace_id)
|
|
return
|
|
|
|
names = {s['spanId']: s['name'] for s in spans}
|
|
|
|
def total(prefix, top_level=False):
|
|
return sum(s['ms'] for s in spans if s['name'].startswith(prefix)
|
|
and not (top_level and names.get(s.get('parentSpanId'), '').startswith('prisma:')))
|
|
db, s3 = total('prisma:', top_level=True), total('S3.')
|
|
log('%s: trace %s %s -> %s, %.0f ms' % (title, trace_id, server['name'], server['attrs'].get('http.status_code'), server['ms']))
|
|
log(' DB calls %.0f ms (pool wait %.0f ms, SQL %.0f ms); S3 %.0f ms; handler itself %.0f ms' % (
|
|
db, total('prisma:engine:connection'), total('prisma:engine:db_query'), s3, max(0, server['ms'] - db - s3)))
|
|
errors = sorted({s['name'] for s in spans if s.get('status', {}).get('code') == 'STATUS_CODE_ERROR'})
|
|
if errors:
|
|
log(' spans with error status:', ', '.join(errors))
|
|
|
|
|
|
def noise_png(width, height):
|
|
raw = b''.join(b'\0' + os.urandom(width * 3) for _ in range(height))
|
|
|
|
def chunk(kind, data):
|
|
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
|
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
|
|
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b'')
|
|
|
|
|
|
def docker(*args):
|
|
log('$ docker', ' '.join(args))
|
|
subprocess.run(['docker', *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
|
|
|
|
def container(service):
|
|
return subprocess.run(['docker', 'compose', 'ps', '-q', service], check=True, capture_output=True, text=True).stdout.split()
|
|
|
|
|
|
def scenario_worker_stop():
|
|
"""Incident 1: worker stopped -> jobs age, MessengerJobQueueLag fires -> worker started -> backlog drains, resolved."""
|
|
js.wait_ready()
|
|
auth, chat = js.session()
|
|
columns = [
|
|
('due', 'max(jobs_queue_depth{state="due"})', ''),
|
|
('processing', 'max(jobs_queue_depth{state="processing"})', ''),
|
|
('oldest_age_s', 'messenger:jobs_queue_oldest_age:max', ''),
|
|
('worker_busy', 'sum(worker_jobs_busy)', ''),
|
|
('worker_up', 'count(timestamp(worker_jobs_capacity) > time() - 30)', ''),
|
|
('ready_per_s', 'sum(rate(jobs_processed_total{outcome="success"}[1m]))', ''),
|
|
]
|
|
alerts = ['MessengerJobQueueLag']
|
|
js.compose('stop', 'worker')
|
|
stopped = time.time()
|
|
uploads = []
|
|
|
|
def upload_more():
|
|
if len(uploads) < 12 and time.time() - stopped < 150:
|
|
uploads.append(js.upload(auth, chat))
|
|
return alert_state('MessengerJobQueueLag') == 'firing'
|
|
fired = watch(columns, alerts, upload_more, timeout=300)
|
|
log('alert MessengerJobQueueLag firing after %.0f s of worker downtime: %s' % (time.time() - stopped, fired))
|
|
js.start_only('worker')
|
|
started = time.time()
|
|
resolved = watch(columns, alerts, lambda: alert_state('MessengerJobQueueLag') == 'inactive', timeout=240)
|
|
log('backlog drained and alert resolved %.0f s after worker start: %s' % (time.time() - started, resolved))
|
|
ready = sum(1 for a in uploads if js.metadata(auth, a)[1]['status'] == 'ready')
|
|
log('uploads during downtime: %d, ready now: %d' % (len(uploads), ready))
|
|
time.sleep(15)
|
|
log('time to ready, histogram over 5m: p50=%s s, p99=%s s' % tuple(fmt(prom(
|
|
'histogram_quantile(%s, sum by (le) (increase(attachments_time_to_ready_seconds_bucket[5m])))' % q)) for q in (0.5, 0.99)))
|
|
log('time to ready from the database (finished_at - created_at), s:', js.psql(
|
|
"SELECT round(min(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) || ' .. ' || "
|
|
"round(max(extract(epoch FROM j.finished_at - a.created_at))::numeric, 1) FROM attachments a "
|
|
"JOIN attachment_jobs j ON j.attachment_id = a.id WHERE a.id IN (%s)" % ','.join("'%s'" % a for a in uploads)))
|
|
if not (fired and resolved and ready == len(uploads)):
|
|
sys.exit('FAIL')
|
|
log('PASS')
|
|
|
|
|
|
def scenario_db():
|
|
"""Incident 2: slow DB (CPU-throttled postgres) vs slow handler (large upload); then DB unavailable (paused)."""
|
|
js.wait_ready()
|
|
auth, chat = js.session()
|
|
columns = [
|
|
('p95_1m', 'histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api",http_route!~"/health.*"}[1m])))', 'ms'),
|
|
('5xx_1m', '(sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*",http_response_status_class="5xx"}[1m])) or vector(0)) / sum(rate(http_server_requests_total{job="messenger-api",http_route!~"/health.*"}[1m]))', '%'),
|
|
('pool_wait', 'sum(db_client_connection_pending_requests{job="messenger-api"})', ''),
|
|
('pool_used', 'sum(db_client_connection_count{job="messenger-api",db_client_connection_state="used"})', ''),
|
|
('core_5xx_1m', 'sum(increase(http_server_requests_total{http_route="/health/core",http_response_status_class="5xx"}[1m]))', ''),
|
|
('api_cpu', 'sum(rate(process_cpu_time_seconds_total{job="messenger-api"}[1m]))', ''),
|
|
]
|
|
alerts = ['MessengerSlowApi', 'MessengerHighErrorRate', 'MessengerApiNotReady']
|
|
traffic = Traffic(auth, chat).run()
|
|
try:
|
|
log('== baseline: 60 s of traffic')
|
|
watch(columns, alerts, lambda: False, timeout=60, every=20)
|
|
traffic.snapshot('baseline')
|
|
|
|
log('== slow handler (not DB): 1700x1700 noise PNG upload — decoding and S3 dominate')
|
|
big = noise_png(1700, 1700)
|
|
since = time.time()
|
|
code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=big.png' % chat,
|
|
headers={**auth, 'Content-Type': 'image/png'}, raw=big)
|
|
log('POST %.1f MiB attachment -> %s' % (len(big) / 2 ** 20, code))
|
|
time.sleep(15)
|
|
for t in find_traces('{resource.service.name="messenger-api" && kind=server && span.http.route="/api/v1/chats/{chat_id}/attachments"}', since, 1):
|
|
breakdown(t['traceID'], 'slow handler')
|
|
|
|
log('== slow DB: postgres limited to 0.05 CPU for 150 s')
|
|
postgres = container('postgres')
|
|
since = time.time()
|
|
docker('update', '--cpus', '0.05', *postgres)
|
|
watch(columns, alerts, lambda: alert_state('MessengerSlowApi') == 'firing', timeout=150, every=15)
|
|
traffic.snapshot('slow DB')
|
|
slow = find_traces('{resource.service.name="messenger-api" && kind=server && duration > 500ms}', since, 3)
|
|
for t in slow[:2]:
|
|
breakdown(t['traceID'], 'slow DB')
|
|
docker('update', '--cpus', '1.0', *postgres)
|
|
watch(columns, alerts, lambda: alert_state('MessengerSlowApi') != 'firing', timeout=240, every=15)
|
|
traffic.snapshot('after slow DB')
|
|
|
|
log('== DB unavailable: postgres paused for 120 s')
|
|
since = time.time()
|
|
docker('pause', *postgres)
|
|
not_ready = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'firing', timeout=120, every=15)
|
|
traffic.snapshot('DB paused')
|
|
for t in find_traces('{resource.service.name="messenger-api" && kind=server && status=error}', since, 2):
|
|
breakdown(t['traceID'], 'DB unavailable')
|
|
logs = get(LOKI + 'query_range', query='{service_name="messenger-api"} | event="http.error"', limit=3, since='3m')
|
|
for stream in logs['data']['result'][:3]:
|
|
log(' loki:', stream['values'][0][1][:200])
|
|
docker('unpause', *postgres)
|
|
recovered = watch(columns, alerts, lambda: alert_state('MessengerApiNotReady') == 'inactive', timeout=240, every=15)
|
|
traffic.snapshot('after DB unpaused')
|
|
finally:
|
|
traffic.close()
|
|
subprocess.run(['docker', 'unpause', *container('postgres')], capture_output=True)
|
|
subprocess.run(['docker', 'update', '--cpus', '1.0', *container('postgres')], capture_output=True)
|
|
log('MessengerApiNotReady fired: %s, resolved after recovery: %s' % (not_ready, recovered))
|
|
log('PASS' if not_ready and recovered else 'FAIL')
|
|
|
|
|
|
def scenario_telemetry():
|
|
"""Incident 3: Loki and Tempo stopped, then the Collector stopped; API keeps serving, memory stays bounded."""
|
|
js.wait_ready()
|
|
auth, chat = js.session()
|
|
columns = [
|
|
('spans_failed_1m', 'sum(increase(otelcol_exporter_send_failed_spans[1m]))', ''),
|
|
('logs_failed_1m', 'sum(increase(otelcol_exporter_send_failed_log_records[1m]))', ''),
|
|
('queue', 'max(otelcol_exporter_queue_size)', ''),
|
|
('queue_cap', 'max(otelcol_exporter_queue_capacity)', ''),
|
|
('refused_1m', 'sum(increase(otelcol_receiver_refused_spans[1m])) + sum(increase(otelcol_receiver_refused_log_records[1m]))', ''),
|
|
('collector_rss', 'max(otelcol_process_memory_rss)', 'MiB'),
|
|
('api_rss', 'max(process_memory_usage_bytes{job="messenger-api"})', 'MiB'),
|
|
]
|
|
alerts = ['TelemetryExportFailing', 'MessengerHighErrorRate']
|
|
traffic = Traffic(auth, chat, threads=6, pause=0.05).run()
|
|
try:
|
|
watch(columns, alerts, lambda: False, timeout=30, every=15)
|
|
traffic.snapshot('baseline')
|
|
log('== Loki and Tempo stopped until TelemetryExportFailing fires (max 240 s)')
|
|
js.compose('stop', 'loki', 'tempo')
|
|
fired = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'firing', timeout=240, every=15)
|
|
traffic.snapshot('Loki+Tempo down')
|
|
js.compose('start', 'loki', 'tempo')
|
|
resolved = watch(columns, alerts, lambda: alert_state('TelemetryExportFailing') == 'inactive', timeout=240, every=15)
|
|
traffic.snapshot('after start')
|
|
|
|
log('== Collector stopped for 60 s: SDK export fails fast (5 s timeout, bounded queues)')
|
|
js.compose('stop', 'otel-collector')
|
|
for _ in range(4):
|
|
time.sleep(15)
|
|
log('docker stats:', subprocess.run(
|
|
['docker', 'stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', *container('api'), *container('worker')],
|
|
capture_output=True, text=True).stdout.replace('\n', '; '))
|
|
traffic.snapshot('collector down')
|
|
js.compose('start', 'otel-collector')
|
|
time.sleep(30)
|
|
traffic.snapshot('collector back')
|
|
finally:
|
|
traffic.close()
|
|
subprocess.run(['docker', 'compose', 'start', 'loki', 'tempo', 'otel-collector'], capture_output=True)
|
|
log('TelemetryExportFailing fired: %s, resolved: %s' % (fired, resolved))
|
|
log('PASS' if fired and resolved else 'FAIL')
|
|
|
|
|
|
def scenario_retry_trace():
|
|
"""Level 4 trace: one upload whose first attempt fails on S3, then succeeds; all attempts in one trace."""
|
|
js.wait_ready()
|
|
auth, chat = js.session()
|
|
trace_id = os.urandom(16).hex()
|
|
js.compose('stop', 'worker')
|
|
code, raw, _ = js.call('POST', '/api/v1/chats/%s/attachments?filename=sample.png' % chat,
|
|
headers={**auth, 'Content-Type': 'image/png', 'traceparent': '00-%s-%s-01' % (trace_id, os.urandom(8).hex())},
|
|
raw=js.SAMPLE)
|
|
attachment = js.json_of(raw)['id']
|
|
log('upload ->', code, attachment, 'trace', trace_id)
|
|
js.compose('stop', 's3')
|
|
js.start_only('worker')
|
|
time.sleep(5)
|
|
js.start_only('s3')
|
|
js.wait_status(auth, attachment, ('ready', 'failed'))
|
|
js.job_state(attachment)
|
|
time.sleep(15)
|
|
spans = trace_spans(trace_id)
|
|
for span in sorted(spans, key=lambda s: int(s['startTimeUnixNano'])):
|
|
if span.get('kind') in ('SPAN_KIND_SERVER', 'SPAN_KIND_PRODUCER', 'SPAN_KIND_CONSUMER') or span['name'].startswith('S3.'):
|
|
log(' %-17s %-20s %-36s attempt=%-2s %8.1f ms %s' % (
|
|
span['service'], span.get('kind', '')[10:], span['name'], span['attrs'].get('messaging.job.attempt', ''),
|
|
span['ms'], span.get('status', {}).get('code', '')[12:]))
|
|
consumers = [s for s in spans if s.get('kind') == 'SPAN_KIND_CONSUMER']
|
|
log('consumer spans in one trace: %d (attempts %s)' % (
|
|
len(consumers), sorted({s['attrs'].get('messaging.job.attempt') for s in consumers})))
|
|
log('PASS' if len(consumers) >= 2 else 'FAIL')
|
|
|
|
|
|
SCENARIOS = {
|
|
'worker-stop': scenario_worker_stop,
|
|
'db': scenario_db,
|
|
'telemetry': scenario_telemetry,
|
|
'retry-trace': scenario_retry_trace,
|
|
}
|
|
|
|
if __name__ == '__main__':
|
|
if len(sys.argv) != 2 or sys.argv[1] not in SCENARIOS:
|
|
sys.exit('usage: incidents.py %s' % '|'.join(SCENARIOS))
|
|
SCENARIOS[sys.argv[1]]()
|