Files
mtusi2026lab6/tests/observability.py
T

409 lines
23 KiB
Python

#!/usr/bin/env python3
"""Lab6 experiments on a running stack: `slo` (track 2: burn-rate alerts vs a spike, a long degradation and
an outage) and `incidents` (level 4: worker stop, slow DB, telemetry backends down). Reads telemetry through
the Grafana datasource proxy (admin password from secrets/, never printed); drives faults with docker compose,
so run it with the same COMPOSE_PROJECT_NAME/COMPOSE_FILE as the stack. `traces` (level 3/4 trace+log correlation,
upload chain with one retry, leak check) and `slo` need FAULT_INJECTION=1."""
import argparse
import base64
import json
import subprocess
import threading
import time
import urllib.parse
import urllib.request
from client import ROOT, Client, Failure, check, run
T0 = time.monotonic()
def now():
return round(time.monotonic() - T0, 1)
class Grafana:
def __init__(self, url):
password = (ROOT / 'secrets/grafana_admin_password').read_text().strip()
self.url = url.rstrip('/')
self.auth = 'Basic ' + base64.b64encode(('admin:' + password).encode()).decode()
def get(self, path):
req = urllib.request.Request(self.url + path, headers={'Authorization': self.auth})
with urllib.request.urlopen(req, timeout=20) as r:
return json.loads(r.read())
def prom(self, query):
return self.get('/api/datasources/proxy/uid/prometheus/api/v1/query?query=' + urllib.parse.quote(query))['data']['result']
def value(self, query):
result = self.prom(query)
return round(float(result[0]['value'][1]), 4) if result else None
def alerts(self):
data = self.get('/api/datasources/proxy/uid/prometheus/api/v1/alerts')['data']['alerts']
return {a['labels']['alertname']: a['state'] for a in data}
class AlertLog(threading.Thread):
"""Polls Prometheus alert states every 2 s and keeps transitions with experiment-relative time."""
def __init__(self, grafana):
super().__init__(daemon=True)
self.grafana, self.state, self.events, self.stop = grafana, {}, [], threading.Event()
def run(self):
while not self.stop.is_set():
try:
current = self.grafana.alerts()
except OSError:
current = self.state
for name in set(current) | set(self.state):
old, new = self.state.get(name, 'inactive'), current.get(name, 'inactive')
if old != new:
self.events.append({'t': now(), 'alert': name, 'from': old, 'to': new})
self.state = current
self.stop.wait(2)
def intervals(self, name):
"""Firing intervals [start, end) of one alert; end None = still firing."""
out, start = [], None
for e in self.events:
if e['alert'] != name:
continue
if e['to'] == 'firing':
start = e['t']
elif e['from'] == 'firing' and start is not None:
out.append([start, e['t']])
start = None
if start is not None:
out.append([start, None])
return out
class Load:
"""Paced POST-message load: `rps` total; share `fault` of requests carries X-Fault-Inject: http-500."""
def __init__(self, base_url, rps, threads=5):
self.client = Client(base_url)
self.alice, _, _, self.chat = self.client.bootstrap()
self.fault, self.phase, self.stop = 0.0, 'warmup', threading.Event()
self.counts, self.lock = {}, threading.Lock()
self.interval = threads / rps
self.threads = [threading.Thread(target=self.worker, args=(Client(base_url, timeout=5),), daemon=True) for _ in range(threads)]
def start(self):
for t in self.threads:
t.start()
def worker(self, client):
due, debt, n = time.monotonic(), 0.0, 0
path = '/api/v1/chats/' + self.chat['id'] + '/messages'
while not self.stop.is_set():
debt += self.fault
inject = debt >= 1
if inject:
debt -= 1
headers = {'Idempotency-Key': 'obs-%d-%d-%f' % (id(client), n, time.time())}
if inject:
headers['X-Fault-Inject'] = 'http-500'
try:
status = client.call('POST', path, [201, 500], body={'text': 'obs %d' % n}, actor=self.alice, headers=headers)['status']
outcome = 'injected_500' if status == 500 and inject else ('ok' if status == 201 else 'unexpected_500')
except Failure:
outcome = 'failed'
with self.lock:
bucket = self.counts.setdefault(self.phase, {})
bucket[outcome] = bucket.get(outcome, 0) + 1
n += 1
due += self.interval
time.sleep(max(0.0, due - time.monotonic()))
def compose(*args, check_rc=True):
return subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True, check=check_rc).stdout
def slo(args):
g = Grafana(args.grafana_url)
probe = Client(args.base_url)
a, _, _, chat = probe.bootstrap()
status = probe.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', [201, 500], body={'text': 'probe'}, actor=a,
headers={'Idempotency-Key': 'probe-%f' % time.time(), 'X-Fault-Inject': 'http-500'})['status']
check(status == 500, 'fault injection is off: start the stack with FAULT_INJECTION=1')
check(wait_alerts(g, list(g.alerts()), False, 600) is not None, 'alerts still active before the experiment')
phases = [('baseline', args.baseline, 0.0), ('spike', args.spike, 1.0), ('after_spike', args.quiet, 0.0),
('degradation', args.degradation, args.degradation_ratio), ('after_degradation', args.quiet + 30, 0.0),
('outage', args.outage, 1.0), ('after_outage', args.quiet, 0.0)]
load = Load(args.base_url, args.rps)
watcher = AlertLog(g)
load.start()
watcher.start()
timeline, samples = [], []
for name, seconds, ratio in phases:
load.phase, load.fault = name, ratio
start = now()
print('%7.1fs phase %-18s %4ds fault=%.2f' % (start, name, seconds, ratio), flush=True)
while now() - start < seconds:
time.sleep(10)
samples.append({'t': now(), 'phase': name, **{w: g.value('slo:api_errors:ratio_rate' + w) for w in ('30s', '1m', '1m30s', '2m30s', '15m')}})
timeline.append({'phase': name, 'start': start, 'end': now(), 'fault_ratio': ratio})
load.stop.set()
watcher.stop.set()
watcher.join()
alerts = {name: watcher.intervals(name) for name in ('ApiAvailabilityFastBurn', 'ApiAvailabilitySlowBurn', 'ApiHighErrorRate')}
expected = {'ApiAvailabilityFastBurn': {'outage'}, 'ApiAvailabilitySlowBurn': {'degradation', 'outage'},
'ApiHighErrorRate': {'degradation', 'outage'}} # по политике SLO: всплеск < 2 % бюджета — не повод для реакции
phase_of = lambda t: next((p['phase'] for p in timeline if p['start'] <= t < p['end']), 'end')
fault_phase = {'spike', 'degradation', 'outage'}
analysis = {}
for name, spans in alerts.items():
rows = []
for start, end in spans:
cause = phase_of(start)
cause = cause.replace('after_', '') if cause.startswith('after_') else cause
origin = next(p for p in timeline if p['phase'] == cause) if cause in fault_phase else None
fault_end = origin['end'] if origin else None
rows.append({'fired_at': start, 'resolved_at': end, 'caused_by': cause,
'detect_s': round(start - origin['start'], 1) if origin else None,
'resolve_after_fault_end_s': round(end - fault_end, 1) if origin and end else None,
'expected': cause in expected[name]})
missed = sorted(expected[name] - {r['caused_by'] for r in rows})
analysis[name] = {'firings': rows, 'false_positives': sum(not r['expected'] for r in rows), 'missed': missed}
total = {k: sum(c.get(k, 0) for c in load.counts.values()) for k in ('ok', 'injected_500', 'unexpected_500', 'failed')}
result = {'design': {'rps': args.rps, 'slo': '99% non-5xx', 'compression': 24, 'phases': timeline},
'requests': load.counts, 'totals': total, 'alerts': analysis, 'events': watcher.events, 'ratios': samples}
out = ROOT / 'evidence/slo_experiment.json'
out.write_text(json.dumps(result, ensure_ascii=False, indent=1))
for name, a in analysis.items():
print(name, 'FP=%d missed=%s' % (a['false_positives'], a['missed']),
['%s: detect %ss, resolve +%ss' % (r['caused_by'], r['detect_s'], r['resolve_after_fault_end_s']) for r in a['firings']])
print('requests', total, '->', out.relative_to(ROOT))
check(total['unexpected_500'] == 0 and total['failed'] <= 3, 'unexpected request failures: %s' % total)
def wait_alerts(g, names, firing, timeout):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
active = {k for k, v in g.alerts().items() if v == 'firing'}
if (set(names) <= active) if firing else not (set(names) & set(g.alerts())):
return now()
time.sleep(2)
return None
def incidents(args):
g = Grafana(args.grafana_url)
report = {}
load = Load(args.base_url, 5, threads=2)
uploader = Client(args.base_url, timeout=10)
owner, _, _, chat = uploader.bootstrap()
uploads = {'accepted': 0, 'failed': 0}
stop_uploads = threading.Event()
def upload_loop():
while not stop_uploads.is_set():
try:
uploader.attachment(owner, chat)
uploads['accepted'] += 1
except Failure:
uploads['failed'] += 1
stop_uploads.wait(2)
watcher = AlertLog(g)
load.start()
watcher.start()
threading.Thread(target=upload_loop, daemon=True).start()
time.sleep(40)
# 1. Worker stopped: backlog grows, QueueLagHigh/WorkerAbsent fire; after start backlog drains, alerts resolve.
load.phase = 'worker_stopped'
t_stop = now()
compose('stop', 'worker')
queue = []
while now() - t_stop < args.worker_down:
time.sleep(10)
queue.append({'t': now(), 'depth': g.value('max(messenger_queue_depth)'), 'oldest_age_s': g.value('max(messenger_queue_oldest_age_seconds)')})
load.phase = 'worker_recovering'
t_start = now()
compose('start', 'worker')
drained = None
while now() - t_start < 300:
time.sleep(5)
depth = g.value('max(messenger_queue_depth)')
queue.append({'t': now(), 'depth': depth, 'oldest_age_s': g.value('max(messenger_queue_oldest_age_seconds)')})
if drained is None and depth == 0 and now() - t_start > 20:
drained = now()
if drained and wait_alerts(g, ['QueueLagHigh', 'WorkerAbsent'], False, 1):
break
report['worker_stop'] = {'stopped_at': t_stop, 'started_at': t_start, 'backlog_drained_at': drained, 'queue': queue}
print('worker stop: drained at', drained, json.dumps({n: watcher.intervals(n) for n in ('QueueLagHigh', 'WorkerAbsent')}), flush=True)
# 2. Slow DB: an exclusive lock on messages for N s; POST /messages waits in the INSERT span, not in the handler.
load.phase = 'db_locked'
t_lock = now()
compose('exec', '-T', 'postgres', 'psql', '-q', '-U', 'messenger', '-d', 'messenger', '-c',
'begin; lock table messages in access exclusive mode; select pg_sleep(%d); commit;' % args.db_lock)
load.phase = 'db_recovered'
time.sleep(25)
end_ns, start_ns = time.time_ns(), time.time_ns() - int((now() - t_lock + 30) * 1e9)
search = g.get('/api/datasources/proxy/uid/tempo/api/search?' + urllib.parse.urlencode(
{'tags': 'service.name=messenger-api', 'minDuration': '%ds' % max(2, args.db_lock // 2), 'limit': 5,
'start': start_ns // 10**9, 'end': end_ns // 10**9}))
slow = []
for t in search.get('traces', [])[:3]:
trace = g.get('/api/datasources/proxy/uid/tempo/api/traces/' + t['traceID'])
spans = [s for b in trace.get('batches') or trace.get('resourceSpans', []) for ss in b.get('scopeSpans', []) for s in ss['spans']]
dur = lambda s: (int(s['endTimeUnixNano']) - int(s['startTimeUnixNano'])) / 1e9
server = max((s for s in spans if s.get('kind') in ('SPAN_KIND_SERVER', 2)), key=dur, default=None)
db = max((s for s in spans if s.get('kind') in ('SPAN_KIND_CLIENT', 3) and any(a['key'].startswith('db.') for a in s.get('attributes', []))), key=dur, default=None)
slow.append({'trace_id': t['traceID'], 'root': t.get('rootTraceName'), 'server_s': server and round(dur(server), 2),
'slowest_db_span': db and db['name'], 'db_s': db and round(dur(db), 2)})
report['db_lock'] = {'locked_at': t_lock, 'lock_s': args.db_lock, 'slow_traces': slow,
'p99_messages_s_2m': g.value('histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api",http_route="/api/v1/chats/{id}/messages",http_request_method="POST"}[2m])))'),
'max_pending_connections_2m': g.value('max(max_over_time(db_client_connections_pending_requests[2m]))'),
'requests': {k: load.counts.get(k) for k in ('db_locked', 'db_recovered')}}
print('db lock:', json.dumps(slow), flush=True)
# 3. Telemetry backends down: API keeps serving; Collector memory stays bounded; losses are counted.
load.phase = 'telemetry_down'
# Собственные метрики Collector 0.115 приходят без суффикса _total; счётчик появляется при первой ошибке: нет = 0.
lost = lambda k: sum(g.value('sum(otelcol_exporter_%s_%s)' % (kind, k)) or 0 for kind in ('send_failed', 'enqueue_failed'))
before = {k: lost(k) for k in ('spans', 'log_records')}
compose('stop', 'loki', 'tempo')
t_down, memory = now(), []
while now() - t_down < args.telemetry_down:
time.sleep(10)
stats = compose('stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', 'otel-collector', check_rc=False).strip()
memory.append({'t': now(), 'collector_mem': stats.split(' ', 1)[-1],
'queue_batches': g.value('sum(otelcol_exporter_queue_size)'),
'refused_spans': g.value('sum(otelcol_receiver_refused_spans)')})
compose('start', 'loki', 'tempo')
load.phase = 'telemetry_recovered'
time.sleep(75)
after = {k: lost(k) for k in ('spans', 'log_records')}
report['telemetry_down'] = {'down_s': args.telemetry_down, 'collector': memory, 'lost_before': before, 'lost_after': after,
'requests': {k: load.counts.get(k) for k in ('telemetry_down', 'telemetry_recovered')}}
print('telemetry down:', json.dumps(memory[-1]), 'lost', before, '->', after, flush=True)
stop_uploads.set()
load.stop.set()
watcher.stop.set()
time.sleep(2)
report['alerts'] = {n: watcher.intervals(n) for n in sorted({e['alert'] for e in watcher.events})}
report['uploads'] = uploads
report['requests'] = load.counts
report['events'] = watcher.events
out = ROOT / 'evidence/incidents.json'
out.write_text(json.dumps(report, ensure_ascii=False, indent=1))
print('requests', load.counts, 'uploads', uploads, '->', out.relative_to(ROOT))
def trace_tree(g, trace_id, wait=60):
"""Spans of one trace from Tempo (waits for ingestion): service, kind, name, parent, status, key attributes."""
deadline = time.monotonic() + wait
while True:
try:
data = g.get('/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
except OSError:
data = {}
spans = []
for b in data.get('batches') or data.get('resourceSpans', []):
service = next((a['value'].get('stringValue') for a in b['resource']['attributes'] if a['key'] == 'service.name'), '?')
for ss in b.get('scopeSpans', []):
for sp in ss['spans']:
attrs = {a['key']: next(iter(a['value'].values()), None) for a in sp.get('attributes', [])}
spans.append({'service': service, 'kind': str(sp.get('kind', '')).replace('SPAN_KIND_', ''), 'name': sp['name'],
'span_id': sp['spanId'], 'parent': sp.get('parentSpanId') or None,
'start_ns': int(sp['startTimeUnixNano']), 'ms': round((int(sp['endTimeUnixNano']) - int(sp['startTimeUnixNano'])) / 1e6, 1),
'status': sp.get('status', {}).get('code', 'UNSET'),
**{k: attrs[k] for k in ('job.attempt', 'job.outcome', 'http.response.status_code', 'messaging.destination.name') if k in attrs}})
if any(s['kind'] == 'CONSUMER' and s.get('job.outcome') == 'done' for s in spans) or time.monotonic() > deadline:
return sorted(spans, key=lambda s: s['start_ns']), json.dumps(data)
time.sleep(5)
def logs_for(g, service, trace_id):
end = time.time_ns()
q = urllib.parse.quote('{service_name="%s"} | trace_id="%s"' % (service, trace_id))
data = g.get('/api/datasources/proxy/uid/loki/loki/api/v1/query_range?query=%s&limit=50&start=%d&end=%d&direction=forward' % (q, end - 900 * 10**9, end))
return [{'ts': v[0], 'line': v[1][:160], **{k: s['stream'].get(k) for k in ('event', 'severity_text', 'trace_id', 'span_id', 'request_id', 'http_route', 'http_response_status_code')}}
for s in data['data']['result'] for v in s['values']]
def traces(args):
"""Level 3/4 evidence: a controlled request → its trace + log; one upload with one retry → one trace to S3/DB."""
import os
g = Grafana(args.grafana_url)
c = Client(args.base_url)
alice, _, _, chat = c.bootstrap()
tid = lambda: os.urandom(16).hex()
marker = 'leak-marker-' + os.urandom(6).hex()
req_trace = tid()
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, body={'text': marker}, actor=alice, schema='Message',
headers={'Idempotency-Key': os.urandom(8).hex(), 'traceparent': '00-%s-%s-01' % (req_trace, os.urandom(8).hex())})
request_id = r['headers'].get('X-Request-Id')
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
up_trace = tid()
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=fault-flaky-trace.png', 202, actor=alice, raw=content, schema='Attachment',
headers={'Content-Type': 'image/png', 'traceparent': '00-%s-%s-01' % (up_trace, os.urandom(8).hex())})['json']
ready = c.wait_ready(alice, att, 90)
time.sleep(15)
req_spans, req_raw = trace_tree(g, req_trace, 30)
up_spans, up_raw = trace_tree(g, up_trace, 90)
req_logs = logs_for(g, 'messenger-api', req_trace)
up_logs = logs_for(g, 'messenger-api', up_trace) + logs_for(g, 'messenger-worker', up_trace)
by_id = {s['span_id']: s for s in up_spans}
chain = [{'service': s['service'], 'kind': s['kind'], 'name': s['name'], 'parent': by_id[s['parent']]['name'] if s['parent'] in by_id else None,
'ms': s['ms'], 'status': s['status'], **{k: s[k] for k in ('job.attempt', 'job.outcome') if k in s}}
for s in up_spans if s['kind'] in ('SERVER', 'PRODUCER', 'CONSUMER')]
effects = sorted({s['name'] for s in up_spans if s['service'] == 'messenger-worker' and s['kind'] == 'CLIENT' and (s['name'].startswith('S3.') or s['name'].split(' ')[0] in ('UPDATE', 'INSERT'))})
token = alice['token']
leaks = {'message_text_in_logs_or_traces': sum(x.count(marker) for x in (json.dumps(req_logs), req_raw)),
'session_token_in_logs_or_traces': sum(x.count(token) for x in (json.dumps(req_logs + up_logs), req_raw, up_raw))}
lq = urllib.parse.quote('{service_name=~"messenger-api|messenger-worker"} |= "%s"' % marker)
end = time.time_ns()
leaks['message_text_in_loki_all_logs'] = len(g.get('/api/datasources/proxy/uid/loki/loki/api/v1/query_range?query=%s&limit=10&start=%d&end=%d' % (lq, end - 900 * 10**9, end))['data']['result'])
result = {'controlled_request': {'trace_id': req_trace, 'request_id': request_id, 'spans': [{k: s[k] for k in ('service', 'kind', 'name', 'ms', 'status')} for s in req_spans], 'logs': req_logs},
'upload_with_retry': {'trace_id': up_trace, 'attachment_status': ready['status'], 'chain': chain, 'worker_effect_spans': effects,
'logs': [{k: l[k] for k in ('event', 'severity_text', 'line', 'span_id')} for l in up_logs]},
'leak_check': leaks}
out = ROOT / 'evidence/trace_chain.json'
out.write_text(json.dumps(result, ensure_ascii=False, indent=1))
print(json.dumps({'request_spans': len(req_spans), 'request_logs': len(req_logs), 'chain': [(x['kind'], x['name'], x.get('job.outcome')) for x in chain],
'effects': effects, 'leaks': leaks}, ensure_ascii=False))
check(req_spans and req_logs, 'controlled request: trace or log not found')
check(any(x.get('job.outcome') == 'retry' for x in chain) and any(x.get('job.outcome') == 'done' for x in chain), 'upload trace lacks retry→done chain')
check(not any(leaks.values()), 'telemetry leak: %s' % leaks)
print('PASS: trace/log correlation and upload chain ->', out.relative_to(ROOT))
def ttr(args):
"""Baseline time to ready under light load: N uploads one per second, client-side (request start → ready, poll 0.3 s)."""
c = Client(args.base_url)
owner, _, _, chat = c.bootstrap()
seconds, lock = [], threading.Lock()
def one():
start = time.monotonic()
att, _ = c.attachment(owner, chat)
c.wait_ready(owner, att, 120)
with lock:
seconds.append(round(time.monotonic() - start, 2))
threads = []
for _ in range(args.uploads):
threads.append(threading.Thread(target=one))
threads[-1].start()
time.sleep(1)
for t in threads:
t.join()
seconds.sort()
q = lambda f: seconds[min(len(seconds) - 1, int(f * len(seconds)))]
result = {'uploads': len(seconds), 'p50_s': q(0.5), 'p95_s': q(0.95), 'max_s': seconds[-1], 'over_10s': sum(x > 10 for x in seconds), 'all_s': seconds}
out = ROOT / 'evidence/ttr_baseline.json'
out.write_text(json.dumps(result, indent=1))
print({k: v for k, v in result.items() if k != 'all_s'}, '->', out.relative_to(ROOT))
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('mode', choices=['traces', 'slo', 'incidents', 'ttr'])
p.add_argument('--base-url', default=None)
p.add_argument('--grafana-url', default='http://localhost:3000')
p.add_argument('--rps', type=float, default=10)
p.add_argument('--baseline', type=int, default=300)
p.add_argument('--spike', type=int, default=15)
p.add_argument('--quiet', type=int, default=120)
p.add_argument('--degradation', type=int, default=360)
p.add_argument('--degradation-ratio', type=float, default=0.1)
p.add_argument('--outage', type=int, default=60)
p.add_argument('--worker-down', type=int, default=150)
p.add_argument('--db-lock', type=int, default=20)
p.add_argument('--telemetry-down', type=int, default=120)
p.add_argument('--uploads', type=int, default=30)
args = p.parse_args()
{'traces': traces, 'slo': slo, 'incidents': incidents, 'ttr': ttr}[args.mode](args)
if __name__ == '__main__':
run(main)