Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
409 lines
23 KiB
Python
409 lines
23 KiB
Python
#!/usr/bin/env python3
|
|
"""Lab6 experiments on a running stack: `slo` (track 2: burn-rate alerts vs a spike, a long degradation and
|
|
an outage) and `incidents` (level 4: worker stop, slow DB, telemetry backends down). Reads telemetry through
|
|
the Grafana datasource proxy (admin password from secrets/, never printed); drives faults with docker compose,
|
|
so run it with the same COMPOSE_PROJECT_NAME/COMPOSE_FILE as the stack. `traces` (level 3/4 trace+log correlation,
|
|
upload chain with one retry, leak check) and `slo` need FAULT_INJECTION=1."""
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from client import ROOT, Client, Failure, check, run
|
|
|
|
T0 = time.monotonic()
|
|
def now():
|
|
return round(time.monotonic() - T0, 1)
|
|
|
|
class Grafana:
|
|
def __init__(self, url):
|
|
password = (ROOT / 'secrets/grafana_admin_password').read_text().strip()
|
|
self.url = url.rstrip('/')
|
|
self.auth = 'Basic ' + base64.b64encode(('admin:' + password).encode()).decode()
|
|
|
|
def get(self, path):
|
|
req = urllib.request.Request(self.url + path, headers={'Authorization': self.auth})
|
|
with urllib.request.urlopen(req, timeout=20) as r:
|
|
return json.loads(r.read())
|
|
|
|
def prom(self, query):
|
|
return self.get('/api/datasources/proxy/uid/prometheus/api/v1/query?query=' + urllib.parse.quote(query))['data']['result']
|
|
|
|
def value(self, query):
|
|
result = self.prom(query)
|
|
return round(float(result[0]['value'][1]), 4) if result else None
|
|
|
|
def alerts(self):
|
|
data = self.get('/api/datasources/proxy/uid/prometheus/api/v1/alerts')['data']['alerts']
|
|
return {a['labels']['alertname']: a['state'] for a in data}
|
|
|
|
class AlertLog(threading.Thread):
|
|
"""Polls Prometheus alert states every 2 s and keeps transitions with experiment-relative time."""
|
|
def __init__(self, grafana):
|
|
super().__init__(daemon=True)
|
|
self.grafana, self.state, self.events, self.stop = grafana, {}, [], threading.Event()
|
|
|
|
def run(self):
|
|
while not self.stop.is_set():
|
|
try:
|
|
current = self.grafana.alerts()
|
|
except OSError:
|
|
current = self.state
|
|
for name in set(current) | set(self.state):
|
|
old, new = self.state.get(name, 'inactive'), current.get(name, 'inactive')
|
|
if old != new:
|
|
self.events.append({'t': now(), 'alert': name, 'from': old, 'to': new})
|
|
self.state = current
|
|
self.stop.wait(2)
|
|
|
|
def intervals(self, name):
|
|
"""Firing intervals [start, end) of one alert; end None = still firing."""
|
|
out, start = [], None
|
|
for e in self.events:
|
|
if e['alert'] != name:
|
|
continue
|
|
if e['to'] == 'firing':
|
|
start = e['t']
|
|
elif e['from'] == 'firing' and start is not None:
|
|
out.append([start, e['t']])
|
|
start = None
|
|
if start is not None:
|
|
out.append([start, None])
|
|
return out
|
|
|
|
class Load:
|
|
"""Paced POST-message load: `rps` total; share `fault` of requests carries X-Fault-Inject: http-500."""
|
|
def __init__(self, base_url, rps, threads=5):
|
|
self.client = Client(base_url)
|
|
self.alice, _, _, self.chat = self.client.bootstrap()
|
|
self.fault, self.phase, self.stop = 0.0, 'warmup', threading.Event()
|
|
self.counts, self.lock = {}, threading.Lock()
|
|
self.interval = threads / rps
|
|
self.threads = [threading.Thread(target=self.worker, args=(Client(base_url, timeout=5),), daemon=True) for _ in range(threads)]
|
|
|
|
def start(self):
|
|
for t in self.threads:
|
|
t.start()
|
|
|
|
def worker(self, client):
|
|
due, debt, n = time.monotonic(), 0.0, 0
|
|
path = '/api/v1/chats/' + self.chat['id'] + '/messages'
|
|
while not self.stop.is_set():
|
|
debt += self.fault
|
|
inject = debt >= 1
|
|
if inject:
|
|
debt -= 1
|
|
headers = {'Idempotency-Key': 'obs-%d-%d-%f' % (id(client), n, time.time())}
|
|
if inject:
|
|
headers['X-Fault-Inject'] = 'http-500'
|
|
try:
|
|
status = client.call('POST', path, [201, 500], body={'text': 'obs %d' % n}, actor=self.alice, headers=headers)['status']
|
|
outcome = 'injected_500' if status == 500 and inject else ('ok' if status == 201 else 'unexpected_500')
|
|
except Failure:
|
|
outcome = 'failed'
|
|
with self.lock:
|
|
bucket = self.counts.setdefault(self.phase, {})
|
|
bucket[outcome] = bucket.get(outcome, 0) + 1
|
|
n += 1
|
|
due += self.interval
|
|
time.sleep(max(0.0, due - time.monotonic()))
|
|
|
|
def compose(*args, check_rc=True):
|
|
return subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True, check=check_rc).stdout
|
|
|
|
def slo(args):
|
|
g = Grafana(args.grafana_url)
|
|
probe = Client(args.base_url)
|
|
a, _, _, chat = probe.bootstrap()
|
|
status = probe.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', [201, 500], body={'text': 'probe'}, actor=a,
|
|
headers={'Idempotency-Key': 'probe-%f' % time.time(), 'X-Fault-Inject': 'http-500'})['status']
|
|
check(status == 500, 'fault injection is off: start the stack with FAULT_INJECTION=1')
|
|
check(wait_alerts(g, list(g.alerts()), False, 600) is not None, 'alerts still active before the experiment')
|
|
phases = [('baseline', args.baseline, 0.0), ('spike', args.spike, 1.0), ('after_spike', args.quiet, 0.0),
|
|
('degradation', args.degradation, args.degradation_ratio), ('after_degradation', args.quiet + 30, 0.0),
|
|
('outage', args.outage, 1.0), ('after_outage', args.quiet, 0.0)]
|
|
load = Load(args.base_url, args.rps)
|
|
watcher = AlertLog(g)
|
|
load.start()
|
|
watcher.start()
|
|
timeline, samples = [], []
|
|
for name, seconds, ratio in phases:
|
|
load.phase, load.fault = name, ratio
|
|
start = now()
|
|
print('%7.1fs phase %-18s %4ds fault=%.2f' % (start, name, seconds, ratio), flush=True)
|
|
while now() - start < seconds:
|
|
time.sleep(10)
|
|
samples.append({'t': now(), 'phase': name, **{w: g.value('slo:api_errors:ratio_rate' + w) for w in ('30s', '1m', '1m30s', '2m30s', '15m')}})
|
|
timeline.append({'phase': name, 'start': start, 'end': now(), 'fault_ratio': ratio})
|
|
load.stop.set()
|
|
watcher.stop.set()
|
|
watcher.join()
|
|
alerts = {name: watcher.intervals(name) for name in ('ApiAvailabilityFastBurn', 'ApiAvailabilitySlowBurn', 'ApiHighErrorRate')}
|
|
expected = {'ApiAvailabilityFastBurn': {'outage'}, 'ApiAvailabilitySlowBurn': {'degradation', 'outage'},
|
|
'ApiHighErrorRate': {'degradation', 'outage'}} # по политике SLO: всплеск < 2 % бюджета — не повод для реакции
|
|
phase_of = lambda t: next((p['phase'] for p in timeline if p['start'] <= t < p['end']), 'end')
|
|
fault_phase = {'spike', 'degradation', 'outage'}
|
|
analysis = {}
|
|
for name, spans in alerts.items():
|
|
rows = []
|
|
for start, end in spans:
|
|
cause = phase_of(start)
|
|
cause = cause.replace('after_', '') if cause.startswith('after_') else cause
|
|
origin = next(p for p in timeline if p['phase'] == cause) if cause in fault_phase else None
|
|
fault_end = origin['end'] if origin else None
|
|
rows.append({'fired_at': start, 'resolved_at': end, 'caused_by': cause,
|
|
'detect_s': round(start - origin['start'], 1) if origin else None,
|
|
'resolve_after_fault_end_s': round(end - fault_end, 1) if origin and end else None,
|
|
'expected': cause in expected[name]})
|
|
missed = sorted(expected[name] - {r['caused_by'] for r in rows})
|
|
analysis[name] = {'firings': rows, 'false_positives': sum(not r['expected'] for r in rows), 'missed': missed}
|
|
total = {k: sum(c.get(k, 0) for c in load.counts.values()) for k in ('ok', 'injected_500', 'unexpected_500', 'failed')}
|
|
result = {'design': {'rps': args.rps, 'slo': '99% non-5xx', 'compression': 24, 'phases': timeline},
|
|
'requests': load.counts, 'totals': total, 'alerts': analysis, 'events': watcher.events, 'ratios': samples}
|
|
out = ROOT / 'evidence/slo_experiment.json'
|
|
out.write_text(json.dumps(result, ensure_ascii=False, indent=1))
|
|
for name, a in analysis.items():
|
|
print(name, 'FP=%d missed=%s' % (a['false_positives'], a['missed']),
|
|
['%s: detect %ss, resolve +%ss' % (r['caused_by'], r['detect_s'], r['resolve_after_fault_end_s']) for r in a['firings']])
|
|
print('requests', total, '->', out.relative_to(ROOT))
|
|
check(total['unexpected_500'] == 0 and total['failed'] <= 3, 'unexpected request failures: %s' % total)
|
|
|
|
def wait_alerts(g, names, firing, timeout):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
active = {k for k, v in g.alerts().items() if v == 'firing'}
|
|
if (set(names) <= active) if firing else not (set(names) & set(g.alerts())):
|
|
return now()
|
|
time.sleep(2)
|
|
return None
|
|
|
|
def incidents(args):
|
|
g = Grafana(args.grafana_url)
|
|
report = {}
|
|
load = Load(args.base_url, 5, threads=2)
|
|
uploader = Client(args.base_url, timeout=10)
|
|
owner, _, _, chat = uploader.bootstrap()
|
|
uploads = {'accepted': 0, 'failed': 0}
|
|
stop_uploads = threading.Event()
|
|
def upload_loop():
|
|
while not stop_uploads.is_set():
|
|
try:
|
|
uploader.attachment(owner, chat)
|
|
uploads['accepted'] += 1
|
|
except Failure:
|
|
uploads['failed'] += 1
|
|
stop_uploads.wait(2)
|
|
watcher = AlertLog(g)
|
|
load.start()
|
|
watcher.start()
|
|
threading.Thread(target=upload_loop, daemon=True).start()
|
|
time.sleep(40)
|
|
|
|
# 1. Worker stopped: backlog grows, QueueLagHigh/WorkerAbsent fire; after start backlog drains, alerts resolve.
|
|
load.phase = 'worker_stopped'
|
|
t_stop = now()
|
|
compose('stop', 'worker')
|
|
queue = []
|
|
while now() - t_stop < args.worker_down:
|
|
time.sleep(10)
|
|
queue.append({'t': now(), 'depth': g.value('max(messenger_queue_depth)'), 'oldest_age_s': g.value('max(messenger_queue_oldest_age_seconds)')})
|
|
load.phase = 'worker_recovering'
|
|
t_start = now()
|
|
compose('start', 'worker')
|
|
drained = None
|
|
while now() - t_start < 300:
|
|
time.sleep(5)
|
|
depth = g.value('max(messenger_queue_depth)')
|
|
queue.append({'t': now(), 'depth': depth, 'oldest_age_s': g.value('max(messenger_queue_oldest_age_seconds)')})
|
|
if drained is None and depth == 0 and now() - t_start > 20:
|
|
drained = now()
|
|
if drained and wait_alerts(g, ['QueueLagHigh', 'WorkerAbsent'], False, 1):
|
|
break
|
|
report['worker_stop'] = {'stopped_at': t_stop, 'started_at': t_start, 'backlog_drained_at': drained, 'queue': queue}
|
|
print('worker stop: drained at', drained, json.dumps({n: watcher.intervals(n) for n in ('QueueLagHigh', 'WorkerAbsent')}), flush=True)
|
|
|
|
# 2. Slow DB: an exclusive lock on messages for N s; POST /messages waits in the INSERT span, not in the handler.
|
|
load.phase = 'db_locked'
|
|
t_lock = now()
|
|
compose('exec', '-T', 'postgres', 'psql', '-q', '-U', 'messenger', '-d', 'messenger', '-c',
|
|
'begin; lock table messages in access exclusive mode; select pg_sleep(%d); commit;' % args.db_lock)
|
|
load.phase = 'db_recovered'
|
|
time.sleep(25)
|
|
end_ns, start_ns = time.time_ns(), time.time_ns() - int((now() - t_lock + 30) * 1e9)
|
|
search = g.get('/api/datasources/proxy/uid/tempo/api/search?' + urllib.parse.urlencode(
|
|
{'tags': 'service.name=messenger-api', 'minDuration': '%ds' % max(2, args.db_lock // 2), 'limit': 5,
|
|
'start': start_ns // 10**9, 'end': end_ns // 10**9}))
|
|
slow = []
|
|
for t in search.get('traces', [])[:3]:
|
|
trace = g.get('/api/datasources/proxy/uid/tempo/api/traces/' + t['traceID'])
|
|
spans = [s for b in trace.get('batches') or trace.get('resourceSpans', []) for ss in b.get('scopeSpans', []) for s in ss['spans']]
|
|
dur = lambda s: (int(s['endTimeUnixNano']) - int(s['startTimeUnixNano'])) / 1e9
|
|
server = max((s for s in spans if s.get('kind') in ('SPAN_KIND_SERVER', 2)), key=dur, default=None)
|
|
db = max((s for s in spans if s.get('kind') in ('SPAN_KIND_CLIENT', 3) and any(a['key'].startswith('db.') for a in s.get('attributes', []))), key=dur, default=None)
|
|
slow.append({'trace_id': t['traceID'], 'root': t.get('rootTraceName'), 'server_s': server and round(dur(server), 2),
|
|
'slowest_db_span': db and db['name'], 'db_s': db and round(dur(db), 2)})
|
|
report['db_lock'] = {'locked_at': t_lock, 'lock_s': args.db_lock, 'slow_traces': slow,
|
|
'p99_messages_s_2m': g.value('histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job="messenger-api",http_route="/api/v1/chats/{id}/messages",http_request_method="POST"}[2m])))'),
|
|
'max_pending_connections_2m': g.value('max(max_over_time(db_client_connections_pending_requests[2m]))'),
|
|
'requests': {k: load.counts.get(k) for k in ('db_locked', 'db_recovered')}}
|
|
print('db lock:', json.dumps(slow), flush=True)
|
|
|
|
# 3. Telemetry backends down: API keeps serving; Collector memory stays bounded; losses are counted.
|
|
load.phase = 'telemetry_down'
|
|
# Собственные метрики Collector 0.115 приходят без суффикса _total; счётчик появляется при первой ошибке: нет = 0.
|
|
lost = lambda k: sum(g.value('sum(otelcol_exporter_%s_%s)' % (kind, k)) or 0 for kind in ('send_failed', 'enqueue_failed'))
|
|
before = {k: lost(k) for k in ('spans', 'log_records')}
|
|
compose('stop', 'loki', 'tempo')
|
|
t_down, memory = now(), []
|
|
while now() - t_down < args.telemetry_down:
|
|
time.sleep(10)
|
|
stats = compose('stats', '--no-stream', '--format', '{{.Name}} {{.MemUsage}}', 'otel-collector', check_rc=False).strip()
|
|
memory.append({'t': now(), 'collector_mem': stats.split(' ', 1)[-1],
|
|
'queue_batches': g.value('sum(otelcol_exporter_queue_size)'),
|
|
'refused_spans': g.value('sum(otelcol_receiver_refused_spans)')})
|
|
compose('start', 'loki', 'tempo')
|
|
load.phase = 'telemetry_recovered'
|
|
time.sleep(75)
|
|
after = {k: lost(k) for k in ('spans', 'log_records')}
|
|
report['telemetry_down'] = {'down_s': args.telemetry_down, 'collector': memory, 'lost_before': before, 'lost_after': after,
|
|
'requests': {k: load.counts.get(k) for k in ('telemetry_down', 'telemetry_recovered')}}
|
|
print('telemetry down:', json.dumps(memory[-1]), 'lost', before, '->', after, flush=True)
|
|
|
|
stop_uploads.set()
|
|
load.stop.set()
|
|
watcher.stop.set()
|
|
time.sleep(2)
|
|
report['alerts'] = {n: watcher.intervals(n) for n in sorted({e['alert'] for e in watcher.events})}
|
|
report['uploads'] = uploads
|
|
report['requests'] = load.counts
|
|
report['events'] = watcher.events
|
|
out = ROOT / 'evidence/incidents.json'
|
|
out.write_text(json.dumps(report, ensure_ascii=False, indent=1))
|
|
print('requests', load.counts, 'uploads', uploads, '->', out.relative_to(ROOT))
|
|
|
|
def trace_tree(g, trace_id, wait=60):
|
|
"""Spans of one trace from Tempo (waits for ingestion): service, kind, name, parent, status, key attributes."""
|
|
deadline = time.monotonic() + wait
|
|
while True:
|
|
try:
|
|
data = g.get('/api/datasources/proxy/uid/tempo/api/traces/' + trace_id)
|
|
except OSError:
|
|
data = {}
|
|
spans = []
|
|
for b in data.get('batches') or data.get('resourceSpans', []):
|
|
service = next((a['value'].get('stringValue') for a in b['resource']['attributes'] if a['key'] == 'service.name'), '?')
|
|
for ss in b.get('scopeSpans', []):
|
|
for sp in ss['spans']:
|
|
attrs = {a['key']: next(iter(a['value'].values()), None) for a in sp.get('attributes', [])}
|
|
spans.append({'service': service, 'kind': str(sp.get('kind', '')).replace('SPAN_KIND_', ''), 'name': sp['name'],
|
|
'span_id': sp['spanId'], 'parent': sp.get('parentSpanId') or None,
|
|
'start_ns': int(sp['startTimeUnixNano']), 'ms': round((int(sp['endTimeUnixNano']) - int(sp['startTimeUnixNano'])) / 1e6, 1),
|
|
'status': sp.get('status', {}).get('code', 'UNSET'),
|
|
**{k: attrs[k] for k in ('job.attempt', 'job.outcome', 'http.response.status_code', 'messaging.destination.name') if k in attrs}})
|
|
if any(s['kind'] == 'CONSUMER' and s.get('job.outcome') == 'done' for s in spans) or time.monotonic() > deadline:
|
|
return sorted(spans, key=lambda s: s['start_ns']), json.dumps(data)
|
|
time.sleep(5)
|
|
|
|
def logs_for(g, service, trace_id):
|
|
end = time.time_ns()
|
|
q = urllib.parse.quote('{service_name="%s"} | trace_id="%s"' % (service, trace_id))
|
|
data = g.get('/api/datasources/proxy/uid/loki/loki/api/v1/query_range?query=%s&limit=50&start=%d&end=%d&direction=forward' % (q, end - 900 * 10**9, end))
|
|
return [{'ts': v[0], 'line': v[1][:160], **{k: s['stream'].get(k) for k in ('event', 'severity_text', 'trace_id', 'span_id', 'request_id', 'http_route', 'http_response_status_code')}}
|
|
for s in data['data']['result'] for v in s['values']]
|
|
|
|
def traces(args):
|
|
"""Level 3/4 evidence: a controlled request → its trace + log; one upload with one retry → one trace to S3/DB."""
|
|
import os
|
|
g = Grafana(args.grafana_url)
|
|
c = Client(args.base_url)
|
|
alice, _, _, chat = c.bootstrap()
|
|
tid = lambda: os.urandom(16).hex()
|
|
marker = 'leak-marker-' + os.urandom(6).hex()
|
|
req_trace = tid()
|
|
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 201, body={'text': marker}, actor=alice, schema='Message',
|
|
headers={'Idempotency-Key': os.urandom(8).hex(), 'traceparent': '00-%s-%s-01' % (req_trace, os.urandom(8).hex())})
|
|
request_id = r['headers'].get('X-Request-Id')
|
|
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
up_trace = tid()
|
|
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=fault-flaky-trace.png', 202, actor=alice, raw=content, schema='Attachment',
|
|
headers={'Content-Type': 'image/png', 'traceparent': '00-%s-%s-01' % (up_trace, os.urandom(8).hex())})['json']
|
|
ready = c.wait_ready(alice, att, 90)
|
|
time.sleep(15)
|
|
req_spans, req_raw = trace_tree(g, req_trace, 30)
|
|
up_spans, up_raw = trace_tree(g, up_trace, 90)
|
|
req_logs = logs_for(g, 'messenger-api', req_trace)
|
|
up_logs = logs_for(g, 'messenger-api', up_trace) + logs_for(g, 'messenger-worker', up_trace)
|
|
by_id = {s['span_id']: s for s in up_spans}
|
|
chain = [{'service': s['service'], 'kind': s['kind'], 'name': s['name'], 'parent': by_id[s['parent']]['name'] if s['parent'] in by_id else None,
|
|
'ms': s['ms'], 'status': s['status'], **{k: s[k] for k in ('job.attempt', 'job.outcome') if k in s}}
|
|
for s in up_spans if s['kind'] in ('SERVER', 'PRODUCER', 'CONSUMER')]
|
|
effects = sorted({s['name'] for s in up_spans if s['service'] == 'messenger-worker' and s['kind'] == 'CLIENT' and (s['name'].startswith('S3.') or s['name'].split(' ')[0] in ('UPDATE', 'INSERT'))})
|
|
token = alice['token']
|
|
leaks = {'message_text_in_logs_or_traces': sum(x.count(marker) for x in (json.dumps(req_logs), req_raw)),
|
|
'session_token_in_logs_or_traces': sum(x.count(token) for x in (json.dumps(req_logs + up_logs), req_raw, up_raw))}
|
|
lq = urllib.parse.quote('{service_name=~"messenger-api|messenger-worker"} |= "%s"' % marker)
|
|
end = time.time_ns()
|
|
leaks['message_text_in_loki_all_logs'] = len(g.get('/api/datasources/proxy/uid/loki/loki/api/v1/query_range?query=%s&limit=10&start=%d&end=%d' % (lq, end - 900 * 10**9, end))['data']['result'])
|
|
result = {'controlled_request': {'trace_id': req_trace, 'request_id': request_id, 'spans': [{k: s[k] for k in ('service', 'kind', 'name', 'ms', 'status')} for s in req_spans], 'logs': req_logs},
|
|
'upload_with_retry': {'trace_id': up_trace, 'attachment_status': ready['status'], 'chain': chain, 'worker_effect_spans': effects,
|
|
'logs': [{k: l[k] for k in ('event', 'severity_text', 'line', 'span_id')} for l in up_logs]},
|
|
'leak_check': leaks}
|
|
out = ROOT / 'evidence/trace_chain.json'
|
|
out.write_text(json.dumps(result, ensure_ascii=False, indent=1))
|
|
print(json.dumps({'request_spans': len(req_spans), 'request_logs': len(req_logs), 'chain': [(x['kind'], x['name'], x.get('job.outcome')) for x in chain],
|
|
'effects': effects, 'leaks': leaks}, ensure_ascii=False))
|
|
check(req_spans and req_logs, 'controlled request: trace or log not found')
|
|
check(any(x.get('job.outcome') == 'retry' for x in chain) and any(x.get('job.outcome') == 'done' for x in chain), 'upload trace lacks retry→done chain')
|
|
check(not any(leaks.values()), 'telemetry leak: %s' % leaks)
|
|
print('PASS: trace/log correlation and upload chain ->', out.relative_to(ROOT))
|
|
|
|
def ttr(args):
|
|
"""Baseline time to ready under light load: N uploads one per second, client-side (request start → ready, poll 0.3 s)."""
|
|
c = Client(args.base_url)
|
|
owner, _, _, chat = c.bootstrap()
|
|
seconds, lock = [], threading.Lock()
|
|
def one():
|
|
start = time.monotonic()
|
|
att, _ = c.attachment(owner, chat)
|
|
c.wait_ready(owner, att, 120)
|
|
with lock:
|
|
seconds.append(round(time.monotonic() - start, 2))
|
|
threads = []
|
|
for _ in range(args.uploads):
|
|
threads.append(threading.Thread(target=one))
|
|
threads[-1].start()
|
|
time.sleep(1)
|
|
for t in threads:
|
|
t.join()
|
|
seconds.sort()
|
|
q = lambda f: seconds[min(len(seconds) - 1, int(f * len(seconds)))]
|
|
result = {'uploads': len(seconds), 'p50_s': q(0.5), 'p95_s': q(0.95), 'max_s': seconds[-1], 'over_10s': sum(x > 10 for x in seconds), 'all_s': seconds}
|
|
out = ROOT / 'evidence/ttr_baseline.json'
|
|
out.write_text(json.dumps(result, indent=1))
|
|
print({k: v for k, v in result.items() if k != 'all_s'}, '->', out.relative_to(ROOT))
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('mode', choices=['traces', 'slo', 'incidents', 'ttr'])
|
|
p.add_argument('--base-url', default=None)
|
|
p.add_argument('--grafana-url', default='http://localhost:3000')
|
|
p.add_argument('--rps', type=float, default=10)
|
|
p.add_argument('--baseline', type=int, default=300)
|
|
p.add_argument('--spike', type=int, default=15)
|
|
p.add_argument('--quiet', type=int, default=120)
|
|
p.add_argument('--degradation', type=int, default=360)
|
|
p.add_argument('--degradation-ratio', type=float, default=0.1)
|
|
p.add_argument('--outage', type=int, default=60)
|
|
p.add_argument('--worker-down', type=int, default=150)
|
|
p.add_argument('--db-lock', type=int, default=20)
|
|
p.add_argument('--telemetry-down', type=int, default=120)
|
|
p.add_argument('--uploads', type=int, default=30)
|
|
args = p.parse_args()
|
|
{'traces': traces, 'slo': slo, 'incidents': incidents, 'ttr': ttr}[args.mode](args)
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|