Files
mtusi2026lab6/scripts/lab6/incidents.py

677 lines
32 KiB
Python
Executable File

#!/usr/bin/env python3
"""lab6 controlled incidents: worker | db | telemetry. Stdlib only.
Every scenario runs inside obs.app_environment (SDK on, sampling 1,
FAULT_INJECTION_ENABLED=1) and always undoes its own faults/locks/stopped
services in `finally`. Times are UTC + monotonic offsets; an unobserved
transition is null with a reason, never a guess."""
import argparse
import calendar
import subprocess
import sys
import threading
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import measurement # noqa: E402
import obs # noqa: E402
import reconcile # noqa: E402
import resources # noqa: E402
import verify # noqa: E402
import workload # noqa: E402
from obs import Failure, stand, utc_now # noqa: E402
ROOT = obs.ROOT
def utc_epoch(text):
return calendar.timegm(time.strptime(text[:19], '%Y-%m-%dT%H:%M:%S'))
ALERTS = ('MessengerApiHighErrorRate', 'MessengerJobsQueueLagGrowing')
ENV = {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1',
'FAULT_INJECTION_ENABLED': '1', 'LOG_LEVEL': 'INFO'}
class Ctx:
def __init__(self, args):
self.args = args
self.obs = verify.make_obs(args)
self.out = Path(args.output_dir)
self.t0 = time.monotonic()
def mono(self):
return round(time.monotonic() - self.t0, 2)
def mark(self, timeline, key, extra=None):
timeline[key] = {'utc': utc_now(), 'mono_s': self.mono()}
if extra:
timeline[key].update(extra)
print('[%s] %s' % (timeline[key]['utc'], key), flush=True)
def actor_chat(self):
c = stand.new_client(self.args.base_url, self.args.ca_file)
alice, _, _, chat = c.bootstrap()
return alice, chat
def alert_state(ctx, name):
result = ctx.obs.prom_query('ALERTS{alertname="%s"}' % name)
states = {r['metric'].get('alertstate') for r in result}
if 'firing' in states:
return 'firing'
return 'pending' if 'pending' in states else None
def wait_state(ctx, name, want, timeout, timeline, polls):
"""want: 'firing' or None (resolved). Returns True when reached."""
end = time.monotonic() + timeout
while time.monotonic() < end:
state = alert_state(ctx, name)
polls.append({'utc': utc_now(), 'mono_s': ctx.mono(), 'alert': name, 'state': state})
if state == 'pending' and 'pending' not in timeline:
ctx.mark(timeline, 'pending')
if state == want:
return True
time.sleep(5)
return False
class BackgroundLoad:
def __init__(self, ctx, actor, chat, rate, upload_every, phase):
self.stop_event = threading.Event()
self.result = None
self.error = None
def body():
try:
self.result = workload.run_window(
ctx.args.base_url, ctx.args.ca_file, actor, chat, rate, 1800,
phase, upload_every=upload_every, stop_event=self.stop_event,
settle=False)
except Exception as exc: # reported by stop()
self.error = exc
self.thread = threading.Thread(target=body, daemon=True)
self.thread.start()
def stop(self):
self.stop_event.set()
self.thread.join(timeout=120)
if self.error:
raise self.error
if self.result is None:
raise Failure('background load did not finish')
return self.result
def baseline(ctx, actor, chat, rate, upload_every):
ctx.obs.require_no_foreign_faults()
for name in ALERTS:
if alert_state(ctx, name) is not None:
raise Failure('alert already active before baseline: ' + name)
result = workload.run_window(ctx.args.base_url, ctx.args.ca_file, actor, chat,
rate, 60, 'baseline', upload_every=upload_every,
settle=False)
drain = obs.wait_drained(result['uploads'], obs=ctx.obs)
for name in ALERTS:
if alert_state(ctx, name) is not None:
raise Failure('alert active at the end of baseline: ' + name)
if not drain['drained']:
raise Failure('baseline jobs did not drain')
return {'load': result['load'], 'drain': drain, 'uploads': result['uploads']}
def series(ctx, expr, start, end, step=10):
out = []
for item in ctx.obs.prom_range(expr, start, end, step):
out.append({'labels': item['metric'],
'points': [[float(t), v] for t, v in item['values']]})
return out
def peak(rows):
values = [float(v) for r in rows for _, v in r['points'] if v != 'NaN']
return max(values) if values else None
def ready_seconds(ids):
"""max/median finished_at - created_at of the jobs of these attachments."""
if not ids:
return {'n': 0, 'max': None, 'median': None}
values = []
for i in range(0, len(ids), 200):
chunk = ids[i:i + 200]
arr = "ARRAY[" + ",".join("'" + x + "'" for x in chunk) + "]::uuid[]"
out = obs.psql(
"SELECT COALESCE(string_agg(extract(epoch from (finished_at - created_at))::text, ';'),'') "
"FROM attachment_jobs WHERE status='succeeded' AND attachment_id = ANY(" + arr + ")")
values += [float(x) for x in out.split(';') if x]
return {'n': len(values), 'max': max(values) if values else None,
'median': measurement.median(values)}
def finish(ctx, name, result):
result['passed'] = not result['problems']
verify.write_json(ctx.out, name, result)
if result['problems']:
raise Failure(name + ': ' + '; '.join(result['problems']))
print('PASS: ' + name)
# ------------------------------------------------------------------- worker
def scenario_worker(ctx, label=''):
alice, chat = ctx.actor_chat()
result = {'scenario': 'worker', 'profile': {'rate': 20, 'upload_every': 20},
'timeline': {}, 'problems': [], 'polls': []}
tl = result['timeline']
base = baseline(ctx, alice, chat, 20, 20)
result['baseline'] = {'load': base['load'], 'ready_seconds': ready_seconds(base['uploads'])}
load = None
worker_stopped = False
try:
load = BackgroundLoad(ctx, alice, chat, 20, 20, 'incident')
time.sleep(10)
ctx.mark(tl, 'inject')
stand.compose('stop', 'worker')
worker_stopped = True
fired = wait_state(ctx, 'MessengerJobsQueueLagGrowing', 'firing', 150, tl, result['polls'])
if fired:
ctx.mark(tl, 'firing')
else:
tl['firing'] = None
result['problems'].append('queue-lag alert did not fire within 150 s')
# Capture the evidence while the backlog exists.
t_now = time.time()
result['queue_at_firing'] = {
'oldest_age_max': ctx.obs.prom_scalar('max(messenger_jobs_queue_oldest_age_seconds)'),
'depth_ready_max': ctx.obs.prom_scalar('max(messenger_jobs_queue_depth{state="ready"})'),
'depth_ready_sum_over_replicas': ctx.obs.prom_scalar('sum(messenger_jobs_queue_depth{state="ready"})'),
'depth_ready_per_replica': {
r['metric']['instance']: float(r['value'][1])
for r in ctx.obs.prom_query('messenger_jobs_queue_depth{state="ready"}')},
'outbox_pending_max': ctx.obs.prom_scalar('max(messenger_outbox_pending)'),
}
q = result['queue_at_firing']
per = list(q['depth_ready_per_replica'].values())
if len(per) >= 2 and q['depth_ready_max'] != max(per):
result['problems'].append('max() does not equal the per-replica snapshot')
if len(per) >= 2 and q['depth_ready_sum_over_replicas'] == q['depth_ready_max'] and per[0] > 0:
result['problems'].append('replicas not both reporting (cannot show non-doubling)')
finally:
ctx.mark(tl, 'recover_start')
if worker_stopped:
stand.compose('up', '-d', '--no-deps', '--wait', 'worker')
ctx.mark(tl, 'worker_back')
incident_load = load.stop() if load else None
ctx.mark(tl, 'load_stopped')
ids = incident_load['uploads']
result['incident_load'] = incident_load['load']
drain = obs.wait_drained(ids + base['uploads'], obs=ctx.obs, timeout=180)
result['drain'] = drain
ctx.mark(tl, 'backlog_drained' if drain['drained'] else 'drain_deadline')
if not drain['drained']:
tl['backlog_drained'] = None
result['problems'].append('backlog not drained within 180 s')
elif any(s != 'ready' for s in drain['states'].values()):
result['problems'].append('some accepted attachments are not ready')
resolved = wait_state(ctx, 'MessengerJobsQueueLagGrowing', None, 180, tl, result['polls'])
if resolved:
ctx.mark(tl, 'resolved')
else:
tl['resolved'] = None
result['problems'].append('queue-lag alert not resolved within 180 s')
end = time.time()
t_inject = utc_epoch(tl['inject']['utc'])
window = (t_inject - 60, end)
result['queries'] = {
'oldest_age': 'max(messenger_jobs_queue_oldest_age_seconds)',
'depth': 'max by (state)(messenger_jobs_queue_depth)',
'outbox': 'max(messenger_outbox_pending)',
'worker_active': 'sum(messenger_worker_jobs_active)'}
result['series'] = {k: series(ctx, v, *window) for k, v in result['queries'].items()}
peak_age = peak(result['series']['oldest_age'])
result['peak_oldest_age_s'] = peak_age
if peak_age is None or peak_age < 30:
result['problems'].append('queue age did not grow past 30 s')
result['incident_ready_seconds'] = ready_seconds(ids)
if (result['incident_ready_seconds']['max'] or 0) <= (result['baseline']['ready_seconds']['max'] or 0):
result['problems'].append('time-to-ready did not exceed the baseline')
http_ok = result['incident_load']['http_5xx'] == 0 and \
sum(result['incident_load']['transport_failures'].values()) == 0
result['http_acceptance_continued'] = http_ok
if not http_ok:
result['problems'].append('HTTP acceptance degraded during worker outage')
finish(ctx, 'incident-worker.json', result)
# ----------------------------------------------------------------------- db
def trace_gap(spans, server):
"""Server duration minus the union of its direct child intervals (ms)."""
children = sorted((s['start_ns'], s['end_ns']) for s in spans
if s['parent_span_id'] == server['span_id'])
# Descendants of the transaction span are covered by their ancestor.
covered, last = 0, None
for start, end in children:
if last is None or start > last:
covered += end - start
last = end
elif end > last:
covered += end - last
last = end
return ((server['end_ns'] - server['start_ns']) - covered) / 1e6
def db_time_ms(spans, server):
return sum((s['end_ns'] - s['start_ns']) for s in spans
if s['kind'] == 'CLIENT' and 'db.operation.name' in s['attributes']
and s['parent_span_id'] == server['span_id']) / 1e6
def sequential_uploads(ctx, actor, chat, count, delay):
c = stand.new_client(ctx.args.base_url, ctx.args.ca_file)
c.timeout = 30
key = 'fault:api.after_commit_before_publish'
rows = []
for _ in range(count):
if delay:
stand.set_fault(key, 'delay:%d' % delay)
started = time.monotonic()
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
range(200, 600), actor=actor, raw=stand.sample_bytes(),
headers={'Content-Type': 'image/png'}, schema='Attachment')
elapsed = (time.monotonic() - started) * 1000
consumed = True
if delay:
out = stand.compose('exec', '-T', 'redis', 'redis-cli', 'GET', key).stdout.strip()
consumed = out == ''
rows.append({'request_id': r['headers']['X-Request-Id'], 'status': r['status'],
'elapsed_ms': elapsed, 'consumed': consumed,
'attachment_id': r['json']['id'] if r['status'] == 202 else None})
return rows
def server_view(ctx, request_id):
log = verify.http_log(ctx.obs, request_id)
tid, sid = log['labels']['trace_id'], log['labels']['span_id']
spans = verify.trace_with(ctx.obs, tid, lambda sp: any(
s['span_id'] == sid for s in sp), description='server trace')
server = next(s for s in spans if s['span_id'] == sid)
return spans, server
def scenario_db(ctx, label=''):
alice, chat = ctx.actor_chat()
result = {'scenario': 'db', 'profile': {'rate': 2, 'upload_every': 1},
'timeline': {}, 'problems': [], 'polls': []}
tl = result['timeline']
base = baseline(ctx, alice, chat, 2, 1)
result['baseline'] = {'load': base['load']}
load = None
lock_released = False
try:
load = BackgroundLoad(ctx, alice, chat, 2, 1, 'incident')
time.sleep(10)
# Own client: 30 s timeout is set inside workload (REQUEST_TIMEOUT).
with obs.hold_table_lock('attachments') as pid:
ctx.mark(tl, 'inject', {'lock_backend_pid': pid})
fired = wait_state(ctx, 'MessengerApiHighErrorRate', 'firing', 120, tl, result['polls'])
if fired:
ctx.mark(tl, 'firing')
else:
tl['firing'] = None
result['problems'].append('error-rate alert did not fire within 120 s')
result['at_firing'] = {
'api_5xx_rate_1m': ctx.obs.prom_scalar('messenger:api_errors:rate1m'),
'api_request_rate_1m': ctx.obs.prom_scalar('messenger:api_requests:rate1m'),
'haproxy_5xx_rate_1m': ctx.obs.prom_scalar(
'sum(rate(haproxy_backend_http_responses_total{proxy="api",code="5xx"}[1m]))', 0.0),
'db_insert_attachments_p95_s': ctx.obs.prom_scalar(
'histogram_quantile(0.95, sum by (le)(rate(db_client_operation_duration_seconds_bucket{db_operation_name="INSERT"}[1m])))'),
'pool_pending': ctx.obs.prom_scalar('sum(db_client_connection_pending_requests)'),
'pool_used': ctx.obs.prom_scalar('sum(db_client_connection_count{db_client_connection_state="used"})'),
}
lock_released = True
ctx.mark(tl, 'lock_released')
finally:
if not lock_released:
ctx.mark(tl, 'lock_released_in_finally')
incident_load = load.stop() if load else None
ctx.mark(tl, 'load_stopped')
result['incident_load'] = incident_load['load']
if incident_load['load']['http_5xx'] == 0:
result['problems'].append('no real server 5xx observed in the load')
ids = incident_load['uploads']
drain = obs.wait_drained(ids + base['uploads'], obs=ctx.obs, timeout=180)
result['drain'] = drain
if drain['drained']:
ctx.mark(tl, 'backlog_drained')
else:
tl['backlog_drained'] = None
result['problems'].append('jobs did not drain after recovery')
if wait_state(ctx, 'MessengerApiHighErrorRate', None, 120, tl, result['polls']):
ctx.mark(tl, 'resolved')
else:
tl['resolved'] = None
result['problems'].append('error-rate alert not resolved within 120 s')
# -- localisation: a 5xx trace where the time went to the DB.
inject_ns = int(utc_epoch(tl['inject']['utc']) - 5) * 10 ** 9
rows = verify.wait_rows(
ctx.obs, '{service_name="messenger-api"} | event_name="http.request" '
'| http_response_status_code=~"5.."', timeout=60, back=900)
rows = [r for r in rows if r['ts'] >= inject_ns]
if rows:
row = rows[0]
spans = verify.trace_with(ctx.obs, row['labels']['trace_id'], lambda sp: any(
s['span_id'] == row['labels']['span_id'] for s in sp), description='5xx trace')
server = next(s for s in spans if s['span_id'] == row['labels']['span_id'])
dur = (server['end_ns'] - server['start_ns']) / 1e6
db_ms = db_time_ms(spans, server)
result['trace_5xx'] = {
'request_id': row['labels'].get('request_id'), 'trace_id': row['labels']['trace_id'],
'status': row['labels'].get('http_response_status_code'),
'server_ms': dur, 'db_children_ms': db_ms,
'handler_gap_ms': trace_gap(spans, server),
'db_children': [verify.safe_span(s) | {'duration_ms': (s['end_ns'] - s['start_ns']) / 1e6}
for s in spans if s['kind'] == 'CLIENT' and 'db.operation.name' in s['attributes']
and s['parent_span_id'] == server['span_id']][:6]}
if db_ms < 0.5 * dur:
result['problems'].append('5xx trace is not dominated by DB spans')
else:
result['problems'].append('no 5xx http.request log found in Loki')
# -- control: handler delay (not DB) shows up as a gap, not as DB time.
ctx.obs.require_no_foreign_faults()
ctrl_chat = ctx.actor_chat()
base_rows = sequential_uploads(ctx, ctrl_chat[0], ctrl_chat[1], 20, 0)
delay_rows = sequential_uploads(ctx, ctrl_chat[0], ctrl_chat[1], 20, 500)
stand.clear_fault('fault:api.after_commit_before_publish')
def gaps(rows_):
out = []
for r in rows_:
spans, server = server_view(ctx, r['request_id'])
out.append({'request_id': r['request_id'],
'server_ms': (server['end_ns'] - server['start_ns']) / 1e6,
'db_children_ms': db_time_ms(spans, server),
'handler_gap_ms': trace_gap(spans, server)})
return out
base_g, delay_g = gaps(base_rows), gaps(delay_rows)
med = measurement.median
shift = med([g['handler_gap_ms'] for g in delay_g]) - med([g['handler_gap_ms'] for g in base_g])
result['control'] = {
'baseline_uploads': base_g, 'delayed_uploads': delay_g,
'handler_gap_shift_ms_median': shift,
'db_children_ms_median_baseline': med([g['db_children_ms'] for g in base_g]),
'db_children_ms_median_delayed': med([g['db_children_ms'] for g in delay_g]),
'all_keys_consumed': all(r['consumed'] for r in delay_rows),
'all_http_202': all(r['status'] == 202 for r in base_rows + delay_rows),
'note': 'timer/scheduling noise is part of the shift; the check is ~500 ms +/- 250 ms'}
if not result['control']['all_keys_consumed']:
result['problems'].append('delay fault key was not consumed')
if not result['control']['all_http_202']:
result['problems'].append('control uploads were not all 202')
if not (250 <= shift <= 750):
result['problems'].append('handler gap shift %.0f ms is not ~500 ms' % shift)
cdb = result['control']
if abs(cdb['db_children_ms_median_delayed'] - cdb['db_children_ms_median_baseline']) > 100:
result['problems'].append('DB time changed in the handler-delay control')
stand.wait_for(lambda: obs.wait_drained(
[r['attachment_id'] for r in base_rows + delay_rows], obs=None, timeout=60,
flush_telemetry=False)['drained'], 90, 'control uploads ready')
end = time.time()
t_inject = utc_epoch(tl['inject']['utc'])
window = (t_inject - 60, t_inject + 240)
result['queries'] = {
'db_operation_p95_by_op': 'histogram_quantile(0.95, sum by (le, db_operation_name)(rate(db_client_operation_duration_seconds_bucket[1m])))',
'pool_by_state': 'sum by (db_client_connection_state)(db_client_connection_count{db_client_connection_pool_name="main"})',
'pool_pending': 'sum(db_client_connection_pending_requests)',
'api_5xx_ratio': 'messenger:api_errors:rate1m / messenger:api_requests:rate1m'}
result['series'] = {k: series(ctx, v, *window) for k, v in result['queries'].items()}
finish(ctx, 'incident-db.json', result)
# ---------------------------------------------------------------- telemetry
def regression_slope(points):
n = len(points)
if n < 3:
return None
mx = sum(x for x, _ in points) / n
my = sum(y for _, y in points) / n
den = sum((x - mx) ** 2 for x, _ in points)
return sum((x - mx) * (y - my) for x, y in points) / den if den else None
def rss_trend(samples, service, after_s, start_mono):
pts = [(s['mono'] - start_mono, s['rss_bytes']) for s in samples
if s['service'] == service and s.get('rss_bytes') is not None
and s['mono'] - start_mono >= after_s]
by_tick = {}
for t, v in pts:
by_tick.setdefault(round(t / 5), []).append((t, v))
summed = [(sum(t for t, _ in g) / len(g), sum(v for _, v in g)) for g in by_tick.values()]
values = [v for _, v in summed]
return {'points': len(summed), 'min': min(values) if values else None,
'max': max(values) if values else None,
'slope_bytes_per_s': regression_slope(summed),
'first': values[0] if values else None, 'last': values[-1] if values else None}
def sdk_overflow_logs(since_utc):
result = subprocess.run(
['docker', 'compose', '--env-file', stand.ENV_FILE, 'logs', '--no-color',
'--no-log-prefix', '--since', since_utc, 'api', 'worker'],
cwd=ROOT, text=True, capture_output=True, timeout=180)
count = 0
for line in result.stdout.splitlines():
if '"event":"otel.sdk"' in line and ('queue' in line.lower() or 'dropped' in line.lower()):
count += 1
return count
def resume_probe(ctx, actor, chat, count):
"""After recovery: fresh requests must reach Loki (logs) and Tempo (traces)."""
c = stand.new_client(ctx.args.base_url, ctx.args.ca_file)
since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time() - 1))
start_ns = time.time_ns() - 5 * 10 ** 9
requests = []
for i in range(count):
r = c.message(actor, chat, 'lab6-resume-%d' % i)
requests.append({'request_id': r['headers']['X-Request-Id'], 'status': r['status']})
_, summary = reconcile.reconcile(ctx.obs, 'full', requests, since, start_ns,
time.time_ns() + 60 * 10 ** 9)
return summary
def telemetry_outage(ctx, service, alice, chat, repeat):
out = {'service': service, 'timeline': {}, 'problems': []}
tl = out['timeline']
ctx.obs.require_no_foreign_faults()
base = baseline(ctx, alice, chat, 20, 20)
out['baseline'] = {'load': base['load']}
state_before = ctx.obs.app_state()
ids = resources.container_map()
sampler = resources.ResourceSampler(ids, interval=5.0)
sampler.start()
sampler_start = time.monotonic()
since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time() - 1))
pre_failed = ctx.obs.prom_scalar('sum(messenger_telemetry_export_items_total{outcome="failed"})', 0.0)
load = BackgroundLoad(ctx, alice, chat, 20, 20, service + '-outage')
stopped = False
t_smoke = None
try:
time.sleep(10)
ctx.mark(tl, 'inject')
t_inject = time.time()
stand.compose('stop', service)
stopped = True
t_start = time.monotonic()
smoke = subprocess.run(['make', 'test', 'BASE_URL=' + ctx.args.base_url], cwd=ROOT,
capture_output=True, text=True, timeout=80)
t_smoke = time.monotonic() - t_start
out['smoke_during_outage'] = {'returncode': smoke.returncode,
'seconds': round(t_smoke, 1),
'tail': smoke.stdout.strip().splitlines()[-1:] if smoke.stdout else []}
if smoke.returncode != 0:
out['problems'].append('make test failed during the outage')
remaining = 90 - (time.monotonic() - t_start)
if remaining > 0:
time.sleep(remaining)
if service == 'tempo':
out['collector_during_outage'] = {
'queue_size_max': ctx.obs.prom_scalar('max(max_over_time(otelcol_exporter_queue_size{exporter="otlp/tempo"}[90s]))'),
'queue_capacity': ctx.obs.prom_scalar('max(otelcol_exporter_queue_capacity{exporter="otlp/tempo"})'),
'send_failed_spans_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_send_failed_spans[90s]))', 0.0),
'enqueue_failed_spans_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_enqueue_failed_spans[90s]))', 0.0),
'logs_sent_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_sent_log_records[90s]))', 0.0),
'metrics_sent_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_sent_metric_points[90s]))', 0.0),
'collector_rss_bytes': ctx.obs.prom_scalar('otelcol_process_memory_rss'),
'loki_up': ctx.obs.prom_scalar('up{job="loki"}'), 'prometheus_ready': True}
ctx.mark(tl, 'outage_end')
finally:
if stopped:
stand.compose('start', service)
ctx.mark(tl, 'recover')
try:
incident_load = load.stop()
finally:
ctx.mark(tl, 'load_stopped')
t_restore = time.time()
ctx.obs.wait_ready(timeout=180, app_telemetry=True, provisioning=True)
ctx.mark(tl, 'backends_ready')
drain = obs.wait_drained(incident_load['uploads'] + base['uploads'], obs=ctx.obs, timeout=180)
out['drain'] = drain
if drain['drained']:
ctx.mark(tl, 'backlog_drained')
else:
out['problems'].append('jobs did not drain')
sampler.stop()
out['resume_probe'] = resume_probe(ctx, alice, chat, 10)
probe = out['resume_probe']
if probe.get('sampled_present') != 10 or probe.get('log_exported') != 10:
out['problems'].append('traces/logs did not resume after recovery: %s' % probe)
out['incident_load'] = incident_load['load']
if incident_load['load']['http_5xx'] or sum(incident_load['load']['transport_failures'].values()):
out['problems'].append('business traffic degraded during telemetry outage')
# -- no restarts/OOM, same instances.
state_after = ctx.obs.app_state()
for svc in ('api', 'worker'):
if set(state_before[svc]) != set(state_after[svc]):
out['problems'].append('instance IDs changed for ' + svc)
for cid, info in state_after[svc].items():
if info['restarts'] != state_before[svc].get(cid, info)['restarts'] or info['oom_killed']:
out['problems'].append('restart/OOM in ' + svc)
summary = resources.summarize(sampler.samples)
out['resources'] = {k: v for k, v in summary.items() if k != 'sample_errors'}
out['sample_errors'] = summary['sample_errors']
out['memory_vs_limit'] = {
svc: (v['container_memory_bytes']['max'] / v['mem_limit_bytes'])
for svc, v in summary.items() if svc != 'sample_errors'
and v['mem_limit_bytes'] and v['container_memory_bytes']['max'] is not None}
for svc, ratio in out['memory_vs_limit'].items():
if ratio >= 1.0:
out['problems'].append(svc + ' memory at/over limit')
out['rss_trend_after_first_minute'] = {
svc: rss_trend(sampler.samples, svc, 60, sampler_start)
for svc in (('otel-collector', 'api', 'worker') if service == 'collector' or service == 'otel-collector'
else ('otel-collector',))}
out['sdk_queue_diagnostic_messages'] = sdk_overflow_logs(since)
# -- stdout vs backends for requests accepted during the outage.
inject_unix = utc_epoch(tl['inject']['utc'])
during = [s for s in incident_load['slots'] if s.get('outcome') == 'http'
and s.get('request_id') and s['kind'] == 'message']
stdout = reconcile.stdout_http_logs(since)
# slots are relative to the load t0; classify by stdout timestamp instead.
in_window = []
for s in during:
row = stdout.get(s['request_id'])
if row is None:
continue
ts = utc_epoch(row['timestamp'])
if inject_unix <= ts <= inject_unix + 90:
in_window.append((s, row))
sampled_window = [(s, r) for s, r in in_window if r.get('trace_sampled') is True]
step = max(1, len(sampled_window) // 100)
probe = sampled_window[::step][:100]
targets = {s['request_id']: (r['trace_id'], r['span_id']) for s, r in probe}
found = reconcile.tempo_server_spans(ctx.obs, targets, deadline_s=60)
out['stdout_vs_tempo'] = {
'requests_in_outage_window_with_stdout': len(in_window),
'sampled_in_window_per_stdout': len(sampled_window),
'tempo_probe_size': len(probe),
'tempo_probe_missing_server_spans': sum(1 for v in found.values() if not v),
'note': 'loss is measured for SERVER root spans only, on a systematic probe '
'sample; it does not count lost child spans'}
if not in_window:
out['problems'].append('no stdout http.request lines in the outage window')
if any(not r.get('trace_id') or r.get('trace_sampled') is not True for _, r in sampled_window):
out['problems'].append('stdout trace fields missing')
if service == 'tempo':
c = out['collector_during_outage']
if (c['queue_size_max'] or 0) > (c['queue_capacity'] or 128):
out['problems'].append('Collector queue exceeded capacity')
if not (c['send_failed_spans_increase'] or 0) > 0:
out['problems'].append('Collector did not count failed span exports')
if not (c['logs_sent_increase'] or 0) > 0 or not (c['metrics_sent_increase'] or 0) > 0:
out['problems'].append('Loki/Prometheus paths stopped during the Tempo outage')
else:
post_failed = ctx.obs.prom_scalar('sum(messenger_telemetry_export_items_total{outcome="failed"})', 0.0)
out['app_export_failed_items'] = {'before': pre_failed, 'after': post_failed,
'delta': post_failed - pre_failed if post_failed >= pre_failed else None,
'reason': None if post_failed >= pre_failed else 'counter_reset'}
if not (out['app_export_failed_items']['delta'] or 0) > 0:
out['problems'].append('app failed-export counter did not grow')
resumed = {name: ctx.obs.prom_scalar(expr, 0.0) for name, expr in (
('spans_sent_300s', 'sum(increase(otelcol_exporter_sent_spans[300s]))'),
('logs_sent_300s', 'sum(increase(otelcol_exporter_sent_log_records[300s]))'),
('metrics_sent_300s', 'sum(increase(otelcol_exporter_sent_metric_points[300s]))'))}
out['signals_resumed'] = resumed
if not all(v > 0 for v in resumed.values()):
out['problems'].append('not all three signals resumed')
return out
def scenario_telemetry(ctx, label=''):
alice, chat = ctx.actor_chat()
result = {'scenario': 'telemetry', 'repeat': label, 'problems': [], 'outages': []}
for service in ('tempo', 'otel-collector'):
outage = telemetry_outage(ctx, service, alice, chat, label)
result['outages'].append(outage)
result['problems'] += [service + ': ' + p for p in outage['problems']]
name = 'incident-telemetry-%s.json' % label
finish(ctx, name, result)
SCENARIOS = {'worker': scenario_worker, 'db': scenario_db, 'telemetry': scenario_telemetry}
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--scenario', required=True, choices=sorted(SCENARIOS))
parser.add_argument('--repeat', type=int, default=1)
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--ca-file')
parser.add_argument('--prometheus-url', default='http://localhost:9090')
parser.add_argument('--grafana-url', default='http://localhost:3000')
parser.add_argument('--output-dir', default='evidence/lab6')
args = parser.parse_args(argv)
ctx = Ctx(args)
failed = False
try:
with ctx.obs.app_environment(ENV):
for repeat in range(1, args.repeat + 1):
try:
SCENARIOS[args.scenario](ctx, str(repeat))
except Failure as error:
failed = True
print('FAIL: ' + str(error), file=sys.stderr)
break
except Failure as error:
print('FAIL: ' + str(error), file=sys.stderr)
return 1
return 1 if failed else 0
if __name__ == '__main__':
sys.exit(main())