forked from mtusi2026/mtusi2026lab6
677 lines
32 KiB
Python
Executable File
677 lines
32 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""lab6 controlled incidents: worker | db | telemetry. Stdlib only.
|
|
|
|
Every scenario runs inside obs.app_environment (SDK on, sampling 1,
|
|
FAULT_INJECTION_ENABLED=1) and always undoes its own faults/locks/stopped
|
|
services in `finally`. Times are UTC + monotonic offsets; an unobserved
|
|
transition is null with a reason, never a guess."""
|
|
import argparse
|
|
import calendar
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import measurement # noqa: E402
|
|
import obs # noqa: E402
|
|
import reconcile # noqa: E402
|
|
import resources # noqa: E402
|
|
import verify # noqa: E402
|
|
import workload # noqa: E402
|
|
from obs import Failure, stand, utc_now # noqa: E402
|
|
|
|
ROOT = obs.ROOT
|
|
|
|
|
|
def utc_epoch(text):
|
|
return calendar.timegm(time.strptime(text[:19], '%Y-%m-%dT%H:%M:%S'))
|
|
ALERTS = ('MessengerApiHighErrorRate', 'MessengerJobsQueueLagGrowing')
|
|
ENV = {'OTEL_SDK_DISABLED': 'false', 'OTEL_TRACES_SAMPLER_ARG': '1',
|
|
'FAULT_INJECTION_ENABLED': '1', 'LOG_LEVEL': 'INFO'}
|
|
|
|
|
|
class Ctx:
|
|
def __init__(self, args):
|
|
self.args = args
|
|
self.obs = verify.make_obs(args)
|
|
self.out = Path(args.output_dir)
|
|
self.t0 = time.monotonic()
|
|
|
|
def mono(self):
|
|
return round(time.monotonic() - self.t0, 2)
|
|
|
|
def mark(self, timeline, key, extra=None):
|
|
timeline[key] = {'utc': utc_now(), 'mono_s': self.mono()}
|
|
if extra:
|
|
timeline[key].update(extra)
|
|
print('[%s] %s' % (timeline[key]['utc'], key), flush=True)
|
|
|
|
def actor_chat(self):
|
|
c = stand.new_client(self.args.base_url, self.args.ca_file)
|
|
alice, _, _, chat = c.bootstrap()
|
|
return alice, chat
|
|
|
|
|
|
def alert_state(ctx, name):
|
|
result = ctx.obs.prom_query('ALERTS{alertname="%s"}' % name)
|
|
states = {r['metric'].get('alertstate') for r in result}
|
|
if 'firing' in states:
|
|
return 'firing'
|
|
return 'pending' if 'pending' in states else None
|
|
|
|
|
|
def wait_state(ctx, name, want, timeout, timeline, polls):
|
|
"""want: 'firing' or None (resolved). Returns True when reached."""
|
|
end = time.monotonic() + timeout
|
|
while time.monotonic() < end:
|
|
state = alert_state(ctx, name)
|
|
polls.append({'utc': utc_now(), 'mono_s': ctx.mono(), 'alert': name, 'state': state})
|
|
if state == 'pending' and 'pending' not in timeline:
|
|
ctx.mark(timeline, 'pending')
|
|
if state == want:
|
|
return True
|
|
time.sleep(5)
|
|
return False
|
|
|
|
|
|
class BackgroundLoad:
|
|
def __init__(self, ctx, actor, chat, rate, upload_every, phase):
|
|
self.stop_event = threading.Event()
|
|
self.result = None
|
|
self.error = None
|
|
|
|
def body():
|
|
try:
|
|
self.result = workload.run_window(
|
|
ctx.args.base_url, ctx.args.ca_file, actor, chat, rate, 1800,
|
|
phase, upload_every=upload_every, stop_event=self.stop_event,
|
|
settle=False)
|
|
except Exception as exc: # reported by stop()
|
|
self.error = exc
|
|
self.thread = threading.Thread(target=body, daemon=True)
|
|
self.thread.start()
|
|
|
|
def stop(self):
|
|
self.stop_event.set()
|
|
self.thread.join(timeout=120)
|
|
if self.error:
|
|
raise self.error
|
|
if self.result is None:
|
|
raise Failure('background load did not finish')
|
|
return self.result
|
|
|
|
|
|
def baseline(ctx, actor, chat, rate, upload_every):
|
|
ctx.obs.require_no_foreign_faults()
|
|
for name in ALERTS:
|
|
if alert_state(ctx, name) is not None:
|
|
raise Failure('alert already active before baseline: ' + name)
|
|
result = workload.run_window(ctx.args.base_url, ctx.args.ca_file, actor, chat,
|
|
rate, 60, 'baseline', upload_every=upload_every,
|
|
settle=False)
|
|
drain = obs.wait_drained(result['uploads'], obs=ctx.obs)
|
|
for name in ALERTS:
|
|
if alert_state(ctx, name) is not None:
|
|
raise Failure('alert active at the end of baseline: ' + name)
|
|
if not drain['drained']:
|
|
raise Failure('baseline jobs did not drain')
|
|
return {'load': result['load'], 'drain': drain, 'uploads': result['uploads']}
|
|
|
|
|
|
def series(ctx, expr, start, end, step=10):
|
|
out = []
|
|
for item in ctx.obs.prom_range(expr, start, end, step):
|
|
out.append({'labels': item['metric'],
|
|
'points': [[float(t), v] for t, v in item['values']]})
|
|
return out
|
|
|
|
|
|
def peak(rows):
|
|
values = [float(v) for r in rows for _, v in r['points'] if v != 'NaN']
|
|
return max(values) if values else None
|
|
|
|
|
|
def ready_seconds(ids):
|
|
"""max/median finished_at - created_at of the jobs of these attachments."""
|
|
if not ids:
|
|
return {'n': 0, 'max': None, 'median': None}
|
|
values = []
|
|
for i in range(0, len(ids), 200):
|
|
chunk = ids[i:i + 200]
|
|
arr = "ARRAY[" + ",".join("'" + x + "'" for x in chunk) + "]::uuid[]"
|
|
out = obs.psql(
|
|
"SELECT COALESCE(string_agg(extract(epoch from (finished_at - created_at))::text, ';'),'') "
|
|
"FROM attachment_jobs WHERE status='succeeded' AND attachment_id = ANY(" + arr + ")")
|
|
values += [float(x) for x in out.split(';') if x]
|
|
return {'n': len(values), 'max': max(values) if values else None,
|
|
'median': measurement.median(values)}
|
|
|
|
|
|
def finish(ctx, name, result):
|
|
result['passed'] = not result['problems']
|
|
verify.write_json(ctx.out, name, result)
|
|
if result['problems']:
|
|
raise Failure(name + ': ' + '; '.join(result['problems']))
|
|
print('PASS: ' + name)
|
|
|
|
|
|
# ------------------------------------------------------------------- worker
|
|
|
|
def scenario_worker(ctx, label=''):
|
|
alice, chat = ctx.actor_chat()
|
|
result = {'scenario': 'worker', 'profile': {'rate': 20, 'upload_every': 20},
|
|
'timeline': {}, 'problems': [], 'polls': []}
|
|
tl = result['timeline']
|
|
base = baseline(ctx, alice, chat, 20, 20)
|
|
result['baseline'] = {'load': base['load'], 'ready_seconds': ready_seconds(base['uploads'])}
|
|
load = None
|
|
worker_stopped = False
|
|
try:
|
|
load = BackgroundLoad(ctx, alice, chat, 20, 20, 'incident')
|
|
time.sleep(10)
|
|
ctx.mark(tl, 'inject')
|
|
stand.compose('stop', 'worker')
|
|
worker_stopped = True
|
|
fired = wait_state(ctx, 'MessengerJobsQueueLagGrowing', 'firing', 150, tl, result['polls'])
|
|
if fired:
|
|
ctx.mark(tl, 'firing')
|
|
else:
|
|
tl['firing'] = None
|
|
result['problems'].append('queue-lag alert did not fire within 150 s')
|
|
# Capture the evidence while the backlog exists.
|
|
t_now = time.time()
|
|
result['queue_at_firing'] = {
|
|
'oldest_age_max': ctx.obs.prom_scalar('max(messenger_jobs_queue_oldest_age_seconds)'),
|
|
'depth_ready_max': ctx.obs.prom_scalar('max(messenger_jobs_queue_depth{state="ready"})'),
|
|
'depth_ready_sum_over_replicas': ctx.obs.prom_scalar('sum(messenger_jobs_queue_depth{state="ready"})'),
|
|
'depth_ready_per_replica': {
|
|
r['metric']['instance']: float(r['value'][1])
|
|
for r in ctx.obs.prom_query('messenger_jobs_queue_depth{state="ready"}')},
|
|
'outbox_pending_max': ctx.obs.prom_scalar('max(messenger_outbox_pending)'),
|
|
}
|
|
q = result['queue_at_firing']
|
|
per = list(q['depth_ready_per_replica'].values())
|
|
if len(per) >= 2 and q['depth_ready_max'] != max(per):
|
|
result['problems'].append('max() does not equal the per-replica snapshot')
|
|
if len(per) >= 2 and q['depth_ready_sum_over_replicas'] == q['depth_ready_max'] and per[0] > 0:
|
|
result['problems'].append('replicas not both reporting (cannot show non-doubling)')
|
|
finally:
|
|
ctx.mark(tl, 'recover_start')
|
|
if worker_stopped:
|
|
stand.compose('up', '-d', '--no-deps', '--wait', 'worker')
|
|
ctx.mark(tl, 'worker_back')
|
|
incident_load = load.stop() if load else None
|
|
ctx.mark(tl, 'load_stopped')
|
|
ids = incident_load['uploads']
|
|
result['incident_load'] = incident_load['load']
|
|
drain = obs.wait_drained(ids + base['uploads'], obs=ctx.obs, timeout=180)
|
|
result['drain'] = drain
|
|
ctx.mark(tl, 'backlog_drained' if drain['drained'] else 'drain_deadline')
|
|
if not drain['drained']:
|
|
tl['backlog_drained'] = None
|
|
result['problems'].append('backlog not drained within 180 s')
|
|
elif any(s != 'ready' for s in drain['states'].values()):
|
|
result['problems'].append('some accepted attachments are not ready')
|
|
resolved = wait_state(ctx, 'MessengerJobsQueueLagGrowing', None, 180, tl, result['polls'])
|
|
if resolved:
|
|
ctx.mark(tl, 'resolved')
|
|
else:
|
|
tl['resolved'] = None
|
|
result['problems'].append('queue-lag alert not resolved within 180 s')
|
|
end = time.time()
|
|
t_inject = utc_epoch(tl['inject']['utc'])
|
|
window = (t_inject - 60, end)
|
|
result['queries'] = {
|
|
'oldest_age': 'max(messenger_jobs_queue_oldest_age_seconds)',
|
|
'depth': 'max by (state)(messenger_jobs_queue_depth)',
|
|
'outbox': 'max(messenger_outbox_pending)',
|
|
'worker_active': 'sum(messenger_worker_jobs_active)'}
|
|
result['series'] = {k: series(ctx, v, *window) for k, v in result['queries'].items()}
|
|
peak_age = peak(result['series']['oldest_age'])
|
|
result['peak_oldest_age_s'] = peak_age
|
|
if peak_age is None or peak_age < 30:
|
|
result['problems'].append('queue age did not grow past 30 s')
|
|
result['incident_ready_seconds'] = ready_seconds(ids)
|
|
if (result['incident_ready_seconds']['max'] or 0) <= (result['baseline']['ready_seconds']['max'] or 0):
|
|
result['problems'].append('time-to-ready did not exceed the baseline')
|
|
http_ok = result['incident_load']['http_5xx'] == 0 and \
|
|
sum(result['incident_load']['transport_failures'].values()) == 0
|
|
result['http_acceptance_continued'] = http_ok
|
|
if not http_ok:
|
|
result['problems'].append('HTTP acceptance degraded during worker outage')
|
|
finish(ctx, 'incident-worker.json', result)
|
|
|
|
|
|
# ----------------------------------------------------------------------- db
|
|
|
|
def trace_gap(spans, server):
|
|
"""Server duration minus the union of its direct child intervals (ms)."""
|
|
children = sorted((s['start_ns'], s['end_ns']) for s in spans
|
|
if s['parent_span_id'] == server['span_id'])
|
|
# Descendants of the transaction span are covered by their ancestor.
|
|
covered, last = 0, None
|
|
for start, end in children:
|
|
if last is None or start > last:
|
|
covered += end - start
|
|
last = end
|
|
elif end > last:
|
|
covered += end - last
|
|
last = end
|
|
return ((server['end_ns'] - server['start_ns']) - covered) / 1e6
|
|
|
|
|
|
def db_time_ms(spans, server):
|
|
return sum((s['end_ns'] - s['start_ns']) for s in spans
|
|
if s['kind'] == 'CLIENT' and 'db.operation.name' in s['attributes']
|
|
and s['parent_span_id'] == server['span_id']) / 1e6
|
|
|
|
|
|
def sequential_uploads(ctx, actor, chat, count, delay):
|
|
c = stand.new_client(ctx.args.base_url, ctx.args.ca_file)
|
|
c.timeout = 30
|
|
key = 'fault:api.after_commit_before_publish'
|
|
rows = []
|
|
for _ in range(count):
|
|
if delay:
|
|
stand.set_fault(key, 'delay:%d' % delay)
|
|
started = time.monotonic()
|
|
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
|
|
range(200, 600), actor=actor, raw=stand.sample_bytes(),
|
|
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
elapsed = (time.monotonic() - started) * 1000
|
|
consumed = True
|
|
if delay:
|
|
out = stand.compose('exec', '-T', 'redis', 'redis-cli', 'GET', key).stdout.strip()
|
|
consumed = out == ''
|
|
rows.append({'request_id': r['headers']['X-Request-Id'], 'status': r['status'],
|
|
'elapsed_ms': elapsed, 'consumed': consumed,
|
|
'attachment_id': r['json']['id'] if r['status'] == 202 else None})
|
|
return rows
|
|
|
|
|
|
def server_view(ctx, request_id):
|
|
log = verify.http_log(ctx.obs, request_id)
|
|
tid, sid = log['labels']['trace_id'], log['labels']['span_id']
|
|
spans = verify.trace_with(ctx.obs, tid, lambda sp: any(
|
|
s['span_id'] == sid for s in sp), description='server trace')
|
|
server = next(s for s in spans if s['span_id'] == sid)
|
|
return spans, server
|
|
|
|
|
|
def scenario_db(ctx, label=''):
|
|
alice, chat = ctx.actor_chat()
|
|
result = {'scenario': 'db', 'profile': {'rate': 2, 'upload_every': 1},
|
|
'timeline': {}, 'problems': [], 'polls': []}
|
|
tl = result['timeline']
|
|
base = baseline(ctx, alice, chat, 2, 1)
|
|
result['baseline'] = {'load': base['load']}
|
|
load = None
|
|
lock_released = False
|
|
try:
|
|
load = BackgroundLoad(ctx, alice, chat, 2, 1, 'incident')
|
|
time.sleep(10)
|
|
# Own client: 30 s timeout is set inside workload (REQUEST_TIMEOUT).
|
|
with obs.hold_table_lock('attachments') as pid:
|
|
ctx.mark(tl, 'inject', {'lock_backend_pid': pid})
|
|
fired = wait_state(ctx, 'MessengerApiHighErrorRate', 'firing', 120, tl, result['polls'])
|
|
if fired:
|
|
ctx.mark(tl, 'firing')
|
|
else:
|
|
tl['firing'] = None
|
|
result['problems'].append('error-rate alert did not fire within 120 s')
|
|
result['at_firing'] = {
|
|
'api_5xx_rate_1m': ctx.obs.prom_scalar('messenger:api_errors:rate1m'),
|
|
'api_request_rate_1m': ctx.obs.prom_scalar('messenger:api_requests:rate1m'),
|
|
'haproxy_5xx_rate_1m': ctx.obs.prom_scalar(
|
|
'sum(rate(haproxy_backend_http_responses_total{proxy="api",code="5xx"}[1m]))', 0.0),
|
|
'db_insert_attachments_p95_s': ctx.obs.prom_scalar(
|
|
'histogram_quantile(0.95, sum by (le)(rate(db_client_operation_duration_seconds_bucket{db_operation_name="INSERT"}[1m])))'),
|
|
'pool_pending': ctx.obs.prom_scalar('sum(db_client_connection_pending_requests)'),
|
|
'pool_used': ctx.obs.prom_scalar('sum(db_client_connection_count{db_client_connection_state="used"})'),
|
|
}
|
|
lock_released = True
|
|
ctx.mark(tl, 'lock_released')
|
|
finally:
|
|
if not lock_released:
|
|
ctx.mark(tl, 'lock_released_in_finally')
|
|
incident_load = load.stop() if load else None
|
|
ctx.mark(tl, 'load_stopped')
|
|
result['incident_load'] = incident_load['load']
|
|
if incident_load['load']['http_5xx'] == 0:
|
|
result['problems'].append('no real server 5xx observed in the load')
|
|
ids = incident_load['uploads']
|
|
drain = obs.wait_drained(ids + base['uploads'], obs=ctx.obs, timeout=180)
|
|
result['drain'] = drain
|
|
if drain['drained']:
|
|
ctx.mark(tl, 'backlog_drained')
|
|
else:
|
|
tl['backlog_drained'] = None
|
|
result['problems'].append('jobs did not drain after recovery')
|
|
if wait_state(ctx, 'MessengerApiHighErrorRate', None, 120, tl, result['polls']):
|
|
ctx.mark(tl, 'resolved')
|
|
else:
|
|
tl['resolved'] = None
|
|
result['problems'].append('error-rate alert not resolved within 120 s')
|
|
# -- localisation: a 5xx trace where the time went to the DB.
|
|
inject_ns = int(utc_epoch(tl['inject']['utc']) - 5) * 10 ** 9
|
|
rows = verify.wait_rows(
|
|
ctx.obs, '{service_name="messenger-api"} | event_name="http.request" '
|
|
'| http_response_status_code=~"5.."', timeout=60, back=900)
|
|
rows = [r for r in rows if r['ts'] >= inject_ns]
|
|
if rows:
|
|
row = rows[0]
|
|
spans = verify.trace_with(ctx.obs, row['labels']['trace_id'], lambda sp: any(
|
|
s['span_id'] == row['labels']['span_id'] for s in sp), description='5xx trace')
|
|
server = next(s for s in spans if s['span_id'] == row['labels']['span_id'])
|
|
dur = (server['end_ns'] - server['start_ns']) / 1e6
|
|
db_ms = db_time_ms(spans, server)
|
|
result['trace_5xx'] = {
|
|
'request_id': row['labels'].get('request_id'), 'trace_id': row['labels']['trace_id'],
|
|
'status': row['labels'].get('http_response_status_code'),
|
|
'server_ms': dur, 'db_children_ms': db_ms,
|
|
'handler_gap_ms': trace_gap(spans, server),
|
|
'db_children': [verify.safe_span(s) | {'duration_ms': (s['end_ns'] - s['start_ns']) / 1e6}
|
|
for s in spans if s['kind'] == 'CLIENT' and 'db.operation.name' in s['attributes']
|
|
and s['parent_span_id'] == server['span_id']][:6]}
|
|
if db_ms < 0.5 * dur:
|
|
result['problems'].append('5xx trace is not dominated by DB spans')
|
|
else:
|
|
result['problems'].append('no 5xx http.request log found in Loki')
|
|
# -- control: handler delay (not DB) shows up as a gap, not as DB time.
|
|
ctx.obs.require_no_foreign_faults()
|
|
ctrl_chat = ctx.actor_chat()
|
|
base_rows = sequential_uploads(ctx, ctrl_chat[0], ctrl_chat[1], 20, 0)
|
|
delay_rows = sequential_uploads(ctx, ctrl_chat[0], ctrl_chat[1], 20, 500)
|
|
stand.clear_fault('fault:api.after_commit_before_publish')
|
|
|
|
def gaps(rows_):
|
|
out = []
|
|
for r in rows_:
|
|
spans, server = server_view(ctx, r['request_id'])
|
|
out.append({'request_id': r['request_id'],
|
|
'server_ms': (server['end_ns'] - server['start_ns']) / 1e6,
|
|
'db_children_ms': db_time_ms(spans, server),
|
|
'handler_gap_ms': trace_gap(spans, server)})
|
|
return out
|
|
base_g, delay_g = gaps(base_rows), gaps(delay_rows)
|
|
med = measurement.median
|
|
shift = med([g['handler_gap_ms'] for g in delay_g]) - med([g['handler_gap_ms'] for g in base_g])
|
|
result['control'] = {
|
|
'baseline_uploads': base_g, 'delayed_uploads': delay_g,
|
|
'handler_gap_shift_ms_median': shift,
|
|
'db_children_ms_median_baseline': med([g['db_children_ms'] for g in base_g]),
|
|
'db_children_ms_median_delayed': med([g['db_children_ms'] for g in delay_g]),
|
|
'all_keys_consumed': all(r['consumed'] for r in delay_rows),
|
|
'all_http_202': all(r['status'] == 202 for r in base_rows + delay_rows),
|
|
'note': 'timer/scheduling noise is part of the shift; the check is ~500 ms +/- 250 ms'}
|
|
if not result['control']['all_keys_consumed']:
|
|
result['problems'].append('delay fault key was not consumed')
|
|
if not result['control']['all_http_202']:
|
|
result['problems'].append('control uploads were not all 202')
|
|
if not (250 <= shift <= 750):
|
|
result['problems'].append('handler gap shift %.0f ms is not ~500 ms' % shift)
|
|
cdb = result['control']
|
|
if abs(cdb['db_children_ms_median_delayed'] - cdb['db_children_ms_median_baseline']) > 100:
|
|
result['problems'].append('DB time changed in the handler-delay control')
|
|
stand.wait_for(lambda: obs.wait_drained(
|
|
[r['attachment_id'] for r in base_rows + delay_rows], obs=None, timeout=60,
|
|
flush_telemetry=False)['drained'], 90, 'control uploads ready')
|
|
end = time.time()
|
|
t_inject = utc_epoch(tl['inject']['utc'])
|
|
window = (t_inject - 60, t_inject + 240)
|
|
result['queries'] = {
|
|
'db_operation_p95_by_op': 'histogram_quantile(0.95, sum by (le, db_operation_name)(rate(db_client_operation_duration_seconds_bucket[1m])))',
|
|
'pool_by_state': 'sum by (db_client_connection_state)(db_client_connection_count{db_client_connection_pool_name="main"})',
|
|
'pool_pending': 'sum(db_client_connection_pending_requests)',
|
|
'api_5xx_ratio': 'messenger:api_errors:rate1m / messenger:api_requests:rate1m'}
|
|
result['series'] = {k: series(ctx, v, *window) for k, v in result['queries'].items()}
|
|
finish(ctx, 'incident-db.json', result)
|
|
|
|
|
|
# ---------------------------------------------------------------- telemetry
|
|
|
|
def regression_slope(points):
|
|
n = len(points)
|
|
if n < 3:
|
|
return None
|
|
mx = sum(x for x, _ in points) / n
|
|
my = sum(y for _, y in points) / n
|
|
den = sum((x - mx) ** 2 for x, _ in points)
|
|
return sum((x - mx) * (y - my) for x, y in points) / den if den else None
|
|
|
|
|
|
def rss_trend(samples, service, after_s, start_mono):
|
|
pts = [(s['mono'] - start_mono, s['rss_bytes']) for s in samples
|
|
if s['service'] == service and s.get('rss_bytes') is not None
|
|
and s['mono'] - start_mono >= after_s]
|
|
by_tick = {}
|
|
for t, v in pts:
|
|
by_tick.setdefault(round(t / 5), []).append((t, v))
|
|
summed = [(sum(t for t, _ in g) / len(g), sum(v for _, v in g)) for g in by_tick.values()]
|
|
values = [v for _, v in summed]
|
|
return {'points': len(summed), 'min': min(values) if values else None,
|
|
'max': max(values) if values else None,
|
|
'slope_bytes_per_s': regression_slope(summed),
|
|
'first': values[0] if values else None, 'last': values[-1] if values else None}
|
|
|
|
|
|
def sdk_overflow_logs(since_utc):
|
|
result = subprocess.run(
|
|
['docker', 'compose', '--env-file', stand.ENV_FILE, 'logs', '--no-color',
|
|
'--no-log-prefix', '--since', since_utc, 'api', 'worker'],
|
|
cwd=ROOT, text=True, capture_output=True, timeout=180)
|
|
count = 0
|
|
for line in result.stdout.splitlines():
|
|
if '"event":"otel.sdk"' in line and ('queue' in line.lower() or 'dropped' in line.lower()):
|
|
count += 1
|
|
return count
|
|
|
|
|
|
def resume_probe(ctx, actor, chat, count):
|
|
"""After recovery: fresh requests must reach Loki (logs) and Tempo (traces)."""
|
|
c = stand.new_client(ctx.args.base_url, ctx.args.ca_file)
|
|
since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time() - 1))
|
|
start_ns = time.time_ns() - 5 * 10 ** 9
|
|
requests = []
|
|
for i in range(count):
|
|
r = c.message(actor, chat, 'lab6-resume-%d' % i)
|
|
requests.append({'request_id': r['headers']['X-Request-Id'], 'status': r['status']})
|
|
_, summary = reconcile.reconcile(ctx.obs, 'full', requests, since, start_ns,
|
|
time.time_ns() + 60 * 10 ** 9)
|
|
return summary
|
|
|
|
|
|
def telemetry_outage(ctx, service, alice, chat, repeat):
|
|
out = {'service': service, 'timeline': {}, 'problems': []}
|
|
tl = out['timeline']
|
|
ctx.obs.require_no_foreign_faults()
|
|
base = baseline(ctx, alice, chat, 20, 20)
|
|
out['baseline'] = {'load': base['load']}
|
|
state_before = ctx.obs.app_state()
|
|
ids = resources.container_map()
|
|
sampler = resources.ResourceSampler(ids, interval=5.0)
|
|
sampler.start()
|
|
sampler_start = time.monotonic()
|
|
since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time() - 1))
|
|
pre_failed = ctx.obs.prom_scalar('sum(messenger_telemetry_export_items_total{outcome="failed"})', 0.0)
|
|
load = BackgroundLoad(ctx, alice, chat, 20, 20, service + '-outage')
|
|
stopped = False
|
|
t_smoke = None
|
|
try:
|
|
time.sleep(10)
|
|
ctx.mark(tl, 'inject')
|
|
t_inject = time.time()
|
|
stand.compose('stop', service)
|
|
stopped = True
|
|
t_start = time.monotonic()
|
|
smoke = subprocess.run(['make', 'test', 'BASE_URL=' + ctx.args.base_url], cwd=ROOT,
|
|
capture_output=True, text=True, timeout=80)
|
|
t_smoke = time.monotonic() - t_start
|
|
out['smoke_during_outage'] = {'returncode': smoke.returncode,
|
|
'seconds': round(t_smoke, 1),
|
|
'tail': smoke.stdout.strip().splitlines()[-1:] if smoke.stdout else []}
|
|
if smoke.returncode != 0:
|
|
out['problems'].append('make test failed during the outage')
|
|
remaining = 90 - (time.monotonic() - t_start)
|
|
if remaining > 0:
|
|
time.sleep(remaining)
|
|
if service == 'tempo':
|
|
out['collector_during_outage'] = {
|
|
'queue_size_max': ctx.obs.prom_scalar('max(max_over_time(otelcol_exporter_queue_size{exporter="otlp/tempo"}[90s]))'),
|
|
'queue_capacity': ctx.obs.prom_scalar('max(otelcol_exporter_queue_capacity{exporter="otlp/tempo"})'),
|
|
'send_failed_spans_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_send_failed_spans[90s]))', 0.0),
|
|
'enqueue_failed_spans_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_enqueue_failed_spans[90s]))', 0.0),
|
|
'logs_sent_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_sent_log_records[90s]))', 0.0),
|
|
'metrics_sent_increase': ctx.obs.prom_scalar('sum(increase(otelcol_exporter_sent_metric_points[90s]))', 0.0),
|
|
'collector_rss_bytes': ctx.obs.prom_scalar('otelcol_process_memory_rss'),
|
|
'loki_up': ctx.obs.prom_scalar('up{job="loki"}'), 'prometheus_ready': True}
|
|
ctx.mark(tl, 'outage_end')
|
|
finally:
|
|
if stopped:
|
|
stand.compose('start', service)
|
|
ctx.mark(tl, 'recover')
|
|
try:
|
|
incident_load = load.stop()
|
|
finally:
|
|
ctx.mark(tl, 'load_stopped')
|
|
t_restore = time.time()
|
|
ctx.obs.wait_ready(timeout=180, app_telemetry=True, provisioning=True)
|
|
ctx.mark(tl, 'backends_ready')
|
|
drain = obs.wait_drained(incident_load['uploads'] + base['uploads'], obs=ctx.obs, timeout=180)
|
|
out['drain'] = drain
|
|
if drain['drained']:
|
|
ctx.mark(tl, 'backlog_drained')
|
|
else:
|
|
out['problems'].append('jobs did not drain')
|
|
sampler.stop()
|
|
out['resume_probe'] = resume_probe(ctx, alice, chat, 10)
|
|
probe = out['resume_probe']
|
|
if probe.get('sampled_present') != 10 or probe.get('log_exported') != 10:
|
|
out['problems'].append('traces/logs did not resume after recovery: %s' % probe)
|
|
out['incident_load'] = incident_load['load']
|
|
if incident_load['load']['http_5xx'] or sum(incident_load['load']['transport_failures'].values()):
|
|
out['problems'].append('business traffic degraded during telemetry outage')
|
|
# -- no restarts/OOM, same instances.
|
|
state_after = ctx.obs.app_state()
|
|
for svc in ('api', 'worker'):
|
|
if set(state_before[svc]) != set(state_after[svc]):
|
|
out['problems'].append('instance IDs changed for ' + svc)
|
|
for cid, info in state_after[svc].items():
|
|
if info['restarts'] != state_before[svc].get(cid, info)['restarts'] or info['oom_killed']:
|
|
out['problems'].append('restart/OOM in ' + svc)
|
|
summary = resources.summarize(sampler.samples)
|
|
out['resources'] = {k: v for k, v in summary.items() if k != 'sample_errors'}
|
|
out['sample_errors'] = summary['sample_errors']
|
|
out['memory_vs_limit'] = {
|
|
svc: (v['container_memory_bytes']['max'] / v['mem_limit_bytes'])
|
|
for svc, v in summary.items() if svc != 'sample_errors'
|
|
and v['mem_limit_bytes'] and v['container_memory_bytes']['max'] is not None}
|
|
for svc, ratio in out['memory_vs_limit'].items():
|
|
if ratio >= 1.0:
|
|
out['problems'].append(svc + ' memory at/over limit')
|
|
out['rss_trend_after_first_minute'] = {
|
|
svc: rss_trend(sampler.samples, svc, 60, sampler_start)
|
|
for svc in (('otel-collector', 'api', 'worker') if service == 'collector' or service == 'otel-collector'
|
|
else ('otel-collector',))}
|
|
out['sdk_queue_diagnostic_messages'] = sdk_overflow_logs(since)
|
|
# -- stdout vs backends for requests accepted during the outage.
|
|
inject_unix = utc_epoch(tl['inject']['utc'])
|
|
during = [s for s in incident_load['slots'] if s.get('outcome') == 'http'
|
|
and s.get('request_id') and s['kind'] == 'message']
|
|
stdout = reconcile.stdout_http_logs(since)
|
|
# slots are relative to the load t0; classify by stdout timestamp instead.
|
|
in_window = []
|
|
for s in during:
|
|
row = stdout.get(s['request_id'])
|
|
if row is None:
|
|
continue
|
|
ts = utc_epoch(row['timestamp'])
|
|
if inject_unix <= ts <= inject_unix + 90:
|
|
in_window.append((s, row))
|
|
sampled_window = [(s, r) for s, r in in_window if r.get('trace_sampled') is True]
|
|
step = max(1, len(sampled_window) // 100)
|
|
probe = sampled_window[::step][:100]
|
|
targets = {s['request_id']: (r['trace_id'], r['span_id']) for s, r in probe}
|
|
found = reconcile.tempo_server_spans(ctx.obs, targets, deadline_s=60)
|
|
out['stdout_vs_tempo'] = {
|
|
'requests_in_outage_window_with_stdout': len(in_window),
|
|
'sampled_in_window_per_stdout': len(sampled_window),
|
|
'tempo_probe_size': len(probe),
|
|
'tempo_probe_missing_server_spans': sum(1 for v in found.values() if not v),
|
|
'note': 'loss is measured for SERVER root spans only, on a systematic probe '
|
|
'sample; it does not count lost child spans'}
|
|
if not in_window:
|
|
out['problems'].append('no stdout http.request lines in the outage window')
|
|
if any(not r.get('trace_id') or r.get('trace_sampled') is not True for _, r in sampled_window):
|
|
out['problems'].append('stdout trace fields missing')
|
|
if service == 'tempo':
|
|
c = out['collector_during_outage']
|
|
if (c['queue_size_max'] or 0) > (c['queue_capacity'] or 128):
|
|
out['problems'].append('Collector queue exceeded capacity')
|
|
if not (c['send_failed_spans_increase'] or 0) > 0:
|
|
out['problems'].append('Collector did not count failed span exports')
|
|
if not (c['logs_sent_increase'] or 0) > 0 or not (c['metrics_sent_increase'] or 0) > 0:
|
|
out['problems'].append('Loki/Prometheus paths stopped during the Tempo outage')
|
|
else:
|
|
post_failed = ctx.obs.prom_scalar('sum(messenger_telemetry_export_items_total{outcome="failed"})', 0.0)
|
|
out['app_export_failed_items'] = {'before': pre_failed, 'after': post_failed,
|
|
'delta': post_failed - pre_failed if post_failed >= pre_failed else None,
|
|
'reason': None if post_failed >= pre_failed else 'counter_reset'}
|
|
if not (out['app_export_failed_items']['delta'] or 0) > 0:
|
|
out['problems'].append('app failed-export counter did not grow')
|
|
resumed = {name: ctx.obs.prom_scalar(expr, 0.0) for name, expr in (
|
|
('spans_sent_300s', 'sum(increase(otelcol_exporter_sent_spans[300s]))'),
|
|
('logs_sent_300s', 'sum(increase(otelcol_exporter_sent_log_records[300s]))'),
|
|
('metrics_sent_300s', 'sum(increase(otelcol_exporter_sent_metric_points[300s]))'))}
|
|
out['signals_resumed'] = resumed
|
|
if not all(v > 0 for v in resumed.values()):
|
|
out['problems'].append('not all three signals resumed')
|
|
return out
|
|
|
|
|
|
def scenario_telemetry(ctx, label=''):
|
|
alice, chat = ctx.actor_chat()
|
|
result = {'scenario': 'telemetry', 'repeat': label, 'problems': [], 'outages': []}
|
|
for service in ('tempo', 'otel-collector'):
|
|
outage = telemetry_outage(ctx, service, alice, chat, label)
|
|
result['outages'].append(outage)
|
|
result['problems'] += [service + ': ' + p for p in outage['problems']]
|
|
name = 'incident-telemetry-%s.json' % label
|
|
finish(ctx, name, result)
|
|
|
|
|
|
SCENARIOS = {'worker': scenario_worker, 'db': scenario_db, 'telemetry': scenario_telemetry}
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--scenario', required=True, choices=sorted(SCENARIOS))
|
|
parser.add_argument('--repeat', type=int, default=1)
|
|
parser.add_argument('--base-url', default='http://localhost:8080')
|
|
parser.add_argument('--ca-file')
|
|
parser.add_argument('--prometheus-url', default='http://localhost:9090')
|
|
parser.add_argument('--grafana-url', default='http://localhost:3000')
|
|
parser.add_argument('--output-dir', default='evidence/lab6')
|
|
args = parser.parse_args(argv)
|
|
ctx = Ctx(args)
|
|
failed = False
|
|
try:
|
|
with ctx.obs.app_environment(ENV):
|
|
for repeat in range(1, args.repeat + 1):
|
|
try:
|
|
SCENARIOS[args.scenario](ctx, str(repeat))
|
|
except Failure as error:
|
|
failed = True
|
|
print('FAIL: ' + str(error), file=sys.stderr)
|
|
break
|
|
except Failure as error:
|
|
print('FAIL: ' + str(error), file=sys.stderr)
|
|
return 1
|
|
return 1 if failed else 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|