Files
mtusi2026lab6/scripts/lab6/track1.py

458 lines
24 KiB
Python
Executable File

#!/usr/bin/env python3
"""lab6 track 1 "cost of observability": off / full / p25 / p10 under the same
fixed-rate workload. Stdlib only. The protocol (rate calibration, 3x4 matrix,
diagnostic cohorts, p25 acceptance, one independent validation) is frozen in
this file; see REPORT.md for the description."""
import argparse
import json
import os
import resource
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import incidents # noqa: E402
import measurement as m # noqa: E402
import obs # noqa: E402
import reconcile # noqa: E402
import resources # noqa: E402
import verify # noqa: E402
import workload # noqa: E402
from obs import Failure, stand, utc_now # noqa: E402
BASELINE_ENV = dict(obs.SDK_DEFAULTS)
BASELINE_ENV.update({'CACHE_MODE': 'protected', 'FAULT_INJECTION_ENABLED': '1',
'LOG_LEVEL': 'INFO'})
WARMUP_S, MEASURE_S = 60, 180
CALIBRATION = ((20, 20, 40), (10, 20, 40), (5, 20, 40)) # rate, warmup, measure
N_SLOW, ERR_RATE, ERR_SECONDS = 40, 4, 10
COUNTERS = (
('accepted_spans', 'sum(otelcol_receiver_accepted_spans)', False),
('accepted_log_records', 'sum(otelcol_receiver_accepted_log_records)', False),
('accepted_metric_points', 'sum(otelcol_receiver_accepted_metric_points)', False),
('sent_spans', 'sum(otelcol_exporter_sent_spans)', False),
('sent_log_records', 'sum(otelcol_exporter_sent_log_records)', False),
('sent_metric_points', 'sum(otelcol_exporter_sent_metric_points)', False),
('send_failed_spans', 'sum(otelcol_exporter_send_failed_spans)', True),
('send_failed_log_records', 'sum(otelcol_exporter_send_failed_log_records)', True),
('send_failed_metric_points', 'sum(otelcol_exporter_send_failed_metric_points)', True),
('enqueue_failed_spans', 'sum(otelcol_exporter_enqueue_failed_spans)', True),
('enqueue_failed_log_records', 'sum(otelcol_exporter_enqueue_failed_log_records)', True),
('enqueue_failed_metric_points', 'sum(otelcol_exporter_enqueue_failed_metric_points)', True),
('tempo_bytes_received', 'sum(tempo_distributor_bytes_received_total)', False),
('loki_bytes_received', 'sum(loki_distributor_bytes_received_total)', False),
)
def write(path, data):
Path(path).parent.mkdir(parents=True, exist_ok=True)
Path(path).write_text(json.dumps(data, indent=2, ensure_ascii=False, sort_keys=True) + '\n')
def unique_dir(base):
base = Path(base)
if not base.exists():
return base
n = 2
while Path('%s-attempt-%d' % (base, n)).exists():
n += 1
return Path('%s-attempt-%d' % (base, n))
def snapshot(client, instance_ids, sdk_on):
snap = {'utc': utc_now(), 'values': {}, 'sample_ts': {}}
for name, expr, _ in COUNTERS:
result = client.prom_query(expr)
snap['values'][name] = float(result[0]['value'][1]) if result else None
snap['sample_ts'][name] = float(result[0]['value'][0]) if result else None
if sdk_on:
selector = 'instance=~"%s"' % '|'.join(instance_ids)
for outcome in ('failed', 'success'):
result = client.prom_query(
'sum(messenger_telemetry_export_items_total{outcome="%s",%s})' % (outcome, selector))
snap['values']['app_export_' + outcome] = float(result[0]['value'][1]) if result else None
return snap
def deltas(a, b, operations, sdk_on):
out = {}
for name, _, failure_only in COUNTERS:
delta, reason = m.counter_delta(a['values'][name], b['values'][name], failure_only)
if name.endswith('_bytes_received') and reason == 'missing_series':
out[name] = {'status': 'not_exposed'}
continue
out[name] = {'start': a['values'][name], 'end': b['values'][name],
'delta': delta, 'reason': reason,
'per_operation': (delta / operations) if delta is not None and operations else None}
for outcome in ('failed', 'success'):
key = 'app_export_' + outcome
if not sdk_on:
out[key] = {'status': 'not_applicable'}
else:
delta, reason = m.counter_delta(a['values'].get(key), b['values'].get(key),
failure_only=(outcome == 'failed'))
out[key] = {'start': a['values'].get(key), 'end': b['values'].get(key),
'delta': delta, 'reason': reason}
return out
def container_health(ids):
"""{short_id: (service, restarts, oom)} from docker inspect."""
out = {}
for cid, service in ids.items():
data = json.loads(stand.docker('inspect', cid).stdout)[0]
out[cid] = (service, data['RestartCount'], bool(data['State'].get('OOMKilled')),
data['State'].get('Running'))
return out
def rusage():
own, kids = resource.getrusage(resource.RUSAGE_SELF), resource.getrusage(resource.RUSAGE_CHILDREN)
return {'self_cpu_s': own.ru_utime + own.ru_stime, 'children_cpu_s': kids.ru_utime + kids.ru_stime,
'loadavg': os.getloadavg()}
class Track:
def __init__(self, args):
self.args = args
self.obs = verify.make_obs(args)
self.out = Path(args.output_dir)
self.image_ids = None
self.set_env = None
# -- environment
def configure(self, mode, force=True):
env = dict(BASELINE_ENV)
env.update(m.MODES[mode])
state = self.set_env(env, force=force)
for service in ('api', 'worker'):
for cid, info in state[service].items():
for key, value in env.items():
if info['env'][key] != value:
raise Failure('effective %s=%s in %s %s' % (key, info['env'][key], service, cid))
images = {svc: sorted(i['image'] for i in state[svc].values()) for svc in ('api', 'worker')}
if self.image_ids is None:
self.image_ids = images
ok = images == self.image_ids
instance_ids = sorted(list(state['api']) + list(state['worker']))
return state, instance_ids, ok, {k: env[k] for k in obs.MUTABLE_KEYS if k in env}
# -- one protocol run
def execute_run(self, mode, rate, run_dir, repeat=None, warmup=WARMUP_S, measure=MEASURE_S,
n_slow=N_SLOW, err_rate=ERR_RATE, err_seconds=ERR_SECONDS, label=None):
run_dir = Path(run_dir)
run_dir.mkdir(parents=True, exist_ok=True)
sdk_on = mode != 'off'
defects, stamps = [], {}
self.obs.require_no_foreign_faults()
pre = obs.wait_drained([], obs=self.obs, timeout=180)
if not pre['drained']:
raise Failure('stack did not drain before the run (%s)' % mode)
state, instance_ids, image_ok, effective = self.configure(mode)
ids = resources.container_map()
health0 = container_health(ids)
c = stand.new_client(self.args.base_url, self.args.ca_file)
alice, _, _, chat = c.bootstrap()
stamps['warmup_start'] = utc_now()
warm = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate,
warmup, 'warmup', settle=False)
warm_drain = obs.wait_drained(warm['uploads'], obs=self.obs, timeout=180)
stamps['warmup_end'] = utc_now()
if not warm_drain['drained']:
defects.append('warmup_did_not_drain')
snap0 = snapshot(self.obs, instance_ids, sdk_on)
usage0 = rusage()
sampler = resources.ResourceSampler(ids, 5.0, run_dir / 'docker-stats.jsonl')
sampler.start()
stamps['measure_start'] = utc_now()
result = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate,
measure, 'measure', settle=False)
sampler.stop()
stamps['measure_dispatch_end'] = utc_now()
usage1 = rusage()
drain = obs.wait_drained(result['uploads'], obs=self.obs, timeout=180)
snap1 = snapshot(self.obs, instance_ids, sdk_on)
stamps['measure_drained'] = utc_now()
load = result['load']
# -- integrity after the final ingest snapshot (HTTP is allowed now)
integrity = 0
reference = stand.sha256(stand.sample_bytes())
for upload_id in result['uploads']:
meta = stand.metadata(c, alice, upload_id)
if meta['status'] != 'ready' or meta['sha256'] != reference:
integrity += 1
not_ready = sum(1 for s in drain['states'].values() if s != 'ready')
write(run_dir / 'drain.json', {'warmup': warm_drain, 'measurement': drain})
# -- diagnostic cohorts, never mixed with the healthy statistics
cohort_since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time() - 2))
cohort_start_ns = time.time_ns() - 10 * 10 ** 9
stamps['cohorts_start'] = utc_now()
slow_rows = incidents.sequential_uploads(self, alice, chat, n_slow, 500)
slow_valid = all(r['consumed'] and r['status'] == 202 for r in slow_rows)
err_slots = []
with obs.hold_table_lock('messages'):
err = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat,
err_rate, err_seconds, 'error', upload_every=0, settle=False)
err_slots = err['slots']
err_expected = err_rate * err_seconds
err_valid = (len(err_slots) == err_expected and all(
s['outcome'] == 'http' and s['status'] >= 500 and s.get('instance_id') and s.get('request_id')
for s in err_slots))
stamps['cohorts_end'] = utc_now()
slow_drain = obs.wait_drained([r['attachment_id'] for r in slow_rows if r['attachment_id']],
obs=self.obs, timeout=180)
snap2 = snapshot(self.obs, instance_ids, sdk_on)
end_ns = time.time_ns() + 60 * 10 ** 9
slow_records, slow_summary = reconcile.reconcile(
self.obs, mode, [{'request_id': r['request_id'], 'status': r['status']} for r in slow_rows],
cohort_since, cohort_start_ns, end_ns)
err_records, err_summary = reconcile.reconcile(
self.obs, mode, [{'request_id': s.get('request_id'), 'status': s.get('status')}
for s in err_slots if s.get('request_id')],
cohort_since, cohort_start_ns, end_ns)
slow_summary['invalid'] = not slow_valid
err_summary['invalid'] = not err_valid
err_summary['responses'] = len(err_slots)
write(run_dir / 'cohorts.json', {'slow': {'requests': slow_rows, 'records': slow_records,
'summary': slow_summary},
'error': {'slots': [{k: s.get(k) for k in (
'i', 'outcome', 'status', 'request_id', 'instance_id', 'service_ms')}
for s in err_slots],
'records': err_records, 'summary': err_summary}})
health1 = container_health(ids)
restarts = [health1[cid][0] for cid in ids
if health1[cid][1] != health0[cid][1] or health1[cid][2]]
# -- independent proof that SDK-off sends nothing
d_final = deltas(snap0, snap2, load['completed'], sdk_on)
if not sdk_on:
for name in ('accepted_spans', 'accepted_log_records', 'accepted_metric_points'):
if (d_final[name]['delta'] or 0) != 0:
defects.append('sdk_off_but_collector_ingest:' + name)
ingest = deltas(snap0, snap1, load['completed'], sdk_on)
loss = m.export_loss(
ingest['app_export_failed'].get('delta') if sdk_on else None,
sum(ingest[k]['delta'] or 0 for k in ingest if k.startswith('send_failed')),
sum(ingest[k]['delta'] or 0 for k in ingest if k.startswith('enqueue_failed')))
res_summary = resources.summarize(sampler.samples)
spans_op = ingest['accepted_spans']['per_operation']
summary = {
'mode': mode, 'sampler_ratio': m.SAMPLER_RATIO[mode], 'repeat': repeat, 'label': label,
'rate_rps': rate, 'warmup_s': warmup, 'measurement_s': measure,
'effective_env': effective, 'effective_mode_ok': True,
'image_ids': self.image_ids, 'image_ids_ok': image_ok, 'instance_ids': instance_ids,
'load': load, 'resources': res_summary,
'controller_usage': {'start': usage0, 'end': usage1,
'cpu_s_during_dispatch': (usage1['self_cpu_s'] + usage1['children_cpu_s']
- usage0['self_cpu_s'] - usage0['children_cpu_s'])},
'ingest': ingest, 'ingest_through_cohorts': d_final, 'export_failures': loss,
'spans_per_operation': spans_op,
'log_records_per_operation': ingest['accepted_log_records']['per_operation'],
'metric_points_per_operation': ingest['accepted_metric_points']['per_operation'],
'cohorts': {'slow': slow_summary, 'error': err_summary},
'not_ready_at_deadline': not_ready, 'integrity_errors': integrity,
'restarts': restarts, 'measurement_defects': defects, 'timestamps': stamps,
'foreign_faults': self.obs.foreign_faults(), 'dir': str(run_dir),
'slow_drain': {k: slow_drain[k] for k in ('drained', 'seconds')},
}
valid, invalid, failures = m.evaluate_run(summary)
summary.update(valid=valid, invalid_reasons=invalid, configuration_failures=failures)
with (run_dir / 'requests.jsonl').open('w') as handle:
for slot in result['slots']:
handle.write(json.dumps({k: slot.get(k) for k in (
'i', 'kind', 'scheduled', 'start', 'end', 'service_ms', 'scheduled_ms',
'start_lag_ms', 'status', 'outcome', 'request_id', 'attachment_id', 'error')}) + '\n')
write(run_dir / 'summary.json', summary)
print('run %s -> valid=%s invalid=%s failures=%s spans/op=%s p95=%s' % (
run_dir, valid, invalid, failures, spans_op, load['scheduled_ms']['p95']), flush=True)
return summary
# -- calibration
def calibrate(self):
rows = []
self.configure('full')
for rate, warm, measure in CALIBRATION:
ctx_dir = self.out / 'calibration' / ('rate-%d' % rate)
ctx_dir.mkdir(parents=True, exist_ok=True)
c = stand.new_client(self.args.base_url, self.args.ca_file)
alice, _, _, chat = c.bootstrap()
workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate, warm,
'cal-warmup', settle=False)
result = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate,
measure, 'cal-measure', settle=True, obs_client=self.obs)
load = result['load']
drain = result['drain']
checks = {
'zero_client_dropped': load['client_dropped'] == 0,
'zero_http_errors': m.load_errors(load) == 0 and load['http_4xx'] == 0,
'all_uploads_ready': drain['drained'] and all(s == 'ready' for s in drain['states'].values()),
'start_lag_p95_le_50ms': (load['start_lag_ms_p95'] or 1e9) <= m.START_LAG_LIMIT_MS,
'completed_ge_99pct': (load['completed_ratio'] or 0) >= m.COMPLETION_LIMIT,
}
row = {'rate_rps': rate, 'warmup_s': warm, 'measure_s': measure, 'load': load,
'drain': drain, 'checks': checks, 'passed': all(checks.values())}
rows.append(row)
write(ctx_dir / 'calibration.json', row)
print('calibration rate=%d passed=%s %s' % (rate, row['passed'], checks), flush=True)
if row['passed']:
write(self.out / 'calibration.json', {'selected_rate_rps': rate, 'candidates': rows})
return rate
write(self.out / 'calibration.json', {'selected_rate_rps': None, 'candidates': rows})
raise Failure('no candidate rate (20/10/5) passed; fix the stand before the study')
# -- smoke
def smoke(self):
out = self.out / 'smoke'
results = {}
problems = []
for mode in ('off', 'full', 'p25', 'p10'):
s = self.execute_run(mode, 5, unique_dir(out / mode), warmup=5, measure=15,
n_slow=3, err_rate=2, err_seconds=2, label='smoke')
results[mode] = {'valid': s['valid'], 'invalid_reasons': s['invalid_reasons'],
'spans_per_operation': s['spans_per_operation'],
'cohorts': s['cohorts'],
'effective_otel': {k: s['effective_env'][k] for k in
('OTEL_SDK_DISABLED', 'OTEL_TRACES_SAMPLER_ARG')}}
if not s['valid']:
problems.append('%s: invalid %s' % (mode, s['invalid_reasons']))
if mode == 'off' and (s['ingest_through_cohorts']['accepted_spans']['delta'] or 0) != 0:
problems.append('off sent spans')
if mode == 'full' and not s['cohorts']['error'].get('sampled_present'):
problems.append('full has no sampled roots in Tempo')
write(out / 'smoke.json', {'note': 'technical smoke only; not a study result',
'results': results, 'problems': problems})
if problems:
raise Failure('smoke: ' + '; '.join(problems))
print('PASS: smoke (technical only, not a study result)')
# -- full protocol
def full(self, attempt_label):
rate = self.calibrate()
base = self.out / 'runs'
runs, entries = [], []
for repeat, order in enumerate(m.ORDERS, start=1):
for mode in order:
s = self.execute_run(mode, rate, unique_dir(base / ('%d-%s' % (repeat, mode))),
repeat=repeat)
runs.append(s)
entries.append(m.matrix_entry(s))
write(self.out / 'matrix.json', {'rate_rps': rate, 'runs': entries,
'complete': False})
if not s['valid']:
raise Failure('protocol-invalid run %s: %s (fix the measurement defect and '
'rerun the whole block; raw evidence kept)' % (s['dir'], s['invalid_reasons']))
by_mode = {mode: [r for r in runs if r['mode'] == mode] for mode in m.MODES}
write(self.out / 'matrix.json', {'rate_rps': rate, 'runs': entries, 'complete': True,
'comparisons': comparisons(by_mode)})
selection = m.select_p25(by_mode['full'], by_mode['p25'])
ref = selection['reference']
selected = 'p25' if selection['passed'] else 'full'
write(self.out / 'selected.json', {
'selected': selected, 'candidate': 'p25', 'candidate_passed': selection['passed'],
'selection': selection, 'rate_rps': rate,
'rare_event_miss_all_probability_model': rare_events(by_mode)})
validations = []
verdict = None
order = [selected] if selected == 'full' else ['p25', 'full']
for mode in order:
s = self.execute_run(mode, rate, unique_dir(self.out / 'validation' / mode), label='validation')
result = m.validate_single(s, mode, ref) if ref['complete'] else {
'passed': False, 'checks': {'frozen_full_reference_complete': False}, 'mode': mode}
validations.append({'mode': mode, 'run_dir': s['dir'], **result})
if result['passed']:
verdict = mode
break
final = {'selected_before_validation': selected, 'validated_mode': verdict,
'validations': validations, 'passed': verdict is not None,
'frozen_full_reference': ref}
write(self.out / 'selected-validation.json', final)
if verdict is None:
raise Failure('selected-validation FAIL: neither p25 nor full passed independent validation')
print('PASS: track1 selected=%s validated=%s' % (selected, verdict))
def metric_paths(run):
r = run['resources']
api = r.get('api', {})
return {
'scheduled_p95_ms': run['load']['scheduled_ms']['p95'],
'service_p95_ms': run['load']['service_ms']['p95'],
'scheduled_p99_ms': run['load']['scheduled_ms']['p99'],
'success_scheduled_p95_ms': run['load']['success_scheduled_ms']['p95'],
'api_cpu_pct_sum_replicas': api.get('cpu_pct_time_weighted_mean'),
'worker_cpu_pct': r.get('worker', {}).get('cpu_pct_time_weighted_mean'),
'collector_cpu_pct': r.get('otel-collector', {}).get('cpu_pct_time_weighted_mean'),
'api_container_mem_median': api.get('container_memory_bytes', {}).get('median'),
'api_rss_median': api.get('process_rss_bytes', {}).get('median'),
'worker_container_mem_median': r.get('worker', {}).get('container_memory_bytes', {}).get('median'),
'spans_per_operation': run.get('spans_per_operation'),
'log_records_per_operation': run.get('log_records_per_operation'),
'metric_points_per_operation': run.get('metric_points_per_operation'),
'retained_error_traces': run['cohorts']['error'].get('retained_traces'),
'retained_slow_traces': run['cohorts']['slow'].get('retained_traces'),
}
def comparisons(by_mode):
table = {mode: {} for mode in by_mode}
for mode, runs in by_mode.items():
keys = metric_paths(runs[0]).keys() if runs else []
for key in keys:
table[mode][key] = m.spread([metric_paths(r)[key] for r in runs])
versus = {}
for mode in by_mode:
versus[mode] = {}
for key in ('scheduled_p95_ms', 'service_p95_ms', 'api_cpu_pct_sum_replicas',
'worker_cpu_pct', 'api_container_mem_median', 'api_rss_median'):
versus[mode][key + '_vs_off'] = m.relative(
table[mode].get(key, {}).get('median'), table['off'].get(key, {}).get('median'))
for key in ('spans_per_operation', 'log_records_per_operation', 'metric_points_per_operation'):
versus[mode][key + '_vs_full'] = m.relative(
table[mode].get(key, {}).get('median'), table['full'].get(key, {}).get('median'))
failures = {mode: [r['configuration_failures'] for r in runs] for mode, runs in by_mode.items()}
return {'per_mode': table, 'relative': versus, 'configuration_failures': failures}
def rare_events(by_mode):
out = {}
for mode, runs in by_mode.items():
ratio = m.SAMPLER_RATIO[mode]
if ratio is None:
continue
for cls in ('error', 'slow'):
kept = sum(r['cohorts'][cls].get('retained_traces', 0) for r in runs)
n = sum(r['cohorts'][cls].get('n', 0) for r in runs)
out['%s/%s' % (mode, cls)] = {
'N_total_over_runs': n, 'observed_kept': kept,
'per_run_N': 40, 'miss_all_probability_per_run_model': m.miss_all_probability(ratio, 40),
'note': '(1-r)^N assumes independent root sampling; a model, not a measurement'}
return out
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--ca-file')
parser.add_argument('--prometheus-url', default='http://localhost:9090')
parser.add_argument('--grafana-url', default='http://localhost:3000')
parser.add_argument('--output-dir', default='evidence/lab6/track1')
parser.add_argument('--smoke', action='store_true')
args = parser.parse_args(argv)
track = Track(args)
try:
with track.obs.app_environment(BASELINE_ENV) as set_env:
track.set_env = set_env
if args.smoke:
track.smoke()
else:
track.full('attempt')
except Failure as error:
print('FAIL: ' + str(error), file=sys.stderr)
return 1
return 0
if __name__ == '__main__':
sys.exit(main())