#!/usr/bin/env python3 """lab6 track 1 "cost of observability": off / full / p25 / p10 under the same fixed-rate workload. Stdlib only. The protocol (rate calibration, 3x4 matrix, diagnostic cohorts, p25 acceptance, one independent validation) is frozen in this file; see REPORT.md for the description.""" import argparse import json import os import resource import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import incidents # noqa: E402 import measurement as m # noqa: E402 import obs # noqa: E402 import reconcile # noqa: E402 import resources # noqa: E402 import verify # noqa: E402 import workload # noqa: E402 from obs import Failure, stand, utc_now # noqa: E402 BASELINE_ENV = dict(obs.SDK_DEFAULTS) BASELINE_ENV.update({'CACHE_MODE': 'protected', 'FAULT_INJECTION_ENABLED': '1', 'LOG_LEVEL': 'INFO'}) WARMUP_S, MEASURE_S = 60, 180 CALIBRATION = ((20, 20, 40), (10, 20, 40), (5, 20, 40)) # rate, warmup, measure N_SLOW, ERR_RATE, ERR_SECONDS = 40, 4, 10 COUNTERS = ( ('accepted_spans', 'sum(otelcol_receiver_accepted_spans)', False), ('accepted_log_records', 'sum(otelcol_receiver_accepted_log_records)', False), ('accepted_metric_points', 'sum(otelcol_receiver_accepted_metric_points)', False), ('sent_spans', 'sum(otelcol_exporter_sent_spans)', False), ('sent_log_records', 'sum(otelcol_exporter_sent_log_records)', False), ('sent_metric_points', 'sum(otelcol_exporter_sent_metric_points)', False), ('send_failed_spans', 'sum(otelcol_exporter_send_failed_spans)', True), ('send_failed_log_records', 'sum(otelcol_exporter_send_failed_log_records)', True), ('send_failed_metric_points', 'sum(otelcol_exporter_send_failed_metric_points)', True), ('enqueue_failed_spans', 'sum(otelcol_exporter_enqueue_failed_spans)', True), ('enqueue_failed_log_records', 'sum(otelcol_exporter_enqueue_failed_log_records)', True), ('enqueue_failed_metric_points', 'sum(otelcol_exporter_enqueue_failed_metric_points)', True), ('tempo_bytes_received', 'sum(tempo_distributor_bytes_received_total)', False), ('loki_bytes_received', 'sum(loki_distributor_bytes_received_total)', False), ) def write(path, data): Path(path).parent.mkdir(parents=True, exist_ok=True) Path(path).write_text(json.dumps(data, indent=2, ensure_ascii=False, sort_keys=True) + '\n') def unique_dir(base): base = Path(base) if not base.exists(): return base n = 2 while Path('%s-attempt-%d' % (base, n)).exists(): n += 1 return Path('%s-attempt-%d' % (base, n)) def snapshot(client, instance_ids, sdk_on): snap = {'utc': utc_now(), 'values': {}, 'sample_ts': {}} for name, expr, _ in COUNTERS: result = client.prom_query(expr) snap['values'][name] = float(result[0]['value'][1]) if result else None snap['sample_ts'][name] = float(result[0]['value'][0]) if result else None if sdk_on: selector = 'instance=~"%s"' % '|'.join(instance_ids) for outcome in ('failed', 'success'): result = client.prom_query( 'sum(messenger_telemetry_export_items_total{outcome="%s",%s})' % (outcome, selector)) snap['values']['app_export_' + outcome] = float(result[0]['value'][1]) if result else None return snap def deltas(a, b, operations, sdk_on): out = {} for name, _, failure_only in COUNTERS: delta, reason = m.counter_delta(a['values'][name], b['values'][name], failure_only) if name.endswith('_bytes_received') and reason == 'missing_series': out[name] = {'status': 'not_exposed'} continue out[name] = {'start': a['values'][name], 'end': b['values'][name], 'delta': delta, 'reason': reason, 'per_operation': (delta / operations) if delta is not None and operations else None} for outcome in ('failed', 'success'): key = 'app_export_' + outcome if not sdk_on: out[key] = {'status': 'not_applicable'} else: delta, reason = m.counter_delta(a['values'].get(key), b['values'].get(key), failure_only=(outcome == 'failed')) out[key] = {'start': a['values'].get(key), 'end': b['values'].get(key), 'delta': delta, 'reason': reason} return out def container_health(ids): """{short_id: (service, restarts, oom)} from docker inspect.""" out = {} for cid, service in ids.items(): data = json.loads(stand.docker('inspect', cid).stdout)[0] out[cid] = (service, data['RestartCount'], bool(data['State'].get('OOMKilled')), data['State'].get('Running')) return out def rusage(): own, kids = resource.getrusage(resource.RUSAGE_SELF), resource.getrusage(resource.RUSAGE_CHILDREN) return {'self_cpu_s': own.ru_utime + own.ru_stime, 'children_cpu_s': kids.ru_utime + kids.ru_stime, 'loadavg': os.getloadavg()} class Track: def __init__(self, args): self.args = args self.obs = verify.make_obs(args) self.out = Path(args.output_dir) self.image_ids = None self.set_env = None # -- environment def configure(self, mode, force=True): env = dict(BASELINE_ENV) env.update(m.MODES[mode]) state = self.set_env(env, force=force) for service in ('api', 'worker'): for cid, info in state[service].items(): for key, value in env.items(): if info['env'][key] != value: raise Failure('effective %s=%s in %s %s' % (key, info['env'][key], service, cid)) images = {svc: sorted(i['image'] for i in state[svc].values()) for svc in ('api', 'worker')} if self.image_ids is None: self.image_ids = images ok = images == self.image_ids instance_ids = sorted(list(state['api']) + list(state['worker'])) return state, instance_ids, ok, {k: env[k] for k in obs.MUTABLE_KEYS if k in env} # -- one protocol run def execute_run(self, mode, rate, run_dir, repeat=None, warmup=WARMUP_S, measure=MEASURE_S, n_slow=N_SLOW, err_rate=ERR_RATE, err_seconds=ERR_SECONDS, label=None): run_dir = Path(run_dir) run_dir.mkdir(parents=True, exist_ok=True) sdk_on = mode != 'off' defects, stamps = [], {} self.obs.require_no_foreign_faults() pre = obs.wait_drained([], obs=self.obs, timeout=180) if not pre['drained']: raise Failure('stack did not drain before the run (%s)' % mode) state, instance_ids, image_ok, effective = self.configure(mode) ids = resources.container_map() health0 = container_health(ids) c = stand.new_client(self.args.base_url, self.args.ca_file) alice, _, _, chat = c.bootstrap() stamps['warmup_start'] = utc_now() warm = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate, warmup, 'warmup', settle=False) warm_drain = obs.wait_drained(warm['uploads'], obs=self.obs, timeout=180) stamps['warmup_end'] = utc_now() if not warm_drain['drained']: defects.append('warmup_did_not_drain') snap0 = snapshot(self.obs, instance_ids, sdk_on) usage0 = rusage() sampler = resources.ResourceSampler(ids, 5.0, run_dir / 'docker-stats.jsonl') sampler.start() stamps['measure_start'] = utc_now() result = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate, measure, 'measure', settle=False) sampler.stop() stamps['measure_dispatch_end'] = utc_now() usage1 = rusage() drain = obs.wait_drained(result['uploads'], obs=self.obs, timeout=180) snap1 = snapshot(self.obs, instance_ids, sdk_on) stamps['measure_drained'] = utc_now() load = result['load'] # -- integrity after the final ingest snapshot (HTTP is allowed now) integrity = 0 reference = stand.sha256(stand.sample_bytes()) for upload_id in result['uploads']: meta = stand.metadata(c, alice, upload_id) if meta['status'] != 'ready' or meta['sha256'] != reference: integrity += 1 not_ready = sum(1 for s in drain['states'].values() if s != 'ready') write(run_dir / 'drain.json', {'warmup': warm_drain, 'measurement': drain}) # -- diagnostic cohorts, never mixed with the healthy statistics cohort_since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(time.time() - 2)) cohort_start_ns = time.time_ns() - 10 * 10 ** 9 stamps['cohorts_start'] = utc_now() slow_rows = incidents.sequential_uploads(self, alice, chat, n_slow, 500) slow_valid = all(r['consumed'] and r['status'] == 202 for r in slow_rows) err_slots = [] with obs.hold_table_lock('messages'): err = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, err_rate, err_seconds, 'error', upload_every=0, settle=False) err_slots = err['slots'] err_expected = err_rate * err_seconds err_valid = (len(err_slots) == err_expected and all( s['outcome'] == 'http' and s['status'] >= 500 and s.get('instance_id') and s.get('request_id') for s in err_slots)) stamps['cohorts_end'] = utc_now() slow_drain = obs.wait_drained([r['attachment_id'] for r in slow_rows if r['attachment_id']], obs=self.obs, timeout=180) snap2 = snapshot(self.obs, instance_ids, sdk_on) end_ns = time.time_ns() + 60 * 10 ** 9 slow_records, slow_summary = reconcile.reconcile( self.obs, mode, [{'request_id': r['request_id'], 'status': r['status']} for r in slow_rows], cohort_since, cohort_start_ns, end_ns) err_records, err_summary = reconcile.reconcile( self.obs, mode, [{'request_id': s.get('request_id'), 'status': s.get('status')} for s in err_slots if s.get('request_id')], cohort_since, cohort_start_ns, end_ns) slow_summary['invalid'] = not slow_valid err_summary['invalid'] = not err_valid err_summary['responses'] = len(err_slots) write(run_dir / 'cohorts.json', {'slow': {'requests': slow_rows, 'records': slow_records, 'summary': slow_summary}, 'error': {'slots': [{k: s.get(k) for k in ( 'i', 'outcome', 'status', 'request_id', 'instance_id', 'service_ms')} for s in err_slots], 'records': err_records, 'summary': err_summary}}) health1 = container_health(ids) restarts = [health1[cid][0] for cid in ids if health1[cid][1] != health0[cid][1] or health1[cid][2]] # -- independent proof that SDK-off sends nothing d_final = deltas(snap0, snap2, load['completed'], sdk_on) if not sdk_on: for name in ('accepted_spans', 'accepted_log_records', 'accepted_metric_points'): if (d_final[name]['delta'] or 0) != 0: defects.append('sdk_off_but_collector_ingest:' + name) ingest = deltas(snap0, snap1, load['completed'], sdk_on) loss = m.export_loss( ingest['app_export_failed'].get('delta') if sdk_on else None, sum(ingest[k]['delta'] or 0 for k in ingest if k.startswith('send_failed')), sum(ingest[k]['delta'] or 0 for k in ingest if k.startswith('enqueue_failed'))) res_summary = resources.summarize(sampler.samples) spans_op = ingest['accepted_spans']['per_operation'] summary = { 'mode': mode, 'sampler_ratio': m.SAMPLER_RATIO[mode], 'repeat': repeat, 'label': label, 'rate_rps': rate, 'warmup_s': warmup, 'measurement_s': measure, 'effective_env': effective, 'effective_mode_ok': True, 'image_ids': self.image_ids, 'image_ids_ok': image_ok, 'instance_ids': instance_ids, 'load': load, 'resources': res_summary, 'controller_usage': {'start': usage0, 'end': usage1, 'cpu_s_during_dispatch': (usage1['self_cpu_s'] + usage1['children_cpu_s'] - usage0['self_cpu_s'] - usage0['children_cpu_s'])}, 'ingest': ingest, 'ingest_through_cohorts': d_final, 'export_failures': loss, 'spans_per_operation': spans_op, 'log_records_per_operation': ingest['accepted_log_records']['per_operation'], 'metric_points_per_operation': ingest['accepted_metric_points']['per_operation'], 'cohorts': {'slow': slow_summary, 'error': err_summary}, 'not_ready_at_deadline': not_ready, 'integrity_errors': integrity, 'restarts': restarts, 'measurement_defects': defects, 'timestamps': stamps, 'foreign_faults': self.obs.foreign_faults(), 'dir': str(run_dir), 'slow_drain': {k: slow_drain[k] for k in ('drained', 'seconds')}, } valid, invalid, failures = m.evaluate_run(summary) summary.update(valid=valid, invalid_reasons=invalid, configuration_failures=failures) with (run_dir / 'requests.jsonl').open('w') as handle: for slot in result['slots']: handle.write(json.dumps({k: slot.get(k) for k in ( 'i', 'kind', 'scheduled', 'start', 'end', 'service_ms', 'scheduled_ms', 'start_lag_ms', 'status', 'outcome', 'request_id', 'attachment_id', 'error')}) + '\n') write(run_dir / 'summary.json', summary) print('run %s -> valid=%s invalid=%s failures=%s spans/op=%s p95=%s' % ( run_dir, valid, invalid, failures, spans_op, load['scheduled_ms']['p95']), flush=True) return summary # -- calibration def calibrate(self): rows = [] self.configure('full') for rate, warm, measure in CALIBRATION: ctx_dir = self.out / 'calibration' / ('rate-%d' % rate) ctx_dir.mkdir(parents=True, exist_ok=True) c = stand.new_client(self.args.base_url, self.args.ca_file) alice, _, _, chat = c.bootstrap() workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate, warm, 'cal-warmup', settle=False) result = workload.run_window(self.args.base_url, self.args.ca_file, alice, chat, rate, measure, 'cal-measure', settle=True, obs_client=self.obs) load = result['load'] drain = result['drain'] checks = { 'zero_client_dropped': load['client_dropped'] == 0, 'zero_http_errors': m.load_errors(load) == 0 and load['http_4xx'] == 0, 'all_uploads_ready': drain['drained'] and all(s == 'ready' for s in drain['states'].values()), 'start_lag_p95_le_50ms': (load['start_lag_ms_p95'] or 1e9) <= m.START_LAG_LIMIT_MS, 'completed_ge_99pct': (load['completed_ratio'] or 0) >= m.COMPLETION_LIMIT, } row = {'rate_rps': rate, 'warmup_s': warm, 'measure_s': measure, 'load': load, 'drain': drain, 'checks': checks, 'passed': all(checks.values())} rows.append(row) write(ctx_dir / 'calibration.json', row) print('calibration rate=%d passed=%s %s' % (rate, row['passed'], checks), flush=True) if row['passed']: write(self.out / 'calibration.json', {'selected_rate_rps': rate, 'candidates': rows}) return rate write(self.out / 'calibration.json', {'selected_rate_rps': None, 'candidates': rows}) raise Failure('no candidate rate (20/10/5) passed; fix the stand before the study') # -- smoke def smoke(self): out = self.out / 'smoke' results = {} problems = [] for mode in ('off', 'full', 'p25', 'p10'): s = self.execute_run(mode, 5, unique_dir(out / mode), warmup=5, measure=15, n_slow=3, err_rate=2, err_seconds=2, label='smoke') results[mode] = {'valid': s['valid'], 'invalid_reasons': s['invalid_reasons'], 'spans_per_operation': s['spans_per_operation'], 'cohorts': s['cohorts'], 'effective_otel': {k: s['effective_env'][k] for k in ('OTEL_SDK_DISABLED', 'OTEL_TRACES_SAMPLER_ARG')}} if not s['valid']: problems.append('%s: invalid %s' % (mode, s['invalid_reasons'])) if mode == 'off' and (s['ingest_through_cohorts']['accepted_spans']['delta'] or 0) != 0: problems.append('off sent spans') if mode == 'full' and not s['cohorts']['error'].get('sampled_present'): problems.append('full has no sampled roots in Tempo') write(out / 'smoke.json', {'note': 'technical smoke only; not a study result', 'results': results, 'problems': problems}) if problems: raise Failure('smoke: ' + '; '.join(problems)) print('PASS: smoke (technical only, not a study result)') # -- full protocol def full(self, attempt_label): rate = self.calibrate() base = self.out / 'runs' runs, entries = [], [] for repeat, order in enumerate(m.ORDERS, start=1): for mode in order: s = self.execute_run(mode, rate, unique_dir(base / ('%d-%s' % (repeat, mode))), repeat=repeat) runs.append(s) entries.append(m.matrix_entry(s)) write(self.out / 'matrix.json', {'rate_rps': rate, 'runs': entries, 'complete': False}) if not s['valid']: raise Failure('protocol-invalid run %s: %s (fix the measurement defect and ' 'rerun the whole block; raw evidence kept)' % (s['dir'], s['invalid_reasons'])) by_mode = {mode: [r for r in runs if r['mode'] == mode] for mode in m.MODES} write(self.out / 'matrix.json', {'rate_rps': rate, 'runs': entries, 'complete': True, 'comparisons': comparisons(by_mode)}) selection = m.select_p25(by_mode['full'], by_mode['p25']) ref = selection['reference'] selected = 'p25' if selection['passed'] else 'full' write(self.out / 'selected.json', { 'selected': selected, 'candidate': 'p25', 'candidate_passed': selection['passed'], 'selection': selection, 'rate_rps': rate, 'rare_event_miss_all_probability_model': rare_events(by_mode)}) validations = [] verdict = None order = [selected] if selected == 'full' else ['p25', 'full'] for mode in order: s = self.execute_run(mode, rate, unique_dir(self.out / 'validation' / mode), label='validation') result = m.validate_single(s, mode, ref) if ref['complete'] else { 'passed': False, 'checks': {'frozen_full_reference_complete': False}, 'mode': mode} validations.append({'mode': mode, 'run_dir': s['dir'], **result}) if result['passed']: verdict = mode break final = {'selected_before_validation': selected, 'validated_mode': verdict, 'validations': validations, 'passed': verdict is not None, 'frozen_full_reference': ref} write(self.out / 'selected-validation.json', final) if verdict is None: raise Failure('selected-validation FAIL: neither p25 nor full passed independent validation') print('PASS: track1 selected=%s validated=%s' % (selected, verdict)) def metric_paths(run): r = run['resources'] api = r.get('api', {}) return { 'scheduled_p95_ms': run['load']['scheduled_ms']['p95'], 'service_p95_ms': run['load']['service_ms']['p95'], 'scheduled_p99_ms': run['load']['scheduled_ms']['p99'], 'success_scheduled_p95_ms': run['load']['success_scheduled_ms']['p95'], 'api_cpu_pct_sum_replicas': api.get('cpu_pct_time_weighted_mean'), 'worker_cpu_pct': r.get('worker', {}).get('cpu_pct_time_weighted_mean'), 'collector_cpu_pct': r.get('otel-collector', {}).get('cpu_pct_time_weighted_mean'), 'api_container_mem_median': api.get('container_memory_bytes', {}).get('median'), 'api_rss_median': api.get('process_rss_bytes', {}).get('median'), 'worker_container_mem_median': r.get('worker', {}).get('container_memory_bytes', {}).get('median'), 'spans_per_operation': run.get('spans_per_operation'), 'log_records_per_operation': run.get('log_records_per_operation'), 'metric_points_per_operation': run.get('metric_points_per_operation'), 'retained_error_traces': run['cohorts']['error'].get('retained_traces'), 'retained_slow_traces': run['cohorts']['slow'].get('retained_traces'), } def comparisons(by_mode): table = {mode: {} for mode in by_mode} for mode, runs in by_mode.items(): keys = metric_paths(runs[0]).keys() if runs else [] for key in keys: table[mode][key] = m.spread([metric_paths(r)[key] for r in runs]) versus = {} for mode in by_mode: versus[mode] = {} for key in ('scheduled_p95_ms', 'service_p95_ms', 'api_cpu_pct_sum_replicas', 'worker_cpu_pct', 'api_container_mem_median', 'api_rss_median'): versus[mode][key + '_vs_off'] = m.relative( table[mode].get(key, {}).get('median'), table['off'].get(key, {}).get('median')) for key in ('spans_per_operation', 'log_records_per_operation', 'metric_points_per_operation'): versus[mode][key + '_vs_full'] = m.relative( table[mode].get(key, {}).get('median'), table['full'].get(key, {}).get('median')) failures = {mode: [r['configuration_failures'] for r in runs] for mode, runs in by_mode.items()} return {'per_mode': table, 'relative': versus, 'configuration_failures': failures} def rare_events(by_mode): out = {} for mode, runs in by_mode.items(): ratio = m.SAMPLER_RATIO[mode] if ratio is None: continue for cls in ('error', 'slow'): kept = sum(r['cohorts'][cls].get('retained_traces', 0) for r in runs) n = sum(r['cohorts'][cls].get('n', 0) for r in runs) out['%s/%s' % (mode, cls)] = { 'N_total_over_runs': n, 'observed_kept': kept, 'per_run_N': 40, 'miss_all_probability_per_run_model': m.miss_all_probability(ratio, 40), 'note': '(1-r)^N assumes independent root sampling; a model, not a measurement'} return out def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--base-url', default='http://localhost:8080') parser.add_argument('--ca-file') parser.add_argument('--prometheus-url', default='http://localhost:9090') parser.add_argument('--grafana-url', default='http://localhost:3000') parser.add_argument('--output-dir', default='evidence/lab6/track1') parser.add_argument('--smoke', action='store_true') args = parser.parse_args(argv) track = Track(args) try: with track.obs.app_environment(BASELINE_ENV) as set_env: track.set_env = set_env if args.smoke: track.smoke() else: track.full('attempt') except Failure as error: print('FAIL: ' + str(error), file=sys.stderr) return 1 return 0 if __name__ == '__main__': sys.exit(main())