import sys import unittest from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) import measurement as m # noqa: E402 def slot(i, status=201, service=10.0, lag=1.0, outcome='http'): if outcome != 'http': return {'i': i, 'outcome': outcome, 'start_lag_ms': None} return {'i': i, 'outcome': 'http', 'status': status, 'service_ms': service, 'scheduled_ms': service + lag, 'start_lag_ms': lag} def cohort(**kw): base = {'n': 40, 'retained_traces': 5, 'sampled_present': 5, 'intentional_unsampled': 35} base.update(kw) return base def run(mode='p25', spans=10.0, p95=100.0, failures=(), valid=True, error=None, slow=None): return { 'mode': mode, 'valid': valid, 'invalid_reasons': [], 'configuration_failures': list(failures), 'spans_per_operation': spans, 'load': {'scheduled_ms': {'p95': p95}}, 'cohorts': {'error': error or cohort(), 'slow': slow or cohort()}, 'dir': mode, } def summary(load_slots, **extra): load = m.summarize_load(load_slots, 10) base = {'mode': 'off', 'load': load, 'cohorts': {}, 'ingest': {}, 'export_failures': {'loss': False}} base.update(extra) return base class SamplingIsNotLoss(unittest.TestCase): def test_false_flag_is_not_export_loss_or_missing_span(self): rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16, 'trace_sampled': False}, 'loki': True, 'span': False} self.assertEqual(m.classify_record('p25', rec), ('intentional_unsampled', 'exported')) counts = m.summarize_cohort('p25', [rec] * 3) self.assertEqual(counts.get('sampled_missing', 0), 0) self.assertFalse(m.export_loss(0, 0.0, 0.0)['loss']) def test_sampled_without_span_is_a_measured_loss(self): rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16, 'trace_sampled': True}, 'loki': True, 'span': False} self.assertEqual(m.classify_record('full', rec)[0], 'sampled_missing') def test_missing_stdout_is_not_intentional_sampling(self): self.assertEqual(m.classify_record('p25', {'stdout': None})[0], 'unmatched_stdout') no_flag = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16}} self.assertEqual(m.classify_record('p25', no_flag)[0], 'unmatched_stdout') incomplete = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16, 'trace_sampled': True}, 'loki': None, 'span': None} self.assertEqual(m.classify_record('p25', incomplete), ('incomplete_observation', 'incomplete_observation')) def test_full_observation_without_log_is_missing_exported_log(self): rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16, 'trace_sampled': False}, 'loki': False} self.assertEqual(m.classify_record('p10', rec)[1], 'missing_exported_logs') def test_off_needs_neither_loki_nor_app_counters(self): rec = {'stdout': {'event': 'http.request'}, 'loki': None, 'span': None} self.assertEqual(m.classify_record('off', rec), ('not_applicable', 'not_applicable')) leaked = {'stdout': {'trace_id': 'x'}} self.assertEqual(m.classify_record('off', leaked)[0], 'ids_present_with_sdk_off') s = summary([slot(i) for i in range(100)], cohorts={ 'error': m.summarize_cohort('off', [rec] * 4)}) valid, invalid, failures = m.evaluate_run(s) self.assertTrue(valid, invalid) self.assertEqual(failures, []) class Counters(unittest.TestCase): def test_reset_is_not_a_negative_loss(self): self.assertEqual(m.counter_delta(10.0, 3.0), (None, 'counter_reset')) loss = m.export_loss(None, 0.0, 2.0) self.assertEqual(loss['total_known'], 2.0) self.assertTrue(all(v >= 0 for v in loss['parts'].values() if v is not None)) def test_missing_positive_series_is_not_zero_failure_series_may_be(self): self.assertEqual(m.counter_delta(None, None), (None, 'missing_series')) self.assertEqual(m.counter_delta(None, None, failure_only=True)[0], 0.0) self.assertEqual(m.counter_delta(5.0, 9.0), (4.0, None)) def test_unexplained_reset_invalidates_but_restart_explains_it(self): s = summary([slot(i) for i in range(100)], ingest={'spans': {'delta': None, 'reason': 'counter_reset'}}) self.assertFalse(m.evaluate_run(s)[0]) s['restarts'] = ['collector'] valid, _, failures = m.evaluate_run(s) self.assertTrue(valid) self.assertIn('restart_or_oom', failures) class LoadAccounting(unittest.TestCase): def test_dropped_and_late_slots_stay_in_denominator(self): slots = [slot(i) for i in range(90)] + \ [slot(90 + i, outcome='client_dropped') for i in range(5)] + \ [slot(95 + i, lag=300.0) for i in range(5)] load = m.summarize_load(slots, 10) self.assertEqual(load['scheduled'], 100) self.assertEqual(load['completed'], 95) self.assertAlmostEqual(load['completed_ratio'], 0.95) valid, invalid, _ = m.evaluate_run(summary(slots)) self.assertFalse(valid) self.assertTrue(any('client_dropped' in r for r in invalid)) self.assertTrue(any('completed_ratio' in r for r in invalid)) def test_late_scheduler_fails_start_lag_even_without_drops(self): slots = [slot(i, lag=80.0) for i in range(100)] self.assertFalse(m.evaluate_run(summary(slots))[0]) def test_overall_p95_includes_error_responses(self): slots = [slot(i, service=10.0) for i in range(90)] + \ [slot(90 + i, status=500, service=900.0) for i in range(10)] load = m.summarize_load(slots, 10) self.assertEqual(load['service_ms']['p95'], 900.0) self.assertEqual(load['success_service_ms']['p95'], 10.0) self.assertEqual(m.load_errors(load), 10) def test_nearest_rank(self): self.assertEqual(m.nearest_rank(list(range(1, 101)), 0.95), 95) self.assertEqual(m.nearest_rank([7], 0.99), 7) self.assertIsNone(m.nearest_rank([], 0.5)) class MatrixAndSelection(unittest.TestCase): def test_valid_adverse_run_stays_in_matrix(self): s = summary([slot(i) for i in range(100)], mode='p25', export_failures={'loss': True}, cohorts={'error': cohort(sampled_missing=2)}) valid, invalid, failures = m.evaluate_run(s) self.assertTrue(valid, invalid) self.assertIn('export_loss', failures) self.assertTrue(any(f.startswith('sampled_missing') for f in failures)) s.update(valid=valid, invalid_reasons=invalid, configuration_failures=failures, repeat=1) entry = m.matrix_entry(s) self.assertTrue(entry['valid']) self.assertTrue(entry['configuration_failures']) def full_runs(self): return [run('full', spans=100.0, p95=100.0) for _ in range(3)] def test_p25_selected_only_against_complete_full_reference(self): good = [run('p25', spans=40.0, p95=105.0) for _ in range(3)] self.assertTrue(m.select_p25(self.full_runs(), good)['passed']) broken_full = self.full_runs() broken_full[1]['spans_per_operation'] = None self.assertFalse(m.select_p25(broken_full, good)['passed']) heavy = [run('p25', spans=60.0, p95=105.0) for _ in range(3)] self.assertFalse(m.select_p25(self.full_runs(), heavy)['passed']) slow = [run('p25', spans=40.0, p95=120.0) for _ in range(3)] self.assertFalse(m.select_p25(self.full_runs(), slow)['passed']) def test_p25_rejected_when_an_error_class_has_no_retained_trace(self): runs = [run('p25', spans=40.0, p95=100.0) for _ in range(3)] runs[2]['cohorts']['slow'] = cohort(retained_traces=0, sampled_present=0) self.assertFalse(m.select_p25(self.full_runs(), runs)['passed']) def test_single_validation_uses_frozen_full_reference(self): ref = m.full_reference(self.full_runs()) ok = run('p25', spans=45.0, p95=108.0) self.assertTrue(m.validate_single(ok, 'p25', ref)['passed']) # Would pass a "median of its own" comparison, fails the frozen one. bad = run('p25', spans=55.0, p95=108.0) result = m.validate_single(bad, 'p25', ref) self.assertFalse(result['passed']) self.assertFalse(result['checks']['spans_le_50pct_frozen_full']) def test_full_validation_needs_all_eighty_server_spans(self): ref = m.full_reference(self.full_runs()) ok = run('full', error=cohort(sampled_present=40, retained_traces=40), slow=cohort(sampled_present=40, retained_traces=40)) self.assertTrue(m.validate_single(ok, 'full', ref)['passed']) short = run('full', error=cohort(sampled_present=39, retained_traces=39), slow=cohort(sampled_present=40, retained_traces=40)) self.assertFalse(m.validate_single(short, 'full', ref)['passed']) def test_miss_all_probability_model(self): self.assertAlmostEqual(m.miss_all_probability(0.25, 40), 0.75 ** 40) if __name__ == '__main__': unittest.main()