forked from mtusi2026/mtusi2026lab6
200 lines
9.2 KiB
Python
200 lines
9.2 KiB
Python
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import measurement as m # noqa: E402
|
|
|
|
|
|
def slot(i, status=201, service=10.0, lag=1.0, outcome='http'):
|
|
if outcome != 'http':
|
|
return {'i': i, 'outcome': outcome, 'start_lag_ms': None}
|
|
return {'i': i, 'outcome': 'http', 'status': status,
|
|
'service_ms': service, 'scheduled_ms': service + lag,
|
|
'start_lag_ms': lag}
|
|
|
|
|
|
def cohort(**kw):
|
|
base = {'n': 40, 'retained_traces': 5, 'sampled_present': 5,
|
|
'intentional_unsampled': 35}
|
|
base.update(kw)
|
|
return base
|
|
|
|
|
|
def run(mode='p25', spans=10.0, p95=100.0, failures=(), valid=True,
|
|
error=None, slow=None):
|
|
return {
|
|
'mode': mode, 'valid': valid, 'invalid_reasons': [],
|
|
'configuration_failures': list(failures),
|
|
'spans_per_operation': spans,
|
|
'load': {'scheduled_ms': {'p95': p95}},
|
|
'cohorts': {'error': error or cohort(), 'slow': slow or cohort()},
|
|
'dir': mode,
|
|
}
|
|
|
|
|
|
def summary(load_slots, **extra):
|
|
load = m.summarize_load(load_slots, 10)
|
|
base = {'mode': 'off', 'load': load, 'cohorts': {}, 'ingest': {},
|
|
'export_failures': {'loss': False}}
|
|
base.update(extra)
|
|
return base
|
|
|
|
|
|
class SamplingIsNotLoss(unittest.TestCase):
|
|
def test_false_flag_is_not_export_loss_or_missing_span(self):
|
|
rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
|
|
'trace_sampled': False}, 'loki': True, 'span': False}
|
|
self.assertEqual(m.classify_record('p25', rec),
|
|
('intentional_unsampled', 'exported'))
|
|
counts = m.summarize_cohort('p25', [rec] * 3)
|
|
self.assertEqual(counts.get('sampled_missing', 0), 0)
|
|
self.assertFalse(m.export_loss(0, 0.0, 0.0)['loss'])
|
|
|
|
def test_sampled_without_span_is_a_measured_loss(self):
|
|
rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
|
|
'trace_sampled': True}, 'loki': True, 'span': False}
|
|
self.assertEqual(m.classify_record('full', rec)[0], 'sampled_missing')
|
|
|
|
def test_missing_stdout_is_not_intentional_sampling(self):
|
|
self.assertEqual(m.classify_record('p25', {'stdout': None})[0],
|
|
'unmatched_stdout')
|
|
no_flag = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16}}
|
|
self.assertEqual(m.classify_record('p25', no_flag)[0], 'unmatched_stdout')
|
|
incomplete = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
|
|
'trace_sampled': True}, 'loki': None, 'span': None}
|
|
self.assertEqual(m.classify_record('p25', incomplete),
|
|
('incomplete_observation', 'incomplete_observation'))
|
|
|
|
def test_full_observation_without_log_is_missing_exported_log(self):
|
|
rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
|
|
'trace_sampled': False}, 'loki': False}
|
|
self.assertEqual(m.classify_record('p10', rec)[1], 'missing_exported_logs')
|
|
|
|
def test_off_needs_neither_loki_nor_app_counters(self):
|
|
rec = {'stdout': {'event': 'http.request'}, 'loki': None, 'span': None}
|
|
self.assertEqual(m.classify_record('off', rec),
|
|
('not_applicable', 'not_applicable'))
|
|
leaked = {'stdout': {'trace_id': 'x'}}
|
|
self.assertEqual(m.classify_record('off', leaked)[0], 'ids_present_with_sdk_off')
|
|
s = summary([slot(i) for i in range(100)], cohorts={
|
|
'error': m.summarize_cohort('off', [rec] * 4)})
|
|
valid, invalid, failures = m.evaluate_run(s)
|
|
self.assertTrue(valid, invalid)
|
|
self.assertEqual(failures, [])
|
|
|
|
|
|
class Counters(unittest.TestCase):
|
|
def test_reset_is_not_a_negative_loss(self):
|
|
self.assertEqual(m.counter_delta(10.0, 3.0), (None, 'counter_reset'))
|
|
loss = m.export_loss(None, 0.0, 2.0)
|
|
self.assertEqual(loss['total_known'], 2.0)
|
|
self.assertTrue(all(v >= 0 for v in loss['parts'].values() if v is not None))
|
|
|
|
def test_missing_positive_series_is_not_zero_failure_series_may_be(self):
|
|
self.assertEqual(m.counter_delta(None, None), (None, 'missing_series'))
|
|
self.assertEqual(m.counter_delta(None, None, failure_only=True)[0], 0.0)
|
|
self.assertEqual(m.counter_delta(5.0, 9.0), (4.0, None))
|
|
|
|
def test_unexplained_reset_invalidates_but_restart_explains_it(self):
|
|
s = summary([slot(i) for i in range(100)],
|
|
ingest={'spans': {'delta': None, 'reason': 'counter_reset'}})
|
|
self.assertFalse(m.evaluate_run(s)[0])
|
|
s['restarts'] = ['collector']
|
|
valid, _, failures = m.evaluate_run(s)
|
|
self.assertTrue(valid)
|
|
self.assertIn('restart_or_oom', failures)
|
|
|
|
|
|
class LoadAccounting(unittest.TestCase):
|
|
def test_dropped_and_late_slots_stay_in_denominator(self):
|
|
slots = [slot(i) for i in range(90)] + \
|
|
[slot(90 + i, outcome='client_dropped') for i in range(5)] + \
|
|
[slot(95 + i, lag=300.0) for i in range(5)]
|
|
load = m.summarize_load(slots, 10)
|
|
self.assertEqual(load['scheduled'], 100)
|
|
self.assertEqual(load['completed'], 95)
|
|
self.assertAlmostEqual(load['completed_ratio'], 0.95)
|
|
valid, invalid, _ = m.evaluate_run(summary(slots))
|
|
self.assertFalse(valid)
|
|
self.assertTrue(any('client_dropped' in r for r in invalid))
|
|
self.assertTrue(any('completed_ratio' in r for r in invalid))
|
|
|
|
def test_late_scheduler_fails_start_lag_even_without_drops(self):
|
|
slots = [slot(i, lag=80.0) for i in range(100)]
|
|
self.assertFalse(m.evaluate_run(summary(slots))[0])
|
|
|
|
def test_overall_p95_includes_error_responses(self):
|
|
slots = [slot(i, service=10.0) for i in range(90)] + \
|
|
[slot(90 + i, status=500, service=900.0) for i in range(10)]
|
|
load = m.summarize_load(slots, 10)
|
|
self.assertEqual(load['service_ms']['p95'], 900.0)
|
|
self.assertEqual(load['success_service_ms']['p95'], 10.0)
|
|
self.assertEqual(m.load_errors(load), 10)
|
|
|
|
def test_nearest_rank(self):
|
|
self.assertEqual(m.nearest_rank(list(range(1, 101)), 0.95), 95)
|
|
self.assertEqual(m.nearest_rank([7], 0.99), 7)
|
|
self.assertIsNone(m.nearest_rank([], 0.5))
|
|
|
|
|
|
class MatrixAndSelection(unittest.TestCase):
|
|
def test_valid_adverse_run_stays_in_matrix(self):
|
|
s = summary([slot(i) for i in range(100)], mode='p25',
|
|
export_failures={'loss': True},
|
|
cohorts={'error': cohort(sampled_missing=2)})
|
|
valid, invalid, failures = m.evaluate_run(s)
|
|
self.assertTrue(valid, invalid)
|
|
self.assertIn('export_loss', failures)
|
|
self.assertTrue(any(f.startswith('sampled_missing') for f in failures))
|
|
s.update(valid=valid, invalid_reasons=invalid,
|
|
configuration_failures=failures, repeat=1)
|
|
entry = m.matrix_entry(s)
|
|
self.assertTrue(entry['valid'])
|
|
self.assertTrue(entry['configuration_failures'])
|
|
|
|
def full_runs(self):
|
|
return [run('full', spans=100.0, p95=100.0) for _ in range(3)]
|
|
|
|
def test_p25_selected_only_against_complete_full_reference(self):
|
|
good = [run('p25', spans=40.0, p95=105.0) for _ in range(3)]
|
|
self.assertTrue(m.select_p25(self.full_runs(), good)['passed'])
|
|
broken_full = self.full_runs()
|
|
broken_full[1]['spans_per_operation'] = None
|
|
self.assertFalse(m.select_p25(broken_full, good)['passed'])
|
|
heavy = [run('p25', spans=60.0, p95=105.0) for _ in range(3)]
|
|
self.assertFalse(m.select_p25(self.full_runs(), heavy)['passed'])
|
|
slow = [run('p25', spans=40.0, p95=120.0) for _ in range(3)]
|
|
self.assertFalse(m.select_p25(self.full_runs(), slow)['passed'])
|
|
|
|
def test_p25_rejected_when_an_error_class_has_no_retained_trace(self):
|
|
runs = [run('p25', spans=40.0, p95=100.0) for _ in range(3)]
|
|
runs[2]['cohorts']['slow'] = cohort(retained_traces=0, sampled_present=0)
|
|
self.assertFalse(m.select_p25(self.full_runs(), runs)['passed'])
|
|
|
|
def test_single_validation_uses_frozen_full_reference(self):
|
|
ref = m.full_reference(self.full_runs())
|
|
ok = run('p25', spans=45.0, p95=108.0)
|
|
self.assertTrue(m.validate_single(ok, 'p25', ref)['passed'])
|
|
# Would pass a "median of its own" comparison, fails the frozen one.
|
|
bad = run('p25', spans=55.0, p95=108.0)
|
|
result = m.validate_single(bad, 'p25', ref)
|
|
self.assertFalse(result['passed'])
|
|
self.assertFalse(result['checks']['spans_le_50pct_frozen_full'])
|
|
|
|
def test_full_validation_needs_all_eighty_server_spans(self):
|
|
ref = m.full_reference(self.full_runs())
|
|
ok = run('full', error=cohort(sampled_present=40, retained_traces=40),
|
|
slow=cohort(sampled_present=40, retained_traces=40))
|
|
self.assertTrue(m.validate_single(ok, 'full', ref)['passed'])
|
|
short = run('full', error=cohort(sampled_present=39, retained_traces=39),
|
|
slow=cohort(sampled_present=40, retained_traces=40))
|
|
self.assertFalse(m.validate_single(short, 'full', ref)['passed'])
|
|
|
|
def test_miss_all_probability_model(self):
|
|
self.assertAlmostEqual(m.miss_all_probability(0.25, 40), 0.75 ** 40)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|