Files
mtusi2026lab6/scripts/lab6/test_measurement.py

200 lines
9.2 KiB
Python

import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import measurement as m # noqa: E402
def slot(i, status=201, service=10.0, lag=1.0, outcome='http'):
if outcome != 'http':
return {'i': i, 'outcome': outcome, 'start_lag_ms': None}
return {'i': i, 'outcome': 'http', 'status': status,
'service_ms': service, 'scheduled_ms': service + lag,
'start_lag_ms': lag}
def cohort(**kw):
base = {'n': 40, 'retained_traces': 5, 'sampled_present': 5,
'intentional_unsampled': 35}
base.update(kw)
return base
def run(mode='p25', spans=10.0, p95=100.0, failures=(), valid=True,
error=None, slow=None):
return {
'mode': mode, 'valid': valid, 'invalid_reasons': [],
'configuration_failures': list(failures),
'spans_per_operation': spans,
'load': {'scheduled_ms': {'p95': p95}},
'cohorts': {'error': error or cohort(), 'slow': slow or cohort()},
'dir': mode,
}
def summary(load_slots, **extra):
load = m.summarize_load(load_slots, 10)
base = {'mode': 'off', 'load': load, 'cohorts': {}, 'ingest': {},
'export_failures': {'loss': False}}
base.update(extra)
return base
class SamplingIsNotLoss(unittest.TestCase):
def test_false_flag_is_not_export_loss_or_missing_span(self):
rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
'trace_sampled': False}, 'loki': True, 'span': False}
self.assertEqual(m.classify_record('p25', rec),
('intentional_unsampled', 'exported'))
counts = m.summarize_cohort('p25', [rec] * 3)
self.assertEqual(counts.get('sampled_missing', 0), 0)
self.assertFalse(m.export_loss(0, 0.0, 0.0)['loss'])
def test_sampled_without_span_is_a_measured_loss(self):
rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
'trace_sampled': True}, 'loki': True, 'span': False}
self.assertEqual(m.classify_record('full', rec)[0], 'sampled_missing')
def test_missing_stdout_is_not_intentional_sampling(self):
self.assertEqual(m.classify_record('p25', {'stdout': None})[0],
'unmatched_stdout')
no_flag = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16}}
self.assertEqual(m.classify_record('p25', no_flag)[0], 'unmatched_stdout')
incomplete = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
'trace_sampled': True}, 'loki': None, 'span': None}
self.assertEqual(m.classify_record('p25', incomplete),
('incomplete_observation', 'incomplete_observation'))
def test_full_observation_without_log_is_missing_exported_log(self):
rec = {'stdout': {'trace_id': 'a' * 32, 'span_id': 'b' * 16,
'trace_sampled': False}, 'loki': False}
self.assertEqual(m.classify_record('p10', rec)[1], 'missing_exported_logs')
def test_off_needs_neither_loki_nor_app_counters(self):
rec = {'stdout': {'event': 'http.request'}, 'loki': None, 'span': None}
self.assertEqual(m.classify_record('off', rec),
('not_applicable', 'not_applicable'))
leaked = {'stdout': {'trace_id': 'x'}}
self.assertEqual(m.classify_record('off', leaked)[0], 'ids_present_with_sdk_off')
s = summary([slot(i) for i in range(100)], cohorts={
'error': m.summarize_cohort('off', [rec] * 4)})
valid, invalid, failures = m.evaluate_run(s)
self.assertTrue(valid, invalid)
self.assertEqual(failures, [])
class Counters(unittest.TestCase):
def test_reset_is_not_a_negative_loss(self):
self.assertEqual(m.counter_delta(10.0, 3.0), (None, 'counter_reset'))
loss = m.export_loss(None, 0.0, 2.0)
self.assertEqual(loss['total_known'], 2.0)
self.assertTrue(all(v >= 0 for v in loss['parts'].values() if v is not None))
def test_missing_positive_series_is_not_zero_failure_series_may_be(self):
self.assertEqual(m.counter_delta(None, None), (None, 'missing_series'))
self.assertEqual(m.counter_delta(None, None, failure_only=True)[0], 0.0)
self.assertEqual(m.counter_delta(5.0, 9.0), (4.0, None))
def test_unexplained_reset_invalidates_but_restart_explains_it(self):
s = summary([slot(i) for i in range(100)],
ingest={'spans': {'delta': None, 'reason': 'counter_reset'}})
self.assertFalse(m.evaluate_run(s)[0])
s['restarts'] = ['collector']
valid, _, failures = m.evaluate_run(s)
self.assertTrue(valid)
self.assertIn('restart_or_oom', failures)
class LoadAccounting(unittest.TestCase):
def test_dropped_and_late_slots_stay_in_denominator(self):
slots = [slot(i) for i in range(90)] + \
[slot(90 + i, outcome='client_dropped') for i in range(5)] + \
[slot(95 + i, lag=300.0) for i in range(5)]
load = m.summarize_load(slots, 10)
self.assertEqual(load['scheduled'], 100)
self.assertEqual(load['completed'], 95)
self.assertAlmostEqual(load['completed_ratio'], 0.95)
valid, invalid, _ = m.evaluate_run(summary(slots))
self.assertFalse(valid)
self.assertTrue(any('client_dropped' in r for r in invalid))
self.assertTrue(any('completed_ratio' in r for r in invalid))
def test_late_scheduler_fails_start_lag_even_without_drops(self):
slots = [slot(i, lag=80.0) for i in range(100)]
self.assertFalse(m.evaluate_run(summary(slots))[0])
def test_overall_p95_includes_error_responses(self):
slots = [slot(i, service=10.0) for i in range(90)] + \
[slot(90 + i, status=500, service=900.0) for i in range(10)]
load = m.summarize_load(slots, 10)
self.assertEqual(load['service_ms']['p95'], 900.0)
self.assertEqual(load['success_service_ms']['p95'], 10.0)
self.assertEqual(m.load_errors(load), 10)
def test_nearest_rank(self):
self.assertEqual(m.nearest_rank(list(range(1, 101)), 0.95), 95)
self.assertEqual(m.nearest_rank([7], 0.99), 7)
self.assertIsNone(m.nearest_rank([], 0.5))
class MatrixAndSelection(unittest.TestCase):
def test_valid_adverse_run_stays_in_matrix(self):
s = summary([slot(i) for i in range(100)], mode='p25',
export_failures={'loss': True},
cohorts={'error': cohort(sampled_missing=2)})
valid, invalid, failures = m.evaluate_run(s)
self.assertTrue(valid, invalid)
self.assertIn('export_loss', failures)
self.assertTrue(any(f.startswith('sampled_missing') for f in failures))
s.update(valid=valid, invalid_reasons=invalid,
configuration_failures=failures, repeat=1)
entry = m.matrix_entry(s)
self.assertTrue(entry['valid'])
self.assertTrue(entry['configuration_failures'])
def full_runs(self):
return [run('full', spans=100.0, p95=100.0) for _ in range(3)]
def test_p25_selected_only_against_complete_full_reference(self):
good = [run('p25', spans=40.0, p95=105.0) for _ in range(3)]
self.assertTrue(m.select_p25(self.full_runs(), good)['passed'])
broken_full = self.full_runs()
broken_full[1]['spans_per_operation'] = None
self.assertFalse(m.select_p25(broken_full, good)['passed'])
heavy = [run('p25', spans=60.0, p95=105.0) for _ in range(3)]
self.assertFalse(m.select_p25(self.full_runs(), heavy)['passed'])
slow = [run('p25', spans=40.0, p95=120.0) for _ in range(3)]
self.assertFalse(m.select_p25(self.full_runs(), slow)['passed'])
def test_p25_rejected_when_an_error_class_has_no_retained_trace(self):
runs = [run('p25', spans=40.0, p95=100.0) for _ in range(3)]
runs[2]['cohorts']['slow'] = cohort(retained_traces=0, sampled_present=0)
self.assertFalse(m.select_p25(self.full_runs(), runs)['passed'])
def test_single_validation_uses_frozen_full_reference(self):
ref = m.full_reference(self.full_runs())
ok = run('p25', spans=45.0, p95=108.0)
self.assertTrue(m.validate_single(ok, 'p25', ref)['passed'])
# Would pass a "median of its own" comparison, fails the frozen one.
bad = run('p25', spans=55.0, p95=108.0)
result = m.validate_single(bad, 'p25', ref)
self.assertFalse(result['passed'])
self.assertFalse(result['checks']['spans_le_50pct_frozen_full'])
def test_full_validation_needs_all_eighty_server_spans(self):
ref = m.full_reference(self.full_runs())
ok = run('full', error=cohort(sampled_present=40, retained_traces=40),
slow=cohort(sampled_present=40, retained_traces=40))
self.assertTrue(m.validate_single(ok, 'full', ref)['passed'])
short = run('full', error=cohort(sampled_present=39, retained_traces=39),
slow=cohort(sampled_present=40, retained_traces=40))
self.assertFalse(m.validate_single(short, 'full', ref)['passed'])
def test_miss_all_probability_model(self):
self.assertAlmostEqual(m.miss_all_probability(0.25, 40), 0.75 ** 40)
if __name__ == '__main__':
unittest.main()