forked from mtusi2026/mtusi2026lab3
271 lines
11 KiB
Python
Executable File
271 lines
11 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Регрессия аналитики трека 1: успешный RPS отделён от частоты ответов.
|
|
|
|
Проверяются два дефекта, найденные в разборе серии роста и A/B:
|
|
|
|
1. Пропускная способность считалась как `requests / duration`, поэтому ответы об
|
|
ошибках выглядели выполненной работой. На шаге `abdiag#A` 400 req/s клиент
|
|
получил 1978 ответов, из них 853 успешных: старая формула давала 131.87 req/s
|
|
вместо 56.87, и это же число шло в вердикт A/B и в потолок.
|
|
2. Серия роста — блок why# на одном темпе. Точка, отличающаяся от контроля больше
|
|
чем одним ключом бюджета или снятая на другом темпе, исключается с причиной.
|
|
|
|
Тест детерминированный и оффлайн: сырые сводки k6 подставляются фикстурами в
|
|
временный каталог, стенд и Docker не нужны.
|
|
|
|
Запуск: make test-analytics
|
|
python3 -m unittest discover -s tests -p 'test_*.py' -v
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def load_lab3():
|
|
"""Загружает драйвер экспериментов как модуль, не запуская CLI."""
|
|
path = ROOT / 'scripts' / 'lab3' / 'lab3.py'
|
|
spec = importlib.util.spec_from_file_location('lab3', path)
|
|
if spec is None or spec.loader is None:
|
|
raise SystemExit(f'cannot load {path}')
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules['lab3'] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
lab3 = load_lab3()
|
|
|
|
# Дефектный шаг из abdiag#A: 1978 ответов за 15 s, 853 успешных, 1125 ответов 503.
|
|
DEGRADED_RATE = 400
|
|
DEGRADED_REQUESTS = 1978
|
|
DEGRADED_OK = 853
|
|
DEGRADED_FAILED = 1125
|
|
STEP_SECONDS = 15
|
|
DEGRADED_SUCCESSFUL_RPS = 56.87
|
|
DEGRADED_ATTEMPTED_RPS = 131.87
|
|
|
|
|
|
def step(
|
|
rate: int,
|
|
*,
|
|
ok: int,
|
|
requests: int | None = None,
|
|
failed: int = 0,
|
|
duration: int = STEP_SECONDS,
|
|
p95: float = 10.0,
|
|
) -> dict:
|
|
total = requests if requests is not None else ok + failed
|
|
return {
|
|
'rate': rate,
|
|
'duration_s': duration,
|
|
'requests': total,
|
|
'successful_rps': round(ok / duration, 2),
|
|
'attempted_rps': round(total / duration, 2),
|
|
'failed': failed,
|
|
'p95': p95,
|
|
'vus_active': 0,
|
|
'status': {'ok': ok, 'client_4xx': 0, 'server_5xx': failed, 'other': 0},
|
|
}
|
|
|
|
|
|
def summary(label: str, kind: str, steps: list[dict], profile: str = 'spread') -> dict:
|
|
return {
|
|
'label': label,
|
|
'profile': profile,
|
|
'mode': 'write',
|
|
'kind': kind,
|
|
'steps': steps,
|
|
'dropped_iterations': 0,
|
|
}
|
|
|
|
|
|
class AnalyticsTestCase(unittest.TestCase):
|
|
"""Изолирует raw/каталоги вывода: аналитика читает их как глобальные пути."""
|
|
|
|
def setUp(self) -> None:
|
|
self._temporary = tempfile.TemporaryDirectory()
|
|
root = Path(self._temporary.name)
|
|
self.raw_dir = root / 'raw'
|
|
self.raw_dir.mkdir()
|
|
self.track1_dir = root / 'track1'
|
|
self.track1_dir.mkdir()
|
|
self._original = (lab3.RAW_DIR, lab3.TRACK1_DIR, lab3.GRAPH_DIR)
|
|
lab3.RAW_DIR = self.raw_dir
|
|
lab3.TRACK1_DIR = self.track1_dir
|
|
lab3.GRAPH_DIR = self.track1_dir / 'graphs'
|
|
|
|
def tearDown(self) -> None:
|
|
lab3.RAW_DIR, lab3.TRACK1_DIR, lab3.GRAPH_DIR = self._original
|
|
self._temporary.cleanup()
|
|
|
|
def write_raw(
|
|
self,
|
|
label: str,
|
|
payload: dict,
|
|
*,
|
|
profile: str = 'spread',
|
|
mode: str = 'write',
|
|
manifest: dict | None = None,
|
|
) -> None:
|
|
(self.raw_dir / f'{label}.k6.json').write_text(json.dumps(payload, ensure_ascii=False, indent=1))
|
|
(self.raw_dir / f'{label}.manifest.json').write_text(
|
|
json.dumps(
|
|
{'label': label, 'profile': profile, 'mode': mode, 'kind': payload.get('kind'), **(manifest or {})},
|
|
ensure_ascii=False,
|
|
)
|
|
)
|
|
|
|
def capture_log(self) -> list[str]:
|
|
messages: list[str] = []
|
|
self._log = lab3.log
|
|
lab3.log = messages.append
|
|
self.addCleanup(lambda: setattr(lab3, 'log', self._log))
|
|
return messages
|
|
|
|
|
|
class SuccessfulThroughputTest(AnalyticsTestCase):
|
|
def test_error_answers_are_not_throughput(self) -> None:
|
|
label = 'ab#A#rep1'
|
|
self.write_raw(
|
|
label,
|
|
summary(
|
|
label,
|
|
'ladder',
|
|
[
|
|
step(
|
|
DEGRADED_RATE,
|
|
ok=DEGRADED_OK,
|
|
requests=DEGRADED_REQUESTS,
|
|
failed=DEGRADED_FAILED,
|
|
p95=5200.88,
|
|
)
|
|
],
|
|
),
|
|
)
|
|
series = lab3.summarise_series([label])
|
|
row = series['rows'][0]
|
|
self.assertEqual(row['successful_rps'], DEGRADED_SUCCESSFUL_RPS)
|
|
self.assertEqual(row['attempted_rps'], DEGRADED_ATTEMPTED_RPS)
|
|
self.assertEqual(row['successful_min'], DEGRADED_SUCCESSFUL_RPS)
|
|
self.assertEqual(row['successful_max'], DEGRADED_SUCCESSFUL_RPS)
|
|
# Потолок и сатурация считаются по успешным ответам, а не по всем.
|
|
self.assertEqual(series['ceiling_rps'], DEGRADED_SUCCESSFUL_RPS)
|
|
self.assertEqual(series['saturation']['rate'], DEGRADED_RATE)
|
|
self.assertEqual(series['saturation']['successful_rps'], DEGRADED_SUCCESSFUL_RPS)
|
|
|
|
|
|
class AbVerdictTest(AnalyticsTestCase):
|
|
def test_verdict_delta_uses_successful_rps(self) -> None:
|
|
for rep in (1, 2, 3):
|
|
label = f'ab#A#rep{rep}'
|
|
self.write_raw(
|
|
label,
|
|
summary(
|
|
label,
|
|
'ladder',
|
|
[step(DEGRADED_RATE, ok=DEGRADED_OK, requests=DEGRADED_REQUESTS, failed=DEGRADED_FAILED, p95=5200.88)],
|
|
),
|
|
)
|
|
label_b = f'ab#B#rep{rep}'
|
|
self.write_raw(label_b, summary(label_b, 'ladder', [step(DEGRADED_RATE, ok=6000, requests=6000, failed=0, p95=24.4)]))
|
|
|
|
lab3.report_ab()
|
|
verdict = json.loads((self.track1_dir / 'ab-verdict.json').read_text())
|
|
entry = next(item for item in verdict['verdicts'] if item['rate'] == DEGRADED_RATE)
|
|
expected_change = round((400.0 - DEGRADED_SUCCESSFUL_RPS) / DEGRADED_SUCCESSFUL_RPS * 100, 2)
|
|
self.assertEqual(entry['change_percent'], expected_change)
|
|
self.assertEqual(entry['a']['successful_rps'], DEGRADED_SUCCESSFUL_RPS)
|
|
self.assertEqual(entry['a']['attempted_rps'], DEGRADED_ATTEMPTED_RPS)
|
|
self.assertEqual(entry['a']['status_5xx'], DEGRADED_FAILED)
|
|
self.assertEqual(entry['b']['successful_rps'], 400.0)
|
|
self.assertTrue(entry['confirmed'])
|
|
# Потолок стороны тоже успешный, а не «все ответы».
|
|
self.assertEqual(verdict['ceiling'], {'A': DEGRADED_SUCCESSFUL_RPS, 'B': 400.0})
|
|
|
|
summary_md = (self.track1_dir / 'summary-ab.md').read_text()
|
|
self.assertIn('успешный RPS', summary_md)
|
|
self.assertIn(str(DEGRADED_SUCCESSFUL_RPS), summary_md)
|
|
self.assertIn(str(DEGRADED_ATTEMPTED_RPS), summary_md)
|
|
|
|
|
|
class GrowthBlockTest(AnalyticsTestCase):
|
|
"""Серия роста — блок why# на одном темпе: несопоставимые точки исключаются."""
|
|
|
|
COMMON = {
|
|
'commit': 'c',
|
|
'generator': {'max_vus': '900', 'cpus': '1.5', 'memory': '1024m'},
|
|
}
|
|
BASE_STAND = {
|
|
'API_REPLICAS': '4',
|
|
'API_TOTAL_CPUS': '3.0',
|
|
'PG_CPUS': '1.5',
|
|
'DATABASE_POOL_MAX': '10',
|
|
'LB_CPUS': '0.25',
|
|
'API_IMAGE': 'img',
|
|
}
|
|
|
|
def write_point(self, label: str, successful: int, stand: dict | None = None, generator: dict | None = None) -> None:
|
|
manifest = {
|
|
'commit': self.COMMON['commit'],
|
|
'generator': {**self.COMMON['generator'], **(generator or {})},
|
|
'stand': {**self.BASE_STAND, **(stand or {})},
|
|
}
|
|
self.write_raw(label, summary(label, 'steady', [step(1500, ok=successful * 60, duration=60)]), manifest=manifest)
|
|
|
|
def setUp(self) -> None:
|
|
super().setUp()
|
|
self.write_point('why#n4#rep1', 943)
|
|
self.write_point('why#n4-pg3#rep1', 1480, {'PG_CPUS': '3.0'})
|
|
self.write_point('why#n4-api6#rep1', 891, {'API_TOTAL_CPUS': '6.0'})
|
|
self.write_point('why#n4-pool20#rep1', 853, {'DATABASE_POOL_MAX': '20'})
|
|
self.write_point('why#n4-lb1#rep1', 922, {'LB_CPUS': '1.0'}, {'max_vus': '200'})
|
|
self.groups = lab3.group_configurations(lab3.raw_labels())
|
|
|
|
def test_only_comparable_points_are_compared(self) -> None:
|
|
lab3.analyse_growth(self.groups)
|
|
text = (self.track1_dir / 'summary-growth.md').read_text()
|
|
rows = {line.split(' | ')[0].lstrip('| '): line for line in text.splitlines() if line.startswith('| why#')}
|
|
self.assertIn('рычаг поднимает потолок', rows['why#n4-pg3'])
|
|
self.assertIn('в пределах разброса', rows['why#n4-api6'])
|
|
self.assertIn('ухудшает', rows['why#n4-pool20'])
|
|
self.assertNotIn('why#n4-lb1', rows)
|
|
omitted = text.split('Вне сравнения:')[1]
|
|
lb1 = next(line for line in omitted.splitlines() if 'why#n4-lb1' in line)
|
|
self.assertIn('max_vus', lb1)
|
|
pool5 = next(line for line in omitted.splitlines() if 'why#n4-pool5' in line)
|
|
self.assertIn('не снята на этом темпе', pool5)
|
|
|
|
def test_graph_built_from_same_block(self) -> None:
|
|
try:
|
|
import matplotlib # noqa: F401
|
|
except ImportError:
|
|
self.skipTest('matplotlib не установлен: график не строится')
|
|
lab3.make_graphs(self.groups)
|
|
self.assertTrue((lab3.GRAPH_DIR / 'growth.png').is_file())
|
|
|
|
|
|
class RawSchemaTest(unittest.TestCase):
|
|
"""Сырые сводки k6 обязаны быть в текущей схеме: без achieved_rps."""
|
|
|
|
def test_committed_raw_uses_current_fields(self) -> None:
|
|
files = sorted((ROOT / 'evidence' / 'track1' / 'raw').glob('*.k6.json'))
|
|
self.assertTrue(files, 'нет сырых сводок k6')
|
|
for path in files:
|
|
text = path.read_text()
|
|
self.assertNotIn('achieved_rps', text, path.name)
|
|
for s in json.loads(text).get('steps', []):
|
|
self.assertIn('successful_rps', s, path.name)
|
|
self.assertIn('attempted_rps', s, path.name)
|
|
self.assertLessEqual(s['successful_rps'], s['attempted_rps'], path.name)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|