"""Трек 1: цена масштабирования и отдельная проверка размера пула соединений.""" import argparse import json import os from pathlib import Path import statistics import subprocess import sys import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from scripts.benchmark import context, measure from scripts.monitor import Monitor from scripts.stack import scale, save, command, containers def seed(ctx): data = json.dumps(dict(chat_id=ctx['chat']['id'], sender_id=ctx['actor']['id'])) running = [row for row in containers('api') if row['running']] if not running: raise RuntimeError('Нет запущенной API для заполнения истории') subprocess.run(['docker', 'exec', '-i', running[0]['id'], 'python', 'scripts/seed_history.py'], input=data, text=True, cwd=ROOT, check=True, timeout=60) def summarize(rows): groups = {} for row in rows: key = (row['series'], row['replicas'], row['concurrency'], row['pool']) groups.setdefault(key, []).append(row) result = [] for key, group in groups.items(): result.append(dict(series=key[0], replicas=key[1], concurrency=key[2], pool=key[3], repetitions=len(group), rps=round(statistics.median(r['successful_rps'] for r in group), 3), p95_ms=round(statistics.median(r['successful_latency_ms']['p95'] for r in group), 3), error_ratio=statistics.mean(r['error_ratio'] for r in group))) return result def plots(summary): import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt for metric, name in [('rps', 'Successful RPS'), ('p95_ms', 'p95, ms')]: fig, axes = plt.subplots(1, 2, figsize=(10, 4)) for ax, series in zip(axes, ['fixed', 'growth']): for n in [1, 2, 4]: points = sorted((r for r in summary if r['series'] == series and r['replicas'] == n), key=lambda r: r['concurrency']) ax.plot([r['concurrency'] for r in points], [r[metric] for r in points], marker='o', label=f'{n} replicas') ax.set(title=series, xlabel='Concurrent clients', ylabel=name) ax.grid(alpha=.3); ax.legend() fig.tight_layout() fig.savefig(ROOT / 'evidence' / f'scaling_{metric}.png', dpi=150) plt.close(fig) def conclusions(summary, rows): lines = ['# Трек 1: фактические результаты', '', 'Медианы повторов; p95 считается только по успешным запросам. Ошибки показаны отдельно.', '', '| Серия | API | Clients | Pool/API | RPS | p95 ms | Ошибки |', '| --- | ---: | ---: | ---: | ---: | ---: | ---: |'] for r in summary: lines.append(f"| {r['series']} | {r['replicas']} | {r['concurrency']} | {r['pool']} | " f"{r['rps']} | {r['p95_ms']} | {r['error_ratio']:.2%} |") lines += ['', '## Где рост нагрузки перестал давать заметный прирост RPS', ''] plateau = [] for series in ['fixed', 'growth']: for n in [1, 2, 4]: points = sorted((r for r in summary if r['series'] == series and r['replicas'] == n), key=lambda r: r['concurrency']) pairs = [(a, b) for a, b in zip(points, points[1:]) if b['rps'] <= a['rps'] * 1.10] if pairs: a, b = pairs[0] plateau.append(dict(series=series, replicas=n, clients_from=a['concurrency'], clients_to=b['concurrency'])) lines.append(f"- {series}, {n} API: clients {a['concurrency']} → {b['concurrency']}, " f"RPS {a['rps']} → {b['rps']}; p95 {a['p95_ms']} → {b['p95_ms']} ms.") if not plateau: lines.append('Плато не найдено: увеличить список concurrency и повторить. Насыщение пока не доказано.') before = next(r for r in summary if r['series'] == 'pool_before') after = next(r for r in summary if r['series'] == 'pool_after') increase_gain = after['rps'] / before['rps'] - 1 # Имена серий отражают исходную гипотезу 1 -> 8, а не заранее известный победитель. # Сравниваем оба направления, сохраняя исходные замеры и номера повторов. baseline, candidate = (after, before) if before['rps'] > after['rps'] else (before, after) gain = candidate['rps'] / baseline['rps'] - 1 p95_reduction = 1 - candidate['p95_ms'] / baseline['p95_ms'] baseline_runs = {r['repetition']: r for r in rows if r['series'] == baseline['series']} candidate_runs = {r['repetition']: r for r in rows if r['series'] == candidate['series']} repeats = sorted(baseline_runs.keys() & candidate_runs.keys()) consistent = len(repeats) >= 3 and all( candidate_runs[n]['successful_rps'] > baseline_runs[n]['successful_rps'] and candidate_runs[n]['successful_latency_ms']['p95'] < baseline_runs[n]['successful_latency_ms']['p95'] and candidate_runs[n]['error_ratio'] <= baseline_runs[n]['error_ratio'] + .005 for n in repeats) improved = gain >= .05 and p95_reduction > 0 and consistent queued = max((pool['stats'].get('requests_waiting', 0) for r in rows if r['series'] == 'pool_before' for sample in r['monitor'] for pool in sample.get('pools', [])), default=0) lines += ['', '## Проверка одного изменения', '', 'Одинаковые 4 API, по 0.5 CPU и 256 MiB. Сравниваются DB_POOL_MAX=1 и 8; остальные лимиты неизменны.', 'Исходная гипотеза — увеличение пула 1 → 8; названия pool_before/pool_after сохранены.', f'Изменение median RPS при 1 → 8: {increase_gain:+.1%}.', f'Лучший измеренный вариант: pool={candidate["pool"]}; сравнение {baseline["pool"]} → {candidate["pool"]}.', f'Median RPS: {baseline["rps"]} → {candidate["rps"]}; изменение {gain:+.1%}.', f'Median p95: {baseline["p95_ms"]} → {candidate["p95_ms"]} ms; снижение {p95_reduction:.1%}.', f'Число пар повторов: {len(repeats)}; одинаковое направление RPS/p95 без роста ошибок: {"да" if consistent else "нет"}.', f'Максимум наблюдаемой очереди при pool=1: {queued}.', '', 'Наличие очереди не доказывает, что нужно увеличивать пул. ' 'Вывод о механизме проверяется по CPU/RAM и ожиданиям PG из monitor. ' 'Результат относится к измеренному профилю и лимитам, не к любому серверу.'] confirmed = improved and queued > 0 and bool(plateau) status = 'MEASURED_IMPROVEMENT' if confirmed else 'NEEDS_REVIEW' lines += ['', f'Статус эксперимента: {status}.'] (ROOT / 'evidence' / 'track1_summary.md').write_text('\n'.join(lines) + '\n') return dict(status=status, original_increase_rps_gain=increase_gain, tested_change=f'{baseline["pool"]}_to_{candidate["pool"]}', recommended_pool=candidate['pool'], rps_gain=gain, p95_reduction=p95_reduction, paired_repetitions=len(repeats), consistent_pairs=consistent, maximum_pool_queue_before=queued, plateau=plateau) def main(): p = argparse.ArgumentParser() p.add_argument('--duration', type=int, default=15) p.add_argument('--warmup', type=int, default=3) p.add_argument('--repeats', type=int, default=2) p.add_argument('--concurrency', nargs='+', type=int, default=[1, 4, 16, 32]) p.add_argument('--summarize-only', action='store_true', help='Analyse saved raw data without load or Docker changes') args = p.parse_args() if args.summarize_only: rows = json.loads((ROOT / 'evidence' / 'track1_raw.json').read_text())['runs'] summary = summarize(rows) verdict = conclusions(summary, rows) save('track1_summary.json', dict(groups=summary, conclusion=verdict)) print('Experiment status:', verdict['status']) return if min(args.duration, args.warmup, args.repeats, *args.concurrency) < 1: p.error('All parameters must be positive') base = os.getenv('BASE_URL', 'http://localhost:8080') rows = [] metadata = dict(parameters=vars(args), docker_version=command(['docker', 'version', '--format', '{{.Server.Version}}']), compose_version=command(['docker', 'compose', 'version', '--short']), logical_cpus=os.cpu_count(), started_at=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), database_and_lb_limits='Unchanged throughout; actual values in containers', initial_history_per_case=10000, monitoring_interval_seconds=3, pool_before_is_deliberately_constrained=True) cases = [] for series in ['fixed', 'growth']: for n in [1, 2, 4]: cpu, memory = (2 / n, f'{1024 // n}m') if series == 'fixed' else (1, '256m') for clients in args.concurrency: for repeat in range(args.repeats): cases.append((series, n, cpu, memory, 8, clients, repeat)) # Порядок AB/BA чередуется, чтобы рост базы и прогрев не помогали всегда одному варианту. for repeat in range(max(3, args.repeats)): for pool in ([1, 8] if repeat % 2 == 0 else [8, 1]): cases.append(('pool_before' if pool == 1 else 'pool_after', 4, .5, '256m', pool, max(args.concurrency), repeat)) try: for index, (series, n, cpu, memory, pool, clients, repeat) in enumerate(cases, 1): print(f'[{index}/{len(cases)}] {series}, API={n}, clients={clients}, pool={pool}', flush=True) topology = scale(n, cpu, memory, pool) ctx = context(base) seed(ctx) measure(base, ctx, args.warmup, clients) with Monitor() as monitor: report, _ = measure(base, ctx, args.duration, clients) report.update(series=series, replicas=n, cpu_per_api=cpu, ram_per_api=memory, pool=pool, repetition=repeat + 1, containers=containers(), topology=topology, monitor=monitor.samples) rows.append(report) save('track1_raw.json', dict(metadata=metadata, runs=rows)) summary = summarize(rows) verdict = conclusions(summary, rows) save('track1_summary.json', dict(groups=summary, conclusion=verdict)) plots(summary) print('Experiment status:', verdict['status']) if verdict['status'] == 'NEEDS_REVIEW': print('Изменение не подтверждено: проверьте ошибки, мониторинг и условия измерения.') finally: scale(2, 1, '256m', 8) if __name__ == '__main__': main()