Files
mtusi2026lab3/scripts/experiment.py
T
2026-10-13 19:30:32 +03:00

192 lines
12 KiB
Python

"""Трек 1: цена масштабирования и отдельная проверка размера пула соединений."""
import argparse
import json
import os
from pathlib import Path
import statistics
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from scripts.benchmark import context, measure
from scripts.monitor import Monitor
from scripts.stack import scale, save, command, containers
def seed(ctx):
data = json.dumps(dict(chat_id=ctx['chat']['id'], sender_id=ctx['actor']['id']))
running = [row for row in containers('api') if row['running']]
if not running:
raise RuntimeError('Нет запущенной API для заполнения истории')
subprocess.run(['docker', 'exec', '-i', running[0]['id'],
'python', 'scripts/seed_history.py'], input=data, text=True,
cwd=ROOT, check=True, timeout=60)
def summarize(rows):
groups = {}
for row in rows:
key = (row['series'], row['replicas'], row['concurrency'], row['pool'])
groups.setdefault(key, []).append(row)
result = []
for key, group in groups.items():
result.append(dict(series=key[0], replicas=key[1], concurrency=key[2], pool=key[3],
repetitions=len(group),
rps=round(statistics.median(r['successful_rps'] for r in group), 3),
p95_ms=round(statistics.median(r['successful_latency_ms']['p95'] for r in group), 3),
error_ratio=statistics.mean(r['error_ratio'] for r in group)))
return result
def plots(summary):
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
for metric, name in [('rps', 'Successful RPS'), ('p95_ms', 'p95, ms')]:
fig, axes = plt.subplots(1, 2, figsize=(10, 4))
for ax, series in zip(axes, ['fixed', 'growth']):
for n in [1, 2, 4]:
points = sorted((r for r in summary if r['series'] == series and r['replicas'] == n),
key=lambda r: r['concurrency'])
ax.plot([r['concurrency'] for r in points], [r[metric] for r in points],
marker='o', label=f'{n} replicas')
ax.set(title=series, xlabel='Concurrent clients', ylabel=name)
ax.grid(alpha=.3); ax.legend()
fig.tight_layout()
fig.savefig(ROOT / 'evidence' / f'scaling_{metric}.png', dpi=150)
plt.close(fig)
def conclusions(summary, rows):
lines = ['# Трек 1: фактические результаты', '',
'Медианы повторов; p95 считается только по успешным запросам. Ошибки показаны отдельно.', '',
'| Серия | API | Clients | Pool/API | RPS | p95 ms | Ошибки |',
'| --- | ---: | ---: | ---: | ---: | ---: | ---: |']
for r in summary:
lines.append(f"| {r['series']} | {r['replicas']} | {r['concurrency']} | {r['pool']} | "
f"{r['rps']} | {r['p95_ms']} | {r['error_ratio']:.2%} |")
lines += ['', '## Где рост нагрузки перестал давать заметный прирост RPS', '']
plateau = []
for series in ['fixed', 'growth']:
for n in [1, 2, 4]:
points = sorted((r for r in summary if r['series'] == series and r['replicas'] == n),
key=lambda r: r['concurrency'])
pairs = [(a, b) for a, b in zip(points, points[1:]) if b['rps'] <= a['rps'] * 1.10]
if pairs:
a, b = pairs[0]
plateau.append(dict(series=series, replicas=n, clients_from=a['concurrency'],
clients_to=b['concurrency']))
lines.append(f"- {series}, {n} API: clients {a['concurrency']} → {b['concurrency']}, "
f"RPS {a['rps']} → {b['rps']}; p95 {a['p95_ms']} → {b['p95_ms']} ms.")
if not plateau:
lines.append('Плато не найдено: увеличить список concurrency и повторить. Насыщение пока не доказано.')
before = next(r for r in summary if r['series'] == 'pool_before')
after = next(r for r in summary if r['series'] == 'pool_after')
increase_gain = after['rps'] / before['rps'] - 1
# Имена серий отражают исходную гипотезу 1 -> 8, а не заранее известный победитель.
# Сравниваем оба направления, сохраняя исходные замеры и номера повторов.
baseline, candidate = (after, before) if before['rps'] > after['rps'] else (before, after)
gain = candidate['rps'] / baseline['rps'] - 1
p95_reduction = 1 - candidate['p95_ms'] / baseline['p95_ms']
baseline_runs = {r['repetition']: r for r in rows if r['series'] == baseline['series']}
candidate_runs = {r['repetition']: r for r in rows if r['series'] == candidate['series']}
repeats = sorted(baseline_runs.keys() & candidate_runs.keys())
consistent = len(repeats) >= 3 and all(
candidate_runs[n]['successful_rps'] > baseline_runs[n]['successful_rps'] and
candidate_runs[n]['successful_latency_ms']['p95'] < baseline_runs[n]['successful_latency_ms']['p95'] and
candidate_runs[n]['error_ratio'] <= baseline_runs[n]['error_ratio'] + .005
for n in repeats)
improved = gain >= .05 and p95_reduction > 0 and consistent
queued = max((pool['stats'].get('requests_waiting', 0)
for r in rows if r['series'] == 'pool_before'
for sample in r['monitor'] for pool in sample.get('pools', [])), default=0)
lines += ['', '## Проверка одного изменения', '',
'Одинаковые 4 API, по 0.5 CPU и 256 MiB. Сравниваются DB_POOL_MAX=1 и 8; остальные лимиты неизменны.',
'Исходная гипотеза — увеличение пула 1 → 8; названия pool_before/pool_after сохранены.',
f'Изменение median RPS при 1 → 8: {increase_gain:+.1%}.',
f'Лучший измеренный вариант: pool={candidate["pool"]}; сравнение {baseline["pool"]} → {candidate["pool"]}.',
f'Median RPS: {baseline["rps"]} → {candidate["rps"]}; изменение {gain:+.1%}.',
f'Median p95: {baseline["p95_ms"]} → {candidate["p95_ms"]} ms; снижение {p95_reduction:.1%}.',
f'Число пар повторов: {len(repeats)}; одинаковое направление RPS/p95 без роста ошибок: {"да" if consistent else "нет"}.',
f'Максимум наблюдаемой очереди при pool=1: {queued}.',
'', 'Наличие очереди не доказывает, что нужно увеличивать пул. '
'Вывод о механизме проверяется по CPU/RAM и ожиданиям PG из monitor. '
'Результат относится к измеренному профилю и лимитам, не к любому серверу.']
confirmed = improved and queued > 0 and bool(plateau)
status = 'MEASURED_IMPROVEMENT' if confirmed else 'NEEDS_REVIEW'
lines += ['', f'Статус эксперимента: {status}.']
(ROOT / 'evidence' / 'track1_summary.md').write_text('\n'.join(lines) + '\n')
return dict(status=status, original_increase_rps_gain=increase_gain,
tested_change=f'{baseline["pool"]}_to_{candidate["pool"]}',
recommended_pool=candidate['pool'], rps_gain=gain, p95_reduction=p95_reduction,
paired_repetitions=len(repeats), consistent_pairs=consistent,
maximum_pool_queue_before=queued, plateau=plateau)
def main():
p = argparse.ArgumentParser()
p.add_argument('--duration', type=int, default=15)
p.add_argument('--warmup', type=int, default=3)
p.add_argument('--repeats', type=int, default=2)
p.add_argument('--concurrency', nargs='+', type=int, default=[1, 4, 16, 32])
p.add_argument('--summarize-only', action='store_true', help='Analyse saved raw data without load or Docker changes')
args = p.parse_args()
if args.summarize_only:
rows = json.loads((ROOT / 'evidence' / 'track1_raw.json').read_text())['runs']
summary = summarize(rows)
verdict = conclusions(summary, rows)
save('track1_summary.json', dict(groups=summary, conclusion=verdict))
print('Experiment status:', verdict['status'])
return
if min(args.duration, args.warmup, args.repeats, *args.concurrency) < 1:
p.error('All parameters must be positive')
base = os.getenv('BASE_URL', 'http://localhost:8080')
rows = []
metadata = dict(parameters=vars(args), docker_version=command(['docker', 'version', '--format', '{{.Server.Version}}']),
compose_version=command(['docker', 'compose', 'version', '--short']),
logical_cpus=os.cpu_count(), started_at=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
database_and_lb_limits='Unchanged throughout; actual values in containers',
initial_history_per_case=10000, monitoring_interval_seconds=3,
pool_before_is_deliberately_constrained=True)
cases = []
for series in ['fixed', 'growth']:
for n in [1, 2, 4]:
cpu, memory = (2 / n, f'{1024 // n}m') if series == 'fixed' else (1, '256m')
for clients in args.concurrency:
for repeat in range(args.repeats):
cases.append((series, n, cpu, memory, 8, clients, repeat))
# Порядок AB/BA чередуется, чтобы рост базы и прогрев не помогали всегда одному варианту.
for repeat in range(max(3, args.repeats)):
for pool in ([1, 8] if repeat % 2 == 0 else [8, 1]):
cases.append(('pool_before' if pool == 1 else 'pool_after', 4, .5, '256m', pool,
max(args.concurrency), repeat))
try:
for index, (series, n, cpu, memory, pool, clients, repeat) in enumerate(cases, 1):
print(f'[{index}/{len(cases)}] {series}, API={n}, clients={clients}, pool={pool}', flush=True)
topology = scale(n, cpu, memory, pool)
ctx = context(base)
seed(ctx)
measure(base, ctx, args.warmup, clients)
with Monitor() as monitor:
report, _ = measure(base, ctx, args.duration, clients)
report.update(series=series, replicas=n, cpu_per_api=cpu, ram_per_api=memory,
pool=pool, repetition=repeat + 1, containers=containers(),
topology=topology, monitor=monitor.samples)
rows.append(report)
save('track1_raw.json', dict(metadata=metadata, runs=rows))
summary = summarize(rows)
verdict = conclusions(summary, rows)
save('track1_summary.json', dict(groups=summary, conclusion=verdict))
plots(summary)
print('Experiment status:', verdict['status'])
if verdict['status'] == 'NEEDS_REVIEW':
print('Изменение не подтверждено: проверьте ошибки, мониторинг и условия измерения.')
finally:
scale(2, 1, '256m', 8)
if __name__ == '__main__':
main()