forked from mtusi2026/mtusi2026lab3
192 lines
12 KiB
Python
192 lines
12 KiB
Python
"""Трек 1: цена масштабирования и отдельная проверка размера пула соединений."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
from scripts.benchmark import context, measure
|
|
from scripts.monitor import Monitor
|
|
from scripts.stack import scale, save, command, containers
|
|
|
|
|
|
def seed(ctx):
|
|
data = json.dumps(dict(chat_id=ctx['chat']['id'], sender_id=ctx['actor']['id']))
|
|
running = [row for row in containers('api') if row['running']]
|
|
if not running:
|
|
raise RuntimeError('Нет запущенной API для заполнения истории')
|
|
subprocess.run(['docker', 'exec', '-i', running[0]['id'],
|
|
'python', 'scripts/seed_history.py'], input=data, text=True,
|
|
cwd=ROOT, check=True, timeout=60)
|
|
|
|
|
|
def summarize(rows):
|
|
groups = {}
|
|
for row in rows:
|
|
key = (row['series'], row['replicas'], row['concurrency'], row['pool'])
|
|
groups.setdefault(key, []).append(row)
|
|
result = []
|
|
for key, group in groups.items():
|
|
result.append(dict(series=key[0], replicas=key[1], concurrency=key[2], pool=key[3],
|
|
repetitions=len(group),
|
|
rps=round(statistics.median(r['successful_rps'] for r in group), 3),
|
|
p95_ms=round(statistics.median(r['successful_latency_ms']['p95'] for r in group), 3),
|
|
error_ratio=statistics.mean(r['error_ratio'] for r in group)))
|
|
return result
|
|
|
|
|
|
def plots(summary):
|
|
import matplotlib
|
|
matplotlib.use('Agg')
|
|
import matplotlib.pyplot as plt
|
|
for metric, name in [('rps', 'Successful RPS'), ('p95_ms', 'p95, ms')]:
|
|
fig, axes = plt.subplots(1, 2, figsize=(10, 4))
|
|
for ax, series in zip(axes, ['fixed', 'growth']):
|
|
for n in [1, 2, 4]:
|
|
points = sorted((r for r in summary if r['series'] == series and r['replicas'] == n),
|
|
key=lambda r: r['concurrency'])
|
|
ax.plot([r['concurrency'] for r in points], [r[metric] for r in points],
|
|
marker='o', label=f'{n} replicas')
|
|
ax.set(title=series, xlabel='Concurrent clients', ylabel=name)
|
|
ax.grid(alpha=.3); ax.legend()
|
|
fig.tight_layout()
|
|
fig.savefig(ROOT / 'evidence' / f'scaling_{metric}.png', dpi=150)
|
|
plt.close(fig)
|
|
|
|
|
|
def conclusions(summary, rows):
|
|
lines = ['# Трек 1: фактические результаты', '',
|
|
'Медианы повторов; p95 считается только по успешным запросам. Ошибки показаны отдельно.', '',
|
|
'| Серия | API | Clients | Pool/API | RPS | p95 ms | Ошибки |',
|
|
'| --- | ---: | ---: | ---: | ---: | ---: | ---: |']
|
|
for r in summary:
|
|
lines.append(f"| {r['series']} | {r['replicas']} | {r['concurrency']} | {r['pool']} | "
|
|
f"{r['rps']} | {r['p95_ms']} | {r['error_ratio']:.2%} |")
|
|
lines += ['', '## Где рост нагрузки перестал давать заметный прирост RPS', '']
|
|
plateau = []
|
|
for series in ['fixed', 'growth']:
|
|
for n in [1, 2, 4]:
|
|
points = sorted((r for r in summary if r['series'] == series and r['replicas'] == n),
|
|
key=lambda r: r['concurrency'])
|
|
pairs = [(a, b) for a, b in zip(points, points[1:]) if b['rps'] <= a['rps'] * 1.10]
|
|
if pairs:
|
|
a, b = pairs[0]
|
|
plateau.append(dict(series=series, replicas=n, clients_from=a['concurrency'],
|
|
clients_to=b['concurrency']))
|
|
lines.append(f"- {series}, {n} API: clients {a['concurrency']} → {b['concurrency']}, "
|
|
f"RPS {a['rps']} → {b['rps']}; p95 {a['p95_ms']} → {b['p95_ms']} ms.")
|
|
if not plateau:
|
|
lines.append('Плато не найдено: увеличить список concurrency и повторить. Насыщение пока не доказано.')
|
|
before = next(r for r in summary if r['series'] == 'pool_before')
|
|
after = next(r for r in summary if r['series'] == 'pool_after')
|
|
increase_gain = after['rps'] / before['rps'] - 1
|
|
# Имена серий отражают исходную гипотезу 1 -> 8, а не заранее известный победитель.
|
|
# Сравниваем оба направления, сохраняя исходные замеры и номера повторов.
|
|
baseline, candidate = (after, before) if before['rps'] > after['rps'] else (before, after)
|
|
gain = candidate['rps'] / baseline['rps'] - 1
|
|
p95_reduction = 1 - candidate['p95_ms'] / baseline['p95_ms']
|
|
baseline_runs = {r['repetition']: r for r in rows if r['series'] == baseline['series']}
|
|
candidate_runs = {r['repetition']: r for r in rows if r['series'] == candidate['series']}
|
|
repeats = sorted(baseline_runs.keys() & candidate_runs.keys())
|
|
consistent = len(repeats) >= 3 and all(
|
|
candidate_runs[n]['successful_rps'] > baseline_runs[n]['successful_rps'] and
|
|
candidate_runs[n]['successful_latency_ms']['p95'] < baseline_runs[n]['successful_latency_ms']['p95'] and
|
|
candidate_runs[n]['error_ratio'] <= baseline_runs[n]['error_ratio'] + .005
|
|
for n in repeats)
|
|
improved = gain >= .05 and p95_reduction > 0 and consistent
|
|
queued = max((pool['stats'].get('requests_waiting', 0)
|
|
for r in rows if r['series'] == 'pool_before'
|
|
for sample in r['monitor'] for pool in sample.get('pools', [])), default=0)
|
|
lines += ['', '## Проверка одного изменения', '',
|
|
'Одинаковые 4 API, по 0.5 CPU и 256 MiB. Сравниваются DB_POOL_MAX=1 и 8; остальные лимиты неизменны.',
|
|
'Исходная гипотеза — увеличение пула 1 → 8; названия pool_before/pool_after сохранены.',
|
|
f'Изменение median RPS при 1 → 8: {increase_gain:+.1%}.',
|
|
f'Лучший измеренный вариант: pool={candidate["pool"]}; сравнение {baseline["pool"]} → {candidate["pool"]}.',
|
|
f'Median RPS: {baseline["rps"]} → {candidate["rps"]}; изменение {gain:+.1%}.',
|
|
f'Median p95: {baseline["p95_ms"]} → {candidate["p95_ms"]} ms; снижение {p95_reduction:.1%}.',
|
|
f'Число пар повторов: {len(repeats)}; одинаковое направление RPS/p95 без роста ошибок: {"да" if consistent else "нет"}.',
|
|
f'Максимум наблюдаемой очереди при pool=1: {queued}.',
|
|
'', 'Наличие очереди не доказывает, что нужно увеличивать пул. '
|
|
'Вывод о механизме проверяется по CPU/RAM и ожиданиям PG из monitor. '
|
|
'Результат относится к измеренному профилю и лимитам, не к любому серверу.']
|
|
confirmed = improved and queued > 0 and bool(plateau)
|
|
status = 'MEASURED_IMPROVEMENT' if confirmed else 'NEEDS_REVIEW'
|
|
lines += ['', f'Статус эксперимента: {status}.']
|
|
(ROOT / 'evidence' / 'track1_summary.md').write_text('\n'.join(lines) + '\n')
|
|
return dict(status=status, original_increase_rps_gain=increase_gain,
|
|
tested_change=f'{baseline["pool"]}_to_{candidate["pool"]}',
|
|
recommended_pool=candidate['pool'], rps_gain=gain, p95_reduction=p95_reduction,
|
|
paired_repetitions=len(repeats), consistent_pairs=consistent,
|
|
maximum_pool_queue_before=queued, plateau=plateau)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('--duration', type=int, default=15)
|
|
p.add_argument('--warmup', type=int, default=3)
|
|
p.add_argument('--repeats', type=int, default=2)
|
|
p.add_argument('--concurrency', nargs='+', type=int, default=[1, 4, 16, 32])
|
|
p.add_argument('--summarize-only', action='store_true', help='Analyse saved raw data without load or Docker changes')
|
|
args = p.parse_args()
|
|
if args.summarize_only:
|
|
rows = json.loads((ROOT / 'evidence' / 'track1_raw.json').read_text())['runs']
|
|
summary = summarize(rows)
|
|
verdict = conclusions(summary, rows)
|
|
save('track1_summary.json', dict(groups=summary, conclusion=verdict))
|
|
print('Experiment status:', verdict['status'])
|
|
return
|
|
if min(args.duration, args.warmup, args.repeats, *args.concurrency) < 1:
|
|
p.error('All parameters must be positive')
|
|
base = os.getenv('BASE_URL', 'http://localhost:8080')
|
|
rows = []
|
|
metadata = dict(parameters=vars(args), docker_version=command(['docker', 'version', '--format', '{{.Server.Version}}']),
|
|
compose_version=command(['docker', 'compose', 'version', '--short']),
|
|
logical_cpus=os.cpu_count(), started_at=time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
|
database_and_lb_limits='Unchanged throughout; actual values in containers',
|
|
initial_history_per_case=10000, monitoring_interval_seconds=3,
|
|
pool_before_is_deliberately_constrained=True)
|
|
cases = []
|
|
for series in ['fixed', 'growth']:
|
|
for n in [1, 2, 4]:
|
|
cpu, memory = (2 / n, f'{1024 // n}m') if series == 'fixed' else (1, '256m')
|
|
for clients in args.concurrency:
|
|
for repeat in range(args.repeats):
|
|
cases.append((series, n, cpu, memory, 8, clients, repeat))
|
|
# Порядок AB/BA чередуется, чтобы рост базы и прогрев не помогали всегда одному варианту.
|
|
for repeat in range(max(3, args.repeats)):
|
|
for pool in ([1, 8] if repeat % 2 == 0 else [8, 1]):
|
|
cases.append(('pool_before' if pool == 1 else 'pool_after', 4, .5, '256m', pool,
|
|
max(args.concurrency), repeat))
|
|
try:
|
|
for index, (series, n, cpu, memory, pool, clients, repeat) in enumerate(cases, 1):
|
|
print(f'[{index}/{len(cases)}] {series}, API={n}, clients={clients}, pool={pool}', flush=True)
|
|
topology = scale(n, cpu, memory, pool)
|
|
ctx = context(base)
|
|
seed(ctx)
|
|
measure(base, ctx, args.warmup, clients)
|
|
with Monitor() as monitor:
|
|
report, _ = measure(base, ctx, args.duration, clients)
|
|
report.update(series=series, replicas=n, cpu_per_api=cpu, ram_per_api=memory,
|
|
pool=pool, repetition=repeat + 1, containers=containers(),
|
|
topology=topology, monitor=monitor.samples)
|
|
rows.append(report)
|
|
save('track1_raw.json', dict(metadata=metadata, runs=rows))
|
|
summary = summarize(rows)
|
|
verdict = conclusions(summary, rows)
|
|
save('track1_summary.json', dict(groups=summary, conclusion=verdict))
|
|
plots(summary)
|
|
print('Experiment status:', verdict['status'])
|
|
if verdict['status'] == 'NEEDS_REVIEW':
|
|
print('Изменение не подтверждено: проверьте ошибки, мониторинг и условия измерения.')
|
|
finally:
|
|
scale(2, 1, '256m', 8)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|