Files
mtusi2026lab3/scripts/track1_report.py
T
Weinmaier_AndrewandClaude Opus 5.5 2e7a80be92 lab3: балансировка, идемпотентность, отказ реплики, трек 1
- X-Instance-Id, обязательный Idempotency-Key (PostgreSQL, одна транзакция)
- HAProxy + 2..N реплик API, readiness/drain по SIGTERM, однократные миграции
- скрипты stop-one/restore/restart/rolling-update/collect-evidence
- трек 1: параметризованный стенд, cmd/bench, оркестратор и анализ
- workflow реального прогона Docker-стенда на GitHub Actions

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-23 17:24:54 +03:00

305 lines
18 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Анализ трека 1: evidence/track1/raw.jsonl → summary.csv, summary.md, report.html.
Правила анализа (фиксированы заранее, чтобы не подгонять выводы под данные):
* агрегат по повторам — медиана; разброс — min..max;
* точка насыщения конфигурации — первый уровень параллелизма, на котором
медианный ok_rps вырос меньше чем на 10% относительно предыдущего уровня;
* узкое место — роль (api/postgres/redis/lb) с наибольшей утилизацией CPU
от своего лимита на уровне насыщения; ≥0.85 — «насыщена»; дополнительно
признак пула: доля acquire без свободного соединения и среднее ожидание;
* улучшение «подтверждено», если на уровне хотя бы одного параллелизма ≥ уровня
насыщения baseline min(ok_rps кандидата) > max(ok_rps baseline) по всем повторам,
медиана p95 не хуже и доля ошибок не выше.
Только stdlib.
"""
import argparse
import csv
import html
import json
from pathlib import Path
import statistics
ROOT = Path(__file__).resolve().parents[1]
ROLES = ('api', 'postgres', 'redis', 'lb')
SAT_GAIN = 0.10
SAT_UTIL = 0.85
def load(path):
rows = []
for line in path.read_text(encoding='utf-8').splitlines():
if line.strip():
rows.append(json.loads(line))
return rows
def med(xs):
xs = [x for x in xs if x is not None]
return statistics.median(xs) if xs else None
def aggregate(rows):
groups = {}
for r in rows:
groups.setdefault((r['series'], r['config'], r['concurrency']), []).append(r)
agg = {}
for key, rs in groups.items():
b = [r['bench'] for r in rs]
a = {
'repeats': len(rs),
'env': rs[0]['env'],
'ok_rps': med(x['ok_rps'] for x in b),
'ok_rps_min': min(x['ok_rps'] for x in b),
'ok_rps_max': max(x['ok_rps'] for x in b),
'write_rps': med(x['ok_write_rps'] for x in b),
'read_rps': med(x['ok_read_rps'] for x in b),
'p50': med(x['latency_ms']['p50'] for x in b),
'p95': med(x['latency_ms']['p95'] for x in b),
'p99': med(x['latency_ms']['p99'] for x in b),
'write_p95': med(x['write_ms']['p95'] for x in b),
'error_ratio': med(x['error_ratio'] for x in b),
'empty_acquire_ratio': med(r['pool'].get('empty_acquire_ratio') for r in rs),
'acquire_wait_ms': med(r['pool'].get('avg_acquire_wait_ms') for r in rs),
'pg_active_max': med(r['resources'].get('postgres_connections', {}).get('active_max') for r in rs),
}
for role in ROLES:
a['util_' + role] = med(r['resources'].get(role, {}).get('cpu_utilization_of_limit') for r in rs)
agg[key] = a
return agg
def series_order(rows):
seen, order = set(), []
for r in rows:
k = (r['series'], r['config'])
if k not in seen:
seen.add(k)
order.append(k)
return order
def configs_of(order, series):
return [c for s, c in order if s == series]
def levels_of(agg, series, config):
return sorted(l for s, c, l in agg if s == series and c == config)
def saturation(agg, series, config):
lv = levels_of(agg, series, config)
peak_l = max(lv, key=lambda l: agg[(series, config, l)]['ok_rps'])
knee = lv[-1]
for prev, cur in zip(lv, lv[1:]):
if agg[(series, config, cur)]['ok_rps'] < agg[(series, config, prev)]['ok_rps'] * (1 + SAT_GAIN):
knee = cur
break
a = agg[(series, config, knee)]
utils = {r: a['util_' + r] for r in ROLES if a['util_' + r] is not None}
bottleneck = max(utils, key=utils.get) if utils else None
return {'peak_level': peak_l, 'peak_rps': agg[(series, config, peak_l)]['ok_rps'], 'knee_level': knee,
'bottleneck': bottleneck, 'bottleneck_util': utils.get(bottleneck),
'saturated': bool(bottleneck and utils[bottleneck] >= SAT_UTIL), 'utils': utils,
'pool_empty_ratio': a['empty_acquire_ratio'], 'pool_wait_ms': a['acquire_wait_ms']}
def improvement(rows, agg, series, baseline, candidate, from_level):
res = []
for l in levels_of(agg, series, baseline):
if l < from_level or (series, candidate, l) not in agg:
continue
b = [r['bench'] for r in rows if (r['series'], r['config'], r['concurrency']) == (series, baseline, l)]
c = [r['bench'] for r in rows if (r['series'], r['config'], r['concurrency']) == (series, candidate, l)]
ab, ac = agg[(series, baseline, l)], agg[(series, candidate, l)]
confirmed = (min(x['ok_rps'] for x in c) > max(x['ok_rps'] for x in b)
and ac['p95'] <= ab['p95'] and ac['error_ratio'] <= ab['error_ratio'])
res.append({'level': l, 'rps_delta_pct': 100 * (ac['ok_rps'] / ab['ok_rps'] - 1) if ab['ok_rps'] else None,
'p95_delta_pct': 100 * (ac['p95'] / ab['p95'] - 1) if ab['p95'] else None,
'repeats': (len(b), len(c)), 'confirmed': confirmed})
return res
def fmt(x, nd=1):
return '—' if x is None else ('%.*f' % (nd, x))
# ---------- SVG ----------
COLORS = ['--series-1', '--series-2', '--series-3', '--series-4']
DASH = ['', '6 3', '2 3', '8 3 2 3']
def line_chart(title, xlabels, series, ylabel):
"""series: [(name, [y or None per x])]. Категориальная ось X (уровни параллелизма)."""
W, H, L, R, T, B = 640, 300, 56, 120, 16, 40
pw, ph = W - L - R, H - T - B
ys = [y for _, vals in series for y in vals if y is not None]
ymax = max(ys) * 1.1 if ys else 1
step = 10 ** max(0, len(str(int(ymax))) - 1)
for m in (1, 2, 5, 10):
if ymax / (step * m) <= 6:
step *= m
break
xs = [L + (pw * i / (len(xlabels) - 1) if len(xlabels) > 1 else pw / 2) for i in range(len(xlabels))]
yv = lambda v: T + ph - ph * v / ymax
o = ['<svg viewBox="0 0 %d %d" role="img" aria-label="%s">' % (W, H, html.escape(title))]
v = 0
while v <= ymax:
o.append('<line class="grid" x1="%d" x2="%d" y1="%.1f" y2="%.1f"/>' % (L, L + pw, yv(v), yv(v)))
o.append('<text class="tick" x="%d" y="%.1f" text-anchor="end" dy="4">%g</text>' % (L - 6, yv(v), v))
v += step
o.append('<line class="axis" x1="%d" x2="%d" y1="%d" y2="%d"/>' % (L, L + pw, T + ph, T + ph))
for x, lab in zip(xs, xlabels):
o.append('<text class="tick" x="%.1f" y="%d" text-anchor="middle">%s</text>' % (x, T + ph + 16, lab))
o.append('<text class="tick" x="%d" y="%d" text-anchor="middle">параллелизм (closed-loop воркеры)</text>' % (L + pw / 2, H - 4))
o.append('<text class="tick" x="%d" y="%d">%s</text>' % (L, T - 4 if T > 12 else 10, html.escape(ylabel)))
for i, (name, vals) in enumerate(series):
col, dash = COLORS[i % 4], DASH[i % 4]
pts = [(x, yv(y), y, lab) for x, y, lab in zip(xs, vals, xlabels) if y is not None]
if not pts:
continue
d = ' '.join(('M' if j == 0 else 'L') + '%.1f %.1f' % (p[0], p[1]) for j, p in enumerate(pts))
o.append('<path d="%s" fill="none" stroke="var(%s)" stroke-width="2" stroke-dasharray="%s" '
'stroke-linejoin="round" stroke-linecap="round"/>' % (d, col, dash))
for x, y, val, lab in pts:
tip = '%s · c=%s · %s' % (name, lab, fmt(val))
o.append('<circle class="pt" cx="%.1f" cy="%.1f" r="4" fill="var(%s)" data-tip="%s"/>' % (x, y, col, html.escape(tip)))
o.append('<circle class="hit" cx="%.1f" cy="%.1f" r="12" data-tip="%s"/>' % (x, y, html.escape(tip)))
lx, ly = pts[-1][0], pts[-1][1]
o.append('<text class="dlabel" x="%.1f" y="%.1f" dy="4">%s</text>' % (lx + 8, ly + (i - 1.5) * 3, html.escape(name)))
o.append('</svg>')
legend = ''.join('<span class="key"><svg width="22" height="10"><line x1="1" x2="21" y1="5" y2="5" stroke="var(%s)" '
'stroke-width="2" stroke-dasharray="%s"/></svg>%s</span>' % (COLORS[i % 4], DASH[i % 4], html.escape(n))
for i, (n, _) in enumerate(series))
return '<figure><figcaption>%s</figcaption><div class="legend">%s</div>%s</figure>' % (html.escape(title), legend, ''.join(o))
CSS = '''
.viz-root{color-scheme:light;--surface-1:#fcfcfb;--page:#f9f9f7;--text-primary:#0b0b0b;--text-secondary:#52514e;
--muted:#898781;--grid:#e1e0d9;--axis:#c3c2b7;--border:rgba(11,11,11,.10);
--series-1:#2a78d6;--series-2:#eb6834;--series-3:#1baf7a;--series-4:#eda100}
@media (prefers-color-scheme:dark){:root:where(:not([data-theme="light"])) .viz-root{color-scheme:dark;--surface-1:#1a1a19;
--page:#0d0d0d;--text-primary:#f0efec;--text-secondary:#c3c2b7;--grid:#2c2c2a;--axis:#383835;--border:rgba(255,255,255,.10);
--series-1:#3987e5;--series-2:#d95926;--series-3:#199e70;--series-4:#c98500}}
:root[data-theme="dark"] .viz-root{color-scheme:dark;--surface-1:#1a1a19;--page:#0d0d0d;--text-primary:#f0efec;
--text-secondary:#c3c2b7;--grid:#2c2c2a;--axis:#383835;--border:rgba(255,255,255,.10);
--series-1:#3987e5;--series-2:#d95926;--series-3:#199e70;--series-4:#c98500}
body{margin:0;background:var(--page)}
.viz-root{font:14px/1.45 system-ui,sans-serif;color:var(--text-primary);background:var(--page);padding:16px;max-width:1100px;margin:auto}
h1{font-size:22px}h2{font-size:18px;margin-top:32px}p,li{color:var(--text-secondary)}
.charts{display:grid;grid-template-columns:repeat(auto-fit,minmax(min(320px,100%),1fr));gap:16px}
figure{margin:0;min-width:0;background:var(--surface-1);border:1px solid var(--border);border-radius:8px;padding:12px}
figcaption{font-weight:600;margin-bottom:6px}svg{width:100%;height:auto;display:block}
.legend{display:flex;flex-wrap:wrap;gap:12px;font-size:12px;color:var(--text-secondary);margin-bottom:4px}
.key{display:inline-flex;align-items:center;gap:4px}
.grid{stroke:var(--grid);stroke-width:1}.axis{stroke:var(--axis);stroke-width:1}
.tick{fill:var(--muted);font-size:11px}.dlabel{fill:var(--text-secondary);font-size:11px}
.pt{stroke:var(--surface-1);stroke-width:2}.hit{fill:transparent;cursor:default}
table{border-collapse:collapse;width:100%;font-size:12px;background:var(--surface-1);font-variant-numeric:tabular-nums}
th,td{border-bottom:1px solid var(--grid);padding:4px 6px;text-align:right}th:first-child,td:first-child{text-align:left}
.wrap{overflow-x:auto}
#tip{position:fixed;pointer-events:none;background:var(--surface-1);color:var(--text-primary);border:1px solid var(--border);
border-radius:6px;padding:4px 8px;font-size:12px;display:none;box-shadow:0 2px 8px rgba(0,0,0,.15)}
'''
JS = '''
const tip=document.getElementById('tip');
document.querySelectorAll('[data-tip]').forEach(el=>{
el.addEventListener('mousemove',e=>{tip.textContent=el.dataset.tip;tip.style.display='block';
tip.style.left=Math.min(e.clientX+12,innerWidth-220)+'px';tip.style.top=(e.clientY+12)+'px'});
el.addEventListener('mouseleave',()=>tip.style.display='none')});
'''
def table(headers, rows):
h = ''.join('<th>%s</th>' % html.escape(x) for x in headers)
b = ''.join('<tr>%s</tr>' % ''.join('<td>%s</td>' % html.escape(str(c)) for c in r) for r in rows)
return '<div class="wrap"><table><thead><tr>%s</tr></thead><tbody>%s</tbody></table></div>' % (h, b)
def main():
p = argparse.ArgumentParser()
p.add_argument('--input', default='evidence/track1/raw.jsonl')
p.add_argument('--experiments', default='experiments/track1.json')
p.add_argument('--outdir', default='evidence/track1')
args = p.parse_args()
src = ROOT / args.input
if not src.exists() or not src.read_text(encoding='utf-8').strip():
raise SystemExit('Нет данных: %s отсутствует или пуст. Сначала успешно выполните scripts/track1.py '
'(его вывод должен содержать строки "c=... ok_rps=...").' % src)
rows = load(src)
exp = json.loads((ROOT / args.experiments).read_text(encoding='utf-8'))
titles = {s['name']: s for s in exp['series']}
agg = aggregate(rows)
order = series_order(rows)
outdir = ROOT / args.outdir
outdir.mkdir(parents=True, exist_ok=True)
headers = ['series', 'config', 'c', 'repeats', 'ok_rps', 'rps_min', 'rps_max', 'write_rps', 'read_rps', 'p50_ms',
'p95_ms', 'p99_ms', 'err', 'util_api', 'util_pg', 'util_redis', 'util_lb', 'pool_empty', 'pool_wait_ms', 'pg_active']
csv_rows = []
for s, c in order:
for l in levels_of(agg, s, c):
a = agg[(s, c, l)]
csv_rows.append([s, c, l, a['repeats'], fmt(a['ok_rps']), fmt(a['ok_rps_min']), fmt(a['ok_rps_max']),
fmt(a['write_rps']), fmt(a['read_rps']), fmt(a['p50'], 2), fmt(a['p95'], 2), fmt(a['p99'], 2),
fmt(a['error_ratio'], 4), fmt(a['util_api'], 2), fmt(a['util_postgres'], 2),
fmt(a['util_redis'], 2), fmt(a['util_lb'], 2), fmt(a['empty_acquire_ratio'], 3),
fmt(a['acquire_wait_ms'], 2), fmt(a['pg_active_max'], 0)])
with (outdir / 'summary.csv').open('w', newline='', encoding='utf-8') as f:
csv.writer(f).writerows([headers] + csv_rows)
md = ['# Трек 1 — сводка (сгенерировано scripts/track1_report.py)', '',
'Медианы по повторам; util_* — доля CPU-лимита контейнера роли; pool_empty — доля acquire без свободного соединения.', '']
body = ['<h1>Трек 1 — цена масштабирования</h1>',
'<p>Медианы по повторам. Наведите на точку — значение. Полные данные — таблицы под графиками и summary.csv.</p>']
for name in dict.fromkeys(s for s, _ in order):
meta = titles.get(name, {'title': name, 'hypothesis': ''})
cfgs = configs_of(order, name)
lv = sorted({l for c in cfgs for l in levels_of(agg, name, c)})
xl = [str(l) for l in lv]
get = lambda c, k: [agg[(name, c, l)][k] if (name, c, l) in agg else None for l in lv]
body.append('<h2>%s — %s</h2><p><b>Гипотеза:</b> %s</p>' % (name, html.escape(meta['title']), html.escape(meta['hypothesis'])))
body.append('<div class="charts">%s%s</div>' % (
line_chart('Успешные RPS', xl, [(c, get(c, 'ok_rps')) for c in cfgs[:4]], 'ok RPS'),
line_chart('p95 задержки, мс', xl, [(c, get(c, 'p95')) for c in cfgs[:4]], 'мс')))
sat_rows = []
md += ['## %s — %s' % (name, meta['title']), '', '**Гипотеза:** ' + meta['hypothesis'], '',
'| config | пик RPS | c пика | c насыщения | узкое место (util) | pool empty / wait ms |', '|---|---|---|---|---|---|']
base_peak = None
for c in cfgs:
st = saturation(agg, name, c)
base_peak = base_peak or st['peak_rps']
bn = '%s (%s)%s' % (st['bottleneck'], fmt(st['bottleneck_util'], 2), ' — насыщена' if st['saturated'] else '')
pool = '%s / %s' % (fmt(st['pool_empty_ratio'], 3), fmt(st['pool_wait_ms'], 2))
md.append('| %s | %s | %s | %s | %s | %s |' % (c, fmt(st['peak_rps']), st['peak_level'], st['knee_level'], bn, pool))
sat_rows.append([c, fmt(st['peak_rps']), '%.2f×' % (st['peak_rps'] / base_peak) if base_peak else '—',
st['peak_level'], st['knee_level'], bn, pool])
body.append(table(['config', 'пик RPS', 'к первой', 'c пика', 'c насыщения', 'узкое место', 'pool empty / wait ms'], sat_rows))
if name.startswith('D') and cfgs:
baseline = cfgs[0]
from_level = saturation(agg, name, baseline)['knee_level']
imp_rows = []
md += ['', 'Сравнение с `%s` (уровни ≥ насыщения baseline):' % baseline, '',
'| кандидат | c | ΔRPS % | Δp95 % | повторы b/c | подтверждено |', '|---|---|---|---|---|---|']
for c in cfgs[1:]:
for r in improvement(rows, agg, name, baseline, c, min(from_level, max(levels_of(agg, name, baseline)))):
row = [c, r['level'], fmt(r['rps_delta_pct']), fmt(r['p95_delta_pct']), '%d/%d' % r['repeats'],
'да' if r['confirmed'] else 'нет']
imp_rows.append(row)
md.append('| %s |' % ' | '.join(str(x) for x in row))
body.append('<h3>Кандидаты против %s</h3>' % html.escape(baseline))
body.append(table(['кандидат', 'c', 'ΔRPS %', 'Δp95 %', 'повторы b/c', 'подтверждено'], imp_rows))
md.append('')
series_rows = [r for r in csv_rows if r[0] == name]
body.append('<details><summary>Все точки серии</summary>%s</details>' % table(headers[1:], [r[1:] for r in series_rows]))
(outdir / 'summary.md').write_text('\n'.join(md) + '\n', encoding='utf-8')
page = ('<!doctype html><html lang="ru"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">'
'<title>Трек 1 отчёт</title><style>%s</style></head><body><div class="viz-root">%s<div id="tip"></div></div>'
'<script>%s</script></body></html>') % (CSS, ''.join(body), JS)
(outdir / 'report.html').write_text(page, encoding='utf-8')
print('written:', outdir / 'summary.csv', outdir / 'summary.md', outdir / 'report.html')
if __name__ == '__main__':
main()