- доказательства уровней 3–4 (прогон 2 на GitHub runner) - трек 1: 129 прогонов серий A–E, анализ, графики, подтверждённое улучшение - Dockerfile: базовые образы закреплены по digest - собственные тесты перенесены в student-tests/ Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
337 lines
20 KiB
Python
337 lines
20 KiB
Python
#!/usr/bin/env python3
|
||
"""Анализ трека 1: evidence/track1/raw.jsonl → summary.csv, summary.md, report.html.
|
||
|
||
Правила анализа (фиксированы заранее, чтобы не подгонять выводы под данные):
|
||
* агрегат по повторам — медиана; разброс — min..max;
|
||
* точка насыщения конфигурации — первый уровень параллелизма, на котором
|
||
медианный ok_rps вырос меньше чем на 10% относительно предыдущего уровня;
|
||
* узкое место — роль (api/postgres/redis/lb) с наибольшей утилизацией CPU
|
||
от своего лимита на уровне насыщения; ≥0.85 — «насыщена»; дополнительно
|
||
признак пула: доля acquire без свободного соединения и среднее ожидание;
|
||
* если ни одна роль не насыщена, а сумма занятых ядер (util × лимит × число
|
||
контейнеров по всем ролям) ≥ 0.85 × CPU хоста — узкое место «CPU хоста»
|
||
(критерий добавлен после прогона 2: сумма лимитов превысила 2 vCPU runner);
|
||
* улучшение «подтверждено», если на уровне хотя бы одного параллелизма ≥ уровня
|
||
насыщения baseline min(ok_rps кандидата) > max(ok_rps baseline) по всем повторам,
|
||
медиана p95 не хуже и доля ошибок не выше.
|
||
Только stdlib.
|
||
"""
|
||
import argparse
|
||
import csv
|
||
import html
|
||
import json
|
||
from pathlib import Path
|
||
import statistics
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
ROLES = ('api', 'postgres', 'redis', 'lb')
|
||
SAT_GAIN = 0.10
|
||
SAT_UTIL = 0.85
|
||
|
||
|
||
def load(path):
|
||
rows = []
|
||
for line in path.read_text(encoding='utf-8').splitlines():
|
||
if line.strip():
|
||
rows.append(json.loads(line))
|
||
return rows
|
||
|
||
|
||
LIMIT_ENV = {'api': 'API_CPUS', 'postgres': 'PG_CPUS', 'redis': 'REDIS_CPUS', 'lb': 'LB_CPUS'}
|
||
DEFAULT_LIMIT = {'postgres': 1.0, 'redis': 0.5, 'lb': 0.5}
|
||
|
||
|
||
def cores_used(r):
|
||
total = 0.0
|
||
for role in ROLES:
|
||
x = r['resources'].get(role)
|
||
if not x or x.get('cpu_utilization_of_limit') is None:
|
||
continue
|
||
lim = float(r['env'].get(LIMIT_ENV[role]) or DEFAULT_LIMIT.get(role, 0.5))
|
||
total += x['cpu_utilization_of_limit'] * lim * x.get('containers', 1)
|
||
return total
|
||
|
||
|
||
def host_cpus(r):
|
||
try:
|
||
return int(r.get('host', {}).get('docker_cpus') or r.get('host', {}).get('host_cpus'))
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def med(xs):
|
||
xs = [x for x in xs if x is not None]
|
||
return statistics.median(xs) if xs else None
|
||
|
||
|
||
def aggregate(rows):
|
||
groups = {}
|
||
for r in rows:
|
||
groups.setdefault((r['series'], r['config'], r['concurrency']), []).append(r)
|
||
agg = {}
|
||
for key, rs in groups.items():
|
||
b = [r['bench'] for r in rs]
|
||
a = {
|
||
'repeats': len(rs),
|
||
'env': rs[0]['env'],
|
||
'ok_rps': med(x['ok_rps'] for x in b),
|
||
'ok_rps_min': min(x['ok_rps'] for x in b),
|
||
'ok_rps_max': max(x['ok_rps'] for x in b),
|
||
'write_rps': med(x['ok_write_rps'] for x in b),
|
||
'read_rps': med(x['ok_read_rps'] for x in b),
|
||
'p50': med(x['latency_ms']['p50'] for x in b),
|
||
'p95': med(x['latency_ms']['p95'] for x in b),
|
||
'p99': med(x['latency_ms']['p99'] for x in b),
|
||
'write_p95': med(x['write_ms']['p95'] for x in b),
|
||
'error_ratio': med(x['error_ratio'] for x in b),
|
||
'empty_acquire_ratio': med(r['pool'].get('empty_acquire_ratio') for r in rs),
|
||
'acquire_wait_ms': med(r['pool'].get('avg_acquire_wait_ms') for r in rs),
|
||
'host_cores': med(cores_used(r) for r in rs),
|
||
'host_cpus': host_cpus(rs[0]),
|
||
'pg_active_max': med(r['resources'].get('postgres_connections', {}).get('active_max') for r in rs),
|
||
}
|
||
for role in ROLES:
|
||
a['util_' + role] = med(r['resources'].get(role, {}).get('cpu_utilization_of_limit') for r in rs)
|
||
agg[key] = a
|
||
return agg
|
||
|
||
|
||
def series_order(rows):
|
||
seen, order = set(), []
|
||
for r in rows:
|
||
k = (r['series'], r['config'])
|
||
if k not in seen:
|
||
seen.add(k)
|
||
order.append(k)
|
||
return order
|
||
|
||
|
||
def configs_of(order, series):
|
||
return [c for s, c in order if s == series]
|
||
|
||
|
||
def levels_of(agg, series, config):
|
||
return sorted(l for s, c, l in agg if s == series and c == config)
|
||
|
||
|
||
def saturation(agg, series, config):
|
||
lv = levels_of(agg, series, config)
|
||
peak_l = max(lv, key=lambda l: agg[(series, config, l)]['ok_rps'])
|
||
knee = lv[-1]
|
||
for prev, cur in zip(lv, lv[1:]):
|
||
if agg[(series, config, cur)]['ok_rps'] < agg[(series, config, prev)]['ok_rps'] * (1 + SAT_GAIN):
|
||
knee = cur
|
||
break
|
||
a = agg[(series, config, knee)]
|
||
utils = {r: a['util_' + r] for r in ROLES if a['util_' + r] is not None}
|
||
bottleneck = max(utils, key=utils.get) if utils else None
|
||
host_share = a['host_cores'] / a['host_cpus'] if a['host_cpus'] and a['host_cores'] is not None else None
|
||
if bottleneck and utils[bottleneck] < SAT_UTIL and host_share is not None and host_share >= SAT_UTIL:
|
||
bottleneck, utils = 'host_cpu', dict(utils, host_cpu=host_share)
|
||
return {'host_share': host_share,'peak_level': peak_l, 'peak_rps': agg[(series, config, peak_l)]['ok_rps'], 'knee_level': knee,
|
||
'bottleneck': bottleneck, 'bottleneck_util': utils.get(bottleneck),
|
||
'saturated': bool(bottleneck and utils[bottleneck] >= SAT_UTIL), 'utils': utils,
|
||
'pool_empty_ratio': a['empty_acquire_ratio'], 'pool_wait_ms': a['acquire_wait_ms']}
|
||
|
||
|
||
def improvement(rows, agg, series, baseline, candidate, from_level):
|
||
res = []
|
||
for l in levels_of(agg, series, baseline):
|
||
if l < from_level or (series, candidate, l) not in agg:
|
||
continue
|
||
b = [r['bench'] for r in rows if (r['series'], r['config'], r['concurrency']) == (series, baseline, l)]
|
||
c = [r['bench'] for r in rows if (r['series'], r['config'], r['concurrency']) == (series, candidate, l)]
|
||
ab, ac = agg[(series, baseline, l)], agg[(series, candidate, l)]
|
||
confirmed = (min(x['ok_rps'] for x in c) > max(x['ok_rps'] for x in b)
|
||
and ac['p95'] <= ab['p95'] and ac['error_ratio'] <= ab['error_ratio'])
|
||
res.append({'level': l, 'rps_delta_pct': 100 * (ac['ok_rps'] / ab['ok_rps'] - 1) if ab['ok_rps'] else None,
|
||
'p95_delta_pct': 100 * (ac['p95'] / ab['p95'] - 1) if ab['p95'] else None,
|
||
'repeats': (len(b), len(c)), 'confirmed': confirmed})
|
||
return res
|
||
|
||
|
||
def fmt(x, nd=1):
|
||
return '—' if x is None else ('%.*f' % (nd, x))
|
||
|
||
|
||
# ---------- SVG ----------
|
||
COLORS = ['--series-1', '--series-2', '--series-3', '--series-4']
|
||
DASH = ['', '6 3', '2 3', '8 3 2 3']
|
||
|
||
|
||
def line_chart(title, xlabels, series, ylabel):
|
||
"""series: [(name, [y or None per x])]. Категориальная ось X (уровни параллелизма)."""
|
||
W, H, L, R, T, B = 640, 300, 56, 120, 16, 40
|
||
pw, ph = W - L - R, H - T - B
|
||
ys = [y for _, vals in series for y in vals if y is not None]
|
||
ymax = max(ys) * 1.1 if ys else 1
|
||
step = 10 ** max(0, len(str(int(ymax))) - 1)
|
||
for m in (1, 2, 5, 10):
|
||
if ymax / (step * m) <= 6:
|
||
step *= m
|
||
break
|
||
xs = [L + (pw * i / (len(xlabels) - 1) if len(xlabels) > 1 else pw / 2) for i in range(len(xlabels))]
|
||
yv = lambda v: T + ph - ph * v / ymax
|
||
o = ['<svg viewBox="0 0 %d %d" role="img" aria-label="%s">' % (W, H, html.escape(title))]
|
||
v = 0
|
||
while v <= ymax:
|
||
o.append('<line class="grid" x1="%d" x2="%d" y1="%.1f" y2="%.1f"/>' % (L, L + pw, yv(v), yv(v)))
|
||
o.append('<text class="tick" x="%d" y="%.1f" text-anchor="end" dy="4">%g</text>' % (L - 6, yv(v), v))
|
||
v += step
|
||
o.append('<line class="axis" x1="%d" x2="%d" y1="%d" y2="%d"/>' % (L, L + pw, T + ph, T + ph))
|
||
for x, lab in zip(xs, xlabels):
|
||
o.append('<text class="tick" x="%.1f" y="%d" text-anchor="middle">%s</text>' % (x, T + ph + 16, lab))
|
||
o.append('<text class="tick" x="%d" y="%d" text-anchor="middle">параллелизм (closed-loop воркеры)</text>' % (L + pw / 2, H - 4))
|
||
o.append('<text class="tick" x="%d" y="%d">%s</text>' % (L, T - 4 if T > 12 else 10, html.escape(ylabel)))
|
||
for i, (name, vals) in enumerate(series):
|
||
col, dash = COLORS[i % 4], DASH[i % 4]
|
||
pts = [(x, yv(y), y, lab) for x, y, lab in zip(xs, vals, xlabels) if y is not None]
|
||
if not pts:
|
||
continue
|
||
d = ' '.join(('M' if j == 0 else 'L') + '%.1f %.1f' % (p[0], p[1]) for j, p in enumerate(pts))
|
||
o.append('<path d="%s" fill="none" stroke="var(%s)" stroke-width="2" stroke-dasharray="%s" '
|
||
'stroke-linejoin="round" stroke-linecap="round"/>' % (d, col, dash))
|
||
for x, y, val, lab in pts:
|
||
tip = '%s · c=%s · %s' % (name, lab, fmt(val))
|
||
o.append('<circle class="pt" cx="%.1f" cy="%.1f" r="4" fill="var(%s)" data-tip="%s"/>' % (x, y, col, html.escape(tip)))
|
||
o.append('<circle class="hit" cx="%.1f" cy="%.1f" r="12" data-tip="%s"/>' % (x, y, html.escape(tip)))
|
||
lx, ly = pts[-1][0], pts[-1][1]
|
||
o.append('<text class="dlabel" x="%.1f" y="%.1f" dy="4">%s</text>' % (lx + 8, ly + (i - 1.5) * 3, html.escape(name)))
|
||
o.append('</svg>')
|
||
legend = ''.join('<span class="key"><svg width="22" height="10"><line x1="1" x2="21" y1="5" y2="5" stroke="var(%s)" '
|
||
'stroke-width="2" stroke-dasharray="%s"/></svg>%s</span>' % (COLORS[i % 4], DASH[i % 4], html.escape(n))
|
||
for i, (n, _) in enumerate(series))
|
||
return '<figure><figcaption>%s</figcaption><div class="legend">%s</div>%s</figure>' % (html.escape(title), legend, ''.join(o))
|
||
|
||
|
||
CSS = '''
|
||
.viz-root{color-scheme:light;--surface-1:#fcfcfb;--page:#f9f9f7;--text-primary:#0b0b0b;--text-secondary:#52514e;
|
||
--muted:#898781;--grid:#e1e0d9;--axis:#c3c2b7;--border:rgba(11,11,11,.10);
|
||
--series-1:#2a78d6;--series-2:#eb6834;--series-3:#1baf7a;--series-4:#eda100}
|
||
@media (prefers-color-scheme:dark){:root:where(:not([data-theme="light"])) .viz-root{color-scheme:dark;--surface-1:#1a1a19;
|
||
--page:#0d0d0d;--text-primary:#f0efec;--text-secondary:#c3c2b7;--grid:#2c2c2a;--axis:#383835;--border:rgba(255,255,255,.10);
|
||
--series-1:#3987e5;--series-2:#d95926;--series-3:#199e70;--series-4:#c98500}}
|
||
:root[data-theme="dark"] .viz-root{color-scheme:dark;--surface-1:#1a1a19;--page:#0d0d0d;--text-primary:#f0efec;
|
||
--text-secondary:#c3c2b7;--grid:#2c2c2a;--axis:#383835;--border:rgba(255,255,255,.10);
|
||
--series-1:#3987e5;--series-2:#d95926;--series-3:#199e70;--series-4:#c98500}
|
||
body{margin:0;background:var(--page)}
|
||
.viz-root{font:14px/1.45 system-ui,sans-serif;color:var(--text-primary);background:var(--page);padding:16px;max-width:1100px;margin:auto}
|
||
h1{font-size:22px}h2{font-size:18px;margin-top:32px}p,li{color:var(--text-secondary)}
|
||
.charts{display:grid;grid-template-columns:repeat(auto-fit,minmax(min(320px,100%),1fr));gap:16px}
|
||
figure{margin:0;min-width:0;background:var(--surface-1);border:1px solid var(--border);border-radius:8px;padding:12px}
|
||
figcaption{font-weight:600;margin-bottom:6px}svg{width:100%;height:auto;display:block}
|
||
.legend{display:flex;flex-wrap:wrap;gap:12px;font-size:12px;color:var(--text-secondary);margin-bottom:4px}
|
||
.key{display:inline-flex;align-items:center;gap:4px}
|
||
.grid{stroke:var(--grid);stroke-width:1}.axis{stroke:var(--axis);stroke-width:1}
|
||
.tick{fill:var(--muted);font-size:11px}.dlabel{fill:var(--text-secondary);font-size:11px}
|
||
.pt{stroke:var(--surface-1);stroke-width:2}.hit{fill:transparent;cursor:default}
|
||
table{border-collapse:collapse;width:100%;font-size:12px;background:var(--surface-1);font-variant-numeric:tabular-nums}
|
||
th,td{border-bottom:1px solid var(--grid);padding:4px 6px;text-align:right}th:first-child,td:first-child{text-align:left}
|
||
.wrap{overflow-x:auto}
|
||
#tip{position:fixed;pointer-events:none;background:var(--surface-1);color:var(--text-primary);border:1px solid var(--border);
|
||
border-radius:6px;padding:4px 8px;font-size:12px;display:none;box-shadow:0 2px 8px rgba(0,0,0,.15)}
|
||
'''
|
||
JS = '''
|
||
const tip=document.getElementById('tip');
|
||
document.querySelectorAll('[data-tip]').forEach(el=>{
|
||
el.addEventListener('mousemove',e=>{tip.textContent=el.dataset.tip;tip.style.display='block';
|
||
tip.style.left=Math.min(e.clientX+12,innerWidth-220)+'px';tip.style.top=(e.clientY+12)+'px'});
|
||
el.addEventListener('mouseleave',()=>tip.style.display='none')});
|
||
'''
|
||
|
||
|
||
def table(headers, rows):
|
||
h = ''.join('<th>%s</th>' % html.escape(x) for x in headers)
|
||
b = ''.join('<tr>%s</tr>' % ''.join('<td>%s</td>' % html.escape(str(c)) for c in r) for r in rows)
|
||
return '<div class="wrap"><table><thead><tr>%s</tr></thead><tbody>%s</tbody></table></div>' % (h, b)
|
||
|
||
|
||
def main():
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument('--input', default='evidence/track1/raw.jsonl')
|
||
p.add_argument('--experiments', default='experiments/track1.json')
|
||
p.add_argument('--outdir', default='evidence/track1')
|
||
args = p.parse_args()
|
||
src = ROOT / args.input
|
||
if not src.exists() or not src.read_text(encoding='utf-8').strip():
|
||
raise SystemExit('Нет данных: %s отсутствует или пуст. Сначала успешно выполните scripts/track1.py '
|
||
'(его вывод должен содержать строки "c=... ok_rps=...").' % src)
|
||
rows = load(src)
|
||
exp = json.loads((ROOT / args.experiments).read_text(encoding='utf-8'))
|
||
titles = {s['name']: s for s in exp['series']}
|
||
agg = aggregate(rows)
|
||
order = series_order(rows)
|
||
outdir = ROOT / args.outdir
|
||
outdir.mkdir(parents=True, exist_ok=True)
|
||
|
||
headers = ['series', 'config', 'c', 'repeats', 'ok_rps', 'rps_min', 'rps_max', 'write_rps', 'read_rps', 'p50_ms',
|
||
'p95_ms', 'p99_ms', 'err', 'util_api', 'util_pg', 'util_redis', 'util_lb', 'host_cores', 'pool_empty', 'pool_wait_ms', 'pg_active']
|
||
csv_rows = []
|
||
for s, c in order:
|
||
for l in levels_of(agg, s, c):
|
||
a = agg[(s, c, l)]
|
||
csv_rows.append([s, c, l, a['repeats'], fmt(a['ok_rps']), fmt(a['ok_rps_min']), fmt(a['ok_rps_max']),
|
||
fmt(a['write_rps']), fmt(a['read_rps']), fmt(a['p50'], 2), fmt(a['p95'], 2), fmt(a['p99'], 2),
|
||
fmt(a['error_ratio'], 4), fmt(a['util_api'], 2), fmt(a['util_postgres'], 2),
|
||
fmt(a['util_redis'], 2), fmt(a['util_lb'], 2),
|
||
'%s/%s' % (fmt(a['host_cores'], 2), a['host_cpus']), fmt(a['empty_acquire_ratio'], 3),
|
||
fmt(a['acquire_wait_ms'], 2), fmt(a['pg_active_max'], 0)])
|
||
with (outdir / 'summary.csv').open('w', newline='', encoding='utf-8') as f:
|
||
csv.writer(f).writerows([headers] + csv_rows)
|
||
|
||
md = ['# Трек 1 — сводка (сгенерировано scripts/track1_report.py)', '',
|
||
'Медианы по повторам; util_* — доля CPU-лимита контейнера роли; pool_empty — доля acquire без свободного соединения.', '']
|
||
body = ['<h1>Трек 1 — цена масштабирования</h1>',
|
||
'<p>Медианы по повторам. Наведите на точку — значение. Полные данные — таблицы под графиками и summary.csv.</p>']
|
||
for name in dict.fromkeys(s for s, _ in order):
|
||
meta = titles.get(name, {'title': name, 'hypothesis': ''})
|
||
cfgs = configs_of(order, name)
|
||
lv = sorted({l for c in cfgs for l in levels_of(agg, name, c)})
|
||
xl = [str(l) for l in lv]
|
||
get = lambda c, k: [agg[(name, c, l)][k] if (name, c, l) in agg else None for l in lv]
|
||
body.append('<h2>%s — %s</h2><p><b>Гипотеза:</b> %s</p>' % (name, html.escape(meta['title']), html.escape(meta['hypothesis'])))
|
||
body.append('<div class="charts">%s%s</div>' % (
|
||
line_chart('Успешные RPS', xl, [(c, get(c, 'ok_rps')) for c in cfgs[:4]], 'ok RPS'),
|
||
line_chart('p95 задержки, мс', xl, [(c, get(c, 'p95')) for c in cfgs[:4]], 'мс')))
|
||
sat_rows = []
|
||
md += ['## %s — %s' % (name, meta['title']), '', '**Гипотеза:** ' + meta['hypothesis'], '',
|
||
'| config | пик RPS | c пика | c насыщения | узкое место (util) | CPU хоста | pool empty / wait ms |', '|---|---|---|---|---|---|---|']
|
||
base_peak = None
|
||
for c in cfgs:
|
||
st = saturation(agg, name, c)
|
||
base_peak = base_peak or st['peak_rps']
|
||
bn = '%s (%s)%s' % (st['bottleneck'], fmt(st['bottleneck_util'], 2), ' — насыщена' if st['saturated'] else '')
|
||
pool = '%s / %s' % (fmt(st['pool_empty_ratio'], 3), fmt(st['pool_wait_ms'], 2))
|
||
hs = fmt(100 * st['host_share'], 0) + '%' if st['host_share'] is not None else '—'
|
||
md.append('| %s | %s | %s | %s | %s | %s | %s |' % (c, fmt(st['peak_rps']), st['peak_level'], st['knee_level'], bn, hs, pool))
|
||
sat_rows.append([c, fmt(st['peak_rps']), '%.2f×' % (st['peak_rps'] / base_peak) if base_peak else '—',
|
||
st['peak_level'], st['knee_level'], bn, hs, pool])
|
||
body.append(table(['config', 'пик RPS', 'к первой', 'c пика', 'c насыщения', 'узкое место', 'CPU хоста', 'pool empty / wait ms'], sat_rows))
|
||
if name.startswith('D') and cfgs:
|
||
baseline = cfgs[0]
|
||
from_level = saturation(agg, name, baseline)['knee_level']
|
||
imp_rows = []
|
||
md += ['', 'Сравнение с `%s` (уровни ≥ насыщения baseline):' % baseline, '',
|
||
'| кандидат | c | ΔRPS % | Δp95 % | повторы b/c | подтверждено |', '|---|---|---|---|---|---|']
|
||
for c in cfgs[1:]:
|
||
for r in improvement(rows, agg, name, baseline, c, min(from_level, max(levels_of(agg, name, baseline)))):
|
||
row = [c, r['level'], fmt(r['rps_delta_pct']), fmt(r['p95_delta_pct']), '%d/%d' % r['repeats'],
|
||
'да' if r['confirmed'] else 'нет']
|
||
imp_rows.append(row)
|
||
md.append('| %s |' % ' | '.join(str(x) for x in row))
|
||
body.append('<h3>Кандидаты против %s</h3>' % html.escape(baseline))
|
||
body.append(table(['кандидат', 'c', 'ΔRPS %', 'Δp95 %', 'повторы b/c', 'подтверждено'], imp_rows))
|
||
md.append('')
|
||
series_rows = [r for r in csv_rows if r[0] == name]
|
||
body.append('<details><summary>Все точки серии</summary>%s</details>' % table(headers[1:], [r[1:] for r in series_rows]))
|
||
(outdir / 'summary.md').write_text('\n'.join(md) + '\n', encoding='utf-8')
|
||
page = ('<!doctype html><html lang="ru"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">'
|
||
'<title>Трек 1 отчёт</title><style>%s</style></head><body><div class="viz-root">%s<div id="tip"></div></div>'
|
||
'<script>%s</script></body></html>') % (CSS, ''.join(body), JS)
|
||
(outdir / 'report.html').write_text(page, encoding='utf-8')
|
||
print('written:', outdir / 'summary.csv', outdir / 'summary.md', outdir / 'report.html')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|