Files
mtusi2026lab3/scripts/track1_report.py
T
Weinmaier_AndrewandClaude Opus 5.5 7366f430ca lab3: результаты Docker-стенда и трека 1, отчёт
- доказательства уровней 3–4 (прогон 2 на GitHub runner)
- трек 1: 129 прогонов серий A–E, анализ, графики, подтверждённое улучшение
- Dockerfile: базовые образы закреплены по digest
- собственные тесты перенесены в student-tests/

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-23 18:06:45 +03:00

337 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Анализ трека 1: evidence/track1/raw.jsonl → summary.csv, summary.md, report.html.
Правила анализа (фиксированы заранее, чтобы не подгонять выводы под данные):
* агрегат по повторам — медиана; разброс — min..max;
* точка насыщения конфигурации — первый уровень параллелизма, на котором
медианный ok_rps вырос меньше чем на 10% относительно предыдущего уровня;
* узкое место — роль (api/postgres/redis/lb) с наибольшей утилизацией CPU
от своего лимита на уровне насыщения; ≥0.85 — «насыщена»; дополнительно
признак пула: доля acquire без свободного соединения и среднее ожидание;
* если ни одна роль не насыщена, а сумма занятых ядер (util × лимит × число
контейнеров по всем ролям) ≥ 0.85 × CPU хоста — узкое место «CPU хоста»
(критерий добавлен после прогона 2: сумма лимитов превысила 2 vCPU runner);
* улучшение «подтверждено», если на уровне хотя бы одного параллелизма ≥ уровня
насыщения baseline min(ok_rps кандидата) > max(ok_rps baseline) по всем повторам,
медиана p95 не хуже и доля ошибок не выше.
Только stdlib.
"""
import argparse
import csv
import html
import json
from pathlib import Path
import statistics
ROOT = Path(__file__).resolve().parents[1]
ROLES = ('api', 'postgres', 'redis', 'lb')
SAT_GAIN = 0.10
SAT_UTIL = 0.85
def load(path):
rows = []
for line in path.read_text(encoding='utf-8').splitlines():
if line.strip():
rows.append(json.loads(line))
return rows
LIMIT_ENV = {'api': 'API_CPUS', 'postgres': 'PG_CPUS', 'redis': 'REDIS_CPUS', 'lb': 'LB_CPUS'}
DEFAULT_LIMIT = {'postgres': 1.0, 'redis': 0.5, 'lb': 0.5}
def cores_used(r):
total = 0.0
for role in ROLES:
x = r['resources'].get(role)
if not x or x.get('cpu_utilization_of_limit') is None:
continue
lim = float(r['env'].get(LIMIT_ENV[role]) or DEFAULT_LIMIT.get(role, 0.5))
total += x['cpu_utilization_of_limit'] * lim * x.get('containers', 1)
return total
def host_cpus(r):
try:
return int(r.get('host', {}).get('docker_cpus') or r.get('host', {}).get('host_cpus'))
except (TypeError, ValueError):
return None
def med(xs):
xs = [x for x in xs if x is not None]
return statistics.median(xs) if xs else None
def aggregate(rows):
groups = {}
for r in rows:
groups.setdefault((r['series'], r['config'], r['concurrency']), []).append(r)
agg = {}
for key, rs in groups.items():
b = [r['bench'] for r in rs]
a = {
'repeats': len(rs),
'env': rs[0]['env'],
'ok_rps': med(x['ok_rps'] for x in b),
'ok_rps_min': min(x['ok_rps'] for x in b),
'ok_rps_max': max(x['ok_rps'] for x in b),
'write_rps': med(x['ok_write_rps'] for x in b),
'read_rps': med(x['ok_read_rps'] for x in b),
'p50': med(x['latency_ms']['p50'] for x in b),
'p95': med(x['latency_ms']['p95'] for x in b),
'p99': med(x['latency_ms']['p99'] for x in b),
'write_p95': med(x['write_ms']['p95'] for x in b),
'error_ratio': med(x['error_ratio'] for x in b),
'empty_acquire_ratio': med(r['pool'].get('empty_acquire_ratio') for r in rs),
'acquire_wait_ms': med(r['pool'].get('avg_acquire_wait_ms') for r in rs),
'host_cores': med(cores_used(r) for r in rs),
'host_cpus': host_cpus(rs[0]),
'pg_active_max': med(r['resources'].get('postgres_connections', {}).get('active_max') for r in rs),
}
for role in ROLES:
a['util_' + role] = med(r['resources'].get(role, {}).get('cpu_utilization_of_limit') for r in rs)
agg[key] = a
return agg
def series_order(rows):
seen, order = set(), []
for r in rows:
k = (r['series'], r['config'])
if k not in seen:
seen.add(k)
order.append(k)
return order
def configs_of(order, series):
return [c for s, c in order if s == series]
def levels_of(agg, series, config):
return sorted(l for s, c, l in agg if s == series and c == config)
def saturation(agg, series, config):
lv = levels_of(agg, series, config)
peak_l = max(lv, key=lambda l: agg[(series, config, l)]['ok_rps'])
knee = lv[-1]
for prev, cur in zip(lv, lv[1:]):
if agg[(series, config, cur)]['ok_rps'] < agg[(series, config, prev)]['ok_rps'] * (1 + SAT_GAIN):
knee = cur
break
a = agg[(series, config, knee)]
utils = {r: a['util_' + r] for r in ROLES if a['util_' + r] is not None}
bottleneck = max(utils, key=utils.get) if utils else None
host_share = a['host_cores'] / a['host_cpus'] if a['host_cpus'] and a['host_cores'] is not None else None
if bottleneck and utils[bottleneck] < SAT_UTIL and host_share is not None and host_share >= SAT_UTIL:
bottleneck, utils = 'host_cpu', dict(utils, host_cpu=host_share)
return {'host_share': host_share,'peak_level': peak_l, 'peak_rps': agg[(series, config, peak_l)]['ok_rps'], 'knee_level': knee,
'bottleneck': bottleneck, 'bottleneck_util': utils.get(bottleneck),
'saturated': bool(bottleneck and utils[bottleneck] >= SAT_UTIL), 'utils': utils,
'pool_empty_ratio': a['empty_acquire_ratio'], 'pool_wait_ms': a['acquire_wait_ms']}
def improvement(rows, agg, series, baseline, candidate, from_level):
res = []
for l in levels_of(agg, series, baseline):
if l < from_level or (series, candidate, l) not in agg:
continue
b = [r['bench'] for r in rows if (r['series'], r['config'], r['concurrency']) == (series, baseline, l)]
c = [r['bench'] for r in rows if (r['series'], r['config'], r['concurrency']) == (series, candidate, l)]
ab, ac = agg[(series, baseline, l)], agg[(series, candidate, l)]
confirmed = (min(x['ok_rps'] for x in c) > max(x['ok_rps'] for x in b)
and ac['p95'] <= ab['p95'] and ac['error_ratio'] <= ab['error_ratio'])
res.append({'level': l, 'rps_delta_pct': 100 * (ac['ok_rps'] / ab['ok_rps'] - 1) if ab['ok_rps'] else None,
'p95_delta_pct': 100 * (ac['p95'] / ab['p95'] - 1) if ab['p95'] else None,
'repeats': (len(b), len(c)), 'confirmed': confirmed})
return res
def fmt(x, nd=1):
return '—' if x is None else ('%.*f' % (nd, x))
# ---------- SVG ----------
COLORS = ['--series-1', '--series-2', '--series-3', '--series-4']
DASH = ['', '6 3', '2 3', '8 3 2 3']
def line_chart(title, xlabels, series, ylabel):
"""series: [(name, [y or None per x])]. Категориальная ось X (уровни параллелизма)."""
W, H, L, R, T, B = 640, 300, 56, 120, 16, 40
pw, ph = W - L - R, H - T - B
ys = [y for _, vals in series for y in vals if y is not None]
ymax = max(ys) * 1.1 if ys else 1
step = 10 ** max(0, len(str(int(ymax))) - 1)
for m in (1, 2, 5, 10):
if ymax / (step * m) <= 6:
step *= m
break
xs = [L + (pw * i / (len(xlabels) - 1) if len(xlabels) > 1 else pw / 2) for i in range(len(xlabels))]
yv = lambda v: T + ph - ph * v / ymax
o = ['<svg viewBox="0 0 %d %d" role="img" aria-label="%s">' % (W, H, html.escape(title))]
v = 0
while v <= ymax:
o.append('<line class="grid" x1="%d" x2="%d" y1="%.1f" y2="%.1f"/>' % (L, L + pw, yv(v), yv(v)))
o.append('<text class="tick" x="%d" y="%.1f" text-anchor="end" dy="4">%g</text>' % (L - 6, yv(v), v))
v += step
o.append('<line class="axis" x1="%d" x2="%d" y1="%d" y2="%d"/>' % (L, L + pw, T + ph, T + ph))
for x, lab in zip(xs, xlabels):
o.append('<text class="tick" x="%.1f" y="%d" text-anchor="middle">%s</text>' % (x, T + ph + 16, lab))
o.append('<text class="tick" x="%d" y="%d" text-anchor="middle">параллелизм (closed-loop воркеры)</text>' % (L + pw / 2, H - 4))
o.append('<text class="tick" x="%d" y="%d">%s</text>' % (L, T - 4 if T > 12 else 10, html.escape(ylabel)))
for i, (name, vals) in enumerate(series):
col, dash = COLORS[i % 4], DASH[i % 4]
pts = [(x, yv(y), y, lab) for x, y, lab in zip(xs, vals, xlabels) if y is not None]
if not pts:
continue
d = ' '.join(('M' if j == 0 else 'L') + '%.1f %.1f' % (p[0], p[1]) for j, p in enumerate(pts))
o.append('<path d="%s" fill="none" stroke="var(%s)" stroke-width="2" stroke-dasharray="%s" '
'stroke-linejoin="round" stroke-linecap="round"/>' % (d, col, dash))
for x, y, val, lab in pts:
tip = '%s · c=%s · %s' % (name, lab, fmt(val))
o.append('<circle class="pt" cx="%.1f" cy="%.1f" r="4" fill="var(%s)" data-tip="%s"/>' % (x, y, col, html.escape(tip)))
o.append('<circle class="hit" cx="%.1f" cy="%.1f" r="12" data-tip="%s"/>' % (x, y, html.escape(tip)))
lx, ly = pts[-1][0], pts[-1][1]
o.append('<text class="dlabel" x="%.1f" y="%.1f" dy="4">%s</text>' % (lx + 8, ly + (i - 1.5) * 3, html.escape(name)))
o.append('</svg>')
legend = ''.join('<span class="key"><svg width="22" height="10"><line x1="1" x2="21" y1="5" y2="5" stroke="var(%s)" '
'stroke-width="2" stroke-dasharray="%s"/></svg>%s</span>' % (COLORS[i % 4], DASH[i % 4], html.escape(n))
for i, (n, _) in enumerate(series))
return '<figure><figcaption>%s</figcaption><div class="legend">%s</div>%s</figure>' % (html.escape(title), legend, ''.join(o))
CSS = '''
.viz-root{color-scheme:light;--surface-1:#fcfcfb;--page:#f9f9f7;--text-primary:#0b0b0b;--text-secondary:#52514e;
--muted:#898781;--grid:#e1e0d9;--axis:#c3c2b7;--border:rgba(11,11,11,.10);
--series-1:#2a78d6;--series-2:#eb6834;--series-3:#1baf7a;--series-4:#eda100}
@media (prefers-color-scheme:dark){:root:where(:not([data-theme="light"])) .viz-root{color-scheme:dark;--surface-1:#1a1a19;
--page:#0d0d0d;--text-primary:#f0efec;--text-secondary:#c3c2b7;--grid:#2c2c2a;--axis:#383835;--border:rgba(255,255,255,.10);
--series-1:#3987e5;--series-2:#d95926;--series-3:#199e70;--series-4:#c98500}}
:root[data-theme="dark"] .viz-root{color-scheme:dark;--surface-1:#1a1a19;--page:#0d0d0d;--text-primary:#f0efec;
--text-secondary:#c3c2b7;--grid:#2c2c2a;--axis:#383835;--border:rgba(255,255,255,.10);
--series-1:#3987e5;--series-2:#d95926;--series-3:#199e70;--series-4:#c98500}
body{margin:0;background:var(--page)}
.viz-root{font:14px/1.45 system-ui,sans-serif;color:var(--text-primary);background:var(--page);padding:16px;max-width:1100px;margin:auto}
h1{font-size:22px}h2{font-size:18px;margin-top:32px}p,li{color:var(--text-secondary)}
.charts{display:grid;grid-template-columns:repeat(auto-fit,minmax(min(320px,100%),1fr));gap:16px}
figure{margin:0;min-width:0;background:var(--surface-1);border:1px solid var(--border);border-radius:8px;padding:12px}
figcaption{font-weight:600;margin-bottom:6px}svg{width:100%;height:auto;display:block}
.legend{display:flex;flex-wrap:wrap;gap:12px;font-size:12px;color:var(--text-secondary);margin-bottom:4px}
.key{display:inline-flex;align-items:center;gap:4px}
.grid{stroke:var(--grid);stroke-width:1}.axis{stroke:var(--axis);stroke-width:1}
.tick{fill:var(--muted);font-size:11px}.dlabel{fill:var(--text-secondary);font-size:11px}
.pt{stroke:var(--surface-1);stroke-width:2}.hit{fill:transparent;cursor:default}
table{border-collapse:collapse;width:100%;font-size:12px;background:var(--surface-1);font-variant-numeric:tabular-nums}
th,td{border-bottom:1px solid var(--grid);padding:4px 6px;text-align:right}th:first-child,td:first-child{text-align:left}
.wrap{overflow-x:auto}
#tip{position:fixed;pointer-events:none;background:var(--surface-1);color:var(--text-primary);border:1px solid var(--border);
border-radius:6px;padding:4px 8px;font-size:12px;display:none;box-shadow:0 2px 8px rgba(0,0,0,.15)}
'''
JS = '''
const tip=document.getElementById('tip');
document.querySelectorAll('[data-tip]').forEach(el=>{
el.addEventListener('mousemove',e=>{tip.textContent=el.dataset.tip;tip.style.display='block';
tip.style.left=Math.min(e.clientX+12,innerWidth-220)+'px';tip.style.top=(e.clientY+12)+'px'});
el.addEventListener('mouseleave',()=>tip.style.display='none')});
'''
def table(headers, rows):
h = ''.join('<th>%s</th>' % html.escape(x) for x in headers)
b = ''.join('<tr>%s</tr>' % ''.join('<td>%s</td>' % html.escape(str(c)) for c in r) for r in rows)
return '<div class="wrap"><table><thead><tr>%s</tr></thead><tbody>%s</tbody></table></div>' % (h, b)
def main():
p = argparse.ArgumentParser()
p.add_argument('--input', default='evidence/track1/raw.jsonl')
p.add_argument('--experiments', default='experiments/track1.json')
p.add_argument('--outdir', default='evidence/track1')
args = p.parse_args()
src = ROOT / args.input
if not src.exists() or not src.read_text(encoding='utf-8').strip():
raise SystemExit('Нет данных: %s отсутствует или пуст. Сначала успешно выполните scripts/track1.py '
'(его вывод должен содержать строки "c=... ok_rps=...").' % src)
rows = load(src)
exp = json.loads((ROOT / args.experiments).read_text(encoding='utf-8'))
titles = {s['name']: s for s in exp['series']}
agg = aggregate(rows)
order = series_order(rows)
outdir = ROOT / args.outdir
outdir.mkdir(parents=True, exist_ok=True)
headers = ['series', 'config', 'c', 'repeats', 'ok_rps', 'rps_min', 'rps_max', 'write_rps', 'read_rps', 'p50_ms',
'p95_ms', 'p99_ms', 'err', 'util_api', 'util_pg', 'util_redis', 'util_lb', 'host_cores', 'pool_empty', 'pool_wait_ms', 'pg_active']
csv_rows = []
for s, c in order:
for l in levels_of(agg, s, c):
a = agg[(s, c, l)]
csv_rows.append([s, c, l, a['repeats'], fmt(a['ok_rps']), fmt(a['ok_rps_min']), fmt(a['ok_rps_max']),
fmt(a['write_rps']), fmt(a['read_rps']), fmt(a['p50'], 2), fmt(a['p95'], 2), fmt(a['p99'], 2),
fmt(a['error_ratio'], 4), fmt(a['util_api'], 2), fmt(a['util_postgres'], 2),
fmt(a['util_redis'], 2), fmt(a['util_lb'], 2),
'%s/%s' % (fmt(a['host_cores'], 2), a['host_cpus']), fmt(a['empty_acquire_ratio'], 3),
fmt(a['acquire_wait_ms'], 2), fmt(a['pg_active_max'], 0)])
with (outdir / 'summary.csv').open('w', newline='', encoding='utf-8') as f:
csv.writer(f).writerows([headers] + csv_rows)
md = ['# Трек 1 — сводка (сгенерировано scripts/track1_report.py)', '',
'Медианы по повторам; util_* — доля CPU-лимита контейнера роли; pool_empty — доля acquire без свободного соединения.', '']
body = ['<h1>Трек 1 — цена масштабирования</h1>',
'<p>Медианы по повторам. Наведите на точку — значение. Полные данные — таблицы под графиками и summary.csv.</p>']
for name in dict.fromkeys(s for s, _ in order):
meta = titles.get(name, {'title': name, 'hypothesis': ''})
cfgs = configs_of(order, name)
lv = sorted({l for c in cfgs for l in levels_of(agg, name, c)})
xl = [str(l) for l in lv]
get = lambda c, k: [agg[(name, c, l)][k] if (name, c, l) in agg else None for l in lv]
body.append('<h2>%s — %s</h2><p><b>Гипотеза:</b> %s</p>' % (name, html.escape(meta['title']), html.escape(meta['hypothesis'])))
body.append('<div class="charts">%s%s</div>' % (
line_chart('Успешные RPS', xl, [(c, get(c, 'ok_rps')) for c in cfgs[:4]], 'ok RPS'),
line_chart('p95 задержки, мс', xl, [(c, get(c, 'p95')) for c in cfgs[:4]], 'мс')))
sat_rows = []
md += ['## %s — %s' % (name, meta['title']), '', '**Гипотеза:** ' + meta['hypothesis'], '',
'| config | пик RPS | c пика | c насыщения | узкое место (util) | CPU хоста | pool empty / wait ms |', '|---|---|---|---|---|---|---|']
base_peak = None
for c in cfgs:
st = saturation(agg, name, c)
base_peak = base_peak or st['peak_rps']
bn = '%s (%s)%s' % (st['bottleneck'], fmt(st['bottleneck_util'], 2), ' — насыщена' if st['saturated'] else '')
pool = '%s / %s' % (fmt(st['pool_empty_ratio'], 3), fmt(st['pool_wait_ms'], 2))
hs = fmt(100 * st['host_share'], 0) + '%' if st['host_share'] is not None else '—'
md.append('| %s | %s | %s | %s | %s | %s | %s |' % (c, fmt(st['peak_rps']), st['peak_level'], st['knee_level'], bn, hs, pool))
sat_rows.append([c, fmt(st['peak_rps']), '%.2f×' % (st['peak_rps'] / base_peak) if base_peak else '—',
st['peak_level'], st['knee_level'], bn, hs, pool])
body.append(table(['config', 'пик RPS', 'к первой', 'c пика', 'c насыщения', 'узкое место', 'CPU хоста', 'pool empty / wait ms'], sat_rows))
if name.startswith('D') and cfgs:
baseline = cfgs[0]
from_level = saturation(agg, name, baseline)['knee_level']
imp_rows = []
md += ['', 'Сравнение с `%s` (уровни ≥ насыщения baseline):' % baseline, '',
'| кандидат | c | ΔRPS % | Δp95 % | повторы b/c | подтверждено |', '|---|---|---|---|---|---|']
for c in cfgs[1:]:
for r in improvement(rows, agg, name, baseline, c, min(from_level, max(levels_of(agg, name, baseline)))):
row = [c, r['level'], fmt(r['rps_delta_pct']), fmt(r['p95_delta_pct']), '%d/%d' % r['repeats'],
'да' if r['confirmed'] else 'нет']
imp_rows.append(row)
md.append('| %s |' % ' | '.join(str(x) for x in row))
body.append('<h3>Кандидаты против %s</h3>' % html.escape(baseline))
body.append(table(['кандидат', 'c', 'ΔRPS %', 'Δp95 %', 'повторы b/c', 'подтверждено'], imp_rows))
md.append('')
series_rows = [r for r in csv_rows if r[0] == name]
body.append('<details><summary>Все точки серии</summary>%s</details>' % table(headers[1:], [r[1:] for r in series_rows]))
(outdir / 'summary.md').write_text('\n'.join(md) + '\n', encoding='utf-8')
page = ('<!doctype html><html lang="ru"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">'
'<title>Трек 1 отчёт</title><style>%s</style></head><body><div class="viz-root">%s<div id="tip"></div></div>'
'<script>%s</script></body></html>') % (CSS, ''.join(body), JS)
(outdir / 'report.html').write_text(page, encoding='utf-8')
print('written:', outdir / 'summary.csv', outdir / 'summary.md', outdir / 'report.html')
if __name__ == '__main__':
main()