#!/usr/bin/env python3 """Builds evidence/track1/summary.md from the raw results of run-all.sh.""" import csv import glob import json import os import re import statistics from datetime import datetime ROOT = "evidence/track1" def load_k6(path): metrics = json.load(open(path))["metrics"] duration = metrics["http_req_duration{phase:measure}"]["values"] requests = metrics["http_reqs{phase:measure}"]["values"]["count"] failed = metrics["http_req_failed{phase:measure}"]["values"] return { "requests": requests, "failed": failed["passes"], "p50": duration["med"], "p95": duration["p(95)"], "p99": duration["p(99)"], "max": duration["max"], } def median_range(values, fmt="{:.0f}"): values = [v for v in values if v is not None] if not values: return "—" med = statistics.median(values) if len(values) == 1: return fmt.format(med) return f"{fmt.format(med)} ({fmt.format(min(values))}–{fmt.format(max(values))})" def stampede_rows(): runs = [] for run_dir in sorted(glob.glob(f"{ROOT}/stampede/*/*")): run = json.load(open(f"{run_dir}/run.json")) sessions, users, total = map(int, open(f"{run_dir}/sql-calls.csv").read().strip().split(",")) redis = dict(line.strip().split(":") for line in open(f"{run_dir}/redis-stats.txt") if ":" in line) k6 = load_k6(f"{run_dir}/k6-summary.json") runs.append({**run, **k6, "sessions": sessions, "users": users, "total_sql": total, "misses": int(redis["keyspace_misses"]), "hits": int(redis["keyspace_hits"])}) with open(f"{ROOT}/stampede-runs.csv", "w", newline="") as f: writer = csv.DictWriter(f, fieldnames=list(runs[0].keys())) writer.writeheader() writer.writerows(runs) return runs def stampede_table(runs): lines = [ "| Сессий | Сбросов кэша | Single-flight | Запросов к `sessions` | на один сброс | на сессию за сброс | Промахов Redis | Всего SQL | RPS | p50 / p95 / p99, мс | max, мс | Ошибки |", "| ---: | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: |", ] keys = sorted({(r["tokens"], r["flush_interval_s"], r["variant"]) for r in runs}, key=lambda k: (-k[1], -k[0], k[2])) for tokens, flush, variant in keys: group = [r for r in runs if (r["tokens"], r["flush_interval_s"], r["variant"]) == (tokens, flush, variant)] flushes = group[0]["flushes"] per_flush = [r["sessions"] / r["flushes"] for r in group] if flushes else [None] per_session = [r["sessions"] / r["flushes"] / tokens for r in group] if flushes else [None] lines.append( f"| {tokens} | {flushes if flushes else 'нет'} | {'да' if variant == 'on' else 'нет'} " f"| {median_range([r['sessions'] for r in group])} " f"| {median_range(per_flush, '{:.1f}')} | {median_range(per_session, '{:.2f}')} " f"| {median_range([r['misses'] for r in group])} | {median_range([r['total_sql'] for r in group])} " f"| {median_range([r['requests'] / r['duration_s'] for r in group])} " f"| {statistics.median(r['p50'] for r in group):.1f} / {statistics.median(r['p95'] for r in group):.1f} / " f"{statistics.median(r['p99'] for r in group):.1f} " f"| {median_range([r['max'] for r in group], '{:.0f}')} | {sum(r['failed'] for r in group)} |" ) return lines def parse_log_time(line): stamp = line.split()[0] return datetime.fromisoformat(stamp.replace("Z", "+00:00")).timestamp() def restart_rows(): runs = [] for run_dir in sorted(glob.glob(f"{ROOT}/redis-restart/*/*")): run = json.load(open(f"{run_dir}/run.json")) k6 = load_k6(f"{run_dir}/k6-summary.json") seconds = list(csv.DictReader(open(f"{run_dir}/per-second.csv"))) samples = list(csv.DictReader(open(f"{run_dir}/samples.csv"))) lookups = max(int(s["session_lookups_total"]) for s in samples if s["session_lookups_total"]) log = [line for line in open(f"{run_dir}/cache-availability.log")] down = [parse_log_time(l) for l in log if "unavailable" in l and parse_log_time(l) >= run["measure_start"]] up = [parse_log_time(l) for l in log if "available again" in l and parse_log_time(l) >= run["measure_start"]] outage = (up[0] - down[0]) if down and up else None restart_second = int(run["restart_begin"] - run["measure_start"]) window = [s for s in seconds if restart_second - 1 <= int(s["second"]) <= restart_second + 1 and s["p99_ms"]] before = [s for s in seconds if int(s["second"]) < restart_second - 1 and s["p99_ms"]] runs.append({ **run, **k6, "session_lookups": lookups, "cache_outage_s": outage, "recovered_after_restart_s": (up[0] - run["restart_end"]) if up else None, "restart_s": run["restart_end"] - run["restart_begin"], "peak_p99_ms": max(float(s["p99_ms"]) for s in window), "peak_max_ms": max(float(s["max_ms"]) for s in window), "p99_before_ms": statistics.median(float(s["p99_ms"]) for s in before), "non_200": sum(int(s["non_200"]) for s in seconds), }) with open(f"{ROOT}/redis-restart-runs.csv", "w", newline="") as f: writer = csv.DictWriter(f, fieldnames=list(runs[0].keys())) writer.writeheader() writer.writerows(runs) return runs def restart_table(runs): lines = [ "| Single-flight | `docker compose restart redis`, с | Кэш недоступен по логу API, с | Запросов к `sessions` за прогон | p99 до рестарта, мс | p99 в секунду рестарта, мс | max в секунду рестарта, мс | Не 200 |", "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", ] for variant in ("off", "on"): group = [r for r in runs if r["variant"] == variant] if not group: continue lines.append( f"| {'да' if variant == 'on' else 'нет'} | {median_range([r['restart_s'] for r in group], '{:.2f}')} " f"| {median_range([r['cache_outage_s'] for r in group], '{:.2f}')} " f"| {median_range([r['session_lookups'] for r in group])} " f"| {median_range([r['p99_before_ms'] for r in group], '{:.0f}')} " f"| {median_range([r['peak_p99_ms'] for r in group], '{:.0f}')} " f"| {median_range([r['peak_max_ms'] for r in group], '{:.0f}')} | {sum(r['non_200'] for r in group)} |" ) return lines def read_after_write_table(): lines = [ "| Single-flight | Redis | Итераций | Первый запрос после входа не 200 | Запросов до ответа logout | Запросов после ответа logout | Из них 200 (нарушение) | Другие коды | Токен жив через 1 с |", "| --- | --- | ---: | ---: | ---: | ---: | ---: | --- | ---: |", ] for path in sorted(glob.glob(f"{ROOT}/read-after-write/*.json")): variant, redis = re.match(r".*/(on|off)_redis-(up|down)\.json", path).groups() d = json.load(open(path)) lines.append( f"| {'да' if variant == 'on' else 'нет'} | {'работает' if redis == 'up' else 'остановлен'} | {d['iterations']} " f"| {d['first_request_after_login_not_200']} | {d['requests_started_before_logout_response']} " f"| {d['requests_started_after_logout_response']} | {d['violations_200_after_logout']} " f"| {d['unexpected_statuses'] or 'нет'} | {d['token_alive_one_second_later']} |" ) return lines def main(): out = ["# Track 1 summary", "", "Generated by `python3 scripts/track1/aggregate.py`. Medians of the repeats, (min–max) in brackets.", ""] if os.path.isdir(f"{ROOT}/stampede"): out += ["## Cache stampede: FLUSHALL every second under 200 VUs", ""] + stampede_table(stampede_rows()) + [""] if os.path.isdir(f"{ROOT}/redis-restart"): out += ["## Redis restart under 200 VUs, 50 sessions", ""] + restart_table(restart_rows()) + [""] if os.path.isdir(f"{ROOT}/read-after-write"): out += ["## Read-after-write of login and logout, 16 concurrent readers", ""] + read_after_write_table() + [""] open(f"{ROOT}/summary.md", "w").write("\n".join(out)) print("\n".join(out)) if __name__ == "__main__": main()