168 lines
8.4 KiB
Python
168 lines
8.4 KiB
Python
#!/usr/bin/env python3
|
||
"""Builds evidence/track1/summary.md from the raw results of run-all.sh."""
|
||
import csv
|
||
import glob
|
||
import json
|
||
import os
|
||
import re
|
||
import statistics
|
||
from datetime import datetime
|
||
|
||
ROOT = "evidence/track1"
|
||
|
||
|
||
def load_k6(path):
|
||
metrics = json.load(open(path))["metrics"]
|
||
duration = metrics["http_req_duration{phase:measure}"]["values"]
|
||
requests = metrics["http_reqs{phase:measure}"]["values"]["count"]
|
||
failed = metrics["http_req_failed{phase:measure}"]["values"]
|
||
return {
|
||
"requests": requests,
|
||
"failed": failed["passes"],
|
||
"p50": duration["med"],
|
||
"p95": duration["p(95)"],
|
||
"p99": duration["p(99)"],
|
||
"max": duration["max"],
|
||
}
|
||
|
||
|
||
def median_range(values, fmt="{:.0f}"):
|
||
values = [v for v in values if v is not None]
|
||
if not values:
|
||
return "—"
|
||
med = statistics.median(values)
|
||
if len(values) == 1:
|
||
return fmt.format(med)
|
||
return f"{fmt.format(med)} ({fmt.format(min(values))}–{fmt.format(max(values))})"
|
||
|
||
|
||
def stampede_rows():
|
||
runs = []
|
||
for run_dir in sorted(glob.glob(f"{ROOT}/stampede/*/*")):
|
||
run = json.load(open(f"{run_dir}/run.json"))
|
||
sessions, users, total = map(int, open(f"{run_dir}/sql-calls.csv").read().strip().split(","))
|
||
redis = dict(line.strip().split(":") for line in open(f"{run_dir}/redis-stats.txt") if ":" in line)
|
||
k6 = load_k6(f"{run_dir}/k6-summary.json")
|
||
runs.append({**run, **k6, "sessions": sessions, "users": users, "total_sql": total,
|
||
"misses": int(redis["keyspace_misses"]), "hits": int(redis["keyspace_hits"])})
|
||
with open(f"{ROOT}/stampede-runs.csv", "w", newline="") as f:
|
||
writer = csv.DictWriter(f, fieldnames=list(runs[0].keys()))
|
||
writer.writeheader()
|
||
writer.writerows(runs)
|
||
return runs
|
||
|
||
|
||
def stampede_table(runs):
|
||
lines = [
|
||
"| Сессий | Сбросов кэша | Single-flight | Запросов к `sessions` | на один сброс | на сессию за сброс | Промахов Redis | Всего SQL | RPS | p50 / p95 / p99, мс | max, мс | Ошибки |",
|
||
"| ---: | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: |",
|
||
]
|
||
keys = sorted({(r["tokens"], r["flush_interval_s"], r["variant"]) for r in runs}, key=lambda k: (-k[1], -k[0], k[2]))
|
||
for tokens, flush, variant in keys:
|
||
group = [r for r in runs if (r["tokens"], r["flush_interval_s"], r["variant"]) == (tokens, flush, variant)]
|
||
flushes = group[0]["flushes"]
|
||
per_flush = [r["sessions"] / r["flushes"] for r in group] if flushes else [None]
|
||
per_session = [r["sessions"] / r["flushes"] / tokens for r in group] if flushes else [None]
|
||
lines.append(
|
||
f"| {tokens} | {flushes if flushes else 'нет'} | {'да' if variant == 'on' else 'нет'} "
|
||
f"| {median_range([r['sessions'] for r in group])} "
|
||
f"| {median_range(per_flush, '{:.1f}')} | {median_range(per_session, '{:.2f}')} "
|
||
f"| {median_range([r['misses'] for r in group])} | {median_range([r['total_sql'] for r in group])} "
|
||
f"| {median_range([r['requests'] / r['duration_s'] for r in group])} "
|
||
f"| {statistics.median(r['p50'] for r in group):.1f} / {statistics.median(r['p95'] for r in group):.1f} / "
|
||
f"{statistics.median(r['p99'] for r in group):.1f} "
|
||
f"| {median_range([r['max'] for r in group], '{:.0f}')} | {sum(r['failed'] for r in group)} |"
|
||
)
|
||
return lines
|
||
|
||
|
||
def parse_log_time(line):
|
||
stamp = line.split()[0]
|
||
return datetime.fromisoformat(stamp.replace("Z", "+00:00")).timestamp()
|
||
|
||
|
||
def restart_rows():
|
||
runs = []
|
||
for run_dir in sorted(glob.glob(f"{ROOT}/redis-restart/*/*")):
|
||
run = json.load(open(f"{run_dir}/run.json"))
|
||
k6 = load_k6(f"{run_dir}/k6-summary.json")
|
||
seconds = list(csv.DictReader(open(f"{run_dir}/per-second.csv")))
|
||
samples = list(csv.DictReader(open(f"{run_dir}/samples.csv")))
|
||
lookups = max(int(s["session_lookups_total"]) for s in samples if s["session_lookups_total"])
|
||
log = [line for line in open(f"{run_dir}/cache-availability.log")]
|
||
down = [parse_log_time(l) for l in log if "unavailable" in l and parse_log_time(l) >= run["measure_start"]]
|
||
up = [parse_log_time(l) for l in log if "available again" in l and parse_log_time(l) >= run["measure_start"]]
|
||
outage = (up[0] - down[0]) if down and up else None
|
||
restart_second = int(run["restart_begin"] - run["measure_start"])
|
||
window = [s for s in seconds if restart_second - 1 <= int(s["second"]) <= restart_second + 1 and s["p99_ms"]]
|
||
before = [s for s in seconds if int(s["second"]) < restart_second - 1 and s["p99_ms"]]
|
||
runs.append({
|
||
**run, **k6,
|
||
"session_lookups": lookups,
|
||
"cache_outage_s": outage,
|
||
"recovered_after_restart_s": (up[0] - run["restart_end"]) if up else None,
|
||
"restart_s": run["restart_end"] - run["restart_begin"],
|
||
"peak_p99_ms": max(float(s["p99_ms"]) for s in window),
|
||
"peak_max_ms": max(float(s["max_ms"]) for s in window),
|
||
"p99_before_ms": statistics.median(float(s["p99_ms"]) for s in before),
|
||
"non_200": sum(int(s["non_200"]) for s in seconds),
|
||
})
|
||
with open(f"{ROOT}/redis-restart-runs.csv", "w", newline="") as f:
|
||
writer = csv.DictWriter(f, fieldnames=list(runs[0].keys()))
|
||
writer.writeheader()
|
||
writer.writerows(runs)
|
||
return runs
|
||
|
||
|
||
def restart_table(runs):
|
||
lines = [
|
||
"| Single-flight | `docker compose restart redis`, с | Кэш недоступен по логу API, с | Запросов к `sessions` за прогон | p99 до рестарта, мс | p99 в секунду рестарта, мс | max в секунду рестарта, мс | Не 200 |",
|
||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||
]
|
||
for variant in ("off", "on"):
|
||
group = [r for r in runs if r["variant"] == variant]
|
||
if not group:
|
||
continue
|
||
lines.append(
|
||
f"| {'да' if variant == 'on' else 'нет'} | {median_range([r['restart_s'] for r in group], '{:.2f}')} "
|
||
f"| {median_range([r['cache_outage_s'] for r in group], '{:.2f}')} "
|
||
f"| {median_range([r['session_lookups'] for r in group])} "
|
||
f"| {median_range([r['p99_before_ms'] for r in group], '{:.0f}')} "
|
||
f"| {median_range([r['peak_p99_ms'] for r in group], '{:.0f}')} "
|
||
f"| {median_range([r['peak_max_ms'] for r in group], '{:.0f}')} | {sum(r['non_200'] for r in group)} |"
|
||
)
|
||
return lines
|
||
|
||
|
||
def read_after_write_table():
|
||
lines = [
|
||
"| Single-flight | Redis | Итераций | Первый запрос после входа не 200 | Запросов до ответа logout | Запросов после ответа logout | Из них 200 (нарушение) | Другие коды | Токен жив через 1 с |",
|
||
"| --- | --- | ---: | ---: | ---: | ---: | ---: | --- | ---: |",
|
||
]
|
||
for path in sorted(glob.glob(f"{ROOT}/read-after-write/*.json")):
|
||
variant, redis = re.match(r".*/(on|off)_redis-(up|down)\.json", path).groups()
|
||
d = json.load(open(path))
|
||
lines.append(
|
||
f"| {'да' if variant == 'on' else 'нет'} | {'работает' if redis == 'up' else 'остановлен'} | {d['iterations']} "
|
||
f"| {d['first_request_after_login_not_200']} | {d['requests_started_before_logout_response']} "
|
||
f"| {d['requests_started_after_logout_response']} | {d['violations_200_after_logout']} "
|
||
f"| {d['unexpected_statuses'] or 'нет'} | {d['token_alive_one_second_later']} |"
|
||
)
|
||
return lines
|
||
|
||
|
||
def main():
|
||
out = ["# Track 1 summary", "", "Generated by `python3 scripts/track1/aggregate.py`. Medians of the repeats, (min–max) in brackets.", ""]
|
||
if os.path.isdir(f"{ROOT}/stampede"):
|
||
out += ["## Cache stampede: FLUSHALL every second under 200 VUs", ""] + stampede_table(stampede_rows()) + [""]
|
||
if os.path.isdir(f"{ROOT}/redis-restart"):
|
||
out += ["## Redis restart under 200 VUs, 50 sessions", ""] + restart_table(restart_rows()) + [""]
|
||
if os.path.isdir(f"{ROOT}/read-after-write"):
|
||
out += ["## Read-after-write of login and logout, 16 concurrent readers", ""] + read_after_write_table() + [""]
|
||
open(f"{ROOT}/summary.md", "w").write("\n".join(out))
|
||
print("\n".join(out))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|