This commit is contained in:
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Builds evidence/track1/summary.md from the raw results of run-all.sh."""
|
||||
import csv
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
from datetime import datetime
|
||||
|
||||
ROOT = "evidence/track1"
|
||||
|
||||
|
||||
def load_k6(path):
|
||||
metrics = json.load(open(path))["metrics"]
|
||||
duration = metrics["http_req_duration{phase:measure}"]["values"]
|
||||
requests = metrics["http_reqs{phase:measure}"]["values"]["count"]
|
||||
failed = metrics["http_req_failed{phase:measure}"]["values"]
|
||||
return {
|
||||
"requests": requests,
|
||||
"failed": failed["passes"],
|
||||
"p50": duration["med"],
|
||||
"p95": duration["p(95)"],
|
||||
"p99": duration["p(99)"],
|
||||
"max": duration["max"],
|
||||
}
|
||||
|
||||
|
||||
def median_range(values, fmt="{:.0f}"):
|
||||
values = [v for v in values if v is not None]
|
||||
if not values:
|
||||
return "—"
|
||||
med = statistics.median(values)
|
||||
if len(values) == 1:
|
||||
return fmt.format(med)
|
||||
return f"{fmt.format(med)} ({fmt.format(min(values))}–{fmt.format(max(values))})"
|
||||
|
||||
|
||||
def stampede_rows():
|
||||
runs = []
|
||||
for run_dir in sorted(glob.glob(f"{ROOT}/stampede/*/*")):
|
||||
run = json.load(open(f"{run_dir}/run.json"))
|
||||
sessions, users, total = map(int, open(f"{run_dir}/sql-calls.csv").read().strip().split(","))
|
||||
redis = dict(line.strip().split(":") for line in open(f"{run_dir}/redis-stats.txt") if ":" in line)
|
||||
k6 = load_k6(f"{run_dir}/k6-summary.json")
|
||||
runs.append({**run, **k6, "sessions": sessions, "users": users, "total_sql": total,
|
||||
"misses": int(redis["keyspace_misses"]), "hits": int(redis["keyspace_hits"])})
|
||||
with open(f"{ROOT}/stampede-runs.csv", "w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(runs[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(runs)
|
||||
return runs
|
||||
|
||||
|
||||
def stampede_table(runs):
|
||||
lines = [
|
||||
"| Сессий | Сбросов кэша | Single-flight | Запросов к `sessions` | на один сброс | на сессию за сброс | Промахов Redis | Всего SQL | RPS | p50 / p95 / p99, мс | max, мс | Ошибки |",
|
||||
"| ---: | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: |",
|
||||
]
|
||||
keys = sorted({(r["tokens"], r["flush_interval_s"], r["variant"]) for r in runs}, key=lambda k: (-k[1], -k[0], k[2]))
|
||||
for tokens, flush, variant in keys:
|
||||
group = [r for r in runs if (r["tokens"], r["flush_interval_s"], r["variant"]) == (tokens, flush, variant)]
|
||||
flushes = group[0]["flushes"]
|
||||
per_flush = [r["sessions"] / r["flushes"] for r in group] if flushes else [None]
|
||||
per_session = [r["sessions"] / r["flushes"] / tokens for r in group] if flushes else [None]
|
||||
lines.append(
|
||||
f"| {tokens} | {flushes if flushes else 'нет'} | {'да' if variant == 'on' else 'нет'} "
|
||||
f"| {median_range([r['sessions'] for r in group])} "
|
||||
f"| {median_range(per_flush, '{:.1f}')} | {median_range(per_session, '{:.2f}')} "
|
||||
f"| {median_range([r['misses'] for r in group])} | {median_range([r['total_sql'] for r in group])} "
|
||||
f"| {median_range([r['requests'] / r['duration_s'] for r in group])} "
|
||||
f"| {statistics.median(r['p50'] for r in group):.1f} / {statistics.median(r['p95'] for r in group):.1f} / "
|
||||
f"{statistics.median(r['p99'] for r in group):.1f} "
|
||||
f"| {median_range([r['max'] for r in group], '{:.0f}')} | {sum(r['failed'] for r in group)} |"
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def parse_log_time(line):
|
||||
stamp = line.split()[0]
|
||||
return datetime.fromisoformat(stamp.replace("Z", "+00:00")).timestamp()
|
||||
|
||||
|
||||
def restart_rows():
|
||||
runs = []
|
||||
for run_dir in sorted(glob.glob(f"{ROOT}/redis-restart/*/*")):
|
||||
run = json.load(open(f"{run_dir}/run.json"))
|
||||
k6 = load_k6(f"{run_dir}/k6-summary.json")
|
||||
seconds = list(csv.DictReader(open(f"{run_dir}/per-second.csv")))
|
||||
samples = list(csv.DictReader(open(f"{run_dir}/samples.csv")))
|
||||
lookups = max(int(s["session_lookups_total"]) for s in samples if s["session_lookups_total"])
|
||||
log = [line for line in open(f"{run_dir}/cache-availability.log")]
|
||||
down = [parse_log_time(l) for l in log if "unavailable" in l and parse_log_time(l) >= run["measure_start"]]
|
||||
up = [parse_log_time(l) for l in log if "available again" in l and parse_log_time(l) >= run["measure_start"]]
|
||||
outage = (up[0] - down[0]) if down and up else None
|
||||
restart_second = int(run["restart_begin"] - run["measure_start"])
|
||||
window = [s for s in seconds if restart_second - 1 <= int(s["second"]) <= restart_second + 1 and s["p99_ms"]]
|
||||
before = [s for s in seconds if int(s["second"]) < restart_second - 1 and s["p99_ms"]]
|
||||
runs.append({
|
||||
**run, **k6,
|
||||
"session_lookups": lookups,
|
||||
"cache_outage_s": outage,
|
||||
"recovered_after_restart_s": (up[0] - run["restart_end"]) if up else None,
|
||||
"restart_s": run["restart_end"] - run["restart_begin"],
|
||||
"peak_p99_ms": max(float(s["p99_ms"]) for s in window),
|
||||
"peak_max_ms": max(float(s["max_ms"]) for s in window),
|
||||
"p99_before_ms": statistics.median(float(s["p99_ms"]) for s in before),
|
||||
"non_200": sum(int(s["non_200"]) for s in seconds),
|
||||
})
|
||||
with open(f"{ROOT}/redis-restart-runs.csv", "w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(runs[0].keys()))
|
||||
writer.writeheader()
|
||||
writer.writerows(runs)
|
||||
return runs
|
||||
|
||||
|
||||
def restart_table(runs):
|
||||
lines = [
|
||||
"| Single-flight | `docker compose restart redis`, с | Кэш недоступен по логу API, с | Запросов к `sessions` за прогон | p99 до рестарта, мс | p99 в секунду рестарта, мс | max в секунду рестарта, мс | Не 200 |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
for variant in ("off", "on"):
|
||||
group = [r for r in runs if r["variant"] == variant]
|
||||
if not group:
|
||||
continue
|
||||
lines.append(
|
||||
f"| {'да' if variant == 'on' else 'нет'} | {median_range([r['restart_s'] for r in group], '{:.2f}')} "
|
||||
f"| {median_range([r['cache_outage_s'] for r in group], '{:.2f}')} "
|
||||
f"| {median_range([r['session_lookups'] for r in group])} "
|
||||
f"| {median_range([r['p99_before_ms'] for r in group], '{:.0f}')} "
|
||||
f"| {median_range([r['peak_p99_ms'] for r in group], '{:.0f}')} "
|
||||
f"| {median_range([r['peak_max_ms'] for r in group], '{:.0f}')} | {sum(r['non_200'] for r in group)} |"
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def read_after_write_table():
|
||||
lines = [
|
||||
"| Single-flight | Redis | Итераций | Первый запрос после входа не 200 | Запросов до ответа logout | Запросов после ответа logout | Из них 200 (нарушение) | Другие коды | Токен жив через 1 с |",
|
||||
"| --- | --- | ---: | ---: | ---: | ---: | ---: | --- | ---: |",
|
||||
]
|
||||
for path in sorted(glob.glob(f"{ROOT}/read-after-write/*.json")):
|
||||
variant, redis = re.match(r".*/(on|off)_redis-(up|down)\.json", path).groups()
|
||||
d = json.load(open(path))
|
||||
lines.append(
|
||||
f"| {'да' if variant == 'on' else 'нет'} | {'работает' if redis == 'up' else 'остановлен'} | {d['iterations']} "
|
||||
f"| {d['first_request_after_login_not_200']} | {d['requests_started_before_logout_response']} "
|
||||
f"| {d['requests_started_after_logout_response']} | {d['violations_200_after_logout']} "
|
||||
f"| {d['unexpected_statuses'] or 'нет'} | {d['token_alive_one_second_later']} |"
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def main():
|
||||
out = ["# Track 1 summary", "", "Generated by `python3 scripts/track1/aggregate.py`. Medians of the repeats, (min–max) in brackets.", ""]
|
||||
if os.path.isdir(f"{ROOT}/stampede"):
|
||||
out += ["## Cache stampede: FLUSHALL every second under 200 VUs", ""] + stampede_table(stampede_rows()) + [""]
|
||||
if os.path.isdir(f"{ROOT}/redis-restart"):
|
||||
out += ["## Redis restart under 200 VUs, 50 sessions", ""] + restart_table(restart_rows()) + [""]
|
||||
if os.path.isdir(f"{ROOT}/read-after-write"):
|
||||
out += ["## Read-after-write of login and logout, 16 concurrent readers", ""] + read_after_write_table() + [""]
|
||||
open(f"{ROOT}/summary.md", "w").write("\n".join(out))
|
||||
print("\n".join(out))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user