Files
mtusi2026lab2/scripts/track1/aggregate.py
T
Mraksimus dc5abb6fff
Skeleton contracts / contracts (pull_request) Canceled after 0s
Completed for a grade of five
2026-10-09 11:50:01 +03:00

168 lines
8.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Builds evidence/track1/summary.md from the raw results of run-all.sh."""
import csv
import glob
import json
import os
import re
import statistics
from datetime import datetime
ROOT = "evidence/track1"
def load_k6(path):
metrics = json.load(open(path))["metrics"]
duration = metrics["http_req_duration{phase:measure}"]["values"]
requests = metrics["http_reqs{phase:measure}"]["values"]["count"]
failed = metrics["http_req_failed{phase:measure}"]["values"]
return {
"requests": requests,
"failed": failed["passes"],
"p50": duration["med"],
"p95": duration["p(95)"],
"p99": duration["p(99)"],
"max": duration["max"],
}
def median_range(values, fmt="{:.0f}"):
values = [v for v in values if v is not None]
if not values:
return "—"
med = statistics.median(values)
if len(values) == 1:
return fmt.format(med)
return f"{fmt.format(med)} ({fmt.format(min(values))}–{fmt.format(max(values))})"
def stampede_rows():
runs = []
for run_dir in sorted(glob.glob(f"{ROOT}/stampede/*/*")):
run = json.load(open(f"{run_dir}/run.json"))
sessions, users, total = map(int, open(f"{run_dir}/sql-calls.csv").read().strip().split(","))
redis = dict(line.strip().split(":") for line in open(f"{run_dir}/redis-stats.txt") if ":" in line)
k6 = load_k6(f"{run_dir}/k6-summary.json")
runs.append({**run, **k6, "sessions": sessions, "users": users, "total_sql": total,
"misses": int(redis["keyspace_misses"]), "hits": int(redis["keyspace_hits"])})
with open(f"{ROOT}/stampede-runs.csv", "w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=list(runs[0].keys()))
writer.writeheader()
writer.writerows(runs)
return runs
def stampede_table(runs):
lines = [
"| Сессий | Сбросов кэша | Single-flight | Запросов к `sessions` | на один сброс | на сессию за сброс | Промахов Redis | Всего SQL | RPS | p50 / p95 / p99, мс | max, мс | Ошибки |",
"| ---: | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: |",
]
keys = sorted({(r["tokens"], r["flush_interval_s"], r["variant"]) for r in runs}, key=lambda k: (-k[1], -k[0], k[2]))
for tokens, flush, variant in keys:
group = [r for r in runs if (r["tokens"], r["flush_interval_s"], r["variant"]) == (tokens, flush, variant)]
flushes = group[0]["flushes"]
per_flush = [r["sessions"] / r["flushes"] for r in group] if flushes else [None]
per_session = [r["sessions"] / r["flushes"] / tokens for r in group] if flushes else [None]
lines.append(
f"| {tokens} | {flushes if flushes else 'нет'} | {'да' if variant == 'on' else 'нет'} "
f"| {median_range([r['sessions'] for r in group])} "
f"| {median_range(per_flush, '{:.1f}')} | {median_range(per_session, '{:.2f}')} "
f"| {median_range([r['misses'] for r in group])} | {median_range([r['total_sql'] for r in group])} "
f"| {median_range([r['requests'] / r['duration_s'] for r in group])} "
f"| {statistics.median(r['p50'] for r in group):.1f} / {statistics.median(r['p95'] for r in group):.1f} / "
f"{statistics.median(r['p99'] for r in group):.1f} "
f"| {median_range([r['max'] for r in group], '{:.0f}')} | {sum(r['failed'] for r in group)} |"
)
return lines
def parse_log_time(line):
stamp = line.split()[0]
return datetime.fromisoformat(stamp.replace("Z", "+00:00")).timestamp()
def restart_rows():
runs = []
for run_dir in sorted(glob.glob(f"{ROOT}/redis-restart/*/*")):
run = json.load(open(f"{run_dir}/run.json"))
k6 = load_k6(f"{run_dir}/k6-summary.json")
seconds = list(csv.DictReader(open(f"{run_dir}/per-second.csv")))
samples = list(csv.DictReader(open(f"{run_dir}/samples.csv")))
lookups = max(int(s["session_lookups_total"]) for s in samples if s["session_lookups_total"])
log = [line for line in open(f"{run_dir}/cache-availability.log")]
down = [parse_log_time(l) for l in log if "unavailable" in l and parse_log_time(l) >= run["measure_start"]]
up = [parse_log_time(l) for l in log if "available again" in l and parse_log_time(l) >= run["measure_start"]]
outage = (up[0] - down[0]) if down and up else None
restart_second = int(run["restart_begin"] - run["measure_start"])
window = [s for s in seconds if restart_second - 1 <= int(s["second"]) <= restart_second + 1 and s["p99_ms"]]
before = [s for s in seconds if int(s["second"]) < restart_second - 1 and s["p99_ms"]]
runs.append({
**run, **k6,
"session_lookups": lookups,
"cache_outage_s": outage,
"recovered_after_restart_s": (up[0] - run["restart_end"]) if up else None,
"restart_s": run["restart_end"] - run["restart_begin"],
"peak_p99_ms": max(float(s["p99_ms"]) for s in window),
"peak_max_ms": max(float(s["max_ms"]) for s in window),
"p99_before_ms": statistics.median(float(s["p99_ms"]) for s in before),
"non_200": sum(int(s["non_200"]) for s in seconds),
})
with open(f"{ROOT}/redis-restart-runs.csv", "w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=list(runs[0].keys()))
writer.writeheader()
writer.writerows(runs)
return runs
def restart_table(runs):
lines = [
"| Single-flight | `docker compose restart redis`, с | Кэш недоступен по логу API, с | Запросов к `sessions` за прогон | p99 до рестарта, мс | p99 в секунду рестарта, мс | max в секунду рестарта, мс | Не 200 |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
]
for variant in ("off", "on"):
group = [r for r in runs if r["variant"] == variant]
if not group:
continue
lines.append(
f"| {'да' if variant == 'on' else 'нет'} | {median_range([r['restart_s'] for r in group], '{:.2f}')} "
f"| {median_range([r['cache_outage_s'] for r in group], '{:.2f}')} "
f"| {median_range([r['session_lookups'] for r in group])} "
f"| {median_range([r['p99_before_ms'] for r in group], '{:.0f}')} "
f"| {median_range([r['peak_p99_ms'] for r in group], '{:.0f}')} "
f"| {median_range([r['peak_max_ms'] for r in group], '{:.0f}')} | {sum(r['non_200'] for r in group)} |"
)
return lines
def read_after_write_table():
lines = [
"| Single-flight | Redis | Итераций | Первый запрос после входа не 200 | Запросов до ответа logout | Запросов после ответа logout | Из них 200 (нарушение) | Другие коды | Токен жив через 1 с |",
"| --- | --- | ---: | ---: | ---: | ---: | ---: | --- | ---: |",
]
for path in sorted(glob.glob(f"{ROOT}/read-after-write/*.json")):
variant, redis = re.match(r".*/(on|off)_redis-(up|down)\.json", path).groups()
d = json.load(open(path))
lines.append(
f"| {'да' if variant == 'on' else 'нет'} | {'работает' if redis == 'up' else 'остановлен'} | {d['iterations']} "
f"| {d['first_request_after_login_not_200']} | {d['requests_started_before_logout_response']} "
f"| {d['requests_started_after_logout_response']} | {d['violations_200_after_logout']} "
f"| {d['unexpected_statuses'] or 'нет'} | {d['token_alive_one_second_later']} |"
)
return lines
def main():
out = ["# Track 1 summary", "", "Generated by `python3 scripts/track1/aggregate.py`. Medians of the repeats, (min–max) in brackets.", ""]
if os.path.isdir(f"{ROOT}/stampede"):
out += ["## Cache stampede: FLUSHALL every second under 200 VUs", ""] + stampede_table(stampede_rows()) + [""]
if os.path.isdir(f"{ROOT}/redis-restart"):
out += ["## Redis restart under 200 VUs, 50 sessions", ""] + restart_table(restart_rows()) + [""]
if os.path.isdir(f"{ROOT}/read-after-write"):
out += ["## Read-after-write of login and logout, 16 concurrent readers", ""] + read_after_write_table() + [""]
open(f"{ROOT}/summary.md", "w").write("\n".join(out))
print("\n".join(out))
if __name__ == "__main__":
main()