forked from mtusi2026/mtusi2026lab3
244 lines
15 KiB
Python
244 lines
15 KiB
Python
|
||
import json
|
||
import os
|
||
import platform
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
EVIDENCE = ROOT / "evidence"
|
||
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||
PY = sys.executable
|
||
|
||
STEPS = [
|
||
("check", "offline contract check (make check)", [PY, "scripts/check_contract.py"]),
|
||
("stack", "stand layout: replicas, ports, limits, networks, secrets", [PY, "student-tests/check_stack.py"]),
|
||
("smoke", "public smoke via the balancer (make test)", [PY, "tests/smoke.py", "--base-url", BASE_URL]),
|
||
("balancing", "balancing, shared sessions and data, X-Instance-Id", [PY, "student-tests/test_balancing.py"]),
|
||
("idempotency", "Idempotency-Key, parallel same key via replicas", [PY, "student-tests/test_idempotency.py"]),
|
||
("student-tests", "lab1/lab2 API tests via the balancer", [PY, "student-tests/test_api.py"]),
|
||
("race", "lab2 races via the balancer", [PY, "student-tests/test_race.py"]),
|
||
("db", "PostgreSQL: rollback, FK, Argon2id, unique keys, migrations", [PY, "student-tests/test_db.py"]),
|
||
("redis", "Redis as the session store", [PY, "student-tests/redis_info.py"]),
|
||
("explain", "EXPLAIN of the history page (full plans: explain-history.txt)", [PY, "student-tests/explain.py"]),
|
||
("failover", "public failover test for every replica (make test-failover)", [PY, "student-tests/run_failover.py"]),
|
||
("failover-metrics", "stop/kill every replica under load, metrics", [PY, "student-tests/failover_metrics.py"]),
|
||
("load", "public load generator, 30 s, concurrency 4", [PY, "tests/load.py", "--base-url", BASE_URL,
|
||
"--duration", "30", "--concurrency", "4",
|
||
"--output", "evidence/load.json"]),
|
||
("expiry", "session TTL=2 s, then normal TTL", [PY, "student-tests/test_expiry.py"]),
|
||
("failures", "Redis and PostgreSQL outages via the balancer", [PY, "student-tests/test_failures.py"]),
|
||
("persistence", "public persistence test + proof of real restarts (make test-persistence)",
|
||
[PY, "student-tests/run_persistence.py"]),
|
||
("rolling-update", "rolling update under load", [PY, "student-tests/rolling_update.py"]),
|
||
]
|
||
|
||
|
||
def git(*args):
|
||
try:
|
||
return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
|
||
except (OSError, subprocess.CalledProcessError):
|
||
return ""
|
||
|
||
|
||
def commit():
|
||
head = git("rev-parse", "--short", "HEAD") or "unknown"
|
||
changed = [line for line in git("status", "--porcelain").splitlines()
|
||
if not line[3:].startswith(("evidence/", "REPORT.md"))]
|
||
return head + (" + uncommitted changes" if changed else "")
|
||
|
||
|
||
def run_step(name, description, command):
|
||
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", BASE_URL=BASE_URL)
|
||
print(f"=== {name}: {description}", flush=True)
|
||
started = time.monotonic()
|
||
result = subprocess.run(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||
seconds = time.monotonic() - started
|
||
output = result.stdout.decode("utf-8", "replace").replace("\r\n", "\n")
|
||
shown = " ".join(Path(part).name if part == PY else part for part in command)
|
||
header = (f"# {name}: {description}\n# date: {time.strftime('%Y-%m-%d %H:%M:%S')}, commit: {commit()}\n"
|
||
f"# command: {shown}\n# exit code: {result.returncode}, {seconds:.1f} s\n\n")
|
||
(EVIDENCE / f"{name}.txt").write_text(header + output, encoding="utf-8")
|
||
tail = [line for line in output.splitlines() if line.strip()][-3:]
|
||
for line in tail:
|
||
print(" " + line)
|
||
print(f" -> {'PASS' if result.returncode == 0 else 'FAIL'} ({seconds:.0f} s), evidence/{name}.txt", flush=True)
|
||
return result.returncode == 0, seconds
|
||
|
||
|
||
def load_json(name):
|
||
path = EVIDENCE / name
|
||
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None
|
||
|
||
|
||
def measurements():
|
||
"""Markdown-таблицы замеров из evidence/*.json и evidence/*.txt (для REPORT.md и summary.md)."""
|
||
lines = []
|
||
try:
|
||
info = json.loads(subprocess.run(["docker", "info", "--format", "{{json .}}"], capture_output=True,
|
||
text=True, check=True).stdout)
|
||
docker_vm = f"{info.get('NCPU')} CPU, {int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM"
|
||
except (OSError, ValueError, subprocess.CalledProcessError):
|
||
docker_vm = "unknown"
|
||
lines.append(f"Машина: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; "
|
||
f"Docker VM: {docker_vm}; клиент Python {platform.python_version()}; дата прогона "
|
||
f"{time.strftime('%Y-%m-%d')}, commit {commit()}. Лимиты контейнеров: api 1 CPU / 512 MiB "
|
||
f"каждая, postgres 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB.")
|
||
lines.append("")
|
||
|
||
failover = EVIDENCE / "failover.txt"
|
||
results = []
|
||
if failover.exists():
|
||
text = failover.read_text(encoding="utf-8")
|
||
stopped = re.findall(r"the hook stopped (\w+)", text)
|
||
results = [json.loads(m) for m in re.findall(r"^(\{\"observed_instances_before\".*\})$", text, re.M)]
|
||
if results:
|
||
lines.append("Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):")
|
||
lines.append("")
|
||
lines.append("| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток "
|
||
"(повторены с тем же ключом) | Первая запись после hook, с |")
|
||
lines.append("| --- | --- | --- | --- | --- | --- |")
|
||
for number, (result, container) in enumerate(zip(results, stopped), start=1):
|
||
lines.append(f"| {number} | {container} | {result['observed_instances_before']} | "
|
||
f"{result['successful_writes']} | {result['failed_attempts']} | "
|
||
f"{result['first_recovery_seconds']} |")
|
||
lines.append("")
|
||
|
||
metrics = load_json("failover-metrics.json")
|
||
if metrics:
|
||
lines.append("Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи "
|
||
"без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа):")
|
||
lines.append("")
|
||
lines.append("| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок "
|
||
"(коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | "
|
||
"Вернулась в балансировку после docker start, с |")
|
||
lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |")
|
||
for r in metrics["runs"]:
|
||
how = "docker stop (SIGTERM)" if r["mode"] == "stop" else "docker kill (SIGKILL)"
|
||
codes = ", ".join(f"{k}: {v}" for k, v in r["error_codes"].items()) or "-"
|
||
lines.append(f"| {r['replica']} | {how} | {r['command_seconds']} | {r['lb_excluded_after_seconds']} | "
|
||
f"{r['errors']} ({codes}) | {r['errors_stopped_after_seconds']} | {r['before']['write_rps']} | "
|
||
f"{r['after']['write_rps']} | {r['after']['write_p95_ms']} | {r['rejoin_seconds']} |")
|
||
lines.append("")
|
||
lines.append("Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика "
|
||
"после остановки не ответила ни разу.")
|
||
lines.append("")
|
||
|
||
rolling = load_json("rolling-update.json")
|
||
if rolling:
|
||
during = rolling["during"]
|
||
codes = ", ".join(f"{k}: {v}" for k, v in during["error_codes"].items()) or "-"
|
||
lines.append(f"Rolling update (`student-tests/rolling_update.py`): версия {rolling['from_version']} -> "
|
||
f"{rolling['to_version']}, {rolling['replicas']} реплики заменены по одной за "
|
||
f"{rolling['update_seconds']} с под той же нагрузкой. Записей/с до: {rolling['before']['write_rps']}, "
|
||
f"во время замены: {during['write_rps']}; ошибок: {during['errors']} ({codes}); "
|
||
f"p95 {during['write_p95_ms']} мс, максимум {during['write_max_ms']} мс; подтверждено "
|
||
f"{rolling['integrity']['confirmed_writes']} сообщений, потерь и дублей нет.")
|
||
lines.append("")
|
||
|
||
load = load_json("load.json")
|
||
if load:
|
||
latency = load["successful_latency_ms"]
|
||
lines.append(f"Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency {load['concurrency']}, "
|
||
f"{load['requested_duration_seconds']:.0f} с, 2 реплики): {load['successful_rps']} успешных "
|
||
f"POST/с, ошибок {load['failed_requests']} (доля {load['error_ratio']}), задержка успешных "
|
||
f"p50 {latency['p50']} мс, p95 {latency['p95']} мс, p99 {latency['p99']} мс.")
|
||
lines.append("")
|
||
|
||
persistence = EVIDENCE / "persistence.txt"
|
||
restart_seconds = None
|
||
if persistence.exists():
|
||
found = re.search(r"took ([\d.]+) s", persistence.read_text(encoding="utf-8"))
|
||
restart_seconds = found.group(1) if found else None
|
||
|
||
impacts = []
|
||
if metrics:
|
||
for mode, what, expected in (
|
||
("stop", "`docker stop` реплики (SIGTERM) под нагрузкой",
|
||
"ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём"),
|
||
("kill", "`docker kill` реплики (SIGKILL, авария) под нагрузкой",
|
||
"единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом")):
|
||
runs = [r for r in metrics["runs"] if r["mode"] == mode]
|
||
if runs:
|
||
impacts.append(
|
||
f"| {what} | {expected} | ошибок: {', '.join(str(r['errors']) for r in runs)} (по репликам); "
|
||
f"HAProxy вывел реплику через {', '.join(str(r['lb_excluded_after_seconds']) for r in runs)} с | "
|
||
f"ошибки прекратились через {max(r['errors_stopped_after_seconds'] for r in runs)} с; "
|
||
f"потерь 0, дублей 0; реплика вернулась через "
|
||
f"{max(r['rejoin_seconds'] for r in runs)} с после `docker start` |")
|
||
if failover.exists() and results:
|
||
impacts.append(
|
||
f"| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, "
|
||
f"подтверждённые сообщения на месте, дублей нет | PASS {len(results)} раз(а), неудачных попыток: "
|
||
f"{', '.join(str(r['failed_attempts']) for r in results)} | первая запись после hook через "
|
||
f"{', '.join(str(r['first_recovery_seconds']) for r in results)} с (hook ждёт остановки ~4 с); потерь нет |")
|
||
if rolling:
|
||
impacts.append(
|
||
f"| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше {rolling['replicas']} | "
|
||
f"ошибок: {rolling['during']['errors']}, замена заняла {rolling['update_seconds']} с | потерь 0, дублей 0 |")
|
||
if restart_seconds:
|
||
impacts.append(
|
||
f"| перезапуск всего стенда (`restart` hook: postgres, redis, api, lb) | данные и прежний токен живы | "
|
||
f"PASS, те же контейнеры с новым временем старта | {restart_seconds} с до готовности; потерь нет |")
|
||
if impacts:
|
||
lines.append("| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |")
|
||
lines.append("| --- | --- | --- | --- |")
|
||
lines.extend(impacts)
|
||
lines.append("")
|
||
return "\n".join(lines).strip()
|
||
|
||
|
||
def update_report(block):
|
||
report = ROOT / "REPORT.md"
|
||
text = report.read_text(encoding="utf-8")
|
||
begin, end = "<!-- measurements:begin -->", "<!-- measurements:end -->"
|
||
if begin not in text or end not in text:
|
||
print("REPORT.md has no measurement markers, only evidence/summary.md was written")
|
||
return
|
||
head, rest = text.split(begin, 1)
|
||
_, tail = rest.split(end, 1)
|
||
report.write_text(f"{head}{begin}\n{block}\n{end}{tail}", encoding="utf-8")
|
||
print("REPORT.md: measurement tables updated")
|
||
|
||
|
||
def main():
|
||
args = sys.argv[1:]
|
||
names = [name for name, _, _ in STEPS]
|
||
if args[:1] == ["--from"]:
|
||
if len(args) < 2 or args[1] not in names:
|
||
raise SystemExit("usage: --from <step>; steps: " + ", ".join(names))
|
||
selected = names[names.index(args[1]):]
|
||
else:
|
||
unknown = [a for a in args if a not in names]
|
||
if unknown:
|
||
raise SystemExit(f"unknown steps {unknown}; steps: " + ", ".join(names))
|
||
selected = args or names
|
||
EVIDENCE.mkdir(exist_ok=True)
|
||
results = []
|
||
for name, description, command in STEPS:
|
||
if name not in selected:
|
||
continue
|
||
ok, seconds = run_step(name, description, command)
|
||
results.append((name, description, ok, seconds))
|
||
if not ok:
|
||
print(f"\nSTOP: step '{name}' failed, see evidence/{name}.txt.")
|
||
print(f"After fixing: python student-tests/collect_evidence.py --from {name}")
|
||
break
|
||
block = measurements()
|
||
summary = ["# Сводка проверок", "", "| Шаг | Что проверяет | Результат | Время, с | Файл |",
|
||
"| --- | --- | --- | --- | --- |"]
|
||
summary += [f"| {n} | {d} | {'PASS' if ok else 'FAIL'} | {s:.0f} | evidence/{n}.txt |" for n, d, ok, s in results]
|
||
(EVIDENCE / "summary.md").write_text("\n".join(summary) + "\n\n# Замеры\n\n" + block + "\n", encoding="utf-8")
|
||
print("evidence/summary.md written")
|
||
passed = all(ok for _, _, ok, _ in results)
|
||
if passed:
|
||
update_report(block) # таблицы строятся из всех файлов evidence/, в том числе от прошлых запусков
|
||
raise SystemExit(0 if passed else 1)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|