Files

244 lines
15 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import json
import os
import platform
import re
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
PY = sys.executable
STEPS = [
("check", "offline contract check (make check)", [PY, "scripts/check_contract.py"]),
("stack", "stand layout: replicas, ports, limits, networks, secrets", [PY, "student-tests/check_stack.py"]),
("smoke", "public smoke via the balancer (make test)", [PY, "tests/smoke.py", "--base-url", BASE_URL]),
("balancing", "balancing, shared sessions and data, X-Instance-Id", [PY, "student-tests/test_balancing.py"]),
("idempotency", "Idempotency-Key, parallel same key via replicas", [PY, "student-tests/test_idempotency.py"]),
("student-tests", "lab1/lab2 API tests via the balancer", [PY, "student-tests/test_api.py"]),
("race", "lab2 races via the balancer", [PY, "student-tests/test_race.py"]),
("db", "PostgreSQL: rollback, FK, Argon2id, unique keys, migrations", [PY, "student-tests/test_db.py"]),
("redis", "Redis as the session store", [PY, "student-tests/redis_info.py"]),
("explain", "EXPLAIN of the history page (full plans: explain-history.txt)", [PY, "student-tests/explain.py"]),
("failover", "public failover test for every replica (make test-failover)", [PY, "student-tests/run_failover.py"]),
("failover-metrics", "stop/kill every replica under load, metrics", [PY, "student-tests/failover_metrics.py"]),
("load", "public load generator, 30 s, concurrency 4", [PY, "tests/load.py", "--base-url", BASE_URL,
"--duration", "30", "--concurrency", "4",
"--output", "evidence/load.json"]),
("expiry", "session TTL=2 s, then normal TTL", [PY, "student-tests/test_expiry.py"]),
("failures", "Redis and PostgreSQL outages via the balancer", [PY, "student-tests/test_failures.py"]),
("persistence", "public persistence test + proof of real restarts (make test-persistence)",
[PY, "student-tests/run_persistence.py"]),
("rolling-update", "rolling update under load", [PY, "student-tests/rolling_update.py"]),
]
def git(*args):
try:
return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
except (OSError, subprocess.CalledProcessError):
return ""
def commit():
head = git("rev-parse", "--short", "HEAD") or "unknown"
changed = [line for line in git("status", "--porcelain").splitlines()
if not line[3:].startswith(("evidence/", "REPORT.md"))]
return head + (" + uncommitted changes" if changed else "")
def run_step(name, description, command):
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", BASE_URL=BASE_URL)
print(f"=== {name}: {description}", flush=True)
started = time.monotonic()
result = subprocess.run(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
seconds = time.monotonic() - started
output = result.stdout.decode("utf-8", "replace").replace("\r\n", "\n")
shown = " ".join(Path(part).name if part == PY else part for part in command)
header = (f"# {name}: {description}\n# date: {time.strftime('%Y-%m-%d %H:%M:%S')}, commit: {commit()}\n"
f"# command: {shown}\n# exit code: {result.returncode}, {seconds:.1f} s\n\n")
(EVIDENCE / f"{name}.txt").write_text(header + output, encoding="utf-8")
tail = [line for line in output.splitlines() if line.strip()][-3:]
for line in tail:
print(" " + line)
print(f" -> {'PASS' if result.returncode == 0 else 'FAIL'} ({seconds:.0f} s), evidence/{name}.txt", flush=True)
return result.returncode == 0, seconds
def load_json(name):
path = EVIDENCE / name
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None
def measurements():
"""Markdown-таблицы замеров из evidence/*.json и evidence/*.txt (для REPORT.md и summary.md)."""
lines = []
try:
info = json.loads(subprocess.run(["docker", "info", "--format", "{{json .}}"], capture_output=True,
text=True, check=True).stdout)
docker_vm = f"{info.get('NCPU')} CPU, {int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM"
except (OSError, ValueError, subprocess.CalledProcessError):
docker_vm = "unknown"
lines.append(f"Машина: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; "
f"Docker VM: {docker_vm}; клиент Python {platform.python_version()}; дата прогона "
f"{time.strftime('%Y-%m-%d')}, commit {commit()}. Лимиты контейнеров: api 1 CPU / 512 MiB "
f"каждая, postgres 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB.")
lines.append("")
failover = EVIDENCE / "failover.txt"
results = []
if failover.exists():
text = failover.read_text(encoding="utf-8")
stopped = re.findall(r"the hook stopped (\w+)", text)
results = [json.loads(m) for m in re.findall(r"^(\{\"observed_instances_before\".*\})$", text, re.M)]
if results:
lines.append("Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):")
lines.append("")
lines.append("| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток "
"(повторены с тем же ключом) | Первая запись после hook, с |")
lines.append("| --- | --- | --- | --- | --- | --- |")
for number, (result, container) in enumerate(zip(results, stopped), start=1):
lines.append(f"| {number} | {container} | {result['observed_instances_before']} | "
f"{result['successful_writes']} | {result['failed_attempts']} | "
f"{result['first_recovery_seconds']} |")
lines.append("")
metrics = load_json("failover-metrics.json")
if metrics:
lines.append("Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи "
"без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа):")
lines.append("")
lines.append("| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок "
"(коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | "
"Вернулась в балансировку после docker start, с |")
lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |")
for r in metrics["runs"]:
how = "docker stop (SIGTERM)" if r["mode"] == "stop" else "docker kill (SIGKILL)"
codes = ", ".join(f"{k}: {v}" for k, v in r["error_codes"].items()) or "-"
lines.append(f"| {r['replica']} | {how} | {r['command_seconds']} | {r['lb_excluded_after_seconds']} | "
f"{r['errors']} ({codes}) | {r['errors_stopped_after_seconds']} | {r['before']['write_rps']} | "
f"{r['after']['write_rps']} | {r['after']['write_p95_ms']} | {r['rejoin_seconds']} |")
lines.append("")
lines.append("Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика "
"после остановки не ответила ни разу.")
lines.append("")
rolling = load_json("rolling-update.json")
if rolling:
during = rolling["during"]
codes = ", ".join(f"{k}: {v}" for k, v in during["error_codes"].items()) or "-"
lines.append(f"Rolling update (`student-tests/rolling_update.py`): версия {rolling['from_version']} -> "
f"{rolling['to_version']}, {rolling['replicas']} реплики заменены по одной за "
f"{rolling['update_seconds']} с под той же нагрузкой. Записей/с до: {rolling['before']['write_rps']}, "
f"во время замены: {during['write_rps']}; ошибок: {during['errors']} ({codes}); "
f"p95 {during['write_p95_ms']} мс, максимум {during['write_max_ms']} мс; подтверждено "
f"{rolling['integrity']['confirmed_writes']} сообщений, потерь и дублей нет.")
lines.append("")
load = load_json("load.json")
if load:
latency = load["successful_latency_ms"]
lines.append(f"Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency {load['concurrency']}, "
f"{load['requested_duration_seconds']:.0f} с, 2 реплики): {load['successful_rps']} успешных "
f"POST/с, ошибок {load['failed_requests']} (доля {load['error_ratio']}), задержка успешных "
f"p50 {latency['p50']} мс, p95 {latency['p95']} мс, p99 {latency['p99']} мс.")
lines.append("")
persistence = EVIDENCE / "persistence.txt"
restart_seconds = None
if persistence.exists():
found = re.search(r"took ([\d.]+) s", persistence.read_text(encoding="utf-8"))
restart_seconds = found.group(1) if found else None
impacts = []
if metrics:
for mode, what, expected in (
("stop", "`docker stop` реплики (SIGTERM) под нагрузкой",
"ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём"),
("kill", "`docker kill` реплики (SIGKILL, авария) под нагрузкой",
"единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом")):
runs = [r for r in metrics["runs"] if r["mode"] == mode]
if runs:
impacts.append(
f"| {what} | {expected} | ошибок: {', '.join(str(r['errors']) for r in runs)} (по репликам); "
f"HAProxy вывел реплику через {', '.join(str(r['lb_excluded_after_seconds']) for r in runs)} с | "
f"ошибки прекратились через {max(r['errors_stopped_after_seconds'] for r in runs)} с; "
f"потерь 0, дублей 0; реплика вернулась через "
f"{max(r['rejoin_seconds'] for r in runs)} с после `docker start` |")
if failover.exists() and results:
impacts.append(
f"| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, "
f"подтверждённые сообщения на месте, дублей нет | PASS {len(results)} раз(а), неудачных попыток: "
f"{', '.join(str(r['failed_attempts']) for r in results)} | первая запись после hook через "
f"{', '.join(str(r['first_recovery_seconds']) for r in results)} с (hook ждёт остановки ~4 с); потерь нет |")
if rolling:
impacts.append(
f"| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше {rolling['replicas']} | "
f"ошибок: {rolling['during']['errors']}, замена заняла {rolling['update_seconds']} с | потерь 0, дублей 0 |")
if restart_seconds:
impacts.append(
f"| перезапуск всего стенда (`restart` hook: postgres, redis, api, lb) | данные и прежний токен живы | "
f"PASS, те же контейнеры с новым временем старта | {restart_seconds} с до готовности; потерь нет |")
if impacts:
lines.append("| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |")
lines.append("| --- | --- | --- | --- |")
lines.extend(impacts)
lines.append("")
return "\n".join(lines).strip()
def update_report(block):
report = ROOT / "REPORT.md"
text = report.read_text(encoding="utf-8")
begin, end = "<!-- measurements:begin -->", "<!-- measurements:end -->"
if begin not in text or end not in text:
print("REPORT.md has no measurement markers, only evidence/summary.md was written")
return
head, rest = text.split(begin, 1)
_, tail = rest.split(end, 1)
report.write_text(f"{head}{begin}\n{block}\n{end}{tail}", encoding="utf-8")
print("REPORT.md: measurement tables updated")
def main():
args = sys.argv[1:]
names = [name for name, _, _ in STEPS]
if args[:1] == ["--from"]:
if len(args) < 2 or args[1] not in names:
raise SystemExit("usage: --from <step>; steps: " + ", ".join(names))
selected = names[names.index(args[1]):]
else:
unknown = [a for a in args if a not in names]
if unknown:
raise SystemExit(f"unknown steps {unknown}; steps: " + ", ".join(names))
selected = args or names
EVIDENCE.mkdir(exist_ok=True)
results = []
for name, description, command in STEPS:
if name not in selected:
continue
ok, seconds = run_step(name, description, command)
results.append((name, description, ok, seconds))
if not ok:
print(f"\nSTOP: step '{name}' failed, see evidence/{name}.txt.")
print(f"After fixing: python student-tests/collect_evidence.py --from {name}")
break
block = measurements()
summary = ["# Сводка проверок", "", "| Шаг | Что проверяет | Результат | Время, с | Файл |",
"| --- | --- | --- | --- | --- |"]
summary += [f"| {n} | {d} | {'PASS' if ok else 'FAIL'} | {s:.0f} | evidence/{n}.txt |" for n, d, ok, s in results]
(EVIDENCE / "summary.md").write_text("\n".join(summary) + "\n\n# Замеры\n\n" + block + "\n", encoding="utf-8")
print("evidence/summary.md written")
passed = all(ok for _, _, ok, _ in results)
if passed:
update_report(block) # таблицы строятся из всех файлов evidence/, в том числе от прошлых запусков
raise SystemExit(0 if passed else 1)
if __name__ == "__main__":
main()