Files
mtusi2026lab3/scripts/collect_evidence.py
T
Alexey BodanyukandClaude Opus 5.5 27087326d0 fix: LB и на [::1] — localhost на Windows стоил 2 с на запрос; сдвиг цели failover
Найдено первым прогоном из чистого clone: make test-failover (BASE_URL
по умолчанию http://localhost:8080) успел сделать 8 записей за 15 с, и
первая успешная запись после хука пришла на 8,4 с. Windows сначала
стучится на ::1, где порт не был опубликован, и каждое новое соединение
ждёт ~2 с. LB теперь опубликован на 127.0.0.1 и [::1] (оба loopback):
0,013 с на запрос. Тест политики портов допускает только loopback.

collect_evidence: публичный тест всегда выбирает реплику, обслужившую
его первый POST, а последовательность запросов фиксирована — перед
каждым следующим прогоном rotation сдвигается лишними GET, чтобы
по очереди останавливались обе реплики.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 18:09:12 +03:00

163 lines
8.6 KiB
Python

#!/usr/bin/env python3
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
python scripts/collect_evidence.py # ~25 min; the stand must NOT be running yet
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the
level 4 experiments, and records container states before/after every hook. Prints no secrets
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
"""
from __future__ import annotations
import datetime as dt
import json
import os
import shutil
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
WINDOWS = os.name == "nt"
HOOK = ".cmd" if WINDOWS else ".sh"
PYTHON = sys.executable
VENV_PY = str(ROOT / ".venv" / ("Scripts/python.exe" if WINDOWS else "bin/python"))
# Makefile recipes use $(PYTHON) unquoted: pass interpreters WITHOUT spaces in the path.
MAKE_PY = "python" if WINDOWS else "python3"
MAKE_VENV = ".venv/Scripts/python.exe" if WINDOWS else ".venv/bin/python"
ENV = {**os.environ, "PYTHONUTF8": "1"}
if WINDOWS:
ENV["PATH"] = ENV["PATH"] + os.pathsep + r"C:\Program Files\Git\usr\bin"
def now() -> str:
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def sh(cmd: list[str], timeout: float = 1800, env: dict | None = None) -> tuple[int, str]:
started = time.monotonic()
result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
timeout=timeout, env={**ENV, **(env or {})})
took = time.monotonic() - started
return result.returncode, (result.stdout + result.stderr).rstrip() + f"\n[exit {result.returncode}, {took:.1f} s]\n"
class Log:
def __init__(self, name: str, title: str):
self.path = EVIDENCE / name
self.path.parent.mkdir(parents=True, exist_ok=True)
commit = subprocess.run(["git", "log", "-1", "--format=%h %s"], cwd=ROOT, capture_output=True, text=True,
encoding="utf-8").stdout.strip()
self.text = f"# {title}\n# {now()}, commit {commit}\n"
def run(self, cmd: list[str], shown: str | None = None, **kwargs) -> int:
code, out = sh(cmd, **kwargs)
self.text += f"\n$ {shown or ' '.join(cmd)}\n{out}"
self.save()
print(f" {self.path.name}: {shown or ' '.join(cmd[:4])} -> exit {code}", flush=True)
return code
def note(self, text: str) -> None:
self.text += f"\n{text}\n"
self.save()
def save(self) -> None:
self.path.write_text(self.text, encoding="utf-8")
def states() -> list[str]:
ids = subprocess.run(["docker", "compose", "ps", "-a", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
rows = []
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout or "[]"):
st = item["State"]
rows.append(f"{item['Name'].lstrip('/'):24} host={item['Config']['Hostname']:13} {st['Status']:8} "
f"exit={st['ExitCode']:<3} pid={st['Pid']:<6} started={st['StartedAt'][:23]}")
return sorted(rows)
def main() -> int:
EVIDENCE.mkdir(exist_ok=True)
make = shutil.which("make", path=ENV["PATH"]) or "make"
# 1. Clean clone -> secrets -> build -> start
log = Log("clean-clone.txt", "Clean clone: secrets, config, build, migrations from zero, start, versions")
log.run(["git", "status", "--short", "--branch"])
log.run([PYTHON, "--version"]); log.run(["docker", "version", "--format", "client {{.Client.Version}} / server {{.Server.Version}}"])
log.run(["docker", "compose", "version"]); log.run([make, "--version"], shown="make --version")
log.run([PYTHON, "scripts/init_secrets.py"], shown="python scripts/init_secrets.py")
if not (ROOT / ".env").exists():
shutil.copy(ROOT / ".env.example", ROOT / ".env")
log.note("$ cp .env.example .env")
log.run(["docker", "compose", "--env-file", ".env", "config", "--quiet"])
log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800)
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
# venv for validate / own tests
if not Path(VENV_PY).exists():
log.run([PYTHON, "-m", "venv", ".venv"])
log.run([VENV_PY, "-m", "pip", "install", "-q", "--require-hashes", "--no-deps", "-r", "requirements.txt"], timeout=1800)
log.run([VENV_PY, "-m", "pip", "install", "-q", "-r", "requirements-test.txt", "-r", "requirements-dev.txt"], timeout=1800)
# 2. make check / validate / test
log = Log("make-check-validate-test.txt", "make check, make validate (venv), make test")
log.run([make, "check", f"PYTHON={MAKE_PY}"], shown="make check")
log.run([make, "validate", f"PYTHON={MAKE_VENV}"], shown=f"make validate PYTHON={MAKE_VENV}")
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
# 3. persistence with states before/after
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey")
log.note("Containers BEFORE:\n" + "\n".join(states()))
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey; volumes kept):\n" + "\n".join(states()))
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
log = Log("test-failover.txt", "make test-failover with scripts/stop-one: which container is really stopped, restore, repeat")
stopped: set[str] = set()
import urllib.request
for attempt in range(1, 7):
# The public test stops the replica that served its first POST; its request sequence is
# fixed, so with round robin it hits the same replica every time. Shift the rotation by
# (attempt - 1) extra requests so that the other replica becomes the target as well.
for _ in range(attempt - 1):
urllib.request.urlopen("http://127.0.0.1:8080/health/live", timeout=5).read()
before = {r for r in states() if "-api-" in r and " running " in r}
log.note(f"--- run {attempt}; running API replicas BEFORE:\n" + "\n".join(sorted(before)))
log.run([make, "test-failover", f"ACTION_SCRIPT=scripts/stop-one{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-failover ACTION_SCRIPT=scripts/stop-one{HOOK}")
after = states()
gone = [r for r in after if "-api-" in r and " exited " in r]
log.note("AFTER (the stopped replica, left stopped by the hook):\n" + "\n".join(gone))
log.run([PYTHON, "scripts/lbctl.py", "status"])
stopped.update(r.split()[0] for r in gone)
log.run(["sh" if not WINDOWS else r"C:\Program Files\Git\bin\bash.exe", "scripts/restore-api.sh"],
shown="sh scripts/restore-api.sh")
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
if len(stopped) >= 2:
break
log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}")
# 5. own tests
log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally")
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
timeout=3600)
# 6. level 4 experiments and the public load script
log = Log("level4-runs.txt", "Level 4: failover series, rolling update under load, public load.py")
log.run([PYTHON, "bench/failover_series.py", "--output", "evidence/failover"], timeout=1800)
log.run([PYTHON, "bench/rolling_under_load.py", "--output", "evidence/rolling-update"], timeout=1800)
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
print("done:", EVIDENCE)
return 0
if __name__ == "__main__":
raise SystemExit(main())