Files
mtusi2026lab3/scripts/collect_evidence.py
T
Alexey BodanyukandClaude Opus 5.5 55210c6e9c docs: DEFENSE и WINDOWS под lab3, модель данных, CI, сбор доказательств
- DEFENSE: что изменилось с lab2, путь POST с ключом, ответы на вопросы
  README, ловушки из чужих PR, периоды проверок и время обнаружения.
- WINDOWS: запуск на ноутбуке, make из PowerShell (нужен sh из Git),
  сценарий защиты по шагам README.
- DATA_MODEL: idempotency_keys и функция post_message.
- CI: test-failover с восстановлением, свои тесты.
- scripts/collect_evidence.py: все проверки на чистом clone -> evidence/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 16:03:23 +03:00

154 lines
7.9 KiB
Python

#!/usr/bin/env python3
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
python scripts/collect_evidence.py # ~25 min; the stand must NOT be running yet
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the
level 4 experiments, and records container states before/after every hook. Prints no secrets
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
"""
from __future__ import annotations
import datetime as dt
import json
import os
import shutil
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
WINDOWS = os.name == "nt"
HOOK = ".cmd" if WINDOWS else ".sh"
PYTHON = sys.executable
VENV_PY = str(ROOT / ".venv" / ("Scripts/python.exe" if WINDOWS else "bin/python"))
ENV = {**os.environ, "PYTHONUTF8": "1"}
if WINDOWS:
ENV["PATH"] = ENV["PATH"] + os.pathsep + r"C:\Program Files\Git\usr\bin"
def now() -> str:
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def sh(cmd: list[str], timeout: float = 1800, env: dict | None = None) -> tuple[int, str]:
started = time.monotonic()
result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
timeout=timeout, env={**ENV, **(env or {})})
took = time.monotonic() - started
return result.returncode, (result.stdout + result.stderr).rstrip() + f"\n[exit {result.returncode}, {took:.1f} s]\n"
class Log:
def __init__(self, name: str, title: str):
self.path = EVIDENCE / name
self.path.parent.mkdir(parents=True, exist_ok=True)
commit = subprocess.run(["git", "log", "-1", "--format=%h %s"], cwd=ROOT, capture_output=True, text=True,
encoding="utf-8").stdout.strip()
self.text = f"# {title}\n# {now()}, commit {commit}\n"
def run(self, cmd: list[str], shown: str | None = None, **kwargs) -> int:
code, out = sh(cmd, **kwargs)
self.text += f"\n$ {shown or ' '.join(cmd)}\n{out}"
self.save()
print(f" {self.path.name}: {shown or ' '.join(cmd[:4])} -> exit {code}", flush=True)
return code
def note(self, text: str) -> None:
self.text += f"\n{text}\n"
self.save()
def save(self) -> None:
self.path.write_text(self.text, encoding="utf-8")
def states() -> list[str]:
ids = subprocess.run(["docker", "compose", "ps", "-a", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
rows = []
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout or "[]"):
st = item["State"]
rows.append(f"{item['Name'].lstrip('/'):24} host={item['Config']['Hostname']:13} {st['Status']:8} "
f"exit={st['ExitCode']:<3} pid={st['Pid']:<6} started={st['StartedAt'][:23]}")
return sorted(rows)
def main() -> int:
EVIDENCE.mkdir(exist_ok=True)
make = shutil.which("make", path=ENV["PATH"]) or "make"
# 1. Clean clone -> secrets -> build -> start
log = Log("clean-clone.txt", "Clean clone: secrets, config, build, migrations from zero, start, versions")
log.run(["git", "status", "--short", "--branch"])
log.run([PYTHON, "--version"]); log.run(["docker", "version", "--format", "client {{.Client.Version}} / server {{.Server.Version}}"])
log.run(["docker", "compose", "version"]); log.run([make, "--version"], shown="make --version")
log.run([PYTHON, "scripts/init_secrets.py"], shown="python scripts/init_secrets.py")
if not (ROOT / ".env").exists():
shutil.copy(ROOT / ".env.example", ROOT / ".env")
log.note("$ cp .env.example .env")
log.run(["docker", "compose", "--env-file", ".env", "config", "--quiet"])
log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800)
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
# venv for validate / own tests
if not Path(VENV_PY).exists():
log.run([PYTHON, "-m", "venv", ".venv"])
log.run([VENV_PY, "-m", "pip", "install", "-q", "--require-hashes", "--no-deps", "-r", "requirements.txt"], timeout=1800)
log.run([VENV_PY, "-m", "pip", "install", "-q", "-r", "requirements-test.txt", "-r", "requirements-dev.txt"], timeout=1800)
# 2. make check / validate / test
log = Log("make-check-validate-test.txt", "make check, make validate (venv), make test")
log.run([make, "check", f"PYTHON={PYTHON}"], shown="make check")
log.run([make, "validate", f"PYTHON={VENV_PY}"], shown="make validate PYTHON=.venv/python")
log.run([make, "test", f"PYTHON={PYTHON}"], shown="make test")
# 3. persistence with states before/after
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey")
log.note("Containers BEFORE:\n" + "\n".join(states()))
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={PYTHON}"],
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey; volumes kept):\n" + "\n".join(states()))
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
log = Log("test-failover.txt", "make test-failover with scripts/stop-one: which container is really stopped, restore, repeat")
stopped: set[str] = set()
for attempt in range(1, 7):
before = {r for r in states() if "-api-" in r and " running " in r}
log.note(f"--- run {attempt}; running API replicas BEFORE:\n" + "\n".join(sorted(before)))
log.run([make, "test-failover", f"ACTION_SCRIPT=scripts/stop-one{HOOK}", f"PYTHON={PYTHON}"],
shown=f"make test-failover ACTION_SCRIPT=scripts/stop-one{HOOK}")
after = states()
gone = [r for r in after if "-api-" in r and " exited " in r]
log.note("AFTER (the stopped replica, left stopped by the hook):\n" + "\n".join(gone))
log.run([PYTHON, "scripts/lbctl.py", "status"])
stopped.update(r.split()[0] for r in gone)
log.run(["sh" if not WINDOWS else r"C:\Program Files\Git\bin\bash.exe", "scripts/restore-api.sh"],
shown="sh scripts/restore-api.sh")
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
if len(stopped) >= 2:
break
log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}")
# 5. own tests
log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally")
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
timeout=3600)
# 6. level 4 experiments and the public load script
log = Log("level4-runs.txt", "Level 4: failover series, rolling update under load, public load.py")
log.run([PYTHON, "bench/failover_series.py", "--output", "evidence/failover"], timeout=1800)
log.run([PYTHON, "bench/rolling_under_load.py", "--output", "evidence/rolling-update"], timeout=1800)
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
print("done:", EVIDENCE)
return 0
if __name__ == "__main__":
raise SystemExit(main())