forked from mtusi2026/mtusi2026lab3
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
157 lines
8.1 KiB
Python
157 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
|
|
|
|
python scripts/collect_evidence.py # ~25 min; the stand must NOT be running yet
|
|
|
|
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
|
|
stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the
|
|
level 4 experiments, and records container states before/after every hook. Prints no secrets
|
|
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
|
|
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
EVIDENCE = ROOT / "evidence"
|
|
WINDOWS = os.name == "nt"
|
|
HOOK = ".cmd" if WINDOWS else ".sh"
|
|
PYTHON = sys.executable
|
|
VENV_PY = str(ROOT / ".venv" / ("Scripts/python.exe" if WINDOWS else "bin/python"))
|
|
# Makefile recipes use $(PYTHON) unquoted: pass interpreters WITHOUT spaces in the path.
|
|
MAKE_PY = "python" if WINDOWS else "python3"
|
|
MAKE_VENV = ".venv/Scripts/python.exe" if WINDOWS else ".venv/bin/python"
|
|
ENV = {**os.environ, "PYTHONUTF8": "1"}
|
|
if WINDOWS:
|
|
ENV["PATH"] = ENV["PATH"] + os.pathsep + r"C:\Program Files\Git\usr\bin"
|
|
|
|
|
|
def now() -> str:
|
|
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
def sh(cmd: list[str], timeout: float = 1800, env: dict | None = None) -> tuple[int, str]:
|
|
started = time.monotonic()
|
|
result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
|
|
timeout=timeout, env={**ENV, **(env or {})})
|
|
took = time.monotonic() - started
|
|
return result.returncode, (result.stdout + result.stderr).rstrip() + f"\n[exit {result.returncode}, {took:.1f} s]\n"
|
|
|
|
|
|
class Log:
|
|
def __init__(self, name: str, title: str):
|
|
self.path = EVIDENCE / name
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
commit = subprocess.run(["git", "log", "-1", "--format=%h %s"], cwd=ROOT, capture_output=True, text=True,
|
|
encoding="utf-8").stdout.strip()
|
|
self.text = f"# {title}\n# {now()}, commit {commit}\n"
|
|
|
|
def run(self, cmd: list[str], shown: str | None = None, **kwargs) -> int:
|
|
code, out = sh(cmd, **kwargs)
|
|
self.text += f"\n$ {shown or ' '.join(cmd)}\n{out}"
|
|
self.save()
|
|
print(f" {self.path.name}: {shown or ' '.join(cmd[:4])} -> exit {code}", flush=True)
|
|
return code
|
|
|
|
def note(self, text: str) -> None:
|
|
self.text += f"\n{text}\n"
|
|
self.save()
|
|
|
|
def save(self) -> None:
|
|
self.path.write_text(self.text, encoding="utf-8")
|
|
|
|
|
|
def states() -> list[str]:
|
|
ids = subprocess.run(["docker", "compose", "ps", "-a", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
|
|
rows = []
|
|
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout or "[]"):
|
|
st = item["State"]
|
|
rows.append(f"{item['Name'].lstrip('/'):24} host={item['Config']['Hostname']:13} {st['Status']:8} "
|
|
f"exit={st['ExitCode']:<3} pid={st['Pid']:<6} started={st['StartedAt'][:23]}")
|
|
return sorted(rows)
|
|
|
|
|
|
def main() -> int:
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
make = shutil.which("make", path=ENV["PATH"]) or "make"
|
|
# 1. Clean clone -> secrets -> build -> start
|
|
log = Log("clean-clone.txt", "Clean clone: secrets, config, build, migrations from zero, start, versions")
|
|
log.run(["git", "status", "--short", "--branch"])
|
|
log.run([PYTHON, "--version"]); log.run(["docker", "version", "--format", "client {{.Client.Version}} / server {{.Server.Version}}"])
|
|
log.run(["docker", "compose", "version"]); log.run([make, "--version"], shown="make --version")
|
|
log.run([PYTHON, "scripts/init_secrets.py"], shown="python scripts/init_secrets.py")
|
|
if not (ROOT / ".env").exists():
|
|
shutil.copy(ROOT / ".env.example", ROOT / ".env")
|
|
log.note("$ cp .env.example .env")
|
|
log.run(["docker", "compose", "--env-file", ".env", "config", "--quiet"])
|
|
log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800)
|
|
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"])
|
|
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
|
|
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
|
|
# venv for validate / own tests
|
|
if not Path(VENV_PY).exists():
|
|
log.run([PYTHON, "-m", "venv", ".venv"])
|
|
log.run([VENV_PY, "-m", "pip", "install", "-q", "--require-hashes", "--no-deps", "-r", "requirements.txt"], timeout=1800)
|
|
log.run([VENV_PY, "-m", "pip", "install", "-q", "-r", "requirements-test.txt", "-r", "requirements-dev.txt"], timeout=1800)
|
|
|
|
# 2. make check / validate / test
|
|
log = Log("make-check-validate-test.txt", "make check, make validate (venv), make test")
|
|
log.run([make, "check", f"PYTHON={MAKE_PY}"], shown="make check")
|
|
log.run([make, "validate", f"PYTHON={MAKE_VENV}"], shown=f"make validate PYTHON={MAKE_VENV}")
|
|
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
|
|
|
|
# 3. persistence with states before/after
|
|
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey")
|
|
log.note("Containers BEFORE:\n" + "\n".join(states()))
|
|
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
|
|
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
|
|
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey; volumes kept):\n" + "\n".join(states()))
|
|
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
|
|
|
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
|
|
log = Log("test-failover.txt", "make test-failover with scripts/stop-one: which container is really stopped, restore, repeat")
|
|
stopped: set[str] = set()
|
|
for attempt in range(1, 7):
|
|
before = {r for r in states() if "-api-" in r and " running " in r}
|
|
log.note(f"--- run {attempt}; running API replicas BEFORE:\n" + "\n".join(sorted(before)))
|
|
log.run([make, "test-failover", f"ACTION_SCRIPT=scripts/stop-one{HOOK}", f"PYTHON={MAKE_PY}"],
|
|
shown=f"make test-failover ACTION_SCRIPT=scripts/stop-one{HOOK}")
|
|
after = states()
|
|
gone = [r for r in after if "-api-" in r and " exited " in r]
|
|
log.note("AFTER (the stopped replica, left stopped by the hook):\n" + "\n".join(gone))
|
|
log.run([PYTHON, "scripts/lbctl.py", "status"])
|
|
stopped.update(r.split()[0] for r in gone)
|
|
log.run(["sh" if not WINDOWS else r"C:\Program Files\Git\bin\bash.exe", "scripts/restore-api.sh"],
|
|
shown="sh scripts/restore-api.sh")
|
|
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
|
if len(stopped) >= 2:
|
|
break
|
|
log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}")
|
|
|
|
# 5. own tests
|
|
log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally")
|
|
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
|
|
timeout=3600)
|
|
|
|
# 6. level 4 experiments and the public load script
|
|
log = Log("level4-runs.txt", "Level 4: failover series, rolling update under load, public load.py")
|
|
log.run([PYTHON, "bench/failover_series.py", "--output", "evidence/failover"], timeout=1800)
|
|
log.run([PYTHON, "bench/rolling_under_load.py", "--output", "evidence/rolling-update"], timeout=1800)
|
|
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
|
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
|
|
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
|
|
print("done:", EVIDENCE)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|