#!/usr/bin/env python3 """Run every check of REPORT.md on the current checkout and save the real output in evidence/. python scripts/collect_evidence.py # ~60 min; the stand must NOT be running yet Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the lab 4 experiments (memory under concurrent uploads, reconciler faults and cost), and records container states before/after every hook. Prints no secrets (hook output is shown by the public tests as suppressed; our scripts print names and states only). On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin. """ from __future__ import annotations import datetime as dt import json import os import shutil import subprocess import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] EVIDENCE = ROOT / "evidence" WINDOWS = os.name == "nt" HOOK = ".cmd" if WINDOWS else ".sh" PYTHON = sys.executable VENV_PY = str(ROOT / ".venv" / ("Scripts/python.exe" if WINDOWS else "bin/python")) # Makefile recipes use $(PYTHON) unquoted: pass interpreters WITHOUT spaces in the path. MAKE_PY = "python" if WINDOWS else "python3" MAKE_VENV = ".venv/Scripts/python.exe" if WINDOWS else ".venv/bin/python" ENV = {**os.environ, "PYTHONUTF8": "1"} if WINDOWS: ENV["PATH"] = ENV["PATH"] + os.pathsep + r"C:\Program Files\Git\usr\bin" def now() -> str: return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def sh(cmd: list[str], timeout: float = 1800, env: dict | None = None) -> tuple[int, str]: started = time.monotonic() result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, env={**ENV, **(env or {})}) took = time.monotonic() - started return result.returncode, (result.stdout + result.stderr).rstrip() + f"\n[exit {result.returncode}, {took:.1f} s]\n" class Log: def __init__(self, name: str, title: str): self.path = EVIDENCE / name self.path.parent.mkdir(parents=True, exist_ok=True) commit = subprocess.run(["git", "log", "-1", "--format=%h %s"], cwd=ROOT, capture_output=True, text=True, encoding="utf-8").stdout.strip() self.text = f"# {title}\n# {now()}, commit {commit}\n" def run(self, cmd: list[str], shown: str | None = None, **kwargs) -> int: code, out = sh(cmd, **kwargs) self.text += f"\n$ {shown or ' '.join(cmd)}\n{out}" self.save() print(f" {self.path.name}: {shown or ' '.join(cmd[:4])} -> exit {code}", flush=True) return code def note(self, text: str) -> None: self.text += f"\n{text}\n" self.save() def save(self) -> None: self.path.write_text(self.text, encoding="utf-8") def states() -> list[str]: ids = subprocess.run(["docker", "compose", "ps", "-a", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split() rows = [] for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout or "[]"): st = item["State"] rows.append(f"{item['Name'].lstrip('/'):24} host={item['Config']['Hostname']:13} {st['Status']:8} " f"exit={st['ExitCode']:<3} pid={st['Pid']:<6} started={st['StartedAt'][:23]}") return sorted(rows) def main() -> int: EVIDENCE.mkdir(exist_ok=True) make = shutil.which("make", path=ENV["PATH"]) or "make" # 1. Clean clone -> secrets -> build -> start log = Log("clean-clone.txt", "Clean clone: secrets, config, build, migrations from zero, start, versions") log.run(["git", "status", "--short", "--branch"]) log.run([PYTHON, "--version"]); log.run(["docker", "version", "--format", "client {{.Client.Version}} / server {{.Server.Version}}"]) log.run(["docker", "compose", "version"]); log.run([make, "--version"], shown="make --version") log.run([PYTHON, "scripts/init_secrets.py"], shown="python scripts/init_secrets.py") if not (ROOT / ".env").exists(): shutil.copy(ROOT / ".env.example", ROOT / ".env") log.note("$ cp .env.example .env") log.run(["docker", "compose", "--env-file", ".env", "config", "--quiet"]) log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800) log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"]) log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"]) log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"]) # venv for validate / own tests if not Path(VENV_PY).exists(): log.run([PYTHON, "-m", "venv", ".venv"]) log.run([VENV_PY, "-m", "pip", "install", "-q", "--require-hashes", "--no-deps", "-r", "requirements.txt"], timeout=1800) log.run([VENV_PY, "-m", "pip", "install", "-q", "-r", "requirements-test.txt", "-r", "requirements-dev.txt"], timeout=1800) # 2. make check / validate / test log = Log("make-check-validate-test.txt", "make check, make validate (venv), make test") log.run([make, "check", f"PYTHON={MAKE_PY}"], shown="make check") log.run([make, "validate", f"PYTHON={MAKE_VENV}"], shown=f"make validate PYTHON={MAKE_VENV}") log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test") # 3. persistence with states before/after log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3") log.note("Containers BEFORE:\n" + "\n".join(states())) log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"], shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}") log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):\n" + "\n".join(states())) log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"]) # 4. failover: until BOTH replicas have been the stopped one (each run is a real public test) log = Log("test-failover.txt", "make test-failover with scripts/stop-one: which container is really stopped, restore, repeat") stopped: set[str] = set() import urllib.request for attempt in range(1, 7): # The public test stops the replica that served its first POST; its request sequence is # fixed, so with round robin it hits the same replica every time. Shift the rotation by # (attempt - 1) extra requests so that the other replica becomes the target as well. for _ in range(attempt - 1): urllib.request.urlopen("http://127.0.0.1:8080/health/live", timeout=5).read() before = {r for r in states() if "-api-" in r and " running " in r} log.note(f"--- run {attempt}; running API replicas BEFORE:\n" + "\n".join(sorted(before))) log.run([make, "test-failover", f"ACTION_SCRIPT=scripts/stop-one{HOOK}", f"PYTHON={MAKE_PY}"], shown=f"make test-failover ACTION_SCRIPT=scripts/stop-one{HOOK}") after = states() gone = [r for r in after if "-api-" in r and " exited " in r] log.note("AFTER (the stopped replica, left stopped by the hook):\n" + "\n".join(gone)) log.run([PYTHON, "scripts/lbctl.py", "status"]) stopped.update(r.split()[0] for r in gone) log.run(["sh" if not WINDOWS else r"C:\Program Files\Git\bin\bash.exe", "scripts/restore-api.sh"], shown="sh scripts/restore-api.sh") log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"]) if len(stopped) >= 2: break log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}") # 5. own tests log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally") log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v", timeout=3600) # 6. lab 4 experiments (level 4 memory, track 2 reconciler) and the public load script log = Log("lab4-runs.txt", "Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py") log.run([VENV_PY, "bench/upload_memory.py", "--output", "evidence/memory"], timeout=3600) log.run([VENV_PY, "bench/reconcile_faults.py", "--output", "evidence/reconcile"], timeout=1800) log.run([VENV_PY, "bench/reconcile_cost.py", "--output", "evidence/reconcile"], timeout=3600) log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"]) log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"]) log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"]) print("done:", EVIDENCE) return 0 if __name__ == "__main__": raise SystemExit(main())