Files
mtusi2026lab5/scripts/collect_evidence.py
T

185 lines
10 KiB
Python

#!/usr/bin/env python3
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
python scripts/collect_evidence.py # ~90 min; the stand must NOT be running yet
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
stand, then runs make check/validate/test/test-persistence/test-failover, the lab 5 defense
scenario (scripts/demo_pipeline.py), the own tests, the track 2 fault-injection matrix
(bench/pipeline_faults.py) and the invariant audit, and records container states before/after
every hook and the real resource limits (docker inspect). Prints no secrets
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
"""
from __future__ import annotations
import datetime as dt
import json
import os
import shutil
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
WINDOWS = os.name == "nt"
HOOK = ".cmd" if WINDOWS else ".sh"
PYTHON = sys.executable
VENV_PY = str(ROOT / ".venv" / ("Scripts/python.exe" if WINDOWS else "bin/python"))
# Makefile recipes use $(PYTHON) unquoted: pass interpreters WITHOUT spaces in the path.
MAKE_PY = "python" if WINDOWS else "python3"
MAKE_VENV = ".venv/Scripts/python.exe" if WINDOWS else ".venv/bin/python"
ENV = {**os.environ, "PYTHONUTF8": "1"}
if WINDOWS:
ENV["PATH"] = ENV["PATH"] + os.pathsep + r"C:\Program Files\Git\usr\bin"
def now() -> str:
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def sh(cmd: list[str], timeout: float = 1800, env: dict | None = None) -> tuple[int, str]:
started = time.monotonic()
result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
timeout=timeout, env={**ENV, **(env or {})})
took = time.monotonic() - started
return result.returncode, (result.stdout + result.stderr).rstrip() + f"\n[exit {result.returncode}, {took:.1f} s]\n"
class Log:
def __init__(self, name: str, title: str):
self.path = EVIDENCE / name
self.path.parent.mkdir(parents=True, exist_ok=True)
commit = subprocess.run(["git", "log", "-1", "--format=%h %s"], cwd=ROOT, capture_output=True, text=True,
encoding="utf-8").stdout.strip()
self.text = f"# {title}\n# {now()}, commit {commit}\n"
def run(self, cmd: list[str], shown: str | None = None, **kwargs) -> int:
code, out = sh(cmd, **kwargs)
self.text += f"\n$ {shown or ' '.join(cmd)}\n{out}"
self.save()
print(f" {self.path.name}: {shown or ' '.join(cmd[:4])} -> exit {code}", flush=True)
return code
def note(self, text: str) -> None:
self.text += f"\n{text}\n"
self.save()
def save(self) -> None:
self.path.write_text(self.text, encoding="utf-8")
def states() -> list[str]:
ids = subprocess.run(["docker", "compose", "ps", "-a", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
rows = []
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout or "[]"):
st = item["State"]
rows.append(f"{item['Name'].lstrip('/'):24} host={item['Config']['Hostname']:13} {st['Status']:8} "
f"exit={st['ExitCode']:<3} pid={st['Pid']:<6} started={st['StartedAt'][:23]}")
return sorted(rows)
def main() -> int:
EVIDENCE.mkdir(exist_ok=True)
make = shutil.which("make", path=ENV["PATH"]) or "make"
# 1. Clean clone -> secrets -> build -> start
log = Log("clean-clone.txt", "Clean clone: secrets, config, build, migrations from zero, start, versions")
log.run(["git", "status", "--short", "--branch"])
log.run([PYTHON, "--version"]); log.run(["docker", "version", "--format", "client {{.Client.Version}} / server {{.Server.Version}}"])
log.run(["docker", "compose", "version"]); log.run([make, "--version"], shown="make --version")
log.run([PYTHON, "scripts/init_secrets.py"], shown="python scripts/init_secrets.py")
if not (ROOT / ".env").exists():
shutil.copy(ROOT / ".env.example", ROOT / ".env")
log.note("$ cp .env.example .env")
log.run(["docker", "compose", "--env-file", ".env", "config", "--quiet"])
log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800)
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_users"])
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "type",
"durable", "messages", "consumers", "arguments"])
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_permissions"])
ids = subprocess.run(["docker", "compose", "ps", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
log.run(["docker", "inspect", "-f", "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} "
"memswap={{.HostConfig.MemorySwap}} image={{.Config.Image}}", *ids], shown="docker inspect (limits)")
# venv for validate / own tests
if not Path(VENV_PY).exists():
log.run([PYTHON, "-m", "venv", ".venv"])
log.run([VENV_PY, "-m", "pip", "install", "-q", "--require-hashes", "--no-deps", "-r", "requirements.txt"], timeout=1800)
log.run([VENV_PY, "-m", "pip", "install", "-q", "-r", "requirements-test.txt", "-r", "requirements-dev.txt"], timeout=1800)
# 2. make check / validate / test
log = Log("make-check-validate-test.txt", "make check, make validate (venv), make test")
log.run([make, "check", f"PYTHON={MAKE_PY}"], shown="make check")
log.run([make, "validate", f"PYTHON={MAKE_VENV}"], shown=f"make validate PYTHON={MAKE_VENV}")
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
# 3. persistence with states before/after
log = Log("test-persistence.txt", "make test-persistence: restart of LB, API replicas, relay, worker, PostgreSQL, "
"Valkey, S3, RabbitMQ")
log.note("Containers BEFORE:\n" + "\n".join(states()))
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
log.note("Containers AFTER (new pid/started for lb, api-*, relay, worker, postgres, valkey, s3, broker; volumes "
"kept):\n" + "\n".join(states()))
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
log = Log("test-failover.txt", "make test-failover with scripts/stop-one: which container is really stopped, restore, repeat")
stopped: set[str] = set()
import urllib.request
for attempt in range(1, 7):
# The public test stops the replica that served its first POST; its request sequence is
# fixed, so with round robin it hits the same replica every time. Shift the rotation by
# (attempt - 1) extra requests so that the other replica becomes the target as well.
for _ in range(attempt - 1):
urllib.request.urlopen("http://127.0.0.1:8080/health/live", timeout=5).read()
before = {r for r in states() if "-api-" in r and " running " in r}
log.note(f"--- run {attempt}; running API replicas BEFORE:\n" + "\n".join(sorted(before)))
log.run([make, "test-failover", f"ACTION_SCRIPT=scripts/stop-one{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-failover ACTION_SCRIPT=scripts/stop-one{HOOK}")
after = states()
gone = [r for r in after if "-api-" in r and " exited " in r]
log.note("AFTER (the stopped replica, left stopped by the hook):\n" + "\n".join(gone))
log.run([PYTHON, "scripts/lbctl.py", "status"])
stopped.update(r.split()[0] for r in gone)
log.run(["sh" if not WINDOWS else r"C:\Program Files\Git\bin\bash.exe", "scripts/restore-api.sh"],
shown="sh scripts/restore-api.sh")
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
if len(stopped) >= 2:
break
log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}")
# 5. lab 5 defense scenario: worker stopped -> 202 -> 409 -> restart api+broker -> worker -> ready
log = Log("demo-pipeline.txt", "Defense steps 2-3: worker stopped, upload 202, content 409, restart API and broker "
"with volumes, worker started, ready + thumbnail")
log.note("Containers BEFORE:\n" + "\n".join(states()))
log.run([VENV_PY, "scripts/demo_pipeline.py"], shown="python scripts/demo_pipeline.py", timeout=900)
log.note("Containers AFTER (api-*, broker restarted; worker stopped and started):\n" + "\n".join(states()))
# 6. own tests
log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally")
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
timeout=5400)
# 7. track 2: fault-injection matrix + audit; lab 3 load script
log = Log("track2-runs.txt", "Track 2: bench/pipeline_faults.py (duplicate, reorder, crashes, stale versions, hung "
"worker) + read-only audit of the invariants")
log.run([VENV_PY, "bench/pipeline_faults.py", "--output", "evidence/track2", "--uploads", "5"], timeout=5400)
log.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "auditor", "--verify-checksums"],
shown="docker compose run --rm auditor --verify-checksums", timeout=900)
log.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "jobctl", "stats"], shown="docker compose run --rm jobctl stats")
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
print("done:", EVIDENCE)
return 0
if __name__ == "__main__":
raise SystemExit(main())