Files
mtusi2026lab4/student-tests/verify.py

72 lines
3.9 KiB
Python

#!/usr/bin/env python3
"""One entry point: existing Go checks, runner regression checks, public and student scenarios."""
import argparse
from collections import Counter
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--config", default="student-tests/lab4/config.example.json")
p.add_argument("--project", required=True, help="explicit synthetic Compose project to test")
p.add_argument("--output", required=True, help="new, empty directory for original results")
args = p.parse_args()
output = Path(args.output).resolve()
output.mkdir(parents=True, exist_ok=True)
if any(output.iterdir()):
p.error("output must be empty; never overwrite a previous experiment")
env = dict(os.environ, COMPOSE_PROJECT_NAME=args.project)
env.setdefault("GOCACHE", "/private/tmp/highload-lab4-go-cache" if sys.platform == "darwin"
else "/tmp/highload-lab4-go-cache")
summary = {"time_utc": datetime.now(timezone.utc).isoformat(), "project": args.project,
"scope": "lab4 student checks; no automatic grade", "checks": {}}
commands = [
("contract_structure", ["make", "check", "PYTHON=" + sys.executable], 30),
("go_tests", ["go", "test", "-count=1", "-json", "./..."], 180),
("runner_regressions", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests/lab4", "-p", "test_*.py", "-v"], 30),
("lab4_scenarios", [sys.executable, "student-tests/lab4/run.py", "--config", args.config,
"--output", str(output / "lab4"), "--scenario", "all"], 1800)]
for name, argv, timeout in commands:
print("Running " + name, flush=True)
try:
with (output / (name + ".stdout.txt")).open("wb") as stdout, (output / (name + ".stderr.txt")).open("wb") as stderr:
proc = subprocess.run(argv, cwd=ROOT, env=env, stdout=stdout, stderr=stderr, timeout=timeout, check=False)
item = {"status": "PASS" if proc.returncode == 0 else "FAIL", "exit_code": proc.returncode}
except (OSError, subprocess.TimeoutExpired) as exc:
item = {"status": "FAIL", "detail": type(exc).__name__}
if name == "go_tests":
events = []
for line in (output / (name + ".stdout.txt")).read_text(errors="replace").splitlines():
try:
events.append(json.loads(line))
except ValueError:
pass
counts = Counter(e["Action"] for e in events if "Test" in e and e.get("Action") in ("pass", "skip", "fail"))
item["test_events"] = dict(counts)
item["pending_tests"] = [e["Package"] + "/" + e["Test"] for e in events if e.get("Action") == "skip" and "Test" in e]
if counts["skip"]:
summary["checks"]["go_storage_and_restart_coverage"] = {
"status": "PENDING_INFRA", "detail": "Go skip events are not proof of storage/restart behavior"}
if name == "lab4_scenarios" and (output / "lab4/summary.json").is_file():
statuses = Counter(v["status"] for v in json.loads((output / "lab4/summary.json").read_text()).values())
item["case_statuses"] = dict(statuses)
if not statuses["FAIL"] and statuses["PENDING_INFRA"]:
item["status"] = "PENDING_INFRA"
summary["checks"][name] = item
print(item["status"] + " " + name, flush=True)
summary["status"] = "PASS" if all(v["status"] == "PASS" for v in summary["checks"].values()) else "INCOMPLETE"
(output / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n")
print(summary["status"] + ": " + str(output / "summary.json"), flush=True)
return 0 if summary["status"] == "PASS" else 1
if __name__ == "__main__":
raise SystemExit(main())