forked from mtusi2026/mtusi2026lab4
72 lines
3.9 KiB
Python
72 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
"""One entry point: existing Go checks, runner regression checks, public and student scenarios."""
|
|
import argparse
|
|
from collections import Counter
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("--config", default="student-tests/lab4/config.example.json")
|
|
p.add_argument("--project", required=True, help="explicit synthetic Compose project to test")
|
|
p.add_argument("--output", required=True, help="new, empty directory for original results")
|
|
args = p.parse_args()
|
|
output = Path(args.output).resolve()
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
if any(output.iterdir()):
|
|
p.error("output must be empty; never overwrite a previous experiment")
|
|
env = dict(os.environ, COMPOSE_PROJECT_NAME=args.project)
|
|
env.setdefault("GOCACHE", "/private/tmp/highload-lab4-go-cache" if sys.platform == "darwin"
|
|
else "/tmp/highload-lab4-go-cache")
|
|
summary = {"time_utc": datetime.now(timezone.utc).isoformat(), "project": args.project,
|
|
"scope": "lab4 student checks; no automatic grade", "checks": {}}
|
|
commands = [
|
|
("contract_structure", ["make", "check", "PYTHON=" + sys.executable], 30),
|
|
("go_tests", ["go", "test", "-count=1", "-json", "./..."], 180),
|
|
("runner_regressions", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests/lab4", "-p", "test_*.py", "-v"], 30),
|
|
("lab4_scenarios", [sys.executable, "student-tests/lab4/run.py", "--config", args.config,
|
|
"--output", str(output / "lab4"), "--scenario", "all"], 1800)]
|
|
for name, argv, timeout in commands:
|
|
print("Running " + name, flush=True)
|
|
try:
|
|
with (output / (name + ".stdout.txt")).open("wb") as stdout, (output / (name + ".stderr.txt")).open("wb") as stderr:
|
|
proc = subprocess.run(argv, cwd=ROOT, env=env, stdout=stdout, stderr=stderr, timeout=timeout, check=False)
|
|
item = {"status": "PASS" if proc.returncode == 0 else "FAIL", "exit_code": proc.returncode}
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
item = {"status": "FAIL", "detail": type(exc).__name__}
|
|
if name == "go_tests":
|
|
events = []
|
|
for line in (output / (name + ".stdout.txt")).read_text(errors="replace").splitlines():
|
|
try:
|
|
events.append(json.loads(line))
|
|
except ValueError:
|
|
pass
|
|
counts = Counter(e["Action"] for e in events if "Test" in e and e.get("Action") in ("pass", "skip", "fail"))
|
|
item["test_events"] = dict(counts)
|
|
item["pending_tests"] = [e["Package"] + "/" + e["Test"] for e in events if e.get("Action") == "skip" and "Test" in e]
|
|
if counts["skip"]:
|
|
summary["checks"]["go_storage_and_restart_coverage"] = {
|
|
"status": "PENDING_INFRA", "detail": "Go skip events are not proof of storage/restart behavior"}
|
|
if name == "lab4_scenarios" and (output / "lab4/summary.json").is_file():
|
|
statuses = Counter(v["status"] for v in json.loads((output / "lab4/summary.json").read_text()).values())
|
|
item["case_statuses"] = dict(statuses)
|
|
if not statuses["FAIL"] and statuses["PENDING_INFRA"]:
|
|
item["status"] = "PENDING_INFRA"
|
|
summary["checks"][name] = item
|
|
print(item["status"] + " " + name, flush=True)
|
|
summary["status"] = "PASS" if all(v["status"] == "PASS" for v in summary["checks"].values()) else "INCOMPLETE"
|
|
(output / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n")
|
|
print(summary["status"] + ": " + str(output / "summary.json"), flush=True)
|
|
return 0 if summary["status"] == "PASS" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|