183 lines
7.9 KiB
Python
183 lines
7.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Run lab5 checks from Windows or Linux with only host Python and Docker.
|
|
|
|
Start the Compose stand first. Run ``python scripts/test_lab5.py --quick``
|
|
for image/API checks, or omit --quick to include deliberate service failures.
|
|
Commands run from the repository root regardless of the current directory.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import os
|
|
from pathlib import Path
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
EVIDENCE = ROOT / "evidence"
|
|
COMPOSE = ["docker", "compose", "--profile", "tools"]
|
|
SERVICES = ("db", "valkey", "s3", "broker", "api", "lb", "publisher", "worker")
|
|
|
|
|
|
class CheckFailed(Exception):
|
|
"""A check or a Docker command returned a nonzero exit code."""
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
|
|
|
|
|
|
def command_text(command: list[str]) -> str:
|
|
return subprocess.list2cmdline(command) if os.name == "nt" else shlex.join(command)
|
|
|
|
|
|
def run(command: list[str], evidence: str, *, append: bool = False,
|
|
stdin_file: Path | None = None) -> str:
|
|
"""Stream combined stdout/stderr to the terminal and a UTF-8 evidence log."""
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
with (EVIDENCE / evidence).open("a" if append else "w", encoding="utf-8", newline="\n") as log:
|
|
def write(value: str) -> None:
|
|
print(value, end="", flush=True)
|
|
log.write(value)
|
|
log.flush()
|
|
|
|
display = command_text(command)
|
|
if stdin_file is not None:
|
|
display += " < " + stdin_file.relative_to(ROOT).as_posix()
|
|
write(f"\nUTC: {utc_now()}\nCommand: {display}\n")
|
|
output: list[str] = []
|
|
try:
|
|
# Pass an actual file as stdin: no shell redirection, quoting or
|
|
# PowerShell encoding conversions are involved.
|
|
with stdin_file.open("rb") if stdin_file else open(os.devnull, "rb") as source:
|
|
process = subprocess.Popen(
|
|
command, cwd=ROOT, stdin=source, stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace",
|
|
)
|
|
try:
|
|
assert process.stdout is not None
|
|
for line in process.stdout:
|
|
write(line)
|
|
output.append(line)
|
|
code = process.wait()
|
|
except BaseException:
|
|
process.terminate()
|
|
try:
|
|
process.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
process.wait()
|
|
raise
|
|
finally:
|
|
if process.stdout is not None:
|
|
process.stdout.close()
|
|
except (OSError, KeyboardInterrupt) as error:
|
|
write(f"\nRESULT: ERROR ({type(error).__name__}) at {utc_now()}\n")
|
|
raise
|
|
write(f"\nRESULT: {'PASS' if code == 0 else 'FAIL'} (exit {code}) at {utc_now()}\n")
|
|
if code:
|
|
raise CheckFailed(f"{display}: exit {code}; see evidence/{evidence}")
|
|
return "".join(output)
|
|
|
|
|
|
def tester(*command: str) -> list[str]:
|
|
return [*COMPOSE, "run", "--rm", "-T", "--no-deps", "tester", *command]
|
|
|
|
|
|
def snapshot(stage: str, *, append: bool = True) -> None:
|
|
"""Record hook scope without dumping container environment or secrets."""
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
with (EVIDENCE / "container-events.txt").open("a" if append else "w", encoding="utf-8") as log:
|
|
log.write(f"\n{stage} — {utc_now()}\n")
|
|
identifiers = run([*COMPOSE, "ps", "-a", "-q"], "container-events.txt", append=True).split()
|
|
if identifiers:
|
|
run(["docker", "inspect", "--format",
|
|
"{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}",
|
|
*identifiers], "container-events.txt", append=True)
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--quick", action="store_true",
|
|
help="skip recovery, full-stack restart and API failover checks")
|
|
args = parser.parse_args()
|
|
for stream in (sys.stdout, sys.stderr):
|
|
if hasattr(stream, "reconfigure"):
|
|
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
|
|
destructive_started = False
|
|
complete = False
|
|
exit_code = 0
|
|
try:
|
|
running = run([*COMPOSE, "ps", "--status", "running", "--services"], "runner-setup.txt")
|
|
missing = set(SERVICES) - set(running.split())
|
|
if missing:
|
|
raise CheckFailed(
|
|
"Start the Compose stand first: missing running services "
|
|
+ ", ".join(sorted(missing))
|
|
+ ". Run docker compose up -d --build --wait --wait-timeout 180."
|
|
)
|
|
run([*COMPOSE, "ps", "-a"], "stand.txt")
|
|
run([*COMPOSE, "build", "tester"], "runner-setup.txt", append=True)
|
|
run(tester("python3", "-u", "scripts/check_contract.py"), "check.txt")
|
|
run(tester("python3", "-u", "tests/smoke.py"), "smoke.txt")
|
|
run(tester("python3", "-u", "student-tests/image_validation_unit.py"), "image-validation.txt")
|
|
run(tester("python3", "-u", "student-tests/lab5_images.py"), "images.txt")
|
|
run([*COMPOSE, "exec", "-T", "worker", "python", "-u", "-"], "limits.txt",
|
|
stdin_file=ROOT / "student-tests" / "lab5_limits.py")
|
|
|
|
if not args.quick:
|
|
# Git clones on Linux may lack executable bits on newly added
|
|
# hooks; the public test explicitly requires executable scripts.
|
|
run(tester("chmod", "+x", "scripts/restart.sh", "scripts/stop-one.sh"),
|
|
"runner-setup.txt", append=True)
|
|
destructive_started = True
|
|
run(tester("python3", "-u", "student-tests/lab5_recovery.py"), "recovery.txt")
|
|
snapshot("Before public persistence", append=False)
|
|
try:
|
|
run(tester("python3", "-u", "tests/resilience.py", "--mode", "persistence",
|
|
"--action-script", "scripts/restart.sh"), "persistence.txt")
|
|
finally:
|
|
snapshot("After public persistence")
|
|
snapshot("Before public failover")
|
|
try:
|
|
run(tester("python3", "-u", "tests/resilience.py", "--mode", "failover",
|
|
"--action-script", "scripts/stop-one.sh"), "failover.txt")
|
|
finally:
|
|
# Failover deliberately leaves an API replica stopped.
|
|
# Restore it after success as well as after a failed check.
|
|
try:
|
|
snapshot("After public failover, before API restoration")
|
|
finally:
|
|
run(tester("sh", "scripts/start-api.sh"), "failover.txt", append=True)
|
|
snapshot("After API restoration")
|
|
run([*COMPOSE, "ps", "-a"], "stand.txt", append=True)
|
|
complete = True
|
|
except KeyboardInterrupt:
|
|
print("\nInterrupted. Completed command logs remain in evidence/.", file=sys.stderr)
|
|
exit_code = 130
|
|
except (CheckFailed, OSError) as error:
|
|
print(f"\nFAIL: {error}", file=sys.stderr)
|
|
exit_code = 1
|
|
finally:
|
|
if destructive_started and not complete:
|
|
try:
|
|
# Preserve all volumes. Compose may also restart the
|
|
# idempotent migration/bootstrap dependencies.
|
|
run([*COMPOSE, "start", *SERVICES], "runner-setup.txt", append=True)
|
|
except (CheckFailed, OSError, KeyboardInterrupt) as error:
|
|
print(f"Service restoration failed: {error}", file=sys.stderr)
|
|
exit_code = exit_code or 1
|
|
|
|
if exit_code == 0:
|
|
mode = "quick" if args.quick else "full"
|
|
print(f"\nPASS: lab5 {mode} checks. UTF-8 logs: {EVIDENCE}")
|
|
return exit_code
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|