585 lines
33 KiB
Python
585 lines
33 KiB
Python
#!/usr/bin/env python3
|
|
"""Portable Compose acceptance runner and explicit persistence/outage hooks."""
|
|
import argparse
|
|
from collections import Counter
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import secrets
|
|
import shlex
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
class CheckFailure(RuntimeError):
|
|
pass
|
|
|
|
|
|
class PersistenceOnlyComplete(Exception):
|
|
pass
|
|
|
|
|
|
def command(args, *, env=None, timeout=90):
|
|
result = subprocess.run(args, cwd=ROOT, env=env, capture_output=True,
|
|
text=True, timeout=timeout)
|
|
if result.returncode:
|
|
raise CheckFailure("Command failed: " + " ".join(str(x) for x in args[:3]))
|
|
return result.stdout
|
|
|
|
|
|
def compose_base(env):
|
|
project = env.get("COMPOSE_PROJECT_NAME", "")
|
|
if not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", project):
|
|
raise CheckFailure("Set COMPOSE_PROJECT_NAME explicitly to the intended test project")
|
|
files = env.get("COMPOSE_FILE", str(ROOT / "compose.yaml")).split(os.pathsep)
|
|
base = ["docker", "compose", "--project-directory", str(ROOT), "-p", project]
|
|
for file in files:
|
|
if not file:
|
|
raise CheckFailure("Empty COMPOSE_FILE entry")
|
|
path = Path(file)
|
|
if not path.is_absolute():
|
|
path = ROOT / path
|
|
base.extend(("-f", str(path.resolve())))
|
|
return base
|
|
|
|
|
|
def wait_ready(url, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(url.rstrip("/") + "/health/ready", timeout=2) as response:
|
|
if response.status == 200:
|
|
return
|
|
except (OSError, urllib.error.URLError):
|
|
pass
|
|
time.sleep(0.25)
|
|
raise CheckFailure("API did not become ready within the configured budget")
|
|
|
|
|
|
def snapshot(base, env, services):
|
|
result = {}
|
|
for service in services:
|
|
ids = command(base + ["ps", "-aq", service], env=env).split()
|
|
if not ids or (service != "api" and len(ids) != 1):
|
|
raise CheckFailure("Expected existing container(s) for " + service)
|
|
containers = []
|
|
for identifier in ids:
|
|
raw = json.loads(command(["docker", "inspect", identifier], env=env))[0]
|
|
labels = raw["Config"]["Labels"]
|
|
if labels.get("com.docker.compose.project") != env["COMPOSE_PROJECT_NAME"] or \
|
|
labels.get("com.docker.compose.service") != service:
|
|
raise CheckFailure("Container belongs to another Compose project/service")
|
|
containers.append({
|
|
"id": raw["Id"], "started_at": raw["State"]["StartedAt"],
|
|
"running": raw["State"]["Running"],
|
|
"volumes": sorted(m["Name"] for m in raw["Mounts"] if m["Type"] == "volume"),
|
|
})
|
|
containers.sort(key=lambda item: item["id"])
|
|
result[service] = containers if service == "api" else containers[0]
|
|
return result
|
|
|
|
|
|
def validate_transition(action, before, after):
|
|
if not before or set(before) != set(after):
|
|
raise CheckFailure("Missing lifecycle evidence")
|
|
for service, old_group in before.items():
|
|
new_group = after[service]
|
|
old_items = old_group if isinstance(old_group, list) else [old_group]
|
|
new_items = new_group if isinstance(new_group, list) else [new_group]
|
|
old_by_id = {item["id"]: item for item in old_items}
|
|
new_by_id = {item["id"]: item for item in new_items}
|
|
if len(old_by_id) != len(old_items) or len(new_by_id) != len(new_items) or \
|
|
set(old_by_id) != set(new_by_id):
|
|
raise CheckFailure("Lifecycle action changed container set for " + service)
|
|
for identifier, old in old_by_id.items():
|
|
new = new_by_id[identifier]
|
|
if old["volumes"] != new["volumes"]:
|
|
raise CheckFailure("Lifecycle action changed volumes for " + service)
|
|
if action in ("stop-db", "stop-cache"):
|
|
if not old["running"] or new["running"]:
|
|
raise CheckFailure("Service was not actually stopped: " + service)
|
|
elif not new["running"] or old["started_at"] == new["started_at"]:
|
|
raise CheckFailure("Service was not actually started/restarted: " + service)
|
|
|
|
|
|
def record_lifecycle(path, action, project, before, after, source=None):
|
|
validate_transition(action, before, after)
|
|
if path:
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("a") as output:
|
|
output.write(json.dumps({"action": action, "project": project,
|
|
"source": source, "before": before, "after": after}) + "\n")
|
|
|
|
|
|
def lifecycle(action, env=None):
|
|
env = dict(os.environ if env is None else env)
|
|
base = compose_base(env)
|
|
config = json.loads(command(base + ["config", "--format", "json"], env=env))
|
|
services = config["services"]
|
|
stores = env.get("HL_STATE_SERVICES", "").split()
|
|
if not stores:
|
|
stores = [name for name in ("postgres", "redis", "valkey") if name in services]
|
|
if action == "restart":
|
|
targets = ["api"] + stores + (["lb"] if "lb" in services else [])
|
|
elif "cache" in action:
|
|
targets = [name for name in ("redis", "valkey") if name in services]
|
|
if len(targets) != 1:
|
|
raise CheckFailure("Expected exactly one configured Redis/Valkey service")
|
|
else:
|
|
targets = ["postgres"]
|
|
if "postgres" not in stores or any(name not in services for name in targets):
|
|
raise CheckFailure("Configured state stores do not match Compose services")
|
|
before = snapshot(base, env, targets)
|
|
verb = {"restart": "restart", "stop-db": "stop", "start-db": "start",
|
|
"restart-cache": "restart", "stop-cache": "stop", "start-cache": "start"}[action]
|
|
args = [verb] + (["--timeout", "20"] if verb in ("restart", "stop") else []) + targets
|
|
command(base + args, env=env, timeout=65)
|
|
if not action.startswith("stop-"):
|
|
wait_ready(env.get("BASE_URL", "http://127.0.0.1:8080"), timeout=55)
|
|
after = snapshot(base, env, targets)
|
|
record_lifecycle(env.get("HL_LIFECYCLE_EVIDENCE"), action,
|
|
env["COMPOSE_PROJECT_NAME"], before, after, source="stack.py")
|
|
print("PASS: " + action + "; container state and volumes verified", flush=True)
|
|
|
|
|
|
def go_results(output):
|
|
events = []
|
|
for line in output.splitlines():
|
|
try:
|
|
event = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if isinstance(event, dict):
|
|
events.append(event)
|
|
counts = Counter(x["Action"] for x in events if x.get("Test") and "/" not in x["Test"]
|
|
and x.get("Action") in ("pass", "fail", "skip"))
|
|
skipped = any(x.get("Test") and x.get("Action") == "skip" for x in events)
|
|
if not counts["pass"] or counts["fail"] or skipped:
|
|
raise CheckFailure("Go tests failed, were skipped, or did not run")
|
|
return dict(counts)
|
|
|
|
|
|
def free_ports():
|
|
# Reserve both ports together so the two selections cannot collide.
|
|
with socket.socket() as api, socket.socket() as postgres:
|
|
api.bind(("127.0.0.1", 0))
|
|
postgres.bind(("127.0.0.1", 0))
|
|
return api.getsockname()[1], postgres.getsockname()[1]
|
|
|
|
|
|
def verify(output, pull, measure=False, operation_budget_ms=None, persistence_only=False,
|
|
lab3_only=False, lab3_scenario="all"):
|
|
if not (persistence_only or lab3_only) and importlib.util.find_spec("openapi_spec_validator") is None:
|
|
raise CheckFailure("Install test dependencies first: scripts/setup-tests.sh")
|
|
output = output.resolve()
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
project = ("lab3-test-" if lab3_only else "lab2-test-") + secrets.token_hex(6)
|
|
results = []
|
|
summary = {"project": project, "status": "FAIL", "results": results,
|
|
"scope": ("lab3-rolling-dual-diagnostic" if lab3_only and lab3_scenario == "rolling-dual" else
|
|
"lab3-idempotency-only" if lab3_only and lab3_scenario == "idempotency" else
|
|
"lab3-rolling-only" if lab3_only and lab3_scenario == "rolling" else
|
|
"lab3-level4" if lab3_only else
|
|
"persistence-only" if persistence_only else "full")}
|
|
with tempfile.TemporaryDirectory(prefix=project + "-") as directory:
|
|
tmp = Path(directory)
|
|
api_port, pg_port = free_ports()
|
|
direct_api_port = free_ports()[0]
|
|
while direct_api_port in (api_port, pg_port):
|
|
direct_api_port = free_ports()[0]
|
|
password = secrets.token_hex(24)
|
|
env = os.environ.copy()
|
|
# Do not inherit an operator's application/database/Compose settings.
|
|
for key in list(env):
|
|
if key.startswith(("COMPOSE_", "DATABASE_", "AUTH_TEST_", "MIGRATE_TEST_", "LAB2_", "HL_")):
|
|
del env[key]
|
|
env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env.get("PATH", "")
|
|
env.update(COMPOSE_PROJECT_NAME=project,
|
|
COMPOSE_FILE=os.pathsep.join((str(ROOT / "compose.yaml"), str(tmp / "compose.test.yaml"))),
|
|
BASE_URL=f"http://127.0.0.1:{api_port}",
|
|
HL_LIFECYCLE_EVIDENCE=str(output / "lifecycle.jsonl"))
|
|
env["LAB2_BASE_URL"] = env["BASE_URL"]
|
|
# Old lab2 dependency/TTL tests need an API-generated response even
|
|
# when the LB has excluded every unready backend. This port exists only
|
|
# in the disposable test project, never in the production Compose file.
|
|
env["AUTH_TEST_BASE_URL"] = f"http://127.0.0.1:{direct_api_port}"
|
|
env["AUTH_TEST_STACK_RESTART_HOOK"] = str(ROOT / "scripts/restart.sh")
|
|
env["AUTH_TEST_DIAGNOSTICS_DIR"] = str(output / "diagnostics")
|
|
if operation_budget_ms is not None:
|
|
env["AUTH_TEST_OPERATION_BUDGET_MS"] = str(operation_budget_ms)
|
|
database = f"postgres://messenger:{password}@127.0.0.1:{pg_port}/messenger?sslmode=disable"
|
|
env.update(AUTH_TEST_DATABASE_URL=database, MIGRATE_TEST_DATABASE_URL=database, LAB2_TEST_DATABASE_URL=database)
|
|
for name, value in {
|
|
"postgres_password": password,
|
|
"database_url": f"postgres://messenger:{password}@postgres:5432/messenger?sslmode=disable",
|
|
".env": (ROOT / ".env.example").read_text(),
|
|
}.items():
|
|
path = tmp / name
|
|
path.write_text(value)
|
|
path.chmod(0o600 if name == ".env" else 0o644)
|
|
override = tmp / "compose.test.yaml"
|
|
redis_port = None
|
|
|
|
def configure(ttl):
|
|
# JSON strings are valid quoted YAML scalars, including paths with spaces.
|
|
q = json.dumps
|
|
api_scale = 2 if lab3_only else 1
|
|
api_ports = (" ports: !override []" if lab3_only else
|
|
f' ports: !override\n - "127.0.0.1:{direct_api_port}:8080"')
|
|
segmented = lab3_only and lab3_scenario != "rolling"
|
|
api_networks = " networks: [front, state]\n" if segmented else ""
|
|
lb_networks = " networks: [front]\n" if segmented else ""
|
|
state_networks = " networks: [state]\n" if segmented else ""
|
|
network_definitions = "networks:\n front:\n state:\n" if segmented else ""
|
|
override.write_text(f'''services:
|
|
api:
|
|
image: {project}-api:check
|
|
scale: {api_scale}
|
|
env_file: !override
|
|
- {q(str(tmp / '.env'))}
|
|
{api_ports}
|
|
{api_networks} environment:
|
|
APP_HOST: 0.0.0.0
|
|
APP_PORT: "8080"
|
|
SESSION_TTL_SECONDS: "{ttl}"
|
|
DATABASE_URL_FILE: /run/secrets/database_url
|
|
lb:
|
|
{lb_networks} ports: !override
|
|
- "127.0.0.1:{api_port}:8080"
|
|
migrate:
|
|
{state_networks} image: {project}-migrate:check
|
|
postgres:
|
|
{state_networks} ports: !override
|
|
- "127.0.0.1:{pg_port}:5432"
|
|
secrets:
|
|
database_url:
|
|
file: {q(str(tmp / 'database_url'))}
|
|
postgres_password:
|
|
file: {q(str(tmp / 'postgres_password'))}
|
|
{network_definitions}
|
|
''')
|
|
if redis_port is not None:
|
|
text = override.read_text().replace("secrets:\n database_url:", f''' redis:
|
|
{state_networks} ports: !override
|
|
- "127.0.0.1:{redis_port}:6379"
|
|
secrets:
|
|
database_url:''')
|
|
text = text.replace(" APP_HOST: 0.0.0.0", " REDIS_ADDR: redis:6379\n SESSION_STORE: redis\n APP_HOST: 0.0.0.0")
|
|
override.write_text(text)
|
|
if measure:
|
|
text = override.read_text().replace(' image: ' + project + '-api:check', ' cpus: 2.0\n mem_limit: 1g\n image: ' + project + '-api:check')
|
|
text = text.replace(' postgres:\n ports:', ' postgres:\n cpus: 1.0\n mem_limit: 512m\n ports:')
|
|
text = text.replace(' redis:\n ports:', ' redis:\n cpus: 0.5\n mem_limit: 256m\n ports:')
|
|
override.write_text(text)
|
|
|
|
configure(3600)
|
|
lab3_config = tmp / "lab3-restart.json"
|
|
lab3_config.write_text(json.dumps({"base_url": env["BASE_URL"],
|
|
"hooks": {"restart": [str(ROOT / "scripts/restart.sh")]}}))
|
|
base = compose_base(env)
|
|
services = command(base + ["config", "--services"], env=env).split()
|
|
if "redis" in services:
|
|
redis_port = free_ports()[0]
|
|
while redis_port in (api_port, pg_port, direct_api_port):
|
|
redis_port = free_ports()[0]
|
|
configure(3600)
|
|
env["AUTH_TEST_REDIS_ADDR"] = f"127.0.0.1:{redis_port}"
|
|
restart = tmp / "restart-redis"
|
|
restart.write_text("#!/bin/sh\nexec " + shlex.join([sys.executable, str(ROOT / "student-tests/stack.py"), "restart-cache"]) + "\n")
|
|
restart.chmod(0o700)
|
|
env["AUTH_TEST_REDIS_RESTART_HOOK"] = str(restart)
|
|
summary["redis_fixture"] = "critical session store; explicit REDIS_ADDR=redis:6379 in test override"
|
|
|
|
if lab3_only:
|
|
actions = [sys.executable, str(ROOT / "student-tests/lab3/docker_actions.py")]
|
|
lab3_config.write_text(json.dumps({
|
|
"base_url": env["BASE_URL"], "api_services": ["api"], "lb_services": ["lb"],
|
|
"compose_base_argv": base, "compose_config_argv": base + ["config", "--format", "json"],
|
|
"rolling_telemetry": lab3_scenario == "rolling-dual",
|
|
"hooks": {
|
|
"restart": [str(ROOT / "scripts/restart.sh")],
|
|
"stop_one": [str(ROOT / "scripts/stop-one.sh")],
|
|
"rolling_update": [str(ROOT / "scripts/rolling-update.sh")],
|
|
**{name: actions + [name] for name in
|
|
("start_one", "recover_all", "sigterm_one", "probe_stopped",
|
|
"make_unready", "restore_ready", "probe_health",
|
|
"lb_timing_evidence", "revoke_access", "restore_access")},
|
|
},
|
|
}))
|
|
if lab3_scenario != "rolling":
|
|
env["HL_STATE_NETWORK"] = project + "_state"
|
|
if lab3_scenario in ("all", "rolling-dual"):
|
|
# The readiness fault needs front+state networks. The working
|
|
# rolling script runs on a one-network production stack and
|
|
# asks Docker for all IPs concatenated. Resolve that one
|
|
# inspect format to the actual front-network IP in this
|
|
# test-only topology; every other Docker call is unchanged.
|
|
real_docker = shutil.which("docker")
|
|
if not real_docker:
|
|
raise CheckFailure("Docker CLI unavailable for rolling IP adapter")
|
|
shim = tmp / "docker"
|
|
shim.write_text('''#!/bin/sh
|
|
set -eu
|
|
trace() {
|
|
[ "${HL_ROLLING_TRACE_ACTIVE:-}" = 1 ] || return 0
|
|
"$HL_TRACE_PYTHON" - "$HL_ROLLING_STEP_LOG" "$@" <<'PYTRACE'
|
|
import datetime, json, sys
|
|
record = {"time_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="milliseconds"),
|
|
"phase": sys.argv[2], "argv": sys.argv[3:]}
|
|
with open(sys.argv[1], "a") as output:
|
|
output.write(json.dumps(record) + "\\n")
|
|
PYTRACE
|
|
}
|
|
trace begin "$@"
|
|
if [ "$#" -eq 4 ] && [ "$1" = inspect ] && [ "$2" = --format ] &&
|
|
[ "$3" = '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ]; then
|
|
owner=$("$HL_REAL_DOCKER" inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' "$4")
|
|
[ "$owner" = "$COMPOSE_PROJECT_NAME" ] || exit 1
|
|
format="{{(index .NetworkSettings.Networks \\"$HL_FRONT_NETWORK\\").IPAddress}}"
|
|
if "$HL_REAL_DOCKER" inspect --format "$format" "$4"; then code=0; else code=$?; fi
|
|
else
|
|
if "$HL_REAL_DOCKER" "$@"; then code=0; else code=$?; fi
|
|
fi
|
|
trace end "$code" "$@"
|
|
exit "$code"
|
|
''')
|
|
shim.chmod(0o700)
|
|
env["HL_REAL_DOCKER"] = real_docker
|
|
env["HL_FRONT_NETWORK"] = project + "_front"
|
|
env["PATH"] = str(tmp) + os.pathsep + env["PATH"]
|
|
summary["rolling_ip_adapter"] = "test-only Docker inspect selects the real front-network IP"
|
|
if lab3_scenario == "rolling-dual":
|
|
env["HL_TRACE_PYTHON"] = sys.executable
|
|
env["HL_ROLLING_STEP_LOG"] = str(output / "lab3" / "rolling-docker-steps.jsonl")
|
|
|
|
def stage(name, args, timeout=180, parse_go=False, required=True, lifecycle_targets=None):
|
|
started = time.monotonic()
|
|
try:
|
|
before_lifecycle = snapshot(base, env, lifecycle_targets) if lifecycle_targets else None
|
|
process = subprocess.run(args, cwd=ROOT, env=env, capture_output=True,
|
|
text=True, timeout=timeout)
|
|
text = process.stdout + process.stderr
|
|
# Never put the generated fixture credential or DSN in artifacts.
|
|
text = text.replace(database, "<test-database-url>").replace(password, "<test-password>")
|
|
(output / (name + ".log")).write_text(text)
|
|
if process.returncode:
|
|
raise CheckFailure(name + " failed; see its sanitized test log")
|
|
if lifecycle_targets:
|
|
after_lifecycle = snapshot(base, env, lifecycle_targets)
|
|
record_lifecycle(env.get("HL_LIFECYCLE_EVIDENCE"), "restart",
|
|
project, before_lifecycle, after_lifecycle, source=name)
|
|
detail = go_results(process.stdout) if parse_go else None
|
|
results.append({"check": name, "status": "PASS", "seconds": round(time.monotonic() - started, 3),
|
|
**({"tests": detail} if detail else {})})
|
|
print("PASS: " + name, flush=True)
|
|
except subprocess.TimeoutExpired as error:
|
|
def decode(value):
|
|
return value.decode(errors="replace") if isinstance(value, bytes) else (value or "")
|
|
partial = decode(error.stdout) + decode(error.stderr)
|
|
partial = partial.replace(database, "<test-database-url>").replace(password, "<test-password>")
|
|
(output / (name + ".log")).write_text(partial)
|
|
results.append({"check": name, "status": "FAIL", "detail": "TimeoutExpired",
|
|
"seconds": round(time.monotonic() - started, 3), "runner_budget_seconds": timeout})
|
|
if required:
|
|
raise CheckFailure(name + " timed out; see partial sanitized log") from None
|
|
print("FAIL: " + name + "; partial log retained; continuing independent checks", flush=True)
|
|
except (OSError, subprocess.SubprocessError, CheckFailure) as error:
|
|
results.append({"check": name, "status": "FAIL", "detail": type(error).__name__})
|
|
if required:
|
|
raise CheckFailure(name + " failed; see test artifacts") from None
|
|
print("FAIL: " + name + "; continuing independent checks", flush=True)
|
|
|
|
try:
|
|
if not (persistence_only or lab3_only):
|
|
stage("contracts", ["make", "check", "validate", "PYTHON=" + Path(sys.executable).name])
|
|
stage("harness", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_stack.py", "-v"])
|
|
stage("lab2-harness", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests/lab2", "-p", "test_harness.py", "-v"])
|
|
if lab3_only:
|
|
# Subscribe before the clean Compose project is created. Docker
|
|
# Desktop may not retain historical events for a later query.
|
|
events_path = output / "docker-start-events.jsonl"
|
|
errors_path = output / "docker-events.stderr.log"
|
|
with events_path.open("wb") as events_stream, errors_path.open("wb") as errors_stream:
|
|
watcher = subprocess.Popen(
|
|
["docker", "events", "--filter", "type=container", "--filter", "event=start",
|
|
"--filter", "label=com.docker.compose.project=" + project,
|
|
"--format", "{{json .}}"], cwd=ROOT, env=env,
|
|
stdout=events_stream, stderr=errors_stream)
|
|
time.sleep(.5)
|
|
try:
|
|
stage("compose-up", base + ["up", "-d", "--build", "--pull", pull,
|
|
"--wait", "--wait-timeout", "90"], 300)
|
|
finally:
|
|
watcher.terminate()
|
|
watcher.wait(timeout=5)
|
|
events = []
|
|
for line in events_path.read_text().splitlines():
|
|
try:
|
|
events.append(json.loads(line))
|
|
except ValueError:
|
|
pass
|
|
migrate_starts = sum(
|
|
(event.get("Actor") or {}).get("Attributes", {}).get("com.docker.compose.service") == "migrate"
|
|
and event.get("Action") == "start" for event in events)
|
|
migrate_ids = command(base + ["ps", "-aq", "migrate"], env=env).split()
|
|
api_ids = command(base + ["ps", "-q", "api"], env=env).split()
|
|
if len(migrate_ids) == 1 and len(api_ids) == 2:
|
|
migrator = json.loads(command(["docker", "inspect", migrate_ids[0]], env=env))[0]
|
|
apis = [json.loads(command(["docker", "inspect", identifier], env=env))[0]
|
|
for identifier in api_ids]
|
|
finished = migrator["State"]["FinishedAt"]
|
|
migration = {
|
|
"project": project, "migration_starts": migrate_starts,
|
|
"migration_exit_code": migrator["State"]["ExitCode"],
|
|
"migration_container_id": migrator["Id"],
|
|
"migration_finished_at": finished,
|
|
"api_containers": [{"id": item["Id"], "started_at": item["State"]["StartedAt"]}
|
|
for item in apis],
|
|
"api_started_before_migration_complete": any(
|
|
item["State"]["StartedAt"] < finished for item in apis),
|
|
"recorded_start_events": len(events),
|
|
}
|
|
evidence_path = output / "migration-evidence.json"
|
|
evidence_path.write_text(json.dumps(migration, indent=2) + "\n")
|
|
if events:
|
|
config_value = json.loads(lab3_config.read_text())
|
|
config_value["hooks"]["migration_evidence"] = actions + ["migration_evidence"]
|
|
lab3_config.write_text(json.dumps(config_value))
|
|
env["HL_MIGRATION_EVIDENCE"] = str(evidence_path)
|
|
else:
|
|
stage("compose-up", base + ["up", "-d", "--build", "--pull", pull,
|
|
"--wait", "--wait-timeout", "90"], 300)
|
|
wait_ready(env["BASE_URL"])
|
|
if lab3_only:
|
|
stage("lab3-" + lab3_scenario if lab3_scenario != "all" else "lab3-level4",
|
|
[sys.executable, "student-tests/lab3/run.py",
|
|
"--config", str(lab3_config),
|
|
"--output", str(output / "lab3"),
|
|
"--scenario", "rolling" if lab3_scenario == "rolling-dual" else lab3_scenario],
|
|
timeout=420, required=False)
|
|
summary["status"] = "FAIL" if any(x["status"] == "FAIL" for x in results) else "PASS"
|
|
raise PersistenceOnlyComplete
|
|
if persistence_only:
|
|
stage("teacher-persistence", ["make", "test-persistence", "BASE_URL=" + env["BASE_URL"],
|
|
"PYTHON=" + Path(sys.executable).name,
|
|
"ACTION_SCRIPT=" + str(ROOT / "scripts/restart.sh")],
|
|
required=False, lifecycle_targets=["api", "postgres", "redis", "lb"])
|
|
stage("lab3-restart-idempotency", [sys.executable, "student-tests/lab3/run.py",
|
|
"--config", str(lab3_config),
|
|
"--output", str(output / "lab3-restart"),
|
|
"--scenario", "restart"],
|
|
required=False, lifecycle_targets=["api", "postgres", "redis", "lb"])
|
|
summary["status"] = "FAIL" if any(x["status"] == "FAIL" for x in results) else "PASS"
|
|
raise PersistenceOnlyComplete
|
|
# Compile race-instrumented packages separately. Cold compilation is
|
|
# not an application-operation latency test and must not consume
|
|
# the execution budget of the actual integration tests.
|
|
stage("go-race-build", ["go", "test", "-race", "-run", "^$", "./..."], timeout=300)
|
|
stage("go-race", ["go", "test", "-race", "./...", "-count=1", "-json"], parse_go=True, timeout=600, required=False)
|
|
stage("go-build", ["go", "build", "./..."], required=False)
|
|
stage("go-vet", ["go", "vet", "./..."], required=False)
|
|
stage("teacher-smoke", ["make", "test", "BASE_URL=" + env["BASE_URL"], "PYTHON=" + Path(sys.executable).name], required=False)
|
|
auth = [sys.executable, "student-tests/auth_contract.py", "--base-url", env["AUTH_TEST_BASE_URL"], "--ttl", "3600"]
|
|
if redis_port is not None:
|
|
auth.extend(("--session-store", "redis"))
|
|
stage("auth-normal", auth, required=False)
|
|
stage("business", [sys.executable, "student-tests/lab2/run.py", "--suite", "business-boundaries", "--suite", "business-cursors", "--suite", "read-after-write", "--output", str(output / "business.json")], required=False)
|
|
stage("sql-plan-10000", [sys.executable, "student-tests/lab2/run.py", "--suite", "sql-plan", "--output", str(output / "sql-plan.json")], timeout=180, required=False)
|
|
if measure:
|
|
resources = {}
|
|
for role, cpus in (("api", 2.0), ("postgres", 1.0), ("redis", 0.5)):
|
|
identifier = command(base + ["ps", "-q", role], env=env).strip()
|
|
resources[role] = {"name": identifier, "cpus": cpus, "memory_limit_mib": {"api": 1024, "postgres": 512, "redis": 256}[role]}
|
|
env["LAB2_RESOURCE_CONTAINERS"] = json.dumps(resources)
|
|
stage("latency-matrix", [sys.executable, "student-tests/lab2/budgets.py", "--base-url", env["BASE_URL"], "--output", str(output / "latency")], timeout=300, required=False)
|
|
stage("teacher-persistence", ["make", "test-persistence", "BASE_URL=" + env["BASE_URL"],
|
|
"PYTHON=" + Path(sys.executable).name, "ACTION_SCRIPT=" + str(ROOT / "scripts/restart.sh")],
|
|
required=False, lifecycle_targets=["api", "postgres", "redis", "lb"])
|
|
stage("lab3-restart-idempotency", [sys.executable, "student-tests/lab3/run.py",
|
|
"--config", str(lab3_config),
|
|
"--output", str(output / "lab3-restart"),
|
|
"--scenario", "restart"],
|
|
required=False, lifecycle_targets=["api", "postgres", "redis", "lb"])
|
|
stage("auth-persistence", auth + ["--suite", "persistence", "--restart-hook", str(ROOT / "scripts/restart.sh")],
|
|
required=False, lifecycle_targets=["api", "postgres", "redis", "lb"])
|
|
stage("postgres-outage", auth + ["--suite", "dependency", "--dependency", "postgres", "--timeout", "5",
|
|
"--failure-hook", str(ROOT / "scripts/stop-postgres.sh"),
|
|
"--recovery-hook", str(ROOT / "scripts/start-postgres.sh")], required=False)
|
|
if redis_port is not None:
|
|
stage("redis-outage", auth + ["--suite", "dependency", "--dependency", "redis", "--cache-policy", "critical", "--timeout", "5",
|
|
"--failure-hook", str(ROOT / "scripts/stop-redis.sh"),
|
|
"--recovery-hook", str(ROOT / "scripts/start-redis.sh")], required=False)
|
|
configure(2)
|
|
stage("short-ttl-api", base + ["up", "-d", "--no-build", "--no-deps", "api"])
|
|
wait_ready(env["BASE_URL"])
|
|
stage("auth-expiry", [sys.executable, "student-tests/auth_contract.py", "--base-url", env["AUTH_TEST_BASE_URL"], "--suite", "expiry", "--ttl", "2"], required=False)
|
|
summary["status"] = "FAIL" if any(x["status"] == "FAIL" for x in results) else "PASS"
|
|
except PersistenceOnlyComplete:
|
|
pass
|
|
except (OSError, subprocess.SubprocessError, CheckFailure) as error:
|
|
summary["failure"] = str(error) if isinstance(error, CheckFailure) else type(error).__name__
|
|
finally:
|
|
# Only this newly generated project's own resources may be removed.
|
|
try:
|
|
stage("cleanup", base + ["down", "--volumes", "--remove-orphans", "--timeout", "20"], 90)
|
|
containers = command(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + project], env=env)
|
|
volumes = command(["docker", "volume", "ls", "-q", "--filter", "label=com.docker.compose.project=" + project], env=env)
|
|
if containers.strip() or volumes.strip():
|
|
raise CheckFailure("Disposable project resources remain after cleanup")
|
|
except (OSError, subprocess.SubprocessError, CheckFailure):
|
|
summary["status"] = "FAIL"
|
|
summary["cleanup_failed"] = True
|
|
(output / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
print(summary["status"] + ": " + str(output / "summary.json"), flush=True)
|
|
return 0 if summary["status"] == "PASS" else 1
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("action", choices=("verify", "restart", "stop-db", "start-db", "restart-cache", "stop-cache", "start-cache"))
|
|
parser.add_argument("--output", type=Path, default=ROOT / "student-tests/output" / time.strftime("%Y%m%d-%H%M%S"))
|
|
parser.add_argument("--measure-budgets", action="store_true", help="bounded 1/5/10 HTTP matrix with explicit container limits")
|
|
parser.add_argument("--persistence-only", action="store_true", help="same isolated Compose project, only public persistence and lab3 restart replay")
|
|
parser.add_argument("--lab3-only", action="store_true", help="disposable two-replica lab3/LB level-4 run; saves evidence and cleans only its own project")
|
|
parser.add_argument("--lab3-scenario", choices=("all", "rolling", "rolling-dual", "idempotency"), default="all",
|
|
help="with --lab3-only: run one focused scenario in a fresh project, or the full suite")
|
|
parser.add_argument("--operation-budget-ms", type=int, help="explicitly agreed operation budget; omitted keeps provisional 3000ms")
|
|
parser.add_argument("--pull", choices=("missing", "never", "always"), default="missing")
|
|
args = parser.parse_args()
|
|
if args.persistence_only and args.lab3_only:
|
|
parser.error("choose either --persistence-only or --lab3-only")
|
|
if args.lab3_scenario != "all" and not args.lab3_only:
|
|
parser.error("--lab3-scenario requires --lab3-only")
|
|
if args.operation_budget_ms is not None and not 500 <= args.operation_budget_ms <= 10000:
|
|
parser.error("operation budget must be 500..10000 ms")
|
|
if args.action == "verify":
|
|
# Normal CI cancellation must run the same finally cleanup as failures.
|
|
def interrupted(_number, _frame):
|
|
raise CheckFailure("Test run interrupted")
|
|
signal.signal(signal.SIGTERM, interrupted)
|
|
signal.signal(signal.SIGINT, interrupted)
|
|
return verify(args.output, args.pull, args.measure_budgets, args.operation_budget_ms,
|
|
args.persistence_only, args.lab3_only, args.lab3_scenario)
|
|
lifecycle(args.action)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
raise SystemExit(main())
|
|
except (OSError, ValueError, KeyError, subprocess.SubprocessError, CheckFailure) as error:
|
|
print("FAIL: " + (str(error) if isinstance(error, CheckFailure) else type(error).__name__), file=sys.stderr)
|
|
raise SystemExit(1)
|