Implement lab 3 level 4 failover and idempotency

This commit is contained in:
2026-10-29 23:35:04 +03:00
parent 0ab3a341aa
commit a2cb1ad2ae
338 changed files with 37821 additions and 22 deletions
+584
View File
@@ -0,0 +1,584 @@
#!/usr/bin/env python3
"""Portable Compose acceptance runner and explicit persistence/outage hooks."""
import argparse
from collections import Counter
import importlib.util
import json
import os
from pathlib import Path
import re
import secrets
import shlex
import shutil
import signal
import socket
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
class CheckFailure(RuntimeError):
pass
class PersistenceOnlyComplete(Exception):
pass
def command(args, *, env=None, timeout=90):
result = subprocess.run(args, cwd=ROOT, env=env, capture_output=True,
text=True, timeout=timeout)
if result.returncode:
raise CheckFailure("Command failed: " + " ".join(str(x) for x in args[:3]))
return result.stdout
def compose_base(env):
project = env.get("COMPOSE_PROJECT_NAME", "")
if not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", project):
raise CheckFailure("Set COMPOSE_PROJECT_NAME explicitly to the intended test project")
files = env.get("COMPOSE_FILE", str(ROOT / "compose.yaml")).split(os.pathsep)
base = ["docker", "compose", "--project-directory", str(ROOT), "-p", project]
for file in files:
if not file:
raise CheckFailure("Empty COMPOSE_FILE entry")
path = Path(file)
if not path.is_absolute():
path = ROOT / path
base.extend(("-f", str(path.resolve())))
return base
def wait_ready(url, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(url.rstrip("/") + "/health/ready", timeout=2) as response:
if response.status == 200:
return
except (OSError, urllib.error.URLError):
pass
time.sleep(0.25)
raise CheckFailure("API did not become ready within the configured budget")
def snapshot(base, env, services):
result = {}
for service in services:
ids = command(base + ["ps", "-aq", service], env=env).split()
if not ids or (service != "api" and len(ids) != 1):
raise CheckFailure("Expected existing container(s) for " + service)
containers = []
for identifier in ids:
raw = json.loads(command(["docker", "inspect", identifier], env=env))[0]
labels = raw["Config"]["Labels"]
if labels.get("com.docker.compose.project") != env["COMPOSE_PROJECT_NAME"] or \
labels.get("com.docker.compose.service") != service:
raise CheckFailure("Container belongs to another Compose project/service")
containers.append({
"id": raw["Id"], "started_at": raw["State"]["StartedAt"],
"running": raw["State"]["Running"],
"volumes": sorted(m["Name"] for m in raw["Mounts"] if m["Type"] == "volume"),
})
containers.sort(key=lambda item: item["id"])
result[service] = containers if service == "api" else containers[0]
return result
def validate_transition(action, before, after):
if not before or set(before) != set(after):
raise CheckFailure("Missing lifecycle evidence")
for service, old_group in before.items():
new_group = after[service]
old_items = old_group if isinstance(old_group, list) else [old_group]
new_items = new_group if isinstance(new_group, list) else [new_group]
old_by_id = {item["id"]: item for item in old_items}
new_by_id = {item["id"]: item for item in new_items}
if len(old_by_id) != len(old_items) or len(new_by_id) != len(new_items) or \
set(old_by_id) != set(new_by_id):
raise CheckFailure("Lifecycle action changed container set for " + service)
for identifier, old in old_by_id.items():
new = new_by_id[identifier]
if old["volumes"] != new["volumes"]:
raise CheckFailure("Lifecycle action changed volumes for " + service)
if action in ("stop-db", "stop-cache"):
if not old["running"] or new["running"]:
raise CheckFailure("Service was not actually stopped: " + service)
elif not new["running"] or old["started_at"] == new["started_at"]:
raise CheckFailure("Service was not actually started/restarted: " + service)
def record_lifecycle(path, action, project, before, after, source=None):
validate_transition(action, before, after)
if path:
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("a") as output:
output.write(json.dumps({"action": action, "project": project,
"source": source, "before": before, "after": after}) + "\n")
def lifecycle(action, env=None):
env = dict(os.environ if env is None else env)
base = compose_base(env)
config = json.loads(command(base + ["config", "--format", "json"], env=env))
services = config["services"]
stores = env.get("HL_STATE_SERVICES", "").split()
if not stores:
stores = [name for name in ("postgres", "redis", "valkey") if name in services]
if action == "restart":
targets = ["api"] + stores + (["lb"] if "lb" in services else [])
elif "cache" in action:
targets = [name for name in ("redis", "valkey") if name in services]
if len(targets) != 1:
raise CheckFailure("Expected exactly one configured Redis/Valkey service")
else:
targets = ["postgres"]
if "postgres" not in stores or any(name not in services for name in targets):
raise CheckFailure("Configured state stores do not match Compose services")
before = snapshot(base, env, targets)
verb = {"restart": "restart", "stop-db": "stop", "start-db": "start",
"restart-cache": "restart", "stop-cache": "stop", "start-cache": "start"}[action]
args = [verb] + (["--timeout", "20"] if verb in ("restart", "stop") else []) + targets
command(base + args, env=env, timeout=65)
if not action.startswith("stop-"):
wait_ready(env.get("BASE_URL", "http://127.0.0.1:8080"), timeout=55)
after = snapshot(base, env, targets)
record_lifecycle(env.get("HL_LIFECYCLE_EVIDENCE"), action,
env["COMPOSE_PROJECT_NAME"], before, after, source="stack.py")
print("PASS: " + action + "; container state and volumes verified", flush=True)
def go_results(output):
events = []
for line in output.splitlines():
try:
event = json.loads(line)
except ValueError:
continue
if isinstance(event, dict):
events.append(event)
counts = Counter(x["Action"] for x in events if x.get("Test") and "/" not in x["Test"]
and x.get("Action") in ("pass", "fail", "skip"))
skipped = any(x.get("Test") and x.get("Action") == "skip" for x in events)
if not counts["pass"] or counts["fail"] or skipped:
raise CheckFailure("Go tests failed, were skipped, or did not run")
return dict(counts)
def free_ports():
# Reserve both ports together so the two selections cannot collide.
with socket.socket() as api, socket.socket() as postgres:
api.bind(("127.0.0.1", 0))
postgres.bind(("127.0.0.1", 0))
return api.getsockname()[1], postgres.getsockname()[1]
def verify(output, pull, measure=False, operation_budget_ms=None, persistence_only=False,
lab3_only=False, lab3_scenario="all"):
if not (persistence_only or lab3_only) and importlib.util.find_spec("openapi_spec_validator") is None:
raise CheckFailure("Install test dependencies first: scripts/setup-tests.sh")
output = output.resolve()
output.mkdir(parents=True, exist_ok=True)
project = ("lab3-test-" if lab3_only else "lab2-test-") + secrets.token_hex(6)
results = []
summary = {"project": project, "status": "FAIL", "results": results,
"scope": ("lab3-rolling-dual-diagnostic" if lab3_only and lab3_scenario == "rolling-dual" else
"lab3-idempotency-only" if lab3_only and lab3_scenario == "idempotency" else
"lab3-rolling-only" if lab3_only and lab3_scenario == "rolling" else
"lab3-level4" if lab3_only else
"persistence-only" if persistence_only else "full")}
with tempfile.TemporaryDirectory(prefix=project + "-") as directory:
tmp = Path(directory)
api_port, pg_port = free_ports()
direct_api_port = free_ports()[0]
while direct_api_port in (api_port, pg_port):
direct_api_port = free_ports()[0]
password = secrets.token_hex(24)
env = os.environ.copy()
# Do not inherit an operator's application/database/Compose settings.
for key in list(env):
if key.startswith(("COMPOSE_", "DATABASE_", "AUTH_TEST_", "MIGRATE_TEST_", "LAB2_", "HL_")):
del env[key]
env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env.get("PATH", "")
env.update(COMPOSE_PROJECT_NAME=project,
COMPOSE_FILE=os.pathsep.join((str(ROOT / "compose.yaml"), str(tmp / "compose.test.yaml"))),
BASE_URL=f"http://127.0.0.1:{api_port}",
HL_LIFECYCLE_EVIDENCE=str(output / "lifecycle.jsonl"))
env["LAB2_BASE_URL"] = env["BASE_URL"]
# Old lab2 dependency/TTL tests need an API-generated response even
# when the LB has excluded every unready backend. This port exists only
# in the disposable test project, never in the production Compose file.
env["AUTH_TEST_BASE_URL"] = f"http://127.0.0.1:{direct_api_port}"
env["AUTH_TEST_STACK_RESTART_HOOK"] = str(ROOT / "scripts/restart.sh")
env["AUTH_TEST_DIAGNOSTICS_DIR"] = str(output / "diagnostics")
if operation_budget_ms is not None:
env["AUTH_TEST_OPERATION_BUDGET_MS"] = str(operation_budget_ms)
database = f"postgres://messenger:{password}@127.0.0.1:{pg_port}/messenger?sslmode=disable"
env.update(AUTH_TEST_DATABASE_URL=database, MIGRATE_TEST_DATABASE_URL=database, LAB2_TEST_DATABASE_URL=database)
for name, value in {
"postgres_password": password,
"database_url": f"postgres://messenger:{password}@postgres:5432/messenger?sslmode=disable",
".env": (ROOT / ".env.example").read_text(),
}.items():
path = tmp / name
path.write_text(value)
path.chmod(0o600 if name == ".env" else 0o644)
override = tmp / "compose.test.yaml"
redis_port = None
def configure(ttl):
# JSON strings are valid quoted YAML scalars, including paths with spaces.
q = json.dumps
api_scale = 2 if lab3_only else 1
api_ports = (" ports: !override []" if lab3_only else
f' ports: !override\n - "127.0.0.1:{direct_api_port}:8080"')
segmented = lab3_only and lab3_scenario != "rolling"
api_networks = " networks: [front, state]\n" if segmented else ""
lb_networks = " networks: [front]\n" if segmented else ""
state_networks = " networks: [state]\n" if segmented else ""
network_definitions = "networks:\n front:\n state:\n" if segmented else ""
override.write_text(f'''services:
api:
image: {project}-api:check
scale: {api_scale}
env_file: !override
- {q(str(tmp / '.env'))}
{api_ports}
{api_networks} environment:
APP_HOST: 0.0.0.0
APP_PORT: "8080"
SESSION_TTL_SECONDS: "{ttl}"
DATABASE_URL_FILE: /run/secrets/database_url
lb:
{lb_networks} ports: !override
- "127.0.0.1:{api_port}:8080"
migrate:
{state_networks} image: {project}-migrate:check
postgres:
{state_networks} ports: !override
- "127.0.0.1:{pg_port}:5432"
secrets:
database_url:
file: {q(str(tmp / 'database_url'))}
postgres_password:
file: {q(str(tmp / 'postgres_password'))}
{network_definitions}
''')
if redis_port is not None:
text = override.read_text().replace("secrets:\n database_url:", f''' redis:
{state_networks} ports: !override
- "127.0.0.1:{redis_port}:6379"
secrets:
database_url:''')
text = text.replace(" APP_HOST: 0.0.0.0", " REDIS_ADDR: redis:6379\n SESSION_STORE: redis\n APP_HOST: 0.0.0.0")
override.write_text(text)
if measure:
text = override.read_text().replace(' image: ' + project + '-api:check', ' cpus: 2.0\n mem_limit: 1g\n image: ' + project + '-api:check')
text = text.replace(' postgres:\n ports:', ' postgres:\n cpus: 1.0\n mem_limit: 512m\n ports:')
text = text.replace(' redis:\n ports:', ' redis:\n cpus: 0.5\n mem_limit: 256m\n ports:')
override.write_text(text)
configure(3600)
lab3_config = tmp / "lab3-restart.json"
lab3_config.write_text(json.dumps({"base_url": env["BASE_URL"],
"hooks": {"restart": [str(ROOT / "scripts/restart.sh")]}}))
base = compose_base(env)
services = command(base + ["config", "--services"], env=env).split()
if "redis" in services:
redis_port = free_ports()[0]
while redis_port in (api_port, pg_port, direct_api_port):
redis_port = free_ports()[0]
configure(3600)
env["AUTH_TEST_REDIS_ADDR"] = f"127.0.0.1:{redis_port}"
restart = tmp / "restart-redis"
restart.write_text("#!/bin/sh\nexec " + shlex.join([sys.executable, str(ROOT / "student-tests/stack.py"), "restart-cache"]) + "\n")
restart.chmod(0o700)
env["AUTH_TEST_REDIS_RESTART_HOOK"] = str(restart)
summary["redis_fixture"] = "critical session store; explicit REDIS_ADDR=redis:6379 in test override"
if lab3_only:
actions = [sys.executable, str(ROOT / "student-tests/lab3/docker_actions.py")]
lab3_config.write_text(json.dumps({
"base_url": env["BASE_URL"], "api_services": ["api"], "lb_services": ["lb"],
"compose_base_argv": base, "compose_config_argv": base + ["config", "--format", "json"],
"rolling_telemetry": lab3_scenario == "rolling-dual",
"hooks": {
"restart": [str(ROOT / "scripts/restart.sh")],
"stop_one": [str(ROOT / "scripts/stop-one.sh")],
"rolling_update": [str(ROOT / "scripts/rolling-update.sh")],
**{name: actions + [name] for name in
("start_one", "recover_all", "sigterm_one", "probe_stopped",
"make_unready", "restore_ready", "probe_health",
"lb_timing_evidence", "revoke_access", "restore_access")},
},
}))
if lab3_scenario != "rolling":
env["HL_STATE_NETWORK"] = project + "_state"
if lab3_scenario in ("all", "rolling-dual"):
# The readiness fault needs front+state networks. The working
# rolling script runs on a one-network production stack and
# asks Docker for all IPs concatenated. Resolve that one
# inspect format to the actual front-network IP in this
# test-only topology; every other Docker call is unchanged.
real_docker = shutil.which("docker")
if not real_docker:
raise CheckFailure("Docker CLI unavailable for rolling IP adapter")
shim = tmp / "docker"
shim.write_text('''#!/bin/sh
set -eu
trace() {
[ "${HL_ROLLING_TRACE_ACTIVE:-}" = 1 ] || return 0
"$HL_TRACE_PYTHON" - "$HL_ROLLING_STEP_LOG" "$@" <<'PYTRACE'
import datetime, json, sys
record = {"time_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="milliseconds"),
"phase": sys.argv[2], "argv": sys.argv[3:]}
with open(sys.argv[1], "a") as output:
output.write(json.dumps(record) + "\\n")
PYTRACE
}
trace begin "$@"
if [ "$#" -eq 4 ] && [ "$1" = inspect ] && [ "$2" = --format ] &&
[ "$3" = '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ]; then
owner=$("$HL_REAL_DOCKER" inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' "$4")
[ "$owner" = "$COMPOSE_PROJECT_NAME" ] || exit 1
format="{{(index .NetworkSettings.Networks \\"$HL_FRONT_NETWORK\\").IPAddress}}"
if "$HL_REAL_DOCKER" inspect --format "$format" "$4"; then code=0; else code=$?; fi
else
if "$HL_REAL_DOCKER" "$@"; then code=0; else code=$?; fi
fi
trace end "$code" "$@"
exit "$code"
''')
shim.chmod(0o700)
env["HL_REAL_DOCKER"] = real_docker
env["HL_FRONT_NETWORK"] = project + "_front"
env["PATH"] = str(tmp) + os.pathsep + env["PATH"]
summary["rolling_ip_adapter"] = "test-only Docker inspect selects the real front-network IP"
if lab3_scenario == "rolling-dual":
env["HL_TRACE_PYTHON"] = sys.executable
env["HL_ROLLING_STEP_LOG"] = str(output / "lab3" / "rolling-docker-steps.jsonl")
def stage(name, args, timeout=180, parse_go=False, required=True, lifecycle_targets=None):
started = time.monotonic()
try:
before_lifecycle = snapshot(base, env, lifecycle_targets) if lifecycle_targets else None
process = subprocess.run(args, cwd=ROOT, env=env, capture_output=True,
text=True, timeout=timeout)
text = process.stdout + process.stderr
# Never put the generated fixture credential or DSN in artifacts.
text = text.replace(database, "<test-database-url>").replace(password, "<test-password>")
(output / (name + ".log")).write_text(text)
if process.returncode:
raise CheckFailure(name + " failed; see its sanitized test log")
if lifecycle_targets:
after_lifecycle = snapshot(base, env, lifecycle_targets)
record_lifecycle(env.get("HL_LIFECYCLE_EVIDENCE"), "restart",
project, before_lifecycle, after_lifecycle, source=name)
detail = go_results(process.stdout) if parse_go else None
results.append({"check": name, "status": "PASS", "seconds": round(time.monotonic() - started, 3),
**({"tests": detail} if detail else {})})
print("PASS: " + name, flush=True)
except subprocess.TimeoutExpired as error:
def decode(value):
return value.decode(errors="replace") if isinstance(value, bytes) else (value or "")
partial = decode(error.stdout) + decode(error.stderr)
partial = partial.replace(database, "<test-database-url>").replace(password, "<test-password>")
(output / (name + ".log")).write_text(partial)
results.append({"check": name, "status": "FAIL", "detail": "TimeoutExpired",
"seconds": round(time.monotonic() - started, 3), "runner_budget_seconds": timeout})
if required:
raise CheckFailure(name + " timed out; see partial sanitized log") from None
print("FAIL: " + name + "; partial log retained; continuing independent checks", flush=True)
except (OSError, subprocess.SubprocessError, CheckFailure) as error:
results.append({"check": name, "status": "FAIL", "detail": type(error).__name__})
if required:
raise CheckFailure(name + " failed; see test artifacts") from None
print("FAIL: " + name + "; continuing independent checks", flush=True)
try:
if not (persistence_only or lab3_only):
stage("contracts", ["make", "check", "validate", "PYTHON=" + Path(sys.executable).name])
stage("harness", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_stack.py", "-v"])
stage("lab2-harness", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests/lab2", "-p", "test_harness.py", "-v"])
if lab3_only:
# Subscribe before the clean Compose project is created. Docker
# Desktop may not retain historical events for a later query.
events_path = output / "docker-start-events.jsonl"
errors_path = output / "docker-events.stderr.log"
with events_path.open("wb") as events_stream, errors_path.open("wb") as errors_stream:
watcher = subprocess.Popen(
["docker", "events", "--filter", "type=container", "--filter", "event=start",
"--filter", "label=com.docker.compose.project=" + project,
"--format", "{{json .}}"], cwd=ROOT, env=env,
stdout=events_stream, stderr=errors_stream)
time.sleep(.5)
try:
stage("compose-up", base + ["up", "-d", "--build", "--pull", pull,
"--wait", "--wait-timeout", "90"], 300)
finally:
watcher.terminate()
watcher.wait(timeout=5)
events = []
for line in events_path.read_text().splitlines():
try:
events.append(json.loads(line))
except ValueError:
pass
migrate_starts = sum(
(event.get("Actor") or {}).get("Attributes", {}).get("com.docker.compose.service") == "migrate"
and event.get("Action") == "start" for event in events)
migrate_ids = command(base + ["ps", "-aq", "migrate"], env=env).split()
api_ids = command(base + ["ps", "-q", "api"], env=env).split()
if len(migrate_ids) == 1 and len(api_ids) == 2:
migrator = json.loads(command(["docker", "inspect", migrate_ids[0]], env=env))[0]
apis = [json.loads(command(["docker", "inspect", identifier], env=env))[0]
for identifier in api_ids]
finished = migrator["State"]["FinishedAt"]
migration = {
"project": project, "migration_starts": migrate_starts,
"migration_exit_code": migrator["State"]["ExitCode"],
"migration_container_id": migrator["Id"],
"migration_finished_at": finished,
"api_containers": [{"id": item["Id"], "started_at": item["State"]["StartedAt"]}
for item in apis],
"api_started_before_migration_complete": any(
item["State"]["StartedAt"] < finished for item in apis),
"recorded_start_events": len(events),
}
evidence_path = output / "migration-evidence.json"
evidence_path.write_text(json.dumps(migration, indent=2) + "\n")
if events:
config_value = json.loads(lab3_config.read_text())
config_value["hooks"]["migration_evidence"] = actions + ["migration_evidence"]
lab3_config.write_text(json.dumps(config_value))
env["HL_MIGRATION_EVIDENCE"] = str(evidence_path)
else:
stage("compose-up", base + ["up", "-d", "--build", "--pull", pull,
"--wait", "--wait-timeout", "90"], 300)
wait_ready(env["BASE_URL"])
if lab3_only:
stage("lab3-" + lab3_scenario if lab3_scenario != "all" else "lab3-level4",
[sys.executable, "student-tests/lab3/run.py",
"--config", str(lab3_config),
"--output", str(output / "lab3"),
"--scenario", "rolling" if lab3_scenario == "rolling-dual" else lab3_scenario],
timeout=420, required=False)
summary["status"] = "FAIL" if any(x["status"] == "FAIL" for x in results) else "PASS"
raise PersistenceOnlyComplete
if persistence_only:
stage("teacher-persistence", ["make", "test-persistence", "BASE_URL=" + env["BASE_URL"],
"PYTHON=" + Path(sys.executable).name,
"ACTION_SCRIPT=" + str(ROOT / "scripts/restart.sh")],
required=False, lifecycle_targets=["api", "postgres", "redis", "lb"])
stage("lab3-restart-idempotency", [sys.executable, "student-tests/lab3/run.py",
"--config", str(lab3_config),
"--output", str(output / "lab3-restart"),
"--scenario", "restart"],
required=False, lifecycle_targets=["api", "postgres", "redis", "lb"])
summary["status"] = "FAIL" if any(x["status"] == "FAIL" for x in results) else "PASS"
raise PersistenceOnlyComplete
# Compile race-instrumented packages separately. Cold compilation is
# not an application-operation latency test and must not consume
# the execution budget of the actual integration tests.
stage("go-race-build", ["go", "test", "-race", "-run", "^$", "./..."], timeout=300)
stage("go-race", ["go", "test", "-race", "./...", "-count=1", "-json"], parse_go=True, timeout=600, required=False)
stage("go-build", ["go", "build", "./..."], required=False)
stage("go-vet", ["go", "vet", "./..."], required=False)
stage("teacher-smoke", ["make", "test", "BASE_URL=" + env["BASE_URL"], "PYTHON=" + Path(sys.executable).name], required=False)
auth = [sys.executable, "student-tests/auth_contract.py", "--base-url", env["AUTH_TEST_BASE_URL"], "--ttl", "3600"]
if redis_port is not None:
auth.extend(("--session-store", "redis"))
stage("auth-normal", auth, required=False)
stage("business", [sys.executable, "student-tests/lab2/run.py", "--suite", "business-boundaries", "--suite", "business-cursors", "--suite", "read-after-write", "--output", str(output / "business.json")], required=False)
stage("sql-plan-10000", [sys.executable, "student-tests/lab2/run.py", "--suite", "sql-plan", "--output", str(output / "sql-plan.json")], timeout=180, required=False)
if measure:
resources = {}
for role, cpus in (("api", 2.0), ("postgres", 1.0), ("redis", 0.5)):
identifier = command(base + ["ps", "-q", role], env=env).strip()
resources[role] = {"name": identifier, "cpus": cpus, "memory_limit_mib": {"api": 1024, "postgres": 512, "redis": 256}[role]}
env["LAB2_RESOURCE_CONTAINERS"] = json.dumps(resources)
stage("latency-matrix", [sys.executable, "student-tests/lab2/budgets.py", "--base-url", env["BASE_URL"], "--output", str(output / "latency")], timeout=300, required=False)
stage("teacher-persistence", ["make", "test-persistence", "BASE_URL=" + env["BASE_URL"],
"PYTHON=" + Path(sys.executable).name, "ACTION_SCRIPT=" + str(ROOT / "scripts/restart.sh")],
required=False, lifecycle_targets=["api", "postgres", "redis", "lb"])
stage("lab3-restart-idempotency", [sys.executable, "student-tests/lab3/run.py",
"--config", str(lab3_config),
"--output", str(output / "lab3-restart"),
"--scenario", "restart"],
required=False, lifecycle_targets=["api", "postgres", "redis", "lb"])
stage("auth-persistence", auth + ["--suite", "persistence", "--restart-hook", str(ROOT / "scripts/restart.sh")],
required=False, lifecycle_targets=["api", "postgres", "redis", "lb"])
stage("postgres-outage", auth + ["--suite", "dependency", "--dependency", "postgres", "--timeout", "5",
"--failure-hook", str(ROOT / "scripts/stop-postgres.sh"),
"--recovery-hook", str(ROOT / "scripts/start-postgres.sh")], required=False)
if redis_port is not None:
stage("redis-outage", auth + ["--suite", "dependency", "--dependency", "redis", "--cache-policy", "critical", "--timeout", "5",
"--failure-hook", str(ROOT / "scripts/stop-redis.sh"),
"--recovery-hook", str(ROOT / "scripts/start-redis.sh")], required=False)
configure(2)
stage("short-ttl-api", base + ["up", "-d", "--no-build", "--no-deps", "api"])
wait_ready(env["BASE_URL"])
stage("auth-expiry", [sys.executable, "student-tests/auth_contract.py", "--base-url", env["AUTH_TEST_BASE_URL"], "--suite", "expiry", "--ttl", "2"], required=False)
summary["status"] = "FAIL" if any(x["status"] == "FAIL" for x in results) else "PASS"
except PersistenceOnlyComplete:
pass
except (OSError, subprocess.SubprocessError, CheckFailure) as error:
summary["failure"] = str(error) if isinstance(error, CheckFailure) else type(error).__name__
finally:
# Only this newly generated project's own resources may be removed.
try:
stage("cleanup", base + ["down", "--volumes", "--remove-orphans", "--timeout", "20"], 90)
containers = command(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + project], env=env)
volumes = command(["docker", "volume", "ls", "-q", "--filter", "label=com.docker.compose.project=" + project], env=env)
if containers.strip() or volumes.strip():
raise CheckFailure("Disposable project resources remain after cleanup")
except (OSError, subprocess.SubprocessError, CheckFailure):
summary["status"] = "FAIL"
summary["cleanup_failed"] = True
(output / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2))
print(summary["status"] + ": " + str(output / "summary.json"), flush=True)
return 0 if summary["status"] == "PASS" else 1
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=("verify", "restart", "stop-db", "start-db", "restart-cache", "stop-cache", "start-cache"))
parser.add_argument("--output", type=Path, default=ROOT / "student-tests/output" / time.strftime("%Y%m%d-%H%M%S"))
parser.add_argument("--measure-budgets", action="store_true", help="bounded 1/5/10 HTTP matrix with explicit container limits")
parser.add_argument("--persistence-only", action="store_true", help="same isolated Compose project, only public persistence and lab3 restart replay")
parser.add_argument("--lab3-only", action="store_true", help="disposable two-replica lab3/LB level-4 run; saves evidence and cleans only its own project")
parser.add_argument("--lab3-scenario", choices=("all", "rolling", "rolling-dual", "idempotency"), default="all",
help="with --lab3-only: run one focused scenario in a fresh project, or the full suite")
parser.add_argument("--operation-budget-ms", type=int, help="explicitly agreed operation budget; omitted keeps provisional 3000ms")
parser.add_argument("--pull", choices=("missing", "never", "always"), default="missing")
args = parser.parse_args()
if args.persistence_only and args.lab3_only:
parser.error("choose either --persistence-only or --lab3-only")
if args.lab3_scenario != "all" and not args.lab3_only:
parser.error("--lab3-scenario requires --lab3-only")
if args.operation_budget_ms is not None and not 500 <= args.operation_budget_ms <= 10000:
parser.error("operation budget must be 500..10000 ms")
if args.action == "verify":
# Normal CI cancellation must run the same finally cleanup as failures.
def interrupted(_number, _frame):
raise CheckFailure("Test run interrupted")
signal.signal(signal.SIGTERM, interrupted)
signal.signal(signal.SIGINT, interrupted)
return verify(args.output, args.pull, args.measure_budgets, args.operation_budget_ms,
args.persistence_only, args.lab3_only, args.lab3_scenario)
lifecycle(args.action)
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except (OSError, ValueError, KeyError, subprocess.SubprocessError, CheckFailure) as error:
print("FAIL: " + (str(error) if isinstance(error, CheckFailure) else type(error).__name__), file=sys.stderr)
raise SystemExit(1)