#!/usr/bin/env python3 """Portable Compose acceptance runner and explicit persistence/outage hooks.""" import argparse from collections import Counter import importlib.util import json import os from pathlib import Path import re import secrets import shlex import shutil import signal import socket import subprocess import sys import tempfile import time import urllib.error import urllib.request ROOT = Path(__file__).resolve().parents[1] class CheckFailure(RuntimeError): pass class PersistenceOnlyComplete(Exception): pass def command(args, *, env=None, timeout=90): result = subprocess.run(args, cwd=ROOT, env=env, capture_output=True, text=True, timeout=timeout) if result.returncode: raise CheckFailure("Command failed: " + " ".join(str(x) for x in args[:3])) return result.stdout def compose_base(env): project = env.get("COMPOSE_PROJECT_NAME", "") if not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", project): raise CheckFailure("Set COMPOSE_PROJECT_NAME explicitly to the intended test project") files = env.get("COMPOSE_FILE", str(ROOT / "compose.yaml")).split(os.pathsep) base = ["docker", "compose", "--project-directory", str(ROOT), "-p", project] for file in files: if not file: raise CheckFailure("Empty COMPOSE_FILE entry") path = Path(file) if not path.is_absolute(): path = ROOT / path base.extend(("-f", str(path.resolve()))) return base def wait_ready(url, timeout=60): deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: with urllib.request.urlopen(url.rstrip("/") + "/health/ready", timeout=2) as response: if response.status == 200: return except (OSError, urllib.error.URLError): pass time.sleep(0.25) raise CheckFailure("API did not become ready within the configured budget") def snapshot(base, env, services): result = {} for service in services: ids = command(base + ["ps", "-aq", service], env=env).split() if not ids or (service != "api" and len(ids) != 1): raise CheckFailure("Expected existing container(s) for " + service) containers = [] for identifier in ids: raw = json.loads(command(["docker", "inspect", identifier], env=env))[0] labels = raw["Config"]["Labels"] if labels.get("com.docker.compose.project") != env["COMPOSE_PROJECT_NAME"] or \ labels.get("com.docker.compose.service") != service: raise CheckFailure("Container belongs to another Compose project/service") containers.append({ "id": raw["Id"], "started_at": raw["State"]["StartedAt"], "running": raw["State"]["Running"], "volumes": sorted(m["Name"] for m in raw["Mounts"] if m["Type"] == "volume"), }) containers.sort(key=lambda item: item["id"]) result[service] = containers if service == "api" else containers[0] return result def validate_transition(action, before, after): if not before or set(before) != set(after): raise CheckFailure("Missing lifecycle evidence") for service, old_group in before.items(): new_group = after[service] old_items = old_group if isinstance(old_group, list) else [old_group] new_items = new_group if isinstance(new_group, list) else [new_group] old_by_id = {item["id"]: item for item in old_items} new_by_id = {item["id"]: item for item in new_items} if len(old_by_id) != len(old_items) or len(new_by_id) != len(new_items) or \ set(old_by_id) != set(new_by_id): raise CheckFailure("Lifecycle action changed container set for " + service) for identifier, old in old_by_id.items(): new = new_by_id[identifier] if old["volumes"] != new["volumes"]: raise CheckFailure("Lifecycle action changed volumes for " + service) if action in ("stop-db", "stop-cache"): if not old["running"] or new["running"]: raise CheckFailure("Service was not actually stopped: " + service) elif not new["running"] or old["started_at"] == new["started_at"]: raise CheckFailure("Service was not actually started/restarted: " + service) def record_lifecycle(path, action, project, before, after, source=None): validate_transition(action, before, after) if path: path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) with path.open("a") as output: output.write(json.dumps({"action": action, "project": project, "source": source, "before": before, "after": after}) + "\n") def lifecycle(action, env=None): env = dict(os.environ if env is None else env) base = compose_base(env) config = json.loads(command(base + ["config", "--format", "json"], env=env)) services = config["services"] stores = env.get("HL_STATE_SERVICES", "").split() if not stores: stores = [name for name in ("postgres", "redis", "valkey") if name in services] if action == "restart": targets = ["api"] + stores + (["lb"] if "lb" in services else []) elif "cache" in action: targets = [name for name in ("redis", "valkey") if name in services] if len(targets) != 1: raise CheckFailure("Expected exactly one configured Redis/Valkey service") else: targets = ["postgres"] if "postgres" not in stores or any(name not in services for name in targets): raise CheckFailure("Configured state stores do not match Compose services") before = snapshot(base, env, targets) verb = {"restart": "restart", "stop-db": "stop", "start-db": "start", "restart-cache": "restart", "stop-cache": "stop", "start-cache": "start"}[action] args = [verb] + (["--timeout", "20"] if verb in ("restart", "stop") else []) + targets command(base + args, env=env, timeout=65) if not action.startswith("stop-"): wait_ready(env.get("BASE_URL", "http://127.0.0.1:8080"), timeout=55) after = snapshot(base, env, targets) record_lifecycle(env.get("HL_LIFECYCLE_EVIDENCE"), action, env["COMPOSE_PROJECT_NAME"], before, after, source="stack.py") print("PASS: " + action + "; container state and volumes verified", flush=True) def go_results(output): events = [] for line in output.splitlines(): try: event = json.loads(line) except ValueError: continue if isinstance(event, dict): events.append(event) counts = Counter(x["Action"] for x in events if x.get("Test") and "/" not in x["Test"] and x.get("Action") in ("pass", "fail", "skip")) skipped = any(x.get("Test") and x.get("Action") == "skip" for x in events) if not counts["pass"] or counts["fail"] or skipped: raise CheckFailure("Go tests failed, were skipped, or did not run") return dict(counts) def free_ports(): # Reserve both ports together so the two selections cannot collide. with socket.socket() as api, socket.socket() as postgres: api.bind(("127.0.0.1", 0)) postgres.bind(("127.0.0.1", 0)) return api.getsockname()[1], postgres.getsockname()[1] def verify(output, pull, measure=False, operation_budget_ms=None, persistence_only=False, lab3_only=False, lab3_scenario="all"): if not (persistence_only or lab3_only) and importlib.util.find_spec("openapi_spec_validator") is None: raise CheckFailure("Install test dependencies first: scripts/setup-tests.sh") output = output.resolve() output.mkdir(parents=True, exist_ok=True) project = ("lab3-test-" if lab3_only else "lab2-test-") + secrets.token_hex(6) results = [] summary = {"project": project, "status": "FAIL", "results": results, "scope": ("lab3-rolling-dual-diagnostic" if lab3_only and lab3_scenario == "rolling-dual" else "lab3-idempotency-only" if lab3_only and lab3_scenario == "idempotency" else "lab3-rolling-only" if lab3_only and lab3_scenario == "rolling" else "lab3-level4" if lab3_only else "persistence-only" if persistence_only else "full")} with tempfile.TemporaryDirectory(prefix=project + "-") as directory: tmp = Path(directory) api_port, pg_port = free_ports() direct_api_port = free_ports()[0] while direct_api_port in (api_port, pg_port): direct_api_port = free_ports()[0] password = secrets.token_hex(24) env = os.environ.copy() # Do not inherit an operator's application/database/Compose settings. for key in list(env): if key.startswith(("COMPOSE_", "DATABASE_", "AUTH_TEST_", "MIGRATE_TEST_", "LAB2_", "HL_")): del env[key] env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env.get("PATH", "") env.update(COMPOSE_PROJECT_NAME=project, COMPOSE_FILE=os.pathsep.join((str(ROOT / "compose.yaml"), str(tmp / "compose.test.yaml"))), BASE_URL=f"http://127.0.0.1:{api_port}", HL_LIFECYCLE_EVIDENCE=str(output / "lifecycle.jsonl")) env["LAB2_BASE_URL"] = env["BASE_URL"] # Old lab2 dependency/TTL tests need an API-generated response even # when the LB has excluded every unready backend. This port exists only # in the disposable test project, never in the production Compose file. env["AUTH_TEST_BASE_URL"] = f"http://127.0.0.1:{direct_api_port}" env["AUTH_TEST_STACK_RESTART_HOOK"] = str(ROOT / "scripts/restart.sh") env["AUTH_TEST_DIAGNOSTICS_DIR"] = str(output / "diagnostics") if operation_budget_ms is not None: env["AUTH_TEST_OPERATION_BUDGET_MS"] = str(operation_budget_ms) database = f"postgres://messenger:{password}@127.0.0.1:{pg_port}/messenger?sslmode=disable" env.update(AUTH_TEST_DATABASE_URL=database, MIGRATE_TEST_DATABASE_URL=database, LAB2_TEST_DATABASE_URL=database) for name, value in { "postgres_password": password, "database_url": f"postgres://messenger:{password}@postgres:5432/messenger?sslmode=disable", ".env": (ROOT / ".env.example").read_text(), }.items(): path = tmp / name path.write_text(value) path.chmod(0o600 if name == ".env" else 0o644) override = tmp / "compose.test.yaml" redis_port = None def configure(ttl): # JSON strings are valid quoted YAML scalars, including paths with spaces. q = json.dumps api_scale = 2 if lab3_only else 1 api_ports = (" ports: !override []" if lab3_only else f' ports: !override\n - "127.0.0.1:{direct_api_port}:8080"') segmented = lab3_only and lab3_scenario != "rolling" api_networks = " networks: [front, state]\n" if segmented else "" lb_networks = " networks: [front]\n" if segmented else "" state_networks = " networks: [state]\n" if segmented else "" network_definitions = "networks:\n front:\n state:\n" if segmented else "" override.write_text(f'''services: api: image: {project}-api:check scale: {api_scale} env_file: !override - {q(str(tmp / '.env'))} {api_ports} {api_networks} environment: APP_HOST: 0.0.0.0 APP_PORT: "8080" SESSION_TTL_SECONDS: "{ttl}" DATABASE_URL_FILE: /run/secrets/database_url lb: {lb_networks} ports: !override - "127.0.0.1:{api_port}:8080" migrate: {state_networks} image: {project}-migrate:check postgres: {state_networks} ports: !override - "127.0.0.1:{pg_port}:5432" secrets: database_url: file: {q(str(tmp / 'database_url'))} postgres_password: file: {q(str(tmp / 'postgres_password'))} {network_definitions} ''') if redis_port is not None: text = override.read_text().replace("secrets:\n database_url:", f''' redis: {state_networks} ports: !override - "127.0.0.1:{redis_port}:6379" secrets: database_url:''') text = text.replace(" APP_HOST: 0.0.0.0", " REDIS_ADDR: redis:6379\n SESSION_STORE: redis\n APP_HOST: 0.0.0.0") override.write_text(text) if measure: text = override.read_text().replace(' image: ' + project + '-api:check', ' cpus: 2.0\n mem_limit: 1g\n image: ' + project + '-api:check') text = text.replace(' postgres:\n ports:', ' postgres:\n cpus: 1.0\n mem_limit: 512m\n ports:') text = text.replace(' redis:\n ports:', ' redis:\n cpus: 0.5\n mem_limit: 256m\n ports:') override.write_text(text) configure(3600) lab3_config = tmp / "lab3-restart.json" lab3_config.write_text(json.dumps({"base_url": env["BASE_URL"], "hooks": {"restart": [str(ROOT / "scripts/restart.sh")]}})) base = compose_base(env) services = command(base + ["config", "--services"], env=env).split() if "redis" in services: redis_port = free_ports()[0] while redis_port in (api_port, pg_port, direct_api_port): redis_port = free_ports()[0] configure(3600) env["AUTH_TEST_REDIS_ADDR"] = f"127.0.0.1:{redis_port}" restart = tmp / "restart-redis" restart.write_text("#!/bin/sh\nexec " + shlex.join([sys.executable, str(ROOT / "student-tests/stack.py"), "restart-cache"]) + "\n") restart.chmod(0o700) env["AUTH_TEST_REDIS_RESTART_HOOK"] = str(restart) summary["redis_fixture"] = "critical session store; explicit REDIS_ADDR=redis:6379 in test override" if lab3_only: actions = [sys.executable, str(ROOT / "student-tests/lab3/docker_actions.py")] lab3_config.write_text(json.dumps({ "base_url": env["BASE_URL"], "api_services": ["api"], "lb_services": ["lb"], "compose_base_argv": base, "compose_config_argv": base + ["config", "--format", "json"], "rolling_telemetry": lab3_scenario == "rolling-dual", "hooks": { "restart": [str(ROOT / "scripts/restart.sh")], "stop_one": [str(ROOT / "scripts/stop-one.sh")], "rolling_update": [str(ROOT / "scripts/rolling-update.sh")], **{name: actions + [name] for name in ("start_one", "recover_all", "sigterm_one", "probe_stopped", "make_unready", "restore_ready", "probe_health", "lb_timing_evidence", "revoke_access", "restore_access")}, }, })) if lab3_scenario != "rolling": env["HL_STATE_NETWORK"] = project + "_state" if lab3_scenario in ("all", "rolling-dual"): # The readiness fault needs front+state networks. The working # rolling script runs on a one-network production stack and # asks Docker for all IPs concatenated. Resolve that one # inspect format to the actual front-network IP in this # test-only topology; every other Docker call is unchanged. real_docker = shutil.which("docker") if not real_docker: raise CheckFailure("Docker CLI unavailable for rolling IP adapter") shim = tmp / "docker" shim.write_text('''#!/bin/sh set -eu trace() { [ "${HL_ROLLING_TRACE_ACTIVE:-}" = 1 ] || return 0 "$HL_TRACE_PYTHON" - "$HL_ROLLING_STEP_LOG" "$@" <<'PYTRACE' import datetime, json, sys record = {"time_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="milliseconds"), "phase": sys.argv[2], "argv": sys.argv[3:]} with open(sys.argv[1], "a") as output: output.write(json.dumps(record) + "\\n") PYTRACE } trace begin "$@" if [ "$#" -eq 4 ] && [ "$1" = inspect ] && [ "$2" = --format ] && [ "$3" = '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ]; then owner=$("$HL_REAL_DOCKER" inspect --format '{{index .Config.Labels "com.docker.compose.project"}}' "$4") [ "$owner" = "$COMPOSE_PROJECT_NAME" ] || exit 1 format="{{(index .NetworkSettings.Networks \\"$HL_FRONT_NETWORK\\").IPAddress}}" if "$HL_REAL_DOCKER" inspect --format "$format" "$4"; then code=0; else code=$?; fi else if "$HL_REAL_DOCKER" "$@"; then code=0; else code=$?; fi fi trace end "$code" "$@" exit "$code" ''') shim.chmod(0o700) env["HL_REAL_DOCKER"] = real_docker env["HL_FRONT_NETWORK"] = project + "_front" env["PATH"] = str(tmp) + os.pathsep + env["PATH"] summary["rolling_ip_adapter"] = "test-only Docker inspect selects the real front-network IP" if lab3_scenario == "rolling-dual": env["HL_TRACE_PYTHON"] = sys.executable env["HL_ROLLING_STEP_LOG"] = str(output / "lab3" / "rolling-docker-steps.jsonl") def stage(name, args, timeout=180, parse_go=False, required=True, lifecycle_targets=None): started = time.monotonic() try: before_lifecycle = snapshot(base, env, lifecycle_targets) if lifecycle_targets else None process = subprocess.run(args, cwd=ROOT, env=env, capture_output=True, text=True, timeout=timeout) text = process.stdout + process.stderr # Never put the generated fixture credential or DSN in artifacts. text = text.replace(database, "").replace(password, "") (output / (name + ".log")).write_text(text) if process.returncode: raise CheckFailure(name + " failed; see its sanitized test log") if lifecycle_targets: after_lifecycle = snapshot(base, env, lifecycle_targets) record_lifecycle(env.get("HL_LIFECYCLE_EVIDENCE"), "restart", project, before_lifecycle, after_lifecycle, source=name) detail = go_results(process.stdout) if parse_go else None results.append({"check": name, "status": "PASS", "seconds": round(time.monotonic() - started, 3), **({"tests": detail} if detail else {})}) print("PASS: " + name, flush=True) except subprocess.TimeoutExpired as error: def decode(value): return value.decode(errors="replace") if isinstance(value, bytes) else (value or "") partial = decode(error.stdout) + decode(error.stderr) partial = partial.replace(database, "").replace(password, "") (output / (name + ".log")).write_text(partial) results.append({"check": name, "status": "FAIL", "detail": "TimeoutExpired", "seconds": round(time.monotonic() - started, 3), "runner_budget_seconds": timeout}) if required: raise CheckFailure(name + " timed out; see partial sanitized log") from None print("FAIL: " + name + "; partial log retained; continuing independent checks", flush=True) except (OSError, subprocess.SubprocessError, CheckFailure) as error: results.append({"check": name, "status": "FAIL", "detail": type(error).__name__}) if required: raise CheckFailure(name + " failed; see test artifacts") from None print("FAIL: " + name + "; continuing independent checks", flush=True) try: if not (persistence_only or lab3_only): stage("contracts", ["make", "check", "validate", "PYTHON=" + Path(sys.executable).name]) stage("harness", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_stack.py", "-v"]) stage("lab2-harness", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests/lab2", "-p", "test_harness.py", "-v"]) if lab3_only: # Subscribe before the clean Compose project is created. Docker # Desktop may not retain historical events for a later query. events_path = output / "docker-start-events.jsonl" errors_path = output / "docker-events.stderr.log" with events_path.open("wb") as events_stream, errors_path.open("wb") as errors_stream: watcher = subprocess.Popen( ["docker", "events", "--filter", "type=container", "--filter", "event=start", "--filter", "label=com.docker.compose.project=" + project, "--format", "{{json .}}"], cwd=ROOT, env=env, stdout=events_stream, stderr=errors_stream) time.sleep(.5) try: stage("compose-up", base + ["up", "-d", "--build", "--pull", pull, "--wait", "--wait-timeout", "90"], 300) finally: watcher.terminate() watcher.wait(timeout=5) events = [] for line in events_path.read_text().splitlines(): try: events.append(json.loads(line)) except ValueError: pass migrate_starts = sum( (event.get("Actor") or {}).get("Attributes", {}).get("com.docker.compose.service") == "migrate" and event.get("Action") == "start" for event in events) migrate_ids = command(base + ["ps", "-aq", "migrate"], env=env).split() api_ids = command(base + ["ps", "-q", "api"], env=env).split() if len(migrate_ids) == 1 and len(api_ids) == 2: migrator = json.loads(command(["docker", "inspect", migrate_ids[0]], env=env))[0] apis = [json.loads(command(["docker", "inspect", identifier], env=env))[0] for identifier in api_ids] finished = migrator["State"]["FinishedAt"] migration = { "project": project, "migration_starts": migrate_starts, "migration_exit_code": migrator["State"]["ExitCode"], "migration_container_id": migrator["Id"], "migration_finished_at": finished, "api_containers": [{"id": item["Id"], "started_at": item["State"]["StartedAt"]} for item in apis], "api_started_before_migration_complete": any( item["State"]["StartedAt"] < finished for item in apis), "recorded_start_events": len(events), } evidence_path = output / "migration-evidence.json" evidence_path.write_text(json.dumps(migration, indent=2) + "\n") if events: config_value = json.loads(lab3_config.read_text()) config_value["hooks"]["migration_evidence"] = actions + ["migration_evidence"] lab3_config.write_text(json.dumps(config_value)) env["HL_MIGRATION_EVIDENCE"] = str(evidence_path) else: stage("compose-up", base + ["up", "-d", "--build", "--pull", pull, "--wait", "--wait-timeout", "90"], 300) wait_ready(env["BASE_URL"]) if lab3_only: stage("lab3-" + lab3_scenario if lab3_scenario != "all" else "lab3-level4", [sys.executable, "student-tests/lab3/run.py", "--config", str(lab3_config), "--output", str(output / "lab3"), "--scenario", "rolling" if lab3_scenario == "rolling-dual" else lab3_scenario], timeout=420, required=False) summary["status"] = "FAIL" if any(x["status"] == "FAIL" for x in results) else "PASS" raise PersistenceOnlyComplete if persistence_only: stage("teacher-persistence", ["make", "test-persistence", "BASE_URL=" + env["BASE_URL"], "PYTHON=" + Path(sys.executable).name, "ACTION_SCRIPT=" + str(ROOT / "scripts/restart.sh")], required=False, lifecycle_targets=["api", "postgres", "redis", "lb"]) stage("lab3-restart-idempotency", [sys.executable, "student-tests/lab3/run.py", "--config", str(lab3_config), "--output", str(output / "lab3-restart"), "--scenario", "restart"], required=False, lifecycle_targets=["api", "postgres", "redis", "lb"]) summary["status"] = "FAIL" if any(x["status"] == "FAIL" for x in results) else "PASS" raise PersistenceOnlyComplete # Compile race-instrumented packages separately. Cold compilation is # not an application-operation latency test and must not consume # the execution budget of the actual integration tests. stage("go-race-build", ["go", "test", "-race", "-run", "^$", "./..."], timeout=300) stage("go-race", ["go", "test", "-race", "./...", "-count=1", "-json"], parse_go=True, timeout=600, required=False) stage("go-build", ["go", "build", "./..."], required=False) stage("go-vet", ["go", "vet", "./..."], required=False) stage("teacher-smoke", ["make", "test", "BASE_URL=" + env["BASE_URL"], "PYTHON=" + Path(sys.executable).name], required=False) auth = [sys.executable, "student-tests/auth_contract.py", "--ttl", "3600"] if redis_port is not None: auth.extend(("--session-store", "redis")) # Normal contract traffic follows the same public LB path as users. # Only the later dependency/TTL diagnostics need direct API access. stage("auth-normal", auth + ["--base-url", env["BASE_URL"]], required=False) auth.extend(("--base-url", env["AUTH_TEST_BASE_URL"])) stage("business", [sys.executable, "student-tests/lab2/run.py", "--suite", "business-boundaries", "--suite", "business-cursors", "--suite", "read-after-write", "--output", str(output / "business.json")], required=False) stage("sql-plan-10000", [sys.executable, "student-tests/lab2/run.py", "--suite", "sql-plan", "--output", str(output / "sql-plan.json")], timeout=180, required=False) if measure: resources = {} for role, cpus in (("api", 2.0), ("postgres", 1.0), ("redis", 0.5)): identifier = command(base + ["ps", "-q", role], env=env).strip() resources[role] = {"name": identifier, "cpus": cpus, "memory_limit_mib": {"api": 1024, "postgres": 512, "redis": 256}[role]} env["LAB2_RESOURCE_CONTAINERS"] = json.dumps(resources) stage("latency-matrix", [sys.executable, "student-tests/lab2/budgets.py", "--base-url", env["BASE_URL"], "--output", str(output / "latency")], timeout=300, required=False) stage("teacher-persistence", ["make", "test-persistence", "BASE_URL=" + env["BASE_URL"], "PYTHON=" + Path(sys.executable).name, "ACTION_SCRIPT=" + str(ROOT / "scripts/restart.sh")], required=False, lifecycle_targets=["api", "postgres", "redis", "lb"]) stage("lab3-restart-idempotency", [sys.executable, "student-tests/lab3/run.py", "--config", str(lab3_config), "--output", str(output / "lab3-restart"), "--scenario", "restart"], required=False, lifecycle_targets=["api", "postgres", "redis", "lb"]) stage("auth-persistence", auth + ["--suite", "persistence", "--restart-hook", str(ROOT / "scripts/restart.sh")], required=False, lifecycle_targets=["api", "postgres", "redis", "lb"]) stage("postgres-outage", auth + ["--suite", "dependency", "--dependency", "postgres", "--timeout", "5", "--failure-hook", str(ROOT / "scripts/stop-postgres.sh"), "--recovery-hook", str(ROOT / "scripts/start-postgres.sh")], required=False) if redis_port is not None: stage("redis-outage", auth + ["--suite", "dependency", "--dependency", "redis", "--cache-policy", "critical", "--timeout", "5", "--failure-hook", str(ROOT / "scripts/stop-redis.sh"), "--recovery-hook", str(ROOT / "scripts/start-redis.sh")], required=False) configure(2) stage("short-ttl-api", base + ["up", "-d", "--no-build", "--no-deps", "api"]) wait_ready(env["BASE_URL"]) stage("auth-expiry", [sys.executable, "student-tests/auth_contract.py", "--base-url", env["AUTH_TEST_BASE_URL"], "--suite", "expiry", "--ttl", "2"], required=False) summary["status"] = "FAIL" if any(x["status"] == "FAIL" for x in results) else "PASS" except PersistenceOnlyComplete: pass except (OSError, subprocess.SubprocessError, CheckFailure) as error: summary["failure"] = str(error) if isinstance(error, CheckFailure) else type(error).__name__ finally: # Retain startup/migration diagnostics before deleting the fixture. # A failed one-shot migrator otherwise disappears with its cause. try: logs = command(base + ["logs", "--no-color", "--tail", "200", "migrate", "postgres", "api", "lb"], env=env, timeout=30) logs = logs.replace(database, "").replace(password, "") (output / "service-logs.log").write_text(logs) except (OSError, subprocess.SubprocessError, CheckFailure) as error: (output / "service-logs.log").write_text("Diagnostics unavailable: " + type(error).__name__ + "\n") # Only this newly generated project's own resources may be removed. try: stage("cleanup", base + ["down", "--volumes", "--remove-orphans", "--timeout", "20"], 90) containers = command(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + project], env=env) volumes = command(["docker", "volume", "ls", "-q", "--filter", "label=com.docker.compose.project=" + project], env=env) if containers.strip() or volumes.strip(): raise CheckFailure("Disposable project resources remain after cleanup") except (OSError, subprocess.SubprocessError, CheckFailure): summary["status"] = "FAIL" summary["cleanup_failed"] = True (output / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2)) print(summary["status"] + ": " + str(output / "summary.json"), flush=True) return 0 if summary["status"] == "PASS" else 1 def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("action", choices=("verify", "restart", "stop-db", "start-db", "restart-cache", "stop-cache", "start-cache")) parser.add_argument("--output", type=Path, default=ROOT / "student-tests/output" / time.strftime("%Y%m%d-%H%M%S")) parser.add_argument("--measure-budgets", action="store_true", help="bounded 1/5/10 HTTP matrix with explicit container limits") parser.add_argument("--persistence-only", action="store_true", help="same isolated Compose project, only public persistence and lab3 restart replay") parser.add_argument("--lab3-only", action="store_true", help="disposable two-replica lab3/LB level-4 run; saves evidence and cleans only its own project") parser.add_argument("--lab3-scenario", choices=("all", "rolling", "rolling-dual", "idempotency"), default="all", help="with --lab3-only: run one focused scenario in a fresh project, or the full suite") parser.add_argument("--operation-budget-ms", type=int, help="explicitly agreed operation budget; omitted keeps provisional 3000ms") parser.add_argument("--pull", choices=("missing", "never", "always"), default="missing") args = parser.parse_args() if args.persistence_only and args.lab3_only: parser.error("choose either --persistence-only or --lab3-only") if args.lab3_scenario != "all" and not args.lab3_only: parser.error("--lab3-scenario requires --lab3-only") if args.operation_budget_ms is not None and not 500 <= args.operation_budget_ms <= 10000: parser.error("operation budget must be 500..10000 ms") if args.action == "verify": # Normal CI cancellation must run the same finally cleanup as failures. def interrupted(_number, _frame): raise CheckFailure("Test run interrupted") signal.signal(signal.SIGTERM, interrupted) signal.signal(signal.SIGINT, interrupted) return verify(args.output, args.pull, args.measure_budgets, args.operation_budget_ms, args.persistence_only, args.lab3_only, args.lab3_scenario) lifecycle(args.action) return 0 if __name__ == "__main__": try: raise SystemExit(main()) except (OSError, ValueError, KeyError, subprocess.SubprocessError, CheckFailure) as error: print("FAIL: " + (str(error) if isinstance(error, CheckFailure) else type(error).__name__), file=sys.stderr) raise SystemExit(1)