forked from mtusi2026/mtusi2026lab2
Завершить PostgreSQL, Redis-сессии, миграции, таймауты и диагностику. Зафиксировать собственные тесты и доказательства финального прогона: 43 PASS, 0 FAIL, 0 SKIP; 768 HTTP-запросов без ошибок. Добавить официальный отчёт с запуском, архитектурой, SQL-планами и ограничениями. PR отложен до завершения уровня 5.
359 lines
19 KiB
Python
359 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Portable Compose acceptance runner and explicit persistence/outage hooks."""
|
|
import argparse
|
|
from collections import Counter
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import secrets
|
|
import shlex
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
class CheckFailure(RuntimeError):
|
|
pass
|
|
|
|
|
|
def command(args, *, env=None, timeout=90):
|
|
result = subprocess.run(args, cwd=ROOT, env=env, capture_output=True,
|
|
text=True, timeout=timeout)
|
|
if result.returncode:
|
|
raise CheckFailure("Command failed: " + " ".join(str(x) for x in args[:3]))
|
|
return result.stdout
|
|
|
|
|
|
def compose_base(env):
|
|
project = env.get("COMPOSE_PROJECT_NAME", "")
|
|
if not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", project):
|
|
raise CheckFailure("Set COMPOSE_PROJECT_NAME explicitly to the intended test project")
|
|
files = env.get("COMPOSE_FILE", str(ROOT / "compose.yaml")).split(os.pathsep)
|
|
base = ["docker", "compose", "--project-directory", str(ROOT), "-p", project]
|
|
for file in files:
|
|
if not file:
|
|
raise CheckFailure("Empty COMPOSE_FILE entry")
|
|
path = Path(file)
|
|
if not path.is_absolute():
|
|
path = ROOT / path
|
|
base.extend(("-f", str(path.resolve())))
|
|
return base
|
|
|
|
|
|
def wait_ready(url, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(url.rstrip("/") + "/health/ready", timeout=2) as response:
|
|
if response.status == 200:
|
|
return
|
|
except (OSError, urllib.error.URLError):
|
|
pass
|
|
time.sleep(0.25)
|
|
raise CheckFailure("API did not become ready within the configured budget")
|
|
|
|
|
|
def snapshot(base, env, services):
|
|
result = {}
|
|
for service in services:
|
|
ids = command(base + ["ps", "-aq", service], env=env).split()
|
|
if len(ids) != 1:
|
|
raise CheckFailure("Expected one existing container for " + service)
|
|
raw = json.loads(command(["docker", "inspect", ids[0]], env=env))[0]
|
|
if raw["Config"]["Labels"].get("com.docker.compose.project") != env["COMPOSE_PROJECT_NAME"]:
|
|
raise CheckFailure("Container belongs to another Compose project")
|
|
result[service] = {
|
|
"id": raw["Id"], "started_at": raw["State"]["StartedAt"],
|
|
"running": raw["State"]["Running"],
|
|
"volumes": sorted(m["Name"] for m in raw["Mounts"] if m["Type"] == "volume"),
|
|
}
|
|
return result
|
|
|
|
|
|
def validate_transition(action, before, after):
|
|
if not before or set(before) != set(after):
|
|
raise CheckFailure("Missing lifecycle evidence")
|
|
for service, old in before.items():
|
|
new = after[service]
|
|
if old["id"] != new["id"] or old["volumes"] != new["volumes"]:
|
|
raise CheckFailure("Lifecycle action recreated a container or changed volumes")
|
|
if action in ("stop-db", "stop-cache"):
|
|
if not old["running"] or new["running"]:
|
|
raise CheckFailure("Database was not actually stopped")
|
|
else:
|
|
if not new["running"] or old["started_at"] == new["started_at"]:
|
|
raise CheckFailure("Service was not actually started/restarted: " + service)
|
|
|
|
|
|
def lifecycle(action, env=None):
|
|
env = dict(os.environ if env is None else env)
|
|
base = compose_base(env)
|
|
config = json.loads(command(base + ["config", "--format", "json"], env=env))
|
|
services = config["services"]
|
|
stores = env.get("HL_STATE_SERVICES", "").split()
|
|
if not stores:
|
|
stores = [name for name in ("postgres", "redis", "valkey") if name in services]
|
|
if action == "restart":
|
|
targets = ["api"] + stores
|
|
elif "cache" in action:
|
|
targets = [name for name in ("redis", "valkey") if name in services]
|
|
if len(targets) != 1:
|
|
raise CheckFailure("Expected exactly one configured Redis/Valkey service")
|
|
else:
|
|
targets = ["postgres"]
|
|
if "postgres" not in stores or any(name not in services for name in targets):
|
|
raise CheckFailure("Configured state stores do not match Compose services")
|
|
before = snapshot(base, env, targets)
|
|
verb = {"restart": "restart", "stop-db": "stop", "start-db": "start",
|
|
"restart-cache": "restart", "stop-cache": "stop", "start-cache": "start"}[action]
|
|
args = [verb] + (["--timeout", "20"] if verb in ("restart", "stop") else []) + targets
|
|
command(base + args, env=env, timeout=65)
|
|
if not action.startswith("stop-"):
|
|
wait_ready(env.get("BASE_URL", "http://127.0.0.1:8080"), timeout=55)
|
|
after = snapshot(base, env, targets)
|
|
validate_transition(action, before, after)
|
|
evidence = env.get("HL_LIFECYCLE_EVIDENCE")
|
|
if evidence:
|
|
path = Path(evidence)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("a") as output:
|
|
output.write(json.dumps({"action": action, "project": env["COMPOSE_PROJECT_NAME"],
|
|
"before": before, "after": after}) + "\n")
|
|
print("PASS: " + action + "; container state and volumes verified", flush=True)
|
|
|
|
|
|
def go_results(output):
|
|
events = []
|
|
for line in output.splitlines():
|
|
try:
|
|
event = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if isinstance(event, dict):
|
|
events.append(event)
|
|
counts = Counter(x["Action"] for x in events if x.get("Test") and "/" not in x["Test"]
|
|
and x.get("Action") in ("pass", "fail", "skip"))
|
|
skipped = any(x.get("Test") and x.get("Action") == "skip" for x in events)
|
|
if not counts["pass"] or counts["fail"] or skipped:
|
|
raise CheckFailure("Go tests failed, were skipped, or did not run")
|
|
return dict(counts)
|
|
|
|
|
|
def free_ports():
|
|
# Reserve both ports together so the two selections cannot collide.
|
|
with socket.socket() as api, socket.socket() as postgres:
|
|
api.bind(("127.0.0.1", 0))
|
|
postgres.bind(("127.0.0.1", 0))
|
|
return api.getsockname()[1], postgres.getsockname()[1]
|
|
|
|
|
|
def verify(output, pull, measure=False, operation_budget_ms=None):
|
|
if importlib.util.find_spec("openapi_spec_validator") is None:
|
|
raise CheckFailure("Install test dependencies first: scripts/setup-tests.sh")
|
|
output = output.resolve()
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
project = "lab2-test-" + secrets.token_hex(6)
|
|
results = []
|
|
summary = {"project": project, "status": "FAIL", "results": results}
|
|
with tempfile.TemporaryDirectory(prefix=project + "-") as directory:
|
|
tmp = Path(directory)
|
|
api_port, pg_port = free_ports()
|
|
password = secrets.token_hex(24)
|
|
env = os.environ.copy()
|
|
# Do not inherit an operator's application/database/Compose settings.
|
|
for key in list(env):
|
|
if key.startswith(("COMPOSE_", "DATABASE_", "AUTH_TEST_", "MIGRATE_TEST_", "LAB2_", "HL_")):
|
|
del env[key]
|
|
env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env.get("PATH", "")
|
|
env.update(COMPOSE_PROJECT_NAME=project,
|
|
COMPOSE_FILE=os.pathsep.join((str(ROOT / "compose.yaml"), str(tmp / "compose.test.yaml"))),
|
|
BASE_URL=f"http://127.0.0.1:{api_port}",
|
|
HL_LIFECYCLE_EVIDENCE=str(output / "lifecycle.jsonl"))
|
|
env["LAB2_BASE_URL"] = env["BASE_URL"]
|
|
env["AUTH_TEST_BASE_URL"] = env["BASE_URL"]
|
|
env["AUTH_TEST_STACK_RESTART_HOOK"] = str(ROOT / "scripts/restart.sh")
|
|
env["AUTH_TEST_DIAGNOSTICS_DIR"] = str(output / "diagnostics")
|
|
if operation_budget_ms is not None:
|
|
env["AUTH_TEST_OPERATION_BUDGET_MS"] = str(operation_budget_ms)
|
|
database = f"postgres://messenger:{password}@127.0.0.1:{pg_port}/messenger?sslmode=disable"
|
|
env.update(AUTH_TEST_DATABASE_URL=database, MIGRATE_TEST_DATABASE_URL=database, LAB2_TEST_DATABASE_URL=database)
|
|
for name, value in {
|
|
"postgres_password": password,
|
|
"database_url": f"postgres://messenger:{password}@postgres:5432/messenger?sslmode=disable",
|
|
".env": (ROOT / ".env.example").read_text(),
|
|
}.items():
|
|
path = tmp / name
|
|
path.write_text(value)
|
|
path.chmod(0o600 if name == ".env" else 0o644)
|
|
override = tmp / "compose.test.yaml"
|
|
redis_port = None
|
|
|
|
def configure(ttl):
|
|
# JSON strings are valid quoted YAML scalars, including paths with spaces.
|
|
q = json.dumps
|
|
override.write_text(f'''services:
|
|
api:
|
|
image: {project}-api:check
|
|
env_file: !override
|
|
- {q(str(tmp / '.env'))}
|
|
ports: !override
|
|
- "127.0.0.1:{api_port}:8080"
|
|
environment:
|
|
APP_HOST: 0.0.0.0
|
|
APP_PORT: "8080"
|
|
SESSION_TTL_SECONDS: "{ttl}"
|
|
DATABASE_URL_FILE: /run/secrets/database_url
|
|
migrate:
|
|
image: {project}-migrate:check
|
|
postgres:
|
|
ports: !override
|
|
- "127.0.0.1:{pg_port}:5432"
|
|
secrets:
|
|
database_url:
|
|
file: {q(str(tmp / 'database_url'))}
|
|
postgres_password:
|
|
file: {q(str(tmp / 'postgres_password'))}
|
|
''')
|
|
if redis_port is not None:
|
|
text = override.read_text().replace("secrets:\n database_url:", f''' redis:
|
|
ports: !override
|
|
- "127.0.0.1:{redis_port}:6379"
|
|
secrets:
|
|
database_url:''')
|
|
text = text.replace(" APP_HOST: 0.0.0.0", " REDIS_ADDR: redis:6379\n SESSION_STORE: redis\n APP_HOST: 0.0.0.0")
|
|
override.write_text(text)
|
|
if measure:
|
|
text = override.read_text().replace(' image: ' + project + '-api:check', ' cpus: 2.0\n mem_limit: 1g\n image: ' + project + '-api:check')
|
|
text = text.replace(' postgres:\n ports:', ' postgres:\n cpus: 1.0\n mem_limit: 512m\n ports:')
|
|
text = text.replace(' redis:\n ports:', ' redis:\n cpus: 0.5\n mem_limit: 256m\n ports:')
|
|
override.write_text(text)
|
|
|
|
configure(3600)
|
|
base = compose_base(env)
|
|
services = command(base + ["config", "--services"], env=env).split()
|
|
if "redis" in services:
|
|
redis_port = free_ports()[0]
|
|
while redis_port in (api_port, pg_port):
|
|
redis_port = free_ports()[0]
|
|
configure(3600)
|
|
env["AUTH_TEST_REDIS_ADDR"] = f"127.0.0.1:{redis_port}"
|
|
restart = tmp / "restart-redis"
|
|
restart.write_text("#!/bin/sh\nexec " + shlex.join([sys.executable, str(ROOT / "student-tests/stack.py"), "restart-cache"]) + "\n")
|
|
restart.chmod(0o700)
|
|
env["AUTH_TEST_REDIS_RESTART_HOOK"] = str(restart)
|
|
summary["redis_fixture"] = "critical session store; explicit REDIS_ADDR=redis:6379 in test override"
|
|
|
|
def stage(name, args, timeout=180, parse_go=False, required=True):
|
|
started = time.monotonic()
|
|
try:
|
|
process = subprocess.run(args, cwd=ROOT, env=env, capture_output=True,
|
|
text=True, timeout=timeout)
|
|
text = process.stdout + process.stderr
|
|
# Never put the generated fixture credential or DSN in artifacts.
|
|
text = text.replace(database, "<test-database-url>").replace(password, "<test-password>")
|
|
(output / (name + ".log")).write_text(text)
|
|
if process.returncode:
|
|
raise CheckFailure(name + " failed; see its sanitized test log")
|
|
detail = go_results(process.stdout) if parse_go else None
|
|
results.append({"check": name, "status": "PASS", "seconds": round(time.monotonic() - started, 3),
|
|
**({"tests": detail} if detail else {})})
|
|
print("PASS: " + name, flush=True)
|
|
except (OSError, subprocess.SubprocessError, CheckFailure) as error:
|
|
results.append({"check": name, "status": "FAIL", "detail": type(error).__name__})
|
|
if required:
|
|
raise CheckFailure(name + " failed; see test artifacts") from None
|
|
print("FAIL: " + name + "; continuing independent checks", flush=True)
|
|
|
|
try:
|
|
stage("contracts", ["make", "check", "validate", "PYTHON=" + Path(sys.executable).name])
|
|
stage("harness", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_stack.py", "-v"])
|
|
stage("lab2-harness", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests/lab2", "-p", "test_harness.py", "-v"])
|
|
stage("compose-up", base + ["up", "-d", "--build", "--pull", pull, "--wait", "--wait-timeout", "90"], 300)
|
|
wait_ready(env["BASE_URL"])
|
|
stage("go-race", ["go", "test", "-race", "./...", "-count=1", "-json"], parse_go=True, timeout=300, required=False)
|
|
stage("go-build", ["go", "build", "./..."], required=False)
|
|
stage("go-vet", ["go", "vet", "./..."], required=False)
|
|
stage("teacher-smoke", ["make", "test", "BASE_URL=" + env["BASE_URL"], "PYTHON=" + Path(sys.executable).name], required=False)
|
|
auth = [sys.executable, "student-tests/auth_contract.py", "--base-url", env["BASE_URL"], "--ttl", "3600"]
|
|
if redis_port is not None:
|
|
auth.extend(("--session-store", "redis"))
|
|
stage("auth-normal", auth, required=False)
|
|
stage("business", [sys.executable, "student-tests/lab2/run.py", "--suite", "business-boundaries", "--suite", "business-cursors", "--suite", "read-after-write", "--output", str(output / "business.json")], required=False)
|
|
stage("sql-plan-10000", [sys.executable, "student-tests/lab2/run.py", "--suite", "sql-plan", "--output", str(output / "sql-plan.json")], timeout=180, required=False)
|
|
if measure:
|
|
resources = {}
|
|
for role, cpus in (("api", 2.0), ("postgres", 1.0), ("redis", 0.5)):
|
|
identifier = command(base + ["ps", "-q", role], env=env).strip()
|
|
resources[role] = {"name": identifier, "cpus": cpus, "memory_limit_mib": {"api": 1024, "postgres": 512, "redis": 256}[role]}
|
|
env["LAB2_RESOURCE_CONTAINERS"] = json.dumps(resources)
|
|
stage("latency-matrix", [sys.executable, "student-tests/lab2/budgets.py", "--base-url", env["BASE_URL"], "--output", str(output / "latency")], timeout=300, required=False)
|
|
stage("teacher-persistence", ["make", "test-persistence", "BASE_URL=" + env["BASE_URL"],
|
|
"PYTHON=" + Path(sys.executable).name, "ACTION_SCRIPT=" + str(ROOT / "scripts/restart.sh")], required=False)
|
|
stage("auth-persistence", auth + ["--suite", "persistence", "--restart-hook", str(ROOT / "scripts/restart.sh")], required=False)
|
|
stage("postgres-outage", auth + ["--suite", "dependency", "--dependency", "postgres", "--timeout", "5",
|
|
"--failure-hook", str(ROOT / "scripts/stop-postgres.sh"),
|
|
"--recovery-hook", str(ROOT / "scripts/start-postgres.sh")], required=False)
|
|
if redis_port is not None:
|
|
stage("redis-outage", auth + ["--suite", "dependency", "--dependency", "redis", "--cache-policy", "critical", "--timeout", "5",
|
|
"--failure-hook", str(ROOT / "scripts/stop-redis.sh"),
|
|
"--recovery-hook", str(ROOT / "scripts/start-redis.sh")], required=False)
|
|
configure(2)
|
|
stage("short-ttl-api", base + ["up", "-d", "--no-build", "--no-deps", "api"])
|
|
wait_ready(env["BASE_URL"])
|
|
stage("auth-expiry", [sys.executable, "student-tests/auth_contract.py", "--base-url", env["BASE_URL"], "--suite", "expiry", "--ttl", "2"], required=False)
|
|
summary["status"] = "FAIL" if any(x["status"] == "FAIL" for x in results) else "PASS"
|
|
except (OSError, subprocess.SubprocessError, CheckFailure) as error:
|
|
summary["failure"] = str(error) if isinstance(error, CheckFailure) else type(error).__name__
|
|
finally:
|
|
# Only this newly generated project's own resources may be removed.
|
|
try:
|
|
stage("cleanup", base + ["down", "--volumes", "--remove-orphans", "--timeout", "20"], 90)
|
|
containers = command(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + project], env=env)
|
|
volumes = command(["docker", "volume", "ls", "-q", "--filter", "label=com.docker.compose.project=" + project], env=env)
|
|
if containers.strip() or volumes.strip():
|
|
raise CheckFailure("Disposable project resources remain after cleanup")
|
|
except (OSError, subprocess.SubprocessError, CheckFailure):
|
|
summary["status"] = "FAIL"
|
|
summary["cleanup_failed"] = True
|
|
(output / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
print(summary["status"] + ": " + str(output / "summary.json"), flush=True)
|
|
return 0 if summary["status"] == "PASS" else 1
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("action", choices=("verify", "restart", "stop-db", "start-db", "restart-cache", "stop-cache", "start-cache"))
|
|
parser.add_argument("--output", type=Path, default=ROOT / "student-tests/output" / time.strftime("%Y%m%d-%H%M%S"))
|
|
parser.add_argument("--measure-budgets", action="store_true", help="bounded 1/5/10 HTTP matrix with explicit container limits")
|
|
parser.add_argument("--operation-budget-ms", type=int, help="explicitly agreed operation budget; omitted keeps provisional 3000ms")
|
|
parser.add_argument("--pull", choices=("missing", "never", "always"), default="missing")
|
|
args = parser.parse_args()
|
|
if args.operation_budget_ms is not None and not 500 <= args.operation_budget_ms <= 10000:
|
|
parser.error("operation budget must be 500..10000 ms")
|
|
if args.action == "verify":
|
|
# Normal CI cancellation must run the same finally cleanup as failures.
|
|
def interrupted(_number, _frame):
|
|
raise CheckFailure("Test run interrupted")
|
|
signal.signal(signal.SIGTERM, interrupted)
|
|
signal.signal(signal.SIGINT, interrupted)
|
|
return verify(args.output, args.pull, args.measure_budgets, args.operation_budget_ms)
|
|
lifecycle(args.action)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
raise SystemExit(main())
|
|
except (OSError, ValueError, KeyError, subprocess.SubprocessError, CheckFailure) as error:
|
|
print("FAIL: " + (str(error) if isinstance(error, CheckFailure) else type(error).__name__), file=sys.stderr)
|
|
raise SystemExit(1)
|