Files
mtusi2026lab2/student-tests/stack.py
T
OlegRood 834876c938 ЛР 2: закрыть уровень 4
Завершить PostgreSQL, Redis-сессии, миграции, таймауты и диагностику. Зафиксировать собственные тесты и доказательства финального прогона: 43 PASS, 0 FAIL, 0 SKIP; 768 HTTP-запросов без ошибок. Добавить официальный отчёт с запуском, архитектурой, SQL-планами и ограничениями. PR отложен до завершения уровня 5.
2026-10-28 13:30:26 +03:00

359 lines
19 KiB
Python

#!/usr/bin/env python3
"""Portable Compose acceptance runner and explicit persistence/outage hooks."""
import argparse
from collections import Counter
import importlib.util
import json
import os
from pathlib import Path
import re
import secrets
import shlex
import signal
import socket
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
class CheckFailure(RuntimeError):
pass
def command(args, *, env=None, timeout=90):
result = subprocess.run(args, cwd=ROOT, env=env, capture_output=True,
text=True, timeout=timeout)
if result.returncode:
raise CheckFailure("Command failed: " + " ".join(str(x) for x in args[:3]))
return result.stdout
def compose_base(env):
project = env.get("COMPOSE_PROJECT_NAME", "")
if not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", project):
raise CheckFailure("Set COMPOSE_PROJECT_NAME explicitly to the intended test project")
files = env.get("COMPOSE_FILE", str(ROOT / "compose.yaml")).split(os.pathsep)
base = ["docker", "compose", "--project-directory", str(ROOT), "-p", project]
for file in files:
if not file:
raise CheckFailure("Empty COMPOSE_FILE entry")
path = Path(file)
if not path.is_absolute():
path = ROOT / path
base.extend(("-f", str(path.resolve())))
return base
def wait_ready(url, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(url.rstrip("/") + "/health/ready", timeout=2) as response:
if response.status == 200:
return
except (OSError, urllib.error.URLError):
pass
time.sleep(0.25)
raise CheckFailure("API did not become ready within the configured budget")
def snapshot(base, env, services):
result = {}
for service in services:
ids = command(base + ["ps", "-aq", service], env=env).split()
if len(ids) != 1:
raise CheckFailure("Expected one existing container for " + service)
raw = json.loads(command(["docker", "inspect", ids[0]], env=env))[0]
if raw["Config"]["Labels"].get("com.docker.compose.project") != env["COMPOSE_PROJECT_NAME"]:
raise CheckFailure("Container belongs to another Compose project")
result[service] = {
"id": raw["Id"], "started_at": raw["State"]["StartedAt"],
"running": raw["State"]["Running"],
"volumes": sorted(m["Name"] for m in raw["Mounts"] if m["Type"] == "volume"),
}
return result
def validate_transition(action, before, after):
if not before or set(before) != set(after):
raise CheckFailure("Missing lifecycle evidence")
for service, old in before.items():
new = after[service]
if old["id"] != new["id"] or old["volumes"] != new["volumes"]:
raise CheckFailure("Lifecycle action recreated a container or changed volumes")
if action in ("stop-db", "stop-cache"):
if not old["running"] or new["running"]:
raise CheckFailure("Database was not actually stopped")
else:
if not new["running"] or old["started_at"] == new["started_at"]:
raise CheckFailure("Service was not actually started/restarted: " + service)
def lifecycle(action, env=None):
env = dict(os.environ if env is None else env)
base = compose_base(env)
config = json.loads(command(base + ["config", "--format", "json"], env=env))
services = config["services"]
stores = env.get("HL_STATE_SERVICES", "").split()
if not stores:
stores = [name for name in ("postgres", "redis", "valkey") if name in services]
if action == "restart":
targets = ["api"] + stores
elif "cache" in action:
targets = [name for name in ("redis", "valkey") if name in services]
if len(targets) != 1:
raise CheckFailure("Expected exactly one configured Redis/Valkey service")
else:
targets = ["postgres"]
if "postgres" not in stores or any(name not in services for name in targets):
raise CheckFailure("Configured state stores do not match Compose services")
before = snapshot(base, env, targets)
verb = {"restart": "restart", "stop-db": "stop", "start-db": "start",
"restart-cache": "restart", "stop-cache": "stop", "start-cache": "start"}[action]
args = [verb] + (["--timeout", "20"] if verb in ("restart", "stop") else []) + targets
command(base + args, env=env, timeout=65)
if not action.startswith("stop-"):
wait_ready(env.get("BASE_URL", "http://127.0.0.1:8080"), timeout=55)
after = snapshot(base, env, targets)
validate_transition(action, before, after)
evidence = env.get("HL_LIFECYCLE_EVIDENCE")
if evidence:
path = Path(evidence)
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("a") as output:
output.write(json.dumps({"action": action, "project": env["COMPOSE_PROJECT_NAME"],
"before": before, "after": after}) + "\n")
print("PASS: " + action + "; container state and volumes verified", flush=True)
def go_results(output):
events = []
for line in output.splitlines():
try:
event = json.loads(line)
except ValueError:
continue
if isinstance(event, dict):
events.append(event)
counts = Counter(x["Action"] for x in events if x.get("Test") and "/" not in x["Test"]
and x.get("Action") in ("pass", "fail", "skip"))
skipped = any(x.get("Test") and x.get("Action") == "skip" for x in events)
if not counts["pass"] or counts["fail"] or skipped:
raise CheckFailure("Go tests failed, were skipped, or did not run")
return dict(counts)
def free_ports():
# Reserve both ports together so the two selections cannot collide.
with socket.socket() as api, socket.socket() as postgres:
api.bind(("127.0.0.1", 0))
postgres.bind(("127.0.0.1", 0))
return api.getsockname()[1], postgres.getsockname()[1]
def verify(output, pull, measure=False, operation_budget_ms=None):
if importlib.util.find_spec("openapi_spec_validator") is None:
raise CheckFailure("Install test dependencies first: scripts/setup-tests.sh")
output = output.resolve()
output.mkdir(parents=True, exist_ok=True)
project = "lab2-test-" + secrets.token_hex(6)
results = []
summary = {"project": project, "status": "FAIL", "results": results}
with tempfile.TemporaryDirectory(prefix=project + "-") as directory:
tmp = Path(directory)
api_port, pg_port = free_ports()
password = secrets.token_hex(24)
env = os.environ.copy()
# Do not inherit an operator's application/database/Compose settings.
for key in list(env):
if key.startswith(("COMPOSE_", "DATABASE_", "AUTH_TEST_", "MIGRATE_TEST_", "LAB2_", "HL_")):
del env[key]
env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env.get("PATH", "")
env.update(COMPOSE_PROJECT_NAME=project,
COMPOSE_FILE=os.pathsep.join((str(ROOT / "compose.yaml"), str(tmp / "compose.test.yaml"))),
BASE_URL=f"http://127.0.0.1:{api_port}",
HL_LIFECYCLE_EVIDENCE=str(output / "lifecycle.jsonl"))
env["LAB2_BASE_URL"] = env["BASE_URL"]
env["AUTH_TEST_BASE_URL"] = env["BASE_URL"]
env["AUTH_TEST_STACK_RESTART_HOOK"] = str(ROOT / "scripts/restart.sh")
env["AUTH_TEST_DIAGNOSTICS_DIR"] = str(output / "diagnostics")
if operation_budget_ms is not None:
env["AUTH_TEST_OPERATION_BUDGET_MS"] = str(operation_budget_ms)
database = f"postgres://messenger:{password}@127.0.0.1:{pg_port}/messenger?sslmode=disable"
env.update(AUTH_TEST_DATABASE_URL=database, MIGRATE_TEST_DATABASE_URL=database, LAB2_TEST_DATABASE_URL=database)
for name, value in {
"postgres_password": password,
"database_url": f"postgres://messenger:{password}@postgres:5432/messenger?sslmode=disable",
".env": (ROOT / ".env.example").read_text(),
}.items():
path = tmp / name
path.write_text(value)
path.chmod(0o600 if name == ".env" else 0o644)
override = tmp / "compose.test.yaml"
redis_port = None
def configure(ttl):
# JSON strings are valid quoted YAML scalars, including paths with spaces.
q = json.dumps
override.write_text(f'''services:
api:
image: {project}-api:check
env_file: !override
- {q(str(tmp / '.env'))}
ports: !override
- "127.0.0.1:{api_port}:8080"
environment:
APP_HOST: 0.0.0.0
APP_PORT: "8080"
SESSION_TTL_SECONDS: "{ttl}"
DATABASE_URL_FILE: /run/secrets/database_url
migrate:
image: {project}-migrate:check
postgres:
ports: !override
- "127.0.0.1:{pg_port}:5432"
secrets:
database_url:
file: {q(str(tmp / 'database_url'))}
postgres_password:
file: {q(str(tmp / 'postgres_password'))}
''')
if redis_port is not None:
text = override.read_text().replace("secrets:\n database_url:", f''' redis:
ports: !override
- "127.0.0.1:{redis_port}:6379"
secrets:
database_url:''')
text = text.replace(" APP_HOST: 0.0.0.0", " REDIS_ADDR: redis:6379\n SESSION_STORE: redis\n APP_HOST: 0.0.0.0")
override.write_text(text)
if measure:
text = override.read_text().replace(' image: ' + project + '-api:check', ' cpus: 2.0\n mem_limit: 1g\n image: ' + project + '-api:check')
text = text.replace(' postgres:\n ports:', ' postgres:\n cpus: 1.0\n mem_limit: 512m\n ports:')
text = text.replace(' redis:\n ports:', ' redis:\n cpus: 0.5\n mem_limit: 256m\n ports:')
override.write_text(text)
configure(3600)
base = compose_base(env)
services = command(base + ["config", "--services"], env=env).split()
if "redis" in services:
redis_port = free_ports()[0]
while redis_port in (api_port, pg_port):
redis_port = free_ports()[0]
configure(3600)
env["AUTH_TEST_REDIS_ADDR"] = f"127.0.0.1:{redis_port}"
restart = tmp / "restart-redis"
restart.write_text("#!/bin/sh\nexec " + shlex.join([sys.executable, str(ROOT / "student-tests/stack.py"), "restart-cache"]) + "\n")
restart.chmod(0o700)
env["AUTH_TEST_REDIS_RESTART_HOOK"] = str(restart)
summary["redis_fixture"] = "critical session store; explicit REDIS_ADDR=redis:6379 in test override"
def stage(name, args, timeout=180, parse_go=False, required=True):
started = time.monotonic()
try:
process = subprocess.run(args, cwd=ROOT, env=env, capture_output=True,
text=True, timeout=timeout)
text = process.stdout + process.stderr
# Never put the generated fixture credential or DSN in artifacts.
text = text.replace(database, "<test-database-url>").replace(password, "<test-password>")
(output / (name + ".log")).write_text(text)
if process.returncode:
raise CheckFailure(name + " failed; see its sanitized test log")
detail = go_results(process.stdout) if parse_go else None
results.append({"check": name, "status": "PASS", "seconds": round(time.monotonic() - started, 3),
**({"tests": detail} if detail else {})})
print("PASS: " + name, flush=True)
except (OSError, subprocess.SubprocessError, CheckFailure) as error:
results.append({"check": name, "status": "FAIL", "detail": type(error).__name__})
if required:
raise CheckFailure(name + " failed; see test artifacts") from None
print("FAIL: " + name + "; continuing independent checks", flush=True)
try:
stage("contracts", ["make", "check", "validate", "PYTHON=" + Path(sys.executable).name])
stage("harness", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_stack.py", "-v"])
stage("lab2-harness", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests/lab2", "-p", "test_harness.py", "-v"])
stage("compose-up", base + ["up", "-d", "--build", "--pull", pull, "--wait", "--wait-timeout", "90"], 300)
wait_ready(env["BASE_URL"])
stage("go-race", ["go", "test", "-race", "./...", "-count=1", "-json"], parse_go=True, timeout=300, required=False)
stage("go-build", ["go", "build", "./..."], required=False)
stage("go-vet", ["go", "vet", "./..."], required=False)
stage("teacher-smoke", ["make", "test", "BASE_URL=" + env["BASE_URL"], "PYTHON=" + Path(sys.executable).name], required=False)
auth = [sys.executable, "student-tests/auth_contract.py", "--base-url", env["BASE_URL"], "--ttl", "3600"]
if redis_port is not None:
auth.extend(("--session-store", "redis"))
stage("auth-normal", auth, required=False)
stage("business", [sys.executable, "student-tests/lab2/run.py", "--suite", "business-boundaries", "--suite", "business-cursors", "--suite", "read-after-write", "--output", str(output / "business.json")], required=False)
stage("sql-plan-10000", [sys.executable, "student-tests/lab2/run.py", "--suite", "sql-plan", "--output", str(output / "sql-plan.json")], timeout=180, required=False)
if measure:
resources = {}
for role, cpus in (("api", 2.0), ("postgres", 1.0), ("redis", 0.5)):
identifier = command(base + ["ps", "-q", role], env=env).strip()
resources[role] = {"name": identifier, "cpus": cpus, "memory_limit_mib": {"api": 1024, "postgres": 512, "redis": 256}[role]}
env["LAB2_RESOURCE_CONTAINERS"] = json.dumps(resources)
stage("latency-matrix", [sys.executable, "student-tests/lab2/budgets.py", "--base-url", env["BASE_URL"], "--output", str(output / "latency")], timeout=300, required=False)
stage("teacher-persistence", ["make", "test-persistence", "BASE_URL=" + env["BASE_URL"],
"PYTHON=" + Path(sys.executable).name, "ACTION_SCRIPT=" + str(ROOT / "scripts/restart.sh")], required=False)
stage("auth-persistence", auth + ["--suite", "persistence", "--restart-hook", str(ROOT / "scripts/restart.sh")], required=False)
stage("postgres-outage", auth + ["--suite", "dependency", "--dependency", "postgres", "--timeout", "5",
"--failure-hook", str(ROOT / "scripts/stop-postgres.sh"),
"--recovery-hook", str(ROOT / "scripts/start-postgres.sh")], required=False)
if redis_port is not None:
stage("redis-outage", auth + ["--suite", "dependency", "--dependency", "redis", "--cache-policy", "critical", "--timeout", "5",
"--failure-hook", str(ROOT / "scripts/stop-redis.sh"),
"--recovery-hook", str(ROOT / "scripts/start-redis.sh")], required=False)
configure(2)
stage("short-ttl-api", base + ["up", "-d", "--no-build", "--no-deps", "api"])
wait_ready(env["BASE_URL"])
stage("auth-expiry", [sys.executable, "student-tests/auth_contract.py", "--base-url", env["BASE_URL"], "--suite", "expiry", "--ttl", "2"], required=False)
summary["status"] = "FAIL" if any(x["status"] == "FAIL" for x in results) else "PASS"
except (OSError, subprocess.SubprocessError, CheckFailure) as error:
summary["failure"] = str(error) if isinstance(error, CheckFailure) else type(error).__name__
finally:
# Only this newly generated project's own resources may be removed.
try:
stage("cleanup", base + ["down", "--volumes", "--remove-orphans", "--timeout", "20"], 90)
containers = command(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + project], env=env)
volumes = command(["docker", "volume", "ls", "-q", "--filter", "label=com.docker.compose.project=" + project], env=env)
if containers.strip() or volumes.strip():
raise CheckFailure("Disposable project resources remain after cleanup")
except (OSError, subprocess.SubprocessError, CheckFailure):
summary["status"] = "FAIL"
summary["cleanup_failed"] = True
(output / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2))
print(summary["status"] + ": " + str(output / "summary.json"), flush=True)
return 0 if summary["status"] == "PASS" else 1
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=("verify", "restart", "stop-db", "start-db", "restart-cache", "stop-cache", "start-cache"))
parser.add_argument("--output", type=Path, default=ROOT / "student-tests/output" / time.strftime("%Y%m%d-%H%M%S"))
parser.add_argument("--measure-budgets", action="store_true", help="bounded 1/5/10 HTTP matrix with explicit container limits")
parser.add_argument("--operation-budget-ms", type=int, help="explicitly agreed operation budget; omitted keeps provisional 3000ms")
parser.add_argument("--pull", choices=("missing", "never", "always"), default="missing")
args = parser.parse_args()
if args.operation_budget_ms is not None and not 500 <= args.operation_budget_ms <= 10000:
parser.error("operation budget must be 500..10000 ms")
if args.action == "verify":
# Normal CI cancellation must run the same finally cleanup as failures.
def interrupted(_number, _frame):
raise CheckFailure("Test run interrupted")
signal.signal(signal.SIGTERM, interrupted)
signal.signal(signal.SIGINT, interrupted)
return verify(args.output, args.pull, args.measure_budgets, args.operation_budget_ms)
lifecycle(args.action)
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except (OSError, ValueError, KeyError, subprocess.SubprocessError, CheckFailure) as error:
print("FAIL: " + (str(error) if isinstance(error, CheckFailure) else type(error).__name__), file=sys.stderr)
raise SystemExit(1)