395 lines
15 KiB
Python
395 lines
15 KiB
Python
"""Отдельные стенды собственных тестов: `make test-isolated`, `make failure-checks`.
|
||
|
||
python3 student-tests/stand.py isolated
|
||
python3 student-tests/stand.py failures [аргументы pytest]
|
||
|
||
Оба поднимают тот же `compose.yaml` под своим проектом
|
||
(`<рабочий>-isolated`, `<рабочий>-failures`) с override
|
||
`student-tests/stand.compose.yml`: свои PostgreSQL и Valkey со своим
|
||
volume, порты хранилищ не публикуются, образ API — свой тег. Подстановки
|
||
— `student-tests/stand.env`, а не `.env` рабочего стенда; пароли
|
||
хранилищ генерируются на прогон и никуда не пишутся. Затем:
|
||
|
||
- isolated — стенд лабы 3 целиком: HAProxy на свободном порту 127.0.0.1
|
||
и две реплики за ним. Через LB: `make test`, `pytest` (без проверок
|
||
отказов, `EXPECT_REPLICAS=2` — тесты реплик выполняются), `make
|
||
test-failover` (hook `scripts/stop-one.sh` в проекте стенда), возврат
|
||
реплик и `make test-persistence` (hook `scripts/restart.sh`);
|
||
- failures — без LB: обычный API и «тесный» (сроки сессии и ключа
|
||
идемпотентности 2 с, пул из одного соединения, короткие сроки) —
|
||
контейнеры `docker compose run` на свободных портах, `pytest -m fault`
|
||
с hook `student-tests/fault_hook.py`;
|
||
- удалить стенд вместе с volume при любом исходе, в том числе по Ctrl+C.
|
||
|
||
Рабочий стенд, его база и кэш не затрагиваются: каждый вызов compose —
|
||
только с `-p <проект стенда>`; hooks skeleton получают тот же проект
|
||
через `COMPOSE_PROJECT_NAME`, `COMPOSE_FILE` и `COMPOSE_ENV_FILES`. Два
|
||
прогона одного вида одновременно не запускать: уборка в начале второго
|
||
удалит стенд первого. Только stdlib и Python 3.9+: скрипт запускается
|
||
`python3`, как проверки skeleton.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import secrets
|
||
import signal
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
COMPOSE_FILES = (ROOT / "compose.yaml", ROOT / "student-tests" / "stand.compose.yml")
|
||
ENV_FILE = ROOT / "student-tests" / "stand.env"
|
||
FAULT_HOOK = ROOT / "student-tests" / "fault_hook.py"
|
||
# что из окружения хоста нужно самому docker compose и uv; остальное
|
||
# (в том числе POSTGRES_* из shell) в стенд не попадает
|
||
HOST_ENV = ("PATH", "HOME", "USER", "TMPDIR", "LANG", "DOCKER_HOST", "DOCKER_CONFIG")
|
||
READY_TIMEOUT = 60
|
||
LOG_TAIL_LINES = 50
|
||
REPLICAS = 2
|
||
# «тесный» API стенда отказов: сроки сессии и ключа 2 с, пул из одного
|
||
# соединения, короткие сроки ожидания пула и операции
|
||
TIGHT_ENV = {
|
||
"SESSION_TTL_SECONDS": "2",
|
||
"IDEMPOTENCY_TTL_SECONDS": "2",
|
||
"DB_POOL_SIZE": "1",
|
||
"DB_POOL_OVERFLOW": "0",
|
||
"DB_POOL_TIMEOUT": "1",
|
||
"DB_OPERATION_TIMEOUT": "2",
|
||
}
|
||
KINDS = ("isolated", "failures")
|
||
|
||
|
||
def say(kind: str, message: str) -> None:
|
||
print(f"[{kind}] {message}", flush=True)
|
||
|
||
|
||
def working_project() -> str:
|
||
"""Имя проекта рабочего стенда — из compose.yaml (`name:`)."""
|
||
config = subprocess.run(
|
||
[
|
||
"docker",
|
||
"compose",
|
||
"--env-file",
|
||
str(ENV_FILE),
|
||
"-f",
|
||
str(COMPOSE_FILES[0]),
|
||
"config",
|
||
"--format",
|
||
"json",
|
||
],
|
||
cwd=ROOT,
|
||
capture_output=True,
|
||
text=True,
|
||
check=True,
|
||
# подстановки не важны: нужно только имя проекта
|
||
env={**stand_env(), "POSTGRES_PASSWORD": "x", "VALKEY_PASSWORD": "x"},
|
||
)
|
||
return json.loads(config.stdout)["name"]
|
||
|
||
|
||
def stand_env() -> dict[str, str]:
|
||
"""Окружение docker compose стенда, собранное с нуля.
|
||
|
||
Пароли хранилищ новые на каждый прогон: живут только здесь, в
|
||
контейнерах стенда и у дочерних процессов (hooks вызывают compose с
|
||
ними же). В вывод и файлы не попадают.
|
||
"""
|
||
env = {key: os.environ[key] for key in HOST_ENV if key in os.environ}
|
||
return {
|
||
**env,
|
||
"POSTGRES_PASSWORD": secrets.token_hex(16),
|
||
"VALKEY_PASSWORD": secrets.token_hex(16),
|
||
# порт LB стенда isolated: постоянный на весь прогон, переживает
|
||
# stop/start из restart.sh
|
||
"STAND_LB_PORT": str(free_port()),
|
||
}
|
||
|
||
|
||
def free_port() -> int:
|
||
with socket.socket() as sock:
|
||
sock.bind(("127.0.0.1", 0))
|
||
return sock.getsockname()[1]
|
||
|
||
|
||
class Stand:
|
||
def __init__(self, kind: str) -> None:
|
||
self.kind = kind
|
||
self.env = stand_env()
|
||
working = working_project()
|
||
self.project = f"{working}-{kind}"
|
||
if self.project == working:
|
||
raise SystemExit(f"refusing to use the working project {working!r}")
|
||
self.containers: list[str] = []
|
||
|
||
def run(self, *args: str, check: bool = True, quiet: bool = False, env=None):
|
||
return subprocess.run(
|
||
args,
|
||
cwd=ROOT,
|
||
check=check,
|
||
text=True,
|
||
capture_output=quiet,
|
||
env=env or self.env,
|
||
)
|
||
|
||
def compose(self, *args: str, check: bool = True, quiet: bool = False):
|
||
"""docker compose стенда; без `-p <проект>` не вызывается никогда."""
|
||
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
|
||
return self.run(
|
||
"docker",
|
||
"compose",
|
||
"-p",
|
||
self.project,
|
||
"--env-file",
|
||
str(ENV_FILE),
|
||
*files,
|
||
*args,
|
||
check=check,
|
||
quiet=quiet,
|
||
)
|
||
|
||
def hook_env(self, base_url: str) -> dict[str, str]:
|
||
"""Окружение make и hooks skeleton: docker compose без `-p` внутри
|
||
`scripts/*.sh` попадает в проект стенда, а не в рабочий."""
|
||
return {
|
||
**os.environ,
|
||
**self.env,
|
||
"BASE_URL": base_url,
|
||
"COMPOSE_PROJECT_NAME": self.project,
|
||
"COMPOSE_FILE": os.pathsep.join(str(path) for path in COMPOSE_FILES),
|
||
"COMPOSE_ENV_FILES": str(ENV_FILE),
|
||
}
|
||
|
||
def leftovers(self) -> list[str]:
|
||
"""Контейнеры и volume проекта стенда, которые ещё остались."""
|
||
label = f"label=com.docker.compose.project={self.project}"
|
||
found = []
|
||
for command in (("ps", "-aq"), ("volume", "ls", "-q")):
|
||
out = self.run(
|
||
"docker", *command, "--filter", label, check=False, quiet=True
|
||
)
|
||
found += out.stdout.split()
|
||
return found
|
||
|
||
def cleanup(self) -> bool:
|
||
for name in self.containers:
|
||
self.run("docker", "rm", "-f", name, check=False, quiet=True)
|
||
down = self.compose("down", "-v", "--remove-orphans", check=False, quiet=True)
|
||
return down.returncode == 0 and not self.leftovers()
|
||
|
||
def wait_ready(self, base_url: str, container: str | None) -> None:
|
||
deadline = time.monotonic() + READY_TIMEOUT
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
with urllib.request.urlopen(
|
||
f"{base_url}/health/ready", timeout=2
|
||
) as resp:
|
||
if resp.status == 200:
|
||
return
|
||
except (urllib.error.URLError, OSError):
|
||
pass
|
||
if container:
|
||
state = self.run(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}}",
|
||
container,
|
||
check=False,
|
||
quiet=True,
|
||
)
|
||
if state.stdout.strip() != "true":
|
||
raise RuntimeError(f"container {container} is not running")
|
||
time.sleep(0.5)
|
||
raise RuntimeError(f"{base_url} not ready in {READY_TIMEOUT}s")
|
||
|
||
def start_api(self, suffix: str, extra_env: dict[str, str]) -> str:
|
||
"""Один процесс API без LB (стенд отказов) на свободном порту."""
|
||
name = f"{self.project}-{suffix}"
|
||
self.containers.append(name)
|
||
port = free_port()
|
||
env_args = [
|
||
arg for key, value in extra_env.items() for arg in ("-e", f"{key}={value}")
|
||
]
|
||
self.compose(
|
||
"run",
|
||
"-d",
|
||
"--no-deps",
|
||
"--name",
|
||
name,
|
||
*env_args,
|
||
"-p",
|
||
f"127.0.0.1:{port}:8080",
|
||
"api",
|
||
quiet=True,
|
||
)
|
||
base_url = f"http://127.0.0.1:{port}"
|
||
self.wait_ready(base_url, name)
|
||
return base_url
|
||
|
||
def start_lb(self) -> str:
|
||
"""HAProxy и REPLICAS реплик за ним; адрес LB на 127.0.0.1."""
|
||
self.compose("up", "-d", "--no-build", "--wait", "api", "haproxy")
|
||
base_url = f"http://127.0.0.1:{self.env['STAND_LB_PORT']}"
|
||
self.wait_ready(base_url, None)
|
||
return base_url
|
||
|
||
def prepare(self) -> None:
|
||
say(self.kind, "build image")
|
||
self.compose("build", "api")
|
||
say(self.kind, "start postgres and valkey")
|
||
self.compose("up", "-d", "--wait", "postgres", "valkey")
|
||
say(self.kind, "migrations")
|
||
self.compose("run", "--rm", "migrate")
|
||
|
||
def skeleton(self, script: str, base_url: str, *args: str) -> int:
|
||
"""Проверка skeleton (`tests/<script>`) — то же, что её цель make.
|
||
|
||
Не через make: Makefile подключает `.env` рабочего стенда, и его
|
||
пароли перекрыли бы пароли стенда в окружении hooks — `up` в
|
||
`restart.sh` пересоздал бы хранилища с чужими паролями.
|
||
"""
|
||
return self.run(
|
||
sys.executable,
|
||
f"tests/{script}",
|
||
"--base-url",
|
||
base_url,
|
||
*args,
|
||
check=False,
|
||
env=self.hook_env(base_url),
|
||
).returncode
|
||
|
||
def isolated(self, pytest_args: list[str]) -> dict[str, int]:
|
||
self.prepare()
|
||
base_url = self.start_lb()
|
||
say(self.kind, f"ready: lb={base_url} replicas={REPLICAS}")
|
||
results = {}
|
||
say(self.kind, "make test")
|
||
results["test"] = self.skeleton("smoke.py", base_url)
|
||
say(self.kind, "pytest")
|
||
results["student-test"] = self.run(
|
||
"uv",
|
||
"run",
|
||
"pytest",
|
||
*pytest_args,
|
||
check=False,
|
||
env={
|
||
**os.environ,
|
||
"BASE_URL": base_url,
|
||
"EXPECT_REPLICAS": str(REPLICAS),
|
||
},
|
||
).returncode
|
||
say(self.kind, "make test-failover (scripts/stop-one.sh)")
|
||
results["test-failover"] = self.skeleton(
|
||
"resilience.py",
|
||
base_url,
|
||
"--mode",
|
||
"failover",
|
||
"--action-script",
|
||
"scripts/stop-one.sh",
|
||
)
|
||
say(self.kind, "restore replicas")
|
||
self.compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--no-recreate",
|
||
"--no-build",
|
||
"--wait",
|
||
"--scale",
|
||
f"api={REPLICAS}",
|
||
"api",
|
||
)
|
||
say(self.kind, "make test-persistence (scripts/restart.sh)")
|
||
results["test-persistence"] = self.skeleton(
|
||
"resilience.py",
|
||
base_url,
|
||
"--mode",
|
||
"persistence",
|
||
"--action-script",
|
||
"scripts/restart.sh",
|
||
)
|
||
return results
|
||
|
||
def failures(self, pytest_args: list[str]) -> dict[str, int]:
|
||
self.prepare()
|
||
base_url = self.start_api("api", {})
|
||
tight_url = self.start_api("api-tight", TIGHT_ENV)
|
||
say(self.kind, f"ready: api={base_url} tight={tight_url}")
|
||
env = {
|
||
**os.environ,
|
||
**self.env,
|
||
"BASE_URL": base_url,
|
||
"FAULT_TIGHT_BASE_URL": tight_url,
|
||
"FAULT_HOOK": str(FAULT_HOOK),
|
||
"FAULT_PROJECT": self.project,
|
||
}
|
||
say(self.kind, "pytest -m fault")
|
||
code = self.run(
|
||
"uv",
|
||
"run",
|
||
"pytest",
|
||
"-m",
|
||
"fault",
|
||
*pytest_args,
|
||
check=False,
|
||
env=env,
|
||
).returncode
|
||
return {"fault": code}
|
||
|
||
def failed_logs(self) -> None:
|
||
for name in self.containers:
|
||
say(self.kind, f"last {LOG_TAIL_LINES} lines of {name}")
|
||
self.run("docker", "logs", "--tail", str(LOG_TAIL_LINES), name, check=False)
|
||
if self.kind == "isolated":
|
||
say(self.kind, f"last {LOG_TAIL_LINES} lines of api and haproxy")
|
||
self.compose(
|
||
"logs", "--tail", str(LOG_TAIL_LINES), "api", "haproxy", check=False
|
||
)
|
||
|
||
|
||
def main() -> int:
|
||
if len(sys.argv) < 2 or sys.argv[1] not in KINDS:
|
||
print(__doc__, file=sys.stderr)
|
||
return 2
|
||
kind, pytest_args = sys.argv[1], sys.argv[2:]
|
||
stand = Stand(kind)
|
||
say(kind, f"project={stand.project}")
|
||
if not stand.cleanup():
|
||
say(kind, "leftovers of a previous run could not be removed")
|
||
return 1
|
||
|
||
results: dict[str, int] = {}
|
||
failed = True
|
||
try:
|
||
results = getattr(stand, kind)(pytest_args)
|
||
failed = any(results.values())
|
||
except KeyboardInterrupt:
|
||
say(kind, "interrupted")
|
||
results["interrupted"] = 130
|
||
except (subprocess.CalledProcessError, RuntimeError) as exc:
|
||
say(kind, f"setup failed: {exc}")
|
||
results["setup"] = 1
|
||
finally:
|
||
# повторный Ctrl+C не должен оборвать уборку на полпути
|
||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||
if failed:
|
||
stand.failed_logs()
|
||
say(kind, "cleanup")
|
||
if not stand.cleanup():
|
||
say(kind, "cleanup failed")
|
||
results["cleanup"] = 1
|
||
|
||
say(kind, f"results: {results}")
|
||
return next((code for code in results.values() if code), 0)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|