- carry the lab 2 solution over and plan lab 3 (PLAN.md, OpenSpec change instance-id-idempotency) - rename the compose project to mtusi-lab3 and the image to :lab3 - add X-Instance-Id to every response and instance_id to every log record ([app] instance_id, hostname by default) - require Idempotency-Key on POST messages: keys table in Postgres, key and message in one transaction, replay returns the same 201, another body gets 409, TTL from [idempotency] ttl_seconds - send the key from student-tests, k6 scenarios and the skill driver; add idempotency, X-Instance-Id, key race and fault tests Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
268 lines
8.4 KiB
Python
268 lines
8.4 KiB
Python
"""`make failure_checks`: проверки отказов на отдельном стенде.
|
||
|
||
Поднимает тот же `docker-compose.yml` под проектом `<рабочий>-failures`
|
||
с override `student-tests/failures.compose.yml`: свои PostgreSQL и Valkey
|
||
со своим volume, порты хранилищ не публикуются. Затем:
|
||
- собирает образ из рабочего дерева и применяет миграции;
|
||
- запускает два контейнера API: обычный и «тесный» (сроки сессии и ключа
|
||
идемпотентности 2 с, пул из одного соединения);
|
||
- прогоняет тесты с маркером `fault` (`pytest -m fault`), передав им hook
|
||
`student-tests/fault_hook.py`;
|
||
- удаляет стенд вместе с volume при любом исходе, в том числе по Ctrl+C.
|
||
|
||
Рабочий стенд, его база и кэш не затрагиваются. Только stdlib. Решения
|
||
1, 2, 5 design change `failure-checks`. Запускается `$(PYTHON)`, а не
|
||
`uv run`, по той же причине, что `run_isolated.py`.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import signal
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
ENV_FILE = ROOT / ".env"
|
||
COMPOSE_FILES = (
|
||
ROOT / "docker-compose.yml",
|
||
ROOT / "student-tests" / "failures.compose.yml",
|
||
)
|
||
FAULT_HOOK = ROOT / "student-tests" / "fault_hook.py"
|
||
PROJECT_SUFFIX = "-failures"
|
||
READY_TIMEOUT_S = 60
|
||
LOG_TAIL_LINES = 50
|
||
# «тесный» API: короткие сроки сессии и ключа идемпотентности, пул из
|
||
# одного соединения
|
||
TIGHT_ENV = {
|
||
"AUTH__SESSION_TTL_SECONDS": "2",
|
||
"IDEMPOTENCY__TTL_SECONDS": "2",
|
||
"DATABASE__POOL_SIZE": "1",
|
||
"DATABASE__MAX_OVERFLOW": "0",
|
||
"DATABASE__POOL_TIMEOUT_SECONDS": "1",
|
||
"DATABASE__REQUEST_TIMEOUT_SECONDS": "2",
|
||
}
|
||
|
||
|
||
def say(message: str) -> None:
|
||
print(f"[failure_checks] {message}", flush=True)
|
||
|
||
|
||
def run(
|
||
*args: str,
|
||
check: bool = True,
|
||
quiet: bool = False,
|
||
env: dict[str, str] | None = None,
|
||
) -> subprocess.CompletedProcess[str]:
|
||
"""Команда из корня репозитория; вывод — в терминал, если не quiet."""
|
||
return subprocess.run(
|
||
args,
|
||
cwd=ROOT,
|
||
check=check,
|
||
text=True,
|
||
capture_output=quiet,
|
||
env=env,
|
||
)
|
||
|
||
|
||
def dc(
|
||
project: str,
|
||
*args: str,
|
||
check: bool = True,
|
||
quiet: bool = False,
|
||
) -> subprocess.CompletedProcess[str]:
|
||
"""docker compose проекта отказов; без `-p` не вызывается никогда."""
|
||
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
|
||
return run(
|
||
"docker",
|
||
"compose",
|
||
"-p",
|
||
project,
|
||
*files,
|
||
*args,
|
||
check=check,
|
||
quiet=quiet,
|
||
)
|
||
|
||
|
||
def working_project() -> str:
|
||
config = run("docker", "compose", "config", "--format", "json", quiet=True)
|
||
return json.loads(config.stdout)["name"]
|
||
|
||
|
||
def app_port() -> str:
|
||
"""`APP__PORT` из `.env`: на нём слушает API внутри контейнера."""
|
||
if ENV_FILE.exists():
|
||
for line in ENV_FILE.read_text(encoding="utf-8").splitlines():
|
||
if line.strip().startswith("APP__PORT="):
|
||
return line.split("=", 1)[1].strip()
|
||
return "8080"
|
||
|
||
|
||
def free_port() -> int:
|
||
with socket.socket() as sock:
|
||
sock.bind(("127.0.0.1", 0))
|
||
return sock.getsockname()[1]
|
||
|
||
|
||
def wait_ready(url: str, container: str) -> None:
|
||
deadline = time.monotonic() + READY_TIMEOUT_S
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
with urllib.request.urlopen(url, timeout=2) as response:
|
||
if response.status == 200:
|
||
return
|
||
except (urllib.error.URLError, OSError):
|
||
pass
|
||
state = run(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}}",
|
||
container,
|
||
check=False,
|
||
quiet=True,
|
||
)
|
||
if state.stdout.strip() != "true":
|
||
raise RuntimeError(f"container {container} is not running")
|
||
time.sleep(1)
|
||
raise RuntimeError(f"{url} not ready in {READY_TIMEOUT_S}s")
|
||
|
||
|
||
def leftovers(project: str, kind: str) -> str:
|
||
"""Контейнеры (`ps -a`) или volume проекта, которые ещё остались."""
|
||
command = ("ps", "-aq") if kind == "container" else ("volume", "ls", "-q")
|
||
return run(
|
||
"docker",
|
||
*command,
|
||
"--filter",
|
||
f"label=com.docker.compose.project={project}",
|
||
quiet=True,
|
||
check=False,
|
||
).stdout.strip()
|
||
|
||
|
||
def cleanup(project: str, containers: list[str]) -> bool:
|
||
"""Контейнеры API, сервисы и volume проекта отказов; True — чисто."""
|
||
for name in containers:
|
||
run("docker", "rm", "-f", name, check=False, quiet=True)
|
||
down = dc(project, "down", "-v", "--remove-orphans", check=False)
|
||
return (
|
||
down.returncode == 0
|
||
and not leftovers(project, "container")
|
||
and not leftovers(project, "volume")
|
||
)
|
||
|
||
|
||
def start_api(
|
||
project: str,
|
||
name: str,
|
||
port: int,
|
||
env: dict[str, str],
|
||
) -> str:
|
||
env_args = [
|
||
arg for key, value in env.items() for arg in ("-e", f"{key}={value}")
|
||
]
|
||
dc(
|
||
project,
|
||
"run",
|
||
"-d",
|
||
"--no-deps",
|
||
"--name",
|
||
name,
|
||
*env_args,
|
||
"-p",
|
||
f"127.0.0.1:{port}:{app_port()}",
|
||
"api",
|
||
quiet=True,
|
||
)
|
||
base_url = f"http://127.0.0.1:{port}"
|
||
wait_ready(f"{base_url}/health/ready", name)
|
||
return base_url
|
||
|
||
|
||
def main() -> int:
|
||
pytest_args = sys.argv[1:]
|
||
working = working_project()
|
||
project = f"{working}{PROJECT_SUFFIX}"
|
||
if project == working or not project.endswith(PROJECT_SUFFIX):
|
||
say(f"refusing to use project {project!r}")
|
||
return 2
|
||
default_name = f"{project}-api"
|
||
tight_name = f"{project}-api-tight"
|
||
containers = [default_name, tight_name]
|
||
|
||
say(f"project={project} (working stand: {working})")
|
||
if not cleanup(project, containers):
|
||
say("leftovers of a previous run could not be removed")
|
||
return 1
|
||
|
||
results: dict[str, int] = {}
|
||
failed = True
|
||
try:
|
||
say("build image")
|
||
dc(project, "build", "api")
|
||
say("start postgres and valkey")
|
||
dc(project, "up", "-d", "--wait", "postgres", "valkey")
|
||
say("migrations")
|
||
dc(project, "run", "--rm", "migrate")
|
||
say("start api containers")
|
||
base_url = start_api(project, default_name, free_port(), {})
|
||
tight_url = start_api(project, tight_name, free_port(), TIGHT_ENV)
|
||
say(f"ready: api={base_url} tight={tight_url}")
|
||
env = os.environ | {
|
||
"BASE_URL": base_url,
|
||
"FAULT_TIGHT_BASE_URL": tight_url,
|
||
"FAULT_HOOK": str(FAULT_HOOK),
|
||
"FAULT_PROJECT": project,
|
||
}
|
||
say("pytest -m fault")
|
||
results["fault"] = run(
|
||
"uv",
|
||
"run",
|
||
"pytest",
|
||
"-m",
|
||
"fault",
|
||
*pytest_args,
|
||
check=False,
|
||
env=env,
|
||
).returncode
|
||
failed = bool(results["fault"])
|
||
except KeyboardInterrupt:
|
||
say("interrupted")
|
||
results["interrupted"] = 130
|
||
except (subprocess.CalledProcessError, RuntimeError) as e:
|
||
say(f"setup failed: {e}")
|
||
results["setup"] = 1
|
||
finally:
|
||
# повторный Ctrl+C не должен оборвать уборку на полпути
|
||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||
if failed:
|
||
for name in containers:
|
||
say(f"last {LOG_TAIL_LINES} lines of {name}")
|
||
run(
|
||
"docker",
|
||
"logs",
|
||
"--tail",
|
||
str(LOG_TAIL_LINES),
|
||
name,
|
||
check=False,
|
||
)
|
||
say("cleanup")
|
||
if not cleanup(project, containers):
|
||
say("cleanup failed")
|
||
results["cleanup"] = 1
|
||
|
||
say(f"results: {results}")
|
||
return next((code for code in results.values() if code), 0)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|