Files
mtusi2026lab3/student-tests/run_failures.py
T
ricoromaandClaude Opus 5.5 1758790e5c fix: name postgres log read failures in failure checks
docker compose logs runs with check=False; a non-zero exit prints
"postgres log unavailable" and fails the run instead of being reported
as a setup failure.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 18:23:23 +03:00

319 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""`make failure_checks`: проверки отказов на отдельном стенде.
Поднимает тот же `docker-compose.yml` под проектом `<рабочий>-failures`
с override `student-tests/failures.compose.yml`: свои PostgreSQL и Valkey
со своим volume, порты хранилищ не публикуются. Затем:
- собирает образ из рабочего дерева и применяет миграции;
- запускает два контейнера API: обычный и «тесный» (сроки сессии и ключа
идемпотентности 2 с, пул из одного соединения);
- прогоняет тесты с маркером `fault` (`pytest -m fault`), передав им hook
`student-tests/fault_hook.py`;
- после тестов читает журнал PostgreSQL стенда: печатает сводку и падает
на признаках аварийного перезапуска (решение 12 design change
`fix-known-issues`);
- удаляет стенд вместе с volume при любом исходе, в том числе по Ctrl+C.
Рабочий стенд, его база и кэш не затрагиваются. Только stdlib. Решения
1, 2, 5 design change `failure-checks`. Запускается `$(PYTHON)`, а не
`uv run`, по той же причине, что `run_isolated.py`.
"""
from __future__ import annotations
import json
import os
import signal
import socket
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
ENV_FILE = ROOT / ".env"
COMPOSE_FILES = (
ROOT / "docker-compose.yml",
ROOT / "student-tests" / "failures.compose.yml",
)
FAULT_HOOK = ROOT / "student-tests" / "fault_hook.py"
PROJECT_SUFFIX = "-failures"
READY_TIMEOUT_S = 60
LOG_TAIL_LINES = 50
# признаки аварийного перезапуска PostgreSQL: postmaster счёл аварией выход
# неизвестного потомка, завершил все процессы и восстанавливается. Штатные
# stop / start дают «database system was shut down», обрыв backend — FATAL
CRASH_MARKERS = (
"untracked child process",
"terminating any other active server processes",
"reinitializing",
"automatic recovery in progress",
)
CRASH_LINES_MAX = 20
# «тесный» API: короткие сроки сессии и ключа идемпотентности, пул из
# одного соединения
TIGHT_ENV = {
"AUTH__SESSION_TTL_SECONDS": "2",
"IDEMPOTENCY__TTL_SECONDS": "2",
"DATABASE__POOL_SIZE": "1",
"DATABASE__MAX_OVERFLOW": "0",
"DATABASE__POOL_TIMEOUT_SECONDS": "1",
"DATABASE__REQUEST_TIMEOUT_SECONDS": "2",
}
def say(message: str) -> None:
print(f"[failure_checks] {message}", flush=True)
def run(
*args: str,
check: bool = True,
quiet: bool = False,
env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess[str]:
"""Команда из корня репозитория; вывод — в терминал, если не quiet."""
return subprocess.run(
args,
cwd=ROOT,
check=check,
text=True,
capture_output=quiet,
env=env,
)
def dc(
project: str,
*args: str,
check: bool = True,
quiet: bool = False,
) -> subprocess.CompletedProcess[str]:
"""docker compose проекта отказов; без `-p` не вызывается никогда."""
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
return run(
"docker",
"compose",
"-p",
project,
*files,
*args,
check=check,
quiet=quiet,
)
def working_project() -> str:
config = run("docker", "compose", "config", "--format", "json", quiet=True)
return json.loads(config.stdout)["name"]
def app_port() -> str:
"""`APP__PORT` из `.env`: на нём слушает API внутри контейнера."""
if ENV_FILE.exists():
for line in ENV_FILE.read_text(encoding="utf-8").splitlines():
if line.strip().startswith("APP__PORT="):
return line.split("=", 1)[1].strip()
return "8080"
def free_port() -> int:
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
return sock.getsockname()[1]
def wait_ready(url: str, container: str) -> None:
deadline = time.monotonic() + READY_TIMEOUT_S
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(url, timeout=2) as response:
if response.status == 200:
return
except (urllib.error.URLError, OSError):
pass
state = run(
"docker",
"inspect",
"-f",
"{{.State.Running}}",
container,
check=False,
quiet=True,
)
if state.stdout.strip() != "true":
raise RuntimeError(f"container {container} is not running")
time.sleep(1)
raise RuntimeError(f"{url} not ready in {READY_TIMEOUT_S}s")
def leftovers(project: str, kind: str) -> str:
"""Контейнеры (`ps -a`) или volume проекта, которые ещё остались."""
command = ("ps", "-aq") if kind == "container" else ("volume", "ls", "-q")
return run(
"docker",
*command,
"--filter",
f"label=com.docker.compose.project={project}",
quiet=True,
check=False,
).stdout.strip()
def cleanup(project: str, containers: list[str]) -> bool:
"""Контейнеры API, сервисы и volume проекта отказов; True — чисто."""
for name in containers:
run("docker", "rm", "-f", name, check=False, quiet=True)
down = dc(project, "down", "-v", "--remove-orphans", check=False)
return (
down.returncode == 0
and not leftovers(project, "container")
and not leftovers(project, "volume")
)
def postgres_log_check(project: str) -> int:
"""Журнал PostgreSQL стенда отказов: сводка всегда, признаки аварии —
строками; 1 — признаки есть или журнал не прочитан. Два счётчика
FATAL показывают, что заполнение и освобождение слотов правда были."""
log = dc(
project,
"logs",
"--no-log-prefix",
"postgres",
check=False,
quiet=True,
)
if log.returncode != 0:
# сбой чтения журнала — не сбой подготовки стенда
tail = log.stderr.strip().splitlines()[-1:] or ["no stderr"]
say(
f"postgres log unavailable: docker compose logs exited "
f"{log.returncode}: {tail[0]}",
)
return 1
lines = (log.stdout + log.stderr).splitlines()
crashes = [line for line in lines if any(m in line for m in CRASH_MARKERS)]
too_many = sum("too many clients" in line for line in lines)
admin = sum("administrator command" in line for line in lines)
say(
f"postgres log: {len(lines)} lines, crash markers {len(crashes)}; "
f'"too many clients" {too_many}, "administrator command" {admin}',
)
if not crashes:
return 0
say("crash markers in the postgres log:")
for line in crashes[:CRASH_LINES_MAX]:
print(line, flush=True)
return 1
def start_api(
project: str,
name: str,
port: int,
env: dict[str, str],
) -> str:
env_args = [
arg for key, value in env.items() for arg in ("-e", f"{key}={value}")
]
dc(
project,
"run",
"-d",
"--no-deps",
"--name",
name,
*env_args,
"-p",
f"127.0.0.1:{port}:{app_port()}",
"api",
quiet=True,
)
base_url = f"http://127.0.0.1:{port}"
wait_ready(f"{base_url}/health/ready", name)
return base_url
def main() -> int:
pytest_args = sys.argv[1:]
working = working_project()
project = f"{working}{PROJECT_SUFFIX}"
if project == working or not project.endswith(PROJECT_SUFFIX):
say(f"refusing to use project {project!r}")
return 2
default_name = f"{project}-api"
tight_name = f"{project}-api-tight"
containers = [default_name, tight_name]
say(f"project={project} (working stand: {working})")
if not cleanup(project, containers):
say("leftovers of a previous run could not be removed")
return 1
results: dict[str, int] = {}
failed = True
try:
say("build image")
dc(project, "build", "api")
say("start postgres and valkey")
dc(project, "up", "-d", "--wait", "postgres", "valkey")
say("migrations")
dc(project, "run", "--rm", "migrate")
say("start api containers")
base_url = start_api(project, default_name, free_port(), {})
tight_url = start_api(project, tight_name, free_port(), TIGHT_ENV)
say(f"ready: api={base_url} tight={tight_url}")
env = os.environ | {
"BASE_URL": base_url,
"FAULT_TIGHT_BASE_URL": tight_url,
"FAULT_HOOK": str(FAULT_HOOK),
"FAULT_PROJECT": project,
}
say("pytest -m fault")
results["fault"] = run(
"uv",
"run",
"pytest",
"-m",
"fault",
*pytest_args,
check=False,
env=env,
).returncode
failed = bool(results["fault"])
# стенд ещё жив: журнал PostgreSQL до уборки
results["postgres"] = postgres_log_check(project)
except KeyboardInterrupt:
say("interrupted")
results["interrupted"] = 130
except (subprocess.CalledProcessError, RuntimeError) as e:
say(f"setup failed: {e}")
results["setup"] = 1
finally:
# повторный Ctrl+C не должен оборвать уборку на полпути
signal.signal(signal.SIGINT, signal.SIG_IGN)
if failed:
for name in containers:
say(f"last {LOG_TAIL_LINES} lines of {name}")
run(
"docker",
"logs",
"--tail",
str(LOG_TAIL_LINES),
name,
check=False,
)
say("cleanup")
if not cleanup(project, containers):
say("cleanup failed")
results["cleanup"] = 1
say(f"results: {results}")
return next((code for code in results.values() if code), 0)
if __name__ == "__main__":
sys.exit(main())