- s3 and s3-bootstrap services: SeaweedFS 4.48 pinned by digest, internals on loopback, signing key, private bucket, API user policy, privacy checks on every bootstrap - [s3] settings and aiobotocore client in src/infra/s3: deadline per call, no retries, 503 on outage or rejected credentials - make start_s3, stop_s3, s3_check; run_dev, test_isolated, wipe_data and restart hook include S3 - failure_checks with own S3 and readiness check without it; bench generates S3 secrets and filters them out of the passport - rules, AGENTS.md, skill; sync health delta; KNOWN_ISSUES: rollout_from Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
330 lines
11 KiB
Python
330 lines
11 KiB
Python
"""`make failure_checks`: проверки отказов на отдельном стенде.
|
||
|
||
Поднимает тот же `docker-compose.yml` под проектом `<рабочий>-failures`
|
||
с override `student-tests/failures.compose.yml`: свои PostgreSQL, Valkey и
|
||
S3 со своими volumes, порты хранилищ не публикуются. Затем:
|
||
- собирает образ из рабочего дерева, применяет миграции и bootstrap S3;
|
||
- запускает два контейнера API: обычный и «тесный» (сроки сессии и ключа
|
||
идемпотентности 2 с, пул из одного соединения);
|
||
- прогоняет тесты с маркером `fault` (`pytest -m fault`), передав им hook
|
||
`student-tests/fault_hook.py`;
|
||
- после тестов читает журнал PostgreSQL стенда: печатает сводку и падает
|
||
на признаках аварийного перезапуска (решение 12 design change
|
||
`fix-known-issues`);
|
||
- удаляет стенд вместе с volume при любом исходе, в том числе по Ctrl+C.
|
||
|
||
Рабочий стенд, его база, кэш и хранилище объектов не затрагиваются. Только stdlib. Решения
|
||
1, 2, 5 design change `failure-checks`. Запускается `$(PYTHON)`, а не
|
||
`uv run`, по той же причине, что `run_isolated.py`.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import signal
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
ENV_FILE = ROOT / ".env"
|
||
COMPOSE_FILES = (
|
||
ROOT / "docker-compose.yml",
|
||
ROOT / "student-tests" / "failures.compose.yml",
|
||
)
|
||
FAULT_HOOK = ROOT / "student-tests" / "fault_hook.py"
|
||
PROJECT_SUFFIX = "-failures"
|
||
READY_TIMEOUT_S = 60
|
||
LOG_TAIL_LINES = 50
|
||
# признаки аварийного перезапуска PostgreSQL: postmaster счёл аварией выход
|
||
# неизвестного потомка, завершил все процессы и восстанавливается. Штатные
|
||
# stop / start дают «database system was shut down», обрыв backend — FATAL
|
||
CRASH_MARKERS = (
|
||
"untracked child process",
|
||
"terminating any other active server processes",
|
||
"reinitializing",
|
||
"automatic recovery in progress",
|
||
)
|
||
CRASH_LINES_MAX = 20
|
||
# «тесный» API: короткие сроки сессии и ключа идемпотентности, пул из
|
||
# одного соединения
|
||
TIGHT_ENV = {
|
||
"AUTH__SESSION_TTL_SECONDS": "2",
|
||
"IDEMPOTENCY__TTL_SECONDS": "2",
|
||
"DATABASE__POOL_SIZE": "1",
|
||
"DATABASE__MAX_OVERFLOW": "0",
|
||
"DATABASE__POOL_TIMEOUT_SECONDS": "1",
|
||
"DATABASE__REQUEST_TIMEOUT_SECONDS": "2",
|
||
}
|
||
|
||
|
||
def say(message: str) -> None:
|
||
print(f"[failure_checks] {message}", flush=True)
|
||
|
||
|
||
def run(
|
||
*args: str,
|
||
check: bool = True,
|
||
quiet: bool = False,
|
||
env: dict[str, str] | None = None,
|
||
) -> subprocess.CompletedProcess[str]:
|
||
"""Команда из корня репозитория; вывод — в терминал, если не quiet."""
|
||
return subprocess.run(
|
||
args,
|
||
cwd=ROOT,
|
||
check=check,
|
||
text=True,
|
||
capture_output=quiet,
|
||
env=env,
|
||
)
|
||
|
||
|
||
def dc(
|
||
project: str,
|
||
*args: str,
|
||
check: bool = True,
|
||
quiet: bool = False,
|
||
) -> subprocess.CompletedProcess[str]:
|
||
"""docker compose проекта отказов; без `-p` не вызывается никогда."""
|
||
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
|
||
return run(
|
||
"docker",
|
||
"compose",
|
||
"-p",
|
||
project,
|
||
*files,
|
||
*args,
|
||
check=check,
|
||
quiet=quiet,
|
||
)
|
||
|
||
|
||
def working_project() -> str:
|
||
config = run("docker", "compose", "config", "--format", "json", quiet=True)
|
||
return json.loads(config.stdout)["name"]
|
||
|
||
|
||
def app_port() -> str:
|
||
"""`APP__PORT` из `.env`: на нём слушает API внутри контейнера."""
|
||
if ENV_FILE.exists():
|
||
for line in ENV_FILE.read_text(encoding="utf-8").splitlines():
|
||
if line.strip().startswith("APP__PORT="):
|
||
return line.split("=", 1)[1].strip()
|
||
return "8080"
|
||
|
||
|
||
def free_port() -> int:
|
||
with socket.socket() as sock:
|
||
sock.bind(("127.0.0.1", 0))
|
||
return sock.getsockname()[1]
|
||
|
||
|
||
def wait_ready(url: str, container: str) -> None:
|
||
deadline = time.monotonic() + READY_TIMEOUT_S
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
with urllib.request.urlopen(url, timeout=2) as response:
|
||
if response.status == 200:
|
||
return
|
||
except (urllib.error.URLError, OSError):
|
||
pass
|
||
state = run(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}}",
|
||
container,
|
||
check=False,
|
||
quiet=True,
|
||
)
|
||
if state.stdout.strip() != "true":
|
||
raise RuntimeError(f"container {container} is not running")
|
||
time.sleep(1)
|
||
raise RuntimeError(f"{url} not ready in {READY_TIMEOUT_S}s")
|
||
|
||
|
||
def leftovers(project: str, kind: str) -> str:
|
||
"""Контейнеры (`ps -a`) или volume проекта, которые ещё остались."""
|
||
command = ("ps", "-aq") if kind == "container" else ("volume", "ls", "-q")
|
||
return run(
|
||
"docker",
|
||
*command,
|
||
"--filter",
|
||
f"label=com.docker.compose.project={project}",
|
||
quiet=True,
|
||
check=False,
|
||
).stdout.strip()
|
||
|
||
|
||
def cleanup(project: str, containers: list[str]) -> bool:
|
||
"""Контейнеры API, сервисы и volume проекта отказов; True — чисто."""
|
||
for name in containers:
|
||
run("docker", "rm", "-f", name, check=False, quiet=True)
|
||
down = dc(project, "down", "-v", "--remove-orphans", check=False)
|
||
return (
|
||
down.returncode == 0
|
||
and not leftovers(project, "container")
|
||
and not leftovers(project, "volume")
|
||
)
|
||
|
||
|
||
def postgres_log_check(project: str) -> int:
|
||
"""Журнал PostgreSQL стенда отказов: сводка всегда, признаки аварии —
|
||
строками; 1 — признаки есть или журнал не прочитан. Два счётчика
|
||
FATAL показывают, что заполнение и освобождение слотов правда были."""
|
||
log = dc(
|
||
project,
|
||
"logs",
|
||
"--no-log-prefix",
|
||
"postgres",
|
||
check=False,
|
||
quiet=True,
|
||
)
|
||
if log.returncode != 0:
|
||
# сбой чтения журнала — не сбой подготовки стенда
|
||
tail = log.stderr.strip().splitlines()[-1:] or ["no stderr"]
|
||
say(
|
||
f"postgres log unavailable: docker compose logs exited "
|
||
f"{log.returncode}: {tail[0]}",
|
||
)
|
||
return 1
|
||
lines = (log.stdout + log.stderr).splitlines()
|
||
crashes = [line for line in lines if any(m in line for m in CRASH_MARKERS)]
|
||
too_many = sum("too many clients" in line for line in lines)
|
||
admin = sum("administrator command" in line for line in lines)
|
||
say(
|
||
f"postgres log: {len(lines)} lines, crash markers {len(crashes)}; "
|
||
f'"too many clients" {too_many}, "administrator command" {admin}',
|
||
)
|
||
if not crashes:
|
||
return 0
|
||
say("crash markers in the postgres log:")
|
||
for line in crashes[:CRASH_LINES_MAX]:
|
||
print(line, flush=True)
|
||
return 1
|
||
|
||
|
||
def start_api(
|
||
project: str,
|
||
name: str,
|
||
port: int,
|
||
env: dict[str, str],
|
||
) -> str:
|
||
env_args = [
|
||
arg for key, value in env.items() for arg in ("-e", f"{key}={value}")
|
||
]
|
||
dc(
|
||
project,
|
||
"run",
|
||
"-d",
|
||
"--no-deps",
|
||
"--name",
|
||
name,
|
||
*env_args,
|
||
"-p",
|
||
f"127.0.0.1:{port}:{app_port()}",
|
||
"api",
|
||
quiet=True,
|
||
)
|
||
base_url = f"http://127.0.0.1:{port}"
|
||
wait_ready(f"{base_url}/health/ready", name)
|
||
return base_url
|
||
|
||
|
||
def main() -> int:
|
||
pytest_args = sys.argv[1:]
|
||
working = working_project()
|
||
project = f"{working}{PROJECT_SUFFIX}"
|
||
if project == working or not project.endswith(PROJECT_SUFFIX):
|
||
say(f"refusing to use project {project!r}")
|
||
return 2
|
||
default_name = f"{project}-api"
|
||
tight_name = f"{project}-api-tight"
|
||
containers = [default_name, tight_name]
|
||
|
||
say(f"project={project} (working stand: {working})")
|
||
if not cleanup(project, containers):
|
||
say("leftovers of a previous run could not be removed")
|
||
return 1
|
||
|
||
results: dict[str, int] = {}
|
||
failed = True
|
||
try:
|
||
say("build image")
|
||
dc(project, "build", "api")
|
||
say("start postgres, valkey and s3")
|
||
dc(project, "up", "-d", "--wait", "postgres", "valkey", "s3")
|
||
say("migrations")
|
||
dc(project, "run", "--rm", "migrate")
|
||
# --wait одноразовый сервис считает готовым уже запущенным: код
|
||
# выхода bootstrap — отдельным up, как make start_s3
|
||
say("s3 bootstrap")
|
||
dc(
|
||
project,
|
||
"up",
|
||
"--no-deps",
|
||
"--exit-code-from",
|
||
"s3-bootstrap",
|
||
"s3-bootstrap",
|
||
)
|
||
say("start api containers")
|
||
base_url = start_api(project, default_name, free_port(), {})
|
||
tight_url = start_api(project, tight_name, free_port(), TIGHT_ENV)
|
||
say(f"ready: api={base_url} tight={tight_url}")
|
||
env = os.environ | {
|
||
"BASE_URL": base_url,
|
||
"FAULT_TIGHT_BASE_URL": tight_url,
|
||
"FAULT_HOOK": str(FAULT_HOOK),
|
||
"FAULT_PROJECT": project,
|
||
}
|
||
say("pytest -m fault")
|
||
results["fault"] = run(
|
||
"uv",
|
||
"run",
|
||
"pytest",
|
||
"-m",
|
||
"fault",
|
||
*pytest_args,
|
||
check=False,
|
||
env=env,
|
||
).returncode
|
||
failed = bool(results["fault"])
|
||
# стенд ещё жив: журнал PostgreSQL до уборки
|
||
results["postgres"] = postgres_log_check(project)
|
||
except KeyboardInterrupt:
|
||
say("interrupted")
|
||
results["interrupted"] = 130
|
||
except (subprocess.CalledProcessError, RuntimeError) as e:
|
||
say(f"setup failed: {e}")
|
||
results["setup"] = 1
|
||
finally:
|
||
# повторный Ctrl+C не должен оборвать уборку на полпути
|
||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||
if failed:
|
||
for name in containers:
|
||
say(f"last {LOG_TAIL_LINES} lines of {name}")
|
||
run(
|
||
"docker",
|
||
"logs",
|
||
"--tail",
|
||
str(LOG_TAIL_LINES),
|
||
name,
|
||
check=False,
|
||
)
|
||
say("cleanup")
|
||
if not cleanup(project, containers):
|
||
say("cleanup failed")
|
||
results["cleanup"] = 1
|
||
|
||
say(f"results: {results}")
|
||
return next((code for code in results.values() if code), 0)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|