- upload: delete the stored object when ready fails before COMMIT or the pending row is gone (503 instead of 500); unknown COMMIT outcome keeps it - cleaner service (python -m src.cleaner): stale pending rows older than grace (900 s, must exceed 17 s live-upload bound) under FOR UPDATE SKIP LOCKED, object before row, own S3 user with read/delete only - shared session_scope for API and cleaner; early_stop process name - partial index on pending attachments, CREATE INDEX CONCURRENTLY - s3-bootstrap: cleaner user, policy and checks; make env_setup appends missing template variables to an existing .env - failure checks: cleaner on the failure stand, id-list hook actions, five new and three updated tests; driver upload-consistency - sync upload-consistency deltas into main specs (storage-consistency) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
393 lines
14 KiB
Python
393 lines
14 KiB
Python
"""`make failure_checks`: проверки отказов на отдельном стенде.
|
||
|
||
Поднимает тот же `docker-compose.yml` под проектом `<рабочий>-failures`
|
||
с override `student-tests/failures.compose.yml`: свои PostgreSQL, Valkey и
|
||
S3 со своими volumes, порты хранилищ не публикуются. Затем:
|
||
- собирает образ из рабочего дерева, применяет миграции и bootstrap S3;
|
||
- запускает уборщика незавершённых загрузок с grace 20 с и интервалом
|
||
1 с (`failures.compose.yml`; design change `upload-consistency`,
|
||
решение 10) и передаёт оба значения тестам;
|
||
- запускает два контейнера API: обычный (с `MAX_UPLOAD_BYTES=1` и
|
||
`MAX_IMAGE_PIXELS=1` в окружении — лимиты вложений от них не зависят) и
|
||
«тесный» (сроки сессии и ключа идемпотентности 2 с, пул из одного
|
||
соединения);
|
||
- прогоняет тесты с маркером `fault` (`pytest -m fault`), передав им hook
|
||
`student-tests/fault_hook.py`; при падении печатает журналы API и
|
||
уборщика;
|
||
- после тестов читает журнал PostgreSQL стенда: печатает сводку и падает
|
||
на признаках аварийного перезапуска (решение 12 design change
|
||
`fix-known-issues`);
|
||
- удаляет стенд вместе с volume при любом исходе, в том числе по Ctrl+C.
|
||
|
||
Рабочий стенд, его база, кэш и хранилище объектов не затрагиваются. Только stdlib. Решения
|
||
1, 2, 5 design change `failure-checks`. Запускается `$(PYTHON)`, а не
|
||
`uv run`, по той же причине, что `run_isolated.py`.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import signal
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
ENV_FILE = ROOT / ".env"
|
||
COMPOSE_FILES = (
|
||
ROOT / "docker-compose.yml",
|
||
ROOT / "student-tests" / "failures.compose.yml",
|
||
)
|
||
FAULT_HOOK = ROOT / "student-tests" / "fault_hook.py"
|
||
PROJECT_SUFFIX = "-failures"
|
||
READY_TIMEOUT_S = 60
|
||
LOG_TAIL_LINES = 50
|
||
# признаки аварийного перезапуска PostgreSQL: postmaster счёл аварией выход
|
||
# неизвестного потомка, завершил все процессы и восстанавливается. Штатные
|
||
# stop / start дают «database system was shut down», обрыв backend — FATAL
|
||
CRASH_MARKERS = (
|
||
"untracked child process",
|
||
"terminating any other active server processes",
|
||
"reinitializing",
|
||
"automatic recovery in progress",
|
||
)
|
||
CRASH_LINES_MAX = 20
|
||
# «тесный» API: короткие сроки сессии и ключа идемпотентности, пул из
|
||
# одного соединения
|
||
TIGHT_ENV = {
|
||
"AUTH__SESSION_TTL_SECONDS": "2",
|
||
"IDEMPOTENCY__TTL_SECONDS": "2",
|
||
"DATABASE__POOL_SIZE": "1",
|
||
"DATABASE__MAX_OVERFLOW": "0",
|
||
"DATABASE__POOL_TIMEOUT_SECONDS": "1",
|
||
"DATABASE__REQUEST_TIMEOUT_SECONDS": "2",
|
||
}
|
||
# обычный API: имена лимитов skeleton с заведомо малыми значениями. Лимиты
|
||
# вложений — константы контракта, и окружение их не меняет (спек
|
||
# `service-configuration`, «Окружение не меняет лимит»): тест грузит
|
||
# `sample.png` и ждёт 201. Те же пары тест получает в
|
||
# `FAULT_LIMITS_ENV` и сверяет, что стенд запущен с ними
|
||
LIMITS_ENV = {
|
||
"MAX_UPLOAD_BYTES": "1",
|
||
"MAX_IMAGE_PIXELS": "1",
|
||
}
|
||
|
||
|
||
def say(message: str) -> None:
|
||
print(f"[failure_checks] {message}", flush=True)
|
||
|
||
|
||
def run(
|
||
*args: str,
|
||
check: bool = True,
|
||
quiet: bool = False,
|
||
env: dict[str, str] | None = None,
|
||
) -> subprocess.CompletedProcess[str]:
|
||
"""Команда из корня репозитория; вывод — в терминал, если не quiet."""
|
||
return subprocess.run(
|
||
args,
|
||
cwd=ROOT,
|
||
check=check,
|
||
text=True,
|
||
capture_output=quiet,
|
||
env=env,
|
||
)
|
||
|
||
|
||
def dc(
|
||
project: str,
|
||
*args: str,
|
||
check: bool = True,
|
||
quiet: bool = False,
|
||
) -> subprocess.CompletedProcess[str]:
|
||
"""docker compose проекта отказов; без `-p` не вызывается никогда."""
|
||
files = [arg for path in COMPOSE_FILES for arg in ("-f", str(path))]
|
||
return run(
|
||
"docker",
|
||
"compose",
|
||
"-p",
|
||
project,
|
||
*files,
|
||
*args,
|
||
check=check,
|
||
quiet=quiet,
|
||
)
|
||
|
||
|
||
def working_project() -> str:
|
||
config = run("docker", "compose", "config", "--format", "json", quiet=True)
|
||
return json.loads(config.stdout)["name"]
|
||
|
||
|
||
def app_port() -> str:
|
||
"""`APP__PORT` из `.env`: на нём слушает API внутри контейнера."""
|
||
if ENV_FILE.exists():
|
||
for line in ENV_FILE.read_text(encoding="utf-8").splitlines():
|
||
if line.strip().startswith("APP__PORT="):
|
||
return line.split("=", 1)[1].strip()
|
||
return "8080"
|
||
|
||
|
||
def storage_names(project: str) -> dict[str, str]:
|
||
"""Bucket и имя базы стенда — из окружения сервиса `api` в его
|
||
конфигурации compose: по ним hook считает объекты вложений."""
|
||
config = dc(project, "config", "--format", "json", quiet=True)
|
||
environment = json.loads(config.stdout)["services"]["api"]["environment"]
|
||
return {
|
||
"FAULT_S3_BUCKET": environment["S3__BUCKET"],
|
||
"FAULT_DATABASE": environment["DATABASE__NAME"],
|
||
}
|
||
|
||
|
||
def cleaner_timings(project: str) -> dict[str, str]:
|
||
"""Grace и интервал уборщика стенда — из окружения сервиса `cleaner`
|
||
в конфигурации compose (`failures.compose.yml`): тесты ждут по ним."""
|
||
config = dc(project, "config", "--format", "json", quiet=True)
|
||
environment = json.loads(config.stdout)["services"]["cleaner"][
|
||
"environment"
|
||
]
|
||
return {
|
||
"FAULT_CLEANER_GRACE_S": environment["CLEANER__PENDING_GRACE_SECONDS"],
|
||
"FAULT_CLEANER_INTERVAL_S": environment["CLEANER__INTERVAL_SECONDS"],
|
||
}
|
||
|
||
|
||
def free_port() -> int:
|
||
with socket.socket() as sock:
|
||
sock.bind(("127.0.0.1", 0))
|
||
return sock.getsockname()[1]
|
||
|
||
|
||
def wait_ready(url: str, container: str) -> None:
|
||
deadline = time.monotonic() + READY_TIMEOUT_S
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
with urllib.request.urlopen(url, timeout=2) as response:
|
||
if response.status == 200:
|
||
return
|
||
except (urllib.error.URLError, OSError):
|
||
pass
|
||
state = run(
|
||
"docker",
|
||
"inspect",
|
||
"-f",
|
||
"{{.State.Running}}",
|
||
container,
|
||
check=False,
|
||
quiet=True,
|
||
)
|
||
if state.stdout.strip() != "true":
|
||
raise RuntimeError(f"container {container} is not running")
|
||
time.sleep(1)
|
||
raise RuntimeError(f"{url} not ready in {READY_TIMEOUT_S}s")
|
||
|
||
|
||
def leftovers(project: str, kind: str) -> str:
|
||
"""Контейнеры (`ps -a`) или volume проекта, которые ещё остались."""
|
||
command = ("ps", "-aq") if kind == "container" else ("volume", "ls", "-q")
|
||
return run(
|
||
"docker",
|
||
*command,
|
||
"--filter",
|
||
f"label=com.docker.compose.project={project}",
|
||
quiet=True,
|
||
check=False,
|
||
).stdout.strip()
|
||
|
||
|
||
def cleanup(project: str, containers: list[str]) -> bool:
|
||
"""Контейнеры API, сервисы и volume проекта отказов; True — чисто."""
|
||
for name in containers:
|
||
run("docker", "rm", "-f", name, check=False, quiet=True)
|
||
down = dc(project, "down", "-v", "--remove-orphans", check=False)
|
||
return (
|
||
down.returncode == 0
|
||
and not leftovers(project, "container")
|
||
and not leftovers(project, "volume")
|
||
)
|
||
|
||
|
||
def postgres_log_check(project: str) -> int:
|
||
"""Журнал PostgreSQL стенда отказов: сводка всегда, признаки аварии —
|
||
строками; 1 — признаки есть или журнал не прочитан. Два счётчика
|
||
FATAL показывают, что заполнение и освобождение слотов правда были."""
|
||
log = dc(
|
||
project,
|
||
"logs",
|
||
"--no-log-prefix",
|
||
"postgres",
|
||
check=False,
|
||
quiet=True,
|
||
)
|
||
if log.returncode != 0:
|
||
# сбой чтения журнала — не сбой подготовки стенда
|
||
tail = log.stderr.strip().splitlines()[-1:] or ["no stderr"]
|
||
say(
|
||
f"postgres log unavailable: docker compose logs exited "
|
||
f"{log.returncode}: {tail[0]}",
|
||
)
|
||
return 1
|
||
lines = (log.stdout + log.stderr).splitlines()
|
||
crashes = [line for line in lines if any(m in line for m in CRASH_MARKERS)]
|
||
too_many = sum("too many clients" in line for line in lines)
|
||
admin = sum("administrator command" in line for line in lines)
|
||
say(
|
||
f"postgres log: {len(lines)} lines, crash markers {len(crashes)}; "
|
||
f'"too many clients" {too_many}, "administrator command" {admin}',
|
||
)
|
||
if not crashes:
|
||
return 0
|
||
say("crash markers in the postgres log:")
|
||
for line in crashes[:CRASH_LINES_MAX]:
|
||
print(line, flush=True)
|
||
return 1
|
||
|
||
|
||
def start_api(
|
||
project: str,
|
||
name: str,
|
||
port: int,
|
||
env: dict[str, str],
|
||
) -> str:
|
||
env_args = [
|
||
arg for key, value in env.items() for arg in ("-e", f"{key}={value}")
|
||
]
|
||
dc(
|
||
project,
|
||
"run",
|
||
"-d",
|
||
"--no-deps",
|
||
"--name",
|
||
name,
|
||
*env_args,
|
||
"-p",
|
||
f"127.0.0.1:{port}:{app_port()}",
|
||
"api",
|
||
quiet=True,
|
||
)
|
||
base_url = f"http://127.0.0.1:{port}"
|
||
wait_ready(f"{base_url}/health/ready", name)
|
||
return base_url
|
||
|
||
|
||
def main() -> int:
|
||
pytest_args = sys.argv[1:]
|
||
working = working_project()
|
||
project = f"{working}{PROJECT_SUFFIX}"
|
||
if project == working or not project.endswith(PROJECT_SUFFIX):
|
||
say(f"refusing to use project {project!r}")
|
||
return 2
|
||
default_name = f"{project}-api"
|
||
tight_name = f"{project}-api-tight"
|
||
containers = [default_name, tight_name]
|
||
|
||
say(f"project={project} (working stand: {working})")
|
||
if not cleanup(project, containers):
|
||
say("leftovers of a previous run could not be removed")
|
||
return 1
|
||
|
||
results: dict[str, int] = {}
|
||
failed = True
|
||
try:
|
||
say("build image")
|
||
dc(project, "build", "api")
|
||
say("start postgres, valkey and s3")
|
||
dc(project, "up", "-d", "--wait", "postgres", "valkey", "s3")
|
||
say("migrations")
|
||
dc(project, "run", "--rm", "migrate")
|
||
# --wait одноразовый сервис считает готовым уже запущенным: код
|
||
# выхода bootstrap — отдельным up, как make start_s3
|
||
say("s3 bootstrap")
|
||
dc(
|
||
project,
|
||
"up",
|
||
"--no-deps",
|
||
"--exit-code-from",
|
||
"s3-bootstrap",
|
||
"s3-bootstrap",
|
||
)
|
||
timings = cleaner_timings(project)
|
||
say(
|
||
f"start cleaner: grace {timings['FAULT_CLEANER_GRACE_S']} s, "
|
||
f"interval {timings['FAULT_CLEANER_INTERVAL_S']} s",
|
||
)
|
||
dc(project, "up", "-d", "--no-deps", "cleaner")
|
||
say("start api containers")
|
||
base_url = start_api(project, default_name, free_port(), LIMITS_ENV)
|
||
tight_url = start_api(project, tight_name, free_port(), TIGHT_ENV)
|
||
say(f"ready: api={base_url} tight={tight_url}")
|
||
env = (
|
||
os.environ
|
||
| {
|
||
"BASE_URL": base_url,
|
||
"FAULT_TIGHT_BASE_URL": tight_url,
|
||
"FAULT_HOOK": str(FAULT_HOOK),
|
||
"FAULT_PROJECT": project,
|
||
"FAULT_LIMITS_ENV": " ".join(
|
||
f"{key}={value}" for key, value in LIMITS_ENV.items()
|
||
),
|
||
}
|
||
| storage_names(project)
|
||
| timings
|
||
)
|
||
say("pytest -m fault")
|
||
results["fault"] = run(
|
||
"uv",
|
||
"run",
|
||
"pytest",
|
||
"-m",
|
||
"fault",
|
||
*pytest_args,
|
||
check=False,
|
||
env=env,
|
||
).returncode
|
||
failed = bool(results["fault"])
|
||
# стенд ещё жив: журнал PostgreSQL до уборки
|
||
results["postgres"] = postgres_log_check(project)
|
||
except KeyboardInterrupt:
|
||
say("interrupted")
|
||
results["interrupted"] = 130
|
||
except (subprocess.CalledProcessError, RuntimeError) as e:
|
||
say(f"setup failed: {e}")
|
||
results["setup"] = 1
|
||
finally:
|
||
# повторный Ctrl+C не должен оборвать уборку на полпути
|
||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||
if failed:
|
||
for name in containers:
|
||
say(f"last {LOG_TAIL_LINES} lines of {name}")
|
||
run(
|
||
"docker",
|
||
"logs",
|
||
"--tail",
|
||
str(LOG_TAIL_LINES),
|
||
name,
|
||
check=False,
|
||
)
|
||
say(f"last {LOG_TAIL_LINES} lines of cleaner")
|
||
dc(
|
||
project,
|
||
"logs",
|
||
"--no-log-prefix",
|
||
"--tail",
|
||
str(LOG_TAIL_LINES),
|
||
"cleaner",
|
||
check=False,
|
||
)
|
||
say("cleanup")
|
||
if not cleanup(project, containers):
|
||
say("cleanup failed")
|
||
results["cleanup"] = 1
|
||
|
||
say(f"results: {results}")
|
||
return next((code for code in results.values() if code), 0)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|