This reverts commit53eb38e(the pool change9c3959d,623a973,80e64cd). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
426 lines
16 KiB
Python
426 lines
16 KiB
Python
"""Проверки отказов: выполняет только `make failure_checks` (маркер `fault`).
|
|
|
|
Стенд и hook готовит `student-tests/run_failures.py`: свои Postgres и
|
|
Valkey, обычный API (`BASE_URL`) и «тесный» (сроки сессии и ключа
|
|
идемпотентности 2 с, пул из одного соединения). На хранилища тесты действуют только через фикстуру
|
|
`stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в
|
|
рабочее состояние. Решение 4 design change `failure-checks`.
|
|
"""
|
|
|
|
import statistics
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from collections.abc import Callable, Iterator
|
|
from contextlib import contextmanager
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from api_client import API, Actor, Api, new_key
|
|
from contract import (
|
|
BEARER_CHALLENGE,
|
|
assert_error,
|
|
assert_ok,
|
|
assert_unauthorized,
|
|
)
|
|
|
|
pytestmark = pytest.mark.fault
|
|
|
|
type Stand = Callable[..., str]
|
|
|
|
RESTORE = {"stop": "start", "pause": "unpause"}
|
|
# запас к границам времени: планировщик Docker Desktop
|
|
SLACK_S = 0.3
|
|
# [cache] timeout_seconds и retry_after_seconds по умолчанию
|
|
CACHE_TIMEOUT_S = 0.2
|
|
CACHE_RETRY_AFTER_S = 5
|
|
# окружение «тесного» API (TIGHT_ENV в run_failures.py)
|
|
SESSION_TTL_S = 2
|
|
IDEMPOTENCY_TTL_S = 2
|
|
POOL_TIMEOUT_S = 1
|
|
# срок проверки Postgres в /health/ready
|
|
READINESS_TIMEOUT_S = 2
|
|
# сколько секунд hook держит слоты max_connections; clear-connections
|
|
# освобождает их раньше
|
|
FILL_HOLD_S = 30
|
|
ME = f"{API}/users/me"
|
|
|
|
|
|
@contextmanager
|
|
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
|
|
"""`stop` или `pause` хранилища; в `finally` — обратно."""
|
|
stand(action, service)
|
|
try:
|
|
yield
|
|
finally:
|
|
stand(RESTORE[action], service)
|
|
|
|
|
|
def timed(
|
|
call: Callable[[], httpx.Response],
|
|
) -> tuple[httpx.Response, float]:
|
|
started = time.monotonic()
|
|
resp = call()
|
|
return resp, time.monotonic() - started
|
|
|
|
|
|
def read_chat(api: Api, actor: Actor, chat: dict[str, Any]) -> httpx.Response:
|
|
return api.request("GET", f"{API}/chats/{chat['id']}", actor=actor)
|
|
|
|
|
|
def wait_cache_ready(
|
|
api: Api,
|
|
stand: Stand,
|
|
actor: Actor,
|
|
chat: dict[str, Any],
|
|
) -> None:
|
|
"""Дождаться, пока чтение снова пишет карточку в кэш.
|
|
|
|
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
|
|
"""
|
|
deadline = time.monotonic() + CACHE_RETRY_AFTER_S + 2
|
|
stand("flush-cache")
|
|
while time.monotonic() < deadline:
|
|
assert_ok(read_chat(api, actor, chat), 200, "Chat")
|
|
if stand("cache-has-chat", chat["id"]) == "1":
|
|
return
|
|
time.sleep(0.5)
|
|
raise AssertionError("cache did not come back")
|
|
|
|
|
|
def chat_titles(api: Api, actor: Actor) -> set[str]:
|
|
return {
|
|
chat["title"]
|
|
for page in api.chat_pages(actor)
|
|
for chat in page["items"]
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Сессии: срок 2 с на «тесном» API, БД остановлена — на обычном
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.spec("sessions", "Истёкший токен")
|
|
def test_expired_token(tight_api: Api) -> None:
|
|
actor = tight_api.register()
|
|
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
|
|
time.sleep(SESSION_TTL_S + 1)
|
|
assert_unauthorized(
|
|
tight_api.request("GET", ME, actor=actor),
|
|
BEARER_CHALLENGE,
|
|
)
|
|
|
|
|
|
@pytest.mark.spec("sessions", "Срок не продлевается")
|
|
def test_use_does_not_extend_session(tight_api: Api) -> None:
|
|
actor = tight_api.register()
|
|
logged_in = time.monotonic()
|
|
successes = 0
|
|
while time.monotonic() - logged_in < SESSION_TTL_S - SLACK_S:
|
|
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
|
|
successes += 1
|
|
time.sleep(0.5)
|
|
assert successes >= 3, "too few requests before expiry"
|
|
time.sleep(max(0.0, logged_in + SESSION_TTL_S + 1 - time.monotonic()))
|
|
assert_unauthorized(
|
|
tight_api.request("GET", ME, actor=actor),
|
|
BEARER_CHALLENGE,
|
|
)
|
|
|
|
|
|
@pytest.mark.spec("sessions", "БД остановлена при запросе с токеном")
|
|
def test_token_while_database_stopped(api: Api, stand: Stand) -> None:
|
|
actor = api.register()
|
|
with outage(stand, "stop", "postgres"):
|
|
resp = api.request("GET", ME, actor=actor)
|
|
assert_error(resp, 503)
|
|
# недоступность хранилища — не отказ в аутентификации
|
|
assert "WWW-Authenticate" not in resp.headers
|
|
assert_ok(api.request("GET", ME, actor=actor), 200, "User")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Хранение: откат, остановленная БД, исчерпанный пул
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.spec("data-storage", "Сбой при добавлении участников")
|
|
def test_members_failure_rolls_back_chat(
|
|
api: Api,
|
|
stand: Stand,
|
|
alice: Actor,
|
|
bob: Actor,
|
|
) -> None:
|
|
title = f"fault-{uuid.uuid4().hex}"
|
|
stand("fail-members", title)
|
|
try:
|
|
resp = api.request(
|
|
"POST",
|
|
f"{API}/chats",
|
|
actor=alice,
|
|
json={"title": title, "member_ids": [alice.id, bob.id]},
|
|
)
|
|
assert_error(resp, 500)
|
|
finally:
|
|
stand("clear-fail-members")
|
|
for actor in (alice, bob):
|
|
assert title not in chat_titles(api, actor)
|
|
assert stand("count-chats", title) == "0"
|
|
|
|
|
|
@pytest.mark.spec("data-storage", "БД остановлена")
|
|
@pytest.mark.spec("data-storage", "БД вернулась")
|
|
def test_database_stopped_and_back(
|
|
api: Api,
|
|
stand: Stand,
|
|
alice: Actor,
|
|
chat: dict[str, Any],
|
|
) -> None:
|
|
with outage(stand, "stop", "postgres"):
|
|
resp, took = timed(
|
|
lambda: api.post_message(
|
|
alice,
|
|
chat["id"],
|
|
json={"text": "во время отказа"},
|
|
),
|
|
)
|
|
error = assert_error(resp, 503)
|
|
assert took < 2, f"503 took {took:.2f}s"
|
|
for word in ("postgres", "sql", "connect"):
|
|
assert word not in error["message"].lower()
|
|
message = api.send_message(alice, chat["id"], "после возврата")
|
|
# запись во время отказа не зафиксирована, новая — видна
|
|
assert api.all_messages(alice, chat["id"]) == [message]
|
|
|
|
|
|
@pytest.mark.spec("data-storage", "Соединения исчерпаны")
|
|
def test_pool_exhausted(tight_api: Api, stand: Stand) -> None:
|
|
actor = tight_api.register()
|
|
holder = Api(str(tight_api.client.base_url), timeout=15)
|
|
first: list[int] = []
|
|
|
|
def hold_connection() -> None:
|
|
first.append(holder.request("GET", ME, actor=actor).status_code)
|
|
|
|
with outage(stand, "pause", "postgres"):
|
|
thread = threading.Thread(target=hold_connection)
|
|
thread.start()
|
|
time.sleep(SLACK_S)
|
|
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
|
|
assert_error(resp, 503)
|
|
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
|
|
thread.join()
|
|
holder.close()
|
|
assert first == [503]
|
|
# срок сессии 2 с мог истечь за время паузы: проверяем пул новым входом
|
|
fresh = tight_api.register()
|
|
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
|
|
|
|
|
|
@pytest.mark.spec("data-storage", "Предел соединений PostgreSQL исчерпан")
|
|
def test_connection_limit_reached(tight_api: Api, stand: Stand) -> None:
|
|
actor = tight_api.register()
|
|
# hook оборвал соединение пула: при выдаче pool_pre_ping
|
|
# переподключается и получает «too many clients» (SQLSTATE 53300)
|
|
stand("fill-connections", str(FILL_HOLD_S))
|
|
try:
|
|
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
|
|
error = assert_error(resp, 503)
|
|
# отказ PostgreSQL мгновенный: ответ не ждёт срока пула
|
|
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
|
|
# недоступность хранилища — не отказ в аутентификации
|
|
assert "WWW-Authenticate" not in resp.headers
|
|
for word in ("postgres", "sql", "client", "connect"):
|
|
assert word not in error["message"].lower()
|
|
finally:
|
|
stand("clear-connections")
|
|
# слоты свободны: сервис снова обслуживает запросы без рестарта; срок
|
|
# сессии 2 с за время заполнения истёк — новый вход
|
|
fresh = tight_api.register()
|
|
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Readiness
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.spec("health", "Хранилище не готово")
|
|
def test_readiness_without_database(api: Api, stand: Stand) -> None:
|
|
with outage(stand, "stop", "postgres"):
|
|
resp, took = timed(lambda: api.request("GET", "/health/ready"))
|
|
assert_error(resp, 503)
|
|
assert took < READINESS_TIMEOUT_S + 0.5, f"ready took {took:.2f}s"
|
|
live = api.request("GET", "/health/live")
|
|
assert live.status_code == 200
|
|
assert api.request("GET", "/health/ready").status_code == 200
|
|
|
|
|
|
@pytest.mark.spec("health", "Кэш недоступен")
|
|
def test_readiness_without_cache(api: Api, stand: Stand) -> None:
|
|
with outage(stand, "stop", "valkey"):
|
|
assert api.request("GET", "/health/ready").status_code == 200
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Кэш карточки чата
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.spec("chat-cache", "Очистка кэша")
|
|
def test_flush_cache(
|
|
api: Api,
|
|
stand: Stand,
|
|
bob: Actor,
|
|
chat: dict[str, Any],
|
|
) -> None:
|
|
wait_cache_ready(api, stand, bob, chat)
|
|
stand("flush-cache")
|
|
assert stand("cache-has-chat", chat["id"]) == "0"
|
|
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
|
|
assert stand("cache-has-chat", chat["id"]) == "1"
|
|
|
|
|
|
@pytest.mark.spec("chat-cache", "Valkey остановлен")
|
|
def test_valkey_stopped(
|
|
api: Api,
|
|
stand: Stand,
|
|
alice: Actor,
|
|
bob: Actor,
|
|
chat: dict[str, Any],
|
|
) -> None:
|
|
sent = [api.send_message(alice, chat["id"], "до отказа")]
|
|
with outage(stand, "stop", "valkey"):
|
|
resp, took = timed(lambda: read_chat(api, bob, chat))
|
|
assert assert_ok(resp, 200, "Chat") == chat
|
|
assert took < CACHE_TIMEOUT_S + SLACK_S, f"read took {took:.2f}s"
|
|
resp, took = timed(
|
|
lambda: api.post_message(
|
|
alice,
|
|
chat["id"],
|
|
json={"text": "во время отказа"},
|
|
),
|
|
)
|
|
sent.append(assert_ok(resp, 201, "Message"))
|
|
assert took < CACHE_TIMEOUT_S + SLACK_S, f"send took {took:.2f}s"
|
|
assert api.all_messages(bob, chat["id"]) == sent
|
|
|
|
|
|
@pytest.mark.spec("chat-cache", "Valkey завис")
|
|
def test_valkey_paused(
|
|
api: Api,
|
|
stand: Stand,
|
|
bob: Actor,
|
|
chat: dict[str, Any],
|
|
) -> None:
|
|
wait_cache_ready(api, stand, bob, chat)
|
|
baseline = statistics.median(
|
|
timed(lambda: read_chat(api, bob, chat))[1] for _ in range(5)
|
|
)
|
|
with outage(stand, "pause", "valkey"):
|
|
resp, took = timed(lambda: read_chat(api, bob, chat))
|
|
assert assert_ok(resp, 200, "Chat") == chat
|
|
assert took < baseline + CACHE_TIMEOUT_S + SLACK_S, (
|
|
f"read took {took:.2f}s, baseline {baseline:.3f}s"
|
|
)
|
|
# окно пропуска: следующий запрос таймаута не ждёт
|
|
resp, took = timed(lambda: read_chat(api, bob, chat))
|
|
assert assert_ok(resp, 200, "Chat") == chat
|
|
assert took < CACHE_TIMEOUT_S, f"second read took {took:.2f}s"
|
|
|
|
|
|
@pytest.mark.spec("chat-cache", "Valkey вернулся")
|
|
def test_valkey_back(
|
|
api: Api,
|
|
stand: Stand,
|
|
bob: Actor,
|
|
chat: dict[str, Any],
|
|
) -> None:
|
|
with outage(stand, "stop", "valkey"):
|
|
assert_ok(read_chat(api, bob, chat), 200, "Chat")
|
|
# API не перезапускался; кэш снова в работе после окна пропуска
|
|
time.sleep(CACHE_RETRY_AFTER_S + 0.5)
|
|
assert_ok(read_chat(api, bob, chat), 200, "Chat")
|
|
assert stand("cache-has-chat", chat["id"]) == "1"
|
|
hits = int(stand("cache-hits"))
|
|
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
|
|
assert int(stand("cache-hits")) > hits
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Идемпотентность: ключ только вместе с сообщением, срок 2 с на «тесном» API
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.spec("idempotency", "Сбой при записи сообщения")
|
|
def test_message_failure_keeps_key_free(
|
|
api: Api,
|
|
stand: Stand,
|
|
alice: Actor,
|
|
chat: dict[str, Any],
|
|
) -> None:
|
|
# ключ вставляется первым: триггер роняет транзакцию уже после него
|
|
text = f"fault-{uuid.uuid4().hex}"
|
|
key = new_key()
|
|
stand("fail-messages", text)
|
|
try:
|
|
resp = api.post_message(
|
|
alice,
|
|
chat["id"],
|
|
key=key,
|
|
json={"text": text},
|
|
)
|
|
assert_error(resp, 500)
|
|
finally:
|
|
stand("clear-fail-messages")
|
|
# ключ не занят: повтор выполняется заново, а не 500 и не 409
|
|
message = api.send_message(alice, chat["id"], text, key=key)
|
|
assert api.all_messages(alice, chat["id"]) == [message]
|
|
|
|
|
|
@pytest.mark.spec("idempotency", "Отказ БД при отправке")
|
|
def test_database_outage_keeps_key_free(
|
|
api: Api,
|
|
stand: Stand,
|
|
alice: Actor,
|
|
chat: dict[str, Any],
|
|
) -> None:
|
|
text = "при отказе БД"
|
|
key = new_key()
|
|
with outage(stand, "stop", "postgres"):
|
|
resp = api.post_message(
|
|
alice,
|
|
chat["id"],
|
|
key=key,
|
|
json={"text": text},
|
|
)
|
|
assert_error(resp, 503)
|
|
message = api.send_message(alice, chat["id"], text, key=key)
|
|
assert api.all_messages(alice, chat["id"]) == [message]
|
|
|
|
|
|
@pytest.mark.spec("idempotency", "Ключ после срока")
|
|
def test_key_free_after_ttl(tight_api: Api) -> None:
|
|
alice = tight_api.register()
|
|
chat = tight_api.create_chat(alice)
|
|
key = new_key()
|
|
first = tight_api.send_message(alice, chat["id"], "до срока", key=key)
|
|
# в пределах срока ключ занят: другое тело — 409
|
|
resp = tight_api.post_message(
|
|
alice,
|
|
chat["id"],
|
|
key=key,
|
|
json={"text": "после срока"},
|
|
)
|
|
assert_error(resp, 409)
|
|
time.sleep(IDEMPOTENCY_TTL_S + SLACK_S)
|
|
# сессия «тесного» API тоже истекла: новый вход, ключ тот же
|
|
alice = tight_api.login(alice.user, alice.password)
|
|
second = tight_api.send_message(alice, chat["id"], "после срока", key=key)
|
|
assert second["id"] != first["id"]
|
|
history = tight_api.all_messages(alice, chat["id"])
|
|
assert [item["id"] for item in history] == [first["id"], second["id"]]
|