"""Проверки отказов: выполняет только `make failure_checks` (маркер `fault`). Стенд и hook готовит `student-tests/run_failures.py`: свои Postgres и Valkey, обычный API (`BASE_URL`) и «тесный» (сроки сессии и ключа идемпотентности 2 с, пул из одного соединения). На хранилища тесты действуют только через фикстуру `stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в рабочее состояние. Решение 4 design change `failure-checks`. """ import statistics import threading import time import uuid from collections.abc import Callable, Iterator from contextlib import contextmanager from typing import Any import httpx import pytest from api_client import API, Actor, Api, new_key from contract import ( BEARER_CHALLENGE, assert_error, assert_ok, assert_unauthorized, ) pytestmark = pytest.mark.fault type Stand = Callable[..., str] RESTORE = {"stop": "start", "pause": "unpause"} # запас к границам времени: планировщик Docker Desktop SLACK_S = 0.3 # [cache] timeout_seconds и retry_after_seconds по умолчанию CACHE_TIMEOUT_S = 0.2 CACHE_RETRY_AFTER_S = 5 # окружение «тесного» API (TIGHT_ENV в run_failures.py) SESSION_TTL_S = 2 IDEMPOTENCY_TTL_S = 2 POOL_TIMEOUT_S = 1 # срок проверки Postgres в /health/ready READINESS_TIMEOUT_S = 2 ME = f"{API}/users/me" @contextmanager def outage(stand: Stand, action: str, service: str) -> Iterator[None]: """`stop` или `pause` хранилища; в `finally` — обратно.""" stand(action, service) try: yield finally: stand(RESTORE[action], service) def timed( call: Callable[[], httpx.Response], ) -> tuple[httpx.Response, float]: started = time.monotonic() resp = call() return resp, time.monotonic() - started def read_chat(api: Api, actor: Actor, chat: dict[str, Any]) -> httpx.Response: return api.request("GET", f"{API}/chats/{chat['id']}", actor=actor) def wait_cache_ready( api: Api, stand: Stand, actor: Actor, chat: dict[str, Any], ) -> None: """Дождаться, пока чтение снова пишет карточку в кэш. После отказа в прошлом тесте API пропускает кэш ещё до 5 с. """ deadline = time.monotonic() + CACHE_RETRY_AFTER_S + 2 stand("flush-cache") while time.monotonic() < deadline: assert_ok(read_chat(api, actor, chat), 200, "Chat") if stand("cache-has-chat", chat["id"]) == "1": return time.sleep(0.5) raise AssertionError("cache did not come back") def chat_titles(api: Api, actor: Actor) -> set[str]: return { chat["title"] for page in api.chat_pages(actor) for chat in page["items"] } # --------------------------------------------------------------------------- # Сессии: срок 2 с на «тесном» API, БД остановлена — на обычном # --------------------------------------------------------------------------- @pytest.mark.spec("sessions", "Истёкший токен") def test_expired_token(tight_api: Api) -> None: actor = tight_api.register() assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User") time.sleep(SESSION_TTL_S + 1) assert_unauthorized( tight_api.request("GET", ME, actor=actor), BEARER_CHALLENGE, ) @pytest.mark.spec("sessions", "Срок не продлевается") def test_use_does_not_extend_session(tight_api: Api) -> None: actor = tight_api.register() logged_in = time.monotonic() successes = 0 while time.monotonic() - logged_in < SESSION_TTL_S - SLACK_S: assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User") successes += 1 time.sleep(0.5) assert successes >= 3, "too few requests before expiry" time.sleep(max(0.0, logged_in + SESSION_TTL_S + 1 - time.monotonic())) assert_unauthorized( tight_api.request("GET", ME, actor=actor), BEARER_CHALLENGE, ) @pytest.mark.spec("sessions", "БД остановлена при запросе с токеном") def test_token_while_database_stopped(api: Api, stand: Stand) -> None: actor = api.register() with outage(stand, "stop", "postgres"): resp = api.request("GET", ME, actor=actor) assert_error(resp, 503) # недоступность хранилища — не отказ в аутентификации assert "WWW-Authenticate" not in resp.headers assert_ok(api.request("GET", ME, actor=actor), 200, "User") # --------------------------------------------------------------------------- # Хранение: откат, остановленная БД, исчерпанный пул # --------------------------------------------------------------------------- @pytest.mark.spec("data-storage", "Сбой при добавлении участников") def test_members_failure_rolls_back_chat( api: Api, stand: Stand, alice: Actor, bob: Actor, ) -> None: title = f"fault-{uuid.uuid4().hex}" stand("fail-members", title) try: resp = api.request( "POST", f"{API}/chats", actor=alice, json={"title": title, "member_ids": [alice.id, bob.id]}, ) assert_error(resp, 500) finally: stand("clear-fail-members") for actor in (alice, bob): assert title not in chat_titles(api, actor) assert stand("count-chats", title) == "0" @pytest.mark.spec("data-storage", "БД остановлена") @pytest.mark.spec("data-storage", "БД вернулась") def test_database_stopped_and_back( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: with outage(stand, "stop", "postgres"): resp, took = timed( lambda: api.post_message( alice, chat["id"], json={"text": "во время отказа"}, ), ) error = assert_error(resp, 503) assert took < 2, f"503 took {took:.2f}s" for word in ("postgres", "sql", "connect"): assert word not in error["message"].lower() message = api.send_message(alice, chat["id"], "после возврата") # запись во время отказа не зафиксирована, новая — видна assert api.all_messages(alice, chat["id"]) == [message] @pytest.mark.spec("data-storage", "Соединения исчерпаны") def test_pool_exhausted(tight_api: Api, stand: Stand) -> None: actor = tight_api.register() holder = Api(str(tight_api.client.base_url), timeout=15) first: list[int] = [] def hold_connection() -> None: first.append(holder.request("GET", ME, actor=actor).status_code) with outage(stand, "pause", "postgres"): thread = threading.Thread(target=hold_connection) thread.start() time.sleep(SLACK_S) resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor)) assert_error(resp, 503) assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s" thread.join() holder.close() assert first == [503] # срок сессии 2 с мог истечь за время паузы: проверяем пул новым входом fresh = tight_api.register() assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User") # --------------------------------------------------------------------------- # Readiness # --------------------------------------------------------------------------- @pytest.mark.spec("health", "Хранилище не готово") def test_readiness_without_database(api: Api, stand: Stand) -> None: with outage(stand, "stop", "postgres"): resp, took = timed(lambda: api.request("GET", "/health/ready")) assert_error(resp, 503) assert took < READINESS_TIMEOUT_S + 0.5, f"ready took {took:.2f}s" live = api.request("GET", "/health/live") assert live.status_code == 200 assert api.request("GET", "/health/ready").status_code == 200 @pytest.mark.spec("health", "Кэш недоступен") def test_readiness_without_cache(api: Api, stand: Stand) -> None: with outage(stand, "stop", "valkey"): assert api.request("GET", "/health/ready").status_code == 200 # --------------------------------------------------------------------------- # Кэш карточки чата # --------------------------------------------------------------------------- @pytest.mark.spec("chat-cache", "Очистка кэша") def test_flush_cache( api: Api, stand: Stand, bob: Actor, chat: dict[str, Any], ) -> None: wait_cache_ready(api, stand, bob, chat) stand("flush-cache") assert stand("cache-has-chat", chat["id"]) == "0" assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat assert stand("cache-has-chat", chat["id"]) == "1" @pytest.mark.spec("chat-cache", "Valkey остановлен") def test_valkey_stopped( api: Api, stand: Stand, alice: Actor, bob: Actor, chat: dict[str, Any], ) -> None: sent = [api.send_message(alice, chat["id"], "до отказа")] with outage(stand, "stop", "valkey"): resp, took = timed(lambda: read_chat(api, bob, chat)) assert assert_ok(resp, 200, "Chat") == chat assert took < CACHE_TIMEOUT_S + SLACK_S, f"read took {took:.2f}s" resp, took = timed( lambda: api.post_message( alice, chat["id"], json={"text": "во время отказа"}, ), ) sent.append(assert_ok(resp, 201, "Message")) assert took < CACHE_TIMEOUT_S + SLACK_S, f"send took {took:.2f}s" assert api.all_messages(bob, chat["id"]) == sent @pytest.mark.spec("chat-cache", "Valkey завис") def test_valkey_paused( api: Api, stand: Stand, bob: Actor, chat: dict[str, Any], ) -> None: wait_cache_ready(api, stand, bob, chat) baseline = statistics.median( timed(lambda: read_chat(api, bob, chat))[1] for _ in range(5) ) with outage(stand, "pause", "valkey"): resp, took = timed(lambda: read_chat(api, bob, chat)) assert assert_ok(resp, 200, "Chat") == chat assert took < baseline + CACHE_TIMEOUT_S + SLACK_S, ( f"read took {took:.2f}s, baseline {baseline:.3f}s" ) # окно пропуска: следующий запрос таймаута не ждёт resp, took = timed(lambda: read_chat(api, bob, chat)) assert assert_ok(resp, 200, "Chat") == chat assert took < CACHE_TIMEOUT_S, f"second read took {took:.2f}s" @pytest.mark.spec("chat-cache", "Valkey вернулся") def test_valkey_back( api: Api, stand: Stand, bob: Actor, chat: dict[str, Any], ) -> None: with outage(stand, "stop", "valkey"): assert_ok(read_chat(api, bob, chat), 200, "Chat") # API не перезапускался; кэш снова в работе после окна пропуска time.sleep(CACHE_RETRY_AFTER_S + 0.5) assert_ok(read_chat(api, bob, chat), 200, "Chat") assert stand("cache-has-chat", chat["id"]) == "1" hits = int(stand("cache-hits")) assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat assert int(stand("cache-hits")) > hits # --------------------------------------------------------------------------- # Идемпотентность: ключ только вместе с сообщением, срок 2 с на «тесном» API # --------------------------------------------------------------------------- @pytest.mark.spec("idempotency", "Сбой при записи сообщения") def test_message_failure_keeps_key_free( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: # ключ вставляется первым: триггер роняет транзакцию уже после него text = f"fault-{uuid.uuid4().hex}" key = new_key() stand("fail-messages", text) try: resp = api.post_message( alice, chat["id"], key=key, json={"text": text}, ) assert_error(resp, 500) finally: stand("clear-fail-messages") # ключ не занят: повтор выполняется заново, а не 500 и не 409 message = api.send_message(alice, chat["id"], text, key=key) assert api.all_messages(alice, chat["id"]) == [message] @pytest.mark.spec("idempotency", "Отказ БД при отправке") def test_database_outage_keeps_key_free( api: Api, stand: Stand, alice: Actor, chat: dict[str, Any], ) -> None: text = "при отказе БД" key = new_key() with outage(stand, "stop", "postgres"): resp = api.post_message( alice, chat["id"], key=key, json={"text": text}, ) assert_error(resp, 503) message = api.send_message(alice, chat["id"], text, key=key) assert api.all_messages(alice, chat["id"]) == [message] @pytest.mark.spec("idempotency", "Ключ после срока") def test_key_free_after_ttl(tight_api: Api) -> None: alice = tight_api.register() chat = tight_api.create_chat(alice) key = new_key() first = tight_api.send_message(alice, chat["id"], "до срока", key=key) # в пределах срока ключ занят: другое тело — 409 resp = tight_api.post_message( alice, chat["id"], key=key, json={"text": "после срока"}, ) assert_error(resp, 409) time.sleep(IDEMPOTENCY_TTL_S + SLACK_S) # сессия «тесного» API тоже истекла: новый вход, ключ тот же alice = tight_api.login(alice.user, alice.password) second = tight_api.send_message(alice, chat["id"], "после срока", key=key) assert second["id"] != first["id"] history = tight_api.all_messages(alice, chat["id"]) assert [item["id"] for item in history] == [first["id"], second["id"]]