Files
mtusi2026lab3/student-tests/test_failures.py
T
2026-10-11 14:10:43 +03:00

426 lines
16 KiB
Python

"""Проверки отказов: выполняет только `make failure_checks` (маркер `fault`).
Стенд и hook готовит `student-tests/run_failures.py`: свои Postgres и
Valkey, обычный API (`BASE_URL`) и «тесный» (сроки сессии и ключа
идемпотентности 2 с, пул из одного соединения). На хранилища тесты действуют только через фикстуру
`stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в
рабочее состояние. Решение 4 design change `failure-checks`.
"""
import statistics
import threading
import time
import uuid
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from typing import Any
import httpx
import pytest
from api_client import API, Actor, Api, new_key
from contract import (
BEARER_CHALLENGE,
assert_error,
assert_ok,
assert_unauthorized,
)
pytestmark = pytest.mark.fault
type Stand = Callable[..., str]
RESTORE = {"stop": "start", "pause": "unpause"}
# запас к границам времени: планировщик Docker Desktop
SLACK_S = 0.3
# [cache] timeout_seconds и retry_after_seconds по умолчанию
CACHE_TIMEOUT_S = 0.2
CACHE_RETRY_AFTER_S = 5
# окружение «тесного» API (TIGHT_ENV в run_failures.py)
SESSION_TTL_S = 2
IDEMPOTENCY_TTL_S = 2
POOL_TIMEOUT_S = 1
# срок проверки Postgres в /health/ready
READINESS_TIMEOUT_S = 2
# сколько секунд hook держит слоты max_connections; clear-connections
# освобождает их раньше
FILL_HOLD_S = 30
ME = f"{API}/users/me"
@contextmanager
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
"""`stop` или `pause` хранилища; в `finally` — обратно."""
stand(action, service)
try:
yield
finally:
stand(RESTORE[action], service)
def timed(
call: Callable[[], httpx.Response],
) -> tuple[httpx.Response, float]:
started = time.monotonic()
resp = call()
return resp, time.monotonic() - started
def read_chat(api: Api, actor: Actor, chat: dict[str, Any]) -> httpx.Response:
return api.request("GET", f"{API}/chats/{chat['id']}", actor=actor)
def wait_cache_ready(
api: Api,
stand: Stand,
actor: Actor,
chat: dict[str, Any],
) -> None:
"""Дождаться, пока чтение снова пишет карточку в кэш.
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
"""
deadline = time.monotonic() + CACHE_RETRY_AFTER_S + 2
stand("flush-cache")
while time.monotonic() < deadline:
assert_ok(read_chat(api, actor, chat), 200, "Chat")
if stand("cache-has-chat", chat["id"]) == "1":
return
time.sleep(0.5)
raise AssertionError("cache did not come back")
def chat_titles(api: Api, actor: Actor) -> set[str]:
return {
chat["title"]
for page in api.chat_pages(actor)
for chat in page["items"]
}
# ---------------------------------------------------------------------------
# Сессии: срок 2 с на «тесном» API, БД остановлена — на обычном
# ---------------------------------------------------------------------------
@pytest.mark.spec("sessions", "Истёкший токен")
def test_expired_token(tight_api: Api) -> None:
actor = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
time.sleep(SESSION_TTL_S + 1)
assert_unauthorized(
tight_api.request("GET", ME, actor=actor),
BEARER_CHALLENGE,
)
@pytest.mark.spec("sessions", "Срок не продлевается")
def test_use_does_not_extend_session(tight_api: Api) -> None:
actor = tight_api.register()
logged_in = time.monotonic()
successes = 0
while time.monotonic() - logged_in < SESSION_TTL_S - SLACK_S:
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
successes += 1
time.sleep(0.5)
assert successes >= 3, "too few requests before expiry"
time.sleep(max(0.0, logged_in + SESSION_TTL_S + 1 - time.monotonic()))
assert_unauthorized(
tight_api.request("GET", ME, actor=actor),
BEARER_CHALLENGE,
)
@pytest.mark.spec("sessions", "БД остановлена при запросе с токеном")
def test_token_while_database_stopped(api: Api, stand: Stand) -> None:
actor = api.register()
with outage(stand, "stop", "postgres"):
resp = api.request("GET", ME, actor=actor)
assert_error(resp, 503)
# недоступность хранилища — не отказ в аутентификации
assert "WWW-Authenticate" not in resp.headers
assert_ok(api.request("GET", ME, actor=actor), 200, "User")
# ---------------------------------------------------------------------------
# Хранение: откат, остановленная БД, исчерпанный пул
# ---------------------------------------------------------------------------
@pytest.mark.spec("data-storage", "Сбой при добавлении участников")
def test_members_failure_rolls_back_chat(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
) -> None:
title = f"fault-{uuid.uuid4().hex}"
stand("fail-members", title)
try:
resp = api.request(
"POST",
f"{API}/chats",
actor=alice,
json={"title": title, "member_ids": [alice.id, bob.id]},
)
assert_error(resp, 500)
finally:
stand("clear-fail-members")
for actor in (alice, bob):
assert title not in chat_titles(api, actor)
assert stand("count-chats", title) == "0"
@pytest.mark.spec("data-storage", "БД остановлена")
@pytest.mark.spec("data-storage", "БД вернулась")
def test_database_stopped_and_back(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
with outage(stand, "stop", "postgres"):
resp, took = timed(
lambda: api.post_message(
alice,
chat["id"],
json={"text": "во время отказа"},
),
)
error = assert_error(resp, 503)
assert took < 2, f"503 took {took:.2f}s"
for word in ("postgres", "sql", "connect"):
assert word not in error["message"].lower()
message = api.send_message(alice, chat["id"], "после возврата")
# запись во время отказа не зафиксирована, новая — видна
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("data-storage", "Соединения исчерпаны")
def test_pool_exhausted(tight_api: Api, stand: Stand) -> None:
actor = tight_api.register()
holder = Api(str(tight_api.client.base_url), timeout=15)
first: list[int] = []
def hold_connection() -> None:
first.append(holder.request("GET", ME, actor=actor).status_code)
with outage(stand, "pause", "postgres"):
thread = threading.Thread(target=hold_connection)
thread.start()
time.sleep(SLACK_S)
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
assert_error(resp, 503)
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
thread.join()
holder.close()
assert first == [503]
# срок сессии 2 с мог истечь за время паузы: проверяем пул новым входом
fresh = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
@pytest.mark.spec("data-storage", "Предел соединений PostgreSQL исчерпан")
def test_connection_limit_reached(tight_api: Api, stand: Stand) -> None:
actor = tight_api.register()
# hook оборвал соединение пула: при выдаче pool_pre_ping
# переподключается и получает «too many clients» (SQLSTATE 53300)
stand("fill-connections", str(FILL_HOLD_S))
try:
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
error = assert_error(resp, 503)
# отказ PostgreSQL мгновенный: ответ не ждёт срока пула
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
# недоступность хранилища — не отказ в аутентификации
assert "WWW-Authenticate" not in resp.headers
for word in ("postgres", "sql", "client", "connect"):
assert word not in error["message"].lower()
finally:
stand("clear-connections")
# слоты свободны: сервис снова обслуживает запросы без рестарта; срок
# сессии 2 с за время заполнения истёк — новый вход
fresh = tight_api.register()
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
# ---------------------------------------------------------------------------
# Readiness
# ---------------------------------------------------------------------------
@pytest.mark.spec("health", "Хранилище не готово")
def test_readiness_without_database(api: Api, stand: Stand) -> None:
with outage(stand, "stop", "postgres"):
resp, took = timed(lambda: api.request("GET", "/health/ready"))
assert_error(resp, 503)
assert took < READINESS_TIMEOUT_S + 0.5, f"ready took {took:.2f}s"
live = api.request("GET", "/health/live")
assert live.status_code == 200
assert api.request("GET", "/health/ready").status_code == 200
@pytest.mark.spec("health", "Кэш недоступен")
def test_readiness_without_cache(api: Api, stand: Stand) -> None:
with outage(stand, "stop", "valkey"):
assert api.request("GET", "/health/ready").status_code == 200
# ---------------------------------------------------------------------------
# Кэш карточки чата
# ---------------------------------------------------------------------------
@pytest.mark.spec("chat-cache", "Очистка кэша")
def test_flush_cache(
api: Api,
stand: Stand,
bob: Actor,
chat: dict[str, Any],
) -> None:
wait_cache_ready(api, stand, bob, chat)
stand("flush-cache")
assert stand("cache-has-chat", chat["id"]) == "0"
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
assert stand("cache-has-chat", chat["id"]) == "1"
@pytest.mark.spec("chat-cache", "Valkey остановлен")
def test_valkey_stopped(
api: Api,
stand: Stand,
alice: Actor,
bob: Actor,
chat: dict[str, Any],
) -> None:
sent = [api.send_message(alice, chat["id"], "до отказа")]
with outage(stand, "stop", "valkey"):
resp, took = timed(lambda: read_chat(api, bob, chat))
assert assert_ok(resp, 200, "Chat") == chat
assert took < CACHE_TIMEOUT_S + SLACK_S, f"read took {took:.2f}s"
resp, took = timed(
lambda: api.post_message(
alice,
chat["id"],
json={"text": "во время отказа"},
),
)
sent.append(assert_ok(resp, 201, "Message"))
assert took < CACHE_TIMEOUT_S + SLACK_S, f"send took {took:.2f}s"
assert api.all_messages(bob, chat["id"]) == sent
@pytest.mark.spec("chat-cache", "Valkey завис")
def test_valkey_paused(
api: Api,
stand: Stand,
bob: Actor,
chat: dict[str, Any],
) -> None:
wait_cache_ready(api, stand, bob, chat)
baseline = statistics.median(
timed(lambda: read_chat(api, bob, chat))[1] for _ in range(5)
)
with outage(stand, "pause", "valkey"):
resp, took = timed(lambda: read_chat(api, bob, chat))
assert assert_ok(resp, 200, "Chat") == chat
assert took < baseline + CACHE_TIMEOUT_S + SLACK_S, (
f"read took {took:.2f}s, baseline {baseline:.3f}s"
)
# окно пропуска: следующий запрос таймаута не ждёт
resp, took = timed(lambda: read_chat(api, bob, chat))
assert assert_ok(resp, 200, "Chat") == chat
assert took < CACHE_TIMEOUT_S, f"second read took {took:.2f}s"
@pytest.mark.spec("chat-cache", "Valkey вернулся")
def test_valkey_back(
api: Api,
stand: Stand,
bob: Actor,
chat: dict[str, Any],
) -> None:
with outage(stand, "stop", "valkey"):
assert_ok(read_chat(api, bob, chat), 200, "Chat")
# API не перезапускался; кэш снова в работе после окна пропуска
time.sleep(CACHE_RETRY_AFTER_S + 0.5)
assert_ok(read_chat(api, bob, chat), 200, "Chat")
assert stand("cache-has-chat", chat["id"]) == "1"
hits = int(stand("cache-hits"))
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
assert int(stand("cache-hits")) > hits
# ---------------------------------------------------------------------------
# Идемпотентность: ключ только вместе с сообщением, срок 2 с на «тесном» API
# ---------------------------------------------------------------------------
@pytest.mark.spec("idempotency", "Сбой при записи сообщения")
def test_message_failure_keeps_key_free(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
# ключ вставляется первым: триггер роняет транзакцию уже после него
text = f"fault-{uuid.uuid4().hex}"
key = new_key()
stand("fail-messages", text)
try:
resp = api.post_message(
alice,
chat["id"],
key=key,
json={"text": text},
)
assert_error(resp, 500)
finally:
stand("clear-fail-messages")
# ключ не занят: повтор выполняется заново, а не 500 и не 409
message = api.send_message(alice, chat["id"], text, key=key)
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("idempotency", "Отказ БД при отправке")
def test_database_outage_keeps_key_free(
api: Api,
stand: Stand,
alice: Actor,
chat: dict[str, Any],
) -> None:
text = "при отказе БД"
key = new_key()
with outage(stand, "stop", "postgres"):
resp = api.post_message(
alice,
chat["id"],
key=key,
json={"text": text},
)
assert_error(resp, 503)
message = api.send_message(alice, chat["id"], text, key=key)
assert api.all_messages(alice, chat["id"]) == [message]
@pytest.mark.spec("idempotency", "Ключ после срока")
def test_key_free_after_ttl(tight_api: Api) -> None:
alice = tight_api.register()
chat = tight_api.create_chat(alice)
key = new_key()
first = tight_api.send_message(alice, chat["id"], "до срока", key=key)
# в пределах срока ключ занят: другое тело — 409
resp = tight_api.post_message(
alice,
chat["id"],
key=key,
json={"text": "после срока"},
)
assert_error(resp, 409)
time.sleep(IDEMPOTENCY_TTL_S + SLACK_S)
# сессия «тесного» API тоже истекла: новый вход, ключ тот же
alice = tight_api.login(alice.user, alice.password)
second = tight_api.send_message(alice, chat["id"], "после срока", key=key)
assert second["id"] != first["id"]
history = tight_api.all_messages(alice, chat["id"])
assert [item["id"] for item in history] == [first["id"], second["id"]]