510 lines
19 KiB
Python
510 lines
19 KiB
Python
"""Проверки отказов: выполняет только `make failure_checks` (маркер `fault`).
|
||
|
||
Стенд и hook готовит `student-tests/run_failures.py`: свои Postgres и
|
||
Valkey, обычный API (`BASE_URL`) и «тесный» (сроки сессии и ключа
|
||
идемпотентности 2 с, пул из одного соединения). На хранилища тесты действуют только через фикстуру
|
||
`stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в
|
||
рабочее состояние. Решение 4 design change `failure-checks`.
|
||
"""
|
||
|
||
import statistics
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from collections.abc import Callable, Iterator
|
||
from contextlib import contextmanager
|
||
from typing import Any
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
from api_client import API, Actor, Api, new_key
|
||
from contract import (
|
||
BEARER_CHALLENGE,
|
||
assert_error,
|
||
assert_ok,
|
||
assert_unauthorized,
|
||
)
|
||
|
||
pytestmark = pytest.mark.fault
|
||
|
||
type Stand = Callable[..., str]
|
||
|
||
RESTORE = {"stop": "start", "pause": "unpause"}
|
||
# запас к границам времени: планировщик Docker Desktop
|
||
SLACK_S = 0.3
|
||
# [cache] timeout_seconds и retry_after_seconds по умолчанию
|
||
CACHE_TIMEOUT_S = 0.2
|
||
CACHE_RETRY_AFTER_S = 5
|
||
# окружение «тесного» API (TIGHT_ENV в run_failures.py)
|
||
SESSION_TTL_S = 2
|
||
IDEMPOTENCY_TTL_S = 2
|
||
POOL_TIMEOUT_S = 1
|
||
# срок проверки Postgres в /health/ready
|
||
READINESS_TIMEOUT_S = 2
|
||
# сколько секунд hook держит слоты max_connections; clear-connections
|
||
# освобождает их раньше
|
||
FILL_HOLD_S = 30
|
||
ME = f"{API}/users/me"
|
||
|
||
|
||
@contextmanager
|
||
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
|
||
"""`stop` или `pause` хранилища; в `finally` — обратно."""
|
||
stand(action, service)
|
||
try:
|
||
yield
|
||
finally:
|
||
stand(RESTORE[action], service)
|
||
|
||
|
||
def timed(
|
||
call: Callable[[], httpx.Response],
|
||
) -> tuple[httpx.Response, float]:
|
||
started = time.monotonic()
|
||
resp = call()
|
||
return resp, time.monotonic() - started
|
||
|
||
|
||
def read_chat(api: Api, actor: Actor, chat: dict[str, Any]) -> httpx.Response:
|
||
return api.request("GET", f"{API}/chats/{chat['id']}", actor=actor)
|
||
|
||
|
||
def wait_cache_ready(
|
||
api: Api,
|
||
stand: Stand,
|
||
actor: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
"""Дождаться, пока чтение снова пишет карточку в кэш.
|
||
|
||
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
|
||
"""
|
||
deadline = time.monotonic() + CACHE_RETRY_AFTER_S + 2
|
||
stand("flush-cache")
|
||
while time.monotonic() < deadline:
|
||
assert_ok(read_chat(api, actor, chat), 200, "Chat")
|
||
if stand("cache-has-chat", chat["id"]) == "1":
|
||
return
|
||
time.sleep(0.5)
|
||
raise AssertionError("cache did not come back")
|
||
|
||
|
||
def chat_titles(api: Api, actor: Actor) -> set[str]:
|
||
return {
|
||
chat["title"]
|
||
for page in api.chat_pages(actor)
|
||
for chat in page["items"]
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Сессии: срок 2 с на «тесном» API, БД остановлена — на обычном
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("sessions", "Истёкший токен")
|
||
def test_expired_token(tight_api: Api) -> None:
|
||
actor = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
|
||
time.sleep(SESSION_TTL_S + 1)
|
||
assert_unauthorized(
|
||
tight_api.request("GET", ME, actor=actor),
|
||
BEARER_CHALLENGE,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("sessions", "Срок не продлевается")
|
||
def test_use_does_not_extend_session(tight_api: Api) -> None:
|
||
actor = tight_api.register()
|
||
logged_in = time.monotonic()
|
||
successes = 0
|
||
while time.monotonic() - logged_in < SESSION_TTL_S - SLACK_S:
|
||
assert_ok(tight_api.request("GET", ME, actor=actor), 200, "User")
|
||
successes += 1
|
||
time.sleep(0.5)
|
||
assert successes >= 3, "too few requests before expiry"
|
||
time.sleep(max(0.0, logged_in + SESSION_TTL_S + 1 - time.monotonic()))
|
||
assert_unauthorized(
|
||
tight_api.request("GET", ME, actor=actor),
|
||
BEARER_CHALLENGE,
|
||
)
|
||
|
||
|
||
@pytest.mark.spec("sessions", "БД остановлена при запросе с токеном")
|
||
def test_token_while_database_stopped(api: Api, stand: Stand) -> None:
|
||
actor = api.register()
|
||
with outage(stand, "stop", "postgres"):
|
||
resp = api.request("GET", ME, actor=actor)
|
||
assert_error(resp, 503)
|
||
# недоступность хранилища — не отказ в аутентификации
|
||
assert "WWW-Authenticate" not in resp.headers
|
||
assert_ok(api.request("GET", ME, actor=actor), 200, "User")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Хранение: откат, остановленная БД, исчерпанный пул
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Сбой при добавлении участников")
|
||
def test_members_failure_rolls_back_chat(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
) -> None:
|
||
title = f"fault-{uuid.uuid4().hex}"
|
||
stand("fail-members", title)
|
||
try:
|
||
resp = api.request(
|
||
"POST",
|
||
f"{API}/chats",
|
||
actor=alice,
|
||
json={"title": title, "member_ids": [alice.id, bob.id]},
|
||
)
|
||
assert_error(resp, 500)
|
||
finally:
|
||
stand("clear-fail-members")
|
||
for actor in (alice, bob):
|
||
assert title not in chat_titles(api, actor)
|
||
assert stand("count-chats", title) == "0"
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "БД остановлена")
|
||
@pytest.mark.spec("data-storage", "БД вернулась")
|
||
def test_database_stopped_and_back(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with outage(stand, "stop", "postgres"):
|
||
resp, took = timed(
|
||
lambda: api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "во время отказа"},
|
||
),
|
||
)
|
||
error = assert_error(resp, 503)
|
||
assert took < 2, f"503 took {took:.2f}s"
|
||
for word in ("postgres", "sql", "connect"):
|
||
assert word not in error["message"].lower()
|
||
message = api.send_message(alice, chat["id"], "после возврата")
|
||
# запись во время отказа не зафиксирована, новая — видна
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Соединения исчерпаны")
|
||
def test_pool_exhausted(tight_api: Api, stand: Stand) -> None:
|
||
actor = tight_api.register()
|
||
holder = Api(str(tight_api.client.base_url), timeout=15)
|
||
first: list[int] = []
|
||
|
||
def hold_connection() -> None:
|
||
first.append(holder.request("GET", ME, actor=actor).status_code)
|
||
|
||
with outage(stand, "pause", "postgres"):
|
||
thread = threading.Thread(target=hold_connection)
|
||
thread.start()
|
||
time.sleep(SLACK_S)
|
||
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
|
||
assert_error(resp, 503)
|
||
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
|
||
thread.join()
|
||
holder.close()
|
||
assert first == [503]
|
||
# срок сессии 2 с мог истечь за время паузы: проверяем пул новым входом
|
||
fresh = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Предел соединений PostgreSQL исчерпан")
|
||
def test_connection_limit_reached(tight_api: Api, stand: Stand) -> None:
|
||
actor = tight_api.register()
|
||
# hook оборвал соединение пула: при выдаче pool_pre_ping
|
||
# переподключается и получает «too many clients» (SQLSTATE 53300)
|
||
stand("fill-connections", str(FILL_HOLD_S))
|
||
try:
|
||
resp, took = timed(lambda: tight_api.request("GET", ME, actor=actor))
|
||
error = assert_error(resp, 503)
|
||
# отказ PostgreSQL мгновенный: ответ не ждёт срока пула
|
||
assert took < POOL_TIMEOUT_S + 0.5, f"503 took {took:.2f}s"
|
||
# недоступность хранилища — не отказ в аутентификации
|
||
assert "WWW-Authenticate" not in resp.headers
|
||
for word in ("postgres", "sql", "client", "connect"):
|
||
assert word not in error["message"].lower()
|
||
finally:
|
||
stand("clear-connections")
|
||
# слоты свободны: сервис снова обслуживает запросы без рестарта; срок
|
||
# сессии 2 с за время заполнения истёк — новый вход
|
||
fresh = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=fresh), 200, "User")
|
||
|
||
|
||
# пачка одновременных запросов больше пула обычного API (10)
|
||
BURST_THREADS = 48
|
||
BURST_AFTER_BARRIER = 4
|
||
# PGSTAT_IDLE_INTERVAL PostgreSQL (10 с) и запас
|
||
STATS_FLUSH_S = 11
|
||
|
||
|
||
def burst(api: Api, actor: Actor) -> None:
|
||
"""Пачка: у каждого потока свой клиент (keep-alive), один запрос до
|
||
барьера и BURST_AFTER_BARRIER после — около BURST_THREADS
|
||
одновременно. Все ответы — 200."""
|
||
barrier = threading.Barrier(BURST_THREADS)
|
||
clients = [
|
||
Api(str(api.client.base_url), timeout=15) for _ in range(BURST_THREADS)
|
||
]
|
||
responses: list[httpx.Response] = []
|
||
errors: list[BaseException] = []
|
||
lock = threading.Lock()
|
||
|
||
def worker(client: Api) -> None:
|
||
try:
|
||
got = [client.request("GET", ME, actor=actor)]
|
||
barrier.wait()
|
||
got += [
|
||
client.request("GET", ME, actor=actor)
|
||
for _ in range(BURST_AFTER_BARRIER)
|
||
]
|
||
except BaseException as e: # noqa: BLE001 — поднимется в тесте
|
||
barrier.abort()
|
||
with lock:
|
||
errors.append(e)
|
||
return
|
||
with lock:
|
||
responses.extend(got)
|
||
|
||
threads = [
|
||
threading.Thread(target=worker, args=(client,)) for client in clients
|
||
]
|
||
for thread in threads:
|
||
thread.start()
|
||
for thread in threads:
|
||
thread.join()
|
||
for client in clients:
|
||
client.close()
|
||
assert not errors, f"burst failed: {errors[0]!r}"
|
||
assert len(responses) == BURST_THREADS * (1 + BURST_AFTER_BARRIER)
|
||
for resp in responses:
|
||
assert_ok(resp, 200, "User")
|
||
|
||
|
||
@pytest.mark.spec("data-storage", "Соединения пула не переоткрываются")
|
||
def test_pool_keeps_connections(
|
||
api: Api,
|
||
tight_api: Api,
|
||
stand: Stand,
|
||
) -> None:
|
||
actor = api.register()
|
||
# соединение «тесного» API могли оборвать прежние тесты
|
||
# (fill-connections): его переподключение не должно попасть в окно
|
||
tight = tight_api.register()
|
||
assert_ok(tight_api.request("GET", ME, actor=tight), 200, "User")
|
||
# прогрев: пул обычного API набирает свой размер
|
||
for _ in range(2):
|
||
burst(api, actor)
|
||
# новый backend учитывает себя в sessions не сразу: его счётчики уходят
|
||
# в общую статистику без ожидания блокировки и при занятой блокировке
|
||
# откладываются. Простаивающий backend сбрасывает их принудительно по
|
||
# таймеру PGSTAT_IDLE_INTERVAL (10 с): без паузы соединения прогрева
|
||
# попадали в окно по одному
|
||
time.sleep(STATS_FLUSH_S)
|
||
# калибровка: второй вызов видит сессию первого вызова hook, и только её
|
||
s1 = int(stand("db-sessions"))
|
||
s2 = int(stand("db-sessions"))
|
||
assert s2 - s1 == 1, f"stand is noisy: {s2 - s1} sessions between calls"
|
||
for _ in range(3):
|
||
burst(api, actor)
|
||
s3 = int(stand("db-sessions"))
|
||
opened = s3 - s2 - 1
|
||
assert opened == 0, (
|
||
f"PostgreSQL accepted {opened} new sessions during "
|
||
f"{3 * BURST_THREADS * (1 + BURST_AFTER_BARRIER)} requests"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Readiness
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("health", "Хранилище не готово")
|
||
def test_readiness_without_database(api: Api, stand: Stand) -> None:
|
||
with outage(stand, "stop", "postgres"):
|
||
resp, took = timed(lambda: api.request("GET", "/health/ready"))
|
||
assert_error(resp, 503)
|
||
assert took < READINESS_TIMEOUT_S + 0.5, f"ready took {took:.2f}s"
|
||
live = api.request("GET", "/health/live")
|
||
assert live.status_code == 200
|
||
assert api.request("GET", "/health/ready").status_code == 200
|
||
|
||
|
||
@pytest.mark.spec("health", "Кэш недоступен")
|
||
def test_readiness_without_cache(api: Api, stand: Stand) -> None:
|
||
with outage(stand, "stop", "valkey"):
|
||
assert api.request("GET", "/health/ready").status_code == 200
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Кэш карточки чата
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Очистка кэша")
|
||
def test_flush_cache(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
wait_cache_ready(api, stand, bob, chat)
|
||
stand("flush-cache")
|
||
assert stand("cache-has-chat", chat["id"]) == "0"
|
||
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
|
||
assert stand("cache-has-chat", chat["id"]) == "1"
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey остановлен")
|
||
def test_valkey_stopped(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
sent = [api.send_message(alice, chat["id"], "до отказа")]
|
||
with outage(stand, "stop", "valkey"):
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < CACHE_TIMEOUT_S + SLACK_S, f"read took {took:.2f}s"
|
||
resp, took = timed(
|
||
lambda: api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
json={"text": "во время отказа"},
|
||
),
|
||
)
|
||
sent.append(assert_ok(resp, 201, "Message"))
|
||
assert took < CACHE_TIMEOUT_S + SLACK_S, f"send took {took:.2f}s"
|
||
assert api.all_messages(bob, chat["id"]) == sent
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey завис")
|
||
def test_valkey_paused(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
wait_cache_ready(api, stand, bob, chat)
|
||
baseline = statistics.median(
|
||
timed(lambda: read_chat(api, bob, chat))[1] for _ in range(5)
|
||
)
|
||
with outage(stand, "pause", "valkey"):
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < baseline + CACHE_TIMEOUT_S + SLACK_S, (
|
||
f"read took {took:.2f}s, baseline {baseline:.3f}s"
|
||
)
|
||
# окно пропуска: следующий запрос таймаута не ждёт
|
||
resp, took = timed(lambda: read_chat(api, bob, chat))
|
||
assert assert_ok(resp, 200, "Chat") == chat
|
||
assert took < CACHE_TIMEOUT_S, f"second read took {took:.2f}s"
|
||
|
||
|
||
@pytest.mark.spec("chat-cache", "Valkey вернулся")
|
||
def test_valkey_back(
|
||
api: Api,
|
||
stand: Stand,
|
||
bob: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
with outage(stand, "stop", "valkey"):
|
||
assert_ok(read_chat(api, bob, chat), 200, "Chat")
|
||
# API не перезапускался; кэш снова в работе после окна пропуска
|
||
time.sleep(CACHE_RETRY_AFTER_S + 0.5)
|
||
assert_ok(read_chat(api, bob, chat), 200, "Chat")
|
||
assert stand("cache-has-chat", chat["id"]) == "1"
|
||
hits = int(stand("cache-hits"))
|
||
assert assert_ok(read_chat(api, bob, chat), 200, "Chat") == chat
|
||
assert int(stand("cache-hits")) > hits
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Идемпотентность: ключ только вместе с сообщением, срок 2 с на «тесном» API
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Сбой при записи сообщения")
|
||
def test_message_failure_keeps_key_free(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
# ключ вставляется первым: триггер роняет транзакцию уже после него
|
||
text = f"fault-{uuid.uuid4().hex}"
|
||
key = new_key()
|
||
stand("fail-messages", text)
|
||
try:
|
||
resp = api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": text},
|
||
)
|
||
assert_error(resp, 500)
|
||
finally:
|
||
stand("clear-fail-messages")
|
||
# ключ не занят: повтор выполняется заново, а не 500 и не 409
|
||
message = api.send_message(alice, chat["id"], text, key=key)
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Отказ БД при отправке")
|
||
def test_database_outage_keeps_key_free(
|
||
api: Api,
|
||
stand: Stand,
|
||
alice: Actor,
|
||
chat: dict[str, Any],
|
||
) -> None:
|
||
text = "при отказе БД"
|
||
key = new_key()
|
||
with outage(stand, "stop", "postgres"):
|
||
resp = api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": text},
|
||
)
|
||
assert_error(resp, 503)
|
||
message = api.send_message(alice, chat["id"], text, key=key)
|
||
assert api.all_messages(alice, chat["id"]) == [message]
|
||
|
||
|
||
@pytest.mark.spec("idempotency", "Ключ после срока")
|
||
def test_key_free_after_ttl(tight_api: Api) -> None:
|
||
alice = tight_api.register()
|
||
chat = tight_api.create_chat(alice)
|
||
key = new_key()
|
||
first = tight_api.send_message(alice, chat["id"], "до срока", key=key)
|
||
# в пределах срока ключ занят: другое тело — 409
|
||
resp = tight_api.post_message(
|
||
alice,
|
||
chat["id"],
|
||
key=key,
|
||
json={"text": "после срока"},
|
||
)
|
||
assert_error(resp, 409)
|
||
time.sleep(IDEMPOTENCY_TTL_S + SLACK_S)
|
||
# сессия «тесного» API тоже истекла: новый вход, ключ тот же
|
||
alice = tight_api.login(alice.user, alice.password)
|
||
second = tight_api.send_message(alice, chat["id"], "после срока", key=key)
|
||
assert second["id"] != first["id"]
|
||
history = tight_api.all_messages(alice, chat["id"])
|
||
assert [item["id"] for item in history] == [first["id"], second["id"]]
|