Files
mtusi2026lab3/student-tests/test_failures.py
T

361 lines
16 KiB
Python

"""Проверки отказов (уровень «4»): выполняет только `make failure-checks`.
Стенд и hook готовит `student-tests/stand.py failures`: свои PostgreSQL и
Valkey, обычный API (`BASE_URL`) и «тесный» (сроки сессии и ключа
идемпотентности 2 с, пул из одного соединения, срок ожидания пула 1 с,
срок операции 2 с). На хранилища тесты действуют только через фикстуру
`stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в
рабочее состояние (`outage`): упавшая проверка не ломает следующие.
Что проверяется: истечение сессии и то, что запросы её не продлевают;
недоступная БД — 503 без деталей и без вызова аутентификации, а не 401 и
не анонимный доступ; ограниченное ожидание при замёрзшей БД и
исчерпанном пуле; откат создания чата; readiness; обход кэша при отказе
Valkey и его возврат без рестарта API. С лабы 3 — ключ идемпотентности:
сбой записи сообщения откатывает и ключ, транзиентная 503 не фиксируется
результатом (повтор с тем же ключом проходит один раз), срок ключа.
Границы времени — с запасом 0,3–1 с, но на порядок ниже «плохого»
поведения (секунды повторов redis-py, зависший до разморозки запрос).
"""
import statistics
import threading
import time
import uuid
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from typing import Any
import httpx
import pytest
from helpers import (
BEARER_CHALLENGE,
CHATS,
ME,
assert_error,
assert_unauthorized,
chat_ids,
create_chat,
headers,
login,
new_key,
public,
read_history,
register,
send,
)
pytestmark = pytest.mark.fault
type Stand = Callable[..., str]
RESTORE = {"stop": "start", "pause": "unpause"}
# запас к границам времени: планировщик Docker Desktop
SLACK = 0.3
# значения по умолчанию CACHE_TIMEOUT и CACHE_RETRY_AFTER
CACHE_TIMEOUT = 0.2
CACHE_RETRY_AFTER = 5.0
# окружение «тесного» API (TIGHT_ENV в stand.py)
SESSION_TTL = 2
IDEMPOTENCY_TTL = 2
POOL_TIMEOUT = 1.0
OPERATION_TIMEOUT = 2.0
# ограниченное закрытие соединения (DB_CLOSE_TIMEOUT) и срок readiness
CLOSE_TIMEOUT = 2.0
READINESS_TIMEOUT = 2.0
@contextmanager
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
"""`stop` или `pause` хранилища; в `finally` — обратно."""
stand(action, service)
try:
yield
finally:
stand(RESTORE[action], service)
def timed(call: Callable[[], httpx.Response]) -> tuple[httpx.Response, float]:
started = time.monotonic()
response = call()
return response, time.monotonic() - started
def read_chat(client: httpx.Client, actor: dict[str, Any], chat: dict[str, Any]):
return client.get(f"{CHATS}/{chat['id']}", headers=headers(actor))
def wait_cache_ready(
client: httpx.Client,
stand: Stand,
actor: dict[str, Any],
chat: dict[str, Any],
) -> None:
"""Дождаться, пока чтение снова пишет карточку в кэш.
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
"""
deadline = time.monotonic() + CACHE_RETRY_AFTER + 2
stand("flush-cache")
while time.monotonic() < deadline:
assert read_chat(client, actor, chat).status_code == 200
if stand("cache-has-chat", chat["id"]) == "1":
return
time.sleep(0.5)
raise AssertionError("cache did not come back")
# --- сессии: срок 2 с на «тесном» API, БД остановлена — на обычном -----------
def test_expired_token_is_401(tight_client: httpx.Client) -> None:
actor = register(tight_client, "ttl")
assert tight_client.get(ME, headers=headers(actor)).status_code == 200
time.sleep(SESSION_TTL + 1)
assert_unauthorized(tight_client.get(ME, headers=headers(actor)), BEARER_CHALLENGE)
def test_requests_do_not_extend_session(tight_client: httpx.Client) -> None:
actor = register(tight_client, "ttl")
logged_in = time.monotonic()
successes = 0
while time.monotonic() - logged_in < SESSION_TTL - SLACK:
assert tight_client.get(ME, headers=headers(actor)).status_code == 200
successes += 1
time.sleep(0.5)
assert successes >= 3, "too few requests before expiry"
time.sleep(max(0.0, logged_in + SESSION_TTL + 1 - time.monotonic()))
assert_unauthorized(tight_client.get(ME, headers=headers(actor)), BEARER_CHALLENGE)
def test_token_while_database_stopped_is_503(
client: httpx.Client, stand: Stand
) -> None:
actor = register(client, "db")
with outage(stand, "stop", "postgres"):
response = client.get(ME, headers=headers(actor))
assert_error(response, 503, "unavailable")
# недоступность хранилища — не отказ в аутентификации
assert "WWW-Authenticate" not in response.headers
# тот же токен после возврата БД
assert client.get(ME, headers=headers(actor)).json() == public(actor)
# --- хранение: откат, остановленная и замёрзшая БД, исчерпанный пул ------------
def test_members_failure_rolls_back_chat(client: httpx.Client, stand: Stand) -> None:
alice, bob = register(client, "alice"), register(client, "bob")
title = f"fault-{uuid.uuid4().hex}"
stand("fail-members", title)
try:
response = client.post(
CHATS,
json={"title": title, "member_ids": [alice["id"], bob["id"]]},
headers=headers(alice),
)
assert_error(response, 500, "internal_error")
finally:
stand("clear-fail-members")
# ни чата, ни участников: вставка строки чата откатилась вместе с ними
for actor in (alice, bob):
assert chat_ids(client, actor) == []
assert stand("count-chats", title) == "0"
def test_database_stopped_and_back(client: httpx.Client, stand: Stand) -> None:
alice = register(client, "alice")
chat = create_chat(client, alice)
with outage(stand, "stop", "postgres"):
response, took = timed(lambda: send(client, alice, chat, "во время отказа"))
error = assert_error(response, 503, "unavailable")
print(f"postgres stopped: POST message -> 503 in {took:.3f}s")
assert took < 2, f"503 took {took:.2f}s"
for word in ("postgres", "sql", "connect", "asyncpg"):
assert word not in error["message"].lower()
after = send(client, alice, chat, "после возврата")
assert after.status_code == 201, after.text
# запись во время отказа не зафиксирована, новая — видна
assert read_history(client, alice, chat, limit=50)[0] == [after.json()]
def test_frozen_database_and_exhausted_pool(
tight_client: httpx.Client,
stand: Stand,
) -> None:
"""Замёрзшая БД держит запрос не дольше срока операции и закрытия
соединения; пока пул занят, следующий запрос ждёт не дольше срока пула."""
actor = register(tight_client, "pool")
first: list[tuple[int, float]] = []
def hold_connection() -> None:
with httpx.Client(base_url=str(tight_client.base_url), timeout=20) as holder:
response, took = timed(lambda: holder.get(ME, headers=headers(actor)))
first.append((response.status_code, took))
with outage(stand, "pause", "postgres"):
thread = threading.Thread(target=hold_connection)
thread.start()
time.sleep(SLACK)
response, took = timed(lambda: tight_client.get(ME, headers=headers(actor)))
assert_error(response, 503, "unavailable")
assert took < POOL_TIMEOUT + 0.5, f"pool wait took {took:.2f}s"
thread.join()
status, held = first[0]
print(
f"postgres paused: first request -> {status} in {held:.2f}s, "
f"second (pool of 1 busy) -> {response.status_code} in {took:.2f}s",
)
assert status == 503
assert held < OPERATION_TIMEOUT + CLOSE_TIMEOUT + 1, f"frozen db held {held:.2f}s"
# срок сессии 2 с мог истечь за время паузы: пул проверяем новым входом
fresh = register(tight_client, "pool")
assert tight_client.get(ME, headers=headers(fresh)).status_code == 200
# --- ключ идемпотентности (лаба 3) ----------------------------------------------
def test_message_failure_rolls_back_key(client: httpx.Client, stand: Stand) -> None:
"""Ключ вставлен первым, сообщение упало — откат убирает оба: повтор с
тем же ключом создаёт сообщение, а не 409 и не «пустой» повтор."""
alice = register(client, "alice")
chat = create_chat(client, alice)
text = f"fault-{uuid.uuid4().hex}"
key = new_key()
stand("fail-messages", text)
try:
assert_error(send(client, alice, chat, text, key=key), 500, "internal_error")
finally:
stand("clear-fail-messages")
assert stand("count-keys", chat["id"]) == "0"
assert read_history(client, alice, chat, limit=50)[0] == []
retry = send(client, alice, chat, text, key=key)
assert retry.status_code == 201, retry.text
assert read_history(client, alice, chat, limit=50)[0] == [retry.json()]
assert stand("count-keys", chat["id"]) == "1"
def test_unavailable_is_not_saved_as_result(client: httpx.Client, stand: Stand) -> None:
"""503 — транзиентный результат: тот же ключ после возврата БД даёт 201,
повтор — то же сообщение, в истории одно."""
alice = register(client, "alice")
chat = create_chat(client, alice)
key = new_key()
with outage(stand, "stop", "postgres"):
assert_error(
send(client, alice, chat, "неопределённо", key=key), 503, "unavailable"
)
first = send(client, alice, chat, "неопределённо", key=key)
assert first.status_code == 201, first.text
again = send(client, alice, chat, "неопределённо", key=key)
assert again.json() == first.json()
assert read_history(client, alice, chat, limit=50)[0] == [first.json()]
def test_key_expires_after_ttl(tight_client: httpx.Client) -> None:
"""Срок ключа 2 с на «тесном» API: в срок — повтор и 409, после срока
тот же ключ занимается заново (другое тело — новое сообщение, не 409)."""
alice = register(tight_client, "ttl")
chat = create_chat(tight_client, alice)
key = new_key()
first = send(tight_client, alice, chat, "в срок", key=key).json()
assert send(tight_client, alice, chat, "в срок", key=key).json() == first
assert_error(send(tight_client, alice, chat, "другое", key=key), 409, "conflict")
time.sleep(IDEMPOTENCY_TTL + 1)
# срок сессии на этом API тоже 2 с: новый вход
alice["token"] = login(tight_client, alice)
late = send(tight_client, alice, chat, "после срока", key=key)
assert late.status_code == 201, late.text
assert late.json()["id"] != first["id"]
items, _ = read_history(tight_client, alice, chat, limit=50)
assert [item["text"] for item in items] == ["в срок", "после срока"]
# --- readiness -------------------------------------------------------------------
def test_readiness_without_database(client: httpx.Client, stand: Stand) -> None:
with outage(stand, "stop", "postgres"):
response, took = timed(lambda: client.get("/health/ready"))
assert_error(response, 503, "unavailable")
print(f"postgres stopped: /health/ready -> 503 in {took:.3f}s")
assert took < READINESS_TIMEOUT + 0.5, f"ready took {took:.2f}s"
assert client.get("/health/live").status_code == 200
assert client.get("/health/ready").status_code == 200
def test_readiness_without_cache(client: httpx.Client, stand: Stand) -> None:
with outage(stand, "stop", "valkey"):
assert client.get("/health/ready").status_code == 200
# --- кэш карточки чата -------------------------------------------------------------
def test_flush_cache_refills_on_read(client: httpx.Client, stand: Stand) -> None:
alice, bob = register(client, "alice"), register(client, "bob")
chat = create_chat(client, alice, bob)
wait_cache_ready(client, stand, bob, chat)
stand("flush-cache")
assert stand("cache-has-chat", chat["id"]) == "0"
assert read_chat(client, bob, chat).json() == chat
assert stand("cache-has-chat", chat["id"]) == "1"
def test_valkey_stopped_is_bypassed(client: httpx.Client, stand: Stand) -> None:
alice, bob = register(client, "alice"), register(client, "bob")
chat = create_chat(client, alice, bob)
sent = [send(client, alice, chat, "до отказа").json()]
with outage(stand, "stop", "valkey"):
response, read_took = timed(lambda: read_chat(client, bob, chat))
assert response.json() == chat
assert read_took < CACHE_TIMEOUT + SLACK, f"read took {read_took:.2f}s"
response, took = timed(lambda: send(client, alice, chat, "во время отказа"))
assert response.status_code == 201, response.text
print(
f"valkey stopped: GET chat -> 200 in {read_took:.3f}s, "
f"POST message -> 201 in {took:.3f}s",
)
assert took < CACHE_TIMEOUT + SLACK, f"send took {took:.2f}s"
sent.append(response.json())
assert read_history(client, bob, chat, limit=50)[0] == sent
def test_valkey_paused_costs_one_timeout(client: httpx.Client, stand: Stand) -> None:
alice, bob = register(client, "alice"), register(client, "bob")
chat = create_chat(client, alice, bob)
wait_cache_ready(client, stand, bob, chat)
baseline = statistics.median(
timed(lambda: read_chat(client, bob, chat))[1] for _ in range(5)
)
with outage(stand, "pause", "valkey"):
response, took = timed(lambda: read_chat(client, bob, chat))
assert response.json() == chat
assert took < baseline + CACHE_TIMEOUT + SLACK, (
f"read took {took:.2f}s, baseline {baseline:.3f}s"
)
# окно пропуска: следующий запрос срок не ждёт
response, second = timed(lambda: read_chat(client, bob, chat))
assert response.json() == chat
print(
f"valkey paused: baseline {baseline:.3f}s, first read {took:.3f}s, "
f"second read {second:.3f}s",
)
assert second < CACHE_TIMEOUT, f"second read took {second:.2f}s"
def test_valkey_back_without_api_restart(client: httpx.Client, stand: Stand) -> None:
alice, bob = register(client, "alice"), register(client, "bob")
chat = create_chat(client, alice, bob)
with outage(stand, "stop", "valkey"):
assert read_chat(client, bob, chat).status_code == 200
# API не перезапускался; кэш снова в работе после окна пропуска
time.sleep(CACHE_RETRY_AFTER + 0.5)
assert read_chat(client, bob, chat).status_code == 200
assert stand("cache-has-chat", chat["id"]) == "1"
hits = int(stand("cache-hits"))
assert read_chat(client, bob, chat).json() == chat
assert int(stand("cache-hits")) > hits