361 lines
16 KiB
Python
361 lines
16 KiB
Python
"""Проверки отказов (уровень «4»): выполняет только `make failure-checks`.
|
|
|
|
Стенд и hook готовит `student-tests/stand.py failures`: свои PostgreSQL и
|
|
Valkey, обычный API (`BASE_URL`) и «тесный» (сроки сессии и ключа
|
|
идемпотентности 2 с, пул из одного соединения, срок ожидания пула 1 с,
|
|
срок операции 2 с). На хранилища тесты действуют только через фикстуру
|
|
`stand` (hook `student-tests/fault_hook.py`) и всегда возвращают их в
|
|
рабочее состояние (`outage`): упавшая проверка не ломает следующие.
|
|
|
|
Что проверяется: истечение сессии и то, что запросы её не продлевают;
|
|
недоступная БД — 503 без деталей и без вызова аутентификации, а не 401 и
|
|
не анонимный доступ; ограниченное ожидание при замёрзшей БД и
|
|
исчерпанном пуле; откат создания чата; readiness; обход кэша при отказе
|
|
Valkey и его возврат без рестарта API. С лабы 3 — ключ идемпотентности:
|
|
сбой записи сообщения откатывает и ключ, транзиентная 503 не фиксируется
|
|
результатом (повтор с тем же ключом проходит один раз), срок ключа.
|
|
Границы времени — с запасом 0,3–1 с, но на порядок ниже «плохого»
|
|
поведения (секунды повторов redis-py, зависший до разморозки запрос).
|
|
"""
|
|
|
|
import statistics
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from collections.abc import Callable, Iterator
|
|
from contextlib import contextmanager
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from helpers import (
|
|
BEARER_CHALLENGE,
|
|
CHATS,
|
|
ME,
|
|
assert_error,
|
|
assert_unauthorized,
|
|
chat_ids,
|
|
create_chat,
|
|
headers,
|
|
login,
|
|
new_key,
|
|
public,
|
|
read_history,
|
|
register,
|
|
send,
|
|
)
|
|
|
|
pytestmark = pytest.mark.fault
|
|
|
|
type Stand = Callable[..., str]
|
|
|
|
RESTORE = {"stop": "start", "pause": "unpause"}
|
|
# запас к границам времени: планировщик Docker Desktop
|
|
SLACK = 0.3
|
|
# значения по умолчанию CACHE_TIMEOUT и CACHE_RETRY_AFTER
|
|
CACHE_TIMEOUT = 0.2
|
|
CACHE_RETRY_AFTER = 5.0
|
|
# окружение «тесного» API (TIGHT_ENV в stand.py)
|
|
SESSION_TTL = 2
|
|
IDEMPOTENCY_TTL = 2
|
|
POOL_TIMEOUT = 1.0
|
|
OPERATION_TIMEOUT = 2.0
|
|
# ограниченное закрытие соединения (DB_CLOSE_TIMEOUT) и срок readiness
|
|
CLOSE_TIMEOUT = 2.0
|
|
READINESS_TIMEOUT = 2.0
|
|
|
|
|
|
@contextmanager
|
|
def outage(stand: Stand, action: str, service: str) -> Iterator[None]:
|
|
"""`stop` или `pause` хранилища; в `finally` — обратно."""
|
|
stand(action, service)
|
|
try:
|
|
yield
|
|
finally:
|
|
stand(RESTORE[action], service)
|
|
|
|
|
|
def timed(call: Callable[[], httpx.Response]) -> tuple[httpx.Response, float]:
|
|
started = time.monotonic()
|
|
response = call()
|
|
return response, time.monotonic() - started
|
|
|
|
|
|
def read_chat(client: httpx.Client, actor: dict[str, Any], chat: dict[str, Any]):
|
|
return client.get(f"{CHATS}/{chat['id']}", headers=headers(actor))
|
|
|
|
|
|
def wait_cache_ready(
|
|
client: httpx.Client,
|
|
stand: Stand,
|
|
actor: dict[str, Any],
|
|
chat: dict[str, Any],
|
|
) -> None:
|
|
"""Дождаться, пока чтение снова пишет карточку в кэш.
|
|
|
|
После отказа в прошлом тесте API пропускает кэш ещё до 5 с.
|
|
"""
|
|
deadline = time.monotonic() + CACHE_RETRY_AFTER + 2
|
|
stand("flush-cache")
|
|
while time.monotonic() < deadline:
|
|
assert read_chat(client, actor, chat).status_code == 200
|
|
if stand("cache-has-chat", chat["id"]) == "1":
|
|
return
|
|
time.sleep(0.5)
|
|
raise AssertionError("cache did not come back")
|
|
|
|
|
|
# --- сессии: срок 2 с на «тесном» API, БД остановлена — на обычном -----------
|
|
|
|
|
|
def test_expired_token_is_401(tight_client: httpx.Client) -> None:
|
|
actor = register(tight_client, "ttl")
|
|
assert tight_client.get(ME, headers=headers(actor)).status_code == 200
|
|
time.sleep(SESSION_TTL + 1)
|
|
assert_unauthorized(tight_client.get(ME, headers=headers(actor)), BEARER_CHALLENGE)
|
|
|
|
|
|
def test_requests_do_not_extend_session(tight_client: httpx.Client) -> None:
|
|
actor = register(tight_client, "ttl")
|
|
logged_in = time.monotonic()
|
|
successes = 0
|
|
while time.monotonic() - logged_in < SESSION_TTL - SLACK:
|
|
assert tight_client.get(ME, headers=headers(actor)).status_code == 200
|
|
successes += 1
|
|
time.sleep(0.5)
|
|
assert successes >= 3, "too few requests before expiry"
|
|
time.sleep(max(0.0, logged_in + SESSION_TTL + 1 - time.monotonic()))
|
|
assert_unauthorized(tight_client.get(ME, headers=headers(actor)), BEARER_CHALLENGE)
|
|
|
|
|
|
def test_token_while_database_stopped_is_503(
|
|
client: httpx.Client, stand: Stand
|
|
) -> None:
|
|
actor = register(client, "db")
|
|
with outage(stand, "stop", "postgres"):
|
|
response = client.get(ME, headers=headers(actor))
|
|
assert_error(response, 503, "unavailable")
|
|
# недоступность хранилища — не отказ в аутентификации
|
|
assert "WWW-Authenticate" not in response.headers
|
|
# тот же токен после возврата БД
|
|
assert client.get(ME, headers=headers(actor)).json() == public(actor)
|
|
|
|
|
|
# --- хранение: откат, остановленная и замёрзшая БД, исчерпанный пул ------------
|
|
|
|
|
|
def test_members_failure_rolls_back_chat(client: httpx.Client, stand: Stand) -> None:
|
|
alice, bob = register(client, "alice"), register(client, "bob")
|
|
title = f"fault-{uuid.uuid4().hex}"
|
|
stand("fail-members", title)
|
|
try:
|
|
response = client.post(
|
|
CHATS,
|
|
json={"title": title, "member_ids": [alice["id"], bob["id"]]},
|
|
headers=headers(alice),
|
|
)
|
|
assert_error(response, 500, "internal_error")
|
|
finally:
|
|
stand("clear-fail-members")
|
|
# ни чата, ни участников: вставка строки чата откатилась вместе с ними
|
|
for actor in (alice, bob):
|
|
assert chat_ids(client, actor) == []
|
|
assert stand("count-chats", title) == "0"
|
|
|
|
|
|
def test_database_stopped_and_back(client: httpx.Client, stand: Stand) -> None:
|
|
alice = register(client, "alice")
|
|
chat = create_chat(client, alice)
|
|
with outage(stand, "stop", "postgres"):
|
|
response, took = timed(lambda: send(client, alice, chat, "во время отказа"))
|
|
error = assert_error(response, 503, "unavailable")
|
|
print(f"postgres stopped: POST message -> 503 in {took:.3f}s")
|
|
assert took < 2, f"503 took {took:.2f}s"
|
|
for word in ("postgres", "sql", "connect", "asyncpg"):
|
|
assert word not in error["message"].lower()
|
|
after = send(client, alice, chat, "после возврата")
|
|
assert after.status_code == 201, after.text
|
|
# запись во время отказа не зафиксирована, новая — видна
|
|
assert read_history(client, alice, chat, limit=50)[0] == [after.json()]
|
|
|
|
|
|
def test_frozen_database_and_exhausted_pool(
|
|
tight_client: httpx.Client,
|
|
stand: Stand,
|
|
) -> None:
|
|
"""Замёрзшая БД держит запрос не дольше срока операции и закрытия
|
|
соединения; пока пул занят, следующий запрос ждёт не дольше срока пула."""
|
|
actor = register(tight_client, "pool")
|
|
first: list[tuple[int, float]] = []
|
|
|
|
def hold_connection() -> None:
|
|
with httpx.Client(base_url=str(tight_client.base_url), timeout=20) as holder:
|
|
response, took = timed(lambda: holder.get(ME, headers=headers(actor)))
|
|
first.append((response.status_code, took))
|
|
|
|
with outage(stand, "pause", "postgres"):
|
|
thread = threading.Thread(target=hold_connection)
|
|
thread.start()
|
|
time.sleep(SLACK)
|
|
response, took = timed(lambda: tight_client.get(ME, headers=headers(actor)))
|
|
assert_error(response, 503, "unavailable")
|
|
assert took < POOL_TIMEOUT + 0.5, f"pool wait took {took:.2f}s"
|
|
thread.join()
|
|
status, held = first[0]
|
|
print(
|
|
f"postgres paused: first request -> {status} in {held:.2f}s, "
|
|
f"second (pool of 1 busy) -> {response.status_code} in {took:.2f}s",
|
|
)
|
|
assert status == 503
|
|
assert held < OPERATION_TIMEOUT + CLOSE_TIMEOUT + 1, f"frozen db held {held:.2f}s"
|
|
# срок сессии 2 с мог истечь за время паузы: пул проверяем новым входом
|
|
fresh = register(tight_client, "pool")
|
|
assert tight_client.get(ME, headers=headers(fresh)).status_code == 200
|
|
|
|
|
|
# --- ключ идемпотентности (лаба 3) ----------------------------------------------
|
|
|
|
|
|
def test_message_failure_rolls_back_key(client: httpx.Client, stand: Stand) -> None:
|
|
"""Ключ вставлен первым, сообщение упало — откат убирает оба: повтор с
|
|
тем же ключом создаёт сообщение, а не 409 и не «пустой» повтор."""
|
|
alice = register(client, "alice")
|
|
chat = create_chat(client, alice)
|
|
text = f"fault-{uuid.uuid4().hex}"
|
|
key = new_key()
|
|
stand("fail-messages", text)
|
|
try:
|
|
assert_error(send(client, alice, chat, text, key=key), 500, "internal_error")
|
|
finally:
|
|
stand("clear-fail-messages")
|
|
assert stand("count-keys", chat["id"]) == "0"
|
|
assert read_history(client, alice, chat, limit=50)[0] == []
|
|
retry = send(client, alice, chat, text, key=key)
|
|
assert retry.status_code == 201, retry.text
|
|
assert read_history(client, alice, chat, limit=50)[0] == [retry.json()]
|
|
assert stand("count-keys", chat["id"]) == "1"
|
|
|
|
|
|
def test_unavailable_is_not_saved_as_result(client: httpx.Client, stand: Stand) -> None:
|
|
"""503 — транзиентный результат: тот же ключ после возврата БД даёт 201,
|
|
повтор — то же сообщение, в истории одно."""
|
|
alice = register(client, "alice")
|
|
chat = create_chat(client, alice)
|
|
key = new_key()
|
|
with outage(stand, "stop", "postgres"):
|
|
assert_error(
|
|
send(client, alice, chat, "неопределённо", key=key), 503, "unavailable"
|
|
)
|
|
first = send(client, alice, chat, "неопределённо", key=key)
|
|
assert first.status_code == 201, first.text
|
|
again = send(client, alice, chat, "неопределённо", key=key)
|
|
assert again.json() == first.json()
|
|
assert read_history(client, alice, chat, limit=50)[0] == [first.json()]
|
|
|
|
|
|
def test_key_expires_after_ttl(tight_client: httpx.Client) -> None:
|
|
"""Срок ключа 2 с на «тесном» API: в срок — повтор и 409, после срока
|
|
тот же ключ занимается заново (другое тело — новое сообщение, не 409)."""
|
|
alice = register(tight_client, "ttl")
|
|
chat = create_chat(tight_client, alice)
|
|
key = new_key()
|
|
first = send(tight_client, alice, chat, "в срок", key=key).json()
|
|
assert send(tight_client, alice, chat, "в срок", key=key).json() == first
|
|
assert_error(send(tight_client, alice, chat, "другое", key=key), 409, "conflict")
|
|
time.sleep(IDEMPOTENCY_TTL + 1)
|
|
# срок сессии на этом API тоже 2 с: новый вход
|
|
alice["token"] = login(tight_client, alice)
|
|
late = send(tight_client, alice, chat, "после срока", key=key)
|
|
assert late.status_code == 201, late.text
|
|
assert late.json()["id"] != first["id"]
|
|
items, _ = read_history(tight_client, alice, chat, limit=50)
|
|
assert [item["text"] for item in items] == ["в срок", "после срока"]
|
|
|
|
|
|
# --- readiness -------------------------------------------------------------------
|
|
|
|
|
|
def test_readiness_without_database(client: httpx.Client, stand: Stand) -> None:
|
|
with outage(stand, "stop", "postgres"):
|
|
response, took = timed(lambda: client.get("/health/ready"))
|
|
assert_error(response, 503, "unavailable")
|
|
print(f"postgres stopped: /health/ready -> 503 in {took:.3f}s")
|
|
assert took < READINESS_TIMEOUT + 0.5, f"ready took {took:.2f}s"
|
|
assert client.get("/health/live").status_code == 200
|
|
assert client.get("/health/ready").status_code == 200
|
|
|
|
|
|
def test_readiness_without_cache(client: httpx.Client, stand: Stand) -> None:
|
|
with outage(stand, "stop", "valkey"):
|
|
assert client.get("/health/ready").status_code == 200
|
|
|
|
|
|
# --- кэш карточки чата -------------------------------------------------------------
|
|
|
|
|
|
def test_flush_cache_refills_on_read(client: httpx.Client, stand: Stand) -> None:
|
|
alice, bob = register(client, "alice"), register(client, "bob")
|
|
chat = create_chat(client, alice, bob)
|
|
wait_cache_ready(client, stand, bob, chat)
|
|
stand("flush-cache")
|
|
assert stand("cache-has-chat", chat["id"]) == "0"
|
|
assert read_chat(client, bob, chat).json() == chat
|
|
assert stand("cache-has-chat", chat["id"]) == "1"
|
|
|
|
|
|
def test_valkey_stopped_is_bypassed(client: httpx.Client, stand: Stand) -> None:
|
|
alice, bob = register(client, "alice"), register(client, "bob")
|
|
chat = create_chat(client, alice, bob)
|
|
sent = [send(client, alice, chat, "до отказа").json()]
|
|
with outage(stand, "stop", "valkey"):
|
|
response, read_took = timed(lambda: read_chat(client, bob, chat))
|
|
assert response.json() == chat
|
|
assert read_took < CACHE_TIMEOUT + SLACK, f"read took {read_took:.2f}s"
|
|
response, took = timed(lambda: send(client, alice, chat, "во время отказа"))
|
|
assert response.status_code == 201, response.text
|
|
print(
|
|
f"valkey stopped: GET chat -> 200 in {read_took:.3f}s, "
|
|
f"POST message -> 201 in {took:.3f}s",
|
|
)
|
|
assert took < CACHE_TIMEOUT + SLACK, f"send took {took:.2f}s"
|
|
sent.append(response.json())
|
|
assert read_history(client, bob, chat, limit=50)[0] == sent
|
|
|
|
|
|
def test_valkey_paused_costs_one_timeout(client: httpx.Client, stand: Stand) -> None:
|
|
alice, bob = register(client, "alice"), register(client, "bob")
|
|
chat = create_chat(client, alice, bob)
|
|
wait_cache_ready(client, stand, bob, chat)
|
|
baseline = statistics.median(
|
|
timed(lambda: read_chat(client, bob, chat))[1] for _ in range(5)
|
|
)
|
|
with outage(stand, "pause", "valkey"):
|
|
response, took = timed(lambda: read_chat(client, bob, chat))
|
|
assert response.json() == chat
|
|
assert took < baseline + CACHE_TIMEOUT + SLACK, (
|
|
f"read took {took:.2f}s, baseline {baseline:.3f}s"
|
|
)
|
|
# окно пропуска: следующий запрос срок не ждёт
|
|
response, second = timed(lambda: read_chat(client, bob, chat))
|
|
assert response.json() == chat
|
|
print(
|
|
f"valkey paused: baseline {baseline:.3f}s, first read {took:.3f}s, "
|
|
f"second read {second:.3f}s",
|
|
)
|
|
assert second < CACHE_TIMEOUT, f"second read took {second:.2f}s"
|
|
|
|
|
|
def test_valkey_back_without_api_restart(client: httpx.Client, stand: Stand) -> None:
|
|
alice, bob = register(client, "alice"), register(client, "bob")
|
|
chat = create_chat(client, alice, bob)
|
|
with outage(stand, "stop", "valkey"):
|
|
assert read_chat(client, bob, chat).status_code == 200
|
|
# API не перезапускался; кэш снова в работе после окна пропуска
|
|
time.sleep(CACHE_RETRY_AFTER + 0.5)
|
|
assert read_chat(client, bob, chat).status_code == 200
|
|
assert stand("cache-has-chat", chat["id"]) == "1"
|
|
hits = int(stand("cache-hits"))
|
|
assert read_chat(client, bob, chat).json() == chat
|
|
assert int(stand("cache-hits")) > hits
|