Files
mtusi2026lab3/student-tests/test_failures.py
Alexey BodanyukandClaude Opus 5.5 78b95acabf test: строгие тесты lab3 — идемпотентность, балансировка, отказы реплик
- test_idempotency: точные коды и тела повторов через разные реплики, 24
  одновременных запроса с одним ключом -> ровно одно сообщение и одна строка
  ключа в БД, 409 при другом body, scope user+chat+key, 404 постороннему,
  откат после сбоя на втором INSERT (триггер, n_tup_ins), неопределённый
  POST (клиентский timeout) и lock_timeout с повтором тем же ключом.
- test_balancing: X-Instance-Id = реальные контейнеры, нет sticky,
  LB исключает живую, но неготовую реплику (pg_hba reject), graceful stop
  каждой реплики под нагрузкой без единой ошибки, SIGKILL — ограниченные
  ошибки и восстановление, полный restart и scale up/down без рестарта LB,
  политика compose (лимиты, порты, secrets, digest), миграции один раз.
- Входы из замечаний к чужим PR: курсоры с []/{}/числами/пустые/пробел,
  байт FF, TEXT/text, дубли ключей, Bearer é, UUID в разном регистре, тело
  при входе, пароль "я:"*64. Тесты lab2 переведены на Idempotency-Key и LB;
  все отказы восстанавливаются в finally и ждут UP всех реплик в HAProxy.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 13:32:13 +03:00

112 lines
5.4 KiB
Python

"""Dependency failures with a VALID session: bounded 503, never anonymous access, recovery afterwards.
Every test stops/pauses a real container and restores it in `finally` (service_down).
Timing bounds: DB_TIMEOUT_SECONDS=3 (client deadline), CACHE_TIMEOUT_SECONDS=0.5.
Lab 3: requests go through HAProxy. While the outage is being detected the replicas answer
503 themselves; once every replica fails /health/ready, HAProxy answers 503 in the same
contract format (without X-Instance-Id). Both are "unavailable"; never 200 and never 401
for a valid token. Liveness/readiness of the processes are checked inside each container.
"""
from __future__ import annotations
import subprocess
import time
import pytest
from conftest import ROOT, probe_inside, restart_services, running_replicas, service_down
pytestmark = pytest.mark.stack
DB_BOUND = 3.6 # DB_TIMEOUT_SECONDS + network/HTTP overhead
CACHE_BOUND = 1.6 # 2 x CACHE_TIMEOUT_SECONDS + overhead
def timed(api, *args, **kwargs):
started = time.monotonic()
response = api.call(*args, lb_error_ok=True, **kwargs)
return response, time.monotonic() - started
def processes_alive_but_not_ready() -> None:
for replica in running_replicas():
assert probe_inside(replica, "/health/live") == 200
assert probe_inside(replica, "/health/ready") == 503
@pytest.fixture
def prepared(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
message = api.send(alice, chat, "до отказа")
return alice, bob, chat, message
def check_intact(api, alice, bob, chat, message):
"""After recovery the SAME tokens work and confirmed data is unchanged."""
assert api.call("GET", "/api/v1/users/me", 200, token=alice["token"]).json() == alice["user"]
items = api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json()["items"]
assert items == [message]
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_valkey_failure(api, prepared, how):
alice, bob, chat, message = prepared
with service_down("valkey", how):
response, took = timed(api, "GET", "/api/v1/users/me", 503, token=alice["token"])
assert response.json()["error"]["code"] == "unavailable" and took < CACHE_BOUND, took
api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
api.call("GET", "/health/ready", 503, lb_error_ok=True)
processes_alive_but_not_ready()
check_intact(api, alice, bob, chat, message)
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_postgres_failure(api, prepared, how):
alice, bob, chat, message = prepared
with service_down("postgres", how):
for path in ("/api/v1/users/me", f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"):
response, took = timed(api, "GET", path, 503, token=alice["token"])
assert response.json()["error"]["code"] == "unavailable", path
assert took < DB_BOUND, (path, took) # bounded even when PostgreSQL is frozen (pause)
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
api.call("GET", "/health/ready", 503, lb_error_ok=True)
processes_alive_but_not_ready()
check_intact(api, alice, bob, chat, message)
def test_locked_table_gives_bounded_503(api, prepared):
"""A statement stuck behind a lock is stopped by lock_timeout, not after minutes."""
alice, bob, chat, message = prepared
locker = subprocess.Popen(
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c",
"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(6); COMMIT;"],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
time.sleep(1.5) # let the lock be taken
response, took = timed(api, "GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
assert response.json()["error"]["code"] == "unavailable" and took < DB_BOUND, took
assert response.headers.get("x-instance-id"), "a lock is the API's own 503, the replicas stay ready"
finally:
locker.wait(timeout=30)
check_intact(api, alice, bob, chat, message)
def test_logout_during_valkey_outage_is_never_lost(api, prepared):
"""Valkey is the single source of truth for sessions, so a logout cannot be 'forgotten':
during the outage logout answers 503 (nothing revoked, the client knows it failed);
after recovery logout works, and the revocation survives a Valkey and API restart (AOF)."""
alice, bob, chat, message = prepared
with service_down("valkey", "stop"):
api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"], lb_error_ok=True)
api.call("GET", "/api/v1/users/me", 200, token=alice["token"]) # logout did not happen
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
restart_services("valkey", "api")
for _ in range(10): # repeated checks: the revoked token must never come back
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], json_body={"text": "x"})
api.call("GET", "/api/v1/users/me", 200, token=bob["token"]) # other sessions survived the restart