forked from mtusi2026/mtusi2026lab3
- test_idempotency: точные коды и тела повторов через разные реплики, 24
одновременных запроса с одним ключом -> ровно одно сообщение и одна строка
ключа в БД, 409 при другом body, scope user+chat+key, 404 постороннему,
откат после сбоя на втором INSERT (триггер, n_tup_ins), неопределённый
POST (клиентский timeout) и lock_timeout с повтором тем же ключом.
- test_balancing: X-Instance-Id = реальные контейнеры, нет sticky,
LB исключает живую, но неготовую реплику (pg_hba reject), graceful stop
каждой реплики под нагрузкой без единой ошибки, SIGKILL — ограниченные
ошибки и восстановление, полный restart и scale up/down без рестарта LB,
политика compose (лимиты, порты, secrets, digest), миграции один раз.
- Входы из замечаний к чужим PR: курсоры с []/{}/числами/пустые/пробел,
байт FF, TEXT/text, дубли ключей, Bearer é, UUID в разном регистре, тело
при входе, пароль "я:"*64. Тесты lab2 переведены на Idempotency-Key и LB;
все отказы восстанавливаются в finally и ждут UP всех реплик в HAProxy.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
112 lines
5.4 KiB
Python
112 lines
5.4 KiB
Python
"""Dependency failures with a VALID session: bounded 503, never anonymous access, recovery afterwards.
|
|
|
|
Every test stops/pauses a real container and restores it in `finally` (service_down).
|
|
Timing bounds: DB_TIMEOUT_SECONDS=3 (client deadline), CACHE_TIMEOUT_SECONDS=0.5.
|
|
|
|
Lab 3: requests go through HAProxy. While the outage is being detected the replicas answer
|
|
503 themselves; once every replica fails /health/ready, HAProxy answers 503 in the same
|
|
contract format (without X-Instance-Id). Both are "unavailable"; never 200 and never 401
|
|
for a valid token. Liveness/readiness of the processes are checked inside each container.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import subprocess
|
|
import time
|
|
|
|
import pytest
|
|
|
|
from conftest import ROOT, probe_inside, restart_services, running_replicas, service_down
|
|
|
|
pytestmark = pytest.mark.stack
|
|
|
|
DB_BOUND = 3.6 # DB_TIMEOUT_SECONDS + network/HTTP overhead
|
|
CACHE_BOUND = 1.6 # 2 x CACHE_TIMEOUT_SECONDS + overhead
|
|
|
|
|
|
def timed(api, *args, **kwargs):
|
|
started = time.monotonic()
|
|
response = api.call(*args, lb_error_ok=True, **kwargs)
|
|
return response, time.monotonic() - started
|
|
|
|
|
|
def processes_alive_but_not_ready() -> None:
|
|
for replica in running_replicas():
|
|
assert probe_inside(replica, "/health/live") == 200
|
|
assert probe_inside(replica, "/health/ready") == 503
|
|
|
|
|
|
@pytest.fixture
|
|
def prepared(api):
|
|
alice, bob = api.user("alice"), api.user("bob")
|
|
chat = api.chat(alice, bob)
|
|
message = api.send(alice, chat, "до отказа")
|
|
return alice, bob, chat, message
|
|
|
|
|
|
def check_intact(api, alice, bob, chat, message):
|
|
"""After recovery the SAME tokens work and confirmed data is unchanged."""
|
|
assert api.call("GET", "/api/v1/users/me", 200, token=alice["token"]).json() == alice["user"]
|
|
items = api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json()["items"]
|
|
assert items == [message]
|
|
|
|
|
|
@pytest.mark.parametrize("how", ["stop", "pause"])
|
|
def test_valkey_failure(api, prepared, how):
|
|
alice, bob, chat, message = prepared
|
|
with service_down("valkey", how):
|
|
response, took = timed(api, "GET", "/api/v1/users/me", 503, token=alice["token"])
|
|
assert response.json()["error"]["code"] == "unavailable" and took < CACHE_BOUND, took
|
|
api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
|
|
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
|
|
api.call("GET", "/health/ready", 503, lb_error_ok=True)
|
|
processes_alive_but_not_ready()
|
|
check_intact(api, alice, bob, chat, message)
|
|
|
|
|
|
@pytest.mark.parametrize("how", ["stop", "pause"])
|
|
def test_postgres_failure(api, prepared, how):
|
|
alice, bob, chat, message = prepared
|
|
with service_down("postgres", how):
|
|
for path in ("/api/v1/users/me", f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"):
|
|
response, took = timed(api, "GET", path, 503, token=alice["token"])
|
|
assert response.json()["error"]["code"] == "unavailable", path
|
|
assert took < DB_BOUND, (path, took) # bounded even when PostgreSQL is frozen (pause)
|
|
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
|
|
api.call("GET", "/health/ready", 503, lb_error_ok=True)
|
|
processes_alive_but_not_ready()
|
|
check_intact(api, alice, bob, chat, message)
|
|
|
|
|
|
def test_locked_table_gives_bounded_503(api, prepared):
|
|
"""A statement stuck behind a lock is stopped by lock_timeout, not after minutes."""
|
|
alice, bob, chat, message = prepared
|
|
locker = subprocess.Popen(
|
|
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c",
|
|
"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(6); COMMIT;"],
|
|
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
try:
|
|
time.sleep(1.5) # let the lock be taken
|
|
response, took = timed(api, "GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
|
|
assert response.json()["error"]["code"] == "unavailable" and took < DB_BOUND, took
|
|
assert response.headers.get("x-instance-id"), "a lock is the API's own 503, the replicas stay ready"
|
|
finally:
|
|
locker.wait(timeout=30)
|
|
check_intact(api, alice, bob, chat, message)
|
|
|
|
|
|
def test_logout_during_valkey_outage_is_never_lost(api, prepared):
|
|
"""Valkey is the single source of truth for sessions, so a logout cannot be 'forgotten':
|
|
during the outage logout answers 503 (nothing revoked, the client knows it failed);
|
|
after recovery logout works, and the revocation survives a Valkey and API restart (AOF)."""
|
|
alice, bob, chat, message = prepared
|
|
with service_down("valkey", "stop"):
|
|
api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"], lb_error_ok=True)
|
|
api.call("GET", "/api/v1/users/me", 200, token=alice["token"]) # logout did not happen
|
|
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
|
|
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
|
|
restart_services("valkey", "api")
|
|
for _ in range(10): # repeated checks: the revoked token must never come back
|
|
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
|
|
api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], json_body={"text": "x"})
|
|
api.call("GET", "/api/v1/users/me", 200, token=bob["token"]) # other sessions survived the restart
|