Files
mtusi2026lab5/student-tests/stack/test_faults.py
T

280 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import asyncio
import time
from collections.abc import AsyncIterator
from uuid import uuid4
import pytest
from httpx import AsyncClient
from stand import (
BASE_URL,
BASIC,
BEARER,
basic,
bearer,
compose,
enroll,
inside_replica,
lb_up,
new_user,
psql,
report,
restart_stack,
started_at,
timed,
valkey,
wait_ready,
waited,
)
pytestmark = pytest.mark.stack
RACERS = 50
@pytest.fixture
async def http() -> AsyncIterator[AsyncClient]:
async with AsyncClient(base_url=BASE_URL, timeout=10, trust_env=False) as client:
yield client
async def test_registration_race_yields_exactly_one_user(http: AsyncClient) -> None:
body = new_user("race")
responses = await asyncio.gather(
*(http.post("/api/v1/users", json=body) for _ in range(RACERS))
)
codes = sorted(response.status_code for response in responses)
print(f"гонка {RACERS} регистраций: 201×{codes.count(201)}, 409×{codes.count(409)}")
assert codes.count(201) == 1
assert codes.count(409) == RACERS - 1
assert psql("SELECT count(*) FROM users WHERE username = :'name'", name=body["username"]) == "1"
async def test_chat_creation_rolls_back_on_unknown_member(http: AsyncClient) -> None:
alice, token = await enroll(http, "alice")
title = f"Откат {uuid4().hex}"
response = await http.post(
"/api/v1/chats",
json={"title": title, "member_ids": [alice["id"], str(uuid4())]},
headers=bearer(token),
)
code = response.json()["error"]["code"]
print(f"чат с несуществующим участником: {response.status_code} {code}")
assert response.status_code == 400
assert response.json()["error"]["code"] == "invalid_request"
assert psql("SELECT count(*) FROM chats WHERE title = :'title'", title=title) == "0"
members = "SELECT count(*) FROM chat_members WHERE user_id = :'id'::uuid"
assert psql(members, id=alice["id"]) == "0"
async def test_wrong_password_and_unknown_user_get_same_basic_challenge(http: AsyncClient) -> None:
alice, _ = await enroll(http, "alice")
wrong, wrong_time = await timed(
http.post("/api/v1/auth/sessions", headers=basic(alice["username"], "x" * 20))
)
unknown, unknown_time = await timed(
http.post("/api/v1/auth/sessions", headers=basic(f"u_ghost_{uuid4().hex[:8]}"))
)
report("неверный пароль", wrong, wrong_time)
report("неизвестный пользователь", unknown, unknown_time)
for response in (wrong, unknown):
assert response.status_code == 401
assert response.headers["WWW-Authenticate"] == BASIC
assert wrong.json()["error"]["message"] == unknown.json()["error"]["message"]
async def test_logout_then_token_is_dead(http: AsyncClient) -> None:
_, token = await enroll(http, "alice")
assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 200
logout = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
after = await http.get("/api/v1/users/me", headers=bearer(token))
again = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
print(f"logout: {logout.status_code}, затем {after.status_code}, повтор {again.status_code}")
assert (logout.status_code, logout.content) == (204, b"")
assert after.status_code == again.status_code == 401
assert again.headers["WWW-Authenticate"] == BEARER
async def test_valkey_serves_repeated_checks(http: AsyncClient) -> None:
_, token = await enroll(http, "cache")
await http.get("/api/v1/users/me", headers=bearer(token))
before = int(valkey("info stats").split("keyspace_hits:")[1].split()[0])
for _ in range(5):
assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 200
after = int(valkey("info stats").split("keyspace_hits:")[1].split()[0])
print(f"5 проверок токена: keyspace_hits {before} → {after}")
assert after - before >= 5
async def test_valkey_stopped_is_bypassed_but_logout_waits(http: AsyncClient) -> None:
user, token = await enroll(http, "nocache")
try:
compose("stop", "valkey")
me, me_time = await timed(http.get("/api/v1/users/me", headers=bearer(token)))
stranger, stranger_time = await timed(
http.get("/api/v1/users/me", headers=bearer("x" * 43))
)
ready = await http.get("/health/ready")
login = await http.post("/api/v1/auth/sessions", headers=basic(user["username"]))
logout = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
report("valkey остановлен, действующий токен", me, me_time)
report("valkey остановлен, неизвестный токен", stranger, stranger_time)
codes = (ready.status_code, login.status_code, logout.status_code)
print("valkey остановлен: ready {}, вход {}, выход {}".format(*codes))
assert me.json() == user and me_time < 1
assert stranger.status_code == 401 and stranger.headers["WWW-Authenticate"] == BEARER
assert ready.status_code == 200
assert login.status_code == 201
assert logout.status_code == 503 and logout.json()["error"]["code"] == "unavailable"
finally:
compose("start", "valkey")
compose("up", "-d", "--wait", "valkey")
await asyncio.sleep(1.2)
retried = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
print(f"valkey снова запущен: повторный выход {retried.status_code}")
assert retried.status_code == 204
assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 401
async def test_valkey_restart_keeps_cache_and_revocations(http: AsyncClient) -> None:
_, alive = await enroll(http, "alive")
_, revoked = await enroll(http, "revoked")
assert (await http.get("/api/v1/users/me", headers=bearer(alive))).status_code == 200
assert (await http.get("/api/v1/users/me", headers=bearer(revoked))).status_code == 200
assert (
await http.delete("/api/v1/auth/sessions/current", headers=bearer(revoked))
).status_code == 204
keys_before = valkey("dbsize")
compose("restart", "valkey")
compose("up", "-d", "--wait", "valkey")
keys_after = valkey("dbsize")
await asyncio.sleep(1.2)
still = await http.get("/api/v1/users/me", headers=bearer(alive))
dead = await http.get("/api/v1/users/me", headers=bearer(revoked))
print(f"рестарт valkey: ключей {keys_before} → {keys_after}")
print(f"рестарт valkey: живой {still.status_code}, отозванный {dead.status_code}")
assert keys_after == keys_before
assert still.status_code == 200
assert dead.status_code == 401
async def test_valkey_flushall_loses_nothing(http: AsyncClient) -> None:
_, alive = await enroll(http, "alive")
_, revoked = await enroll(http, "revoked")
assert (
await http.delete("/api/v1/auth/sessions/current", headers=bearer(revoked))
).status_code == 204
valkey("flushall")
still = await http.get("/api/v1/users/me", headers=bearer(alive))
dead = await http.get("/api/v1/users/me", headers=bearer(revoked))
print(f"FLUSHALL: живой {still.status_code}, отозванный {dead.status_code}")
assert still.status_code == 200
assert dead.status_code == 401
async def test_postgres_stopped_gives_bounded_503_and_recovers(http: AsyncClient) -> None:
user, token = await enroll(http, "nodb")
assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 200
api_started = started_at("api")
try:
compose("stop", "postgres")
checks = {
"ready": await timed(http.get("/health/ready")),
"me": await timed(http.get("/api/v1/users/me", headers=bearer(token))),
"stranger": await timed(http.get("/api/v1/users/me", headers=bearer("x" * 43))),
"login": await timed(
http.post("/api/v1/auth/sessions", headers=basic(user["username"]))
),
"register": await timed(http.post("/api/v1/users", json=new_user("nodb"))),
}
live_inside = inside_replica("/health/live")
excluded = await waited(lambda: lb_up() == 0, 5)
through_lb = await http.get("/health/live")
for name, (response, elapsed) in checks.items():
report(f"postgres остановлен, {name}", response, elapsed)
assert elapsed < 2
assert response.status_code == 503
assert response.json()["error"]["code"] == "unavailable"
print(
f"live внутри реплики {live_inside}; LB исключил все реплики ещё через {excluded:.2f} с"
)
report("live через LB без реплик", through_lb, 0)
assert live_inside == 200
assert lb_up() == 0
assert excluded < 2
assert through_lb.status_code == 503
assert through_lb.json()["error"]["request_id"] == through_lb.headers["X-Request-Id"]
assert "X-Instance-Id" not in through_lb.headers
finally:
compose("start", "postgres")
recovered = await wait_ready(http)
await waited(lambda: lb_up() == len(compose("ps", "-q", "api").split()))
me = await http.get("/api/v1/users/me", headers=bearer(token))
print(f"postgres запущен: готовность через {recovered:.1f} с, прежний токен {me.status_code}")
assert me.json() == user
instances = set()
for _ in range(40):
response = await http.post("/api/v1/users", json=new_user("recovered"))
assert response.status_code == 201
instances.add(response.headers["X-Instance-Id"])
print(f"postgres запущен: 40 регистраций 201 через {len(instances)} реплики без рестарта API")
assert len(instances) >= 2
assert started_at("api") == api_started
async def test_api_restart_keeps_session_without_extending_it(http: AsyncClient) -> None:
body = new_user("restart")
assert (await http.post("/api/v1/users", json=body)).status_code == 201
session = (await http.post("/api/v1/auth/sessions", headers=basic(body["username"]))).json()
compose("restart", "api")
recovered = await wait_ready(http)
me = await http.get("/api/v1/users/me", headers=bearer(session["token"]))
stored = psql(
"SELECT to_char(expires_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') "
"FROM sessions s JOIN users u ON u.id = s.user_id WHERE u.username = :'name'",
name=body["username"],
)
print(f"рестарт api: готовность через {recovered:.1f} с, токен {me.status_code}")
print(f"expires_at при входе {session['expires_at']}, в БД после рестарта {stored}")
assert me.status_code == 200
assert stored.rstrip("Z").rstrip("0") == session["expires_at"].rstrip("Z").rstrip("0")
async def test_full_restart_keeps_session_until_logout(http: AsyncClient) -> None:
user, token = await enroll(http, "fullrestart")
restart_stack()
recovered = await wait_ready(http)
me = await http.get("/api/v1/users/me", headers=bearer(token))
logout = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
after = await http.get("/api/v1/users/me", headers=bearer(token))
print(
f"restart.sh: готовность через {recovered:.1f} с, прежний токен {me.status_code}, "
f"выход {logout.status_code}, затем {after.status_code}"
)
assert me.json() == user
assert logout.status_code == 204
assert after.status_code == 401
async def test_session_expires_after_restart_at_original_time(http: AsyncClient) -> None:
user, token = await enroll(http, "shortrestart")
expires_at = float(
psql(
"UPDATE sessions s SET expires_at = now() + interval '20 seconds' FROM users u "
"WHERE u.id = s.user_id AND u.username = :'name' "
"RETURNING extract(epoch FROM s.expires_at)",
name=user["username"],
).splitlines()[0]
)
compose("restart", "api")
recovered = await wait_ready(http)
fresh = await http.get("/api/v1/users/me", headers=bearer(token))
await asyncio.sleep(max(0.0, expires_at - time.time()) + 0.5)
stale = await http.get("/api/v1/users/me", headers=bearer(token))
print(
f"срок сессии сокращён до 20 с, рестарт api: готовность через {recovered:.1f} с, "
f"до срока {fresh.status_code}, после срока {stale.status_code}"
)
assert fresh.status_code == 200
assert stale.status_code == 401
assert stale.headers["WWW-Authenticate"] == BEARER