Files
mtusi2026lab2/student-tests/stack/test_faults.py
T
ArtyomandClaude Opus 5.5 dc2015473f Run the API as replicas behind HAProxy
compose.yaml now runs one api service with API_REPLICAS replicas and
no published ports, container name or hostname, behind lb, HAProxy
3.2.26 LTS pinned by digest and the only service with ports, bound to
127.0.0.1 (LB_HTTP_PORT for HTTP, LB_STATS_PORT for read-only stats).
The balancer finds replicas through Docker DNS with a server template,
checks GET /health/ready every second (fall 2, rise 2), balances round
robin without stickiness, retries connection failures on another
replica and allows a retry after a sent request only for GET and for
message posts carrying Idempotency-Key, never for users, chats or
login. Its own 502/503/504 answers use the contract error body with
the forwarded X-Request-Id.

The network is split into edge and an internal backend, every service
gets CPU and memory limits, the API runs read-only with a tmpfs /tmp,
two workers per replica, an 8 s request limit and a 5 s graceful stop
inside an 8 s stop_grace_period. Only api builds the image, migrate
reuses it once before the replicas start. PostgreSQL loads
pg_stat_statements and allows 200 connections. The short TTL instance
of lab 2 published the API around the balancer and is removed; its
expiry check stays as an in-process test.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-28 11:43:43 +03:00

216 lines
9.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import asyncio
from collections.abc import AsyncIterator
from uuid import uuid4
import pytest
from httpx import AsyncClient
from stand import (
BASE_URL,
BASIC,
BEARER,
basic,
bearer,
compose,
enroll,
new_user,
psql,
report,
started_at,
timed,
valkey,
wait_ready,
)
pytestmark = pytest.mark.stack
RACERS = 50
@pytest.fixture
async def http() -> AsyncIterator[AsyncClient]:
async with AsyncClient(base_url=BASE_URL, timeout=10, trust_env=False) as client:
yield client
async def test_registration_race_yields_exactly_one_user(http: AsyncClient) -> None:
body = new_user("race")
responses = await asyncio.gather(
*(http.post("/api/v1/users", json=body) for _ in range(RACERS))
)
codes = sorted(response.status_code for response in responses)
print(f"гонка {RACERS} регистраций: 201×{codes.count(201)}, 409×{codes.count(409)}")
assert codes.count(201) == 1
assert codes.count(409) == RACERS - 1
assert psql("SELECT count(*) FROM users WHERE username = :'name'", name=body["username"]) == "1"
async def test_chat_creation_rolls_back_on_unknown_member(http: AsyncClient) -> None:
alice, token = await enroll(http, "alice")
title = f"Откат {uuid4().hex}"
response = await http.post(
"/api/v1/chats",
json={"title": title, "member_ids": [alice["id"], str(uuid4())]},
headers=bearer(token),
)
print(f"чат с несуществующим участником: {response.status_code}")
assert response.status_code == 400
assert response.json()["error"]["code"] == "invalid_request"
assert psql("SELECT count(*) FROM chats WHERE title = :'title'", title=title) == "0"
members = "SELECT count(*) FROM chat_members WHERE user_id = :'id'::uuid"
assert psql(members, id=alice["id"]) == "0"
async def test_wrong_password_and_unknown_user_get_same_basic_challenge(http: AsyncClient) -> None:
alice, _ = await enroll(http, "alice")
wrong, wrong_time = await timed(
http.post("/api/v1/auth/sessions", headers=basic(alice["username"], "x" * 20))
)
unknown, unknown_time = await timed(
http.post("/api/v1/auth/sessions", headers=basic(f"u_ghost_{uuid4().hex[:8]}"))
)
report("неверный пароль", wrong, wrong_time)
report("неизвестный пользователь", unknown, unknown_time)
for response in (wrong, unknown):
assert response.status_code == 401
assert response.headers["WWW-Authenticate"] == BASIC
assert wrong.json()["error"]["message"] == unknown.json()["error"]["message"]
async def test_logout_then_token_is_dead(http: AsyncClient) -> None:
_, token = await enroll(http, "alice")
assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 200
logout = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
after = await http.get("/api/v1/users/me", headers=bearer(token))
again = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
print(f"logout: {logout.status_code}, затем {after.status_code}, повтор {again.status_code}")
assert (logout.status_code, logout.content) == (204, b"")
assert after.status_code == again.status_code == 401
assert again.headers["WWW-Authenticate"] == BEARER
async def test_valkey_serves_repeated_checks(http: AsyncClient) -> None:
_, token = await enroll(http, "cache")
await http.get("/api/v1/users/me", headers=bearer(token))
before = int(valkey("info stats").split("keyspace_hits:")[1].split()[0])
for _ in range(5):
assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 200
after = int(valkey("info stats").split("keyspace_hits:")[1].split()[0])
print(f"5 проверок токена: keyspace_hits {before} → {after}")
assert after - before >= 5
async def test_valkey_stopped_is_bypassed_but_logout_waits(http: AsyncClient) -> None:
user, token = await enroll(http, "nocache")
try:
compose("stop", "valkey")
me, me_time = await timed(http.get("/api/v1/users/me", headers=bearer(token)))
stranger, stranger_time = await timed(
http.get("/api/v1/users/me", headers=bearer("x" * 43))
)
ready = await http.get("/health/ready")
login = await http.post("/api/v1/auth/sessions", headers=basic(user["username"]))
logout = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
report("valkey остановлен, действующий токен", me, me_time)
report("valkey остановлен, неизвестный токен", stranger, stranger_time)
codes = (ready.status_code, login.status_code, logout.status_code)
print("valkey остановлен: ready {}, вход {}, выход {}".format(*codes))
assert me.json() == user and me_time < 1
assert stranger.status_code == 401 and stranger.headers["WWW-Authenticate"] == BEARER
assert ready.status_code == 200
assert login.status_code == 201
assert logout.status_code == 503 and logout.json()["error"]["code"] == "unavailable"
finally:
compose("start", "valkey")
compose("up", "-d", "--wait", "valkey")
await asyncio.sleep(1.2)
retried = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
print(f"valkey снова запущен: повторный выход {retried.status_code}")
assert retried.status_code == 204
assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 401
async def test_valkey_restart_keeps_cache_and_revocations(http: AsyncClient) -> None:
_, alive = await enroll(http, "alive")
_, revoked = await enroll(http, "revoked")
assert (await http.get("/api/v1/users/me", headers=bearer(alive))).status_code == 200
assert (await http.get("/api/v1/users/me", headers=bearer(revoked))).status_code == 200
assert (
await http.delete("/api/v1/auth/sessions/current", headers=bearer(revoked))
).status_code == 204
keys_before = valkey("dbsize")
compose("restart", "valkey")
compose("up", "-d", "--wait", "valkey")
keys_after = valkey("dbsize")
await asyncio.sleep(1.2)
still = await http.get("/api/v1/users/me", headers=bearer(alive))
dead = await http.get("/api/v1/users/me", headers=bearer(revoked))
print(f"рестарт valkey: ключей {keys_before} → {keys_after}")
print(f"рестарт valkey: живой {still.status_code}, отозванный {dead.status_code}")
assert keys_after == keys_before
assert still.status_code == 200
assert dead.status_code == 401
async def test_valkey_flushall_loses_nothing(http: AsyncClient) -> None:
_, alive = await enroll(http, "alive")
_, revoked = await enroll(http, "revoked")
assert (
await http.delete("/api/v1/auth/sessions/current", headers=bearer(revoked))
).status_code == 204
valkey("flushall")
still = await http.get("/api/v1/users/me", headers=bearer(alive))
dead = await http.get("/api/v1/users/me", headers=bearer(revoked))
print(f"FLUSHALL: живой {still.status_code}, отозванный {dead.status_code}")
assert still.status_code == 200
assert dead.status_code == 401
async def test_postgres_stopped_gives_bounded_503_and_recovers(http: AsyncClient) -> None:
user, token = await enroll(http, "nodb")
assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 200
api_started = started_at("api")
try:
compose("stop", "postgres")
checks = {
"ready": await timed(http.get("/health/ready")),
"live": await timed(http.get("/health/live")),
"me": await timed(http.get("/api/v1/users/me", headers=bearer(token))),
"stranger": await timed(http.get("/api/v1/users/me", headers=bearer("x" * 43))),
"login": await timed(
http.post("/api/v1/auth/sessions", headers=basic(user["username"]))
),
"register": await timed(http.post("/api/v1/users", json=new_user("nodb"))),
}
for name, (response, elapsed) in checks.items():
report(f"postgres остановлен, {name}", response, elapsed)
assert elapsed < 2
assert checks["live"][0].status_code == 200
for name in ("ready", "me", "stranger", "login", "register"):
assert checks[name][0].status_code == 503
assert checks[name][0].json()["error"]["code"] == "unavailable"
finally:
compose("start", "postgres")
recovered = await wait_ready(http)
me = await http.get("/api/v1/users/me", headers=bearer(token))
print(f"postgres запущен: готовность через {recovered:.1f} с, прежний токен {me.status_code}")
assert me.json() == user
assert started_at("api") == api_started
async def test_api_restart_keeps_session_without_extending_it(http: AsyncClient) -> None:
body = new_user("restart")
assert (await http.post("/api/v1/users", json=body)).status_code == 201
session = (await http.post("/api/v1/auth/sessions", headers=basic(body["username"]))).json()
compose("restart", "api")
recovered = await wait_ready(http)
me = await http.get("/api/v1/users/me", headers=bearer(session["token"]))
stored = psql(
"SELECT to_char(expires_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') "
"FROM sessions s JOIN users u ON u.id = s.user_id WHERE u.username = :'name'",
name=body["username"],
)
print(f"рестарт api: готовность через {recovered:.1f} с, токен {me.status_code}")
print(f"expires_at при входе {session['expires_at']}, в БД после рестарта {stored}")
assert me.status_code == 200
assert stored.rstrip("Z").rstrip("0") == session["expires_at"].rstrip("Z").rstrip("0")