forked from mtusi2026/mtusi2026lab2
Bring the system changes of lab 3 from 61c0c28..72a1e57 onto the lab 4 code, resolving each conflict by meaning and keeping everything lab 4 added: - a message is posted with one SQL statement under the chat row lock, and the attachment links go into the same statement; a foreign key violation on the attachment link still answers 404 and leaves nothing behind. Text and the attachment list are checked before the write, an outsider gets 404 for both, as before the carry over; - the request pipeline counts the body bytes actually received against the limit of the route, so a chunked body over 1 MiB gets 413 and the upload route keeps its MAX_UPLOAD_BYTES limit; - a request cancelled at shutdown gets 503 unavailable in the contract format, the worker waits a second less than the rest of the shutdown budget, SHUTDOWN_TIMEOUT_SECONDS becomes 6 s and the stop hooks 7 s; - readiness sits behind a Readiness port with the draining flag in shared infrastructure, and the drain window is logged through Logger; - logout revokes through SessionRevocations and withdraws the mark when PostgreSQL rejects the delete; an empty cursor is 400; - EventBus.project is gone, the sweep test subscribes and waits on a queue; domain tests check attachment_list instead of Chat.post; - make up waits for every replica in the balancer, BASE_URL follows the lb port, the image is tagged by the compose project, retry-check and faults-evidence are added, detection runs three times, failover-each and rolling-update record the stand usage, CI checks numeric uids. down also takes the tools and reconcile profiles of lab 4. Track 1 tooling of lab 3 (scaling.py, its samplers, tools/ and the CPU pinning of api, lb, valkey and k6) stays out; stand-usage.sh queries the database size itself instead of the track 1 sampler. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
273 lines
12 KiB
Python
273 lines
12 KiB
Python
import asyncio
|
||
import time
|
||
from collections.abc import AsyncIterator
|
||
from uuid import uuid4
|
||
|
||
import pytest
|
||
from httpx import AsyncClient
|
||
|
||
from stand import (
|
||
BASE_URL,
|
||
BASIC,
|
||
BEARER,
|
||
basic,
|
||
bearer,
|
||
compose,
|
||
enroll,
|
||
inside_replica,
|
||
lb_up,
|
||
new_user,
|
||
psql,
|
||
report,
|
||
restart_stack,
|
||
started_at,
|
||
timed,
|
||
valkey,
|
||
wait_ready,
|
||
waited,
|
||
)
|
||
|
||
pytestmark = pytest.mark.stack
|
||
RACERS = 50
|
||
|
||
|
||
@pytest.fixture
|
||
async def http() -> AsyncIterator[AsyncClient]:
|
||
async with AsyncClient(base_url=BASE_URL, timeout=10, trust_env=False) as client:
|
||
yield client
|
||
|
||
|
||
async def test_registration_race_yields_exactly_one_user(http: AsyncClient) -> None:
|
||
body = new_user("race")
|
||
responses = await asyncio.gather(
|
||
*(http.post("/api/v1/users", json=body) for _ in range(RACERS))
|
||
)
|
||
codes = sorted(response.status_code for response in responses)
|
||
print(f"гонка {RACERS} регистраций: 201×{codes.count(201)}, 409×{codes.count(409)}")
|
||
assert codes.count(201) == 1
|
||
assert codes.count(409) == RACERS - 1
|
||
assert psql("SELECT count(*) FROM users WHERE username = :'name'", name=body["username"]) == "1"
|
||
|
||
|
||
async def test_chat_creation_rolls_back_on_unknown_member(http: AsyncClient) -> None:
|
||
alice, token = await enroll(http, "alice")
|
||
title = f"Откат {uuid4().hex}"
|
||
response = await http.post(
|
||
"/api/v1/chats",
|
||
json={"title": title, "member_ids": [alice["id"], str(uuid4())]},
|
||
headers=bearer(token),
|
||
)
|
||
code = response.json()["error"]["code"]
|
||
print(f"чат с несуществующим участником: {response.status_code} {code}")
|
||
assert response.status_code == 400
|
||
assert response.json()["error"]["code"] == "invalid_request"
|
||
assert psql("SELECT count(*) FROM chats WHERE title = :'title'", title=title) == "0"
|
||
members = "SELECT count(*) FROM chat_members WHERE user_id = :'id'::uuid"
|
||
assert psql(members, id=alice["id"]) == "0"
|
||
|
||
|
||
async def test_wrong_password_and_unknown_user_get_same_basic_challenge(http: AsyncClient) -> None:
|
||
alice, _ = await enroll(http, "alice")
|
||
wrong, wrong_time = await timed(
|
||
http.post("/api/v1/auth/sessions", headers=basic(alice["username"], "x" * 20))
|
||
)
|
||
unknown, unknown_time = await timed(
|
||
http.post("/api/v1/auth/sessions", headers=basic(f"u_ghost_{uuid4().hex[:8]}"))
|
||
)
|
||
report("неверный пароль", wrong, wrong_time)
|
||
report("неизвестный пользователь", unknown, unknown_time)
|
||
for response in (wrong, unknown):
|
||
assert response.status_code == 401
|
||
assert response.headers["WWW-Authenticate"] == BASIC
|
||
assert wrong.json()["error"]["message"] == unknown.json()["error"]["message"]
|
||
|
||
|
||
async def test_logout_then_token_is_dead(http: AsyncClient) -> None:
|
||
_, token = await enroll(http, "alice")
|
||
assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 200
|
||
logout = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
|
||
after = await http.get("/api/v1/users/me", headers=bearer(token))
|
||
again = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
|
||
print(f"logout: {logout.status_code}, затем {after.status_code}, повтор {again.status_code}")
|
||
assert (logout.status_code, logout.content) == (204, b"")
|
||
assert after.status_code == again.status_code == 401
|
||
assert again.headers["WWW-Authenticate"] == BEARER
|
||
|
||
|
||
async def test_valkey_serves_repeated_checks(http: AsyncClient) -> None:
|
||
_, token = await enroll(http, "cache")
|
||
await http.get("/api/v1/users/me", headers=bearer(token))
|
||
before = int(valkey("info stats").split("keyspace_hits:")[1].split()[0])
|
||
for _ in range(5):
|
||
assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 200
|
||
after = int(valkey("info stats").split("keyspace_hits:")[1].split()[0])
|
||
print(f"5 проверок токена: keyspace_hits {before} → {after}")
|
||
assert after - before >= 5
|
||
|
||
|
||
async def test_valkey_stopped_is_bypassed_but_logout_waits(http: AsyncClient) -> None:
|
||
user, token = await enroll(http, "nocache")
|
||
try:
|
||
compose("stop", "valkey")
|
||
me, me_time = await timed(http.get("/api/v1/users/me", headers=bearer(token)))
|
||
stranger, stranger_time = await timed(
|
||
http.get("/api/v1/users/me", headers=bearer("x" * 43))
|
||
)
|
||
ready = await http.get("/health/ready")
|
||
login = await http.post("/api/v1/auth/sessions", headers=basic(user["username"]))
|
||
logout = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
|
||
report("valkey остановлен, действующий токен", me, me_time)
|
||
report("valkey остановлен, неизвестный токен", stranger, stranger_time)
|
||
codes = (ready.status_code, login.status_code, logout.status_code)
|
||
print("valkey остановлен: ready {}, вход {}, выход {}".format(*codes))
|
||
assert me.json() == user and me_time < 1
|
||
assert stranger.status_code == 401 and stranger.headers["WWW-Authenticate"] == BEARER
|
||
assert ready.status_code == 200
|
||
assert login.status_code == 201
|
||
assert logout.status_code == 503 and logout.json()["error"]["code"] == "unavailable"
|
||
finally:
|
||
compose("start", "valkey")
|
||
compose("up", "-d", "--wait", "valkey")
|
||
await asyncio.sleep(1.2)
|
||
retried = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
|
||
print(f"valkey снова запущен: повторный выход {retried.status_code}")
|
||
assert retried.status_code == 204
|
||
assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 401
|
||
|
||
|
||
async def test_valkey_restart_keeps_cache_and_revocations(http: AsyncClient) -> None:
|
||
_, alive = await enroll(http, "alive")
|
||
_, revoked = await enroll(http, "revoked")
|
||
assert (await http.get("/api/v1/users/me", headers=bearer(alive))).status_code == 200
|
||
assert (await http.get("/api/v1/users/me", headers=bearer(revoked))).status_code == 200
|
||
assert (
|
||
await http.delete("/api/v1/auth/sessions/current", headers=bearer(revoked))
|
||
).status_code == 204
|
||
keys_before = valkey("dbsize")
|
||
compose("restart", "valkey")
|
||
compose("up", "-d", "--wait", "valkey")
|
||
keys_after = valkey("dbsize")
|
||
await asyncio.sleep(1.2)
|
||
still = await http.get("/api/v1/users/me", headers=bearer(alive))
|
||
dead = await http.get("/api/v1/users/me", headers=bearer(revoked))
|
||
print(f"рестарт valkey: ключей {keys_before} → {keys_after}")
|
||
print(f"рестарт valkey: живой {still.status_code}, отозванный {dead.status_code}")
|
||
assert keys_after == keys_before
|
||
assert still.status_code == 200
|
||
assert dead.status_code == 401
|
||
|
||
|
||
async def test_valkey_flushall_loses_nothing(http: AsyncClient) -> None:
|
||
_, alive = await enroll(http, "alive")
|
||
_, revoked = await enroll(http, "revoked")
|
||
assert (
|
||
await http.delete("/api/v1/auth/sessions/current", headers=bearer(revoked))
|
||
).status_code == 204
|
||
valkey("flushall")
|
||
still = await http.get("/api/v1/users/me", headers=bearer(alive))
|
||
dead = await http.get("/api/v1/users/me", headers=bearer(revoked))
|
||
print(f"FLUSHALL: живой {still.status_code}, отозванный {dead.status_code}")
|
||
assert still.status_code == 200
|
||
assert dead.status_code == 401
|
||
|
||
|
||
async def test_postgres_stopped_gives_bounded_503_and_recovers(http: AsyncClient) -> None:
|
||
user, token = await enroll(http, "nodb")
|
||
assert (await http.get("/api/v1/users/me", headers=bearer(token))).status_code == 200
|
||
api_started = started_at("api")
|
||
try:
|
||
compose("stop", "postgres")
|
||
checks = {
|
||
"ready": await timed(http.get("/health/ready")),
|
||
"me": await timed(http.get("/api/v1/users/me", headers=bearer(token))),
|
||
"stranger": await timed(http.get("/api/v1/users/me", headers=bearer("x" * 43))),
|
||
"login": await timed(
|
||
http.post("/api/v1/auth/sessions", headers=basic(user["username"]))
|
||
),
|
||
"register": await timed(http.post("/api/v1/users", json=new_user("nodb"))),
|
||
}
|
||
live_inside = inside_replica("/health/live")
|
||
excluded = await waited(lambda: lb_up() == 0, 5)
|
||
through_lb = await http.get("/health/live")
|
||
for name, (response, elapsed) in checks.items():
|
||
report(f"postgres остановлен, {name}", response, elapsed)
|
||
assert elapsed < 2
|
||
assert response.status_code == 503
|
||
assert response.json()["error"]["code"] == "unavailable"
|
||
print(
|
||
f"live внутри реплики {live_inside}; LB исключил все реплики ещё через {excluded:.2f} с"
|
||
)
|
||
report("live через LB без реплик", through_lb, 0)
|
||
assert live_inside == 200
|
||
assert lb_up() == 0
|
||
assert excluded < 2
|
||
assert through_lb.status_code == 503
|
||
assert through_lb.json()["error"]["request_id"] == through_lb.headers["X-Request-Id"]
|
||
assert "X-Instance-Id" not in through_lb.headers
|
||
finally:
|
||
compose("start", "postgres")
|
||
recovered = await wait_ready(http)
|
||
await waited(lambda: lb_up() == len(compose("ps", "-q", "api").split()))
|
||
me = await http.get("/api/v1/users/me", headers=bearer(token))
|
||
print(f"postgres запущен: готовность через {recovered:.1f} с, прежний токен {me.status_code}")
|
||
assert me.json() == user
|
||
assert started_at("api") == api_started
|
||
|
||
|
||
async def test_api_restart_keeps_session_without_extending_it(http: AsyncClient) -> None:
|
||
body = new_user("restart")
|
||
assert (await http.post("/api/v1/users", json=body)).status_code == 201
|
||
session = (await http.post("/api/v1/auth/sessions", headers=basic(body["username"]))).json()
|
||
compose("restart", "api")
|
||
recovered = await wait_ready(http)
|
||
me = await http.get("/api/v1/users/me", headers=bearer(session["token"]))
|
||
stored = psql(
|
||
"SELECT to_char(expires_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.US\"Z\"') "
|
||
"FROM sessions s JOIN users u ON u.id = s.user_id WHERE u.username = :'name'",
|
||
name=body["username"],
|
||
)
|
||
print(f"рестарт api: готовность через {recovered:.1f} с, токен {me.status_code}")
|
||
print(f"expires_at при входе {session['expires_at']}, в БД после рестарта {stored}")
|
||
assert me.status_code == 200
|
||
assert stored.rstrip("Z").rstrip("0") == session["expires_at"].rstrip("Z").rstrip("0")
|
||
|
||
|
||
async def test_full_restart_keeps_session_until_logout(http: AsyncClient) -> None:
|
||
user, token = await enroll(http, "fullrestart")
|
||
restart_stack()
|
||
recovered = await wait_ready(http)
|
||
me = await http.get("/api/v1/users/me", headers=bearer(token))
|
||
logout = await http.delete("/api/v1/auth/sessions/current", headers=bearer(token))
|
||
after = await http.get("/api/v1/users/me", headers=bearer(token))
|
||
print(
|
||
f"restart.sh: готовность через {recovered:.1f} с, прежний токен {me.status_code}, "
|
||
f"выход {logout.status_code}, затем {after.status_code}"
|
||
)
|
||
assert me.json() == user
|
||
assert logout.status_code == 204
|
||
assert after.status_code == 401
|
||
|
||
|
||
async def test_session_expires_after_restart_at_original_time(http: AsyncClient) -> None:
|
||
user, token = await enroll(http, "shortrestart")
|
||
expires_at = float(
|
||
psql(
|
||
"UPDATE sessions s SET expires_at = now() + interval '20 seconds' FROM users u "
|
||
"WHERE u.id = s.user_id AND u.username = :'name' "
|
||
"RETURNING extract(epoch FROM s.expires_at)",
|
||
name=user["username"],
|
||
).splitlines()[0]
|
||
)
|
||
compose("restart", "api")
|
||
recovered = await wait_ready(http)
|
||
fresh = await http.get("/api/v1/users/me", headers=bearer(token))
|
||
await asyncio.sleep(max(0.0, expires_at - time.time()) + 0.5)
|
||
stale = await http.get("/api/v1/users/me", headers=bearer(token))
|
||
print(
|
||
f"срок сессии сокращён до 20 с, рестарт api: готовность через {recovered:.1f} с, "
|
||
f"до срока {fresh.status_code}, после срока {stale.status_code}"
|
||
)
|
||
assert fresh.status_code == 200
|
||
assert stale.status_code == 401
|
||
assert stale.headers["WWW-Authenticate"] == BEARER
|