forked from mtusi2026/mtusi2026lab2
Флаг очистки кэша жил только в памяти процесса. Сценарий: logout при недоступном Valkey (отзыв записан в PostgreSQL, tombstone нет) -> рестарт API -> Valkey вернулся. Новый процесс читал устаревшую положительную запись, и отозванный токен получал 200 на /users/me и 201 на отправку сообщения до истечения TTL записи (до 60 s). Теперь процесс стартует с включённым флагом и выполняет FLUSHDB перед первым использованием кэша. Добавлен регрессионный тест test_logout_during_outage_then_api_restart, evidence и отчёт обновлены. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
379 lines
14 KiB
Python
379 lines
14 KiB
Python
"""Automatic lab2 level-4 checks against the running compose stand.
|
|
|
|
docker compose --profile tests up -d --build api_short_ttl
|
|
python student-tests/level4.py
|
|
|
|
Covers: wrong password, Basic/X-User-Id/garbage tokens, logout, session
|
|
expiry with a short SESSION_TTL_SECONDS (api_short_ttl on :8081), the Valkey
|
|
session cache role, registration race on one username, Valkey stop and hang
|
|
(pause), PostgreSQL stop. The script stops/pauses containers itself and
|
|
always starts them again; volumes are never removed. Tokens, passwords and
|
|
secrets are never printed.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PASSWORD = "correct-horse-battery-staple"
|
|
failures: list[str] = []
|
|
|
|
|
|
def check(condition: bool, message: str) -> None:
|
|
print(("PASS: " if condition else "FAIL: ") + message, flush=True)
|
|
if not condition:
|
|
failures.append(message)
|
|
|
|
|
|
class Api:
|
|
def __init__(self, base_url: str) -> None:
|
|
self.base_url = base_url.rstrip("/")
|
|
|
|
def call(
|
|
self,
|
|
method: str,
|
|
path: str,
|
|
*,
|
|
body: dict | None = None,
|
|
headers: dict[str, str] | None = None,
|
|
timeout: float = 15,
|
|
) -> tuple[int, dict, dict[str, str], float]:
|
|
request_headers = {"Accept": "application/json"}
|
|
data = None
|
|
if body is not None:
|
|
data = json.dumps(body).encode("utf-8")
|
|
request_headers["Content-Type"] = "application/json"
|
|
request_headers.update(headers or {})
|
|
request = urllib.request.Request(
|
|
self.base_url + path, data=data, headers=request_headers, method=method
|
|
)
|
|
started = time.monotonic()
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
raw = response.read().decode("utf-8")
|
|
status, response_headers = response.status, response.headers
|
|
except urllib.error.HTTPError as exc:
|
|
raw = exc.read().decode("utf-8")
|
|
status, response_headers = exc.code, exc.headers
|
|
elapsed = time.monotonic() - started
|
|
return status, json.loads(raw) if raw else {}, response_headers, elapsed
|
|
|
|
def register(self, username: str | None = None) -> dict:
|
|
username = username or f"l4_{uuid.uuid4().hex[:20]}"
|
|
status, body, _, _ = self.call(
|
|
"POST", "/api/v1/users",
|
|
body={"username": username, "display_name": "Level 4",
|
|
"password": PASSWORD},
|
|
)
|
|
assert status == 201, f"register returned {status}"
|
|
return body
|
|
|
|
def login(self, username: str, password: str = PASSWORD):
|
|
raw = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
return self.call(
|
|
"POST", "/api/v1/auth/sessions",
|
|
headers={"Authorization": f"Basic {raw}"},
|
|
)
|
|
|
|
def token(self, username: str) -> str:
|
|
status, body, _, _ = self.login(username)
|
|
assert status == 201, f"login returned {status}"
|
|
return body["token"]
|
|
|
|
def me(self, token: str, timeout: float = 15):
|
|
return self.call(
|
|
"GET", "/api/v1/users/me",
|
|
headers={"Authorization": f"Bearer {token}"}, timeout=timeout,
|
|
)
|
|
|
|
def create_chat(self, token: str, user_id: str):
|
|
return self.call(
|
|
"POST", "/api/v1/chats",
|
|
body={"title": "Level 4", "member_ids": [user_id]},
|
|
headers={"Authorization": f"Bearer {token}"},
|
|
)
|
|
|
|
def logout(self, token: str):
|
|
return self.call(
|
|
"DELETE", "/api/v1/auth/sessions/current",
|
|
headers={"Authorization": f"Bearer {token}"},
|
|
)
|
|
|
|
def wait_ready(self, seconds: float = 90) -> bool:
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if self.call("GET", "/health/ready", timeout=3)[0] == 200:
|
|
return True
|
|
except OSError:
|
|
pass
|
|
time.sleep(1)
|
|
return False
|
|
|
|
|
|
def compose(*args: str) -> str:
|
|
result = subprocess.run(
|
|
["docker", "compose", *args],
|
|
cwd=ROOT, capture_output=True, text=True, check=True,
|
|
)
|
|
return result.stdout
|
|
|
|
|
|
def valkey_has_session(token: str) -> bool:
|
|
"""The cache key is sha256(token); the raw token is never stored."""
|
|
key = "messenger:session:" + hashlib.sha256(token.encode()).hexdigest()
|
|
output = compose(
|
|
"exec", "-T", "valkey", "sh", "-c",
|
|
'valkey-cli -a "$(cat /run/secrets/valkey_password)" --no-auth-warning '
|
|
f"exists {key}",
|
|
)
|
|
return output.strip() == "1"
|
|
|
|
|
|
def is_error(body: dict, code: str) -> bool:
|
|
error = body.get("error", {})
|
|
return error.get("code") == code and bool(error.get("request_id"))
|
|
|
|
|
|
def test_credentials(api: Api) -> None:
|
|
print("\n== credentials and anonymous requests")
|
|
user = api.register()
|
|
|
|
status, body, headers, _ = api.login(user["username"], "wrong-password-123")
|
|
check(
|
|
status == 401 and is_error(body, "unauthorized")
|
|
and headers.get("WWW-Authenticate", "").startswith("Basic"),
|
|
"wrong password -> 401 with Basic challenge",
|
|
)
|
|
status, _, _, _ = api.login(f"nobody_{uuid.uuid4().hex[:10]}")
|
|
check(status == 401, "unknown user -> 401")
|
|
|
|
status, _, headers, _ = api.call("GET", "/api/v1/users/me")
|
|
check(
|
|
status == 401 and headers.get("WWW-Authenticate", "").startswith("Bearer"),
|
|
"no Authorization -> 401 with Bearer challenge",
|
|
)
|
|
status, _, _, _ = api.call(
|
|
"GET", "/api/v1/users/me", headers={"X-User-Id": user["id"]}
|
|
)
|
|
check(status == 401, "X-User-Id alone gives no access")
|
|
raw = base64.b64encode(f"{user['username']}:{PASSWORD}".encode()).decode()
|
|
status, _, _, _ = api.call(
|
|
"GET", "/api/v1/users/me", headers={"Authorization": f"Basic {raw}"}
|
|
)
|
|
check(status == 401, "Basic is not accepted on business endpoints")
|
|
status, _, _, _ = api.me("forged-" + uuid.uuid4().hex)
|
|
check(status == 401, "unknown Bearer token -> 401")
|
|
|
|
|
|
def test_logout_and_cache(api: Api) -> None:
|
|
print("\n== logout and Valkey session cache")
|
|
user = api.register()
|
|
token = api.token(user["username"])
|
|
|
|
check(not valkey_has_session(token), "no cache entry before first use")
|
|
check(api.me(token)[0] == 200, "fresh token works (cache miss -> PostgreSQL)")
|
|
check(valkey_has_session(token), "session is cached in Valkey under sha256(token)")
|
|
check(api.me(token)[0] == 200, "second request works (cache hit)")
|
|
|
|
check(api.logout(token)[0] == 204, "logout -> 204")
|
|
check(api.me(token)[0] == 401, "revoked token -> 401 right after logout")
|
|
check(api.me(token)[0] == 401, "revoked token stays 401 (cache tombstone)")
|
|
check(api.logout(token)[0] == 401, "second logout with same token -> 401")
|
|
|
|
other = api.token(user["username"])
|
|
check(api.me(other)[0] == 200, "another session of the user is unaffected")
|
|
|
|
|
|
def test_expiry(short_api: Api, ttl: float) -> None:
|
|
print(f"\n== expiry with SESSION_TTL_SECONDS={ttl:g} (api_short_ttl)")
|
|
if not short_api.wait_ready(30):
|
|
check(False, "api_short_ttl is ready (start it with --profile tests)")
|
|
return
|
|
user = short_api.register()
|
|
status, body, _, _ = short_api.login(user["username"])
|
|
check(status == 201, "login on short-TTL instance")
|
|
token = body["token"]
|
|
check(short_api.me(token)[0] == 200, "token valid before expiry (now cached)")
|
|
time.sleep(ttl + 1.5)
|
|
check(short_api.me(token)[0] == 401, "token rejected after expiry, cache too")
|
|
check(short_api.logout(token)[0] == 401, "logout of expired token -> 401")
|
|
|
|
|
|
def test_registration_race(api: Api, parallel: int = 20) -> None:
|
|
print(f"\n== registration race: {parallel} parallel requests, one username")
|
|
username = f"race_{uuid.uuid4().hex[:20]}"
|
|
body = {"username": username, "display_name": "Race", "password": PASSWORD}
|
|
|
|
def attempt(_):
|
|
return api.call("POST", "/api/v1/users", body=body, timeout=30)[0]
|
|
|
|
with ThreadPoolExecutor(max_workers=parallel) as pool:
|
|
statuses = list(pool.map(attempt, range(parallel)))
|
|
|
|
summary = {code: statuses.count(code) for code in sorted(set(statuses))}
|
|
print(f"statuses: {summary}")
|
|
check(statuses.count(201) == 1, "exactly one registration wins")
|
|
check(statuses.count(409) == parallel - 1, "all others get 409, no 500")
|
|
|
|
|
|
def test_valkey_outage(api: Api, mode: str) -> None:
|
|
stop, start = ("stop", "start") if mode == "stop" else ("pause", "unpause")
|
|
print(f"\n== Valkey {mode}: cache is bypassed, PostgreSQL decides")
|
|
user = api.register()
|
|
cached_token = api.token(user["username"])
|
|
check(api.me(cached_token)[0] == 200, "token cached before outage")
|
|
|
|
compose(stop, "valkey")
|
|
try:
|
|
worst = 0.0
|
|
status, body, _, elapsed = api.login(user["username"])
|
|
worst = max(worst, elapsed)
|
|
check(status == 201, "login works without Valkey")
|
|
token = body.get("token", "")
|
|
|
|
status, _, _, elapsed = api.me(token)
|
|
worst = max(worst, elapsed)
|
|
check(status == 200, "Bearer request works without Valkey")
|
|
|
|
status, _, _, elapsed = api.create_chat(token, user["id"])
|
|
worst = max(worst, elapsed)
|
|
check(status == 201, "write works without Valkey")
|
|
|
|
status, _, _, elapsed = api.me("forged-" + uuid.uuid4().hex)
|
|
worst = max(worst, elapsed)
|
|
check(status == 401, "forged token still 401 without Valkey")
|
|
check(api.call("GET", "/api/v1/users/me")[0] == 401,
|
|
"anonymous still 401 without Valkey")
|
|
|
|
# Invalidation is lost while Valkey is unreachable.
|
|
check(api.logout(cached_token)[0] == 204, "logout works without Valkey")
|
|
status, _, _, elapsed = api.me(cached_token)
|
|
worst = max(worst, elapsed)
|
|
check(status == 401, "token revoked during outage -> 401")
|
|
check(worst < 2.0, f"bounded latency during outage (max {worst:.2f}s)")
|
|
finally:
|
|
compose(start, "valkey")
|
|
|
|
time.sleep(7) # > CACHE_RETRY_AFTER_SECONDS: API uses Valkey again
|
|
check(api.me(cached_token)[0] == 401,
|
|
"token revoked during outage stays 401 after Valkey returns "
|
|
"(stale cache flushed)")
|
|
check(api.me(token)[0] == 200, "live session works after recovery")
|
|
|
|
|
|
def test_logout_during_outage_then_api_restart(api: Api) -> None:
|
|
print("\n== logout while Valkey hangs, API restart, Valkey returns")
|
|
user = api.register()
|
|
token = api.token(user["username"])
|
|
check(api.me(token)[0] == 200, "token cached before outage")
|
|
check(valkey_has_session(token), "positive cache entry exists in Valkey")
|
|
_, chat, _, _ = api.create_chat(token, user["id"])
|
|
|
|
compose("pause", "valkey")
|
|
try:
|
|
check(api.logout(token)[0] == 204,
|
|
"logout committed in PostgreSQL, tombstone not written")
|
|
# A new API process knows nothing about the failed invalidation.
|
|
compose("restart", "api")
|
|
check(api.wait_ready(60), "API restarted while Valkey is unreachable")
|
|
finally:
|
|
compose("unpause", "valkey")
|
|
|
|
# The stale positive entry (TTL <= 60 s) is still in Valkey now.
|
|
status, _, _, _ = api.me(token)
|
|
check(status == 401, f"revoked token -> 401 after API restart (got {status})")
|
|
status, _, _, _ = api.call(
|
|
"POST", f"/api/v1/chats/{chat.get('id')}/messages",
|
|
body={"text": "must be rejected"},
|
|
headers={"Authorization": f"Bearer {token}"},
|
|
)
|
|
check(status == 401, f"revoked token cannot send messages (got {status})")
|
|
|
|
|
|
def test_database_outage(api: Api) -> None:
|
|
print("\n== PostgreSQL stop: limited 503, anonymous is never trusted")
|
|
user = api.register()
|
|
token = api.token(user["username"])
|
|
check(api.me(token)[0] == 200, "token cached before outage")
|
|
|
|
compose("stop", "db")
|
|
try:
|
|
status, body, headers, elapsed = api.create_chat(token, user["id"])
|
|
check(
|
|
status == 503 and is_error(body, "unavailable")
|
|
and bool(headers.get("X-Request-Id")),
|
|
f"write -> 503 unavailable with request id ({elapsed:.2f}s)",
|
|
)
|
|
check(elapsed < 10, "503 is returned within the configured timeouts")
|
|
|
|
status, _, _, _ = api.call("GET", "/health/ready")
|
|
check(status == 503, "/health/ready -> 503")
|
|
check(api.call("GET", "/health/live")[0] == 200, "/health/live -> 200")
|
|
|
|
status, body, _, _ = api.login(user["username"])
|
|
check(status == 503 and is_error(body, "unavailable"), "login -> 503")
|
|
|
|
check(api.call("GET", "/api/v1/users/me")[0] == 401,
|
|
"anonymous -> 401 without touching the DB")
|
|
status, _, _, _ = api.me("forged-" + uuid.uuid4().hex)
|
|
check(status == 503, "unverifiable token -> 503, never 2xx")
|
|
status, _, _, _ = api.call(
|
|
"GET", "/api/v1/chats", headers={"X-User-Id": user["id"]}
|
|
)
|
|
check(status == 401, "X-User-Id -> 401 during outage")
|
|
status, _, _, _ = api.me(token)
|
|
check(status == 200, "cached valid session still identifies the user")
|
|
finally:
|
|
compose("start", "db")
|
|
|
|
check(api.wait_ready(90), "API ready again after PostgreSQL start")
|
|
status, _, _, _ = api.create_chat(token, user["id"])
|
|
check(status == 201, "writes work again with the old session")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
parser.add_argument("--short-ttl-url", default="http://localhost:8081")
|
|
parser.add_argument("--short-ttl", type=float, default=3)
|
|
parser.add_argument("--skip-outages", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
api = Api(args.base_url)
|
|
if not api.wait_ready(30):
|
|
print("FAIL: API is not ready")
|
|
return 1
|
|
|
|
test_credentials(api)
|
|
test_logout_and_cache(api)
|
|
test_expiry(Api(args.short_ttl_url), args.short_ttl)
|
|
test_registration_race(api)
|
|
if not args.skip_outages:
|
|
test_valkey_outage(api, "stop")
|
|
test_valkey_outage(api, "pause")
|
|
test_logout_during_outage_then_api_restart(api)
|
|
test_database_outage(api)
|
|
|
|
if failures:
|
|
print(f"\nFAIL: {len(failures)} check(s) failed")
|
|
return 1
|
|
print("\nPASS: level 4 checks")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|