import sys import time import uuid from common import PASSWORD, basic, call, check, compose, new_chat, new_key, new_user, read_history, run_tests, send, wait_ready FAKE_TOKEN = "A" * 43 def timed(method, path, expected, **kwargs): started = time.monotonic() reply = call(method, path, expected, **kwargs) return reply, (time.monotonic() - started) * 1000 def prepare(): alice, bob = new_user("alice"), new_user("bob") chat = new_chat(alice, bob) before = send(alice, chat["id"], "confirmed before outage").json return alice, bob, chat, before def recovered(alice, chat, before): wait_ready() history, _ = read_history(alice, chat["id"]) check(before in history, "message confirmed before the outage is lost or changed") call("GET", "/api/v1/users/me", 200, token=alice["token"]) print("after start: the old token works, confirmed messages are in place") def test_redis_down(): alice, bob, chat, before = prepare() print("stopping redis") compose("stop", "redis") try: call("GET", "/health/live", 200) _, ms = timed("GET", "/health/ready", 503) print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") reply, ms = timed("GET", "/api/v1/users/me", 503, token=alice["token"]) check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") print(f"valid token: GET /users/me -> 503 unavailable ({ms:.0f} ms), session cannot be checked") call("GET", "/api/v1/chats", 503, token=FAKE_TOKEN) print("unknown token -> 503, not 200: nobody is trusted without a check") call("GET", "/api/v1/chats", 401) print("no token -> 401") call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) print("login -> 503: there is nowhere to store the session") call("POST", "/api/v1/users", 201, body={"username": "rd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) print("registration -> 201: it needs only PostgreSQL") finally: print("starting redis") compose("start", "redis") recovered(alice, chat, before) print("PASS: Redis outage gives 503 and does not let anyone in without a session check") def test_redis_crash(): alice = new_user("crash") print("killing redis with SIGKILL (docker compose kill redis): no graceful shutdown") compose("kill", "redis") try: # Обычно 503; 200 возможен, если Docker уже успел поднять Redis по restart policy. status = call("GET", "/api/v1/users/me", (503, 200), token=alice["token"]).status print(f"right after the crash: GET /users/me -> {status}") finally: print("starting redis") compose("start", "redis") wait_ready() call("GET", "/api/v1/users/me", 200, token=alice["token"]) print("after start: the session issued before the crash still works (AOF, appendfsync always)") print("PASS: sessions survive a Redis crash, not only a clean restart") def test_postgres_down(): alice, bob, chat, before = prepare() print("stopping postgres") compose("stop", "postgres") try: call("GET", "/health/live", 200) _, ms = timed("GET", "/health/ready", 503) print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") for method, path, body in (("GET", "/api/v1/users/me", None), ("GET", "/api/v1/chats", None), ("GET", f"/api/v1/chats/{chat['id']}/messages", None), ("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "will not be saved"})): reply = call(method, path, 503, token=alice["token"], body=body, headers={"Idempotency-Key": new_key()}) check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") check("postgres" not in reply.data.decode().lower(), "error must not show database details") print("valid token: users/me, chats, history, new message -> 503 unavailable, no details") call("GET", "/api/v1/chats", 401) call("GET", "/api/v1/chats", 401, token=FAKE_TOKEN) print("no token -> 401, unknown token -> 401: anonymous users are not let in") call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) call("POST", "/api/v1/users", 503, body={"username": "pd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) print("login -> 503, registration -> 503") finally: print("starting postgres") compose("start", "postgres") recovered(alice, chat, before) print("PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out") if __name__ == "__main__": target = sys.argv[1] if len(sys.argv) > 1 else "all" tests = {"redis": [test_redis_down, test_redis_crash], "postgres": [test_postgres_down], "all": [test_redis_down, test_redis_crash, test_postgres_down]} check(target in tests, "usage: python student-tests/test_failures.py [redis|postgres]") run_tests(tests[target])