Files

97 lines
5.1 KiB
Python

#!/usr/bin/env python3
"""Отказы зависимостей: Redis (кэш) и PostgreSQL (источник истины).
ВНИМАНИЕ: тест останавливает и снова запускает контейнеры redis и postgres (по умолчанию через
docker compose; команды переопределяются STOP_DB/START_DB/STOP_CACHE/START_CACHE). Volumes не трогает.
"""
import time
import uuid
from common import BASE_URL, Client, Failure, Suite, basic, check, login, new_user, sh, sql, wait_until
c = Client(BASE_URL, timeout=10)
s = Suite("dependency failures (" + BASE_URL + ")")
alice = new_user(c, "fa"); login(c, alice)
bob = new_user(c, "fb"); login(c, bob)
chat = c.call("POST", "/api/v1/chats", 201, actor=alice, schema="Chat",
body={"title": "failures", "member_ids": [alice["id"], bob["id"]]})["json"]
PATH = "/api/v1/chats/" + chat["id"] + "/messages"
def timed(fn):
t = time.monotonic(); r = fn(); return r, time.monotonic() - t
@s.case("Redis down: API bypasses cache via PostgreSQL, no anonymous trust")
def _():
c.call("GET", "/api/v1/users/me", actor=alice, schema="User") # запись в кэше есть
victim = new_user(c, "fv"); login(c, victim)
c.call("GET", "/api/v1/users/me", actor=victim, schema="User")
sh("stop_cache")
try:
_, took = timed(lambda: c.call("GET", "/api/v1/users/me", actor=alice, schema="User"))
check(took < 2.0, "fallback too slow: %.2fs" % took)
c.call("GET", "/health/ready", schema="Health") # кэш не критичен
c.message(alice, chat, "while redis is down")
fresh = new_user(c, "fn"); login(c, fresh) # вход без Redis
c.call("GET", "/api/v1/users/me", actor=fresh, schema="User")
c.call("GET", "/api/v1/users/me", 401) # аноним — 401
c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Bearer " + "B" * 43})
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=victim) # logout при мёртвом Redis
c.call("GET", "/api/v1/users/me", 401, actor=victim)
pending = sql("SELECT count(*) FROM session_cache_invalidations")[0]
finally:
sh("start_cache")
# После возврата Redis старая запись кэша victim не должна «воскресить» отозванный токен.
wait_until(lambda: sql("SELECT count(*) FROM session_cache_invalidations") == ["0"], timeout=30)
for _ in range(10):
c.call("GET", "/api/v1/users/me", 401, actor=victim)
time.sleep(0.3)
c.call("GET", "/api/v1/users/me", actor=alice, schema="User")
return "fallback latency %.3fs; outbox rows during outage=%s, drained after recovery" % (took, pending)
@s.case("PostgreSQL down: bounded 503 in error format; anonymous still 401")
def _():
before = len(c.call("GET", PATH + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"])
sh("stop_db")
observed = {}
try:
c.call("GET", "/health/live", schema="Health")
r, took = timed(lambda: c.call("GET", "/health/ready", 503))
observed["ready"] = round(took, 2)
for name, call in {
"post_message": lambda: c.call("POST", PATH, 503, actor=alice, body={"text": "db down"},
headers={"Idempotency-Key": uuid.uuid4().hex}),
"history": lambda: c.call("GET", PATH, 503, actor=bob),
"login": lambda: c.call("POST", "/api/v1/auth/sessions", 503,
headers={"Authorization": basic(alice["username"], alice["password"])}),
"register": lambda: c.call("POST", "/api/v1/users", 503,
body={"username": "x" + uuid.uuid4().hex[:10], "display_name": "x", "password": "p" * 12}),
}.items():
r, took = timed(call)
check(r["json"]["error"]["code"] == "unavailable", name + ": wrong code")
check(took < 5.0, name + ": not bounded (%.2fs)" % took)
check("psycopg" not in r["json"]["error"]["message"].lower(), "leaks internals")
observed[name] = round(took, 2)
c.call("GET", "/api/v1/users/me", 401) # без токена — 401
r = c.call("GET", "/api/v1/chats", [401, 503], headers={"Authorization": "Bearer " + "C" * 43})
check(r["status"] != 200, "unknown token accepted")
finally:
sh("start_db")
t0 = time.monotonic()
wait_until(lambda: c.call("GET", "/health/ready", [200, 503])["status"] == 200, timeout=90)
recovered = time.monotonic() - t0
after = c.call("GET", PATH + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"]
check(len(after) == before, "data changed across DB outage")
c.message(alice, chat, "after db recovery")
return "latency(s)=%s; ready again in %.1fs" % (observed, recovered)
if __name__ == "__main__":
try:
s.run()
except Failure as exc:
print("FAIL", exc)
raise SystemExit(1)