forked from mtusi2026/mtusi2026lab6
207 lines
11 KiB
Python
207 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""Автоматическая проверка недоступности Redis и PostgreSQL.
|
||
|
||
Сценарий явно останавливает зависимости, поэтому не входит в discovery
|
||
unittest. По умолчанию использует docker compose из корня репозитория:
|
||
|
||
python student-tests/dependency_failure.py --base-url http://127.0.0.1:8080
|
||
|
||
Команды остановки/запуска можно переопределить (--stop-redis и т.д.), например
|
||
для стенда без Docker. Результат — JSON-сводка (без токенов) в stdout и,
|
||
при --output, в файл.
|
||
|
||
Ожидаемое поведение:
|
||
* Redis недоступен -> API обходит кэш, всё работает через PostgreSQL,
|
||
readiness = 200, задержка ограничена таймаутом Redis (250 мс) и
|
||
circuit breaker'ом; logout по-прежнему отзывает сессию.
|
||
* Разрыв сети API <-> Redis БЕЗ рестарта Redis (данные кэша живы) и logout
|
||
во время разрыва -> logout 204, и отозванный токен получает 401 и во время
|
||
разрыва, и сразу после восстановления связи, и после закрытия circuit
|
||
breaker (регрессия: раньше доверяли старой положительной записи ~60 с).
|
||
* PostgreSQL недоступна -> /health/live = 200, /health/ready = 503,
|
||
запросы к данным -> 503 unavailable за ограниченное время; анонимный
|
||
запрос -> 401; неизвестный токен не принимается (503, а не 200).
|
||
* После возврата зависимости тот же токен снова работает, данные на месте.
|
||
"""
|
||
import argparse
|
||
import json
|
||
import shlex
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
import common # noqa: E402
|
||
from common import actor, basic, create_chat, login, request # noqa: E402
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
MAX_FAIL_SECONDS = 6.0 # верхняя граница ответа при отказе зависимости
|
||
MAX_DEGRADED_SECONDS = 1.5 # верхняя граница ответа при отказе Redis
|
||
|
||
|
||
class Check:
|
||
def __init__(self):
|
||
self.results = []
|
||
|
||
def __call__(self, name, condition, detail=""):
|
||
self.results.append({"check": name, "ok": bool(condition), "detail": detail})
|
||
print(("PASS " if condition else "FAIL ") + name + (" — " + detail if detail else ""), flush=True)
|
||
return condition
|
||
|
||
@property
|
||
def ok(self):
|
||
return all(r["ok"] for r in self.results)
|
||
|
||
|
||
def run(cmd):
|
||
print("$ " + cmd, flush=True)
|
||
subprocess.run(shlex.split(cmd), cwd=ROOT, check=True,
|
||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||
|
||
|
||
def wait_ready(timeout=90):
|
||
deadline = time.monotonic() + timeout
|
||
started = time.monotonic()
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
if request("GET", "/health/ready", timeout=3).status == 200:
|
||
return time.monotonic() - started
|
||
except OSError:
|
||
pass
|
||
time.sleep(0.5)
|
||
return None
|
||
|
||
|
||
def is_unavailable(r):
|
||
return (r.status == 503 and r.json and r.json["error"]["code"] == "unavailable"
|
||
and r.json["error"]["request_id"] == r.headers.get("X-Request-Id"))
|
||
|
||
|
||
def main():
|
||
p = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||
p.add_argument("--stop-redis", default="docker compose stop redis")
|
||
p.add_argument("--start-redis", default="docker compose up -d --wait redis")
|
||
p.add_argument("--partition-redis",
|
||
default="docker network disconnect messenger-lab3_backend messenger-lab3-redis-1",
|
||
help="cut API<->Redis network WITHOUT restarting Redis")
|
||
p.add_argument("--heal-redis",
|
||
default="docker network connect --alias redis messenger-lab3_backend messenger-lab3-redis-1")
|
||
p.add_argument("--stop-postgres", default="docker compose stop postgres")
|
||
p.add_argument("--start-postgres", default="docker compose up -d --wait postgres")
|
||
p.add_argument("--output")
|
||
args = p.parse_args()
|
||
common.BASE_URL = args.base_url.rstrip("/")
|
||
|
||
check = Check()
|
||
summary = {"base_url": common.BASE_URL, "timings": {}}
|
||
|
||
alice, bob = actor("dep"), actor("dep")
|
||
chat = create_chat(alice, bob)
|
||
path = "/api/v1/chats/%s/messages" % chat["id"]
|
||
first = request("POST", path, token=alice["_token"], payload={"text": "до отказов"}).json
|
||
request("GET", "/api/v1/users/me", token=alice["_token"]) # прогрев кэша
|
||
|
||
# 1. Redis недоступен
|
||
run(args.stop_redis)
|
||
try:
|
||
r = request("GET", "/api/v1/users/me", token=alice["_token"])
|
||
check("redis down: existing token still valid via PostgreSQL", r.status == 200, "%.3fs" % r.elapsed)
|
||
summary["timings"]["redis_down_first_request_s"] = round(r.elapsed, 3)
|
||
timings = []
|
||
for i in range(10):
|
||
r = request("POST", path, token=bob["_token"], payload={"text": "redis down %d" % i})
|
||
timings.append(r.elapsed)
|
||
if r.status != 201:
|
||
break
|
||
check("redis down: writes succeed", r.status == 201)
|
||
check("redis down: bounded latency", max(timings) < MAX_DEGRADED_SECONDS,
|
||
"max %.3fs over %d requests" % (max(timings), len(timings)))
|
||
summary["timings"]["redis_down_max_write_s"] = round(max(timings), 3)
|
||
check("redis down: readiness stays 200 (cache is not critical)",
|
||
request("GET", "/health/ready").status == 200)
|
||
token = login(bob)
|
||
check("redis down: login works", bool(token))
|
||
check("redis down: logout works",
|
||
request("DELETE", "/api/v1/auth/sessions/current", token=token).status == 204)
|
||
check("redis down: revoked token rejected",
|
||
request("GET", "/api/v1/users/me", token=token).status == 401)
|
||
check("redis down: anonymous rejected", request("GET", "/api/v1/users/me").status == 401)
|
||
check("redis down: unknown token rejected",
|
||
request("GET", "/api/v1/users/me", token="x" * 43).status == 401)
|
||
finally:
|
||
run(args.start_redis)
|
||
time.sleep(6) # больше CACHE_RETRY_AFTER_SECONDS: API снова пробует Redis
|
||
check("redis back: requests ok",
|
||
request("GET", "/api/v1/users/me", token=alice["_token"]).status == 200)
|
||
|
||
# 2. Разрыв связи с живым Redis + logout (замечание ревью №1)
|
||
carol = actor("dep")
|
||
for _ in range(2): # промах -> запись в кэш, затем попадание
|
||
request("GET", "/api/v1/users/me", token=carol["_token"])
|
||
run(args.partition_redis)
|
||
try:
|
||
r = request("DELETE", "/api/v1/auth/sessions/current", token=carol["_token"])
|
||
check("redis partition: logout -> 204", r.status == 204, "status %d, %.3fs" % (r.status, r.elapsed))
|
||
check("redis partition: revoked token -> 401",
|
||
request("GET", "/api/v1/users/me", token=carol["_token"]).status == 401)
|
||
finally:
|
||
run(args.heal_redis)
|
||
# Опрашиваем дольше CACHE_RETRY_AFTER_SECONDS (5 с): после закрытия circuit
|
||
# breaker API снова читает Redis, где жила положительная запись этого токена.
|
||
statuses = set()
|
||
deadline = time.monotonic() + 8
|
||
while time.monotonic() < deadline:
|
||
statuses.add(request("GET", "/api/v1/users/me", token=carol["_token"]).status)
|
||
statuses.add(request("POST", path, token=carol["_token"], payload={"text": "revoked"}).status)
|
||
time.sleep(0.25)
|
||
check("redis healed (same data): revoked token stays 401 for 8 s", statuses == {401}, str(sorted(statuses)))
|
||
check("redis healed: valid token works", request("GET", "/api/v1/users/me", token=alice["_token"]).status == 200)
|
||
|
||
# 3. PostgreSQL недоступна
|
||
run(args.stop_postgres)
|
||
try:
|
||
check("postgres down: liveness 200", request("GET", "/health/live").status == 200)
|
||
r = request("GET", "/health/ready")
|
||
check("postgres down: readiness 503 unavailable", is_unavailable(r), "%.3fs" % r.elapsed)
|
||
r = request("GET", "/api/v1/users/me")
|
||
check("postgres down: anonymous -> 401 (not trusted)", r.status == 401, "status %d" % r.status)
|
||
r = request("GET", "/api/v1/users/me", token="y" * 43)
|
||
check("postgres down: unknown token is not accepted (503)", is_unavailable(r),
|
||
"status %d, %.3fs" % (r.status, r.elapsed))
|
||
r = request("POST", path, token=alice["_token"], payload={"text": "должно не записаться"})
|
||
check("postgres down: write -> 503 within bound", is_unavailable(r) and r.elapsed < MAX_FAIL_SECONDS,
|
||
"status %d, %.3fs" % (r.status, r.elapsed))
|
||
summary["timings"]["postgres_down_write_s"] = round(r.elapsed, 3)
|
||
r = request("GET", path, token=alice["_token"])
|
||
check("postgres down: history read -> 503", is_unavailable(r), "status %d" % r.status)
|
||
r = request("POST", "/api/v1/auth/sessions",
|
||
headers={"Authorization": basic(alice["username"], alice["_password"])})
|
||
check("postgres down: login -> 503 (not 401, not 201)", is_unavailable(r), "status %d" % r.status)
|
||
body = json.dumps(r.json, ensure_ascii=False)
|
||
check("postgres down: error hides internals",
|
||
not any(s in body.lower() for s in ("psycopg", "postgres", "select", "traceback", "5432")))
|
||
finally:
|
||
run(args.start_postgres)
|
||
recovered = wait_ready()
|
||
check("postgres back: readiness recovered", recovered is not None,
|
||
"%.1fs" % recovered if recovered is not None else "timeout")
|
||
summary["timings"]["postgres_recovery_s"] = round(recovered or -1, 2)
|
||
r = request("GET", path + "?limit=100", token=alice["_token"])
|
||
ok = r.status == 200 and r.json["items"][0] == first
|
||
texts = [m["text"] for m in r.json["items"]] if r.status == 200 else []
|
||
check("postgres back: same token reads intact history", ok)
|
||
check("postgres back: rejected write was not stored", "должно не записаться" not in texts)
|
||
|
||
summary["checks"] = check.results
|
||
summary["passed"] = check.ok
|
||
if args.output:
|
||
Path(args.output).write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print(("PASS" if check.ok else "FAIL") + ": dependency failure scenario")
|
||
sys.exit(0 if check.ok else 1)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|