143 lines
5.7 KiB
Python
143 lines
5.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Уровень 4: деградация при потере сервисов.
|
|
Redis down -> API обслуживает запросы через SQL (200), логи warn.
|
|
Postgres down -> /health/ready 503 и защищённые запросы 503 (не 200, не «аноним доверен»).
|
|
Запуск из корня репозитория:
|
|
python student-tests/failover_tests.py --base-url http://localhost:8080
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import subprocess
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
|
|
FAILURES = []
|
|
BASE = "http://localhost:8080"
|
|
PASSWORD = "Example-only-not-a-real-secret-123"
|
|
|
|
|
|
def check(name, cond, detail=""):
|
|
if not cond:
|
|
FAILURES.append(name)
|
|
print(f"[{'PASS' if cond else 'FAIL'}] {name}" + (f" -- {detail}" if not cond and detail else ""))
|
|
|
|
|
|
def sh(cmd):
|
|
subprocess.run(cmd, shell=True, check=True, capture_output=True)
|
|
|
|
|
|
def req(method, path, body=None, token=None, basic=None):
|
|
headers, data = {}, None
|
|
if body is not None:
|
|
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
|
headers["Content-Type"] = "application/json"
|
|
if token:
|
|
headers["Authorization"] = "Bearer " + token
|
|
if basic:
|
|
headers["Authorization"] = "Basic " + base64.b64encode(basic.encode()).decode()
|
|
r = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
|
|
try:
|
|
with urllib.request.urlopen(r, timeout=10) as resp:
|
|
payload = resp.read().decode()
|
|
return resp.status, (json.loads(payload) if payload else None)
|
|
except urllib.error.HTTPError as e:
|
|
payload = e.read().decode()
|
|
try:
|
|
return e.code, json.loads(payload)
|
|
except ValueError:
|
|
return e.code, None
|
|
|
|
|
|
def wait_ready(timeout=60, want=200):
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
st, _ = req("GET", "/health/ready")
|
|
if st == want:
|
|
return True
|
|
time.sleep(1)
|
|
return False
|
|
|
|
|
|
def main():
|
|
global BASE
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--base-url", default=BASE)
|
|
BASE = ap.parse_args().base_url
|
|
|
|
# Создаём пользователя
|
|
uname = "fo" + uuid.uuid4().hex[:10] # без дефиса
|
|
st, body = req("POST", "/api/v1/users", {"username": uname, "display_name": "FO", "password": PASSWORD})
|
|
print(f"DEBUG register: {st} {body}")
|
|
assert st == 201, f"register failed: {st} {body}"
|
|
|
|
st, sess = req("POST", "/api/v1/auth/sessions", basic=f"{uname}:{PASSWORD}")
|
|
print(f"DEBUG login: {st} {sess}")
|
|
assert st == 201, f"login failed: {st} {sess}"
|
|
token = sess["token"]
|
|
|
|
# Узнаём свой id через /me
|
|
me_st, me = req("GET", "/api/v1/users/me", token=token)
|
|
print(f"DEBUG /me: {me_st} {me}")
|
|
assert me_st == 200, f"/me failed: {me_st} {me}"
|
|
|
|
# Создаём чат с самим собой
|
|
st, chat = req("POST", "/api/v1/chats", {"title": "failover", "member_ids": [me["id"]]}, token=token)
|
|
print(f"DEBUG create chat: {st} {chat}")
|
|
assert st == 201, f"create chat failed: {st} {chat}"
|
|
|
|
# ── Redis down: деградация в SQL, не отказ ──
|
|
sh("docker compose stop redis")
|
|
time.sleep(2)
|
|
st, _ = req("GET", "/health/ready")
|
|
check("redis down: ready stays 200 (cache optional)", st == 200, str(st))
|
|
st, _ = req("GET", "/api/v1/users/me", token=token)
|
|
check("redis down: auth via postgres 200", st == 200, str(st))
|
|
st, _ = req("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "during redis outage"}, token=token)
|
|
check("redis down: write 201", st == 201, str(st))
|
|
st, _ = req("GET", f"/api/v1/chats/{chat['id']}/messages?limit=10", token=token)
|
|
check("redis down: read 200", st == 200, str(st))
|
|
sh("docker compose start redis")
|
|
check("redis back: ready 200", wait_ready(60, 200))
|
|
|
|
# ── Postgres down: ограниченные ошибки, аноним не доверен ──
|
|
sh("docker compose stop postgres")
|
|
check("postgres down: ready becomes 503", wait_ready(60, 503))
|
|
st, _ = req("GET", "/api/v1/users/me", token=token)
|
|
check("postgres down: protected request 503 (not 200/401)", st == 503, str(st))
|
|
st, _ = req("GET", "/api/v1/users/me")
|
|
check("postgres down: anonymous still 401", st == 401, str(st))
|
|
sh("docker compose start postgres")
|
|
check("postgres back: ready 200", wait_ready(90, 200))
|
|
st, _ = req("GET", "/api/v1/users/me", token=token)
|
|
check("postgres back: same session valid 200", st == 200, str(st))
|
|
|
|
# ── Logout при сбое Redis: никаких «тихих» 204, оставляющих токен живым ──
|
|
st, sess2 = req("POST", "/api/v1/auth/sessions", basic=f"{uname}:{PASSWORD}")
|
|
assert st == 201, f"second login failed: {st} {sess2}"
|
|
token2 = sess2["token"]
|
|
|
|
sh("docker compose stop redis")
|
|
time.sleep(2)
|
|
st, _ = req("DELETE", "/api/v1/auth/sessions/current", token=token2)
|
|
check("redis down: logout returns 503, not silent 204", st == 503, str(st))
|
|
st, _ = req("GET", "/api/v1/users/me", token=token2)
|
|
check("redis down: token already revoked in postgres -> 401", st == 401, str(st))
|
|
|
|
sh("docker compose start redis")
|
|
check("redis back (2): ready 200", wait_ready(60, 200))
|
|
st, _ = req("DELETE", "/api/v1/auth/sessions/current", token=token2)
|
|
check("redis back: logout retry 204 (DEL confirmed)", st == 204, str(st))
|
|
st, _ = req("GET", "/api/v1/users/me", token=token2)
|
|
check("after retry logout: token still 401", st == 401, str(st))
|
|
|
|
if FAILURES:
|
|
print(f"FAILED {len(FAILURES)}: {FAILURES}")
|
|
raise SystemExit(1)
|
|
print("ALL FAILOVER TESTS PASSED")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |