forked from mtusi2026/mtusi2026lab3
161 lines
6.9 KiB
Python
161 lines
6.9 KiB
Python
"""Проверка отказов только локального Compose-стенда; сервисы восстанавливаются."""
|
|
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from client import Client, Failure, arguments, run
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def compose(*args):
|
|
subprocess.run(
|
|
["docker", "compose", *args], cwd=ROOT, check=True, stdout=subprocess.DEVNULL, timeout=60
|
|
)
|
|
|
|
|
|
def main():
|
|
args = arguments("Отказы Redis и PostgreSQL")
|
|
c = Client(args.base_url, args.ca_file)
|
|
actor = c.user("outage")
|
|
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
key = "user:v1:" + actor["id"]
|
|
|
|
def redis(*args):
|
|
return subprocess.check_output(
|
|
["docker", "compose", "exec", "-T", "redis", "redis-cli", "--raw", *args],
|
|
cwd=ROOT,
|
|
text=True,
|
|
timeout=10,
|
|
).strip()
|
|
|
|
assert redis("EXISTS", key) == "1", "Профиль не попал в Redis"
|
|
assert 0 < int(redis("TTL", key)) <= 60
|
|
|
|
def hits():
|
|
return int(
|
|
next(
|
|
line.split(":")[1]
|
|
for line in redis("INFO", "stats").splitlines()
|
|
if line.startswith("keyspace_hits:")
|
|
)
|
|
)
|
|
|
|
before = hits()
|
|
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
assert hits() > before, "Повторное чтение не использовало Redis"
|
|
redis("DEL", key)
|
|
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
assert redis("EXISTS", key) == "1", "Кэш не восстановился из PostgreSQL"
|
|
token_key = "session:v1:" + hashlib.sha256(actor["token"].encode()).hexdigest()
|
|
assert redis("EXISTS", token_key) == "1", "Сессия отсутствует в Redis"
|
|
ttl_before = int(redis("PTTL", token_key))
|
|
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
assert 0 < int(redis("PTTL", token_key)) <= ttl_before, "Чтение продлило сессию"
|
|
print("PASS: сессия хранится в Redis с TTL; чтение не продлевает срок")
|
|
print(
|
|
"PASS: профиль кэшируется с TTL<=60; повторный GET даёт cache hit; потеря ключа восполняется"
|
|
)
|
|
lost = c.user("lost")
|
|
c.call("GET", "/api/v1/users/me", actor=lost, schema="User")
|
|
lost_key = "session:v1:" + hashlib.sha256(lost["token"].encode()).hexdigest()
|
|
redis("DEL", lost_key)
|
|
c.call("GET", "/api/v1/users/me", 401, actor=lost)
|
|
assert redis("EXISTS", lost_key) == "0", "Удалённая сессия воскресла"
|
|
print("PASS: отсутствующая сессия не восстанавливается из профиля/БД; Redis — источник истины")
|
|
# При общей неисправности LB исключает все API. Проверку отдельного
|
|
# процесса выполняем из контейнера; внешние API-порты не открываем.
|
|
container = subprocess.check_output(
|
|
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
|
|
).split()[0]
|
|
|
|
def probe(method, path, expected=200, actor=None, body=None, schema=None):
|
|
code = r"""
|
|
import json, sys, urllib.request, urllib.error
|
|
p=json.load(sys.stdin)
|
|
h={"Content-Type":"application/json"}
|
|
if p["actor"]: h["Authorization"]="Bearer "+p["actor"]["token"]
|
|
r=urllib.request.Request("http://127.0.0.1:8080"+p["path"], method=p["method"], headers=h,
|
|
data=json.dumps(p["body"]).encode() if p["body"] is not None else None)
|
|
try: response=urllib.request.urlopen(r, timeout=6)
|
|
except urllib.error.HTTPError as e: response=e
|
|
with response:
|
|
assert response.headers.get("X-Request-Id") and response.headers.get("X-Instance-Id")
|
|
print(response.status)
|
|
"""
|
|
result = subprocess.check_output(
|
|
["docker", "exec", "-i", container, "python", "-c", code],
|
|
cwd=ROOT,
|
|
text=True,
|
|
input=json.dumps(dict(method=method, path=path, actor=actor, body=body)),
|
|
)
|
|
assert int(result.strip()) == expected, (method, path, expected, result)
|
|
|
|
def wait_lb():
|
|
deadline = time.monotonic() + 30
|
|
while True:
|
|
try:
|
|
c.call("GET", "/health/ready", schema="Health")
|
|
return
|
|
except Failure:
|
|
if time.monotonic() > deadline:
|
|
raise
|
|
time.sleep(0.25)
|
|
|
|
try:
|
|
compose("stop", "redis")
|
|
probe("GET", "/health/live", schema="Health")
|
|
probe("GET", "/api/v1/users/me", 503, actor=actor)
|
|
probe("DELETE", "/api/v1/auth/sessions/current", 503, actor=actor)
|
|
probe("GET", "/health/ready", 503)
|
|
probe("GET", "/api/v1/users/me", 503, actor={"token": "invalid-token"})
|
|
probe("GET", "/api/v1/users/me", 401)
|
|
print(
|
|
"PASS: Redis выключен; live=200, ready=503; проверка сессий недоступна, доступ не выдаётся"
|
|
)
|
|
finally:
|
|
compose("up", "-d", "--wait", "redis")
|
|
wait_lb()
|
|
probe("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
print("PASS: сессия сохранилась после перезапуска Redis")
|
|
revoked = c.user("revoked")
|
|
probe("GET", "/api/v1/users/me", actor=revoked, schema="User")
|
|
probe("DELETE", "/api/v1/auth/sessions/current", 204, actor=revoked)
|
|
compose("restart", "redis")
|
|
compose("up", "-d", "--wait", "redis")
|
|
probe("GET", "/api/v1/users/me", 401, actor=revoked)
|
|
probe(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
401,
|
|
actor=revoked,
|
|
body={"title": "Нет доступа", "member_ids": [revoked["id"]]},
|
|
)
|
|
print(
|
|
"PASS: logout при отказе Redis возвращает 503; успешный отзыв переживает AOF-рестарт и не даёт доступ"
|
|
)
|
|
try:
|
|
compose("stop", "postgres")
|
|
probe("GET", "/health/live", schema="Health")
|
|
probe("GET", "/health/ready", 503)
|
|
probe("GET", "/api/v1/chats", 503, actor=actor)
|
|
probe("GET", "/api/v1/users/me", 401)
|
|
probe("GET", "/api/v1/users/me", 401, actor={"token": "invalid-token"})
|
|
probe("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
print(
|
|
"PASS: PostgreSQL выключен; live=200, ready=503, чаты=503; действующая сессия и прогретый профиль доступны в Redis"
|
|
)
|
|
finally:
|
|
compose("up", "-d", "--wait", "postgres")
|
|
probe("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
wait_lb()
|
|
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
print("PASS: после восстановления БД прежняя сессия снова работает через HAProxy")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|