90 lines
2.9 KiB
Python
90 lines
2.9 KiB
Python
"""Dependency failure and Redis restart (AOF + cache rebuild)."""
|
|
import os
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from client import Client, Failure
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def compose(*args):
|
|
env_file = ROOT / ".env"
|
|
cmd = ["docker", "compose"]
|
|
if env_file.is_file():
|
|
cmd.extend(["--env-file", str(env_file)])
|
|
cmd.extend(args)
|
|
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
|
|
if proc.returncode != 0:
|
|
raise AssertionError("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or "")[-500:]))
|
|
return proc
|
|
|
|
|
|
def wait_ready(c: Client, seconds=90):
|
|
deadline = time.monotonic() + seconds
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
c.call("GET", "/health/ready", schema="Health")
|
|
return
|
|
except Failure as exc:
|
|
last = exc
|
|
time.sleep(0.5)
|
|
raise AssertionError("ready not restored: %s" % last)
|
|
|
|
|
|
def wait_status(c: Client, path, expected, seconds=30, actor=None):
|
|
deadline = time.monotonic() + seconds
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
schema = "Health" if expected == 200 and path.startswith("/health") else None
|
|
c.call("GET", path, expected, actor=actor, schema=schema)
|
|
return
|
|
except Failure as exc:
|
|
last = exc
|
|
time.sleep(0.4)
|
|
raise AssertionError("did not observe %s for %s: %s" % (expected, path, last))
|
|
|
|
|
|
@pytest.mark.infra
|
|
def test_redis_and_postgres_unavailable(live: Client):
|
|
if os.environ.get("RUN_INFRA") != "1":
|
|
pytest.skip("set RUN_INFRA=1 to stop compose services")
|
|
alice = live.user("infra")
|
|
compose("stop", "redis")
|
|
wait_status(live, "/health/live", 200)
|
|
wait_status(live, "/health/ready", 503)
|
|
wait_status(live, "/api/v1/users/me", 503, actor=alice)
|
|
compose("start", "redis")
|
|
wait_ready(live)
|
|
live.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
|
|
|
compose("stop", "postgres")
|
|
wait_status(live, "/health/live", 200)
|
|
wait_status(live, "/health/ready", 503)
|
|
compose("start", "postgres")
|
|
wait_ready(live)
|
|
live.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
|
|
|
|
|
@pytest.mark.infra
|
|
def test_redis_restart_keeps_session_and_history(live: Client):
|
|
if os.environ.get("RUN_INFRA") != "1":
|
|
pytest.skip("set RUN_INFRA=1 to restart redis")
|
|
alice, bob, _eve, chat = live.bootstrap()
|
|
msg = live.message(alice, chat, "before-redis-restart")["json"]
|
|
compose("restart", "redis")
|
|
wait_ready(live)
|
|
me = live.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"]
|
|
assert me["id"] == alice["id"]
|
|
page = live.call(
|
|
"GET",
|
|
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
|
|
actor=bob,
|
|
schema="MessagePage",
|
|
)["json"]
|
|
assert any(m["id"] == msg["id"] for m in page["items"])
|