325 lines
6.9 KiB
Python
325 lines
6.9 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
import time
|
|
|
|
import lab2_checks as checks
|
|
|
|
|
|
PROJECT_DIR = checks.PROJECT_DIR
|
|
client = checks.Lab2Checks()
|
|
|
|
COMPOSE = [
|
|
"docker",
|
|
"compose",
|
|
"--env-file",
|
|
".env",
|
|
]
|
|
|
|
|
|
def compose(*args, ttl=None):
|
|
env = os.environ.copy()
|
|
|
|
if ttl is not None:
|
|
env["SESSION_TTL_SECONDS"] = str(ttl)
|
|
|
|
result = subprocess.run(
|
|
COMPOSE + list(args),
|
|
cwd=PROJECT_DIR,
|
|
env=env,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=90,
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
raise RuntimeError(
|
|
"Compose command failed: "
|
|
+ " ".join(args)
|
|
+ ". Check docker compose ps -a and service logs."
|
|
)
|
|
|
|
return result.stdout.strip()
|
|
|
|
|
|
def start_storage(service):
|
|
compose(
|
|
"up",
|
|
"-d",
|
|
"--no-deps",
|
|
"--wait",
|
|
"--wait-timeout",
|
|
"45",
|
|
service,
|
|
)
|
|
|
|
|
|
def recreate_api(ttl):
|
|
compose(
|
|
"up",
|
|
"-d",
|
|
"--no-deps",
|
|
"--force-recreate",
|
|
"--wait",
|
|
"--wait-timeout",
|
|
"45",
|
|
"api",
|
|
ttl=ttl,
|
|
)
|
|
|
|
|
|
def expect_status(
|
|
method,
|
|
path,
|
|
expected,
|
|
token=None,
|
|
headers=None,
|
|
):
|
|
started = time.monotonic()
|
|
|
|
status, body, response_headers = client.request(
|
|
method,
|
|
path,
|
|
token=token,
|
|
headers=headers,
|
|
)
|
|
|
|
elapsed = time.monotonic() - started
|
|
|
|
client.assertEqual(
|
|
status,
|
|
expected,
|
|
f"Unexpected status for {method} {path}",
|
|
)
|
|
|
|
client.assertLess(
|
|
elapsed,
|
|
10,
|
|
f"Request exceeded local test budget: {path}",
|
|
)
|
|
|
|
return body, response_headers, elapsed
|
|
|
|
|
|
def wait_for_working_token(token, timeout=20):
|
|
deadline = time.monotonic() + timeout
|
|
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
status, _, _ = client.request(
|
|
"GET",
|
|
"/api/v1/users/me",
|
|
token=token,
|
|
)
|
|
except OSError:
|
|
time.sleep(0.5)
|
|
continue
|
|
|
|
if status == 200:
|
|
return
|
|
|
|
time.sleep(0.5)
|
|
|
|
raise AssertionError(
|
|
"Existing session did not recover after database restart"
|
|
)
|
|
|
|
|
|
def main():
|
|
original_ttl = int(
|
|
compose(
|
|
"exec",
|
|
"-T",
|
|
"api",
|
|
"python",
|
|
"-c",
|
|
(
|
|
"import os; "
|
|
"print(os.getenv('SESSION_TTL_SECONDS', '3600'))"
|
|
),
|
|
)
|
|
)
|
|
|
|
client.assertGreaterEqual(
|
|
original_ttl,
|
|
300,
|
|
"Start this scenario with normal SESSION_TTL_SECONDS >= 300",
|
|
)
|
|
|
|
user, token = client.create_logged_in_user()
|
|
|
|
try:
|
|
# Warm the user cache before dependency failures.
|
|
expect_status(
|
|
"GET",
|
|
"/api/v1/users/me",
|
|
200,
|
|
token=token,
|
|
)
|
|
expect_status(
|
|
"GET",
|
|
"/api/v1/users/me",
|
|
200,
|
|
token=token,
|
|
)
|
|
|
|
compose("stop", "--timeout", "10", "redis")
|
|
|
|
_, _, elapsed = expect_status(
|
|
"GET",
|
|
"/api/v1/users/me",
|
|
200,
|
|
token=token,
|
|
)
|
|
|
|
expect_status(
|
|
"GET",
|
|
"/api/v1/users/me",
|
|
401,
|
|
headers={"X-User-Id": str(user["id"])},
|
|
)
|
|
|
|
expect_status("GET", "/health/live", 200)
|
|
expect_status("GET", "/health/ready", 200)
|
|
|
|
print(
|
|
f"PASS: Redis outage fallback; "
|
|
f"authenticated request took {elapsed:.3f}s",
|
|
flush=True,
|
|
)
|
|
|
|
start_storage("redis")
|
|
|
|
# Refill the cache, then prove it cannot replace session checks.
|
|
expect_status(
|
|
"GET",
|
|
"/api/v1/users/me",
|
|
200,
|
|
token=token,
|
|
)
|
|
|
|
compose("stop", "--timeout", "10", "postgres")
|
|
|
|
body, _, elapsed = expect_status(
|
|
"GET",
|
|
"/api/v1/users/me",
|
|
503,
|
|
token=token,
|
|
)
|
|
client.assertEqual(
|
|
body["error"]["code"],
|
|
"unavailable",
|
|
)
|
|
|
|
expect_status("GET", "/health/live", 200)
|
|
expect_status("GET", "/health/ready", 503)
|
|
|
|
expect_status(
|
|
"GET",
|
|
"/api/v1/users/me",
|
|
401,
|
|
headers={"X-User-Id": str(user["id"])},
|
|
)
|
|
|
|
print(
|
|
f"PASS: PostgreSQL outage fails closed; "
|
|
f"authenticated request took {elapsed:.3f}s",
|
|
flush=True,
|
|
)
|
|
|
|
start_storage("postgres")
|
|
wait_for_working_token(token)
|
|
|
|
print(
|
|
"PASS: existing session recovered after PostgreSQL restart",
|
|
flush=True,
|
|
)
|
|
|
|
recreate_api(ttl=2)
|
|
|
|
_, short_token = client.create_logged_in_user()
|
|
|
|
expect_status(
|
|
"GET",
|
|
"/api/v1/users/me",
|
|
200,
|
|
token=short_token,
|
|
)
|
|
|
|
started = time.monotonic()
|
|
deadline = started + 10
|
|
|
|
while True:
|
|
status, body, headers = client.request(
|
|
"GET",
|
|
"/api/v1/users/me",
|
|
token=short_token,
|
|
)
|
|
|
|
if status == 401:
|
|
client.assertEqual(
|
|
body["error"]["code"],
|
|
"unauthorized",
|
|
)
|
|
client.assertEqual(
|
|
headers.get("WWW-Authenticate"),
|
|
'Bearer realm="messenger"',
|
|
)
|
|
break
|
|
|
|
client.assertEqual(status, 200)
|
|
|
|
if time.monotonic() >= deadline:
|
|
raise AssertionError(
|
|
"Session did not expire with TTL=2"
|
|
)
|
|
|
|
time.sleep(0.25)
|
|
|
|
print(
|
|
f"PASS: short session expired; "
|
|
f"observed after {time.monotonic() - started:.3f}s",
|
|
flush=True,
|
|
)
|
|
|
|
finally:
|
|
restoration_errors = []
|
|
|
|
for service in ("postgres", "redis"):
|
|
try:
|
|
start_storage(service)
|
|
except Exception as exc:
|
|
restoration_errors.append(
|
|
f"{service}: {type(exc).__name__}"
|
|
)
|
|
|
|
try:
|
|
recreate_api(ttl=original_ttl)
|
|
except Exception as exc:
|
|
restoration_errors.append(
|
|
f"api: {type(exc).__name__}"
|
|
)
|
|
|
|
if restoration_errors:
|
|
raise RuntimeError(
|
|
"Automatic restoration failed: "
|
|
+ ", ".join(restoration_errors)
|
|
)
|
|
|
|
print(
|
|
f"RESTORED: services running; "
|
|
f"SESSION_TTL_SECONDS={original_ttl}",
|
|
flush=True,
|
|
)
|
|
|
|
expect_status("GET", "/health/ready", 200)
|
|
|
|
print(
|
|
"PASS: dependency outages and short session expiry",
|
|
flush=True,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |