Files
mtusi2026lab2/tests/lab2_faults.py
T

325 lines
6.9 KiB
Python

from __future__ import annotations
import os
import subprocess
import time
import lab2_checks as checks
PROJECT_DIR = checks.PROJECT_DIR
client = checks.Lab2Checks()
COMPOSE = [
"docker",
"compose",
"--env-file",
".env",
]
def compose(*args, ttl=None):
env = os.environ.copy()
if ttl is not None:
env["SESSION_TTL_SECONDS"] = str(ttl)
result = subprocess.run(
COMPOSE + list(args),
cwd=PROJECT_DIR,
env=env,
capture_output=True,
text=True,
timeout=90,
)
if result.returncode != 0:
raise RuntimeError(
"Compose command failed: "
+ " ".join(args)
+ ". Check docker compose ps -a and service logs."
)
return result.stdout.strip()
def start_storage(service):
compose(
"up",
"-d",
"--no-deps",
"--wait",
"--wait-timeout",
"45",
service,
)
def recreate_api(ttl):
compose(
"up",
"-d",
"--no-deps",
"--force-recreate",
"--wait",
"--wait-timeout",
"45",
"api",
ttl=ttl,
)
def expect_status(
method,
path,
expected,
token=None,
headers=None,
):
started = time.monotonic()
status, body, response_headers = client.request(
method,
path,
token=token,
headers=headers,
)
elapsed = time.monotonic() - started
client.assertEqual(
status,
expected,
f"Unexpected status for {method} {path}",
)
client.assertLess(
elapsed,
10,
f"Request exceeded local test budget: {path}",
)
return body, response_headers, elapsed
def wait_for_working_token(token, timeout=20):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
status, _, _ = client.request(
"GET",
"/api/v1/users/me",
token=token,
)
except OSError:
time.sleep(0.5)
continue
if status == 200:
return
time.sleep(0.5)
raise AssertionError(
"Existing session did not recover after database restart"
)
def main():
original_ttl = int(
compose(
"exec",
"-T",
"api",
"python",
"-c",
(
"import os; "
"print(os.getenv('SESSION_TTL_SECONDS', '3600'))"
),
)
)
client.assertGreaterEqual(
original_ttl,
300,
"Start this scenario with normal SESSION_TTL_SECONDS >= 300",
)
user, token = client.create_logged_in_user()
try:
# Warm the user cache before dependency failures.
expect_status(
"GET",
"/api/v1/users/me",
200,
token=token,
)
expect_status(
"GET",
"/api/v1/users/me",
200,
token=token,
)
compose("stop", "--timeout", "10", "redis")
_, _, elapsed = expect_status(
"GET",
"/api/v1/users/me",
200,
token=token,
)
expect_status(
"GET",
"/api/v1/users/me",
401,
headers={"X-User-Id": str(user["id"])},
)
expect_status("GET", "/health/live", 200)
expect_status("GET", "/health/ready", 200)
print(
f"PASS: Redis outage fallback; "
f"authenticated request took {elapsed:.3f}s",
flush=True,
)
start_storage("redis")
# Refill the cache, then prove it cannot replace session checks.
expect_status(
"GET",
"/api/v1/users/me",
200,
token=token,
)
compose("stop", "--timeout", "10", "postgres")
body, _, elapsed = expect_status(
"GET",
"/api/v1/users/me",
503,
token=token,
)
client.assertEqual(
body["error"]["code"],
"unavailable",
)
expect_status("GET", "/health/live", 200)
expect_status("GET", "/health/ready", 503)
expect_status(
"GET",
"/api/v1/users/me",
401,
headers={"X-User-Id": str(user["id"])},
)
print(
f"PASS: PostgreSQL outage fails closed; "
f"authenticated request took {elapsed:.3f}s",
flush=True,
)
start_storage("postgres")
wait_for_working_token(token)
print(
"PASS: existing session recovered after PostgreSQL restart",
flush=True,
)
recreate_api(ttl=2)
_, short_token = client.create_logged_in_user()
expect_status(
"GET",
"/api/v1/users/me",
200,
token=short_token,
)
started = time.monotonic()
deadline = started + 10
while True:
status, body, headers = client.request(
"GET",
"/api/v1/users/me",
token=short_token,
)
if status == 401:
client.assertEqual(
body["error"]["code"],
"unauthorized",
)
client.assertEqual(
headers.get("WWW-Authenticate"),
'Bearer realm="messenger"',
)
break
client.assertEqual(status, 200)
if time.monotonic() >= deadline:
raise AssertionError(
"Session did not expire with TTL=2"
)
time.sleep(0.25)
print(
f"PASS: short session expired; "
f"observed after {time.monotonic() - started:.3f}s",
flush=True,
)
finally:
restoration_errors = []
for service in ("postgres", "redis"):
try:
start_storage(service)
except Exception as exc:
restoration_errors.append(
f"{service}: {type(exc).__name__}"
)
try:
recreate_api(ttl=original_ttl)
except Exception as exc:
restoration_errors.append(
f"api: {type(exc).__name__}"
)
if restoration_errors:
raise RuntimeError(
"Automatic restoration failed: "
+ ", ".join(restoration_errors)
)
print(
f"RESTORED: services running; "
f"SESSION_TTL_SECONDS={original_ttl}",
flush=True,
)
expect_status("GET", "/health/ready", 200)
print(
"PASS: dependency outages and short session expiry",
flush=True,
)
if __name__ == "__main__":
main()