Files
mtusi2026lab2/tests/database_resilience.py

121 lines
3.4 KiB
Python
Executable File

#!/usr/bin/env python3
"""Verify bounded behavior while mandatory PostgreSQL is unavailable."""
import os
from pathlib import Path
import subprocess
import time
import uuid
from client import Client, Failure, arguments, check, run
def execute_hook(script, action):
env = dict(os.environ)
env['DATABASE_ACTION'] = action
try:
result = subprocess.run(
[str(script)],
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=90,
check=False,
)
except subprocess.TimeoutExpired:
raise Failure('PostgreSQL outage hook exceeded 90 seconds') from None
check(
result.returncode == 0,
'PostgreSQL outage hook failed for action ' + action,
)
def main():
def configure(parser):
parser.add_argument(
'--action-script',
required=True,
help='Executable hook controlled by DATABASE_ACTION=stop|start',
)
args = arguments(
'PostgreSQL failure scenario; creates only synthetic API data',
configure,
)
script = Path(args.action_script).expanduser().resolve()
check(
script.is_file() and os.access(script, os.X_OK),
'action script must exist and be executable (chmod +x)',
)
client = Client(args.base_url, args.ca_file, timeout=8)
alice, _, _, _ = client.bootstrap()
print('Stopping only PostgreSQL; hook output is suppressed.')
execute_hook(script, 'stop')
try:
client.call('GET', '/health/live', schema='Health')
readiness = client.call('GET', '/health/ready', 503)
check(
readiness['json']['error']['code'] == 'unavailable',
'readiness did not return a bounded unavailable error',
)
client.call('GET', '/api/v1/users/me', 401)
authenticated = client.call(
'GET',
'/api/v1/users/me',
503,
actor=alice,
)
check(
authenticated['json']['error']['code'] == 'unavailable',
'database auth failure did not return unavailable',
)
registration = client.call(
'POST',
'/api/v1/users',
503,
body={
'username': 'db_down_' + uuid.uuid4().hex[:16],
'display_name': 'Database outage',
'password': 'T3st-' + uuid.uuid4().hex,
},
)
check(
registration['json']['error']['code'] == 'unavailable',
'write during database outage did not return unavailable',
)
finally:
execute_hook(script, 'start')
deadline = time.monotonic() + 20
while True:
try:
recovered = client.call(
'GET',
'/api/v1/users/me',
actor=alice,
schema='User',
)['json']
break
except Failure:
check(
time.monotonic() < deadline,
'PostgreSQL pool did not recover within 20 seconds',
)
time.sleep(0.5)
check(
recovered['id'] == alice['id'],
'existing session changed after PostgreSQL recovery',
)
print(
'PASS: PostgreSQL outage returned bounded 503 responses, '
'anonymous access stayed rejected, and the existing session recovered.'
)
if __name__ == '__main__':
run(main)