121 lines
3.4 KiB
Python
Executable File
121 lines
3.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Verify bounded behavior while mandatory PostgreSQL is unavailable."""
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
|
|
def execute_hook(script, action):
|
|
env = dict(os.environ)
|
|
env['DATABASE_ACTION'] = action
|
|
try:
|
|
result = subprocess.run(
|
|
[str(script)],
|
|
env=env,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=90,
|
|
check=False,
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
raise Failure('PostgreSQL outage hook exceeded 90 seconds') from None
|
|
check(
|
|
result.returncode == 0,
|
|
'PostgreSQL outage hook failed for action ' + action,
|
|
)
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument(
|
|
'--action-script',
|
|
required=True,
|
|
help='Executable hook controlled by DATABASE_ACTION=stop|start',
|
|
)
|
|
|
|
args = arguments(
|
|
'PostgreSQL failure scenario; creates only synthetic API data',
|
|
configure,
|
|
)
|
|
script = Path(args.action_script).expanduser().resolve()
|
|
check(
|
|
script.is_file() and os.access(script, os.X_OK),
|
|
'action script must exist and be executable (chmod +x)',
|
|
)
|
|
|
|
client = Client(args.base_url, args.ca_file, timeout=8)
|
|
alice, _, _, _ = client.bootstrap()
|
|
|
|
print('Stopping only PostgreSQL; hook output is suppressed.')
|
|
execute_hook(script, 'stop')
|
|
try:
|
|
client.call('GET', '/health/live', schema='Health')
|
|
readiness = client.call('GET', '/health/ready', 503)
|
|
check(
|
|
readiness['json']['error']['code'] == 'unavailable',
|
|
'readiness did not return a bounded unavailable error',
|
|
)
|
|
|
|
client.call('GET', '/api/v1/users/me', 401)
|
|
authenticated = client.call(
|
|
'GET',
|
|
'/api/v1/users/me',
|
|
503,
|
|
actor=alice,
|
|
)
|
|
check(
|
|
authenticated['json']['error']['code'] == 'unavailable',
|
|
'database auth failure did not return unavailable',
|
|
)
|
|
|
|
registration = client.call(
|
|
'POST',
|
|
'/api/v1/users',
|
|
503,
|
|
body={
|
|
'username': 'db_down_' + uuid.uuid4().hex[:16],
|
|
'display_name': 'Database outage',
|
|
'password': 'T3st-' + uuid.uuid4().hex,
|
|
},
|
|
)
|
|
check(
|
|
registration['json']['error']['code'] == 'unavailable',
|
|
'write during database outage did not return unavailable',
|
|
)
|
|
finally:
|
|
execute_hook(script, 'start')
|
|
|
|
deadline = time.monotonic() + 20
|
|
while True:
|
|
try:
|
|
recovered = client.call(
|
|
'GET',
|
|
'/api/v1/users/me',
|
|
actor=alice,
|
|
schema='User',
|
|
)['json']
|
|
break
|
|
except Failure:
|
|
check(
|
|
time.monotonic() < deadline,
|
|
'PostgreSQL pool did not recover within 20 seconds',
|
|
)
|
|
time.sleep(0.5)
|
|
|
|
check(
|
|
recovered['id'] == alice['id'],
|
|
'existing session changed after PostgreSQL recovery',
|
|
)
|
|
print(
|
|
'PASS: PostgreSQL outage returned bounded 503 responses, '
|
|
'anonymous access stayed rejected, and the existing session recovered.'
|
|
)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|