#!/usr/bin/env python3 """Verify bounded behavior while mandatory PostgreSQL is unavailable.""" import os from pathlib import Path import subprocess import time import uuid from client import Client, Failure, arguments, check, run def execute_hook(script, action): env = dict(os.environ) env['DATABASE_ACTION'] = action try: result = subprocess.run( [str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=90, check=False, ) except subprocess.TimeoutExpired: raise Failure('PostgreSQL outage hook exceeded 90 seconds') from None check( result.returncode == 0, 'PostgreSQL outage hook failed for action ' + action, ) def main(): def configure(parser): parser.add_argument( '--action-script', required=True, help='Executable hook controlled by DATABASE_ACTION=stop|start', ) args = arguments( 'PostgreSQL failure scenario; creates only synthetic API data', configure, ) script = Path(args.action_script).expanduser().resolve() check( script.is_file() and os.access(script, os.X_OK), 'action script must exist and be executable (chmod +x)', ) client = Client(args.base_url, args.ca_file, timeout=8) alice, _, _, _ = client.bootstrap() print('Stopping only PostgreSQL; hook output is suppressed.') execute_hook(script, 'stop') try: client.call('GET', '/health/live', schema='Health') readiness = client.call('GET', '/health/ready', 503) check( readiness['json']['error']['code'] == 'unavailable', 'readiness did not return a bounded unavailable error', ) client.call('GET', '/api/v1/users/me', 401) authenticated = client.call( 'GET', '/api/v1/users/me', 503, actor=alice, ) check( authenticated['json']['error']['code'] == 'unavailable', 'database auth failure did not return unavailable', ) registration = client.call( 'POST', '/api/v1/users', 503, body={ 'username': 'db_down_' + uuid.uuid4().hex[:16], 'display_name': 'Database outage', 'password': 'T3st-' + uuid.uuid4().hex, }, ) check( registration['json']['error']['code'] == 'unavailable', 'write during database outage did not return unavailable', ) finally: execute_hook(script, 'start') deadline = time.monotonic() + 20 while True: try: recovered = client.call( 'GET', '/api/v1/users/me', actor=alice, schema='User', )['json'] break except Failure: check( time.monotonic() < deadline, 'PostgreSQL pool did not recover within 20 seconds', ) time.sleep(0.5) check( recovered['id'] == alice['id'], 'existing session changed after PostgreSQL recovery', ) print( 'PASS: PostgreSQL outage returned bounded 503 responses, ' 'anonymous access stayed rejected, and the existing session recovered.' ) if __name__ == '__main__': run(main)