"""Explicitly stop each storage service on the local lab stack, then restore it.""" import subprocess import sys import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client def compose(*args): subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, timeout=60) def main(): client = Client(timeout=6) actor = client.user('failure') chat = client.call('POST', '/api/v1/chats', 201, actor=actor, body={'title': 'Failure test', 'member_ids': [actor['id']]}, schema='Chat')['json'] message = client.message(actor, chat, 'Must survive storage failure')['json'] for service in ('redis', 'postgres'): try: compose('stop', service) elapsed = [] for _ in range(3): result = client.call('GET', '/api/v1/users/me', 503, actor=actor) assert result['elapsed'] < 5, 'Dependency error took too long' elapsed.append(round(result['elapsed'], 3)) client.call('GET', '/health/ready', 503) client.call('GET', '/health/live', 200, schema='Health') client.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': actor['id']}) client.call('GET', '/api/v1/users/me', 401, actor={'token': 'unknown'}) print(f'PASS {service} unavailable: 503; seconds={elapsed}; anonymous=401; live=200', flush=True) finally: compose('start', service) start = time.monotonic() while time.monotonic() - start < 60: try: client.call('GET', '/health/ready') break except AssertionError: time.sleep(0.5) else: raise AssertionError('Storage recovery timed out') client.call('GET', '/api/v1/users/me', actor=actor, schema='User') page = client.call('GET', f"/api/v1/chats/{chat['id']}/messages", actor=actor, schema='MessagePage')['json'] assert page['items'] == [message] print(f'PASS {service} recovered in {time.monotonic() - start:.2f}s: same session and message', flush=True) if __name__ == '__main__': main()