forked from mtusi2026/mtusi2026lab3
54 lines
2.2 KiB
Python
54 lines
2.2 KiB
Python
"""Explicitly stop each storage service on the local lab stack, then restore it."""
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client
|
|
|
|
|
|
def compose(*args):
|
|
subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, timeout=60)
|
|
|
|
|
|
def main():
|
|
client = Client(timeout=6)
|
|
actor = client.user('failure')
|
|
chat = client.call('POST', '/api/v1/chats', 201, actor=actor,
|
|
body={'title': 'Failure test', 'member_ids': [actor['id']]}, schema='Chat')['json']
|
|
message = client.message(actor, chat, 'Must survive storage failure')['json']
|
|
for service in ('redis', 'postgres'):
|
|
try:
|
|
compose('stop', service)
|
|
elapsed = []
|
|
for _ in range(3):
|
|
result = client.call('GET', '/api/v1/users/me', 503, actor=actor)
|
|
assert result['elapsed'] < 5, 'Dependency error took too long'
|
|
elapsed.append(round(result['elapsed'], 3))
|
|
client.call('GET', '/health/ready', 503)
|
|
client.call('GET', '/health/live', 200, schema='Health')
|
|
client.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': actor['id']})
|
|
client.call('GET', '/api/v1/users/me', 401, actor={'token': 'unknown'})
|
|
print(f'PASS {service} unavailable: 503; seconds={elapsed}; anonymous=401; live=200', flush=True)
|
|
finally:
|
|
compose('start', service)
|
|
start = time.monotonic()
|
|
while time.monotonic() - start < 60:
|
|
try:
|
|
client.call('GET', '/health/ready')
|
|
break
|
|
except AssertionError:
|
|
time.sleep(0.5)
|
|
else:
|
|
raise AssertionError('Storage recovery timed out')
|
|
client.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
|
page = client.call('GET', f"/api/v1/chats/{chat['id']}/messages", actor=actor, schema='MessagePage')['json']
|
|
assert page['items'] == [message]
|
|
print(f'PASS {service} recovered in {time.monotonic() - start:.2f}s: same session and message', flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|