"""Exercise both API replica failures against the same session, data and files.""" from pathlib import Path import os import subprocess import sys import time ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure, arguments, check, run REPLICAS = {'api1', 'api2'} def compose(*args): subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, timeout=40) def wait(client, expected, timeout): observed = set() consecutive = 0 required = 5 if len(expected) == 1 else 1 start = time.monotonic() while time.monotonic() - start < timeout: try: response = client.call('GET', '/health/ready', schema='Health') replica = response['headers']['X-Instance-Id'] except Failure: replica = None if replica in expected: observed.add(replica) consecutive += 1 if observed == expected and consecutive >= required: return time.monotonic() - start else: consecutive = 0 time.sleep(0.25) raise Failure('expected replicas did not become ready') def main(): args = arguments(__doc__) client = Client(args.base_url, args.ca_file or os.environ.get('CA_FILE', str(ROOT / 'secrets/ca/ca.crt')), timeout=2) wait(client, REPLICAS, 45) alice, _, _, chat = client.bootstrap() first = client.message(alice, chat, 'Проверка сохранности')['json'] attachment, content = client.attachment(alice, chat) client.wait_ready(alice, attachment) for stopped in sorted(REPLICAS): other = (REPLICAS - {stopped}).pop() compose('stop', '-t', '2', stopped) try: elapsed = wait(client, {other}, 10) for _ in range(5): response = client.call('GET', '/api/v1/users/me', actor=alice, schema='User') check(response['headers']['X-Instance-Id'] == other, 'unexpected replica while one is stopped') message = client.message(alice, chat, 'Запись через оставшуюся реплику')['json'] history = client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=alice, schema='MessagePage')['json']['items'] check({first['id'], message['id']} <= {item['id'] for item in history}, 'message missing') data = client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=alice)['bytes'] check(data == content, 'original file changed') print(f'PASS: {stopped} stopped, {other} serves existing session, messages and file. ' f'Readiness after stop returned in {elapsed:.2f}s') finally: compose('start', stopped) wait(client, REPLICAS, 45) if __name__ == '__main__': run(main)