#!/usr/bin/env python3 import argparse import subprocess import sys import threading import time import uuid import os sys.path.insert(0, os.path.dirname(__file__)) from client import Client, Failure, run, check def main(): ap = argparse.ArgumentParser() ap.add_argument('--base-url', default='http://localhost:8080') ap.add_argument('--action-script', required=True) args = ap.parse_args() c = Client(args.base_url) alice, bob, eve, chat = c.bootstrap() path = '/api/v1/chats/' + chat['id'] instances = set() messages = [] errors = 0 # 1. Собираем имена реплик for _ in range(10): r = c.call('GET', '/health/live') instances.add(r['headers'].get('X-Instance-Id')) time.sleep(0.05) check(len(instances) >= 2, f'Expected at least 2 API replicas, got: {instances}') print('Observed replicas:', instances) # 2. Фоновая нагрузка stop_event = threading.Event() def sender(thread_id): nonlocal errors local_c = Client(args.base_url, timeout=5) i = 0 while not stop_event.is_set(): key = uuid.uuid4().hex text = f'msg_{thread_id}_{i}' try: r = local_c.message(alice, chat, text, key=key) messages.append(r['json']['id']) except Failure: # При 502/503 во время failover делаем retry с тем же ключом try: time.sleep(0.1) r = local_c.message(alice, chat, text, key=key) messages.append(r['json']['id']) except Failure: errors += 1 i += 1 time.sleep(0.02) threads = [threading.Thread(target=sender, args=(t,)) for t in range(4)] for t in threads: t.start() time.sleep(1) # Даем нагрузке раскрутиться # 3. Останавливаем реплику print(f'Running action script: {args.action_script}') subprocess.run(args.action_script, shell=True, check=True) # 4. Ждем и останавливаем time.sleep(4) stop_event.set() for t in threads: t.join() # 5. Проверяем, что нет дублей и потерь history_ids = [] cursor = None while True: q = path + '/messages?limit=100' if cursor: q += '&cursor=' + cursor page = c.call('GET', q, actor=alice, schema='MessagePage')['json'] for m in page['items']: history_ids.append(m['id']) cursor = page['next_cursor'] if not cursor: break check(len(set(history_ids)) == len(history_ids), 'Duplicate messages found!') check(len(messages) > 0, 'No messages were successfully sent') print(f'Success! Messages sent: {len(messages)}, Network errors (retried safely): {errors}') print('PASS: failover verified') if __name__ == '__main__': run(main)