Files
mtusi2026lab4/tests/failover.py
T

93 lines
2.9 KiB
Python

#!/usr/bin/env python3
import argparse
import subprocess
import sys
import threading
import time
import uuid
import os
sys.path.insert(0, os.path.dirname(__file__))
from client import Client, Failure, run, check
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--base-url', default='http://localhost:8080')
ap.add_argument('--action-script', required=True)
args = ap.parse_args()
c = Client(args.base_url)
alice, bob, eve, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id']
instances = set()
messages = []
errors = 0
# 1. Собираем имена реплик
for _ in range(10):
r = c.call('GET', '/health/live')
instances.add(r['headers'].get('X-Instance-Id'))
time.sleep(0.05)
check(len(instances) >= 2, f'Expected at least 2 API replicas, got: {instances}')
print('Observed replicas:', instances)
# 2. Фоновая нагрузка
stop_event = threading.Event()
def sender(thread_id):
nonlocal errors
local_c = Client(args.base_url, timeout=5)
i = 0
while not stop_event.is_set():
key = uuid.uuid4().hex
text = f'msg_{thread_id}_{i}'
try:
r = local_c.message(alice, chat, text, key=key)
messages.append(r['json']['id'])
except Failure:
# При 502/503 во время failover делаем retry с тем же ключом
try:
time.sleep(0.1)
r = local_c.message(alice, chat, text, key=key)
messages.append(r['json']['id'])
except Failure:
errors += 1
i += 1
time.sleep(0.02)
threads = [threading.Thread(target=sender, args=(t,)) for t in range(4)]
for t in threads: t.start()
time.sleep(1) # Даем нагрузке раскрутиться
# 3. Останавливаем реплику
print(f'Running action script: {args.action_script}')
subprocess.run(args.action_script, shell=True, check=True)
# 4. Ждем и останавливаем
time.sleep(4)
stop_event.set()
for t in threads: t.join()
# 5. Проверяем, что нет дублей и потерь
history_ids = []
cursor = None
while True:
q = path + '/messages?limit=100'
if cursor: q += '&cursor=' + cursor
page = c.call('GET', q, actor=alice, schema='MessagePage')['json']
for m in page['items']:
history_ids.append(m['id'])
cursor = page['next_cursor']
if not cursor: break
check(len(set(history_ids)) == len(history_ids), 'Duplicate messages found!')
check(len(messages) > 0, 'No messages were successfully sent')
print(f'Success! Messages sent: {len(messages)}, Network errors (retried safely): {errors}')
print('PASS: failover verified')
if __name__ == '__main__':
run(main)