93 lines
2.9 KiB
Python
93 lines
2.9 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import os
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
from client import Client, Failure, run, check
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('--base-url', default='http://localhost:8080')
|
|
ap.add_argument('--action-script', required=True)
|
|
args = ap.parse_args()
|
|
|
|
c = Client(args.base_url)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
path = '/api/v1/chats/' + chat['id']
|
|
|
|
instances = set()
|
|
messages = []
|
|
errors = 0
|
|
|
|
# 1. Собираем имена реплик
|
|
for _ in range(10):
|
|
r = c.call('GET', '/health/live')
|
|
instances.add(r['headers'].get('X-Instance-Id'))
|
|
time.sleep(0.05)
|
|
|
|
check(len(instances) >= 2, f'Expected at least 2 API replicas, got: {instances}')
|
|
print('Observed replicas:', instances)
|
|
|
|
# 2. Фоновая нагрузка
|
|
stop_event = threading.Event()
|
|
|
|
def sender(thread_id):
|
|
nonlocal errors
|
|
local_c = Client(args.base_url, timeout=5)
|
|
i = 0
|
|
while not stop_event.is_set():
|
|
key = uuid.uuid4().hex
|
|
text = f'msg_{thread_id}_{i}'
|
|
try:
|
|
r = local_c.message(alice, chat, text, key=key)
|
|
messages.append(r['json']['id'])
|
|
except Failure:
|
|
# При 502/503 во время failover делаем retry с тем же ключом
|
|
try:
|
|
time.sleep(0.1)
|
|
r = local_c.message(alice, chat, text, key=key)
|
|
messages.append(r['json']['id'])
|
|
except Failure:
|
|
errors += 1
|
|
i += 1
|
|
time.sleep(0.02)
|
|
|
|
threads = [threading.Thread(target=sender, args=(t,)) for t in range(4)]
|
|
for t in threads: t.start()
|
|
|
|
time.sleep(1) # Даем нагрузке раскрутиться
|
|
|
|
# 3. Останавливаем реплику
|
|
print(f'Running action script: {args.action_script}')
|
|
subprocess.run(args.action_script, shell=True, check=True)
|
|
|
|
# 4. Ждем и останавливаем
|
|
time.sleep(4)
|
|
stop_event.set()
|
|
for t in threads: t.join()
|
|
|
|
# 5. Проверяем, что нет дублей и потерь
|
|
history_ids = []
|
|
cursor = None
|
|
while True:
|
|
q = path + '/messages?limit=100'
|
|
if cursor: q += '&cursor=' + cursor
|
|
page = c.call('GET', q, actor=alice, schema='MessagePage')['json']
|
|
for m in page['items']:
|
|
history_ids.append(m['id'])
|
|
cursor = page['next_cursor']
|
|
if not cursor: break
|
|
|
|
check(len(set(history_ids)) == len(history_ids), 'Duplicate messages found!')
|
|
check(len(messages) > 0, 'No messages were successfully sent')
|
|
|
|
print(f'Success! Messages sent: {len(messages)}, Network errors (retried safely): {errors}')
|
|
print('PASS: failover verified')
|
|
|
|
if __name__ == '__main__':
|
|
run(main) |