65 lines
3.2 KiB
Python
65 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Проверка второй реплики, если публичный сценарий повторно выбирает тот же target."""
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from uuid import uuid4
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
from client import Client, Failure, ROOT, arguments, check, run
|
|
from resilience import history
|
|
from worker_compromise import cmd
|
|
|
|
|
|
def main():
|
|
args = arguments('Отказ выбранной API-реплики', lambda p: p.add_argument('--replica', choices=('1', '2'), required=True))
|
|
c = Client(args.base_url, args.ca_file or ROOT / 'secrets/tls/service-ca/ca.crt', timeout=2)
|
|
container = cmd('docker', 'compose', 'ps', '-q', 'api', capture=True).splitlines()
|
|
target, = (cid[:12] for cid in container if cmd('docker', 'inspect', '-f',
|
|
'{{index .Config.Labels "com.docker.compose.container-number"}}', cid, capture=True) == args.replica)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
initial = c.message(alice, chat, 'Подтверждено до отказа')['json']
|
|
attachment, image = c.attachment(alice, chat)
|
|
c.wait_ready(alice, attachment)
|
|
seen = {c.call('GET', '/api/v1/users/me', actor=alice)['headers']['X-Instance-Id'] for _ in range(10)}
|
|
check(len(seen) == 2 and target in seen, 'Две реплики не обнаружены')
|
|
|
|
started = time.monotonic()
|
|
proc = subprocess.Popen([str(ROOT / 'scripts/stop-one.sh')], cwd=ROOT,
|
|
env={**os.environ, 'TARGET_INSTANCE_ID': target})
|
|
results, failures, recovered = [], 0, None
|
|
key = uuid4().hex
|
|
try:
|
|
while time.monotonic() < started + 15:
|
|
try:
|
|
response = c.message(alice, chat, key, key=key)
|
|
except Failure:
|
|
failures += 1
|
|
else:
|
|
results.append(response['json'])
|
|
if proc.poll() == 0:
|
|
check(response['headers']['X-Instance-Id'] != target, 'Остановленная реплика ответила')
|
|
if recovered is None:
|
|
recovered = time.monotonic() - started
|
|
key = uuid4().hex
|
|
time.sleep(0.1)
|
|
check(proc.wait(timeout=2) == 0 and recovered is not None and recovered <= 10, 'Долгое восстановление')
|
|
check(len(results) >= 5, 'Мало успешных записей')
|
|
messages = history(c, bob, chat)
|
|
check(all(row in messages for row in [initial, *results]), 'Подтверждённое сообщение потеряно')
|
|
check(len({row['text'] for row in messages}) == len(messages), 'Дубли сообщений')
|
|
result = c.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob)
|
|
check(result['bytes'] == image, 'Оригинал изменился')
|
|
print(f'PASS api-{args.replica}: {len(results)} successful writes, {failures} failures, recovery {recovered:.3f}s')
|
|
finally:
|
|
if proc.poll() is None:
|
|
proc.terminate()
|
|
proc.wait()
|
|
cmd('docker', 'compose', 'start', 'api')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|