Files
mtusi2026lab7/student-tests/alternate_failover.py
T

65 lines
3.2 KiB
Python

#!/usr/bin/env python3
"""Проверка второй реплики, если публичный сценарий повторно выбирает тот же target."""
import os
from pathlib import Path
import subprocess
import sys
import time
from uuid import uuid4
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, Failure, ROOT, arguments, check, run
from resilience import history
from worker_compromise import cmd
def main():
args = arguments('Отказ выбранной API-реплики', lambda p: p.add_argument('--replica', choices=('1', '2'), required=True))
c = Client(args.base_url, args.ca_file or ROOT / 'secrets/tls/service-ca/ca.crt', timeout=2)
container = cmd('docker', 'compose', 'ps', '-q', 'api', capture=True).splitlines()
target, = (cid[:12] for cid in container if cmd('docker', 'inspect', '-f',
'{{index .Config.Labels "com.docker.compose.container-number"}}', cid, capture=True) == args.replica)
alice, bob, _, chat = c.bootstrap()
initial = c.message(alice, chat, 'Подтверждено до отказа')['json']
attachment, image = c.attachment(alice, chat)
c.wait_ready(alice, attachment)
seen = {c.call('GET', '/api/v1/users/me', actor=alice)['headers']['X-Instance-Id'] for _ in range(10)}
check(len(seen) == 2 and target in seen, 'Две реплики не обнаружены')
started = time.monotonic()
proc = subprocess.Popen([str(ROOT / 'scripts/stop-one.sh')], cwd=ROOT,
env={**os.environ, 'TARGET_INSTANCE_ID': target})
results, failures, recovered = [], 0, None
key = uuid4().hex
try:
while time.monotonic() < started + 15:
try:
response = c.message(alice, chat, key, key=key)
except Failure:
failures += 1
else:
results.append(response['json'])
if proc.poll() == 0:
check(response['headers']['X-Instance-Id'] != target, 'Остановленная реплика ответила')
if recovered is None:
recovered = time.monotonic() - started
key = uuid4().hex
time.sleep(0.1)
check(proc.wait(timeout=2) == 0 and recovered is not None and recovered <= 10, 'Долгое восстановление')
check(len(results) >= 5, 'Мало успешных записей')
messages = history(c, bob, chat)
check(all(row in messages for row in [initial, *results]), 'Подтверждённое сообщение потеряно')
check(len({row['text'] for row in messages}) == len(messages), 'Дубли сообщений')
result = c.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob)
check(result['bytes'] == image, 'Оригинал изменился')
print(f'PASS api-{args.replica}: {len(results)} successful writes, {failures} failures, recovery {recovered:.3f}s')
finally:
if proc.poll() is None:
proc.terminate()
proc.wait()
cmd('docker', 'compose', 'start', 'api')
if __name__ == '__main__':
run(main)