198 lines
7.7 KiB
Python
Executable File
198 lines
7.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Последовательная остановка каждой API-реплики с измерениями (lab3).
|
|
|
|
Отличие от публичного `make test-failover`: здесь реплика выбирается явно
|
|
(сначала первая, затем вторая), печатается реально остановленный контейнер и
|
|
собираются метрики: успешные записи, ошибки, время восстановления записи через
|
|
LB, набор реплик после отказа и отсутствие дублей при повторе одного ключа.
|
|
|
|
Запуск (стенд поднят, LB на 8080, hooks в scripts/):
|
|
python3 student-tests/replica_failover.py --action-script scripts/stop-one.sh
|
|
"""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
|
|
from client import Client, LAB, Failure, arguments, check, run # noqa: E402
|
|
from tests_helpers import compose, containers_by_hostname, messages_path # noqa: E402
|
|
|
|
WINDOW_SECONDS = 15.0
|
|
RECOVERY_DEADLINE_SECONDS = 10.0
|
|
STOP_DELAY_SECONDS = 0.5
|
|
|
|
|
|
def observed_instances(client, actor, tries=40):
|
|
seen = {}
|
|
|
|
for _ in range(tries):
|
|
response = client.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
|
instance = response['headers']['X-Instance-Id']
|
|
seen[instance] = seen.get(instance, 0) + 1
|
|
|
|
return seen
|
|
|
|
|
|
def stop_replica(script, instance):
|
|
started = time.monotonic()
|
|
env = dict(os.environ, TARGET_INSTANCE_ID=instance)
|
|
proc = subprocess.run([str(script)], env=env, capture_output=True, text=True)
|
|
|
|
return {
|
|
'returncode': proc.returncode,
|
|
'seconds': round(time.monotonic() - started, 3),
|
|
'stdout': proc.stdout.strip(),
|
|
'stderr': proc.stderr.strip(),
|
|
}
|
|
|
|
|
|
def measure_outage(client, actor, chat, target, script):
|
|
"""Пишет сообщения сквозь LB, пока hook останавливает выбранную реплику."""
|
|
hook = ThreadPoolExecutor(max_workers=1)
|
|
stop_started = time.monotonic()
|
|
hook_future = hook.submit(lambda: (time.sleep(STOP_DELAY_SECONDS), stop_replica(script, target))[1])
|
|
|
|
successes, failures, uncertain = 0, 0, 0
|
|
instances_after = []
|
|
first_recovery = None
|
|
key, text = None, None
|
|
stop_finished = None
|
|
|
|
try:
|
|
while time.monotonic() - stop_started < WINDOW_SECONDS:
|
|
if key is None:
|
|
key = uuid.uuid4().hex
|
|
text = 'Отказ ' + target + ' ' + key
|
|
|
|
try:
|
|
response = client.message(actor, chat, text, key=key)
|
|
except Failure:
|
|
failures += 1
|
|
uncertain += 1
|
|
continue
|
|
|
|
successes += 1
|
|
instances_after.append(response['headers']['X-Instance-Id'])
|
|
key, text = None, None
|
|
|
|
if hook_future.done() and stop_finished is None:
|
|
stop_finished = time.monotonic() - stop_started
|
|
first_recovery = first_recovery or round(stop_finished, 3)
|
|
|
|
time.sleep(0.1)
|
|
finally:
|
|
stopped = hook_future.result()
|
|
|
|
if first_recovery is None:
|
|
first_recovery = round(time.monotonic() - stop_started, 3)
|
|
|
|
elapsed = time.monotonic() - stop_started
|
|
|
|
return {
|
|
'stopped_container': stopped['stdout'],
|
|
'hook_returncode': stopped['returncode'],
|
|
'hook_seconds': stopped['seconds'],
|
|
'hook_error': stopped['stderr'],
|
|
'successful_writes': successes,
|
|
'successful_rps': round(successes / elapsed, 2) if elapsed else None,
|
|
'failed_attempts': failures,
|
|
'uncertain_retries': uncertain,
|
|
'first_recovery_seconds': first_recovery,
|
|
'instances_after_recovery': sorted(set(instances_after[-5:])),
|
|
}
|
|
|
|
|
|
def all_messages(client, actor, chat, limit=2000):
|
|
"""Вся история чата с keyset-пагинацией (проверка дублей не должна зависеть от одной страницы)."""
|
|
items = []
|
|
cursor = None
|
|
|
|
while True:
|
|
query = messages_path(chat) + '?limit=100'
|
|
if cursor:
|
|
query += '&cursor=' + quote(cursor, safe='')
|
|
|
|
page = client.call('GET', query, actor=actor, schema='MessagePage')['json']
|
|
items.extend(page['items'])
|
|
cursor = page['next_cursor']
|
|
|
|
if not cursor:
|
|
return items
|
|
|
|
check(len(items) <= limit, 'история больше защитного предела проверки')
|
|
|
|
|
|
def ensure_no_duplicates(client, actor, chat):
|
|
items = all_messages(client, actor, chat)
|
|
texts = [message['text'] for message in items]
|
|
ids = [message['id'] for message in items]
|
|
check(len(texts) == len(set(texts)), 'повтор с тем же ключом создал дубль')
|
|
check(len(ids) == len(set(ids)), 'дубли ID в истории')
|
|
|
|
return len(items)
|
|
|
|
|
|
def restore(instance, timeout=90):
|
|
started = time.monotonic()
|
|
compose('up', '-d', '--no-deps', 'api')
|
|
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
hosts = containers_by_hostname()
|
|
if instance in hosts:
|
|
return {'seconds': round(time.monotonic() - started, 3), 'instances': sorted(hosts)}
|
|
|
|
time.sleep(1)
|
|
|
|
raise Failure('реплика ' + instance + ' не вернулась в строй')
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument('--action-script', required=True, help='stop-one hook, executable, authored by student')
|
|
|
|
args = arguments('Последовательная остановка реплик с метриками (lab3)', configure)
|
|
check(LAB >= 3, 'отказ API-реплики требует lab3+')
|
|
|
|
script = Path(args.action_script).expanduser().resolve()
|
|
check(script.is_file() and os.access(script, os.X_OK), 'hook должен существовать и быть исполняемым')
|
|
|
|
client = Client(args.base_url, args.ca_file)
|
|
alice, _, _, chat = client.bootstrap()
|
|
|
|
observed = observed_instances(client, alice)
|
|
check(len(observed) >= 2, 'через LB видно меньше двух реплик: ' + str(sorted(observed)))
|
|
|
|
report = {'before': {'instances': sorted(observed)}, 'replicas': []}
|
|
|
|
for instance in sorted(observed):
|
|
result = measure_outage(client, alice, chat, instance, script)
|
|
check(result['hook_returncode'] == 0, 'hook завершился неуспешно: ' + result['hook_error'])
|
|
check(
|
|
result['first_recovery_seconds'] <= RECOVERY_DEADLINE_SECONDS,
|
|
'запись не восстановилась за ' + str(RECOVERY_DEADLINE_SECONDS) + ' c',
|
|
)
|
|
check(result['successful_writes'] >= 5, 'слишком мало подтверждённых записей после отказа')
|
|
check(instance not in result['instances_after_recovery'], 'остановленная реплика продолжает обслуживать')
|
|
result['messages_in_history'] = ensure_no_duplicates(client, alice, chat)
|
|
|
|
restored = restore(instance)
|
|
result['restored_seconds'] = restored['seconds']
|
|
result['instances_restored'] = restored['instances']
|
|
|
|
report['replicas'].append({'stopped': instance, **result})
|
|
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
print('PASS: каждая реплика остановлена по очереди; записи продолжились, дублей и потерь нет')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|