Files
mtusi2026lab3/student_tests/failover_each.py
T
2026-11-03 20:49:07 +03:00

136 lines
5.0 KiB
Python

#!/usr/bin/env python3
"""Stop every observed API replica in turn and record recovery metrics."""
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import uuid
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, arguments, check, run
def executable(path, label):
value = Path(path).expanduser().resolve()
check(value.is_file() and os.access(value, os.X_OK), label + ' must be executable')
return value
def observe_instances(client, actor, attempts=60):
result = set()
for _ in range(attempts):
result.add(client.call(
'GET', '/api/v1/users/me', actor=actor, schema='User'
)['headers']['X-Instance-Id'])
return result
def message_history(client, actor, chat_id):
path = '/api/v1/chats/' + chat_id + '/messages?limit=100'
result = []
for _ in range(1000):
page = client.call(
'GET', path, actor=actor, schema='MessagePage')['json']
result.extend(page['items'])
if page['next_cursor'] is None:
return result
path = ('/api/v1/chats/' + chat_id + '/messages?limit=100&cursor='
+ quote(page['next_cursor'], safe=''))
raise Failure('history pagination exceeded safety bound')
def main():
def configure(parser):
parser.add_argument('--action-script', required=True)
parser.add_argument('--restore-script', required=True)
parser.add_argument('--window', type=float, default=15)
parser.add_argument('--output', default=None)
args = arguments('Sequential failover of every API replica', configure)
check(5 <= args.window <= 60, 'window must be 5..60 seconds')
action = executable(args.action_script, 'action script')
restore = executable(args.restore_script, 'restore script')
client = Client(args.base_url, args.ca_file, timeout=2)
alice, bob, _, chat = client.bootstrap()
targets = sorted(observe_instances(client, alice))
check(len(targets) >= 2, 'expected at least two API replicas')
targets = targets[:2]
summaries = []
for target in targets:
check(target in observe_instances(client, alice),
'target replica is not observable before stop')
env = dict(os.environ)
env['TARGET_INSTANCE_ID'] = target
process = subprocess.Popen(
[str(action)], env=env,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
started = time.monotonic()
first_recovery = None
reads = writes = failures = 0
confirmed = []
key = uuid.uuid4().hex
text = 'failover-each-' + target + '-' + key
while time.monotonic() - started < args.window:
try:
if (reads + writes) % 2 == 0:
response = client.message(alice, chat, text, key=key)
confirmed.append(response['json'])
writes += 1
key = uuid.uuid4().hex
text = 'failover-each-' + target + '-' + key
else:
client.call('GET', '/api/v1/users/me', actor=bob, schema='User')
reads += 1
if process.poll() == 0 and first_recovery is None:
first_recovery = time.monotonic() - started
except Failure:
failures += 1
time.sleep(0.05)
check(process.wait(timeout=5) == 0, 'stop-one action failed')
elapsed = time.monotonic() - started
check(first_recovery is not None and first_recovery <= 10,
'recovery exceeded 10 seconds')
history = message_history(client, bob, chat['id'])
history_ids = {message['id'] for message in history}
check(all(message['id'] in history_ids for message in confirmed),
'confirmed message disappeared after failover')
summaries.append({
'target_instance_id': target,
'window_seconds': round(elapsed, 3),
'successful_reads': reads,
'successful_writes': writes,
'failed_attempts': failures,
'successful_rps': round((reads + writes) / elapsed, 3),
'first_recovery_seconds': round(first_recovery, 3),
})
check(subprocess.run(
[str(restore)], stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, timeout=100).returncode == 0,
'restore action failed')
check(target in observe_instances(client, alice),
'stopped replica did not return after restore')
report = {
'scenario': 'sequential failover of each API replica',
'replicas': len(targets),
'results': summaries,
}
output = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
destination = Path(args.output)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(output + '\n', encoding='utf-8')
print(output)
if __name__ == '__main__':
run(main)