forked from mtusi2026/mtusi2026lab5
136 lines
5.0 KiB
Python
136 lines
5.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Stop every observed API replica in turn and record recovery metrics."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from urllib.parse import quote
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
|
|
def executable(path, label):
|
|
value = Path(path).expanduser().resolve()
|
|
check(value.is_file() and os.access(value, os.X_OK), label + ' must be executable')
|
|
return value
|
|
|
|
|
|
def observe_instances(client, actor, attempts=60):
|
|
result = set()
|
|
for _ in range(attempts):
|
|
result.add(client.call(
|
|
'GET', '/api/v1/users/me', actor=actor, schema='User'
|
|
)['headers']['X-Instance-Id'])
|
|
return result
|
|
|
|
|
|
def message_history(client, actor, chat_id):
|
|
path = '/api/v1/chats/' + chat_id + '/messages?limit=100'
|
|
result = []
|
|
for _ in range(1000):
|
|
page = client.call(
|
|
'GET', path, actor=actor, schema='MessagePage')['json']
|
|
result.extend(page['items'])
|
|
if page['next_cursor'] is None:
|
|
return result
|
|
path = ('/api/v1/chats/' + chat_id + '/messages?limit=100&cursor='
|
|
+ quote(page['next_cursor'], safe=''))
|
|
raise Failure('history pagination exceeded safety bound')
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument('--action-script', required=True)
|
|
parser.add_argument('--restore-script', required=True)
|
|
parser.add_argument('--window', type=float, default=15)
|
|
parser.add_argument('--output', default=None)
|
|
args = arguments('Sequential failover of every API replica', configure)
|
|
check(5 <= args.window <= 60, 'window must be 5..60 seconds')
|
|
action = executable(args.action_script, 'action script')
|
|
restore = executable(args.restore_script, 'restore script')
|
|
client = Client(args.base_url, args.ca_file, timeout=2)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
targets = sorted(observe_instances(client, alice))
|
|
check(len(targets) >= 2, 'expected at least two API replicas')
|
|
targets = targets[:2]
|
|
summaries = []
|
|
|
|
for target in targets:
|
|
check(target in observe_instances(client, alice),
|
|
'target replica is not observable before stop')
|
|
env = dict(os.environ)
|
|
env['TARGET_INSTANCE_ID'] = target
|
|
process = subprocess.Popen(
|
|
[str(action)], env=env,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
started = time.monotonic()
|
|
first_recovery = None
|
|
reads = writes = failures = 0
|
|
confirmed = []
|
|
key = uuid.uuid4().hex
|
|
text = 'failover-each-' + target + '-' + key
|
|
|
|
while time.monotonic() - started < args.window:
|
|
try:
|
|
if (reads + writes) % 2 == 0:
|
|
response = client.message(alice, chat, text, key=key)
|
|
confirmed.append(response['json'])
|
|
writes += 1
|
|
key = uuid.uuid4().hex
|
|
text = 'failover-each-' + target + '-' + key
|
|
else:
|
|
client.call('GET', '/api/v1/users/me', actor=bob, schema='User')
|
|
reads += 1
|
|
if process.poll() == 0 and first_recovery is None:
|
|
first_recovery = time.monotonic() - started
|
|
except Failure:
|
|
failures += 1
|
|
time.sleep(0.05)
|
|
|
|
check(process.wait(timeout=5) == 0, 'stop-one action failed')
|
|
elapsed = time.monotonic() - started
|
|
check(first_recovery is not None and first_recovery <= 10,
|
|
'recovery exceeded 10 seconds')
|
|
history = message_history(client, bob, chat['id'])
|
|
history_ids = {message['id'] for message in history}
|
|
check(all(message['id'] in history_ids for message in confirmed),
|
|
'confirmed message disappeared after failover')
|
|
|
|
summaries.append({
|
|
'target_instance_id': target,
|
|
'window_seconds': round(elapsed, 3),
|
|
'successful_reads': reads,
|
|
'successful_writes': writes,
|
|
'failed_attempts': failures,
|
|
'successful_rps': round((reads + writes) / elapsed, 3),
|
|
'first_recovery_seconds': round(first_recovery, 3),
|
|
})
|
|
|
|
check(subprocess.run(
|
|
[str(restore)], stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL, timeout=100).returncode == 0,
|
|
'restore action failed')
|
|
check(target in observe_instances(client, alice),
|
|
'stopped replica did not return after restore')
|
|
|
|
report = {
|
|
'scenario': 'sequential failover of each API replica',
|
|
'replicas': len(targets),
|
|
'results': summaries,
|
|
}
|
|
output = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
destination = Path(args.output)
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(output + '\n', encoding='utf-8')
|
|
print(output)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|