Files
mtusi2026lab5/student_tests/node_failover.py
T
2026-11-18 20:33:27 +03:00

183 lines
7.2 KiB
Python

#!/usr/bin/env python3
"""Hard-stop one Swarm worker and prove API task relocation and data safety."""
import json
import atexit
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import subprocess
import sys
import time
import uuid
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, arguments, check, run
def command(*parts, timeout=30):
result = subprocess.run(
parts, capture_output=True, text=True, timeout=timeout)
check(result.returncode == 0, 'command failed: ' + ' '.join(parts[:3]))
return result.stdout.strip()
def api_tasks(manager):
output = command(
'docker', 'exec', manager, 'docker', 'service', 'ps', 'hla-app_api',
'--filter', 'desired-state=running',
'--format', '{{.ID}}\t{{.Node}}\t{{.CurrentState}}')
result = []
for line in output.splitlines():
fields = line.split('\t')
if len(fields) == 3 and fields[2].startswith('Running'):
result.append({'task_id': fields[0], 'node': fields[1]})
return result
def history(client, actor, chat_id):
path = '/api/v1/chats/' + chat_id + '/messages?limit=100'
result = []
for _ in range(1000):
page = client.call('GET', path, actor=actor, schema='MessagePage')['json']
result.extend(page['items'])
if page['next_cursor'] is None:
return result
path = ('/api/v1/chats/' + chat_id + '/messages?limit=100&cursor='
+ quote(page['next_cursor'], safe=''))
raise Failure('history pagination exceeded safety bound')
def observe(client, actor, attempts=40):
def request(_):
local = Client(client.base, timeout=2)
return local.call(
'GET', '/api/v1/users/me', actor=actor, schema='User')
with ThreadPoolExecutor(max_workers=16) as pool:
responses = list(pool.map(request, range(attempts)))
return {response['headers']['X-Instance-Id'] for response in responses}
def restore_worker(manager, node):
subprocess.run(
['docker', 'start', node], stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, timeout=30)
deadline = time.monotonic() + 90
status = ''
while time.monotonic() < deadline:
result = subprocess.run(
['docker', 'exec', manager, 'docker', 'node', 'ls',
'--filter', 'name=' + node, '--format', '{{.Status}}'],
capture_output=True, text=True, timeout=30)
status = result.stdout.strip()
if result.returncode == 0 and status == 'Ready':
return True
time.sleep(2)
return False
def main():
def configure(parser):
parser.add_argument('--manager-container', default='hla-swarm-manager')
parser.add_argument('--window', type=float, default=30)
parser.add_argument('--output', default=None)
args = arguments('Swarm worker loss and API relocation', configure)
check(15 <= args.window <= 120, 'window must be 15..120 seconds')
client = Client(args.base_url, args.ca_file, timeout=2)
alice, bob, _, chat = client.bootstrap()
before_instances = observe(client, alice)
check(len(before_instances) >= 2, 'expected two API instances before node loss')
before_tasks = api_tasks(args.manager_container)
check(len(before_tasks) == 2, 'expected two running API tasks')
candidates = [task for task in before_tasks
if task['node'].startswith('hla-swarm-worker')]
check(bool(candidates), 'no API task is running on a worker node')
victim = candidates[0]
confirmed_before = client.message(
alice, chat, 'confirmed-before-node-loss-' + uuid.uuid4().hex)['json']
started = time.monotonic()
command('docker', 'stop', '--time', '1', victim['node'], timeout=15)
restore_at_exit = lambda: restore_worker(args.manager_container, victim['node'])
atexit.register(restore_at_exit)
first_recovery = None
reads = writes = failures = 0
confirmed_during = []
key = uuid.uuid4().hex
text = 'during-node-loss-' + key
while time.monotonic() - started < args.window:
try:
if (reads + writes) % 2 == 0:
response = client.message(alice, chat, text, key=key)
confirmed_during.append(response['json'])
writes += 1
key = uuid.uuid4().hex
text = 'during-node-loss-' + key
else:
client.call('GET', '/api/v1/users/me', actor=bob, schema='User')
reads += 1
if first_recovery is None:
first_recovery = time.monotonic() - started
except Failure:
failures += 1
time.sleep(0.05)
deadline = time.monotonic() + 90
after_tasks = []
while time.monotonic() < deadline:
after_tasks = api_tasks(args.manager_container)
if (len(after_tasks) == 2
and all(task['node'] != victim['node'] for task in after_tasks)
and any(task['task_id'] != victim['task_id'] for task in after_tasks)):
break
time.sleep(2)
check(len(after_tasks) == 2, 'API service did not return to two tasks')
check(all(task['node'] != victim['node'] for task in after_tasks),
'API task was not relocated away from failed node')
check(first_recovery is not None and first_recovery <= 10,
'API did not recover within 10 seconds')
stored = history(client, bob, chat['id'])
stored_ids = {message['id'] for message in stored}
check(confirmed_before['id'] in stored_ids,
'pre-failure confirmed message was lost')
check(all(message['id'] in stored_ids for message in confirmed_during),
'message confirmed during node loss was lost')
check(restore_worker(args.manager_container, victim['node']),
'failed worker did not rejoin the Swarm')
atexit.unregister(restore_at_exit)
after_instances = observe(client, alice, attempts=96)
check(len(after_instances - before_instances) >= 1,
'replacement API process was not visible through X-Instance-Id')
elapsed = time.monotonic() - started
report = {
'scenario': 'hard loss of one Swarm worker',
'failed_node': victim['node'],
'old_task_id': victim['task_id'],
'tasks_before': before_tasks,
'tasks_after_relocation': after_tasks,
'instance_ids_before': sorted(before_instances),
'instance_ids_after': sorted(after_instances),
'successful_reads': reads,
'successful_writes': writes,
'failed_attempts': failures,
'first_recovery_seconds': round(first_recovery, 3),
'successful_rps': round((reads + writes) / args.window, 3),
'experiment_seconds_including_rejoin': round(elapsed, 3),
'confirmed_data_preserved': True,
'database_node_failed': False,
'note': ('PostgreSQL, Nginx, manager quorum and local volumes are not '
'protected by this experiment.'),
}
output = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
destination = Path(args.output)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(output + '\n', encoding='utf-8')
print(output)
if __name__ == '__main__':
run(main)