forked from mtusi2026/mtusi2026lab5
183 lines
7.2 KiB
Python
183 lines
7.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Hard-stop one Swarm worker and prove API task relocation and data safety."""
|
|
import json
|
|
import atexit
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from urllib.parse import quote
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
|
|
def command(*parts, timeout=30):
|
|
result = subprocess.run(
|
|
parts, capture_output=True, text=True, timeout=timeout)
|
|
check(result.returncode == 0, 'command failed: ' + ' '.join(parts[:3]))
|
|
return result.stdout.strip()
|
|
|
|
|
|
def api_tasks(manager):
|
|
output = command(
|
|
'docker', 'exec', manager, 'docker', 'service', 'ps', 'hla-app_api',
|
|
'--filter', 'desired-state=running',
|
|
'--format', '{{.ID}}\t{{.Node}}\t{{.CurrentState}}')
|
|
result = []
|
|
for line in output.splitlines():
|
|
fields = line.split('\t')
|
|
if len(fields) == 3 and fields[2].startswith('Running'):
|
|
result.append({'task_id': fields[0], 'node': fields[1]})
|
|
return result
|
|
|
|
|
|
def history(client, actor, chat_id):
|
|
path = '/api/v1/chats/' + chat_id + '/messages?limit=100'
|
|
result = []
|
|
for _ in range(1000):
|
|
page = client.call('GET', path, actor=actor, schema='MessagePage')['json']
|
|
result.extend(page['items'])
|
|
if page['next_cursor'] is None:
|
|
return result
|
|
path = ('/api/v1/chats/' + chat_id + '/messages?limit=100&cursor='
|
|
+ quote(page['next_cursor'], safe=''))
|
|
raise Failure('history pagination exceeded safety bound')
|
|
|
|
|
|
def observe(client, actor, attempts=40):
|
|
def request(_):
|
|
local = Client(client.base, timeout=2)
|
|
return local.call(
|
|
'GET', '/api/v1/users/me', actor=actor, schema='User')
|
|
with ThreadPoolExecutor(max_workers=16) as pool:
|
|
responses = list(pool.map(request, range(attempts)))
|
|
return {response['headers']['X-Instance-Id'] for response in responses}
|
|
|
|
|
|
def restore_worker(manager, node):
|
|
subprocess.run(
|
|
['docker', 'start', node], stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL, timeout=30)
|
|
deadline = time.monotonic() + 90
|
|
status = ''
|
|
while time.monotonic() < deadline:
|
|
result = subprocess.run(
|
|
['docker', 'exec', manager, 'docker', 'node', 'ls',
|
|
'--filter', 'name=' + node, '--format', '{{.Status}}'],
|
|
capture_output=True, text=True, timeout=30)
|
|
status = result.stdout.strip()
|
|
if result.returncode == 0 and status == 'Ready':
|
|
return True
|
|
time.sleep(2)
|
|
return False
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument('--manager-container', default='hla-swarm-manager')
|
|
parser.add_argument('--window', type=float, default=30)
|
|
parser.add_argument('--output', default=None)
|
|
args = arguments('Swarm worker loss and API relocation', configure)
|
|
check(15 <= args.window <= 120, 'window must be 15..120 seconds')
|
|
client = Client(args.base_url, args.ca_file, timeout=2)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
before_instances = observe(client, alice)
|
|
check(len(before_instances) >= 2, 'expected two API instances before node loss')
|
|
before_tasks = api_tasks(args.manager_container)
|
|
check(len(before_tasks) == 2, 'expected two running API tasks')
|
|
candidates = [task for task in before_tasks
|
|
if task['node'].startswith('hla-swarm-worker')]
|
|
check(bool(candidates), 'no API task is running on a worker node')
|
|
victim = candidates[0]
|
|
confirmed_before = client.message(
|
|
alice, chat, 'confirmed-before-node-loss-' + uuid.uuid4().hex)['json']
|
|
|
|
started = time.monotonic()
|
|
command('docker', 'stop', '--time', '1', victim['node'], timeout=15)
|
|
restore_at_exit = lambda: restore_worker(args.manager_container, victim['node'])
|
|
atexit.register(restore_at_exit)
|
|
first_recovery = None
|
|
reads = writes = failures = 0
|
|
confirmed_during = []
|
|
key = uuid.uuid4().hex
|
|
text = 'during-node-loss-' + key
|
|
while time.monotonic() - started < args.window:
|
|
try:
|
|
if (reads + writes) % 2 == 0:
|
|
response = client.message(alice, chat, text, key=key)
|
|
confirmed_during.append(response['json'])
|
|
writes += 1
|
|
key = uuid.uuid4().hex
|
|
text = 'during-node-loss-' + key
|
|
else:
|
|
client.call('GET', '/api/v1/users/me', actor=bob, schema='User')
|
|
reads += 1
|
|
if first_recovery is None:
|
|
first_recovery = time.monotonic() - started
|
|
except Failure:
|
|
failures += 1
|
|
time.sleep(0.05)
|
|
|
|
deadline = time.monotonic() + 90
|
|
after_tasks = []
|
|
while time.monotonic() < deadline:
|
|
after_tasks = api_tasks(args.manager_container)
|
|
if (len(after_tasks) == 2
|
|
and all(task['node'] != victim['node'] for task in after_tasks)
|
|
and any(task['task_id'] != victim['task_id'] for task in after_tasks)):
|
|
break
|
|
time.sleep(2)
|
|
check(len(after_tasks) == 2, 'API service did not return to two tasks')
|
|
check(all(task['node'] != victim['node'] for task in after_tasks),
|
|
'API task was not relocated away from failed node')
|
|
check(first_recovery is not None and first_recovery <= 10,
|
|
'API did not recover within 10 seconds')
|
|
|
|
stored = history(client, bob, chat['id'])
|
|
stored_ids = {message['id'] for message in stored}
|
|
check(confirmed_before['id'] in stored_ids,
|
|
'pre-failure confirmed message was lost')
|
|
check(all(message['id'] in stored_ids for message in confirmed_during),
|
|
'message confirmed during node loss was lost')
|
|
check(restore_worker(args.manager_container, victim['node']),
|
|
'failed worker did not rejoin the Swarm')
|
|
atexit.unregister(restore_at_exit)
|
|
after_instances = observe(client, alice, attempts=96)
|
|
check(len(after_instances - before_instances) >= 1,
|
|
'replacement API process was not visible through X-Instance-Id')
|
|
|
|
elapsed = time.monotonic() - started
|
|
report = {
|
|
'scenario': 'hard loss of one Swarm worker',
|
|
'failed_node': victim['node'],
|
|
'old_task_id': victim['task_id'],
|
|
'tasks_before': before_tasks,
|
|
'tasks_after_relocation': after_tasks,
|
|
'instance_ids_before': sorted(before_instances),
|
|
'instance_ids_after': sorted(after_instances),
|
|
'successful_reads': reads,
|
|
'successful_writes': writes,
|
|
'failed_attempts': failures,
|
|
'first_recovery_seconds': round(first_recovery, 3),
|
|
'successful_rps': round((reads + writes) / args.window, 3),
|
|
'experiment_seconds_including_rejoin': round(elapsed, 3),
|
|
'confirmed_data_preserved': True,
|
|
'database_node_failed': False,
|
|
'note': ('PostgreSQL, Nginx, manager quorum and local volumes are not '
|
|
'protected by this experiment.'),
|
|
}
|
|
output = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
destination = Path(args.output)
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(output + '\n', encoding='utf-8')
|
|
print(output)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|