#!/usr/bin/env python3 """Hard-stop one Swarm worker and prove API task relocation and data safety.""" import json import atexit from concurrent.futures import ThreadPoolExecutor from pathlib import Path import subprocess import sys import time import uuid from urllib.parse import quote ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure, arguments, check, run def command(*parts, timeout=30): result = subprocess.run( parts, capture_output=True, text=True, timeout=timeout) check(result.returncode == 0, 'command failed: ' + ' '.join(parts[:3])) return result.stdout.strip() def api_tasks(manager): output = command( 'docker', 'exec', manager, 'docker', 'service', 'ps', 'hla-app_api', '--filter', 'desired-state=running', '--format', '{{.ID}}\t{{.Node}}\t{{.CurrentState}}') result = [] for line in output.splitlines(): fields = line.split('\t') if len(fields) == 3 and fields[2].startswith('Running'): result.append({'task_id': fields[0], 'node': fields[1]}) return result def history(client, actor, chat_id): path = '/api/v1/chats/' + chat_id + '/messages?limit=100' result = [] for _ in range(1000): page = client.call('GET', path, actor=actor, schema='MessagePage')['json'] result.extend(page['items']) if page['next_cursor'] is None: return result path = ('/api/v1/chats/' + chat_id + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='')) raise Failure('history pagination exceeded safety bound') def observe(client, actor, attempts=40): def request(_): local = Client(client.base, timeout=2) return local.call( 'GET', '/api/v1/users/me', actor=actor, schema='User') with ThreadPoolExecutor(max_workers=16) as pool: responses = list(pool.map(request, range(attempts))) return {response['headers']['X-Instance-Id'] for response in responses} def restore_worker(manager, node): subprocess.run( ['docker', 'start', node], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30) deadline = time.monotonic() + 90 status = '' while time.monotonic() < deadline: result = subprocess.run( ['docker', 'exec', manager, 'docker', 'node', 'ls', '--filter', 'name=' + node, '--format', '{{.Status}}'], capture_output=True, text=True, timeout=30) status = result.stdout.strip() if result.returncode == 0 and status == 'Ready': return True time.sleep(2) return False def main(): def configure(parser): parser.add_argument('--manager-container', default='hla-swarm-manager') parser.add_argument('--window', type=float, default=30) parser.add_argument('--output', default=None) args = arguments('Swarm worker loss and API relocation', configure) check(15 <= args.window <= 120, 'window must be 15..120 seconds') client = Client(args.base_url, args.ca_file, timeout=2) alice, bob, _, chat = client.bootstrap() before_instances = observe(client, alice) check(len(before_instances) >= 2, 'expected two API instances before node loss') before_tasks = api_tasks(args.manager_container) check(len(before_tasks) == 2, 'expected two running API tasks') candidates = [task for task in before_tasks if task['node'].startswith('hla-swarm-worker')] check(bool(candidates), 'no API task is running on a worker node') victim = candidates[0] confirmed_before = client.message( alice, chat, 'confirmed-before-node-loss-' + uuid.uuid4().hex)['json'] started = time.monotonic() command('docker', 'stop', '--time', '1', victim['node'], timeout=15) restore_at_exit = lambda: restore_worker(args.manager_container, victim['node']) atexit.register(restore_at_exit) first_recovery = None reads = writes = failures = 0 confirmed_during = [] key = uuid.uuid4().hex text = 'during-node-loss-' + key while time.monotonic() - started < args.window: try: if (reads + writes) % 2 == 0: response = client.message(alice, chat, text, key=key) confirmed_during.append(response['json']) writes += 1 key = uuid.uuid4().hex text = 'during-node-loss-' + key else: client.call('GET', '/api/v1/users/me', actor=bob, schema='User') reads += 1 if first_recovery is None: first_recovery = time.monotonic() - started except Failure: failures += 1 time.sleep(0.05) deadline = time.monotonic() + 90 after_tasks = [] while time.monotonic() < deadline: after_tasks = api_tasks(args.manager_container) if (len(after_tasks) == 2 and all(task['node'] != victim['node'] for task in after_tasks) and any(task['task_id'] != victim['task_id'] for task in after_tasks)): break time.sleep(2) check(len(after_tasks) == 2, 'API service did not return to two tasks') check(all(task['node'] != victim['node'] for task in after_tasks), 'API task was not relocated away from failed node') check(first_recovery is not None and first_recovery <= 10, 'API did not recover within 10 seconds') stored = history(client, bob, chat['id']) stored_ids = {message['id'] for message in stored} check(confirmed_before['id'] in stored_ids, 'pre-failure confirmed message was lost') check(all(message['id'] in stored_ids for message in confirmed_during), 'message confirmed during node loss was lost') check(restore_worker(args.manager_container, victim['node']), 'failed worker did not rejoin the Swarm') atexit.unregister(restore_at_exit) after_instances = observe(client, alice, attempts=96) check(len(after_instances - before_instances) >= 1, 'replacement API process was not visible through X-Instance-Id') elapsed = time.monotonic() - started report = { 'scenario': 'hard loss of one Swarm worker', 'failed_node': victim['node'], 'old_task_id': victim['task_id'], 'tasks_before': before_tasks, 'tasks_after_relocation': after_tasks, 'instance_ids_before': sorted(before_instances), 'instance_ids_after': sorted(after_instances), 'successful_reads': reads, 'successful_writes': writes, 'failed_attempts': failures, 'first_recovery_seconds': round(first_recovery, 3), 'successful_rps': round((reads + writes) / args.window, 3), 'experiment_seconds_including_rejoin': round(elapsed, 3), 'confirmed_data_preserved': True, 'database_node_failed': False, 'note': ('PostgreSQL, Nginx, manager quorum and local volumes are not ' 'protected by this experiment.'), } output = json.dumps(report, ensure_ascii=False, indent=2) if args.output: destination = Path(args.output) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(output + '\n', encoding='utf-8') print(output) if __name__ == '__main__': run(main)