forked from mtusi2026/mtusi2026lab3
create tasks from 5
This commit is contained in:
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Hard-stop one Swarm worker and prove API task relocation and data safety."""
|
||||
import json
|
||||
import atexit
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def command(*parts, timeout=30):
|
||||
result = subprocess.run(
|
||||
parts, capture_output=True, text=True, timeout=timeout)
|
||||
check(result.returncode == 0, 'command failed: ' + ' '.join(parts[:3]))
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def api_tasks(manager):
|
||||
output = command(
|
||||
'docker', 'exec', manager, 'docker', 'service', 'ps', 'hla-app_api',
|
||||
'--filter', 'desired-state=running',
|
||||
'--format', '{{.ID}}\t{{.Node}}\t{{.CurrentState}}')
|
||||
result = []
|
||||
for line in output.splitlines():
|
||||
fields = line.split('\t')
|
||||
if len(fields) == 3 and fields[2].startswith('Running'):
|
||||
result.append({'task_id': fields[0], 'node': fields[1]})
|
||||
return result
|
||||
|
||||
|
||||
def history(client, actor, chat_id):
|
||||
path = '/api/v1/chats/' + chat_id + '/messages?limit=100'
|
||||
result = []
|
||||
for _ in range(1000):
|
||||
page = client.call('GET', path, actor=actor, schema='MessagePage')['json']
|
||||
result.extend(page['items'])
|
||||
if page['next_cursor'] is None:
|
||||
return result
|
||||
path = ('/api/v1/chats/' + chat_id + '/messages?limit=100&cursor='
|
||||
+ quote(page['next_cursor'], safe=''))
|
||||
raise Failure('history pagination exceeded safety bound')
|
||||
|
||||
|
||||
def observe(client, actor, attempts=40):
|
||||
def request(_):
|
||||
local = Client(client.base, timeout=2)
|
||||
return local.call(
|
||||
'GET', '/api/v1/users/me', actor=actor, schema='User')
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
responses = list(pool.map(request, range(attempts)))
|
||||
return {response['headers']['X-Instance-Id'] for response in responses}
|
||||
|
||||
|
||||
def restore_worker(manager, node):
|
||||
subprocess.run(
|
||||
['docker', 'start', node], stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, timeout=30)
|
||||
deadline = time.monotonic() + 90
|
||||
status = ''
|
||||
while time.monotonic() < deadline:
|
||||
result = subprocess.run(
|
||||
['docker', 'exec', manager, 'docker', 'node', 'ls',
|
||||
'--filter', 'name=' + node, '--format', '{{.Status}}'],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
status = result.stdout.strip()
|
||||
if result.returncode == 0 and status == 'Ready':
|
||||
return True
|
||||
time.sleep(2)
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument('--manager-container', default='hla-swarm-manager')
|
||||
parser.add_argument('--window', type=float, default=30)
|
||||
parser.add_argument('--output', default=None)
|
||||
args = arguments('Swarm worker loss and API relocation', configure)
|
||||
check(15 <= args.window <= 120, 'window must be 15..120 seconds')
|
||||
client = Client(args.base_url, args.ca_file, timeout=2)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
before_instances = observe(client, alice)
|
||||
check(len(before_instances) >= 2, 'expected two API instances before node loss')
|
||||
before_tasks = api_tasks(args.manager_container)
|
||||
check(len(before_tasks) == 2, 'expected two running API tasks')
|
||||
candidates = [task for task in before_tasks
|
||||
if task['node'].startswith('hla-swarm-worker')]
|
||||
check(bool(candidates), 'no API task is running on a worker node')
|
||||
victim = candidates[0]
|
||||
confirmed_before = client.message(
|
||||
alice, chat, 'confirmed-before-node-loss-' + uuid.uuid4().hex)['json']
|
||||
|
||||
started = time.monotonic()
|
||||
command('docker', 'stop', '--time', '1', victim['node'], timeout=15)
|
||||
restore_at_exit = lambda: restore_worker(args.manager_container, victim['node'])
|
||||
atexit.register(restore_at_exit)
|
||||
first_recovery = None
|
||||
reads = writes = failures = 0
|
||||
confirmed_during = []
|
||||
key = uuid.uuid4().hex
|
||||
text = 'during-node-loss-' + key
|
||||
while time.monotonic() - started < args.window:
|
||||
try:
|
||||
if (reads + writes) % 2 == 0:
|
||||
response = client.message(alice, chat, text, key=key)
|
||||
confirmed_during.append(response['json'])
|
||||
writes += 1
|
||||
key = uuid.uuid4().hex
|
||||
text = 'during-node-loss-' + key
|
||||
else:
|
||||
client.call('GET', '/api/v1/users/me', actor=bob, schema='User')
|
||||
reads += 1
|
||||
if first_recovery is None:
|
||||
first_recovery = time.monotonic() - started
|
||||
except Failure:
|
||||
failures += 1
|
||||
time.sleep(0.05)
|
||||
|
||||
deadline = time.monotonic() + 90
|
||||
after_tasks = []
|
||||
while time.monotonic() < deadline:
|
||||
after_tasks = api_tasks(args.manager_container)
|
||||
if (len(after_tasks) == 2
|
||||
and all(task['node'] != victim['node'] for task in after_tasks)
|
||||
and any(task['task_id'] != victim['task_id'] for task in after_tasks)):
|
||||
break
|
||||
time.sleep(2)
|
||||
check(len(after_tasks) == 2, 'API service did not return to two tasks')
|
||||
check(all(task['node'] != victim['node'] for task in after_tasks),
|
||||
'API task was not relocated away from failed node')
|
||||
check(first_recovery is not None and first_recovery <= 10,
|
||||
'API did not recover within 10 seconds')
|
||||
|
||||
stored = history(client, bob, chat['id'])
|
||||
stored_ids = {message['id'] for message in stored}
|
||||
check(confirmed_before['id'] in stored_ids,
|
||||
'pre-failure confirmed message was lost')
|
||||
check(all(message['id'] in stored_ids for message in confirmed_during),
|
||||
'message confirmed during node loss was lost')
|
||||
check(restore_worker(args.manager_container, victim['node']),
|
||||
'failed worker did not rejoin the Swarm')
|
||||
atexit.unregister(restore_at_exit)
|
||||
after_instances = observe(client, alice, attempts=96)
|
||||
check(len(after_instances - before_instances) >= 1,
|
||||
'replacement API process was not visible through X-Instance-Id')
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
report = {
|
||||
'scenario': 'hard loss of one Swarm worker',
|
||||
'failed_node': victim['node'],
|
||||
'old_task_id': victim['task_id'],
|
||||
'tasks_before': before_tasks,
|
||||
'tasks_after_relocation': after_tasks,
|
||||
'instance_ids_before': sorted(before_instances),
|
||||
'instance_ids_after': sorted(after_instances),
|
||||
'successful_reads': reads,
|
||||
'successful_writes': writes,
|
||||
'failed_attempts': failures,
|
||||
'first_recovery_seconds': round(first_recovery, 3),
|
||||
'successful_rps': round((reads + writes) / args.window, 3),
|
||||
'experiment_seconds_including_rejoin': round(elapsed, 3),
|
||||
'confirmed_data_preserved': True,
|
||||
'database_node_failed': False,
|
||||
'note': ('PostgreSQL, Nginx, manager quorum and local volumes are not '
|
||||
'protected by this experiment.'),
|
||||
}
|
||||
output = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
destination = Path(args.output)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(output + '\n', encoding='utf-8')
|
||||
print(output)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user