forked from mtusi2026/mtusi2026lab3
127 lines
6.7 KiB
Python
127 lines
6.7 KiB
Python
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
from client import Client, Failure, check
|
|
from operate import containers
|
|
|
|
OUT = ROOT / 'evidence'
|
|
|
|
|
|
def run(args, name):
|
|
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding='utf-8')
|
|
(OUT / (name + '.txt')).write_text(result.stdout + result.stderr, encoding='utf-8')
|
|
if result.returncode:
|
|
raise RuntimeError(name + ': ' + result.stdout[-1000:] + result.stderr[-1000:])
|
|
print(name + ': PASS', flush=True)
|
|
|
|
|
|
def restore():
|
|
subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--scale', 'api=2', '--wait', 'api'], cwd=ROOT, check=True, capture_output=True)
|
|
time.sleep(4)
|
|
|
|
|
|
def snapshot():
|
|
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split()
|
|
inspected = json.loads(subprocess.check_output(['docker', 'inspect', *ids], cwd=ROOT, text=True))
|
|
return [{'id': item['Id'][:12], 'service': item['Config']['Labels']['com.docker.compose.service'], 'hostname': item['Config']['Hostname'], 'started_at': item['State']['StartedAt'], 'status': item['State']['Status'], 'cpu_nanoseconds': item['HostConfig']['NanoCpus'], 'memory_bytes': item['HostConfig']['Memory'], 'pool_size': next((value.split('=', 1)[1] for value in item['Config']['Env'] if value.startswith('DB_POOL_SIZE=')), None), 'published_ports': item['HostConfig']['PortBindings']} for item in inspected]
|
|
|
|
|
|
def matrix():
|
|
c = Client(timeout=2)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
confirmed = c.message(alice, chat, 'before matrix')['json']
|
|
rows = []
|
|
for item in containers():
|
|
restore()
|
|
target = item['Config']['Hostname']
|
|
started = time.monotonic()
|
|
errors, successes, reads, recovery, observed = 0, [], 0, None, set()
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
action = pool.submit(subprocess.run, ['docker', 'kill', item['Id']], cwd=ROOT, check=True, capture_output=True)
|
|
key = os.urandom(16).hex()
|
|
while time.monotonic() - started < 12:
|
|
try:
|
|
result = c.message(alice, chat, 'matrix-' + key, key=key)
|
|
successes.append(result['json'])
|
|
observed.add(result['headers']['X-Instance-Id'])
|
|
key = os.urandom(16).hex()
|
|
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=bob, schema='MessagePage')['json']
|
|
check(confirmed in page['items'], 'confirmed data missing during failure')
|
|
reads += 1
|
|
if action.done() and recovery is None:
|
|
recovery = time.monotonic() - started
|
|
except Failure:
|
|
errors += 1
|
|
time.sleep(0.1)
|
|
action.result()
|
|
state = json.loads(subprocess.check_output(['docker', 'inspect', '--format', '{{json .State}}', item['Id']], text=True))
|
|
check(not state['Running'], 'target still running')
|
|
check(recovery is not None and recovery <= 10, 'recovery deadline exceeded')
|
|
from resilience import history
|
|
stored = history(c, bob, chat)
|
|
check(all(message in stored for message in successes), 'lost confirmed write')
|
|
texts = [message['text'] for message in stored]
|
|
check(len(texts) == len(set(texts)), 'retry duplicated data')
|
|
elapsed = time.monotonic() - started
|
|
rows.append({'target': target, 'container': item['Id'][:12], 'state': state['Status'], 'successful_writes': len(successes), 'successful_reads': reads, 'failed_attempts': errors, 'elapsed_seconds': round(elapsed, 3), 'write_rps': round(len(successes)/elapsed, 3), 'read_rps': round(reads/elapsed, 3), 'recovery_seconds': round(recovery, 3), 'observed_instances': sorted(observed), 'lost_confirmations': 0})
|
|
print('Hard failure verified: ' + target, flush=True)
|
|
restore()
|
|
(OUT / 'failover-matrix.json').write_text(json.dumps(rows, indent=2), encoding='utf-8')
|
|
|
|
|
|
def rolling():
|
|
ready_file = OUT / 'rolling-ready.json'
|
|
if ready_file.exists():
|
|
ready_file.unlink()
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
load = pool.submit(run, [sys.executable, 'scripts/load_active.py', '--duration', '45', '--concurrency', '16', '--output', 'evidence/rolling-load.json', '--ready-file', str(ready_file)], 'rolling-load')
|
|
deadline = time.monotonic() + 90
|
|
while not ready_file.exists():
|
|
if load.done():
|
|
load.result()
|
|
raise RuntimeError('Load ended before measurement window started')
|
|
check(time.monotonic() < deadline, 'load setup exceeded 90 seconds')
|
|
time.sleep(0.2)
|
|
run([sys.executable, 'scripts/operate.py', 'rolling'], 'rolling-action')
|
|
load.result()
|
|
summary = json.loads((OUT / 'rolling-load.json').read_text(encoding='utf-8'))
|
|
replacements = [json.loads(line) for line in (OUT / 'rolling-action.txt').read_text(encoding='utf-8').splitlines() if line.startswith('{')]
|
|
check(len(replacements) == 2, 'rolling must replace exactly two processes')
|
|
check(all(summary['measurement_started_epoch'] <= row['started_epoch'] <= row['finished_epoch'] <= summary['measurement_finished_epoch'] for row in replacements), 'replacement happened outside active load window')
|
|
run([sys.executable, 'tests/smoke.py'], 'smoke-after-rolling')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--rolling-only', action='store_true')
|
|
args = parser.parse_args()
|
|
OUT.mkdir(exist_ok=True)
|
|
if not args.rolling_only:
|
|
run([sys.executable, 'scripts/check_contract.py'], 'contract')
|
|
run([sys.executable, 'tests/smoke.py'], 'smoke')
|
|
run([sys.executable, 'student-tests/acceptance.py'], 'concurrent')
|
|
suffix = '.cmd' if os.name == 'nt' else '.sh'
|
|
before = snapshot()
|
|
run([sys.executable, 'tests/resilience.py', '--mode', 'persistence', '--action-script', 'scripts/restart' + suffix], 'persistence')
|
|
after = snapshot()
|
|
check(all(any(new['id'] == old['id'] and new['started_at'] != old['started_at'] for new in after) for old in before), 'restart hook did not restart all running services')
|
|
(OUT / 'restart-processes.json').write_text(json.dumps({'before': before, 'after': after}, indent=2), encoding='utf-8')
|
|
restore()
|
|
run([sys.executable, 'tests/resilience.py', '--mode', 'failover', '--action-script', 'scripts/stop-one' + suffix], 'public-failover')
|
|
restore()
|
|
matrix()
|
|
rolling()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|