Files

127 lines
6.7 KiB
Python

import argparse
from concurrent.futures import ThreadPoolExecutor
import json
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'scripts'))
from client import Client, Failure, check
from operate import containers
OUT = ROOT / 'evidence'
def run(args, name):
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding='utf-8')
(OUT / (name + '.txt')).write_text(result.stdout + result.stderr, encoding='utf-8')
if result.returncode:
raise RuntimeError(name + ': ' + result.stdout[-1000:] + result.stderr[-1000:])
print(name + ': PASS', flush=True)
def restore():
subprocess.run(['docker', 'compose', 'up', '-d', '--no-deps', '--scale', 'api=2', '--wait', 'api'], cwd=ROOT, check=True, capture_output=True)
time.sleep(4)
def snapshot():
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split()
inspected = json.loads(subprocess.check_output(['docker', 'inspect', *ids], cwd=ROOT, text=True))
return [{'id': item['Id'][:12], 'service': item['Config']['Labels']['com.docker.compose.service'], 'hostname': item['Config']['Hostname'], 'started_at': item['State']['StartedAt'], 'status': item['State']['Status'], 'cpu_nanoseconds': item['HostConfig']['NanoCpus'], 'memory_bytes': item['HostConfig']['Memory'], 'pool_size': next((value.split('=', 1)[1] for value in item['Config']['Env'] if value.startswith('DB_POOL_SIZE=')), None), 'published_ports': item['HostConfig']['PortBindings']} for item in inspected]
def matrix():
c = Client(timeout=2)
alice, bob, eve, chat = c.bootstrap()
confirmed = c.message(alice, chat, 'before matrix')['json']
rows = []
for item in containers():
restore()
target = item['Config']['Hostname']
started = time.monotonic()
errors, successes, reads, recovery, observed = 0, [], 0, None, set()
with ThreadPoolExecutor(max_workers=1) as pool:
action = pool.submit(subprocess.run, ['docker', 'kill', item['Id']], cwd=ROOT, check=True, capture_output=True)
key = os.urandom(16).hex()
while time.monotonic() - started < 12:
try:
result = c.message(alice, chat, 'matrix-' + key, key=key)
successes.append(result['json'])
observed.add(result['headers']['X-Instance-Id'])
key = os.urandom(16).hex()
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=bob, schema='MessagePage')['json']
check(confirmed in page['items'], 'confirmed data missing during failure')
reads += 1
if action.done() and recovery is None:
recovery = time.monotonic() - started
except Failure:
errors += 1
time.sleep(0.1)
action.result()
state = json.loads(subprocess.check_output(['docker', 'inspect', '--format', '{{json .State}}', item['Id']], text=True))
check(not state['Running'], 'target still running')
check(recovery is not None and recovery <= 10, 'recovery deadline exceeded')
from resilience import history
stored = history(c, bob, chat)
check(all(message in stored for message in successes), 'lost confirmed write')
texts = [message['text'] for message in stored]
check(len(texts) == len(set(texts)), 'retry duplicated data')
elapsed = time.monotonic() - started
rows.append({'target': target, 'container': item['Id'][:12], 'state': state['Status'], 'successful_writes': len(successes), 'successful_reads': reads, 'failed_attempts': errors, 'elapsed_seconds': round(elapsed, 3), 'write_rps': round(len(successes)/elapsed, 3), 'read_rps': round(reads/elapsed, 3), 'recovery_seconds': round(recovery, 3), 'observed_instances': sorted(observed), 'lost_confirmations': 0})
print('Hard failure verified: ' + target, flush=True)
restore()
(OUT / 'failover-matrix.json').write_text(json.dumps(rows, indent=2), encoding='utf-8')
def rolling():
ready_file = OUT / 'rolling-ready.json'
if ready_file.exists():
ready_file.unlink()
with ThreadPoolExecutor(max_workers=1) as pool:
load = pool.submit(run, [sys.executable, 'scripts/load_active.py', '--duration', '45', '--concurrency', '16', '--output', 'evidence/rolling-load.json', '--ready-file', str(ready_file)], 'rolling-load')
deadline = time.monotonic() + 90
while not ready_file.exists():
if load.done():
load.result()
raise RuntimeError('Load ended before measurement window started')
check(time.monotonic() < deadline, 'load setup exceeded 90 seconds')
time.sleep(0.2)
run([sys.executable, 'scripts/operate.py', 'rolling'], 'rolling-action')
load.result()
summary = json.loads((OUT / 'rolling-load.json').read_text(encoding='utf-8'))
replacements = [json.loads(line) for line in (OUT / 'rolling-action.txt').read_text(encoding='utf-8').splitlines() if line.startswith('{')]
check(len(replacements) == 2, 'rolling must replace exactly two processes')
check(all(summary['measurement_started_epoch'] <= row['started_epoch'] <= row['finished_epoch'] <= summary['measurement_finished_epoch'] for row in replacements), 'replacement happened outside active load window')
run([sys.executable, 'tests/smoke.py'], 'smoke-after-rolling')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--rolling-only', action='store_true')
args = parser.parse_args()
OUT.mkdir(exist_ok=True)
if not args.rolling_only:
run([sys.executable, 'scripts/check_contract.py'], 'contract')
run([sys.executable, 'tests/smoke.py'], 'smoke')
run([sys.executable, 'student-tests/acceptance.py'], 'concurrent')
suffix = '.cmd' if os.name == 'nt' else '.sh'
before = snapshot()
run([sys.executable, 'tests/resilience.py', '--mode', 'persistence', '--action-script', 'scripts/restart' + suffix], 'persistence')
after = snapshot()
check(all(any(new['id'] == old['id'] and new['started_at'] != old['started_at'] for new in after) for old in before), 'restart hook did not restart all running services')
(OUT / 'restart-processes.json').write_text(json.dumps({'before': before, 'after': after}, indent=2), encoding='utf-8')
restore()
run([sys.executable, 'tests/resilience.py', '--mode', 'failover', '--action-script', 'scripts/stop-one' + suffix], 'public-failover')
restore()
matrix()
rolling()
if __name__ == '__main__':
main()