178 lines
7.6 KiB
Python
178 lines
7.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Run from tests/ in lab3. Uses the unchanged skeleton client and resilience helpers.
|
|
Creates synthetic users/messages, stops only API containers, restores in finally.
|
|
Metrics are a small closed-loop workload, not a capacity benchmark.
|
|
"""
|
|
import csv
|
|
import json
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from client import Client, check
|
|
from resilience import history
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OUT = ROOT / 'results' / ('level4-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'))
|
|
|
|
|
|
def docker(*args):
|
|
p = subprocess.run(['docker', *args], cwd=ROOT, text=True,
|
|
capture_output=True, timeout=210)
|
|
check(p.returncode == 0, 'Docker command failed: ' + ' '.join(args))
|
|
return p.stdout.strip()
|
|
|
|
|
|
def state(cid):
|
|
return json.loads(docker('inspect', '--format', '{{json .State}}', cid))
|
|
|
|
|
|
def ready(cid):
|
|
deadline = time.monotonic() + 180
|
|
while time.monotonic() < deadline:
|
|
s = state(cid)
|
|
if s.get('Running') and s.get('Health', {}).get('Status') == 'healthy':
|
|
return
|
|
time.sleep(1)
|
|
raise RuntimeError('API did not become healthy: ' + cid[:12])
|
|
|
|
|
|
def observe(c, actor, expected):
|
|
deadline = time.monotonic() + 15
|
|
seen = set()
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
r = c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
|
seen.add(r['headers']['X-Instance-Id'])
|
|
if expected <= seen:
|
|
return
|
|
except Exception:
|
|
pass
|
|
time.sleep(.1)
|
|
raise RuntimeError('Both actual API hostnames must be observed through LB')
|
|
|
|
|
|
def main():
|
|
ids = docker('compose', 'ps', '--status', 'running', '-q', 'api').splitlines()
|
|
check(len(ids) == 2, 'Start exactly two healthy API replicas first')
|
|
names = {cid: docker('inspect', '--format', '{{.Config.Hostname}}', cid) for cid in ids}
|
|
for cid in ids:
|
|
ready(cid)
|
|
OUT.mkdir(parents=True)
|
|
summary = {'workload': '4 closed-loop workers, POST then GET, 50ms pause; HTTP timeout 2s',
|
|
'replicas': names, 'experiments': []}
|
|
c = Client(timeout=2)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
|
observe(c, alice, set(names.values()))
|
|
|
|
# One key, eight requests released together; a fresh chat isolates this test.
|
|
barrier = threading.Barrier(8)
|
|
key = uuid.uuid4().hex
|
|
def replay(_):
|
|
local = Client(timeout=10)
|
|
barrier.wait(timeout=15)
|
|
r = local.message(alice, chat, 'parallel-' + key, key=key)
|
|
return {'instance': r['headers']['X-Instance-Id'], 'message': r['json']}
|
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
replies = list(pool.map(replay, range(8)))
|
|
check(all(r['message'] == replies[0]['message'] for r in replies), 'Concurrent replies differ')
|
|
check(len({r['instance'] for r in replies}) == 2, 'Concurrency did not cover both replicas; rerun')
|
|
check(history(c, bob, chat) == [replies[0]['message']], 'Concurrent replay created duplicates')
|
|
summary['concurrency'] = {'pass': True, 'responses': replies}
|
|
print('PASS: parallel replay, two replicas, one message', flush=True)
|
|
|
|
for cid in ids:
|
|
observe(c, alice, set(names.values()))
|
|
rows, confirmed = [], {}
|
|
halt = threading.Event()
|
|
lock = threading.Lock()
|
|
start = time.monotonic()
|
|
def worker():
|
|
local = Client(timeout=2)
|
|
key = uuid.uuid4().hex
|
|
while not halt.is_set():
|
|
# Preserve the key/body after any uncertain POST result.
|
|
for method in ('POST', 'GET'):
|
|
began = time.monotonic()
|
|
row = {'method': method, 'start': began - start, 'ok': False, 'instance': '', 'error': ''}
|
|
try:
|
|
if method == 'POST':
|
|
r = local.message(alice, chat, 'load-' + key, key=key)
|
|
with lock:
|
|
confirmed[r['json']['id']] = r['json']
|
|
key = uuid.uuid4().hex
|
|
else:
|
|
r = local.call('GET', path, actor=alice, schema='MessagePage')
|
|
row.update(ok=True, instance=r['headers']['X-Instance-Id'])
|
|
except Exception as exc:
|
|
row['error'] = type(exc).__name__
|
|
row['end'] = time.monotonic() - start
|
|
with lock:
|
|
rows.append(row)
|
|
halt.wait(.05)
|
|
pool = ThreadPoolExecutor(max_workers=4)
|
|
jobs = [pool.submit(worker) for _ in range(4)]
|
|
try:
|
|
time.sleep(5)
|
|
impact = time.monotonic() - start
|
|
docker('stop', '--timeout', '20', cid)
|
|
stopped = time.monotonic() - start
|
|
stopped_state = state(cid)
|
|
check(not stopped_state['Running'], 'Target is still running')
|
|
print('Stopped ' + names[cid] + '; measuring for 15 seconds', flush=True)
|
|
time.sleep(15)
|
|
finally:
|
|
halt.set()
|
|
pool.shutdown(wait=True)
|
|
# Restore even if HTTP validation or measurement fails.
|
|
docker('start', cid)
|
|
ready(cid)
|
|
for job in jobs:
|
|
job.result()
|
|
observe(c, alice, set(names.values()))
|
|
stored = history(c, bob, chat)
|
|
by_id = {m['id']: m for m in stored}
|
|
check(all(by_id.get(k) == v for k, v in confirmed.items()), 'Confirmed message lost')
|
|
texts = [m['text'] for m in stored]
|
|
check(len(texts) == len(set(texts)), 'Uncertain retry created duplicate')
|
|
end = max(r['end'] for r in rows)
|
|
def metrics(lo, hi):
|
|
selected = [r for r in rows if lo <= r['start'] < hi]
|
|
good = sum(r['ok'] for r in selected)
|
|
return {'seconds': round(hi-lo, 3), 'attempts': len(selected), 'errors': len(selected)-good,
|
|
'attempt_rps': round(len(selected)/(hi-lo), 2),
|
|
'success_rps': round(good/(hi-lo), 2)}
|
|
recovery = {}
|
|
for method in ('POST', 'GET'):
|
|
hits = [r['end']-impact for r in rows if r['method'] == method and r['ok']
|
|
and r['start'] >= stopped and r['instance'] != names[cid]]
|
|
recovery[method] = round(min(hits), 3) if hits else None
|
|
result = {'target': names[cid], 'exit_code': stopped_state['ExitCode'],
|
|
'stop_started_seconds': impact, 'stop_finished_seconds': stopped,
|
|
'baseline': metrics(0, impact), 'during_and_after_stop': metrics(impact, end),
|
|
'first_success_after_stop_seconds_from_action_start': recovery,
|
|
'confirmed_messages_preserved': True}
|
|
summary['experiments'].append(result)
|
|
with (OUT / (names[cid] + '.csv')).open('w', newline='') as f:
|
|
w = csv.DictWriter(f, fieldnames=list(rows[0]))
|
|
w.writeheader()
|
|
w.writerows(rows)
|
|
(OUT / 'summary.json').write_text(json.dumps(summary, indent=2), encoding='utf-8')
|
|
print(json.dumps(result, indent=2), flush=True)
|
|
check(all(v is not None and v <= 10 for v in recovery.values()), 'Read/write recovery exceeded 10s')
|
|
print('PASS: replica restored, session and data preserved', flush=True)
|
|
print('Results:', OUT)
|
|
print('This checks SIGTERM failover and concurrency, NOT rolling replacement.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except Exception as exc:
|
|
print('FAIL:', str(exc))
|
|
raise SystemExit(1)
|