Files
mtusi2026lab4/tests/level4_check.py
T
2026-11-07 16:26:47 +03:00

178 lines
7.6 KiB
Python

#!/usr/bin/env python3
"""Run from tests/ in lab3. Uses the unchanged skeleton client and resilience helpers.
Creates synthetic users/messages, stops only API containers, restores in finally.
Metrics are a small closed-loop workload, not a capacity benchmark.
"""
import csv
import json
import subprocess
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from client import Client, check
from resilience import history
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / 'results' / ('level4-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'))
def docker(*args):
p = subprocess.run(['docker', *args], cwd=ROOT, text=True,
capture_output=True, timeout=210)
check(p.returncode == 0, 'Docker command failed: ' + ' '.join(args))
return p.stdout.strip()
def state(cid):
return json.loads(docker('inspect', '--format', '{{json .State}}', cid))
def ready(cid):
deadline = time.monotonic() + 180
while time.monotonic() < deadline:
s = state(cid)
if s.get('Running') and s.get('Health', {}).get('Status') == 'healthy':
return
time.sleep(1)
raise RuntimeError('API did not become healthy: ' + cid[:12])
def observe(c, actor, expected):
deadline = time.monotonic() + 15
seen = set()
while time.monotonic() < deadline:
try:
r = c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
seen.add(r['headers']['X-Instance-Id'])
if expected <= seen:
return
except Exception:
pass
time.sleep(.1)
raise RuntimeError('Both actual API hostnames must be observed through LB')
def main():
ids = docker('compose', 'ps', '--status', 'running', '-q', 'api').splitlines()
check(len(ids) == 2, 'Start exactly two healthy API replicas first')
names = {cid: docker('inspect', '--format', '{{.Config.Hostname}}', cid) for cid in ids}
for cid in ids:
ready(cid)
OUT.mkdir(parents=True)
summary = {'workload': '4 closed-loop workers, POST then GET, 50ms pause; HTTP timeout 2s',
'replicas': names, 'experiments': []}
c = Client(timeout=2)
alice, bob, _, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
observe(c, alice, set(names.values()))
# One key, eight requests released together; a fresh chat isolates this test.
barrier = threading.Barrier(8)
key = uuid.uuid4().hex
def replay(_):
local = Client(timeout=10)
barrier.wait(timeout=15)
r = local.message(alice, chat, 'parallel-' + key, key=key)
return {'instance': r['headers']['X-Instance-Id'], 'message': r['json']}
with ThreadPoolExecutor(max_workers=8) as pool:
replies = list(pool.map(replay, range(8)))
check(all(r['message'] == replies[0]['message'] for r in replies), 'Concurrent replies differ')
check(len({r['instance'] for r in replies}) == 2, 'Concurrency did not cover both replicas; rerun')
check(history(c, bob, chat) == [replies[0]['message']], 'Concurrent replay created duplicates')
summary['concurrency'] = {'pass': True, 'responses': replies}
print('PASS: parallel replay, two replicas, one message', flush=True)
for cid in ids:
observe(c, alice, set(names.values()))
rows, confirmed = [], {}
halt = threading.Event()
lock = threading.Lock()
start = time.monotonic()
def worker():
local = Client(timeout=2)
key = uuid.uuid4().hex
while not halt.is_set():
# Preserve the key/body after any uncertain POST result.
for method in ('POST', 'GET'):
began = time.monotonic()
row = {'method': method, 'start': began - start, 'ok': False, 'instance': '', 'error': ''}
try:
if method == 'POST':
r = local.message(alice, chat, 'load-' + key, key=key)
with lock:
confirmed[r['json']['id']] = r['json']
key = uuid.uuid4().hex
else:
r = local.call('GET', path, actor=alice, schema='MessagePage')
row.update(ok=True, instance=r['headers']['X-Instance-Id'])
except Exception as exc:
row['error'] = type(exc).__name__
row['end'] = time.monotonic() - start
with lock:
rows.append(row)
halt.wait(.05)
pool = ThreadPoolExecutor(max_workers=4)
jobs = [pool.submit(worker) for _ in range(4)]
try:
time.sleep(5)
impact = time.monotonic() - start
docker('stop', '--timeout', '20', cid)
stopped = time.monotonic() - start
stopped_state = state(cid)
check(not stopped_state['Running'], 'Target is still running')
print('Stopped ' + names[cid] + '; measuring for 15 seconds', flush=True)
time.sleep(15)
finally:
halt.set()
pool.shutdown(wait=True)
# Restore even if HTTP validation or measurement fails.
docker('start', cid)
ready(cid)
for job in jobs:
job.result()
observe(c, alice, set(names.values()))
stored = history(c, bob, chat)
by_id = {m['id']: m for m in stored}
check(all(by_id.get(k) == v for k, v in confirmed.items()), 'Confirmed message lost')
texts = [m['text'] for m in stored]
check(len(texts) == len(set(texts)), 'Uncertain retry created duplicate')
end = max(r['end'] for r in rows)
def metrics(lo, hi):
selected = [r for r in rows if lo <= r['start'] < hi]
good = sum(r['ok'] for r in selected)
return {'seconds': round(hi-lo, 3), 'attempts': len(selected), 'errors': len(selected)-good,
'attempt_rps': round(len(selected)/(hi-lo), 2),
'success_rps': round(good/(hi-lo), 2)}
recovery = {}
for method in ('POST', 'GET'):
hits = [r['end']-impact for r in rows if r['method'] == method and r['ok']
and r['start'] >= stopped and r['instance'] != names[cid]]
recovery[method] = round(min(hits), 3) if hits else None
result = {'target': names[cid], 'exit_code': stopped_state['ExitCode'],
'stop_started_seconds': impact, 'stop_finished_seconds': stopped,
'baseline': metrics(0, impact), 'during_and_after_stop': metrics(impact, end),
'first_success_after_stop_seconds_from_action_start': recovery,
'confirmed_messages_preserved': True}
summary['experiments'].append(result)
with (OUT / (names[cid] + '.csv')).open('w', newline='') as f:
w = csv.DictWriter(f, fieldnames=list(rows[0]))
w.writeheader()
w.writerows(rows)
(OUT / 'summary.json').write_text(json.dumps(summary, indent=2), encoding='utf-8')
print(json.dumps(result, indent=2), flush=True)
check(all(v is not None and v <= 10 for v in recovery.values()), 'Read/write recovery exceeded 10s')
print('PASS: replica restored, session and data preserved', flush=True)
print('Results:', OUT)
print('This checks SIGTERM failover and concurrency, NOT rolling replacement.')
if __name__ == '__main__':
try:
main()
except Exception as exc:
print('FAIL:', str(exc))
raise SystemExit(1)