Files
mtusi2026lab3-fork/student-tests/failures.py
T

115 lines
4.8 KiB
Python

"""Sequential real-container failures and rolling replacement under read/write load."""
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import statistics
import subprocess
import sys
import threading
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT/'tests'), str(ROOT/'scripts')]
from client import Client, Failure, check
from resilience import history
from manage import docker, ready, replicas
def exercise(label, action):
c = Client(timeout=2)
alice, bob, _, chat = c.bootstrap()
original = c.message(alice, chat, 'Before '+label)['json']
start = time.monotonic()
stop = threading.Event()
confirmations, latencies, successes, error_times = [], [], [], []
lock = threading.Lock()
def worker():
local = Client(timeout=2)
key = uuid.uuid4().hex
while not stop.is_set():
began = time.monotonic()
try:
message = local.message(alice, chat, key, key=key)['json']
local.call('GET', '/api/v1/chats/'+chat['id'], actor=bob, schema='Chat')
with lock:
confirmations.append(message)
successes.append(time.monotonic()-start)
latencies.append((time.monotonic()-began)*1000)
key = uuid.uuid4().hex
except Failure:
with lock:
error_times.append(time.monotonic()-start)
stop.wait(.03)
before = replicas()
with ThreadPoolExecutor(max_workers=4) as pool:
workers = [pool.submit(worker) for _ in range(4)]
time.sleep(2)
action_start = time.monotonic()-start
try:
action()
action_end = time.monotonic()-start
time.sleep(4)
finally:
stop.set()
for worker_result in workers:
worker_result.result()
elapsed = time.monotonic()-start
after = replicas()
stored = history(c, bob, chat)
by_id = {m['id']:m for m in stored}
check(by_id.get(original['id'])==original, 'pre-failure confirmation lost')
check(all(by_id.get(m['id'])==m for m in confirmations), 'confirmation lost')
check(len({m['text'] for m in stored})==len(stored), 'uncertain retry duplicated message')
post = [t for t in successes if t>=action_end]
check(bool(post), 'no reads/writes after action')
recovery = min(post)-action_start
# The total rolling action spans two replacements; each stop is checked separately.
if label != 'rolling':
check(recovery<=10, 'recovery exceeded 10 seconds')
times = sorted(successes)
gaps = [b-a for a,b in zip(times,times[1:])]
report = {'scenario':label, 'before':before, 'after':after,
'successful_read_write_cycles':len(successes), 'failed_cycles':len(error_times),
'elapsed_seconds':round(elapsed,3), 'successful_cycles_per_second':round(len(successes)/elapsed,3),
'confirmed_pair_http_rps':round(2*len(successes)/elapsed,3),
'first_success_after_action_seconds':round(recovery,3),
'max_success_gap_seconds':round(max(gaps,default=0),3),
'cycle_latency_ms':{'p50':statistics.median(latencies), 'p95':sorted(latencies)[int(.95*(len(latencies)-1))]},
'all_confirmations_preserved':True, 'duplicate_messages':0}
print(json.dumps(report))
return report
def main():
ready(2)
results = []
original = replicas()
check(len(original)==2, 'expected two API containers')
try:
for selected in original:
def action(selected=selected):
docker('stop', '--time', '9', selected['id'])
state = docker('inspect', '--format', '{{.State.Status}} {{.State.ExitCode}}', selected['id'], capture=True)
check(state=='exited 0', 'target did not shut down cleanly')
result = exercise('stop '+selected['instance'], action)
check(len(result['after'])==1 and result['after'][0]['id']!=selected['id'], 'wrong container stopped')
results.append(result)
docker('compose', 'start', 'api')
ready(2)
def rolling():
subprocess.run([sys.executable, '-X', 'utf8', 'scripts/manage.py', 'rolling'], cwd=ROOT, check=True)
results.append(exercise('rolling', rolling))
check(not {r['id'] for r in results[-1]['before']} & {r['id'] for r in results[-1]['after']}, 'rolling did not replace both containers')
(ROOT/'evidence/failures.json').write_text(json.dumps(results,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
finally:
docker('compose', 'up', '-d', '--no-deps', '--scale', 'api=2', 'api')
ready(2)
print('PASS: each replica stopped, real containers verified, rolling replacement preserved sessions/data')
if __name__=='__main__':
main()