Files
mtusi2026lab3/student-tests/mixed_failover.py

90 lines
4.0 KiB
Python

#!/usr/bin/env python3
"""Read AND write during a selected real API process failure, through LB only."""
import argparse
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import time
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, Failure, check
from resilience import history
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--target-hostname', required=True)
args = parser.parse_args()
c = Client(timeout=2)
alice, bob, _, chat = c.bootstrap()
original = c.message(alice, chat, 'confirmed before selected failure')['json']
target = None
for _ in range(100):
alias = c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
if alias.startswith(args.target_hostname + '-'):
target = alias
break
check(target is not None, 'Selected target was not observed through LB')
env = dict(os.environ, TARGET_INSTANCE_ID=target)
start = time.monotonic()
proc = subprocess.Popen(['scripts/stop-one.sh'], env=env,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
records = {'read': [], 'write': []}
errors = {'read': 0, 'write': 0}
recover = {'read': None, 'write': None}
confirmed, instances = [], []
key = uuid.uuid4().hex
try:
while time.monotonic() - start < 15:
check(proc.poll() in (None, 0), 'stop-one hook failed')
for operation in ('read', 'write'):
before = time.monotonic()
try:
if operation == 'read':
r = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100',
actor=bob, schema='MessagePage')
check(original in r['json']['items'], 'Confirmed message unavailable during failure')
else:
r = c.message(alice, chat, 'during-' + key, key=key)
confirmed.append(r['json'])
key = uuid.uuid4().hex
records[operation].append(1000 * (time.monotonic() - before))
if proc.poll() == 0:
if recover[operation] is None:
recover[operation] = time.monotonic() - start
instances.append(r['headers']['X-Instance-Id'])
except Failure:
errors[operation] += 1
time.sleep(.15)
check(proc.poll() == 0, 'Hook did not complete')
check(all(v is not None and v <= 10 for v in recover.values()), 'Read/write recovery >10s')
check(target not in instances[-10:], 'Stopped target still serving')
stored = history(c, bob, chat)
check(all(m in stored for m in [original, *confirmed]), 'Confirmed data lost')
texts = [m['text'] for m in stored]
check(len(texts) == len(set(texts)), 'Uncertain retries duplicated messages')
elapsed = time.monotonic() - start
metrics = {}
for op, latencies in records.items():
ordered = sorted(latencies)
metrics[op] = {'successes': len(ordered), 'errors': errors[op],
'successful_rps': round(len(ordered) / elapsed, 3),
'first_recovery_seconds': round(recover[op], 3),
'successful_latency_ms': {label: round(ordered[max(0, math.ceil(p * len(ordered)) - 1)], 3)
for label, p in [('p50', .5), ('p95', .95), ('p99', .99)]}}
print(json.dumps({'result': 'PASS', 'target_alias': target, 'duration_seconds': round(elapsed, 3),
'metrics': metrics, 'preserved_confirmations': len(confirmed) + 1,
'duplicate_texts': 0}, indent=2))
finally:
if proc.poll() is None:
proc.terminate()
proc.wait(timeout=5)
if __name__ == '__main__':
main()