forked from mtusi2026/mtusi2026lab3
90 lines
4.0 KiB
Python
90 lines
4.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Read AND write during a selected real API process failure, through LB only."""
|
|
import argparse
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
from client import Client, Failure, check
|
|
from resilience import history
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--target-hostname', required=True)
|
|
args = parser.parse_args()
|
|
c = Client(timeout=2)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
original = c.message(alice, chat, 'confirmed before selected failure')['json']
|
|
target = None
|
|
for _ in range(100):
|
|
alias = c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
|
|
if alias.startswith(args.target_hostname + '-'):
|
|
target = alias
|
|
break
|
|
check(target is not None, 'Selected target was not observed through LB')
|
|
env = dict(os.environ, TARGET_INSTANCE_ID=target)
|
|
start = time.monotonic()
|
|
proc = subprocess.Popen(['scripts/stop-one.sh'], env=env,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
records = {'read': [], 'write': []}
|
|
errors = {'read': 0, 'write': 0}
|
|
recover = {'read': None, 'write': None}
|
|
confirmed, instances = [], []
|
|
key = uuid.uuid4().hex
|
|
try:
|
|
while time.monotonic() - start < 15:
|
|
check(proc.poll() in (None, 0), 'stop-one hook failed')
|
|
for operation in ('read', 'write'):
|
|
before = time.monotonic()
|
|
try:
|
|
if operation == 'read':
|
|
r = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100',
|
|
actor=bob, schema='MessagePage')
|
|
check(original in r['json']['items'], 'Confirmed message unavailable during failure')
|
|
else:
|
|
r = c.message(alice, chat, 'during-' + key, key=key)
|
|
confirmed.append(r['json'])
|
|
key = uuid.uuid4().hex
|
|
records[operation].append(1000 * (time.monotonic() - before))
|
|
if proc.poll() == 0:
|
|
if recover[operation] is None:
|
|
recover[operation] = time.monotonic() - start
|
|
instances.append(r['headers']['X-Instance-Id'])
|
|
except Failure:
|
|
errors[operation] += 1
|
|
time.sleep(.15)
|
|
check(proc.poll() == 0, 'Hook did not complete')
|
|
check(all(v is not None and v <= 10 for v in recover.values()), 'Read/write recovery >10s')
|
|
check(target not in instances[-10:], 'Stopped target still serving')
|
|
stored = history(c, bob, chat)
|
|
check(all(m in stored for m in [original, *confirmed]), 'Confirmed data lost')
|
|
texts = [m['text'] for m in stored]
|
|
check(len(texts) == len(set(texts)), 'Uncertain retries duplicated messages')
|
|
elapsed = time.monotonic() - start
|
|
metrics = {}
|
|
for op, latencies in records.items():
|
|
ordered = sorted(latencies)
|
|
metrics[op] = {'successes': len(ordered), 'errors': errors[op],
|
|
'successful_rps': round(len(ordered) / elapsed, 3),
|
|
'first_recovery_seconds': round(recover[op], 3),
|
|
'successful_latency_ms': {label: round(ordered[max(0, math.ceil(p * len(ordered)) - 1)], 3)
|
|
for label, p in [('p50', .5), ('p95', .95), ('p99', .99)]}}
|
|
print(json.dumps({'result': 'PASS', 'target_alias': target, 'duration_seconds': round(elapsed, 3),
|
|
'metrics': metrics, 'preserved_confirmations': len(confirmed) + 1,
|
|
'duplicate_texts': 0}, indent=2))
|
|
finally:
|
|
if proc.poll() is None:
|
|
proc.terminate()
|
|
proc.wait(timeout=5)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|