forked from mtusi2026/mtusi2026lab3
65 lines
2.9 KiB
Python
65 lines
2.9 KiB
Python
"""Stop every real API process in turn while reading/writing through LB."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def main():
|
|
args = arguments(__doc__)
|
|
c = Client(args.base_url, timeout=10)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
c.timeout = 2
|
|
observed = {c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id'] for _ in range(20)}
|
|
check(len(observed) >= 2, 'need two replicas')
|
|
rows = []
|
|
for target in sorted(observed):
|
|
before = c.message(alice, chat, 'before ' + target)['json']
|
|
env = dict(os.environ, TARGET_INSTANCE_ID=target)
|
|
started = time.monotonic()
|
|
action = subprocess.Popen([str(ROOT / 'scripts/stop-one.sh')], env=env)
|
|
successes, errors, first, confirmations = 0, 0, None, [before]
|
|
key = uuid.uuid4().hex
|
|
while time.monotonic() - started < 15:
|
|
try:
|
|
result = c.message(alice, chat, 'during ' + key, key=key)
|
|
confirmations.append(result['json'])
|
|
c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
|
successes += 1
|
|
if action.poll() == 0 and first is None:
|
|
first = time.monotonic() - started
|
|
key = uuid.uuid4().hex
|
|
except Failure:
|
|
errors += 1
|
|
time.sleep(.05)
|
|
check(action.wait(timeout=5) == 0, 'stop hook failed')
|
|
check(first is not None and first <= 10, 'recovery exceeded 10s')
|
|
from resilience import history
|
|
messages = history(c, bob, chat)
|
|
check(all(m in messages for m in confirmations), 'confirmation lost')
|
|
texts = [m['text'] for m in messages]
|
|
check(len(texts) == len(set(texts)), 'duplicate effects')
|
|
snapshots = subprocess.check_output([str(ROOT / 'scripts/compose.sh'), 'ps', '-a', '--format', 'json'], text=True)
|
|
rows.append({'target': target, 'successful_read_write_pairs': successes, 'failed_attempts': errors, 'successful_pairs_rps': round(successes/(time.monotonic()-started), 3), 'first_recovery_seconds': round(first, 3), 'containers_after_stop': snapshots})
|
|
subprocess.run([str(ROOT / 'scripts/compose.sh'), 'start', 'api'], check=True, stdout=subprocess.DEVNULL)
|
|
deadline = time.monotonic()+45
|
|
while True:
|
|
try:
|
|
seen = {c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id'] for _ in range(12)}
|
|
if observed <= seen:
|
|
break
|
|
except Failure:
|
|
pass
|
|
check(time.monotonic() < deadline, 'replica did not return')
|
|
time.sleep(.5)
|
|
(ROOT / 'evidence/replica-matrix.json').write_text(json.dumps(rows, indent=2)+'\n')
|
|
print(json.dumps(rows, indent=2))
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|