Files

65 lines
2.9 KiB
Python

"""Stop every real API process in turn while reading/writing through LB."""
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
from client import Client, Failure, arguments, check, run
ROOT = Path(__file__).resolve().parents[1]
def main():
args = arguments(__doc__)
c = Client(args.base_url, timeout=10)
alice, bob, eve, chat = c.bootstrap()
c.timeout = 2
observed = {c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id'] for _ in range(20)}
check(len(observed) >= 2, 'need two replicas')
rows = []
for target in sorted(observed):
before = c.message(alice, chat, 'before ' + target)['json']
env = dict(os.environ, TARGET_INSTANCE_ID=target)
started = time.monotonic()
action = subprocess.Popen([str(ROOT / 'scripts/stop-one.sh')], env=env)
successes, errors, first, confirmations = 0, 0, None, [before]
key = uuid.uuid4().hex
while time.monotonic() - started < 15:
try:
result = c.message(alice, chat, 'during ' + key, key=key)
confirmations.append(result['json'])
c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
successes += 1
if action.poll() == 0 and first is None:
first = time.monotonic() - started
key = uuid.uuid4().hex
except Failure:
errors += 1
time.sleep(.05)
check(action.wait(timeout=5) == 0, 'stop hook failed')
check(first is not None and first <= 10, 'recovery exceeded 10s')
from resilience import history
messages = history(c, bob, chat)
check(all(m in messages for m in confirmations), 'confirmation lost')
texts = [m['text'] for m in messages]
check(len(texts) == len(set(texts)), 'duplicate effects')
snapshots = subprocess.check_output([str(ROOT / 'scripts/compose.sh'), 'ps', '-a', '--format', 'json'], text=True)
rows.append({'target': target, 'successful_read_write_pairs': successes, 'failed_attempts': errors, 'successful_pairs_rps': round(successes/(time.monotonic()-started), 3), 'first_recovery_seconds': round(first, 3), 'containers_after_stop': snapshots})
subprocess.run([str(ROOT / 'scripts/compose.sh'), 'start', 'api'], check=True, stdout=subprocess.DEVNULL)
deadline = time.monotonic()+45
while True:
try:
seen = {c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id'] for _ in range(12)}
if observed <= seen:
break
except Failure:
pass
check(time.monotonic() < deadline, 'replica did not return')
time.sleep(.5)
(ROOT / 'evidence/replica-matrix.json').write_text(json.dumps(rows, indent=2)+'\n')
print(json.dumps(rows, indent=2))
if __name__ == '__main__':
run(main)