115 lines
4.8 KiB
Python
115 lines
4.8 KiB
Python
"""Sequential real-container failures and rolling replacement under read/write load."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
from pathlib import Path
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path[:0] = [str(ROOT/'tests'), str(ROOT/'scripts')]
|
|
from client import Client, Failure, check
|
|
from resilience import history
|
|
from manage import docker, ready, replicas
|
|
|
|
|
|
def exercise(label, action):
|
|
c = Client(timeout=2)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
original = c.message(alice, chat, 'Before '+label)['json']
|
|
start = time.monotonic()
|
|
stop = threading.Event()
|
|
confirmations, latencies, successes, error_times = [], [], [], []
|
|
lock = threading.Lock()
|
|
|
|
def worker():
|
|
local = Client(timeout=2)
|
|
key = uuid.uuid4().hex
|
|
while not stop.is_set():
|
|
began = time.monotonic()
|
|
try:
|
|
message = local.message(alice, chat, key, key=key)['json']
|
|
local.call('GET', '/api/v1/chats/'+chat['id'], actor=bob, schema='Chat')
|
|
with lock:
|
|
confirmations.append(message)
|
|
successes.append(time.monotonic()-start)
|
|
latencies.append((time.monotonic()-began)*1000)
|
|
key = uuid.uuid4().hex
|
|
except Failure:
|
|
with lock:
|
|
error_times.append(time.monotonic()-start)
|
|
stop.wait(.03)
|
|
|
|
before = replicas()
|
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
|
workers = [pool.submit(worker) for _ in range(4)]
|
|
time.sleep(2)
|
|
action_start = time.monotonic()-start
|
|
try:
|
|
action()
|
|
action_end = time.monotonic()-start
|
|
time.sleep(4)
|
|
finally:
|
|
stop.set()
|
|
for worker_result in workers:
|
|
worker_result.result()
|
|
elapsed = time.monotonic()-start
|
|
after = replicas()
|
|
stored = history(c, bob, chat)
|
|
by_id = {m['id']:m for m in stored}
|
|
check(by_id.get(original['id'])==original, 'pre-failure confirmation lost')
|
|
check(all(by_id.get(m['id'])==m for m in confirmations), 'confirmation lost')
|
|
check(len({m['text'] for m in stored})==len(stored), 'uncertain retry duplicated message')
|
|
post = [t for t in successes if t>=action_end]
|
|
check(bool(post), 'no reads/writes after action')
|
|
recovery = min(post)-action_start
|
|
# The total rolling action spans two replacements; each stop is checked separately.
|
|
if label != 'rolling':
|
|
check(recovery<=10, 'recovery exceeded 10 seconds')
|
|
times = sorted(successes)
|
|
gaps = [b-a for a,b in zip(times,times[1:])]
|
|
report = {'scenario':label, 'before':before, 'after':after,
|
|
'successful_read_write_cycles':len(successes), 'failed_cycles':len(error_times),
|
|
'elapsed_seconds':round(elapsed,3), 'successful_cycles_per_second':round(len(successes)/elapsed,3),
|
|
'confirmed_pair_http_rps':round(2*len(successes)/elapsed,3),
|
|
'first_success_after_action_seconds':round(recovery,3),
|
|
'max_success_gap_seconds':round(max(gaps,default=0),3),
|
|
'cycle_latency_ms':{'p50':statistics.median(latencies), 'p95':sorted(latencies)[int(.95*(len(latencies)-1))]},
|
|
'all_confirmations_preserved':True, 'duplicate_messages':0}
|
|
print(json.dumps(report))
|
|
return report
|
|
|
|
|
|
def main():
|
|
ready(2)
|
|
results = []
|
|
original = replicas()
|
|
check(len(original)==2, 'expected two API containers')
|
|
try:
|
|
for selected in original:
|
|
def action(selected=selected):
|
|
docker('stop', '--time', '9', selected['id'])
|
|
state = docker('inspect', '--format', '{{.State.Status}} {{.State.ExitCode}}', selected['id'], capture=True)
|
|
check(state=='exited 0', 'target did not shut down cleanly')
|
|
result = exercise('stop '+selected['instance'], action)
|
|
check(len(result['after'])==1 and result['after'][0]['id']!=selected['id'], 'wrong container stopped')
|
|
results.append(result)
|
|
docker('compose', 'start', 'api')
|
|
ready(2)
|
|
def rolling():
|
|
subprocess.run([sys.executable, '-X', 'utf8', 'scripts/manage.py', 'rolling'], cwd=ROOT, check=True)
|
|
results.append(exercise('rolling', rolling))
|
|
check(not {r['id'] for r in results[-1]['before']} & {r['id'] for r in results[-1]['after']}, 'rolling did not replace both containers')
|
|
(ROOT/'evidence/failures.json').write_text(json.dumps(results,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
|
finally:
|
|
docker('compose', 'up', '-d', '--no-deps', '--scale', 'api=2', 'api')
|
|
ready(2)
|
|
print('PASS: each replica stopped, real containers verified, rolling replacement preserved sessions/data')
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|