Files

71 lines
3.2 KiB
Python

"""Read/write under actual rolling container replacement, safe retry and history."""
from concurrent.futures import ThreadPoolExecutor
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
from client import Client, Failure, arguments, check, run
from resilience import history
ROOT = Path(__file__).resolve().parents[1]
def main():
args = arguments(__doc__)
c = Client(args.base_url, timeout=10)
alice, bob, eve, chat = c.bootstrap()
before = c.message(alice, chat, 'before rolling')['json']
started = time.monotonic()
proc = subprocess.Popen([str(ROOT / 'scripts/rolling-update.sh')], cwd=ROOT,
env=dict(os.environ, BASE_URL=c.base))
def worker(index):
local = Client(c.base, timeout=2)
confirmations, failures, samples, instances = [], 0, [], set()
key = uuid.uuid4().hex
after_deadline = None
while time.monotonic() - started < 120:
if proc.poll() is not None and after_deadline is None:
after_deadline = time.monotonic() + 3
if after_deadline and time.monotonic() >= after_deadline:
break
at = time.monotonic()
try:
result = local.message(alice, chat, f'rolling-{index}-{key}', key=key)
confirmations.append(result['json'])
local.call('GET', '/api/v1/users/me', actor=alice, schema='User')
instances.add(result['headers']['X-Instance-Id'])
samples.append(round(1000*(time.monotonic()-at), 3))
key = uuid.uuid4().hex
except Failure:
failures += 1
time.sleep(.02)
return confirmations, failures, samples, instances
with ThreadPoolExecutor(max_workers=4) as pool:
results = list(pool.map(worker, range(4)))
check(proc.wait(timeout=5) == 0, 'rolling hook failed')
messages = history(c, bob, chat)
check(before in messages, 'pre-update message lost')
for confirmations, _, _, _ in results:
check(all(m in messages for m in confirmations), 'confirmed message lost')
texts = [m['text'] for m in messages]
check(len(texts) == len(set(texts)), 'retry duplicated effects')
meta = json.loads((ROOT / 'evidence/rolling-containers.json').read_text())
check({c['image_id'] for c in meta['before']} != {c['image_id'] for c in meta['after']}, 'image was not updated')
elapsed = time.monotonic() - started
latencies = sorted(x for _, _, values, _ in results for x in values)
report = {'status': 'PASS', 'concurrency': 4, 'elapsed_seconds': round(elapsed, 3),
'successful_read_write_pairs': len(latencies), 'failed_attempts': sum(r[1] for r in results),
'successful_pairs_rps': round(len(latencies)/elapsed, 3),
'successful_pair_latency_ms': {f'p{p}': latencies[min(len(latencies)-1, int(len(latencies)*p/100))] for p in (50,95,99)},
'observed_instances': sorted(set.union(*(r[3] for r in results))),
'confirmed_data_preserved': True, 'duplicates': 0}
(ROOT / 'evidence/rolling-load.json').write_text(json.dumps(report, indent=2)+'\n')
print(json.dumps(report, indent=2))
if __name__ == '__main__':
run(main)