200 lines
9.8 KiB
Python
200 lines
9.8 KiB
Python
"""Repeatable grade-4 experiments. Run on host with Docker access and UTF-8 mode."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from uuid import uuid4
|
|
|
|
from ops import ROOT, compose, restore, rolling, save, snapshot, stop_instance, wait_api
|
|
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, check
|
|
from resilience import history
|
|
|
|
|
|
def command(name, args):
|
|
result = subprocess.run(args, cwd=ROOT, text=True, encoding='utf-8',
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=180)
|
|
(ROOT / 'evidence' / name).write_text(result.stdout, encoding='utf-8')
|
|
print(result.stdout, flush=True)
|
|
check(result.returncode == 0, 'Command failed: ' + name)
|
|
|
|
|
|
def concurrency(base):
|
|
c = Client(base)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
|
key = uuid4().hex
|
|
barrier = threading.Barrier(8)
|
|
|
|
def send(_):
|
|
client = Client(base)
|
|
barrier.wait(timeout=10)
|
|
return client.message(alice, chat, 'one concurrent message', key=key)
|
|
|
|
responses = []
|
|
with ThreadPoolExecutor(max_workers=8) as executor:
|
|
for _ in range(3):
|
|
responses.extend(executor.map(send, range(8)))
|
|
first = responses[0]['json']
|
|
check(all(r['json'] == first and r['status'] == 201 for r in responses), 'Concurrent replies differ')
|
|
instances = sorted({r['headers']['X-Instance-Id'] for r in responses})
|
|
check(len(instances) >= 2, 'Concurrent test did not reach both replicas')
|
|
check(history(c, alice, chat) == [first], 'Concurrent retry duplicated message')
|
|
c.call('POST', path, 409, actor=alice, body={'text': 'different'}, headers={'Idempotency-Key': key})
|
|
c.call('POST', path, 401, body={'text': 'one concurrent message'}, headers={'Idempotency-Key': key})
|
|
c.call('POST', path, 404, actor=eve, body={'text': 'one concurrent message'}, headers={'Idempotency-Key': key})
|
|
for invalid in (None, '', 'bad key', 'x' * 129):
|
|
c.call('POST', path, 400, actor=alice, body={'text': 'invalid key'},
|
|
headers={} if invalid is None else {'Idempotency-Key': invalid})
|
|
other = c.message(bob, chat, 'independent user', key=key)['json']
|
|
check(other['id'] != first['id'], 'Key scope must include actor')
|
|
chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice,
|
|
body={'title': 'Scope', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
|
other = c.message(alice, chat2, 'independent chat', key=key)['json']
|
|
check(other['id'] != first['id'], 'Key scope must include chat')
|
|
save('idempotency.json', {'requests': 24, 'concurrency': 8, 'instances': instances,
|
|
'unique_messages': 1, 'same_response': True,
|
|
'conflict_auth_access_key_scope': 'PASS'})
|
|
return c, alice, chat, key, first
|
|
|
|
|
|
def experiment(base, name, action, target=None):
|
|
c = Client(base, timeout=9)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
confirmed_before = c.message(alice, chat, 'before ' + name)['json']
|
|
events = []
|
|
stop = threading.Event()
|
|
started = time.monotonic()
|
|
|
|
def worker():
|
|
client = Client(base, timeout=9)
|
|
key = uuid4().hex
|
|
while not stop.is_set():
|
|
begin = time.monotonic()
|
|
try:
|
|
response = client.message(alice, chat, key, key=key)
|
|
msg = response['json']
|
|
events.append({'kind': 'write', 'time': time.monotonic() - started,
|
|
'latency': response['elapsed'], 'instance': response['headers']['X-Instance-Id'],
|
|
'message': msg})
|
|
key = uuid4().hex
|
|
except Failure as exc:
|
|
events.append({'kind': 'error', 'operation': 'write', 'time': time.monotonic() - started,
|
|
'latency': time.monotonic() - begin, 'error': str(exc)})
|
|
continue # Keep original key/body after an uncertain write.
|
|
begin = time.monotonic()
|
|
try:
|
|
response = client.call('GET', '/api/v1/chats/' + chat['id'], actor=bob, schema='Chat')
|
|
events.append({'kind': 'read', 'time': time.monotonic() - started,
|
|
'latency': response['elapsed'], 'instance': response['headers']['X-Instance-Id']})
|
|
except Failure as exc:
|
|
events.append({'kind': 'error', 'operation': 'read', 'time': time.monotonic() - started,
|
|
'latency': time.monotonic() - begin, 'error': str(exc)})
|
|
time.sleep(0.02)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
futures = [executor.submit(worker) for _ in range(2)]
|
|
try:
|
|
time.sleep(3)
|
|
action_started = time.monotonic() - started
|
|
effect = action()
|
|
action_finished = time.monotonic() - started
|
|
time.sleep(max(5, 20 - (time.monotonic() - started)))
|
|
finally:
|
|
stop.set()
|
|
for future in futures:
|
|
future.result()
|
|
duration = time.monotonic() - started
|
|
successes = [e for e in events if e['kind'] != 'error']
|
|
errors = [e for e in events if e['kind'] == 'error']
|
|
writes = [e for e in successes if e['kind'] == 'write']
|
|
recovery = {}
|
|
for kind in ('read', 'write'):
|
|
after = [e for e in successes if e['kind'] == kind and e['time'] >= action_finished]
|
|
check(bool(after), 'No ' + kind + ' after action')
|
|
recovery[kind] = round(min(e['time'] for e in after) - action_started, 3)
|
|
if target:
|
|
check(all(e['instance'] != target for e in after), 'Stopped target still serves')
|
|
check(recovery[kind] <= 10, 'Recovery exceeded 10 seconds')
|
|
messages = history(c, bob, chat)
|
|
by_id = {m['id']: m for m in messages}
|
|
check(by_id.get(confirmed_before['id']) == confirmed_before, 'Pre-failure message lost')
|
|
check(all(by_id.get(e['message']['id']) == e['message'] for e in writes), 'Confirmed write lost')
|
|
check(len({m['text'] for m in messages}) == len(messages), 'Retry duplicated a message')
|
|
# Retry a pre-failure key through the surviving/new processes.
|
|
if writes:
|
|
item = writes[0]['message']
|
|
check(c.message(alice, chat, item['text'], key=item['text'])['json'] == item, 'Replay changed after action')
|
|
latencies = sorted(e['latency'] for e in successes)
|
|
gaps = {}
|
|
for kind in ('read', 'write'):
|
|
times = sorted(e['time'] for e in successes if e['kind'] == kind)
|
|
gaps[kind] = round(max((b - a for a, b in zip(times, times[1:])), default=0), 3)
|
|
summary = {'duration_seconds': round(duration, 3), 'concurrency': 2,
|
|
'successful_http_rps': round(len(successes) / duration, 2),
|
|
'successful_write_rps': round(len(writes) / duration, 2),
|
|
'successful_requests': len(successes), 'errors': len(errors),
|
|
'action_started_seconds': round(action_started, 3),
|
|
'action_finished_seconds': round(action_finished, 3),
|
|
'first_read_write_after_action_seconds_from_start': recovery,
|
|
'max_success_gap_seconds': gaps,
|
|
'latency_ms': {str(p): round(latencies[min(int(len(latencies)*p/100), len(latencies)-1)]*1000, 2)
|
|
for p in (50, 95, 99)},
|
|
'confirmed_messages_preserved': True, 'duplicate_messages': 0,
|
|
'instances': sorted({e['instance'] for e in successes}), 'target': target}
|
|
save(name + '.json', {'summary': summary, 'containers': effect, 'errors': errors})
|
|
print(name + ': ' + json.dumps(summary), flush=True)
|
|
return summary
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
|
args = parser.parse_args()
|
|
os.environ['PYTHONUTF8'] = '1'
|
|
base = args.base_url
|
|
wait_api()
|
|
command('check.txt', [sys.executable, '-X', 'utf8', 'scripts/check_contract.py'])
|
|
command('final-smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', base])
|
|
c, actor, chat, key, original = concurrency(base)
|
|
hook = 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh'
|
|
command('persistence.txt', [sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'persistence',
|
|
'--base-url', base, '--action-script', hook])
|
|
wait_api()
|
|
check(c.message(actor, chat, original['text'], key=key)['json'] == original,
|
|
'Idempotency/session not preserved across full restart')
|
|
save('idempotency-restart.json', {'same_token': True, 'same_201_message': True})
|
|
hook = 'scripts/stop-one.cmd' if os.name == 'nt' else 'scripts/stop-one.sh'
|
|
for index in range(2):
|
|
try:
|
|
command('public-failover-' + str(index+1) + '.txt',
|
|
[sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'failover',
|
|
'--base-url', base, '--action-script', hook])
|
|
finally:
|
|
restore()
|
|
results = {}
|
|
for index, row in enumerate(snapshot('api'), start=1):
|
|
target = row['instance']
|
|
try:
|
|
results['failover-' + str(index)] = experiment(base, 'failover-' + str(index),
|
|
lambda: stop_instance(target), target)
|
|
finally:
|
|
restore()
|
|
# Build one image first; apply compatible migrations once before replacing replicas.
|
|
compose('build', 'api')
|
|
compose('run', '--rm', '--no-deps', 'migrate')
|
|
results['rolling'] = experiment(base, 'rolling-load', rolling)
|
|
save('grade4-summary.json', results)
|
|
save('final-containers.json', snapshot())
|
|
command('final-smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', base])
|
|
print('PASS: all grade-4 scenarios', flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|