"""Repeatable grade-4 experiments. Run on host with Docker access and UTF-8 mode.""" import argparse import json import os import subprocess import sys import threading import time from concurrent.futures import ThreadPoolExecutor from uuid import uuid4 from ops import ROOT, compose, restore, rolling, save, snapshot, stop_instance, wait_api sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure, check from resilience import history def command(name, args): result = subprocess.run(args, cwd=ROOT, text=True, encoding='utf-8', stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=180) (ROOT / 'evidence' / name).write_text(result.stdout, encoding='utf-8') print(result.stdout, flush=True) check(result.returncode == 0, 'Command failed: ' + name) def concurrency(base): c = Client(base) alice, bob, eve, chat = c.bootstrap() path = '/api/v1/chats/' + chat['id'] + '/messages' key = uuid4().hex barrier = threading.Barrier(8) def send(_): client = Client(base) barrier.wait(timeout=10) return client.message(alice, chat, 'one concurrent message', key=key) responses = [] with ThreadPoolExecutor(max_workers=8) as executor: for _ in range(3): responses.extend(executor.map(send, range(8))) first = responses[0]['json'] check(all(r['json'] == first and r['status'] == 201 for r in responses), 'Concurrent replies differ') instances = sorted({r['headers']['X-Instance-Id'] for r in responses}) check(len(instances) >= 2, 'Concurrent test did not reach both replicas') check(history(c, alice, chat) == [first], 'Concurrent retry duplicated message') c.call('POST', path, 409, actor=alice, body={'text': 'different'}, headers={'Idempotency-Key': key}) c.call('POST', path, 401, body={'text': 'one concurrent message'}, headers={'Idempotency-Key': key}) c.call('POST', path, 404, actor=eve, body={'text': 'one concurrent message'}, headers={'Idempotency-Key': key}) for invalid in (None, '', 'bad key', 'x' * 129): c.call('POST', path, 400, actor=alice, body={'text': 'invalid key'}, headers={} if invalid is None else {'Idempotency-Key': invalid}) other = c.message(bob, chat, 'independent user', key=key)['json'] check(other['id'] != first['id'], 'Key scope must include actor') chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'Scope', 'member_ids': [alice['id']]}, schema='Chat')['json'] other = c.message(alice, chat2, 'independent chat', key=key)['json'] check(other['id'] != first['id'], 'Key scope must include chat') save('idempotency.json', {'requests': 24, 'concurrency': 8, 'instances': instances, 'unique_messages': 1, 'same_response': True, 'conflict_auth_access_key_scope': 'PASS'}) return c, alice, chat, key, first def experiment(base, name, action, target=None): c = Client(base, timeout=9) alice, bob, _, chat = c.bootstrap() confirmed_before = c.message(alice, chat, 'before ' + name)['json'] events = [] stop = threading.Event() started = time.monotonic() def worker(): client = Client(base, timeout=9) key = uuid4().hex while not stop.is_set(): begin = time.monotonic() try: response = client.message(alice, chat, key, key=key) msg = response['json'] events.append({'kind': 'write', 'time': time.monotonic() - started, 'latency': response['elapsed'], 'instance': response['headers']['X-Instance-Id'], 'message': msg}) key = uuid4().hex except Failure as exc: events.append({'kind': 'error', 'operation': 'write', 'time': time.monotonic() - started, 'latency': time.monotonic() - begin, 'error': str(exc)}) continue # Keep original key/body after an uncertain write. begin = time.monotonic() try: response = client.call('GET', '/api/v1/chats/' + chat['id'], actor=bob, schema='Chat') events.append({'kind': 'read', 'time': time.monotonic() - started, 'latency': response['elapsed'], 'instance': response['headers']['X-Instance-Id']}) except Failure as exc: events.append({'kind': 'error', 'operation': 'read', 'time': time.monotonic() - started, 'latency': time.monotonic() - begin, 'error': str(exc)}) time.sleep(0.02) with ThreadPoolExecutor(max_workers=2) as executor: futures = [executor.submit(worker) for _ in range(2)] try: time.sleep(3) action_started = time.monotonic() - started effect = action() action_finished = time.monotonic() - started time.sleep(max(5, 20 - (time.monotonic() - started))) finally: stop.set() for future in futures: future.result() duration = time.monotonic() - started successes = [e for e in events if e['kind'] != 'error'] errors = [e for e in events if e['kind'] == 'error'] writes = [e for e in successes if e['kind'] == 'write'] recovery = {} for kind in ('read', 'write'): after = [e for e in successes if e['kind'] == kind and e['time'] >= action_finished] check(bool(after), 'No ' + kind + ' after action') recovery[kind] = round(min(e['time'] for e in after) - action_started, 3) if target: check(all(e['instance'] != target for e in after), 'Stopped target still serves') check(recovery[kind] <= 10, 'Recovery exceeded 10 seconds') messages = history(c, bob, chat) by_id = {m['id']: m for m in messages} check(by_id.get(confirmed_before['id']) == confirmed_before, 'Pre-failure message lost') check(all(by_id.get(e['message']['id']) == e['message'] for e in writes), 'Confirmed write lost') check(len({m['text'] for m in messages}) == len(messages), 'Retry duplicated a message') # Retry a pre-failure key through the surviving/new processes. if writes: item = writes[0]['message'] check(c.message(alice, chat, item['text'], key=item['text'])['json'] == item, 'Replay changed after action') latencies = sorted(e['latency'] for e in successes) gaps = {} for kind in ('read', 'write'): times = sorted(e['time'] for e in successes if e['kind'] == kind) gaps[kind] = round(max((b - a for a, b in zip(times, times[1:])), default=0), 3) summary = {'duration_seconds': round(duration, 3), 'concurrency': 2, 'successful_http_rps': round(len(successes) / duration, 2), 'successful_write_rps': round(len(writes) / duration, 2), 'successful_requests': len(successes), 'errors': len(errors), 'action_started_seconds': round(action_started, 3), 'action_finished_seconds': round(action_finished, 3), 'first_read_write_after_action_seconds_from_start': recovery, 'max_success_gap_seconds': gaps, 'latency_ms': {str(p): round(latencies[min(int(len(latencies)*p/100), len(latencies)-1)]*1000, 2) for p in (50, 95, 99)}, 'confirmed_messages_preserved': True, 'duplicate_messages': 0, 'instances': sorted({e['instance'] for e in successes}), 'target': target} save(name + '.json', {'summary': summary, 'containers': effect, 'errors': errors}) print(name + ': ' + json.dumps(summary), flush=True) return summary def main(): parser = argparse.ArgumentParser() parser.add_argument('--base-url', default='http://127.0.0.1:8080') args = parser.parse_args() os.environ['PYTHONUTF8'] = '1' base = args.base_url wait_api() command('check.txt', [sys.executable, '-X', 'utf8', 'scripts/check_contract.py']) command('final-smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', base]) c, actor, chat, key, original = concurrency(base) hook = 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh' command('persistence.txt', [sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'persistence', '--base-url', base, '--action-script', hook]) wait_api() check(c.message(actor, chat, original['text'], key=key)['json'] == original, 'Idempotency/session not preserved across full restart') save('idempotency-restart.json', {'same_token': True, 'same_201_message': True}) hook = 'scripts/stop-one.cmd' if os.name == 'nt' else 'scripts/stop-one.sh' for index in range(2): try: command('public-failover-' + str(index+1) + '.txt', [sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', 'failover', '--base-url', base, '--action-script', hook]) finally: restore() results = {} for index, row in enumerate(snapshot('api'), start=1): target = row['instance'] try: results['failover-' + str(index)] = experiment(base, 'failover-' + str(index), lambda: stop_instance(target), target) finally: restore() # Build one image first; apply compatible migrations once before replacing replicas. compose('build', 'api') compose('run', '--rm', '--no-deps', 'migrate') results['rolling'] = experiment(base, 'rolling-load', rolling) save('grade4-summary.json', results) save('final-containers.json', snapshot()) command('final-smoke.txt', [sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', base]) print('PASS: all grade-4 scenarios', flush=True) if __name__ == '__main__': main()