"""Real LB/replica integration, fault injection and rolling replacement evidence.""" from concurrent.futures import ThreadPoolExecutor import json import os from pathlib import Path import subprocess import sys import tempfile import threading import time import uuid import httpx from stack import ROOT, docker, instance, ready, ready_replicas, replicas, state OUT = ROOT / 'evidence' OUT.mkdir(exist_ok=True) evidence = {} def user(c): name = 'v_' + uuid.uuid4().hex[:20] password = 'test-password-only-123' r = c.post('/api/v1/users', json={'username': name, 'display_name': 'Проверка', 'password': password}) assert r.status_code == 201, r.text uid = r.json()['id'] r = c.post('/api/v1/auth/sessions', auth=(name, password)) assert r.status_code == 201, r.text return uid, {'Authorization': 'Bearer ' + r.json()['token']} def chat(c, headers, members): r = c.post('/api/v1/chats', headers=headers, json={'title': 'Verify', 'member_ids': members}) assert r.status_code == 201, r.text return '/api/v1/chats/' + r.json()['id'] + '/messages' def history(c, headers, path): rows, cursor = [], None for _ in range(1000): r = c.get(path, headers=headers, params={'limit': 100, **({'cursor': cursor} if cursor else {})}) assert r.status_code == 200, r.text page = r.json() rows.extend(page['items']) cursor = page['next_cursor'] if cursor is None: return rows raise AssertionError('Pagination exceeded bound') def restore(c): docker('compose', 'up', '-d', '--no-deps', '--scale', 'api=2', 'api') ready_replicas() ready(c) deadline = time.monotonic() + 20 while time.monotonic() < deadline: ids = {c.get('/health/ready').headers.get('X-Instance-Id') for _ in range(12)} - {None} if len(ids) == 2: return time.sleep(.2) raise AssertionError('LB did not observe both actual replicas') def public(mode): script = 'restart' if mode == 'persistence' else 'stop-one' with tempfile.TemporaryDirectory(prefix='lab3-hook-') as temp: if os.name == 'nt': hook = Path(temp) / (script + '.cmd') hook.write_text('@echo off\n"' + sys.executable + '" "' + str(ROOT / ('scripts/' + script + '.py')) + '"\n', encoding='utf-8') else: hook = ROOT / ('scripts/' + script + '.sh') return subprocess.check_output([sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', mode, '--action-script', str(hook)], cwd=ROOT, text=True, encoding='utf-8').strip() def fault_scenario(c, uid, headers, target, rolling=False): path = chat(c, headers, [uid]) before = state() alias = instance(target) rows, confirmed = [], {} stop = threading.Event() start = time.monotonic() def traffic(worker): with httpx.Client(base_url='http://127.0.0.1:8080', timeout=3) as client: key = uuid.uuid4().hex while not stop.is_set(): tick = time.monotonic() body = {'text': key} try: r = client.post(path, headers={**headers, 'Idempotency-Key': key}, json=body) status, serving = r.status_code, r.headers.get('X-Instance-Id') if status == 201: msg = r.json() assert key not in confirmed or confirmed[key] == msg confirmed[key] = msg key = uuid.uuid4().hex except httpx.HTTPError: status, serving = 0, None rows.append({'t': round(tick - start, 6), 'method': 'POST', 'status': status, 'instance': serving, 'ms': round((time.monotonic() - tick) * 1000, 3)}) tick = time.monotonic() try: r = client.get(path, headers=headers, params={'limit': 10}) status, serving = r.status_code, r.headers.get('X-Instance-Id') except httpx.HTTPError: status, serving = 0, None rows.append({'t': round(tick - start, 6), 'method': 'GET', 'status': status, 'instance': serving, 'ms': round((time.monotonic() - tick) * 1000, 3)}) time.sleep(.03) events = [] with ThreadPoolExecutor(max_workers=3) as pool: futures = [pool.submit(traffic, w) for w in range(3)] try: time.sleep(2) targets = replicas() if rolling else [target] for cid in targets: event = {'container': cid, 'alias': instance(cid), 'signal_at': time.monotonic() - start} docker('stop', '-t', '5', cid) event['stopped_at'] = time.monotonic() - start assert json.loads(docker('inspect', '--format', '{{json .State.Running}}', cid)) is False if rolling: docker('rm', cid) docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=2', 'api') ready_replicas() ready(c) event['replacement_ready_at'] = time.monotonic() - start events.append(event) time.sleep(3) time.sleep(3) finally: stop.set() for future in futures: future.result() elapsed = time.monotonic() - start actual = history(c, headers, path) assert len({m['text'] for m in actual}) == len(actual), 'Duplicate retry side effect' by_id = {m['id']: m for m in actual} assert all(by_id.get(m['id']) == m for m in confirmed.values()), 'Lost confirmation' for event in events: for method, good in [('GET', 200), ('POST', 201)]: successful = [r for r in rows if r['method'] == method and r['status'] == good and r['t'] >= event['stopped_at']] assert successful, 'No successful operation after stop' event[method + '_recovery_seconds'] = round(successful[0]['t'] - event['signal_at'], 3) assert event[method + '_recovery_seconds'] <= 10 assert all(r['instance'] != event['alias'] for r in successful) errors = [r for r in rows if r['status'] not in (200, 201)] result = {'events': events, 'duration_seconds': round(elapsed, 3), 'attempts': len(rows), 'successful_rps': round((len(rows) - len(errors)) / elapsed, 3), 'errors': len(errors), 'confirmed_messages': len(confirmed), 'stored_messages': len(actual), 'all_confirmed_retained': True, 'before': before, 'after': state(), 'requests': sorted(rows, key=lambda r: r['t'])} name = 'rolling' if rolling else 'stop-' + alias (OUT / (name + '.json')).write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8') return {k: v for k, v in result.items() if k not in ('requests', 'before', 'after')} with httpx.Client(base_url='http://127.0.0.1:8080', timeout=12) as c: restore(c) evidence['public_smoke'] = subprocess.check_output([sys.executable, '-X', 'utf8', 'tests/smoke.py'], cwd=ROOT, text=True).strip() uid, headers = user(c) uid2, headers2 = user(c) path = chat(c, headers, [uid, uid2]) key = uuid.uuid4().hex same_headers = {**headers, 'Idempotency-Key': key} with ThreadPoolExecutor(max_workers=16) as pool: replies = list(pool.map(lambda _: c.post(path, headers=same_headers, json={'text': 'Один результат'}), range(32))) assert all(r.status_code == 201 for r in replies), [r.status_code for r in replies] assert len({r.headers['X-Instance-Id'] for r in replies}) == 2 original = replies[0].json() assert all(r.json() == original for r in replies) assert len(history(c, headers, path)) == 1 evidence['parallel_idempotency'] = {'attempts': 32, 'replicas': 2, 'messages': 1} assert c.post(path, headers=same_headers, json={'text': 'Другой результат'}).status_code == 409 r = c.post(path, headers={**headers2, 'Idempotency-Key': key}, json={'text': 'Другой пользователь'}) assert r.status_code == 201 and r.json()['id'] != original['id'] path2 = chat(c, headers, [uid]) assert c.post(path2, headers=same_headers, json={'text': 'Другой чат'}).status_code == 201 assert c.post(path, headers=headers, json={'text': 'Без ключа'}).status_code == 400 assert c.post(path, headers={**headers, 'Idempotency-Key': 'bad key'}, json={'text': 'Ключ'}).status_code == 400 # Invalid request must not reserve a key. fresh = {**headers, 'Idempotency-Key': uuid.uuid4().hex} assert c.post(path, headers=fresh, json={'text': ''}).status_code == 400 assert c.post(path, headers=fresh, json={'text': 'После 400'}).status_code == 201 evidence['key_scopes_validation'] = '409 mismatch; independent users/chats; missing/invalid key 400; 400 does not reserve key' evidence['public_persistence'] = public('persistence') ready(c) assert c.post(path, headers=same_headers, json={'text': 'Один результат'}).json() == original assert c.delete('/api/v1/auth/sessions/current', headers=headers2).status_code == 204 assert c.post(path, headers={**headers2, 'Idempotency-Key': key}, json={'text': 'Другой пользователь'}).status_code == 401 evidence['replay_after_restart_logout'] = 'same response after all-service restart; revoked session cannot replay' restore(c) evidence['public_failover'] = public('failover') restore(c) targets = replicas() evidence['each_replica_failure'] = [] for target in targets: try: evidence['each_replica_failure'].append(fault_scenario(c, uid, headers, target)) finally: restore(c) evidence['rolling'] = fault_scenario(c, uid, headers, replicas()[0], rolling=True) restore(c) evidence['migration_repeat'] = docker('compose', 'run', '--rm', '--no-deps', 'migrate') evidence['passed'] = True (OUT / 'integration.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') print(json.dumps(evidence, ensure_ascii=False, indent=2))