208 lines
10 KiB
Python
208 lines
10 KiB
Python
"""Real LB/replica integration, fault injection and rolling replacement evidence."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
import httpx
|
|
from stack import ROOT, docker, instance, ready, ready_replicas, replicas, state
|
|
|
|
OUT = ROOT / 'evidence'
|
|
OUT.mkdir(exist_ok=True)
|
|
evidence = {}
|
|
|
|
|
|
def user(c):
|
|
name = 'v_' + uuid.uuid4().hex[:20]
|
|
password = 'test-password-only-123'
|
|
r = c.post('/api/v1/users', json={'username': name, 'display_name': 'Проверка', 'password': password})
|
|
assert r.status_code == 201, r.text
|
|
uid = r.json()['id']
|
|
r = c.post('/api/v1/auth/sessions', auth=(name, password))
|
|
assert r.status_code == 201, r.text
|
|
return uid, {'Authorization': 'Bearer ' + r.json()['token']}
|
|
|
|
|
|
def chat(c, headers, members):
|
|
r = c.post('/api/v1/chats', headers=headers, json={'title': 'Verify', 'member_ids': members})
|
|
assert r.status_code == 201, r.text
|
|
return '/api/v1/chats/' + r.json()['id'] + '/messages'
|
|
|
|
|
|
def history(c, headers, path):
|
|
rows, cursor = [], None
|
|
for _ in range(1000):
|
|
r = c.get(path, headers=headers, params={'limit': 100, **({'cursor': cursor} if cursor else {})})
|
|
assert r.status_code == 200, r.text
|
|
page = r.json()
|
|
rows.extend(page['items'])
|
|
cursor = page['next_cursor']
|
|
if cursor is None:
|
|
return rows
|
|
raise AssertionError('Pagination exceeded bound')
|
|
|
|
|
|
def restore(c):
|
|
docker('compose', 'up', '-d', '--no-deps', '--scale', 'api=2', 'api')
|
|
ready_replicas()
|
|
ready(c)
|
|
deadline = time.monotonic() + 20
|
|
while time.monotonic() < deadline:
|
|
ids = {c.get('/health/ready').headers.get('X-Instance-Id') for _ in range(12)} - {None}
|
|
if len(ids) == 2:
|
|
return
|
|
time.sleep(.2)
|
|
raise AssertionError('LB did not observe both actual replicas')
|
|
|
|
|
|
def public(mode):
|
|
script = 'restart' if mode == 'persistence' else 'stop-one'
|
|
with tempfile.TemporaryDirectory(prefix='lab3-hook-') as temp:
|
|
if os.name == 'nt':
|
|
hook = Path(temp) / (script + '.cmd')
|
|
hook.write_text('@echo off\n"' + sys.executable + '" "' + str(ROOT / ('scripts/' + script + '.py')) + '"\n', encoding='utf-8')
|
|
else:
|
|
hook = ROOT / ('scripts/' + script + '.sh')
|
|
return subprocess.check_output([sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', mode,
|
|
'--action-script', str(hook)], cwd=ROOT, text=True, encoding='utf-8').strip()
|
|
|
|
|
|
def fault_scenario(c, uid, headers, target, rolling=False):
|
|
path = chat(c, headers, [uid])
|
|
before = state()
|
|
alias = instance(target)
|
|
rows, confirmed = [], {}
|
|
stop = threading.Event()
|
|
start = time.monotonic()
|
|
|
|
def traffic(worker):
|
|
with httpx.Client(base_url='http://127.0.0.1:8080', timeout=3) as client:
|
|
key = uuid.uuid4().hex
|
|
while not stop.is_set():
|
|
tick = time.monotonic()
|
|
body = {'text': key}
|
|
try:
|
|
r = client.post(path, headers={**headers, 'Idempotency-Key': key}, json=body)
|
|
status, serving = r.status_code, r.headers.get('X-Instance-Id')
|
|
if status == 201:
|
|
msg = r.json()
|
|
assert key not in confirmed or confirmed[key] == msg
|
|
confirmed[key] = msg
|
|
key = uuid.uuid4().hex
|
|
except httpx.HTTPError:
|
|
status, serving = 0, None
|
|
rows.append({'t': round(tick - start, 6), 'method': 'POST', 'status': status,
|
|
'instance': serving, 'ms': round((time.monotonic() - tick) * 1000, 3)})
|
|
tick = time.monotonic()
|
|
try:
|
|
r = client.get(path, headers=headers, params={'limit': 10})
|
|
status, serving = r.status_code, r.headers.get('X-Instance-Id')
|
|
except httpx.HTTPError:
|
|
status, serving = 0, None
|
|
rows.append({'t': round(tick - start, 6), 'method': 'GET', 'status': status,
|
|
'instance': serving, 'ms': round((time.monotonic() - tick) * 1000, 3)})
|
|
time.sleep(.03)
|
|
|
|
events = []
|
|
with ThreadPoolExecutor(max_workers=3) as pool:
|
|
futures = [pool.submit(traffic, w) for w in range(3)]
|
|
try:
|
|
time.sleep(2)
|
|
targets = replicas() if rolling else [target]
|
|
for cid in targets:
|
|
event = {'container': cid, 'alias': instance(cid), 'signal_at': time.monotonic() - start}
|
|
docker('stop', '-t', '5', cid)
|
|
event['stopped_at'] = time.monotonic() - start
|
|
assert json.loads(docker('inspect', '--format', '{{json .State.Running}}', cid)) is False
|
|
if rolling:
|
|
docker('rm', cid)
|
|
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=2', 'api')
|
|
ready_replicas()
|
|
ready(c)
|
|
event['replacement_ready_at'] = time.monotonic() - start
|
|
events.append(event)
|
|
time.sleep(3)
|
|
time.sleep(3)
|
|
finally:
|
|
stop.set()
|
|
for future in futures:
|
|
future.result()
|
|
elapsed = time.monotonic() - start
|
|
actual = history(c, headers, path)
|
|
assert len({m['text'] for m in actual}) == len(actual), 'Duplicate retry side effect'
|
|
by_id = {m['id']: m for m in actual}
|
|
assert all(by_id.get(m['id']) == m for m in confirmed.values()), 'Lost confirmation'
|
|
for event in events:
|
|
for method, good in [('GET', 200), ('POST', 201)]:
|
|
successful = [r for r in rows if r['method'] == method and r['status'] == good and r['t'] >= event['stopped_at']]
|
|
assert successful, 'No successful operation after stop'
|
|
event[method + '_recovery_seconds'] = round(successful[0]['t'] - event['signal_at'], 3)
|
|
assert event[method + '_recovery_seconds'] <= 10
|
|
assert all(r['instance'] != event['alias'] for r in successful)
|
|
errors = [r for r in rows if r['status'] not in (200, 201)]
|
|
result = {'events': events, 'duration_seconds': round(elapsed, 3), 'attempts': len(rows),
|
|
'successful_rps': round((len(rows) - len(errors)) / elapsed, 3), 'errors': len(errors),
|
|
'confirmed_messages': len(confirmed), 'stored_messages': len(actual), 'all_confirmed_retained': True,
|
|
'before': before, 'after': state(), 'requests': sorted(rows, key=lambda r: r['t'])}
|
|
name = 'rolling' if rolling else 'stop-' + alias
|
|
(OUT / (name + '.json')).write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
|
|
return {k: v for k, v in result.items() if k not in ('requests', 'before', 'after')}
|
|
|
|
|
|
with httpx.Client(base_url='http://127.0.0.1:8080', timeout=12) as c:
|
|
restore(c)
|
|
evidence['public_smoke'] = subprocess.check_output([sys.executable, '-X', 'utf8', 'tests/smoke.py'], cwd=ROOT, text=True).strip()
|
|
uid, headers = user(c)
|
|
uid2, headers2 = user(c)
|
|
path = chat(c, headers, [uid, uid2])
|
|
key = uuid.uuid4().hex
|
|
same_headers = {**headers, 'Idempotency-Key': key}
|
|
with ThreadPoolExecutor(max_workers=16) as pool:
|
|
replies = list(pool.map(lambda _: c.post(path, headers=same_headers, json={'text': 'Один результат'}), range(32)))
|
|
assert all(r.status_code == 201 for r in replies), [r.status_code for r in replies]
|
|
assert len({r.headers['X-Instance-Id'] for r in replies}) == 2
|
|
original = replies[0].json()
|
|
assert all(r.json() == original for r in replies)
|
|
assert len(history(c, headers, path)) == 1
|
|
evidence['parallel_idempotency'] = {'attempts': 32, 'replicas': 2, 'messages': 1}
|
|
assert c.post(path, headers=same_headers, json={'text': 'Другой результат'}).status_code == 409
|
|
r = c.post(path, headers={**headers2, 'Idempotency-Key': key}, json={'text': 'Другой пользователь'})
|
|
assert r.status_code == 201 and r.json()['id'] != original['id']
|
|
path2 = chat(c, headers, [uid])
|
|
assert c.post(path2, headers=same_headers, json={'text': 'Другой чат'}).status_code == 201
|
|
assert c.post(path, headers=headers, json={'text': 'Без ключа'}).status_code == 400
|
|
assert c.post(path, headers={**headers, 'Idempotency-Key': 'bad key'}, json={'text': 'Ключ'}).status_code == 400
|
|
# Invalid request must not reserve a key.
|
|
fresh = {**headers, 'Idempotency-Key': uuid.uuid4().hex}
|
|
assert c.post(path, headers=fresh, json={'text': ''}).status_code == 400
|
|
assert c.post(path, headers=fresh, json={'text': 'После 400'}).status_code == 201
|
|
evidence['key_scopes_validation'] = '409 mismatch; independent users/chats; missing/invalid key 400; 400 does not reserve key'
|
|
evidence['public_persistence'] = public('persistence')
|
|
ready(c)
|
|
assert c.post(path, headers=same_headers, json={'text': 'Один результат'}).json() == original
|
|
assert c.delete('/api/v1/auth/sessions/current', headers=headers2).status_code == 204
|
|
assert c.post(path, headers={**headers2, 'Idempotency-Key': key}, json={'text': 'Другой пользователь'}).status_code == 401
|
|
evidence['replay_after_restart_logout'] = 'same response after all-service restart; revoked session cannot replay'
|
|
restore(c)
|
|
evidence['public_failover'] = public('failover')
|
|
restore(c)
|
|
targets = replicas()
|
|
evidence['each_replica_failure'] = []
|
|
for target in targets:
|
|
try:
|
|
evidence['each_replica_failure'].append(fault_scenario(c, uid, headers, target))
|
|
finally:
|
|
restore(c)
|
|
evidence['rolling'] = fault_scenario(c, uid, headers, replicas()[0], rolling=True)
|
|
restore(c)
|
|
evidence['migration_repeat'] = docker('compose', 'run', '--rm', '--no-deps', 'migrate')
|
|
evidence['passed'] = True
|
|
(OUT / 'integration.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
print(json.dumps(evidence, ensure_ascii=False, indent=2))
|