Files
mtusi2026lab3/scripts/verify_stack.py
T

206 lines
10 KiB
Python

"""Real LB/replica integration, fault injection and rolling replacement evidence."""
from concurrent.futures import ThreadPoolExecutor
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import threading
import time
import uuid
import httpx
from stack import ROOT, docker, instance, ready, replicas, state
OUT = ROOT / 'evidence'
OUT.mkdir(exist_ok=True)
evidence = {}
def user(c):
name = 'v_' + uuid.uuid4().hex[:20]
password = 'test-password-only-123'
r = c.post('/api/v1/users', json={'username': name, 'display_name': 'Проверка', 'password': password})
assert r.status_code == 201, r.text
uid = r.json()['id']
r = c.post('/api/v1/auth/sessions', auth=(name, password))
assert r.status_code == 201, r.text
return uid, {'Authorization': 'Bearer ' + r.json()['token']}
def chat(c, headers, members):
r = c.post('/api/v1/chats', headers=headers, json={'title': 'Verify', 'member_ids': members})
assert r.status_code == 201, r.text
return '/api/v1/chats/' + r.json()['id'] + '/messages'
def history(c, headers, path):
rows, cursor = [], None
for _ in range(1000):
r = c.get(path, headers=headers, params={'limit': 100, **({'cursor': cursor} if cursor else {})})
assert r.status_code == 200, r.text
page = r.json()
rows.extend(page['items'])
cursor = page['next_cursor']
if cursor is None:
return rows
raise AssertionError('Pagination exceeded bound')
def restore(c):
docker('compose', 'up', '-d', '--no-deps', '--wait', '--scale', 'api=2', 'api')
ready(c)
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
ids = {c.get('/health/ready').headers.get('X-Instance-Id') for _ in range(12)} - {None}
if len(ids) == 2:
return
time.sleep(.2)
raise AssertionError('LB did not observe both actual replicas')
def public(mode):
script = 'restart' if mode == 'persistence' else 'stop-one'
with tempfile.TemporaryDirectory(prefix='lab3-hook-') as temp:
if os.name == 'nt':
hook = Path(temp) / (script + '.cmd')
hook.write_text('@echo off\n"' + sys.executable + '" "' + str(ROOT / ('scripts/' + script + '.py')) + '"\n', encoding='utf-8')
else:
hook = ROOT / ('scripts/' + script + '.sh')
return subprocess.check_output([sys.executable, '-X', 'utf8', 'tests/resilience.py', '--mode', mode,
'--action-script', str(hook)], cwd=ROOT, text=True, encoding='utf-8').strip()
def fault_scenario(c, uid, headers, target, rolling=False):
path = chat(c, headers, [uid])
before = state()
alias = instance(target)
rows, confirmed = [], {}
stop = threading.Event()
start = time.monotonic()
def traffic(worker):
with httpx.Client(base_url='http://127.0.0.1:8080', timeout=3) as client:
key = uuid.uuid4().hex
while not stop.is_set():
tick = time.monotonic()
body = {'text': key}
try:
r = client.post(path, headers={**headers, 'Idempotency-Key': key}, json=body)
status, serving = r.status_code, r.headers.get('X-Instance-Id')
if status == 201:
msg = r.json()
assert key not in confirmed or confirmed[key] == msg
confirmed[key] = msg
key = uuid.uuid4().hex
except httpx.HTTPError:
status, serving = 0, None
rows.append({'t': round(tick - start, 6), 'method': 'POST', 'status': status,
'instance': serving, 'ms': round((time.monotonic() - tick) * 1000, 3)})
tick = time.monotonic()
try:
r = client.get(path, headers=headers, params={'limit': 10})
status, serving = r.status_code, r.headers.get('X-Instance-Id')
except httpx.HTTPError:
status, serving = 0, None
rows.append({'t': round(tick - start, 6), 'method': 'GET', 'status': status,
'instance': serving, 'ms': round((time.monotonic() - tick) * 1000, 3)})
time.sleep(.03)
events = []
with ThreadPoolExecutor(max_workers=3) as pool:
futures = [pool.submit(traffic, w) for w in range(3)]
try:
time.sleep(2)
targets = replicas() if rolling else [target]
for cid in targets:
event = {'container': cid, 'alias': instance(cid), 'signal_at': time.monotonic() - start}
docker('stop', '-t', '5', cid)
event['stopped_at'] = time.monotonic() - start
assert json.loads(docker('inspect', '--format', '{{json .State.Running}}', cid)) is False
if rolling:
docker('rm', cid)
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--wait', '--scale', 'api=2', 'api')
ready(c)
event['replacement_ready_at'] = time.monotonic() - start
events.append(event)
time.sleep(3)
time.sleep(3)
finally:
stop.set()
for future in futures:
future.result()
elapsed = time.monotonic() - start
actual = history(c, headers, path)
assert len({m['text'] for m in actual}) == len(actual), 'Duplicate retry side effect'
by_id = {m['id']: m for m in actual}
assert all(by_id.get(m['id']) == m for m in confirmed.values()), 'Lost confirmation'
for event in events:
for method, good in [('GET', 200), ('POST', 201)]:
successful = [r for r in rows if r['method'] == method and r['status'] == good and r['t'] >= event['stopped_at']]
assert successful, 'No successful operation after stop'
event[method + '_recovery_seconds'] = round(successful[0]['t'] - event['signal_at'], 3)
assert event[method + '_recovery_seconds'] <= 10
assert all(r['instance'] != event['alias'] for r in successful)
errors = [r for r in rows if r['status'] not in (200, 201)]
result = {'events': events, 'duration_seconds': round(elapsed, 3), 'attempts': len(rows),
'successful_rps': round((len(rows) - len(errors)) / elapsed, 3), 'errors': len(errors),
'confirmed_messages': len(confirmed), 'stored_messages': len(actual), 'all_confirmed_retained': True,
'before': before, 'after': state(), 'requests': sorted(rows, key=lambda r: r['t'])}
name = 'rolling' if rolling else 'stop-' + alias
(OUT / (name + '.json')).write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8')
return {k: v for k, v in result.items() if k not in ('requests', 'before', 'after')}
with httpx.Client(base_url='http://127.0.0.1:8080', timeout=12) as c:
restore(c)
evidence['public_smoke'] = subprocess.check_output([sys.executable, '-X', 'utf8', 'tests/smoke.py'], cwd=ROOT, text=True).strip()
uid, headers = user(c)
uid2, headers2 = user(c)
path = chat(c, headers, [uid, uid2])
key = uuid.uuid4().hex
same_headers = {**headers, 'Idempotency-Key': key}
with ThreadPoolExecutor(max_workers=16) as pool:
replies = list(pool.map(lambda _: c.post(path, headers=same_headers, json={'text': 'Один результат'}), range(32)))
assert all(r.status_code == 201 for r in replies), [r.status_code for r in replies]
assert len({r.headers['X-Instance-Id'] for r in replies}) == 2
original = replies[0].json()
assert all(r.json() == original for r in replies)
assert len(history(c, headers, path)) == 1
evidence['parallel_idempotency'] = {'attempts': 32, 'replicas': 2, 'messages': 1}
assert c.post(path, headers=same_headers, json={'text': 'Другой результат'}).status_code == 409
r = c.post(path, headers={**headers2, 'Idempotency-Key': key}, json={'text': 'Другой пользователь'})
assert r.status_code == 201 and r.json()['id'] != original['id']
path2 = chat(c, headers, [uid])
assert c.post(path2, headers=same_headers, json={'text': 'Другой чат'}).status_code == 201
assert c.post(path, headers=headers, json={'text': 'Без ключа'}).status_code == 400
assert c.post(path, headers={**headers, 'Idempotency-Key': 'bad key'}, json={'text': 'Ключ'}).status_code == 400
# Invalid request must not reserve a key.
fresh = {**headers, 'Idempotency-Key': uuid.uuid4().hex}
assert c.post(path, headers=fresh, json={'text': ''}).status_code == 400
assert c.post(path, headers=fresh, json={'text': 'После 400'}).status_code == 201
evidence['key_scopes_validation'] = '409 mismatch; independent users/chats; missing/invalid key 400; 400 does not reserve key'
evidence['public_persistence'] = public('persistence')
ready(c)
assert c.post(path, headers=same_headers, json={'text': 'Один результат'}).json() == original
assert c.delete('/api/v1/auth/sessions/current', headers=headers2).status_code == 204
assert c.post(path, headers={**headers2, 'Idempotency-Key': key}, json={'text': 'Другой пользователь'}).status_code == 401
evidence['replay_after_restart_logout'] = 'same response after all-service restart; revoked session cannot replay'
restore(c)
evidence['public_failover'] = public('failover')
restore(c)
targets = replicas()
evidence['each_replica_failure'] = []
for target in targets:
try:
evidence['each_replica_failure'].append(fault_scenario(c, uid, headers, target))
finally:
restore(c)
evidence['rolling'] = fault_scenario(c, uid, headers, replicas()[0], rolling=True)
restore(c)
evidence['migration_repeat'] = docker('compose', 'run', '--rm', '--no-deps', 'migrate')
evidence['passed'] = True
(OUT / 'integration.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print(json.dumps(evidence, ensure_ascii=False, indent=2))