175 lines
7.8 KiB
Python
175 lines
7.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Серия отказов под нагрузкой (уровень 4): поочерёдно останавливает КАЖДУЮ
|
|
API-реплику (docker stop → SIGTERM → drain), восстанавливает её и переходит
|
|
к следующей. Во время всей серии идёт closed-loop нагрузка через LB:
|
|
POST сообщения (при ошибке повтор с тем же Idempotency-Key) и GET истории.
|
|
|
|
Измеряется на каждую реплику: RPS до/во время/после, число ошибок, время от
|
|
начала остановки до последней ошибки (recovery), время до исчезновения реплики
|
|
из ответов LB и время восстановления реплики после docker start.
|
|
В конце проверяется: все подтверждённые сообщения на месте, дублей нет.
|
|
|
|
Скрипт сам вызывает docker CLI (это не публичный тест), запускать из корня репозитория.
|
|
"""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import sys
|
|
from pathlib import Path
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) # публичный client.py
|
|
from client import Client, Failure, arguments, check, run
|
|
from resilience import history
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def docker(*args):
|
|
return subprocess.run(['docker', *args], cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip()
|
|
|
|
|
|
def api_containers():
|
|
return docker('compose', 'ps', '-q', 'api').split()
|
|
|
|
|
|
def health(cid):
|
|
return docker('inspect', '-f', '{{.State.Health.Status}}', cid)
|
|
|
|
|
|
class Load:
|
|
def __init__(self, args, actor, reader, chat, concurrency):
|
|
self.args, self.actor, self.reader, self.chat = args, actor, reader, chat
|
|
self.concurrency = concurrency
|
|
self.events = [] # (t, kind, ok, instance)
|
|
self.confirmed = [] # подтверждённые Message
|
|
self.lock = threading.Lock()
|
|
self.stop = threading.Event()
|
|
|
|
def record(self, kind, ok, instance=None):
|
|
with self.lock:
|
|
self.events.append((time.monotonic(), kind, ok, instance))
|
|
|
|
def worker(self, n):
|
|
c = Client(self.args.base_url, self.args.ca_file, timeout=3)
|
|
path = '/api/v1/chats/' + self.chat['id'] + '/messages?limit=20'
|
|
key, text, i = None, None, 0
|
|
while not self.stop.is_set():
|
|
if key is None:
|
|
key = uuid.uuid4().hex
|
|
text = 'series-%d-%d-%s' % (n, i, key)
|
|
i += 1
|
|
try:
|
|
r = c.message(self.actor, self.chat, text, key=key)
|
|
self.record('write', True, r['headers'].get('X-Instance-Id'))
|
|
with self.lock:
|
|
self.confirmed.append(r['json'])
|
|
key = None
|
|
except Failure:
|
|
self.record('write', False) # неопределённый результат: тот же ключ
|
|
try:
|
|
r = c.call('GET', path, actor=self.reader, schema='MessagePage')
|
|
self.record('read', True, r['headers'].get('X-Instance-Id'))
|
|
except Failure:
|
|
self.record('read', False)
|
|
|
|
def __enter__(self):
|
|
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
|
|
self.futures = [self.pool.submit(self.worker, n) for n in range(self.concurrency)]
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
self.stop.set()
|
|
for f in self.futures:
|
|
f.result()
|
|
self.pool.shutdown()
|
|
|
|
def window(self, a, b):
|
|
with self.lock:
|
|
ev = [e for e in self.events if a <= e[0] < b]
|
|
ok = sum(1 for e in ev if e[2])
|
|
return {'seconds': round(b - a, 2), 'ok': ok, 'errors': len(ev) - ok,
|
|
'ok_rps': round(ok / (b - a), 1) if b > a else None}
|
|
|
|
|
|
def main():
|
|
def configure(p):
|
|
p.add_argument('--concurrency', type=int, default=4)
|
|
p.add_argument('--warmup', type=float, default=5)
|
|
p.add_argument('--observe', type=float, default=15)
|
|
p.add_argument('--output', default=None)
|
|
args = arguments('Stop each API replica in turn under load; collect RPS/errors/recovery', configure)
|
|
c = Client(args.base_url, args.ca_file)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
replicas = api_containers()
|
|
check(len(replicas) >= 2, 'need at least 2 api containers')
|
|
results = []
|
|
with Load(args, alice, bob, chat, args.concurrency) as load:
|
|
time.sleep(args.warmup)
|
|
for cid in replicas:
|
|
target = cid[:12]
|
|
before_start = time.monotonic() - args.warmup
|
|
t0 = time.monotonic()
|
|
docker('stop', '-t', '15', cid)
|
|
stop_seconds = time.monotonic() - t0
|
|
time.sleep(max(0.0, args.observe - stop_seconds))
|
|
t_end = time.monotonic()
|
|
with load.lock:
|
|
after = [e for e in load.events if e[0] >= t0]
|
|
errs = [e[0] for e in after if not e[2]]
|
|
served = [e[0] for e in after if e[2] and e[3] == target]
|
|
row = {
|
|
'target_instance': target,
|
|
'docker_stop_seconds': round(stop_seconds, 2),
|
|
'before': load.window(before_start, t0),
|
|
'during_stop_and_observe': load.window(t0, t_end),
|
|
'errors_after_stop': len(errs),
|
|
'recovery_seconds': round(max(errs) - t0, 2) if errs else 0.0,
|
|
'target_last_served_after_stop_seconds': round(max(served) - t0, 2) if served else None,
|
|
'other_instances_after': sorted({e[3] for e in after if e[2] and e[3] != target}),
|
|
}
|
|
# Восстановление реплики и её возврат в балансировку.
|
|
r0 = time.monotonic()
|
|
docker('start', cid)
|
|
while health(cid) != 'healthy':
|
|
check(time.monotonic() - r0 < 60, 'replica did not become healthy')
|
|
time.sleep(0.5)
|
|
healthy_at = time.monotonic() - r0
|
|
while True:
|
|
with load.lock:
|
|
seen = any(e[0] > r0 and e[2] and e[3] == target for e in load.events)
|
|
if seen:
|
|
break
|
|
check(time.monotonic() - r0 < 60, 'replica not seen behind LB after restore')
|
|
time.sleep(0.2)
|
|
row['restore_healthy_seconds'] = round(healthy_at, 2)
|
|
row['restore_serving_via_lb_seconds'] = round(time.monotonic() - r0, 2)
|
|
results.append(row)
|
|
print(json.dumps(row, ensure_ascii=False))
|
|
time.sleep(3)
|
|
total = load.window(load.events[0][0], time.monotonic())
|
|
confirmed = list(load.confirmed)
|
|
|
|
messages = history(c, bob, chat)
|
|
ids = [m['id'] for m in messages]
|
|
texts = [m['text'] for m in messages]
|
|
check(len(ids) == len(set(ids)), 'duplicated message IDs')
|
|
check(len(texts) == len(set(texts)), 'retry with same key created a duplicate')
|
|
by_id = {m['id']: m for m in messages}
|
|
lost = [m for m in confirmed if by_id.get(m['id']) != m]
|
|
check(not lost, '%d confirmed messages lost/changed' % len(lost))
|
|
summary = {'concurrency': args.concurrency, 'replicas': len(replicas), 'total': total,
|
|
'confirmed_writes': len(confirmed), 'stored_messages': len(messages),
|
|
'duplicates': 0, 'lost_confirmed': 0, 'per_replica': results}
|
|
text = json.dumps(summary, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + '\n', encoding='utf-8')
|
|
print(text)
|
|
print('PASS: every replica stopped and restored under load; no lost or duplicated messages.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|