Files
mtusi2026lab6/student-tests/failover_series.py
T

175 lines
7.8 KiB
Python

#!/usr/bin/env python3
"""Серия отказов под нагрузкой (уровень 4): поочерёдно останавливает КАЖДУЮ
API-реплику (docker stop → SIGTERM → drain), восстанавливает её и переходит
к следующей. Во время всей серии идёт closed-loop нагрузка через LB:
POST сообщения (при ошибке повтор с тем же Idempotency-Key) и GET истории.
Измеряется на каждую реплику: RPS до/во время/после, число ошибок, время от
начала остановки до последней ошибки (recovery), время до исчезновения реплики
из ответов LB и время восстановления реплики после docker start.
В конце проверяется: все подтверждённые сообщения на месте, дублей нет.
Скрипт сам вызывает docker CLI (это не публичный тест), запускать из корня репозитория.
"""
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import subprocess
import threading
import time
import uuid
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) # публичный client.py
from client import Client, Failure, arguments, check, run
from resilience import history
ROOT = Path(__file__).resolve().parents[1]
def docker(*args):
return subprocess.run(['docker', *args], cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip()
def api_containers():
return docker('compose', 'ps', '-q', 'api').split()
def health(cid):
return docker('inspect', '-f', '{{.State.Health.Status}}', cid)
class Load:
def __init__(self, args, actor, reader, chat, concurrency):
self.args, self.actor, self.reader, self.chat = args, actor, reader, chat
self.concurrency = concurrency
self.events = [] # (t, kind, ok, instance)
self.confirmed = [] # подтверждённые Message
self.lock = threading.Lock()
self.stop = threading.Event()
def record(self, kind, ok, instance=None):
with self.lock:
self.events.append((time.monotonic(), kind, ok, instance))
def worker(self, n):
c = Client(self.args.base_url, self.args.ca_file, timeout=3)
path = '/api/v1/chats/' + self.chat['id'] + '/messages?limit=20'
key, text, i = None, None, 0
while not self.stop.is_set():
if key is None:
key = uuid.uuid4().hex
text = 'series-%d-%d-%s' % (n, i, key)
i += 1
try:
r = c.message(self.actor, self.chat, text, key=key)
self.record('write', True, r['headers'].get('X-Instance-Id'))
with self.lock:
self.confirmed.append(r['json'])
key = None
except Failure:
self.record('write', False) # неопределённый результат: тот же ключ
try:
r = c.call('GET', path, actor=self.reader, schema='MessagePage')
self.record('read', True, r['headers'].get('X-Instance-Id'))
except Failure:
self.record('read', False)
def __enter__(self):
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
self.futures = [self.pool.submit(self.worker, n) for n in range(self.concurrency)]
return self
def __exit__(self, *exc):
self.stop.set()
for f in self.futures:
f.result()
self.pool.shutdown()
def window(self, a, b):
with self.lock:
ev = [e for e in self.events if a <= e[0] < b]
ok = sum(1 for e in ev if e[2])
return {'seconds': round(b - a, 2), 'ok': ok, 'errors': len(ev) - ok,
'ok_rps': round(ok / (b - a), 1) if b > a else None}
def main():
def configure(p):
p.add_argument('--concurrency', type=int, default=4)
p.add_argument('--warmup', type=float, default=5)
p.add_argument('--observe', type=float, default=15)
p.add_argument('--output', default=None)
args = arguments('Stop each API replica in turn under load; collect RPS/errors/recovery', configure)
c = Client(args.base_url, args.ca_file)
alice, bob, _, chat = c.bootstrap()
replicas = api_containers()
check(len(replicas) >= 2, 'need at least 2 api containers')
results = []
with Load(args, alice, bob, chat, args.concurrency) as load:
time.sleep(args.warmup)
for cid in replicas:
target = cid[:12]
before_start = time.monotonic() - args.warmup
t0 = time.monotonic()
docker('stop', '-t', '15', cid)
stop_seconds = time.monotonic() - t0
time.sleep(max(0.0, args.observe - stop_seconds))
t_end = time.monotonic()
with load.lock:
after = [e for e in load.events if e[0] >= t0]
errs = [e[0] for e in after if not e[2]]
served = [e[0] for e in after if e[2] and e[3] == target]
row = {
'target_instance': target,
'docker_stop_seconds': round(stop_seconds, 2),
'before': load.window(before_start, t0),
'during_stop_and_observe': load.window(t0, t_end),
'errors_after_stop': len(errs),
'recovery_seconds': round(max(errs) - t0, 2) if errs else 0.0,
'target_last_served_after_stop_seconds': round(max(served) - t0, 2) if served else None,
'other_instances_after': sorted({e[3] for e in after if e[2] and e[3] != target}),
}
# Восстановление реплики и её возврат в балансировку.
r0 = time.monotonic()
docker('start', cid)
while health(cid) != 'healthy':
check(time.monotonic() - r0 < 60, 'replica did not become healthy')
time.sleep(0.5)
healthy_at = time.monotonic() - r0
while True:
with load.lock:
seen = any(e[0] > r0 and e[2] and e[3] == target for e in load.events)
if seen:
break
check(time.monotonic() - r0 < 60, 'replica not seen behind LB after restore')
time.sleep(0.2)
row['restore_healthy_seconds'] = round(healthy_at, 2)
row['restore_serving_via_lb_seconds'] = round(time.monotonic() - r0, 2)
results.append(row)
print(json.dumps(row, ensure_ascii=False))
time.sleep(3)
total = load.window(load.events[0][0], time.monotonic())
confirmed = list(load.confirmed)
messages = history(c, bob, chat)
ids = [m['id'] for m in messages]
texts = [m['text'] for m in messages]
check(len(ids) == len(set(ids)), 'duplicated message IDs')
check(len(texts) == len(set(texts)), 'retry with same key created a duplicate')
by_id = {m['id']: m for m in messages}
lost = [m for m in confirmed if by_id.get(m['id']) != m]
check(not lost, '%d confirmed messages lost/changed' % len(lost))
summary = {'concurrency': args.concurrency, 'replicas': len(replicas), 'total': total,
'confirmed_writes': len(confirmed), 'stored_messages': len(messages),
'duplicates': 0, 'lost_confirmed': 0, 'per_replica': results}
text = json.dumps(summary, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + '\n', encoding='utf-8')
print(text)
print('PASS: every replica stopped and restored under load; no lost or duplicated messages.')
if __name__ == '__main__':
run(main)