#!/usr/bin/env python3 """Нагрузка во время отказа/замены реплики с метриками по времени. Потоки непрерывно пишут сообщения (как настоящий клиент: при неопределённом результате повторяют POST с ТЕМ ЖЕ Idempotency-Key) и читают историю. Через --action-after секунд запускается action-скрипт (stop-one.sh / kill-one.sh / rolling-update.sh) с TARGET_INSTANCE_ID. Отчёт: успешные RPS до/во время/после, ошибки, p50/p95/p99, время до последней ошибки, максимальный разрыв между успешными записями, распределение по репликам, проверка сохранности подтверждённых сообщений и отсутствия дублей. """ from concurrent.futures import ThreadPoolExecutor import json import math import os from pathlib import Path import subprocess import sys import threading import time import uuid from urllib.parse import quote sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) from client import Client, Failure, arguments, check, run # noqa: E402 def pct(values, p): if not values: return None values = sorted(values) return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 2) def history(c, actor, chat): path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' items = [] while True: page = c.call('GET', path, actor=actor, schema='MessagePage')['json'] items.extend(page['items']) if page['next_cursor'] is None: return items path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='') def main(): def configure(p): p.add_argument('--duration', type=float, default=30) p.add_argument('--concurrency', type=int, default=4) p.add_argument('--action-after', type=float, default=8) p.add_argument('--action-script', default=None) p.add_argument('--target', default='auto', help='X-Instance-Id реплики для TARGET_INSTANCE_ID; auto = первая наблюдённая') p.add_argument('--label', default='') p.add_argument('--output', default=None) args = arguments('Load during API replica failure; creates synthetic data', configure) c = Client(args.base_url, args.ca_file, timeout=5) alice, bob, _eve, chat = c.bootstrap() observed = {} for _ in range(40): inst = c.call('GET', '/api/v1/users/me', actor=alice)['headers']['X-Instance-Id'] observed[inst] = observed.get(inst, 0) + 1 target = args.target if args.target != 'auto' else sorted(observed)[0] check(len(observed) >= 2, 'less than two replicas observed before the experiment') events = [] # (t, op, ok, latency, instance) confirmed = [] lock = threading.Lock() start = time.monotonic() started_wall = time.time() stop_at = start + args.duration def record(op, ok, latency, instance): with lock: events.append((time.monotonic() - start, op, ok, latency, instance)) def worker(wid): local = Client(args.base_url, args.ca_file, timeout=3) n = 0 key, text = uuid.uuid4().hex, None while time.monotonic() < stop_at: if text is None: text = 'fo-%s-%d-%d' % (args.label or 'x', wid, n) t0 = time.monotonic() try: if n % 2 == 0: r = local.message(alice, chat, text, key=key) record('write', True, time.monotonic() - t0, r['headers']['X-Instance-Id']) with lock: confirmed.append(r['json']) key, text = uuid.uuid4().hex, None # следующий запрос — новое сообщение else: r = local.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=20', actor=bob) record('read', True, time.monotonic() - t0, r['headers']['X-Instance-Id']) n += 1 except Failure: # Неопределённый результат: повторяем ТОТ ЖЕ key+text (запись) после паузы. record('write' if n % 2 == 0 else 'read', False, time.monotonic() - t0, None) if n % 2 == 1: n += 1 time.sleep(0.05) action = {} def run_action(): if not args.action_script: return time.sleep(args.action_after) env = dict(os.environ, TARGET_INSTANCE_ID=target) action['start'] = time.monotonic() - start proc = subprocess.run([args.action_script], env=env, capture_output=True, text=True) action['end'] = time.monotonic() - start action['exit_code'] = proc.returncode action['stdout_tail'] = proc.stdout.strip().splitlines()[-6:] with ThreadPoolExecutor(max_workers=args.concurrency + 1) as pool: futures = [pool.submit(worker, i) for i in range(args.concurrency)] act = pool.submit(run_action) for f in futures + [act]: f.result() elapsed = time.monotonic() - start check(action.get('exit_code', 0) == 0, 'action script failed: %s' % action.get('stdout_tail')) a0 = action.get('start', elapsed) a1 = action.get('end', a0) def phase(lo, hi): ev = [e for e in events if lo <= e[0] < hi] ok = [e for e in ev if e[2]] dur = max(hi - lo, 1e-9) return {'seconds': round(dur, 2), 'successful_rps': round(len(ok) / dur, 1), 'errors': len(ev) - len(ok), 'error_ratio': round((len(ev) - len(ok)) / len(ev), 4) if ev else None, 'p50_ms': pct([e[3] for e in ok], .5), 'p95_ms': pct([e[3] for e in ok], .95), 'p99_ms': pct([e[3] for e in ok], .99)} errors = [e for e in events if not e[2]] writes_ok = sorted(e[0] for e in events if e[1] == 'write' and e[2]) gaps = [b - a for a, b in zip(writes_ok, writes_ok[1:]) if b >= a0 - 1] from_target_after = [e[0] for e in events if e[2] and e[4] == target and e[0] >= a0] per_second = [] for s in range(int(math.ceil(elapsed))): ev = [e for e in events if s <= e[0] < s + 1] per_second.append({'t': s, 'ok': sum(1 for e in ev if e[2]), 'err': sum(1 for e in ev if not e[2])}) instances_after = {} for e in events: if e[2] and e[0] >= a1: instances_after[e[4]] = instances_after.get(e[4], 0) + 1 # Сохранность: каждое подтверждённое сообщение есть в истории, дублей текстов нет. msgs = history(c, bob, chat) ids = [m['id'] for m in msgs] texts = [m['text'] for m in msgs] check(len(ids) == len(set(ids)), 'duplicate message ids') check(len(texts) == len(set(texts)), 'retry with the same key created a duplicate message') by_id = {m['id']: m for m in msgs} lost = [m for m in confirmed if by_id.get(m['id']) != m] check(not lost, '%d confirmed messages lost/changed' % len(lost)) report = { 'label': args.label, 'action_script': args.action_script, 'target_instance': target, 'replicas_observed_before': len(observed), 'concurrency': args.concurrency, 'workload': 'closed loop, 50% POST message (retry same key on failure) / 50% GET history, timeout 3 s', 'started_at_utc': time.strftime('%H:%M:%S', time.gmtime(started_wall)), 'duration_seconds': round(elapsed, 2), 'action_window_seconds': [round(a0, 2), round(a1, 2)], 'action_stdout': action.get('stdout_tail'), 'before': phase(0, a0), 'during_action': phase(a0, a1), 'after_action': phase(a1, elapsed), 'total_errors': len(errors), 'last_error_after_action_start_s': round(max(e[0] for e in errors) - a0, 2) if errors else 0.0, 'max_gap_between_successful_writes_s': round(max(gaps), 3) if gaps else None, 'target_last_served_after_action_start_s': round(max(from_target_after) - a0, 2) if from_target_after else None, 'instances_after_action': instances_after, 'confirmed_writes': len(confirmed), 'lost_confirmed': 0, 'duplicates': 0, 'per_second': per_second, } out = json.dumps(report, ensure_ascii=False, indent=2) if args.output: Path(args.output).write_text(out + '\n', encoding='utf-8') brief = {k: report[k] for k in ('label', 'target_instance', 'before', 'during_action', 'after_action', 'total_errors', 'last_error_after_action_start_s', 'max_gap_between_successful_writes_s', 'target_last_served_after_action_start_s', 'instances_after_action', 'confirmed_writes')} print(json.dumps(brief, ensure_ascii=False)) print('PASS: no confirmed message lost, no duplicates') if __name__ == '__main__': run(main)