#!/usr/bin/env python3 """Эксперимент отказа хранилищ под нагрузкой (уровень 4). Стенд должен быть запущен. redis Redis останавливается на --down секунд: ожидаем 0 ошибок (сессии из PostgreSQL), после запуска кэш снова заполняется. postgres PostgreSQL останавливается: ожидаем быстрый ограниченный 503 (не зависание), живой /health/live, 503 /health/ready и восстановление без перезапуска API. Нагрузка: потоки по кругу делают GET /users/me (путь проверки сессии) и POST сообщения. Результат (без токенов) пишется в evidence/failure-.json. """ import argparse import json import math import subprocess import sys import threading import time import uuid from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure # noqa: E402 REDIS_CLI = 'redis-cli -a "$(cat /run/secrets/redis_password)" --no-auth-warning' def compose(*args, timeout=120): proc = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout) return proc.returncode, (proc.stdout + proc.stderr).strip() def redis_keys(): code, out = compose('exec', '-T', 'redis', 'sh', '-c', REDIS_CLI + ' --scan --pattern "session:*" | wc -l') return int(out.split()[-1]) if code == 0 and out.split() and out.split()[-1].isdigit() else None def percentile(values, p): values = sorted(values) return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 1) if values else None def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument('service', choices=['redis', 'postgres']) p.add_argument('--base-url', default='http://localhost:8080') p.add_argument('--workers', type=int, default=4) p.add_argument('--before', type=float, default=5) p.add_argument('--down', type=float, default=10) p.add_argument('--after', type=float, default=10) p.add_argument('--output', default=str(ROOT / 'evidence')) args = p.parse_args() c = Client(args.base_url) alice, bob, eve, chat = c.bootstrap() path = '/api/v1/chats/' + chat['id'] + '/messages' events, lock, stop = [], threading.Lock(), threading.Event() t0 = time.monotonic() def record(kind, start, status): with lock: events.append((round(start - t0, 3), round(time.monotonic() - t0, 3), kind, status)) def worker(): local = Client(args.base_url, timeout=5) while not stop.is_set(): for kind, call in ( ('me', lambda: local.call('GET', '/api/v1/users/me', [200, 503], actor=alice)), ('write', lambda: local.call('POST', path, [201, 503], actor=alice, body={'text': 'отказ ' + uuid.uuid4().hex})), ): start = time.monotonic() try: record(kind, start, call()['status']) except Failure: record(kind, start, 'error') threads = [threading.Thread(target=worker, daemon=True) for _ in range(args.workers)] for t in threads: t.start() time.sleep(args.before) keys_before = redis_keys() if args.service == 'redis' else None t_stop = time.monotonic() - t0 code, out = compose('stop', args.service) stop_seconds = round(time.monotonic() - t0 - t_stop, 2) health_down = {} time.sleep(2) for name in ('live', 'ready'): try: health_down[name] = c.call('GET', '/health/' + name, [200, 503])['status'] except Failure as exc: health_down[name] = str(exc) time.sleep(max(0, args.down - 2)) t_start = time.monotonic() - t0 compose('start', args.service) if args.service == 'postgres': compose('up', '-d', '--wait', 'postgres', timeout=120) time.sleep(args.after) stop.set() for t in threads: t.join(timeout=10) end = time.monotonic() - t0 keys_after = redis_keys() if args.service == 'redis' else None _, logs = compose('logs', '--since', '2m', 'api') warnings = [line.split('] ', 3)[-1] for line in logs.splitlines() if 'Redis недоступен' in line] def phase(lo, hi): sel = [e for e in events if lo <= e[0] < hi] result = {'seconds': round(hi - lo, 1)} for kind in ('me', 'write'): k = [e for e in sel if e[2] == kind] statuses = {} for e in k: statuses[str(e[3])] = statuses.get(str(e[3]), 0) + 1 ok = [e[1] - e[0] for e in k if e[3] in (200, 201)] fail = [e[1] - e[0] for e in k if e[3] not in (200, 201)] result[kind] = {'requests': len(k), 'statuses': statuses, 'ok_latency_ms_p50': percentile(ok, .5), 'ok_latency_ms_p95': percentile(ok, .95), 'failed_latency_ms_max': percentile(fail, 1.0)} return result failures_after_start = [e for e in events if e[0] >= t_start and e[3] not in (200, 201)] first_ok_after_start = min((e[1] for e in events if e[0] >= t_start and e[2] == 'write' and e[3] == 201), default=None) first_fail = min((e[1] for e in events if e[0] >= t_stop and e[3] not in (200, 201)), default=None) report = { 'service': args.service, 'method': f'{args.workers} closed-loop workers: GET /users/me + POST message; client timeout 5 s; ' f'docker compose stop {args.service} at t={t_stop:.1f}s, start at t={t_start:.1f}s', 'docker_stop_seconds': stop_seconds, 'health_while_down': health_down, 'before': phase(0, t_stop), 'while_down': phase(t_stop, t_start), 'after_start': phase(t_start, end), 'first_failure_seconds_after_stop': round(first_fail - t_stop, 2) if first_fail else None, 'last_failure_seconds_after_start': round(max(e[1] for e in failures_after_start) - t_start, 2) if failures_after_start else 0.0, 'first_successful_write_seconds_after_start': round(first_ok_after_start - t_start, 2) if first_ok_after_start else None, 'redis_session_keys_before_after': [keys_before, keys_after], 'api_log_redis_warnings': warnings[:5], } out_dir = Path(args.output) out_dir.mkdir(parents=True, exist_ok=True) dest = out_dir / f'failure-{args.service}.json' dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') print(json.dumps(report, ensure_ascii=False, indent=2)) print('written', dest) if __name__ == '__main__': main()