forked from mtusi2026/mtusi2026lab2
Уровень 4: - Redis как кэш сессий (cache-aside, источник истины — PostgreSQL): TTL <= остатка сессии, метка revoked при logout + SET NX против гонки, при отказе Redis — работа через базу и фоновая проверка доступности - тесты: гонка регистрации, откат посреди транзакции, недоступность БД и Redis (44 теста) - EXPLAIN (ANALYZE, BUFFERS) на 22 000 сообщений, history_query общий для API и скрипта - эксперименты отказа Redis/PostgreSQL под нагрузкой; dns_opt у API, чтобы отказ БД давал быстрый 503, а не зависание на поиске имени остановленного контейнера Уровень 5, трек 2: - 105 000 сообщений в неравномерных чатах, 4 варианта индекса + OFFSET против keyset, стоимость записи (время, WAL), два независимых прогона - pg_dump -> pg_restore в новый контейнер -> сверка md5 всех таблиц и выборки сообщений, RTO 14 с, демонстрация RPO
155 lines
6.9 KiB
Python
155 lines
6.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Эксперимент отказа хранилищ под нагрузкой (уровень 4). Стенд должен быть запущен.
|
|
|
|
redis Redis останавливается на --down секунд: ожидаем 0 ошибок (сессии из PostgreSQL),
|
|
после запуска кэш снова заполняется.
|
|
postgres PostgreSQL останавливается: ожидаем быстрый ограниченный 503 (не зависание), живой
|
|
/health/live, 503 /health/ready и восстановление без перезапуска API.
|
|
|
|
Нагрузка: потоки по кругу делают GET /users/me (путь проверки сессии) и POST сообщения.
|
|
Результат (без токенов) пишется в evidence/failure-<service>.json.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import math
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure # noqa: E402
|
|
|
|
REDIS_CLI = 'redis-cli -a "$(cat /run/secrets/redis_password)" --no-auth-warning'
|
|
|
|
|
|
def compose(*args, timeout=120):
|
|
proc = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True,
|
|
encoding='utf-8', errors='replace', timeout=timeout)
|
|
return proc.returncode, (proc.stdout + proc.stderr).strip()
|
|
|
|
|
|
def redis_keys():
|
|
code, out = compose('exec', '-T', 'redis', 'sh', '-c', REDIS_CLI + ' --scan --pattern "session:*" | wc -l')
|
|
return int(out.split()[-1]) if code == 0 and out.split() and out.split()[-1].isdigit() else None
|
|
|
|
|
|
def percentile(values, p):
|
|
values = sorted(values)
|
|
return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 1) if values else None
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument('service', choices=['redis', 'postgres'])
|
|
p.add_argument('--base-url', default='http://localhost:8080')
|
|
p.add_argument('--workers', type=int, default=4)
|
|
p.add_argument('--before', type=float, default=5)
|
|
p.add_argument('--down', type=float, default=10)
|
|
p.add_argument('--after', type=float, default=10)
|
|
p.add_argument('--output', default=str(ROOT / 'evidence'))
|
|
args = p.parse_args()
|
|
|
|
c = Client(args.base_url)
|
|
alice, bob, eve, chat = c.bootstrap()
|
|
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
|
events, lock, stop = [], threading.Lock(), threading.Event()
|
|
t0 = time.monotonic()
|
|
|
|
def record(kind, start, status):
|
|
with lock:
|
|
events.append((round(start - t0, 3), round(time.monotonic() - t0, 3), kind, status))
|
|
|
|
def worker():
|
|
local = Client(args.base_url, timeout=5)
|
|
while not stop.is_set():
|
|
for kind, call in (
|
|
('me', lambda: local.call('GET', '/api/v1/users/me', [200, 503], actor=alice)),
|
|
('write', lambda: local.call('POST', path, [201, 503], actor=alice,
|
|
body={'text': 'отказ ' + uuid.uuid4().hex})),
|
|
):
|
|
start = time.monotonic()
|
|
try:
|
|
record(kind, start, call()['status'])
|
|
except Failure:
|
|
record(kind, start, 'error')
|
|
|
|
threads = [threading.Thread(target=worker, daemon=True) for _ in range(args.workers)]
|
|
for t in threads:
|
|
t.start()
|
|
time.sleep(args.before)
|
|
keys_before = redis_keys() if args.service == 'redis' else None
|
|
t_stop = time.monotonic() - t0
|
|
code, out = compose('stop', args.service)
|
|
stop_seconds = round(time.monotonic() - t0 - t_stop, 2)
|
|
health_down = {}
|
|
time.sleep(2)
|
|
for name in ('live', 'ready'):
|
|
try:
|
|
health_down[name] = c.call('GET', '/health/' + name, [200, 503])['status']
|
|
except Failure as exc:
|
|
health_down[name] = str(exc)
|
|
time.sleep(max(0, args.down - 2))
|
|
t_start = time.monotonic() - t0
|
|
compose('start', args.service)
|
|
if args.service == 'postgres':
|
|
compose('up', '-d', '--wait', 'postgres', timeout=120)
|
|
time.sleep(args.after)
|
|
stop.set()
|
|
for t in threads:
|
|
t.join(timeout=10)
|
|
end = time.monotonic() - t0
|
|
keys_after = redis_keys() if args.service == 'redis' else None
|
|
_, logs = compose('logs', '--since', '2m', 'api')
|
|
warnings = [line.split('] ', 3)[-1] for line in logs.splitlines() if 'Redis недоступен' in line]
|
|
|
|
def phase(lo, hi):
|
|
sel = [e for e in events if lo <= e[0] < hi]
|
|
result = {'seconds': round(hi - lo, 1)}
|
|
for kind in ('me', 'write'):
|
|
k = [e for e in sel if e[2] == kind]
|
|
statuses = {}
|
|
for e in k:
|
|
statuses[str(e[3])] = statuses.get(str(e[3]), 0) + 1
|
|
ok = [e[1] - e[0] for e in k if e[3] in (200, 201)]
|
|
fail = [e[1] - e[0] for e in k if e[3] not in (200, 201)]
|
|
result[kind] = {'requests': len(k), 'statuses': statuses,
|
|
'ok_latency_ms_p50': percentile(ok, .5), 'ok_latency_ms_p95': percentile(ok, .95),
|
|
'failed_latency_ms_max': percentile(fail, 1.0)}
|
|
return result
|
|
|
|
failures_after_start = [e for e in events if e[0] >= t_start and e[3] not in (200, 201)]
|
|
first_ok_after_start = min((e[1] for e in events if e[0] >= t_start and e[2] == 'write'
|
|
and e[3] == 201), default=None)
|
|
first_fail = min((e[1] for e in events if e[0] >= t_stop and e[3] not in (200, 201)), default=None)
|
|
report = {
|
|
'service': args.service,
|
|
'method': f'{args.workers} closed-loop workers: GET /users/me + POST message; client timeout 5 s; '
|
|
f'docker compose stop {args.service} at t={t_stop:.1f}s, start at t={t_start:.1f}s',
|
|
'docker_stop_seconds': stop_seconds,
|
|
'health_while_down': health_down,
|
|
'before': phase(0, t_stop),
|
|
'while_down': phase(t_stop, t_start),
|
|
'after_start': phase(t_start, end),
|
|
'first_failure_seconds_after_stop': round(first_fail - t_stop, 2) if first_fail else None,
|
|
'last_failure_seconds_after_start': round(max(e[1] for e in failures_after_start) - t_start, 2)
|
|
if failures_after_start else 0.0,
|
|
'first_successful_write_seconds_after_start': round(first_ok_after_start - t_start, 2)
|
|
if first_ok_after_start else None,
|
|
'redis_session_keys_before_after': [keys_before, keys_after],
|
|
'api_log_redis_warnings': warnings[:5],
|
|
}
|
|
out_dir = Path(args.output)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
dest = out_dir / f'failure-{args.service}.json'
|
|
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
print('written', dest)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|