Files
mtusi2026lab3/student-tests/failover_load.py
T
2026-10-13 19:11:11 +03:00

189 lines
9.0 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Нагрузка во время отказа/замены реплики с метриками по времени.
Потоки непрерывно пишут сообщения (как настоящий клиент: при неопределённом результате
повторяют POST с ТЕМ ЖЕ Idempotency-Key) и читают историю. Через --action-after секунд
запускается action-скрипт (stop-one.sh / kill-one.sh / rolling-update.sh) с TARGET_INSTANCE_ID.
Отчёт: успешные RPS до/во время/после, ошибки, p50/p95/p99, время до последней ошибки,
максимальный разрыв между успешными записями, распределение по репликам, проверка
сохранности подтверждённых сообщений и отсутствия дублей.
"""
from concurrent.futures import ThreadPoolExecutor
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import uuid
from urllib.parse import quote
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, Failure, arguments, check, run # noqa: E402
def pct(values, p):
if not values:
return None
values = sorted(values)
return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 2)
def history(c, actor, chat):
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
items = []
while True:
page = c.call('GET', path, actor=actor, schema='MessagePage')['json']
items.extend(page['items'])
if page['next_cursor'] is None:
return items
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='')
def main():
def configure(p):
p.add_argument('--duration', type=float, default=30)
p.add_argument('--concurrency', type=int, default=4)
p.add_argument('--action-after', type=float, default=8)
p.add_argument('--action-script', default=None)
p.add_argument('--target', default='auto',
help='X-Instance-Id реплики для TARGET_INSTANCE_ID; auto = первая наблюдённая')
p.add_argument('--label', default='')
p.add_argument('--output', default=None)
args = arguments('Load during API replica failure; creates synthetic data', configure)
c = Client(args.base_url, args.ca_file, timeout=5)
alice, bob, _eve, chat = c.bootstrap()
observed = {}
for _ in range(40):
inst = c.call('GET', '/api/v1/users/me', actor=alice)['headers']['X-Instance-Id']
observed[inst] = observed.get(inst, 0) + 1
target = args.target if args.target != 'auto' else sorted(observed)[0]
check(len(observed) >= 2, 'less than two replicas observed before the experiment')
events = [] # (t, op, ok, latency, instance)
confirmed = []
lock = threading.Lock()
start = time.monotonic()
started_wall = time.time()
stop_at = start + args.duration
def record(op, ok, latency, instance):
with lock:
events.append((time.monotonic() - start, op, ok, latency, instance))
def worker(wid):
local = Client(args.base_url, args.ca_file, timeout=3)
n = 0
key, text = uuid.uuid4().hex, None
while time.monotonic() < stop_at:
if text is None:
text = 'fo-%s-%d-%d' % (args.label or 'x', wid, n)
t0 = time.monotonic()
try:
if n % 2 == 0:
r = local.message(alice, chat, text, key=key)
record('write', True, time.monotonic() - t0, r['headers']['X-Instance-Id'])
with lock:
confirmed.append(r['json'])
key, text = uuid.uuid4().hex, None # следующий запрос — новое сообщение
else:
r = local.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=20', actor=bob)
record('read', True, time.monotonic() - t0, r['headers']['X-Instance-Id'])
n += 1
except Failure:
# Неопределённый результат: повторяем ТОТ ЖЕ key+text (запись) после паузы.
record('write' if n % 2 == 0 else 'read', False, time.monotonic() - t0, None)
if n % 2 == 1:
n += 1
time.sleep(0.05)
action = {}
def run_action():
if not args.action_script:
return
time.sleep(args.action_after)
env = dict(os.environ, TARGET_INSTANCE_ID=target)
action['start'] = time.monotonic() - start
proc = subprocess.run([args.action_script], env=env, capture_output=True, text=True)
action['end'] = time.monotonic() - start
action['exit_code'] = proc.returncode
action['stdout_tail'] = proc.stdout.strip().splitlines()[-6:]
with ThreadPoolExecutor(max_workers=args.concurrency + 1) as pool:
futures = [pool.submit(worker, i) for i in range(args.concurrency)]
act = pool.submit(run_action)
for f in futures + [act]:
f.result()
elapsed = time.monotonic() - start
check(action.get('exit_code', 0) == 0, 'action script failed: %s' % action.get('stdout_tail'))
a0 = action.get('start', elapsed)
a1 = action.get('end', a0)
def phase(lo, hi):
ev = [e for e in events if lo <= e[0] < hi]
ok = [e for e in ev if e[2]]
dur = max(hi - lo, 1e-9)
return {'seconds': round(dur, 2), 'successful_rps': round(len(ok) / dur, 1),
'errors': len(ev) - len(ok), 'error_ratio': round((len(ev) - len(ok)) / len(ev), 4) if ev else None,
'p50_ms': pct([e[3] for e in ok], .5), 'p95_ms': pct([e[3] for e in ok], .95),
'p99_ms': pct([e[3] for e in ok], .99)}
errors = [e for e in events if not e[2]]
writes_ok = sorted(e[0] for e in events if e[1] == 'write' and e[2])
gaps = [b - a for a, b in zip(writes_ok, writes_ok[1:]) if b >= a0 - 1]
from_target_after = [e[0] for e in events if e[2] and e[4] == target and e[0] >= a0]
per_second = []
for s in range(int(math.ceil(elapsed))):
ev = [e for e in events if s <= e[0] < s + 1]
per_second.append({'t': s, 'ok': sum(1 for e in ev if e[2]), 'err': sum(1 for e in ev if not e[2])})
instances_after = {}
for e in events:
if e[2] and e[0] >= a1:
instances_after[e[4]] = instances_after.get(e[4], 0) + 1
# Сохранность: каждое подтверждённое сообщение есть в истории, дублей текстов нет.
msgs = history(c, bob, chat)
ids = [m['id'] for m in msgs]
texts = [m['text'] for m in msgs]
check(len(ids) == len(set(ids)), 'duplicate message ids')
check(len(texts) == len(set(texts)), 'retry with the same key created a duplicate message')
by_id = {m['id']: m for m in msgs}
lost = [m for m in confirmed if by_id.get(m['id']) != m]
check(not lost, '%d confirmed messages lost/changed' % len(lost))
report = {
'label': args.label, 'action_script': args.action_script, 'target_instance': target,
'replicas_observed_before': len(observed), 'concurrency': args.concurrency,
'workload': 'closed loop, 50% POST message (retry same key on failure) / 50% GET history, timeout 3 s',
'started_at_utc': time.strftime('%H:%M:%S', time.gmtime(started_wall)),
'duration_seconds': round(elapsed, 2),
'action_window_seconds': [round(a0, 2), round(a1, 2)], 'action_stdout': action.get('stdout_tail'),
'before': phase(0, a0), 'during_action': phase(a0, a1), 'after_action': phase(a1, elapsed),
'total_errors': len(errors),
'last_error_after_action_start_s': round(max(e[0] for e in errors) - a0, 2) if errors else 0.0,
'max_gap_between_successful_writes_s': round(max(gaps), 3) if gaps else None,
'target_last_served_after_action_start_s': round(max(from_target_after) - a0, 2) if from_target_after else None,
'instances_after_action': instances_after,
'confirmed_writes': len(confirmed), 'lost_confirmed': 0, 'duplicates': 0,
'per_second': per_second,
}
out = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(out + '\n', encoding='utf-8')
brief = {k: report[k] for k in ('label', 'target_instance', 'before', 'during_action', 'after_action',
'total_errors', 'last_error_after_action_start_s',
'max_gap_between_successful_writes_s', 'target_last_served_after_action_start_s',
'instances_after_action', 'confirmed_writes')}
print(json.dumps(brief, ensure_ascii=False))
print('PASS: no confirmed message lost, no duplicates')
if __name__ == '__main__':
run(main)