189 lines
9.0 KiB
Python
189 lines
9.0 KiB
Python
#!/usr/bin/env python3
|
||
"""Нагрузка во время отказа/замены реплики с метриками по времени.
|
||
|
||
Потоки непрерывно пишут сообщения (как настоящий клиент: при неопределённом результате
|
||
повторяют POST с ТЕМ ЖЕ Idempotency-Key) и читают историю. Через --action-after секунд
|
||
запускается action-скрипт (stop-one.sh / kill-one.sh / rolling-update.sh) с TARGET_INSTANCE_ID.
|
||
|
||
Отчёт: успешные RPS до/во время/после, ошибки, p50/p95/p99, время до последней ошибки,
|
||
максимальный разрыв между успешными записями, распределение по репликам, проверка
|
||
сохранности подтверждённых сообщений и отсутствия дублей.
|
||
"""
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
import json
|
||
import math
|
||
import os
|
||
from pathlib import Path
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from urllib.parse import quote
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||
from client import Client, Failure, arguments, check, run # noqa: E402
|
||
|
||
|
||
def pct(values, p):
|
||
if not values:
|
||
return None
|
||
values = sorted(values)
|
||
return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 2)
|
||
|
||
|
||
def history(c, actor, chat):
|
||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
||
items = []
|
||
while True:
|
||
page = c.call('GET', path, actor=actor, schema='MessagePage')['json']
|
||
items.extend(page['items'])
|
||
if page['next_cursor'] is None:
|
||
return items
|
||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='')
|
||
|
||
|
||
def main():
|
||
def configure(p):
|
||
p.add_argument('--duration', type=float, default=30)
|
||
p.add_argument('--concurrency', type=int, default=4)
|
||
p.add_argument('--action-after', type=float, default=8)
|
||
p.add_argument('--action-script', default=None)
|
||
p.add_argument('--target', default='auto',
|
||
help='X-Instance-Id реплики для TARGET_INSTANCE_ID; auto = первая наблюдённая')
|
||
p.add_argument('--label', default='')
|
||
p.add_argument('--output', default=None)
|
||
args = arguments('Load during API replica failure; creates synthetic data', configure)
|
||
c = Client(args.base_url, args.ca_file, timeout=5)
|
||
alice, bob, _eve, chat = c.bootstrap()
|
||
|
||
observed = {}
|
||
for _ in range(40):
|
||
inst = c.call('GET', '/api/v1/users/me', actor=alice)['headers']['X-Instance-Id']
|
||
observed[inst] = observed.get(inst, 0) + 1
|
||
target = args.target if args.target != 'auto' else sorted(observed)[0]
|
||
check(len(observed) >= 2, 'less than two replicas observed before the experiment')
|
||
|
||
events = [] # (t, op, ok, latency, instance)
|
||
confirmed = []
|
||
lock = threading.Lock()
|
||
start = time.monotonic()
|
||
started_wall = time.time()
|
||
stop_at = start + args.duration
|
||
|
||
def record(op, ok, latency, instance):
|
||
with lock:
|
||
events.append((time.monotonic() - start, op, ok, latency, instance))
|
||
|
||
def worker(wid):
|
||
local = Client(args.base_url, args.ca_file, timeout=3)
|
||
n = 0
|
||
key, text = uuid.uuid4().hex, None
|
||
while time.monotonic() < stop_at:
|
||
if text is None:
|
||
text = 'fo-%s-%d-%d' % (args.label or 'x', wid, n)
|
||
t0 = time.monotonic()
|
||
try:
|
||
if n % 2 == 0:
|
||
r = local.message(alice, chat, text, key=key)
|
||
record('write', True, time.monotonic() - t0, r['headers']['X-Instance-Id'])
|
||
with lock:
|
||
confirmed.append(r['json'])
|
||
key, text = uuid.uuid4().hex, None # следующий запрос — новое сообщение
|
||
else:
|
||
r = local.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=20', actor=bob)
|
||
record('read', True, time.monotonic() - t0, r['headers']['X-Instance-Id'])
|
||
n += 1
|
||
except Failure:
|
||
# Неопределённый результат: повторяем ТОТ ЖЕ key+text (запись) после паузы.
|
||
record('write' if n % 2 == 0 else 'read', False, time.monotonic() - t0, None)
|
||
if n % 2 == 1:
|
||
n += 1
|
||
time.sleep(0.05)
|
||
|
||
action = {}
|
||
|
||
def run_action():
|
||
if not args.action_script:
|
||
return
|
||
time.sleep(args.action_after)
|
||
env = dict(os.environ, TARGET_INSTANCE_ID=target)
|
||
action['start'] = time.monotonic() - start
|
||
proc = subprocess.run([args.action_script], env=env, capture_output=True, text=True)
|
||
action['end'] = time.monotonic() - start
|
||
action['exit_code'] = proc.returncode
|
||
action['stdout_tail'] = proc.stdout.strip().splitlines()[-6:]
|
||
|
||
with ThreadPoolExecutor(max_workers=args.concurrency + 1) as pool:
|
||
futures = [pool.submit(worker, i) for i in range(args.concurrency)]
|
||
act = pool.submit(run_action)
|
||
for f in futures + [act]:
|
||
f.result()
|
||
elapsed = time.monotonic() - start
|
||
check(action.get('exit_code', 0) == 0, 'action script failed: %s' % action.get('stdout_tail'))
|
||
|
||
a0 = action.get('start', elapsed)
|
||
a1 = action.get('end', a0)
|
||
|
||
def phase(lo, hi):
|
||
ev = [e for e in events if lo <= e[0] < hi]
|
||
ok = [e for e in ev if e[2]]
|
||
dur = max(hi - lo, 1e-9)
|
||
return {'seconds': round(dur, 2), 'successful_rps': round(len(ok) / dur, 1),
|
||
'errors': len(ev) - len(ok), 'error_ratio': round((len(ev) - len(ok)) / len(ev), 4) if ev else None,
|
||
'p50_ms': pct([e[3] for e in ok], .5), 'p95_ms': pct([e[3] for e in ok], .95),
|
||
'p99_ms': pct([e[3] for e in ok], .99)}
|
||
|
||
errors = [e for e in events if not e[2]]
|
||
writes_ok = sorted(e[0] for e in events if e[1] == 'write' and e[2])
|
||
gaps = [b - a for a, b in zip(writes_ok, writes_ok[1:]) if b >= a0 - 1]
|
||
from_target_after = [e[0] for e in events if e[2] and e[4] == target and e[0] >= a0]
|
||
per_second = []
|
||
for s in range(int(math.ceil(elapsed))):
|
||
ev = [e for e in events if s <= e[0] < s + 1]
|
||
per_second.append({'t': s, 'ok': sum(1 for e in ev if e[2]), 'err': sum(1 for e in ev if not e[2])})
|
||
instances_after = {}
|
||
for e in events:
|
||
if e[2] and e[0] >= a1:
|
||
instances_after[e[4]] = instances_after.get(e[4], 0) + 1
|
||
|
||
# Сохранность: каждое подтверждённое сообщение есть в истории, дублей текстов нет.
|
||
msgs = history(c, bob, chat)
|
||
ids = [m['id'] for m in msgs]
|
||
texts = [m['text'] for m in msgs]
|
||
check(len(ids) == len(set(ids)), 'duplicate message ids')
|
||
check(len(texts) == len(set(texts)), 'retry with the same key created a duplicate message')
|
||
by_id = {m['id']: m for m in msgs}
|
||
lost = [m for m in confirmed if by_id.get(m['id']) != m]
|
||
check(not lost, '%d confirmed messages lost/changed' % len(lost))
|
||
|
||
report = {
|
||
'label': args.label, 'action_script': args.action_script, 'target_instance': target,
|
||
'replicas_observed_before': len(observed), 'concurrency': args.concurrency,
|
||
'workload': 'closed loop, 50% POST message (retry same key on failure) / 50% GET history, timeout 3 s',
|
||
'started_at_utc': time.strftime('%H:%M:%S', time.gmtime(started_wall)),
|
||
'duration_seconds': round(elapsed, 2),
|
||
'action_window_seconds': [round(a0, 2), round(a1, 2)], 'action_stdout': action.get('stdout_tail'),
|
||
'before': phase(0, a0), 'during_action': phase(a0, a1), 'after_action': phase(a1, elapsed),
|
||
'total_errors': len(errors),
|
||
'last_error_after_action_start_s': round(max(e[0] for e in errors) - a0, 2) if errors else 0.0,
|
||
'max_gap_between_successful_writes_s': round(max(gaps), 3) if gaps else None,
|
||
'target_last_served_after_action_start_s': round(max(from_target_after) - a0, 2) if from_target_after else None,
|
||
'instances_after_action': instances_after,
|
||
'confirmed_writes': len(confirmed), 'lost_confirmed': 0, 'duplicates': 0,
|
||
'per_second': per_second,
|
||
}
|
||
out = json.dumps(report, ensure_ascii=False, indent=2)
|
||
if args.output:
|
||
Path(args.output).write_text(out + '\n', encoding='utf-8')
|
||
brief = {k: report[k] for k in ('label', 'target_instance', 'before', 'during_action', 'after_action',
|
||
'total_errors', 'last_error_after_action_start_s',
|
||
'max_gap_between_successful_writes_s', 'target_last_served_after_action_start_s',
|
||
'instances_after_action', 'confirmed_writes')}
|
||
print(json.dumps(brief, ensure_ascii=False))
|
||
print('PASS: no confirmed message lost, no duplicates')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
run(main)
|