157 lines
7.7 KiB
Python
157 lines
7.7 KiB
Python
"""Один профиль нагрузки для всех серий. Только синтетические данные, без токенов в отчёте."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from collections import Counter
|
|
import math
|
|
from pathlib import Path
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
from client import Client, Failure
|
|
|
|
|
|
def context(base):
|
|
c = Client(base)
|
|
alice, bob, _, chat = c.bootstrap()
|
|
return dict(actor=alice, reader=bob, chat=chat)
|
|
|
|
|
|
def percentile(values, p):
|
|
if not values:
|
|
return None
|
|
return round(sorted(values)[max(0, math.ceil(p * len(values)) - 1)] * 1000, 3)
|
|
|
|
|
|
def measure(base, ctx, duration, concurrency, action=None, retry_writes=False):
|
|
gate = threading.Event()
|
|
started = time.monotonic()
|
|
deadline = started + duration
|
|
action_events = []
|
|
|
|
def perform_action():
|
|
nonlocal deadline
|
|
time.sleep(2)
|
|
start = time.monotonic()
|
|
try:
|
|
info = action()
|
|
deadline = max(deadline, time.monotonic() + 5)
|
|
action_events.append(dict(start_seconds=start-started, end_seconds=time.monotonic()-started,
|
|
result=info))
|
|
except Exception as exc:
|
|
action_events.append(dict(start_seconds=start-started, error=str(exc)))
|
|
|
|
def worker(worker):
|
|
c = Client(base, timeout=3)
|
|
gate.wait()
|
|
latencies, confirmed, bins = [], [], Counter()
|
|
errors, instances, failures, operation_counts = Counter(), Counter(), [], Counter()
|
|
step, pending = worker, None
|
|
while time.monotonic() < deadline or (action_thread and action_thread.is_alive()):
|
|
if time.monotonic() - started > 180:
|
|
raise RuntimeError('Infrastructure action exceeded workload time limit')
|
|
begin = time.monotonic()
|
|
# Один и тот же цикл: 60% POST, 20% история, 20% профиль.
|
|
operation = 'write' if pending or step % 5 < 3 else ('history' if step % 5 == 3 else 'profile')
|
|
operation_counts[operation] += 1
|
|
try:
|
|
if operation == 'write':
|
|
key, text = pending or (uuid.uuid4().hex, 'x' * 128)
|
|
response = c.message(ctx['actor'], ctx['chat'], text, key=key)
|
|
confirmed.append(response['json'])
|
|
pending = None
|
|
elif operation == 'history':
|
|
response = c.call('GET', '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=50',
|
|
actor=ctx['reader'], schema='MessagePage')
|
|
else:
|
|
response = c.call('GET', '/api/v1/users/me', actor=ctx['actor'], schema='User')
|
|
elapsed = time.monotonic() - begin
|
|
latencies.append(elapsed)
|
|
instances[response['headers']['X-Instance-Id']] += 1
|
|
bins[int(time.monotonic() - started)] += 1
|
|
except Failure as exc:
|
|
# Сообщения ошибки клиента не содержат credentials/body.
|
|
errors[str(exc)] += 1
|
|
failures.append(round(time.monotonic()-started, 3))
|
|
if operation == 'write' and retry_writes:
|
|
pending = (key, text)
|
|
step += 1
|
|
# Последний timeout мог произойти после COMMIT. Уточняем результат тем же
|
|
# ключом вне измеряемого окна, иначе проверка ошибочно сочтёт его дубликатом.
|
|
finished = time.monotonic()
|
|
resolved = 0
|
|
if pending:
|
|
for attempt in range(10):
|
|
try:
|
|
confirmed.append(c.message(ctx['actor'], ctx['chat'], pending[1], key=pending[0])['json'])
|
|
pending = None
|
|
resolved = 1
|
|
break
|
|
except Failure:
|
|
time.sleep(.5)
|
|
if pending:
|
|
raise RuntimeError('Final uncertain write could not be resolved')
|
|
return dict(latencies=latencies, confirmed=confirmed, errors=errors, instances=instances,
|
|
failures=failures, bins=bins, operation_counts=operation_counts,
|
|
finished=finished, resolved_outside_window=resolved)
|
|
|
|
action_thread = threading.Thread(target=perform_action) if action else None
|
|
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
|
jobs = [pool.submit(worker, i) for i in range(concurrency)]
|
|
started = time.monotonic()
|
|
deadline = started + duration
|
|
if action_thread:
|
|
action_thread.start()
|
|
gate.set()
|
|
results = [job.result() for job in jobs]
|
|
elapsed = max(r['finished'] for r in results) - started
|
|
if action_thread:
|
|
action_thread.join(timeout=120)
|
|
if action_thread.is_alive():
|
|
raise RuntimeError('Infrastructure action did not finish')
|
|
if any('error' in e for e in action_events):
|
|
raise RuntimeError(str(action_events))
|
|
values = [x for r in results for x in r['latencies']]
|
|
errors, instances, bins, operation_counts = Counter(), Counter(), Counter(), Counter()
|
|
for r in results:
|
|
errors.update(r['errors']); instances.update(r['instances'])
|
|
bins.update(r['bins']); operation_counts.update(r['operation_counts'])
|
|
successes, failed = len(values), sum(errors.values())
|
|
report = dict(duration_requested=duration, elapsed_seconds=round(elapsed, 3), concurrency=concurrency,
|
|
profile='closed-loop: 60% POST text128, 20% first50 history, 20% profile',
|
|
retry_writes=retry_writes, successful_requests=successes, failed_attempts=failed,
|
|
uncertain_writes_resolved_after_window=sum(r['resolved_outside_window'] for r in results),
|
|
successful_rps=round(successes/elapsed, 3), error_ratio=failed/max(1, successes+failed),
|
|
successful_latency_ms={f'p{int(p*100)}': percentile(values, p) for p in [.5, .95, .99]},
|
|
instances=dict(instances), errors=dict(errors), operations=dict(operation_counts),
|
|
successful_per_second=dict(sorted(bins.items())),
|
|
failure_seconds=[x for r in results for x in r['failures']],
|
|
latency_samples_ms=[round(x*1000, 3) for x in values], action_events=action_events)
|
|
if not successes:
|
|
raise RuntimeError('No successful requests')
|
|
return report, [m for r in results for m in r['confirmed']]
|
|
|
|
|
|
def verify_confirmations(base, ctx, confirmations):
|
|
from urllib.parse import quote
|
|
c = Client(base)
|
|
path = '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=100'
|
|
stored = {}
|
|
while True:
|
|
page = c.call('GET', path, actor=ctx['reader'], schema='MessagePage')['json']
|
|
for message in page['items']:
|
|
if message['id'] in stored:
|
|
raise AssertionError('Duplicate ID in history')
|
|
stored[message['id']] = message
|
|
if page['next_cursor'] is None:
|
|
break
|
|
path = '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'])
|
|
if not all(stored.get(m['id']) == m for m in confirmations):
|
|
raise AssertionError('Confirmed message missing after infrastructure action')
|
|
# В fault-сценарии нет seed. Каждый новый ключ должен породить одну строку.
|
|
expected = {m['id'] for m in confirmations}
|
|
if set(stored) - expected:
|
|
raise AssertionError('Retry created an extra message or left an unconfirmed result')
|
|
return dict(confirmed_messages=len(expected), stored_messages=len(stored), status='PASS')
|