Files
mtusi2026lab4/scripts/benchmark.py
T
2026-10-16 22:47:54 +03:00

157 lines
7.7 KiB
Python

"""Один профиль нагрузки для всех серий. Только синтетические данные, без токенов в отчёте."""
from concurrent.futures import ThreadPoolExecutor
from collections import Counter
import math
from pathlib import Path
import sys
import threading
import time
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, Failure
def context(base):
c = Client(base)
alice, bob, _, chat = c.bootstrap()
return dict(actor=alice, reader=bob, chat=chat)
def percentile(values, p):
if not values:
return None
return round(sorted(values)[max(0, math.ceil(p * len(values)) - 1)] * 1000, 3)
def measure(base, ctx, duration, concurrency, action=None, retry_writes=False):
gate = threading.Event()
started = time.monotonic()
deadline = started + duration
action_events = []
def perform_action():
nonlocal deadline
time.sleep(2)
start = time.monotonic()
try:
info = action()
deadline = max(deadline, time.monotonic() + 5)
action_events.append(dict(start_seconds=start-started, end_seconds=time.monotonic()-started,
result=info))
except Exception as exc:
action_events.append(dict(start_seconds=start-started, error=str(exc)))
def worker(worker):
c = Client(base, timeout=3)
gate.wait()
latencies, confirmed, bins = [], [], Counter()
errors, instances, failures, operation_counts = Counter(), Counter(), [], Counter()
step, pending = worker, None
while time.monotonic() < deadline or (action_thread and action_thread.is_alive()):
if time.monotonic() - started > 180:
raise RuntimeError('Infrastructure action exceeded workload time limit')
begin = time.monotonic()
# Один и тот же цикл: 60% POST, 20% история, 20% профиль.
operation = 'write' if pending or step % 5 < 3 else ('history' if step % 5 == 3 else 'profile')
operation_counts[operation] += 1
try:
if operation == 'write':
key, text = pending or (uuid.uuid4().hex, 'x' * 128)
response = c.message(ctx['actor'], ctx['chat'], text, key=key)
confirmed.append(response['json'])
pending = None
elif operation == 'history':
response = c.call('GET', '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=50',
actor=ctx['reader'], schema='MessagePage')
else:
response = c.call('GET', '/api/v1/users/me', actor=ctx['actor'], schema='User')
elapsed = time.monotonic() - begin
latencies.append(elapsed)
instances[response['headers']['X-Instance-Id']] += 1
bins[int(time.monotonic() - started)] += 1
except Failure as exc:
# Сообщения ошибки клиента не содержат credentials/body.
errors[str(exc)] += 1
failures.append(round(time.monotonic()-started, 3))
if operation == 'write' and retry_writes:
pending = (key, text)
step += 1
# Последний timeout мог произойти после COMMIT. Уточняем результат тем же
# ключом вне измеряемого окна, иначе проверка ошибочно сочтёт его дубликатом.
finished = time.monotonic()
resolved = 0
if pending:
for attempt in range(10):
try:
confirmed.append(c.message(ctx['actor'], ctx['chat'], pending[1], key=pending[0])['json'])
pending = None
resolved = 1
break
except Failure:
time.sleep(.5)
if pending:
raise RuntimeError('Final uncertain write could not be resolved')
return dict(latencies=latencies, confirmed=confirmed, errors=errors, instances=instances,
failures=failures, bins=bins, operation_counts=operation_counts,
finished=finished, resolved_outside_window=resolved)
action_thread = threading.Thread(target=perform_action) if action else None
with ThreadPoolExecutor(max_workers=concurrency) as pool:
jobs = [pool.submit(worker, i) for i in range(concurrency)]
started = time.monotonic()
deadline = started + duration
if action_thread:
action_thread.start()
gate.set()
results = [job.result() for job in jobs]
elapsed = max(r['finished'] for r in results) - started
if action_thread:
action_thread.join(timeout=120)
if action_thread.is_alive():
raise RuntimeError('Infrastructure action did not finish')
if any('error' in e for e in action_events):
raise RuntimeError(str(action_events))
values = [x for r in results for x in r['latencies']]
errors, instances, bins, operation_counts = Counter(), Counter(), Counter(), Counter()
for r in results:
errors.update(r['errors']); instances.update(r['instances'])
bins.update(r['bins']); operation_counts.update(r['operation_counts'])
successes, failed = len(values), sum(errors.values())
report = dict(duration_requested=duration, elapsed_seconds=round(elapsed, 3), concurrency=concurrency,
profile='closed-loop: 60% POST text128, 20% first50 history, 20% profile',
retry_writes=retry_writes, successful_requests=successes, failed_attempts=failed,
uncertain_writes_resolved_after_window=sum(r['resolved_outside_window'] for r in results),
successful_rps=round(successes/elapsed, 3), error_ratio=failed/max(1, successes+failed),
successful_latency_ms={f'p{int(p*100)}': percentile(values, p) for p in [.5, .95, .99]},
instances=dict(instances), errors=dict(errors), operations=dict(operation_counts),
successful_per_second=dict(sorted(bins.items())),
failure_seconds=[x for r in results for x in r['failures']],
latency_samples_ms=[round(x*1000, 3) for x in values], action_events=action_events)
if not successes:
raise RuntimeError('No successful requests')
return report, [m for r in results for m in r['confirmed']]
def verify_confirmations(base, ctx, confirmations):
from urllib.parse import quote
c = Client(base)
path = '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=100'
stored = {}
while True:
page = c.call('GET', path, actor=ctx['reader'], schema='MessagePage')['json']
for message in page['items']:
if message['id'] in stored:
raise AssertionError('Duplicate ID in history')
stored[message['id']] = message
if page['next_cursor'] is None:
break
path = '/api/v1/chats/' + ctx['chat']['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'])
if not all(stored.get(m['id']) == m for m in confirmations):
raise AssertionError('Confirmed message missing after infrastructure action')
# В fault-сценарии нет seed. Каждый новый ключ должен породить одну строку.
expected = {m['id'] for m in confirmations}
if set(stored) - expected:
raise AssertionError('Retry created an extra message or left an unconfirmed result')
return dict(confirmed_messages=len(expected), stored_messages=len(stored), status='PASS')