158 lines
6.1 KiB
Python
Executable File
158 lines
6.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Толерантный генератор записи для сценариев отказа (lab3).
|
|
|
|
Публичный `tests/load.py` не переживает разрыв соединения во время удаления
|
|
реплики: он падает на уровне http.client. Этот генератор создан для измерений
|
|
во время rolling update и отказов: он считает успешные запросы, ошибки по видам
|
|
и увиденные реплики, а не падает на обрыве.
|
|
|
|
python3 student-tests/rolling_update_load.py --duration 60 --concurrency 16 --output evidence/rolling-update/load.json
|
|
"""
|
|
import json
|
|
import math
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
|
|
from client import Client, LAB, Failure, arguments, check, run # noqa: E402
|
|
|
|
|
|
def classify(error):
|
|
name = type(error).__name__
|
|
message = str(error)
|
|
|
|
if name == 'Failure':
|
|
# client.call сообщает о проблемах соединения как Failure с этим текстом,
|
|
# отделяя их от HTTP-статусов, которые тоже приходят как Failure.
|
|
if 'connection/TLS failure' in message:
|
|
return 'connection_broken'
|
|
|
|
return 'http_error'
|
|
if name in ('RemoteDisconnected', 'BadStatusLine', 'IncompleteRead', 'ConnectionResetError'):
|
|
return 'connection_broken'
|
|
if name in ('TimeoutError', 'socket.timeout'):
|
|
return 'timeout'
|
|
|
|
return name
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument('--duration', type=float, default=30)
|
|
parser.add_argument('--concurrency', type=int, default=8)
|
|
parser.add_argument('--output', default=None)
|
|
|
|
args = arguments('Запись сообщений сквозь LB с учётом ошибок (lab3)', configure)
|
|
check(LAB >= 3, 'сценарий требует lab3+')
|
|
check(1 <= args.duration <= 3600, 'duration должен быть 1..3600')
|
|
check(1 <= args.concurrency <= 128, 'concurrency должен быть 1..128')
|
|
|
|
bootstrap = Client(args.base_url, args.ca_file)
|
|
|
|
# Стенд в момент старта может быть в переходном состоянии (например, сразу
|
|
# после возврата БД), поэтому однократный 502 на bootstrap не должен ломать
|
|
# измерение: пробуем несколько раз и сообщаем, сколько попыток понадобилось.
|
|
attempts = 0
|
|
while True:
|
|
attempts += 1
|
|
try:
|
|
alice, _, _, chat = bootstrap.bootstrap()
|
|
break
|
|
except Failure as error:
|
|
if attempts >= 10:
|
|
raise
|
|
print('bootstrap attempt %d failed: %s' % (attempts, error), file=sys.stderr)
|
|
time.sleep(2)
|
|
|
|
if attempts > 1:
|
|
print('bootstrap attempts: %d' % attempts, file=sys.stderr)
|
|
|
|
barrier = threading.Barrier(args.concurrency)
|
|
lock = threading.Lock()
|
|
latencies = []
|
|
errors = {}
|
|
instances = {}
|
|
|
|
def worker(worker_id):
|
|
client = Client(args.base_url, args.ca_file, timeout=5)
|
|
local_latencies = []
|
|
local_errors = {}
|
|
local_instances = {}
|
|
barrier.wait()
|
|
|
|
deadline = time.monotonic() + args.duration
|
|
index = 0
|
|
|
|
while time.monotonic() < deadline:
|
|
started = time.monotonic()
|
|
|
|
try:
|
|
response = client.message(alice, chat, 'rolling-%s-%s' % (worker_id, index), key=uuid.uuid4().hex)
|
|
except Exception as error: # noqa: BLE001 - протокол и сеть дают разные типы
|
|
category = classify(error)
|
|
local_errors[category] = local_errors.get(category, 0) + 1
|
|
else:
|
|
local_latencies.append(time.monotonic() - started)
|
|
instance = response['headers']['X-Instance-Id']
|
|
local_instances[instance] = local_instances.get(instance, 0) + 1
|
|
|
|
index += 1
|
|
|
|
with lock:
|
|
latencies.extend(local_latencies)
|
|
|
|
for category, count in local_errors.items():
|
|
errors[category] = errors.get(category, 0) + count
|
|
|
|
for instance, count in local_instances.items():
|
|
instances[instance] = instances.get(instance, 0) + count
|
|
|
|
started = time.monotonic()
|
|
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
list(pool.map(worker, range(args.concurrency)))
|
|
|
|
elapsed = time.monotonic() - started
|
|
ordered = sorted(latencies)
|
|
|
|
def percentile(value):
|
|
return round(1000 * ordered[max(0, math.ceil(value * len(ordered)) - 1)], 3) if ordered else None
|
|
|
|
failed = sum(errors.values())
|
|
|
|
report = {
|
|
'lab': LAB,
|
|
'workload': 'closed-loop POST messages; failures counted, not raised',
|
|
'requested_duration_seconds': args.duration,
|
|
'elapsed_seconds': round(elapsed, 3),
|
|
'concurrency': args.concurrency,
|
|
'successful_requests': len(ordered),
|
|
'failed_requests': failed,
|
|
'error_ratio': round(failed / (failed + len(ordered)), 6) if (failed + len(ordered)) else None,
|
|
'errors_by_kind': errors,
|
|
'successful_rps': round(len(ordered) / elapsed, 3),
|
|
'successful_latency_ms': {'p50': percentile(.50), 'p95': percentile(.95), 'p99': percentile(.99)},
|
|
'instances': instances,
|
|
'bootstrap_attempts': attempts,
|
|
'note': 'Percentiles cover successful writes only; transitional errors are expected during a rolling update.',
|
|
}
|
|
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
print(text)
|
|
|
|
if args.output:
|
|
destination = Path(args.output)
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(text + '\n')
|
|
|
|
check(bool(ordered), 'не было ни одной успешной записи')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|