forked from mtusi2026/mtusi2026lab5
121 lines
6.0 KiB
Python
121 lines
6.0 KiB
Python
"""Фоновая нагрузка для замеров во время отказа и rolling update
|
|
(используют failover_metrics.py и rolling_update.py). Только стандартная библиотека.
|
|
|
|
WRITERS потоков без пауз пишут сообщения в один чат (closed loop: следующий запрос — сразу после
|
|
ответа на предыдущий). Если запрос не подтверждён (обрыв, timeout, 5xx), поток повторяет ТОТ ЖЕ
|
|
запрос с ТЕМ ЖЕ Idempotency-Key — так по контракту делает клиент. Один поток параллельно читает
|
|
историю. Каждая попытка записывается: время, успех, код, реплика, задержка.
|
|
В конце verify() читает всю историю и проверяет, что все подтверждённые сообщения на месте
|
|
и дублей нет (тексты уникальны, поэтому дубль сразу виден).
|
|
"""
|
|
import math
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
from helpers import Client, check, read_history
|
|
from stand import try_call
|
|
|
|
WRITERS = 4
|
|
CLIENT_TIMEOUT_SECONDS = 2
|
|
|
|
|
|
class Load:
|
|
def __init__(self, writers=WRITERS, timeout=CLIENT_TIMEOUT_SECONDS):
|
|
self.api = Client()
|
|
self.alice, self.bob = self.api.user("load_a"), self.api.user("load_b")
|
|
self.chat = self.api.call("POST", "/api/v1/chats", 201, actor=self.alice, schema="Chat",
|
|
body={"title": "load", "member_ids": [self.alice["id"], self.bob["id"]]})["json"]
|
|
self.writers, self.timeout = writers, timeout
|
|
self.events = [] # (время старта, "write"/"read", ok, код или "conn", реплика, задержка в с)
|
|
self.confirmed = [] # сообщения, на которые пришёл 201
|
|
self.lock = threading.Lock()
|
|
self.stopping = threading.Event()
|
|
self.threads = []
|
|
|
|
def _record(self, kind, started, reply, ok):
|
|
code = reply.status if reply is not None else "conn"
|
|
instance = reply.instance if reply is not None else None
|
|
with self.lock:
|
|
self.events.append((started, kind, ok, code, instance, time.monotonic() - started))
|
|
|
|
def _writer(self, number):
|
|
path = f"/api/v1/chats/{self.chat['id']}/messages"
|
|
sent = 0
|
|
key, text = uuid.uuid4().hex, f"load-{number}-{sent}"
|
|
while not self.stopping.is_set():
|
|
started = time.monotonic()
|
|
reply = try_call("POST", path, body={"text": text}, token=self.alice["token"],
|
|
headers={"Idempotency-Key": key}, timeout=self.timeout)
|
|
ok = reply is not None and reply.status == 201
|
|
self._record("write", started, reply, ok)
|
|
if ok:
|
|
with self.lock:
|
|
self.confirmed.append(reply.json)
|
|
sent += 1
|
|
key, text = uuid.uuid4().hex, f"load-{number}-{sent}"
|
|
else:
|
|
time.sleep(0.05) # и повторить тот же text с тем же ключом
|
|
|
|
def _reader(self):
|
|
path = f"/api/v1/chats/{self.chat['id']}/messages?limit=20"
|
|
while not self.stopping.is_set():
|
|
started = time.monotonic()
|
|
reply = try_call("GET", path, token=self.bob["token"], timeout=self.timeout)
|
|
self._record("read", started, reply, reply is not None and reply.status == 200)
|
|
time.sleep(0.05)
|
|
|
|
def start(self):
|
|
self.threads = [threading.Thread(target=self._writer, args=(n,), daemon=True) for n in range(self.writers)]
|
|
self.threads.append(threading.Thread(target=self._reader, daemon=True))
|
|
for thread in self.threads:
|
|
thread.start()
|
|
|
|
def stop(self):
|
|
self.stopping.set()
|
|
for thread in self.threads:
|
|
thread.join(timeout=self.timeout + 5)
|
|
|
|
def window(self, start, end, kind=None):
|
|
with self.lock:
|
|
return [e for e in self.events if start <= e[0] < end and (kind is None or e[1] == kind)]
|
|
|
|
def verify(self):
|
|
"""Все подтверждённые сообщения есть в истории, id и тексты не повторяются."""
|
|
history, _ = read_history(self.api, self.bob, self.chat["id"])
|
|
ids = [message["id"] for message in history]
|
|
texts = [message["text"] for message in history]
|
|
by_id = {message["id"]: message for message in history}
|
|
check(len(ids) == len(set(ids)), "duplicated message ids in history")
|
|
check(len(texts) == len(set(texts)), "a retried request created a duplicate message")
|
|
lost = [m for m in self.confirmed if by_id.get(m["id"]) != m]
|
|
check(not lost, f"{len(lost)} confirmed messages are lost or changed")
|
|
return {"confirmed_writes": len(self.confirmed), "messages_in_history": len(history),
|
|
"duplicates": 0, "lost": 0}
|
|
|
|
|
|
def summary(load, start, end):
|
|
"""Успешные записи в секунду, ошибки по кодам и задержки успешных записей на отрезке [start, end)."""
|
|
events = load.window(start, end)
|
|
writes = [e for e in events if e[1] == "write"]
|
|
ok_writes = sorted(e[5] for e in writes if e[2])
|
|
errors = [e for e in events if not e[2]]
|
|
seconds = max(end - start, 1e-6)
|
|
|
|
def percentile(p):
|
|
return round(1000 * ok_writes[max(0, math.ceil(p * len(ok_writes)) - 1)], 1) if ok_writes else None
|
|
|
|
codes = {}
|
|
for e in errors:
|
|
codes[str(e[3])] = codes.get(str(e[3]), 0) + 1
|
|
return {
|
|
"seconds": round(seconds, 2),
|
|
"write_rps": round(len(ok_writes) / seconds, 1),
|
|
"reads": len([e for e in events if e[1] == "read"]),
|
|
"errors": len(errors),
|
|
"error_codes": codes,
|
|
"write_p50_ms": percentile(0.50),
|
|
"write_p95_ms": percentile(0.95),
|
|
"write_max_ms": round(1000 * ok_writes[-1], 1) if ok_writes else None,
|
|
}
|