forked from mtusi2026/mtusi2026lab3
170 lines
7.3 KiB
Python
170 lines
7.3 KiB
Python
"""Фоновая нагрузка для сценариев отказа/обновления (только стандартная библиотека).
|
||
|
||
Писатели: POST сообщения с Idempotency-Key. Неопределённый результат (обрыв,
|
||
timeout, 5xx) повторяется с ТЕМ ЖЕ ключом и телом, как требует контракт, пока
|
||
не будет 201. Читатели: GET истории и /users/me. Параллельно опрашивается
|
||
страница состояния HAProxy (UP/DOWN реплик), чтобы измерить время обнаружения.
|
||
|
||
Каждая попытка записывается как событие (время от старта, тип, успех, статус,
|
||
X-Instance-Id, задержка). По событиям считаются RPS/ошибки/перцентили в окнах.
|
||
"""
|
||
import csv
|
||
import io
|
||
import math
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from urllib.parse import quote
|
||
from urllib.request import urlopen
|
||
|
||
import common
|
||
from common import actor, create_chat, request
|
||
|
||
|
||
def percentile(values, p):
|
||
if not values:
|
||
return None
|
||
values = sorted(values)
|
||
return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 1)
|
||
|
||
|
||
class Load:
|
||
def __init__(self, base_url, writers=4, readers=2, timeout=2.0, stats_url=None):
|
||
common.BASE_URL = base_url.rstrip("/")
|
||
self.writers, self.readers, self.timeout = writers, readers, timeout
|
||
self.stats_url = stats_url
|
||
self.events = [] # (t, kind, ok, status, instance, latency)
|
||
self.lb = [] # (t, tuple(sorted UP servers))
|
||
self.confirmed = []
|
||
self.same_key_retries = 0
|
||
self._lock = threading.Lock()
|
||
self._stop = threading.Event()
|
||
self._threads = []
|
||
self.alice = actor("lda")
|
||
self.bob = actor("ldb")
|
||
self.chat = create_chat(self.alice, self.bob)
|
||
self.t0 = None
|
||
|
||
def now(self):
|
||
return time.monotonic() - self.t0
|
||
|
||
def _record(self, kind, ok, status, instance, latency):
|
||
with self._lock:
|
||
self.events.append((self.now(), kind, ok, status, instance, latency))
|
||
|
||
def _call(self, kind, method, path, token, payload=None, headers=None):
|
||
started = time.monotonic()
|
||
try:
|
||
r = request(method, path, token=token, payload=payload, headers=headers, timeout=self.timeout)
|
||
ok = r.status < 500
|
||
self._record(kind, ok, r.status, r.headers.get("X-Instance-Id"), time.monotonic() - started)
|
||
return r
|
||
except OSError as exc: # обрыв соединения / timeout — результат неизвестен
|
||
self._record(kind, False, type(exc).__name__, None, time.monotonic() - started)
|
||
return None
|
||
|
||
def _writer(self, wid):
|
||
n = 0
|
||
path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||
while not self._stop.is_set():
|
||
key = "lg-%d-%d-%s" % (wid, n, uuid.uuid4().hex[:8])
|
||
body = {"text": "load %d/%d %s" % (wid, n, key)}
|
||
for attempt in range(200):
|
||
r = self._call("w", "POST", path, self.alice["_token"], body, {"Idempotency-Key": key})
|
||
if r is not None and r.status == 201:
|
||
with self._lock:
|
||
self.confirmed.append(r.json)
|
||
break
|
||
if r is not None and r.status < 500:
|
||
raise AssertionError("unexpected %s on write: %s" % (r.status, r.json))
|
||
with self._lock:
|
||
self.same_key_retries += 1
|
||
time.sleep(0.05) # повтор с тем же ключом и телом
|
||
n += 1
|
||
|
||
def _reader(self, rid):
|
||
i = 0
|
||
while not self._stop.is_set():
|
||
if i % 2:
|
||
self._call("r", "GET", "/api/v1/users/me", self.bob["_token"])
|
||
else:
|
||
self._call("r", "GET", "/api/v1/chats/%s/messages?limit=20" % self.chat["id"], self.bob["_token"])
|
||
i += 1
|
||
|
||
def _lb_poller(self):
|
||
last = None
|
||
while not self._stop.is_set():
|
||
try:
|
||
data = urlopen(self.stats_url, timeout=1).read().decode("utf-8", "replace")
|
||
rows = csv.DictReader(io.StringIO(data.lstrip("# ")))
|
||
up = tuple(sorted(r["svname"] for r in rows if r.get("pxname") == "api"
|
||
and r.get("svname") not in ("BACKEND", "FRONTEND")
|
||
and r.get("status", "").startswith("UP")))
|
||
if up != last:
|
||
with self._lock:
|
||
self.lb.append((self.now(), up))
|
||
last = up
|
||
except Exception:
|
||
pass
|
||
time.sleep(0.1)
|
||
|
||
def start(self):
|
||
self.t0 = time.monotonic()
|
||
targets = [(self._writer, i) for i in range(self.writers)] + [(self._reader, i) for i in range(self.readers)]
|
||
if self.stats_url:
|
||
targets.append((self._lb_poller, None))
|
||
for fn, arg in targets:
|
||
t = threading.Thread(target=fn, args=(() if arg is None else (arg,)), daemon=True)
|
||
t.start()
|
||
self._threads.append(t)
|
||
return self
|
||
|
||
def stop(self):
|
||
self._stop.set()
|
||
for t in self._threads:
|
||
t.join(timeout=30)
|
||
|
||
def window(self, a, b):
|
||
ev = [e for e in self.events if a <= e[0] < b]
|
||
ok = [e for e in ev if e[2]]
|
||
dur = max(b - a, 1e-9)
|
||
lat = [e[5] for e in ok]
|
||
return {
|
||
"from_s": round(a, 2), "to_s": round(b, 2),
|
||
"requests": len(ev), "ok": len(ok), "errors": len(ev) - len(ok),
|
||
"error_ratio": round((len(ev) - len(ok)) / len(ev), 4) if ev else None,
|
||
"ok_rps": round(len(ok) / dur, 1),
|
||
"write_ok_rps": round(sum(1 for e in ok if e[1] == "w") / dur, 1),
|
||
"latency_ms_ok": {"p50": percentile(lat, .5), "p95": percentile(lat, .95), "p99": percentile(lat, .99)},
|
||
"instances": sorted({e[4] for e in ok if e[4]}),
|
||
}
|
||
|
||
def errors_between(self, a, b):
|
||
return [e for e in self.events if a <= e[0] < b and not e[2]]
|
||
|
||
def verify(self):
|
||
"""Все подтверждённые сообщения на месте, дублей (в т.ч. от повторов) нет."""
|
||
items, cursor = [], None
|
||
while True:
|
||
path = "/api/v1/chats/%s/messages?limit=100" % self.chat["id"]
|
||
if cursor:
|
||
path += "&cursor=" + quote(cursor, safe="")
|
||
page = request("GET", path, token=self.bob["_token"], timeout=10).json
|
||
items.extend(page["items"])
|
||
cursor = page["next_cursor"]
|
||
if cursor is None:
|
||
break
|
||
ids = [m["id"] for m in items]
|
||
texts = [m["text"] for m in items]
|
||
by_id = {m["id"]: m for m in items}
|
||
missing = [m for m in self.confirmed if by_id.get(m["id"]) != m]
|
||
return {
|
||
"confirmed_writes": len(self.confirmed),
|
||
"stored_messages": len(items),
|
||
"missing_or_altered": len(missing),
|
||
"duplicate_ids": len(ids) - len(set(ids)),
|
||
"duplicate_texts": len(texts) - len(set(texts)),
|
||
"same_key_retries": self.same_key_retries,
|
||
"ok": not missing and len(ids) == len(set(ids)) and len(texts) == len(set(texts)),
|
||
}
|