Files
mtusi2026lab3/bench/loadgen.py
T

1004 lines
38 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Генератор нагрузки стенда лабы 3. Только стандартная библиотека.
Работает внутри docker-сети стенда, в контейнере `vlad-bench-gen` на
базовом образе Python из `Dockerfile`: запросы идут на HAProxy стенда
(`http://haproxy:8080`) — тем же путём, что у клиента рабочего стенда, и
при одной реплике тоже. Оркестратор `bench/run.py` вызывает его через
`docker exec` и читает одну строку JSON из stdout.
Модель нагрузки — открытая (open model): запросы уходят по расписанию,
не дожидаясь ответов на прежние, — так ведут себя независимые клиенты
мессенджера. Медленный сервер не замедляет генератор: растёт число
запросов в полёте, а задержка считается от момента по расписанию, не от
фактической отправки (без coordinated omission). Если генератор сам не
успевает (запросов в полёте больше предела, отставание от расписания),
это видно в `dropped` и `lag_*` результата, а оркестратор помечает
ступень `generator`.
Процессы (`--procs`): у каждого свой event loop и своя доля
интенсивности; общий старт — по CLOCK_MONOTONIC (один на систему).
Идентификация — как у клиента контракта: `setup` регистрирует
пользователей с паролем, входит по Basic и кладёт токены в файл фикстуры
внутри контейнера генератора. Пароли живут только в памяти `setup`,
токены — только в контейнере генератора: в stdout, результаты и паспорт
метки они не попадают. Отправка сообщения — всегда с `Idempotency-Key`.
Команды:
ready ждать 200 на /health/ready через LB
setup пользователи, читаемый чат с историей, чаты записи, пул курсоров
step ступень открытой модели: смесь чтение/запись/профиль (серии трека 1)
constant постоянная интенсивность записи и чтения с журналом каждого
запроса (профили failover и rollout)
retry повторить неуспешные записи журнала с тем же ключом и телом
"""
import argparse
import array
import asyncio
import base64
import contextlib
import json
import math
import multiprocessing
import os
import random
import secrets
import sys
import time
from pathlib import Path
from urllib.parse import urlsplit
API = "/api/v1"
READ_LIMIT = 50
# одинаковая длина синтетических сообщений; ключ дописывается в конец
TEXT = "Нагрузочный стенд лабы 3: синтетическое сообщение. "
# простаивающее соединение старше этого не берётся: HAProxy закрывает
# сеанс клиента через 10 с простоя (timeout http-keep-alive)
IDLE_REUSE_S = 4.0
KINDS = ("read", "write", "me")
class HttpError(Exception):
pass
# --- HTTP/1.1 ---------------------------------------------------------------------
class Conn:
"""Одно keep-alive соединение HTTP/1.1: запрос — ответ, без конвейера."""
def __init__(self, host: str, port: int) -> None:
self.host = host
self.port = port
self.reader: asyncio.StreamReader | None = None
self.writer: asyncio.StreamWriter | None = None
self.idle_since = 0.0
async def open(self) -> None:
self.reader, self.writer = await asyncio.open_connection(self.host, self.port)
def close(self) -> None:
if self.writer is not None:
self.writer.close()
self.writer = None
async def call(self, raw: bytes) -> tuple[int, bytes, bytes]:
"""Статус, заголовки (байты) и тело ответа."""
assert self.reader is not None
assert self.writer is not None
self.writer.write(raw)
head = await self.reader.readuntil(b"\r\n\r\n")
status = int(head[9:12])
length = header_value(head, b"content-length")
body = await self.reader.readexactly(int(length)) if length else b""
return status, head, body
def header_value(head: bytes, name: bytes) -> bytes:
lowered = head.lower()
start = lowered.find(b"\r\n" + name + b":")
if start < 0:
return b""
start += len(name) + 3
end = lowered.find(b"\r\n", start)
return head[start:end].strip()
def build_request(
method: str,
path: str,
host: str,
*,
token: str | None = None,
authorization: str | None = None,
body: bytes | None = None,
key: str | None = None,
) -> bytes:
lines = [f"{method} {path} HTTP/1.1", f"Host: {host}", "Accept: application/json"]
if token:
lines.append(f"Authorization: Bearer {token}")
elif authorization:
lines.append(f"Authorization: {authorization}")
if key:
lines.append(f"Idempotency-Key: {key}")
if body is not None:
lines.append("Content-Type: application/json")
lines.append(f"Content-Length: {len(body)}")
elif method == "POST":
lines.append("Content-Length: 0")
return ("\r\n".join(lines) + "\r\n\r\n").encode("ascii") + (body or b"")
def target(base_url: str) -> tuple[str, int]:
parts = urlsplit(base_url)
if parts.scheme != "http" or not parts.hostname:
raise SystemExit("base url must look like http://host:port")
return parts.hostname, parts.port or 80
def load_json(path: str):
return json.loads(Path(path).read_text(encoding="ascii"))
def messages_path(chat_id: str) -> str:
return f"{API}/chats/{chat_id}/messages"
def write_request(host: str, chat_id: str, token: str, key: str) -> bytes:
body = json.dumps({"text": TEXT + key}, ensure_ascii=False).encode()
return build_request(
"POST", messages_path(chat_id), host, token=token, body=body, key=key
)
def read_request(host: str, chat_id: str, token: str, cursor: str | None) -> bytes:
path = f"{messages_path(chat_id)}?limit={READ_LIMIT}"
if cursor:
# курсор — base64url, в URL его можно подставлять как есть
path += f"&cursor={cursor}"
return build_request("GET", path, host, token=token)
def me_request(host: str, token: str) -> bytes:
return build_request("GET", f"{API}/users/me", host, token=token)
# --- ready / setup -------------------------------------------------------------------
async def cmd_ready(args: argparse.Namespace) -> dict:
host, port = target(args.base_url)
deadline = time.monotonic() + args.timeout
started = time.monotonic()
raw = build_request("GET", "/health/ready", host)
while time.monotonic() < deadline:
conn = Conn(host, port)
try:
await conn.open()
status, _, _ = await conn.call(raw)
if status == 200:
return {"ready": True, "waited_s": round(time.monotonic() - started, 3)}
except (OSError, asyncio.IncompleteReadError, ValueError):
pass
finally:
conn.close()
await asyncio.sleep(0.2)
return {"ready": False, "waited_s": round(time.monotonic() - started, 3)}
async def _post(
conn: Conn,
host: str,
path: str,
payload: dict,
*,
token: str | None = None,
key: str | None = None,
) -> dict:
body = json.dumps(payload, ensure_ascii=False).encode()
raw = build_request("POST", path, host, token=token, body=body, key=key)
status, _, data = await conn.call(raw)
if status != 201:
raise HttpError(f"POST {path} -> {status}")
return json.loads(data)
async def _register(conn: Conn, host: str, label: str) -> dict:
"""Пользователь с паролем и вход по Basic: id и токен."""
username = f"bench_{label}_{secrets.token_hex(5)}"
password = secrets.token_urlsafe(24)
user = await _post(
conn,
host,
f"{API}/users",
{"username": username, "display_name": "Стенд", "password": password},
)
basic = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
raw = build_request(
"POST", f"{API}/auth/sessions", host, authorization=f"Basic {basic}"
)
status, _, data = await conn.call(raw)
if status != 201:
raise HttpError(f"POST /auth/sessions -> {status}")
return {"id": user["id"], "token": json.loads(data)["token"]}
async def _fill(
host: str,
port: int,
chat: str,
users: list[dict],
total: int,
clients: int,
) -> float:
"""`total` сообщений в чат, `clients` соединений по кругу; секунды."""
counter = {"next": 0}
started = time.monotonic()
async def client(index: int) -> None:
conn = Conn(host, port)
await conn.open()
try:
while counter["next"] < total:
number = counter["next"]
counter["next"] += 1
user = users[number % len(users)]
raw = write_request(host, chat, user["token"], f"fill.{number}")
status, _, _ = await conn.call(raw)
if status != 201:
raise HttpError(f"fill: POST message -> {status}")
finally:
conn.close()
await asyncio.gather(*(client(index) for index in range(clients)))
return time.monotonic() - started
async def cmd_setup(args: argparse.Namespace) -> dict:
"""Пользователи, читаемый чат с историей, чаты записи, пул курсоров.
Все пользователи — участники всех чатов: любой из них читает историю
и пишет в любой чат записи. Пул курсоров — страницы читаемого чата по
`READ_LIMIT`, у которых после курсора полная страница.
"""
host, port = target(args.base_url)
conn = Conn(host, port)
await conn.open()
try:
users = [await _register(conn, host, str(n)) for n in range(args.users)]
member_ids = [user["id"] for user in users]
async def new_chat(title: str) -> str:
chat = await _post(
conn,
host,
f"{API}/chats",
{"title": title, "member_ids": member_ids},
token=users[0]["token"],
)
return chat["id"]
read_chat = await new_chat("bench read")
write_chats = [
await new_chat(f"bench write {n}") for n in range(args.write_chats)
]
finally:
conn.close()
fill_s = await _fill(
host, port, read_chat, users, args.read_messages, args.fill_clients
)
# пул курсоров: обход читаемого чата страницами по READ_LIMIT
conn = Conn(host, port)
await conn.open()
cursors: list[str | None] = [None]
seen = 0
try:
cursor = None
while True:
path = f"{messages_path(read_chat)}?limit={READ_LIMIT}"
if cursor:
path += f"&cursor={cursor}"
status, _, data = await conn.call(
build_request("GET", path, host, token=users[0]["token"])
)
if status != 200:
raise HttpError(f"cursors: GET history -> {status}")
page = json.loads(data)
seen += len(page["items"])
cursor = page["next_cursor"]
if cursor is None:
break
cursors.append(cursor)
finally:
conn.close()
# курсоры, после которых полная страница: последний — только если
# история кратна READ_LIMIT
full = [c for i, c in enumerate(cursors) if seen - i * READ_LIMIT >= READ_LIMIT]
fixture = {
"users": users,
"read_chat": read_chat,
"write_chats": write_chats,
"cursors": full,
}
# токены — только в контейнере генератора; оркестратору — счётчики
Path(args.fixture).write_text(json.dumps(fixture), encoding="ascii") # noqa: ASYNC240
return {
"users": len(users),
"read_chat": read_chat,
"write_chats": write_chats,
"read_messages": seen,
"cursors": len(full),
"fill_s": round(fill_s, 2),
"fill_rps": round(args.read_messages / fill_s, 1) if fill_s else 0.0,
}
# --- расписание и выполнение запросов ---------------------------------------------
def arrival_times(
rate_from: float,
rate_to: float,
ramp_s: float,
hold_s: float,
share: float,
offset: float,
) -> list[float]:
"""Моменты запросов процесса от начала ступени, секунды.
Интенсивность растёт линейно от `rate_from` до `rate_to` за `ramp_s`,
затем держится `hold_s`. Процессу достаётся доля `share`, его моменты
сдвинуты на `offset` от общей сетки: процессы не шлют одновременно.
Число запросов к моменту t — интеграл интенсивности; момент n-го
запроса — корень этого интеграла.
"""
a = rate_from * share
b = (rate_to - rate_from) * share / ramp_s if ramp_s > 0 else 0.0
ramp_count = a * ramp_s + b * ramp_s * ramp_s / 2
total = ramp_count + rate_to * share * hold_s
times = []
n = offset
while n < total:
if n < ramp_count:
if b > 0:
t = (-a + math.sqrt(a * a + 2 * b * n)) / b
else:
t = n / a if a > 0 else 0.0
else:
t = ramp_s + (n - ramp_count) / (rate_to * share)
times.append(t)
n += 1
return times
class Runner:
"""Выполнение запросов одного процесса: пул соединений, сроки, счёт."""
def __init__(self, cfg: dict) -> None:
self.host, self.port = cfg["host"], cfg["port"]
self.timeout = cfg["timeout"]
self.max_inflight = cfg["max_inflight"]
self.idle: list[Conn] = []
self.inflight = 0
self.peak_inflight = 0
async def acquire(self) -> Conn:
while self.idle:
conn = self.idle.pop()
if time.monotonic() - conn.idle_since < IDLE_REUSE_S:
return conn
conn.close()
conn = Conn(self.host, self.port)
await conn.open()
return conn
def release(self, conn: Conn, head: bytes) -> None:
if header_value(head, b"connection").lower() == b"close":
conn.close()
return
conn.idle_since = time.monotonic()
self.idle.append(conn)
async def call(self, raw: bytes) -> tuple[int, bytes, bytes, str]:
"""(статус, заголовки, тело, вид ошибки); статус 0 — ответа нет."""
conn = None
try:
conn = await asyncio.wait_for(self.acquire(), self.timeout)
status, head, body = await asyncio.wait_for(conn.call(raw), self.timeout)
except TimeoutError:
if conn is not None:
conn.close()
return 0, b"", b"", "timeout"
except (OSError, asyncio.IncompleteReadError, ValueError) as exc:
if conn is not None:
conn.close()
return 0, b"", b"", type(exc).__name__
self.release(conn, head)
return status, head, body, ""
def close(self) -> None:
for conn in self.idle:
conn.close()
self.idle.clear()
def percentile(values: list[float] | array.array, fraction: float) -> float | None:
"""Перцентиль методом ближайшего ранга по отсортированным значениям."""
if not values:
return None
rank = max(1, math.ceil(fraction * len(values)))
return values[rank - 1]
def _split_procs(procs: int) -> list[tuple[float, float]]:
"""(доля, сдвиг сетки) каждого процесса."""
return [(1 / procs, index / procs) for index in range(procs)]
def _run_processes(target_fn, cfgs: list[dict]) -> list[dict]:
context = multiprocessing.get_context("fork")
workers = []
for cfg in cfgs:
parent, child = context.Pipe(duplex=False)
process = context.Process(target=target_fn, args=(cfg, child))
process.start()
child.close()
workers.append((process, parent))
results = []
for process, parent in workers:
results.append(parent.recv())
process.join()
fatal = [r["fatal"] for r in results if "fatal" in r]
if fatal:
raise SystemExit("worker failed: " + "; ".join(fatal))
return results
def _entry(fn):
def run(cfg: dict, pipe) -> None:
try:
pipe.send(asyncio.run(fn(cfg)))
except Exception as exc: # noqa: BLE001 - причина уходит родителю строкой
pipe.send({"fatal": f"{type(exc).__name__}: {exc}"})
finally:
pipe.close()
return run
# --- step: ступень открытой модели (серии трека 1) ----------------------------------
def _plan_kinds(count: int, mix: tuple[int, int, int], seed: int) -> bytes:
"""Вид каждого запроса процесса (0 read, 1 write, 2 me) по долям смеси."""
rng = random.Random(seed)
weights = [part / sum(mix) for part in mix]
return bytes(rng.choices(range(3), weights=weights, k=count))
async def _step_worker(cfg: dict) -> dict:
fixture = cfg["fixture"]
users, cursors = fixture["users"], fixture["cursors"]
runner = Runner(cfg)
rng = random.Random(cfg["seed"])
times = arrival_times(
cfg["rate_from"],
cfg["rate"],
cfg["warmup_s"],
cfg["window_s"],
cfg["share"],
cfg["offset"],
)
kinds = _plan_kinds(len(times), cfg["mix"], cfg["seed"])
window_start = cfg["warmup_s"]
key_prefix = cfg["key_prefix"]
stats = {
# всё расписание окна, в том числе не отправленное при досрочном выходе
"scheduled": sum(1 for offset in times if offset >= window_start),
"sent": 0,
"ok": 0,
"dropped": 0,
"short_pages": 0,
"errors": {},
"by_kind": dict.fromkeys(KINDS, 0),
"ok_by_kind": dict.fromkeys(KINDS, 0),
"instances": {},
}
latencies = array.array("d")
service = array.array("d")
lags = array.array("d")
kind_latencies = {kind: array.array("d") for kind in KINDS}
pending: set[asyncio.Task] = set()
abort = {"reason": ""}
def count_error(kind: str) -> None:
stats["errors"][kind] = stats["errors"].get(kind, 0) + 1
async def one(index: int, scheduled: float, in_window: bool) -> None:
kind = KINDS[kinds[index]]
user = users[rng.randrange(len(users))]
if kind == "read":
raw = read_request(
runner.host,
fixture["read_chat"],
user["token"],
cursors[rng.randrange(len(cursors))],
)
elif kind == "write":
chat = fixture["write_chats"][rng.randrange(len(fixture["write_chats"]))]
raw = write_request(
runner.host, chat, user["token"], f"{key_prefix}.{index}"
)
else:
raw = me_request(runner.host, user["token"])
sent_at = time.monotonic()
runner.inflight += 1
runner.peak_inflight = max(runner.peak_inflight, runner.inflight)
try:
status, head, body, error = await runner.call(raw)
finally:
runner.inflight -= 1
done = time.monotonic()
if not in_window:
return
stats["sent"] += 1
stats["by_kind"][kind] += 1
lags.append((sent_at - scheduled) * 1000)
expected = 201 if kind == "write" else 200
if status != expected:
count_error(error or str(status))
return
if kind == "read" and body.count(b'"sender_id"') != READ_LIMIT:
stats["short_pages"] += 1
return
stats["ok"] += 1
stats["ok_by_kind"][kind] += 1
latency = (done - scheduled) * 1000
latencies.append(latency)
kind_latencies[kind].append(latency)
service.append((done - sent_at) * 1000)
instance = header_value(head, b"x-instance-id").decode("ascii", "replace")
stats["instances"][instance] = stats["instances"].get(instance, 0) + 1
async def watchdog() -> None:
"""Досрочный выход ступени: SLO заведомо нарушен, ждать окно зря."""
while True:
await asyncio.sleep(1)
now = time.monotonic() - start
if now < window_start + cfg["abort_after_s"]:
continue
done = stats["sent"]
failed = done - stats["ok"]
if done and failed / done > cfg["abort_error_share"]:
abort["reason"] = "errors"
return
if latencies:
p95 = percentile(sorted(latencies), 0.95)
if p95 is not None and p95 > cfg["abort_p95_ms"]:
abort["reason"] = "p95"
return
delay = cfg["start_at"] - time.monotonic()
if delay > 0:
await asyncio.sleep(delay)
start = cfg["start_at"]
cpu_before = time.process_time()
guard = asyncio.create_task(watchdog())
for index, offset in enumerate(times):
if abort["reason"]:
break
scheduled = start + offset
wait = scheduled - time.monotonic()
if wait > 0:
await asyncio.sleep(wait)
in_window = offset >= window_start
if runner.inflight >= runner.max_inflight:
# генератор не держит столько запросов в полёте: запрос не ушёл
if in_window:
stats["dropped"] += 1
continue
task = asyncio.create_task(one(index, scheduled, in_window))
pending.add(task)
task.add_done_callback(pending.discard)
guard.cancel()
if pending:
await asyncio.wait(pending, timeout=cfg["timeout"] + 1)
for task in list(pending):
task.cancel()
cpu = time.process_time() - cpu_before
runner.close()
return {
"stats": stats,
"latencies": latencies.tobytes(),
"service": service.tobytes(),
"lags": lags.tobytes(),
"kind_latencies": {k: v.tobytes() for k, v in kind_latencies.items()},
"cpu_s": cpu,
"peak_inflight": runner.peak_inflight,
"abort": abort["reason"],
}
def _merge_arrays(results: list[dict], key: str, sub: str | None = None) -> list[float]:
merged = array.array("d")
for result in results:
merged.frombytes(result[key][sub] if sub else result[key])
return sorted(merged)
def cmd_step(args: argparse.Namespace) -> dict:
host, port = target(args.base_url)
fixture = load_json(args.fixture)
mix = tuple(int(part) for part in args.mix.split(","))
if len(mix) != 3 or sum(mix) <= 0:
raise SystemExit("--mix needs three non-negative parts: read,write,me")
start_at = time.monotonic() + 0.3
# часы ВМ Docker: окно ступени для сэмплера
wall_start = time.time() + 0.3
cfgs = []
for index, (share, offset) in enumerate(_split_procs(args.procs)):
cfgs.append(
{
"host": host,
"port": port,
"fixture": fixture,
"rate_from": args.rate_from,
"rate": args.rate,
"warmup_s": args.warmup,
"window_s": args.window,
"share": share,
"offset": offset,
"mix": mix,
"seed": args.seed * 1000 + index,
"start_at": start_at,
"timeout": args.timeout,
"max_inflight": args.max_inflight // args.procs,
"key_prefix": f"{args.key_prefix}.{index}",
"abort_after_s": args.abort_after,
"abort_error_share": args.abort_error_share,
"abort_p95_ms": args.abort_p95_ms,
},
)
results = _run_processes(_entry(_step_worker), cfgs)
total: dict = {
"scheduled": 0,
"sent": 0,
"ok": 0,
"dropped": 0,
"short_pages": 0,
"errors": {},
"by_kind": dict.fromkeys(KINDS, 0),
"ok_by_kind": dict.fromkeys(KINDS, 0),
"instances": {},
}
for result in results:
stats = result["stats"]
for key in ("scheduled", "sent", "ok", "dropped", "short_pages"):
total[key] += stats[key]
for group in ("errors", "by_kind", "ok_by_kind", "instances"):
for key, value in stats[group].items():
total[group][key] = total[group].get(key, 0) + value
latencies = _merge_arrays(results, "latencies")
service = _merge_arrays(results, "service")
lags = _merge_arrays(results, "lags")
errors = sum(total["errors"].values())
scheduled = total["scheduled"]
failed = errors + total["short_pages"] + total["dropped"]
# недосланное расписание при досрочном выходе — тоже провал ступени
unsent = max(0, scheduled - total["sent"] - total["dropped"])
aborts = sorted({r["abort"] for r in results if r["abort"]})
return {
"rate": args.rate,
"rate_from": args.rate_from,
"warmup_s": args.warmup,
"window_s": args.window,
"window_start": round(wall_start + args.warmup, 3),
"window_end": round(wall_start + args.warmup + args.window, 3),
"scheduled": scheduled,
"sent": total["sent"],
"ok": total["ok"],
"errors": errors,
"error_kinds": total["errors"],
"short_pages": total["short_pages"],
"dropped": total["dropped"],
"unsent": unsent,
"fail_share": round((failed + unsent) / scheduled, 5) if scheduled else 1.0,
"achieved_rps": round(total["ok"] / args.window, 1),
"p50_ms": _round(percentile(latencies, 0.50)),
"p95_ms": _round(percentile(latencies, 0.95)),
"p99_ms": _round(percentile(latencies, 0.99)),
"max_ms": _round(latencies[-1] if latencies else None),
"service_p95_ms": _round(percentile(service, 0.95)),
"lag_p99_ms": _round(percentile(lags, 0.99)),
"lag_max_ms": _round(lags[-1] if lags else None),
"by_kind": total["by_kind"],
"ok_by_kind": total["ok_by_kind"],
"kind_p95_ms": {
kind: _round(
percentile(_merge_arrays(results, "kind_latencies", kind), 0.95)
)
for kind in KINDS
},
"instances": total["instances"],
"peak_inflight": sum(r["peak_inflight"] for r in results),
"gen_cpu_s": round(sum(r["cpu_s"] for r in results), 3),
"procs": args.procs,
"abort": ",".join(aborts),
}
def _round(value: float | None) -> float | None:
return None if value is None else round(value, 2)
# --- constant: постоянная интенсивность с журналом (failover, rollout) -----------
async def _constant_worker(cfg: dict) -> dict:
"""Записи и чтения по сетке; строка журнала на каждый запрос.
Строка: `fo,<сдвиг по расписанию, мс>,<отправлен, мс>,<ответ, мс>,
<вид>,<статус>,<X-Instance-Id>,<ключ>,<ошибка>` — времена в мс часов
ВМ (time.time()), ключ — только у записи. Файл — свой у процесса:
строки не перемешиваются.
"""
fixture = cfg["fixture"]
users, cursors = fixture["users"], fixture["cursors"]
runner = Runner(cfg)
rng = random.Random(cfg["seed"])
write_chat = fixture["write_chats"][0]
duration = cfg["duration_s"]
schedule: list[tuple[float, str]] = []
for kind, rate in (("write", cfg["write_rate"]), ("read", cfg["read_rate"])):
if rate > 0:
schedule += [
(t, kind)
for t in arrival_times(
rate, rate, 0, duration, cfg["share"], cfg["offset"]
)
]
schedule.sort()
log = open(cfg["log"], "w", encoding="ascii") # noqa: ASYNC230, SIM115
pending: set[asyncio.Task] = set()
counters = {"write": 0, "read": 0, "dropped": 0}
wall0 = time.time() - time.monotonic()
async def one(scheduled: float, kind: str, number: int) -> None:
user = users[number % len(users)]
key = ""
if kind == "write":
key = f"{cfg['key_prefix']}.{number}"
raw = write_request(runner.host, write_chat, user["token"], key)
else:
raw = read_request(
runner.host,
fixture["read_chat"],
user["token"],
cursors[rng.randrange(len(cursors))],
)
sent = time.monotonic()
runner.inflight += 1
try:
status, head, _, error = await runner.call(raw)
finally:
runner.inflight -= 1
done = time.monotonic()
instance = header_value(head, b"x-instance-id").decode("ascii", "replace")
log.write(
f"fo,{(wall0 + scheduled) * 1000:.1f},{(wall0 + sent) * 1000:.1f},"
f"{(wall0 + done) * 1000:.1f},{kind},{status},{instance},{key},{error}\n",
)
delay = cfg["start_at"] - time.monotonic()
if delay > 0:
await asyncio.sleep(delay)
start = cfg["start_at"]
cpu_before = time.process_time()
for offset, kind in schedule:
scheduled = start + offset
wait = scheduled - time.monotonic()
if wait > 0:
await asyncio.sleep(wait)
number = counters[kind]
counters[kind] += 1
if runner.inflight >= runner.max_inflight:
counters["dropped"] += 1
key = f"{cfg['key_prefix']}.{number}" if kind == "write" else ""
log.write(
f"fo,{(wall0 + scheduled) * 1000:.1f},,,{kind},0,,{key},dropped\n"
)
continue
task = asyncio.create_task(one(scheduled, kind, number))
pending.add(task)
task.add_done_callback(pending.discard)
if pending:
await asyncio.wait(pending, timeout=cfg["timeout"] + 1)
for task in list(pending):
task.cancel()
cpu = time.process_time() - cpu_before
log.close()
runner.close()
return {
"writes": counters["write"],
"reads": counters["read"],
"dropped": counters["dropped"],
"cpu_s": cpu,
}
def cmd_constant(args: argparse.Namespace) -> dict:
host, port = target(args.base_url)
fixture = load_json(args.fixture)
start_at = time.monotonic() + 0.3
started_wall = time.time() + 0.3
cfgs = []
for index, (share, offset) in enumerate(_split_procs(args.procs)):
cfgs.append(
{
"host": host,
"port": port,
"fixture": fixture,
"write_rate": args.write_rate,
"read_rate": args.read_rate,
"duration_s": args.duration,
"share": share,
"offset": offset,
"seed": args.seed * 1000 + index,
"start_at": start_at,
"timeout": args.timeout,
"max_inflight": args.max_inflight // args.procs,
"key_prefix": f"{args.key_prefix}.{index}",
"log": f"{args.log}.{index}",
},
)
results = _run_processes(_entry(_constant_worker), cfgs)
return {
"start": round(started_wall, 3),
"duration_s": args.duration,
"writes": sum(r["writes"] for r in results),
"reads": sum(r["reads"] for r in results),
"dropped": sum(r["dropped"] for r in results),
"gen_cpu_s": round(sum(r["cpu_s"] for r in results), 3),
"logs": [cfg["log"] for cfg in cfgs],
}
# --- retry: повтор неуспешных записей с тем же ключом -------------------------------
async def cmd_retry(args: argparse.Namespace) -> dict:
"""Неуспешные записи журналов — повтор с тем же ключом и телом до 201.
Так клиент обязан поступать с неопределённым результатом POST:
запись, которую сервер успел зафиксировать, вернётся тем же
сообщением (повтор), а не создаст второе.
"""
fixture = load_json(args.fixture)
users = fixture["users"]
write_chat = fixture["write_chats"][0]
host, port = target(args.base_url)
keys: dict[str, bool] = {}
# журналы прочитаны до первого запроса: цикл событий ещё ничего не ждёт
lines = [
line
for path in args.logs
for line in Path(path).read_text(encoding="ascii").splitlines() # noqa: ASYNC240
]
for line in lines:
parts = line.split(",")
if parts[0] != "fo" or parts[4] != "write":
continue
key = parts[7]
keys[key] = keys.get(key, False) or parts[5] == "201"
failed = [key for key, ok in keys.items() if not ok]
conn = Conn(host, port)
recovered = 0
attempts = 0
try:
for key in failed:
number = int(key.rsplit(".", 1)[1])
user = users[number % len(users)]
raw = write_request(host, write_chat, user["token"], key)
for _ in range(args.attempts):
attempts += 1
try:
if conn.writer is None:
await conn.open()
status, _, _ = await conn.call(raw)
except (OSError, asyncio.IncompleteReadError, ValueError):
conn.close()
status = 0
if status == 201:
recovered += 1
break
await asyncio.sleep(0.5)
finally:
conn.close()
return {
"keys": len(keys),
"failed": len(failed),
"recovered": recovered,
"attempts": attempts,
}
# --- CLI -----------------------------------------------------------------------------
def parser() -> argparse.ArgumentParser:
root = argparse.ArgumentParser(description=__doc__.splitlines()[0])
root.add_argument("--base-url", required=True)
commands = root.add_subparsers(dest="command", required=True)
ready = commands.add_parser("ready")
ready.add_argument("--timeout", type=float, default=60)
setup = commands.add_parser("setup")
setup.add_argument("--fixture", required=True)
setup.add_argument("--users", type=int, default=10)
setup.add_argument("--read-messages", type=int, required=True)
setup.add_argument("--write-chats", type=int, default=1)
setup.add_argument("--fill-clients", type=int, default=16)
step = commands.add_parser("step")
step.add_argument("--fixture", required=True)
step.add_argument("--rate", type=float, required=True)
step.add_argument("--rate-from", type=float, default=0.0)
step.add_argument("--warmup", type=float, required=True)
step.add_argument("--window", type=float, required=True)
step.add_argument("--mix", default="70,20,10")
step.add_argument("--procs", type=int, default=4)
step.add_argument("--seed", type=int, default=1)
step.add_argument("--timeout", type=float, default=10)
step.add_argument("--max-inflight", type=int, default=4000)
step.add_argument("--key-prefix", required=True)
step.add_argument("--abort-after", type=float, default=5)
step.add_argument("--abort-error-share", type=float, default=0.5)
step.add_argument("--abort-p95-ms", type=float, default=2000)
constant = commands.add_parser("constant")
constant.add_argument("--fixture", required=True)
constant.add_argument("--write-rate", type=float, required=True)
constant.add_argument("--read-rate", type=float, required=True)
constant.add_argument("--duration", type=float, required=True)
constant.add_argument("--procs", type=int, default=2)
constant.add_argument("--seed", type=int, default=1)
constant.add_argument("--timeout", type=float, default=15)
constant.add_argument("--max-inflight", type=int, default=4000)
constant.add_argument("--key-prefix", required=True)
constant.add_argument("--log", required=True)
retry = commands.add_parser("retry")
retry.add_argument("--fixture", required=True)
retry.add_argument("--attempts", type=int, default=20)
retry.add_argument("logs", nargs="+")
return root
def main() -> None:
args = parser().parse_args()
if args.command == "step":
result = cmd_step(args)
elif args.command == "constant":
result = cmd_constant(args)
else:
handler = {"ready": cmd_ready, "setup": cmd_setup, "retry": cmd_retry}
result = asyncio.run(handler[args.command](args))
json.dump(result, sys.stdout, ensure_ascii=False)
sys.stdout.write("\n")
sys.stdout.flush()
# дочерние процессы уже завершены; без ожидания финализаторов
with contextlib.suppress(Exception):
sys.stdout.close()
os._exit(0)
if __name__ == "__main__":
main()