Files
mtusi2026lab4/bench/loadgen.py
T

1251 lines
49 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Генератор нагрузки стенда лабы 3. Только стандартная библиотека.
Работает внутри docker-сети стенда, в контейнере `vlad-bench-gen` на
базовом образе Python из `Dockerfile`: запросы идут на HAProxy стенда
(`http://haproxy:8080`) — тем же путём, что у клиента рабочего стенда, и
при одной реплике тоже. Оркестратор `bench/run.py` вызывает его через
`docker exec` и читает одну строку JSON из stdout.
Модель нагрузки — открытая (open model): запросы уходят по расписанию,
не дожидаясь ответов на прежние, — так ведут себя независимые клиенты
мессенджера. Медленный сервер не замедляет генератор: растёт число
запросов в полёте, а задержка считается от момента по расписанию, не от
фактической отправки (без coordinated omission). Если генератор сам не
успевает (запросов в полёте больше предела, отставание от расписания),
это видно в `dropped` и `lag_*` результата, а оркестратор помечает
ступень `generator`.
Процессы (`--procs`): у каждого свой event loop и своя доля
интенсивности; общий старт — по CLOCK_MONOTONIC (один на систему).
Идентификация — как у клиента контракта: `setup` регистрирует
пользователей с паролем, входит по Basic и кладёт токены в файл фикстуры
внутри контейнера генератора. Пароли живут только в памяти `setup`,
токены — только в контейнере генератора: в stdout, результаты и паспорт
метки они не попадают. Отправка сообщения — всегда с `Idempotency-Key`.
Команды:
ready ждать 200 на /health/ready через LB
setup пользователи, читаемый чат с историей, чаты записи, пул курсоров
step ступень открытой модели: смесь чтение/запись/профиль (серии трека 1)
constant постоянная интенсивность записи, чтения и (лаба 4) скачивания
вложений с журналом каждого запроса (профили failover, rollout
и фоновая нагрузка профиля reconcile)
retry повторить неуспешные записи журнала с тем же ключом и телом
body PNG из шума заданного размера в файл генератора (лаба 4)
uploads закрытая модель: C клиентов без пауз загружают тело из файла
(на 429 — пауза Retry-After), итог окна по кодам и задержкам
(профиль uploads, лаба 4)
"""
import argparse
import array
import asyncio
import base64
import contextlib
import hashlib
import json
import math
import multiprocessing
import os
import random
import secrets
import struct
import sys
import time
import zlib
from pathlib import Path
from urllib.parse import urlsplit
API = "/api/v1"
READ_LIMIT = 50
# одинаковая длина синтетических сообщений; ключ дописывается в конец
TEXT = "Нагрузочный стенд лабы 3: синтетическое сообщение. "
# простаивающее соединение старше этого не берётся: HAProxy закрывает
# сеанс клиента через 10 с простоя (timeout http-keep-alive)
IDLE_REUSE_S = 4.0
# блок отправки тела загрузки (профиль uploads)
UPLOAD_CHUNK_BYTES = 256 * 1024
KINDS = ("read", "write", "me")
class HttpError(Exception):
pass
# --- HTTP/1.1 ---------------------------------------------------------------------
class Conn:
"""Одно keep-alive соединение HTTP/1.1: запрос — ответ, без конвейера."""
def __init__(self, host: str, port: int) -> None:
self.host = host
self.port = port
self.reader: asyncio.StreamReader | None = None
self.writer: asyncio.StreamWriter | None = None
self.idle_since = 0.0
async def open(self) -> None:
self.reader, self.writer = await asyncio.open_connection(self.host, self.port)
def close(self) -> None:
if self.writer is not None:
self.writer.close()
self.writer = None
async def call(self, raw: bytes) -> tuple[int, bytes, bytes]:
"""Статус, заголовки (байты) и тело ответа."""
assert self.reader is not None
assert self.writer is not None
self.writer.write(raw)
head = await self.reader.readuntil(b"\r\n\r\n")
status = int(head[9:12])
length = header_value(head, b"content-length")
body = await self.reader.readexactly(int(length)) if length else b""
return status, head, body
def header_value(head: bytes, name: bytes) -> bytes:
lowered = head.lower()
start = lowered.find(b"\r\n" + name + b":")
if start < 0:
return b""
start += len(name) + 3
end = lowered.find(b"\r\n", start)
return head[start:end].strip()
def build_request(
method: str,
path: str,
host: str,
*,
token: str | None = None,
authorization: str | None = None,
body: bytes | None = None,
key: str | None = None,
) -> bytes:
lines = [f"{method} {path} HTTP/1.1", f"Host: {host}", "Accept: application/json"]
if token:
lines.append(f"Authorization: Bearer {token}")
elif authorization:
lines.append(f"Authorization: {authorization}")
if key:
lines.append(f"Idempotency-Key: {key}")
if body is not None:
lines.append("Content-Type: application/json")
lines.append(f"Content-Length: {len(body)}")
elif method == "POST":
lines.append("Content-Length: 0")
return ("\r\n".join(lines) + "\r\n\r\n").encode("ascii") + (body or b"")
def target(base_url: str) -> tuple[str, int]:
parts = urlsplit(base_url)
if parts.scheme != "http" or not parts.hostname:
raise SystemExit("base url must look like http://host:port")
return parts.hostname, parts.port or 80
def load_json(path: str):
return json.loads(Path(path).read_text(encoding="ascii"))
def messages_path(chat_id: str) -> str:
return f"{API}/chats/{chat_id}/messages"
def write_request(host: str, chat_id: str, token: str, key: str) -> bytes:
body = json.dumps({"text": TEXT + key}, ensure_ascii=False).encode()
return build_request(
"POST", messages_path(chat_id), host, token=token, body=body, key=key
)
def read_request(host: str, chat_id: str, token: str, cursor: str | None) -> bytes:
path = f"{messages_path(chat_id)}?limit={READ_LIMIT}"
if cursor:
# курсор — base64url, в URL его можно подставлять как есть
path += f"&cursor={cursor}"
return build_request("GET", path, host, token=token)
def me_request(host: str, token: str) -> bytes:
return build_request("GET", f"{API}/users/me", host, token=token)
def download_request(host: str, attachment_id: str, token: str) -> bytes:
return build_request(
"GET", f"{API}/attachments/{attachment_id}/content", host, token=token
)
# --- ready / setup -------------------------------------------------------------------
async def cmd_ready(args: argparse.Namespace) -> dict:
host, port = target(args.base_url)
deadline = time.monotonic() + args.timeout
started = time.monotonic()
raw = build_request("GET", "/health/ready", host)
while time.monotonic() < deadline:
conn = Conn(host, port)
try:
await conn.open()
status, _, _ = await conn.call(raw)
if status == 200:
return {"ready": True, "waited_s": round(time.monotonic() - started, 3)}
except (OSError, asyncio.IncompleteReadError, ValueError):
pass
finally:
conn.close()
await asyncio.sleep(0.2)
return {"ready": False, "waited_s": round(time.monotonic() - started, 3)}
async def _post(
conn: Conn,
host: str,
path: str,
payload: dict,
*,
token: str | None = None,
key: str | None = None,
) -> dict:
body = json.dumps(payload, ensure_ascii=False).encode()
raw = build_request("POST", path, host, token=token, body=body, key=key)
status, _, data = await conn.call(raw)
if status != 201:
raise HttpError(f"POST {path} -> {status}")
return json.loads(data)
async def _register(conn: Conn, host: str, label: str) -> dict:
"""Пользователь с паролем и вход по Basic: id и токен."""
username = f"bench_{label}_{secrets.token_hex(5)}"
password = secrets.token_urlsafe(24)
user = await _post(
conn,
host,
f"{API}/users",
{"username": username, "display_name": "Стенд", "password": password},
)
basic = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
raw = build_request(
"POST", f"{API}/auth/sessions", host, authorization=f"Basic {basic}"
)
status, _, data = await conn.call(raw)
if status != 201:
raise HttpError(f"POST /auth/sessions -> {status}")
return {"id": user["id"], "token": json.loads(data)["token"]}
async def _fill(
host: str,
port: int,
chat: str,
users: list[dict],
total: int,
clients: int,
) -> float:
"""`total` сообщений в чат, `clients` соединений по кругу; секунды."""
counter = {"next": 0}
started = time.monotonic()
async def client(index: int) -> None:
conn = Conn(host, port)
await conn.open()
try:
while counter["next"] < total:
number = counter["next"]
counter["next"] += 1
user = users[number % len(users)]
raw = write_request(host, chat, user["token"], f"fill.{number}")
status, _, _ = await conn.call(raw)
if status != 201:
raise HttpError(f"fill: POST message -> {status}")
finally:
conn.close()
await asyncio.gather(*(client(index) for index in range(clients)))
return time.monotonic() - started
async def cmd_setup(args: argparse.Namespace) -> dict:
"""Пользователи, читаемый чат с историей, чаты записи, пул курсоров.
Все пользователи — участники всех чатов: любой из них читает историю
и пишет в любой чат записи. Пул курсоров — страницы читаемого чата по
`READ_LIMIT`, у которых после курсора полная страница.
"""
host, port = target(args.base_url)
conn = Conn(host, port)
await conn.open()
try:
users = [await _register(conn, host, str(n)) for n in range(args.users)]
member_ids = [user["id"] for user in users]
async def new_chat(title: str) -> str:
chat = await _post(
conn,
host,
f"{API}/chats",
{"title": title, "member_ids": member_ids},
token=users[0]["token"],
)
return chat["id"]
read_chat = await new_chat("bench read")
write_chats = [
await new_chat(f"bench write {n}") for n in range(args.write_chats)
]
finally:
conn.close()
fill_s = await _fill(
host, port, read_chat, users, args.read_messages, args.fill_clients
)
# пул курсоров: обход читаемого чата страницами по READ_LIMIT
conn = Conn(host, port)
await conn.open()
cursors: list[str | None] = [None]
seen = 0
try:
cursor = None
while True:
path = f"{messages_path(read_chat)}?limit={READ_LIMIT}"
if cursor:
path += f"&cursor={cursor}"
status, _, data = await conn.call(
build_request("GET", path, host, token=users[0]["token"])
)
if status != 200:
raise HttpError(f"cursors: GET history -> {status}")
page = json.loads(data)
seen += len(page["items"])
cursor = page["next_cursor"]
if cursor is None:
break
cursors.append(cursor)
finally:
conn.close()
# курсоры, после которых полная страница: последний — только если
# история кратна READ_LIMIT
full = [c for i, c in enumerate(cursors) if seen - i * READ_LIMIT >= READ_LIMIT]
fixture = {
"users": users,
"read_chat": read_chat,
"write_chats": write_chats,
"cursors": full,
}
# токены — только в контейнере генератора; оркестратору — счётчики
Path(args.fixture).write_text(json.dumps(fixture), encoding="ascii") # noqa: ASYNC240
return {
"users": len(users),
# id не секрет: строки вложений профиля reconcile — от его имени
"owner": users[0]["id"],
"read_chat": read_chat,
"write_chats": write_chats,
"read_messages": seen,
"cursors": len(full),
"fill_s": round(fill_s, 2),
"fill_rps": round(args.read_messages / fill_s, 1) if fill_s else 0.0,
}
# --- расписание и выполнение запросов ---------------------------------------------
def arrival_times(
rate_from: float,
rate_to: float,
ramp_s: float,
hold_s: float,
share: float,
offset: float,
) -> list[float]:
"""Моменты запросов процесса от начала ступени, секунды.
Интенсивность растёт линейно от `rate_from` до `rate_to` за `ramp_s`,
затем держится `hold_s`. Процессу достаётся доля `share`, его моменты
сдвинуты на `offset` от общей сетки: процессы не шлют одновременно.
Число запросов к моменту t — интеграл интенсивности; момент n-го
запроса — корень этого интеграла.
"""
a = rate_from * share
b = (rate_to - rate_from) * share / ramp_s if ramp_s > 0 else 0.0
ramp_count = a * ramp_s + b * ramp_s * ramp_s / 2
total = ramp_count + rate_to * share * hold_s
times = []
n = offset
while n < total:
if n < ramp_count:
if b > 0:
t = (-a + math.sqrt(a * a + 2 * b * n)) / b
else:
t = n / a if a > 0 else 0.0
else:
t = ramp_s + (n - ramp_count) / (rate_to * share)
times.append(t)
n += 1
return times
class Runner:
"""Выполнение запросов одного процесса: пул соединений, сроки, счёт."""
def __init__(self, cfg: dict) -> None:
self.host, self.port = cfg["host"], cfg["port"]
self.timeout = cfg["timeout"]
self.max_inflight = cfg["max_inflight"]
self.idle: list[Conn] = []
self.inflight = 0
self.peak_inflight = 0
async def acquire(self) -> Conn:
while self.idle:
conn = self.idle.pop()
if time.monotonic() - conn.idle_since < IDLE_REUSE_S:
return conn
conn.close()
conn = Conn(self.host, self.port)
await conn.open()
return conn
def release(self, conn: Conn, head: bytes) -> None:
if header_value(head, b"connection").lower() == b"close":
conn.close()
return
conn.idle_since = time.monotonic()
self.idle.append(conn)
async def call(self, raw: bytes) -> tuple[int, bytes, bytes, str]:
"""(статус, заголовки, тело, вид ошибки); статус 0 — ответа нет."""
conn = None
try:
conn = await asyncio.wait_for(self.acquire(), self.timeout)
status, head, body = await asyncio.wait_for(conn.call(raw), self.timeout)
except TimeoutError:
if conn is not None:
conn.close()
return 0, b"", b"", "timeout"
except (OSError, asyncio.IncompleteReadError, ValueError) as exc:
if conn is not None:
conn.close()
return 0, b"", b"", type(exc).__name__
self.release(conn, head)
return status, head, body, ""
def close(self) -> None:
for conn in self.idle:
conn.close()
self.idle.clear()
def percentile(values: list[float] | array.array, fraction: float) -> float | None:
"""Перцентиль методом ближайшего ранга по отсортированным значениям."""
if not values:
return None
rank = max(1, math.ceil(fraction * len(values)))
return values[rank - 1]
def _split_procs(procs: int) -> list[tuple[float, float]]:
"""(доля, сдвиг сетки) каждого процесса."""
return [(1 / procs, index / procs) for index in range(procs)]
def _run_processes(target_fn, cfgs: list[dict]) -> list[dict]:
context = multiprocessing.get_context("fork")
workers = []
for cfg in cfgs:
parent, child = context.Pipe(duplex=False)
process = context.Process(target=target_fn, args=(cfg, child))
process.start()
child.close()
workers.append((process, parent))
results = []
for process, parent in workers:
results.append(parent.recv())
process.join()
fatal = [r["fatal"] for r in results if "fatal" in r]
if fatal:
raise SystemExit("worker failed: " + "; ".join(fatal))
return results
def _entry(fn):
def run(cfg: dict, pipe) -> None:
try:
pipe.send(asyncio.run(fn(cfg)))
except Exception as exc: # noqa: BLE001 - причина уходит родителю строкой
pipe.send({"fatal": f"{type(exc).__name__}: {exc}"})
finally:
pipe.close()
return run
# --- step: ступень открытой модели (серии трека 1) ----------------------------------
def _plan_kinds(count: int, mix: tuple[int, int, int], seed: int) -> bytes:
"""Вид каждого запроса процесса (0 read, 1 write, 2 me) по долям смеси."""
rng = random.Random(seed)
weights = [part / sum(mix) for part in mix]
return bytes(rng.choices(range(3), weights=weights, k=count))
async def _step_worker(cfg: dict) -> dict:
fixture = cfg["fixture"]
users, cursors = fixture["users"], fixture["cursors"]
runner = Runner(cfg)
rng = random.Random(cfg["seed"])
times = arrival_times(
cfg["rate_from"],
cfg["rate"],
cfg["warmup_s"],
cfg["window_s"],
cfg["share"],
cfg["offset"],
)
kinds = _plan_kinds(len(times), cfg["mix"], cfg["seed"])
window_start = cfg["warmup_s"]
key_prefix = cfg["key_prefix"]
stats = {
# всё расписание окна, в том числе не отправленное при досрочном выходе
"scheduled": sum(1 for offset in times if offset >= window_start),
"sent": 0,
"ok": 0,
"dropped": 0,
"short_pages": 0,
"errors": {},
"by_kind": dict.fromkeys(KINDS, 0),
"ok_by_kind": dict.fromkeys(KINDS, 0),
"instances": {},
}
latencies = array.array("d")
service = array.array("d")
lags = array.array("d")
kind_latencies = {kind: array.array("d") for kind in KINDS}
pending: set[asyncio.Task] = set()
abort = {"reason": ""}
def count_error(kind: str) -> None:
stats["errors"][kind] = stats["errors"].get(kind, 0) + 1
async def one(index: int, scheduled: float, in_window: bool) -> None:
kind = KINDS[kinds[index]]
user = users[rng.randrange(len(users))]
if kind == "read":
raw = read_request(
runner.host,
fixture["read_chat"],
user["token"],
cursors[rng.randrange(len(cursors))],
)
elif kind == "write":
chat = fixture["write_chats"][rng.randrange(len(fixture["write_chats"]))]
raw = write_request(
runner.host, chat, user["token"], f"{key_prefix}.{index}"
)
else:
raw = me_request(runner.host, user["token"])
sent_at = time.monotonic()
runner.inflight += 1
runner.peak_inflight = max(runner.peak_inflight, runner.inflight)
try:
status, head, body, error = await runner.call(raw)
finally:
runner.inflight -= 1
done = time.monotonic()
if not in_window:
return
stats["sent"] += 1
stats["by_kind"][kind] += 1
lags.append((sent_at - scheduled) * 1000)
expected = 201 if kind == "write" else 200
if status != expected:
count_error(error or str(status))
return
if kind == "read" and body.count(b'"sender_id"') != READ_LIMIT:
stats["short_pages"] += 1
return
stats["ok"] += 1
stats["ok_by_kind"][kind] += 1
latency = (done - scheduled) * 1000
latencies.append(latency)
kind_latencies[kind].append(latency)
service.append((done - sent_at) * 1000)
instance = header_value(head, b"x-instance-id").decode("ascii", "replace")
stats["instances"][instance] = stats["instances"].get(instance, 0) + 1
async def watchdog() -> None:
"""Досрочный выход ступени: SLO заведомо нарушен, ждать окно зря."""
while True:
await asyncio.sleep(1)
now = time.monotonic() - start
if now < window_start + cfg["abort_after_s"]:
continue
done = stats["sent"]
failed = done - stats["ok"]
if done and failed / done > cfg["abort_error_share"]:
abort["reason"] = "errors"
return
if latencies:
p95 = percentile(sorted(latencies), 0.95)
if p95 is not None and p95 > cfg["abort_p95_ms"]:
abort["reason"] = "p95"
return
delay = cfg["start_at"] - time.monotonic()
if delay > 0:
await asyncio.sleep(delay)
start = cfg["start_at"]
cpu_before = time.process_time()
guard = asyncio.create_task(watchdog())
for index, offset in enumerate(times):
if abort["reason"]:
break
scheduled = start + offset
wait = scheduled - time.monotonic()
if wait > 0:
await asyncio.sleep(wait)
in_window = offset >= window_start
if runner.inflight >= runner.max_inflight:
# генератор не держит столько запросов в полёте: запрос не ушёл
if in_window:
stats["dropped"] += 1
continue
task = asyncio.create_task(one(index, scheduled, in_window))
pending.add(task)
task.add_done_callback(pending.discard)
guard.cancel()
if pending:
await asyncio.wait(pending, timeout=cfg["timeout"] + 1)
for task in list(pending):
task.cancel()
cpu = time.process_time() - cpu_before
runner.close()
return {
"stats": stats,
"latencies": latencies.tobytes(),
"service": service.tobytes(),
"lags": lags.tobytes(),
"kind_latencies": {k: v.tobytes() for k, v in kind_latencies.items()},
"cpu_s": cpu,
"peak_inflight": runner.peak_inflight,
"abort": abort["reason"],
}
def _merge_arrays(results: list[dict], key: str, sub: str | None = None) -> list[float]:
merged = array.array("d")
for result in results:
merged.frombytes(result[key][sub] if sub else result[key])
return sorted(merged)
def cmd_step(args: argparse.Namespace) -> dict:
host, port = target(args.base_url)
fixture = load_json(args.fixture)
mix = tuple(int(part) for part in args.mix.split(","))
if len(mix) != 3 or sum(mix) <= 0:
raise SystemExit("--mix needs three non-negative parts: read,write,me")
start_at = time.monotonic() + 0.3
# часы ВМ Docker: окно ступени для сэмплера
wall_start = time.time() + 0.3
cfgs = []
for index, (share, offset) in enumerate(_split_procs(args.procs)):
cfgs.append(
{
"host": host,
"port": port,
"fixture": fixture,
"rate_from": args.rate_from,
"rate": args.rate,
"warmup_s": args.warmup,
"window_s": args.window,
"share": share,
"offset": offset,
"mix": mix,
"seed": args.seed * 1000 + index,
"start_at": start_at,
"timeout": args.timeout,
"max_inflight": args.max_inflight // args.procs,
"key_prefix": f"{args.key_prefix}.{index}",
"abort_after_s": args.abort_after,
"abort_error_share": args.abort_error_share,
"abort_p95_ms": args.abort_p95_ms,
},
)
results = _run_processes(_entry(_step_worker), cfgs)
total: dict = {
"scheduled": 0,
"sent": 0,
"ok": 0,
"dropped": 0,
"short_pages": 0,
"errors": {},
"by_kind": dict.fromkeys(KINDS, 0),
"ok_by_kind": dict.fromkeys(KINDS, 0),
"instances": {},
}
for result in results:
stats = result["stats"]
for key in ("scheduled", "sent", "ok", "dropped", "short_pages"):
total[key] += stats[key]
for group in ("errors", "by_kind", "ok_by_kind", "instances"):
for key, value in stats[group].items():
total[group][key] = total[group].get(key, 0) + value
latencies = _merge_arrays(results, "latencies")
service = _merge_arrays(results, "service")
lags = _merge_arrays(results, "lags")
errors = sum(total["errors"].values())
scheduled = total["scheduled"]
failed = errors + total["short_pages"] + total["dropped"]
# недосланное расписание при досрочном выходе — тоже провал ступени
unsent = max(0, scheduled - total["sent"] - total["dropped"])
aborts = sorted({r["abort"] for r in results if r["abort"]})
return {
"rate": args.rate,
"rate_from": args.rate_from,
"warmup_s": args.warmup,
"window_s": args.window,
"window_start": round(wall_start + args.warmup, 3),
"window_end": round(wall_start + args.warmup + args.window, 3),
"scheduled": scheduled,
"sent": total["sent"],
"ok": total["ok"],
"errors": errors,
"error_kinds": total["errors"],
"short_pages": total["short_pages"],
"dropped": total["dropped"],
"unsent": unsent,
"fail_share": round((failed + unsent) / scheduled, 5) if scheduled else 1.0,
"achieved_rps": round(total["ok"] / args.window, 1),
"p50_ms": _round(percentile(latencies, 0.50)),
"p95_ms": _round(percentile(latencies, 0.95)),
"p99_ms": _round(percentile(latencies, 0.99)),
"max_ms": _round(latencies[-1] if latencies else None),
"service_p95_ms": _round(percentile(service, 0.95)),
"lag_p99_ms": _round(percentile(lags, 0.99)),
"lag_max_ms": _round(lags[-1] if lags else None),
"by_kind": total["by_kind"],
"ok_by_kind": total["ok_by_kind"],
"kind_p95_ms": {
kind: _round(
percentile(_merge_arrays(results, "kind_latencies", kind), 0.95)
)
for kind in KINDS
},
"instances": total["instances"],
"peak_inflight": sum(r["peak_inflight"] for r in results),
"gen_cpu_s": round(sum(r["cpu_s"] for r in results), 3),
"procs": args.procs,
"abort": ",".join(aborts),
}
def _round(value: float | None) -> float | None:
return None if value is None else round(value, 2)
# --- constant: постоянная интенсивность с журналом (failover, rollout) -----------
async def _constant_worker(cfg: dict) -> dict:
"""Записи и чтения по сетке; строка журнала на каждый запрос.
Строка: `fo,<сдвиг по расписанию, мс>,<отправлен, мс>,<ответ, мс>,
<вид>,<статус>,<X-Instance-Id>,<ключ>,<ошибка>` — времена в мс часов
ВМ (time.time()), ключ — только у записи. Файл — свой у процесса:
строки не перемешиваются.
"""
fixture = cfg["fixture"]
users, cursors = fixture["users"], fixture["cursors"]
downloads = fixture.get("downloads", [])
runner = Runner(cfg)
rng = random.Random(cfg["seed"])
write_chat = fixture["write_chats"][0]
duration = cfg["duration_s"]
schedule: list[tuple[float, str]] = []
rates = (
("write", cfg["write_rate"]),
("read", cfg["read_rate"]),
("download", cfg["download_rate"]),
)
for kind, rate in rates:
if rate > 0:
schedule += [
(t, kind)
for t in arrival_times(
rate, rate, 0, duration, cfg["share"], cfg["offset"]
)
]
schedule.sort()
log = open(cfg["log"], "w", encoding="ascii") # noqa: ASYNC230, SIM115
pending: set[asyncio.Task] = set()
counters = {"write": 0, "read": 0, "download": 0, "dropped": 0, "stopped": 0}
wall0 = time.time() - time.monotonic()
async def one(scheduled: float, kind: str, number: int) -> None:
user = users[number % len(users)]
key = ""
if kind == "write":
key = f"{cfg['key_prefix']}.{number}"
raw = write_request(runner.host, write_chat, user["token"], key)
elif kind == "download":
# вложения набора — в чате, где все пользователи фикстуры участники
raw = download_request(
runner.host, downloads[rng.randrange(len(downloads))], user["token"]
)
else:
raw = read_request(
runner.host,
fixture["read_chat"],
user["token"],
cursors[rng.randrange(len(cursors))],
)
sent = time.monotonic()
runner.inflight += 1
try:
status, head, _, error = await runner.call(raw)
finally:
runner.inflight -= 1
done = time.monotonic()
instance = header_value(head, b"x-instance-id").decode("ascii", "replace")
log.write(
f"fo,{(wall0 + scheduled) * 1000:.1f},{(wall0 + sent) * 1000:.1f},"
f"{(wall0 + done) * 1000:.1f},{kind},{status},{instance},{key},{error}\n",
)
delay = cfg["start_at"] - time.monotonic()
if delay > 0:
await asyncio.sleep(delay)
start = cfg["start_at"]
cpu_before = time.process_time()
stop_file = cfg["stop_file"]
next_check = 0.0
for offset, kind in schedule:
scheduled = start + offset
wait = scheduled - time.monotonic()
if wait > 0:
await asyncio.sleep(wait)
# фоновая нагрузка профиля reconcile кончается по файлу: длительность
# прохода заранее неизвестна
if stop_file and time.monotonic() >= next_check:
next_check = time.monotonic() + 0.25
if os.path.exists(stop_file): # noqa: ASYNC240 - stat раз в 0,25 с
counters["stopped"] = 1
break
number = counters[kind]
counters[kind] += 1
if runner.inflight >= runner.max_inflight:
counters["dropped"] += 1
key = f"{cfg['key_prefix']}.{number}" if kind == "write" else ""
log.write(
f"fo,{(wall0 + scheduled) * 1000:.1f},,,{kind},0,,{key},dropped\n"
)
continue
task = asyncio.create_task(one(scheduled, kind, number))
pending.add(task)
task.add_done_callback(pending.discard)
if pending:
await asyncio.wait(pending, timeout=cfg["timeout"] + 1)
for task in list(pending):
task.cancel()
cpu = time.process_time() - cpu_before
log.close()
runner.close()
return {
"writes": counters["write"],
"reads": counters["read"],
"downloads": counters["download"],
"dropped": counters["dropped"],
"stopped": bool(counters["stopped"]),
"cpu_s": cpu,
}
def cmd_constant(args: argparse.Namespace) -> dict:
host, port = target(args.base_url)
fixture = load_json(args.fixture)
if args.downloads:
# id вложений для скачиваний: отдельный файл, его пишет оркестратор
fixture["downloads"] = load_json(args.downloads)
if args.download_rate > 0 and not fixture.get("downloads"):
raise SystemExit("--download-rate needs --downloads with attachment ids")
start_at = time.monotonic() + 0.3
started_wall = time.time() + 0.3
cfgs = []
for index, (share, offset) in enumerate(_split_procs(args.procs)):
cfgs.append(
{
"host": host,
"port": port,
"fixture": fixture,
"write_rate": args.write_rate,
"read_rate": args.read_rate,
"download_rate": args.download_rate,
"duration_s": args.duration,
"share": share,
"offset": offset,
"seed": args.seed * 1000 + index,
"start_at": start_at,
"timeout": args.timeout,
"max_inflight": args.max_inflight // args.procs,
"key_prefix": f"{args.key_prefix}.{index}",
"log": f"{args.log}.{index}",
"stop_file": args.stop_file,
},
)
results = _run_processes(_entry(_constant_worker), cfgs)
return {
"start": round(started_wall, 3),
"end": round(time.time(), 3),
"duration_s": args.duration,
"writes": sum(r["writes"] for r in results),
"reads": sum(r["reads"] for r in results),
"downloads": sum(r["downloads"] for r in results),
"dropped": sum(r["dropped"] for r in results),
"stopped": any(r["stopped"] for r in results),
"gen_cpu_s": round(sum(r["cpu_s"] for r in results), 3),
"logs": [cfg["log"] for cfg in cfgs],
}
# --- retry: повтор неуспешных записей с тем же ключом -------------------------------
async def cmd_retry(args: argparse.Namespace) -> dict:
"""Неуспешные записи журналов — повтор с тем же ключом и телом до 201.
Так клиент обязан поступать с неопределённым результатом POST:
запись, которую сервер успел зафиксировать, вернётся тем же
сообщением (повтор), а не создаст второе.
"""
fixture = load_json(args.fixture)
users = fixture["users"]
write_chat = fixture["write_chats"][0]
host, port = target(args.base_url)
keys: dict[str, bool] = {}
# журналы прочитаны до первого запроса: цикл событий ещё ничего не ждёт
lines = [
line
for path in args.logs
for line in Path(path).read_text(encoding="ascii").splitlines() # noqa: ASYNC240
]
for line in lines:
parts = line.split(",")
if parts[0] != "fo" or parts[4] != "write":
continue
key = parts[7]
keys[key] = keys.get(key, False) or parts[5] == "201"
failed = [key for key, ok in keys.items() if not ok]
conn = Conn(host, port)
recovered = 0
attempts = 0
try:
for key in failed:
number = int(key.rsplit(".", 1)[1])
user = users[number % len(users)]
raw = write_request(host, write_chat, user["token"], key)
for _ in range(args.attempts):
attempts += 1
try:
if conn.writer is None:
await conn.open()
status, _, _ = await conn.call(raw)
except (OSError, asyncio.IncompleteReadError, ValueError):
conn.close()
status = 0
if status == 201:
recovered += 1
break
await asyncio.sleep(0.5)
finally:
conn.close()
return {
"keys": len(keys),
"failed": len(failed),
"recovered": recovered,
"attempts": attempts,
}
# --- uploads: одновременные загрузки (профиль uploads, лаба 4) ----------------------
def _chunk(kind: bytes, body: bytes) -> bytes:
return (
struct.pack(">I", len(body))
+ kind
+ body
+ struct.pack(">I", zlib.crc32(kind + body))
)
def png_noise(width: int, height: int, seed: int) -> bytes:
"""RGB 8 бит из шума от зерна: почти не сжимается, размер ≈ 3 × пиксели."""
rng = random.Random(seed)
rows = b"".join(b"\x00" + rng.randbytes(width * 3) for _ in range(height))
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ _chunk(b"IHDR", header)
+ _chunk(b"IDAT", zlib.compress(rows, 1))
+ _chunk(b"IEND", b"")
)
def cmd_body(args: argparse.Namespace) -> dict:
"""PNG ширины 1024 с высотой под размер не больше `--size`."""
width = 1024
height = max(1, args.size // (width * 3 + 1))
data = png_noise(width, height, args.seed)
while len(data) > args.size and height > 1:
height -= max(1, (len(data) - args.size) // (width * 3) + 1)
data = png_noise(width, max(1, height), args.seed)
Path(args.out).write_bytes(data)
return {
"path": args.out,
"bytes": len(data),
"width": width,
"height": height,
"sha256": hashlib.sha256(data).hexdigest(),
}
def upload_head(host: str, chat_id: str, token: str, size: int) -> bytes:
lines = [
f"POST {API}/chats/{chat_id}/attachments?filename=bench.png HTTP/1.1",
f"Host: {host}",
"Accept: application/json",
f"Authorization: Bearer {token}",
"Content-Type: image/png",
f"Content-Length: {size}",
# своё соединение на загрузку: после раннего ответа (429, 413) LB
# может закрыть сеанс, а тело ещё уходит
"Connection: close",
]
return ("\r\n".join(lines) + "\r\n\r\n").encode("ascii")
async def cmd_uploads(args: argparse.Namespace) -> dict:
"""C клиентов без пауз: загрузка за загрузкой одного тела в один чат.
Окно — от конца прогрева до его конца по времени начала загрузки; в
итог идут ответы загрузок, начатых в окне. 429 — пауза `Retry-After`
(контракт: клиент повторяет после неё), прочие ошибки — пауза 1 с.
Задержка — от отправки до конца ответа (модель закрытая: генератор не
шлёт новую загрузку клиента, пока не пришёл ответ на прежнюю).
"""
host, port = target(args.base_url)
fixture = load_json(args.fixture)
user = fixture["users"][0]
chat = fixture["write_chats"][0]
# до первого запроса: цикл событий ещё ничего не ждёт
body = Path(args.body).read_bytes() # noqa: ASYNC240
head = upload_head(host, chat, user["token"], len(body))
start = time.monotonic() + 0.3
window_start = start + args.warmup
end = window_start + args.window
wall_start = time.time() + 0.3
statuses: dict[str, int] = {}
latencies = array.array("d")
retry_after: list[float] = []
started_uploads = {"count": 0}
view = memoryview(body)
async def send_body(writer: asyncio.StreamWriter) -> None:
# блоками с drain: в буфере транспорта не больше блока, и ранний
# ответ сервера (429, 413) читается, пока тело ещё уходит
writer.write(head)
for offset in range(0, len(view), UPLOAD_CHUNK_BYTES):
writer.write(view[offset : offset + UPLOAD_CHUNK_BYTES])
await writer.drain()
async def client(index: int) -> None:
await asyncio.sleep(max(0.0, start - time.monotonic()) + index * 0.005)
while time.monotonic() < end:
begun = time.monotonic()
started_uploads["count"] += 1
status, response_head = 0, b""
writer = None
sender = None
try:
reader, writer = await asyncio.wait_for(
asyncio.open_connection(host, port), args.timeout
)
sender = asyncio.create_task(send_body(writer))
response_head = await asyncio.wait_for(
reader.readuntil(b"\r\n\r\n"), args.timeout
)
status = int(response_head[9:12])
length = header_value(response_head, b"content-length")
if length:
await asyncio.wait_for(
reader.readexactly(int(length)), args.timeout
)
kind = str(status)
except TimeoutError:
kind = "timeout"
except (OSError, asyncio.IncompleteReadError, ValueError) as exc:
kind = type(exc).__name__
finally:
if sender is not None:
sender.cancel()
with contextlib.suppress(asyncio.CancelledError, OSError):
await sender
if writer is not None:
writer.close()
done = time.monotonic()
if window_start <= begun < end:
statuses[kind] = statuses.get(kind, 0) + 1
if status == 201:
latencies.append((done - begun) * 1000)
if status == 201:
continue
pause = 1.0
if status == 429:
value = header_value(response_head, b"retry-after")
pause = float(value) if value else 1.0
retry_after.append(pause)
await asyncio.sleep(min(pause, max(0.0, end - time.monotonic())))
cpu_before = time.process_time()
await asyncio.gather(*(client(index) for index in range(args.clients)))
ordered = sorted(latencies)
ok = statuses.get("201", 0)
return {
"clients": args.clients,
"body_bytes": len(body),
"warmup_s": args.warmup,
"window_s": args.window,
"window_start": round(wall_start + args.warmup, 3),
"window_end": round(wall_start + args.warmup + args.window, 3),
"statuses": statuses,
"ok": ok,
"rejected_429": statuses.get("429", 0),
"errors_5xx": sum(v for k, v in statuses.items() if k.startswith("5")),
"other_errors": sum(
v
for k, v in statuses.items()
if k not in ("201", "429") and not k.startswith("5")
),
"uploads_per_s": round(ok / args.window, 2),
"mib_per_s": round(ok * len(body) / args.window / 2**20, 2),
"p50_ms": _round(percentile(ordered, 0.50)),
"p95_ms": _round(percentile(ordered, 0.95)),
"p99_ms": _round(percentile(ordered, 0.99)),
"max_ms": _round(ordered[-1] if ordered else None),
"retry_after_values": sorted(set(retry_after)),
"started": started_uploads["count"],
"gen_cpu_s": round(time.process_time() - cpu_before, 3),
}
# --- CLI -----------------------------------------------------------------------------
def parser() -> argparse.ArgumentParser:
root = argparse.ArgumentParser(description=__doc__.splitlines()[0])
root.add_argument("--base-url", required=True)
commands = root.add_subparsers(dest="command", required=True)
ready = commands.add_parser("ready")
ready.add_argument("--timeout", type=float, default=60)
setup = commands.add_parser("setup")
setup.add_argument("--fixture", required=True)
setup.add_argument("--users", type=int, default=10)
setup.add_argument("--read-messages", type=int, required=True)
setup.add_argument("--write-chats", type=int, default=1)
setup.add_argument("--fill-clients", type=int, default=16)
step = commands.add_parser("step")
step.add_argument("--fixture", required=True)
step.add_argument("--rate", type=float, required=True)
step.add_argument("--rate-from", type=float, default=0.0)
step.add_argument("--warmup", type=float, required=True)
step.add_argument("--window", type=float, required=True)
step.add_argument("--mix", default="70,20,10")
step.add_argument("--procs", type=int, default=4)
step.add_argument("--seed", type=int, default=1)
step.add_argument("--timeout", type=float, default=10)
step.add_argument("--max-inflight", type=int, default=4000)
step.add_argument("--key-prefix", required=True)
step.add_argument("--abort-after", type=float, default=5)
step.add_argument("--abort-error-share", type=float, default=0.5)
step.add_argument("--abort-p95-ms", type=float, default=2000)
constant = commands.add_parser("constant")
constant.add_argument("--fixture", required=True)
constant.add_argument("--write-rate", type=float, required=True)
constant.add_argument("--read-rate", type=float, required=True)
constant.add_argument("--download-rate", type=float, default=0.0)
constant.add_argument("--downloads", help="JSON-файл со списком id вложений")
constant.add_argument("--duration", type=float, required=True)
constant.add_argument("--procs", type=int, default=2)
constant.add_argument("--seed", type=int, default=1)
constant.add_argument("--timeout", type=float, default=15)
constant.add_argument("--max-inflight", type=int, default=4000)
constant.add_argument("--key-prefix", required=True)
constant.add_argument("--log", required=True)
# появился файл — новых запросов нет (фоновая нагрузка профиля reconcile)
constant.add_argument("--stop-file")
retry = commands.add_parser("retry")
retry.add_argument("--fixture", required=True)
retry.add_argument("--attempts", type=int, default=20)
retry.add_argument("logs", nargs="+")
body = commands.add_parser("body")
body.add_argument("--size", type=int, required=True)
body.add_argument("--seed", type=int, default=1)
body.add_argument("--out", required=True)
uploads = commands.add_parser("uploads")
uploads.add_argument("--fixture", required=True)
uploads.add_argument("--body", required=True)
uploads.add_argument("--clients", type=int, required=True)
uploads.add_argument("--warmup", type=float, required=True)
uploads.add_argument("--window", type=float, required=True)
uploads.add_argument("--timeout", type=float, default=90)
return root
def main() -> None:
args = parser().parse_args()
if args.command == "step":
result = cmd_step(args)
elif args.command == "constant":
result = cmd_constant(args)
elif args.command == "body":
result = cmd_body(args)
else:
handler = {
"ready": cmd_ready,
"setup": cmd_setup,
"retry": cmd_retry,
"uploads": cmd_uploads,
}
result = asyncio.run(handler[args.command](args))
json.dump(result, sys.stdout, ensure_ascii=False)
sys.stdout.write("\n")
sys.stdout.flush()
# дочерние процессы уже завершены; без ожидания финализаторов
with contextlib.suppress(Exception):
sys.stdout.close()
os._exit(0)
if __name__ == "__main__":
main()