Files
mtusi2026lab3/bench/loadgen.py
Alexey BodanyukandClaude Opus 5.5 c050642ec2 feat: rolling update без простоя и серия отказов реплик под нагрузкой (уровень 4)
- scripts/rolling-update.sh: сборка, однократные миграции новым образом,
  затем для каждой старой реплики: +1 новая (healthy и UP в HAProxy),
  graceful stop старой; ревизия образа видна в label и логе "starting".
- bench/loadgen.py: фоновая нагрузка через LB (POST с Idempotency-Key +
  GET), неопределённый POST повторяется тем же ключом, проверка истории
  на потери и дубли.
- bench/failover_series.py: каждая реплика по очереди в режимах stop /
  kill / pause; RPS, ошибки, время выхода из ротации и возврата.
- bench/rolling_under_load.py: rolling update под нагрузкой.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 13:39:54 +03:00

213 lines
9.4 KiB
Python

#!/usr/bin/env python3
"""Background HTTP traffic through the LB for failover / rolling-update experiments (level 4).
Closed loop, N worker threads, each: POST a message (Idempotency-Key) + GET the last page.
An uncertain POST (timeout, connection error, 5xx) is retried with the SAME key and body,
exactly as a correct client must do; the history is checked afterwards for losses and
duplicates. Every request is recorded as (t, kind, status, instance, latency).
This is a correctness/availability probe, not a capacity benchmark (see bench/track1 for that).
python bench/loadgen.py --duration 30 --workers 4 # prints a summary JSON
"""
from __future__ import annotations
import argparse
import base64
import json
import math
import os
import threading
import time
import uuid
from collections import Counter, defaultdict
import urllib.error
import urllib.request
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
class Http:
"""Minimal stdlib client with keep-alive per thread (http.client under urllib is not)."""
def __init__(self, base_url: str, timeout: float):
import http.client
from urllib.parse import urlsplit
parts = urlsplit(base_url)
self.host, self.port = parts.hostname, parts.port or 80
self.timeout = timeout
self.factory = lambda: http.client.HTTPConnection(self.host, self.port, timeout=self.timeout)
self.conn = self.factory()
def request(self, method: str, path: str, body: bytes | None = None, headers: dict | None = None):
try:
self.conn.request(method, path, body=body, headers=headers or {})
response = self.conn.getresponse()
data = response.read()
if response.getheader("connection", "").lower() == "close":
self.conn.close()
self.conn = self.factory()
return response.status, dict(response.getheaders()), data
except Exception:
self.conn.close()
self.conn = self.factory()
raise
def call(method: str, path: str, body=None, headers=None, timeout=10.0):
data = json.dumps(body).encode() if body is not None else None
h = {"Content-Type": "application/json", **(headers or {})}
req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method)
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.status, json.loads(r.read() or b"null")
except urllib.error.HTTPError as e:
return e.code, json.loads(e.read() or b"null")
def make_user(label: str) -> dict:
username = f"{label}_{uuid.uuid4().hex[:12]}"
password = "Load-password-" + uuid.uuid4().hex
status, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
assert status == 201, (status, user)
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201, (status, session)
return {"id": user["id"], "token": session["token"]}
class Load:
def __init__(self, workers: int = 4, timeout: float = 3.0, pause: float = 0.0):
self.workers = workers
self.timeout = timeout
self.pause = pause
self.alice, self.bob = make_user("load"), make_user("load")
status, chat = call("POST", "/api/v1/chats", {"title": "load", "member_ids": [self.alice["id"], self.bob["id"]]},
headers={"Authorization": "Bearer " + self.alice["token"]})
assert status == 201, chat
self.chat = chat
self.stop_event = threading.Event()
self.lock = threading.Lock()
self.events: list[tuple[float, str, int, str, float]] = []
self.confirmed: dict[str, dict] = {}
self.retries = 0
self.t0 = time.monotonic()
self.threads = [threading.Thread(target=self._worker, args=(i,), daemon=True) for i in range(workers)]
def now(self) -> float:
return time.monotonic() - self.t0
def _record(self, kind: str, status: int, instance: str, latency: float) -> None:
with self.lock:
self.events.append((round(self.now(), 4), kind, status, instance, round(latency * 1000, 2)))
def _worker(self, index: int) -> None:
http = Http(BASE_URL, self.timeout)
auth = {"Authorization": "Bearer " + self.alice["token"]}
path = f"/api/v1/chats/{self.chat['id']}/messages"
n, key = 0, uuid.uuid4().hex
text = f"w{index}-0-{key[:6]}"
while not self.stop_event.is_set():
started = time.monotonic()
try:
status, headers, data = http.request("POST", path, json.dumps({"text": text}).encode(),
{**auth, "Idempotency-Key": key, "Content-Type": "application/json"})
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
except Exception:
status, instance, data = 0, "-", b""
self._record("POST", status, instance, time.monotonic() - started)
if status == 201:
msg = json.loads(data)
with self.lock:
self.confirmed[msg["id"]] = msg
n += 1
key = uuid.uuid4().hex
text = f"w{index}-{n}-{key[:6]}"
elif status in (0, 502, 503, 504, 500):
with self.lock:
self.retries += 1 # same key + same text on the next iteration
started = time.monotonic()
try:
status, headers, _ = http.request("GET", path + "?limit=5", None, auth)
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
except Exception:
status, instance = 0, "-"
self._record("GET", status, instance, time.monotonic() - started)
if self.pause:
time.sleep(self.pause)
def start(self) -> "Load":
self.t0 = time.monotonic()
for t in self.threads:
t.start()
return self
def stop(self) -> None:
self.stop_event.set()
for t in self.threads:
t.join(timeout=15)
def verify_history(self) -> dict:
"""Walk the whole history with the second user: no confirmed message lost, no duplicate."""
items, cursor = [], None
while True:
q = "?limit=100" + (f"&cursor={cursor}" if cursor else "")
status, page = call("GET", f"/api/v1/chats/{self.chat['id']}/messages{q}",
headers={"Authorization": "Bearer " + self.bob["token"]}, timeout=30)
assert status == 200, page
items += page["items"]
cursor = page["next_cursor"]
if not cursor:
break
ids = [m["id"] for m in items]
texts = [m["text"] for m in items]
lost = [i for i in self.confirmed if i not in set(ids)]
changed = [m for m in items if m["id"] in self.confirmed and self.confirmed[m["id"]] != m]
return {"history_messages": len(items), "confirmed_messages": len(self.confirmed),
"lost_confirmed": len(lost), "altered_confirmed": len(changed),
"duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)),
"unconfirmed_but_stored": len(set(ids) - set(self.confirmed))}
# ------------------------------------------------------------------ summaries
def window(self, start: float, end: float) -> dict:
ev = [e for e in self.events if start <= e[0] < end]
ok = [e for e in ev if e[2] in (200, 201)]
lat = sorted(e[4] for e in ok)
def pct(p: float):
return lat[max(0, math.ceil(p * len(lat)) - 1)] if lat else None
span = max(end - start, 1e-9)
return {"seconds": round(span, 2), "requests": len(ev), "ok": len(ok), "failed": len(ev) - len(ok),
"ok_rps": round(len(ok) / span, 1), "error_ratio": round((len(ev) - len(ok)) / len(ev), 5) if ev else None,
"statuses": dict(Counter(e[2] for e in ev)), "instances": dict(Counter(e[3] for e in ok)),
"p50_ms": pct(.5), "p95_ms": pct(.95), "p99_ms": pct(.99)}
def per_second(self) -> list[dict]:
buckets: dict[int, list] = defaultdict(list)
for e in self.events:
buckets[int(e[0])].append(e)
rows = []
for second in sorted(buckets):
ev = buckets[second]
rows.append({"second": second, "ok": sum(1 for e in ev if e[2] in (200, 201)),
"failed": sum(1 for e in ev if e[2] not in (200, 201)),
"instances": "|".join(f"{k}:{v}" for k, v in sorted(Counter(e[3] for e in ev).items()))})
return rows
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--duration", type=float, default=20)
parser.add_argument("--workers", type=int, default=4)
args = parser.parse_args()
load = Load(args.workers).start()
time.sleep(args.duration)
load.stop()
print(json.dumps({"total": load.window(0, load.now()), "history": load.verify_history(),
"client_retries_same_key": load.retries}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())