forked from mtusi2026/mtusi2026lab3
- scripts/rolling-update.sh: сборка, однократные миграции новым образом, затем для каждой старой реплики: +1 новая (healthy и UP в HAProxy), graceful stop старой; ревизия образа видна в label и логе "starting". - bench/loadgen.py: фоновая нагрузка через LB (POST с Idempotency-Key + GET), неопределённый POST повторяется тем же ключом, проверка истории на потери и дубли. - bench/failover_series.py: каждая реплика по очереди в режимах stop / kill / pause; RPS, ошибки, время выхода из ротации и возврата. - bench/rolling_under_load.py: rolling update под нагрузкой. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
213 lines
9.4 KiB
Python
213 lines
9.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Background HTTP traffic through the LB for failover / rolling-update experiments (level 4).
|
|
|
|
Closed loop, N worker threads, each: POST a message (Idempotency-Key) + GET the last page.
|
|
An uncertain POST (timeout, connection error, 5xx) is retried with the SAME key and body,
|
|
exactly as a correct client must do; the history is checked afterwards for losses and
|
|
duplicates. Every request is recorded as (t, kind, status, instance, latency).
|
|
This is a correctness/availability probe, not a capacity benchmark (see bench/track1 for that).
|
|
|
|
python bench/loadgen.py --duration 30 --workers 4 # prints a summary JSON
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import math
|
|
import os
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from collections import Counter, defaultdict
|
|
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
|
|
|
|
|
class Http:
|
|
"""Minimal stdlib client with keep-alive per thread (http.client under urllib is not)."""
|
|
|
|
def __init__(self, base_url: str, timeout: float):
|
|
import http.client
|
|
from urllib.parse import urlsplit
|
|
parts = urlsplit(base_url)
|
|
self.host, self.port = parts.hostname, parts.port or 80
|
|
self.timeout = timeout
|
|
self.factory = lambda: http.client.HTTPConnection(self.host, self.port, timeout=self.timeout)
|
|
self.conn = self.factory()
|
|
|
|
def request(self, method: str, path: str, body: bytes | None = None, headers: dict | None = None):
|
|
try:
|
|
self.conn.request(method, path, body=body, headers=headers or {})
|
|
response = self.conn.getresponse()
|
|
data = response.read()
|
|
if response.getheader("connection", "").lower() == "close":
|
|
self.conn.close()
|
|
self.conn = self.factory()
|
|
return response.status, dict(response.getheaders()), data
|
|
except Exception:
|
|
self.conn.close()
|
|
self.conn = self.factory()
|
|
raise
|
|
|
|
|
|
def call(method: str, path: str, body=None, headers=None, timeout=10.0):
|
|
data = json.dumps(body).encode() if body is not None else None
|
|
h = {"Content-Type": "application/json", **(headers or {})}
|
|
req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return r.status, json.loads(r.read() or b"null")
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, json.loads(e.read() or b"null")
|
|
|
|
|
|
def make_user(label: str) -> dict:
|
|
username = f"{label}_{uuid.uuid4().hex[:12]}"
|
|
password = "Load-password-" + uuid.uuid4().hex
|
|
status, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
|
|
assert status == 201, (status, user)
|
|
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
status, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
|
assert status == 201, (status, session)
|
|
return {"id": user["id"], "token": session["token"]}
|
|
|
|
|
|
class Load:
|
|
def __init__(self, workers: int = 4, timeout: float = 3.0, pause: float = 0.0):
|
|
self.workers = workers
|
|
self.timeout = timeout
|
|
self.pause = pause
|
|
self.alice, self.bob = make_user("load"), make_user("load")
|
|
status, chat = call("POST", "/api/v1/chats", {"title": "load", "member_ids": [self.alice["id"], self.bob["id"]]},
|
|
headers={"Authorization": "Bearer " + self.alice["token"]})
|
|
assert status == 201, chat
|
|
self.chat = chat
|
|
self.stop_event = threading.Event()
|
|
self.lock = threading.Lock()
|
|
self.events: list[tuple[float, str, int, str, float]] = []
|
|
self.confirmed: dict[str, dict] = {}
|
|
self.retries = 0
|
|
self.t0 = time.monotonic()
|
|
self.threads = [threading.Thread(target=self._worker, args=(i,), daemon=True) for i in range(workers)]
|
|
|
|
def now(self) -> float:
|
|
return time.monotonic() - self.t0
|
|
|
|
def _record(self, kind: str, status: int, instance: str, latency: float) -> None:
|
|
with self.lock:
|
|
self.events.append((round(self.now(), 4), kind, status, instance, round(latency * 1000, 2)))
|
|
|
|
def _worker(self, index: int) -> None:
|
|
http = Http(BASE_URL, self.timeout)
|
|
auth = {"Authorization": "Bearer " + self.alice["token"]}
|
|
path = f"/api/v1/chats/{self.chat['id']}/messages"
|
|
n, key = 0, uuid.uuid4().hex
|
|
text = f"w{index}-0-{key[:6]}"
|
|
while not self.stop_event.is_set():
|
|
started = time.monotonic()
|
|
try:
|
|
status, headers, data = http.request("POST", path, json.dumps({"text": text}).encode(),
|
|
{**auth, "Idempotency-Key": key, "Content-Type": "application/json"})
|
|
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
|
|
except Exception:
|
|
status, instance, data = 0, "-", b""
|
|
self._record("POST", status, instance, time.monotonic() - started)
|
|
if status == 201:
|
|
msg = json.loads(data)
|
|
with self.lock:
|
|
self.confirmed[msg["id"]] = msg
|
|
n += 1
|
|
key = uuid.uuid4().hex
|
|
text = f"w{index}-{n}-{key[:6]}"
|
|
elif status in (0, 502, 503, 504, 500):
|
|
with self.lock:
|
|
self.retries += 1 # same key + same text on the next iteration
|
|
started = time.monotonic()
|
|
try:
|
|
status, headers, _ = http.request("GET", path + "?limit=5", None, auth)
|
|
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
|
|
except Exception:
|
|
status, instance = 0, "-"
|
|
self._record("GET", status, instance, time.monotonic() - started)
|
|
if self.pause:
|
|
time.sleep(self.pause)
|
|
|
|
def start(self) -> "Load":
|
|
self.t0 = time.monotonic()
|
|
for t in self.threads:
|
|
t.start()
|
|
return self
|
|
|
|
def stop(self) -> None:
|
|
self.stop_event.set()
|
|
for t in self.threads:
|
|
t.join(timeout=15)
|
|
|
|
def verify_history(self) -> dict:
|
|
"""Walk the whole history with the second user: no confirmed message lost, no duplicate."""
|
|
items, cursor = [], None
|
|
while True:
|
|
q = "?limit=100" + (f"&cursor={cursor}" if cursor else "")
|
|
status, page = call("GET", f"/api/v1/chats/{self.chat['id']}/messages{q}",
|
|
headers={"Authorization": "Bearer " + self.bob["token"]}, timeout=30)
|
|
assert status == 200, page
|
|
items += page["items"]
|
|
cursor = page["next_cursor"]
|
|
if not cursor:
|
|
break
|
|
ids = [m["id"] for m in items]
|
|
texts = [m["text"] for m in items]
|
|
lost = [i for i in self.confirmed if i not in set(ids)]
|
|
changed = [m for m in items if m["id"] in self.confirmed and self.confirmed[m["id"]] != m]
|
|
return {"history_messages": len(items), "confirmed_messages": len(self.confirmed),
|
|
"lost_confirmed": len(lost), "altered_confirmed": len(changed),
|
|
"duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)),
|
|
"unconfirmed_but_stored": len(set(ids) - set(self.confirmed))}
|
|
|
|
# ------------------------------------------------------------------ summaries
|
|
def window(self, start: float, end: float) -> dict:
|
|
ev = [e for e in self.events if start <= e[0] < end]
|
|
ok = [e for e in ev if e[2] in (200, 201)]
|
|
lat = sorted(e[4] for e in ok)
|
|
|
|
def pct(p: float):
|
|
return lat[max(0, math.ceil(p * len(lat)) - 1)] if lat else None
|
|
span = max(end - start, 1e-9)
|
|
return {"seconds": round(span, 2), "requests": len(ev), "ok": len(ok), "failed": len(ev) - len(ok),
|
|
"ok_rps": round(len(ok) / span, 1), "error_ratio": round((len(ev) - len(ok)) / len(ev), 5) if ev else None,
|
|
"statuses": dict(Counter(e[2] for e in ev)), "instances": dict(Counter(e[3] for e in ok)),
|
|
"p50_ms": pct(.5), "p95_ms": pct(.95), "p99_ms": pct(.99)}
|
|
|
|
def per_second(self) -> list[dict]:
|
|
buckets: dict[int, list] = defaultdict(list)
|
|
for e in self.events:
|
|
buckets[int(e[0])].append(e)
|
|
rows = []
|
|
for second in sorted(buckets):
|
|
ev = buckets[second]
|
|
rows.append({"second": second, "ok": sum(1 for e in ev if e[2] in (200, 201)),
|
|
"failed": sum(1 for e in ev if e[2] not in (200, 201)),
|
|
"instances": "|".join(f"{k}:{v}" for k, v in sorted(Counter(e[3] for e in ev).items()))})
|
|
return rows
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
parser.add_argument("--duration", type=float, default=20)
|
|
parser.add_argument("--workers", type=int, default=4)
|
|
args = parser.parse_args()
|
|
load = Load(args.workers).start()
|
|
time.sleep(args.duration)
|
|
load.stop()
|
|
print(json.dumps({"total": load.window(0, load.now()), "history": load.verify_history(),
|
|
"client_retries_same_key": load.retries}, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|