246 lines
10 KiB
Python
246 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Нагрузка записью через балансировщик во время отказа или замены реплик.
|
|
|
|
stop — по очереди останавливает каждую реплику (SIGTERM), восстанавливая предыдущую;
|
|
kill — то же, но SIGKILL, без корректного завершения;
|
|
pause — то же, но реплика зависает (docker pause): соединения открыты, ответов нет;
|
|
rolling — запускает scripts/rolling-update.sh.
|
|
|
|
Клиент ведёт себя как в контракте: при неопределённом ответе повторяет запрос с тем же ключом.
|
|
В конце вся история сверяется с подтверждёнными сообщениями.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import math
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "student-tests"))
|
|
from api import call, history, new_chat, new_user, send
|
|
|
|
WORKERS = 4
|
|
|
|
|
|
def compose(*args):
|
|
command = ["docker", "compose", "--env-file", ".env", *args]
|
|
return subprocess.run(command, cwd=ROOT, check=True, capture_output=True, text=True).stdout
|
|
|
|
|
|
def api_containers():
|
|
return compose("ps", "-q", "api").split()
|
|
|
|
|
|
def wait_ready(seconds=60):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if call("GET", "/health/ready", timeout=2)[0] == 200:
|
|
return
|
|
except OSError:
|
|
pass
|
|
time.sleep(0.2)
|
|
raise SystemExit("стенд не стал готов")
|
|
|
|
|
|
def wait_replicas(count):
|
|
"""Ждёт, пока балансировщик начнёт отдавать запросы на count разных реплик."""
|
|
deadline = time.monotonic() + 30
|
|
while time.monotonic() < deadline:
|
|
seen = {call("GET", "/health/live", timeout=2)[2]["X-Instance-Id"] for _ in range(count * 4)}
|
|
if len(seen) >= count:
|
|
return
|
|
time.sleep(0.2)
|
|
raise SystemExit("балансировщик не видит %d реплик" % count)
|
|
|
|
|
|
class Load:
|
|
"""Несколько клиентов пишут в один чат, каждое событие сохраняется с отметкой времени."""
|
|
|
|
def __init__(self):
|
|
self.user = new_user("chaos")
|
|
self.chat = new_chat(self.user)
|
|
self.stop = threading.Event()
|
|
self.events = [] # (время, ok, реплика, статус)
|
|
self.confirmed = {}
|
|
self.lock = threading.Lock()
|
|
self.threads = [threading.Thread(target=self.worker, args=(n,)) for n in range(WORKERS)]
|
|
|
|
def worker(self, number):
|
|
index = 0
|
|
while not self.stop.is_set():
|
|
text = "worker-%d-%d-%s" % (number, index, uuid.uuid4().hex[:8])
|
|
key = uuid.uuid4().hex
|
|
index += 1
|
|
while True:
|
|
try:
|
|
status, body, headers = send(self.user, self.chat, text, key=key, timeout=3)
|
|
instance = headers.get("X-Instance-Id")
|
|
except OSError:
|
|
status, body, instance = 0, None, None
|
|
now = time.monotonic()
|
|
ok = status == 201
|
|
with self.lock:
|
|
self.events.append((now, ok, instance, status))
|
|
if ok:
|
|
self.confirmed[body["id"]] = body
|
|
if ok or self.stop.is_set():
|
|
break
|
|
time.sleep(0.05) # неопределённый результат: тот же текст и тот же ключ
|
|
|
|
def start(self):
|
|
for thread in self.threads:
|
|
thread.start()
|
|
|
|
def finish(self):
|
|
self.stop.set()
|
|
for thread in self.threads:
|
|
thread.join()
|
|
|
|
|
|
def percentile(values, p):
|
|
ordered = sorted(values)
|
|
return round(ordered[max(0, math.ceil(p * len(ordered)) - 1)], 3) if ordered else None
|
|
|
|
|
|
def summary(events, started, action_at, target=None):
|
|
"""Показатели одного интервала: от action_at считается время до восстановления записи."""
|
|
ok = [e for e in events if e[1]]
|
|
failed = [e for e in events if not e[1]]
|
|
elapsed = events[-1][0] - started
|
|
gaps = [b[0] - a[0] for a, b in zip(ok, ok[1:])]
|
|
after = [e for e in ok if e[0] >= action_at]
|
|
last_on_target = [e[0] for e in ok if target and e[2] and target.startswith(e[2])]
|
|
return {
|
|
"seconds": round(elapsed, 2),
|
|
"successful_requests": len(ok),
|
|
"failed_attempts": len(failed),
|
|
"failed_statuses": sorted({e[3] for e in failed}),
|
|
"successful_rps": round(len(ok) / elapsed, 1),
|
|
"longest_gap_between_successes_s": round(max(gaps), 3) if gaps else None,
|
|
"first_success_after_action_s": round(after[0][0] - action_at, 3) if after else None,
|
|
"last_success_on_target_after_action_s": round(last_on_target[-1] - action_at, 3)
|
|
if last_on_target and last_on_target[-1] >= action_at else None,
|
|
"instances_seen": len({e[2] for e in ok}),
|
|
}
|
|
|
|
|
|
def verify(load):
|
|
items, _ = history(load.user, load.chat, 100)
|
|
by_id = {item["id"]: item for item in items}
|
|
texts = [item["text"] for item in items]
|
|
lost = [mid for mid, message in load.confirmed.items() if by_id.get(mid) != message]
|
|
result = {
|
|
"confirmed": len(load.confirmed),
|
|
"in_history": len(items),
|
|
"lost_confirmed": len(lost),
|
|
"duplicate_ids": len(items) - len(by_id),
|
|
"duplicate_texts": len(texts) - len(set(texts)),
|
|
}
|
|
result["unconfirmed_but_saved"] = len(items) - len(load.confirmed)
|
|
return result
|
|
|
|
|
|
def replicas_up():
|
|
"""Сколько реплик балансировщик считает рабочими, по его странице статистики."""
|
|
csv = compose("exec", "-T", "lb", "wget", "-q", "-O", "-", "http://127.0.0.1:8404/stats;csv")
|
|
return sum(line.startswith("api,api") and ",UP" in line for line in csv.splitlines())
|
|
|
|
|
|
def seconds_until_excluded(replicas, since, limit=8):
|
|
while time.monotonic() - since < limit:
|
|
if replicas_up() < replicas:
|
|
return round(time.monotonic() - since, 2)
|
|
time.sleep(0.1)
|
|
return None
|
|
|
|
|
|
def replica_round(signal):
|
|
containers = api_containers()
|
|
wait_replicas(len(containers))
|
|
rounds = []
|
|
for container in containers:
|
|
load = Load()
|
|
started = time.monotonic()
|
|
load.start()
|
|
time.sleep(5)
|
|
action_at = time.monotonic()
|
|
command = {"SIGTERM": "stop", "SIGKILL": "kill", "HANG": "pause"}[signal]
|
|
subprocess.run(["docker", command, container], check=True, stdout=subprocess.DEVNULL)
|
|
stop_took = time.monotonic() - action_at
|
|
excluded_after = seconds_until_excluded(len(containers), action_at)
|
|
time.sleep(max(0, 10 - (time.monotonic() - action_at)))
|
|
command = "unpause" if signal == "HANG" else "start"
|
|
subprocess.run(["docker", command, container], check=True, stdout=subprocess.DEVNULL)
|
|
wait_replicas(len(containers))
|
|
time.sleep(2)
|
|
load.finish()
|
|
result = summary(load.events, started, action_at, container)
|
|
result.update({"target": container[:12], "signal": signal, "stop_command_s": round(stop_took, 2),
|
|
"excluded_by_lb_after_s": excluded_after})
|
|
result["verify"] = verify(load)
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
rounds.append(result)
|
|
return rounds
|
|
|
|
|
|
def images():
|
|
inspect = ["docker", "inspect", "-f", "{{.Image}}"]
|
|
return {c[:12]: subprocess.check_output(inspect + [c], text=True).strip()[:19] for c in api_containers()}
|
|
|
|
|
|
def rolling_round():
|
|
before = images()
|
|
wait_replicas(len(before))
|
|
load = Load()
|
|
started = time.monotonic()
|
|
load.start()
|
|
time.sleep(5)
|
|
action_at = time.monotonic()
|
|
subprocess.run(["./scripts/rolling-update.sh"], cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
update_took = time.monotonic() - action_at
|
|
time.sleep(5)
|
|
load.finish()
|
|
after = images()
|
|
result = summary(load.events, started, action_at)
|
|
result.update({"update_seconds": round(update_took, 2), "containers_before": before, "containers_after": after,
|
|
"app_version": os.environ.get("APP_VERSION", "dev")})
|
|
result["verify"] = verify(load)
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
# без этой проверки скрипт, который ничего не заменил, выглядел бы как идеальное обновление
|
|
if set(before) & set(after) or set(before.values()) & set(after.values()) or len(after) != len(before):
|
|
raise SystemExit("FAIL: реплики не заменены новым образом")
|
|
return [result]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument("mode", choices=["stop", "kill", "pause", "rolling"])
|
|
parser.add_argument("--output")
|
|
args = parser.parse_args()
|
|
wait_ready()
|
|
if args.mode == "rolling":
|
|
rounds = rolling_round()
|
|
else:
|
|
rounds = replica_round({"stop": "SIGTERM", "kill": "SIGKILL", "pause": "HANG"}[args.mode])
|
|
report = {
|
|
"workload": "%d клиента, closed loop, POST сообщений через балансировщик, повтор с тем же ключом" % WORKERS,
|
|
"mode": args.mode,
|
|
"rounds": rounds,
|
|
}
|
|
if args.output:
|
|
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
|
bad = [r for r in rounds if r["verify"]["lost_confirmed"] or r["verify"]["duplicate_ids"] or r["verify"]["duplicate_texts"]]
|
|
if bad:
|
|
raise SystemExit("FAIL: потеряны или продублированы сообщения")
|
|
print("PASS: подтверждённые сообщения на месте, дублей нет")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|