Files
mtusi2026lab4/scripts/chaos.py

246 lines
10 KiB
Python

#!/usr/bin/env python3
"""Нагрузка записью через балансировщик во время отказа или замены реплик.
stop — по очереди останавливает каждую реплику (SIGTERM), восстанавливая предыдущую;
kill — то же, но SIGKILL, без корректного завершения;
pause — то же, но реплика зависает (docker pause): соединения открыты, ответов нет;
rolling — запускает scripts/rolling-update.sh.
Клиент ведёт себя как в контракте: при неопределённом ответе повторяет запрос с тем же ключом.
В конце вся история сверяется с подтверждёнными сообщениями.
"""
import argparse
import json
import math
import os
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
from api import call, history, new_chat, new_user, send
WORKERS = 4
def compose(*args):
command = ["docker", "compose", "--env-file", ".env", *args]
return subprocess.run(command, cwd=ROOT, check=True, capture_output=True, text=True).stdout
def api_containers():
return compose("ps", "-q", "api").split()
def wait_ready(seconds=60):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
try:
if call("GET", "/health/ready", timeout=2)[0] == 200:
return
except OSError:
pass
time.sleep(0.2)
raise SystemExit("стенд не стал готов")
def wait_replicas(count):
"""Ждёт, пока балансировщик начнёт отдавать запросы на count разных реплик."""
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
seen = {call("GET", "/health/live", timeout=2)[2]["X-Instance-Id"] for _ in range(count * 4)}
if len(seen) >= count:
return
time.sleep(0.2)
raise SystemExit("балансировщик не видит %d реплик" % count)
class Load:
"""Несколько клиентов пишут в один чат, каждое событие сохраняется с отметкой времени."""
def __init__(self):
self.user = new_user("chaos")
self.chat = new_chat(self.user)
self.stop = threading.Event()
self.events = [] # (время, ok, реплика, статус)
self.confirmed = {}
self.lock = threading.Lock()
self.threads = [threading.Thread(target=self.worker, args=(n,)) for n in range(WORKERS)]
def worker(self, number):
index = 0
while not self.stop.is_set():
text = "worker-%d-%d-%s" % (number, index, uuid.uuid4().hex[:8])
key = uuid.uuid4().hex
index += 1
while True:
try:
status, body, headers = send(self.user, self.chat, text, key=key, timeout=3)
instance = headers.get("X-Instance-Id")
except OSError:
status, body, instance = 0, None, None
now = time.monotonic()
ok = status == 201
with self.lock:
self.events.append((now, ok, instance, status))
if ok:
self.confirmed[body["id"]] = body
if ok or self.stop.is_set():
break
time.sleep(0.05) # неопределённый результат: тот же текст и тот же ключ
def start(self):
for thread in self.threads:
thread.start()
def finish(self):
self.stop.set()
for thread in self.threads:
thread.join()
def percentile(values, p):
ordered = sorted(values)
return round(ordered[max(0, math.ceil(p * len(ordered)) - 1)], 3) if ordered else None
def summary(events, started, action_at, target=None):
"""Показатели одного интервала: от action_at считается время до восстановления записи."""
ok = [e for e in events if e[1]]
failed = [e for e in events if not e[1]]
elapsed = events[-1][0] - started
gaps = [b[0] - a[0] for a, b in zip(ok, ok[1:])]
after = [e for e in ok if e[0] >= action_at]
last_on_target = [e[0] for e in ok if target and e[2] and target.startswith(e[2])]
return {
"seconds": round(elapsed, 2),
"successful_requests": len(ok),
"failed_attempts": len(failed),
"failed_statuses": sorted({e[3] for e in failed}),
"successful_rps": round(len(ok) / elapsed, 1),
"longest_gap_between_successes_s": round(max(gaps), 3) if gaps else None,
"first_success_after_action_s": round(after[0][0] - action_at, 3) if after else None,
"last_success_on_target_after_action_s": round(last_on_target[-1] - action_at, 3)
if last_on_target and last_on_target[-1] >= action_at else None,
"instances_seen": len({e[2] for e in ok}),
}
def verify(load):
items, _ = history(load.user, load.chat, 100)
by_id = {item["id"]: item for item in items}
texts = [item["text"] for item in items]
lost = [mid for mid, message in load.confirmed.items() if by_id.get(mid) != message]
result = {
"confirmed": len(load.confirmed),
"in_history": len(items),
"lost_confirmed": len(lost),
"duplicate_ids": len(items) - len(by_id),
"duplicate_texts": len(texts) - len(set(texts)),
}
result["unconfirmed_but_saved"] = len(items) - len(load.confirmed)
return result
def replicas_up():
"""Сколько реплик балансировщик считает рабочими, по его странице статистики."""
csv = compose("exec", "-T", "lb", "wget", "-q", "-O", "-", "http://127.0.0.1:8404/stats;csv")
return sum(line.startswith("api,api") and ",UP" in line for line in csv.splitlines())
def seconds_until_excluded(replicas, since, limit=8):
while time.monotonic() - since < limit:
if replicas_up() < replicas:
return round(time.monotonic() - since, 2)
time.sleep(0.1)
return None
def replica_round(signal):
containers = api_containers()
wait_replicas(len(containers))
rounds = []
for container in containers:
load = Load()
started = time.monotonic()
load.start()
time.sleep(5)
action_at = time.monotonic()
command = {"SIGTERM": "stop", "SIGKILL": "kill", "HANG": "pause"}[signal]
subprocess.run(["docker", command, container], check=True, stdout=subprocess.DEVNULL)
stop_took = time.monotonic() - action_at
excluded_after = seconds_until_excluded(len(containers), action_at)
time.sleep(max(0, 10 - (time.monotonic() - action_at)))
command = "unpause" if signal == "HANG" else "start"
subprocess.run(["docker", command, container], check=True, stdout=subprocess.DEVNULL)
wait_replicas(len(containers))
time.sleep(2)
load.finish()
result = summary(load.events, started, action_at, container)
result.update({"target": container[:12], "signal": signal, "stop_command_s": round(stop_took, 2),
"excluded_by_lb_after_s": excluded_after})
result["verify"] = verify(load)
print(json.dumps(result, ensure_ascii=False))
rounds.append(result)
return rounds
def images():
inspect = ["docker", "inspect", "-f", "{{.Image}}"]
return {c[:12]: subprocess.check_output(inspect + [c], text=True).strip()[:19] for c in api_containers()}
def rolling_round():
before = images()
wait_replicas(len(before))
load = Load()
started = time.monotonic()
load.start()
time.sleep(5)
action_at = time.monotonic()
subprocess.run(["./scripts/rolling-update.sh"], cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
update_took = time.monotonic() - action_at
time.sleep(5)
load.finish()
after = images()
result = summary(load.events, started, action_at)
result.update({"update_seconds": round(update_took, 2), "containers_before": before, "containers_after": after,
"app_version": os.environ.get("APP_VERSION", "dev")})
result["verify"] = verify(load)
print(json.dumps(result, ensure_ascii=False))
# без этой проверки скрипт, который ничего не заменил, выглядел бы как идеальное обновление
if set(before) & set(after) or set(before.values()) & set(after.values()) or len(after) != len(before):
raise SystemExit("FAIL: реплики не заменены новым образом")
return [result]
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("mode", choices=["stop", "kill", "pause", "rolling"])
parser.add_argument("--output")
args = parser.parse_args()
wait_ready()
if args.mode == "rolling":
rounds = rolling_round()
else:
rounds = replica_round({"stop": "SIGTERM", "kill": "SIGKILL", "pause": "HANG"}[args.mode])
report = {
"workload": "%d клиента, closed loop, POST сообщений через балансировщик, повтор с тем же ключом" % WORKERS,
"mode": args.mode,
"rounds": rounds,
}
if args.output:
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
bad = [r for r in rounds if r["verify"]["lost_confirmed"] or r["verify"]["duplicate_ids"] or r["verify"]["duplicate_texts"]]
if bad:
raise SystemExit("FAIL: потеряны или продублированы сообщения")
print("PASS: подтверждённые сообщения на месте, дублей нет")
if __name__ == "__main__":
main()