#!/usr/bin/env python3 """Нагрузка записью через балансировщик во время отказа или замены реплик. stop — по очереди останавливает каждую реплику (SIGTERM), восстанавливая предыдущую; kill — то же, но SIGKILL, без корректного завершения; pause — то же, но реплика зависает (docker pause): соединения открыты, ответов нет; rolling — запускает scripts/rolling-update.sh. Клиент ведёт себя как в контракте: при неопределённом ответе повторяет запрос с тем же ключом. В конце вся история сверяется с подтверждёнными сообщениями. """ import argparse import json import math import os import subprocess import sys import threading import time import uuid from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "student-tests")) from api import call, history, new_chat, new_user, send WORKERS = 4 def compose(*args): command = ["docker", "compose", "--env-file", ".env", *args] return subprocess.run(command, cwd=ROOT, check=True, capture_output=True, text=True).stdout def api_containers(): return compose("ps", "-q", "api").split() def wait_ready(seconds=60): deadline = time.monotonic() + seconds while time.monotonic() < deadline: try: if call("GET", "/health/ready", timeout=2)[0] == 200: return except OSError: pass time.sleep(0.2) raise SystemExit("стенд не стал готов") def wait_replicas(count): """Ждёт, пока балансировщик начнёт отдавать запросы на count разных реплик.""" deadline = time.monotonic() + 30 while time.monotonic() < deadline: seen = {call("GET", "/health/live", timeout=2)[2]["X-Instance-Id"] for _ in range(count * 4)} if len(seen) >= count: return time.sleep(0.2) raise SystemExit("балансировщик не видит %d реплик" % count) class Load: """Несколько клиентов пишут в один чат, каждое событие сохраняется с отметкой времени.""" def __init__(self): self.user = new_user("chaos") self.chat = new_chat(self.user) self.stop = threading.Event() self.events = [] # (время, ok, реплика, статус) self.confirmed = {} self.lock = threading.Lock() self.threads = [threading.Thread(target=self.worker, args=(n,)) for n in range(WORKERS)] def worker(self, number): index = 0 while not self.stop.is_set(): text = "worker-%d-%d-%s" % (number, index, uuid.uuid4().hex[:8]) key = uuid.uuid4().hex index += 1 while True: try: status, body, headers = send(self.user, self.chat, text, key=key, timeout=3) instance = headers.get("X-Instance-Id") except OSError: status, body, instance = 0, None, None now = time.monotonic() ok = status == 201 with self.lock: self.events.append((now, ok, instance, status)) if ok: self.confirmed[body["id"]] = body if ok or self.stop.is_set(): break time.sleep(0.05) # неопределённый результат: тот же текст и тот же ключ def start(self): for thread in self.threads: thread.start() def finish(self): self.stop.set() for thread in self.threads: thread.join() def percentile(values, p): ordered = sorted(values) return round(ordered[max(0, math.ceil(p * len(ordered)) - 1)], 3) if ordered else None def summary(events, started, action_at, target=None): """Показатели одного интервала: от action_at считается время до восстановления записи.""" ok = [e for e in events if e[1]] failed = [e for e in events if not e[1]] elapsed = events[-1][0] - started gaps = [b[0] - a[0] for a, b in zip(ok, ok[1:])] after = [e for e in ok if e[0] >= action_at] last_on_target = [e[0] for e in ok if target and e[2] and target.startswith(e[2])] return { "seconds": round(elapsed, 2), "successful_requests": len(ok), "failed_attempts": len(failed), "failed_statuses": sorted({e[3] for e in failed}), "successful_rps": round(len(ok) / elapsed, 1), "longest_gap_between_successes_s": round(max(gaps), 3) if gaps else None, "first_success_after_action_s": round(after[0][0] - action_at, 3) if after else None, "last_success_on_target_after_action_s": round(last_on_target[-1] - action_at, 3) if last_on_target and last_on_target[-1] >= action_at else None, "instances_seen": len({e[2] for e in ok}), } def verify(load): items, _ = history(load.user, load.chat, 100) by_id = {item["id"]: item for item in items} texts = [item["text"] for item in items] lost = [mid for mid, message in load.confirmed.items() if by_id.get(mid) != message] result = { "confirmed": len(load.confirmed), "in_history": len(items), "lost_confirmed": len(lost), "duplicate_ids": len(items) - len(by_id), "duplicate_texts": len(texts) - len(set(texts)), } result["unconfirmed_but_saved"] = len(items) - len(load.confirmed) return result def replicas_up(): """Сколько реплик балансировщик считает рабочими, по его странице статистики.""" csv = compose("exec", "-T", "lb", "wget", "-q", "-O", "-", "http://127.0.0.1:8404/stats;csv") return sum(line.startswith("api,api") and ",UP" in line for line in csv.splitlines()) def seconds_until_excluded(replicas, since, limit=8): while time.monotonic() - since < limit: if replicas_up() < replicas: return round(time.monotonic() - since, 2) time.sleep(0.1) return None def replica_round(signal): containers = api_containers() wait_replicas(len(containers)) rounds = [] for container in containers: load = Load() started = time.monotonic() load.start() time.sleep(5) action_at = time.monotonic() command = {"SIGTERM": "stop", "SIGKILL": "kill", "HANG": "pause"}[signal] subprocess.run(["docker", command, container], check=True, stdout=subprocess.DEVNULL) stop_took = time.monotonic() - action_at excluded_after = seconds_until_excluded(len(containers), action_at) time.sleep(max(0, 10 - (time.monotonic() - action_at))) command = "unpause" if signal == "HANG" else "start" subprocess.run(["docker", command, container], check=True, stdout=subprocess.DEVNULL) wait_replicas(len(containers)) time.sleep(2) load.finish() result = summary(load.events, started, action_at, container) result.update({"target": container[:12], "signal": signal, "stop_command_s": round(stop_took, 2), "excluded_by_lb_after_s": excluded_after}) result["verify"] = verify(load) print(json.dumps(result, ensure_ascii=False)) rounds.append(result) return rounds def images(): inspect = ["docker", "inspect", "-f", "{{.Image}}"] return {c[:12]: subprocess.check_output(inspect + [c], text=True).strip()[:19] for c in api_containers()} def rolling_round(): before = images() wait_replicas(len(before)) load = Load() started = time.monotonic() load.start() time.sleep(5) action_at = time.monotonic() subprocess.run(["./scripts/rolling-update.sh"], cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) update_took = time.monotonic() - action_at time.sleep(5) load.finish() after = images() result = summary(load.events, started, action_at) result.update({"update_seconds": round(update_took, 2), "containers_before": before, "containers_after": after, "app_version": os.environ.get("APP_VERSION", "dev")}) result["verify"] = verify(load) print(json.dumps(result, ensure_ascii=False)) # без этой проверки скрипт, который ничего не заменил, выглядел бы как идеальное обновление if set(before) & set(after) or set(before.values()) & set(after.values()) or len(after) != len(before): raise SystemExit("FAIL: реплики не заменены новым образом") return [result] def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("mode", choices=["stop", "kill", "pause", "rolling"]) parser.add_argument("--output") args = parser.parse_args() wait_ready() if args.mode == "rolling": rounds = rolling_round() else: rounds = replica_round({"stop": "SIGTERM", "kill": "SIGKILL", "pause": "HANG"}[args.mode]) report = { "workload": "%d клиента, closed loop, POST сообщений через балансировщик, повтор с тем же ключом" % WORKERS, "mode": args.mode, "rounds": rounds, } if args.output: Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") bad = [r for r in rounds if r["verify"]["lost_confirmed"] or r["verify"]["duplicate_ids"] or r["verify"]["duplicate_texts"]] if bad: raise SystemExit("FAIL: потеряны или продублированы сообщения") print("PASS: подтверждённые сообщения на месте, дублей нет") if __name__ == "__main__": main()