#!/usr/bin/env python3 """Нагрузка через LB во время отказа/замены API-реплик; метрики и проверка данных. failover — для каждой API-реплики по очереди: нагрузка → остановка этой реплики (docker stop = SIGTERM, затем docker kill = SIGKILL) → восстановление (scripts/restore.sh) → нагрузка после восстановления. rolling — нагрузка во время scripts/rolling-update.sh (поочерёдная замена реплик). Клиент ведёт себя как настоящий: неподтверждённый POST повторяет с тем же Idempotency-Key и телом. После прогона вся история читается заново: каждое подтверждённое сообщение на месте и не изменено, дублей нет. Python 3.9+, стандартная библиотека; tokens не печатаются и не сохраняются. """ from __future__ import annotations import argparse import json import math import os import subprocess import sys import threading import time import urllib.error import urllib.request import uuid from dataclasses import dataclass from pathlib import Path from urllib.parse import quote ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client # noqa: E402 REQUEST_TIMEOUT_SEC = 5 RETRY_PAUSE_SEC = 0.05 STATS_FORMAT = "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}" @dataclass class Attempt: """Одна HTTP-попытка клиента; время — секунды от начала прогона""" op: str started: float finished: float status: int | None error: str | None instance: str | None retry: bool @property def ok(self) -> bool: return self.status in (200, 201) class Recorder: def __init__(self) -> None: self.started = time.monotonic() self.lock = threading.Lock() self.attempts: list[Attempt] = [] self.confirmed: list[dict] = [] self.events: dict[str, float] = {} def now(self) -> float: return time.monotonic() - self.started def mark(self, name: str) -> None: self.events[name] = round(self.now(), 3) def add(self, attempt: Attempt, message: dict | None = None) -> None: with self.lock: self.attempts.append(attempt) if message is not None: self.confirmed.append(message) def request( base: str, method: str, path: str, token: str, body: dict | None, key: str | None ) -> tuple[int | None, str | None, str | None, dict | None]: headers = {"Authorization": "Bearer " + token, "Accept": "application/json"} data = None if body is not None: data = json.dumps(body, ensure_ascii=False).encode() headers["Content-Type"] = "application/json" if key is not None: headers["Idempotency-Key"] = key req = urllib.request.Request(base + path, data=data, headers=headers, method=method) try: with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_SEC) as response: payload = json.loads(response.read()) return response.status, None, response.headers.get("X-Instance-Id"), payload except urllib.error.HTTPError as error: error.read() return error.code, None, error.headers.get("X-Instance-Id"), None except OSError as error: return None, type(error).__name__, None, None def worker( base: str, token: str, chat_id: str, worker_id: int, recorder: Recorder, stop: threading.Event, ) -> None: """Запись с повтором по тому же ключу до подтверждения, затем чтение""" path = f"/api/v1/chats/{chat_id}/messages" index = 0 while not stop.is_set(): key = uuid.uuid4().hex body = {"text": f"exp-{worker_id}-{index}-{key[:8]}"} retry = False while not stop.is_set(): started = recorder.now() status, error, instance, payload = request( base, "POST", path, token, body, key ) attempt = Attempt( "write", started, recorder.now(), status, error, instance, retry ) recorder.add(attempt, payload if status == 201 else None) if status == 201: break retry = True time.sleep(RETRY_PAUSE_SEC) started = recorder.now() status, error, instance, _ = request( base, "GET", path + "?limit=1", token, None, None ) recorder.add( Attempt("read", started, recorder.now(), status, error, instance, False) ) index += 1 def percentile(values: list[float], p: float) -> float | None: if not values: return None ordered = sorted(values) return round(1000 * ordered[max(0, math.ceil(p * len(ordered)) - 1)], 1) def phase_stats(attempts: list[Attempt], start: float, end: float) -> dict: inside = [a for a in attempts if start <= a.finished < end] duration = max(end - start, 1e-9) result: dict = {"seconds": round(duration, 3)} for op in ("write", "read"): ops = [a for a in inside if a.op == op] ok = [a for a in ops if a.ok] result[op] = { "ok": len(ok), "errors": len(ops) - len(ok), "ok_rps": round(len(ok) / duration, 1), "latency_ms": { "p50": percentile([a.finished - a.started for a in ok], 0.50), "p95": percentile([a.finished - a.started for a in ok], 0.95), "p99": percentile([a.finished - a.started for a in ok], 0.99), }, } return result def recovery_stats(attempts: list[Attempt], action_start: float, end: float) -> dict: """ Время восстановления записи: от первой ошибки после воздействия до первой успешной записи после последней ошибки; без ошибок — 0. Отдельно — наибольший разрыв между подтверждениями записи (задержка без ошибок). """ window = sorted( (a for a in attempts if a.op == "write" and a.finished >= action_start - 1), key=lambda a: a.finished, ) errors = [a for a in window if not a.ok and a.finished >= action_start] kinds: dict[str, int] = {} for a in errors: kind = a.error or f"HTTP {a.status}" kinds[kind] = kinds.get(kind, 0) + 1 if errors: last_error = errors[-1].finished first_ok_after = next( (a.finished for a in window if a.ok and a.finished > last_error), end ) recovery = first_ok_after - errors[0].finished else: recovery = 0.0 oks = [a.finished for a in window if a.ok and a.finished <= end] gap = max((b - a for a, b in zip(oks, oks[1:])), default=None) return { "write_errors": len(errors), "error_kinds": kinds, "write_recovery_seconds": round(recovery, 3), "max_write_gap_seconds": None if gap is None else round(gap, 3), } def timeline(attempts: list[Attempt], end: float) -> list[dict]: rows = [] for second in range(int(end) + 1): inside = [a for a in attempts if second <= a.finished < second + 1] instances: dict[str, int] = {} for a in inside: if a.ok and a.instance: instances[a.instance] = instances.get(a.instance, 0) + 1 rows.append( { "t": second, "ok": sum(a.ok for a in inside), "errors": sum(not a.ok for a in inside), "instances": instances, } ) return rows def read_history(client: Client, actor: dict, chat_id: str) -> list[dict]: path = f"/api/v1/chats/{chat_id}/messages?limit=100" items: list[dict] = [] while True: page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] items.extend(page["items"]) if page["next_cursor"] is None: return items path = f"/api/v1/chats/{chat_id}/messages?limit=100&cursor=" + quote( page["next_cursor"], safe="" ) def integrity(client: Client, actor: dict, chat_id: str, recorder: Recorder) -> dict: history = read_history(client, actor, chat_id) by_id = {m["id"]: m for m in history} texts = [m["text"] for m in history] confirmed = {m["id"]: m for m in recorder.confirmed} return { "messages_in_history": len(history), "confirmed_unique": len(confirmed), "confirmed_missing_or_changed": sum( by_id.get(i) != m for i, m in confirmed.items() ), "duplicate_ids": len(history) - len(by_id), "duplicate_texts": len(texts) - len(set(texts)), "retried_writes_confirmed_as_replay": len(recorder.confirmed) - len(confirmed), } def run(command: list[str], env: dict | None = None) -> str: result = subprocess.run( command, cwd=ROOT, env=env, check=True, capture_output=True, text=True ) return result.stdout.strip() def api_containers() -> list[str]: ids = run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).split() names = [run(["docker", "inspect", "-f", "{{.Name}}", i]).lstrip("/") for i in ids] return sorted(names) def instance_of(container: str) -> str: return run(["docker", "exec", container, "cat", "/tmp/instance-id"]) def container_stats() -> list[str]: names = run( ["docker", "compose", "--env-file", ".env", "ps", "-q", "nginx", "postgres", "api"] ).split() output = run(["docker", "stats", "--no-stream", "--format", STATS_FORMAT, *names]) return output.splitlines() def scenario( args: argparse.Namespace, label: str, action: list[str], restore: list[str] | None, target: str | None = None, env: dict | None = None, ) -> dict: client = Client(args.base_url, timeout=REQUEST_TIMEOUT_SEC) alice, _, _, chat = client.bootstrap() base = client.base recorder = Recorder() stop = threading.Event() threads = [ threading.Thread( target=worker, args=(base, alice["token"], chat["id"], n, recorder, stop), daemon=True, ) for n in range(args.workers) ] for thread in threads: thread.start() time.sleep(args.before / 2) stats = container_stats() time.sleep(args.before / 2) instances_before = sorted( {a.instance for a in recorder.attempts if a.ok and a.instance} ) recorder.mark("action_start") action_output = run(action, env) recorder.mark("action_end") restore_output = "" if restore is not None: time.sleep(args.outage) recorder.mark("restore_start") restore_output = run(restore) recorder.mark("restore_end") time.sleep(args.after) stop.set() for thread in threads: thread.join(timeout=REQUEST_TIMEOUT_SEC + 1) recorder.mark("end") events = recorder.events end = events["end"] after_start = events.get("restore_end", events["action_end"]) attempts = recorder.attempts seen_after = [ a for a in attempts if a.ok and a.finished >= after_start and a.instance ] back_in_rotation = next( ( round(a.finished - after_start, 3) for a in sorted(seen_after, key=lambda a: a.finished) if a.instance not in instances_before ), None, ) served = sorted( (a for a in attempts if a.ok and a.instance), key=lambda a: a.finished ) instances_seen: dict[str, list[float]] = {} for a in served: offset = round(a.finished - events["action_start"], 3) instances_seen.setdefault(a.instance, [offset, offset])[1] = offset phases = { "before": phase_stats(attempts, 0, events["action_start"]), "during": phase_stats(attempts, events["action_start"], after_start), "after": phase_stats(attempts, after_start, end), } target_after_stop = None if target is not None: target_after_stop = sum( 1 for a in attempts if a.ok and a.instance == target and a.finished > events["action_end"] and a.finished < events.get("restore_start", end) ) return { "scenario": label, "target_instance": target, "workers": args.workers, "events_seconds": events, "instances_before": instances_before, "instances_after": sorted({a.instance for a in seen_after}), "target_responses_while_stopped": target_after_stop, "restored_replica_in_rotation_seconds": back_in_rotation if restore is not None else None, # alias → [первый, последний] успешный ответ, секунды от начала воздействия "instances_seen_from_action": instances_seen, "phases": phases, "recovery": recovery_stats(attempts, events["action_start"], end), "integrity": integrity(client, alice, chat["id"], recorder), "container_stats_before_action": stats, "action_output": action_output.splitlines(), "restore_output_tail": restore_output.splitlines()[-2:], "timeline": timeline(attempts, end), } def failover(args: argparse.Namespace) -> list[dict]: results = [] for signal in args.signals: for container in api_containers(): target = instance_of(container) if signal == "SIGTERM": action = ["docker", "stop", "--timeout", "10", container] else: action = ["docker", "kill", "--signal", "SIGKILL", container] label = f"{signal} {container}" print(f"== {label} (instance {target})", flush=True) result = scenario( args, label, action, ["./scripts/restore.sh"], target=target ) print(json.dumps(brief(result), ensure_ascii=False), flush=True) results.append(result) return results def rolling(args: argparse.Namespace) -> list[dict]: env = dict(os.environ) env.setdefault("APP_REVISION", f"exp-{int(time.time())}") print(f"== rolling update to revision {env['APP_REVISION']}", flush=True) result = scenario( args, "rolling update", ["./scripts/rolling-update.sh"], None, env=env ) result["revisions_after"] = run(["./scripts/instances.sh"]).splitlines() print(json.dumps(brief(result), ensure_ascii=False), flush=True) return [result] def brief(result: dict) -> dict: """Краткая строка прогона для терминала; полная версия — в --output""" return { "scenario": result["scenario"], "rps_ok_write": {k: v["write"]["ok_rps"] for k, v in result["phases"].items()}, "rps_ok_read": {k: v["read"]["ok_rps"] for k, v in result["phases"].items()}, "errors": { k: v["write"]["errors"] + v["read"]["errors"] for k, v in result["phases"].items() }, "recovery": result["recovery"], "target_responses_while_stopped": result["target_responses_while_stopped"], "restored_replica_in_rotation_seconds": result[ "restored_replica_in_rotation_seconds" ], "instances_seen_from_action": result["instances_seen_from_action"], "integrity": result["integrity"], } def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("mode", choices=["failover", "rolling"]) parser.add_argument("--base-url", default=None) parser.add_argument("--workers", type=int, default=4) parser.add_argument("--before", type=float, default=8, help="нагрузка до отказа") parser.add_argument("--outage", type=float, default=10, help="реплика остановлена") parser.add_argument("--after", type=float, default=8, help="после восстановления") parser.add_argument( "--signals", nargs="+", default=["SIGTERM", "SIGKILL"], choices=["SIGTERM", "SIGKILL"], ) parser.add_argument("--output", default=None, help="JSON с полными результатами") args = parser.parse_args() started = time.strftime("%Y-%m-%dT%H:%M:%S%z") results = failover(args) if args.mode == "failover" else rolling(args) report = { "mode": args.mode, "started": started, "profile": { "workers": args.workers, "loop": "closed: POST message (retry same key until 201) + GET limit=1", "client_timeout_seconds": REQUEST_TIMEOUT_SEC, "phases_seconds": { "before": args.before, "outage": args.outage, "after": args.after, }, }, "runs": results, } if args.output: destination = ROOT / args.output destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") print(f"saved: {args.output}") failed = [ r["scenario"] for r in results if r["integrity"]["confirmed_missing_or_changed"] or r["integrity"]["duplicate_ids"] or r["integrity"]["duplicate_texts"] or r["target_responses_while_stopped"] ] if failed: raise SystemExit(f"FAIL: потеря/дубль или отказ не изолирован: {failed}") print(f"PASS: прогонов {len(results)}, подтверждённые сообщения целы, дублей нет") if __name__ == "__main__": main()