"""Сценарий поочерёдного отказа всех API-реплик под записью.""" from collections import Counter from concurrent.futures import ThreadPoolExecutor import json import os import re import subprocess from threading import Event, Lock import time from urllib.parse import quote from uuid import uuid4 from scripts.deployment import ComposeProject, ROOT, deployment_lock from tests.client import Client, Failure, arguments, check, run def failure_kind(error: Failure) -> str: """Различает HTTP-отказы, ошибки соединения и проверки контракта.""" match = re.search(r"received (\d{3})$", str(error)) if match: return "http_" + match.group(1) return "connection" if "connection/TLS failure" in str(error) else "contract" def snapshot(project: ComposeProject, phase: str, target: str) -> list: """Печатает состояние API без окружения и секретов.""" replicas = project.replicas() print(json.dumps({ "scenario": "replica_failure", "phase": phase, "target": target, "replicas": [{"instance": item.instance_id, "running": item.running, "health": item.health} for item in replicas], }), flush=True) return replicas def history(client: Client, actor: dict, chat: dict) -> list[dict]: """Читает всю историю без повторов курсора.""" path = f"/api/v1/chats/{chat['id']}/messages?limit=100" items = [] cursors = set() for _ in range(1000): page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] items.extend(page["items"]) cursor = page["next_cursor"] if cursor is None: return items check(cursor not in cursors, "cursor cycle") cursors.add(cursor) path = f"/api/v1/chats/{chat['id']}/messages?limit=100&cursor={quote(cursor, safe='')}" raise Failure("history exceeded safety bound") def exercise(project: ComposeProject, args) -> None: """Останавливает каждую реплику и восстанавливает её перед следующей.""" replicas = project.replicas() check(len(replicas) >= 2, "at least two API replicas are required") project.wait_healthy({item.id for item in replicas}) client = Client(args.base_url, args.ca_file, timeout=2) for item in replicas: project.wait_routed(item.instance_id, client.base, args.ca_file) actor, _, _, chat = client.bootstrap() confirmed = set() for target in replicas: snapshot(project, "before", target.instance_id) stop = Event() lock = Lock() successes = [] errors = [] def worker(): local = Client(client.base, args.ca_file, timeout=2) key = uuid4().hex while not stop.is_set(): started = time.monotonic() try: result = local.message(actor, chat, "failover-" + key, key=key) with lock: successes.append((started, time.monotonic(), result["json"]["id"])) key = uuid4().hex except Failure as error: with lock: errors.append(failure_kind(error)) time.sleep(0.1) started = time.monotonic() with ThreadPoolExecutor(max_workers=args.concurrency) as pool: jobs = [pool.submit(worker) for _ in range(args.concurrency)] try: time.sleep(2) stop_started = time.monotonic() subprocess.run( [str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, env={**os.environ, "TARGET_INSTANCE_ID": target.instance_id}, check=True, timeout=10, capture_output=True, ) stop_finished = time.monotonic() stopped = snapshot(project, "stopped", target.instance_id) check(any(item.id == target.id and not item.running for item in stopped), "target API was not stopped") check(any(item.id != target.id and item.running and item.health == "healthy" for item in stopped), "no healthy surviving API") time.sleep(args.duration) finally: stop.set() try: for job in jobs: job.result(timeout=5) finally: project.docker("start", target.id) project.wait_healthy({item.id for item in replicas}) project.wait_routed(target.instance_id, client.base, args.ca_file) snapshot(project, "restored", target.instance_id) elapsed = max(end for _, end, _ in successes) - started if successes else 0 recovered = [end for begin, end, _ in successes if begin >= stop_finished] check(bool(recovered), "no confirmed writes after failure") recovery = min(recovered) - stop_started check(recovery <= 10, "writes did not recover within 10 seconds") confirmed.update(id for _, _, id in successes) items = history(client, actor, chat) check(confirmed <= {item["id"] for item in items}, "confirmed messages were lost") texts = [item["text"] for item in items] check(len(texts) == len(set(texts)), "uncertain retry created duplicates") print(json.dumps({ "instance": target.instance_id, "successful_writes": len(successes), "failed_attempts": len(errors), "errors_by_kind": dict(Counter(errors)), "elapsed_seconds": round(elapsed, 3), "successful_rps": round(len(successes) / elapsed, 3) if elapsed else 0, "first_recovery_seconds": round(recovery, 3), }), flush=True) def main() -> None: """Готовит параметры эксперимента отказа реплик.""" def configure(parser): parser.add_argument("--duration", type=float, default=15) parser.add_argument("--concurrency", type=int, default=2) args = arguments("Fail each API replica under write load", configure) check(1 <= args.duration <= 120 and 1 <= args.concurrency <= 8, "invalid workload") project = ComposeProject() with deployment_lock(project): exercise(project, args) if __name__ == "__main__": run(main)