"""Проверки реальных процессов, failover, rolling и общей идемпотентности.""" import argparse import json import os import subprocess import threading import time from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from pathlib import Path from uuid import uuid4 import httpx from scripts import cluster from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready def snapshot(): ids = compose("ps", "-q").split() result = [] for c in json.loads(command(["docker", "inspect", *ids])): role = c["Config"]["Labels"]["com.docker.compose.service"] entry = { "role": role, "container": c["Id"][:12], "pid": c["State"]["Pid"], "started_at": c["State"]["StartedAt"], "ports": c["HostConfig"]["PortBindings"], "cpu_nano": c["HostConfig"]["NanoCpus"], "memory_limit": c["HostConfig"]["Memory"], } if role == "api": entry["instance"] = cluster.alias(c["Id"]) assert not entry["ports"] result.append(entry) return result def exercise(client, session, chat, action, *, seconds=15): path = f"/api/v1/chats/{chat['id']}/messages" observations, confirmed, errors, reads = [], {}, 0, 0 started = time.monotonic() finished = threading.Event() action_result = {} def operate(): time.sleep(0.5) try: action() except BaseException as exc: action_result["error"] = type(exc).__name__ finally: action_result["finished"] = round(time.monotonic() - started, 3) finished.set() thread = threading.Thread(target=operate) thread.start() key = uuid4().hex try: while time.monotonic() - started < seconds or not finished.is_set(): h = {**auth(session), "Idempotency-Key": key} try: r = client.post(path, headers=h, json={"text": key}) if r.status_code != 201: errors += 1 continue m = r.json() confirmed[m["id"]] = m observations.append( { "at": round(time.monotonic() - started, 3), "instance": r.headers["x-instance-id"], } ) key = uuid4().hex read = client.get( path, headers=auth(session), params={"limit": 100} ) reads += read.status_code == 200 errors += read.status_code != 200 except httpx.HTTPError: errors += 1 time.sleep(0.04) finally: thread.join(timeout=90) assert not action_result.get("error"), action_result found, cursor = [], None while True: r = client.get( path, headers=auth(session), params={"limit": 100, **({"cursor": cursor} if cursor else {})}, ) assert r.status_code == 200 page = r.json() found.extend(page["items"]) cursor = page["next_cursor"] if not cursor: break by_id = {m["id"]: m for m in found} assert all(by_id.get(k) == v for k, v in confirmed.items()) texts = [m["text"] for m in found] assert len(texts) == len(set(texts)) recovery = next( ( x["at"] for x in observations if x["at"] >= action_result["finished"] ), None, ) assert recovery is not None and (seconds > 20 or recovery <= 10) elapsed = time.monotonic() - started return { "confirmed_writes": len(confirmed), "successful_reads": reads, "errors": errors, "successful_rps": round((len(confirmed) + reads) / elapsed, 2), "first_success_after_action_seconds": recovery, "action": action_result, "observations": observations, "confirmed_preserved": True, "duplicate_texts": False, } def main(): parser = argparse.ArgumentParser() parser.add_argument("--output", default="evidence/cluster-repeat.json") args = parser.parse_args() report = { "date": datetime.now(timezone.utc).isoformat(), "source_commit": command(["git", "rev-parse", "HEAD"]), "complete": False, } output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) try: ready() initial = snapshot() apis = [c for c in initial if c["role"] == "api"] assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2 report["before"] = initial with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client: user, _, session = actor(client) r = client.post( "/api/v1/chats", headers=auth(session), json={ "title": "Cluster verification", "member_ids": [user["id"]], }, ) assert r.status_code == 201 chat = r.json() path = f"/api/v1/chats/{chat['id']}/messages" key = uuid4().hex def repeat(_): with httpx.Client( base_url=BASE, trust_env=False, timeout=8 ) as c: r = c.post( path, headers={**auth(session), "Idempotency-Key": key}, json={"text": "Same concurrent request"}, ) assert r.status_code == 201 return r.json(), r.headers["x-instance-id"] with ThreadPoolExecutor(max_workers=10) as pool: repeated = list(pool.map(repeat, range(30))) assert len({m[0]["id"] for m in repeated}) == 1 assert len({m[1] for m in repeated}) == 2 report["parallel_replay"] = { "requests": 30, "messages": 1, "instances": sorted({m[1] for m in repeated}), } # Отдельный чат каждой серии: проверки дубликатов не смешивают тексты. report["failovers"] = [] for api in apis: def stop(target=api["instance"]): env = {**os.environ, "TARGET_INSTANCE_ID": target} subprocess.run( ["scripts/stop-one.sh"], cwd=ROOT, env=env, check=True, capture_output=True, text=True, ) r = client.post( "/api/v1/chats", headers=auth(session), json={"title": "Failover", "member_ids": [user["id"]]}, ) result = exercise(client, session, r.json(), stop) assert api["instance"] not in [ o["instance"] for o in result["observations"][-5:] ] assert ( command( [ "docker", "inspect", "-f", "{{.State.Running}}", api["container"], ] ) == "false" ) result["stopped"] = api report["failovers"].append(result) compose("up", "-d", "--no-build", "--wait") ready() r = client.post( "/api/v1/chats", headers=auth(session), json={"title": "Rolling", "member_ids": [user["id"]]}, ) before_roll = snapshot() report["rolling"] = exercise( client, session, r.json(), lambda: command(["scripts/rolling-update.sh"]), seconds=45, ) after_roll = snapshot() assert not ( {c["container"] for c in before_roll if c["role"] == "api"} & {c["container"] for c in after_roll if c["role"] == "api"} ) report["rolling"]["before"] = before_roll report["rolling"]["after"] = after_roll replay = repeat(0)[0] assert replay == repeated[0][0] report["replay_after_replacement"] = True before = snapshot() env = { **os.environ, "NO_PROXY": "localhost,127.0.0.1", "no_proxy": "localhost,127.0.0.1", } for mode, hook in [ ("failover", "scripts/stop-one.sh"), ("persistence", "scripts/restart.sh"), ]: if mode == "persistence": compose("up", "-d", "--no-build", "--wait") ready() before = snapshot() run = subprocess.run( [ ".venv/bin/python", "tests/resilience.py", "--mode", mode, "--action-script", hook, "--base-url", BASE, ], cwd=ROOT, env=env, capture_output=True, text=True, ) assert run.returncode == 0, run.stdout + run.stderr report["public_" + mode] = run.stdout.strip() after = snapshot() for role in ["postgres", "cache", "lb"]: old = next(c for c in before if c["role"] == role) new = next(c for c in after if c["role"] == role) assert old["started_at"] != new["started_at"] report["after_persistence"] = after report["complete"] = True finally: output.write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n" ) print("PASS: process failover, rolling, replay and full persistence") if __name__ == "__main__": main()