"""Проверки реальных процессов, failover, rolling и общей идемпотентности.""" import argparse import json import os import subprocess import threading import time from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from pathlib import Path from uuid import uuid4 import httpx from scripts import cluster from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql def snapshot(): ids = compose("ps", "-q").split() result = [] for c in json.loads(command(["docker", "inspect", *ids])): role = c["Config"]["Labels"]["com.docker.compose.service"] entry = { "role": role, "container": c["Id"][:12], "pid": c["State"]["Pid"], "started_at": c["State"]["StartedAt"], "ports": c["HostConfig"]["PortBindings"], "cpu_nano": c["HostConfig"]["NanoCpus"], "memory_limit": c["HostConfig"]["Memory"], } if role == "api": entry["instance"] = cluster.alias(c["Id"]) assert not entry["ports"] result.append(entry) return result def exercise(client, session, chat, action, *, seconds=15): path = f"/api/v1/chats/{chat['id']}/messages" observations, confirmed, errors, reads = [], {}, 0, 0 started = time.monotonic() finished = threading.Event() action_result = {} def operate(): time.sleep(0.5) try: action() except BaseException as exc: action_result["error"] = type(exc).__name__ finally: action_result["finished"] = round(time.monotonic() - started, 3) finished.set() thread = threading.Thread(target=operate) thread.start() key = uuid4().hex try: while time.monotonic() - started < seconds or not finished.is_set(): h = {**auth(session), "Idempotency-Key": key} try: r = client.post(path, headers=h, json={"text": key}) if r.status_code != 201: errors += 1 continue m = r.json() confirmed[m["id"]] = m observations.append( { "at": round(time.monotonic() - started, 3), "instance": r.headers["x-instance-id"], } ) key = uuid4().hex read = client.get( path, headers=auth(session), params={"limit": 100} ) reads += read.status_code == 200 errors += read.status_code != 200 except httpx.HTTPError: errors += 1 time.sleep(0.04) finally: thread.join(timeout=90) assert not action_result.get("error"), action_result found, cursor = [], None while True: r = client.get( path, headers=auth(session), params={"limit": 100, **({"cursor": cursor} if cursor else {})}, ) assert r.status_code == 200 page = r.json() found.extend(page["items"]) cursor = page["next_cursor"] if not cursor: break by_id = {m["id"]: m for m in found} assert all(by_id.get(k) == v for k, v in confirmed.items()) texts = [m["text"] for m in found] assert len(texts) == len(set(texts)) recovery = next( ( x["at"] for x in observations if x["at"] >= action_result["finished"] ), None, ) assert recovery is not None and (seconds > 20 or recovery <= 10) elapsed = time.monotonic() - started return { "confirmed_writes": len(confirmed), "successful_reads": reads, "errors": errors, "successful_rps": round((len(confirmed) + reads) / elapsed, 2), "first_success_after_action_seconds": recovery, "action": action_result, "observations": observations, "confirmed_preserved": True, "duplicate_texts": False, } def main(): parser = argparse.ArgumentParser() parser.add_argument("--output", default="evidence/cluster-repeat.json") args = parser.parse_args() report = { "date": datetime.now(timezone.utc).isoformat(), "source_commit": command(["git", "rev-parse", "HEAD"]), "complete": False, } output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) try: ready() initial = snapshot() apis = [c for c in initial if c["role"] == "api"] assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2 report["before"] = initial with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client: user, _, session = actor(client) r = client.post( "/api/v1/chats", headers=auth(session), json={ "title": "Cluster verification", "member_ids": [user["id"]], }, ) assert r.status_code == 201 chat = r.json() path = f"/api/v1/chats/{chat['id']}/messages" key = uuid4().hex def repeat(_): with httpx.Client( base_url=BASE, trust_env=False, timeout=8 ) as c: r = c.post( path, headers={**auth(session), "Idempotency-Key": key}, json={"text": "Same concurrent request"}, ) assert r.status_code == 201 return r.json(), r.headers["x-instance-id"] with ThreadPoolExecutor(max_workers=10) as pool: repeated = list(pool.map(repeat, range(30))) assert len({m[0]["id"] for m in repeated}) == 1 assert len({m[1] for m in repeated}) == 2 report["parallel_replay"] = { "requests": 30, "messages": 1, "instances": sorted({m[1] for m in repeated}), } # Сбой второй записи транзакции не должен оставить сообщение. fault_key = uuid4().hex fault_text = "Atomic rollback " + fault_key sql( "CREATE FUNCTION student_idempo_fault() RETURNS trigger " "LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='" + fault_key + "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; " "CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency " "FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()" ) try: failed = client.post( path, headers={**auth(session), "Idempotency-Key": fault_key}, json={"text": fault_text}, ) assert failed.status_code == 500 assert ( sql( "SELECT count(*) FROM messages WHERE text='" + fault_text + "'" ) == "0" ) finally: sql( "DROP TRIGGER student_idempo_fault ON idempotency; " "DROP FUNCTION student_idempo_fault()" ) retried = client.post( path, headers={**auth(session), "Idempotency-Key": fault_key}, json={"text": fault_text}, ) assert retried.status_code == 201 report["atomic_rollback_and_5xx_retry"] = True sql( "UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' " "WHERE key='" + fault_key + "'" ) expired = client.post( path, headers={**auth(session), "Idempotency-Key": fault_key}, json={"text": "New retention window"}, ) assert expired.status_code == 201 assert expired.json()["id"] != retried.json()["id"] report["expired_key_can_be_reused"] = True # Отдельный чат каждой серии: проверки дубликатов не смешивают тексты. report["failovers"] = [] for api in apis: def stop(target=api["instance"]): env = {**os.environ, "TARGET_INSTANCE_ID": target} subprocess.run( ["scripts/stop-one.sh"], cwd=ROOT, env=env, check=True, capture_output=True, text=True, ) r = client.post( "/api/v1/chats", headers=auth(session), json={"title": "Failover", "member_ids": [user["id"]]}, ) result = exercise(client, session, r.json(), stop) assert api["instance"] not in [ o["instance"] for o in result["observations"][-5:] ] assert ( command( [ "docker", "inspect", "-f", "{{.State.Running}}", api["container"], ] ) == "false" ) result["stopped"] = api report["failovers"].append(result) compose("up", "-d", "--no-build", "--wait") ready() r = client.post( "/api/v1/chats", headers=auth(session), json={"title": "Rolling", "member_ids": [user["id"]]}, ) before_roll = snapshot() report["rolling"] = exercise( client, session, r.json(), lambda: command(["scripts/rolling-update.sh"]), seconds=45, ) after_roll = snapshot() assert not ( {c["container"] for c in before_roll if c["role"] == "api"} & {c["container"] for c in after_roll if c["role"] == "api"} ) report["rolling"]["before"] = before_roll report["rolling"]["after"] = after_roll replay = repeat(0)[0] assert replay == repeated[0][0] report["replay_after_replacement"] = True before = snapshot() env = { **os.environ, "NO_PROXY": "localhost,127.0.0.1", "no_proxy": "localhost,127.0.0.1", } for mode, hook in [ ("failover", "scripts/stop-one.sh"), ("persistence", "scripts/restart.sh"), ]: if mode == "persistence": compose("up", "-d", "--no-build", "--wait") ready() before = snapshot() run = subprocess.run( [ ".venv/bin/python", "tests/resilience.py", "--mode", mode, "--action-script", hook, "--base-url", BASE, ], cwd=ROOT, env=env, capture_output=True, text=True, ) assert run.returncode == 0, run.stdout + run.stderr report["public_" + mode] = run.stdout.strip() after = snapshot() for role in ["postgres", "cache", "lb"]: old = next(c for c in before if c["role"] == role) new = next(c for c in after if c["role"] == role) assert old["started_at"] != new["started_at"] report["after_persistence"] = after report["complete"] = True finally: output.write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n" ) print("PASS: process failover, rolling, replay and full persistence") if __name__ == "__main__": main()