#!/usr/bin/env python3 """Поочерёдная замена API-реплик под нагрузкой POST сообщений.""" import json import os import subprocess import sys import threading import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client, Failure, check, run EVIDENCE = ROOT / "evidence" BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") def compose(*args, timeout=180): proc = subprocess.run( ["docker", "compose", "--env-file", ".env", *args], cwd=str(ROOT), capture_output=True, text=True, timeout=timeout, encoding="utf-8", errors="replace", ) if proc.returncode != 0: raise Failure("docker compose %s failed" % args[0]) return proc.stdout def api_ids(): return [line.strip() for line in compose("ps", "-q", "api").splitlines() if line.strip()] def inspect_health(container_id): proc = subprocess.run( ["docker", "inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}", container_id], capture_output=True, text=True, timeout=20, encoding="utf-8", errors="replace", ) return proc.stdout.strip() def wait_healthy(container_id, timeout=90): deadline = time.monotonic() + timeout last = "" while time.monotonic() < deadline: last = inspect_health(container_id) if last == "healthy": return time.sleep(0.5) raise Failure("replica %s did not become healthy: %s" % (container_id[:12], last)) def wait_ready(timeout=90): client = Client(BASE, timeout=3) deadline = time.monotonic() + timeout last = "" while time.monotonic() < deadline: try: client.call("GET", "/health/ready", schema="Health") return except Failure as exc: last = str(exc) time.sleep(0.4) raise Failure("LB ready wait failed: " + last) def main(): client = Client(BASE, timeout=8) alice, _bob, _eve, chat = client.bootstrap() halt = threading.Event() latencies = [] errors = [] lock = threading.Lock() def worker(index): local = Client(BASE, timeout=5) n = 0 while not halt.is_set(): started = time.monotonic() try: local.message(alice, chat, "rolling-%s-%s" % (index, n)) with lock: latencies.append(time.monotonic() - started) except Failure: with lock: errors.append(time.monotonic()) n += 1 threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(4)] for thread in threads: thread.start() time.sleep(5) before = api_ids() check(len(before) >= 2, "rolling update needs >=2 replicas") compose("build", "api") replaced = [] for old in list(before): current = api_ids() compose("up", "-d", "--no-deps", "--scale", "api=%d" % (len(current) + 1), "--no-recreate", "api") deadline = time.monotonic() + 60 new_ids = api_ids() while len(new_ids) <= len(current) and time.monotonic() < deadline: time.sleep(0.5) new_ids = api_ids() added = [item for item in new_ids if item not in current] check(added, "new replica was not created") wait_healthy(added[-1]) wait_ready() subprocess.run(["docker", "stop", "-t", "5", old], capture_output=True) subprocess.run(["docker", "rm", "-f", old], capture_output=True) remaining = api_ids() check(old not in remaining, "old replica %s still in service" % old[:12]) compose("up", "-d", "--no-deps", "--scale", "api=%d" % len(before), "api") wait_ready() replaced.append(old[:12]) time.sleep(5) halt.set() for thread in threads: thread.join(timeout=10) after = api_ids() check(len(after) >= 2, "lost replicas after rolling update") check(len(latencies) > 0, "no successful writes during rolling update") report = { "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "replaced": replaced, "replicas_before": [item[:12] for item in before], "replicas_after": [item[:12] for item in after], "successful_writes": len(latencies), "failed_writes": len(errors), "error_ratio": round(len(errors) / max(1, len(latencies) + len(errors)), 6), } EVIDENCE.mkdir(exist_ok=True) (EVIDENCE / "rolling.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print("PASS: rolling update under load") print(json.dumps(report, ensure_ascii=False)) if __name__ == "__main__": run(main)