Files
mtusi2026lab3/student-tests/rolling.py

149 lines
4.8 KiB
Python

#!/usr/bin/env python3
"""Поочерёдная замена API-реплик под нагрузкой POST сообщений."""
import json
import os
import subprocess
import sys
import threading
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check, run
EVIDENCE = ROOT / "evidence"
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
def compose(*args, timeout=180):
proc = subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=str(ROOT),
capture_output=True,
text=True,
timeout=timeout,
encoding="utf-8",
errors="replace",
)
if proc.returncode != 0:
raise Failure("docker compose %s failed" % args[0])
return proc.stdout
def api_ids():
return [line.strip() for line in compose("ps", "-q", "api").splitlines() if line.strip()]
def inspect_health(container_id):
proc = subprocess.run(
["docker", "inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}", container_id],
capture_output=True,
text=True,
timeout=20,
encoding="utf-8",
errors="replace",
)
return proc.stdout.strip()
def wait_healthy(container_id, timeout=90):
deadline = time.monotonic() + timeout
last = ""
while time.monotonic() < deadline:
last = inspect_health(container_id)
if last == "healthy":
return
time.sleep(0.5)
raise Failure("replica %s did not become healthy: %s" % (container_id[:12], last))
def wait_ready(timeout=90):
client = Client(BASE, timeout=3)
deadline = time.monotonic() + timeout
last = ""
while time.monotonic() < deadline:
try:
client.call("GET", "/health/ready", schema="Health")
return
except Failure as exc:
last = str(exc)
time.sleep(0.4)
raise Failure("LB ready wait failed: " + last)
def main():
client = Client(BASE, timeout=8)
alice, _bob, _eve, chat = client.bootstrap()
halt = threading.Event()
latencies = []
errors = []
lock = threading.Lock()
def worker(index):
local = Client(BASE, timeout=5)
n = 0
while not halt.is_set():
started = time.monotonic()
try:
local.message(alice, chat, "rolling-%s-%s" % (index, n))
with lock:
latencies.append(time.monotonic() - started)
except Failure:
with lock:
errors.append(time.monotonic())
n += 1
threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(4)]
for thread in threads:
thread.start()
time.sleep(5)
before = api_ids()
check(len(before) >= 2, "rolling update needs >=2 replicas")
compose("build", "api")
replaced = []
for old in list(before):
current = api_ids()
compose("up", "-d", "--no-deps", "--scale", "api=%d" % (len(current) + 1), "--no-recreate", "api")
deadline = time.monotonic() + 60
new_ids = api_ids()
while len(new_ids) <= len(current) and time.monotonic() < deadline:
time.sleep(0.5)
new_ids = api_ids()
added = [item for item in new_ids if item not in current]
check(added, "new replica was not created")
wait_healthy(added[-1])
wait_ready()
subprocess.run(["docker", "stop", "-t", "5", old], capture_output=True)
subprocess.run(["docker", "rm", "-f", old], capture_output=True)
remaining = api_ids()
check(old not in remaining, "old replica %s still in service" % old[:12])
compose("up", "-d", "--no-deps", "--scale", "api=%d" % len(before), "api")
wait_ready()
replaced.append(old[:12])
time.sleep(5)
halt.set()
for thread in threads:
thread.join(timeout=10)
after = api_ids()
check(len(after) >= 2, "lost replicas after rolling update")
check(len(latencies) > 0, "no successful writes during rolling update")
report = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"replaced": replaced,
"replicas_before": [item[:12] for item in before],
"replicas_after": [item[:12] for item in after],
"successful_writes": len(latencies),
"failed_writes": len(errors),
"error_ratio": round(len(errors) / max(1, len(latencies) + len(errors)), 6),
}
EVIDENCE.mkdir(exist_ok=True)
(EVIDENCE / "rolling.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("PASS: rolling update under load")
print(json.dumps(report, ensure_ascii=False))
if __name__ == "__main__":
run(main)