forked from mtusi2026/mtusi2026lab4
149 lines
4.8 KiB
Python
149 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Поочерёдная замена API-реплик под нагрузкой POST сообщений."""
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, check, run
|
|
|
|
EVIDENCE = ROOT / "evidence"
|
|
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
|
|
|
|
|
def compose(*args, timeout=180):
|
|
proc = subprocess.run(
|
|
["docker", "compose", "--env-file", ".env", *args],
|
|
cwd=str(ROOT),
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
if proc.returncode != 0:
|
|
raise Failure("docker compose %s failed" % args[0])
|
|
return proc.stdout
|
|
|
|
|
|
def api_ids():
|
|
return [line.strip() for line in compose("ps", "-q", "api").splitlines() if line.strip()]
|
|
|
|
|
|
def inspect_health(container_id):
|
|
proc = subprocess.run(
|
|
["docker", "inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}", container_id],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=20,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
return proc.stdout.strip()
|
|
|
|
|
|
def wait_healthy(container_id, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
last = ""
|
|
while time.monotonic() < deadline:
|
|
last = inspect_health(container_id)
|
|
if last == "healthy":
|
|
return
|
|
time.sleep(0.5)
|
|
raise Failure("replica %s did not become healthy: %s" % (container_id[:12], last))
|
|
|
|
|
|
def wait_ready(timeout=90):
|
|
client = Client(BASE, timeout=3)
|
|
deadline = time.monotonic() + timeout
|
|
last = ""
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
return
|
|
except Failure as exc:
|
|
last = str(exc)
|
|
time.sleep(0.4)
|
|
raise Failure("LB ready wait failed: " + last)
|
|
|
|
|
|
def main():
|
|
client = Client(BASE, timeout=8)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
halt = threading.Event()
|
|
latencies = []
|
|
errors = []
|
|
lock = threading.Lock()
|
|
|
|
def worker(index):
|
|
local = Client(BASE, timeout=5)
|
|
n = 0
|
|
while not halt.is_set():
|
|
started = time.monotonic()
|
|
try:
|
|
local.message(alice, chat, "rolling-%s-%s" % (index, n))
|
|
with lock:
|
|
latencies.append(time.monotonic() - started)
|
|
except Failure:
|
|
with lock:
|
|
errors.append(time.monotonic())
|
|
n += 1
|
|
|
|
threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(4)]
|
|
for thread in threads:
|
|
thread.start()
|
|
time.sleep(5)
|
|
before = api_ids()
|
|
check(len(before) >= 2, "rolling update needs >=2 replicas")
|
|
compose("build", "api")
|
|
replaced = []
|
|
for old in list(before):
|
|
current = api_ids()
|
|
compose("up", "-d", "--no-deps", "--scale", "api=%d" % (len(current) + 1), "--no-recreate", "api")
|
|
deadline = time.monotonic() + 60
|
|
new_ids = api_ids()
|
|
while len(new_ids) <= len(current) and time.monotonic() < deadline:
|
|
time.sleep(0.5)
|
|
new_ids = api_ids()
|
|
added = [item for item in new_ids if item not in current]
|
|
check(added, "new replica was not created")
|
|
wait_healthy(added[-1])
|
|
wait_ready()
|
|
subprocess.run(["docker", "stop", "-t", "5", old], capture_output=True)
|
|
subprocess.run(["docker", "rm", "-f", old], capture_output=True)
|
|
remaining = api_ids()
|
|
check(old not in remaining, "old replica %s still in service" % old[:12])
|
|
compose("up", "-d", "--no-deps", "--scale", "api=%d" % len(before), "api")
|
|
wait_ready()
|
|
replaced.append(old[:12])
|
|
time.sleep(5)
|
|
halt.set()
|
|
for thread in threads:
|
|
thread.join(timeout=10)
|
|
after = api_ids()
|
|
check(len(after) >= 2, "lost replicas after rolling update")
|
|
check(len(latencies) > 0, "no successful writes during rolling update")
|
|
report = {
|
|
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"replaced": replaced,
|
|
"replicas_before": [item[:12] for item in before],
|
|
"replicas_after": [item[:12] for item in after],
|
|
"successful_writes": len(latencies),
|
|
"failed_writes": len(errors),
|
|
"error_ratio": round(len(errors) / max(1, len(latencies) + len(errors)), 6),
|
|
}
|
|
EVIDENCE.mkdir(exist_ok=True)
|
|
(EVIDENCE / "rolling.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
print("PASS: rolling update under load")
|
|
print(json.dumps(report, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|