149 lines
5.6 KiB
Python
149 lines
5.6 KiB
Python
#!/usr/bin/env python3
|
||
"""SIGTERM одной API-реплики за балансировщиком.
|
||
|
||
Реплика перестаёт принимать соединения, дожидается текущих запросов и
|
||
завершается не дольше чем за 10 секунд; LB уводит трафик на другие реплики.
|
||
Во время проверки фоновые клиенты непрерывно шлют GET /health/live через LB:
|
||
ошибок (5xx/обрыв) быть не должно, после выхода реплики её X-Instance-Id
|
||
в ответах не появляется.
|
||
"""
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
DEFAULT_BASE_URL = json.loads((ROOT / "lab.json").read_text())["default_base_url"]
|
||
|
||
SHUTDOWN_LIMIT_SEC = 10
|
||
PROBE_INTERVAL_SEC = 0.02
|
||
PROBERS = 4
|
||
|
||
|
||
class Prober(threading.Thread):
|
||
"""Фоновый клиент через LB: время ответа и обслужившая реплика"""
|
||
|
||
def __init__(self, url: str) -> None:
|
||
super().__init__(daemon=True)
|
||
self.url = url
|
||
self.stop = threading.Event()
|
||
self.ok: list[tuple[float, str]] = []
|
||
self.errors: list[tuple[float, str]] = []
|
||
|
||
def run(self) -> None:
|
||
while not self.stop.is_set():
|
||
try:
|
||
with urllib.request.urlopen(self.url, timeout=2) as response:
|
||
self.ok.append(
|
||
(time.monotonic(), response.headers.get("X-Instance-Id", ""))
|
||
)
|
||
except urllib.error.HTTPError as error:
|
||
self.errors.append((time.monotonic(), f"HTTP {error.code}"))
|
||
except OSError as error:
|
||
self.errors.append((time.monotonic(), type(error).__name__))
|
||
time.sleep(PROBE_INTERVAL_SEC)
|
||
|
||
|
||
def docker(*args: str) -> str:
|
||
result = subprocess.run(
|
||
["docker", *args], check=True, capture_output=True, text=True
|
||
)
|
||
return result.stdout.strip()
|
||
|
||
|
||
def first_api_container() -> str:
|
||
ids = docker("compose", "--env-file", ".env", "ps", "-q", "api").split()
|
||
if not ids:
|
||
raise SystemExit("FAIL: нет запущенных API-реплик")
|
||
return docker("inspect", "-f", "{{.Name}}", ids[0]).lstrip("/")
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument(
|
||
"--base-url", default=os.environ.get("BASE_URL") or DEFAULT_BASE_URL
|
||
)
|
||
parser.add_argument("--container", default=os.environ.get("CONTAINER"))
|
||
args = parser.parse_args()
|
||
os.chdir(ROOT)
|
||
container = args.container or first_api_container()
|
||
instance = docker("exec", container, "cat", "/tmp/instance-id")
|
||
|
||
logs = subprocess.Popen(
|
||
["docker", "logs", "--follow", "--since", "0s", container],
|
||
stdout=subprocess.PIPE,
|
||
stderr=subprocess.STDOUT,
|
||
text=True,
|
||
)
|
||
waiter = subprocess.Popen(
|
||
["docker", "wait", container], stdout=subprocess.PIPE, text=True
|
||
)
|
||
probers = [
|
||
Prober(args.base_url.rstrip("/") + "/health/live") for _ in range(PROBERS)
|
||
]
|
||
for prober in probers:
|
||
prober.start()
|
||
time.sleep(1)
|
||
before = {alias for prober in probers for _, alias in prober.ok}
|
||
if instance not in before:
|
||
raise SystemExit("FAIL: целевая реплика не обслуживает запросы до SIGTERM")
|
||
|
||
sent_at = time.monotonic()
|
||
docker("kill", "--signal", "SIGTERM", container)
|
||
exit_code = int(waiter.communicate(timeout=SHUTDOWN_LIMIT_SEC + 5)[0].strip())
|
||
exited_at = time.monotonic()
|
||
time.sleep(1)
|
||
for prober in probers:
|
||
prober.stop.set()
|
||
prober.join()
|
||
logs.terminate()
|
||
shutdown_log = logs.communicate(timeout=5)[0]
|
||
|
||
ok_after = [(t, a) for p in probers for t, a in p.ok if t > sent_at]
|
||
errors_after = [(t, e) for p in probers for t, e in p.errors if t > sent_at]
|
||
target_after_exit = [t for t, a in ok_after if a == instance and t > exited_at]
|
||
last_target = max((t for t, a in ok_after if a == instance), default=None)
|
||
print("Журнал реплики после SIGTERM:")
|
||
print(
|
||
"\n".join(line for line in shutdown_log.splitlines() if "/health/" not in line)
|
||
)
|
||
summary = {
|
||
"container": container,
|
||
"instance": instance,
|
||
"exit_code": exit_code,
|
||
"shutdown_seconds": round(exited_at - sent_at, 3),
|
||
"limit_seconds": SHUTDOWN_LIMIT_SEC,
|
||
"instances_before": sorted(before),
|
||
"probes_ok_after_sigterm": len(ok_after),
|
||
"instances_after_sigterm": sorted({a for _, a in ok_after}),
|
||
"last_target_response_after_sigterm_seconds": None
|
||
if last_target is None
|
||
else round(last_target - sent_at, 3),
|
||
"target_responses_after_exit": len(target_after_exit),
|
||
"errors_after_sigterm": len(errors_after),
|
||
"error_kinds": sorted({e for _, e in errors_after}),
|
||
}
|
||
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
||
if (
|
||
exit_code != 0
|
||
or exited_at - sent_at > SHUTDOWN_LIMIT_SEC
|
||
or errors_after
|
||
or target_after_exit
|
||
or not ok_after
|
||
):
|
||
raise SystemExit("FAIL: завершение по SIGTERM не соответствует требованиям")
|
||
print(
|
||
"PASS: SIGTERM завершил %s за %.2f с, LB обслужил %d запросов без ошибок"
|
||
% (container, exited_at - sent_at, len(ok_after))
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|