Files
mtusi2026lab4/scripts/sigterm_check.py
T

149 lines
5.6 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""SIGTERM одной API-реплики за балансировщиком.
Реплика перестаёт принимать соединения, дожидается текущих запросов и
завершается не дольше чем за 10 секунд; LB уводит трафик на другие реплики.
Во время проверки фоновые клиенты непрерывно шлют GET /health/live через LB:
ошибок (5xx/обрыв) быть не должно, после выхода реплики её X-Instance-Id
в ответах не появляется.
"""
import argparse
import json
import os
import subprocess
import threading
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_BASE_URL = json.loads((ROOT / "lab.json").read_text())["default_base_url"]
SHUTDOWN_LIMIT_SEC = 10
PROBE_INTERVAL_SEC = 0.02
PROBERS = 4
class Prober(threading.Thread):
"""Фоновый клиент через LB: время ответа и обслужившая реплика"""
def __init__(self, url: str) -> None:
super().__init__(daemon=True)
self.url = url
self.stop = threading.Event()
self.ok: list[tuple[float, str]] = []
self.errors: list[tuple[float, str]] = []
def run(self) -> None:
while not self.stop.is_set():
try:
with urllib.request.urlopen(self.url, timeout=2) as response:
self.ok.append(
(time.monotonic(), response.headers.get("X-Instance-Id", ""))
)
except urllib.error.HTTPError as error:
self.errors.append((time.monotonic(), f"HTTP {error.code}"))
except OSError as error:
self.errors.append((time.monotonic(), type(error).__name__))
time.sleep(PROBE_INTERVAL_SEC)
def docker(*args: str) -> str:
result = subprocess.run(
["docker", *args], check=True, capture_output=True, text=True
)
return result.stdout.strip()
def first_api_container() -> str:
ids = docker("compose", "--env-file", ".env", "ps", "-q", "api").split()
if not ids:
raise SystemExit("FAIL: нет запущенных API-реплик")
return docker("inspect", "-f", "{{.Name}}", ids[0]).lstrip("/")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--base-url", default=os.environ.get("BASE_URL") or DEFAULT_BASE_URL
)
parser.add_argument("--container", default=os.environ.get("CONTAINER"))
args = parser.parse_args()
os.chdir(ROOT)
container = args.container or first_api_container()
instance = docker("exec", container, "cat", "/tmp/instance-id")
logs = subprocess.Popen(
["docker", "logs", "--follow", "--since", "0s", container],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
waiter = subprocess.Popen(
["docker", "wait", container], stdout=subprocess.PIPE, text=True
)
probers = [
Prober(args.base_url.rstrip("/") + "/health/live") for _ in range(PROBERS)
]
for prober in probers:
prober.start()
time.sleep(1)
before = {alias for prober in probers for _, alias in prober.ok}
if instance not in before:
raise SystemExit("FAIL: целевая реплика не обслуживает запросы до SIGTERM")
sent_at = time.monotonic()
docker("kill", "--signal", "SIGTERM", container)
exit_code = int(waiter.communicate(timeout=SHUTDOWN_LIMIT_SEC + 5)[0].strip())
exited_at = time.monotonic()
time.sleep(1)
for prober in probers:
prober.stop.set()
prober.join()
logs.terminate()
shutdown_log = logs.communicate(timeout=5)[0]
ok_after = [(t, a) for p in probers for t, a in p.ok if t > sent_at]
errors_after = [(t, e) for p in probers for t, e in p.errors if t > sent_at]
target_after_exit = [t for t, a in ok_after if a == instance and t > exited_at]
last_target = max((t for t, a in ok_after if a == instance), default=None)
print("Журнал реплики после SIGTERM:")
print(
"\n".join(line for line in shutdown_log.splitlines() if "/health/" not in line)
)
summary = {
"container": container,
"instance": instance,
"exit_code": exit_code,
"shutdown_seconds": round(exited_at - sent_at, 3),
"limit_seconds": SHUTDOWN_LIMIT_SEC,
"instances_before": sorted(before),
"probes_ok_after_sigterm": len(ok_after),
"instances_after_sigterm": sorted({a for _, a in ok_after}),
"last_target_response_after_sigterm_seconds": None
if last_target is None
else round(last_target - sent_at, 3),
"target_responses_after_exit": len(target_after_exit),
"errors_after_sigterm": len(errors_after),
"error_kinds": sorted({e for _, e in errors_after}),
}
print(json.dumps(summary, indent=2, ensure_ascii=False))
if (
exit_code != 0
or exited_at - sent_at > SHUTDOWN_LIMIT_SEC
or errors_after
or target_after_exit
or not ok_after
):
raise SystemExit("FAIL: завершение по SIGTERM не соответствует требованиям")
print(
"PASS: SIGTERM завершил %s за %.2f с, LB обслужил %d запросов без ошибок"
% (container, exited_at - sent_at, len(ok_after))
)
if __name__ == "__main__":
main()