Реализация лабораторной работы 4 на 4
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""SIGTERM одной API-реплики за балансировщиком.
|
||||
|
||||
Реплика перестаёт принимать соединения, дожидается текущих запросов и
|
||||
завершается не дольше чем за 10 секунд; LB уводит трафик на другие реплики.
|
||||
Во время проверки фоновые клиенты непрерывно шлют GET /health/live через LB:
|
||||
ошибок (5xx/обрыв) быть не должно, после выхода реплики её X-Instance-Id
|
||||
в ответах не появляется.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_BASE_URL = json.loads((ROOT / "lab.json").read_text())["default_base_url"]
|
||||
|
||||
SHUTDOWN_LIMIT_SEC = 10
|
||||
PROBE_INTERVAL_SEC = 0.02
|
||||
PROBERS = 4
|
||||
|
||||
|
||||
class Prober(threading.Thread):
|
||||
"""Фоновый клиент через LB: время ответа и обслужившая реплика"""
|
||||
|
||||
def __init__(self, url: str) -> None:
|
||||
super().__init__(daemon=True)
|
||||
self.url = url
|
||||
self.stop = threading.Event()
|
||||
self.ok: list[tuple[float, str]] = []
|
||||
self.errors: list[tuple[float, str]] = []
|
||||
|
||||
def run(self) -> None:
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
with urllib.request.urlopen(self.url, timeout=2) as response:
|
||||
self.ok.append(
|
||||
(time.monotonic(), response.headers.get("X-Instance-Id", ""))
|
||||
)
|
||||
except urllib.error.HTTPError as error:
|
||||
self.errors.append((time.monotonic(), f"HTTP {error.code}"))
|
||||
except OSError as error:
|
||||
self.errors.append((time.monotonic(), type(error).__name__))
|
||||
time.sleep(PROBE_INTERVAL_SEC)
|
||||
|
||||
|
||||
def docker(*args: str) -> str:
|
||||
result = subprocess.run(
|
||||
["docker", *args], check=True, capture_output=True, text=True
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def first_api_container() -> str:
|
||||
ids = docker("compose", "--env-file", ".env", "ps", "-q", "api").split()
|
||||
if not ids:
|
||||
raise SystemExit("FAIL: нет запущенных API-реплик")
|
||||
return docker("inspect", "-f", "{{.Name}}", ids[0]).lstrip("/")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--base-url", default=os.environ.get("BASE_URL") or DEFAULT_BASE_URL
|
||||
)
|
||||
parser.add_argument("--container", default=os.environ.get("CONTAINER"))
|
||||
args = parser.parse_args()
|
||||
os.chdir(ROOT)
|
||||
container = args.container or first_api_container()
|
||||
instance = docker("exec", container, "cat", "/tmp/instance-id")
|
||||
|
||||
logs = subprocess.Popen(
|
||||
["docker", "logs", "--follow", "--since", "0s", container],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
waiter = subprocess.Popen(
|
||||
["docker", "wait", container], stdout=subprocess.PIPE, text=True
|
||||
)
|
||||
probers = [
|
||||
Prober(args.base_url.rstrip("/") + "/health/live") for _ in range(PROBERS)
|
||||
]
|
||||
for prober in probers:
|
||||
prober.start()
|
||||
time.sleep(1)
|
||||
before = {alias for prober in probers for _, alias in prober.ok}
|
||||
if instance not in before:
|
||||
raise SystemExit("FAIL: целевая реплика не обслуживает запросы до SIGTERM")
|
||||
|
||||
sent_at = time.monotonic()
|
||||
docker("kill", "--signal", "SIGTERM", container)
|
||||
exit_code = int(waiter.communicate(timeout=SHUTDOWN_LIMIT_SEC + 5)[0].strip())
|
||||
exited_at = time.monotonic()
|
||||
time.sleep(1)
|
||||
for prober in probers:
|
||||
prober.stop.set()
|
||||
prober.join()
|
||||
logs.terminate()
|
||||
shutdown_log = logs.communicate(timeout=5)[0]
|
||||
|
||||
ok_after = [(t, a) for p in probers for t, a in p.ok if t > sent_at]
|
||||
errors_after = [(t, e) for p in probers for t, e in p.errors if t > sent_at]
|
||||
target_after_exit = [t for t, a in ok_after if a == instance and t > exited_at]
|
||||
last_target = max((t for t, a in ok_after if a == instance), default=None)
|
||||
print("Журнал реплики после SIGTERM:")
|
||||
print(
|
||||
"\n".join(line for line in shutdown_log.splitlines() if "/health/" not in line)
|
||||
)
|
||||
summary = {
|
||||
"container": container,
|
||||
"instance": instance,
|
||||
"exit_code": exit_code,
|
||||
"shutdown_seconds": round(exited_at - sent_at, 3),
|
||||
"limit_seconds": SHUTDOWN_LIMIT_SEC,
|
||||
"instances_before": sorted(before),
|
||||
"probes_ok_after_sigterm": len(ok_after),
|
||||
"instances_after_sigterm": sorted({a for _, a in ok_after}),
|
||||
"last_target_response_after_sigterm_seconds": None
|
||||
if last_target is None
|
||||
else round(last_target - sent_at, 3),
|
||||
"target_responses_after_exit": len(target_after_exit),
|
||||
"errors_after_sigterm": len(errors_after),
|
||||
"error_kinds": sorted({e for _, e in errors_after}),
|
||||
}
|
||||
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
||||
if (
|
||||
exit_code != 0
|
||||
or exited_at - sent_at > SHUTDOWN_LIMIT_SEC
|
||||
or errors_after
|
||||
or target_after_exit
|
||||
or not ok_after
|
||||
):
|
||||
raise SystemExit("FAIL: завершение по SIGTERM не соответствует требованиям")
|
||||
print(
|
||||
"PASS: SIGTERM завершил %s за %.2f с, LB обслужил %d запросов без ошибок"
|
||||
% (container, exited_at - sent_at, len(ok_after))
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user