forked from mtusi2026/mtusi2026lab3
92 lines
3.7 KiB
Python
92 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
||
"""Уровень 4: согласованность health/readiness/SIGTERM и время обнаружения отказа балансировщиком.
|
||
|
||
Опыт 1 (штатная остановка, SIGTERM): под нагрузкой ~100 запросов/с на одну реплику посылаем SIGTERM; фиксируем
|
||
t_ready_fail — когда реплика перестала быть в ротации LB (последний ответ с её X-Instance-Id),
|
||
t_exit — когда контейнер завершился, число ошибок клиента за это время (ожидаем 0).
|
||
Опыт 2 (внезапный отказ, SIGKILL): время до прекращения ошибок клиента = время обнаружения отказа LB.
|
||
Повторяется N раз. Результат: evidence/drain_timing.json.
|
||
"""
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.request
|
||
|
||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||
BASE = "http://127.0.0.1:8080"
|
||
PROJECT = "mtusi-lab3"
|
||
|
||
|
||
def sh(*args):
|
||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True).stdout.strip()
|
||
|
||
|
||
def api_ids():
|
||
return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT, "--filter", "label=com.docker.compose.service=api").split()
|
||
|
||
|
||
class Prober(threading.Thread):
|
||
def __init__(self):
|
||
super().__init__(daemon=True)
|
||
self.events, self.stop = [], threading.Event()
|
||
|
||
def run(self):
|
||
while not self.stop.is_set():
|
||
t = time.time()
|
||
try:
|
||
with urllib.request.urlopen(BASE + "/health/live", timeout=2) as r:
|
||
self.events.append((t, r.status, r.headers.get("X-Instance-Id")))
|
||
except Exception as exc:
|
||
self.events.append((t, type(exc).__name__, None))
|
||
time.sleep(0.01)
|
||
|
||
|
||
def trial(kind):
|
||
ids = api_ids()
|
||
target = ids[0]
|
||
prober = Prober()
|
||
prober.start()
|
||
time.sleep(2)
|
||
t0 = time.time()
|
||
if kind == "sigterm":
|
||
sh("docker", "kill", "--signal=SIGTERM", target)
|
||
else:
|
||
sh("docker", "kill", target)
|
||
deadline = time.time() + 30
|
||
t_exit = None
|
||
while time.time() < deadline:
|
||
if sh("docker", "inspect", "-f", "{{.State.Running}}", target) == "false":
|
||
t_exit = time.time() - t0
|
||
break
|
||
time.sleep(0.05)
|
||
time.sleep(3)
|
||
prober.stop.set()
|
||
prober.join()
|
||
served = [e for e in prober.events if e[0] >= t0 and e[2] == target]
|
||
last_served = max((e[0] for e in served), default=t0) - t0
|
||
errors = [e for e in prober.events if e[0] >= t0 and e[1] != 200]
|
||
last_error = max((e[0] for e in errors), default=t0) - t0
|
||
sh("docker", "rm", "-f", target)
|
||
sh("sh", "scripts/restore-replicas.sh")
|
||
time.sleep(8)
|
||
return {"kind": kind, "seconds_until_replica_stops_serving": round(last_served, 2),
|
||
"seconds_until_container_exit": round(t_exit, 2) if t_exit else None, "client_errors": len(errors),
|
||
"seconds_until_errors_stop": round(last_error, 2) if errors else 0.0}
|
||
|
||
|
||
def main():
|
||
n = int(sys.argv[1]) if len(sys.argv) > 1 else 3
|
||
results = [trial(k) for k in ("sigterm", "sigkill") for _ in range(n)]
|
||
report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "lb_health_check": "inter 1s, fall 2, rise 2",
|
||
"drain_seconds": 4, "graceful_shutdown_seconds": 8, "trials": results}
|
||
with open(os.path.join(ROOT, "evidence", "drain_timing.json"), "w", encoding="utf-8") as fh:
|
||
json.dump(report, fh, ensure_ascii=False, indent=2)
|
||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|