116 lines
5.1 KiB
Python
116 lines
5.1 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Поочерёдная остановка КАЖДОЙ реплики API под нагрузкой (lab3, уровень «4»).
|
|
|
|
Для каждой реплики: дождаться, что LB обслуживает все реплики → запустить
|
|
нагрузку записи → через --before секунд остановить реплику (docker stop, SIGTERM)
|
|
→ ещё --after секунд нагрузки → собрать RPS, ошибки, время восстановления →
|
|
проверить сохранность и отсутствие дублей → вернуть реплику и дождаться,
|
|
что LB снова её обслуживает.
|
|
|
|
python student-tests/failover_each.py --base-url http://localhost:8080 --output evidence/failover-each.json
|
|
"""
|
|
import argparse
|
|
import json
|
|
import shlex
|
|
import subprocess
|
|
import time
|
|
|
|
from common import Api, Report
|
|
from loadgen import LoadGen
|
|
|
|
COMPOSE = "docker compose --env-file .env"
|
|
|
|
|
|
def run(cmd: str, **fmt) -> str:
|
|
out = subprocess.run(shlex.split(cmd.format(**fmt)), check=True, capture_output=True, text=True)
|
|
return out.stdout
|
|
|
|
|
|
def served_instances(api: Api, n: int = 30) -> set:
|
|
seen = set()
|
|
for _ in range(n):
|
|
try:
|
|
st, _, rh = api.call("GET", "/health/live", timeout=2)
|
|
if st == 200:
|
|
seen.add(rh.get("X-Instance-Id", ""))
|
|
except OSError:
|
|
pass
|
|
return seen
|
|
|
|
|
|
def wait_served(api: Api, iid: str, seconds: float = 60) -> float:
|
|
start = time.monotonic()
|
|
while time.monotonic() - start < seconds:
|
|
if iid in served_instances(api, 10):
|
|
return round(time.monotonic() - start, 2)
|
|
time.sleep(0.5)
|
|
return -1
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--base-url", default="http://localhost:8080")
|
|
ap.add_argument("--workers", type=int, default=4)
|
|
ap.add_argument("--before", type=float, default=4, help="секунд нагрузки до остановки")
|
|
ap.add_argument("--after", type=float, default=12, help="секунд нагрузки после остановки")
|
|
ap.add_argument("--list-cmd", default=f"{COMPOSE} ps -q api")
|
|
ap.add_argument("--stop-cmd", default="docker stop -t 8 {id}")
|
|
ap.add_argument("--start-cmd", default="docker start {id}")
|
|
ap.add_argument("--output", default=None)
|
|
args = ap.parse_args()
|
|
api, r = Api(args.base_url), Report()
|
|
|
|
replicas = [line.strip() for line in run(args.list_cmd).splitlines() if line.strip()]
|
|
r.check(len(replicas) >= 2, f"at least 2 API replicas running ({len(replicas)})")
|
|
results = []
|
|
user = api.new_user("failover")
|
|
for cid in replicas:
|
|
iid = cid[:12] # X-Instance-Id = hostname контейнера = короткий ID
|
|
print(f"== реплика {iid} ==")
|
|
r.check(wait_served(api, iid) >= 0, f"LB serves {iid} before the test")
|
|
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": f"failover {iid}", "member_ids": [user["id"]]},
|
|
token=user["token"])
|
|
gen = LoadGen(api, user["token"], chat["id"], workers=args.workers)
|
|
gen.start()
|
|
time.sleep(args.before)
|
|
t_stop = time.monotonic()
|
|
run(args.stop_cmd, id=cid)
|
|
stop_cmd_seconds = round(time.monotonic() - t_stop, 2)
|
|
time.sleep(args.after)
|
|
gen.stop()
|
|
rel = t_stop - gen.t0
|
|
later = gen.instances_after(t_stop + 3)
|
|
res = {
|
|
"stopped_instance": iid,
|
|
"stop_command_seconds": stop_cmd_seconds,
|
|
"before": gen.window(0, rel),
|
|
"first_10s_after_stop": gen.window(rel, rel + 10),
|
|
"first_success_after_stop_s": gen.first_success_after(t_stop),
|
|
"max_gap_without_success_after_stop_s": gen.max_gap_after(t_stop),
|
|
"stopped_instance_served_after_3s": later.count(iid),
|
|
"instances_after_stop": sorted(set(later)),
|
|
**gen.verify(),
|
|
}
|
|
t_start = time.monotonic()
|
|
run(args.start_cmd, id=cid)
|
|
res["rejoin_seconds"] = wait_served(api, iid)
|
|
res["rejoin_including_start_s"] = round(time.monotonic() - t_start, 2)
|
|
results.append(res)
|
|
print(json.dumps(res, ensure_ascii=False, indent=2))
|
|
r.check(res["max_gap_without_success_after_stop_s"] <= 10, f"writes resumed within 10 s after stopping {iid}")
|
|
r.check(res["stopped_instance_served_after_3s"] == 0, f"stopped {iid} not served after detection")
|
|
r.check(res["lost_or_changed"] == 0, "confirmed messages preserved")
|
|
r.check(res["duplicate_texts"] == 0, "no duplicates despite retries with the same key")
|
|
r.check(res["rejoin_seconds"] >= 0, f"{iid} rejoined the LB after restart")
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as f:
|
|
json.dump({"workload": f"{args.workers} closed-loop writers, retry same key on failure, "
|
|
f"{args.before}s before + {args.after}s after stop",
|
|
"replicas": results}, f, ensure_ascii=False, indent=2)
|
|
r.finish()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|