forked from mtusi2026/mtusi2026lab5
134 lines
5.6 KiB
Python
134 lines
5.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Поочерёдная замена реплик API (rolling update) под нагрузкой (lab3, уровень «4»).
|
|
|
|
Compose не делает rolling update сам, поэтому замена выполняется по шагам:
|
|
1. собрать новый образ (docker compose build api);
|
|
2. для каждой старой реплики:
|
|
поднять ОДНУ новую (up --scale api=N+1 --no-recreate) → дождаться healthy
|
|
и того, что LB её обслуживает → остановить старую (SIGTERM, graceful) → удалить;
|
|
3. всё это время нагрузка пишет сообщения через LB и повторяет неудачные
|
|
запросы с тем же Idempotency-Key.
|
|
Итог: ошибки/RPS во время замены, все реплики на новом образе, нет потерь и дублей.
|
|
|
|
Миграции: перед заменой выполняется сервис migrate (однократно). Новые миграции
|
|
должны быть совместимы с ещё работающей старой версией (только добавление).
|
|
|
|
python student-tests/rolling_update.py --base-url http://localhost:8080 --output evidence/rolling-update.json
|
|
"""
|
|
import argparse
|
|
import json
|
|
import shlex
|
|
import subprocess
|
|
import time
|
|
|
|
from common import Api, Report
|
|
from failover_each import served_instances
|
|
from loadgen import LoadGen
|
|
|
|
COMPOSE = "docker compose --env-file .env"
|
|
|
|
|
|
def sh(cmd: str) -> str:
|
|
print(f" $ {cmd}")
|
|
return subprocess.run(shlex.split(cmd), check=True, capture_output=True, text=True).stdout
|
|
|
|
|
|
def api_ids() -> list:
|
|
return [x.strip() for x in sh(f"{COMPOSE} ps -q api").splitlines() if x.strip()]
|
|
|
|
|
|
def image_of(cid: str) -> str:
|
|
return sh(f"docker inspect -f {{{{.Image}}}} {cid}").strip()
|
|
|
|
|
|
def wait_healthy(cid: str, seconds: float = 90) -> bool:
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
if sh(f"docker inspect -f {{{{.State.Health.Status}}}} {cid}").strip() == "healthy":
|
|
return True
|
|
time.sleep(1)
|
|
return False
|
|
|
|
|
|
def wait_served(api: Api, iid: str, seconds: float = 60) -> bool:
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
if iid in served_instances(api, 10):
|
|
return True
|
|
time.sleep(0.5)
|
|
return False
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--base-url", default="http://localhost:8080")
|
|
ap.add_argument("--workers", type=int, default=4)
|
|
ap.add_argument("--no-build", action="store_true", help="не пересобирать образ")
|
|
ap.add_argument("--output", default=None)
|
|
args = ap.parse_args()
|
|
api, r = Api(args.base_url), Report()
|
|
|
|
if not args.no_build:
|
|
sh(f"{COMPOSE} build api")
|
|
sh(f"{COMPOSE} run --rm migrate") # однократные миграции до замены
|
|
new_image = sh("docker image inspect -f {{.Id}} mtusi-messenger:lab4").strip()
|
|
old = api_ids()
|
|
n = len(old)
|
|
r.check(n >= 2, f"{n} replicas before update")
|
|
old_images = {cid[:12]: image_of(cid) for cid in old}
|
|
|
|
user = api.new_user("rolling")
|
|
st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "rolling", "member_ids": [user["id"]]},
|
|
token=user["token"])
|
|
gen = LoadGen(api, user["token"], chat["id"], workers=args.workers)
|
|
gen.start()
|
|
time.sleep(3)
|
|
t_begin = time.monotonic()
|
|
steps = []
|
|
for cid in old:
|
|
before = set(api_ids())
|
|
sh(f"{COMPOSE} up -d --no-deps --no-recreate --scale api={len(before) + 1} api")
|
|
new = sorted(set(api_ids()) - before)
|
|
r.check(len(new) == 1, f"one new replica started for {cid[:12]}")
|
|
new_id = new[0]
|
|
healthy = wait_healthy(new_id)
|
|
served = wait_served(api, new_id[:12])
|
|
t0 = time.monotonic()
|
|
sh(f"docker stop -t 15 {cid}")
|
|
sh(f"docker rm {cid}")
|
|
steps.append({"old": cid[:12], "new": new_id[:12], "new_healthy": healthy, "new_served_by_lb": served,
|
|
"old_stop_seconds": round(time.monotonic() - t0, 2)})
|
|
r.check(healthy and served, f"new replica {new_id[:12]} healthy and in LB before old {cid[:12]} stops")
|
|
t_end = time.monotonic()
|
|
time.sleep(3)
|
|
gen.stop()
|
|
|
|
final = api_ids()
|
|
rel_a, rel_b = t_begin - gen.t0, t_end - gen.t0
|
|
result = {
|
|
"replicas_before": sorted(old_images), "replicas_after": sorted(c[:12] for c in final),
|
|
"all_on_new_image": all(image_of(c) == new_image for c in final),
|
|
"steps": steps,
|
|
"update_duration_s": round(t_end - t_begin, 1),
|
|
"before_update": gen.window(0, rel_a),
|
|
"during_update": gen.window(rel_a, rel_b),
|
|
"after_update": gen.window(rel_b, rel_b + 3),
|
|
"max_gap_without_success_s": gen.max_gap_after(t_begin),
|
|
"instances_serving_at_end": sorted(set(gen.instances_after(t_end))),
|
|
**gen.verify(),
|
|
}
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
r.check(len(final) == n, f"replica count restored to {n}")
|
|
r.check(result["all_on_new_image"], "all replicas run the new image")
|
|
r.check(not set(result["instances_serving_at_end"]) & set(old_images), "old replicas no longer serve traffic")
|
|
r.check(result["lost_or_changed"] == 0 and result["duplicate_texts"] == 0, "no lost or duplicated writes")
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as f:
|
|
json.dump(result, f, ensure_ascii=False, indent=2)
|
|
r.finish()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|