Files
mtusi2026lab5/student-tests/rolling_update.py
T
2026-10-25 21:43:03 +03:00

109 lines
6.0 KiB
Python

"""lab3, уровень 4: rolling update под нагрузкой — реплики по одной заменяются новой версией.
make rolling-update (= python student-tests/rolling_update.py)
1. Собирается новый образ с APP_VERSION=rolling-<время> (docker compose build api migrate).
2. Миграции новой версии применяются ОДИН раз отдельным запуском: docker compose run --rm migrate.
Миграции только добавляют (как 004: новые столбцы, которые могут быть NULL), поэтому старые
реплики продолжают работать с новой схемой, пока их не заменят.
3. Для каждой старой реплики: запустить ещё одну, новую (docker compose up --scale api=N+1
--no-recreate), дождаться healthy и UP в HAProxy, затем остановить старую (docker stop:
сначала readiness 503 и выход из балансировки) и удалить её. Реплик всё время не меньше N.
4. Всё это время идёт нагрузка (loadgen.py): ошибки, RPS, задержки; в конце — проверка,
что подтверждённые сообщения на месте и дублей нет.
Итог: таблица в консоли и evidence/rolling-update.json. После скрипта стенд работает на новой
версии. Обычный `docker compose up -d --build` потом вернёт версию dev, пересоздав реплики разом
(это уже не rolling update).
"""
import json
import os
import time
from helpers import ROOT, check, run
from loadgen import CLIENT_TIMEOUT_SECONDS, WRITERS, Load, summary
from stand import api_containers, compose_output, compose_quiet, docker, health, lb_summary, lb_up_count, wait_balanced
OUTPUT = ROOT / "evidence" / "rolling-update.json"
def version_of(container):
lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines()
env = dict(line.split("=", 1) for line in lines if "=" in line)
return env.get("APP_VERSION")
def wait_new_replica(container, up_needed, seconds=90):
deadline = time.monotonic() + seconds
while health(container) != "healthy" or lb_up_count() < up_needed:
check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s")
time.sleep(0.2)
def main():
old = api_containers()
replicas = len(old)
check(replicas >= 2, "need at least 2 running api replicas")
wait_balanced(replicas)
old_version = version_of(old[0])
version = time.strftime("rolling-%Y%m%d-%H%M%S")
env = dict(os.environ, APP_VERSION=version)
print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}")
started = time.monotonic()
compose_quiet("build", "api", "migrate", env=env)
print(f"image built in {time.monotonic() - started:.1f} s")
migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env)
lines = [line for line in migrate_log.splitlines() if "Миграции" in line]
print("migrations (one run, before replacing replicas): " + (lines[-1].split("messenger.db: ")[-1] if lines else "?"))
load = Load()
load.start()
time.sleep(3)
t0 = time.monotonic()
steps = []
for container in old:
step_start = time.monotonic()
before = set(api_containers())
compose_quiet("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env)
new = sorted(set(api_containers()) - before)
check(len(new) == 1, f"expected one new container, got {len(new)}")
wait_new_replica(new[0], replicas + 1)
ready = time.monotonic() - step_start
docker("stop", "--time", "15", container)
docker("rm", container)
steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2),
"step_seconds": round(time.monotonic() - step_start, 2)})
print(f"replaced {container[:12]} -> {new[0][:12]}: new replica healthy and UP in HAProxy after "
f"{ready:.1f} s, then the old one drained and stopped; step {steps[-1]['step_seconds']} s "
f"(HAProxy: {lb_summary()})")
t1 = time.monotonic()
time.sleep(3)
load.stop()
final = api_containers()
versions = sorted({version_of(c) for c in final})
integrity = load.verify()
during = summary(load, t0, t1)
before_stats = summary(load, t0 - 3, t0)
print(f"now: replicas {[c[:12] for c in final]}, versions {versions}")
print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; "
f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, "
f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s")
print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, "
f"lost 0, duplicates 0")
check(len(final) == replicas and not set(final) & set(old), "old replicas are still running")
check(versions == [version], "not every replica runs the new version")
OUTPUT.parent.mkdir(exist_ok=True)
OUTPUT.write_text(json.dumps({
"from_version": old_version, "to_version": version, "replicas": replicas,
"profile": {"writers": WRITERS, "readers": 1, "client_timeout_seconds": CLIENT_TIMEOUT_SECONDS},
"steps": steps, "update_seconds": round(t1 - t0, 2),
"before": before_stats, "during": during, "integrity": integrity,
}, indent=2) + "\n", encoding="utf-8")
print(f"PASS: all {replicas} replicas replaced one by one under load, "
f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT).as_posix()}")
if __name__ == "__main__":
run(main)