"""lab3, уровень 4: rolling update под нагрузкой — реплики по одной заменяются новой версией. make rolling-update (= python student-tests/rolling_update.py) 1. Собирается новый образ с APP_VERSION=rolling-<время> (docker compose build api migrate). 2. Миграции новой версии применяются ОДИН раз отдельным запуском: docker compose run --rm migrate. Миграции только добавляют (как 004: новые столбцы, которые могут быть NULL), поэтому старые реплики продолжают работать с новой схемой, пока их не заменят. 3. Для каждой старой реплики: запустить ещё одну, новую (docker compose up --scale api=N+1 --no-recreate), дождаться healthy и UP в HAProxy, затем остановить старую (docker stop: сначала readiness 503 и выход из балансировки) и удалить её. Реплик всё время не меньше N. 4. Всё это время идёт нагрузка (loadgen.py): ошибки, RPS, задержки; в конце — проверка, что подтверждённые сообщения на месте и дублей нет. Итог: таблица в консоли и evidence/rolling-update.json. После скрипта стенд работает на новой версии. Обычный `docker compose up -d --build` потом вернёт версию dev, пересоздав реплики разом (это уже не rolling update). """ import json import os import time from helpers import ROOT, check, run from loadgen import CLIENT_TIMEOUT_SECONDS, WRITERS, Load, summary from stand import api_containers, compose_output, compose_quiet, docker, health, lb_summary, lb_up_count, wait_balanced OUTPUT = ROOT / "evidence" / "rolling-update.json" def version_of(container): lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines() env = dict(line.split("=", 1) for line in lines if "=" in line) return env.get("APP_VERSION") def wait_new_replica(container, up_needed, seconds=90): deadline = time.monotonic() + seconds while health(container) != "healthy" or lb_up_count() < up_needed: check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s") time.sleep(0.2) def main(): old = api_containers() replicas = len(old) check(replicas >= 2, "need at least 2 running api replicas") wait_balanced(replicas) old_version = version_of(old[0]) version = time.strftime("rolling-%Y%m%d-%H%M%S") env = dict(os.environ, APP_VERSION=version) print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}") started = time.monotonic() compose_quiet("build", "api", "migrate", env=env) print(f"image built in {time.monotonic() - started:.1f} s") migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env) lines = [line for line in migrate_log.splitlines() if "Миграции" in line] print("migrations (one run, before replacing replicas): " + (lines[-1].split("messenger.db: ")[-1] if lines else "?")) load = Load() load.start() time.sleep(3) t0 = time.monotonic() steps = [] for container in old: step_start = time.monotonic() before = set(api_containers()) compose_quiet("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env) new = sorted(set(api_containers()) - before) check(len(new) == 1, f"expected one new container, got {len(new)}") wait_new_replica(new[0], replicas + 1) ready = time.monotonic() - step_start docker("stop", "--time", "15", container) docker("rm", container) steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2), "step_seconds": round(time.monotonic() - step_start, 2)}) print(f"replaced {container[:12]} -> {new[0][:12]}: new replica healthy and UP in HAProxy after " f"{ready:.1f} s, then the old one drained and stopped; step {steps[-1]['step_seconds']} s " f"(HAProxy: {lb_summary()})") t1 = time.monotonic() time.sleep(3) load.stop() final = api_containers() versions = sorted({version_of(c) for c in final}) integrity = load.verify() during = summary(load, t0, t1) before_stats = summary(load, t0 - 3, t0) print(f"now: replicas {[c[:12] for c in final]}, versions {versions}") print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; " f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, " f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s") print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, " f"lost 0, duplicates 0") check(len(final) == replicas and not set(final) & set(old), "old replicas are still running") check(versions == [version], "not every replica runs the new version") OUTPUT.parent.mkdir(exist_ok=True) OUTPUT.write_text(json.dumps({ "from_version": old_version, "to_version": version, "replicas": replicas, "profile": {"writers": WRITERS, "readers": 1, "client_timeout_seconds": CLIENT_TIMEOUT_SECONDS}, "steps": steps, "update_seconds": round(t1 - t0, 2), "before": before_stats, "during": during, "integrity": integrity, }, indent=2) + "\n", encoding="utf-8") print(f"PASS: all {replicas} replicas replaced one by one under load, " f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT).as_posix()}") if __name__ == "__main__": run(main)