import json import os import time from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced from loadgen import Load, summary OUTPUT = ROOT / "evidence" / "rolling-update.json" def env_of(container): lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines() return dict(line.split("=", 1) for line in lines if "=" in line) def wait_new_replica(container, up_needed, seconds=90): deadline = time.monotonic() + seconds while health(container) != "healthy" or lb_up_count() < up_needed: check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s") time.sleep(0.2) def main(): old = api_containers() replicas = len(old) check(replicas >= 2, "need at least 2 running api replicas") wait_balanced(replicas) old_version = env_of(old[0]).get("APP_VERSION") version = time.strftime("rolling-%Y%m%d-%H%M%S") env = dict(os.environ, APP_VERSION=version) print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}") started = time.monotonic() compose("build", "api", "migrate", env=env) print(f"image built in {time.monotonic() - started:.1f} s") migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env) print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1] .split("messenger.db: ")[-1]) load = Load() load.start() time.sleep(3) t0 = time.monotonic() steps = [] for container in old: step_start = time.monotonic() before = set(api_containers()) compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env) new = sorted(set(api_containers()) - before) check(len(new) == 1, f"expected one new container, got {len(new)}") wait_new_replica(new[0], replicas + 1) ready = time.monotonic() - step_start docker("stop", "--time", "15", container) docker("rm", container) steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2), "step_seconds": round(time.monotonic() - step_start, 2)}) print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, " f"step {steps[-1]['step_seconds']} s") t1 = time.monotonic() time.sleep(3) load.stop() final = api_containers() versions = sorted({env_of(c).get("APP_VERSION") for c in final}) integrity = load.verify() during = summary(load, t0, t1) before_stats = summary(load, t0 - 3, t0) print(f"now: replicas {[c[:12] for c in final]}, versions {versions}") print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; " f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, " f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s") print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, " f"lost 0, duplicates 0") check(len(final) == replicas and not set(final) & set(old), "old replicas are still running") check(versions == [version], "not every replica runs the new version") OUTPUT.parent.mkdir(exist_ok=True) OUTPUT.write_text(json.dumps({ "from_version": old_version, "to_version": version, "replicas": replicas, "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2}, "steps": steps, "update_seconds": round(t1 - t0, 2), "before": before_stats, "during": during, "integrity": integrity, }, indent=2) + "\n", encoding="utf-8") print(f"PASS: all {replicas} replicas replaced one by one under load, " f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}") if __name__ == "__main__": main()