Files
mtusi2026lab4/student-tests/rolling_update.py

89 lines
4.0 KiB
Python

import json
import os
import time
from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced
from loadgen import Load, summary
OUTPUT = ROOT / "evidence" / "rolling-update.json"
def env_of(container):
lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines()
return dict(line.split("=", 1) for line in lines if "=" in line)
def wait_new_replica(container, up_needed, seconds=90):
deadline = time.monotonic() + seconds
while health(container) != "healthy" or lb_up_count() < up_needed:
check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s")
time.sleep(0.2)
def main():
old = api_containers()
replicas = len(old)
check(replicas >= 2, "need at least 2 running api replicas")
wait_balanced(replicas)
old_version = env_of(old[0]).get("APP_VERSION")
version = time.strftime("rolling-%Y%m%d-%H%M%S")
env = dict(os.environ, APP_VERSION=version)
print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}")
started = time.monotonic()
compose("build", "api", "migrate", env=env)
print(f"image built in {time.monotonic() - started:.1f} s")
migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env)
print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1]
.split("messenger.db: ")[-1])
load = Load()
load.start()
time.sleep(3)
t0 = time.monotonic()
steps = []
for container in old:
step_start = time.monotonic()
before = set(api_containers())
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env)
new = sorted(set(api_containers()) - before)
check(len(new) == 1, f"expected one new container, got {len(new)}")
wait_new_replica(new[0], replicas + 1)
ready = time.monotonic() - step_start
docker("stop", "--time", "15", container)
docker("rm", container)
steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2),
"step_seconds": round(time.monotonic() - step_start, 2)})
print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, "
f"step {steps[-1]['step_seconds']} s")
t1 = time.monotonic()
time.sleep(3)
load.stop()
final = api_containers()
versions = sorted({env_of(c).get("APP_VERSION") for c in final})
integrity = load.verify()
during = summary(load, t0, t1)
before_stats = summary(load, t0 - 3, t0)
print(f"now: replicas {[c[:12] for c in final]}, versions {versions}")
print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; "
f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, "
f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s")
print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, "
f"lost 0, duplicates 0")
check(len(final) == replicas and not set(final) & set(old), "old replicas are still running")
check(versions == [version], "not every replica runs the new version")
OUTPUT.parent.mkdir(exist_ok=True)
OUTPUT.write_text(json.dumps({
"from_version": old_version, "to_version": version, "replicas": replicas,
"profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2},
"steps": steps, "update_seconds": round(t1 - t0, 2),
"before": before_stats, "during": during, "integrity": integrity,
}, indent=2) + "\n", encoding="utf-8")
print(f"PASS: all {replicas} replicas replaced one by one under load, "
f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}")
if __name__ == "__main__":
main()