forked from mtusi2026/mtusi2026lab7
89 lines
4.0 KiB
Python
89 lines
4.0 KiB
Python
import json
|
|
import os
|
|
import time
|
|
|
|
from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced
|
|
from loadgen import Load, summary
|
|
|
|
OUTPUT = ROOT / "evidence" / "rolling-update.json"
|
|
|
|
|
|
def env_of(container):
|
|
lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines()
|
|
return dict(line.split("=", 1) for line in lines if "=" in line)
|
|
|
|
|
|
def wait_new_replica(container, up_needed, seconds=90):
|
|
deadline = time.monotonic() + seconds
|
|
while health(container) != "healthy" or lb_up_count() < up_needed:
|
|
check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s")
|
|
time.sleep(0.2)
|
|
|
|
|
|
def main():
|
|
old = api_containers()
|
|
replicas = len(old)
|
|
check(replicas >= 2, "need at least 2 running api replicas")
|
|
wait_balanced(replicas)
|
|
old_version = env_of(old[0]).get("APP_VERSION")
|
|
version = time.strftime("rolling-%Y%m%d-%H%M%S")
|
|
env = dict(os.environ, APP_VERSION=version)
|
|
|
|
print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}")
|
|
started = time.monotonic()
|
|
compose("build", "api", "migrate", env=env)
|
|
print(f"image built in {time.monotonic() - started:.1f} s")
|
|
migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env)
|
|
print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1]
|
|
.split("messenger.db: ")[-1])
|
|
|
|
load = Load()
|
|
load.start()
|
|
time.sleep(3)
|
|
t0 = time.monotonic()
|
|
steps = []
|
|
for container in old:
|
|
step_start = time.monotonic()
|
|
before = set(api_containers())
|
|
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env)
|
|
new = sorted(set(api_containers()) - before)
|
|
check(len(new) == 1, f"expected one new container, got {len(new)}")
|
|
wait_new_replica(new[0], replicas + 1)
|
|
ready = time.monotonic() - step_start
|
|
docker("stop", "--time", "15", container)
|
|
docker("rm", container)
|
|
steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2),
|
|
"step_seconds": round(time.monotonic() - step_start, 2)})
|
|
print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, "
|
|
f"step {steps[-1]['step_seconds']} s")
|
|
t1 = time.monotonic()
|
|
time.sleep(3)
|
|
load.stop()
|
|
|
|
final = api_containers()
|
|
versions = sorted({env_of(c).get("APP_VERSION") for c in final})
|
|
integrity = load.verify()
|
|
during = summary(load, t0, t1)
|
|
before_stats = summary(load, t0 - 3, t0)
|
|
print(f"now: replicas {[c[:12] for c in final]}, versions {versions}")
|
|
print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; "
|
|
f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, "
|
|
f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s")
|
|
print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, "
|
|
f"lost 0, duplicates 0")
|
|
check(len(final) == replicas and not set(final) & set(old), "old replicas are still running")
|
|
check(versions == [version], "not every replica runs the new version")
|
|
OUTPUT.parent.mkdir(exist_ok=True)
|
|
OUTPUT.write_text(json.dumps({
|
|
"from_version": old_version, "to_version": version, "replicas": replicas,
|
|
"profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2},
|
|
"steps": steps, "update_seconds": round(t1 - t0, 2),
|
|
"before": before_stats, "during": during, "integrity": integrity,
|
|
}, indent=2) + "\n", encoding="utf-8")
|
|
print(f"PASS: all {replicas} replicas replaced one by one under load, "
|
|
f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|