Files
mtusi2026lab3-fork/scripts/rolling_update.py
T

181 lines
6.3 KiB
Python

import csv
import fcntl
import io
import json
import subprocess
import time
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = [
"docker",
"compose",
"--project-name",
"highload-lab3",
"--env-file",
".env",
"-f",
"compose.yml",
]
def command(args, **kwargs):
return subprocess.check_output(args, cwd=ROOT, text=True, **kwargs).strip()
def configured_replicas():
config = json.loads(command(COMPOSE + ["config", "--format", "json"]))
return int(config["services"]["api"]["deploy"]["replicas"])
def containers():
return command(COMPOSE + ["ps", "-q", "api"]).split()
def identity(container):
# Never dump Docker environment: it contains the database credential.
return json.loads(
command(
[
"docker",
"inspect",
"--format",
'{"hostname":{{json .Config.Hostname}},"state":{{json .State.Status}},"networks":{{json .NetworkSettings.Networks}}}',
container,
]
)
)
def stats():
with urllib.request.urlopen(
"http://localhost:8404/stats;csv", timeout=3
) as response:
rows = csv.DictReader(io.StringIO(response.read().decode().removeprefix("# ")))
return [
row
for row in rows
if row["pxname"] == "api_replicas" and row["svname"].startswith("api")
]
def runtime(slot, state):
proxy = command(COMPOSE + ["ps", "-q", "haproxy"])
reply = command(
["docker", "exec", "-i", proxy, "sh", "-c", "nc -w 1 127.0.0.1 9999"],
input=f"set server api_replicas/{slot} state {state}\n",
)
if reply:
raise RuntimeError("HAProxy runtime command failed: " + reply)
def wait_for(predicate, description, seconds=30):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
if predicate():
return
time.sleep(0.1)
raise RuntimeError("Timed out: " + description)
def replace_all():
ROOT.joinpath(".memory").mkdir(exist_ok=True)
with ROOT.joinpath(".memory/rolling.lock").open("w") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
originals = containers()
expected = configured_replicas()
if expected < 2 or len(originals) != expected:
raise RuntimeError("All configured API replicas must be running; minimum two")
# One explicit migration job per operation, never one per replica.
# Only backward-compatible migrations may be deployed while API is live.
subprocess.run(
COMPOSE + ["run", "--rm", "--no-deps", "migrate"], cwd=ROOT, check=True
)
records = []
for old in originals:
before = containers()
if (
len(before) != expected
or len([row for row in stats() if row["status"].startswith("UP")]) != expected
):
raise RuntimeError(
"Both replicas must be ready before taking one out of service"
)
survivor = next(container for container in before if container != old)
survivors = set(before) - {old}
info = identity(old)
addresses = {network["IPAddress"] for network in info["networks"].values()}
matches = [row for row in stats() if row["addr"].split(":")[0] in addresses]
if len(matches) != 1 or not matches[0]["status"].startswith("UP"):
raise RuntimeError("Target is not uniquely ready in HAProxy")
slot = matches[0]["svname"]
started = time.monotonic()
try:
runtime(slot, "drain")
wait_for(
lambda: any(
row["svname"] == slot
and row["status"].startswith("DRAIN")
and int(row["scur"]) == 0
for row in stats()
),
"drain in-flight requests",
)
command(["docker", "stop", "--timeout", "15", old])
logs = command(["docker", "logs", old], stderr=subprocess.STDOUT)
if "server stopped gracefully" not in logs:
raise RuntimeError("Target did not finish graceful shutdown")
command(["docker", "rm", old])
# Compose fills the missing replica without recreating its survivor.
subprocess.run(
COMPOSE
+ [
"up",
"-d",
"--no-deps",
"--no-recreate",
"--wait",
"--wait-timeout",
"60",
"api",
],
cwd=ROOT,
check=True,
)
finally:
runtime(slot, "ready")
after = containers()
if len(after) != expected or not survivors.issubset(after) or old in after:
raise RuntimeError("Replacement changed the surviving container")
new = next(container for container in after if container not in survivors)
new_info = identity(new)
new_addresses = {
network["IPAddress"] for network in new_info["networks"].values()
}
wait_for(
lambda: (
len([row for row in stats() if row["status"].startswith("UP")]) == expected
and any(
row["addr"].split(":")[0] in new_addresses
and row["status"].startswith("UP")
for row in stats()
)
),
"new replica ready through HAProxy",
)
record = {
"old_instance": info["hostname"],
"new_instance": new_info["hostname"],
"survivor_instance": identity(survivor)["hostname"],
"replacement_seconds": round(time.monotonic() - started, 3),
"graceful_shutdown": True,
}
records.append(record)
print(json.dumps(record), flush=True)
return records
if __name__ == "__main__":
replace_all()