181 lines
6.3 KiB
Python
181 lines
6.3 KiB
Python
import csv
|
|
import fcntl
|
|
import io
|
|
import json
|
|
import subprocess
|
|
import time
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
COMPOSE = [
|
|
"docker",
|
|
"compose",
|
|
"--project-name",
|
|
"highload-lab3",
|
|
"--env-file",
|
|
".env",
|
|
"-f",
|
|
"compose.yml",
|
|
]
|
|
|
|
|
|
def command(args, **kwargs):
|
|
return subprocess.check_output(args, cwd=ROOT, text=True, **kwargs).strip()
|
|
|
|
|
|
def configured_replicas():
|
|
config = json.loads(command(COMPOSE + ["config", "--format", "json"]))
|
|
return int(config["services"]["api"]["deploy"]["replicas"])
|
|
|
|
|
|
def containers():
|
|
return command(COMPOSE + ["ps", "-q", "api"]).split()
|
|
|
|
|
|
def identity(container):
|
|
# Never dump Docker environment: it contains the database credential.
|
|
return json.loads(
|
|
command(
|
|
[
|
|
"docker",
|
|
"inspect",
|
|
"--format",
|
|
'{"hostname":{{json .Config.Hostname}},"state":{{json .State.Status}},"networks":{{json .NetworkSettings.Networks}}}',
|
|
container,
|
|
]
|
|
)
|
|
)
|
|
|
|
|
|
def stats():
|
|
with urllib.request.urlopen(
|
|
"http://localhost:8404/stats;csv", timeout=3
|
|
) as response:
|
|
rows = csv.DictReader(io.StringIO(response.read().decode().removeprefix("# ")))
|
|
return [
|
|
row
|
|
for row in rows
|
|
if row["pxname"] == "api_replicas" and row["svname"].startswith("api")
|
|
]
|
|
|
|
|
|
def runtime(slot, state):
|
|
proxy = command(COMPOSE + ["ps", "-q", "haproxy"])
|
|
reply = command(
|
|
["docker", "exec", "-i", proxy, "sh", "-c", "nc -w 1 127.0.0.1 9999"],
|
|
input=f"set server api_replicas/{slot} state {state}\n",
|
|
)
|
|
if reply:
|
|
raise RuntimeError("HAProxy runtime command failed: " + reply)
|
|
|
|
|
|
def wait_for(predicate, description, seconds=30):
|
|
deadline = time.monotonic() + seconds
|
|
while time.monotonic() < deadline:
|
|
if predicate():
|
|
return
|
|
time.sleep(0.1)
|
|
raise RuntimeError("Timed out: " + description)
|
|
|
|
|
|
def replace_all():
|
|
ROOT.joinpath(".memory").mkdir(exist_ok=True)
|
|
with ROOT.joinpath(".memory/rolling.lock").open("w") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
originals = containers()
|
|
expected = configured_replicas()
|
|
if expected < 2 or len(originals) != expected:
|
|
raise RuntimeError("All configured API replicas must be running; minimum two")
|
|
# One explicit migration job per operation, never one per replica.
|
|
# Only backward-compatible migrations may be deployed while API is live.
|
|
subprocess.run(
|
|
COMPOSE + ["run", "--rm", "--no-deps", "migrate"], cwd=ROOT, check=True
|
|
)
|
|
records = []
|
|
for old in originals:
|
|
before = containers()
|
|
if (
|
|
len(before) != expected
|
|
or len([row for row in stats() if row["status"].startswith("UP")]) != expected
|
|
):
|
|
raise RuntimeError(
|
|
"Both replicas must be ready before taking one out of service"
|
|
)
|
|
survivor = next(container for container in before if container != old)
|
|
survivors = set(before) - {old}
|
|
info = identity(old)
|
|
addresses = {network["IPAddress"] for network in info["networks"].values()}
|
|
matches = [row for row in stats() if row["addr"].split(":")[0] in addresses]
|
|
if len(matches) != 1 or not matches[0]["status"].startswith("UP"):
|
|
raise RuntimeError("Target is not uniquely ready in HAProxy")
|
|
slot = matches[0]["svname"]
|
|
started = time.monotonic()
|
|
try:
|
|
runtime(slot, "drain")
|
|
wait_for(
|
|
lambda: any(
|
|
row["svname"] == slot
|
|
and row["status"].startswith("DRAIN")
|
|
and int(row["scur"]) == 0
|
|
for row in stats()
|
|
),
|
|
"drain in-flight requests",
|
|
)
|
|
command(["docker", "stop", "--timeout", "15", old])
|
|
logs = command(["docker", "logs", old], stderr=subprocess.STDOUT)
|
|
if "server stopped gracefully" not in logs:
|
|
raise RuntimeError("Target did not finish graceful shutdown")
|
|
command(["docker", "rm", old])
|
|
# Compose fills the missing replica without recreating its survivor.
|
|
subprocess.run(
|
|
COMPOSE
|
|
+ [
|
|
"up",
|
|
"-d",
|
|
"--no-deps",
|
|
"--no-recreate",
|
|
"--wait",
|
|
"--wait-timeout",
|
|
"60",
|
|
"api",
|
|
],
|
|
cwd=ROOT,
|
|
check=True,
|
|
)
|
|
finally:
|
|
runtime(slot, "ready")
|
|
after = containers()
|
|
if len(after) != expected or not survivors.issubset(after) or old in after:
|
|
raise RuntimeError("Replacement changed the surviving container")
|
|
new = next(container for container in after if container not in survivors)
|
|
new_info = identity(new)
|
|
new_addresses = {
|
|
network["IPAddress"] for network in new_info["networks"].values()
|
|
}
|
|
wait_for(
|
|
lambda: (
|
|
len([row for row in stats() if row["status"].startswith("UP")]) == expected
|
|
and any(
|
|
row["addr"].split(":")[0] in new_addresses
|
|
and row["status"].startswith("UP")
|
|
for row in stats()
|
|
)
|
|
),
|
|
"new replica ready through HAProxy",
|
|
)
|
|
record = {
|
|
"old_instance": info["hostname"],
|
|
"new_instance": new_info["hostname"],
|
|
"survivor_instance": identity(survivor)["hostname"],
|
|
"replacement_seconds": round(time.monotonic() - started, 3),
|
|
"graceful_shutdown": True,
|
|
}
|
|
records.append(record)
|
|
print(json.dumps(record), flush=True)
|
|
return records
|
|
|
|
|
|
if __name__ == "__main__":
|
|
replace_all()
|