import csv import fcntl import io import json import subprocess import time import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parents[1] COMPOSE = [ "docker", "compose", "--project-name", "highload-lab3", "--env-file", ".env", "-f", "compose.yml", ] def command(args, **kwargs): return subprocess.check_output(args, cwd=ROOT, text=True, **kwargs).strip() def configured_replicas(): config = json.loads(command(COMPOSE + ["config", "--format", "json"])) return int(config["services"]["api"]["deploy"]["replicas"]) def containers(): return command(COMPOSE + ["ps", "-q", "api"]).split() def identity(container): # Never dump Docker environment: it contains the database credential. return json.loads( command( [ "docker", "inspect", "--format", '{"hostname":{{json .Config.Hostname}},"state":{{json .State.Status}},"networks":{{json .NetworkSettings.Networks}}}', container, ] ) ) def stats(): with urllib.request.urlopen( "http://localhost:8404/stats;csv", timeout=3 ) as response: rows = csv.DictReader(io.StringIO(response.read().decode().removeprefix("# "))) return [ row for row in rows if row["pxname"] == "api_replicas" and row["svname"].startswith("api") ] def runtime(slot, state): proxy = command(COMPOSE + ["ps", "-q", "haproxy"]) reply = command( ["docker", "exec", "-i", proxy, "sh", "-c", "nc -w 1 127.0.0.1 9999"], input=f"set server api_replicas/{slot} state {state}\n", ) if reply: raise RuntimeError("HAProxy runtime command failed: " + reply) def wait_for(predicate, description, seconds=30): deadline = time.monotonic() + seconds while time.monotonic() < deadline: if predicate(): return time.sleep(0.1) raise RuntimeError("Timed out: " + description) def replace_all(): ROOT.joinpath(".memory").mkdir(exist_ok=True) with ROOT.joinpath(".memory/rolling.lock").open("w") as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) originals = containers() expected = configured_replicas() if expected < 2 or len(originals) != expected: raise RuntimeError("All configured API replicas must be running; minimum two") # One explicit migration job per operation, never one per replica. # Only backward-compatible migrations may be deployed while API is live. subprocess.run( COMPOSE + ["run", "--rm", "--no-deps", "migrate"], cwd=ROOT, check=True ) records = [] for old in originals: before = containers() if ( len(before) != expected or len([row for row in stats() if row["status"].startswith("UP")]) != expected ): raise RuntimeError( "Both replicas must be ready before taking one out of service" ) survivor = next(container for container in before if container != old) survivors = set(before) - {old} info = identity(old) addresses = {network["IPAddress"] for network in info["networks"].values()} matches = [row for row in stats() if row["addr"].split(":")[0] in addresses] if len(matches) != 1 or not matches[0]["status"].startswith("UP"): raise RuntimeError("Target is not uniquely ready in HAProxy") slot = matches[0]["svname"] started = time.monotonic() try: runtime(slot, "drain") wait_for( lambda: any( row["svname"] == slot and row["status"].startswith("DRAIN") and int(row["scur"]) == 0 for row in stats() ), "drain in-flight requests", ) command(["docker", "stop", "--timeout", "15", old]) logs = command(["docker", "logs", old], stderr=subprocess.STDOUT) if "server stopped gracefully" not in logs: raise RuntimeError("Target did not finish graceful shutdown") command(["docker", "rm", old]) # Compose fills the missing replica without recreating its survivor. subprocess.run( COMPOSE + [ "up", "-d", "--no-deps", "--no-recreate", "--wait", "--wait-timeout", "60", "api", ], cwd=ROOT, check=True, ) finally: runtime(slot, "ready") after = containers() if len(after) != expected or not survivors.issubset(after) or old in after: raise RuntimeError("Replacement changed the surviving container") new = next(container for container in after if container not in survivors) new_info = identity(new) new_addresses = { network["IPAddress"] for network in new_info["networks"].values() } wait_for( lambda: ( len([row for row in stats() if row["status"].startswith("UP")]) == expected and any( row["addr"].split(":")[0] in new_addresses and row["status"].startswith("UP") for row in stats() ) ), "new replica ready through HAProxy", ) record = { "old_instance": info["hostname"], "new_instance": new_info["hostname"], "survivor_instance": identity(survivor)["hostname"], "replacement_seconds": round(time.monotonic() - started, 3), "graceful_shutdown": True, } records.append(record) print(json.dumps(record), flush=True) return records if __name__ == "__main__": replace_all()