forked from mtusi2026/mtusi2026lab3
lab3: haproxy, replicas, idempotency, automated failover tests
This commit is contained in:
+112
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator actions for the local Compose stand; volumes are kept."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERVICES = ("api1", "api2")
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
|
||||
|
||||
|
||||
def compose(*args, capture=False):
|
||||
result = subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True,
|
||||
text=True, stdout=subprocess.PIPE if capture else None)
|
||||
return result.stdout.strip() if capture else None
|
||||
|
||||
|
||||
def state(service):
|
||||
cid = compose("ps", "--all", "--quiet", service, capture=True)
|
||||
if not cid:
|
||||
raise RuntimeError("No container for " + service)
|
||||
result = subprocess.check_output(
|
||||
["docker", "inspect", "--format",
|
||||
'{{json .Config.Hostname}} {{json .State.Status}}', cid], text=True)
|
||||
hostname, status = result.strip().split()
|
||||
return {"service": service, "container": cid[:12],
|
||||
"hostname": json.loads(hostname), "status": json.loads(status)}
|
||||
|
||||
|
||||
def wait_instances(count=2, timeout=40):
|
||||
deadline = time.monotonic() + timeout
|
||||
seen = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=2) as r:
|
||||
instance = r.headers.get("X-Instance-Id")
|
||||
if instance:
|
||||
seen.add(instance)
|
||||
if len(seen) >= count:
|
||||
return seen
|
||||
except (OSError, urllib.error.URLError):
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise RuntimeError("Not enough ready API instances through LB")
|
||||
|
||||
|
||||
def stop_one(service=None):
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "").split(":", 1)[0]
|
||||
if target:
|
||||
service = next((s for s in SERVICES if state(s)["hostname"] == target), None)
|
||||
if not service:
|
||||
raise RuntimeError("TARGET_INSTANCE_ID is not an API container")
|
||||
service = service or "api1"
|
||||
if service not in SERVICES:
|
||||
raise RuntimeError("Only API replicas may be stopped")
|
||||
compose("kill", "--signal", "SIGKILL", service)
|
||||
current = state(service)
|
||||
if current["status"] != "exited":
|
||||
raise RuntimeError("API did not stop")
|
||||
print(json.dumps(current))
|
||||
|
||||
|
||||
def restore():
|
||||
compose("up", "-d", "--no-deps", "--no-build", "--wait", "--wait-timeout", "40", *SERVICES)
|
||||
wait_instances()
|
||||
|
||||
|
||||
def restart():
|
||||
compose("down")
|
||||
compose("up", "-d", "--no-build", "--wait", "--wait-timeout", "60")
|
||||
wait_instances()
|
||||
|
||||
|
||||
def rolling_update():
|
||||
phases = []
|
||||
compose("build", "api1")
|
||||
# This migration only adds tables and is compatible with the old API.
|
||||
compose("run", "--rm", "--no-deps", "migrate")
|
||||
for service in SERVICES:
|
||||
before = state(service)
|
||||
started = time.monotonic()
|
||||
compose("up", "-d", "--no-deps", "--no-build", "--force-recreate",
|
||||
"--wait", "--wait-timeout", "40", service)
|
||||
after = state(service)
|
||||
if before["container"] == after["container"]:
|
||||
raise RuntimeError("Container was not replaced")
|
||||
seen = wait_instances()
|
||||
if not any(i.split(":", 1)[0] == after["hostname"] for i in seen):
|
||||
raise RuntimeError("LB has not discovered the replacement")
|
||||
phases.append({"service": service, "started": started, "finished": time.monotonic()})
|
||||
print(json.dumps({"replaced": service, "before": before, "after": after}))
|
||||
return phases
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=["restart", "stop-one", "restore", "rolling-update"])
|
||||
parser.add_argument("--service", choices=SERVICES)
|
||||
args = parser.parse_args()
|
||||
if args.action == "stop-one":
|
||||
stop_one(args.service)
|
||||
else:
|
||||
{"restart": restart, "restore": restore, "rolling-update": rolling_update}[args.action]()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user