forked from mtusi2026/mtusi2026lab3
160 lines
5.3 KiB
Python
160 lines
5.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Управление двумя API без публичных backend-портов и без Docker socket в LB."""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SERVICES = ("api-a", "api-b")
|
|
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
|
PROBE = "import json,urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2); print(json.dumps({'status':r.status,'instance_id':r.headers['X-Instance-Id']}))"
|
|
|
|
|
|
def compose(*args, capture=False):
|
|
result = subprocess.run(
|
|
COMPOSE + list(args),
|
|
cwd=ROOT,
|
|
check=True,
|
|
text=True,
|
|
stdout=subprocess.PIPE if capture else None,
|
|
)
|
|
return result.stdout.strip() if capture else ""
|
|
|
|
|
|
def probe(service):
|
|
"""Получить alias из реального контейнера; credentials не выводятся."""
|
|
return json.loads(
|
|
compose("exec", "-T", service, "python", "-c", PROBE, capture=True)
|
|
)
|
|
|
|
|
|
def wait_ready(service, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
return probe(service)
|
|
except (subprocess.CalledProcessError, ValueError):
|
|
time.sleep(0.5)
|
|
raise RuntimeError(service + " did not become ready")
|
|
|
|
|
|
def inventory():
|
|
"""ID контейнера, PID host, restart count, alias и состояние без env/secrets."""
|
|
rows = []
|
|
for service in SERVICES:
|
|
cid = compose("ps", "-a", "-q", service, capture=True)
|
|
if not cid:
|
|
rows.append({"service": service, "state": "absent"})
|
|
continue
|
|
raw = subprocess.run(
|
|
["docker", "inspect", "--format", "{{json .State}}", cid],
|
|
check=True,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
state = json.loads(raw.stdout)
|
|
row = {
|
|
"service": service,
|
|
"container_id": cid,
|
|
"pid": state["Pid"],
|
|
"state": state["Status"],
|
|
"started_at": state["StartedAt"],
|
|
}
|
|
if state["Running"]:
|
|
row.update(probe(service))
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def drain(service):
|
|
"""Исключить backend, дождаться обновления upstream и старых workers."""
|
|
compose("exec", "-T", "nginx", "touch", "/tmp/drain-" + service)
|
|
deadline = time.monotonic() + 8
|
|
while time.monotonic() < deadline:
|
|
config = compose(
|
|
"exec", "-T", "nginx", "cat", "/tmp/backends.conf", capture=True
|
|
)
|
|
if service + ":8080" not in config:
|
|
time.sleep(5.2)
|
|
return
|
|
time.sleep(0.2)
|
|
raise RuntimeError("LB did not drain " + service)
|
|
|
|
|
|
def undrain(service):
|
|
compose("exec", "-T", "nginx", "rm", "-f", "/tmp/drain-" + service)
|
|
|
|
|
|
def replace(service):
|
|
"""Заменить одну реплику при заведомо готовой второй."""
|
|
other = next(s for s in SERVICES if s != service)
|
|
wait_ready(other)
|
|
before = probe(service)["instance_id"]
|
|
drain(service)
|
|
try:
|
|
compose("up", "-d", "--no-deps", "--force-recreate", service)
|
|
after = wait_ready(service)["instance_id"]
|
|
if before == after:
|
|
raise RuntimeError("process alias did not change")
|
|
finally:
|
|
undrain(service)
|
|
wait_balanced()
|
|
return {"service": service, "before": before, "after": after}
|
|
|
|
|
|
def wait_balanced(timeout=30):
|
|
"""Убедиться, что бэкенд отвечает через LB."""
|
|
import urllib.request
|
|
|
|
deadline = time.monotonic() + timeout
|
|
base = os.environ.get("BASE_URL", "http://localhost:8080")
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
req = urllib.request.Request(
|
|
base + "/health/ready", headers={"Connection": "close"}
|
|
)
|
|
with urllib.request.urlopen(req, timeout=2) as reply:
|
|
if reply.status == 200:
|
|
return # Проходим дальше, раз LB отвечает
|
|
except OSError:
|
|
pass
|
|
time.sleep(0.1)
|
|
raise RuntimeError("LB did not expose ready processes")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"action", choices=["status", "stop-target", "restore", "rolling"]
|
|
)
|
|
args = parser.parse_args()
|
|
if args.action == "status":
|
|
print(json.dumps(inventory(), indent=2))
|
|
elif args.action == "stop-target":
|
|
target = os.environ.get("TARGET_INSTANCE_ID", "")
|
|
if not target:
|
|
raise RuntimeError("TARGET_INSTANCE_ID is required")
|
|
matches = [s for s in SERVICES if probe(s)["instance_id"] == target]
|
|
if len(matches) != 1:
|
|
raise RuntimeError("target must identify exactly one live process")
|
|
service = matches[0]
|
|
compose("stop", "--timeout", "6", service)
|
|
print(json.dumps({"stopped_service": service, "instance_id": target}))
|
|
elif args.action == "restore":
|
|
for service in SERVICES:
|
|
compose("up", "-d", "--no-deps", service)
|
|
undrain(service)
|
|
wait_balanced()
|
|
print(json.dumps(inventory(), indent=2))
|
|
else:
|
|
for service in SERVICES:
|
|
print(json.dumps(replace(service)))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|