Реализовать лабораторную 6 на 5: SLO и обнаружение инцидентов

This commit is contained in:
Ванек
2026-11-19 18:53:24 +03:00
parent 470d9d6f1e
commit 8dad129edf
127 changed files with 53496 additions and 61 deletions
+90
View File
@@ -0,0 +1,90 @@
"""Управление репликами только текущего Compose project через Docker CLI."""
import argparse
import json
import os
from pathlib import Path
import subprocess
import time
from urllib.request import urlopen
ROOT = Path(__file__).resolve().parents[1]
def command(*args):
return subprocess.check_output(args, cwd=ROOT, text=True, timeout=120).strip()
def replicas():
ids = command("docker", "compose", "ps", "-q", "api").split()
return json.loads(command("docker", "inspect", *ids)) if ids else []
def base_url():
config = json.loads(command("docker", "compose", "config", "--format", "json"))
port = config["services"]["lb"]["ports"][0]["published"]
return os.environ.get("BASE_URL", "http://127.0.0.1:" + str(port)).rstrip("/")
def wait_ready():
url = base_url() + "/health/ready"
deadline = time.monotonic() + 90
seen = set()
while time.monotonic() < deadline:
current = replicas()
expected = {info["Config"]["Hostname"] for info in current}
if expected and all(info["State"].get("Health", {}).get("Status") == "healthy" for info in current):
try:
with urlopen(url, timeout=2) as response:
if response.status == 200:
seen.add(response.headers.get("X-Instance-Id"))
if expected <= seen:
print("API ready through LB:", ", ".join(sorted(expected)), flush=True)
return
except OSError:
pass
time.sleep(0.2)
raise RuntimeError("Не все работающие API-реплики готовы через LB за 90 секунд")
def stop_one():
target = os.environ.get("TARGET_INSTANCE_ID")
if not target:
raise RuntimeError("Задайте TARGET_INSTANCE_ID из X-Instance-Id")
current = replicas()
matches = [info for info in current if info["Config"]["Hostname"] == target]
if len(current) < 2 or len(matches) != 1:
raise RuntimeError("Нужны минимум две работающие реплики и ровно одна выбранная")
selected = matches[0]
command("docker", "stop", "--time", "8", selected["Id"])
print("Stopped API:", target, "container=" + selected["Id"], flush=True)
def rolling():
wait_ready()
originals = replicas()
count = len(originals)
if count < 2:
raise RuntimeError("Для поочерёдной замены нужны минимум две API-реплики")
for old in originals:
active = replicas()
healthy = [info for info in active if info["Id"] != old["Id"]
and info["State"].get("Health", {}).get("Status") == "healthy"]
if not healthy:
raise RuntimeError("Нет готовой реплики, которая продолжит обслуживать запросы")
print("Replacing API:", old["Config"]["Hostname"], flush=True)
command("docker", "stop", "--time", "8", old["Id"])
command("docker", "rm", old["Id"])
# --no-recreate сохраняет остальные процессы; Compose создаёт только недостающий.
command("docker", "compose", "up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={count}", "api")
wait_ready()
assert len(replicas()) == count
final = replicas()
assert not ({info["Id"] for info in originals} & {info["Id"] for info in final})
print("PASS: all API containers replaced sequentially; stores and LB kept running", flush=True)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=["wait", "stop", "rolling"])
action = parser.parse_args().action
{"wait": wait_ready, "stop": stop_one, "rolling": rolling}[action]()