252 lines
9.4 KiB
Python
Executable File
252 lines
9.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Операторские команды стенда лабы 3 (свои, не skeleton). Python 3.9+, stdlib.
|
||
|
||
python3 scripts/lab3.py instances [--base-url URL] [--requests 12]
|
||
вход одним синтетическим пользователем и серия GET /users/me с одним
|
||
токеном через LB: X-Instance-Id каждого ответа и итог по репликам
|
||
(шаг 1 сценария защиты);
|
||
python3 scripts/lab3.py lb-stats
|
||
слоты HAProxy рабочего стенда: адрес, состояние проверки, запросы в
|
||
полёте, всего, ошибки соединения, повторы, перенаправления;
|
||
python3 scripts/lab3.py failover-each [--base-url URL] [--replicas 2]
|
||
[--mode stop|kill]
|
||
make test-failover для каждой реплики по очереди: перед прогоном все
|
||
реплики возвращены (make replicas), тест останавливает реплику,
|
||
ответившую на его первую запись; если это реплика, которую уже
|
||
проверили, очередь LB сдвигается одним запросом и прогон
|
||
повторяется. Печатает docker compose ps до и после, строку
|
||
STOP_ONE_LOG и итог теста — это evidence «какой контейнер
|
||
действительно остановлен».
|
||
|
||
Пароль синтетического пользователя и токен живут только в памяти и не
|
||
печатаются. Работает с проектом compose репозитория (compose.yaml и .env).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import base64
|
||
import json
|
||
import os
|
||
import secrets
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
LB_STATS = (
|
||
"exec 3<>/dev/tcp/127.0.0.1/8404 && "
|
||
"printf 'GET /stats;csv HTTP/1.0\\r\\n\\r\\n' >&3 && cat <&3"
|
||
)
|
||
STAT_FIELDS = ("addr", "status", "scur", "stot", "econ", "eresp", "wretr", "wredis")
|
||
|
||
|
||
def call(base: str, method: str, path: str, headers=None, body=None):
|
||
data = json.dumps(body).encode() if body is not None else None
|
||
request = urllib.request.Request(base + path, data=data, method=method)
|
||
for key, value in (headers or {}).items():
|
||
request.add_header(key, value)
|
||
if data is not None:
|
||
request.add_header("Content-Type", "application/json")
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=10) as response:
|
||
return response.status, dict(response.headers), response.read()
|
||
except urllib.error.HTTPError as error:
|
||
return error.code, dict(error.headers), error.read()
|
||
|
||
|
||
def login(base: str) -> str:
|
||
username = "ops_" + secrets.token_hex(6)
|
||
password = secrets.token_urlsafe(18)
|
||
status, _, _ = call(
|
||
base,
|
||
"POST",
|
||
"/api/v1/users",
|
||
body={"username": username, "display_name": "Оператор", "password": password},
|
||
)
|
||
if status != 201:
|
||
raise SystemExit(f"register -> {status}")
|
||
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||
status, _, data = call(
|
||
base,
|
||
"POST",
|
||
"/api/v1/auth/sessions",
|
||
headers={"Authorization": "Basic " + basic},
|
||
)
|
||
if status != 201:
|
||
raise SystemExit(f"login -> {status}")
|
||
return json.loads(data)["token"]
|
||
|
||
|
||
def instances(args: argparse.Namespace) -> int:
|
||
token = login(args.base_url)
|
||
seen: Counter = Counter()
|
||
print(f"one session, {args.requests} x GET /api/v1/users/me via {args.base_url}")
|
||
for number in range(1, args.requests + 1):
|
||
status, headers, _ = call(
|
||
args.base_url,
|
||
"GET",
|
||
"/api/v1/users/me",
|
||
headers={"Authorization": "Bearer " + token},
|
||
)
|
||
instance = headers.get("X-Instance-Id") or headers.get("x-instance-id") or "-"
|
||
seen[instance] += 1
|
||
print(f" {number:2d}: {status} X-Instance-Id={instance}")
|
||
print(f"replicas seen: {len(seen)} {dict(seen)}")
|
||
return 0 if len(seen) >= 2 else 1
|
||
|
||
|
||
def compose(*args: str, check: bool = True) -> subprocess.CompletedProcess:
|
||
return subprocess.run(
|
||
["docker", "compose", *args],
|
||
cwd=ROOT,
|
||
capture_output=True,
|
||
text=True,
|
||
check=check,
|
||
)
|
||
|
||
|
||
def lb_rows() -> list[dict]:
|
||
done = compose("exec", "-T", "haproxy", "bash", "-c", LB_STATS)
|
||
lines = [
|
||
line for line in done.stdout.splitlines() if line.startswith(("# ", "api,"))
|
||
]
|
||
header = lines[0][2:].split(",")
|
||
rows = []
|
||
for line in lines[1:]:
|
||
# zip(strict=) — только с Python 3.10, скрипт работает и на 3.9
|
||
row = dict(zip(header, line.split(","))) # noqa: B905
|
||
if row["svname"] in ("BACKEND", "FRONTEND") or not row.get("addr"):
|
||
continue
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
def lb_stats(_args: argparse.Namespace) -> int:
|
||
print("slot " + " ".join(f"{field:>12}" for field in STAT_FIELDS))
|
||
for row in lb_rows():
|
||
print(
|
||
f"{row['svname']:<9} "
|
||
+ " ".join(f"{row.get(f, ''):>12}" for f in STAT_FIELDS)
|
||
)
|
||
return 0
|
||
|
||
|
||
def wait_lb_up(replicas: int, timeout: float = 30) -> bool:
|
||
"""Все реплики UP в LB: `--wait` compose ждёт только healthcheck
|
||
контейнера, а слот вернувшейся реплики включается после проверок LB."""
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
up = [row for row in lb_rows() if row.get("status") == "UP"]
|
||
if len(up) >= replicas:
|
||
return True
|
||
time.sleep(0.5)
|
||
return False
|
||
|
||
|
||
def ps() -> str:
|
||
return compose(
|
||
"ps", "-a", "--format", "table {{.Name}}\t{{.ID}}\t{{.State}}\t{{.Status}}"
|
||
).stdout
|
||
|
||
|
||
def failover_each(args: argparse.Namespace) -> int:
|
||
done: set[str] = set()
|
||
log = Path(tempfile.mkdtemp(prefix="stop-one-")) / "stop-one.log"
|
||
env = {**os.environ, "STOP_ONE_LOG": str(log), "STOP_MODE": args.mode}
|
||
attempt = 0
|
||
while len(done) < args.replicas and attempt < args.replicas * 4:
|
||
attempt += 1
|
||
replicas = compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--no-recreate",
|
||
"--wait",
|
||
"--scale",
|
||
f"api={args.replicas}",
|
||
"api",
|
||
check=False,
|
||
)
|
||
if replicas.returncode != 0 or not wait_lb_up(args.replicas):
|
||
print(replicas.stderr or "replicas are not UP in the LB")
|
||
return 2
|
||
# сдвиг очереди round-robin: тест останавливает реплику, ответившую
|
||
# на его первую запись; с каждой попыткой сдвиг на запрос больше
|
||
for _ in range(attempt - 1):
|
||
call(args.base_url, "GET", "/health/live")
|
||
print(f"=== attempt {attempt}: docker compose ps before\n{ps()}", flush=True)
|
||
before = log.read_text().splitlines() if log.exists() else []
|
||
test = subprocess.run(
|
||
[
|
||
sys.executable,
|
||
"tests/resilience.py",
|
||
"--mode",
|
||
"failover",
|
||
"--action-script",
|
||
"scripts/stop-one.sh",
|
||
"--base-url",
|
||
args.base_url,
|
||
],
|
||
cwd=ROOT,
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
print(
|
||
"$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh "
|
||
f"(STOP_MODE={args.mode})"
|
||
)
|
||
print(test.stdout.strip())
|
||
lines = log.read_text().splitlines() if log.exists() else []
|
||
new = lines[len(before) :]
|
||
print("STOP_ONE_LOG: " + (new[-1] if new else "(no line)"))
|
||
print(f"docker compose ps after\n{ps()}", flush=True)
|
||
if test.returncode != 0:
|
||
return test.returncode
|
||
stopped = new[-1].split("container=")[1].split()[0][:12] if new else ""
|
||
if stopped:
|
||
done.add(stopped)
|
||
compose(
|
||
"up",
|
||
"-d",
|
||
"--no-deps",
|
||
"--no-recreate",
|
||
"--wait",
|
||
"--scale",
|
||
f"api={args.replicas}",
|
||
"api",
|
||
)
|
||
print(f"replicas stopped one by one: {sorted(done)}; all restored:\n{ps()}")
|
||
return 0 if len(done) >= args.replicas else 1
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
commands = parser.add_subparsers(dest="command", required=True)
|
||
one = commands.add_parser("instances")
|
||
one.add_argument("--base-url", default="http://localhost:8080")
|
||
one.add_argument("--requests", type=int, default=12)
|
||
commands.add_parser("lb-stats")
|
||
each = commands.add_parser("failover-each")
|
||
each.add_argument("--base-url", default="http://localhost:8080")
|
||
each.add_argument("--replicas", type=int, default=2)
|
||
each.add_argument("--mode", choices=("stop", "kill"), default="stop")
|
||
args = parser.parse_args()
|
||
handler = {
|
||
"instances": instances,
|
||
"lb-stats": lb_stats,
|
||
"failover-each": failover_each,
|
||
}
|
||
return handler[args.command](args)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|