feat: add replicated messenger and reproducible lab3 stand
This commit is contained in:
Executable
+251
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Операторские команды стенда лабы 3 (свои, не skeleton). Python 3.9+, stdlib.
|
||||
|
||||
python3 scripts/lab3.py instances [--base-url URL] [--requests 12]
|
||||
вход одним синтетическим пользователем и серия GET /users/me с одним
|
||||
токеном через LB: X-Instance-Id каждого ответа и итог по репликам
|
||||
(шаг 1 сценария защиты);
|
||||
python3 scripts/lab3.py lb-stats
|
||||
слоты HAProxy рабочего стенда: адрес, состояние проверки, запросы в
|
||||
полёте, всего, ошибки соединения, повторы, перенаправления;
|
||||
python3 scripts/lab3.py failover-each [--base-url URL] [--replicas 2]
|
||||
[--mode stop|kill]
|
||||
make test-failover для каждой реплики по очереди: перед прогоном все
|
||||
реплики возвращены (make replicas), тест останавливает реплику,
|
||||
ответившую на его первую запись; если это реплика, которую уже
|
||||
проверили, очередь LB сдвигается одним запросом и прогон
|
||||
повторяется. Печатает docker compose ps до и после, строку
|
||||
STOP_ONE_LOG и итог теста — это evidence «какой контейнер
|
||||
действительно остановлен».
|
||||
|
||||
Пароль синтетического пользователя и токен живут только в памяти и не
|
||||
печатаются. Работает с проектом compose репозитория (compose.yaml и .env).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LB_STATS = (
|
||||
"exec 3<>/dev/tcp/127.0.0.1/8404 && "
|
||||
"printf 'GET /stats;csv HTTP/1.0\\r\\n\\r\\n' >&3 && cat <&3"
|
||||
)
|
||||
STAT_FIELDS = ("addr", "status", "scur", "stot", "econ", "eresp", "wretr", "wredis")
|
||||
|
||||
|
||||
def call(base: str, method: str, path: str, headers=None, body=None):
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
request = urllib.request.Request(base + path, data=data, method=method)
|
||||
for key, value in (headers or {}).items():
|
||||
request.add_header(key, value)
|
||||
if data is not None:
|
||||
request.add_header("Content-Type", "application/json")
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
return response.status, dict(response.headers), response.read()
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, dict(error.headers), error.read()
|
||||
|
||||
|
||||
def login(base: str) -> str:
|
||||
username = "ops_" + secrets.token_hex(6)
|
||||
password = secrets.token_urlsafe(18)
|
||||
status, _, _ = call(
|
||||
base,
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
body={"username": username, "display_name": "Оператор", "password": password},
|
||||
)
|
||||
if status != 201:
|
||||
raise SystemExit(f"register -> {status}")
|
||||
basic = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
status, _, data = call(
|
||||
base,
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
headers={"Authorization": "Basic " + basic},
|
||||
)
|
||||
if status != 201:
|
||||
raise SystemExit(f"login -> {status}")
|
||||
return json.loads(data)["token"]
|
||||
|
||||
|
||||
def instances(args: argparse.Namespace) -> int:
|
||||
token = login(args.base_url)
|
||||
seen: Counter = Counter()
|
||||
print(f"one session, {args.requests} x GET /api/v1/users/me via {args.base_url}")
|
||||
for number in range(1, args.requests + 1):
|
||||
status, headers, _ = call(
|
||||
args.base_url,
|
||||
"GET",
|
||||
"/api/v1/users/me",
|
||||
headers={"Authorization": "Bearer " + token},
|
||||
)
|
||||
instance = headers.get("X-Instance-Id") or headers.get("x-instance-id") or "-"
|
||||
seen[instance] += 1
|
||||
print(f" {number:2d}: {status} X-Instance-Id={instance}")
|
||||
print(f"replicas seen: {len(seen)} {dict(seen)}")
|
||||
return 0 if len(seen) >= 2 else 1
|
||||
|
||||
|
||||
def compose(*args: str, check: bool = True) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
["docker", "compose", *args],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=check,
|
||||
)
|
||||
|
||||
|
||||
def lb_rows() -> list[dict]:
|
||||
done = compose("exec", "-T", "haproxy", "bash", "-c", LB_STATS)
|
||||
lines = [
|
||||
line for line in done.stdout.splitlines() if line.startswith(("# ", "api,"))
|
||||
]
|
||||
header = lines[0][2:].split(",")
|
||||
rows = []
|
||||
for line in lines[1:]:
|
||||
# zip(strict=) — только с Python 3.10, скрипт работает и на 3.9
|
||||
row = dict(zip(header, line.split(","))) # noqa: B905
|
||||
if row["svname"] in ("BACKEND", "FRONTEND") or not row.get("addr"):
|
||||
continue
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def lb_stats(_args: argparse.Namespace) -> int:
|
||||
print("slot " + " ".join(f"{field:>12}" for field in STAT_FIELDS))
|
||||
for row in lb_rows():
|
||||
print(
|
||||
f"{row['svname']:<9} "
|
||||
+ " ".join(f"{row.get(f, ''):>12}" for f in STAT_FIELDS)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def wait_lb_up(replicas: int, timeout: float = 30) -> bool:
|
||||
"""Все реплики UP в LB: `--wait` compose ждёт только healthcheck
|
||||
контейнера, а слот вернувшейся реплики включается после проверок LB."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
up = [row for row in lb_rows() if row.get("status") == "UP"]
|
||||
if len(up) >= replicas:
|
||||
return True
|
||||
time.sleep(0.5)
|
||||
return False
|
||||
|
||||
|
||||
def ps() -> str:
|
||||
return compose(
|
||||
"ps", "-a", "--format", "table {{.Name}}\t{{.ID}}\t{{.State}}\t{{.Status}}"
|
||||
).stdout
|
||||
|
||||
|
||||
def failover_each(args: argparse.Namespace) -> int:
|
||||
done: set[str] = set()
|
||||
log = Path(tempfile.mkdtemp(prefix="stop-one-")) / "stop-one.log"
|
||||
env = {**os.environ, "STOP_ONE_LOG": str(log), "STOP_MODE": args.mode}
|
||||
attempt = 0
|
||||
while len(done) < args.replicas and attempt < args.replicas * 4:
|
||||
attempt += 1
|
||||
replicas = compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--no-recreate",
|
||||
"--wait",
|
||||
"--scale",
|
||||
f"api={args.replicas}",
|
||||
"api",
|
||||
check=False,
|
||||
)
|
||||
if replicas.returncode != 0 or not wait_lb_up(args.replicas):
|
||||
print(replicas.stderr or "replicas are not UP in the LB")
|
||||
return 2
|
||||
# сдвиг очереди round-robin: тест останавливает реплику, ответившую
|
||||
# на его первую запись; с каждой попыткой сдвиг на запрос больше
|
||||
for _ in range(attempt - 1):
|
||||
call(args.base_url, "GET", "/health/live")
|
||||
print(f"=== attempt {attempt}: docker compose ps before\n{ps()}", flush=True)
|
||||
before = log.read_text().splitlines() if log.exists() else []
|
||||
test = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
"failover",
|
||||
"--action-script",
|
||||
"scripts/stop-one.sh",
|
||||
"--base-url",
|
||||
args.base_url,
|
||||
],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
print(
|
||||
"$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh "
|
||||
f"(STOP_MODE={args.mode})"
|
||||
)
|
||||
print(test.stdout.strip())
|
||||
lines = log.read_text().splitlines() if log.exists() else []
|
||||
new = lines[len(before) :]
|
||||
print("STOP_ONE_LOG: " + (new[-1] if new else "(no line)"))
|
||||
print(f"docker compose ps after\n{ps()}", flush=True)
|
||||
if test.returncode != 0:
|
||||
return test.returncode
|
||||
stopped = new[-1].split("container=")[1].split()[0][:12] if new else ""
|
||||
if stopped:
|
||||
done.add(stopped)
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--no-recreate",
|
||||
"--wait",
|
||||
"--scale",
|
||||
f"api={args.replicas}",
|
||||
"api",
|
||||
)
|
||||
print(f"replicas stopped one by one: {sorted(done)}; all restored:\n{ps()}")
|
||||
return 0 if len(done) >= args.replicas else 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
one = commands.add_parser("instances")
|
||||
one.add_argument("--base-url", default="http://localhost:8080")
|
||||
one.add_argument("--requests", type=int, default=12)
|
||||
commands.add_parser("lb-stats")
|
||||
each = commands.add_parser("failover-each")
|
||||
each.add_argument("--base-url", default="http://localhost:8080")
|
||||
each.add_argument("--replicas", type=int, default=2)
|
||||
each.add_argument("--mode", choices=("stop", "kill"), default="stop")
|
||||
args = parser.parse_args()
|
||||
handler = {
|
||||
"instances": instances,
|
||||
"lb-stats": lb_stats,
|
||||
"failover-each": failover_each,
|
||||
}
|
||||
return handler[args.command](args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user