Files
mtusi2026lab3-fork/scripts/ops.py

399 lines
18 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Операции над compose-стендом lab3. Python 3.9+, только стандартная библиотека.
Работает одинаково в Windows (PowerShell/cmd), Linux и macOS: вызывает
`docker compose` и `docker` без shell-интерполяции. Секреты не печатает.
python scripts/ops.py wait # дождаться N здоровых реплик и LB
python scripts/ops.py ps # реплики: контейнер, X-Instance-Id, health, статус в LB
python scripts/ops.py stop-one # TARGET_INSTANCE_ID=<X-Instance-Id>; оставляет остановленной
python scripts/ops.py restore # запустить остановленные реплики и дождаться UP в LB
python scripts/ops.py restart # hook persistence: API + PostgreSQL + Redis + LB, volumes целы
python scripts/ops.py rolling-update # поочерёдная замена реплик без остановки сервиса
python scripts/ops.py in-network storage # тесты с прямым доступом к БД — внутри compose-сети
python scripts/ops.py in-network expiry # истечение сессии на api-short-ttl (TTL 2 с)
Переменные окружения:
COMPOSE_PROJECT_NAME имя проекта (по умолчанию messenger-lab3, как `name:` в compose.yaml)
ENV_FILE файл переменных (по умолчанию .env, если существует)
API_REPLICAS ожидаемое число реплик (по умолчанию из .env, иначе 2)
APP_PORT, LB_STATS_PORT опубликованные на 127.0.0.1 порты LB (8080, 8404)
STOP_MODE stop-one: stop (SIGTERM, graceful, по умолчанию) | kill (SIGKILL, «падение»)
Каждое действие дописывается в evidence/ops.log (время, контейнер, результат) —
это доказательство того, какой контейнер реально остановлен/заменён.
"""
import csv
import io
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OPS_LOG = ROOT / "evidence" / "ops.log"
try: # консоль Windows может быть не в UTF-8
sys.stdout.reconfigure(errors="replace")
sys.stderr.reconfigure(errors="replace")
except AttributeError:
pass
# ---------------------------------------------------------------- settings
def _dotenv():
path = Path(os.environ.get("ENV_FILE", str(ROOT / ".env")))
values = {}
if path.is_file():
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line and not line.startswith("#") and "=" in line:
k, v = line.split("=", 1)
values[k.strip()] = v.strip().strip('"').strip("'")
return path if path.is_file() else None, values
ENV_PATH, DOTENV = _dotenv()
def setting(name, default):
return os.environ.get(name) or DOTENV.get(name) or default
PROJECT = setting("COMPOSE_PROJECT_NAME", "messenger-lab3")
REPLICAS = int(setting("API_REPLICAS", "2"))
APP_PORT = int(setting("APP_PORT", "8080"))
STATS_PORT = int(setting("LB_STATS_PORT", "8404"))
LB_URL = "http://127.0.0.1:%d" % APP_PORT
STATS_URL = "http://127.0.0.1:%d/stats;csv" % STATS_PORT
def now_iso():
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
def log(msg):
line = "%s %s" % (now_iso(), msg)
print(line, flush=True)
try:
OPS_LOG.parent.mkdir(exist_ok=True)
with OPS_LOG.open("a", encoding="utf-8") as f:
f.write(line + "\n")
except OSError:
pass
# ---------------------------------------------------------------- docker
def compose_cmd(*args):
cmd = ["docker", "compose", "-p", PROJECT]
if ENV_PATH:
cmd += ["--env-file", str(ENV_PATH)]
return cmd + list(args)
def run(cmd, check=True, capture=False, timeout=300, show=False):
"""capture: вернуть вывод; show: показать вывод в консоли; иначе — тихо."""
out = None if show else (subprocess.PIPE if capture else subprocess.DEVNULL)
result = subprocess.run(
cmd, cwd=ROOT, check=False, timeout=timeout,
stdout=out, stderr=out, text=True,
)
if check and result.returncode != 0:
detail = (result.stderr or "").strip()[-400:] if capture else ""
raise SystemExit("command failed (%d): %s %s" % (result.returncode, " ".join(cmd[:4]), detail))
return result
def compose(*args, timeout=300):
return run(compose_cmd(*args), timeout=timeout, capture=True)
def api_containers(include_stopped=True):
"""[{id, short, name, state, health, image, ip}] — только сервис api, без one-off."""
cmd = ["docker", "ps", "--no-trunc", "--format", "{{.ID}}",
"--filter", "label=com.docker.compose.project=" + PROJECT,
"--filter", "label=com.docker.compose.service=api",
"--filter", "label=com.docker.compose.oneoff=False"]
if include_stopped:
cmd.insert(2, "-a")
ids = [x for x in run(cmd, capture=True).stdout.split() if x]
if not ids:
return []
raw = run(["docker", "inspect"] + ids, capture=True).stdout
result = []
for item in json.loads(raw):
state = item["State"]
nets = item.get("NetworkSettings", {}).get("Networks") or {}
ips = [n.get("IPAddress") for n in nets.values() if n.get("IPAddress")]
result.append({
"id": item["Id"],
"short": item["Id"][:12],
"hostname": item["Config"].get("Hostname", ""),
"name": item["Name"].lstrip("/"),
"state": state.get("Status"),
"health": (state.get("Health") or {}).get("Status", "-"),
"image": item["Config"].get("Image"),
"ip": ips[0] if ips else "",
"created": item.get("Created", "")[:19],
})
result.sort(key=lambda c: c["name"])
return result
def running(containers=None):
return [c for c in (containers if containers is not None else api_containers()) if c["state"] == "running"]
# ---------------------------------------------------------------- load balancer
def lb_servers():
"""[{name, status, addr}] backend api из CSV-статистики HAProxy (слоты с адресом)."""
data = urllib.request.urlopen(STATS_URL, timeout=2).read().decode("utf-8", "replace")
rows = csv.DictReader(io.StringIO(data.lstrip("# ")))
servers = []
for r in rows:
if r.get("pxname") != "api" or r.get("svname") in ("BACKEND", "FRONTEND"):
continue
addr = r.get("addr") or ""
status = r.get("status", "")
if status.startswith("MAINT") and not addr:
continue # пустой слот server-template
servers.append({"name": r["svname"], "status": status, "addr": addr})
return servers
def lb_up_count():
return sum(1 for s in lb_servers() if s["status"].startswith("UP"))
def lb_ready():
try:
with urllib.request.urlopen(LB_URL + "/health/ready", timeout=2) as r:
return r.status == 200
except (urllib.error.URLError, OSError):
return False
def wait_until(predicate, timeout, what, interval=0.3):
deadline = time.monotonic() + timeout
last_exc = None
while time.monotonic() < deadline:
try:
if predicate():
return True
except (urllib.error.URLError, OSError, ValueError, KeyError) as exc:
last_exc = exc
time.sleep(interval)
raise SystemExit("timeout after %ss waiting for %s%s" % (
timeout, what, (" (%s)" % type(last_exc).__name__) if last_exc else ""))
def wait_healthy(ids, timeout=90):
def ok():
states = {c["id"]: c for c in api_containers()}
return all(states.get(i, {}).get("health") == "healthy" for i in ids)
wait_until(ok, timeout, "containers healthy: " + ", ".join(i[:12] for i in ids))
def wait_stack(replicas, timeout=120):
wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= replicas,
timeout, "%d healthy api replicas" % replicas)
wait_until(lambda: lb_up_count() >= replicas, 30, "%d replicas UP in HAProxy" % replicas)
wait_until(lb_ready, 30, "LB /health/ready = 200")
# ---------------------------------------------------------------- commands
def cmd_wait(_args):
t0 = time.monotonic()
wait_stack(REPLICAS)
print("ready: %d api replicas healthy and UP in LB (%.1fs)" % (REPLICAS, time.monotonic() - t0))
def cmd_ps(_args):
containers = api_containers()
try:
by_addr = {s["addr"].rsplit(":", 1)[0]: s for s in lb_servers()}
except (urllib.error.URLError, OSError):
by_addr = {}
print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % ("X-Instance-Id", "container", "state", "health", "image", "lb", "lb_status"))
for c in containers:
s = by_addr.get(c["ip"], {})
print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % (
c["hostname"], c["name"], c["state"], c["health"], c["image"], s.get("name", "-"), s.get("status", "-")))
def find_target(target):
for c in api_containers():
# X-Instance-Id = hostname контейнера = первые 12 символов его ID.
if target and (c["hostname"] == target or c["id"].startswith(target)):
return c
return None
def cmd_stop_one(_args):
target = os.environ.get("TARGET_INSTANCE_ID", "").strip()
if not target:
raise SystemExit("TARGET_INSTANCE_ID is required (X-Instance-Id of the replica to stop)")
mode = os.environ.get("STOP_MODE", "stop").strip().lower()
c = find_target(target)
if c is None:
raise SystemExit("no api container of project %s matches X-Instance-Id %s" % (PROJECT, target))
if c["state"] != "running":
raise SystemExit("container %s is already %s" % (c["name"], c["state"]))
t0 = time.monotonic()
log("stop-one: mode=%s target=%s container=%s ip=%s" % (mode, target, c["name"], c["ip"]))
if mode == "kill":
# «Падение» процесса. Политику restart снимаем, чтобы Docker не поднял
# контейнер сам: тест требует, чтобы реплика оставалась остановленной.
run(["docker", "update", "--restart", "no", c["id"]])
run(["docker", "kill", "--signal", "KILL", c["id"]])
else:
# SIGTERM -> осушение 3 с (readiness 503) -> graceful stop; SIGKILL не позже 20 с.
run(["docker", "stop", "--time", "20", c["id"]], timeout=40)
after = find_target(target) or {}
log("stop-one: container=%s state=%s after %.2fs; left stopped (restore: python scripts/ops.py restore)"
% (c["name"], after.get("state", "removed"), time.monotonic() - t0))
def cmd_restore(_args):
t0 = time.monotonic()
stopped = [c for c in api_containers() if c["state"] != "running"]
for c in stopped: # вернуть политику, снятую в режиме kill
run(["docker", "update", "--restart", "unless-stopped", c["id"]], check=False)
# --no-recreate: не пересоздавать работающие; up запускает остановленные
# и добирает число контейнеров до deploy.replicas.
compose("up", "-d", "--no-deps", "--no-recreate", "api")
wait_stack(REPLICAS)
log("restore: started %s; %d replicas healthy and UP in LB after %.1fs"
% ([c["name"] for c in stopped] or "nothing", REPLICAS, time.monotonic() - t0))
def cmd_restart(_args):
"""Hook для make test-persistence: перезапуск API, ВСЕХ хранилищ и LB.
Volumes не удаляются, данные не пересоздаются. Порядок: сначала LB и API
(новые записи не принимаются), затем PostgreSQL (штатный shutdown с
checkpoint) и Redis, затем API и LB в порядке зависимостей.
"""
t0 = time.monotonic()
before = {c["name"]: c["short"] for c in running()}
pg_before = container_started("postgres")
redis_before = container_started("redis")
compose("stop", "lb", "api", timeout=120)
compose("restart", "postgres", "redis", timeout=120)
compose("start", "api", timeout=120)
wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= REPLICAS,
80, "api healthy after restart")
compose("start", "lb", timeout=60)
log("restart: postgres started_at %s -> %s; redis %s -> %s; api %s restarted; lb restarted; %.1fs"
% (pg_before, container_started("postgres"), redis_before, container_started("redis"),
sorted(before), time.monotonic() - t0))
def container_started(service):
ids = run(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + PROJECT,
"--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split()
if not ids:
return "-"
return run(["docker", "inspect", "-f", "{{.State.StartedAt}}", ids[0]], capture=True).stdout.strip()[:23]
def cmd_rolling_update(args):
"""Поочерёдная замена реплик: +1 новая -> healthy и UP в LB -> graceful stop старой.
В любой момент в ротации >= N реплик. Миграции — один раз, до замены, и только
расширяющие (expand): старая версия продолжает работать на новой схеме.
"""
t_all = time.monotonic()
if [c for c in api_containers() if c["state"] != "running"]:
raise SystemExit("some api containers are stopped; run `python scripts/ops.py restore` first")
old = running()
n = len(old)
if n < 1:
raise SystemExit("no running api replicas")
log("rolling-update: start; replicas=%d old=%s" % (n, [c["short"] for c in old]))
if "--no-build" not in args:
t = time.monotonic()
compose("build", "api", "migrate", timeout=900)
log("rolling-update: images built in %.1fs (tag %s)" % (time.monotonic() - t, setting("APP_VERSION", "lab3")))
t = time.monotonic()
out = compose("run", "--rm", "-T", "migrate", timeout=300).stdout.strip().splitlines()
log("rolling-update: migrations once in %.1fs: %s" % (time.monotonic() - t, out[-1] if out else ""))
for i, victim in enumerate(old, 1):
before = {c["id"] for c in api_containers()}
t = time.monotonic()
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=%d" % (n + 1), "api", timeout=180)
new = [c for c in api_containers() if c["id"] not in before]
if len(new) != 1:
raise SystemExit("expected exactly one new container, got %d" % len(new))
fresh = new[0]
wait_healthy([fresh["id"]], timeout=90)
wait_until(lambda: lb_up_count() >= n + 1, 30, "new replica UP in HAProxy")
t_ready = time.monotonic() - t
t = time.monotonic()
run(["docker", "stop", "--time", "20", victim["id"]], timeout=40)
run(["docker", "rm", victim["id"]])
log("rolling-update: step %d/%d new=%s (%s, ready+UP in %.1fs) replaced old=%s (graceful stop %.1fs)"
% (i, n, fresh["short"], fresh["image"], t_ready, victim["short"], time.monotonic() - t))
wait_stack(n)
log("rolling-update: done in %.1fs; replicas now %s" % (time.monotonic() - t_all, [c["short"] for c in running()]))
def cmd_in_network(args):
"""Тесты, которым нужен прямой доступ к PostgreSQL/Redis или экземпляр api-short-ttl.
Запускаются в one-off контейнере образа API внутри сети backend: порты БД и
реплик наружу не публикуются. One-off контейнер не получает DNS-имя `api`,
поэтому LB его не видит.
"""
which = args[0] if args else ""
mount = "%s:/app/student-tests:ro" % (ROOT / "student-tests")
base = compose_cmd("run", "--rm", "-T", "--no-deps", "-v", mount)
if which == "storage":
for pattern in ("test_storage_*.py", "test_session_cache.py"):
run(base + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", "-p", pattern, "-v"],
show=True, timeout=600)
elif which == "expiry":
prof = compose_cmd("--profile", "ttl")
run(prof + ["up", "-d", "--no-recreate", "api-short-ttl"], show=True, timeout=300)
env = ["-e", "EXPIRY_BASE_URL=http://api-short-ttl:8080", "-e", "BASE_URL=http://lb:8080"]
try:
run(base + env + ["api", "python", "student-tests/wait_url.py", "http://api-short-ttl:8080/health/ready"],
show=True, timeout=120)
run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests",
"-p", "test_expiry.py", "-v"], show=True, timeout=300)
finally:
run(prof + ["rm", "-sf", "api-short-ttl"], check=False, timeout=120)
else:
raise SystemExit("usage: python scripts/ops.py in-network storage|expiry")
COMMANDS = {
"wait": cmd_wait,
"ps": cmd_ps,
"stop-one": cmd_stop_one,
"restore": cmd_restore,
"restart": cmd_restart,
"rolling-update": cmd_rolling_update,
"in-network": cmd_in_network,
}
def main():
if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS:
print(__doc__)
sys.exit(2)
COMMANDS[sys.argv[1]](sys.argv[2:])
if __name__ == "__main__":
main()