Files
mtusi2026lab3/scripts/lbctl.py
T
Alexey BodanyukandClaude Opus 5.5 3f12698676 feat: HAProxy и N API-реплик, X-Instance-Id, общая идемпотентность, drain при SIGTERM
- compose: один масштабируемый сервис api (deploy.replicas=2, без ports и
  container_name), HAProxy 3.2 LTS по digest — единственная точка входа
  (127.0.0.1), разовый migrate до старта реплик, CPU/RAM-лимиты у всех сервисов,
  каждому сервису только его secrets.
- HAProxy: server-template + Docker DNS (re-resolve 1 с), активная проверка
  GET /health/ready каждую 1 с (fall 2 / rise 2), без backup-серверов,
  retry только conn-failure, ошибки самого LB — в формате контракта с X-Request-Id.
- API: X-Instance-Id = hostname контейнера на каждом ответе (и на 500/503);
  Idempotency-Key обязателен; ключ, сообщение и результат пишутся в ОДНОЙ
  транзакции (миграция 0002), конкурентный повтор ждёт уникальный индекс;
  другой body — 409, посторонний — всегда 404; attachment_ids=[] равен
  отсутствию поля; дубли ключей JSON и тело у POST /auth/sessions — 400.
- SIGTERM: readiness 503 + DRAIN_SECONDS=3 обслуживания, затем graceful stop.
- scripts: stop-one.sh (+ .cmd для Windows), restore-api.sh, restart.sh
  перезапускает и LB, lbctl.py — состояние бэкендов HAProxy -> контейнеры.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 13:32:13 +03:00

124 lines
5.1 KiB
Python

#!/usr/bin/env python3
"""What the load balancer thinks right now, mapped to real containers.
python scripts/lbctl.py status # table: HAProxy slot, IP, UP/DOWN/MAINT, container, X-Instance-Id
python scripts/lbctl.py wait-up 2 # wait until exactly 2 API replicas are UP in HAProxy
python scripts/lbctl.py replicas # api containers of this compose project: name, hostname, state
Reads HAProxy's runtime API (`show servers state api`) through `docker compose exec lb socat`,
so no admin port is published. Container hostname = X-Instance-Id of that replica.
Prints no secrets. Used by student-tests and bench scripts as a library as well.
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
# srv_op_state of HAProxy (src/server.h): 0 stopped, 1 starting, 2 running, 3 stopping.
OP_STATE = {0: "DOWN", 1: "STARTING", 2: "UP", 3: "STOPPING"}
ADMIN_RMAINT = 0x20 # slot without a resolved address (no running container behind it)
def run(*args: str, timeout: float = 60) -> str:
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding="utf-8",
timeout=timeout)
if result.returncode != 0:
raise RuntimeError(f"{' '.join(args[:3])}... failed: {result.stderr.strip()[:300]}")
return result.stdout
def replicas(all_states: bool = True) -> list[dict]:
"""API containers of this compose project (running and stopped)."""
ids = run("docker", "compose", "ps", "-q", *(["-a"] if all_states else []), "api").split()
if not ids:
return []
data = json.loads(run("docker", "inspect", *ids))
result = []
for item in data:
networks = item["NetworkSettings"]["Networks"] or {}
result.append({
"id": item["Id"][:12],
"name": item["Name"].lstrip("/"),
"hostname": item["Config"]["Hostname"],
"state": item["State"]["Status"],
"health": (item["State"].get("Health") or {}).get("Status", "-"),
"exit_code": item["State"]["ExitCode"],
"started_at": item["State"]["StartedAt"],
"pid": item["State"]["Pid"],
"ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")),
})
return sorted(result, key=lambda r: r["name"])
def lb_servers() -> list[dict]:
raw = run("docker", "compose", "exec", "-T", "lb", "sh", "-c",
'echo "show servers state api" | socat stdio /var/lib/haproxy/admin.sock')
by_ip = {ip: r for r in replicas() for ip in r["ips"]}
servers = []
for line in raw.splitlines():
parts = line.split()
if len(parts) < 7 or not parts[0].isdigit():
continue
name, addr, op, admin = parts[3], parts[4], int(parts[5]), int(parts[6])
if admin & ADMIN_RMAINT or addr == "-":
state = "MAINT"
else:
state = OP_STATE.get(op, str(op))
container = by_ip.get(addr)
servers.append({"slot": name, "addr": addr, "state": state,
"container": container["name"] if container else None,
"instance_id": container["hostname"] if container else None})
return servers
def up_instances() -> set[str]:
return {s["instance_id"] or s["addr"] for s in lb_servers() if s["state"] == "UP"}
def wait_up(count: int, timeout: float = 60) -> float:
"""Seconds until exactly `count` servers are UP (raises TimeoutError otherwise)."""
started = time.monotonic()
while True:
try:
if len(up_instances()) == count:
return time.monotonic() - started
except RuntimeError:
pass # lb restarting
if time.monotonic() - started > timeout:
raise TimeoutError(f"HAProxy does not report {count} UP API replicas after {timeout:.0f} s")
time.sleep(0.25)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
sub = parser.add_subparsers(dest="command", required=True)
sub.add_parser("status")
sub.add_parser("replicas")
wait = sub.add_parser("wait-up")
wait.add_argument("count", type=int)
wait.add_argument("--timeout", type=float, default=60)
args = parser.parse_args()
if args.command == "status":
for s in lb_servers():
if s["state"] != "MAINT" or s["container"]:
print(f"{s['slot']:6} {s['addr']:15} {s['state']:8} {s['container'] or '-':24} X-Instance-Id={s['instance_id'] or '-'}")
print(f"(other slots: MAINT, no container)")
elif args.command == "replicas":
for r in replicas():
print(f"{r['name']:24} X-Instance-Id={r['hostname']} {r['state']:8} health={r['health']:9} "
f"pid={r['pid']} started={r['started_at'][:23]} ip={','.join(r['ips']) or '-'}")
else:
took = wait_up(args.count, args.timeout)
print(f"{args.count} API replica(s) UP in HAProxy after {took:.1f} s: {sorted(up_instances())}")
return 0
if __name__ == "__main__":
sys.exit(main())