The stack test for a stopped PostgreSQL asked for liveness 200 through the balancer, which only held while the balancer had not noticed the outage yet. Now the protected endpoints and readiness through the LB must answer 503 unavailable within 2 s, liveness is checked inside a replica with docker exec, the balancer must report zero UP servers within 2 s and answer liveness itself with 503 in the contract error format, the request id echoed and no X-Instance-Id. After PostgreSQL starts again every replica must be back in rotation without a restart. Stand helpers read the number of UP servers from the HAProxy stats, excluding the backend summary row, and reach a replica by container id, which survives the renumbering after a rolling update. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
133 lines
4.3 KiB
Python
133 lines
4.3 KiB
Python
import asyncio
|
||
import csv
|
||
import os
|
||
import subprocess
|
||
import time
|
||
from base64 import b64encode
|
||
from collections.abc import Awaitable, Callable
|
||
from pathlib import Path
|
||
from typing import Any
|
||
from uuid import uuid4
|
||
|
||
import httpx
|
||
from httpx import AsyncClient, Response, TransportError
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080")
|
||
PASSWORD = "Пароль-для-стенда-01"
|
||
BASIC = 'Basic realm="messenger", charset="UTF-8"'
|
||
BEARER = 'Bearer realm="messenger"'
|
||
VALKEY_AUTH = 'REDISCLI_AUTH="$(sed -n "s/^requirepass //p" /run/secrets/valkey.conf)"'
|
||
|
||
|
||
def compose(*args: str, stdin: str | None = None) -> str:
|
||
done = subprocess.run(
|
||
["docker", "compose", *args],
|
||
cwd=ROOT,
|
||
check=True,
|
||
capture_output=True,
|
||
text=True,
|
||
input=stdin,
|
||
)
|
||
return done.stdout.strip()
|
||
|
||
|
||
def psql(sql: str, **variables: str) -> str:
|
||
assigned = [item for name, value in variables.items() for item in ("-v", f"{name}={value}")]
|
||
command = ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-tA"]
|
||
return compose(*command, *assigned, stdin=sql)
|
||
|
||
|
||
def valkey(command: str) -> str:
|
||
return compose("exec", "-T", "valkey", "sh", "-c", f"{VALKEY_AUTH} valkey-cli {command}")
|
||
|
||
|
||
def started_at(service: str) -> str:
|
||
return subprocess.run(
|
||
[
|
||
"docker",
|
||
"inspect",
|
||
"--format",
|
||
"{{.State.StartedAt}}",
|
||
*compose("ps", "-q", service).split(),
|
||
],
|
||
check=True,
|
||
capture_output=True,
|
||
text=True,
|
||
).stdout.strip()
|
||
|
||
|
||
def inside_replica(path: str) -> int:
|
||
replica = compose("ps", "-q", "api").split()[0]
|
||
probe = f"import urllib.request as u;print(u.urlopen('http://127.0.0.1:8080{path}').status)"
|
||
done = subprocess.run(
|
||
["docker", "exec", replica, "python", "-c", probe],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
return int(done.stdout.strip() or 0)
|
||
|
||
|
||
def lb_up() -> int:
|
||
stats = httpx.get(f"http://{compose('port', 'lb', '8404')}/stats;csv", timeout=2).text
|
||
rows = csv.DictReader(stats.lstrip("# ").splitlines())
|
||
servers = (r for r in rows if r["pxname"] == "api" and r["svname"] != "BACKEND")
|
||
return sum(1 for r in servers if r["status"].startswith("UP"))
|
||
|
||
|
||
def basic(username: str, password: str = PASSWORD) -> dict[str, str]:
|
||
return {"Authorization": "Basic " + b64encode(f"{username}:{password}".encode()).decode()}
|
||
|
||
|
||
def bearer(token: str) -> dict[str, str]:
|
||
return {"Authorization": f"Bearer {token}"}
|
||
|
||
|
||
def new_user(label: str) -> dict[str, str]:
|
||
return {
|
||
"username": f"u_{label}_{uuid4().hex[:12]}",
|
||
"display_name": f"Стенд {label}",
|
||
"password": PASSWORD,
|
||
}
|
||
|
||
|
||
async def enroll(http: AsyncClient, label: str) -> tuple[dict[str, Any], str]:
|
||
body = new_user(label)
|
||
created = await http.post("/api/v1/users", json=body)
|
||
assert created.status_code == 201, created.text
|
||
session = await http.post("/api/v1/auth/sessions", headers=basic(body["username"]))
|
||
assert session.status_code == 201, session.text
|
||
return created.json(), session.json()["token"]
|
||
|
||
|
||
async def timed(call: Awaitable[Response]) -> tuple[Response, float]:
|
||
started = time.perf_counter()
|
||
response = await call
|
||
return response, time.perf_counter() - started
|
||
|
||
|
||
async def wait_ready(http: AsyncClient, seconds: float = 60) -> float:
|
||
started = time.perf_counter()
|
||
while time.perf_counter() - started < seconds:
|
||
try:
|
||
if (await http.get("/health/ready")).status_code == 200:
|
||
return time.perf_counter() - started
|
||
except TransportError:
|
||
pass
|
||
await asyncio.sleep(0.25)
|
||
raise AssertionError("стенд не вернулся в готовность")
|
||
|
||
|
||
async def waited(condition: Callable[[], bool], seconds: float = 30) -> float:
|
||
started = time.perf_counter()
|
||
while time.perf_counter() - started < seconds:
|
||
if condition():
|
||
return time.perf_counter() - started
|
||
await asyncio.sleep(0.05)
|
||
raise AssertionError("условие не выполнилось за отведённое время")
|
||
|
||
|
||
def report(event: str, response: Response, elapsed: float) -> None:
|
||
print(f"{event}: {response.status_code} за {elapsed:.3f} с")
|