Files
mtusi2026lab5/student-tests/test_balancing.py
T
Alexey BodanyukandClaude Opus 5.5 5112be83cc перенос: реализация lab4 без изменений (API, миграции 0001-0004, HAProxy, Valkey, SeaweedFS, тесты)
Скопировано из mtusi2026lab4 (ветка solution/bvt2302-bodanyuk, commit 17c33cd) через
git archive: исходники, миграции, собственные тесты, Dockerfile, compose, конфигурации
infra/, скрипты и bench/. README, COURSE, contracts/, tests/ и отчёт lab4 не копировались.
Makefile и .env.example — версии lab4 (skeleton-часть совпадает с lab5, строки broker
добавляются в следующих коммитах).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-31 15:19:00 +03:00

335 lines
18 KiB
Python

"""Load balancer, replicas and graceful shutdown (lab 3, levels 3 and 4).
Each test that stops, kills, rescales or isolates a replica restores the stand in `finally`
and waits until HAProxy reports every replica UP again (conftest.wait_stand).
"""
from __future__ import annotations
import contextlib
import json
import subprocess
import threading
import time
import uuid
import httpx2 as httpx
import pytest
from conftest import BASE_URL, REPLICAS, Api, compose, lbctl, probe_inside, psql, replica_stopped, run_hook, \
running_replicas, wait_healthy, wait_stand
pytestmark = pytest.mark.stack
def instances_seen(api: Api, token: str, requests: int = 40) -> list[str]:
return [api.call("GET", "/api/v1/users/me", 200, token=token).headers["x-instance-id"] for _ in range(requests)]
# ------------------------------------------------------------------ level 3: replicas, no affinity
def test_one_session_is_served_by_every_real_replica(api):
actor = api.user("rr")
seen = instances_seen(api, actor["token"])
running = {r["hostname"] for r in running_replicas()}
assert len(running) == REPLICAS >= 2
assert set(seen) == running, "X-Instance-Id values must be exactly the hostnames of the running containers"
# Round robin on ONE keep-alive client connection: consecutive requests switch replicas,
# so there is no sticky session hiding split state.
switches = sum(1 for a, b in zip(seen, seen[1:]) if a != b)
assert switches >= len(seen) // 2, seen
assert {s["instance_id"] for s in lbctl.lb_servers() if s["state"] == "UP"} == running
def test_write_on_one_replica_is_read_on_the_other(api):
alice, bob = api.user("rw"), api.user("rw")
chat = api.chat(alice, bob)
for i in range(10):
sent = api.post_message(alice, chat, {"text": f"m{i}"})
read = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"])
assert read.json()["items"][-1] == sent.json()
logout = api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
for _ in range(6): # the revocation is visible on every replica at once (shared Valkey)
r = api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
assert r.headers["x-instance-id"]
assert logout.headers["x-instance-id"]
# ------------------------------------------------------------------ readiness drives the LB
def hba_file() -> str:
return psql("SHOW hba_file")
@contextlib.contextmanager
def isolated_from_postgres(replica: dict):
"""Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its
connections), while it keeps running and answering /health/live. Always undone."""
ip = replica["networks"]["data"] # PostgreSQL sees the replica's address on the data network
path = hba_file()
marker = "lab3-test-isolate"
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}")
try:
psql("SELECT pg_reload_conf()")
psql(f"SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE client_addr = '{ip}'")
yield replica
finally:
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}")
psql("SELECT pg_reload_conf()")
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: no restart of the replica needed
wait_stand()
@contextlib.contextmanager
def disconnected_from_data_network(replica: dict):
"""`docker network disconnect <project>_data <api>`: the replica keeps its edge link to the
LB but loses PostgreSQL AND Valkey. Always reconnected."""
network = next(n for n in json.loads(subprocess.run(
["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", replica["id"]],
capture_output=True, text=True, check=True).stdout) if n.endswith("_data"))
subprocess.run(["docker", "network", "disconnect", network, replica["id"]], check=True, capture_output=True)
try:
yield
finally:
subprocess.run(["docker", "network", "connect", network, replica["id"]], check=True, capture_output=True)
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: dead sockets are dropped (tcp_user_timeout)
wait_stand()
@pytest.mark.parametrize("how", ["pg_hba", "network"])
@pytest.mark.parametrize("position", range(REPLICAS))
def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position, how):
"""The reviewers' scenarios: one API loses PostgreSQL (pg_hba reject) or its whole data
network (docker network disconnect) but keeps running. /health/live 200, /health/ready
503 -> HAProxy must take it out; no business request may reach it."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
actor = api.user("ready")
isolate = isolated_from_postgres if how == "pg_hba" else disconnected_from_data_network
with isolate(replica):
started = time.monotonic()
while replica["hostname"] in lbctl.up_instances():
assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s"
time.sleep(0.1)
detected = time.monotonic() - started
for _ in range(3): # the process stays responsive (event loop never blocked) ...
started_probe = time.monotonic()
assert probe_inside(replica, "/health/live") == 200
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready
assert time.monotonic() - started_probe < 4.5, "replica froze while its dependencies were unreachable"
seen = instances_seen(api, actor["token"], 40)
assert replica["hostname"] not in seen and len(seen) == 40
for i in range(20): # writes keep working, only through the ready replicas
assert api.register(f"r{i}")["id"]
print(f"{replica['name']} ({how}): unready replica out of rotation after {detected:.2f} s")
# ------------------------------------------------------------------ stopping replicas under traffic
class Traffic:
"""Background writers/readers through the LB; uncertain POSTs are retried with the SAME key."""
def __init__(self, api: Api, workers: int = 4):
self.alice, self.bob = api.user("load"), api.user("load")
self.chat = api.chat(self.alice, self.bob)
self.workers = workers
self.stop = threading.Event()
self.lock = threading.Lock()
self.events: list[tuple[float, str, int, str | None]] = [] # (time, kind, status, instance)
self.confirmed: list[dict] = []
self.threads = [threading.Thread(target=self.run, args=(i,), daemon=True) for i in range(workers)]
def record(self, kind: str, status: int, instance: str | None) -> None:
with self.lock:
self.events.append((time.monotonic(), kind, status, instance))
def run(self, index: int) -> None:
client = httpx.Client(base_url=BASE_URL, timeout=3)
auth = {"Authorization": "Bearer " + self.alice["token"]}
n = 0
key, text = uuid.uuid4().hex, f"w{index}-0"
while not self.stop.is_set():
try:
r = client.post(f"/api/v1/chats/{self.chat['id']}/messages", json={"text": text},
headers={**auth, "Idempotency-Key": key})
self.record("POST", r.status_code, r.headers.get("x-instance-id"))
if r.status_code == 201:
with self.lock:
self.confirmed.append(r.json())
n += 1
key, text = uuid.uuid4().hex, f"w{index}-{n}"
# any other outcome: keep key and text, retry the same request
except httpx.HTTPError:
self.record("POST", 0, None)
try:
r = client.get(f"/api/v1/chats/{self.chat['id']}/messages?limit=5", headers=auth)
self.record("GET", r.status_code, r.headers.get("x-instance-id"))
except httpx.HTTPError:
self.record("GET", 0, None)
time.sleep(0.02)
client.close()
def __enter__(self) -> "Traffic":
for t in self.threads:
t.start()
time.sleep(1.0)
return self
def __exit__(self, *exc) -> None:
self.stop.set()
for t in self.threads:
t.join(timeout=10)
def failures(self, since: float = 0.0) -> list[tuple]:
return [e for e in self.events if e[0] >= since and e[2] not in (200, 201)]
def check_history(self, api: Api) -> None:
history = api.history(self.bob, self.chat, 100)[0]
ids, texts = [m["id"] for m in history], [m["text"] for m in history]
assert len(ids) == len(set(ids)) and len(texts) == len(set(texts)), "duplicate after retry"
assert {m["id"] for m in self.confirmed} <= set(ids), "a confirmed message was lost"
@pytest.mark.parametrize("position", range(REPLICAS))
def test_graceful_stop_of_each_replica_loses_no_request(api, position):
"""SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit 0: not one failed request."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
with Traffic(api) as traffic:
started = time.monotonic()
with replica_stopped(replica, "stop"):
stopped_after = time.monotonic() - started
time.sleep(3)
after = [e for e in traffic.events if e[0] > time.monotonic() - 2]
assert after and replica["hostname"] not in {e[3] for e in after}
time.sleep(1)
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
check=True).stdout)[0]["State"]
assert traffic.failures() == [], traffic.failures()[:10]
assert stopped_after < 15, "graceful stop must finish before stop_grace_period (no SIGKILL)"
assert state["Status"] == "running" # restored by replica_stopped
traffic.check_history(api)
print(f"{replica['name']}: stop took {stopped_after:.1f} s, {len(traffic.events)} requests, 0 failed")
@pytest.mark.parametrize("position", range(REPLICAS))
def test_killed_replica_bounded_errors_recovery_and_no_duplicates(api, position):
"""SIGKILL: requests in flight on that replica may fail (counted), new ones succeed within
10 s, the killed replica gets nothing after detection, retries do not duplicate."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
with Traffic(api) as traffic:
killed_at = time.monotonic()
with replica_stopped(replica, "kill"):
time.sleep(4)
failures = traffic.failures(killed_at)
last_failure = max((e[0] for e in failures), default=killed_at) - killed_at
post_ok = [e for e in traffic.events if e[0] > killed_at and e[1] == "POST" and e[2] == 201]
assert post_ok and post_ok[0][0] - killed_at <= 10, "writes did not recover within 10 s"
late = [e for e in traffic.events if e[0] > killed_at + 3.5]
assert late and replica["hostname"] not in {e[3] for e in late}
assert last_failure <= 10
traffic.check_history(api)
print(f"{replica['name']}: killed; {len(failures)} failed of {len(traffic.events)}, last failure +{last_failure:.2f} s")
def test_graceful_shutdown_order_in_logs(api):
"""Logs prove the order: drain_started -> (HAProxy marks DOWN) -> shutdown_requested -> exit 0."""
replica = sorted(running_replicas(), key=lambda r: r["name"])[-1]
since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 1))
with replica_stopped(replica, "stop"):
logs = subprocess.run(["docker", "logs", "--since", since, replica["id"]], capture_output=True,
text=True, encoding="utf-8").stdout
lb_logs = compose("logs", "--no-log-prefix", "--since", since, "lb").stdout
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
check=True).stdout)[0]["State"]
events = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
names = [e["event"] for e in events]
assert names.index("drain_started") < names.index("shutdown_requested") < names.index("stopped")
t = {e["event"]: e["ts"] for e in events}
assert t["drain_started"] < t["shutdown_requested"]
assert state["ExitCode"] == 0 and not state["OOMKilled"]
assert "is DOWN" in lb_logs and ("503" in lb_logs or "Connection refused" in lb_logs)
# ------------------------------------------------------------------ discovery: restart and scaling
def test_full_restart_lb_rediscovers_all_replicas(api):
actor = api.user("restart")
before = {r["hostname"]: r["started_at"] for r in running_replicas()}
run_hook("restart.sh")
wait_healthy("postgres", "valkey", "api")
wait_stand()
after = {r["hostname"]: r["started_at"] for r in running_replicas()}
assert set(after) == set(before) and all(after[h] != before[h] for h in after), "every replica restarted"
assert set(instances_seen(api, actor["token"])) == set(after) # same token, both replicas, no LB restart needed
def test_scale_up_and_down_is_picked_up_without_errors(api):
with Traffic(api, workers=2) as traffic:
try:
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS + 1}", "api",
timeout=180)
lbctl.wait_up(REPLICAS + 1, timeout=30)
actor = api.user("scale")
assert len(set(instances_seen(api, actor["token"], 30))) == REPLICAS + 1
finally:
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS}", "api",
timeout=180)
wait_stand()
time.sleep(1)
assert traffic.failures() == [], traffic.failures()[:10]
traffic.check_history(api)
# ------------------------------------------------------------------ level 4: deployment policy
def test_compose_policy_limits_ports_secrets_images():
config = json.loads(compose("config", "--format", "json").stdout)
services = config["services"]
# The reconciler (profile "tools") is started on demand and not part of the running stand.
assert set(services) == {"postgres", "valkey", "s3", "migrate", "s3-init", "api", "lb"}
for name, svc in services.items():
limits = svc.get("deploy", {}).get("resources", {}).get("limits", {})
assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits"
assert "container_name" not in svc, f"{name}: container_name prevents scaling"
if name in ("api", "migrate"):
assert "build" in svc # built from this repository's source
elif name == "s3-init":
assert svc["image"] == services["api"]["image"] # the same locally built image
else:
assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest"
assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports")
published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")}
assert set(published) == {"lb"} and all(p["host_ip"] in ("127.0.0.1", "::1") for p in published["lb"])
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()}
s3_keys = ['s3_admin_access_key_id', 's3_admin_secret_access_key', 's3_api_access_key_id', 's3_api_secret_access_key', 's3_reconciler_access_key_id', 's3_reconciler_secret_access_key']
assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], "s3": s3_keys,
"migrate": ["database_url"], "s3-init": s3_keys, "lb": [],
# the API: its own S3 key pair only (no admin, no reconciler keys)
"api": ["cache_url", "database_url", "s3_api_access_key_id", "s3_api_secret_access_key"]}
volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"]
for name, svc in services.items()}
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "s3": ["s3data"], "migrate": [],
"s3-init": [], "api": [], "lb": []}
networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()}
assert networks == {"postgres": ["data"], "valkey": ["data"], "s3": ["storage"], "migrate": ["data"],
"s3-init": ["storage"], "api": ["data", "edge", "storage"],
"lb": ["edge"]}, networks # the LB cannot reach the stores at all
assert config["networks"]["data"].get("internal") is True
assert config["networks"]["storage"].get("internal") is True
def test_running_containers_have_limits_and_no_bypass_ports():
ids = compose("ps", "-q").stdout.split()
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True,
check=True).stdout):
name = item["Name"]
assert item["HostConfig"]["NanoCpus"] > 0 and item["HostConfig"]["Memory"] > 0, name
bindings = item["HostConfig"]["PortBindings"] or {}
if "-lb-" in name:
assert all(b["HostIp"] in ("127.0.0.1", "::1") for v in bindings.values() for b in v) # loopback only
else:
assert not bindings, f"{name} publishes ports in bypass of the LB"
def test_migrations_ran_once_by_the_migrate_service():
applied = psql("SELECT version FROM schema_migrations ORDER BY version").split()
assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function", "0004_attachments"]
migrate = compose("ps", "-a", "--format", "json", "migrate").stdout
rows = [json.loads(line) for line in migrate.splitlines() if line.strip()]
assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows)
api_logs = compose("logs", "--no-log-prefix", "api").stdout
assert "migrate:" not in api_logs and "schema_migrations" not in api_logs # replicas never migrate