forked from mtusi2026/mtusi2026lab3
Гипотеза из профиля (py-spy, 2 x 1.0 CPU, 240 req/s): ~57 % CPU реплики уходит в psycopg — накладные расходы на каждый оператор; на одно сообщение ~11 обращений к PostgreSQL в двух транзакциях (проверка членства, BEGIN, ключ, часы чата, сообщение, COMMIT, проверки пула). - migrations/0003: функция post_message — та же логика и тот же SQL, что в многооператорном пути; вызывается в autocommit, поэтому вызов и есть транзакция (ключ, часы чата и сообщение — всё или ничего). - пул: autocommit-соединения; многооператорная работа по-прежнему в явной conn.transaction(); ping — один оператор без транзакции. - MESSAGE_WRITE_PATH=function (по умолчанию) | multi (прежний путь) — A/B на одном образе. Порядок ответов одинаков: постороннему всегда 404, даже при неверном ключе/теле. - Оба режима проходят одни и те же тесты идемпотентности и контракта. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
326 lines
17 KiB
Python
326 lines
17 KiB
Python
"""Load balancer, replicas and graceful shutdown (lab 3, levels 3 and 4).
|
|
|
|
Each test that stops, kills, rescales or isolates a replica restores the stand in `finally`
|
|
and waits until HAProxy reports every replica UP again (conftest.wait_stand).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import json
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
import httpx2 as httpx
|
|
import pytest
|
|
|
|
from conftest import BASE_URL, REPLICAS, Api, compose, lbctl, probe_inside, psql, replica_stopped, run_hook, \
|
|
running_replicas, wait_healthy, wait_stand
|
|
|
|
pytestmark = pytest.mark.stack
|
|
|
|
|
|
def instances_seen(api: Api, token: str, requests: int = 40) -> list[str]:
|
|
return [api.call("GET", "/api/v1/users/me", 200, token=token).headers["x-instance-id"] for _ in range(requests)]
|
|
|
|
|
|
# ------------------------------------------------------------------ level 3: replicas, no affinity
|
|
def test_one_session_is_served_by_every_real_replica(api):
|
|
actor = api.user("rr")
|
|
seen = instances_seen(api, actor["token"])
|
|
running = {r["hostname"] for r in running_replicas()}
|
|
assert len(running) == REPLICAS >= 2
|
|
assert set(seen) == running, "X-Instance-Id values must be exactly the hostnames of the running containers"
|
|
# Round robin on ONE keep-alive client connection: consecutive requests switch replicas,
|
|
# so there is no sticky session hiding split state.
|
|
switches = sum(1 for a, b in zip(seen, seen[1:]) if a != b)
|
|
assert switches >= len(seen) // 2, seen
|
|
assert {s["instance_id"] for s in lbctl.lb_servers() if s["state"] == "UP"} == running
|
|
|
|
|
|
def test_write_on_one_replica_is_read_on_the_other(api):
|
|
alice, bob = api.user("rw"), api.user("rw")
|
|
chat = api.chat(alice, bob)
|
|
for i in range(10):
|
|
sent = api.post_message(alice, chat, {"text": f"m{i}"})
|
|
read = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"])
|
|
assert read.json()["items"][-1] == sent.json()
|
|
logout = api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
|
|
for _ in range(6): # the revocation is visible on every replica at once (shared Valkey)
|
|
r = api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
|
|
assert r.headers["x-instance-id"]
|
|
assert logout.headers["x-instance-id"]
|
|
|
|
|
|
# ------------------------------------------------------------------ readiness drives the LB
|
|
def hba_file() -> str:
|
|
return psql("SHOW hba_file")
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def isolated_from_postgres(replica: dict):
|
|
"""Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its
|
|
connections), while it keeps running and answering /health/live. Always undone."""
|
|
ip = replica["networks"]["data"] # PostgreSQL sees the replica's address on the data network
|
|
path = hba_file()
|
|
marker = "lab3-test-isolate"
|
|
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}")
|
|
try:
|
|
psql("SELECT pg_reload_conf()")
|
|
psql(f"SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE client_addr = '{ip}'")
|
|
yield replica
|
|
finally:
|
|
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}")
|
|
psql("SELECT pg_reload_conf()")
|
|
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: no restart of the replica needed
|
|
wait_stand()
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def disconnected_from_data_network(replica: dict):
|
|
"""`docker network disconnect <project>_data <api>`: the replica keeps its edge link to the
|
|
LB but loses PostgreSQL AND Valkey. Always reconnected."""
|
|
network = next(n for n in json.loads(subprocess.run(
|
|
["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", replica["id"]],
|
|
capture_output=True, text=True, check=True).stdout) if n.endswith("_data"))
|
|
subprocess.run(["docker", "network", "disconnect", network, replica["id"]], check=True, capture_output=True)
|
|
try:
|
|
yield
|
|
finally:
|
|
subprocess.run(["docker", "network", "connect", network, replica["id"]], check=True, capture_output=True)
|
|
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: dead sockets are dropped (tcp_user_timeout)
|
|
wait_stand()
|
|
|
|
|
|
@pytest.mark.parametrize("how", ["pg_hba", "network"])
|
|
@pytest.mark.parametrize("position", range(REPLICAS))
|
|
def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position, how):
|
|
"""The reviewers' scenarios: one API loses PostgreSQL (pg_hba reject) or its whole data
|
|
network (docker network disconnect) but keeps running. /health/live 200, /health/ready
|
|
503 -> HAProxy must take it out; no business request may reach it."""
|
|
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
|
|
actor = api.user("ready")
|
|
isolate = isolated_from_postgres if how == "pg_hba" else disconnected_from_data_network
|
|
with isolate(replica):
|
|
started = time.monotonic()
|
|
while replica["hostname"] in lbctl.up_instances():
|
|
assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s"
|
|
time.sleep(0.1)
|
|
detected = time.monotonic() - started
|
|
for _ in range(3): # the process stays responsive (event loop never blocked) ...
|
|
started_probe = time.monotonic()
|
|
assert probe_inside(replica, "/health/live") == 200
|
|
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready
|
|
assert time.monotonic() - started_probe < 4.5, "replica froze while its dependencies were unreachable"
|
|
seen = instances_seen(api, actor["token"], 40)
|
|
assert replica["hostname"] not in seen and len(seen) == 40
|
|
for i in range(20): # writes keep working, only through the ready replicas
|
|
assert api.register(f"r{i}")["id"]
|
|
print(f"{replica['name']} ({how}): unready replica out of rotation after {detected:.2f} s")
|
|
|
|
|
|
# ------------------------------------------------------------------ stopping replicas under traffic
|
|
class Traffic:
|
|
"""Background writers/readers through the LB; uncertain POSTs are retried with the SAME key."""
|
|
|
|
def __init__(self, api: Api, workers: int = 4):
|
|
self.alice, self.bob = api.user("load"), api.user("load")
|
|
self.chat = api.chat(self.alice, self.bob)
|
|
self.workers = workers
|
|
self.stop = threading.Event()
|
|
self.lock = threading.Lock()
|
|
self.events: list[tuple[float, str, int, str | None]] = [] # (time, kind, status, instance)
|
|
self.confirmed: list[dict] = []
|
|
self.threads = [threading.Thread(target=self.run, args=(i,), daemon=True) for i in range(workers)]
|
|
|
|
def record(self, kind: str, status: int, instance: str | None) -> None:
|
|
with self.lock:
|
|
self.events.append((time.monotonic(), kind, status, instance))
|
|
|
|
def run(self, index: int) -> None:
|
|
client = httpx.Client(base_url=BASE_URL, timeout=3)
|
|
auth = {"Authorization": "Bearer " + self.alice["token"]}
|
|
n = 0
|
|
key, text = uuid.uuid4().hex, f"w{index}-0"
|
|
while not self.stop.is_set():
|
|
try:
|
|
r = client.post(f"/api/v1/chats/{self.chat['id']}/messages", json={"text": text},
|
|
headers={**auth, "Idempotency-Key": key})
|
|
self.record("POST", r.status_code, r.headers.get("x-instance-id"))
|
|
if r.status_code == 201:
|
|
with self.lock:
|
|
self.confirmed.append(r.json())
|
|
n += 1
|
|
key, text = uuid.uuid4().hex, f"w{index}-{n}"
|
|
# any other outcome: keep key and text, retry the same request
|
|
except httpx.HTTPError:
|
|
self.record("POST", 0, None)
|
|
try:
|
|
r = client.get(f"/api/v1/chats/{self.chat['id']}/messages?limit=5", headers=auth)
|
|
self.record("GET", r.status_code, r.headers.get("x-instance-id"))
|
|
except httpx.HTTPError:
|
|
self.record("GET", 0, None)
|
|
time.sleep(0.02)
|
|
client.close()
|
|
|
|
def __enter__(self) -> "Traffic":
|
|
for t in self.threads:
|
|
t.start()
|
|
time.sleep(1.0)
|
|
return self
|
|
|
|
def __exit__(self, *exc) -> None:
|
|
self.stop.set()
|
|
for t in self.threads:
|
|
t.join(timeout=10)
|
|
|
|
def failures(self, since: float = 0.0) -> list[tuple]:
|
|
return [e for e in self.events if e[0] >= since and e[2] not in (200, 201)]
|
|
|
|
def check_history(self, api: Api) -> None:
|
|
history = api.history(self.bob, self.chat, 100)[0]
|
|
ids, texts = [m["id"] for m in history], [m["text"] for m in history]
|
|
assert len(ids) == len(set(ids)) and len(texts) == len(set(texts)), "duplicate after retry"
|
|
assert {m["id"] for m in self.confirmed} <= set(ids), "a confirmed message was lost"
|
|
|
|
|
|
@pytest.mark.parametrize("position", range(REPLICAS))
|
|
def test_graceful_stop_of_each_replica_loses_no_request(api, position):
|
|
"""SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit 0: not one failed request."""
|
|
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
|
|
with Traffic(api) as traffic:
|
|
started = time.monotonic()
|
|
with replica_stopped(replica, "stop"):
|
|
stopped_after = time.monotonic() - started
|
|
time.sleep(3)
|
|
after = [e for e in traffic.events if e[0] > time.monotonic() - 2]
|
|
assert after and replica["hostname"] not in {e[3] for e in after}
|
|
time.sleep(1)
|
|
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
|
|
check=True).stdout)[0]["State"]
|
|
assert traffic.failures() == [], traffic.failures()[:10]
|
|
assert stopped_after < 15, "graceful stop must finish before stop_grace_period (no SIGKILL)"
|
|
assert state["Status"] == "running" # restored by replica_stopped
|
|
traffic.check_history(api)
|
|
print(f"{replica['name']}: stop took {stopped_after:.1f} s, {len(traffic.events)} requests, 0 failed")
|
|
|
|
|
|
@pytest.mark.parametrize("position", range(REPLICAS))
|
|
def test_killed_replica_bounded_errors_recovery_and_no_duplicates(api, position):
|
|
"""SIGKILL: requests in flight on that replica may fail (counted), new ones succeed within
|
|
10 s, the killed replica gets nothing after detection, retries do not duplicate."""
|
|
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
|
|
with Traffic(api) as traffic:
|
|
killed_at = time.monotonic()
|
|
with replica_stopped(replica, "kill"):
|
|
time.sleep(4)
|
|
failures = traffic.failures(killed_at)
|
|
last_failure = max((e[0] for e in failures), default=killed_at) - killed_at
|
|
post_ok = [e for e in traffic.events if e[0] > killed_at and e[1] == "POST" and e[2] == 201]
|
|
assert post_ok and post_ok[0][0] - killed_at <= 10, "writes did not recover within 10 s"
|
|
late = [e for e in traffic.events if e[0] > killed_at + 3.5]
|
|
assert late and replica["hostname"] not in {e[3] for e in late}
|
|
assert last_failure <= 10
|
|
traffic.check_history(api)
|
|
print(f"{replica['name']}: killed; {len(failures)} failed of {len(traffic.events)}, last failure +{last_failure:.2f} s")
|
|
|
|
|
|
def test_graceful_shutdown_order_in_logs(api):
|
|
"""Logs prove the order: drain_started -> (HAProxy marks DOWN) -> shutdown_requested -> exit 0."""
|
|
replica = sorted(running_replicas(), key=lambda r: r["name"])[-1]
|
|
since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 1))
|
|
with replica_stopped(replica, "stop"):
|
|
logs = subprocess.run(["docker", "logs", "--since", since, replica["id"]], capture_output=True,
|
|
text=True, encoding="utf-8").stdout
|
|
lb_logs = compose("logs", "--no-log-prefix", "--since", since, "lb").stdout
|
|
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
|
|
check=True).stdout)[0]["State"]
|
|
events = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
|
|
names = [e["event"] for e in events]
|
|
assert names.index("drain_started") < names.index("shutdown_requested") < names.index("stopped")
|
|
t = {e["event"]: e["ts"] for e in events}
|
|
assert t["drain_started"] < t["shutdown_requested"]
|
|
assert state["ExitCode"] == 0 and not state["OOMKilled"]
|
|
assert "is DOWN" in lb_logs and ("503" in lb_logs or "Connection refused" in lb_logs)
|
|
|
|
|
|
# ------------------------------------------------------------------ discovery: restart and scaling
|
|
def test_full_restart_lb_rediscovers_all_replicas(api):
|
|
actor = api.user("restart")
|
|
before = {r["hostname"]: r["started_at"] for r in running_replicas()}
|
|
run_hook("restart.sh")
|
|
wait_healthy("postgres", "valkey", "api")
|
|
wait_stand()
|
|
after = {r["hostname"]: r["started_at"] for r in running_replicas()}
|
|
assert set(after) == set(before) and all(after[h] != before[h] for h in after), "every replica restarted"
|
|
assert set(instances_seen(api, actor["token"])) == set(after) # same token, both replicas, no LB restart needed
|
|
|
|
|
|
def test_scale_up_and_down_is_picked_up_without_errors(api):
|
|
with Traffic(api, workers=2) as traffic:
|
|
try:
|
|
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS + 1}", "api",
|
|
timeout=180)
|
|
lbctl.wait_up(REPLICAS + 1, timeout=30)
|
|
actor = api.user("scale")
|
|
assert len(set(instances_seen(api, actor["token"], 30))) == REPLICAS + 1
|
|
finally:
|
|
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS}", "api",
|
|
timeout=180)
|
|
wait_stand()
|
|
time.sleep(1)
|
|
assert traffic.failures() == [], traffic.failures()[:10]
|
|
traffic.check_history(api)
|
|
|
|
|
|
# ------------------------------------------------------------------ level 4: deployment policy
|
|
def test_compose_policy_limits_ports_secrets_images():
|
|
config = json.loads(compose("config", "--format", "json").stdout)
|
|
services = config["services"]
|
|
assert set(services) == {"postgres", "valkey", "migrate", "api", "lb"}
|
|
for name, svc in services.items():
|
|
limits = svc.get("deploy", {}).get("resources", {}).get("limits", {})
|
|
assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits"
|
|
assert "container_name" not in svc, f"{name}: container_name prevents scaling"
|
|
if name in ("api", "migrate"):
|
|
assert "build" in svc # built from this repository's source
|
|
else:
|
|
assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest"
|
|
assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports")
|
|
published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")}
|
|
assert set(published) == {"lb"} and all(p["host_ip"] == "127.0.0.1" for p in published["lb"])
|
|
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()}
|
|
assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"],
|
|
"migrate": ["database_url"], "api": ["cache_url", "database_url"], "lb": []}
|
|
volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"]
|
|
for name, svc in services.items()}
|
|
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []}
|
|
networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()}
|
|
assert networks == {"postgres": ["data"], "valkey": ["data"], "migrate": ["data"], "api": ["data", "edge"],
|
|
"lb": ["edge"]}, networks # the LB cannot reach the stores at all
|
|
assert config["networks"]["data"].get("internal") is True
|
|
|
|
|
|
def test_running_containers_have_limits_and_no_bypass_ports():
|
|
ids = compose("ps", "-q").stdout.split()
|
|
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True,
|
|
check=True).stdout):
|
|
name = item["Name"]
|
|
assert item["HostConfig"]["NanoCpus"] > 0 and item["HostConfig"]["Memory"] > 0, name
|
|
bindings = item["HostConfig"]["PortBindings"] or {}
|
|
if "-lb-" in name:
|
|
assert all(b["HostIp"] == "127.0.0.1" for v in bindings.values() for b in v)
|
|
else:
|
|
assert not bindings, f"{name} publishes ports in bypass of the LB"
|
|
|
|
|
|
def test_migrations_ran_once_by_the_migrate_service():
|
|
applied = psql("SELECT version FROM schema_migrations ORDER BY version").split()
|
|
assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function"]
|
|
migrate = compose("ps", "-a", "--format", "json", "migrate").stdout
|
|
rows = [json.loads(line) for line in migrate.splitlines() if line.strip()]
|
|
assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows)
|
|
api_logs = compose("logs", "--no-log-prefix", "api").stdout
|
|
assert "migrate:" not in api_logs and "schema_migrations" not in api_logs # replicas never migrate
|