"""Load balancer, replicas and graceful shutdown (lab 3, levels 3 and 4). Each test that stops, kills, rescales or isolates a replica restores the stand in `finally` and waits until HAProxy reports every replica UP again (conftest.wait_stand). """ from __future__ import annotations import contextlib import json import subprocess import threading import time import uuid import httpx2 as httpx import pytest from conftest import BASE_URL, REPLICAS, Api, compose, lbctl, probe_inside, psql, replica_stopped, run_hook, \ running_replicas, wait_healthy, wait_stand pytestmark = pytest.mark.stack def instances_seen(api: Api, token: str, requests: int = 40) -> list[str]: return [api.call("GET", "/api/v1/users/me", 200, token=token).headers["x-instance-id"] for _ in range(requests)] # ------------------------------------------------------------------ level 3: replicas, no affinity def test_one_session_is_served_by_every_real_replica(api): actor = api.user("rr") seen = instances_seen(api, actor["token"]) running = {r["hostname"] for r in running_replicas()} assert len(running) == REPLICAS >= 2 assert set(seen) == running, "X-Instance-Id values must be exactly the hostnames of the running containers" # Round robin on ONE keep-alive client connection: consecutive requests switch replicas, # so there is no sticky session hiding split state. switches = sum(1 for a, b in zip(seen, seen[1:]) if a != b) assert switches >= len(seen) // 2, seen assert {s["instance_id"] for s in lbctl.lb_servers() if s["state"] == "UP"} == running def test_write_on_one_replica_is_read_on_the_other(api): alice, bob = api.user("rw"), api.user("rw") chat = api.chat(alice, bob) for i in range(10): sent = api.post_message(alice, chat, {"text": f"m{i}"}) read = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]) assert read.json()["items"][-1] == sent.json() logout = api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) for _ in range(6): # the revocation is visible on every replica at once (shared Valkey) r = api.call("GET", "/api/v1/users/me", 401, token=alice["token"]) assert r.headers["x-instance-id"] assert logout.headers["x-instance-id"] # ------------------------------------------------------------------ readiness drives the LB def hba_file() -> str: return psql("SHOW hba_file") @contextlib.contextmanager def isolated_from_postgres(replica: dict): """Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its connections), while it keeps running and answering /health/live. Always undone.""" ip = replica["networks"]["data"] # PostgreSQL sees the replica's address on the data network path = hba_file() marker = "lab3-test-isolate" compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}") try: psql("SELECT pg_reload_conf()") psql(f"SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE client_addr = '{ip}'") yield replica finally: compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}") psql("SELECT pg_reload_conf()") lbctl.wait_up(REPLICAS, timeout=15) # self-heals: no restart of the replica needed wait_stand() @contextlib.contextmanager def disconnected_from_data_network(replica: dict): """`docker network disconnect _data `: the replica keeps its edge link to the LB but loses PostgreSQL AND Valkey. Always reconnected.""" network = next(n for n in json.loads(subprocess.run( ["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", replica["id"]], capture_output=True, text=True, check=True).stdout) if n.endswith("_data")) subprocess.run(["docker", "network", "disconnect", network, replica["id"]], check=True, capture_output=True) try: yield finally: subprocess.run(["docker", "network", "connect", network, replica["id"]], check=True, capture_output=True) lbctl.wait_up(REPLICAS, timeout=15) # self-heals: dead sockets are dropped (tcp_user_timeout) wait_stand() @pytest.mark.parametrize("how", ["pg_hba", "network"]) @pytest.mark.parametrize("position", range(REPLICAS)) def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position, how): """The reviewers' scenarios: one API loses PostgreSQL (pg_hba reject) or its whole data network (docker network disconnect) but keeps running. /health/live 200, /health/ready 503 -> HAProxy must take it out; no business request may reach it.""" replica = sorted(running_replicas(), key=lambda r: r["name"])[position] actor = api.user("ready") isolate = isolated_from_postgres if how == "pg_hba" else disconnected_from_data_network with isolate(replica): started = time.monotonic() while replica["hostname"] in lbctl.up_instances(): assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s" time.sleep(0.1) detected = time.monotonic() - started for _ in range(3): # the process stays responsive (event loop never blocked) ... started_probe = time.monotonic() assert probe_inside(replica, "/health/live") == 200 assert probe_inside(replica, "/health/ready") == 503 # ... but not ready assert time.monotonic() - started_probe < 4.5, "replica froze while its dependencies were unreachable" seen = instances_seen(api, actor["token"], 40) assert replica["hostname"] not in seen and len(seen) == 40 for i in range(20): # writes keep working, only through the ready replicas assert api.register(f"r{i}")["id"] print(f"{replica['name']} ({how}): unready replica out of rotation after {detected:.2f} s") # ------------------------------------------------------------------ stopping replicas under traffic class Traffic: """Background writers/readers through the LB; uncertain POSTs are retried with the SAME key.""" def __init__(self, api: Api, workers: int = 4): self.alice, self.bob = api.user("load"), api.user("load") self.chat = api.chat(self.alice, self.bob) self.workers = workers self.stop = threading.Event() self.lock = threading.Lock() self.events: list[tuple[float, str, int, str | None]] = [] # (time, kind, status, instance) self.confirmed: list[dict] = [] self.threads = [threading.Thread(target=self.run, args=(i,), daemon=True) for i in range(workers)] def record(self, kind: str, status: int, instance: str | None) -> None: with self.lock: self.events.append((time.monotonic(), kind, status, instance)) def run(self, index: int) -> None: client = httpx.Client(base_url=BASE_URL, timeout=3) auth = {"Authorization": "Bearer " + self.alice["token"]} n = 0 key, text = uuid.uuid4().hex, f"w{index}-0" while not self.stop.is_set(): try: r = client.post(f"/api/v1/chats/{self.chat['id']}/messages", json={"text": text}, headers={**auth, "Idempotency-Key": key}) self.record("POST", r.status_code, r.headers.get("x-instance-id")) if r.status_code == 201: with self.lock: self.confirmed.append(r.json()) n += 1 key, text = uuid.uuid4().hex, f"w{index}-{n}" # any other outcome: keep key and text, retry the same request except httpx.HTTPError: self.record("POST", 0, None) try: r = client.get(f"/api/v1/chats/{self.chat['id']}/messages?limit=5", headers=auth) self.record("GET", r.status_code, r.headers.get("x-instance-id")) except httpx.HTTPError: self.record("GET", 0, None) time.sleep(0.02) client.close() def __enter__(self) -> "Traffic": for t in self.threads: t.start() time.sleep(1.0) return self def __exit__(self, *exc) -> None: self.stop.set() for t in self.threads: t.join(timeout=10) def failures(self, since: float = 0.0) -> list[tuple]: return [e for e in self.events if e[0] >= since and e[2] not in (200, 201)] def check_history(self, api: Api) -> None: history = api.history(self.bob, self.chat, 100)[0] ids, texts = [m["id"] for m in history], [m["text"] for m in history] assert len(ids) == len(set(ids)) and len(texts) == len(set(texts)), "duplicate after retry" assert {m["id"] for m in self.confirmed} <= set(ids), "a confirmed message was lost" @pytest.mark.parametrize("position", range(REPLICAS)) def test_graceful_stop_of_each_replica_loses_no_request(api, position): """SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit 0: not one failed request.""" replica = sorted(running_replicas(), key=lambda r: r["name"])[position] with Traffic(api) as traffic: started = time.monotonic() with replica_stopped(replica, "stop"): stopped_after = time.monotonic() - started time.sleep(3) after = [e for e in traffic.events if e[0] > time.monotonic() - 2] assert after and replica["hostname"] not in {e[3] for e in after} time.sleep(1) state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True, check=True).stdout)[0]["State"] assert traffic.failures() == [], traffic.failures()[:10] assert stopped_after < 15, "graceful stop must finish before stop_grace_period (no SIGKILL)" assert state["Status"] == "running" # restored by replica_stopped traffic.check_history(api) print(f"{replica['name']}: stop took {stopped_after:.1f} s, {len(traffic.events)} requests, 0 failed") @pytest.mark.parametrize("position", range(REPLICAS)) def test_killed_replica_bounded_errors_recovery_and_no_duplicates(api, position): """SIGKILL: requests in flight on that replica may fail (counted), new ones succeed within 10 s, the killed replica gets nothing after detection, retries do not duplicate.""" replica = sorted(running_replicas(), key=lambda r: r["name"])[position] with Traffic(api) as traffic: killed_at = time.monotonic() with replica_stopped(replica, "kill"): time.sleep(4) failures = traffic.failures(killed_at) last_failure = max((e[0] for e in failures), default=killed_at) - killed_at post_ok = [e for e in traffic.events if e[0] > killed_at and e[1] == "POST" and e[2] == 201] assert post_ok and post_ok[0][0] - killed_at <= 10, "writes did not recover within 10 s" late = [e for e in traffic.events if e[0] > killed_at + 3.5] assert late and replica["hostname"] not in {e[3] for e in late} assert last_failure <= 10 traffic.check_history(api) print(f"{replica['name']}: killed; {len(failures)} failed of {len(traffic.events)}, last failure +{last_failure:.2f} s") def test_graceful_shutdown_order_in_logs(api): """Logs prove the order: drain_started -> (HAProxy marks DOWN) -> shutdown_requested -> exit 0.""" replica = sorted(running_replicas(), key=lambda r: r["name"])[-1] since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 1)) with replica_stopped(replica, "stop"): logs = subprocess.run(["docker", "logs", "--since", since, replica["id"]], capture_output=True, text=True, encoding="utf-8").stdout lb_logs = compose("logs", "--no-log-prefix", "--since", since, "lb").stdout state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True, check=True).stdout)[0]["State"] events = [json.loads(line) for line in logs.splitlines() if line.startswith("{")] names = [e["event"] for e in events] assert names.index("drain_started") < names.index("shutdown_requested") < names.index("stopped") t = {e["event"]: e["ts"] for e in events} assert t["drain_started"] < t["shutdown_requested"] assert state["ExitCode"] == 0 and not state["OOMKilled"] assert "is DOWN" in lb_logs and ("503" in lb_logs or "Connection refused" in lb_logs) # ------------------------------------------------------------------ discovery: restart and scaling def test_full_restart_lb_rediscovers_all_replicas(api): actor = api.user("restart") before = {r["hostname"]: r["started_at"] for r in running_replicas()} run_hook("restart.sh") wait_healthy("postgres", "valkey", "api") wait_stand() after = {r["hostname"]: r["started_at"] for r in running_replicas()} assert set(after) == set(before) and all(after[h] != before[h] for h in after), "every replica restarted" assert set(instances_seen(api, actor["token"])) == set(after) # same token, both replicas, no LB restart needed def test_scale_up_and_down_is_picked_up_without_errors(api): with Traffic(api, workers=2) as traffic: try: compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS + 1}", "api", timeout=180) lbctl.wait_up(REPLICAS + 1, timeout=30) actor = api.user("scale") assert len(set(instances_seen(api, actor["token"], 30))) == REPLICAS + 1 finally: compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS}", "api", timeout=180) wait_stand() time.sleep(1) assert traffic.failures() == [], traffic.failures()[:10] traffic.check_history(api) # ------------------------------------------------------------------ level 4: deployment policy def test_compose_policy_limits_ports_secrets_images(): config = json.loads(compose("config", "--format", "json").stdout) services = config["services"] # auditor and jobctl (profile "tools") are started on demand and not part of the running stand. assert set(services) == {"postgres", "valkey", "s3", "broker", "migrate", "s3-init", "api", "lb", "relay", "worker"} for name, svc in services.items(): limits = svc.get("deploy", {}).get("resources", {}).get("limits", {}) assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits" assert "container_name" not in svc, f"{name}: container_name prevents scaling" if name in ("api", "migrate"): assert "build" in svc # built from this repository's source elif name in ("s3-init", "relay", "worker"): assert svc["image"] == services["api"]["image"] # the same locally built image else: assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest" assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports") published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")} assert set(published) == {"lb"} and all(p["host_ip"] in ("127.0.0.1", "::1") for p in published["lb"]) secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()} s3_keys = sorted(f"s3_{who}_{part}" for who in ("admin", "api", "worker", "auditor") for part in ("access_key_id", "secret_access_key")) assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], "s3": s3_keys, "migrate": ["database_url"], "s3-init": s3_keys, "lb": [], "broker": ["rabbitmq_users"], # the API: its own S3 key pair only (no admin/worker keys, no broker credentials) "api": ["cache_url", "database_url", "s3_api_access_key_id", "s3_api_secret_access_key"], # relay: database + its broker user only; worker: + its own S3 key pair "relay": ["broker_url_relay", "database_url"], "worker": ["broker_url_worker", "database_url", "s3_worker_access_key_id", "s3_worker_secret_access_key"]} volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"] for name, svc in services.items()} assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "s3": ["s3data"], "migrate": [], "s3-init": [], "api": [], "lb": [], "broker": ["rabbitmqdata"], "relay": [], "worker": []} networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()} assert networks == {"postgres": ["data"], "valkey": ["data"], "s3": ["storage"], "migrate": ["data"], "s3-init": ["storage"], "api": ["data", "edge", "storage"], "broker": ["queue"], "relay": ["data", "queue"], "worker": ["data", "queue", "storage"], "lb": ["edge"]}, networks # the LB cannot reach the stores; the API not the broker for network in ("data", "storage", "queue"): assert config["networks"][network].get("internal") is True, network def test_running_containers_have_limits_and_no_bypass_ports(): ids = compose("ps", "-q").stdout.split() for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True, check=True).stdout): name = item["Name"] assert item["HostConfig"]["NanoCpus"] > 0 and item["HostConfig"]["Memory"] > 0, name bindings = item["HostConfig"]["PortBindings"] or {} if "-lb-" in name: assert all(b["HostIp"] in ("127.0.0.1", "::1") for v in bindings.values() for b in v) # loopback only else: assert not bindings, f"{name} publishes ports in bypass of the LB" def test_migrations_ran_once_by_the_migrate_service(): applied = psql("SELECT version FROM schema_migrations ORDER BY version").split() assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function", "0004_attachments", "0005_pipeline"] migrate = compose("ps", "-a", "--format", "json", "migrate").stdout rows = [json.loads(line) for line in migrate.splitlines() if line.strip()] assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows) api_logs = compose("logs", "--no-log-prefix", "api").stdout assert "migrate:" not in api_logs and "schema_migrations" not in api_logs # replicas never migrate