import json import os import signal import socket import subprocess import sys import time from pathlib import Path import httpx from conftest import DATABASE, free_port, new_user from messenger.shared.infrastructure.readiness import DRAINING DRAIN = 1.0 SHUTDOWN = 3 GUNICORN = ( "-m", "gunicorn", "--config", "python:messenger.gunicorn_conf", "messenger.main:create_app()", ) async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None: DRAINING.set() try: ready = await client.get("/health/ready") live = await client.get("/health/live") finally: DRAINING.clear() assert ready.status_code == 503 assert ready.json()["error"]["code"] == "unavailable" assert ready.headers["X-Instance-Id"] assert live.status_code == 200 assert (await client.get("/health/ready")).status_code == 200 def status(port: int, path: str) -> int: try: return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1, trust_env=False).status_code except httpx.TransportError: return 0 def answer_of(connection: socket.socket) -> tuple[str, dict[str, str], dict]: data = b"" while chunk := connection.recv(4096): data += chunk connection.close() head, _, body = data.partition(b"\r\n\r\n") status, *lines = head.decode("latin-1").split("\r\n") headers = dict(line.lower().split(": ", 1) for line in lines) return status, headers, json.loads(body) if body else {} def begin_registration(port: int) -> tuple[socket.socket, bytes]: body = json.dumps(new_user("drain")).encode() head = ( "POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n" f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n" ) connection = socket.create_connection(("127.0.0.1", port), timeout=5) connection.sendall(head.encode() + body[:8]) return connection, body[8:] def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests(tmp_path: Path) -> None: port = free_port() settings = { "HOST": "127.0.0.1", "PORT": str(port), "WEB_CONCURRENCY": "2", "DATABASE_URL": DATABASE["url"], "CURSOR_SECRET": "test-secret-for-cursors", "LOG_LEVEL": "WARNING", "DRAIN_SECONDS": str(DRAIN), "SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN), } log_path = tmp_path / "gunicorn.log" log = log_path.open("w") server = subprocess.Popen( [sys.executable, *GUNICORN], env={**os.environ, **settings}, stdout=log, stderr=log, ) try: deadline = time.monotonic() + 30 while status(port, "/health/ready") != 200: assert server.poll() is None and time.monotonic() < deadline, log_path.read_text() time.sleep(0.2) pending, rest = begin_registration(port) stuck, _ = begin_registration(port) sent = time.monotonic() server.send_signal(signal.SIGTERM) time.sleep(0.3) during = status(port, "/health/ready"), status(port, "/health/live") while status(port, "/health/live") == 200: assert time.monotonic() - sent < SHUTDOWN + 2, log_path.read_text() time.sleep(0.05) closed = time.monotonic() - sent pending.sendall(rest) answer = pending.recv(64).split(b"\r\n")[0].decode() pending.close() code = server.wait(timeout=SHUTDOWN + 2) exited = time.monotonic() - sent cancelled, headers, error = answer_of(stuck) finally: if server.poll() is None: server.kill() server.wait(timeout=5) log.close() print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с") print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}") print(f"запрос, тело которого так и не пришло: {cancelled or 'соединение закрыто без ответа'}") print(f" content-type {headers.get('content-type')}, тело {error}") assert during == (503, 200) assert answer == "HTTP/1.1 201 Created" assert cancelled == "HTTP/1.1 503 Service Unavailable" assert headers["content-type"] == "application/json" assert headers["x-instance-id"] assert error["error"]["code"] == "unavailable" assert error["error"]["request_id"] == headers["x-request-id"] assert DRAIN - 0.2 <= closed < SHUTDOWN assert exited < SHUTDOWN assert code == 0