From 6fb4587ef641b5af54f33f24b888d4b8eb9f2893 Mon Sep 17 00:00:00 2001 From: Artyom Date: Wed, 28 Oct 2026 19:33:02 +0300 Subject: [PATCH] Leave the rotation before closing the listener on SIGTERM On SIGTERM gunicorn closed the listener at once, so a request already queued on the closing socket reached the balancer as a dropped connection after sending: a retry for GET and keyed messages, a 502 for user, chat and login posts. Now each worker first raises a draining flag and keeps serving for DRAIN_SECONDS (2 s by default, above the 1.5 s worst detection of a 503): readiness answers 503 unavailable, liveness stays 200, and the balancer takes the replica out while it is still healthy. Only then uvicorn closes the listener and finishes accepted requests within SHUTDOWN_TIMEOUT_SECONDS minus the window, so gunicorn graceful_timeout stays 5 s. The worker runs a uvicorn Server subclass whose main loop adds the window after the exit signal; UvicornWorker._serve is overridden to use it. Tests check readiness under the flag and run real gunicorn on a free port: during the window ready is 503 and live 200, the port closes after the window, a registration accepted before SIGTERM and finished after the port closed still gets 201, and the master exits with code 0 before the shutdown limit. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../add-lab3-failover-operations/tasks.md | 8 +- src/messenger/gunicorn_conf.py | 29 ++++- src/messenger/shared/entrypoints/health.py | 6 + student-tests/api/test_drain.py | 108 ++++++++++++++++++ 4 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 student-tests/api/test_drain.py diff --git a/openspec/changes/add-lab3-failover-operations/tasks.md b/openspec/changes/add-lab3-failover-operations/tasks.md index 00916fea..68138284 100644 --- a/openspec/changes/add-lab3-failover-operations/tasks.md +++ b/openspec/changes/add-lab3-failover-operations/tasks.md @@ -1,9 +1,9 @@ ## 1. Остановка и готовность (платформа) -- [ ] 1.1 `DRAINING` в `shared/entrypoints/health.py`: `/health/ready` отвечает `503 unavailable`, пока флаг поднят -- [ ] 1.2 `DrainingServer` и `MessengerWorker._serve` в `gunicorn_conf.py`: окно `DRAIN_SECONDS`, затем штатная остановка; uvicorn дообрабатывает `SHUTDOWN_TIMEOUT_SECONDS − DRAIN_SECONDS` -- [ ] 1.3 Пинг readiness: 0,4 с на соединение и 0,4 с на запрос -- [ ] 1.4 Проверка группы: тесты — ready `503` при флаге и live `200`; настоящий gunicorn по `SIGTERM` держит окно, закрывает порт и выходит с кодом 0 за срок +- [x] 1.1 `DRAINING` в `shared/entrypoints/health.py`: `/health/ready` отвечает `503 unavailable`, пока флаг поднят +- [x] 1.2 `DrainingServer` и `MessengerWorker._serve` в `gunicorn_conf.py`: окно `DRAIN_SECONDS`, затем штатная остановка; uvicorn дообрабатывает `SHUTDOWN_TIMEOUT_SECONDS − DRAIN_SECONDS` +- [x] 1.3 Пинг readiness: 0,4 с на соединение и 0,4 с на запрос +- [x] 1.4 Проверка группы: тесты — ready `503` при флаге и live `200`; настоящий gunicorn по `SIGTERM` держит окно, закрывает порт и выходит с кодом 0 за срок ## 2. Стенд и таймауты diff --git a/src/messenger/gunicorn_conf.py b/src/messenger/gunicorn_conf.py index 1b168029..4ccd4983 100644 --- a/src/messenger/gunicorn_conf.py +++ b/src/messenger/gunicorn_conf.py @@ -1,13 +1,21 @@ +import asyncio +import logging import os +import sys from typing import Any +from gunicorn.arbiter import Arbiter from gunicorn.glogging import Logger as GunicornBaseLogger +from uvicorn.server import Server from uvicorn_worker import UvicornWorker from messenger.main import LOG_FIELDS +from messenger.shared.entrypoints.health import DRAINING from messenger.shared.infrastructure.logging import configure LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") +SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")) +DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "2")) class StructlogGunicornLogger(GunicornBaseLogger): @@ -18,14 +26,31 @@ class StructlogGunicornLogger(GunicornBaseLogger): return +class DrainingServer(Server): + async def main_loop(self) -> None: + await super().main_loop() + if not self.force_exit: + DRAINING.set() + logging.getLogger("uvicorn.error").info("Draining for %.1f s", DRAIN_SECONDS) + await asyncio.sleep(DRAIN_SECONDS) + + class MessengerWorker(UvicornWorker): CONFIG_KWARGS: dict[str, Any] = { "access_log": False, "server_header": False, "date_header": False, - "timeout_graceful_shutdown": int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")), + "timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS)), } + async def _serve(self) -> None: + self.config.app = self.wsgi + server = DrainingServer(config=self.config) + self._install_sigquit_handler() + await server.serve(sockets=self.sockets) + if not server.started: + sys.exit(Arbiter.WORKER_BOOT_ERROR) + bind = f"{os.environ.get('HOST', '0.0.0.0')}:{os.environ.get('PORT', '8080')}" workers = int(os.environ.get("WEB_CONCURRENCY", "2")) @@ -35,7 +60,7 @@ loglevel = LOG_LEVEL.lower() accesslog = None errorlog = "-" timeout = int(float(os.environ.get("REQUEST_TIMEOUT_SECONDS", "8")) + 5) -graceful_timeout = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")) +graceful_timeout = SHUTDOWN_SECONDS keepalive = 5 preload_app = False control_socket_disable = True diff --git a/src/messenger/shared/entrypoints/health.py b/src/messenger/shared/entrypoints/health.py index 7d51bc15..b2da72c9 100644 --- a/src/messenger/shared/entrypoints/health.py +++ b/src/messenger/shared/entrypoints/health.py @@ -1,10 +1,14 @@ +from threading import Event + from dishka.integrations.fastapi import FromDishka, inject from fastapi import APIRouter +from messenger.shared.application.errors import Unavailable from messenger.shared.entrypoints.schemas import HealthOut from messenger.shared.infrastructure.postgres import Database router = APIRouter(tags=["health"]) +DRAINING = Event() @router.get("/health/live") @@ -15,5 +19,7 @@ async def live() -> HealthOut: @router.get("/health/ready") @inject async def ready(database: FromDishka[Database]) -> HealthOut: + if DRAINING.is_set(): + raise Unavailable("Реплика выходит из ротации") await database.ping() return HealthOut() diff --git a/student-tests/api/test_drain.py b/student-tests/api/test_drain.py new file mode 100644 index 00000000..5b400ece --- /dev/null +++ b/student-tests/api/test_drain.py @@ -0,0 +1,108 @@ +import json +import os +import signal +import socket +import subprocess +import sys +import time + +import httpx + +from conftest import DATABASE, new_user +from messenger.shared.entrypoints.health import DRAINING + +DRAIN = 1.5 +SHUTDOWN = 4 +GUNICORN = ( + "-m", + "gunicorn", + "--config", + "python:messenger.gunicorn_conf", + "messenger.main:create_app()", +) + + +async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None: + DRAINING.set() + try: + ready = await client.get("/health/ready") + live = await client.get("/health/live") + finally: + DRAINING.clear() + assert ready.status_code == 503 + assert ready.json()["error"]["code"] == "unavailable" + assert ready.headers["X-Instance-Id"] + assert live.status_code == 200 + assert (await client.get("/health/ready")).status_code == 200 + + +def free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +def status(port: int, path: str) -> int: + try: + return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1).status_code + except httpx.TransportError: + return 0 + + +def begin_registration(port: int) -> tuple[socket.socket, bytes]: + body = json.dumps(new_user("drain")).encode() + head = ( + "POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n" + ) + connection = socket.create_connection(("127.0.0.1", port), timeout=5) + connection.sendall(head.encode() + body[:8]) + return connection, body[8:] + + +def test_sigterm_leaves_rotation_then_closes_and_exits() -> None: + port = free_port() + settings = { + "HOST": "127.0.0.1", + "PORT": str(port), + "WEB_CONCURRENCY": "2", + "DATABASE_URL": DATABASE["url"], + "CURSOR_SECRET": "test-secret-for-cursors", + "LOG_LEVEL": "WARNING", + "DRAIN_SECONDS": str(DRAIN), + "SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN), + } + server = subprocess.Popen( + [sys.executable, *GUNICORN], + env={**os.environ, **settings}, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 30 + while status(port, "/health/ready") != 200: + assert time.monotonic() < deadline, "gunicorn не стал готов" + time.sleep(0.2) + pending, rest = begin_registration(port) + sent = time.monotonic() + server.send_signal(signal.SIGTERM) + time.sleep(0.3) + during = status(port, "/health/ready"), status(port, "/health/live") + while status(port, "/health/live") == 200: + time.sleep(0.05) + closed = time.monotonic() - sent + pending.sendall(rest) + answer = pending.recv(64).split(b"\r\n")[0].decode() + pending.close() + code = server.wait(timeout=SHUTDOWN + 2) + exited = time.monotonic() - sent + finally: + if server.poll() is None: + server.kill() + print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с") + print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}") + assert during == (503, 200) + assert answer == "HTTP/1.1 201 Created" + assert DRAIN - 0.2 <= closed < SHUTDOWN + assert exited < SHUTDOWN + assert code == 0