diff --git a/openspec/changes/add-lab3-failover-operations/tasks.md b/openspec/changes/add-lab3-failover-operations/tasks.md index 00916fea..68138284 100644 --- a/openspec/changes/add-lab3-failover-operations/tasks.md +++ b/openspec/changes/add-lab3-failover-operations/tasks.md @@ -1,9 +1,9 @@ ## 1. Остановка и готовность (платформа) -- [ ] 1.1 `DRAINING` в `shared/entrypoints/health.py`: `/health/ready` отвечает `503 unavailable`, пока флаг поднят -- [ ] 1.2 `DrainingServer` и `MessengerWorker._serve` в `gunicorn_conf.py`: окно `DRAIN_SECONDS`, затем штатная остановка; uvicorn дообрабатывает `SHUTDOWN_TIMEOUT_SECONDS − DRAIN_SECONDS` -- [ ] 1.3 Пинг readiness: 0,4 с на соединение и 0,4 с на запрос -- [ ] 1.4 Проверка группы: тесты — ready `503` при флаге и live `200`; настоящий gunicorn по `SIGTERM` держит окно, закрывает порт и выходит с кодом 0 за срок +- [x] 1.1 `DRAINING` в `shared/entrypoints/health.py`: `/health/ready` отвечает `503 unavailable`, пока флаг поднят +- [x] 1.2 `DrainingServer` и `MessengerWorker._serve` в `gunicorn_conf.py`: окно `DRAIN_SECONDS`, затем штатная остановка; uvicorn дообрабатывает `SHUTDOWN_TIMEOUT_SECONDS − DRAIN_SECONDS` +- [x] 1.3 Пинг readiness: 0,4 с на соединение и 0,4 с на запрос +- [x] 1.4 Проверка группы: тесты — ready `503` при флаге и live `200`; настоящий gunicorn по `SIGTERM` держит окно, закрывает порт и выходит с кодом 0 за срок ## 2. Стенд и таймауты diff --git a/src/messenger/gunicorn_conf.py b/src/messenger/gunicorn_conf.py index 1b168029..4ccd4983 100644 --- a/src/messenger/gunicorn_conf.py +++ b/src/messenger/gunicorn_conf.py @@ -1,13 +1,21 @@ +import asyncio +import logging import os +import sys from typing import Any +from gunicorn.arbiter import Arbiter from gunicorn.glogging import Logger as GunicornBaseLogger +from uvicorn.server import Server from uvicorn_worker import UvicornWorker from messenger.main import LOG_FIELDS +from messenger.shared.entrypoints.health import DRAINING from messenger.shared.infrastructure.logging import configure LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") +SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")) +DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "2")) class StructlogGunicornLogger(GunicornBaseLogger): @@ -18,14 +26,31 @@ class StructlogGunicornLogger(GunicornBaseLogger): return +class DrainingServer(Server): + async def main_loop(self) -> None: + await super().main_loop() + if not self.force_exit: + DRAINING.set() + logging.getLogger("uvicorn.error").info("Draining for %.1f s", DRAIN_SECONDS) + await asyncio.sleep(DRAIN_SECONDS) + + class MessengerWorker(UvicornWorker): CONFIG_KWARGS: dict[str, Any] = { "access_log": False, "server_header": False, "date_header": False, - "timeout_graceful_shutdown": int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")), + "timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS)), } + async def _serve(self) -> None: + self.config.app = self.wsgi + server = DrainingServer(config=self.config) + self._install_sigquit_handler() + await server.serve(sockets=self.sockets) + if not server.started: + sys.exit(Arbiter.WORKER_BOOT_ERROR) + bind = f"{os.environ.get('HOST', '0.0.0.0')}:{os.environ.get('PORT', '8080')}" workers = int(os.environ.get("WEB_CONCURRENCY", "2")) @@ -35,7 +60,7 @@ loglevel = LOG_LEVEL.lower() accesslog = None errorlog = "-" timeout = int(float(os.environ.get("REQUEST_TIMEOUT_SECONDS", "8")) + 5) -graceful_timeout = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")) +graceful_timeout = SHUTDOWN_SECONDS keepalive = 5 preload_app = False control_socket_disable = True diff --git a/src/messenger/shared/entrypoints/health.py b/src/messenger/shared/entrypoints/health.py index 7d51bc15..b2da72c9 100644 --- a/src/messenger/shared/entrypoints/health.py +++ b/src/messenger/shared/entrypoints/health.py @@ -1,10 +1,14 @@ +from threading import Event + from dishka.integrations.fastapi import FromDishka, inject from fastapi import APIRouter +from messenger.shared.application.errors import Unavailable from messenger.shared.entrypoints.schemas import HealthOut from messenger.shared.infrastructure.postgres import Database router = APIRouter(tags=["health"]) +DRAINING = Event() @router.get("/health/live") @@ -15,5 +19,7 @@ async def live() -> HealthOut: @router.get("/health/ready") @inject async def ready(database: FromDishka[Database]) -> HealthOut: + if DRAINING.is_set(): + raise Unavailable("Реплика выходит из ротации") await database.ping() return HealthOut() diff --git a/student-tests/api/test_drain.py b/student-tests/api/test_drain.py new file mode 100644 index 00000000..5b400ece --- /dev/null +++ b/student-tests/api/test_drain.py @@ -0,0 +1,108 @@ +import json +import os +import signal +import socket +import subprocess +import sys +import time + +import httpx + +from conftest import DATABASE, new_user +from messenger.shared.entrypoints.health import DRAINING + +DRAIN = 1.5 +SHUTDOWN = 4 +GUNICORN = ( + "-m", + "gunicorn", + "--config", + "python:messenger.gunicorn_conf", + "messenger.main:create_app()", +) + + +async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None: + DRAINING.set() + try: + ready = await client.get("/health/ready") + live = await client.get("/health/live") + finally: + DRAINING.clear() + assert ready.status_code == 503 + assert ready.json()["error"]["code"] == "unavailable" + assert ready.headers["X-Instance-Id"] + assert live.status_code == 200 + assert (await client.get("/health/ready")).status_code == 200 + + +def free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +def status(port: int, path: str) -> int: + try: + return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1).status_code + except httpx.TransportError: + return 0 + + +def begin_registration(port: int) -> tuple[socket.socket, bytes]: + body = json.dumps(new_user("drain")).encode() + head = ( + "POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n" + ) + connection = socket.create_connection(("127.0.0.1", port), timeout=5) + connection.sendall(head.encode() + body[:8]) + return connection, body[8:] + + +def test_sigterm_leaves_rotation_then_closes_and_exits() -> None: + port = free_port() + settings = { + "HOST": "127.0.0.1", + "PORT": str(port), + "WEB_CONCURRENCY": "2", + "DATABASE_URL": DATABASE["url"], + "CURSOR_SECRET": "test-secret-for-cursors", + "LOG_LEVEL": "WARNING", + "DRAIN_SECONDS": str(DRAIN), + "SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN), + } + server = subprocess.Popen( + [sys.executable, *GUNICORN], + env={**os.environ, **settings}, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 30 + while status(port, "/health/ready") != 200: + assert time.monotonic() < deadline, "gunicorn не стал готов" + time.sleep(0.2) + pending, rest = begin_registration(port) + sent = time.monotonic() + server.send_signal(signal.SIGTERM) + time.sleep(0.3) + during = status(port, "/health/ready"), status(port, "/health/live") + while status(port, "/health/live") == 200: + time.sleep(0.05) + closed = time.monotonic() - sent + pending.sendall(rest) + answer = pending.recv(64).split(b"\r\n")[0].decode() + pending.close() + code = server.wait(timeout=SHUTDOWN + 2) + exited = time.monotonic() - sent + finally: + if server.poll() is None: + server.kill() + print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с") + print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}") + assert during == (503, 200) + assert answer == "HTTP/1.1 201 Created" + assert DRAIN - 0.2 <= closed < SHUTDOWN + assert exited < SHUTDOWN + assert code == 0