Leave the rotation before closing the listener on SIGTERM

On SIGTERM gunicorn closed the listener at once, so a request already
queued on the closing socket reached the balancer as a dropped
connection after sending: a retry for GET and keyed messages, a 502
for user, chat and login posts. Now each worker first raises a
draining flag and keeps serving for DRAIN_SECONDS (2 s by default,
above the 1.5 s worst detection of a 503): readiness answers 503
unavailable, liveness stays 200, and the balancer takes the replica
out while it is still healthy. Only then uvicorn closes the listener
and finishes accepted requests within SHUTDOWN_TIMEOUT_SECONDS minus
the window, so gunicorn graceful_timeout stays 5 s.

The worker runs a uvicorn Server subclass whose main loop adds the
window after the exit signal; UvicornWorker._serve is overridden to
use it. Tests check readiness under the flag and run real gunicorn on
a free port: during the window ready is 503 and live 200, the port
closes after the window, a registration accepted before SIGTERM and
finished after the port closed still gets 201, and the master exits
with code 0 before the shutdown limit.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-28 19:33:02 +03:00
co-authored by Claude Opus 5.5
parent 6e4c387e50
commit 6fb4587ef6
4 changed files with 145 additions and 6 deletions
@@ -1,9 +1,9 @@
## 1. Остановка и готовность (платформа)
- [ ] 1.1 `DRAINING` в `shared/entrypoints/health.py`: `/health/ready` отвечает `503 unavailable`, пока флаг поднят
- [ ] 1.2 `DrainingServer` и `MessengerWorker._serve` в `gunicorn_conf.py`: окно `DRAIN_SECONDS`, затем штатная остановка; uvicorn дообрабатывает `SHUTDOWN_TIMEOUT_SECONDS − DRAIN_SECONDS`
- [ ] 1.3 Пинг readiness: 0,4 с на соединение и 0,4 с на запрос
- [ ] 1.4 Проверка группы: тесты — ready `503` при флаге и live `200`; настоящий gunicorn по `SIGTERM` держит окно, закрывает порт и выходит с кодом 0 за срок
- [x] 1.1 `DRAINING` в `shared/entrypoints/health.py`: `/health/ready` отвечает `503 unavailable`, пока флаг поднят
- [x] 1.2 `DrainingServer` и `MessengerWorker._serve` в `gunicorn_conf.py`: окно `DRAIN_SECONDS`, затем штатная остановка; uvicorn дообрабатывает `SHUTDOWN_TIMEOUT_SECONDS − DRAIN_SECONDS`
- [x] 1.3 Пинг readiness: 0,4 с на соединение и 0,4 с на запрос
- [x] 1.4 Проверка группы: тесты — ready `503` при флаге и live `200`; настоящий gunicorn по `SIGTERM` держит окно, закрывает порт и выходит с кодом 0 за срок
## 2. Стенд и таймауты
+27 -2
View File
@@ -1,13 +1,21 @@
import asyncio
import logging
import os
import sys
from typing import Any
from gunicorn.arbiter import Arbiter
from gunicorn.glogging import Logger as GunicornBaseLogger
from uvicorn.server import Server
from uvicorn_worker import UvicornWorker
from messenger.main import LOG_FIELDS
from messenger.shared.entrypoints.health import DRAINING
from messenger.shared.infrastructure.logging import configure
LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5"))
DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "2"))
class StructlogGunicornLogger(GunicornBaseLogger):
@@ -18,14 +26,31 @@ class StructlogGunicornLogger(GunicornBaseLogger):
return
class DrainingServer(Server):
async def main_loop(self) -> None:
await super().main_loop()
if not self.force_exit:
DRAINING.set()
logging.getLogger("uvicorn.error").info("Draining for %.1f s", DRAIN_SECONDS)
await asyncio.sleep(DRAIN_SECONDS)
class MessengerWorker(UvicornWorker):
CONFIG_KWARGS: dict[str, Any] = {
"access_log": False,
"server_header": False,
"date_header": False,
"timeout_graceful_shutdown": int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")),
"timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS)),
}
async def _serve(self) -> None:
self.config.app = self.wsgi
server = DrainingServer(config=self.config)
self._install_sigquit_handler()
await server.serve(sockets=self.sockets)
if not server.started:
sys.exit(Arbiter.WORKER_BOOT_ERROR)
bind = f"{os.environ.get('HOST', '0.0.0.0')}:{os.environ.get('PORT', '8080')}"
workers = int(os.environ.get("WEB_CONCURRENCY", "2"))
@@ -35,7 +60,7 @@ loglevel = LOG_LEVEL.lower()
accesslog = None
errorlog = "-"
timeout = int(float(os.environ.get("REQUEST_TIMEOUT_SECONDS", "8")) + 5)
graceful_timeout = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5"))
graceful_timeout = SHUTDOWN_SECONDS
keepalive = 5
preload_app = False
control_socket_disable = True
@@ -1,10 +1,14 @@
from threading import Event
from dishka.integrations.fastapi import FromDishka, inject
from fastapi import APIRouter
from messenger.shared.application.errors import Unavailable
from messenger.shared.entrypoints.schemas import HealthOut
from messenger.shared.infrastructure.postgres import Database
router = APIRouter(tags=["health"])
DRAINING = Event()
@router.get("/health/live")
@@ -15,5 +19,7 @@ async def live() -> HealthOut:
@router.get("/health/ready")
@inject
async def ready(database: FromDishka[Database]) -> HealthOut:
if DRAINING.is_set():
raise Unavailable("Реплика выходит из ротации")
await database.ping()
return HealthOut()
+108
View File
@@ -0,0 +1,108 @@
import json
import os
import signal
import socket
import subprocess
import sys
import time
import httpx
from conftest import DATABASE, new_user
from messenger.shared.entrypoints.health import DRAINING
DRAIN = 1.5
SHUTDOWN = 4
GUNICORN = (
"-m",
"gunicorn",
"--config",
"python:messenger.gunicorn_conf",
"messenger.main:create_app()",
)
async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None:
DRAINING.set()
try:
ready = await client.get("/health/ready")
live = await client.get("/health/live")
finally:
DRAINING.clear()
assert ready.status_code == 503
assert ready.json()["error"]["code"] == "unavailable"
assert ready.headers["X-Instance-Id"]
assert live.status_code == 200
assert (await client.get("/health/ready")).status_code == 200
def free_port() -> int:
with socket.socket() as probe:
probe.bind(("127.0.0.1", 0))
return probe.getsockname()[1]
def status(port: int, path: str) -> int:
try:
return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1).status_code
except httpx.TransportError:
return 0
def begin_registration(port: int) -> tuple[socket.socket, bytes]:
body = json.dumps(new_user("drain")).encode()
head = (
"POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n"
f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n"
)
connection = socket.create_connection(("127.0.0.1", port), timeout=5)
connection.sendall(head.encode() + body[:8])
return connection, body[8:]
def test_sigterm_leaves_rotation_then_closes_and_exits() -> None:
port = free_port()
settings = {
"HOST": "127.0.0.1",
"PORT": str(port),
"WEB_CONCURRENCY": "2",
"DATABASE_URL": DATABASE["url"],
"CURSOR_SECRET": "test-secret-for-cursors",
"LOG_LEVEL": "WARNING",
"DRAIN_SECONDS": str(DRAIN),
"SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN),
}
server = subprocess.Popen(
[sys.executable, *GUNICORN],
env={**os.environ, **settings},
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
deadline = time.monotonic() + 30
while status(port, "/health/ready") != 200:
assert time.monotonic() < deadline, "gunicorn не стал готов"
time.sleep(0.2)
pending, rest = begin_registration(port)
sent = time.monotonic()
server.send_signal(signal.SIGTERM)
time.sleep(0.3)
during = status(port, "/health/ready"), status(port, "/health/live")
while status(port, "/health/live") == 200:
time.sleep(0.05)
closed = time.monotonic() - sent
pending.sendall(rest)
answer = pending.recv(64).split(b"\r\n")[0].decode()
pending.close()
code = server.wait(timeout=SHUTDOWN + 2)
exited = time.monotonic() - sent
finally:
if server.poll() is None:
server.kill()
print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с")
print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}")
assert during == (503, 200)
assert answer == "HTTP/1.1 201 Created"
assert DRAIN - 0.2 <= closed < SHUTDOWN
assert exited < SHUTDOWN
assert code == 0