forked from mtusi2026/mtusi2026lab3
Leave the rotation before closing the listener on SIGTERM
On SIGTERM gunicorn closed the listener at once, so a request already queued on the closing socket reached the balancer as a dropped connection after sending: a retry for GET and keyed messages, a 502 for user, chat and login posts. Now each worker first raises a draining flag and keeps serving for DRAIN_SECONDS (2 s by default, above the 1.5 s worst detection of a 503): readiness answers 503 unavailable, liveness stays 200, and the balancer takes the replica out while it is still healthy. Only then uvicorn closes the listener and finishes accepted requests within SHUTDOWN_TIMEOUT_SECONDS minus the window, so gunicorn graceful_timeout stays 5 s. The worker runs a uvicorn Server subclass whose main loop adds the window after the exit signal; UvicornWorker._serve is overridden to use it. Tests check readiness under the flag and run real gunicorn on a free port: during the window ready is 503 and live 200, the port closes after the window, a registration accepted before SIGTERM and finished after the port closed still gets 201, and the master exits with code 0 before the shutdown limit. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
6e4c387e50
commit
6fb4587ef6
@@ -1,9 +1,9 @@
|
||||
## 1. Остановка и готовность (платформа)
|
||||
|
||||
- [ ] 1.1 `DRAINING` в `shared/entrypoints/health.py`: `/health/ready` отвечает `503 unavailable`, пока флаг поднят
|
||||
- [ ] 1.2 `DrainingServer` и `MessengerWorker._serve` в `gunicorn_conf.py`: окно `DRAIN_SECONDS`, затем штатная остановка; uvicorn дообрабатывает `SHUTDOWN_TIMEOUT_SECONDS − DRAIN_SECONDS`
|
||||
- [ ] 1.3 Пинг readiness: 0,4 с на соединение и 0,4 с на запрос
|
||||
- [ ] 1.4 Проверка группы: тесты — ready `503` при флаге и live `200`; настоящий gunicorn по `SIGTERM` держит окно, закрывает порт и выходит с кодом 0 за срок
|
||||
- [x] 1.1 `DRAINING` в `shared/entrypoints/health.py`: `/health/ready` отвечает `503 unavailable`, пока флаг поднят
|
||||
- [x] 1.2 `DrainingServer` и `MessengerWorker._serve` в `gunicorn_conf.py`: окно `DRAIN_SECONDS`, затем штатная остановка; uvicorn дообрабатывает `SHUTDOWN_TIMEOUT_SECONDS − DRAIN_SECONDS`
|
||||
- [x] 1.3 Пинг readiness: 0,4 с на соединение и 0,4 с на запрос
|
||||
- [x] 1.4 Проверка группы: тесты — ready `503` при флаге и live `200`; настоящий gunicorn по `SIGTERM` держит окно, закрывает порт и выходит с кодом 0 за срок
|
||||
|
||||
## 2. Стенд и таймауты
|
||||
|
||||
|
||||
@@ -1,13 +1,21 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from gunicorn.arbiter import Arbiter
|
||||
from gunicorn.glogging import Logger as GunicornBaseLogger
|
||||
from uvicorn.server import Server
|
||||
from uvicorn_worker import UvicornWorker
|
||||
|
||||
from messenger.main import LOG_FIELDS
|
||||
from messenger.shared.entrypoints.health import DRAINING
|
||||
from messenger.shared.infrastructure.logging import configure
|
||||
|
||||
LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO")
|
||||
SHUTDOWN_SECONDS = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5"))
|
||||
DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "2"))
|
||||
|
||||
|
||||
class StructlogGunicornLogger(GunicornBaseLogger):
|
||||
@@ -18,14 +26,31 @@ class StructlogGunicornLogger(GunicornBaseLogger):
|
||||
return
|
||||
|
||||
|
||||
class DrainingServer(Server):
|
||||
async def main_loop(self) -> None:
|
||||
await super().main_loop()
|
||||
if not self.force_exit:
|
||||
DRAINING.set()
|
||||
logging.getLogger("uvicorn.error").info("Draining for %.1f s", DRAIN_SECONDS)
|
||||
await asyncio.sleep(DRAIN_SECONDS)
|
||||
|
||||
|
||||
class MessengerWorker(UvicornWorker):
|
||||
CONFIG_KWARGS: dict[str, Any] = {
|
||||
"access_log": False,
|
||||
"server_header": False,
|
||||
"date_header": False,
|
||||
"timeout_graceful_shutdown": int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5")),
|
||||
"timeout_graceful_shutdown": max(1, int(SHUTDOWN_SECONDS - DRAIN_SECONDS)),
|
||||
}
|
||||
|
||||
async def _serve(self) -> None:
|
||||
self.config.app = self.wsgi
|
||||
server = DrainingServer(config=self.config)
|
||||
self._install_sigquit_handler()
|
||||
await server.serve(sockets=self.sockets)
|
||||
if not server.started:
|
||||
sys.exit(Arbiter.WORKER_BOOT_ERROR)
|
||||
|
||||
|
||||
bind = f"{os.environ.get('HOST', '0.0.0.0')}:{os.environ.get('PORT', '8080')}"
|
||||
workers = int(os.environ.get("WEB_CONCURRENCY", "2"))
|
||||
@@ -35,7 +60,7 @@ loglevel = LOG_LEVEL.lower()
|
||||
accesslog = None
|
||||
errorlog = "-"
|
||||
timeout = int(float(os.environ.get("REQUEST_TIMEOUT_SECONDS", "8")) + 5)
|
||||
graceful_timeout = int(os.environ.get("SHUTDOWN_TIMEOUT_SECONDS", "5"))
|
||||
graceful_timeout = SHUTDOWN_SECONDS
|
||||
keepalive = 5
|
||||
preload_app = False
|
||||
control_socket_disable = True
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
from threading import Event
|
||||
|
||||
from dishka.integrations.fastapi import FromDishka, inject
|
||||
from fastapi import APIRouter
|
||||
|
||||
from messenger.shared.application.errors import Unavailable
|
||||
from messenger.shared.entrypoints.schemas import HealthOut
|
||||
from messenger.shared.infrastructure.postgres import Database
|
||||
|
||||
router = APIRouter(tags=["health"])
|
||||
DRAINING = Event()
|
||||
|
||||
|
||||
@router.get("/health/live")
|
||||
@@ -15,5 +19,7 @@ async def live() -> HealthOut:
|
||||
@router.get("/health/ready")
|
||||
@inject
|
||||
async def ready(database: FromDishka[Database]) -> HealthOut:
|
||||
if DRAINING.is_set():
|
||||
raise Unavailable("Реплика выходит из ротации")
|
||||
await database.ping()
|
||||
return HealthOut()
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
from conftest import DATABASE, new_user
|
||||
from messenger.shared.entrypoints.health import DRAINING
|
||||
|
||||
DRAIN = 1.5
|
||||
SHUTDOWN = 4
|
||||
GUNICORN = (
|
||||
"-m",
|
||||
"gunicorn",
|
||||
"--config",
|
||||
"python:messenger.gunicorn_conf",
|
||||
"messenger.main:create_app()",
|
||||
)
|
||||
|
||||
|
||||
async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None:
|
||||
DRAINING.set()
|
||||
try:
|
||||
ready = await client.get("/health/ready")
|
||||
live = await client.get("/health/live")
|
||||
finally:
|
||||
DRAINING.clear()
|
||||
assert ready.status_code == 503
|
||||
assert ready.json()["error"]["code"] == "unavailable"
|
||||
assert ready.headers["X-Instance-Id"]
|
||||
assert live.status_code == 200
|
||||
assert (await client.get("/health/ready")).status_code == 200
|
||||
|
||||
|
||||
def free_port() -> int:
|
||||
with socket.socket() as probe:
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
return probe.getsockname()[1]
|
||||
|
||||
|
||||
def status(port: int, path: str) -> int:
|
||||
try:
|
||||
return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1).status_code
|
||||
except httpx.TransportError:
|
||||
return 0
|
||||
|
||||
|
||||
def begin_registration(port: int) -> tuple[socket.socket, bytes]:
|
||||
body = json.dumps(new_user("drain")).encode()
|
||||
head = (
|
||||
"POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n"
|
||||
f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
connection = socket.create_connection(("127.0.0.1", port), timeout=5)
|
||||
connection.sendall(head.encode() + body[:8])
|
||||
return connection, body[8:]
|
||||
|
||||
|
||||
def test_sigterm_leaves_rotation_then_closes_and_exits() -> None:
|
||||
port = free_port()
|
||||
settings = {
|
||||
"HOST": "127.0.0.1",
|
||||
"PORT": str(port),
|
||||
"WEB_CONCURRENCY": "2",
|
||||
"DATABASE_URL": DATABASE["url"],
|
||||
"CURSOR_SECRET": "test-secret-for-cursors",
|
||||
"LOG_LEVEL": "WARNING",
|
||||
"DRAIN_SECONDS": str(DRAIN),
|
||||
"SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN),
|
||||
}
|
||||
server = subprocess.Popen(
|
||||
[sys.executable, *GUNICORN],
|
||||
env={**os.environ, **settings},
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
deadline = time.monotonic() + 30
|
||||
while status(port, "/health/ready") != 200:
|
||||
assert time.monotonic() < deadline, "gunicorn не стал готов"
|
||||
time.sleep(0.2)
|
||||
pending, rest = begin_registration(port)
|
||||
sent = time.monotonic()
|
||||
server.send_signal(signal.SIGTERM)
|
||||
time.sleep(0.3)
|
||||
during = status(port, "/health/ready"), status(port, "/health/live")
|
||||
while status(port, "/health/live") == 200:
|
||||
time.sleep(0.05)
|
||||
closed = time.monotonic() - sent
|
||||
pending.sendall(rest)
|
||||
answer = pending.recv(64).split(b"\r\n")[0].decode()
|
||||
pending.close()
|
||||
code = server.wait(timeout=SHUTDOWN + 2)
|
||||
exited = time.monotonic() - sent
|
||||
finally:
|
||||
if server.poll() is None:
|
||||
server.kill()
|
||||
print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с")
|
||||
print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}")
|
||||
assert during == (503, 200)
|
||||
assert answer == "HTTP/1.1 201 Created"
|
||||
assert DRAIN - 0.2 <= closed < SHUTDOWN
|
||||
assert exited < SHUTDOWN
|
||||
assert code == 0
|
||||
Reference in New Issue
Block a user