On SIGTERM gunicorn closed the listener at once, so a request already queued on the closing socket reached the balancer as a dropped connection after sending: a retry for GET and keyed messages, a 502 for user, chat and login posts. Now each worker first raises a draining flag and keeps serving for DRAIN_SECONDS (2 s by default, above the 1.5 s worst detection of a 503): readiness answers 503 unavailable, liveness stays 200, and the balancer takes the replica out while it is still healthy. Only then uvicorn closes the listener and finishes accepted requests within SHUTDOWN_TIMEOUT_SECONDS minus the window, so gunicorn graceful_timeout stays 5 s. The worker runs a uvicorn Server subclass whose main loop adds the window after the exit signal; UvicornWorker._serve is overridden to use it. Tests check readiness under the flag and run real gunicorn on a free port: during the window ready is 503 and live 200, the port closes after the window, a registration accepted before SIGTERM and finished after the port closed still gets 201, and the master exits with code 0 before the shutdown limit. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
109 lines
3.4 KiB
Python
109 lines
3.4 KiB
Python
import json
|
||
import os
|
||
import signal
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
|
||
import httpx
|
||
|
||
from conftest import DATABASE, new_user
|
||
from messenger.shared.entrypoints.health import DRAINING
|
||
|
||
DRAIN = 1.5
|
||
SHUTDOWN = 4
|
||
GUNICORN = (
|
||
"-m",
|
||
"gunicorn",
|
||
"--config",
|
||
"python:messenger.gunicorn_conf",
|
||
"messenger.main:create_app()",
|
||
)
|
||
|
||
|
||
async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None:
|
||
DRAINING.set()
|
||
try:
|
||
ready = await client.get("/health/ready")
|
||
live = await client.get("/health/live")
|
||
finally:
|
||
DRAINING.clear()
|
||
assert ready.status_code == 503
|
||
assert ready.json()["error"]["code"] == "unavailable"
|
||
assert ready.headers["X-Instance-Id"]
|
||
assert live.status_code == 200
|
||
assert (await client.get("/health/ready")).status_code == 200
|
||
|
||
|
||
def free_port() -> int:
|
||
with socket.socket() as probe:
|
||
probe.bind(("127.0.0.1", 0))
|
||
return probe.getsockname()[1]
|
||
|
||
|
||
def status(port: int, path: str) -> int:
|
||
try:
|
||
return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1).status_code
|
||
except httpx.TransportError:
|
||
return 0
|
||
|
||
|
||
def begin_registration(port: int) -> tuple[socket.socket, bytes]:
|
||
body = json.dumps(new_user("drain")).encode()
|
||
head = (
|
||
"POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n"
|
||
f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n"
|
||
)
|
||
connection = socket.create_connection(("127.0.0.1", port), timeout=5)
|
||
connection.sendall(head.encode() + body[:8])
|
||
return connection, body[8:]
|
||
|
||
|
||
def test_sigterm_leaves_rotation_then_closes_and_exits() -> None:
|
||
port = free_port()
|
||
settings = {
|
||
"HOST": "127.0.0.1",
|
||
"PORT": str(port),
|
||
"WEB_CONCURRENCY": "2",
|
||
"DATABASE_URL": DATABASE["url"],
|
||
"CURSOR_SECRET": "test-secret-for-cursors",
|
||
"LOG_LEVEL": "WARNING",
|
||
"DRAIN_SECONDS": str(DRAIN),
|
||
"SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN),
|
||
}
|
||
server = subprocess.Popen(
|
||
[sys.executable, *GUNICORN],
|
||
env={**os.environ, **settings},
|
||
stdout=subprocess.DEVNULL,
|
||
stderr=subprocess.DEVNULL,
|
||
)
|
||
try:
|
||
deadline = time.monotonic() + 30
|
||
while status(port, "/health/ready") != 200:
|
||
assert time.monotonic() < deadline, "gunicorn не стал готов"
|
||
time.sleep(0.2)
|
||
pending, rest = begin_registration(port)
|
||
sent = time.monotonic()
|
||
server.send_signal(signal.SIGTERM)
|
||
time.sleep(0.3)
|
||
during = status(port, "/health/ready"), status(port, "/health/live")
|
||
while status(port, "/health/live") == 200:
|
||
time.sleep(0.05)
|
||
closed = time.monotonic() - sent
|
||
pending.sendall(rest)
|
||
answer = pending.recv(64).split(b"\r\n")[0].decode()
|
||
pending.close()
|
||
code = server.wait(timeout=SHUTDOWN + 2)
|
||
exited = time.monotonic() - sent
|
||
finally:
|
||
if server.poll() is None:
|
||
server.kill()
|
||
print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с")
|
||
print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}")
|
||
assert during == (503, 200)
|
||
assert answer == "HTTP/1.1 201 Created"
|
||
assert DRAIN - 0.2 <= closed < SHUTDOWN
|
||
assert exited < SHUTDOWN
|
||
assert code == 0
|