Files
mtusi2026lab3/student-tests/api/test_drain.py
T
ArtyomandClaude Opus 5.5 6fb4587ef6 Leave the rotation before closing the listener on SIGTERM
On SIGTERM gunicorn closed the listener at once, so a request already
queued on the closing socket reached the balancer as a dropped
connection after sending: a retry for GET and keyed messages, a 502
for user, chat and login posts. Now each worker first raises a
draining flag and keeps serving for DRAIN_SECONDS (2 s by default,
above the 1.5 s worst detection of a 503): readiness answers 503
unavailable, liveness stays 200, and the balancer takes the replica
out while it is still healthy. Only then uvicorn closes the listener
and finishes accepted requests within SHUTDOWN_TIMEOUT_SECONDS minus
the window, so gunicorn graceful_timeout stays 5 s.

The worker runs a uvicorn Server subclass whose main loop adds the
window after the exit signal; UvicornWorker._serve is overridden to
use it. Tests check readiness under the flag and run real gunicorn on
a free port: during the window ready is 503 and live 200, the port
closes after the window, a registration accepted before SIGTERM and
finished after the port closed still gets 201, and the master exits
with code 0 before the shutdown limit.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-28 19:33:02 +03:00

109 lines
3.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import json
import os
import signal
import socket
import subprocess
import sys
import time
import httpx
from conftest import DATABASE, new_user
from messenger.shared.entrypoints.health import DRAINING
DRAIN = 1.5
SHUTDOWN = 4
GUNICORN = (
"-m",
"gunicorn",
"--config",
"python:messenger.gunicorn_conf",
"messenger.main:create_app()",
)
async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None:
DRAINING.set()
try:
ready = await client.get("/health/ready")
live = await client.get("/health/live")
finally:
DRAINING.clear()
assert ready.status_code == 503
assert ready.json()["error"]["code"] == "unavailable"
assert ready.headers["X-Instance-Id"]
assert live.status_code == 200
assert (await client.get("/health/ready")).status_code == 200
def free_port() -> int:
with socket.socket() as probe:
probe.bind(("127.0.0.1", 0))
return probe.getsockname()[1]
def status(port: int, path: str) -> int:
try:
return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1).status_code
except httpx.TransportError:
return 0
def begin_registration(port: int) -> tuple[socket.socket, bytes]:
body = json.dumps(new_user("drain")).encode()
head = (
"POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n"
f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n"
)
connection = socket.create_connection(("127.0.0.1", port), timeout=5)
connection.sendall(head.encode() + body[:8])
return connection, body[8:]
def test_sigterm_leaves_rotation_then_closes_and_exits() -> None:
port = free_port()
settings = {
"HOST": "127.0.0.1",
"PORT": str(port),
"WEB_CONCURRENCY": "2",
"DATABASE_URL": DATABASE["url"],
"CURSOR_SECRET": "test-secret-for-cursors",
"LOG_LEVEL": "WARNING",
"DRAIN_SECONDS": str(DRAIN),
"SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN),
}
server = subprocess.Popen(
[sys.executable, *GUNICORN],
env={**os.environ, **settings},
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
deadline = time.monotonic() + 30
while status(port, "/health/ready") != 200:
assert time.monotonic() < deadline, "gunicorn не стал готов"
time.sleep(0.2)
pending, rest = begin_registration(port)
sent = time.monotonic()
server.send_signal(signal.SIGTERM)
time.sleep(0.3)
during = status(port, "/health/ready"), status(port, "/health/live")
while status(port, "/health/live") == 200:
time.sleep(0.05)
closed = time.monotonic() - sent
pending.sendall(rest)
answer = pending.recv(64).split(b"\r\n")[0].decode()
pending.close()
code = server.wait(timeout=SHUTDOWN + 2)
exited = time.monotonic() - sent
finally:
if server.poll() is None:
server.kill()
print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с")
print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}")
assert during == (503, 200)
assert answer == "HTTP/1.1 201 Created"
assert DRAIN - 0.2 <= closed < SHUTDOWN
assert exited < SHUTDOWN
assert code == 0