Files
mtusi2026lab2/student-tests/api/test_drain.py
T
ArtyomandClaude Opus 5.5 aa1485bd86 Answer requests cancelled at shutdown with 503 in the contract format
After the drain window uvicorn waits for accepted requests for three
seconds and then cancels their tasks. CancelledError is a BaseException,
so neither the pipeline nor ServerErrorMiddleware saw it and uvicorn
answered with its own text/plain 500 without X-Request-Id and
X-Instance-Id. The pipeline now handles the cancellation next to the
request timeout: it logs request.cancelled, sends 503 unavailable in the
common error format when the response has not started and re-raises,
so uvicorn only closes the transport after the buffered answer.

The declared Content-Length check joins the guard as a rejection to keep
the call within the statement limit; behaviour is unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
2026-10-30 05:39:09 +03:00

129 lines
4.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import json
import os
import signal
import socket
import subprocess
import sys
import time
import httpx
from conftest import DATABASE, new_user
from messenger.shared.entrypoints.health import DRAINING
DRAIN = 1.0
SHUTDOWN = 3
GUNICORN = (
"-m",
"gunicorn",
"--config",
"python:messenger.gunicorn_conf",
"messenger.main:create_app()",
)
async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None:
DRAINING.set()
try:
ready = await client.get("/health/ready")
live = await client.get("/health/live")
finally:
DRAINING.clear()
assert ready.status_code == 503
assert ready.json()["error"]["code"] == "unavailable"
assert ready.headers["X-Instance-Id"]
assert live.status_code == 200
assert (await client.get("/health/ready")).status_code == 200
def free_port() -> int:
with socket.socket() as probe:
probe.bind(("127.0.0.1", 0))
return probe.getsockname()[1]
def status(port: int, path: str) -> int:
try:
return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1).status_code
except httpx.TransportError:
return 0
def answer_of(connection: socket.socket) -> tuple[str, dict[str, str], dict]:
data = b""
while chunk := connection.recv(4096):
data += chunk
connection.close()
head, _, body = data.partition(b"\r\n\r\n")
status, *lines = head.decode("latin-1").split("\r\n")
headers = dict(line.lower().split(": ", 1) for line in lines)
return status, headers, json.loads(body) if body else {}
def begin_registration(port: int) -> tuple[socket.socket, bytes]:
body = json.dumps(new_user("drain")).encode()
head = (
"POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n"
f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n"
)
connection = socket.create_connection(("127.0.0.1", port), timeout=5)
connection.sendall(head.encode() + body[:8])
return connection, body[8:]
def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests() -> None:
port = free_port()
settings = {
"HOST": "127.0.0.1",
"PORT": str(port),
"WEB_CONCURRENCY": "2",
"DATABASE_URL": DATABASE["url"],
"CURSOR_SECRET": "test-secret-for-cursors",
"LOG_LEVEL": "WARNING",
"DRAIN_SECONDS": str(DRAIN),
"SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN),
}
server = subprocess.Popen(
[sys.executable, *GUNICORN],
env={**os.environ, **settings},
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
deadline = time.monotonic() + 30
while status(port, "/health/ready") != 200:
assert time.monotonic() < deadline, "gunicorn не стал готов"
time.sleep(0.2)
pending, rest = begin_registration(port)
stuck, _ = begin_registration(port)
sent = time.monotonic()
server.send_signal(signal.SIGTERM)
time.sleep(0.3)
during = status(port, "/health/ready"), status(port, "/health/live")
while status(port, "/health/live") == 200:
time.sleep(0.05)
closed = time.monotonic() - sent
pending.sendall(rest)
answer = pending.recv(64).split(b"\r\n")[0].decode()
pending.close()
code = server.wait(timeout=SHUTDOWN + 2)
exited = time.monotonic() - sent
cancelled, headers, error = answer_of(stuck)
finally:
if server.poll() is None:
server.kill()
print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с")
print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}")
print(f"запрос, тело которого так и не пришло: {cancelled or 'соединение закрыто без ответа'}")
print(f" content-type {headers.get('content-type')}, тело {error}")
assert during == (503, 200)
assert answer == "HTTP/1.1 201 Created"
assert cancelled == "HTTP/1.1 503 Service Unavailable"
assert headers["content-type"] == "application/json"
assert headers["x-instance-id"]
assert error["error"]["code"] == "unavailable"
assert error["error"]["request_id"] == headers["x-request-id"]
assert DRAIN - 0.2 <= closed < SHUTDOWN
assert exited < SHUTDOWN
assert code == 0