forked from mtusi2026/mtusi2026lab5
131 lines
4.6 KiB
Python
131 lines
4.6 KiB
Python
import json
|
||
import os
|
||
import signal
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import httpx
|
||
|
||
from conftest import DATABASE, free_port, new_user
|
||
from messenger.shared.infrastructure.readiness import DRAINING
|
||
|
||
DRAIN = 1.0
|
||
SHUTDOWN = 3
|
||
GUNICORN = (
|
||
"-m",
|
||
"gunicorn",
|
||
"--config",
|
||
"python:messenger.gunicorn_conf",
|
||
"messenger.main:create_app()",
|
||
)
|
||
|
||
|
||
async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None:
|
||
DRAINING.set()
|
||
try:
|
||
ready = await client.get("/health/ready")
|
||
live = await client.get("/health/live")
|
||
finally:
|
||
DRAINING.clear()
|
||
assert ready.status_code == 503
|
||
assert ready.json()["error"]["code"] == "unavailable"
|
||
assert ready.headers["X-Instance-Id"]
|
||
assert live.status_code == 200
|
||
assert (await client.get("/health/ready")).status_code == 200
|
||
|
||
|
||
def status(port: int, path: str) -> int:
|
||
try:
|
||
return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1, trust_env=False).status_code
|
||
except httpx.TransportError:
|
||
return 0
|
||
|
||
|
||
def answer_of(connection: socket.socket) -> tuple[str, dict[str, str], dict]:
|
||
data = b""
|
||
while chunk := connection.recv(4096):
|
||
data += chunk
|
||
connection.close()
|
||
head, _, body = data.partition(b"\r\n\r\n")
|
||
status, *lines = head.decode("latin-1").split("\r\n")
|
||
headers = dict(line.lower().split(": ", 1) for line in lines)
|
||
return status, headers, json.loads(body) if body else {}
|
||
|
||
|
||
def begin_registration(port: int) -> tuple[socket.socket, bytes]:
|
||
body = json.dumps(new_user("drain")).encode()
|
||
head = (
|
||
"POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n"
|
||
f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n"
|
||
)
|
||
connection = socket.create_connection(("127.0.0.1", port), timeout=5)
|
||
connection.sendall(head.encode() + body[:8])
|
||
return connection, body[8:]
|
||
|
||
|
||
def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests(tmp_path: Path) -> None:
|
||
port = free_port()
|
||
settings = {
|
||
"HOST": "127.0.0.1",
|
||
"PORT": str(port),
|
||
"WEB_CONCURRENCY": "2",
|
||
"DATABASE_URL": DATABASE["url"],
|
||
"CURSOR_SECRET": "test-secret-for-cursors",
|
||
"S3_ACCESS_KEY_ID": "test-key",
|
||
"S3_SECRET_ACCESS_KEY": "test-secret-key",
|
||
"LOG_LEVEL": "WARNING",
|
||
"DRAIN_SECONDS": str(DRAIN),
|
||
"SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN),
|
||
}
|
||
log_path = tmp_path / "gunicorn.log"
|
||
log = log_path.open("w")
|
||
server = subprocess.Popen(
|
||
[sys.executable, *GUNICORN],
|
||
env={**os.environ, **settings},
|
||
stdout=log,
|
||
stderr=log,
|
||
)
|
||
try:
|
||
deadline = time.monotonic() + 30
|
||
while status(port, "/health/ready") != 200:
|
||
assert server.poll() is None and time.monotonic() < deadline, log_path.read_text()
|
||
time.sleep(0.2)
|
||
pending, rest = begin_registration(port)
|
||
stuck, _ = begin_registration(port)
|
||
sent = time.monotonic()
|
||
server.send_signal(signal.SIGTERM)
|
||
time.sleep(0.3)
|
||
during = status(port, "/health/ready"), status(port, "/health/live")
|
||
while status(port, "/health/live") == 200:
|
||
assert time.monotonic() - sent < SHUTDOWN + 2, log_path.read_text()
|
||
time.sleep(0.05)
|
||
closed = time.monotonic() - sent
|
||
pending.sendall(rest)
|
||
answer = pending.recv(64).split(b"\r\n")[0].decode()
|
||
pending.close()
|
||
code = server.wait(timeout=SHUTDOWN + 2)
|
||
exited = time.monotonic() - sent
|
||
cancelled, headers, error = answer_of(stuck)
|
||
finally:
|
||
if server.poll() is None:
|
||
server.kill()
|
||
server.wait(timeout=5)
|
||
log.close()
|
||
print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с")
|
||
print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}")
|
||
print(f"запрос, тело которого так и не пришло: {cancelled or 'соединение закрыто без ответа'}")
|
||
print(f" content-type {headers.get('content-type')}, тело {error}")
|
||
assert during == (503, 200)
|
||
assert answer == "HTTP/1.1 201 Created"
|
||
assert cancelled == "HTTP/1.1 503 Service Unavailable"
|
||
assert headers["content-type"] == "application/json"
|
||
assert headers["x-instance-id"]
|
||
assert error["error"]["code"] == "unavailable"
|
||
assert error["error"]["request_id"] == headers["x-request-id"]
|
||
assert DRAIN - 0.2 <= closed < SHUTDOWN
|
||
assert exited < SHUTDOWN
|
||
assert code == 0
|