Carry source f591fa64e525679a31c9db7f942d36c789e1a12f forward from 72a1e57. Preserve lab4 attachment transactions, reconciliation locks, quarantine, and existing schema data through verified legacy adoption. Keep report and evidence work in progress untouched.
131 lines
4.6 KiB
Python
131 lines
4.6 KiB
Python
import json
|
||
import os
|
||
import signal
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import httpx
|
||
|
||
from conftest import DATABASE, free_port, new_user
|
||
from messenger.shared.infrastructure.readiness import DRAINING
|
||
|
||
DRAIN = 1.0
|
||
SHUTDOWN = 3
|
||
GUNICORN = (
|
||
"-m",
|
||
"gunicorn",
|
||
"--config",
|
||
"python:messenger.gunicorn_conf",
|
||
"messenger.main:create_app()",
|
||
)
|
||
|
||
|
||
async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None:
|
||
DRAINING.set()
|
||
try:
|
||
ready = await client.get("/health/ready")
|
||
live = await client.get("/health/live")
|
||
finally:
|
||
DRAINING.clear()
|
||
assert ready.status_code == 503
|
||
assert ready.json()["error"]["code"] == "unavailable"
|
||
assert ready.headers["X-Instance-Id"]
|
||
assert live.status_code == 200
|
||
assert (await client.get("/health/ready")).status_code == 200
|
||
|
||
|
||
def status(port: int, path: str) -> int:
|
||
try:
|
||
return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1, trust_env=False).status_code
|
||
except httpx.TransportError:
|
||
return 0
|
||
|
||
|
||
def answer_of(connection: socket.socket) -> tuple[str, dict[str, str], dict]:
|
||
data = b""
|
||
while chunk := connection.recv(4096):
|
||
data += chunk
|
||
connection.close()
|
||
head, _, body = data.partition(b"\r\n\r\n")
|
||
status, *lines = head.decode("latin-1").split("\r\n")
|
||
headers = dict(line.lower().split(": ", 1) for line in lines)
|
||
return status, headers, json.loads(body) if body else {}
|
||
|
||
|
||
def begin_registration(port: int) -> tuple[socket.socket, bytes]:
|
||
body = json.dumps(new_user("drain")).encode()
|
||
head = (
|
||
"POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n"
|
||
f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n"
|
||
)
|
||
connection = socket.create_connection(("127.0.0.1", port), timeout=5)
|
||
connection.sendall(head.encode() + body[:8])
|
||
return connection, body[8:]
|
||
|
||
|
||
def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests(tmp_path: Path) -> None:
|
||
port = free_port()
|
||
settings = {
|
||
"HOST": "127.0.0.1",
|
||
"PORT": str(port),
|
||
"WEB_CONCURRENCY": "2",
|
||
"DATABASE_URL": DATABASE["url"],
|
||
"CURSOR_SECRET": "test-secret-for-cursors",
|
||
"S3_ACCESS_KEY_ID": "test-key",
|
||
"S3_SECRET_ACCESS_KEY": "test-secret-key",
|
||
"LOG_LEVEL": "WARNING",
|
||
"DRAIN_SECONDS": str(DRAIN),
|
||
"SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN),
|
||
}
|
||
log_path = tmp_path / "gunicorn.log"
|
||
log = log_path.open("w")
|
||
server = subprocess.Popen(
|
||
[sys.executable, *GUNICORN],
|
||
env={**os.environ, **settings},
|
||
stdout=log,
|
||
stderr=log,
|
||
)
|
||
try:
|
||
deadline = time.monotonic() + 30
|
||
while status(port, "/health/ready") != 200:
|
||
assert server.poll() is None and time.monotonic() < deadline, log_path.read_text()
|
||
time.sleep(0.2)
|
||
pending, rest = begin_registration(port)
|
||
stuck, _ = begin_registration(port)
|
||
sent = time.monotonic()
|
||
server.send_signal(signal.SIGTERM)
|
||
time.sleep(0.3)
|
||
during = status(port, "/health/ready"), status(port, "/health/live")
|
||
while status(port, "/health/live") == 200:
|
||
assert time.monotonic() - sent < SHUTDOWN + 2, log_path.read_text()
|
||
time.sleep(0.05)
|
||
closed = time.monotonic() - sent
|
||
pending.sendall(rest)
|
||
answer = pending.recv(64).split(b"\r\n")[0].decode()
|
||
pending.close()
|
||
code = server.wait(timeout=SHUTDOWN + 2)
|
||
exited = time.monotonic() - sent
|
||
cancelled, headers, error = answer_of(stuck)
|
||
finally:
|
||
if server.poll() is None:
|
||
server.kill()
|
||
server.wait(timeout=5)
|
||
log.close()
|
||
print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с")
|
||
print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}")
|
||
print(f"запрос, тело которого так и не пришло: {cancelled or 'соединение закрыто без ответа'}")
|
||
print(f" content-type {headers.get('content-type')}, тело {error}")
|
||
assert during == (503, 200)
|
||
assert answer == "HTTP/1.1 201 Created"
|
||
assert cancelled == "HTTP/1.1 503 Service Unavailable"
|
||
assert headers["content-type"] == "application/json"
|
||
assert headers["x-instance-id"]
|
||
assert error["error"]["code"] == "unavailable"
|
||
assert error["error"]["request_id"] == headers["x-request-id"]
|
||
assert DRAIN - 0.2 <= closed < SHUTDOWN
|
||
assert exited < SHUTDOWN
|
||
assert code == 0
|