Keep the message and idempotency receipt in one Core CTE statement with chat-first locking. Preserve rollback, bounded dependency failures, and domain ports. Replace the SQL runner with Alembic. Adopt legacy databases only after validating their journal and PostgreSQL schema, retain all data, and verify migrations on fresh and populated databases.
128 lines
4.4 KiB
Python
128 lines
4.4 KiB
Python
import json
|
||
import os
|
||
import signal
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import httpx
|
||
|
||
from conftest import DATABASE, new_user
|
||
from messenger.shared.infrastructure.readiness import DRAINING
|
||
|
||
DRAIN = 1.0
|
||
SHUTDOWN = 3
|
||
GUNICORN = (
|
||
"-m",
|
||
"gunicorn",
|
||
"--config",
|
||
"python:messenger.gunicorn_conf",
|
||
"messenger.main:create_app()",
|
||
)
|
||
|
||
|
||
async def test_draining_replica_is_not_ready_but_alive(client: httpx.AsyncClient) -> None:
|
||
DRAINING.set()
|
||
try:
|
||
ready = await client.get("/health/ready")
|
||
live = await client.get("/health/live")
|
||
finally:
|
||
DRAINING.clear()
|
||
assert ready.status_code == 503
|
||
assert ready.json()["error"]["code"] == "unavailable"
|
||
assert ready.headers["X-Instance-Id"]
|
||
assert live.status_code == 200
|
||
assert (await client.get("/health/ready")).status_code == 200
|
||
|
||
|
||
def status(port: int, path: str) -> int:
|
||
try:
|
||
return httpx.get(f"http://127.0.0.1:{port}{path}", timeout=1, trust_env=False).status_code
|
||
except httpx.TransportError:
|
||
return 0
|
||
|
||
|
||
def answer_of(connection: socket.socket) -> tuple[str, dict[str, str], dict]:
|
||
data = b""
|
||
while chunk := connection.recv(4096):
|
||
data += chunk
|
||
connection.close()
|
||
head, _, body = data.partition(b"\r\n\r\n")
|
||
status, *lines = head.decode("latin-1").split("\r\n")
|
||
headers = dict(line.lower().split(": ", 1) for line in lines)
|
||
return status, headers, json.loads(body) if body else {}
|
||
|
||
|
||
def begin_registration(port: int) -> tuple[socket.socket, bytes]:
|
||
body = json.dumps(new_user("drain")).encode()
|
||
head = (
|
||
"POST /api/v1/users HTTP/1.1\r\nHost: replica\r\nContent-Type: application/json\r\n"
|
||
f"Content-Length: {len(body)}\r\nConnection: close\r\n\r\n"
|
||
)
|
||
connection = socket.create_connection(("127.0.0.1", port), timeout=5)
|
||
connection.sendall(head.encode() + body[:8])
|
||
return connection, body[8:]
|
||
|
||
|
||
def test_sigterm_leaves_rotation_then_closes_and_cancels_stuck_requests(tmp_path: Path) -> None:
|
||
port = 18083
|
||
settings = {
|
||
"HOST": "127.0.0.1",
|
||
"PORT": str(port),
|
||
"WEB_CONCURRENCY": "2",
|
||
"DATABASE_URL": DATABASE["url"],
|
||
"CURSOR_SECRET": "test-secret-for-cursors",
|
||
"LOG_LEVEL": "WARNING",
|
||
"DRAIN_SECONDS": str(DRAIN),
|
||
"SHUTDOWN_TIMEOUT_SECONDS": str(SHUTDOWN),
|
||
}
|
||
log_path = tmp_path / "gunicorn.log"
|
||
log = log_path.open("w")
|
||
server = subprocess.Popen(
|
||
[sys.executable, *GUNICORN],
|
||
env={**os.environ, **settings},
|
||
stdout=log,
|
||
stderr=log,
|
||
)
|
||
try:
|
||
deadline = time.monotonic() + 30
|
||
while status(port, "/health/ready") != 200:
|
||
assert server.poll() is None and time.monotonic() < deadline, log_path.read_text()
|
||
time.sleep(0.2)
|
||
pending, rest = begin_registration(port)
|
||
stuck, _ = begin_registration(port)
|
||
sent = time.monotonic()
|
||
server.send_signal(signal.SIGTERM)
|
||
time.sleep(0.3)
|
||
during = status(port, "/health/ready"), status(port, "/health/live")
|
||
while status(port, "/health/live") == 200:
|
||
time.sleep(0.05)
|
||
closed = time.monotonic() - sent
|
||
pending.sendall(rest)
|
||
answer = pending.recv(64).split(b"\r\n")[0].decode()
|
||
pending.close()
|
||
code = server.wait(timeout=SHUTDOWN + 2)
|
||
exited = time.monotonic() - sent
|
||
cancelled, headers, error = answer_of(stuck)
|
||
finally:
|
||
if server.poll() is None:
|
||
server.kill()
|
||
server.wait(timeout=5)
|
||
log.close()
|
||
print(f"окно: ready/live {during}, порт закрыт через {closed:.2f} с, выход {exited:.2f} с")
|
||
print(f"запрос, принятый до SIGTERM и дописанный после закрытия порта: {answer}")
|
||
print(f"запрос, тело которого так и не пришло: {cancelled or 'соединение закрыто без ответа'}")
|
||
print(f" content-type {headers.get('content-type')}, тело {error}")
|
||
assert during == (503, 200)
|
||
assert answer == "HTTP/1.1 201 Created"
|
||
assert cancelled == "HTTP/1.1 503 Service Unavailable"
|
||
assert headers["content-type"] == "application/json"
|
||
assert headers["x-instance-id"]
|
||
assert error["error"]["code"] == "unavailable"
|
||
assert error["error"]["request_id"] == headers["x-request-id"]
|
||
assert DRAIN - 0.2 <= closed < SHUTDOWN
|
||
assert exited < SHUTDOWN
|
||
assert code == 0
|