262 lines
11 KiB
Python
262 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""Проверки реального nginx в отдельном Compose-проекте без БД и приложения."""
|
||
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
import gzip
|
||
import json
|
||
from pathlib import Path
|
||
import subprocess
|
||
import tempfile
|
||
import time
|
||
from urllib.error import HTTPError, URLError
|
||
from urllib.request import Request, urlopen
|
||
from uuid import uuid4
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
IMAGE = "nginx:1.28.0-alpine"
|
||
PAYLOAD = json.dumps({"text": "message " * 256}).encode()
|
||
|
||
|
||
def check(condition: bool, message: str) -> None:
|
||
if not condition:
|
||
raise AssertionError(message)
|
||
|
||
|
||
class Stand:
|
||
def __init__(self, directory: str) -> None:
|
||
self.directory = Path(directory)
|
||
self.project = "messenger-nginx-check-" + uuid4().hex[:12]
|
||
self.base_url = ""
|
||
backend = self.directory / "backend.conf"
|
||
backend.write_text(
|
||
"worker_processes 1;\n"
|
||
"events { worker_connections 1024; }\n"
|
||
"http {\n"
|
||
" log_format backend 'BACKEND $request_method $uri';\n"
|
||
" access_log /dev/stdout backend;\n"
|
||
" server {\n"
|
||
" listen 8080;\n"
|
||
" default_type application/json;\n"
|
||
" add_header X-Instance-Id $hostname always;\n"
|
||
" add_header X-Request-Id $request_id always;\n"
|
||
" add_header Retry-After 7 always;\n"
|
||
' location /health/ { return 200 \'{"status":"ok"}\'; }\n'
|
||
" location = /api/limited {\n"
|
||
' return 429 \'{"error":{"code":"rate_limited",'
|
||
'"message":"upstream limit","request_id":"$request_id"}}\';\n'
|
||
" }\n"
|
||
" location = /api/post-failure {\n"
|
||
' return 503 \'{"error":{"code":"unavailable",'
|
||
'"message":"upstream failure","request_id":"$request_id"}}\';\n'
|
||
" }\n"
|
||
" location / { return 200 '" + PAYLOAD.decode() + "'; }\n"
|
||
" }\n"
|
||
"}\n"
|
||
)
|
||
self.compose_file = self.directory / "compose.json"
|
||
self.compose_file.write_text(
|
||
json.dumps(
|
||
{
|
||
"services": {
|
||
"api": {
|
||
"image": IMAGE,
|
||
"volumes": [f"{backend}:/etc/nginx/nginx.conf:ro"],
|
||
},
|
||
"nginx": {
|
||
"image": IMAGE,
|
||
"volumes": [
|
||
f"{ROOT / 'infra/nginx/nginx.conf'}:/etc/nginx/nginx.conf:ro"
|
||
],
|
||
"ports": ["127.0.0.1::8080"],
|
||
},
|
||
}
|
||
}
|
||
)
|
||
)
|
||
|
||
def compose(self, *args: str) -> str:
|
||
# Явный файл, уникальный проект и пустой env-file исключают основной стенд.
|
||
result = subprocess.run(
|
||
[
|
||
"docker",
|
||
"compose",
|
||
"--env-file",
|
||
"/dev/null",
|
||
"-f",
|
||
str(self.compose_file),
|
||
"-p",
|
||
self.project,
|
||
*args,
|
||
],
|
||
cwd=self.directory,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=45,
|
||
)
|
||
if result.returncode:
|
||
raise RuntimeError(result.stdout + result.stderr)
|
||
return result.stdout + result.stderr
|
||
|
||
def call(self, path: str, *, method: str = "GET", encoding: str = "identity"):
|
||
request = Request(
|
||
self.base_url + path,
|
||
method=method,
|
||
headers={"Accept-Encoding": encoding},
|
||
data=b"{}" if method == "POST" else None,
|
||
)
|
||
try:
|
||
response = urlopen(request, timeout=3)
|
||
except HTTPError as error:
|
||
response = error
|
||
except (URLError, TimeoutError):
|
||
return 0, {}, b""
|
||
with response:
|
||
return (
|
||
response.status,
|
||
{key.lower(): value for key, value in response.headers.items()},
|
||
response.read(),
|
||
)
|
||
|
||
def wait_instances(self, count: int) -> set[str]:
|
||
seen: set[str] = set()
|
||
deadline = time.monotonic() + 15
|
||
while time.monotonic() < deadline:
|
||
status, headers, _ = self.call("/health/live")
|
||
if status == 200:
|
||
seen.add(headers["x-instance-id"])
|
||
if len(seen) == count:
|
||
return seen
|
||
time.sleep(0.05)
|
||
raise AssertionError(f"Ожидалось {count} реплик, наблюдалось {len(seen)}")
|
||
|
||
|
||
def check_error(response, status: int, code: str) -> None:
|
||
actual_status, headers, body = response
|
||
check(actual_status == status, f"Ожидался HTTP {status}, получен {actual_status}")
|
||
check(
|
||
headers.get("content-type", "").startswith("application/json"), "Ошибка не JSON"
|
||
)
|
||
error = json.loads(body)["error"]
|
||
check(error["code"] == code, "Неверный код ошибки")
|
||
check(error["request_id"] == headers.get("x-request-id"), "request_id не совпадает")
|
||
|
||
|
||
def run_checks(stand: Stand) -> None:
|
||
stand.compose("up", "-d", "--pull", "never", "--scale", "api=2", "api", "nginx")
|
||
address = stand.compose("port", "nginx", "8080").strip()
|
||
stand.base_url = "http://" + address
|
||
stand.wait_instances(2)
|
||
print(stand.compose("exec", "-T", "nginx", "nginx", "-v").strip())
|
||
stand.compose("exec", "-T", "nginx", "nginx", "-t")
|
||
print("PASS: nginx -t и балансировка двух реплик")
|
||
|
||
status, headers, body = stand.call("/api/large", encoding="gzip")
|
||
check(
|
||
status == 200 and headers.get("content-encoding") == "gzip", "Нет gzip для JSON"
|
||
)
|
||
check(gzip.decompress(body) == PAYLOAD, "gzip изменил содержимое")
|
||
check(len(body) < len(PAYLOAD), "gzip не уменьшил тестовый ответ")
|
||
check(
|
||
"accept-encoding" in headers.get("vary", "").lower(),
|
||
"Нет Vary: Accept-Encoding",
|
||
)
|
||
check(stand.call("/api/large")[2] == PAYLOAD, "Ответ без gzip изменён")
|
||
small = stand.call("/health/live", encoding="gzip")
|
||
check(
|
||
small[0] == 200 and "content-encoding" not in small[1], "Маленький ответ сжат"
|
||
)
|
||
print(f"PASS: gzip {len(PAYLOAD)} → {len(body)} байт, содержимое сохранено")
|
||
|
||
limited = stand.call("/api/limited")
|
||
check_error(limited, 429, "rate_limited")
|
||
check(limited[1].get("retry-after") == "7", "Подменён Retry-After приложения")
|
||
check("x-instance-id" in limited[1], "Потерян id backend на ошибке API")
|
||
print("PASS: JSON и заголовки ошибки API передаются без подмены")
|
||
|
||
failure = stand.call("/api/post-failure", method="POST")
|
||
check_error(failure, 503, "unavailable")
|
||
backend_logs = stand.compose("logs", "--no-color", "--no-log-prefix", "api")
|
||
check(backend_logs.count("BACKEND POST /api/post-failure") == 1, "POST повторился")
|
||
print("PASS: отправленный POST не повторяется на другой реплике")
|
||
|
||
with ThreadPoolExecutor(max_workers=32) as pool:
|
||
responses = list(pool.map(lambda _: stand.call("/api/large"), range(500)))
|
||
rejected = [response for response in responses if response[0] == 429]
|
||
check(bool(rejected), "Не удалось вызвать ограничение частоты")
|
||
check(
|
||
all(response[0] in (200, 429) for response in responses), "Ошибка при нагрузке"
|
||
)
|
||
for response in rejected:
|
||
check_error(response, 429, "rate_limited")
|
||
check(response[1].get("retry-after") == "1", "Нет Retry-After балансировщика")
|
||
check("x-instance-id" not in response[1], "LB выдал себя за API-реплику")
|
||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||
health = list(pool.map(lambda _: stand.call("/health/ready"), range(100)))
|
||
check(all(response[0] == 200 for response in health), "Лимит затронул health")
|
||
check(stand.call("/lb-health")[0] == 200, "Проверка LB недоступна")
|
||
print(f"PASS: лимит вернул {len(rejected)} ответов 429; health доступен")
|
||
|
||
stand.compose(
|
||
"up", "-d", "--pull", "never", "--scale", "api=3", "--no-recreate", "api"
|
||
)
|
||
instances = stand.wait_instances(3)
|
||
print("PASS: третья реплика обнаружена через DNS без reload nginx")
|
||
|
||
target = sorted(instances)[0]
|
||
containers = stand.compose("ps", "-q", "api").splitlines()
|
||
matching = [container for container in containers if container.startswith(target)]
|
||
check(len(matching) == 1, "Не найдена тестовая реплика для остановки")
|
||
# Все id взяты только из уникального временного Compose-проекта.
|
||
started = time.monotonic()
|
||
subprocess.run(
|
||
["docker", "stop", "--time", "1", matching[0]], check=True, capture_output=True
|
||
)
|
||
recovery = None
|
||
for _ in range(20):
|
||
status, headers, _ = stand.call("/health/live")
|
||
if status == 200:
|
||
check(headers["x-instance-id"] != target, "Остановленная реплика отвечает")
|
||
if recovery is None:
|
||
recovery = time.monotonic() - started
|
||
check(recovery is not None and recovery < 10, "Чтение не восстановилось за 10 с")
|
||
print(f"PASS: отказ реплики, первое успешное чтение через {recovery:.3f} с")
|
||
|
||
stand.compose("stop", "--timeout", "1", "api")
|
||
unavailable = stand.call("/health/ready")
|
||
check_error(unavailable, 503, "unavailable")
|
||
check(
|
||
unavailable[1].get("retry-after") == "1", "Нет Retry-After при отказе всех API"
|
||
)
|
||
check("x-instance-id" not in unavailable[1], "У ошибки LB есть id реплики")
|
||
check(stand.call("/lb-health")[0] == 200, "Отказ API сломал проверку самого LB")
|
||
print("PASS: недоступность всех backend возвращает JSON 503")
|
||
|
||
logs = stand.compose("logs", "--no-color", "--no-log-prefix", "nginx")
|
||
entries = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
|
||
check(bool(entries), "Нет JSON access log")
|
||
check(
|
||
all("request_id" in entry and "upstream_time" in entry for entry in entries),
|
||
"Неполный лог",
|
||
)
|
||
check(
|
||
any(entry["limit_status"] == "REJECTED" for entry in entries),
|
||
"Лимит не отражён в логе",
|
||
)
|
||
print("PASS: JSON access log содержит время backend и результат лимита")
|
||
|
||
|
||
def main() -> None:
|
||
with tempfile.TemporaryDirectory(prefix="messenger-nginx-check-") as directory:
|
||
stand = Stand(directory)
|
||
try:
|
||
run_checks(stand)
|
||
except Exception:
|
||
print(stand.compose("logs", "--no-color", "--tail", "20"), flush=True)
|
||
raise
|
||
finally:
|
||
stand.compose("down", "--volumes", "--remove-orphans", "--timeout", "1")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|