forked from mtusi2026/mtusi2026lab4
325 lines
15 KiB
Python
325 lines
15 KiB
Python
|
||
import base64
|
||
import csv
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||
LB_STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv")
|
||
PASSWORD = "Test-password-123"
|
||
|
||
|
||
class Response:
|
||
def __init__(self, status, headers, data):
|
||
self.status = status
|
||
self.headers = {name.lower(): value for name, value in headers.items()}
|
||
self.data = data
|
||
try:
|
||
self.json = json.loads(data.decode("utf-8")) if data else None
|
||
except ValueError:
|
||
self.json = None
|
||
|
||
@property
|
||
def instance(self):
|
||
"""X-Instance-Id: какая реплика API ответила (ID её контейнера)."""
|
||
return self.headers.get("x-instance-id")
|
||
|
||
|
||
def check(condition, message):
|
||
if not condition:
|
||
raise AssertionError(message)
|
||
|
||
|
||
def call(method, path, expected=200, *, body=None, raw=None, token=None, headers=None, timeout=30):
|
||
"""HTTP-запрос к API. expected — код или кортеж кодов, иначе AssertionError.
|
||
|
||
Заодно проверяет то, что контракт требует от любого ответа:
|
||
X-Request-Id, X-Instance-Id (с lab3), единый формат ошибки, WWW-Authenticate у 401.
|
||
"""
|
||
all_headers = dict(headers or {})
|
||
data = raw
|
||
if body is not None:
|
||
data = json.dumps(body).encode("utf-8")
|
||
if data is not None:
|
||
all_headers.setdefault("Content-Type", "application/json")
|
||
if token is not None:
|
||
all_headers["Authorization"] = "Bearer " + token
|
||
|
||
request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers)
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=timeout) as reply:
|
||
response = Response(reply.status, reply.headers, reply.read())
|
||
except urllib.error.HTTPError as error:
|
||
response = Response(error.code, error.headers, error.read())
|
||
|
||
expected = expected if isinstance(expected, tuple) else (expected,)
|
||
check(response.status in expected,
|
||
f"{method} {path}: expected {expected}, got {response.status} {response.data[:200]!r}")
|
||
request_id = response.headers.get("x-request-id")
|
||
check(request_id, f"{method} {path}: no X-Request-Id header")
|
||
check(response.instance, f"{method} {path}: no X-Instance-Id header")
|
||
if response.status >= 400:
|
||
error = (response.json or {}).get("error") or {}
|
||
check(error.get("code") and error.get("message"), f"{method} {path}: wrong error format")
|
||
check(error.get("request_id") == request_id, f"{method} {path}: request_id differs from header")
|
||
if response.status == 401:
|
||
check(response.headers.get("www-authenticate"), f"{method} {path}: 401 without WWW-Authenticate")
|
||
return response
|
||
|
||
|
||
def try_call(method, path, *, body=None, token=None, headers=None, timeout=2):
|
||
"""Запрос без проверок и без исключений — для нагрузки во время отказа.
|
||
|
||
Возвращает Response или None, если соединение оборвалось / вышел timeout.
|
||
"""
|
||
all_headers = dict(headers or {})
|
||
data = None
|
||
if body is not None:
|
||
data = json.dumps(body).encode("utf-8")
|
||
all_headers["Content-Type"] = "application/json"
|
||
if token is not None:
|
||
all_headers["Authorization"] = "Bearer " + token
|
||
request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers)
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=timeout) as reply:
|
||
return Response(reply.status, reply.headers, reply.read())
|
||
except urllib.error.HTTPError as error:
|
||
try:
|
||
return Response(error.code, error.headers, error.read())
|
||
except OSError:
|
||
return None
|
||
except (urllib.error.URLError, OSError):
|
||
return None
|
||
|
||
|
||
def basic(username, password):
|
||
"""Значение заголовка Authorization: Basic base64(username:password), пароль в UTF-8."""
|
||
return "Basic " + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii")
|
||
|
||
|
||
def login(username, password, expected=201):
|
||
return call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(username, password)})
|
||
|
||
|
||
def new_user(prefix="user", password=PASSWORD):
|
||
"""Регистрирует пользователя и входит. Возвращает dict: id, username, password, token."""
|
||
username = f"{prefix}_{uuid.uuid4().hex[:10]}"
|
||
user = call("POST", "/api/v1/users", 201,
|
||
body={"username": username, "display_name": prefix.title(), "password": password}).json
|
||
session = login(username, password).json
|
||
return {**user, "password": password, "token": session["token"]}
|
||
|
||
|
||
def new_chat(owner, *others, title="Test chat"):
|
||
member_ids = [owner["id"]] + [user["id"] for user in others]
|
||
return call("POST", "/api/v1/chats", 201, token=owner["token"],
|
||
body={"title": title, "member_ids": member_ids}).json
|
||
|
||
|
||
def new_key():
|
||
return uuid.uuid4().hex
|
||
|
||
|
||
def send(user, chat_id, text, expected=201, key=None):
|
||
"""POST сообщения. С lab3 Idempotency-Key обязателен: если не задан, берём новый случайный."""
|
||
return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"],
|
||
body={"text": text}, headers={"Idempotency-Key": key or new_key()})
|
||
|
||
|
||
def post_message(user, chat_id, text="", attachment_ids=None, expected=201, key=None):
|
||
"""POST сообщения с вложениями (lab4). attachment_ids=None — поле не передаётся."""
|
||
body = {"text": text}
|
||
if attachment_ids is not None:
|
||
body["attachment_ids"] = attachment_ids
|
||
return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"],
|
||
body=body, headers={"Idempotency-Key": key or new_key()})
|
||
|
||
|
||
def upload(user, chat_id, data, content_type="image/png", filename="photo.png", expected=201, timeout=60):
|
||
"""POST /chats/{id}/attachments сырыми байтами. filename=None — без параметра."""
|
||
query = "" if filename is None else "?filename=" + urllib.parse.quote(filename, safe="")
|
||
headers = {"Content-Type": content_type} if content_type is not None else {}
|
||
return call("POST", f"/api/v1/chats/{chat_id}/attachments{query}", expected, raw=data,
|
||
token=user["token"] if user else None, headers=headers, timeout=timeout)
|
||
|
||
|
||
def read_history(user, chat_id, limit=100):
|
||
"""Вся история чата по страницам. Возвращает (сообщения, число_страниц)."""
|
||
items, pages, cursor = [], 0, None
|
||
while True:
|
||
query = f"?limit={limit}" + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "")
|
||
page = call("GET", f"/api/v1/chats/{chat_id}/messages{query}", 200, token=user["token"]).json
|
||
items += page["items"]
|
||
pages += 1
|
||
cursor = page["next_cursor"]
|
||
if cursor is None:
|
||
return items, pages
|
||
|
||
|
||
def wait_ready(seconds=90):
|
||
deadline = time.monotonic() + seconds
|
||
while True:
|
||
try:
|
||
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as reply:
|
||
if reply.status == 200:
|
||
return
|
||
except (urllib.error.URLError, OSError):
|
||
pass
|
||
check(time.monotonic() < deadline, f"API is not ready after {seconds} s")
|
||
time.sleep(0.5)
|
||
|
||
|
||
def compose(*args, env=None):
|
||
"""docker compose <args> в корне репозитория. Вывод скрыт: в нём могут быть пути к секретам."""
|
||
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, env=env,
|
||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||
|
||
|
||
def compose_output(*args, env=None):
|
||
"""docker compose <args> и его вывод (для logs, ps)."""
|
||
result = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, check=True, env=env)
|
||
return result.stdout.decode("utf-8", "replace") + result.stderr.decode("utf-8", "replace")
|
||
|
||
|
||
def docker(*args, check_exit=True):
|
||
"""docker <args>, возвращает stdout."""
|
||
result = subprocess.run(["docker", *args], cwd=ROOT, capture_output=True, check=check_exit)
|
||
return result.stdout.decode("utf-8", "replace").strip()
|
||
|
||
|
||
def api_containers(include_stopped=False):
|
||
"""Полные ID контейнеров сервиса api (по умолчанию только запущенные)."""
|
||
args = ["ps", "-q", "-a", "api"] if include_stopped else ["ps", "-q", "api"]
|
||
return sorted(compose_output(*args).split())
|
||
|
||
|
||
def health(container):
|
||
"""Статус Docker healthcheck контейнера: starting / healthy / unhealthy / ''."""
|
||
return docker("inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", container,
|
||
check_exit=False)
|
||
|
||
|
||
def lb_servers():
|
||
"""Состояние слотов backend api в HAProxy (страница статистики, CSV): {имя: статус}."""
|
||
with urllib.request.urlopen(LB_STATS_URL, timeout=2) as reply:
|
||
rows = list(csv.reader(io.StringIO(reply.read().decode("utf-8", "replace"))))
|
||
header = [name.lstrip("# ") for name in rows[0]]
|
||
servers = {}
|
||
for row in rows[1:]:
|
||
item = dict(zip(header, row))
|
||
if item.get("pxname") == "api" and item.get("svname") not in ("FRONTEND", "BACKEND"):
|
||
servers[item["svname"]] = item.get("status", "")
|
||
return servers
|
||
|
||
|
||
def lb_up_count():
|
||
"""Сколько основных слотов (api1..api6, не аварийных) HAProxy считает готовыми (UP)."""
|
||
return sum(1 for name, status in lb_servers().items() if name.startswith("api") and status.startswith("UP"))
|
||
|
||
|
||
def seen_instances(token, requests=20):
|
||
"""Какие реплики отвечают через балансировщик: X-Instance-Id у серии GET /users/me."""
|
||
return {call("GET", "/api/v1/users/me", 200, token=token).instance for _ in range(requests)}
|
||
|
||
|
||
def wait_balanced(replicas, seconds=60):
|
||
"""Ждёт, пока все replicas контейнеров api healthy и HAProxy держит их в балансировке."""
|
||
deadline = time.monotonic() + seconds
|
||
while True:
|
||
running = api_containers()
|
||
healthy = [container for container in running if health(container) == "healthy"]
|
||
try:
|
||
up = lb_up_count()
|
||
except OSError:
|
||
up = 0
|
||
if len(healthy) >= replicas and up >= replicas:
|
||
return running
|
||
check(time.monotonic() < deadline,
|
||
f"after {seconds} s: {len(healthy)} healthy api containers, {up} UP in HAProxy, need {replicas}")
|
||
time.sleep(0.5)
|
||
|
||
|
||
def redis_cli(*args):
|
||
"""redis-cli внутри контейнера redis. Пароль читается там же из секрета и не попадает в вывод."""
|
||
command = "REDISCLI_AUTH=\"$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli " + " ".join(args)
|
||
result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", command],
|
||
cwd=ROOT, capture_output=True, check=True)
|
||
return result.stdout.decode("utf-8", "replace").strip()
|
||
|
||
|
||
def session_key(token):
|
||
"""Ключ сессии в Redis, как его считает auth.py: session:<SHA-256 токена>."""
|
||
return "session:" + hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def psql(sql):
|
||
"""Выполняет SQL в контейнере postgres. Возвращает (код выхода, вывод, текст ошибки).
|
||
|
||
Значения в SQL тестов — только сгенерированные самим тестом (uuid), не ввод пользователя.
|
||
"""
|
||
result = subprocess.run(
|
||
["docker", "compose", "exec", "-T", "postgres",
|
||
"psql", "-U", "messenger", "-d", "messenger", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", sql],
|
||
cwd=ROOT, capture_output=True, check=False,
|
||
)
|
||
output = result.stdout.decode("utf-8", "replace").strip()
|
||
error = result.stderr.decode("utf-8", "replace").strip()
|
||
return result.returncode, output, error
|
||
|
||
|
||
def app_python(code, *args, service="api", env=None):
|
||
"""Выполнить Python-скрипт внутри контейнера приложения (там есть модули API и S3-ключи API).
|
||
|
||
Скрипт передаётся через stdin, секреты остаются в контейнере. Возвращает stdout (последняя строка — JSON).
|
||
"""
|
||
command = ["docker", "compose", "exec", "-T"]
|
||
for name, value in (env or {}).items():
|
||
command += ["-e", f"{name}={value}"]
|
||
command += [service, "python", "-", *map(str, args)]
|
||
result = subprocess.run(command, cwd=ROOT, input=code.encode("utf-8"), capture_output=True, check=False)
|
||
output = result.stdout.decode("utf-8", "replace").strip()
|
||
check(result.returncode == 0, f"script in {service} failed: {result.stderr.decode('utf-8', 'replace')[-600:]}")
|
||
return output
|
||
|
||
|
||
def janitor(*args, env=None):
|
||
"""Один проход janitor.py в контейнере janitor. Возвращает его JSON-отчёт."""
|
||
command = ["docker", "compose", "exec", "-T"]
|
||
for name, value in (env or {}).items():
|
||
command += ["-e", f"{name}={value}"]
|
||
command += ["janitor", "python", "janitor.py", "--once", *map(str, args)]
|
||
result = subprocess.run(command, cwd=ROOT, capture_output=True, check=False)
|
||
output = result.stdout.decode("utf-8", "replace").strip()
|
||
check(result.returncode == 0, f"janitor failed: {result.stderr.decode('utf-8', 'replace')[-600:]}")
|
||
return json.loads(output.splitlines()[-1])
|
||
|
||
|
||
def sql_value(query):
|
||
"""Одно значение из PostgreSQL (psql -A -t)."""
|
||
code, output, error = psql(query)
|
||
check(code == 0, f"psql failed: {error[-300:]}")
|
||
return output
|
||
|
||
|
||
def run_tests(tests):
|
||
"""Запускает функции-тесты по очереди. Код выхода 1, если хоть один упал."""
|
||
failed = 0
|
||
for test in tests:
|
||
try:
|
||
test()
|
||
except Exception as exc: # noqa: BLE001 — нужно напечатать любую ошибку теста
|
||
failed += 1
|
||
print(f"FAIL: {test.__name__}: {exc}")
|
||
print(f"{len(tests) - failed} passed, {failed} failed")
|
||
raise SystemExit(1 if failed else 0)
|