forked from mtusi2026/mtusi2026lab5
feat: перенос логики с 4лабф
This commit is contained in:
Executable
+59
@@ -0,0 +1,59 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Миграции с нуля и повторно, отказ API на непримененной схеме: отдельный
|
||||
# compose-проект с собственным пустым volume. Рабочий стенд и его данные
|
||||
# не затрагиваются; временный volume этого проекта удаляется в конце.
|
||||
cd "$(dirname "$0")/.."
|
||||
PROJECT=messenger-lab5-migrations-check
|
||||
# Свои порты, чтобы не конфликтовать с основным стендом
|
||||
export LB_PORT="${MIGRATIONS_CHECK_LB_PORT:-58080}"
|
||||
export LB_EXPIRY_PORT="${MIGRATIONS_CHECK_LB_EXPIRY_PORT:-58081}"
|
||||
export DB_HOST_PORT="${MIGRATIONS_CHECK_DB_HOST_PORT:-55434}"
|
||||
COMPOSE="docker compose --env-file ${ENV_FILE:-.env} -p $PROJECT"
|
||||
trap '$COMPOSE down -v --remove-orphans >/dev/null 2>&1' EXIT
|
||||
|
||||
$COMPOSE up -d --wait postgres >/dev/null 2>&1
|
||||
# Тот же образ и сеть, что у сервиса migrate; apply_migrations трижды подряд
|
||||
$COMPOSE run --rm --no-deps -T migrate python - <<'EOF'
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "src")
|
||||
|
||||
from infrastructure.dependencies.database import database_pool_manager as db
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
await db.create_pool(30)
|
||||
async with db.acquire() as conn:
|
||||
tables = await conn.fetchval(
|
||||
"SELECT count(*) FROM pg_tables WHERE schemaname = 'public';"
|
||||
)
|
||||
print(f"пустой volume: таблиц в public = {tables}")
|
||||
# Так стартует API-реплика: без migrate она отказывается работать
|
||||
try:
|
||||
await db.check_migrations()
|
||||
except RuntimeError as error:
|
||||
print(f"API до migrate: {error}")
|
||||
else:
|
||||
raise SystemExit("FAIL: API стартовал бы на пустой схеме")
|
||||
for attempt in (1, 2, 3):
|
||||
await db.apply_migrations()
|
||||
async with db.acquire() as conn:
|
||||
versions = await conn.fetch(
|
||||
"SELECT version, name FROM schema_migrations ORDER BY version;"
|
||||
)
|
||||
tables = await conn.fetchval(
|
||||
"SELECT count(*) FROM pg_tables WHERE schemaname = 'public';"
|
||||
)
|
||||
print(
|
||||
f"запуск {attempt}: schema_migrations = "
|
||||
f"{[(r['version'], r['name']) for r in versions]}, таблиц = {tables}"
|
||||
)
|
||||
await db.check_migrations()
|
||||
print("API после migrate: схема совместима")
|
||||
await db.close()
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
EOF
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Совместимость текущей схемы с предыдущей версией API — условие rolling update:
|
||||
# миграции применяются до замены реплик, и старые реплики продолжают работать.
|
||||
# Образ собирается из коммита PREVIOUS_REVISION (по умолчанию lab2-код aa971fb),
|
||||
# запускается одним процессом в сети backend рабочего стенда (наружу порт
|
||||
# не публикуется) и выполняет вход, создание чата, запись и чтение сообщения.
|
||||
# Создаются только синтетические данные; volumes не затрагиваются.
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
REVISION="${PREVIOUS_REVISION:-aa971fb}"
|
||||
IMAGE="mtusi-messenger:compat-$REVISION"
|
||||
PROJECT=$(docker compose --env-file "$ENV_FILE" config --format json | python3 -c \
|
||||
'import json, sys; print(json.load(sys.stdin)["name"])')
|
||||
NETWORK="${PROJECT}_backend"
|
||||
|
||||
git archive "$REVISION" | docker build --quiet --tag "$IMAGE" - >/dev/null
|
||||
echo "previous version image: $IMAGE ($(git log -1 --format='%h %s' "$REVISION"))"
|
||||
echo "schema on stand: $(docker compose --env-file "$ENV_FILE" exec -T postgres sh -c \
|
||||
'psql -X -q -tA -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT string_agg(name, \$\$, \$\$ ORDER BY version) FROM schema_migrations;"')"
|
||||
|
||||
docker run --rm -i --network "$NETWORK" --env-file "$ENV_FILE" \
|
||||
-e DB_HOST=postgres -e DB_PORT=5432 -e VALKEY_HOST=valkey -e VALKEY_PORT=6379 \
|
||||
"$IMAGE" sh -c 'python src/gateway.py >/tmp/api.log 2>&1 & exec python -' <<'EOF'
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None):
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
request = urllib.request.Request(BASE + path, data=data, method=method)
|
||||
request.add_header("Content-Type", "application/json")
|
||||
for name, value in (headers or {}).items():
|
||||
request.add_header(name, value)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
return response.status, json.loads(response.read() or b"null")
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, None
|
||||
|
||||
|
||||
for _ in range(60):
|
||||
try:
|
||||
if call("GET", "/health/ready")[0] == 200:
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
print(open("/tmp/api.log").read()[-2000:])
|
||||
raise SystemExit("FAIL: previous version did not become ready on current schema")
|
||||
|
||||
username = "compat_" + uuid.uuid4().hex[:12]
|
||||
password = "Compat-" + uuid.uuid4().hex
|
||||
status_user, user = call(
|
||||
"POST", "/api/v1/users",
|
||||
{"username": username, "display_name": "Compat", "password": password},
|
||||
)
|
||||
credentials = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
status_login, session = call(
|
||||
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + credentials}
|
||||
)
|
||||
bearer = {"Authorization": "Bearer " + session["token"]}
|
||||
status_chat, chat = call(
|
||||
"POST", "/api/v1/chats", {"title": "compat", "member_ids": [user["id"]]}, bearer
|
||||
)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
status_send, message = call("POST", path, {"text": "старая версия пишет"}, bearer)
|
||||
status_read, page = call("GET", path, headers=bearer)
|
||||
result = {
|
||||
"ready": True,
|
||||
"register": status_user,
|
||||
"login": status_login,
|
||||
"create_chat": status_chat,
|
||||
"send_message_without_key": status_send,
|
||||
"read_history": status_read,
|
||||
"history_matches": page["items"] == [message],
|
||||
}
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
log = open("/tmp/api.log").read()
|
||||
print("startup log:", [line.split(" - ", 1)[-1] for line in log.splitlines() if "миграц" in line.lower() or "Схема" in line])
|
||||
if any(code >= 400 for code in result.values() if isinstance(code, int)) or not result["history_matches"]:
|
||||
raise SystemExit("FAIL: previous version is not compatible with current schema")
|
||||
print("PASS: previous API version works on the current schema (expand-only migration)")
|
||||
EOF
|
||||
@@ -0,0 +1,482 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Нагрузка через LB во время отказа/замены API-реплик; метрики и проверка данных.
|
||||
|
||||
failover — для каждой API-реплики по очереди: нагрузка → остановка этой реплики
|
||||
(docker stop = SIGTERM, затем docker kill = SIGKILL) → восстановление
|
||||
(scripts/restore.sh) → нагрузка после восстановления.
|
||||
rolling — нагрузка во время scripts/rolling-update.sh (поочерёдная замена реплик).
|
||||
|
||||
Клиент ведёт себя как настоящий: неподтверждённый POST повторяет с тем же
|
||||
Idempotency-Key и телом. После прогона вся история читается заново: каждое
|
||||
подтверждённое сообщение на месте и не изменено, дублей нет.
|
||||
Python 3.9+, стандартная библиотека; tokens не печатаются и не сохраняются.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client # noqa: E402
|
||||
|
||||
REQUEST_TIMEOUT_SEC = 5
|
||||
RETRY_PAUSE_SEC = 0.05
|
||||
STATS_FORMAT = "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Attempt:
|
||||
"""Одна HTTP-попытка клиента; время — секунды от начала прогона"""
|
||||
|
||||
op: str
|
||||
started: float
|
||||
finished: float
|
||||
status: int | None
|
||||
error: str | None
|
||||
instance: str | None
|
||||
retry: bool
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.status in (200, 201)
|
||||
|
||||
|
||||
class Recorder:
|
||||
def __init__(self) -> None:
|
||||
self.started = time.monotonic()
|
||||
self.lock = threading.Lock()
|
||||
self.attempts: list[Attempt] = []
|
||||
self.confirmed: list[dict] = []
|
||||
self.events: dict[str, float] = {}
|
||||
|
||||
def now(self) -> float:
|
||||
return time.monotonic() - self.started
|
||||
|
||||
def mark(self, name: str) -> None:
|
||||
self.events[name] = round(self.now(), 3)
|
||||
|
||||
def add(self, attempt: Attempt, message: dict | None = None) -> None:
|
||||
with self.lock:
|
||||
self.attempts.append(attempt)
|
||||
if message is not None:
|
||||
self.confirmed.append(message)
|
||||
|
||||
|
||||
def request(
|
||||
base: str, method: str, path: str, token: str, body: dict | None, key: str | None
|
||||
) -> tuple[int | None, str | None, str | None, dict | None]:
|
||||
headers = {"Authorization": "Bearer " + token, "Accept": "application/json"}
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode()
|
||||
headers["Content-Type"] = "application/json"
|
||||
if key is not None:
|
||||
headers["Idempotency-Key"] = key
|
||||
req = urllib.request.Request(base + path, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_SEC) as response:
|
||||
payload = json.loads(response.read())
|
||||
return response.status, None, response.headers.get("X-Instance-Id"), payload
|
||||
except urllib.error.HTTPError as error:
|
||||
error.read()
|
||||
return error.code, None, error.headers.get("X-Instance-Id"), None
|
||||
except OSError as error:
|
||||
return None, type(error).__name__, None, None
|
||||
|
||||
|
||||
def worker(
|
||||
base: str,
|
||||
token: str,
|
||||
chat_id: str,
|
||||
worker_id: int,
|
||||
recorder: Recorder,
|
||||
stop: threading.Event,
|
||||
) -> None:
|
||||
"""Запись с повтором по тому же ключу до подтверждения, затем чтение"""
|
||||
path = f"/api/v1/chats/{chat_id}/messages"
|
||||
index = 0
|
||||
while not stop.is_set():
|
||||
key = uuid.uuid4().hex
|
||||
body = {"text": f"exp-{worker_id}-{index}-{key[:8]}"}
|
||||
retry = False
|
||||
while not stop.is_set():
|
||||
started = recorder.now()
|
||||
status, error, instance, payload = request(
|
||||
base, "POST", path, token, body, key
|
||||
)
|
||||
attempt = Attempt(
|
||||
"write", started, recorder.now(), status, error, instance, retry
|
||||
)
|
||||
recorder.add(attempt, payload if status == 201 else None)
|
||||
if status == 201:
|
||||
break
|
||||
retry = True
|
||||
time.sleep(RETRY_PAUSE_SEC)
|
||||
started = recorder.now()
|
||||
status, error, instance, _ = request(
|
||||
base, "GET", path + "?limit=1", token, None, None
|
||||
)
|
||||
recorder.add(
|
||||
Attempt("read", started, recorder.now(), status, error, instance, False)
|
||||
)
|
||||
index += 1
|
||||
|
||||
|
||||
def percentile(values: list[float], p: float) -> float | None:
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
return round(1000 * ordered[max(0, math.ceil(p * len(ordered)) - 1)], 1)
|
||||
|
||||
|
||||
def phase_stats(attempts: list[Attempt], start: float, end: float) -> dict:
|
||||
inside = [a for a in attempts if start <= a.finished < end]
|
||||
duration = max(end - start, 1e-9)
|
||||
result: dict = {"seconds": round(duration, 3)}
|
||||
for op in ("write", "read"):
|
||||
ops = [a for a in inside if a.op == op]
|
||||
ok = [a for a in ops if a.ok]
|
||||
result[op] = {
|
||||
"ok": len(ok),
|
||||
"errors": len(ops) - len(ok),
|
||||
"ok_rps": round(len(ok) / duration, 1),
|
||||
"latency_ms": {
|
||||
"p50": percentile([a.finished - a.started for a in ok], 0.50),
|
||||
"p95": percentile([a.finished - a.started for a in ok], 0.95),
|
||||
"p99": percentile([a.finished - a.started for a in ok], 0.99),
|
||||
},
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def recovery_stats(attempts: list[Attempt], action_start: float, end: float) -> dict:
|
||||
"""
|
||||
Время восстановления записи: от первой ошибки после воздействия до первой
|
||||
успешной записи после последней ошибки; без ошибок — 0. Отдельно —
|
||||
наибольший разрыв между подтверждениями записи (задержка без ошибок).
|
||||
"""
|
||||
window = sorted(
|
||||
(a for a in attempts if a.op == "write" and a.finished >= action_start - 1),
|
||||
key=lambda a: a.finished,
|
||||
)
|
||||
errors = [a for a in window if not a.ok and a.finished >= action_start]
|
||||
kinds: dict[str, int] = {}
|
||||
for a in errors:
|
||||
kind = a.error or f"HTTP {a.status}"
|
||||
kinds[kind] = kinds.get(kind, 0) + 1
|
||||
if errors:
|
||||
last_error = errors[-1].finished
|
||||
first_ok_after = next(
|
||||
(a.finished for a in window if a.ok and a.finished > last_error), end
|
||||
)
|
||||
recovery = first_ok_after - errors[0].finished
|
||||
else:
|
||||
recovery = 0.0
|
||||
oks = [a.finished for a in window if a.ok and a.finished <= end]
|
||||
gap = max((b - a for a, b in zip(oks, oks[1:])), default=None)
|
||||
return {
|
||||
"write_errors": len(errors),
|
||||
"error_kinds": kinds,
|
||||
"write_recovery_seconds": round(recovery, 3),
|
||||
"max_write_gap_seconds": None if gap is None else round(gap, 3),
|
||||
}
|
||||
|
||||
|
||||
def timeline(attempts: list[Attempt], end: float) -> list[dict]:
|
||||
rows = []
|
||||
for second in range(int(end) + 1):
|
||||
inside = [a for a in attempts if second <= a.finished < second + 1]
|
||||
instances: dict[str, int] = {}
|
||||
for a in inside:
|
||||
if a.ok and a.instance:
|
||||
instances[a.instance] = instances.get(a.instance, 0) + 1
|
||||
rows.append(
|
||||
{
|
||||
"t": second,
|
||||
"ok": sum(a.ok for a in inside),
|
||||
"errors": sum(not a.ok for a in inside),
|
||||
"instances": instances,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def read_history(client: Client, actor: dict, chat_id: str) -> list[dict]:
|
||||
path = f"/api/v1/chats/{chat_id}/messages?limit=100"
|
||||
items: list[dict] = []
|
||||
while True:
|
||||
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
items.extend(page["items"])
|
||||
if page["next_cursor"] is None:
|
||||
return items
|
||||
path = f"/api/v1/chats/{chat_id}/messages?limit=100&cursor=" + quote(
|
||||
page["next_cursor"], safe=""
|
||||
)
|
||||
|
||||
|
||||
def integrity(client: Client, actor: dict, chat_id: str, recorder: Recorder) -> dict:
|
||||
history = read_history(client, actor, chat_id)
|
||||
by_id = {m["id"]: m for m in history}
|
||||
texts = [m["text"] for m in history]
|
||||
confirmed = {m["id"]: m for m in recorder.confirmed}
|
||||
return {
|
||||
"messages_in_history": len(history),
|
||||
"confirmed_unique": len(confirmed),
|
||||
"confirmed_missing_or_changed": sum(
|
||||
by_id.get(i) != m for i, m in confirmed.items()
|
||||
),
|
||||
"duplicate_ids": len(history) - len(by_id),
|
||||
"duplicate_texts": len(texts) - len(set(texts)),
|
||||
"retried_writes_confirmed_as_replay": len(recorder.confirmed) - len(confirmed),
|
||||
}
|
||||
|
||||
|
||||
def run(command: list[str], env: dict | None = None) -> str:
|
||||
result = subprocess.run(
|
||||
command, cwd=ROOT, env=env, check=True, capture_output=True, text=True
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def api_containers() -> list[str]:
|
||||
ids = run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).split()
|
||||
names = [run(["docker", "inspect", "-f", "{{.Name}}", i]).lstrip("/") for i in ids]
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def instance_of(container: str) -> str:
|
||||
return run(["docker", "exec", container, "cat", "/tmp/instance-id"])
|
||||
|
||||
|
||||
def container_stats() -> list[str]:
|
||||
names = run(
|
||||
["docker", "compose", "--env-file", ".env", "ps", "-q", "nginx", "postgres", "api"]
|
||||
).split()
|
||||
output = run(["docker", "stats", "--no-stream", "--format", STATS_FORMAT, *names])
|
||||
return output.splitlines()
|
||||
|
||||
|
||||
def scenario(
|
||||
args: argparse.Namespace,
|
||||
label: str,
|
||||
action: list[str],
|
||||
restore: list[str] | None,
|
||||
target: str | None = None,
|
||||
env: dict | None = None,
|
||||
) -> dict:
|
||||
client = Client(args.base_url, timeout=REQUEST_TIMEOUT_SEC)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
base = client.base
|
||||
recorder = Recorder()
|
||||
stop = threading.Event()
|
||||
threads = [
|
||||
threading.Thread(
|
||||
target=worker,
|
||||
args=(base, alice["token"], chat["id"], n, recorder, stop),
|
||||
daemon=True,
|
||||
)
|
||||
for n in range(args.workers)
|
||||
]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
time.sleep(args.before / 2)
|
||||
stats = container_stats()
|
||||
time.sleep(args.before / 2)
|
||||
|
||||
instances_before = sorted(
|
||||
{a.instance for a in recorder.attempts if a.ok and a.instance}
|
||||
)
|
||||
recorder.mark("action_start")
|
||||
action_output = run(action, env)
|
||||
recorder.mark("action_end")
|
||||
restore_output = ""
|
||||
if restore is not None:
|
||||
time.sleep(args.outage)
|
||||
recorder.mark("restore_start")
|
||||
restore_output = run(restore)
|
||||
recorder.mark("restore_end")
|
||||
time.sleep(args.after)
|
||||
stop.set()
|
||||
for thread in threads:
|
||||
thread.join(timeout=REQUEST_TIMEOUT_SEC + 1)
|
||||
recorder.mark("end")
|
||||
|
||||
events = recorder.events
|
||||
end = events["end"]
|
||||
after_start = events.get("restore_end", events["action_end"])
|
||||
attempts = recorder.attempts
|
||||
seen_after = [
|
||||
a for a in attempts if a.ok and a.finished >= after_start and a.instance
|
||||
]
|
||||
back_in_rotation = next(
|
||||
(
|
||||
round(a.finished - after_start, 3)
|
||||
for a in sorted(seen_after, key=lambda a: a.finished)
|
||||
if a.instance not in instances_before
|
||||
),
|
||||
None,
|
||||
)
|
||||
served = sorted(
|
||||
(a for a in attempts if a.ok and a.instance), key=lambda a: a.finished
|
||||
)
|
||||
instances_seen: dict[str, list[float]] = {}
|
||||
for a in served:
|
||||
offset = round(a.finished - events["action_start"], 3)
|
||||
instances_seen.setdefault(a.instance, [offset, offset])[1] = offset
|
||||
phases = {
|
||||
"before": phase_stats(attempts, 0, events["action_start"]),
|
||||
"during": phase_stats(attempts, events["action_start"], after_start),
|
||||
"after": phase_stats(attempts, after_start, end),
|
||||
}
|
||||
target_after_stop = None
|
||||
if target is not None:
|
||||
target_after_stop = sum(
|
||||
1
|
||||
for a in attempts
|
||||
if a.ok
|
||||
and a.instance == target
|
||||
and a.finished > events["action_end"]
|
||||
and a.finished < events.get("restore_start", end)
|
||||
)
|
||||
return {
|
||||
"scenario": label,
|
||||
"target_instance": target,
|
||||
"workers": args.workers,
|
||||
"events_seconds": events,
|
||||
"instances_before": instances_before,
|
||||
"instances_after": sorted({a.instance for a in seen_after}),
|
||||
"target_responses_while_stopped": target_after_stop,
|
||||
"restored_replica_in_rotation_seconds": back_in_rotation
|
||||
if restore is not None
|
||||
else None,
|
||||
# alias → [первый, последний] успешный ответ, секунды от начала воздействия
|
||||
"instances_seen_from_action": instances_seen,
|
||||
"phases": phases,
|
||||
"recovery": recovery_stats(attempts, events["action_start"], end),
|
||||
"integrity": integrity(client, alice, chat["id"], recorder),
|
||||
"container_stats_before_action": stats,
|
||||
"action_output": action_output.splitlines(),
|
||||
"restore_output_tail": restore_output.splitlines()[-2:],
|
||||
"timeline": timeline(attempts, end),
|
||||
}
|
||||
|
||||
|
||||
def failover(args: argparse.Namespace) -> list[dict]:
|
||||
results = []
|
||||
for signal in args.signals:
|
||||
for container in api_containers():
|
||||
target = instance_of(container)
|
||||
if signal == "SIGTERM":
|
||||
action = ["docker", "stop", "--timeout", "10", container]
|
||||
else:
|
||||
action = ["docker", "kill", "--signal", "SIGKILL", container]
|
||||
label = f"{signal} {container}"
|
||||
print(f"== {label} (instance {target})", flush=True)
|
||||
result = scenario(
|
||||
args, label, action, ["./scripts/restore.sh"], target=target
|
||||
)
|
||||
print(json.dumps(brief(result), ensure_ascii=False), flush=True)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
def rolling(args: argparse.Namespace) -> list[dict]:
|
||||
env = dict(os.environ)
|
||||
env.setdefault("APP_REVISION", f"exp-{int(time.time())}")
|
||||
print(f"== rolling update to revision {env['APP_REVISION']}", flush=True)
|
||||
result = scenario(
|
||||
args, "rolling update", ["./scripts/rolling-update.sh"], None, env=env
|
||||
)
|
||||
result["revisions_after"] = run(["./scripts/instances.sh"]).splitlines()
|
||||
print(json.dumps(brief(result), ensure_ascii=False), flush=True)
|
||||
return [result]
|
||||
|
||||
|
||||
def brief(result: dict) -> dict:
|
||||
"""Краткая строка прогона для терминала; полная версия — в --output"""
|
||||
return {
|
||||
"scenario": result["scenario"],
|
||||
"rps_ok_write": {k: v["write"]["ok_rps"] for k, v in result["phases"].items()},
|
||||
"rps_ok_read": {k: v["read"]["ok_rps"] for k, v in result["phases"].items()},
|
||||
"errors": {
|
||||
k: v["write"]["errors"] + v["read"]["errors"]
|
||||
for k, v in result["phases"].items()
|
||||
},
|
||||
"recovery": result["recovery"],
|
||||
"target_responses_while_stopped": result["target_responses_while_stopped"],
|
||||
"restored_replica_in_rotation_seconds": result[
|
||||
"restored_replica_in_rotation_seconds"
|
||||
],
|
||||
"instances_seen_from_action": result["instances_seen_from_action"],
|
||||
"integrity": result["integrity"],
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("mode", choices=["failover", "rolling"])
|
||||
parser.add_argument("--base-url", default=None)
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
parser.add_argument("--before", type=float, default=8, help="нагрузка до отказа")
|
||||
parser.add_argument("--outage", type=float, default=10, help="реплика остановлена")
|
||||
parser.add_argument("--after", type=float, default=8, help="после восстановления")
|
||||
parser.add_argument(
|
||||
"--signals",
|
||||
nargs="+",
|
||||
default=["SIGTERM", "SIGKILL"],
|
||||
choices=["SIGTERM", "SIGKILL"],
|
||||
)
|
||||
parser.add_argument("--output", default=None, help="JSON с полными результатами")
|
||||
args = parser.parse_args()
|
||||
started = time.strftime("%Y-%m-%dT%H:%M:%S%z")
|
||||
results = failover(args) if args.mode == "failover" else rolling(args)
|
||||
report = {
|
||||
"mode": args.mode,
|
||||
"started": started,
|
||||
"profile": {
|
||||
"workers": args.workers,
|
||||
"loop": "closed: POST message (retry same key until 201) + GET limit=1",
|
||||
"client_timeout_seconds": REQUEST_TIMEOUT_SEC,
|
||||
"phases_seconds": {
|
||||
"before": args.before,
|
||||
"outage": args.outage,
|
||||
"after": args.after,
|
||||
},
|
||||
},
|
||||
"runs": results,
|
||||
}
|
||||
if args.output:
|
||||
destination = ROOT / args.output
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print(f"saved: {args.output}")
|
||||
failed = [
|
||||
r["scenario"]
|
||||
for r in results
|
||||
if r["integrity"]["confirmed_missing_or_changed"]
|
||||
or r["integrity"]["duplicate_ids"]
|
||||
or r["integrity"]["duplicate_texts"]
|
||||
or r["target_responses_while_stopped"]
|
||||
]
|
||||
if failed:
|
||||
raise SystemExit(f"FAIL: потеря/дубль или отказ не изолирован: {failed}")
|
||||
print(f"PASS: прогонов {len(results)}, подтверждённые сообщения целы, дублей нет")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,111 @@
|
||||
-- Обоснование индекса истории: EXPLAIN (ANALYZE, BUFFERS) на ≥10 000 сообщений.
|
||||
-- Запуск: make explain → evidence/explain_history.txt
|
||||
-- Сид синтетический и идемпотентный: повторный запуск данных не дублирует.
|
||||
-- Пользователь explain_seed не может войти: password_hash не является hash Argon2.
|
||||
|
||||
\pset pager off
|
||||
\timing off
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
seed_user uuid;
|
||||
big_chat uuid;
|
||||
noise uuid;
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM chats WHERE title = 'explain-big') THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
INSERT INTO users (username, display_name, password_hash)
|
||||
VALUES ('explain_seed', 'EXPLAIN seed', 'not-an-argon2-hash')
|
||||
ON CONFLICT (username) DO UPDATE SET display_name = EXCLUDED.display_name
|
||||
RETURNING id INTO seed_user;
|
||||
|
||||
-- Большой чат: 20 000 сообщений
|
||||
INSERT INTO chats (title, created_by) VALUES ('explain-big', seed_user)
|
||||
RETURNING id INTO big_chat;
|
||||
INSERT INTO chat_members (chat_id, user_id, position) VALUES (big_chat, seed_user, 0);
|
||||
INSERT INTO messages (chat_id, sender_id, text, created_at)
|
||||
SELECT big_chat, seed_user, 'synthetic message #' || n,
|
||||
timestamptz '2026-01-01 00:00:00+00' + n * interval '1 second'
|
||||
FROM generate_series(1, 20000) AS n;
|
||||
|
||||
-- Шум: 50 чатов по 1 000 сообщений, чтобы таблица была больше одного чата
|
||||
FOR i IN 1..50 LOOP
|
||||
INSERT INTO chats (title, created_by) VALUES ('explain-noise-' || i, seed_user)
|
||||
RETURNING id INTO noise;
|
||||
INSERT INTO chat_members (chat_id, user_id, position) VALUES (noise, seed_user, 0);
|
||||
INSERT INTO messages (chat_id, sender_id, text, created_at)
|
||||
SELECT noise, seed_user, 'noise #' || n,
|
||||
timestamptz '2026-01-01 00:00:00+00' + n * interval '1 second'
|
||||
FROM generate_series(1, 1000) AS n;
|
||||
END LOOP;
|
||||
END $$;
|
||||
|
||||
ANALYZE users, chats, chat_members, messages;
|
||||
|
||||
SELECT id AS big_chat FROM chats WHERE title = 'explain-big' \gset
|
||||
|
||||
-- Cursor из середины истории: ключ 10 000-го сообщения
|
||||
SELECT created_at AS cursor_created_at, id AS cursor_id
|
||||
FROM messages WHERE chat_id = :'big_chat'
|
||||
ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset
|
||||
|
||||
\echo '== Объём данных'
|
||||
SELECT
|
||||
(SELECT count(*) FROM messages) AS messages_total,
|
||||
(SELECT count(*) FROM messages WHERE chat_id = :'big_chat') AS messages_in_big_chat,
|
||||
pg_size_pretty(pg_relation_size('messages')) AS messages_heap,
|
||||
pg_size_pretty(pg_relation_size('messages_chat_id_created_at_id_idx')) AS history_index;
|
||||
|
||||
\echo ''
|
||||
\echo '== 1. Первая страница истории (limit 50 + 1), как list_for_chat'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = :'big_chat'
|
||||
ORDER BY created_at, id
|
||||
LIMIT 51;
|
||||
|
||||
\echo ''
|
||||
\echo '== 2. Страница из середины по cursor (после 10 000-го), как list_for_chat_after'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = :'big_chat'
|
||||
AND (created_at, id) > (:'cursor_created_at'::timestamptz, :'cursor_id'::uuid)
|
||||
ORDER BY created_at, id
|
||||
LIMIT 51;
|
||||
|
||||
\echo ''
|
||||
\echo '== 3. Тот же запрос как generic plan prepared statement (так его исполняет asyncpg)'
|
||||
SET plan_cache_mode = force_generic_plan;
|
||||
PREPARE history_page(uuid, timestamptz, uuid, int) AS
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
|
||||
ORDER BY created_at, id
|
||||
LIMIT $4;
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
EXECUTE history_page(:'big_chat', :'cursor_created_at', :'cursor_id', 51);
|
||||
DEALLOCATE history_page;
|
||||
RESET plan_cache_mode;
|
||||
|
||||
\echo ''
|
||||
\echo '== 4. MAX(created_at) при вставке сообщения (монотонное время чата)'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT max(created_at) FROM messages WHERE chat_id = :'big_chat';
|
||||
|
||||
\echo ''
|
||||
\echo '== 5. Для сравнения: тот же запрос из середины БЕЗ индекса истории'
|
||||
\echo ' (DROP INDEX внутри транзакции с ROLLBACK — индекс не теряется)'
|
||||
BEGIN;
|
||||
DROP INDEX messages_chat_id_created_at_id_idx;
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = :'big_chat'
|
||||
AND (created_at, id) > (:'cursor_created_at'::timestamptz, :'cursor_id'::uuid)
|
||||
ORDER BY created_at, id
|
||||
LIMIT 51;
|
||||
ROLLBACK;
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Локальные синтетические секреты стенда в secrets/ (каталог в .gitignore)
|
||||
# и .env из .env.example, если их ещё нет. Повторный запуск ничего не меняет:
|
||||
# существующие ключи не перезаписываются, иначе данные bucket станут недоступны.
|
||||
#
|
||||
# s3_admin_* — ключи s3-bootstrap: создание bucket (действие Admin)
|
||||
# s3_access_* — ключи API и janitor: Read/Write/List только своего bucket
|
||||
# s3_identities — конфигурация identities SeaweedFS из этих ключей
|
||||
#
|
||||
# Значения не печатаются. Файлы 0644 внутри каталога 0700: процесс API в
|
||||
# контейнере работает не от root и должен прочитать смонтированный секрет.
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
|
||||
if [ ! -f .env ]; then
|
||||
cp .env.example .env
|
||||
echo "created .env from .env.example"
|
||||
fi
|
||||
|
||||
BUCKET=$(sed -n 's/^S3_BUCKET=//p' .env)
|
||||
: "${BUCKET:?S3_BUCKET is not set in .env}"
|
||||
|
||||
mkdir -p secrets
|
||||
chmod 700 secrets
|
||||
|
||||
random() {
|
||||
LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c "$1"
|
||||
}
|
||||
|
||||
secret() {
|
||||
file="secrets/$1"
|
||||
if [ ! -s "$file" ]; then
|
||||
random "$2" >"$file"
|
||||
echo "created $file"
|
||||
fi
|
||||
chmod 644 "$file"
|
||||
}
|
||||
|
||||
secret s3_admin_access_key_id 20
|
||||
secret s3_admin_secret_access_key 40
|
||||
secret s3_access_key_id 20
|
||||
secret s3_secret_access_key 40
|
||||
|
||||
# Без identity "anonymous" SeaweedFS отклоняет любой неподписанный запрос
|
||||
cat >secrets/s3_identities.json <<EOF
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "bootstrap",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "$(cat secrets/s3_admin_access_key_id)",
|
||||
"secretKey": "$(cat secrets/s3_admin_secret_access_key)"
|
||||
}
|
||||
],
|
||||
"actions": ["Admin"]
|
||||
},
|
||||
{
|
||||
"name": "messenger-api",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "$(cat secrets/s3_access_key_id)",
|
||||
"secretKey": "$(cat secrets/s3_secret_access_key)"
|
||||
}
|
||||
],
|
||||
"actions": ["Read:$BUCKET", "Write:$BUCKET", "List:$BUCKET"]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
chmod 644 secrets/s3_identities.json
|
||||
echo "secrets ready in secrets/ (values are not printed)"
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Операторский вид API-реплик: контейнер ↔ alias X-Instance-Id ↔ ревизия образа.
|
||||
# Нужен, чтобы показать, какой реальный контейнер обслужил запрос или остановлен:
|
||||
# заголовок сам по себе не доказывает разные процессы.
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
|
||||
printf '%-24s %-28s %-14s %-8s %s\n' CONTAINER STATE INSTANCE PID REVISION
|
||||
for container in $(docker compose --env-file "$ENV_FILE" ps -a -q api); do
|
||||
name=$(docker inspect -f '{{.Name}}' "$container")
|
||||
state=$(docker inspect -f '{{.State.Status}}{{if .State.Health}}/{{.State.Health.Status}}{{end}}' "$container")
|
||||
pid=$(docker inspect -f '{{.State.Pid}}' "$container")
|
||||
revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container")
|
||||
alias=$(docker exec "$container" cat /tmp/instance-id 2>/dev/null || echo -)
|
||||
printf '%-24s %-28s %-14s %-8s %s\n' "${name#/}" "$state" "$alias" "$pid" "$revision"
|
||||
done
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Перезапуск всего стенда с сохранением volumes: хранилища (PostgreSQL, Valkey,
|
||||
# S3), затем все API-реплики, janitor и балансировщик. Никаких down -v, flush
|
||||
# или повторного seed; migrate и s3-bootstrap не запускаются — схема и bucket
|
||||
# уже в volumes. Готовность ждёт тест.
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
|
||||
docker compose --env-file "$ENV_FILE" restart --no-deps postgres valkey s3
|
||||
docker compose --env-file "$ENV_FILE" restart --no-deps api janitor
|
||||
docker compose --env-file "$ENV_FILE" restart --no-deps nginx
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Возврат API-реплик к числу API_REPLICAS после stop-one.sh: запуск остановленных
|
||||
# контейнеров и ожидание healthy. Работающие реплики не пересоздаются
|
||||
# (--no-recreate), хранилища, migrate и LB не перезапускаются.
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
|
||||
docker compose --env-file "$ENV_FILE" up -d --no-deps --no-recreate --wait --wait-timeout 60 api
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Поочерёдная замена API-реплик новой сборкой без остановки обслуживания.
|
||||
# 1. build нового образа (ревизия видна в метке образа и журнале старта);
|
||||
# 2. migrate однократно: миграции только расширяют схему, старые реплики
|
||||
# продолжают работать на новой схеме;
|
||||
# 3. для каждой старой реплики: +1 новая → ждать healthy → пауза на обновление
|
||||
# DNS в nginx → graceful stop старой (SIGTERM, LB уводит трафик) → удалить.
|
||||
# Во время замены реплик на одну больше, чем API_REPLICAS: ресурс заложен заранее.
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
COMPOSE="docker compose --env-file $ENV_FILE"
|
||||
APP_REVISION="${APP_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date +%s)}"
|
||||
export APP_REVISION
|
||||
# ID образа зависит только от содержимого (без provenance-attestation)
|
||||
export BUILDX_NO_DEFAULT_ATTESTATIONS=1
|
||||
# Больше resolver valid=2s в nginx: новая реплика уже в upstream до остановки старой
|
||||
LB_DNS_REFRESH_SEC=3
|
||||
STOP_TIMEOUT_SEC=10
|
||||
|
||||
log() {
|
||||
echo "[$(date +%H:%M:%S)] $*"
|
||||
}
|
||||
|
||||
log "build revision $APP_REVISION"
|
||||
$COMPOSE build --quiet api
|
||||
log "migrate"
|
||||
$COMPOSE run --rm --no-deps migrate >/dev/null
|
||||
|
||||
old_containers=$($COMPOSE ps -q api)
|
||||
for container in $old_containers; do
|
||||
name=$(docker inspect -f '{{.Name}}' "$container")
|
||||
running=$($COMPOSE ps -q api | wc -l | tr -d ' ')
|
||||
log "scale api to $((running + 1)), wait healthy"
|
||||
$COMPOSE up -d --no-deps --no-recreate --wait --wait-timeout 60 \
|
||||
--scale "api=$((running + 1))" api
|
||||
sleep "$LB_DNS_REFRESH_SEC"
|
||||
log "graceful stop ${name#/}"
|
||||
docker stop --timeout "$STOP_TIMEOUT_SEC" "$container" >/dev/null
|
||||
docker rm "$container" >/dev/null
|
||||
done
|
||||
log "done: all API replicas run revision $APP_REVISION"
|
||||
Executable
+114
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Временный read-only доступ к Filer UI SeaweedFS (просмотр бакетов) с хоста.
|
||||
|
||||
Filer UI слушает только 127.0.0.1 внутри контейнера s3 и в сеть не открыт
|
||||
(требование: административная консоль не доступна в общей сети). Этот скрипт
|
||||
— операторский инструмент на время демонстрации:
|
||||
|
||||
* слушает только 127.0.0.1 хоста и живёт, пока запущен (Ctrl+C — закрыть);
|
||||
* пропускает только GET/HEAD: удаление/загрузка через UI (POST/PUT/DELETE)
|
||||
отклоняются 405, данные bucket через него не изменить;
|
||||
* каждый запрос выполняется в контейнере s3 (docker exec … curl) к
|
||||
127.0.0.1:8888 — порты контейнера и сети стенда не меняются.
|
||||
|
||||
make s3-ui # http://127.0.0.1:8888/buckets/
|
||||
python3 scripts/s3_ui.py --port 18888
|
||||
|
||||
Python 3.9+, стандартная библиотека.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FILER_URL = "http://127.0.0.1:8888"
|
||||
CURL_TIMEOUT_SEC = 30
|
||||
# Заголовки клиента, передаваемые filer: от Accept зависит HTML или JSON
|
||||
FORWARDED_HEADERS = ("Accept", "Range")
|
||||
# Ответ пересобирается целиком: длина и кодирование задаются заново
|
||||
DROPPED_HEADERS = {"transfer-encoding", "connection", "content-length", "keep-alive"}
|
||||
|
||||
|
||||
def s3_container() -> str:
|
||||
container = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env", "ps", "-q", "s3"],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
if not container:
|
||||
raise SystemExit("контейнер s3 не запущен: сначала make up")
|
||||
return container
|
||||
|
||||
|
||||
class ReadOnlyFilerProxy(BaseHTTPRequestHandler):
|
||||
container = ""
|
||||
|
||||
def do_GET(self) -> None:
|
||||
self._proxy(head=False)
|
||||
|
||||
def do_HEAD(self) -> None:
|
||||
self._proxy(head=True)
|
||||
|
||||
def _reject(self) -> None:
|
||||
self.send_error(405, "Read-only: изменение bucket через UI запрещено")
|
||||
|
||||
do_POST = do_PUT = do_DELETE = do_PATCH = _reject
|
||||
|
||||
def _proxy(self, head: bool) -> None:
|
||||
command = ["docker", "exec", self.container, "curl", "-s", "-i"]
|
||||
if head:
|
||||
command.append("-I")
|
||||
for name in FORWARDED_HEADERS:
|
||||
if self.headers.get(name):
|
||||
command += ["-H", f"{name}: {self.headers[name]}"]
|
||||
command.append(FILER_URL + self.path)
|
||||
try:
|
||||
raw = subprocess.run(
|
||||
command, capture_output=True, timeout=CURL_TIMEOUT_SEC, check=True
|
||||
).stdout
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
self.send_error(502, "Filer недоступен")
|
||||
return
|
||||
head_part, _, body = raw.partition(b"\r\n\r\n")
|
||||
lines = head_part.decode("latin-1").split("\r\n")
|
||||
status = int(lines[0].split()[1])
|
||||
self.send_response(status)
|
||||
for line in lines[1:]:
|
||||
name, _, value = line.partition(":")
|
||||
if name.strip().lower() not in DROPPED_HEADERS:
|
||||
self.send_header(name.strip(), value.strip())
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
if not head:
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
print(f"{self.command} {self.path.split('?')[0]} -> {args[1] if len(args) > 1 else ''}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
parser.add_argument("--port", type=int, default=8888)
|
||||
args = parser.parse_args()
|
||||
|
||||
ReadOnlyFilerProxy.container = s3_container()
|
||||
server = ThreadingHTTPServer(("127.0.0.1", args.port), ReadOnlyFilerProxy)
|
||||
print(f"Filer UI (read-only): http://127.0.0.1:{args.port}/buckets/ — Ctrl+C, чтобы закрыть")
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
server.server_close()
|
||||
print("проброс закрыт")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""SIGTERM одной API-реплики за балансировщиком.
|
||||
|
||||
Реплика перестаёт принимать соединения, дожидается текущих запросов и
|
||||
завершается не дольше чем за 10 секунд; LB уводит трафик на другие реплики.
|
||||
Во время проверки фоновые клиенты непрерывно шлют GET /health/live через LB:
|
||||
ошибок (5xx/обрыв) быть не должно, после выхода реплики её X-Instance-Id
|
||||
в ответах не появляется.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_BASE_URL = json.loads((ROOT / "lab.json").read_text())["default_base_url"]
|
||||
|
||||
SHUTDOWN_LIMIT_SEC = 10
|
||||
PROBE_INTERVAL_SEC = 0.02
|
||||
PROBERS = 4
|
||||
|
||||
|
||||
class Prober(threading.Thread):
|
||||
"""Фоновый клиент через LB: время ответа и обслужившая реплика"""
|
||||
|
||||
def __init__(self, url: str) -> None:
|
||||
super().__init__(daemon=True)
|
||||
self.url = url
|
||||
self.stop = threading.Event()
|
||||
self.ok: list[tuple[float, str]] = []
|
||||
self.errors: list[tuple[float, str]] = []
|
||||
|
||||
def run(self) -> None:
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
with urllib.request.urlopen(self.url, timeout=2) as response:
|
||||
self.ok.append(
|
||||
(time.monotonic(), response.headers.get("X-Instance-Id", ""))
|
||||
)
|
||||
except urllib.error.HTTPError as error:
|
||||
self.errors.append((time.monotonic(), f"HTTP {error.code}"))
|
||||
except OSError as error:
|
||||
self.errors.append((time.monotonic(), type(error).__name__))
|
||||
time.sleep(PROBE_INTERVAL_SEC)
|
||||
|
||||
|
||||
def docker(*args: str) -> str:
|
||||
result = subprocess.run(
|
||||
["docker", *args], check=True, capture_output=True, text=True
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def first_api_container() -> str:
|
||||
ids = docker("compose", "--env-file", ".env", "ps", "-q", "api").split()
|
||||
if not ids:
|
||||
raise SystemExit("FAIL: нет запущенных API-реплик")
|
||||
return docker("inspect", "-f", "{{.Name}}", ids[0]).lstrip("/")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--base-url", default=os.environ.get("BASE_URL") or DEFAULT_BASE_URL
|
||||
)
|
||||
parser.add_argument("--container", default=os.environ.get("CONTAINER"))
|
||||
args = parser.parse_args()
|
||||
os.chdir(ROOT)
|
||||
container = args.container or first_api_container()
|
||||
instance = docker("exec", container, "cat", "/tmp/instance-id")
|
||||
|
||||
logs = subprocess.Popen(
|
||||
["docker", "logs", "--follow", "--since", "0s", container],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
waiter = subprocess.Popen(
|
||||
["docker", "wait", container], stdout=subprocess.PIPE, text=True
|
||||
)
|
||||
probers = [
|
||||
Prober(args.base_url.rstrip("/") + "/health/live") for _ in range(PROBERS)
|
||||
]
|
||||
for prober in probers:
|
||||
prober.start()
|
||||
time.sleep(1)
|
||||
before = {alias for prober in probers for _, alias in prober.ok}
|
||||
if instance not in before:
|
||||
raise SystemExit("FAIL: целевая реплика не обслуживает запросы до SIGTERM")
|
||||
|
||||
sent_at = time.monotonic()
|
||||
docker("kill", "--signal", "SIGTERM", container)
|
||||
exit_code = int(waiter.communicate(timeout=SHUTDOWN_LIMIT_SEC + 5)[0].strip())
|
||||
exited_at = time.monotonic()
|
||||
time.sleep(1)
|
||||
for prober in probers:
|
||||
prober.stop.set()
|
||||
prober.join()
|
||||
logs.terminate()
|
||||
shutdown_log = logs.communicate(timeout=5)[0]
|
||||
|
||||
ok_after = [(t, a) for p in probers for t, a in p.ok if t > sent_at]
|
||||
errors_after = [(t, e) for p in probers for t, e in p.errors if t > sent_at]
|
||||
target_after_exit = [t for t, a in ok_after if a == instance and t > exited_at]
|
||||
last_target = max((t for t, a in ok_after if a == instance), default=None)
|
||||
print("Журнал реплики после SIGTERM:")
|
||||
print(
|
||||
"\n".join(line for line in shutdown_log.splitlines() if "/health/" not in line)
|
||||
)
|
||||
summary = {
|
||||
"container": container,
|
||||
"instance": instance,
|
||||
"exit_code": exit_code,
|
||||
"shutdown_seconds": round(exited_at - sent_at, 3),
|
||||
"limit_seconds": SHUTDOWN_LIMIT_SEC,
|
||||
"instances_before": sorted(before),
|
||||
"probes_ok_after_sigterm": len(ok_after),
|
||||
"instances_after_sigterm": sorted({a for _, a in ok_after}),
|
||||
"last_target_response_after_sigterm_seconds": None
|
||||
if last_target is None
|
||||
else round(last_target - sent_at, 3),
|
||||
"target_responses_after_exit": len(target_after_exit),
|
||||
"errors_after_sigterm": len(errors_after),
|
||||
"error_kinds": sorted({e for _, e in errors_after}),
|
||||
}
|
||||
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
||||
if (
|
||||
exit_code != 0
|
||||
or exited_at - sent_at > SHUTDOWN_LIMIT_SEC
|
||||
or errors_after
|
||||
or target_after_exit
|
||||
or not ok_after
|
||||
):
|
||||
raise SystemExit("FAIL: завершение по SIGTERM не соответствует требованиям")
|
||||
print(
|
||||
"PASS: SIGTERM завершил %s за %.2f с, LB обслужил %d запросов без ошибок"
|
||||
% (container, exited_at - sent_at, len(ok_after))
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Остановка ОДНОЙ API-реплики, обслужившей тест: TARGET_INSTANCE_ID → контейнер.
|
||||
# Каждый процесс API пишет свой alias (X-Instance-Id) в /tmp/instance-id.
|
||||
# docker stop: SIGTERM → graceful shutdown; restart-политика unless-stopped
|
||||
# не поднимает вручную остановленный контейнер, он остаётся остановленным.
|
||||
# БД, LB и volumes не затрагиваются. Восстановление — scripts/restore.sh.
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
# Меньше 10 с лимита hook; больше drain + graceful shutdown (1 + 5 с)
|
||||
STOP_TIMEOUT_SEC=8
|
||||
|
||||
for container in $(docker compose --env-file "$ENV_FILE" ps -q api); do
|
||||
alias=$(docker exec "$container" cat /tmp/instance-id 2>/dev/null || true)
|
||||
if [ "$alias" = "$TARGET_INSTANCE_ID" ]; then
|
||||
name=$(docker inspect -f '{{.Name}}' "$container")
|
||||
docker stop --timeout "$STOP_TIMEOUT_SEC" "$container" >/dev/null
|
||||
echo "stopped ${name#/} (instance $alias)" >&2
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
echo "no running API replica with instance $TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Эксперимент: N одновременных загрузок вложений и пик памяти API-реплик.
|
||||
|
||||
Гипотеза: память реплики на загрузку ограничена (spool ≤ 1 MiB RAM, потоковый
|
||||
PUT в S3, проверка изображения в пуле из IMAGE_INSPECT_WORKERS потоков),
|
||||
поэтому пик RSS растёт не пропорционально N × размер файла.
|
||||
|
||||
Профили:
|
||||
png — PNG ~9 MiB из шума (почти не сжимается): нагрузка на приём тела,
|
||||
spool, SHA-256 и PUT в S3. Генерируется стандартной библиотекой.
|
||||
jpeg — progressive JPEG 8000×5000 = 40 Мпикс (предел): худший случай
|
||||
декодера libjpeg. Нужен Pillow (poetry run python ...).
|
||||
|
||||
Перед каждым прогоном API-реплики перезапускаются: VmHWM (пиковый RSS
|
||||
процесса) начинается заново. Память опрашивается внутри контейнеров каждые
|
||||
0.2 с (cgroup memory.current и VmRSS процесса). Все N запросов стартуют по
|
||||
одному барьеру. Tokens не печатаются и не сохраняются.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import statistics
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client # noqa: E402
|
||||
|
||||
REQUEST_TIMEOUT_SEC = 120
|
||||
READY_TIMEOUT_SEC = 90
|
||||
SAMPLE_INTERVAL_SEC = 0.2
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
SAMPLER = (
|
||||
"while :; do "
|
||||
"echo $(cat /sys/fs/cgroup/memory.current) "
|
||||
"$(awk '/VmRSS/ {print $2}' /proc/1/status); "
|
||||
f"sleep {SAMPLE_INTERVAL_SEC}; done"
|
||||
)
|
||||
|
||||
|
||||
def png_noise(width: int, height: int) -> bytes:
|
||||
"""Настоящий RGB PNG из случайных байт: размер ≈ width × height × 3"""
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return (
|
||||
struct.pack(">I", len(data))
|
||||
+ kind
|
||||
+ data
|
||||
+ struct.pack(">I", zlib.crc32(kind + data))
|
||||
)
|
||||
|
||||
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
|
||||
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", header)
|
||||
+ chunk(b"IDAT", zlib.compress(raw, 1))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def jpeg_worst_case() -> bytes:
|
||||
from PIL import Image
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
image = Image.effect_noise((8000, 5000), 64).convert("RGB")
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, "JPEG", quality=10, progressive=True, subsampling=0)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def compose(*args: str) -> str:
|
||||
return subprocess.run(
|
||||
[*COMPOSE, *args], cwd=ROOT, capture_output=True, text=True, check=True
|
||||
).stdout
|
||||
|
||||
|
||||
def api_containers() -> list[str]:
|
||||
return compose("ps", "-q", "api").split()
|
||||
|
||||
|
||||
def container_name(container: str) -> str:
|
||||
return subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.Name}}", container],
|
||||
capture_output=True, text=True, check=True,
|
||||
).stdout.strip().lstrip("/")
|
||||
|
||||
|
||||
def process_status(container: str) -> dict:
|
||||
"""VmRSS/VmHWM процесса API (PID 1), байты, и его X-Instance-Id"""
|
||||
output = subprocess.run(
|
||||
["docker", "exec", container, "sh", "-c",
|
||||
"grep -E 'VmRSS|VmHWM' /proc/1/status; cat /tmp/instance-id"],
|
||||
capture_output=True, text=True, check=True,
|
||||
).stdout.split("\n")
|
||||
values: dict = {line.split(":")[0]: int(line.split()[1]) * 1024 for line in output[:2]}
|
||||
values["instance"] = output[2].strip()
|
||||
return values
|
||||
|
||||
|
||||
def restart_api(base_url: str) -> None:
|
||||
compose("restart", "api")
|
||||
deadline = time.monotonic() + READY_TIMEOUT_SEC
|
||||
while time.monotonic() < deadline:
|
||||
states = [
|
||||
subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.State.Health.Status}}", c],
|
||||
capture_output=True, text=True,
|
||||
).stdout.strip()
|
||||
for c in api_containers()
|
||||
]
|
||||
if states and all(state == "healthy" for state in states):
|
||||
return
|
||||
time.sleep(1)
|
||||
raise SystemExit("API replicas are not healthy after restart")
|
||||
|
||||
|
||||
class Sampler(threading.Thread):
|
||||
"""Опрос памяти одного контейнера изнутри одним долгим docker exec"""
|
||||
|
||||
def __init__(self, container: str) -> None:
|
||||
super().__init__(daemon=True)
|
||||
self.container = container
|
||||
self.cgroup_max = 0
|
||||
self.rss_max = 0
|
||||
self.samples = 0
|
||||
self._process = subprocess.Popen(
|
||||
["docker", "exec", container, "sh", "-c", SAMPLER],
|
||||
stdout=subprocess.PIPE, text=True,
|
||||
)
|
||||
|
||||
def run(self) -> None:
|
||||
assert self._process.stdout is not None
|
||||
for line in self._process.stdout:
|
||||
parts = line.split()
|
||||
if len(parts) == 2:
|
||||
self.cgroup_max = max(self.cgroup_max, int(parts[0]))
|
||||
self.rss_max = max(self.rss_max, int(parts[1]) * 1024)
|
||||
self.samples += 1
|
||||
|
||||
def stop(self) -> None:
|
||||
self._process.terminate()
|
||||
self._process.wait()
|
||||
|
||||
|
||||
def upload_once(base: str, token: str, chat_id: str, content: bytes, content_type: str) -> dict:
|
||||
request = urllib.request.Request(
|
||||
f"{base}/api/v1/chats/{chat_id}/attachments?filename=experiment",
|
||||
data=content,
|
||||
method="POST",
|
||||
headers={"Authorization": "Bearer " + token, "Content-Type": content_type},
|
||||
)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SEC) as response:
|
||||
body, status, instance = response.read(), response.status, response.headers.get("X-Instance-Id")
|
||||
except urllib.error.HTTPError as error:
|
||||
body, status, instance = error.read(), error.code, error.headers.get("X-Instance-Id")
|
||||
except OSError as error:
|
||||
return {"status": None, "error": type(error).__name__, "seconds": time.monotonic() - started}
|
||||
result = {"status": status, "instance": instance, "seconds": time.monotonic() - started}
|
||||
if status == 201:
|
||||
result["id"] = json.loads(body)["id"]
|
||||
return result
|
||||
|
||||
|
||||
def percentile(values: list[float], share: float) -> float:
|
||||
ordered = sorted(values)
|
||||
return ordered[min(len(ordered) - 1, round(share * (len(ordered) - 1)))]
|
||||
|
||||
|
||||
def run_round(client: Client, actor: dict, chat: dict, profile: str, content: bytes,
|
||||
content_type: str, concurrency: int, restart: bool) -> dict:
|
||||
if restart:
|
||||
restart_api(client.base)
|
||||
containers = api_containers()
|
||||
# Прогрев: первый запрос каждой реплики открывает пулы БД/S3
|
||||
warmup = png_noise(4, 4)
|
||||
for _ in range(2 * len(containers)):
|
||||
upload_once(client.base, actor["token"], chat["id"], warmup, "image/png")
|
||||
before = {container_name(c): process_status(c) for c in containers}
|
||||
samplers = [Sampler(c) for c in containers]
|
||||
for sampler in samplers:
|
||||
sampler.start()
|
||||
time.sleep(1)
|
||||
|
||||
barrier = threading.Barrier(concurrency)
|
||||
results: list[dict] = [{} for _ in range(concurrency)]
|
||||
|
||||
def worker(index: int) -> None:
|
||||
barrier.wait()
|
||||
results[index] = upload_once(client.base, actor["token"], chat["id"], content, content_type)
|
||||
|
||||
started = time.monotonic()
|
||||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
wall = time.monotonic() - started
|
||||
time.sleep(1)
|
||||
for sampler in samplers:
|
||||
sampler.stop()
|
||||
after = {container_name(c): process_status(c) for c in containers}
|
||||
|
||||
ok = [r for r in results if r.get("status") == 201]
|
||||
sample = ok[0]["id"] if ok else None
|
||||
checksum_ok = None
|
||||
if sample:
|
||||
downloaded = client.call("GET", f"/api/v1/attachments/{sample}/content", actor=actor)["bytes"]
|
||||
checksum_ok = hashlib.sha256(downloaded).digest() == hashlib.sha256(content).digest()
|
||||
seconds = [r["seconds"] for r in ok]
|
||||
mib = 1024 * 1024
|
||||
replicas = {}
|
||||
for sampler in samplers:
|
||||
name = container_name(sampler.container)
|
||||
replicas[name] = {
|
||||
"uploads_served": sum(1 for r in ok if r.get("instance") == after[name].get("instance")),
|
||||
"rss_before_mib": round(before[name]["VmRSS"] / mib, 1),
|
||||
"rss_peak_process_vmhwm_mib": round(after[name]["VmHWM"] / mib, 1),
|
||||
"rss_peak_sampled_mib": round(sampler.rss_max / mib, 1),
|
||||
"cgroup_peak_sampled_mib": round(sampler.cgroup_max / mib, 1),
|
||||
"samples": sampler.samples,
|
||||
}
|
||||
by_instance: dict[str, int] = {}
|
||||
for r in ok:
|
||||
by_instance[r["instance"]] = by_instance.get(r["instance"], 0) + 1
|
||||
return {
|
||||
"profile": profile,
|
||||
"file_bytes": len(content),
|
||||
"concurrency": concurrency,
|
||||
"replicas_restarted_before": restart,
|
||||
"status_counts": {str(s): sum(1 for r in results if r.get("status") == s)
|
||||
for s in sorted({r.get("status") for r in results}, key=str)},
|
||||
"uploads_by_instance": by_instance,
|
||||
"wall_seconds": round(wall, 2),
|
||||
"latency_seconds": {
|
||||
"p50": round(percentile(seconds, 0.5), 3),
|
||||
"p95": round(percentile(seconds, 0.95), 3),
|
||||
"max": round(max(seconds), 3),
|
||||
} if seconds else None,
|
||||
"throughput_mib_per_second": round(len(ok) * len(content) / mib / wall, 1),
|
||||
"download_sha256_matches": checksum_ok,
|
||||
"replicas": replicas,
|
||||
}
|
||||
|
||||
|
||||
def environment() -> dict:
|
||||
info = json.loads(subprocess.run(
|
||||
["docker", "info", "--format", "{{json .}}"], capture_output=True, text=True, check=True
|
||||
).stdout)
|
||||
settings = {}
|
||||
for line in (ROOT / ".env").read_text().splitlines():
|
||||
key = line.split("=", 1)[0]
|
||||
if key in {"IMAGE_INSPECT_WORKERS", "UPLOAD_SPOOL_MEMORY_BYTES", "MAX_UPLOAD_BYTES",
|
||||
"MAX_IMAGE_PIXELS", "API_REPLICAS", "S3_POOL_MAX_SIZE"}:
|
||||
settings[key] = line.split("=", 1)[1]
|
||||
return {
|
||||
"host": f"{platform.system()} {platform.machine()}",
|
||||
"docker": info.get("ServerVersion"),
|
||||
"docker_vm_cpus": info.get("NCPU"),
|
||||
"docker_vm_memory_gib": round(info.get("MemTotal", 0) / 2**30, 1),
|
||||
"api_limits": "cpus 1.0, memory 512M per replica (compose.yaml)",
|
||||
"s3": "chrislusf/seaweedfs:4.48",
|
||||
"settings": settings,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
parser.add_argument("--base-url", default=None)
|
||||
parser.add_argument("--output", default="evidence/experiment-uploads.json")
|
||||
parser.add_argument("--concurrency", default="10,20")
|
||||
parser.add_argument("--profiles", default="png,jpeg")
|
||||
parser.add_argument("--no-restart", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
client = Client(args.base_url)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
payloads = {}
|
||||
if "png" in args.profiles:
|
||||
payloads["png"] = (png_noise(1750, 1750), "image/png")
|
||||
if "jpeg" in args.profiles:
|
||||
try:
|
||||
payloads["jpeg"] = (jpeg_worst_case(), "image/jpeg")
|
||||
except ImportError:
|
||||
print("jpeg profile skipped: Pillow is not installed (use poetry run)")
|
||||
report = {"environment": environment(), "rounds": []}
|
||||
for profile, (content, content_type) in payloads.items():
|
||||
for concurrency in (int(x) for x in args.concurrency.split(",")):
|
||||
result = run_round(client, alice, chat, profile, content, content_type,
|
||||
concurrency, not args.no_restart)
|
||||
report["rounds"].append(result)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
output = ROOT / args.output
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print("saved:", args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user