Добавить две реплики API за HAProxy и проверку отказа одной из них.
Общие сессии и ключ идемпотентности позволяют продолжать чтение и запись, когда одна реплика остановлена.
This commit is contained in:
Executable
+143
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Поочерёдный отказ каждой API-реплики и гонка одного Idempotency-Key."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT)
|
||||
|
||||
|
||||
def instances(samples=40):
|
||||
client = Client("http://127.0.0.1:8080", timeout=2)
|
||||
found = []
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
found = []
|
||||
seen = set()
|
||||
try:
|
||||
for _ in range(samples):
|
||||
item = client.call("GET", "/health/live", schema="Health")["headers"]["X-Instance-Id"]
|
||||
if item not in seen:
|
||||
seen.add(item)
|
||||
found.append(item)
|
||||
if len(found) >= 2:
|
||||
return found
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise SystemExit("expected at least two API instances, saw " + ",".join(found))
|
||||
|
||||
|
||||
def restore_api():
|
||||
compose("up", "-d", "--no-deps", "api")
|
||||
ids = subprocess.check_output(
|
||||
["docker", "compose", "--env-file", ".env", "ps", "-aq", "api"], cwd=ROOT, text=True
|
||||
).split()
|
||||
if ids:
|
||||
subprocess.check_call(["docker", "start", *ids], cwd=ROOT)
|
||||
instances()
|
||||
|
||||
|
||||
def stop_and_measure(target):
|
||||
client = Client("http://127.0.0.1:8080", timeout=2)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
end = time.monotonic() + 12
|
||||
stats = {"ok": 0, "fail": 0, "after_ok": 0, "target_after_stop": 0, "first_recovery_s": None}
|
||||
started = {}
|
||||
stopped = {}
|
||||
|
||||
def worker():
|
||||
local = Client("http://127.0.0.1:8080", timeout=2)
|
||||
while time.monotonic() < end:
|
||||
key = uuid.uuid4().hex
|
||||
try:
|
||||
response = local.message(alice, chat, "отказ " + key, key=key)
|
||||
stats["ok"] += 1
|
||||
mark = started.get("at")
|
||||
if mark is not None and stats["first_recovery_s"] is None:
|
||||
stats["first_recovery_s"] = round(time.monotonic() - mark, 3)
|
||||
if stopped.get("at") is not None:
|
||||
stats["after_ok"] += 1
|
||||
if response["headers"].get("X-Instance-Id") == target:
|
||||
stats["target_after_stop"] += 1
|
||||
except Failure:
|
||||
stats["fail"] += 1
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(4)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
time.sleep(2)
|
||||
started["at"] = time.monotonic()
|
||||
env = os.environ.copy()
|
||||
env["TARGET_INSTANCE_ID"] = target
|
||||
proc = subprocess.run([str(ROOT / "scripts" / "stop-one.sh")], cwd=ROOT, env=env, text=True, capture_output=True)
|
||||
stopped["at"] = time.monotonic()
|
||||
stats["hook_exit"] = proc.returncode
|
||||
stats["hook_seconds"] = round(stopped["at"] - started["at"], 3)
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
stats["target"] = target
|
||||
stats["successful_rps"] = round(stats["ok"] / 12, 3)
|
||||
return stats
|
||||
|
||||
|
||||
def idempotency_race():
|
||||
client = Client("http://127.0.0.1:8080", timeout=5)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
ids = []
|
||||
errors = []
|
||||
|
||||
def once():
|
||||
try:
|
||||
ids.append(Client("http://127.0.0.1:8080", timeout=5).message(alice, chat, "гонка", key=key)["json"]["id"])
|
||||
except Failure as exc:
|
||||
errors.append(str(exc))
|
||||
|
||||
threads = [threading.Thread(target=once) for _ in range(12)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
page = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"]
|
||||
texts = [item["text"] for item in page["items"] if item["text"] == "гонка"]
|
||||
return {
|
||||
"parallel_requests": 12,
|
||||
"distinct_ids": len(set(ids)),
|
||||
"successes": len(ids),
|
||||
"errors": errors,
|
||||
"history_copies": len(texts),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
os.chdir(ROOT)
|
||||
before = instances()
|
||||
runs = []
|
||||
for target in before:
|
||||
runs.append(stop_and_measure(target))
|
||||
restore_api()
|
||||
race = idempotency_race()
|
||||
report = {"instances_before": before, "failover": runs, "idempotency_race": race}
|
||||
dest = ROOT / "evidence" / "grade4.json"
|
||||
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if any(item["hook_exit"] != 0 or item["after_ok"] < 1 or item["target_after_stop"] != 0 for item in runs):
|
||||
raise SystemExit("failover measurement failed")
|
||||
if race["distinct_ids"] != 1 or race["history_copies"] != 1 or race["errors"]:
|
||||
raise SystemExit("idempotency race failed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Перезапуск API и всех хранилищ с сохранением volumes. Данные не сидируются заново.
|
||||
# Перезапуск API, балансировщика и всех хранилищ. Volumes не удаляются, данные не сидируются.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env restart api postgres redis
|
||||
docker compose --env-file .env restart postgres redis api haproxy
|
||||
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
# Поочерёдная замена контейнеров api. Оставшаяся реплика продолжает принимать трафик.
|
||||
# Новые контейнеры собираются из текущего Dockerfile; старые заменяются по одному.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
scale="${API_REPLICAS:-2}"
|
||||
docker compose --env-file .env build api
|
||||
ids=$(docker compose --env-file .env ps -q api)
|
||||
for id in $ids; do
|
||||
docker stop -t 8 "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
docker compose --env-file .env up -d --no-deps --no-recreate --scale "api=$scale" api >/dev/null
|
||||
ready=0
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
if curl -fsS http://127.0.0.1:8080/health/ready >/dev/null; then
|
||||
ready=1
|
||||
break
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 1
|
||||
done
|
||||
if [ "$ready" -ne 1 ]; then
|
||||
echo "ready did not return after replacing one replica" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "rolling update finished"
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
# Останавливает одну API-реплику и оставляет её остановленной.
|
||||
# TARGET_INSTANCE_ID — значение X-Instance-Id (hostname контейнера).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
target=$(printf '%s' "$TARGET_INSTANCE_ID" | tr -d '[:space:]')
|
||||
found=""
|
||||
for id in $(docker compose --env-file .env ps -q api); do
|
||||
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
|
||||
full=$(docker inspect -f '{{.Id}}' "$id")
|
||||
short=$(printf '%s' "$full" | cut -c1-12)
|
||||
if [ "$host" = "$target" ] || [ "$short" = "$target" ] || [ "$full" = "$target" ]; then
|
||||
found=$id
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ -z "$found" ]; then
|
||||
echo "API replica not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
docker stop -t 8 "$found" >/dev/null
|
||||
Reference in New Issue
Block a user