320 lines
13 KiB
Bash
Executable File
320 lines
13 KiB
Bash
Executable File
#!/bin/sh
|
||
# Track 1: the price of scaling. Runs one and the same closed-loop write profile
|
||
# against 1/2/4 API replicas while the TOTAL API budget stays fixed, and records
|
||
# raw metrics. The stand has to be built already (make up).
|
||
#
|
||
# scripts/track1_scaling.sh
|
||
# REPLICAS="2 4" DURATION=30 CONCURRENCY=8 scripts/track1_scaling.sh
|
||
#
|
||
# Artifacts: evidence/track1/{load,stats}-r<N>.json|csv plus summary.txt
|
||
set -eu
|
||
|
||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||
cd "$ROOT_DIR"
|
||
|
||
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
|
||
PYTHON=${PYTHON:-python3}
|
||
REPLICAS=${REPLICAS:-"1 2 4"}
|
||
TOTAL_CPU=${TOTAL_CPU:-2.0}
|
||
TOTAL_MEM_MB=${TOTAL_MEM_MB:-1024}
|
||
DURATION=${DURATION:-60}
|
||
CONCURRENCY=${CONCURRENCY:-16}
|
||
WARMUP=${WARMUP:-15}
|
||
BASE_URL=${BASE_URL:-http://localhost:8080}
|
||
LABEL=${LABEL:-}
|
||
OUT=${OUT:-evidence/track1}
|
||
|
||
# Бюджеты общих хранилищ фиксируются в профиле вместе с бюджетом API: без них
|
||
# серию нельзя воспроизвести по артефактам.
|
||
PG_CPU=${POSTGRES_CPU_LIMIT:-0.50}
|
||
PG_MEM=${POSTGRES_MEM_LIMIT:-384M}
|
||
PG_CONNS=${POSTGRES_MAX_CONNS:-10}
|
||
LB_CPU=${LB_CPU_LIMIT:-0.25}
|
||
# Где работает генератор нагрузки: `host` — на хосте через опубликованный порт,
|
||
# `vm` — в контейнере внутри VM, где балансировщик доступен по docker-сети.
|
||
# Вариант `host` упирается в проброс порта: под шквалом новых соединений
|
||
# (клиент ходит без keep-alive) он отказывает, и отказы попадают в метрику
|
||
# ошибок (см. `scripts/path_overhead_check.sh`).
|
||
CLIENT=${CLIENT:-host}
|
||
CLIENT_IMAGE=${CLIENT_IMAGE:-python:3.12-alpine}
|
||
export POSTGRES_CPU_LIMIT="$PG_CPU" POSTGRES_MEM_LIMIT="$PG_MEM" POSTGRES_MAX_CONNS="$PG_CONNS"
|
||
export LB_CPU_LIMIT="$LB_CPU"
|
||
|
||
# Бюджеты хранилищ compose применяет при создании контейнера. Это значения, к
|
||
# которым стенд возвращается после контрольного профиля.
|
||
PG_DEFAULT_CPU=${PG_DEFAULT_CPU:-0.50}
|
||
PG_DEFAULT_MEM=${PG_DEFAULT_MEM:-384M}
|
||
LB_DEFAULT_CPU=${LB_DEFAULT_CPU:-0.25}
|
||
|
||
# Приводим бюджет к тому виду, в котором он снимается с контейнера
|
||
# (`docker inspect`): 1.0 -> 1.00, 1024M -> 1024M. Иначе проверка «применился ли
|
||
# бюджет» сравнивала бы разные записи одного и того же значения.
|
||
normalize_cpu() { awk -v value="$1" 'BEGIN{printf "%.2f", value}'; }
|
||
|
||
normalize_mem() {
|
||
case $1 in
|
||
*[Gg]) awk -v value="$1" 'BEGIN{printf "%.0fM", value * 1024}' ;;
|
||
*[Mm]) awk -v value="$1" 'BEGIN{printf "%.0fM", value}' ;;
|
||
*) printf '%s\n' "$1" ;;
|
||
esac
|
||
}
|
||
|
||
PG_CPU=$(normalize_cpu "$PG_CPU")
|
||
PG_MEM=$(normalize_mem "$PG_MEM")
|
||
PG_DEFAULT_CPU=$(normalize_cpu "$PG_DEFAULT_CPU")
|
||
PG_DEFAULT_MEM=$(normalize_mem "$PG_DEFAULT_MEM")
|
||
LB_CPU=$(normalize_cpu "$LB_CPU")
|
||
LB_DEFAULT_CPU=$(normalize_cpu "$LB_DEFAULT_CPU")
|
||
export POSTGRES_CPU_LIMIT="$PG_CPU" POSTGRES_MEM_LIMIT="$PG_MEM" LB_CPU_LIMIT="$LB_CPU"
|
||
|
||
# The author's shell exports HTTP(S)_PROXY, which breaks localhost clients.
|
||
NO_PROXY_ENV="env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy"
|
||
NO_PROXY_ENV="$NO_PROXY_ENV NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1"
|
||
|
||
mkdir -p "$OUT"
|
||
|
||
compose() {
|
||
# shellcheck disable=SC2086
|
||
$COMPOSE_BIN --env-file .env -f compose.yaml "$@"
|
||
}
|
||
|
||
wait_ready() {
|
||
attempt=0
|
||
while [ "$attempt" -lt 90 ]; do
|
||
if curl -fsS --noproxy '*' -o /dev/null "$BASE_URL/health/ready"; then
|
||
return 0
|
||
fi
|
||
|
||
attempt=$((attempt + 1))
|
||
sleep 1
|
||
done
|
||
|
||
printf 'track1_scaling.sh: %s/health/ready did not become ready\n' "$BASE_URL" >&2
|
||
|
||
return 1
|
||
}
|
||
|
||
per_replica_cpu() {
|
||
awk "BEGIN{printf \"%.2f\", $TOTAL_CPU/$1}"
|
||
}
|
||
|
||
per_replica_mem() {
|
||
awk "BEGIN{printf \"%.0fM\", $TOTAL_MEM_MB/$1}"
|
||
}
|
||
|
||
container_of() {
|
||
compose ps -q "$1"
|
||
}
|
||
|
||
# Сеть стенда: клиент внутри VM обращается к балансировщику по ней, без проброса
|
||
# порта и без сетевого стека хоста.
|
||
stand_network() {
|
||
docker inspect "$(container_of api | head -1)" \
|
||
--format '{{range $name, $network := .NetworkSettings.Networks}}{{$name}}{{end}}'
|
||
}
|
||
|
||
# $1 — длительность, $2 — путь к JSON-метрикам (необязательно).
|
||
load_client() {
|
||
if [ "$CLIENT" = vm ]; then
|
||
network=$(stand_network)
|
||
|
||
if [ -n "$2" ]; then
|
||
# shellcheck disable=SC2086
|
||
docker run --rm -i --network "$network" -v "$ROOT_DIR:/repo:ro" -v "$ROOT_DIR/$OUT:/out" \
|
||
-e NO_PROXY='lb,localhost,127.0.0.1' -e no_proxy='lb,localhost,127.0.0.1' \
|
||
"$CLIENT_IMAGE" python /repo/tests/load.py --base-url http://lb \
|
||
--duration "$1" --concurrency "$CONCURRENCY" --output "/out/$(basename "$2")"
|
||
else
|
||
# shellcheck disable=SC2086
|
||
docker run --rm -i --network "$network" -v "$ROOT_DIR:/repo:ro" \
|
||
-e NO_PROXY='lb,localhost,127.0.0.1' -e no_proxy='lb,localhost,127.0.0.1' \
|
||
"$CLIENT_IMAGE" python /repo/tests/load.py --base-url http://lb \
|
||
--duration "$1" --concurrency "$CONCURRENCY"
|
||
fi
|
||
|
||
return
|
||
fi
|
||
|
||
if [ -n "$2" ]; then
|
||
# shellcheck disable=SC2086
|
||
$NO_PROXY_ENV $PYTHON tests/load.py --base-url "$BASE_URL" \
|
||
--duration "$1" --concurrency "$CONCURRENCY" --output "$2"
|
||
else
|
||
# shellcheck disable=SC2086
|
||
$NO_PROXY_ENV $PYTHON tests/load.py --base-url "$BASE_URL" \
|
||
--duration "$1" --concurrency "$CONCURRENCY"
|
||
fi
|
||
}
|
||
|
||
# Фактический бюджет контейнера, снятый с него самого: в профиль и в сводку
|
||
# попадает измерение, а не переменная, с которой скрипт был запущен.
|
||
container_cpu() {
|
||
nano=$(docker inspect "$(container_of "$1")" --format '{{.HostConfig.NanoCpus}}')
|
||
awk -v nano="$nano" 'BEGIN{printf "%.2f", nano/1000000000}'
|
||
}
|
||
|
||
container_mem() {
|
||
bytes=$(docker inspect "$(container_of "$1")" --format '{{.HostConfig.Memory}}')
|
||
awk -v bytes="$bytes" 'BEGIN{printf "%.0fM", bytes/1024/1024}'
|
||
}
|
||
|
||
wait_postgres_ready() {
|
||
attempt=0
|
||
while [ "$attempt" -lt 90 ]; do
|
||
if compose exec -T postgres pg_isready -U "${POSTGRES_USER:-messenger}" \
|
||
-d "${POSTGRES_DB:-messenger}" >/dev/null 2>&1; then
|
||
return 0
|
||
fi
|
||
|
||
attempt=$((attempt + 1))
|
||
sleep 1
|
||
done
|
||
|
||
printf 'track1_scaling.sh: postgres did not become ready\n' >&2
|
||
|
||
return 1
|
||
}
|
||
|
||
# Лимиты общих хранилищ задаются при создании контейнера, поэтому одного
|
||
# `up -d --no-deps api` для них мало: PostgreSQL и балансировщик приходится
|
||
# пересоздавать, когда бюджет профиля отличается от действующего. Без этого
|
||
# профиль записывал бы намерение, а измерение шло бы на прежнем лимите — именно
|
||
# так контроль удвоения CPU БД перестал быть контролем.
|
||
apply_store_limits() {
|
||
applied_pg_cpu=$(container_cpu postgres)
|
||
applied_pg_mem=$(container_mem postgres)
|
||
applied_lb_cpu=$(container_cpu lb)
|
||
|
||
if [ "$applied_pg_cpu" != "$PG_CPU" ] || [ "$applied_pg_mem" != "$PG_MEM" ]; then
|
||
printf '=== PostgreSQL: бюджет %s/%s -> %s/%s, пересоздаю контейнер ===\n' \
|
||
"$applied_pg_cpu" "$applied_pg_mem" "$PG_CPU" "$PG_MEM" | tee -a "$SUMMARY"
|
||
|
||
POSTGRES_CPU_LIMIT="$PG_CPU" POSTGRES_MEM_LIMIT="$PG_MEM" \
|
||
compose up -d --no-deps postgres >/dev/null
|
||
|
||
wait_postgres_ready
|
||
fi
|
||
|
||
if [ "$applied_lb_cpu" != "$LB_CPU" ]; then
|
||
printf '=== LB: бюджет CPU %s -> %s, пересоздаю контейнер ===\n' \
|
||
"$applied_lb_cpu" "$LB_CPU" | tee -a "$SUMMARY"
|
||
|
||
LB_CPU_LIMIT="$LB_CPU" compose up -d --no-deps lb >/dev/null
|
||
fi
|
||
}
|
||
|
||
SUMMARY="$OUT/summary.txt"
|
||
|
||
if [ ! -f "$SUMMARY" ]; then
|
||
{
|
||
echo "# Track 1: the price of scaling"
|
||
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||
echo "# host: $(uname -sm)"
|
||
echo "# cpu: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || printf unknown)"
|
||
echo "# ram bytes: $(sysctl -n hw.memsize 2>/dev/null || printf unknown)"
|
||
echo "# load: tests/load.py closed loop; postgres/lb budgets are recreated and verified per profile"
|
||
echo "# each profile line below records the budget it was run with"
|
||
docker version --format 'docker engine {{.Server.Version}}'
|
||
# shellcheck disable=SC2086
|
||
$COMPOSE_BIN version
|
||
echo
|
||
} > "$SUMMARY"
|
||
fi
|
||
|
||
{
|
||
echo "=== invocation $(date -u +%Y-%m-%dT%H:%M:%SZ): REPLICAS=\"$REPLICAS\" TOTAL_CPU=$TOTAL_CPU TOTAL_MEM_MB=$TOTAL_MEM_MB DURATION=$DURATION CONCURRENCY=$CONCURRENCY WARMUP=$WARMUP ==="
|
||
echo
|
||
} >> "$SUMMARY"
|
||
|
||
for replicas in $REPLICAS; do
|
||
cpu=$(per_replica_cpu "$replicas")
|
||
mem=$(per_replica_mem "$replicas")
|
||
label=${LABEL:-r$replicas}
|
||
|
||
printf '=== profile label=%s replicas=%s cpu=%s mem=%s postgres=%s/%s lb_cpu=%s max_conns=%s client=%s ===\n' \
|
||
"$label" "$replicas" "$cpu" "$mem" "$PG_CPU" "$PG_MEM" "$LB_CPU" "$PG_CONNS" "$CLIENT" | tee -a "$SUMMARY"
|
||
|
||
apply_store_limits
|
||
|
||
API_REPLICAS="$replicas" API_CPU_LIMIT="$cpu" API_MEM_LIMIT="$mem" compose up -d --no-deps api
|
||
wait_ready
|
||
|
||
# Warm up caches, connection pools and the JIT-free query plans; the warmup
|
||
# data stays in the table, so every profile grows the same history shape.
|
||
load_client "$WARMUP" "" >/dev/null
|
||
|
||
STATS="$OUT/stats-$label.csv"
|
||
printf 'timestamp,container,cpu_percent,mem_usage\n' > "$STATS"
|
||
|
||
# В профиль попадают фактические лимиты, снятые с контейнеров: иначе профиль
|
||
# описывал бы намерение, а не то, на чём действительно шло измерение.
|
||
pg_cpu_applied=$(container_cpu postgres)
|
||
pg_mem_applied=$(container_mem postgres)
|
||
lb_cpu_applied=$(container_cpu lb)
|
||
|
||
if [ "$pg_cpu_applied" != "$PG_CPU" ] || [ "$pg_mem_applied" != "$PG_MEM" ] \
|
||
|| [ "$lb_cpu_applied" != "$LB_CPU" ]; then
|
||
printf 'track1_scaling.sh: лимиты не применились: postgres просили %s/%s, получили %s/%s; lb просили %s, получили %s\n' \
|
||
"$PG_CPU" "$PG_MEM" "$pg_cpu_applied" "$pg_mem_applied" "$LB_CPU" "$lb_cpu_applied" >&2
|
||
|
||
exit 1
|
||
fi
|
||
|
||
printf '=== applied label=%s replicas=%s postgres=%s/%s lb_cpu=%s (снято с контейнеров)\n' \
|
||
"$label" "$replicas" "$pg_cpu_applied" "$pg_mem_applied" "$lb_cpu_applied" | tee -a "$SUMMARY"
|
||
|
||
PROFILE="$OUT/profile-$label.json"
|
||
printf '{"label":"%s","replicas":%s,"total_cpu":%s,"total_mem_mb":%s,"duration":%s,"concurrency":%s,"warmup":%s,"postgres_cpu_limit":"%s","postgres_mem_limit":"%s","postgres_max_conns":%s,"lb_cpu_limit":"%s","client":"%s"}\n' \
|
||
"$label" "$replicas" "$TOTAL_CPU" "$TOTAL_MEM_MB" "$DURATION" "$CONCURRENCY" "$WARMUP" \
|
||
"$pg_cpu_applied" "$pg_mem_applied" "$PG_CONNS" "$lb_cpu_applied" "$CLIENT" > "$PROFILE"
|
||
|
||
(
|
||
while :; do
|
||
docker stats --no-stream \
|
||
--format '{{.Name}},{{.CPUPerc}},{{.MemUsage}}' \
|
||
| sed "s|^|$(date -u +%Y-%m-%dT%H:%M:%SZ),|" >> "$STATS"
|
||
sleep 1
|
||
done
|
||
) &
|
||
|
||
sampler=$!
|
||
trap 'kill "$sampler" 2>/dev/null || true' EXIT INT TERM
|
||
|
||
# shellcheck disable=SC2086
|
||
load_client "$DURATION" "$OUT/load-$label.json" | tee -a "$SUMMARY"
|
||
|
||
kill "$sampler" 2>/dev/null || true
|
||
wait "$sampler" 2>/dev/null || true
|
||
trap - EXIT INT TERM
|
||
|
||
# Shared-dependency view right after the run: connections, pool pressure and
|
||
# how much history the profile wrote into.
|
||
compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -Atc \
|
||
"SELECT 'postgres_connections=' || count(*) FROM pg_stat_activity;
|
||
SELECT 'postgres_max_connections=' || current_setting('max_connections');
|
||
SELECT 'messages_rows=' || count(*) FROM messages;
|
||
SELECT 'idempotency_rows=' || count(*) FROM message_idempotency;" | tee -a "$SUMMARY"
|
||
|
||
printf '\n' >> "$SUMMARY"
|
||
done
|
||
|
||
# Контрольные профили меняют бюджет общих хранилищ; после серии стенд
|
||
# возвращается к бюджетам compose, чтобы следующий сценарий шёл на них.
|
||
if [ "$PG_CPU" != "$PG_DEFAULT_CPU" ] || [ "$PG_MEM" != "$PG_DEFAULT_MEM" ]; then
|
||
printf '=== возвращаю PostgreSQL к бюджету %s/%s ===\n' \
|
||
"$PG_DEFAULT_CPU" "$PG_DEFAULT_MEM" | tee -a "$SUMMARY"
|
||
|
||
POSTGRES_CPU_LIMIT="$PG_DEFAULT_CPU" POSTGRES_MEM_LIMIT="$PG_DEFAULT_MEM" \
|
||
compose up -d --no-deps postgres >/dev/null
|
||
|
||
wait_postgres_ready
|
||
fi
|
||
|
||
if [ "$LB_CPU" != "$LB_DEFAULT_CPU" ]; then
|
||
printf '=== возвращаю LB к бюджету CPU %s ===\n' "$LB_DEFAULT_CPU" | tee -a "$SUMMARY"
|
||
|
||
LB_CPU_LIMIT="$LB_DEFAULT_CPU" compose up -d --no-deps lb >/dev/null
|
||
fi
|
||
|
||
printf 'track1_scaling.sh: artifacts in %s\n' "$OUT"
|