lab3
This commit is contained in:
Executable → Regular
+1
-1
@@ -11,6 +11,6 @@ mkdir -p evidence/cache
|
||||
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy \
|
||||
NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1 \
|
||||
python3 tests/cache_api_performance.py
|
||||
python3 student-tests/cache_api_performance.py
|
||||
} > evidence/cache/api-performance.txt 2>&1
|
||||
cat evidence/cache/api-performance.txt
|
||||
|
||||
Executable → Regular
Executable → Regular
Executable → Regular
Executable
+194
@@ -0,0 +1,194 @@
|
||||
#!/bin/sh
|
||||
# Проверка критерия «LB исключает неготовую реплику» (lab3).
|
||||
#
|
||||
# Рядом с рабочими репликами поднимается третья с тем же сетевым alias, но её
|
||||
# доступ к БД идёт через временный TCP-прокси. Пока прокси жив, реплика готова и
|
||||
# получает трафик через LB. После остановки прокси реплика остаётся доступной по
|
||||
# сети (процесс жив, порт открыт), но /health/ready отвечает 503 — и балансировщик
|
||||
# обязан убрать её из ротации и не отправлять ей ни одного запроса.
|
||||
#
|
||||
# scripts/lb_readiness_check.sh # стенд поднят (make up)
|
||||
# OUT=evidence/lb-readiness.txt scripts/lb_readiness_check.sh
|
||||
set -eu
|
||||
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
# .env определяет, на каком порту публикуется LB (compose.yaml) и какие креды у
|
||||
# хранилищ, поэтому значения берём оттуда же, а не из окружения вызывающего.
|
||||
if [ -f .env ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. ./.env
|
||||
set +a
|
||||
fi
|
||||
|
||||
OUT=${OUT:-evidence/lb-readiness.txt}
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab3}
|
||||
NET=${PROJECT}_default
|
||||
IMAGE=${IMAGE:-mtusi-messenger:lab3}
|
||||
STATS=${STATS:-http://127.0.0.1:8404/stats;csv}
|
||||
BASE_URL=${BASE_URL:-http://localhost:8080}
|
||||
CANDIDATE=lb-readiness-candidate
|
||||
PROXY=lb-readiness-pgproxy
|
||||
REQUESTS=${REQUESTS:-40}
|
||||
|
||||
NO_PROXY_ENV="env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy"
|
||||
NO_PROXY_ENV="$NO_PROXY_ENV NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1"
|
||||
|
||||
cleanup() {
|
||||
docker rm -f "$CANDIDATE" "$PROXY" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
trap cleanup EXIT INT TERM
|
||||
cleanup
|
||||
|
||||
instances() {
|
||||
tries=${1:-$REQUESTS}
|
||||
i=0
|
||||
|
||||
while [ "$i" -lt "$tries" ]; do
|
||||
curl -fsS --noproxy '*' -D - -o /dev/null "$BASE_URL/health/live" 2>/dev/null \
|
||||
| awk 'tolower($1) == "x-instance-id:" { print $2 }'
|
||||
i=$((i + 1))
|
||||
done | sort | uniq -c
|
||||
}
|
||||
|
||||
stats() {
|
||||
curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ { printf "%s=%s ", $2, $18 }'
|
||||
}
|
||||
|
||||
readiness() {
|
||||
docker exec "$CANDIDATE" wget -qO- -S -T 3 http://127.0.0.1:8080/health/ready 2>&1 \
|
||||
| awk '/^ HTTP\//{print $2; exit}'
|
||||
}
|
||||
|
||||
wait_readiness() {
|
||||
want=$1
|
||||
attempt=0
|
||||
|
||||
while [ "$attempt" -lt 30 ]; do
|
||||
[ "$(readiness)" = "$want" ] && return 0
|
||||
attempt=$((attempt + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
up_slots() {
|
||||
curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ && $18 == "UP" && $2 != "BACKEND" { printf "%s ", $2 }'
|
||||
}
|
||||
|
||||
wait_new_up_slot() {
|
||||
before=$1
|
||||
attempt=0
|
||||
|
||||
while [ "$attempt" -lt 30 ]; do
|
||||
for slot in $(up_slots); do
|
||||
case " $before " in
|
||||
*" $slot "*) ;;
|
||||
*) echo "$slot"; return 0 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_slot_status() {
|
||||
slot=$1
|
||||
want=$2
|
||||
attempt=0
|
||||
|
||||
while [ "$attempt" -lt 60 ]; do
|
||||
status=$(curl -s --noproxy '*' "$STATS" | awk -F, -v s="$slot" '$1 ~ /^api/ && $2 == s { print $18; exit }')
|
||||
|
||||
if [ "$status" = "$want" ]; then
|
||||
SWAIT_SECONDS=$((attempt / 2))
|
||||
return 0
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
{
|
||||
echo "# LB excludes a not-ready but reachable replica (lab3)"
|
||||
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "# lb: $(docker inspect -f '{{.Config.Image}}' "$(docker ps -q --filter label=com.docker.compose.service=lb)" 2>/dev/null)"
|
||||
echo
|
||||
echo "## 1. baseline: requests through LB by replica"
|
||||
instances
|
||||
baseline_slots=$(up_slots)
|
||||
echo " stats: $(stats)"
|
||||
echo
|
||||
echo "## 2. start a temporary TCP proxy to postgres and a candidate replica behind the same network alias"
|
||||
docker run -d --name "$PROXY" --network "$NET" alpine/socat@sha256:fb3862c949af7c79996d0e514ee96128644d840835ed5368eeecb14a869de361 \
|
||||
tcp-listen:5432,fork,reuseaddr tcp-connect:postgres:5432 >/dev/null
|
||||
docker run -d --name "$CANDIDATE" --network "$NET" --network-alias api-replica \
|
||||
-e APP_CONFIG=/etc/messenger/dev/config.yaml \
|
||||
-e POSTGRES_HOST="$PROXY" -e POSTGRES_PORT=5432 \
|
||||
-e POSTGRES_DATABASE="${POSTGRES_DATABASE:-messenger}" \
|
||||
-e POSTGRES_USER="${POSTGRES_USER:-messenger}" \
|
||||
-e POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-messenger}" -e POSTGRES_SSL_MODE=disable \
|
||||
-e REDIS_ENABLED=true -e REDIS_HOST=redis -e CACHE_ENABLED=true \
|
||||
"$IMAGE" >/dev/null
|
||||
|
||||
if ! wait_readiness 200; then
|
||||
echo " candidate did not become ready" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo " instance-id: $(docker inspect -f '{{.Config.Hostname}}' "$CANDIDATE"), readiness while its database is reachable: 200"
|
||||
|
||||
slot=$(wait_new_up_slot "$baseline_slots" || true)
|
||||
if [ -z "$slot" ]; then
|
||||
echo " candidate did not appear UP in the load balancer" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo " load balancer slot for the candidate: $slot, stats: $(stats)"
|
||||
echo
|
||||
echo "## 3. cut the candidate's database access (stop only the TCP proxy)"
|
||||
docker stop "$PROXY" >/dev/null
|
||||
|
||||
if ! wait_readiness 503; then
|
||||
echo " candidate readiness did not become 503" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo " candidate is still running: $(docker inspect -f '{{.State.Status}}' "$CANDIDATE"), readiness: 503"
|
||||
echo " waiting for the load balancer to mark $slot DOWN"
|
||||
|
||||
if wait_slot_status "$slot" DOWN; then
|
||||
echo " $slot went DOWN ${SWAIT_SECONDS}s after its database was cut (check: inter 2s fall 2)"
|
||||
else
|
||||
echo " WARNING: $slot is still not DOWN" >&2
|
||||
fi
|
||||
|
||||
echo " stats after failure: $(stats)"
|
||||
echo
|
||||
echo "## 4. ${REQUESTS} requests through LB while the candidate is not ready"
|
||||
instances
|
||||
echo " (the candidate must not appear above)"
|
||||
echo
|
||||
echo "## 5. business writes through LB during the exclusion"
|
||||
# shellcheck disable=SC2086
|
||||
$NO_PROXY_ENV python3 tests/load.py --base-url "$BASE_URL" --duration 5 --concurrency 2 \
|
||||
| grep -E '"successful_rps"|"failed_requests"|"error_ratio"'
|
||||
echo
|
||||
echo "## 6. cleanup"
|
||||
cleanup
|
||||
sleep 5
|
||||
echo " stats after cleanup: $(stats)"
|
||||
} > "$OUT" 2>&1
|
||||
|
||||
cat "$OUT"
|
||||
printf 'lb_readiness_check.sh: evidence in %s\n' "$OUT"
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/bin/sh
|
||||
# Контроль пути «хост -> Colima VM -> LB»: тот же генератор и тот же профиль
|
||||
# запускаются дважды — с хоста через опубликованный порт и изнутри VM, где
|
||||
# клиент говорит с балансировщиком напрямую по docker-сети. Разница показывает
|
||||
# вклад этого пути в потолок closed-loop профиля, а не приписывает его по
|
||||
# догадке. Стенд должен быть поднят (`make up`).
|
||||
#
|
||||
# scripts/path_overhead_check.sh
|
||||
#
|
||||
# Артефакт: evidence/path-overhead/{inside,outside}.txt + summary.txt
|
||||
set -eu
|
||||
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
|
||||
OUT=${OUT:-evidence/path-overhead}
|
||||
DURATION=${DURATION:-60}
|
||||
CONCURRENCY=${CONCURRENCY:-8}
|
||||
WARMUP=${WARMUP:-15}
|
||||
CLIENT_IMAGE=${CLIENT_IMAGE:-python:3.12-alpine}
|
||||
|
||||
compose() {
|
||||
# shellcheck disable=SC2086
|
||||
$COMPOSE_BIN --env-file .env -f compose.yaml "$@"
|
||||
}
|
||||
|
||||
NO_PROXY_ENV="env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy"
|
||||
NO_PROXY_ENV="$NO_PROXY_ENV NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1"
|
||||
|
||||
mkdir -p "$OUT"
|
||||
|
||||
network=$(docker inspect "$(compose ps -q api | head -1)" \
|
||||
--format '{{range $name, $network := .NetworkSettings.Networks}}{{$name}}{{end}}')
|
||||
|
||||
printf '# Контроль пути к балансировщику\n' > "$OUT/summary.txt"
|
||||
{
|
||||
printf '# generated: %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
printf '# profile: tests/load.py duration=%s concurrency=%s warmup=%s\n' "$DURATION" "$CONCURRENCY" "$WARMUP"
|
||||
printf '# docker network: %s\n' "$network"
|
||||
printf '# клиент в VM: %s\n\n' "$CLIENT_IMAGE"
|
||||
} >> "$OUT/summary.txt"
|
||||
|
||||
# Клиент с хоста: путь «хост -> проброс порта -> VM -> LB».
|
||||
printf '=== извне: клиент на хосте, http://127.0.0.1:8080 ===\n' | tee -a "$OUT/summary.txt"
|
||||
# shellcheck disable=SC2086
|
||||
$NO_PROXY_ENV python3 tests/load.py --base-url http://127.0.0.1:8080 \
|
||||
--duration "$WARMUP" --concurrency "$CONCURRENCY" >/dev/null
|
||||
# shellcheck disable=SC2086
|
||||
$NO_PROXY_ENV python3 tests/load.py --base-url http://127.0.0.1:8080 \
|
||||
--duration "$DURATION" --concurrency "$CONCURRENCY" \
|
||||
--output "$OUT/load-outside.json" | tee "$OUT/outside.txt" | tail -12
|
||||
|
||||
# Клиент внутри VM: тот же генератор и профиль, но без проброса порта и без
|
||||
# сетевого стека хоста. Генератор делит 2 vCPU VM со стендом, поэтому его
|
||||
# результат — нижняя оценка, а не потолок.
|
||||
printf '\n=== изнутри: клиент в VM, http://lb ===\n' | tee -a "$OUT/summary.txt"
|
||||
docker run --rm -i --network "$network" -v "$ROOT_DIR:/repo:ro" -v "$ROOT_DIR/$OUT:/out" \
|
||||
-e NO_PROXY='lb,localhost,127.0.0.1' -e no_proxy='lb,localhost,127.0.0.1' \
|
||||
"$CLIENT_IMAGE" python /repo/tests/load.py --base-url http://lb \
|
||||
--duration "$WARMUP" --concurrency "$CONCURRENCY" >/dev/null
|
||||
docker run --rm -i --network "$network" -v "$ROOT_DIR:/repo:ro" -v "$ROOT_DIR/$OUT:/out" \
|
||||
-e NO_PROXY='lb,localhost,127.0.0.1' -e no_proxy='lb,localhost,127.0.0.1' \
|
||||
"$CLIENT_IMAGE" python /repo/tests/load.py --base-url http://lb \
|
||||
--duration "$DURATION" --concurrency "$CONCURRENCY" \
|
||||
--output /out/load-inside.json | tee "$OUT/inside.txt" | tail -12
|
||||
|
||||
printf 'path_overhead_check.sh: артефакты в %s\n' "$OUT"
|
||||
Executable → Regular
+53
-6
@@ -1,11 +1,58 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# Restart hook: bounce API replicas and every state store of the lab3 stand.
|
||||
#
|
||||
# scripts/restart.sh
|
||||
#
|
||||
# Persistent volumes are kept: no `down -v`, no flush, no re-seed. The one-shot
|
||||
# `migrate` service is not re-run (schema stays as it is, data and sessions
|
||||
# survive). The script exits 0 only after /health/ready answers through the LB.
|
||||
# No credentials are printed.
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
COMPOSE_BIN=${COMPOSE_BIN:-docker-compose}
|
||||
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
"$COMPOSE_BIN" -f docker-compose.local.yaml restart postgres redis
|
||||
"$COMPOSE_BIN" -f docker-compose.local.yaml up -d --wait postgres redis
|
||||
"$COMPOSE_BIN" -f docker-compose.local.yaml up -d --force-recreate api
|
||||
# .env определяет, на каком порту публикуется LB (compose.yaml) и какие креды у
|
||||
# хранилищ, поэтому значения берём оттуда же, а не из окружения вызывающего.
|
||||
if [ -f .env ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. ./.env
|
||||
set +a
|
||||
fi
|
||||
|
||||
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
|
||||
PORT=${SERVER_PORT:-8080}
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
compose() {
|
||||
# shellcheck disable=SC2086
|
||||
$COMPOSE_BIN --env-file .env -f compose.yaml "$@"
|
||||
}
|
||||
|
||||
# 1. State stores: restart the running ones, then make sure they are healthy.
|
||||
# Volumes are attached as before, nothing is dropped or re-seeded.
|
||||
compose up -d --wait postgres redis
|
||||
compose restart postgres redis
|
||||
compose up -d --wait postgres redis
|
||||
|
||||
# 2. API replicas: restart in place (same containers, same addresses) and bring
|
||||
# back any replica that was stopped by the fault-injection hook.
|
||||
compose restart api
|
||||
compose up -d --wait --no-deps api
|
||||
|
||||
# 3. Load balancer keeps its upstream; re-resolve in case addresses changed.
|
||||
compose restart lb
|
||||
|
||||
# 4. Wait for readiness through the public entry point.
|
||||
attempt=0
|
||||
while [ "$attempt" -lt 60 ]; do
|
||||
if curl -fsS --noproxy '*' -o /dev/null "http://127.0.0.1:${PORT}/health/ready"; then
|
||||
printf 'restart.sh: stack restarted, http://127.0.0.1:%s/health/ready is ready\n' "$PORT"
|
||||
exit 0
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
printf 'restart.sh: /health/ready did not become ready through the LB\n' >&2
|
||||
exit 1
|
||||
|
||||
Executable
+231
@@ -0,0 +1,231 @@
|
||||
#!/bin/sh
|
||||
# Rolling update of the API replicas under load (lab3, level 4).
|
||||
#
|
||||
# Замена идёт «наплывом» и строго по одной реплике: сначала рядом поднимается
|
||||
# контейнер новой ревизии с тем же сетевым alias, затем снимается ОДИН старый.
|
||||
# Каждый шаг гейтится состоянием балансировщика: новый слот должен стать UP
|
||||
# (активная проверка /health/ready), снятый — перестать быть UP. Гейт по
|
||||
# статистике LB, а не по наблюдению X-Instance-Id в ответах, потому что HAProxy
|
||||
# переиспользует keepalive-соединения к уже работающим репликам и свежая реплика
|
||||
# может какое-то время не получать трафик, оставаясь при этом готовой.
|
||||
#
|
||||
# В конце стенд возвращается под управление compose: поднимаются штатные реплики
|
||||
# из новой ревизии, затем по одной снимаются временные контейнеры наплыва.
|
||||
#
|
||||
# scripts/rolling_update.sh # профиль лимитов берётся из .env/compose.yaml
|
||||
# DURATION=60 CONCURRENCY=8 scripts/rolling_update.sh
|
||||
set -eu
|
||||
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
|
||||
PYTHON=${PYTHON:-python3}
|
||||
DURATION=${DURATION:-60}
|
||||
CONCURRENCY=${CONCURRENCY:-8}
|
||||
BASE_URL=${BASE_URL:-http://localhost:8080}
|
||||
OUT=${OUT:-evidence/rolling-update}
|
||||
IMAGE=${IMAGE:-mtusi-messenger:lab3}
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab3}
|
||||
NET=${PROJECT}_default
|
||||
STATS=${STATS:-http://127.0.0.1:8404/stats;csv}
|
||||
PREFIX=lb-rollout
|
||||
|
||||
NO_PROXY_ENV="env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy"
|
||||
NO_PROXY_ENV="$NO_PROXY_ENV NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1"
|
||||
|
||||
mkdir -p "$OUT"
|
||||
rm -f "$OUT/load.json" "$OUT/load.txt"
|
||||
|
||||
compose() {
|
||||
# shellcheck disable=SC2086
|
||||
$COMPOSE_BIN --env-file .env -f compose.yaml "$@"
|
||||
}
|
||||
|
||||
api_containers() {
|
||||
docker ps -q \
|
||||
--filter "label=com.docker.compose.project=$PROJECT" \
|
||||
--filter 'label=com.docker.compose.service=api'
|
||||
}
|
||||
|
||||
rollout_containers() {
|
||||
docker ps -q --filter "name=^$PREFIX"
|
||||
}
|
||||
|
||||
restarts() {
|
||||
for container in $(api_containers) $(rollout_containers); do
|
||||
printf '%s=%s ' "$(docker inspect --format '{{.Name}}' "$container" | sed 's|^/||')" \
|
||||
"$(docker inspect --format '{{.RestartCount}}' "$container")"
|
||||
done
|
||||
}
|
||||
|
||||
stats_rows() {
|
||||
curl -s --noproxy '*' "$STATS"
|
||||
}
|
||||
|
||||
# В CSV-выгрузке HAProxy колонка addr приходит пустой, поэтому привязка слота к
|
||||
# конкретному контейнеру идёт по двум наблюдаемым фактам: контейнер отвечает на
|
||||
# /health/ready, а число UP-слотов соответствует ожидаемому.
|
||||
container_ready() {
|
||||
docker exec "$1" wget -qO- -T 3 http://127.0.0.1:8080/health/ready 2>/dev/null | grep -q '"status":"ok"'
|
||||
}
|
||||
|
||||
wait_container_ready() {
|
||||
name=$1
|
||||
attempt=0
|
||||
|
||||
while [ "$attempt" -lt 45 ]; do
|
||||
container_ready "$name" && return 0
|
||||
attempt=$((attempt + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
up_slots() {
|
||||
stats_rows | awk -F, '
|
||||
NR == 1 { for (i = 1; i <= NF; i++) if ($i == "status") s = i; next }
|
||||
$1 ~ /^api/ && $s == "UP" { count++ } END { print count + 0 }'
|
||||
}
|
||||
|
||||
wait_up_slots() {
|
||||
want=$1
|
||||
attempt=0
|
||||
|
||||
while [ "$attempt" -lt 45 ]; do
|
||||
[ "$(up_slots)" -ge "$want" ] && return 0
|
||||
attempt=$((attempt + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
lb_5xx() {
|
||||
stats_rows | awk -F, '
|
||||
NR == 1 { for (i = 1; i <= NF; i++) if ($i == "hrsp_5xx") column = i; next }
|
||||
$1 == "http_front" && $2 == "FRONTEND" && column { print $column; exit }'
|
||||
}
|
||||
|
||||
{
|
||||
echo "# Rolling update under load (surge, one replica at a time)"
|
||||
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "# load: tolerant loader, duration=${DURATION}s concurrency=${CONCURRENCY}"
|
||||
echo "# проверки готовности LB: httpchk /health/ready, inter 2s fall 2 rise 2, timeout check 1s"
|
||||
echo
|
||||
|
||||
# Сборка пишет много строк; в артефакт они не нужны.
|
||||
compose build api >/dev/null 2>&1 || compose build api
|
||||
|
||||
before_containers=$(api_containers)
|
||||
replicas=$(printf '%s\n' "$before_containers" | wc -l | tr -d ' ')
|
||||
[ "$replicas" -ge 2 ] || { echo "нужно не менее двух API-реплик" >&2; exit 1; }
|
||||
|
||||
old_image=$(docker inspect --format '{{.Image}}' "$(printf '%s\n' "$before_containers" | head -1)")
|
||||
new_image=$(docker image inspect --format '{{.Id}}' "$IMAGE")
|
||||
echo "реплик до обновления: ${replicas}"
|
||||
echo "образ старых реплик: ${old_image}"
|
||||
echo "образ новой ревизии: ${new_image}"
|
||||
echo "ревизия меняется: $([ "$old_image" = "$new_image" ] && echo 'нет (образ не изменился)' || echo 'да')"
|
||||
echo "слотов UP до: $(up_slots); restarts до: $(restarts)"
|
||||
echo
|
||||
|
||||
envfile=$(mktemp)
|
||||
trap 'rm -f "$envfile"; docker rm -f $(rollout_containers) >/dev/null 2>&1 || true' EXIT INT TERM
|
||||
docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \
|
||||
"$(printf '%s\n' "$before_containers" | head -1)" > "$envfile"
|
||||
|
||||
# Перед измерением убеждаемся, что все ожидаемые реплики видит сам LB: после
|
||||
# экспериментов с отказом хранилищ слоты могут быть в переходном состоянии.
|
||||
if ! wait_up_slots "$replicas"; then
|
||||
echo "перед обновлением LB видит меньше ${replicas} реплик" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
five_before=$(lb_5xx)
|
||||
# shellcheck disable=SC2086
|
||||
$NO_PROXY_ENV $PYTHON student-tests/rolling_update_load.py --base-url "$BASE_URL" \
|
||||
--duration "$DURATION" --concurrency "$CONCURRENCY" \
|
||||
--output "$OUT/load.json" > "$OUT/load.txt" 2>&1 &
|
||||
load_pid=$!
|
||||
|
||||
step=0
|
||||
for container in $before_containers; do
|
||||
step=$((step + 1))
|
||||
name="$PREFIX-$step"
|
||||
old_instance=$(docker inspect --format '{{.Config.Hostname}}' "$container")
|
||||
started=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
|
||||
# 1. поднять реплику новой ревизии рядом со старыми; дождаться, что она
|
||||
# сама готова и что LB видит на один слот больше
|
||||
docker run -d --name "$name" --network "$NET" --network-alias api-replica \
|
||||
--env-file "$envfile" "$IMAGE" >/dev/null
|
||||
new_instance=$(docker inspect --format '{{.Config.Hostname}}' "$name")
|
||||
new_container_image=$(docker inspect --format '{{.Image}}' "$name")
|
||||
|
||||
if ! wait_container_ready "$name"; then
|
||||
echo " step ${step}: реплика ${name} не стала готова" >&2
|
||||
echo " diagnostics: status=$(docker inspect --format '{{.State.Status}}' "$name" 2>&1); stats: $(stats_rows | awk -F, '$1 ~ /^api/ { printf "%s=%s ", $2, $18 }')" >&2
|
||||
docker logs --tail 20 "$name" >&2 2>&1 || true
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! wait_up_slots "$((replicas + step))"; then
|
||||
echo " step ${step}: UP-слотов меньше ожидаемого ($(up_slots) вместо $((replicas + step)))" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 2. снять одну старую реплику и дождаться, что UP-слотов снова ровно
|
||||
# столько, сколько реплик должно остаться
|
||||
docker rm -f "$container" >/dev/null
|
||||
|
||||
if ! wait_up_slots "$((replicas + step - 1))"; then
|
||||
echo " step ${step}: после снятия UP-слотов больше ожидаемого" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf ' step %s: %s обслуживает (образ %s, instance %s), старый контейнер %s (%s) снят; UP-слотов %s; 5xx накоплено %s; старт шага %s\n' \
|
||||
"$step" "$name" "${new_container_image#sha256:}" "$new_instance" \
|
||||
"${container:0:12}" "$old_instance" "$(up_slots)" "$(lb_5xx)" "$started"
|
||||
done
|
||||
|
||||
wait "$load_pid" || true
|
||||
|
||||
five_after=$(lb_5xx)
|
||||
echo
|
||||
echo "load summary (tolerant generator):"
|
||||
cat "$OUT/load.txt"
|
||||
echo
|
||||
echo "5xx от LB за время обновления (hrsp_5xx frontend): $((${five_after:-0} - ${five_before:-0}))"
|
||||
echo "restarts после: $(restarts)"
|
||||
|
||||
# 3. вернуть стенд под управление compose и убрать временные контейнеры
|
||||
echo
|
||||
echo "возврат стенда под управление compose:"
|
||||
compose up -d --no-deps api
|
||||
wait_up_slots "$replicas" || true
|
||||
|
||||
remaining=$(printf '%s\n' $(rollout_containers) | wc -l | tr -d ' ')
|
||||
for container in $(rollout_containers); do
|
||||
docker rm -f "$container" >/dev/null
|
||||
remaining=$((remaining - 1))
|
||||
|
||||
if ! wait_up_slots "$((replicas + remaining))"; then
|
||||
echo " после снятия ${container} UP-слотов больше ожидаемого" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "снято временных контейнеров наплыва: проверено"
|
||||
|
||||
echo "UP-слотов после: $(up_slots); restarts в конце: $(restarts)"
|
||||
echo "API containers после обновления:"
|
||||
docker ps --format '{{.Names}} {{.Status}}' \
|
||||
--filter "label=com.docker.compose.project=$PROJECT" \
|
||||
--filter 'label=com.docker.compose.service=api'
|
||||
} > "$OUT/rolling-update.txt" 2>&1
|
||||
|
||||
cat "$OUT/rolling-update.txt"
|
||||
printf 'rolling_update.sh: evidence in %s\n' "$OUT"
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# TODO: copy to stop-one.sh, map TARGET_INSTANCE_ID to ONE API container/task.
|
||||
# Stop that instance and leave it stopped; no stop of DB/LB/volumes.
|
||||
# Return quickly (within 10 s); the test sends requests during this script.
|
||||
# Restore the replica explicitly after the test. Never print credentials.
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
printf '%s
|
||||
' 'TODO: implement stop-one.sh for your infrastructure' >&2
|
||||
exit 2
|
||||
Executable
+72
@@ -0,0 +1,72 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Fault-injection hook: stop exactly ONE api replica of the lab3 stand.
|
||||
#
|
||||
# TARGET_INSTANCE_ID=<hostname|container name|short id> scripts/stop-one.sh
|
||||
#
|
||||
# The task value is the instance alias the API reports in X-Instance-Id, i.e.
|
||||
# the container hostname (docker's default = short container id). Container
|
||||
# names of the api service are matched too. The container is only stopped
|
||||
# (docker stop); databases, the load balancer and volumes are left untouched,
|
||||
# and the replica stays down until it is started again explicitly, e.g.
|
||||
# docker compose --env-file .env up -d --no-deps api
|
||||
# No credentials are printed.
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab3}
|
||||
|
||||
candidates=$(docker ps -q \
|
||||
--filter "label=com.docker.compose.project=${PROJECT}" \
|
||||
--filter "label=com.docker.compose.service=api")
|
||||
|
||||
if [ -z "$candidates" ]; then
|
||||
printf 'stop-one.sh: no api containers of project %s are running\n' "$PROJECT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Matches the requested alias against every reasonable identifier of a replica:
|
||||
# container hostname (= X-Instance-Id), full/short container id, full container
|
||||
# name (mtusi-messenger-lab3-api-1) and the service-local suffix (api-1).
|
||||
match_replica() {
|
||||
target=$1
|
||||
full_id=$2
|
||||
short_id=$3
|
||||
name=$4
|
||||
hostname=$5
|
||||
suffix=${name#${PROJECT}-}
|
||||
|
||||
[ "$target" = "$hostname" ] && return 0
|
||||
[ "$target" = "$full_id" ] && return 0
|
||||
[ "$target" = "$short_id" ] && return 0
|
||||
[ "$target" = "$name" ] && return 0
|
||||
[ "$target" = "$suffix" ] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
matched=""
|
||||
matched_names=""
|
||||
for id in $candidates; do
|
||||
info=$(docker inspect --format '{{.Id}} {{.Name}} {{.Config.Hostname}}' "$id")
|
||||
full_id=$(printf '%s' "$info" | cut -d' ' -f1)
|
||||
name=$(printf '%s' "$info" | cut -d' ' -f2 | sed 's|^/||')
|
||||
hostname=$(printf '%s' "$info" | cut -d' ' -f3)
|
||||
short_id=$(printf '%s' "$full_id" | cut -c1-12)
|
||||
|
||||
if match_replica "$TARGET_INSTANCE_ID" "$full_id" "$short_id" "$name" "$hostname"; then
|
||||
matched="$matched $id"
|
||||
matched_names="$matched_names $name"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$matched" ]; then
|
||||
printf 'stop-one.sh: no api replica matches TARGET_INSTANCE_ID=%s\n' "$TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Shut down cleanly (SIGTERM) but never hang the caller for long.
|
||||
for id in $matched; do
|
||||
docker stop --time 5 "$id" >/dev/null
|
||||
done
|
||||
|
||||
printf 'stop-one.sh: stopped api replica(s):%s\n' "$matched_names"
|
||||
exit 0
|
||||
@@ -1,267 +0,0 @@
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FAST_PATH = """\tif len(history) == 0 || compareKeyset(
|
||||
\t\thistory[len(history)-1].CreatedAt,
|
||||
\t\thistory[len(history)-1].ID,
|
||||
\t\tmessage.CreatedAt,
|
||||
\t\tmessage.ID,
|
||||
\t) < 0 {
|
||||
\t\ts.messagesByChat[message.ChatID] = append(history, message)
|
||||
|
||||
\t\treturn message, nil
|
||||
\t}
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def request(opener, base, method, path, body=None, user=None):
|
||||
data = json.dumps(body, ensure_ascii=False).encode() if body is not None else None
|
||||
headers = {}
|
||||
if data is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
if user is not None:
|
||||
headers["X-User-Id"] = user
|
||||
req = urllib.request.Request(base + path, data=data, headers=headers, method=method)
|
||||
with opener.open(req, timeout=10) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
def available_port():
|
||||
with socket.socket() as listener:
|
||||
listener.bind(("127.0.0.1", 0))
|
||||
return listener.getsockname()[1]
|
||||
|
||||
|
||||
def descendants(root_pid):
|
||||
result = subprocess.run(
|
||||
["ps", "-axo", "pid=,ppid=,rss=,command="],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
entries = []
|
||||
for line in result.stdout.splitlines():
|
||||
fields = line.strip().split(None, 3)
|
||||
if len(fields) == 4:
|
||||
try:
|
||||
entries.append((int(fields[0]), int(fields[1]), int(fields[2]), fields[3]))
|
||||
except ValueError:
|
||||
continue
|
||||
children = {root_pid}
|
||||
while True:
|
||||
discovered = children | {pid for pid, ppid, _, _ in entries if ppid in children}
|
||||
if discovered == children:
|
||||
break
|
||||
children = discovered
|
||||
return [entry for entry in entries if entry[0] in children and entry[0] != root_pid]
|
||||
|
||||
|
||||
def server_rss_kib(go_run_pid):
|
||||
servers = [rss for _, _, rss, command in descendants(go_run_pid) if command.rstrip().endswith("/messenger")]
|
||||
if not servers:
|
||||
raise RuntimeError("go run server process was not found")
|
||||
return max(servers)
|
||||
|
||||
|
||||
def wait_ready(process, opener, base):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
if process.poll() is not None:
|
||||
raise RuntimeError(f"go run exited with code {process.returncode}")
|
||||
try:
|
||||
request(opener, base, "GET", "/health/live")
|
||||
server_rss_kib(process.pid)
|
||||
return
|
||||
except (OSError, RuntimeError):
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError("go run server did not become ready")
|
||||
|
||||
|
||||
def percentile(latencies, fraction):
|
||||
ordered = sorted(latencies)
|
||||
index = max(0, min(len(ordered) - 1, (len(ordered) * int(fraction * 100) + 99) // 100 - 1))
|
||||
return round(ordered[index] * 1000, 3)
|
||||
|
||||
|
||||
def setup(opener, base, history):
|
||||
users = []
|
||||
for _ in range(10):
|
||||
user = request(opener, base, "POST", "/api/v1/users", {
|
||||
"username": "b" + uuid.uuid4().hex[:20],
|
||||
"display_name": "Benchmark",
|
||||
})
|
||||
users.append(user)
|
||||
|
||||
chat = request(opener, base, "POST", "/api/v1/chats", {
|
||||
"title": "Benchmark",
|
||||
"member_ids": [user["id"] for user in users],
|
||||
}, users[0]["id"])
|
||||
|
||||
for index in range(history):
|
||||
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
|
||||
"text": f"history-{index}",
|
||||
}, users[index % len(users)]["id"])
|
||||
|
||||
return chat, users
|
||||
|
||||
|
||||
def launch(mode, port, overlay):
|
||||
command = ["go", "run"]
|
||||
if mode == "baseline":
|
||||
command.extend(["-overlay", str(overlay)])
|
||||
command.append("./cmd/messenger")
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
env={**os.environ, "APP_HOST": "127.0.0.1", "APP_PORT": str(port)},
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
)
|
||||
return process, command
|
||||
|
||||
|
||||
def run_case(mode, history, concurrency, requests, overlay):
|
||||
port = available_port()
|
||||
base = f"http://127.0.0.1:{port}"
|
||||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
process, command = launch(mode, port, overlay)
|
||||
try:
|
||||
wait_ready(process, opener, base)
|
||||
chat, users = setup(opener, base, history)
|
||||
rss = []
|
||||
stop_sampling = threading.Event()
|
||||
sampling_errors = []
|
||||
|
||||
def sample():
|
||||
while not stop_sampling.is_set():
|
||||
try:
|
||||
rss.append(server_rss_kib(process.pid))
|
||||
except RuntimeError as exc:
|
||||
sampling_errors.append(str(exc))
|
||||
stop_sampling.wait(0.1)
|
||||
|
||||
sampler = threading.Thread(target=sample)
|
||||
sampler.start()
|
||||
warmup = min(100, max(10, requests // 100))
|
||||
|
||||
for index in range(warmup):
|
||||
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
|
||||
"text": f"warmup-{index}",
|
||||
}, users[index % len(users)]["id"])
|
||||
|
||||
started = time.monotonic()
|
||||
|
||||
def worker(worker_id):
|
||||
latencies = []
|
||||
failures = 0
|
||||
worker_requests = requests // concurrency + (1 if worker_id < requests % concurrency else 0)
|
||||
for index in range(worker_requests):
|
||||
request_started = time.perf_counter()
|
||||
try:
|
||||
request(opener, base, "POST", f"/api/v1/chats/{chat['id']}/messages", {
|
||||
"text": f"load-{worker_id}-{index}",
|
||||
}, users[worker_id % len(users)]["id"])
|
||||
latencies.append(time.perf_counter() - request_started)
|
||||
except (OSError, ValueError, urllib.error.HTTPError):
|
||||
failures += 1
|
||||
index += 1
|
||||
return latencies, failures
|
||||
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
results = list(pool.map(worker, range(concurrency)))
|
||||
finally:
|
||||
elapsed = time.monotonic() - started
|
||||
stop_sampling.set()
|
||||
sampler.join()
|
||||
|
||||
latencies = [value for values, _ in results for value in values]
|
||||
failures = sum(value for _, value in results)
|
||||
if not latencies or not rss or sampling_errors:
|
||||
raise RuntimeError(f"invalid sample: successes={len(latencies)} rss={len(rss)} errors={sampling_errors}")
|
||||
|
||||
return {
|
||||
"history": history,
|
||||
"concurrency": concurrency,
|
||||
"duration_seconds": round(elapsed, 3),
|
||||
"requested_requests": requests,
|
||||
"warmup_requests": warmup,
|
||||
"successful_requests": len(latencies),
|
||||
"failed_requests": failures,
|
||||
"rps": round(len(latencies) / elapsed, 3),
|
||||
"p50_ms": percentile(latencies, 0.50),
|
||||
"p95_ms": percentile(latencies, 0.95),
|
||||
"p99_ms": percentile(latencies, 0.99),
|
||||
"rss_min_kib": min(rss),
|
||||
"rss_max_kib": max(rss),
|
||||
"rss_delta_kib": max(rss) - min(rss),
|
||||
}, command
|
||||
finally:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
|
||||
|
||||
def baseline_overlay(directory):
|
||||
source = (ROOT / "internal/storage/memory/message_repository.go").resolve()
|
||||
content = source.read_text()
|
||||
if content.count(FAST_PATH) != 1:
|
||||
raise RuntimeError("expected exactly one optimized append block")
|
||||
replacement = directory / "message_repository_baseline.go"
|
||||
replacement.write_text(content.replace(FAST_PATH, "", 1))
|
||||
overlay = directory / "overlay.json"
|
||||
overlay.write_text(json.dumps({"Replace": {str(source): str(replacement)}}))
|
||||
return overlay
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--mode", choices=("baseline", "optimized"), required=True)
|
||||
parser.add_argument("--requests", type=int, default=10000)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.requests <= 0:
|
||||
parser.error("--requests must be positive")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="messenger-track1-") as temp_dir:
|
||||
overlay = baseline_overlay(Path(temp_dir))
|
||||
rows = []
|
||||
for history in (1000, 10000, 50000):
|
||||
for concurrency in (1, 4, 16, 32):
|
||||
row, command = run_case(args.mode, history, concurrency, args.requests, overlay)
|
||||
rows.append(row)
|
||||
print(json.dumps(row, ensure_ascii=False), flush=True)
|
||||
result = {
|
||||
"mode": args.mode,
|
||||
"command": command,
|
||||
"go_version": subprocess.check_output(["go", "version"], text=True).strip(),
|
||||
"os_arch": subprocess.check_output(["go", "env", "GOOS", "GOARCH"], text=True).split(),
|
||||
"requests_per_case": args.requests,
|
||||
"rss_source": "ps RSS of go run child executable, KiB; sampled every 100 ms",
|
||||
"rows": rows,
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/bin/sh
|
||||
# Контроль бюджетов БД и балансировщика, перемежённый с эталоном.
|
||||
#
|
||||
# По ходу серии пропускная способность дрейфует: таблица сообщений растёт
|
||||
# (~95 тыс. строк на профиль), а вместе с ней и стоимость записи. Поэтому
|
||||
# сравнивать профиль с бюджетом БД 1.00 CPU с эталоном, снятым четверть часа
|
||||
# назад, нельзя — разница бюджетов утонет в дрейфе. Здесь эталон (2 реплики,
|
||||
# PostgreSQL 0.50 CPU, LB 0.25 CPU) повторяется дважды, а каждая контролируемая
|
||||
# ручка меняется ровно в одном профиле рядом с эталоном:
|
||||
#
|
||||
# ctl1base -> ctl2pg (БД 1.00) -> ctl3base -> ctl4lb (LB 1.00)
|
||||
#
|
||||
# Пара ctl1base/ctl3base даёт масштаб дрейфа за два профиля.
|
||||
#
|
||||
# scripts/track1_controls.sh
|
||||
# rm evidence/track1/{load,stats,profile}-ctl*.{json,csv} # перемерить
|
||||
set -eu
|
||||
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
SCALING=scripts/track1_scaling.sh
|
||||
OUT=${OUT:-evidence/track1}
|
||||
export OUT
|
||||
|
||||
export CLIENT=${CLIENT:-vm}
|
||||
export TOTAL_CPU=1.0 TOTAL_MEM_MB=768 DURATION=60 CONCURRENCY=8 WARMUP=15
|
||||
export POSTGRES_CPU_LIMIT=0.50 POSTGRES_MEM_LIMIT=384M POSTGRES_MAX_CONNS=10 LB_CPU_LIMIT=0.25
|
||||
|
||||
run() {
|
||||
label=$1
|
||||
shift
|
||||
|
||||
if [ -f "$OUT/profile-$label.json" ] && [ -f "$OUT/load-$label.json" ]; then
|
||||
printf 'track1_controls.sh: профиль %s пропущен — артефакты уже есть\n' "$label"
|
||||
|
||||
return 0
|
||||
fi
|
||||
|
||||
LABEL="$label" env "$@" "$SCALING"
|
||||
}
|
||||
|
||||
run ctl1base REPLICAS=2
|
||||
run ctl2pg REPLICAS=2 POSTGRES_CPU_LIMIT=1.00
|
||||
run ctl3base REPLICAS=2
|
||||
run ctl4lb REPLICAS=2 LB_CPU_LIMIT=1.00
|
||||
|
||||
printf 'track1_controls.sh: готово, сводка — python3 scripts/track1_report.py\n'
|
||||
Executable
+209
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Сводка трека 1: артефакты `evidence/track1/*` -> таблица и графики.
|
||||
|
||||
Читает `load-<label>.json` (tests/load.py) и `stats-<label>.csv` (docker stats)
|
||||
и печатает метрики профиля: успешный RPS, p50/p95/p99 успешных запросов, доля
|
||||
ошибок, суммарный CPU API, максимум памяти API, CPU PostgreSQL и LB.
|
||||
|
||||
.venv/bin/python scripts/track1_report.py
|
||||
|
||||
Графики (PNG) требуют matplotlib; без него печатается только таблица.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_DIR = ROOT / 'evidence/track1'
|
||||
# Стенд трека 1 — только контейнеры своего проекта: параллельно поднятый стенд
|
||||
# другой лаборатории (тот же Docker-host) иначе попадал бы в те же колонки
|
||||
# (mtusi-messenger-lab2-postgres-1 -> postgres) и искажал бюджет и CPU.
|
||||
DEFAULT_PREFIX = 'mtusi-messenger-lab3-'
|
||||
MEMORY_UNITS = {'B': 1 / 1024 / 1024, 'KiB': 1 / 1024, 'MiB': 1, 'GiB': 1024}
|
||||
|
||||
|
||||
def parse_cpu(value):
|
||||
return float(value.rstrip('%')) if value else 0.0
|
||||
|
||||
|
||||
def parse_memory(value):
|
||||
"""'123.4MiB / 256MiB' -> 123.4 MiB."""
|
||||
match = re.match(r'([\d.]+)\s*([KMG]?i?B)', value.strip())
|
||||
|
||||
if not match:
|
||||
return 0.0
|
||||
|
||||
return float(match.group(1)) * MEMORY_UNITS.get(match.group(2), 1)
|
||||
|
||||
|
||||
def service_of(container):
|
||||
"""mtusi-messenger-lab3-api-12 -> api"""
|
||||
return container.rsplit('-', 1)[0].split('-')[-1] if '-' in container else container
|
||||
|
||||
|
||||
def read_stats(path, prefix=''):
|
||||
"""Профиль -> агрегаты по сервисам.
|
||||
|
||||
Внутри одного timestamp значения всех контейнеров сервиса складываются: две-три
|
||||
реплики — это общий ресурс профиля, а не выборка. Затем CPU усредняется по
|
||||
timestamp'ам, а память берётся пиковой (сумма по репликам).
|
||||
"""
|
||||
samples = defaultdict(lambda: defaultdict(lambda: {'cpu': 0.0, 'mem': 0.0}))
|
||||
|
||||
with path.open() as handle:
|
||||
for row in csv.DictReader(handle):
|
||||
if prefix and not row['container'].startswith(prefix):
|
||||
continue
|
||||
|
||||
service = service_of(row['container'])
|
||||
bucket = samples[service][row['timestamp']]
|
||||
bucket['cpu'] += parse_cpu(row['cpu_percent'])
|
||||
bucket['mem'] += parse_memory(row['mem_usage'])
|
||||
|
||||
cpu = {service: [values['cpu'] for values in by_timestamp.values()] for service, by_timestamp in samples.items()}
|
||||
memory = {service: max(values['mem'] for values in by_timestamp.values()) for service, by_timestamp in samples.items()}
|
||||
|
||||
return cpu, memory
|
||||
|
||||
|
||||
def aggregate(label, directory, prefix=''):
|
||||
load_path = directory / f'load-{label}.json'
|
||||
stats_path = directory / f'stats-{label}.csv'
|
||||
profile_path = directory / f'profile-{label}.json'
|
||||
|
||||
if not load_path.is_file() or not stats_path.is_file():
|
||||
return None
|
||||
|
||||
report = json.loads(load_path.read_text())
|
||||
profile = json.loads(profile_path.read_text()) if profile_path.is_file() else {}
|
||||
cpu, memory = read_stats(stats_path, prefix)
|
||||
|
||||
api_cpu = cpu.get('api', [])
|
||||
api_memory = memory.get('api', 0.0)
|
||||
|
||||
def average(values):
|
||||
return round(sum(values) / len(values), 2) if values else None
|
||||
|
||||
sampled = max((len(values) for values in cpu.values()), default=0)
|
||||
|
||||
return {
|
||||
'label': label,
|
||||
'replicas': profile.get('replicas', 0),
|
||||
'total_cpu': profile.get('total_cpu'),
|
||||
'postgres_budget': '{}/{}'.format(
|
||||
profile.get('postgres_cpu_limit', '?'), profile.get('postgres_mem_limit', '?')
|
||||
),
|
||||
'postgres_max_conns': profile.get('postgres_max_conns', '?'),
|
||||
'lb_cpu_limit': profile.get('lb_cpu_limit', '?'),
|
||||
'cpu_per_replica': round(profile['total_cpu'] / max(1, profile['replicas']), 2)
|
||||
if profile.get('total_cpu') else None,
|
||||
'successful_rps': report['successful_rps'],
|
||||
'p50_ms': report['successful_latency_ms']['p50'],
|
||||
'p95_ms': report['successful_latency_ms']['p95'],
|
||||
'p99_ms': report['successful_latency_ms']['p99'],
|
||||
'errors': report['failed_requests'],
|
||||
'error_ratio': report['error_ratio'],
|
||||
'api_cpu_sum': average(api_cpu),
|
||||
'api_cpu_per_replica': round(average(api_cpu) / max(1, profile.get('replicas') or 1), 2)
|
||||
if api_cpu else None,
|
||||
'api_memory_max_sum_mib': round(api_memory, 1) if api_memory else None,
|
||||
'postgres_cpu': average(cpu.get('postgres', [])),
|
||||
'lb_cpu': average(cpu.get('lb', [])),
|
||||
'stats_samples': sampled,
|
||||
'load_seconds': report['elapsed_seconds'],
|
||||
}
|
||||
|
||||
|
||||
def markdown(rows):
|
||||
header = (
|
||||
'| профиль | реплик | бюджет CPU API | CPU/реплика | бюджет PG CPU/RAM | пул | LB CPU (лимит) | успешный RPS | '
|
||||
'p50 мс | p95 мс | p99 мс | ошибки | API CPU (сумма) | API RAM max (сумма, MiB) | PG CPU | LB CPU |\n'
|
||||
'| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |\n'
|
||||
)
|
||||
|
||||
lines = [
|
||||
'| {label} | {replicas} | {total_cpu} | {cpu_per_replica} | {postgres_budget} | {postgres_max_conns} | '
|
||||
'{lb_cpu_limit} | {successful_rps} | {p50_ms} | {p95_ms} | {p99_ms} | '
|
||||
'{errors} | {api_cpu_sum} | {api_memory_max_sum_mib} | {postgres_cpu} | {lb_cpu} |'.format(**row)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
return header + '\n'.join(lines) + '\n'
|
||||
|
||||
|
||||
def plot(rows, directory):
|
||||
try:
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
print('matplotlib недоступен: графики не построены (pip install matplotlib)')
|
||||
return
|
||||
|
||||
series = {
|
||||
'fixed-budget': [row for row in rows if row['label'].startswith('r') and row['total_cpu']],
|
||||
'growth': [row for row in rows if row['label'].startswith('g') and row['total_cpu']],
|
||||
}
|
||||
|
||||
for name, points in series.items():
|
||||
if len(points) < 2:
|
||||
continue
|
||||
|
||||
axis = 'replicas' if name == 'fixed-budget' else 'total_cpu'
|
||||
points = sorted(points, key=lambda row: row[axis])
|
||||
ticks = [str(row[axis]) for row in points]
|
||||
|
||||
figure, (left, right) = plt.subplots(1, 2, figsize=(10, 4))
|
||||
|
||||
left.plot(ticks, [row['successful_rps'] for row in points], marker='o')
|
||||
left.set_xlabel('API реплик' if axis == 'replicas' else 'суммарный бюджет CPU')
|
||||
left.set_ylabel('успешный RPS')
|
||||
left.grid(True)
|
||||
left.set_title('Пропускная способность')
|
||||
|
||||
right.plot(ticks, [row['p95_ms'] for row in points], marker='o', color='tab:red')
|
||||
right.set_xlabel('API реплик' if axis == 'replicas' else 'суммарный бюджет CPU')
|
||||
right.set_ylabel('p95 успешных, мс')
|
||||
right.grid(True)
|
||||
right.set_title('Задержка p95')
|
||||
|
||||
figure.suptitle(('Фиксированный бюджет' if name == 'fixed-budget' else 'Рост ресурсов') + ' (трек 1)')
|
||||
figure.tight_layout()
|
||||
target = directory / ('scaling-' + name + '.png')
|
||||
figure.savefig(target, dpi=120)
|
||||
print('график:', target)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Сводка трека 1 по артефактам стенда')
|
||||
parser.add_argument('--dir', type=Path, default=DEFAULT_DIR)
|
||||
parser.add_argument('--prefix', default=DEFAULT_PREFIX, help='префикс контейнеров своего стенда')
|
||||
args = parser.parse_args()
|
||||
|
||||
labels = sorted(
|
||||
path.stem.removeprefix('load-') for path in args.dir.glob('load-*.json')
|
||||
)
|
||||
|
||||
rows = []
|
||||
for label in labels:
|
||||
row = aggregate(label, args.dir, args.prefix)
|
||||
if row is None:
|
||||
print('внимание: профиль ' + label + ' пропущен — нет load-*.json или stats-*.csv')
|
||||
continue
|
||||
rows.append(row)
|
||||
|
||||
if not rows:
|
||||
raise SystemExit('нет артефактов в ' + str(args.dir))
|
||||
|
||||
table = markdown(rows)
|
||||
print(table)
|
||||
(args.dir / 'summary.md').write_text('# Трек 1: цена масштабирования\n\n' + table)
|
||||
|
||||
plot(rows, args.dir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+319
@@ -0,0 +1,319 @@
|
||||
#!/bin/sh
|
||||
# Track 1: the price of scaling. Runs one and the same closed-loop write profile
|
||||
# against 1/2/4 API replicas while the TOTAL API budget stays fixed, and records
|
||||
# raw metrics. The stand has to be built already (make up).
|
||||
#
|
||||
# scripts/track1_scaling.sh
|
||||
# REPLICAS="2 4" DURATION=30 CONCURRENCY=8 scripts/track1_scaling.sh
|
||||
#
|
||||
# Artifacts: evidence/track1/{load,stats}-r<N>.json|csv plus summary.txt
|
||||
set -eu
|
||||
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
|
||||
PYTHON=${PYTHON:-python3}
|
||||
REPLICAS=${REPLICAS:-"1 2 4"}
|
||||
TOTAL_CPU=${TOTAL_CPU:-2.0}
|
||||
TOTAL_MEM_MB=${TOTAL_MEM_MB:-1024}
|
||||
DURATION=${DURATION:-60}
|
||||
CONCURRENCY=${CONCURRENCY:-16}
|
||||
WARMUP=${WARMUP:-15}
|
||||
BASE_URL=${BASE_URL:-http://localhost:8080}
|
||||
LABEL=${LABEL:-}
|
||||
OUT=${OUT:-evidence/track1}
|
||||
|
||||
# Бюджеты общих хранилищ фиксируются в профиле вместе с бюджетом API: без них
|
||||
# серию нельзя воспроизвести по артефактам.
|
||||
PG_CPU=${POSTGRES_CPU_LIMIT:-0.50}
|
||||
PG_MEM=${POSTGRES_MEM_LIMIT:-384M}
|
||||
PG_CONNS=${POSTGRES_MAX_CONNS:-10}
|
||||
LB_CPU=${LB_CPU_LIMIT:-0.25}
|
||||
# Где работает генератор нагрузки: `host` — на хосте через опубликованный порт,
|
||||
# `vm` — в контейнере внутри VM, где балансировщик доступен по docker-сети.
|
||||
# Вариант `host` упирается в проброс порта: под шквалом новых соединений
|
||||
# (клиент ходит без keep-alive) он отказывает, и отказы попадают в метрику
|
||||
# ошибок (см. `scripts/path_overhead_check.sh`).
|
||||
CLIENT=${CLIENT:-host}
|
||||
CLIENT_IMAGE=${CLIENT_IMAGE:-python:3.12-alpine}
|
||||
export POSTGRES_CPU_LIMIT="$PG_CPU" POSTGRES_MEM_LIMIT="$PG_MEM" POSTGRES_MAX_CONNS="$PG_CONNS"
|
||||
export LB_CPU_LIMIT="$LB_CPU"
|
||||
|
||||
# Бюджеты хранилищ compose применяет при создании контейнера. Это значения, к
|
||||
# которым стенд возвращается после контрольного профиля.
|
||||
PG_DEFAULT_CPU=${PG_DEFAULT_CPU:-0.50}
|
||||
PG_DEFAULT_MEM=${PG_DEFAULT_MEM:-384M}
|
||||
LB_DEFAULT_CPU=${LB_DEFAULT_CPU:-0.25}
|
||||
|
||||
# Приводим бюджет к тому виду, в котором он снимается с контейнера
|
||||
# (`docker inspect`): 1.0 -> 1.00, 1024M -> 1024M. Иначе проверка «применился ли
|
||||
# бюджет» сравнивала бы разные записи одного и того же значения.
|
||||
normalize_cpu() { awk -v value="$1" 'BEGIN{printf "%.2f", value}'; }
|
||||
|
||||
normalize_mem() {
|
||||
case $1 in
|
||||
*[Gg]) awk -v value="$1" 'BEGIN{printf "%.0fM", value * 1024}' ;;
|
||||
*[Mm]) awk -v value="$1" 'BEGIN{printf "%.0fM", value}' ;;
|
||||
*) printf '%s\n' "$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
PG_CPU=$(normalize_cpu "$PG_CPU")
|
||||
PG_MEM=$(normalize_mem "$PG_MEM")
|
||||
PG_DEFAULT_CPU=$(normalize_cpu "$PG_DEFAULT_CPU")
|
||||
PG_DEFAULT_MEM=$(normalize_mem "$PG_DEFAULT_MEM")
|
||||
LB_CPU=$(normalize_cpu "$LB_CPU")
|
||||
LB_DEFAULT_CPU=$(normalize_cpu "$LB_DEFAULT_CPU")
|
||||
export POSTGRES_CPU_LIMIT="$PG_CPU" POSTGRES_MEM_LIMIT="$PG_MEM" LB_CPU_LIMIT="$LB_CPU"
|
||||
|
||||
# The author's shell exports HTTP(S)_PROXY, which breaks localhost clients.
|
||||
NO_PROXY_ENV="env -u HTTP_PROXY -u HTTPS_PROXY -u ALL_PROXY -u http_proxy -u https_proxy -u all_proxy"
|
||||
NO_PROXY_ENV="$NO_PROXY_ENV NO_PROXY=localhost,127.0.0.1,::1 no_proxy=localhost,127.0.0.1,::1"
|
||||
|
||||
mkdir -p "$OUT"
|
||||
|
||||
compose() {
|
||||
# shellcheck disable=SC2086
|
||||
$COMPOSE_BIN --env-file .env -f compose.yaml "$@"
|
||||
}
|
||||
|
||||
wait_ready() {
|
||||
attempt=0
|
||||
while [ "$attempt" -lt 90 ]; do
|
||||
if curl -fsS --noproxy '*' -o /dev/null "$BASE_URL/health/ready"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
printf 'track1_scaling.sh: %s/health/ready did not become ready\n' "$BASE_URL" >&2
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
per_replica_cpu() {
|
||||
awk "BEGIN{printf \"%.2f\", $TOTAL_CPU/$1}"
|
||||
}
|
||||
|
||||
per_replica_mem() {
|
||||
awk "BEGIN{printf \"%.0fM\", $TOTAL_MEM_MB/$1}"
|
||||
}
|
||||
|
||||
container_of() {
|
||||
compose ps -q "$1"
|
||||
}
|
||||
|
||||
# Сеть стенда: клиент внутри VM обращается к балансировщику по ней, без проброса
|
||||
# порта и без сетевого стека хоста.
|
||||
stand_network() {
|
||||
docker inspect "$(container_of api | head -1)" \
|
||||
--format '{{range $name, $network := .NetworkSettings.Networks}}{{$name}}{{end}}'
|
||||
}
|
||||
|
||||
# $1 — длительность, $2 — путь к JSON-метрикам (необязательно).
|
||||
load_client() {
|
||||
if [ "$CLIENT" = vm ]; then
|
||||
network=$(stand_network)
|
||||
|
||||
if [ -n "$2" ]; then
|
||||
# shellcheck disable=SC2086
|
||||
docker run --rm -i --network "$network" -v "$ROOT_DIR:/repo:ro" -v "$ROOT_DIR/$OUT:/out" \
|
||||
-e NO_PROXY='lb,localhost,127.0.0.1' -e no_proxy='lb,localhost,127.0.0.1' \
|
||||
"$CLIENT_IMAGE" python /repo/tests/load.py --base-url http://lb \
|
||||
--duration "$1" --concurrency "$CONCURRENCY" --output "/out/$(basename "$2")"
|
||||
else
|
||||
# shellcheck disable=SC2086
|
||||
docker run --rm -i --network "$network" -v "$ROOT_DIR:/repo:ro" \
|
||||
-e NO_PROXY='lb,localhost,127.0.0.1' -e no_proxy='lb,localhost,127.0.0.1' \
|
||||
"$CLIENT_IMAGE" python /repo/tests/load.py --base-url http://lb \
|
||||
--duration "$1" --concurrency "$CONCURRENCY"
|
||||
fi
|
||||
|
||||
return
|
||||
fi
|
||||
|
||||
if [ -n "$2" ]; then
|
||||
# shellcheck disable=SC2086
|
||||
$NO_PROXY_ENV $PYTHON tests/load.py --base-url "$BASE_URL" \
|
||||
--duration "$1" --concurrency "$CONCURRENCY" --output "$2"
|
||||
else
|
||||
# shellcheck disable=SC2086
|
||||
$NO_PROXY_ENV $PYTHON tests/load.py --base-url "$BASE_URL" \
|
||||
--duration "$1" --concurrency "$CONCURRENCY"
|
||||
fi
|
||||
}
|
||||
|
||||
# Фактический бюджет контейнера, снятый с него самого: в профиль и в сводку
|
||||
# попадает измерение, а не переменная, с которой скрипт был запущен.
|
||||
container_cpu() {
|
||||
nano=$(docker inspect "$(container_of "$1")" --format '{{.HostConfig.NanoCpus}}')
|
||||
awk -v nano="$nano" 'BEGIN{printf "%.2f", nano/1000000000}'
|
||||
}
|
||||
|
||||
container_mem() {
|
||||
bytes=$(docker inspect "$(container_of "$1")" --format '{{.HostConfig.Memory}}')
|
||||
awk -v bytes="$bytes" 'BEGIN{printf "%.0fM", bytes/1024/1024}'
|
||||
}
|
||||
|
||||
wait_postgres_ready() {
|
||||
attempt=0
|
||||
while [ "$attempt" -lt 90 ]; do
|
||||
if compose exec -T postgres pg_isready -U "${POSTGRES_USER:-messenger}" \
|
||||
-d "${POSTGRES_DB:-messenger}" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
printf 'track1_scaling.sh: postgres did not become ready\n' >&2
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# Лимиты общих хранилищ задаются при создании контейнера, поэтому одного
|
||||
# `up -d --no-deps api` для них мало: PostgreSQL и балансировщик приходится
|
||||
# пересоздавать, когда бюджет профиля отличается от действующего. Без этого
|
||||
# профиль записывал бы намерение, а измерение шло бы на прежнем лимите — именно
|
||||
# так контроль удвоения CPU БД перестал быть контролем.
|
||||
apply_store_limits() {
|
||||
applied_pg_cpu=$(container_cpu postgres)
|
||||
applied_pg_mem=$(container_mem postgres)
|
||||
applied_lb_cpu=$(container_cpu lb)
|
||||
|
||||
if [ "$applied_pg_cpu" != "$PG_CPU" ] || [ "$applied_pg_mem" != "$PG_MEM" ]; then
|
||||
printf '=== PostgreSQL: бюджет %s/%s -> %s/%s, пересоздаю контейнер ===\n' \
|
||||
"$applied_pg_cpu" "$applied_pg_mem" "$PG_CPU" "$PG_MEM" | tee -a "$SUMMARY"
|
||||
|
||||
POSTGRES_CPU_LIMIT="$PG_CPU" POSTGRES_MEM_LIMIT="$PG_MEM" \
|
||||
compose up -d --no-deps postgres >/dev/null
|
||||
|
||||
wait_postgres_ready
|
||||
fi
|
||||
|
||||
if [ "$applied_lb_cpu" != "$LB_CPU" ]; then
|
||||
printf '=== LB: бюджет CPU %s -> %s, пересоздаю контейнер ===\n' \
|
||||
"$applied_lb_cpu" "$LB_CPU" | tee -a "$SUMMARY"
|
||||
|
||||
LB_CPU_LIMIT="$LB_CPU" compose up -d --no-deps lb >/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
SUMMARY="$OUT/summary.txt"
|
||||
|
||||
if [ ! -f "$SUMMARY" ]; then
|
||||
{
|
||||
echo "# Track 1: the price of scaling"
|
||||
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "# host: $(uname -sm)"
|
||||
echo "# cpu: $(sysctl -n machdep.cpu.brand_string 2>/dev/null || printf unknown)"
|
||||
echo "# ram bytes: $(sysctl -n hw.memsize 2>/dev/null || printf unknown)"
|
||||
echo "# load: tests/load.py closed loop; postgres/lb budgets are recreated and verified per profile"
|
||||
echo "# each profile line below records the budget it was run with"
|
||||
docker version --format 'docker engine {{.Server.Version}}'
|
||||
# shellcheck disable=SC2086
|
||||
$COMPOSE_BIN version
|
||||
echo
|
||||
} > "$SUMMARY"
|
||||
fi
|
||||
|
||||
{
|
||||
echo "=== invocation $(date -u +%Y-%m-%dT%H:%M:%SZ): REPLICAS=\"$REPLICAS\" TOTAL_CPU=$TOTAL_CPU TOTAL_MEM_MB=$TOTAL_MEM_MB DURATION=$DURATION CONCURRENCY=$CONCURRENCY WARMUP=$WARMUP ==="
|
||||
echo
|
||||
} >> "$SUMMARY"
|
||||
|
||||
for replicas in $REPLICAS; do
|
||||
cpu=$(per_replica_cpu "$replicas")
|
||||
mem=$(per_replica_mem "$replicas")
|
||||
label=${LABEL:-r$replicas}
|
||||
|
||||
printf '=== profile label=%s replicas=%s cpu=%s mem=%s postgres=%s/%s lb_cpu=%s max_conns=%s client=%s ===\n' \
|
||||
"$label" "$replicas" "$cpu" "$mem" "$PG_CPU" "$PG_MEM" "$LB_CPU" "$PG_CONNS" "$CLIENT" | tee -a "$SUMMARY"
|
||||
|
||||
apply_store_limits
|
||||
|
||||
API_REPLICAS="$replicas" API_CPU_LIMIT="$cpu" API_MEM_LIMIT="$mem" compose up -d --no-deps api
|
||||
wait_ready
|
||||
|
||||
# Warm up caches, connection pools and the JIT-free query plans; the warmup
|
||||
# data stays in the table, so every profile grows the same history shape.
|
||||
load_client "$WARMUP" "" >/dev/null
|
||||
|
||||
STATS="$OUT/stats-$label.csv"
|
||||
printf 'timestamp,container,cpu_percent,mem_usage\n' > "$STATS"
|
||||
|
||||
# В профиль попадают фактические лимиты, снятые с контейнеров: иначе профиль
|
||||
# описывал бы намерение, а не то, на чём действительно шло измерение.
|
||||
pg_cpu_applied=$(container_cpu postgres)
|
||||
pg_mem_applied=$(container_mem postgres)
|
||||
lb_cpu_applied=$(container_cpu lb)
|
||||
|
||||
if [ "$pg_cpu_applied" != "$PG_CPU" ] || [ "$pg_mem_applied" != "$PG_MEM" ] \
|
||||
|| [ "$lb_cpu_applied" != "$LB_CPU" ]; then
|
||||
printf 'track1_scaling.sh: лимиты не применились: postgres просили %s/%s, получили %s/%s; lb просили %s, получили %s\n' \
|
||||
"$PG_CPU" "$PG_MEM" "$pg_cpu_applied" "$pg_mem_applied" "$LB_CPU" "$lb_cpu_applied" >&2
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '=== applied label=%s replicas=%s postgres=%s/%s lb_cpu=%s (снято с контейнеров)\n' \
|
||||
"$label" "$replicas" "$pg_cpu_applied" "$pg_mem_applied" "$lb_cpu_applied" | tee -a "$SUMMARY"
|
||||
|
||||
PROFILE="$OUT/profile-$label.json"
|
||||
printf '{"label":"%s","replicas":%s,"total_cpu":%s,"total_mem_mb":%s,"duration":%s,"concurrency":%s,"warmup":%s,"postgres_cpu_limit":"%s","postgres_mem_limit":"%s","postgres_max_conns":%s,"lb_cpu_limit":"%s","client":"%s"}\n' \
|
||||
"$label" "$replicas" "$TOTAL_CPU" "$TOTAL_MEM_MB" "$DURATION" "$CONCURRENCY" "$WARMUP" \
|
||||
"$pg_cpu_applied" "$pg_mem_applied" "$PG_CONNS" "$lb_cpu_applied" "$CLIENT" > "$PROFILE"
|
||||
|
||||
(
|
||||
while :; do
|
||||
docker stats --no-stream \
|
||||
--format '{{.Name}},{{.CPUPerc}},{{.MemUsage}}' \
|
||||
| sed "s|^|$(date -u +%Y-%m-%dT%H:%M:%SZ),|" >> "$STATS"
|
||||
sleep 1
|
||||
done
|
||||
) &
|
||||
|
||||
sampler=$!
|
||||
trap 'kill "$sampler" 2>/dev/null || true' EXIT INT TERM
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
load_client "$DURATION" "$OUT/load-$label.json" | tee -a "$SUMMARY"
|
||||
|
||||
kill "$sampler" 2>/dev/null || true
|
||||
wait "$sampler" 2>/dev/null || true
|
||||
trap - EXIT INT TERM
|
||||
|
||||
# Shared-dependency view right after the run: connections, pool pressure and
|
||||
# how much history the profile wrote into.
|
||||
compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -Atc \
|
||||
"SELECT 'postgres_connections=' || count(*) FROM pg_stat_activity;
|
||||
SELECT 'postgres_max_connections=' || current_setting('max_connections');
|
||||
SELECT 'messages_rows=' || count(*) FROM messages;
|
||||
SELECT 'idempotency_rows=' || count(*) FROM message_idempotency;" | tee -a "$SUMMARY"
|
||||
|
||||
printf '\n' >> "$SUMMARY"
|
||||
done
|
||||
|
||||
# Контрольные профили меняют бюджет общих хранилищ; после серии стенд
|
||||
# возвращается к бюджетам compose, чтобы следующий сценарий шёл на них.
|
||||
if [ "$PG_CPU" != "$PG_DEFAULT_CPU" ] || [ "$PG_MEM" != "$PG_DEFAULT_MEM" ]; then
|
||||
printf '=== возвращаю PostgreSQL к бюджету %s/%s ===\n' \
|
||||
"$PG_DEFAULT_CPU" "$PG_DEFAULT_MEM" | tee -a "$SUMMARY"
|
||||
|
||||
POSTGRES_CPU_LIMIT="$PG_DEFAULT_CPU" POSTGRES_MEM_LIMIT="$PG_DEFAULT_MEM" \
|
||||
compose up -d --no-deps postgres >/dev/null
|
||||
|
||||
wait_postgres_ready
|
||||
fi
|
||||
|
||||
if [ "$LB_CPU" != "$LB_DEFAULT_CPU" ]; then
|
||||
printf '=== возвращаю LB к бюджету CPU %s ===\n' "$LB_DEFAULT_CPU" | tee -a "$SUMMARY"
|
||||
|
||||
LB_CPU_LIMIT="$LB_DEFAULT_CPU" compose up -d --no-deps lb >/dev/null
|
||||
fi
|
||||
|
||||
printf 'track1_scaling.sh: artifacts in %s\n' "$OUT"
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/bin/sh
|
||||
# Трек 1: вся серия профилей одной командой — так её и воспроизводят.
|
||||
#
|
||||
# scripts/track1_series.sh
|
||||
#
|
||||
# Серия 1: фиксированный суммарный бюджет API (1.0 CPU / 768 MiB) на 1, 2 и 4
|
||||
# репликах. Серия 2: рост бюджета API при 2 репликах (0.5, 1.0, 1.5 CPU).
|
||||
# Контрольные профили меняют одну ручку за раз: пул соединений (10 -> 5),
|
||||
# бюджет CPU PostgreSQL (0.50 -> 1.00) и бюджет CPU балансировщика (0.25 -> 1.00).
|
||||
#
|
||||
# Бюджеты общих хранилищ применяются к работающим контейнерам и проверяются по
|
||||
# `docker inspect` (см. scripts/track1_scaling.sh). Стенд должен быть поднят
|
||||
# (`make up`), артефакты ложатся в evidence/track1/.
|
||||
set -eu
|
||||
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
SCALING=scripts/track1_scaling.sh
|
||||
|
||||
# Генератор работает внутри VM: проброс порта хоста под шквалом новых соединений
|
||||
# отказывает (см. evidence/path-overhead/), и отказы клиента маскировали бы
|
||||
# поведение стенда. Вариант CLIENT=host оставлен для сравнения путей.
|
||||
export CLIENT=${CLIENT:-vm}
|
||||
|
||||
# Общие параметры профиля: closed loop, 60 с нагрузки, 15 с прогрева, 8 клиентов.
|
||||
export TOTAL_CPU=1.0 TOTAL_MEM_MB=768 DURATION=60 CONCURRENCY=8 WARMUP=15
|
||||
|
||||
OUT=${OUT:-evidence/track1}
|
||||
export OUT
|
||||
|
||||
# Профиль с уже снятыми артефактами пропускается: серию можно продолжить после
|
||||
# обрыва (например, если Docker-host отвлёкся), не перемеряя начало.
|
||||
run_profile() {
|
||||
label=$1
|
||||
shift
|
||||
|
||||
if [ -f "$OUT/profile-$label.json" ] && [ -f "$OUT/load-$label.json" ]; then
|
||||
printf 'track1_series.sh: профиль %s пропущен — артефакты уже есть (удалите их, чтобы перемерить)\n' "$label"
|
||||
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Бюджеты профиля передаются именно как переменные окружения: `"$@"` с
|
||||
# присваиваниями превратил бы их в аргументы и профиль пошёл бы на дефолтах.
|
||||
LABEL="$label" env "$@" "$SCALING"
|
||||
}
|
||||
|
||||
# Серия 1: цена реплик при неизменном суммарном бюджете API.
|
||||
run_profile r1 REPLICAS=1
|
||||
run_profile r2 REPLICAS=2
|
||||
run_profile r4 REPLICAS=4
|
||||
|
||||
# Серия 2: рост бюджета API при двух репликах.
|
||||
run_profile g05 REPLICAS=2 TOTAL_CPU=0.5
|
||||
run_profile g10 REPLICAS=2 TOTAL_CPU=1.0
|
||||
run_profile g15 REPLICAS=2 TOTAL_CPU=1.5
|
||||
|
||||
# Контроль: пул соединений одной реплики 10 -> 5 при четырёх репликах.
|
||||
run_profile pool10 REPLICAS=4 POSTGRES_MAX_CONNS=10
|
||||
run_profile pool5 REPLICAS=4 POSTGRES_MAX_CONNS=5
|
||||
|
||||
# Контроль: удвоенный бюджет CPU PostgreSQL (1.00 вместо 0.50) при двух репликах.
|
||||
run_profile db1cpu REPLICAS=2 POSTGRES_CPU_LIMIT=1.00 POSTGRES_MEM_LIMIT=384M
|
||||
|
||||
# Контроль: четырехкратный бюджет CPU балансировщика (1.00 вместо 0.25).
|
||||
run_profile lb1cpu REPLICAS=2 LB_CPU_LIMIT=1.00
|
||||
|
||||
printf 'track1_series.sh: серия завершена, сводка — python3 scripts/track1_report.py\n'
|
||||
Reference in New Issue
Block a user