ops: add balanced replicated stack

This commit is contained in:
2026-10-02 14:50:16 +03:00
parent 563aa26a92
commit 9358ebdc19
14 changed files with 1652 additions and 0 deletions
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
docker compose stop nginx api valkey postgres
docker compose start postgres valkey
docker compose run --rm migrate
docker compose up -d --wait --wait-timeout 90
+57
View File
@@ -0,0 +1,57 @@
#!/bin/sh
set -eu
wait_for_replicas() {
expected=$1
deadline=$(($(date +%s) + 90))
while [ "$(date +%s)" -lt "$deadline" ]; do
containers=$(docker compose ps -q api)
count=0
healthy=0
for container in $containers; do
count=$((count + 1))
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")
if [ "$status" = "healthy" ]; then
healthy=$((healthy + 1))
fi
done
if [ "$count" -eq "$expected" ] && [ "$healthy" -eq "$expected" ]; then
return 0
fi
sleep 1
done
printf '%s\n' "API replicas did not become healthy: expected $expected" >&2
return 1
}
old_containers=$(docker compose ps -q api)
old_count=0
for container in $old_containers; do
old_count=$((old_count + 1))
done
docker compose build
docker compose up -d postgres valkey
docker compose run --rm migrate
if [ "$old_count" -lt 2 ]; then
docker compose up -d --scale api=2 --wait --wait-timeout 120
exit 0
fi
docker compose up -d --no-deps --no-recreate --scale api=3 api
wait_for_replicas 3
for container in $old_containers; do
docker stop --time 6 "$container" >/dev/null
docker rm "$container" >/dev/null
docker compose up -d --no-deps --no-recreate --scale api=3 api
wait_for_replicas 3
done
docker compose up -d --no-deps --no-recreate --scale api=2 api
wait_for_replicas 2
docker compose up -d --no-build --scale api=2 --wait --wait-timeout 120
+104
View File
@@ -0,0 +1,104 @@
#!/bin/sh
set -eu
duration=${DURATION:-15s}
warmup=${WARMUP:-3s}
concurrencies=${CONCURRENCIES:-"8 32 64"}
repeats=${REPEATS:-3}
current_project=
restore_default_stack() {
if [ -n "$current_project" ]; then
env COMPOSE_PROJECT_NAME="$current_project" docker compose --profile research down -v --remove-orphans || true
fi
docker compose up -d --scale api=2 --wait --wait-timeout 120
}
run_configuration() {
series=$1
replicas=$2
api_cpus=$3
api_memory=$4
db_max_conns=$5
repeat=$6
point_concurrencies=$7
project="lab3exp_${series}_r${replicas}_n${repeat}"
pending=false
for concurrency in $point_concurrencies; do
label="${series}-r${replicas}-c${concurrency}-n${repeat}"
if [ ! -s "evidence/lab3-${label}.json" ]; then
pending=true
fi
done
if [ "$pending" = false ]; then
echo "Skipping completed configuration ${series}-r${replicas}-n${repeat}"
return
fi
current_project=$project
compose() {
env \
COMPOSE_PROJECT_NAME="$project" \
API_REPLICAS="$replicas" \
API_CPUS="$api_cpus" \
API_MEMORY="$api_memory" \
DB_MIN_CONNS=0 \
DB_MAX_CONNS="$db_max_conns" \
docker compose "$@"
}
compose up -d --scale api="$replicas" --wait --wait-timeout 120 \
postgres valkey migrate api nginx postgres-exporter valkey-exporter nginx-exporter prometheus
for concurrency in $point_concurrencies; do
label="${series}-r${replicas}-c${concurrency}-n${repeat}"
output="lab3-${label}.json"
if [ -s "evidence/$output" ]; then
echo "Skipping completed point $label"
continue
fi
compose --profile research run --rm experiment \
/app/lab3experiment \
-base-url=http://nginx:8080 \
-prometheus-url=http://prometheus:9090 \
-label="$label" \
-duration="$duration" \
-warmup="$warmup" \
-concurrency="$concurrency" \
-replicas="$replicas" \
-api-cpus="$api_cpus" \
-api-memory="$api_memory" \
-db-max-conns="$db_max_conns" \
-output="/output/$output"
done
compose --profile research down -v --remove-orphans
current_project=
}
trap restore_default_stack EXIT INT TERM
docker compose build api nginx swagger prometheus grafana
docker compose stop
repeat=1
while [ "$repeat" -le "$repeats" ]; do
run_configuration fixed 1 2 1g 16 "$repeat" "$concurrencies"
run_configuration fixed 2 1 512m 8 "$repeat" "$concurrencies"
run_configuration fixed 4 0.5 256m 4 "$repeat" "$concurrencies"
run_configuration growing 1 1 256m 8 "$repeat" "$concurrencies"
run_configuration growing 2 1 256m 8 "$repeat" "$concurrencies"
run_configuration growing 4 1 256m 8 "$repeat" "$concurrencies"
run_configuration pool-baseline 4 0.5 256m 10 "$repeat" "64"
run_configuration pool-tuned 4 0.5 256m 4 "$repeat" "64"
repeat=$((repeat + 1))
done
docker compose --profile research run --no-deps --rm experiment \
/app/lab3summary \
-input=/output \
-output=/output/lab3-scale-results.json \
-chart=/output/lab3-scale-rps.svg
rm -f evidence/lab3-fixed-*.json evidence/lab3-growing-*.json evidence/lab3-pool-*.json
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
for container in $(docker compose ps -q api); do
instance=$(docker inspect --format '{{.Config.Hostname}}' "$container")
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 6 "$container" >/dev/null
exit 0
fi
done
printf '%s\n' "API instance not found: $TARGET_INSTANCE_ID" >&2
exit 1