fix: switch API pools atomically

This commit is contained in:
2026-10-09 13:39:09 +03:00
parent 3fde1c324c
commit 37d93c340e
5 changed files with 144 additions and 59 deletions
+1 -1
View File
@@ -180,7 +180,7 @@ jobs:
git cat-file -e '$DEPLOY_SHA^{commit}'
git reset --hard '$DEPLOY_SHA'
manifest='/tmp/mtusi2026lab4-${{ github.run_id }}.env'
if scripts/rolling-update.sh \"\$manifest\"; then
if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then
rm -f \"\$manifest\"
else
status=\$?
+2
View File
@@ -177,6 +177,8 @@ services:
image: ${NGINX_IMAGE:-mtusi-nginx:lab4}
ports:
- "${API_BIND_ADDRESS}:8080"
volumes:
- ${NGINX_UPSTREAM_FILE:-./nginx/upstream.conf}:/etc/nginx/conf.d/upstream.conf:ro
depends_on:
api:
condition: service_healthy
+1 -1
View File
@@ -20,7 +20,7 @@ http {
upstream messenger {
zone messenger 64k;
server api:8080 resolve max_fails=1 fail_timeout=3s;
include /etc/nginx/conf.d/upstream.conf;
keepalive 32;
}
+1
View File
@@ -0,0 +1 @@
server api:8080 resolve max_fails=1 fail_timeout=3s;
+139 -57
View File
@@ -10,12 +10,21 @@ fi
manifest=$1
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab4"}
current_manifest=$deploy_dir/current-images.env
previous_manifest=$deploy_dir/previous-images.env
active_containers=$deploy_dir/active-api-containers
previous_containers=$deploy_dir/previous-api-containers
upstream_file=$deploy_dir/upstream.conf
upstream_backup=$deploy_dir/upstream.previous.conf
candidate_file=$deploy_dir/candidate-api-containers
replicas=${API_REPLICAS:-2}
rollback_ready=false
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab4}
switched=false
nginx_created=false
upstream_changed=false
old_pool=""
mkdir -p "$deploy_dir"
: > "$candidate_file"
export NGINX_UPSTREAM_FILE=$upstream_file
load_manifest() {
set -a
@@ -28,12 +37,30 @@ compose() {
docker compose -f compose.yaml -f deploy/compose.images.yaml "$@"
}
wait_for_replicas() {
expected=$1
container_name() {
name=$(docker inspect --format '{{.Name}}' "$1")
printf '%s\n' "${name#/}"
}
running_container() {
[ "$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null || true)" = true ]
}
write_upstream() {
output=$1
containers=$2
: > "$output"
for container in $containers; do
name=$(container_name "$container")
printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
done
}
wait_for_containers() {
containers=$1
deadline=$(($(date +%s) + 120))
while [ "$(date +%s)" -lt "$deadline" ]; do
containers=$(compose ps -q api)
count=0
healthy=0
for container in $containers; do
@@ -43,13 +70,13 @@ wait_for_replicas() {
healthy=$((healthy + 1))
fi
done
if [ "$count" -eq "$expected" ] && [ "$healthy" -eq "$expected" ]; then
if [ "$count" -eq "$replicas" ] && [ "$healthy" -eq "$replicas" ]; then
return 0
fi
sleep 1
done
printf 'API replicas did not become healthy: expected %s\n' "$expected" >&2
printf 'candidate API pool did not become healthy: expected %s\n' "$replicas" >&2
return 1
}
@@ -57,53 +84,36 @@ smoke_check() {
compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready
}
running_image() {
containers=$(compose ps -q "$1")
remove_containers() {
containers=$1
for container in $containers; do
docker inspect --format '{{.Config.Image}}' "$container"
return 0
docker rm -f "$container" >/dev/null 2>&1 || true
done
return 1
}
capture_running_images() {
output=$1
api_image=$(running_image api) || return 1
nginx_image=$(running_image nginx) || return 1
swagger_image=$(running_image swagger) || swagger_image=$SWAGGER_IMAGE
prometheus_image=$(running_image prometheus) || prometheus_image=$PROMETHEUS_IMAGE
grafana_image=$(running_image grafana) || grafana_image=$GRAFANA_IMAGE
s3_image=$(running_image s3) || s3_image=$S3_IMAGE
s3_bootstrap_image=$(running_image s3-bootstrap) || s3_bootstrap_image=$S3_BOOTSTRAP_IMAGE
{
printf 'API_IMAGE=%s\n' "$api_image"
printf 'NGINX_IMAGE=%s\n' "$nginx_image"
printf 'SWAGGER_IMAGE=%s\n' "$swagger_image"
printf 'PROMETHEUS_IMAGE=%s\n' "$prometheus_image"
printf 'GRAFANA_IMAGE=%s\n' "$grafana_image"
printf 'S3_IMAGE=%s\n' "$s3_image"
printf 'S3_BOOTSTRAP_IMAGE=%s\n' "$s3_bootstrap_image"
} > "$output"
restore_upstream() {
if [ -s "$upstream_backup" ]; then
cp "$upstream_backup" "$upstream_file"
compose exec -T nginx nginx -t
compose exec -T nginx nginx -s reload
fi
}
rollback() {
status=$?
trap - EXIT INT TERM
candidates=$(cat "$candidate_file" 2>/dev/null || true)
if [ "$switched" = true ] && [ "$rollback_ready" = true ]; then
printf '%s\n' 'Deployment failed; restoring the previous images.' >&2
compose stop nginx api >/dev/null 2>&1 || true
compose rm -f nginx api >/dev/null 2>&1 || true
load_manifest "$previous_manifest"
compose up -d --no-build --no-deps --force-recreate --scale api="$replicas" api
wait_for_replicas "$replicas"
compose up -d --no-build --no-deps --force-recreate nginx
smoke_check
elif [ "$switched" = true ]; then
printf '%s\n' 'Deployment failed and no previous image manifest is available.' >&2
if [ "$upstream_changed" = true ] && [ -n "$old_pool" ]; then
printf '%s\n' 'Deployment failed; switching Nginx back to the previous API pool.' >&2
restore_upstream || true
if [ "$switched" = true ]; then
smoke_check || true
fi
elif [ "$nginx_created" = true ]; then
compose stop nginx >/dev/null 2>&1 || true
fi
remove_containers "$candidates"
exit "$status"
}
@@ -112,30 +122,102 @@ trap 'exit 130' INT
trap 'exit 143' TERM
load_manifest "$manifest"
new_api_image=$API_IMAGE
if [ -f "$current_manifest" ]; then
cp "$current_manifest" "$previous_manifest"
rollback_ready=true
elif capture_running_images "$previous_manifest"; then
rollback_ready=true
if [ -f "$previous_containers" ]; then
remove_containers "$(cat "$previous_containers")"
rm -f "$previous_containers"
fi
if [ -s "$active_containers" ]; then
for container in $(cat "$active_containers"); do
if running_container "$container"; then
old_pool="$old_pool $container"
fi
done
fi
if [ -z "$old_pool" ]; then
old_pool=$(compose ps -q api)
fi
if [ -s "$upstream_file" ]; then
:
elif [ -n "$old_pool" ]; then
write_upstream "$upstream_file" "$old_pool"
else
cp nginx/upstream.conf "$upstream_file"
fi
cp "$upstream_file" "$upstream_backup"
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
compose up -d --no-build --wait --wait-timeout 120 postgres valkey s3
compose run --rm --no-deps s3-bootstrap
compose run --rm --no-deps migrate
# Stop the entry point and every old API replica before starting the new pool.
# This gives an atomic version boundary at the cost of a short maintenance window.
switched=true
compose stop nginx api
compose rm -f nginx api
compose up -d --no-build --no-deps --force-recreate --scale api="$replicas" api
wait_for_replicas "$replicas"
compose up -d --no-build --no-deps --force-recreate nginx
nginx_container=$(compose ps -q nginx)
if [ -n "$nginx_container" ] && ! docker inspect --format '{{range .Mounts}}{{println .Destination}}{{end}}' "$nginx_container" | grep -qx /etc/nginx/conf.d/upstream.conf; then
compose up -d --no-build --no-deps --force-recreate nginx
nginx_container=$(compose ps -q nginx)
fi
if [ -n "$nginx_container" ]; then
running_nginx_image=$(docker inspect --format '{{.Config.Image}}' "$nginx_container")
if [ "$running_nginx_image" != "$NGINX_IMAGE" ]; then
printf 'verified Nginx image differs from the running load balancer; deploy it in a dedicated maintenance operation\n' >&2
exit 1
fi
fi
API_IMAGE=$new_api_image
export API_IMAGE
short_digest=$(printf '%s' "$new_api_image" | sed 's/.*sha256://; s/[^0-9a-f].*//' | cut -c1-12)
if [ -z "$short_digest" ]; then
short_digest=$(date +%s)
fi
generation=${DEPLOY_GENERATION:-"$(date +%s)-$$"}
candidate_pool=""
index=1
while [ "$index" -le "$replicas" ]; do
name="${project}-api-${short_digest}-${generation}-${index}"
container=$(compose run -d --no-deps --name "$name" api)
candidate_pool="$candidate_pool $container"
printf '%s\n' "$container" >> "$candidate_file"
index=$((index + 1))
done
wait_for_containers "$candidate_pool"
write_upstream "$upstream_file.next" "$candidate_pool"
cp "$upstream_file.next" "$upstream_file"
upstream_changed=true
if [ -n "$(compose ps -q nginx)" ]; then
compose exec -T nginx nginx -t
switched=true
compose exec -T nginx nginx -s reload
else
nginx_created=true
compose up -d --no-build --no-deps nginx
switched=true
fi
smoke_check
compose up -d --no-build swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana
application_network="${project}_application"
for container in $candidate_pool; do
docker network disconnect "$application_network" "$container"
docker network connect --alias api "$application_network" "$container"
done
smoke_check
compose up -d --no-build --no-deps swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana
printf '%s\n' "$candidate_pool" > "$active_containers"
cp "$manifest" "$current_manifest"
rm -f "$candidate_file" "$upstream_file.next"
trap - EXIT INT TERM
if [ -n "$old_pool" ]; then
for container in $old_pool; do
docker stop --time 6 "$container" >/dev/null || true
done
printf '%s\n' "$old_pool" > "$previous_containers"
fi