forked from mtusi2026/mtusi2026lab4
fix: switch API pools atomically
This commit is contained in:
@@ -180,7 +180,7 @@ jobs:
|
||||
git cat-file -e '$DEPLOY_SHA^{commit}'
|
||||
git reset --hard '$DEPLOY_SHA'
|
||||
manifest='/tmp/mtusi2026lab4-${{ github.run_id }}.env'
|
||||
if scripts/rolling-update.sh \"\$manifest\"; then
|
||||
if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then
|
||||
rm -f \"\$manifest\"
|
||||
else
|
||||
status=\$?
|
||||
|
||||
@@ -177,6 +177,8 @@ services:
|
||||
image: ${NGINX_IMAGE:-mtusi-nginx:lab4}
|
||||
ports:
|
||||
- "${API_BIND_ADDRESS}:8080"
|
||||
volumes:
|
||||
- ${NGINX_UPSTREAM_FILE:-./nginx/upstream.conf}:/etc/nginx/conf.d/upstream.conf:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ http {
|
||||
|
||||
upstream messenger {
|
||||
zone messenger 64k;
|
||||
server api:8080 resolve max_fails=1 fail_timeout=3s;
|
||||
include /etc/nginx/conf.d/upstream.conf;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
server api:8080 resolve max_fails=1 fail_timeout=3s;
|
||||
+139
-57
@@ -10,12 +10,21 @@ fi
|
||||
manifest=$1
|
||||
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab4"}
|
||||
current_manifest=$deploy_dir/current-images.env
|
||||
previous_manifest=$deploy_dir/previous-images.env
|
||||
active_containers=$deploy_dir/active-api-containers
|
||||
previous_containers=$deploy_dir/previous-api-containers
|
||||
upstream_file=$deploy_dir/upstream.conf
|
||||
upstream_backup=$deploy_dir/upstream.previous.conf
|
||||
candidate_file=$deploy_dir/candidate-api-containers
|
||||
replicas=${API_REPLICAS:-2}
|
||||
rollback_ready=false
|
||||
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab4}
|
||||
switched=false
|
||||
nginx_created=false
|
||||
upstream_changed=false
|
||||
old_pool=""
|
||||
|
||||
mkdir -p "$deploy_dir"
|
||||
: > "$candidate_file"
|
||||
export NGINX_UPSTREAM_FILE=$upstream_file
|
||||
|
||||
load_manifest() {
|
||||
set -a
|
||||
@@ -28,12 +37,30 @@ compose() {
|
||||
docker compose -f compose.yaml -f deploy/compose.images.yaml "$@"
|
||||
}
|
||||
|
||||
wait_for_replicas() {
|
||||
expected=$1
|
||||
container_name() {
|
||||
name=$(docker inspect --format '{{.Name}}' "$1")
|
||||
printf '%s\n' "${name#/}"
|
||||
}
|
||||
|
||||
running_container() {
|
||||
[ "$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null || true)" = true ]
|
||||
}
|
||||
|
||||
write_upstream() {
|
||||
output=$1
|
||||
containers=$2
|
||||
: > "$output"
|
||||
for container in $containers; do
|
||||
name=$(container_name "$container")
|
||||
printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_containers() {
|
||||
containers=$1
|
||||
deadline=$(($(date +%s) + 120))
|
||||
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
containers=$(compose ps -q api)
|
||||
count=0
|
||||
healthy=0
|
||||
for container in $containers; do
|
||||
@@ -43,13 +70,13 @@ wait_for_replicas() {
|
||||
healthy=$((healthy + 1))
|
||||
fi
|
||||
done
|
||||
if [ "$count" -eq "$expected" ] && [ "$healthy" -eq "$expected" ]; then
|
||||
if [ "$count" -eq "$replicas" ] && [ "$healthy" -eq "$replicas" ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
printf 'API replicas did not become healthy: expected %s\n' "$expected" >&2
|
||||
printf 'candidate API pool did not become healthy: expected %s\n' "$replicas" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -57,53 +84,36 @@ smoke_check() {
|
||||
compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready
|
||||
}
|
||||
|
||||
running_image() {
|
||||
containers=$(compose ps -q "$1")
|
||||
remove_containers() {
|
||||
containers=$1
|
||||
for container in $containers; do
|
||||
docker inspect --format '{{.Config.Image}}' "$container"
|
||||
return 0
|
||||
docker rm -f "$container" >/dev/null 2>&1 || true
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
capture_running_images() {
|
||||
output=$1
|
||||
api_image=$(running_image api) || return 1
|
||||
nginx_image=$(running_image nginx) || return 1
|
||||
swagger_image=$(running_image swagger) || swagger_image=$SWAGGER_IMAGE
|
||||
prometheus_image=$(running_image prometheus) || prometheus_image=$PROMETHEUS_IMAGE
|
||||
grafana_image=$(running_image grafana) || grafana_image=$GRAFANA_IMAGE
|
||||
s3_image=$(running_image s3) || s3_image=$S3_IMAGE
|
||||
s3_bootstrap_image=$(running_image s3-bootstrap) || s3_bootstrap_image=$S3_BOOTSTRAP_IMAGE
|
||||
|
||||
{
|
||||
printf 'API_IMAGE=%s\n' "$api_image"
|
||||
printf 'NGINX_IMAGE=%s\n' "$nginx_image"
|
||||
printf 'SWAGGER_IMAGE=%s\n' "$swagger_image"
|
||||
printf 'PROMETHEUS_IMAGE=%s\n' "$prometheus_image"
|
||||
printf 'GRAFANA_IMAGE=%s\n' "$grafana_image"
|
||||
printf 'S3_IMAGE=%s\n' "$s3_image"
|
||||
printf 'S3_BOOTSTRAP_IMAGE=%s\n' "$s3_bootstrap_image"
|
||||
} > "$output"
|
||||
restore_upstream() {
|
||||
if [ -s "$upstream_backup" ]; then
|
||||
cp "$upstream_backup" "$upstream_file"
|
||||
compose exec -T nginx nginx -t
|
||||
compose exec -T nginx nginx -s reload
|
||||
fi
|
||||
}
|
||||
|
||||
rollback() {
|
||||
status=$?
|
||||
trap - EXIT INT TERM
|
||||
candidates=$(cat "$candidate_file" 2>/dev/null || true)
|
||||
|
||||
if [ "$switched" = true ] && [ "$rollback_ready" = true ]; then
|
||||
printf '%s\n' 'Deployment failed; restoring the previous images.' >&2
|
||||
compose stop nginx api >/dev/null 2>&1 || true
|
||||
compose rm -f nginx api >/dev/null 2>&1 || true
|
||||
load_manifest "$previous_manifest"
|
||||
compose up -d --no-build --no-deps --force-recreate --scale api="$replicas" api
|
||||
wait_for_replicas "$replicas"
|
||||
compose up -d --no-build --no-deps --force-recreate nginx
|
||||
smoke_check
|
||||
elif [ "$switched" = true ]; then
|
||||
printf '%s\n' 'Deployment failed and no previous image manifest is available.' >&2
|
||||
if [ "$upstream_changed" = true ] && [ -n "$old_pool" ]; then
|
||||
printf '%s\n' 'Deployment failed; switching Nginx back to the previous API pool.' >&2
|
||||
restore_upstream || true
|
||||
if [ "$switched" = true ]; then
|
||||
smoke_check || true
|
||||
fi
|
||||
elif [ "$nginx_created" = true ]; then
|
||||
compose stop nginx >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
remove_containers "$candidates"
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
@@ -112,30 +122,102 @@ trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
load_manifest "$manifest"
|
||||
new_api_image=$API_IMAGE
|
||||
|
||||
if [ -f "$current_manifest" ]; then
|
||||
cp "$current_manifest" "$previous_manifest"
|
||||
rollback_ready=true
|
||||
elif capture_running_images "$previous_manifest"; then
|
||||
rollback_ready=true
|
||||
if [ -f "$previous_containers" ]; then
|
||||
remove_containers "$(cat "$previous_containers")"
|
||||
rm -f "$previous_containers"
|
||||
fi
|
||||
|
||||
if [ -s "$active_containers" ]; then
|
||||
for container in $(cat "$active_containers"); do
|
||||
if running_container "$container"; then
|
||||
old_pool="$old_pool $container"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ -z "$old_pool" ]; then
|
||||
old_pool=$(compose ps -q api)
|
||||
fi
|
||||
|
||||
if [ -s "$upstream_file" ]; then
|
||||
:
|
||||
elif [ -n "$old_pool" ]; then
|
||||
write_upstream "$upstream_file" "$old_pool"
|
||||
else
|
||||
cp nginx/upstream.conf "$upstream_file"
|
||||
fi
|
||||
cp "$upstream_file" "$upstream_backup"
|
||||
|
||||
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
|
||||
compose up -d --no-build --wait --wait-timeout 120 postgres valkey s3
|
||||
compose run --rm --no-deps s3-bootstrap
|
||||
compose run --rm --no-deps migrate
|
||||
|
||||
# Stop the entry point and every old API replica before starting the new pool.
|
||||
# This gives an atomic version boundary at the cost of a short maintenance window.
|
||||
switched=true
|
||||
compose stop nginx api
|
||||
compose rm -f nginx api
|
||||
compose up -d --no-build --no-deps --force-recreate --scale api="$replicas" api
|
||||
wait_for_replicas "$replicas"
|
||||
compose up -d --no-build --no-deps --force-recreate nginx
|
||||
nginx_container=$(compose ps -q nginx)
|
||||
if [ -n "$nginx_container" ] && ! docker inspect --format '{{range .Mounts}}{{println .Destination}}{{end}}' "$nginx_container" | grep -qx /etc/nginx/conf.d/upstream.conf; then
|
||||
compose up -d --no-build --no-deps --force-recreate nginx
|
||||
nginx_container=$(compose ps -q nginx)
|
||||
fi
|
||||
if [ -n "$nginx_container" ]; then
|
||||
running_nginx_image=$(docker inspect --format '{{.Config.Image}}' "$nginx_container")
|
||||
if [ "$running_nginx_image" != "$NGINX_IMAGE" ]; then
|
||||
printf 'verified Nginx image differs from the running load balancer; deploy it in a dedicated maintenance operation\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
API_IMAGE=$new_api_image
|
||||
export API_IMAGE
|
||||
short_digest=$(printf '%s' "$new_api_image" | sed 's/.*sha256://; s/[^0-9a-f].*//' | cut -c1-12)
|
||||
if [ -z "$short_digest" ]; then
|
||||
short_digest=$(date +%s)
|
||||
fi
|
||||
generation=${DEPLOY_GENERATION:-"$(date +%s)-$$"}
|
||||
|
||||
candidate_pool=""
|
||||
index=1
|
||||
while [ "$index" -le "$replicas" ]; do
|
||||
name="${project}-api-${short_digest}-${generation}-${index}"
|
||||
container=$(compose run -d --no-deps --name "$name" api)
|
||||
candidate_pool="$candidate_pool $container"
|
||||
printf '%s\n' "$container" >> "$candidate_file"
|
||||
index=$((index + 1))
|
||||
done
|
||||
wait_for_containers "$candidate_pool"
|
||||
|
||||
write_upstream "$upstream_file.next" "$candidate_pool"
|
||||
cp "$upstream_file.next" "$upstream_file"
|
||||
upstream_changed=true
|
||||
|
||||
if [ -n "$(compose ps -q nginx)" ]; then
|
||||
compose exec -T nginx nginx -t
|
||||
switched=true
|
||||
compose exec -T nginx nginx -s reload
|
||||
else
|
||||
nginx_created=true
|
||||
compose up -d --no-build --no-deps nginx
|
||||
switched=true
|
||||
fi
|
||||
smoke_check
|
||||
|
||||
compose up -d --no-build swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana
|
||||
application_network="${project}_application"
|
||||
for container in $candidate_pool; do
|
||||
docker network disconnect "$application_network" "$container"
|
||||
docker network connect --alias api "$application_network" "$container"
|
||||
done
|
||||
smoke_check
|
||||
|
||||
compose up -d --no-build --no-deps swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana
|
||||
|
||||
printf '%s\n' "$candidate_pool" > "$active_containers"
|
||||
cp "$manifest" "$current_manifest"
|
||||
rm -f "$candidate_file" "$upstream_file.next"
|
||||
trap - EXIT INT TERM
|
||||
|
||||
if [ -n "$old_pool" ]; then
|
||||
for container in $old_pool; do
|
||||
docker stop --time 6 "$container" >/dev/null || true
|
||||
done
|
||||
printf '%s\n' "$old_pool" > "$previous_containers"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user