Files
mtusi2026lab7/scripts/rolling-update.sh
T

229 lines
6.6 KiB
Bash
Executable File

#!/bin/sh
set -eu
if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
printf 'usage: %s <image-manifest>\n' "$0" >&2
exit 2
fi
manifest=$1
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab7"}
current_manifest=$deploy_dir/current-images.env
active_containers=$deploy_dir/active-api-containers
previous_containers=$deploy_dir/previous-api-containers
upstream_file=$deploy_dir/upstream.conf
upstream_backup=$deploy_dir/upstream.previous.conf
candidate_file=$deploy_dir/candidate-api-containers
replicas=${API_REPLICAS:-2}
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab7}
public_base_url=${PUBLIC_BASE_URL:-https://localhost:8443}
public_ca_file=${PUBLIC_CA_FILE:-.runtime/security/public-ca/ca.crt}
switched=false
nginx_created=false
upstream_changed=false
old_pool=""
mkdir -p "$deploy_dir"
: > "$candidate_file"
export NGINX_UPSTREAM_FILE=$upstream_file
load_manifest() {
set -a
# shellcheck disable=SC1090
. "$1"
set +a
}
compose() {
docker compose -f compose.yaml -f deploy/compose.images.yaml -f deploy/compose.runtime.yaml "$@"
}
container_name() {
name=$(docker inspect --format '{{.Name}}' "$1")
printf '%s\n' "${name#/}"
}
running_container() {
[ "$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null || true)" = true ]
}
write_upstream() {
output=$1
containers=$2
: > "$output"
for container in $containers; do
name=$(container_name "$container")
printf 'server %s:8443 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
done
}
wait_for_containers() {
containers=$1
deadline=$(($(date +%s) + 120))
while [ "$(date +%s)" -lt "$deadline" ]; do
count=0
healthy=0
for container in $containers; do
count=$((count + 1))
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")
if [ "$status" = healthy ]; then
healthy=$((healthy + 1))
fi
done
if [ "$count" -eq "$replicas" ] && [ "$healthy" -eq "$replicas" ]; then
return 0
fi
sleep 1
done
printf 'candidate API pool did not become healthy: expected %s\n' "$replicas" >&2
return 1
}
smoke_check() {
curl --fail --silent --show-error --noproxy '*' \
--cacert "$public_ca_file" "$public_base_url/health/ready" >/dev/null
}
remove_containers() {
containers=$1
for container in $containers; do
docker rm -f "$container" >/dev/null 2>&1 || true
done
}
restore_upstream() {
if [ -s "$upstream_backup" ]; then
cp "$upstream_backup" "$upstream_file"
compose exec -T nginx nginx -t
compose exec -T nginx nginx -s reload
fi
}
rollback() {
status=$?
trap - EXIT INT TERM
candidates=$(cat "$candidate_file" 2>/dev/null || true)
if [ "$upstream_changed" = true ] && [ -n "$old_pool" ]; then
printf '%s\n' 'Deployment failed; switching Nginx back to the previous API pool.' >&2
restore_upstream || true
if [ "$switched" = true ]; then
smoke_check || true
fi
elif [ "$nginx_created" = true ]; then
compose stop nginx >/dev/null 2>&1 || true
fi
remove_containers "$candidates"
exit "$status"
}
trap rollback EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
load_manifest "$manifest"
new_api_image=$API_IMAGE
test -r "$public_ca_file"
if [ -f "$previous_containers" ]; then
remove_containers "$(cat "$previous_containers")"
rm -f "$previous_containers"
fi
if [ -s "$active_containers" ]; then
for container in $(cat "$active_containers"); do
if running_container "$container"; then
old_pool="$old_pool $container"
fi
done
fi
if [ -z "$old_pool" ]; then
old_pool=$(compose ps -q api)
fi
if [ -s "$upstream_file" ]; then
:
elif [ -n "$old_pool" ]; then
write_upstream "$upstream_file" "$old_pool"
else
cp nginx/upstream.conf "$upstream_file"
fi
cp "$upstream_file" "$upstream_backup"
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3
compose run --rm security-bootstrap
compose run --rm --no-deps s3-bootstrap
compose run --rm --no-deps migrate
nginx_container=$(compose ps -q nginx)
if [ -n "$nginx_container" ] && ! docker inspect --format '{{range .Mounts}}{{println .Destination}}{{end}}' "$nginx_container" | grep -qx /etc/nginx/conf.d/upstream.conf; then
compose up -d --no-build --no-deps --force-recreate nginx
nginx_container=$(compose ps -q nginx)
fi
if [ -n "$nginx_container" ]; then
running_nginx_image=$(docker inspect --format '{{.Config.Image}}' "$nginx_container")
if [ "$running_nginx_image" != "$NGINX_IMAGE" ]; then
printf 'verified Nginx image differs from the running load balancer; deploy it in a dedicated maintenance operation\n' >&2
exit 1
fi
fi
API_IMAGE=$new_api_image
export API_IMAGE
short_digest=$(printf '%s' "$new_api_image" | sed 's/.*sha256://; s/[^0-9a-f].*//' | cut -c1-12)
if [ -z "$short_digest" ]; then
short_digest=$(date +%s)
fi
generation=${DEPLOY_GENERATION:-"$(date +%s)-$$"}
candidate_pool=""
index=1
while [ "$index" -le "$replicas" ]; do
name="${project}-api-${short_digest}-${generation}-${index}"
container=$(compose run -d --no-deps --name "$name" api)
candidate_pool="$candidate_pool $container"
printf '%s\n' "$container" >> "$candidate_file"
index=$((index + 1))
done
wait_for_containers "$candidate_pool"
write_upstream "$upstream_file.next" "$candidate_pool"
cp "$upstream_file.next" "$upstream_file"
upstream_changed=true
if [ -n "$(compose ps -q nginx)" ]; then
compose exec -T nginx nginx -t
switched=true
compose exec -T nginx nginx -s reload
else
nginx_created=true
compose up -d --no-build --no-deps nginx
switched=true
fi
smoke_check
application_network="${project}_application"
for container in $candidate_pool; do
docker network disconnect "$application_network" "$container"
docker network connect --alias api "$application_network" "$container"
done
smoke_check
compose up -d --no-build --no-deps dispatcher worker swagger postgres-exporter valkey-exporter redis-broker-exporter nginx-exporter prometheus grafana
printf '%s\n' "$candidate_pool" > "$active_containers"
cp "$manifest" "$current_manifest"
rm -f "$candidate_file" "$upstream_file.next"
trap - EXIT INT TERM
if [ -n "$old_pool" ]; then
for container in $old_pool; do
docker stop --time 6 "$container" >/dev/null || true
done
printf '%s\n' "$old_pool" > "$previous_containers"
fi