forked from mtusi2026/mtusi2026lab7
229 lines
6.6 KiB
Bash
Executable File
229 lines
6.6 KiB
Bash
Executable File
#!/bin/sh
|
|
|
|
set -eu
|
|
|
|
if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
|
|
printf 'usage: %s <image-manifest>\n' "$0" >&2
|
|
exit 2
|
|
fi
|
|
|
|
manifest=$1
|
|
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab7"}
|
|
current_manifest=$deploy_dir/current-images.env
|
|
active_containers=$deploy_dir/active-api-containers
|
|
previous_containers=$deploy_dir/previous-api-containers
|
|
upstream_file=$deploy_dir/upstream.conf
|
|
upstream_backup=$deploy_dir/upstream.previous.conf
|
|
candidate_file=$deploy_dir/candidate-api-containers
|
|
replicas=${API_REPLICAS:-2}
|
|
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab7}
|
|
public_base_url=${PUBLIC_BASE_URL:-https://localhost:8443}
|
|
public_ca_file=${PUBLIC_CA_FILE:-.runtime/security/public-ca/ca.crt}
|
|
switched=false
|
|
nginx_created=false
|
|
upstream_changed=false
|
|
old_pool=""
|
|
|
|
mkdir -p "$deploy_dir"
|
|
: > "$candidate_file"
|
|
export NGINX_UPSTREAM_FILE=$upstream_file
|
|
|
|
load_manifest() {
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
. "$1"
|
|
set +a
|
|
}
|
|
|
|
compose() {
|
|
docker compose -f compose.yaml -f deploy/compose.images.yaml -f deploy/compose.runtime.yaml "$@"
|
|
}
|
|
|
|
container_name() {
|
|
name=$(docker inspect --format '{{.Name}}' "$1")
|
|
printf '%s\n' "${name#/}"
|
|
}
|
|
|
|
running_container() {
|
|
[ "$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null || true)" = true ]
|
|
}
|
|
|
|
write_upstream() {
|
|
output=$1
|
|
containers=$2
|
|
: > "$output"
|
|
for container in $containers; do
|
|
name=$(container_name "$container")
|
|
printf 'server %s:8443 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
|
|
done
|
|
}
|
|
|
|
wait_for_containers() {
|
|
containers=$1
|
|
deadline=$(($(date +%s) + 120))
|
|
|
|
while [ "$(date +%s)" -lt "$deadline" ]; do
|
|
count=0
|
|
healthy=0
|
|
for container in $containers; do
|
|
count=$((count + 1))
|
|
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")
|
|
if [ "$status" = healthy ]; then
|
|
healthy=$((healthy + 1))
|
|
fi
|
|
done
|
|
if [ "$count" -eq "$replicas" ] && [ "$healthy" -eq "$replicas" ]; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
printf 'candidate API pool did not become healthy: expected %s\n' "$replicas" >&2
|
|
return 1
|
|
}
|
|
|
|
smoke_check() {
|
|
curl --fail --silent --show-error --noproxy '*' \
|
|
--cacert "$public_ca_file" "$public_base_url/health/ready" >/dev/null
|
|
}
|
|
|
|
remove_containers() {
|
|
containers=$1
|
|
for container in $containers; do
|
|
docker rm -f "$container" >/dev/null 2>&1 || true
|
|
done
|
|
}
|
|
|
|
restore_upstream() {
|
|
if [ -s "$upstream_backup" ]; then
|
|
cp "$upstream_backup" "$upstream_file"
|
|
compose exec -T nginx nginx -t
|
|
compose exec -T nginx nginx -s reload
|
|
fi
|
|
}
|
|
|
|
rollback() {
|
|
status=$?
|
|
trap - EXIT INT TERM
|
|
candidates=$(cat "$candidate_file" 2>/dev/null || true)
|
|
|
|
if [ "$upstream_changed" = true ] && [ -n "$old_pool" ]; then
|
|
printf '%s\n' 'Deployment failed; switching Nginx back to the previous API pool.' >&2
|
|
restore_upstream || true
|
|
if [ "$switched" = true ]; then
|
|
smoke_check || true
|
|
fi
|
|
elif [ "$nginx_created" = true ]; then
|
|
compose stop nginx >/dev/null 2>&1 || true
|
|
fi
|
|
remove_containers "$candidates"
|
|
exit "$status"
|
|
}
|
|
|
|
trap rollback EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
load_manifest "$manifest"
|
|
new_api_image=$API_IMAGE
|
|
test -r "$public_ca_file"
|
|
|
|
if [ -f "$previous_containers" ]; then
|
|
remove_containers "$(cat "$previous_containers")"
|
|
rm -f "$previous_containers"
|
|
fi
|
|
|
|
if [ -s "$active_containers" ]; then
|
|
for container in $(cat "$active_containers"); do
|
|
if running_container "$container"; then
|
|
old_pool="$old_pool $container"
|
|
fi
|
|
done
|
|
fi
|
|
if [ -z "$old_pool" ]; then
|
|
old_pool=$(compose ps -q api)
|
|
fi
|
|
|
|
if [ -s "$upstream_file" ]; then
|
|
:
|
|
elif [ -n "$old_pool" ]; then
|
|
write_upstream "$upstream_file" "$old_pool"
|
|
else
|
|
cp nginx/upstream.conf "$upstream_file"
|
|
fi
|
|
cp "$upstream_file" "$upstream_backup"
|
|
|
|
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
|
|
compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3
|
|
compose run --rm security-bootstrap
|
|
compose run --rm --no-deps s3-bootstrap
|
|
compose run --rm --no-deps migrate
|
|
|
|
nginx_container=$(compose ps -q nginx)
|
|
if [ -n "$nginx_container" ] && ! docker inspect --format '{{range .Mounts}}{{println .Destination}}{{end}}' "$nginx_container" | grep -qx /etc/nginx/conf.d/upstream.conf; then
|
|
compose up -d --no-build --no-deps --force-recreate nginx
|
|
nginx_container=$(compose ps -q nginx)
|
|
fi
|
|
if [ -n "$nginx_container" ]; then
|
|
running_nginx_image=$(docker inspect --format '{{.Config.Image}}' "$nginx_container")
|
|
if [ "$running_nginx_image" != "$NGINX_IMAGE" ]; then
|
|
printf 'verified Nginx image differs from the running load balancer; deploy it in a dedicated maintenance operation\n' >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
API_IMAGE=$new_api_image
|
|
export API_IMAGE
|
|
short_digest=$(printf '%s' "$new_api_image" | sed 's/.*sha256://; s/[^0-9a-f].*//' | cut -c1-12)
|
|
if [ -z "$short_digest" ]; then
|
|
short_digest=$(date +%s)
|
|
fi
|
|
generation=${DEPLOY_GENERATION:-"$(date +%s)-$$"}
|
|
|
|
candidate_pool=""
|
|
index=1
|
|
while [ "$index" -le "$replicas" ]; do
|
|
name="${project}-api-${short_digest}-${generation}-${index}"
|
|
container=$(compose run -d --no-deps --name "$name" api)
|
|
candidate_pool="$candidate_pool $container"
|
|
printf '%s\n' "$container" >> "$candidate_file"
|
|
index=$((index + 1))
|
|
done
|
|
wait_for_containers "$candidate_pool"
|
|
|
|
write_upstream "$upstream_file.next" "$candidate_pool"
|
|
cp "$upstream_file.next" "$upstream_file"
|
|
upstream_changed=true
|
|
|
|
if [ -n "$(compose ps -q nginx)" ]; then
|
|
compose exec -T nginx nginx -t
|
|
switched=true
|
|
compose exec -T nginx nginx -s reload
|
|
else
|
|
nginx_created=true
|
|
compose up -d --no-build --no-deps nginx
|
|
switched=true
|
|
fi
|
|
smoke_check
|
|
|
|
application_network="${project}_application"
|
|
for container in $candidate_pool; do
|
|
docker network disconnect "$application_network" "$container"
|
|
docker network connect --alias api "$application_network" "$container"
|
|
done
|
|
smoke_check
|
|
|
|
compose up -d --no-build --no-deps dispatcher worker swagger postgres-exporter valkey-exporter redis-broker-exporter nginx-exporter prometheus grafana
|
|
|
|
printf '%s\n' "$candidate_pool" > "$active_containers"
|
|
cp "$manifest" "$current_manifest"
|
|
rm -f "$candidate_file" "$upstream_file.next"
|
|
trap - EXIT INT TERM
|
|
|
|
if [ -n "$old_pool" ]; then
|
|
for container in $old_pool; do
|
|
docker stop --time 6 "$container" >/dev/null || true
|
|
done
|
|
printf '%s\n' "$old_pool" > "$previous_containers"
|
|
fi
|