From 37d93c340e394ed4a923972accc6c872abb96076 Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 9 Oct 2026 13:27:58 +0300 Subject: [PATCH] fix: switch API pools atomically --- .gitea/workflows/deploy.yml | 2 +- compose.yaml | 2 + nginx/nginx.conf | 2 +- nginx/upstream.conf | 1 + scripts/rolling-update.sh | 196 +++++++++++++++++++++++++----------- 5 files changed, 144 insertions(+), 59 deletions(-) create mode 100644 nginx/upstream.conf diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index bcf4cf7..2997748 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -180,7 +180,7 @@ jobs: git cat-file -e '$DEPLOY_SHA^{commit}' git reset --hard '$DEPLOY_SHA' manifest='/tmp/mtusi2026lab4-${{ github.run_id }}.env' - if scripts/rolling-update.sh \"\$manifest\"; then + if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then rm -f \"\$manifest\" else status=\$? diff --git a/compose.yaml b/compose.yaml index 2373efc..72b6ca6 100644 --- a/compose.yaml +++ b/compose.yaml @@ -177,6 +177,8 @@ services: image: ${NGINX_IMAGE:-mtusi-nginx:lab4} ports: - "${API_BIND_ADDRESS}:8080" + volumes: + - ${NGINX_UPSTREAM_FILE:-./nginx/upstream.conf}:/etc/nginx/conf.d/upstream.conf:ro depends_on: api: condition: service_healthy diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 3dde36a..d62dcd6 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -20,7 +20,7 @@ http { upstream messenger { zone messenger 64k; - server api:8080 resolve max_fails=1 fail_timeout=3s; + include /etc/nginx/conf.d/upstream.conf; keepalive 32; } diff --git a/nginx/upstream.conf b/nginx/upstream.conf new file mode 100644 index 0000000..f6131cb --- /dev/null +++ b/nginx/upstream.conf @@ -0,0 +1 @@ +server api:8080 resolve max_fails=1 fail_timeout=3s; diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh index 7d27dd7..cda4309 100755 --- a/scripts/rolling-update.sh +++ b/scripts/rolling-update.sh @@ -10,12 +10,21 @@ fi manifest=$1 deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab4"} current_manifest=$deploy_dir/current-images.env -previous_manifest=$deploy_dir/previous-images.env +active_containers=$deploy_dir/active-api-containers +previous_containers=$deploy_dir/previous-api-containers +upstream_file=$deploy_dir/upstream.conf +upstream_backup=$deploy_dir/upstream.previous.conf +candidate_file=$deploy_dir/candidate-api-containers replicas=${API_REPLICAS:-2} -rollback_ready=false +project=${COMPOSE_PROJECT_NAME:-mtusi2026lab4} switched=false +nginx_created=false +upstream_changed=false +old_pool="" mkdir -p "$deploy_dir" +: > "$candidate_file" +export NGINX_UPSTREAM_FILE=$upstream_file load_manifest() { set -a @@ -28,12 +37,30 @@ compose() { docker compose -f compose.yaml -f deploy/compose.images.yaml "$@" } -wait_for_replicas() { - expected=$1 +container_name() { + name=$(docker inspect --format '{{.Name}}' "$1") + printf '%s\n' "${name#/}" +} + +running_container() { + [ "$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null || true)" = true ] +} + +write_upstream() { + output=$1 + containers=$2 + : > "$output" + for container in $containers; do + name=$(container_name "$container") + printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output" + done +} + +wait_for_containers() { + containers=$1 deadline=$(($(date +%s) + 120)) while [ "$(date +%s)" -lt "$deadline" ]; do - containers=$(compose ps -q api) count=0 healthy=0 for container in $containers; do @@ -43,13 +70,13 @@ wait_for_replicas() { healthy=$((healthy + 1)) fi done - if [ "$count" -eq "$expected" ] && [ "$healthy" -eq "$expected" ]; then + if [ "$count" -eq "$replicas" ] && [ "$healthy" -eq "$replicas" ]; then return 0 fi sleep 1 done - printf 'API replicas did not become healthy: expected %s\n' "$expected" >&2 + printf 'candidate API pool did not become healthy: expected %s\n' "$replicas" >&2 return 1 } @@ -57,53 +84,36 @@ smoke_check() { compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready } -running_image() { - containers=$(compose ps -q "$1") +remove_containers() { + containers=$1 for container in $containers; do - docker inspect --format '{{.Config.Image}}' "$container" - return 0 + docker rm -f "$container" >/dev/null 2>&1 || true done - return 1 } -capture_running_images() { - output=$1 - api_image=$(running_image api) || return 1 - nginx_image=$(running_image nginx) || return 1 - swagger_image=$(running_image swagger) || swagger_image=$SWAGGER_IMAGE - prometheus_image=$(running_image prometheus) || prometheus_image=$PROMETHEUS_IMAGE - grafana_image=$(running_image grafana) || grafana_image=$GRAFANA_IMAGE - s3_image=$(running_image s3) || s3_image=$S3_IMAGE - s3_bootstrap_image=$(running_image s3-bootstrap) || s3_bootstrap_image=$S3_BOOTSTRAP_IMAGE - - { - printf 'API_IMAGE=%s\n' "$api_image" - printf 'NGINX_IMAGE=%s\n' "$nginx_image" - printf 'SWAGGER_IMAGE=%s\n' "$swagger_image" - printf 'PROMETHEUS_IMAGE=%s\n' "$prometheus_image" - printf 'GRAFANA_IMAGE=%s\n' "$grafana_image" - printf 'S3_IMAGE=%s\n' "$s3_image" - printf 'S3_BOOTSTRAP_IMAGE=%s\n' "$s3_bootstrap_image" - } > "$output" +restore_upstream() { + if [ -s "$upstream_backup" ]; then + cp "$upstream_backup" "$upstream_file" + compose exec -T nginx nginx -t + compose exec -T nginx nginx -s reload + fi } rollback() { status=$? trap - EXIT INT TERM + candidates=$(cat "$candidate_file" 2>/dev/null || true) - if [ "$switched" = true ] && [ "$rollback_ready" = true ]; then - printf '%s\n' 'Deployment failed; restoring the previous images.' >&2 - compose stop nginx api >/dev/null 2>&1 || true - compose rm -f nginx api >/dev/null 2>&1 || true - load_manifest "$previous_manifest" - compose up -d --no-build --no-deps --force-recreate --scale api="$replicas" api - wait_for_replicas "$replicas" - compose up -d --no-build --no-deps --force-recreate nginx - smoke_check - elif [ "$switched" = true ]; then - printf '%s\n' 'Deployment failed and no previous image manifest is available.' >&2 + if [ "$upstream_changed" = true ] && [ -n "$old_pool" ]; then + printf '%s\n' 'Deployment failed; switching Nginx back to the previous API pool.' >&2 + restore_upstream || true + if [ "$switched" = true ]; then + smoke_check || true + fi + elif [ "$nginx_created" = true ]; then + compose stop nginx >/dev/null 2>&1 || true fi - + remove_containers "$candidates" exit "$status" } @@ -112,30 +122,102 @@ trap 'exit 130' INT trap 'exit 143' TERM load_manifest "$manifest" +new_api_image=$API_IMAGE -if [ -f "$current_manifest" ]; then - cp "$current_manifest" "$previous_manifest" - rollback_ready=true -elif capture_running_images "$previous_manifest"; then - rollback_ready=true +if [ -f "$previous_containers" ]; then + remove_containers "$(cat "$previous_containers")" + rm -f "$previous_containers" fi +if [ -s "$active_containers" ]; then + for container in $(cat "$active_containers"); do + if running_container "$container"; then + old_pool="$old_pool $container" + fi + done +fi +if [ -z "$old_pool" ]; then + old_pool=$(compose ps -q api) +fi + +if [ -s "$upstream_file" ]; then + : +elif [ -n "$old_pool" ]; then + write_upstream "$upstream_file" "$old_pool" +else + cp nginx/upstream.conf "$upstream_file" +fi +cp "$upstream_file" "$upstream_backup" + compose pull api nginx swagger prometheus grafana s3 s3-bootstrap compose up -d --no-build --wait --wait-timeout 120 postgres valkey s3 compose run --rm --no-deps s3-bootstrap compose run --rm --no-deps migrate -# Stop the entry point and every old API replica before starting the new pool. -# This gives an atomic version boundary at the cost of a short maintenance window. -switched=true -compose stop nginx api -compose rm -f nginx api -compose up -d --no-build --no-deps --force-recreate --scale api="$replicas" api -wait_for_replicas "$replicas" -compose up -d --no-build --no-deps --force-recreate nginx +nginx_container=$(compose ps -q nginx) +if [ -n "$nginx_container" ] && ! docker inspect --format '{{range .Mounts}}{{println .Destination}}{{end}}' "$nginx_container" | grep -qx /etc/nginx/conf.d/upstream.conf; then + compose up -d --no-build --no-deps --force-recreate nginx + nginx_container=$(compose ps -q nginx) +fi +if [ -n "$nginx_container" ]; then + running_nginx_image=$(docker inspect --format '{{.Config.Image}}' "$nginx_container") + if [ "$running_nginx_image" != "$NGINX_IMAGE" ]; then + printf 'verified Nginx image differs from the running load balancer; deploy it in a dedicated maintenance operation\n' >&2 + exit 1 + fi +fi + +API_IMAGE=$new_api_image +export API_IMAGE +short_digest=$(printf '%s' "$new_api_image" | sed 's/.*sha256://; s/[^0-9a-f].*//' | cut -c1-12) +if [ -z "$short_digest" ]; then + short_digest=$(date +%s) +fi +generation=${DEPLOY_GENERATION:-"$(date +%s)-$$"} + +candidate_pool="" +index=1 +while [ "$index" -le "$replicas" ]; do + name="${project}-api-${short_digest}-${generation}-${index}" + container=$(compose run -d --no-deps --name "$name" api) + candidate_pool="$candidate_pool $container" + printf '%s\n' "$container" >> "$candidate_file" + index=$((index + 1)) +done +wait_for_containers "$candidate_pool" + +write_upstream "$upstream_file.next" "$candidate_pool" +cp "$upstream_file.next" "$upstream_file" +upstream_changed=true + +if [ -n "$(compose ps -q nginx)" ]; then + compose exec -T nginx nginx -t + switched=true + compose exec -T nginx nginx -s reload +else + nginx_created=true + compose up -d --no-build --no-deps nginx + switched=true +fi smoke_check -compose up -d --no-build swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana +application_network="${project}_application" +for container in $candidate_pool; do + docker network disconnect "$application_network" "$container" + docker network connect --alias api "$application_network" "$container" +done +smoke_check +compose up -d --no-build --no-deps swagger postgres-exporter valkey-exporter nginx-exporter prometheus grafana + +printf '%s\n' "$candidate_pool" > "$active_containers" cp "$manifest" "$current_manifest" +rm -f "$candidate_file" "$upstream_file.next" trap - EXIT INT TERM + +if [ -n "$old_pool" ]; then + for container in $old_pool; do + docker stop --time 6 "$container" >/dev/null || true + done + printf '%s\n' "$old_pool" > "$previous_containers" +fi