forked from mtusi2026/mtusi2026lab3
feat: rolling update без простоя и серия отказов реплик под нагрузкой (уровень 4)
- scripts/rolling-update.sh: сборка, однократные миграции новым образом, затем для каждой старой реплики: +1 новая (healthy и UP в HAProxy), graceful stop старой; ревизия образа видна в label и логе "starting". - bench/loadgen.py: фоновая нагрузка через LB (POST с Idempotency-Key + GET), неопределённый POST повторяется тем же ключом, проверка истории на потери и дубли. - bench/failover_series.py: каждая реплика по очереди в режимах stop / kill / pause; RPS, ошибки, время выхода из ротации и возврата. - bench/rolling_under_load.py: rolling update под нагрузкой. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
78b95acabf
commit
c050642ec2
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/bin/sh
|
||||
# Rolling update of the API without downtime (Compose has no built-in one for a scaled service).
|
||||
# 1. build the new image from the working tree;
|
||||
# 2. run migrations ONCE with the new image (they must be additive / compatible with the
|
||||
# running old version: expand first, contract in a later release);
|
||||
# 3. for every OLD replica: start one NEW replica (scale N+1, old ones untouched), wait until
|
||||
# Docker says healthy AND HAProxy has N+1 replicas UP, then stop the old one gracefully
|
||||
# (SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit) and remove it.
|
||||
# At every moment at least N replicas serve traffic. Stops on the first error (set -e);
|
||||
# the old replicas that were not replaced yet keep running.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
PY="${PYTHON:-$(command -v python3 || command -v python)}"
|
||||
# New image = new revision label (git commit + UTC time), visible in docker inspect and logs.
|
||||
BUILD_REVISION="${BUILD_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||
export BUILD_REVISION
|
||||
|
||||
replicas="$(docker compose ps -q api | wc -l | tr -d ' ')"
|
||||
[ "$replicas" -ge 1 ] || { echo "rolling-update: no running api replica" >&2; exit 2; }
|
||||
old="$(docker compose ps -q api)"
|
||||
echo "rolling-update: $replicas replica(s) to replace, new revision $BUILD_REVISION"
|
||||
|
||||
docker compose build api
|
||||
docker compose run --rm --no-deps migrate
|
||||
|
||||
for id in $old; do
|
||||
name="$(docker inspect -f '{{.Name}}' "$id" | tr -d '/')"
|
||||
docker compose up -d --no-deps --no-recreate --wait --scale "api=$((replicas + 1))" api
|
||||
"$PY" scripts/lbctl.py wait-up "$((replicas + 1))" --timeout 60
|
||||
docker stop --time 15 "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
"$PY" scripts/lbctl.py wait-up "$replicas" --timeout 60
|
||||
echo "rolling-update: replaced $name"
|
||||
done
|
||||
for id in $(docker compose ps -q api); do
|
||||
docker inspect -f '{{.Name}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} {{.State.Status}}' "$id"
|
||||
done
|
||||
Reference in New Issue
Block a user