61 lines
2.8 KiB
Bash
Executable File
61 lines
2.8 KiB
Bash
Executable File
#!/bin/sh
|
|
# Level 4: a rolling update under a constant write load. The generator sends
|
|
# messages at RATE for DURATION, retrying uncertain sends with the same key;
|
|
# at UPDATE_AT seconds `scripts/rollout.sh` replaces every replica. Afterwards
|
|
# the generator reads the history back: every confirmed message once.
|
|
#
|
|
# sh scripts/bench/rollout-under-load.sh
|
|
#
|
|
# Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (default
|
|
# evidence/rollout). Results in <OUT>/: timeline.json, seconds.csv,
|
|
# rollout.log, update.json (when it ran, which containers it replaced),
|
|
# compose.log, instances-*.txt.
|
|
#
|
|
# Times for the analysis come from the Docker VM's clock, as the generator's
|
|
# do: log timestamps and the start times of the new containers. The host
|
|
# clock of Windows runs apart from it and is kept for reference only. The
|
|
# old replicas are removed by the update; their stop shows in compose.log as
|
|
# their "shutdown started" lines.
|
|
set -eu
|
|
cd "$(dirname "$0")/../.."
|
|
. scripts/bench/lib.sh
|
|
|
|
rate=${RATE:-50}
|
|
duration=${DURATION:-60s}
|
|
update_at=${UPDATE_AT:-15}
|
|
dir=${OUT:-evidence/rollout}
|
|
export MSYS_NO_PATHCONV=1
|
|
|
|
rm -rf "$dir"
|
|
mkdir -p "$dir"
|
|
docker compose up -d --wait >/dev/null 2>&1
|
|
# Build first, as a delivery pipeline would: the update under load then
|
|
# measures the replacement of the replicas, not the compiler.
|
|
docker compose build api >/dev/null 2>&1
|
|
meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at"
|
|
sh scripts/instances.sh >"$dir/instances-before.txt"
|
|
# A margin for the host clock, which may run ahead of the VM's.
|
|
since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ)
|
|
old=$(docker compose ps -q api | cut -c1-12 | tr '\n' ' ')
|
|
|
|
BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \
|
|
-rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 &
|
|
generator=$!
|
|
sleep "$update_at"
|
|
started_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
|
status=0
|
|
sh scripts/rollout.sh >"$dir/rollout.log" 2>&1 || status=$?
|
|
finished_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ)
|
|
wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log"
|
|
|
|
new=""
|
|
for id in $(docker compose ps -q api); do
|
|
new="$new{\"container\": \"$(printf '%s' "$id" | cut -c1-12)\", \"started_vm_utc\": \"$(docker inspect --format '{{.State.StartedAt}}' "$id")\"}, "
|
|
done
|
|
printf '{\n "rollout_exit_code": %s,\n "old_containers": "%s",\n "new_containers": [%s],\n "command_started_host_utc": "%s",\n "command_finished_host_utc": "%s"\n}\n' \
|
|
"$status" "$old" "${new%, }" "$started_host" "$finished_host" >"$dir/update.json"
|
|
docker compose logs --no-color --timestamps --since "$since" lb api migrate >"$dir/compose.log" 2>&1
|
|
sh scripts/instances.sh >"$dir/instances-after.txt"
|
|
cat "$dir/loadgen.log" | tail -n 1
|
|
echo "results in $dir"
|