#!/bin/sh # Level 4: scripts/rollout.sh replaces every replica at UPDATE_AT under a # constant write load that retries uncertain sends with the same key; then # every confirmed message must be in the history once. # # sh scripts/bench/rollout-under-load.sh # # Environment: RATE (50), DURATION (60s), UPDATE_AT (15), OUT (evidence/rollout). # Times come from the Docker VM clock, as in failover.sh. set -eu cd "$(dirname "$0")/../.." . scripts/bench/lib.sh rate=${RATE:-50} duration=${DURATION:-60s} update_at=${UPDATE_AT:-15} dir=${OUT:-evidence/rollout} export MSYS_NO_PATHCONV=1 rm -rf "$dir" mkdir -p "$dir" docker compose up -d --wait >/dev/null 2>&1 # Built beforehand, as by a delivery pipeline: the update replaces replicas. docker compose build api >/dev/null 2>&1 meta "$dir/meta.json" "experiment=rolling-update" "rate=$rate" "duration=$duration" "update_at_s=$update_at" sh scripts/instances.sh >"$dir/instances-before.txt" # A margin for the host clock, which may run ahead of the VM's. since=$(date -u -d '-10 seconds' +%Y-%m-%dT%H:%M:%SZ) old=$(docker compose ps -q api | cut -c1-12 | tr '\n' ' ') BENCH_OUT=$(abspath "$dir") docker compose --profile bench run --rm loadgen timeline \ -rate "$rate" -duration "$duration" >"$dir/loadgen.log" 2>&1 & generator=$! sleep "$update_at" started_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) status=0 sh scripts/rollout.sh >"$dir/rollout.log" 2>&1 || status=$? finished_host=$(date -u +%Y-%m-%dT%H:%M:%S.%3NZ) wait "$generator" || echo "generator reported a failed history check, see $dir/loadgen.log" new="" for id in $(docker compose ps -q api); do new="$new{\"container\": \"$(printf '%s' "$id" | cut -c1-12)\", \"started_vm_utc\": \"$(docker inspect --format '{{.State.StartedAt}}' "$id")\"}, " done printf '{\n "rollout_exit_code": %s,\n "old_containers": "%s",\n "new_containers": [%s],\n "command_started_host_utc": "%s",\n "command_finished_host_utc": "%s"\n}\n' \ "$status" "$old" "${new%, }" "$started_host" "$finished_host" >"$dir/update.json" docker compose logs --no-color --timestamps --since "$since" lb api migrate >"$dir/compose.log" 2>&1 sh scripts/instances.sh >"$dir/instances-after.txt" cat "$dir/loadgen.log" | tail -n 1 echo "results in $dir"