Files
mtusi2026lab6/scripts/ready-slo.sh
T

36 lines
1.8 KiB
Bash
Executable File

#!/bin/sh
# Verify time-to-ready burn against a real suspended worker on an owned test stack.
set -eu
: "${LOCAL_TEST_WORK:?}" "${BASE_URL:?}" "${PROMETHEUS_URL:?}"
export PROBE_WORK="$LOCAL_TEST_WORK/ready-probe"
scripts/prepare-probe.sh
worker=$(cat "$LOCAL_TEST_WORK/worker.pid")
trap 'kill -CONT "$worker" 2>/dev/null || true' EXIT
mkdir -p evidence/ready-slo
: > evidence/ready-slo/alerts.jsonl
chat=$(cat "$PROBE_WORK/chat-id")
# Warm the histogram before suspension so rate() observes a real counter delta.
upload() {
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=slo.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json"
}
upload
sleep 15
kill -STOP "$worker"
printf '%s,worker-stopped\n' "$(date +%s)" > evidence/ready-slo/events.csv
for i in $(seq 1 20); do upload; done
observe() {
phase=$1
for i in $(seq 1 "$2"); do
curl -fsS "$PROMETHEUS_URL/api/v1/alerts" | jq -c --arg phase "$phase" --argjson at "$(date +%s)" '{at:$at,phase:$phase,alerts:.data.alerts}' >> evidence/ready-slo/alerts.jsonl
sleep 5
done
}
observe worker-stopped 18
kill -CONT "$worker"
printf '%s,worker-resumed\n' "$(date +%s)" >> evidence/ready-slo/events.csv
observe worker-recovery 24
jq -s -e '[.[]|.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" and .state=="firing")]|length>0' evidence/ready-slo/alerts.jsonl >/dev/null
tail -n 1 evidence/ready-slo/alerts.jsonl | jq -e '[.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" or .labels.alertname=="MessengerQueueLag")]|length==0' >/dev/null
curl -fsS --get "$PROMETHEUS_URL/api/v1/query" --data-urlencode 'query=max(messenger_queue_depth)' > evidence/ready-slo/final-backlog.json
printf 'PASS: ready burn and queue lag resolved after real worker recovery\n'