forked from mtusi2026/mtusi2026lab6
36 lines
1.8 KiB
Bash
Executable File
36 lines
1.8 KiB
Bash
Executable File
#!/bin/sh
|
|
# Verify time-to-ready burn against a real suspended worker on an owned test stack.
|
|
set -eu
|
|
: "${LOCAL_TEST_WORK:?}" "${BASE_URL:?}" "${PROMETHEUS_URL:?}"
|
|
export PROBE_WORK="$LOCAL_TEST_WORK/ready-probe"
|
|
scripts/prepare-probe.sh
|
|
worker=$(cat "$LOCAL_TEST_WORK/worker.pid")
|
|
trap 'kill -CONT "$worker" 2>/dev/null || true' EXIT
|
|
mkdir -p evidence/ready-slo
|
|
: > evidence/ready-slo/alerts.jsonl
|
|
chat=$(cat "$PROBE_WORK/chat-id")
|
|
# Warm the histogram before suspension so rate() observes a real counter delta.
|
|
upload() {
|
|
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=slo.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json"
|
|
}
|
|
upload
|
|
sleep 15
|
|
kill -STOP "$worker"
|
|
printf '%s,worker-stopped\n' "$(date +%s)" > evidence/ready-slo/events.csv
|
|
for i in $(seq 1 20); do upload; done
|
|
observe() {
|
|
phase=$1
|
|
for i in $(seq 1 "$2"); do
|
|
curl -fsS "$PROMETHEUS_URL/api/v1/alerts" | jq -c --arg phase "$phase" --argjson at "$(date +%s)" '{at:$at,phase:$phase,alerts:.data.alerts}' >> evidence/ready-slo/alerts.jsonl
|
|
sleep 5
|
|
done
|
|
}
|
|
observe worker-stopped 18
|
|
kill -CONT "$worker"
|
|
printf '%s,worker-resumed\n' "$(date +%s)" >> evidence/ready-slo/events.csv
|
|
observe worker-recovery 24
|
|
jq -s -e '[.[]|.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" and .state=="firing")]|length>0' evidence/ready-slo/alerts.jsonl >/dev/null
|
|
tail -n 1 evidence/ready-slo/alerts.jsonl | jq -e '[.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" or .labels.alertname=="MessengerQueueLag")]|length==0' >/dev/null
|
|
curl -fsS --get "$PROMETHEUS_URL/api/v1/query" --data-urlencode 'query=max(messenger_queue_depth)' > evidence/ready-slo/final-backlog.json
|
|
printf 'PASS: ready burn and queue lag resolved after real worker recovery\n'
|