#!/bin/sh # Verify time-to-ready burn against a real suspended worker on an owned test stack. set -eu : "${LOCAL_TEST_WORK:?}" "${BASE_URL:?}" "${PROMETHEUS_URL:?}" export PROBE_WORK="$LOCAL_TEST_WORK/ready-probe" scripts/prepare-probe.sh worker=$(cat "$LOCAL_TEST_WORK/worker.pid") trap 'kill -CONT "$worker" 2>/dev/null || true' EXIT mkdir -p evidence/ready-slo : > evidence/ready-slo/alerts.jsonl chat=$(cat "$PROBE_WORK/chat-id") # Warm the histogram before suspension so rate() observes a real counter delta. upload() { curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=slo.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json" } upload sleep 15 kill -STOP "$worker" printf '%s,worker-stopped\n' "$(date +%s)" > evidence/ready-slo/events.csv for i in $(seq 1 20); do upload; done observe() { phase=$1 for i in $(seq 1 "$2"); do curl -fsS "$PROMETHEUS_URL/api/v1/alerts" | jq -c --arg phase "$phase" --argjson at "$(date +%s)" '{at:$at,phase:$phase,alerts:.data.alerts}' >> evidence/ready-slo/alerts.jsonl sleep 5 done } observe worker-stopped 18 kill -CONT "$worker" printf '%s,worker-resumed\n' "$(date +%s)" >> evidence/ready-slo/events.csv observe worker-recovery 24 jq -s -e '[.[]|.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" and .state=="firing")]|length>0' evidence/ready-slo/alerts.jsonl >/dev/null tail -n 1 evidence/ready-slo/alerts.jsonl | jq -e '[.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" or .labels.alertname=="MessengerQueueLag")]|length==0' >/dev/null curl -fsS --get "$PROMETHEUS_URL/api/v1/query" --data-urlencode 'query=max(messenger_queue_depth)' > evidence/ready-slo/final-backlog.json printf 'PASS: ready burn and queue lag resolved after real worker recovery\n'