Files
mtusi2026lab6/scripts/observability-incidents.sh
T

101 lines
4.8 KiB
Bash
Executable File

#!/bin/sh
# Bounded fault experiments on the disposable test-local.sh stack only.
set -eu
: "${LOCAL_TEST_WORK:?}" "${BASE_URL:?}" "${PROMETHEUS_URL:?}" "${TELEMETRY_TEMPO:?}"
. "$LOCAL_TEST_WORK/environment"
export PROBE_WORK="$LOCAL_TEST_WORK/probe"
mkdir -p evidence/incidents
scripts/prepare-probe.sh
scripts/trace-probe.sh > evidence/current/correlation.txt 2>&1
worker=$(cat "$LOCAL_TEST_WORK/worker.pid")
api_a=$(cat "$LOCAL_TEST_WORK/local-a.pid")
api_b=$(cat "$LOCAL_TEST_WORK/local-b.pid")
chat=$(cat "$PROBE_WORK/chat-id")
load_pids=
paused_db=false
paused_worker=false
stopped_tempo=false
cleanup() {
for pid in $load_pids; do kill "$pid" 2>/dev/null || true; done
if $paused_db; then docker unpause "$LOCAL_TEST_PG" >/dev/null; fi
if $paused_worker; then kill -CONT "$worker"; fi
if $stopped_tempo; then docker start "$TELEMETRY_TEMPO" >/dev/null; fi
}
trap cleanup EXIT
printf 'baseline\n' > "$PROBE_WORK/phase"
for client in 1 2; do
(
while :; do
now=$(date +%s)
phase=$(cat "$PROBE_WORK/phase")
result=$(curl -sS --max-time 4 --config "$PROBE_WORK/auth.conf" -o /dev/null -w '%{http_code},%{time_total}' "$BASE_URL/api/v1/chats/$chat/messages" 2>/dev/null || true)
printf '%s,%s,%s,%s\n' "$now" "$phase" "$client" "$result"
sleep .25
done
) > "evidence/incidents/http-$client.csv" &
load_pids="$load_pids $!"
done
snapshot() {
phase=$1
now=$(date +%s)
curl -fsS "$PROMETHEUS_URL/api/v1/alerts" > "$PROBE_WORK/alerts.json"
jq -c --arg phase "$phase" --argjson at "$now" '{at:$at,phase:$phase,alerts:.data.alerts}' "$PROBE_WORK/alerts.json" >> evidence/incidents/alerts.jsonl
printf '%s %s\n' "$now" "$phase" >> evidence/incidents/rss.txt
ps -p "$api_a,$api_b,$worker" -o pid=,rss=,time= >> evidence/incidents/rss.txt
}
phase() {
name=$1
seconds=$2
printf '%s\n' "$name" > "$PROBE_WORK/phase"
printf '%s,%s,start\n' "$(date +%s)" "$name" >> evidence/incidents/events.csv
deadline=$(($(date +%s)+seconds))
while [ "$(date +%s)" -lt "$deadline" ]; do snapshot "$name"; sleep 5; done
printf '%s,%s,end\n' "$(date +%s)" "$name" >> evidence/incidents/events.csv
}
: > evidence/incidents/events.csv
: > evidence/incidents/alerts.jsonl
: > evidence/incidents/rss.txt
phase baseline 120
docker pause "$LOCAL_TEST_PG" >/dev/null
paused_db=true
phase short-db-spike 5
docker unpause "$LOCAL_TEST_PG" >/dev/null
paused_db=false
phase spike-recovery 90
docker pause "$LOCAL_TEST_PG" >/dev/null
paused_db=true
phase sustained-db-outage 180
docker unpause "$LOCAL_TEST_PG" >/dev/null
paused_db=false
phase db-recovery 90
kill -STOP "$worker"
paused_worker=true
printf 'worker-stopped\n' > "$PROBE_WORK/phase"
printf '%s,worker-stopped,start\n' "$(date +%s)" >> evidence/incidents/events.csv
for attempt in $(seq 1 20); do
status=$(curl -sS --config "$PROBE_WORK/auth.conf" -o "$PROBE_WORK/queued.json" -w '%{http_code}' "$BASE_URL/api/v1/chats/$chat/attachments?filename=probe.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png)
test "$status" = 202
jq -e '.status=="queued"' "$PROBE_WORK/queued.json" >/dev/null
sleep 1
done
phase worker-stopped 90
kill -CONT "$worker"
paused_worker=false
phase worker-recovery 90
docker stop --time 5 "$TELEMETRY_TEMPO" >/dev/null
stopped_tempo=true
phase trace-backend-outage 45
make test > evidence/incidents/smoke-with-tempo-down.txt 2>&1
docker start "$TELEMETRY_TEMPO" >/dev/null
stopped_tempo=false
phase telemetry-recovery 20
for query in 'max(messenger_queue_oldest_seconds)' 'max(messenger_queue_depth)' 'sum(rate(messenger_http_requests_total{status_class="5xx"}[1m]))' 'sum(otelcol_exporter_send_failed_spans_total)' 'sum(otelcol_exporter_queue_size)'; do
curl -fsS --get "$PROMETHEUS_URL/api/v1/query" --data-urlencode "query=$query" | jq -c --arg query "$query" '{query:$query,response:.}' >> evidence/incidents/final-metrics.jsonl
done
jq -s -e '[.[]|select(.phase=="sustained-db-outage" or .phase=="db-recovery")|.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null
jq -s -e '[.[]|select(.phase=="worker-stopped")|.alerts[]|select(.labels.alertname=="MessengerQueueLag" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null
jq -s -e '[.[]|select(.phase=="short-db-spike" or .phase=="spike-recovery")|.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" and .state=="firing")]|length==0' evidence/incidents/alerts.jsonl >/dev/null
jq -s -e '[.[]|select(.phase=="worker-recovery")|.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null
tail -n 1 evidence/incidents/alerts.jsonl | jq -e '[.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" or .labels.alertname=="MessengerQueueLag" or .labels.alertname=="MessengerReadyBurnRate")]|length==0' >/dev/null
printf 'PASS: API, queue and ready alerts resolved\n'