forked from mtusi2026/mtusi2026lab6
101 lines
4.8 KiB
Bash
Executable File
101 lines
4.8 KiB
Bash
Executable File
#!/bin/sh
|
|
# Bounded fault experiments on the disposable test-local.sh stack only.
|
|
set -eu
|
|
: "${LOCAL_TEST_WORK:?}" "${BASE_URL:?}" "${PROMETHEUS_URL:?}" "${TELEMETRY_TEMPO:?}"
|
|
. "$LOCAL_TEST_WORK/environment"
|
|
export PROBE_WORK="$LOCAL_TEST_WORK/probe"
|
|
mkdir -p evidence/incidents
|
|
scripts/prepare-probe.sh
|
|
scripts/trace-probe.sh > evidence/current/correlation.txt 2>&1
|
|
worker=$(cat "$LOCAL_TEST_WORK/worker.pid")
|
|
api_a=$(cat "$LOCAL_TEST_WORK/local-a.pid")
|
|
api_b=$(cat "$LOCAL_TEST_WORK/local-b.pid")
|
|
chat=$(cat "$PROBE_WORK/chat-id")
|
|
load_pids=
|
|
paused_db=false
|
|
paused_worker=false
|
|
stopped_tempo=false
|
|
cleanup() {
|
|
for pid in $load_pids; do kill "$pid" 2>/dev/null || true; done
|
|
if $paused_db; then docker unpause "$LOCAL_TEST_PG" >/dev/null; fi
|
|
if $paused_worker; then kill -CONT "$worker"; fi
|
|
if $stopped_tempo; then docker start "$TELEMETRY_TEMPO" >/dev/null; fi
|
|
}
|
|
trap cleanup EXIT
|
|
printf 'baseline\n' > "$PROBE_WORK/phase"
|
|
for client in 1 2; do
|
|
(
|
|
while :; do
|
|
now=$(date +%s)
|
|
phase=$(cat "$PROBE_WORK/phase")
|
|
result=$(curl -sS --max-time 4 --config "$PROBE_WORK/auth.conf" -o /dev/null -w '%{http_code},%{time_total}' "$BASE_URL/api/v1/chats/$chat/messages" 2>/dev/null || true)
|
|
printf '%s,%s,%s,%s\n' "$now" "$phase" "$client" "$result"
|
|
sleep .25
|
|
done
|
|
) > "evidence/incidents/http-$client.csv" &
|
|
load_pids="$load_pids $!"
|
|
done
|
|
snapshot() {
|
|
phase=$1
|
|
now=$(date +%s)
|
|
curl -fsS "$PROMETHEUS_URL/api/v1/alerts" > "$PROBE_WORK/alerts.json"
|
|
jq -c --arg phase "$phase" --argjson at "$now" '{at:$at,phase:$phase,alerts:.data.alerts}' "$PROBE_WORK/alerts.json" >> evidence/incidents/alerts.jsonl
|
|
printf '%s %s\n' "$now" "$phase" >> evidence/incidents/rss.txt
|
|
ps -p "$api_a,$api_b,$worker" -o pid=,rss=,time= >> evidence/incidents/rss.txt
|
|
}
|
|
phase() {
|
|
name=$1
|
|
seconds=$2
|
|
printf '%s\n' "$name" > "$PROBE_WORK/phase"
|
|
printf '%s,%s,start\n' "$(date +%s)" "$name" >> evidence/incidents/events.csv
|
|
deadline=$(($(date +%s)+seconds))
|
|
while [ "$(date +%s)" -lt "$deadline" ]; do snapshot "$name"; sleep 5; done
|
|
printf '%s,%s,end\n' "$(date +%s)" "$name" >> evidence/incidents/events.csv
|
|
}
|
|
: > evidence/incidents/events.csv
|
|
: > evidence/incidents/alerts.jsonl
|
|
: > evidence/incidents/rss.txt
|
|
phase baseline 120
|
|
docker pause "$LOCAL_TEST_PG" >/dev/null
|
|
paused_db=true
|
|
phase short-db-spike 5
|
|
docker unpause "$LOCAL_TEST_PG" >/dev/null
|
|
paused_db=false
|
|
phase spike-recovery 90
|
|
docker pause "$LOCAL_TEST_PG" >/dev/null
|
|
paused_db=true
|
|
phase sustained-db-outage 180
|
|
docker unpause "$LOCAL_TEST_PG" >/dev/null
|
|
paused_db=false
|
|
phase db-recovery 90
|
|
kill -STOP "$worker"
|
|
paused_worker=true
|
|
printf 'worker-stopped\n' > "$PROBE_WORK/phase"
|
|
printf '%s,worker-stopped,start\n' "$(date +%s)" >> evidence/incidents/events.csv
|
|
for attempt in $(seq 1 20); do
|
|
status=$(curl -sS --config "$PROBE_WORK/auth.conf" -o "$PROBE_WORK/queued.json" -w '%{http_code}' "$BASE_URL/api/v1/chats/$chat/attachments?filename=probe.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png)
|
|
test "$status" = 202
|
|
jq -e '.status=="queued"' "$PROBE_WORK/queued.json" >/dev/null
|
|
sleep 1
|
|
done
|
|
phase worker-stopped 90
|
|
kill -CONT "$worker"
|
|
paused_worker=false
|
|
phase worker-recovery 90
|
|
docker stop --time 5 "$TELEMETRY_TEMPO" >/dev/null
|
|
stopped_tempo=true
|
|
phase trace-backend-outage 45
|
|
make test > evidence/incidents/smoke-with-tempo-down.txt 2>&1
|
|
docker start "$TELEMETRY_TEMPO" >/dev/null
|
|
stopped_tempo=false
|
|
phase telemetry-recovery 20
|
|
for query in 'max(messenger_queue_oldest_seconds)' 'max(messenger_queue_depth)' 'sum(rate(messenger_http_requests_total{status_class="5xx"}[1m]))' 'sum(otelcol_exporter_send_failed_spans_total)' 'sum(otelcol_exporter_queue_size)'; do
|
|
curl -fsS --get "$PROMETHEUS_URL/api/v1/query" --data-urlencode "query=$query" | jq -c --arg query "$query" '{query:$query,response:.}' >> evidence/incidents/final-metrics.jsonl
|
|
done
|
|
jq -s -e '[.[]|select(.phase=="sustained-db-outage" or .phase=="db-recovery")|.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null
|
|
jq -s -e '[.[]|select(.phase=="worker-stopped")|.alerts[]|select(.labels.alertname=="MessengerQueueLag" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null
|
|
jq -s -e '[.[]|select(.phase=="short-db-spike" or .phase=="spike-recovery")|.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" and .state=="firing")]|length==0' evidence/incidents/alerts.jsonl >/dev/null
|
|
jq -s -e '[.[]|select(.phase=="worker-recovery")|.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null
|
|
tail -n 1 evidence/incidents/alerts.jsonl | jq -e '[.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" or .labels.alertname=="MessengerQueueLag" or .labels.alertname=="MessengerReadyBurnRate")]|length==0' >/dev/null
|
|
printf 'PASS: API, queue and ready alerts resolved\n'
|