feat: add bounded observability and verified SLO incident alerts

This commit is contained in:
Vladislav Agapov
2026-10-29 18:01:11 +03:00
parent 470d9d6f1e
commit 6c7601802d
325 changed files with 24988 additions and 44 deletions
+20
View File
@@ -0,0 +1,20 @@
require 'json'
require 'csv'
root=ARGV.fetch(0)
events=CSV.read("#{root}/events.csv").map{|t,p,e| [t.to_i,p,e]}
snapshots=File.readlines("#{root}/alerts.jsonl").map{|l|JSON.parse(l)}
rows=Dir["#{root}/http-*.csv"].flat_map{|p|CSV.read(p)}
summary={phases: {}, alerts: {}}
rows.group_by{|r|r[1]}.each do |phase,rs|
vals=rs.map{|r|r[4].to_f*1000}.sort
start=events.find{|t,p,e|p==phase && e=='start'}&.first
finish=events.reverse.find{|t,p,e|p==phase && e=='end'}&.first
summary[:phases][phase]={requests:rs.size,errors:rs.count{|r|r[3]!='200'},p50_ms:vals[(vals.size*0.5).floor],p95_ms:vals[(vals.size*0.95).floor],p99_ms:vals[(vals.size*0.99).floor],duration_seconds: start && finish ? finish-start : nil}
end
%w[MessengerAPIBurnRate MessengerReadyBurnRate MessengerQueueLag MessengerTelemetryDrops].each do |name|
firing=snapshots.select{|s|s['alerts'].any?{|a|a['labels']['alertname']==name && a['state']=='firing'}}
first=firing.first
resolved=first && snapshots.find{|s|s['at']>firing.last['at'] && !s['alerts'].any?{|a|a['labels']['alertname']==name}}
summary[:alerts][name]={first_firing:first && first['at'],last_firing:firing.last && firing.last['at'],first_resolved:resolved && resolved['at'],firing_samples:firing.size}
end
File.write("#{root}/summary.json",JSON.pretty_generate(summary)+"\n")
+35
View File
@@ -0,0 +1,35 @@
# Aggregate covered statements once per merged block; keep files with no statements as N/A.
FNR == NR {
split($0, fields, "\t")
paths[fields[1]] = fields[2]
hashes[fields[1]] = fields[3]
executable[fields[1]] = fields[4]
next
}
/^mode:/ { next }
{
file = $1
sub(/:[^:]*$/, "", file)
if (!(file in paths)) {
print "unknown coverage source: " file > "/dev/stderr"
invalid = 1
}
total[file] += $2
if ($3 > 0) covered[file] += $2
}
END {
print "{\"threshold_per_file\":90,\"files\":[" > summary
separator = ""
for (file in paths) {
percent = total[file] ? sprintf("%.8f", covered[file] * 100 / total[file]) : "null"
printf "%s{\"file\":\"%s\",\"statements\":%d,\"covered\":%d,\"percent\":%s,\"sha256\":\"%s\"}", separator, paths[file], total[file], covered[file], percent, hashes[file] > summary
separator = ",\n"
printf "%8s %s\n", (total[file] ? sprintf("%.2f%%", covered[file] * 100 / total[file]) : "N/A"), paths[file]
if (covered[file] * 100 < total[file] * 90 || (executable[file] && !total[file])) invalid = 1
}
printf "\n],\"passed\":%s}\n", (invalid ? "false" : "true") > summary
if (invalid) {
print "FAIL: per-file coverage must reach 90%" > "/dev/stderr"
exit 1
}
}
+91
View File
@@ -0,0 +1,91 @@
#!/bin/sh
# Measure unit and real-process coverage, enforcing 90% separately for each Go source file.
set -eu
cd "$(dirname "$0")/.."
GO=${GO:-go}
: "${DATABASE_URL:?Use an isolated test database}" "${VALKEY_SHARDS:?Use test shards}"
: "${TEST_DATABASE_URL:?Use an isolated test database}" "${TEST_VALKEY_ADDRESS:?Use a test shard}"
export APP_PORT=${COVERAGE_PORT:-18080}
export LOG_LEVEL=error
export CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
worker_pid=
server_pid=
watchdog_pid=
mkdir -p coverage
run_dir=$(mktemp -d "$PWD/coverage/run.XXXXXX")
cleanup() {
if [ -n "$worker_pid" ]; then kill "$worker_pid" 2>/dev/null || true; wait "$worker_pid" 2>/dev/null || true; fi
if [ -n "$server_pid" ]; then kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true; fi
if [ -n "$watchdog_pid" ]; then kill "$watchdog_pid" 2>/dev/null || true; wait "$watchdog_pid" 2>/dev/null || true; fi
rm -rf "$run_dir"
}
trap cleanup EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
mkdir "$run_dir/process"
"$GO" build -race -cover -covermode=atomic -coverpkg=./... -o "$run_dir/server" ./cmd/server
"$run_dir/server" migrate
"$GO" test -count=1 -race -coverpkg=./... -coverprofile="$run_dir/unit.out" ./...
export GOCOVERDIR="$run_dir/process"
if "$run_dir/server" unknown >"$run_dir/server-invalid.log" 2>&1; then
echo 'server accepted an unknown command' >&2
exit 1
else
test "$?" -eq 1
fi
"$run_dir/server" worker >"$run_dir/worker.log" 2>&1 &
worker_pid=$!
"$run_dir/server" >"$run_dir/server.log" 2>&1 &
server_pid=$!
ready=false
for attempt in $(seq 1 60); do
kill -0 "$server_pid" 2>/dev/null || { cat "$run_dir/server.log"; exit 1; }
if curl --fail --silent --max-time 1 "http://localhost:$APP_PORT/health/ready" >/dev/null; then ready=true; break; fi
sleep .25
done
$ready || { cat "$run_dir/server.log"; exit 1; }
"$run_dir/server" healthcheck
"$run_dir/server" migrate
"${PYTHON:-python3}" tests/smoke.py --base-url "http://localhost:$APP_PORT"
kill -TERM "$server_pid"
# Bound the real-process shutdown check; a forced kill makes wait fail.
(sleep 12; kill -KILL "$server_pid" 2>/dev/null || true) &
watchdog_pid=$!
wait "$server_pid"
server_pid=
kill "$watchdog_pid" 2>/dev/null || true
wait "$watchdog_pid" 2>/dev/null || true
watchdog_pid=
kill -TERM "$worker_pid"
wait "$worker_pid"
worker_pid=
unset GOCOVERDIR
"$GO" tool covdata textfmt -i="$run_dir/process" -o="$run_dir/process.out"
awk '
/^mode:/ { if ($0 != "mode: atomic") exit 1; next }
{
if (($1 in statements) && statements[$1] != $2) exit 1;
statements[$1] = $2; hits[$1] += $3
}
END {
print "mode: atomic";
for (block in statements) print block, statements[block], hits[block]
}' "$run_dir/unit.out" "$run_dir/process.out" > coverage/coverage.out
"$GO" list -f '{{.ImportPath}} {{.Dir}} {{join .GoFiles ","}}' ./... > "$run_dir/packages.txt"
: > "$run_dir/inventory.tsv"
while read -r import_path directory names; do
old_ifs=$IFS
IFS=,
for name in $names; do
checksum=$(shasum -a 256 "$directory/$name")
checksum=${checksum%% *}
relative=${directory#"$PWD/"}/$name
executable=$(awk '/^func[ (]/ {found=1} END {print found+0}' "$directory/$name")
printf '%s\t%s\t%s\t%s
' "$import_path/$name" "$relative" "$checksum" "$executable" >> "$run_dir/inventory.tsv"
done
IFS=$old_ifs
done < "$run_dir/packages.txt"
awk -v summary=coverage/summary.json -f scripts/coverage.awk "$run_dir/inventory.tsv" coverage/coverage.out
"$GO" version > coverage/go-version.txt
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
# Create lab credentials once, outside Helm values and release history.
set -eu
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
: "${REGISTRY:=asisti.registry.twcstorage.ru}" "${REGISTRY_USER:=token}"
: "${REGISTRY_TOKEN_FILE:=$HOME/.secrets/twc-registry-token}"
test -s "$REGISTRY_TOKEN_FILE" || { echo 'Registry token file is missing or empty' >&2; exit 1; }
umask 077
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
kubectl --context "$KUBE_CONTEXT" create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" apply -f -
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-secrets >/dev/null 2>&1; then
openssl rand -hex 32 | tr -d '\n' > "$work/postgres-password"
printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' "$(cat "$work/postgres-password")" > "$work/database-url"
openssl rand -base64 32 | tr -d '\n' > "$work/cursor-key"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-secrets --from-file="$work/postgres-password" --from-file="$work/database-url" --from-file="$work/cursor-key"
fi
# Add the shared cursor key to an existing lab2 secret without replacing database credentials.
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-secrets -o json | jq -e '.data["cursor-key"] | length > 0' >/dev/null; then
openssl rand -base64 32 | tr -d '\n' > "$work/cursor-key"
jq -n --rawfile key "$work/cursor-key" '{stringData:{"cursor-key":$key}}' > "$work/patch.json"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" patch secret messenger-secrets --type merge --patch-file "$work/patch.json"
fi
# Build a portable Docker config; macOS credential-helper references cannot be used by kubelet.
jq -n --arg registry "$REGISTRY" --arg user "$REGISTRY_USER" --rawfile token "$REGISTRY_TOKEN_FILE" \
'{auths:{($registry):{auth:(($user+":"+($token | sub("[\\r\\n]+$"; "")))|@base64)}}}' > "$work/config.json"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic registry-pull \
--type=kubernetes.io/dockerconfigjson --from-file=".dockerconfigjson=$work/config.json" \
--dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f -
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-s3 >/dev/null 2>&1; then
printf 'operator' > "$work/root-user"
printf 'messenger' > "$work/access-key"
openssl rand -hex 32 | tr -d '\n' > "$work/root-password"
openssl rand -hex 32 | tr -d '\n' > "$work/secret-key"
jq -n --rawfile admin "$work/root-user" --rawfile adminSecret "$work/root-password" --rawfile app "$work/access-key" --rawfile appSecret "$work/secret-key" '{identities:[{name:"operator",credentials:[{accessKey:$admin,secretKey:$adminSecret}],actions:["Admin","Read","Write","List"]},{name:"messenger",credentials:[{accessKey:$app,secretKey:$appSecret}],actions:["Read:attachments","Write:attachments","List:attachments"]}]}' > "$work/identities.json"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-s3 --from-file="$work/root-user" --from-file="$work/root-password" --from-file="$work/access-key" --from-file="$work/secret-key" --from-file="$work/identities.json"
fi
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-grafana >/dev/null 2>&1; then
printf 'admin' > "$work/username"
openssl rand -hex 24 | tr -d '\n' > "$work/password"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-grafana --from-file="$work/username" --from-file="$work/password"
fi
+59
View File
@@ -0,0 +1,59 @@
#!/bin/sh
# Operate only on the isolated processes and containers recorded by test-local.sh.
set -eu
exec 2>> evidence/local/actions-errors.txt
: "${LOCAL_TEST_WORK:?Run through scripts/test-local.sh}"
. "$LOCAL_TEST_WORK/environment"
stop_api() {
label=$1
pid=$(cat "$LOCAL_TEST_WORK/$label.pid")
if kill -0 "$pid" 2>/dev/null; then
state=$(ps -p "$pid" -o stat=)
case "$state" in Z*) return ;; esac
command=$(ps -p "$pid" -o command=)
case "$command" in "$LOCAL_TEST_WORK/server"|"$LOCAL_TEST_WORK/server worker") ;; *) echo 'API PID ownership mismatch' >&2; exit 1 ;; esac
kill -TERM "$pid"
for attempt in $(seq 1 60); do kill -0 "$pid" 2>/dev/null || break; sleep .1; done
fi
}
start_api() {
label=$1
port=$2
INSTANCE_ID="$label" APP_PORT="$port" nohup "$LOCAL_TEST_WORK/server" > "$LOCAL_TEST_WORK/$label.log" 2>&1 < /dev/null &
echo "$!" > "$LOCAL_TEST_WORK/$label.pid"
}
if [ "${LOCAL_ACTION:-restart}" = failover ]; then
case "${TARGET_INSTANCE_ID:?Missing observed replica}" in local-a|local-b) ;; *) echo 'Unknown isolated replica' >&2; exit 1 ;; esac
printf "%s target=%s action=stop\n" "$(date -u +%FT%TZ)" "$TARGET_INSTANCE_ID" >> evidence/local/actions.txt
stop_api "$TARGET_INSTANCE_ID"
exit 0
fi
if [ "${LOCAL_ACTION:-restart}" = pause-worker ]; then
stop_api worker
printf "%s action=stop-worker\n" "$(date -u +%FT%TZ)" >> evidence/local/actions.txt
exit 0
fi
stop_api local-a
stop_api local-b
stop_api worker
printf "%s action=restart-api-worker-postgres-valkey-s3-broker retained-container-filesystems\n" "$(date -u +%FT%TZ)" >> evidence/local/actions.txt
docker restart "$LOCAL_TEST_PG" "$LOCAL_TEST_VK" "$LOCAL_TEST_S3" "$LOCAL_TEST_BROKER" >/dev/null
# Docker may assign new host ports when restarting containers with ephemeral bindings.
export DATABASE_URL="postgres://postgres@$(docker port "$LOCAL_TEST_PG" 5432/tcp)/messenger?sslmode=disable"
export VALKEY_SHARDS="shard0=$(docker port "$LOCAL_TEST_VK" 6379/tcp)"
export S3_ENDPOINT="$(docker port "$LOCAL_TEST_S3" 9000/tcp)"
export BROKER_ADDRESS="$(docker port "$LOCAL_TEST_BROKER" 6379/tcp)"
for attempt in $(seq 1 60); do
if docker exec "$LOCAL_TEST_PG" pg_isready -U postgres >/dev/null 2>&1 && "$LOCAL_TEST_WORK/server" s3-bootstrap >/dev/null 2>&1; then break; fi
sleep 1
done
nohup "$LOCAL_TEST_WORK/server" worker > "$LOCAL_TEST_WORK/worker.log" 2>&1 < /dev/null &
echo "$!" > "$LOCAL_TEST_WORK/worker.pid"
start_api local-a "$LOCAL_TEST_PORT_A"
start_api local-b "$LOCAL_TEST_PORT_B"
for attempt in $(seq 1 60); do
if curl --fail --silent --max-time 1 "http://127.0.0.1:$LOCAL_TEST_PORT_A/health/ready" >/dev/null && curl --fail --silent --max-time 1 "http://127.0.0.1:$LOCAL_TEST_PORT_B/health/ready" >/dev/null; then exit 0; fi
sleep 1
done
echo 'Isolated restart readiness timed out' >&2
exit 1
+100
View File
@@ -0,0 +1,100 @@
#!/bin/sh
# Bounded fault experiments on the disposable test-local.sh stack only.
set -eu
: "${LOCAL_TEST_WORK:?}" "${BASE_URL:?}" "${PROMETHEUS_URL:?}" "${TELEMETRY_TEMPO:?}"
. "$LOCAL_TEST_WORK/environment"
export PROBE_WORK="$LOCAL_TEST_WORK/probe"
mkdir -p evidence/incidents
scripts/prepare-probe.sh
scripts/trace-probe.sh > evidence/current/correlation.txt 2>&1
worker=$(cat "$LOCAL_TEST_WORK/worker.pid")
api_a=$(cat "$LOCAL_TEST_WORK/local-a.pid")
api_b=$(cat "$LOCAL_TEST_WORK/local-b.pid")
chat=$(cat "$PROBE_WORK/chat-id")
load_pids=
paused_db=false
paused_worker=false
stopped_tempo=false
cleanup() {
for pid in $load_pids; do kill "$pid" 2>/dev/null || true; done
if $paused_db; then docker unpause "$LOCAL_TEST_PG" >/dev/null; fi
if $paused_worker; then kill -CONT "$worker"; fi
if $stopped_tempo; then docker start "$TELEMETRY_TEMPO" >/dev/null; fi
}
trap cleanup EXIT
printf 'baseline\n' > "$PROBE_WORK/phase"
for client in 1 2; do
(
while :; do
now=$(date +%s)
phase=$(cat "$PROBE_WORK/phase")
result=$(curl -sS --max-time 4 --config "$PROBE_WORK/auth.conf" -o /dev/null -w '%{http_code},%{time_total}' "$BASE_URL/api/v1/chats/$chat/messages" 2>/dev/null || true)
printf '%s,%s,%s,%s\n' "$now" "$phase" "$client" "$result"
sleep .25
done
) > "evidence/incidents/http-$client.csv" &
load_pids="$load_pids $!"
done
snapshot() {
phase=$1
now=$(date +%s)
curl -fsS "$PROMETHEUS_URL/api/v1/alerts" > "$PROBE_WORK/alerts.json"
jq -c --arg phase "$phase" --argjson at "$now" '{at:$at,phase:$phase,alerts:.data.alerts}' "$PROBE_WORK/alerts.json" >> evidence/incidents/alerts.jsonl
printf '%s %s\n' "$now" "$phase" >> evidence/incidents/rss.txt
ps -p "$api_a,$api_b,$worker" -o pid=,rss=,time= >> evidence/incidents/rss.txt
}
phase() {
name=$1
seconds=$2
printf '%s\n' "$name" > "$PROBE_WORK/phase"
printf '%s,%s,start\n' "$(date +%s)" "$name" >> evidence/incidents/events.csv
deadline=$(($(date +%s)+seconds))
while [ "$(date +%s)" -lt "$deadline" ]; do snapshot "$name"; sleep 5; done
printf '%s,%s,end\n' "$(date +%s)" "$name" >> evidence/incidents/events.csv
}
: > evidence/incidents/events.csv
: > evidence/incidents/alerts.jsonl
: > evidence/incidents/rss.txt
phase baseline 120
docker pause "$LOCAL_TEST_PG" >/dev/null
paused_db=true
phase short-db-spike 5
docker unpause "$LOCAL_TEST_PG" >/dev/null
paused_db=false
phase spike-recovery 90
docker pause "$LOCAL_TEST_PG" >/dev/null
paused_db=true
phase sustained-db-outage 180
docker unpause "$LOCAL_TEST_PG" >/dev/null
paused_db=false
phase db-recovery 90
kill -STOP "$worker"
paused_worker=true
printf 'worker-stopped\n' > "$PROBE_WORK/phase"
printf '%s,worker-stopped,start\n' "$(date +%s)" >> evidence/incidents/events.csv
for attempt in $(seq 1 20); do
status=$(curl -sS --config "$PROBE_WORK/auth.conf" -o "$PROBE_WORK/queued.json" -w '%{http_code}' "$BASE_URL/api/v1/chats/$chat/attachments?filename=probe.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png)
test "$status" = 202
jq -e '.status=="queued"' "$PROBE_WORK/queued.json" >/dev/null
sleep 1
done
phase worker-stopped 90
kill -CONT "$worker"
paused_worker=false
phase worker-recovery 90
docker stop --time 5 "$TELEMETRY_TEMPO" >/dev/null
stopped_tempo=true
phase trace-backend-outage 45
make test > evidence/incidents/smoke-with-tempo-down.txt 2>&1
docker start "$TELEMETRY_TEMPO" >/dev/null
stopped_tempo=false
phase telemetry-recovery 20
for query in 'max(messenger_queue_oldest_seconds)' 'max(messenger_queue_depth)' 'sum(rate(messenger_http_requests_total{status_class="5xx"}[1m]))' 'sum(otelcol_exporter_send_failed_spans_total)' 'sum(otelcol_exporter_queue_size)'; do
curl -fsS --get "$PROMETHEUS_URL/api/v1/query" --data-urlencode "query=$query" | jq -c --arg query "$query" '{query:$query,response:.}' >> evidence/incidents/final-metrics.jsonl
done
jq -s -e '[.[]|select(.phase=="sustained-db-outage" or .phase=="db-recovery")|.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null
jq -s -e '[.[]|select(.phase=="worker-stopped")|.alerts[]|select(.labels.alertname=="MessengerQueueLag" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null
jq -s -e '[.[]|select(.phase=="short-db-spike" or .phase=="spike-recovery")|.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" and .state=="firing")]|length==0' evidence/incidents/alerts.jsonl >/dev/null
jq -s -e '[.[]|select(.phase=="worker-recovery")|.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null
tail -n 1 evidence/incidents/alerts.jsonl | jq -e '[.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" or .labels.alertname=="MessengerQueueLag" or .labels.alertname=="MessengerReadyBurnRate")]|length==0' >/dev/null
printf 'PASS: API, queue and ready alerts resolved\n'
+21
View File
@@ -0,0 +1,21 @@
#!/bin/sh
# Reconnect the loopback tunnel after the API pod is replaced during restart tests.
set -eu
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
LOCAL_PORT=${LOCAL_PORT:-8083}
child=
cleanup() {
if [ -n "$child" ]; then
kill "$child" 2>/dev/null || true
wait "$child" 2>/dev/null || true
fi
exit 0
}
trap cleanup INT TERM
while :; do
kubectl --context "$KUBE_CONTEXT" -n traefik port-forward service/traefik "$LOCAL_PORT:80" --address 127.0.0.1 &
child=$!
wait "$child" || true
child=
sleep 1
done
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
# Create synthetic probe credentials in an explicit private directory; print no secrets.
set -eu
: "${BASE_URL:?}" "${PROBE_WORK:?}"
umask 077
mkdir -p "$PROBE_WORK"
username="observe_$(openssl rand -hex 6)"
password=$(openssl rand -hex 16)
jq -n --arg username "$username" --arg password "$password" '{username:$username,display_name:"Observability probe",password:$password}' > "$PROBE_WORK/user.json"
curl -fsS "$BASE_URL/api/v1/users" -H 'Content-Type: application/json' --data-binary @"$PROBE_WORK/user.json" > "$PROBE_WORK/user-response.json"
printf 'user = "%s:%s"\n' "$username" "$password" > "$PROBE_WORK/basic.conf"
curl -fsS --config "$PROBE_WORK/basic.conf" -X POST "$BASE_URL/api/v1/auth/sessions" > "$PROBE_WORK/session.json"
printf 'header = "Authorization: Bearer %s"\n' "$(jq -r .token "$PROBE_WORK/session.json")" > "$PROBE_WORK/auth.conf"
jq -n --arg id "$(jq -r .id "$PROBE_WORK/user-response.json")" '{title:"Observability probe",member_ids:[$id]}' > "$PROBE_WORK/chat.json"
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats" -H 'Content-Type: application/json' --data-binary @"$PROBE_WORK/chat.json" > "$PROBE_WORK/chat-response.json"
printf '%s\n' "$(jq -r .id "$PROBE_WORK/chat-response.json")" > "$PROBE_WORK/chat-id"
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Verify time-to-ready burn against a real suspended worker on an owned test stack.
set -eu
: "${LOCAL_TEST_WORK:?}" "${BASE_URL:?}" "${PROMETHEUS_URL:?}"
export PROBE_WORK="$LOCAL_TEST_WORK/ready-probe"
scripts/prepare-probe.sh
worker=$(cat "$LOCAL_TEST_WORK/worker.pid")
trap 'kill -CONT "$worker" 2>/dev/null || true' EXIT
mkdir -p evidence/ready-slo
: > evidence/ready-slo/alerts.jsonl
chat=$(cat "$PROBE_WORK/chat-id")
# Warm the histogram before suspension so rate() observes a real counter delta.
upload() {
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=slo.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json"
}
upload
sleep 15
kill -STOP "$worker"
printf '%s,worker-stopped\n' "$(date +%s)" > evidence/ready-slo/events.csv
for i in $(seq 1 20); do upload; done
observe() {
phase=$1
for i in $(seq 1 "$2"); do
curl -fsS "$PROMETHEUS_URL/api/v1/alerts" | jq -c --arg phase "$phase" --argjson at "$(date +%s)" '{at:$at,phase:$phase,alerts:.data.alerts}' >> evidence/ready-slo/alerts.jsonl
sleep 5
done
}
observe worker-stopped 18
kill -CONT "$worker"
printf '%s,worker-resumed\n' "$(date +%s)" >> evidence/ready-slo/events.csv
observe worker-recovery 24
jq -s -e '[.[]|.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" and .state=="firing")]|length>0' evidence/ready-slo/alerts.jsonl >/dev/null
tail -n 1 evidence/ready-slo/alerts.jsonl | jq -e '[.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" or .labels.alertname=="MessengerQueueLag")]|length==0' >/dev/null
curl -fsS --get "$PROMETHEUS_URL/api/v1/query" --data-urlencode 'query=max(messenger_queue_depth)' > evidence/ready-slo/final-backlog.json
printf 'PASS: ready burn and queue lag resolved after real worker recovery\n'
+27
View File
@@ -0,0 +1,27 @@
#!/bin/sh
# Restart this lab's API, worker and persistent stores without removing PVCs.
set -eu
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
if kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get hpa api >/dev/null 2>&1; then
echo 'Disable autoscaling in the lab release before this restart test, then restore it.' >&2
exit 1
fi
replicas=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get deployment api -o jsonpath='{.spec.replicas}')
worker_replicas=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get deployment worker -o jsonpath='{.spec.replicas}')
restore() {
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/api --replicas="$replicas" >/dev/null
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/worker --replicas="$worker_replicas" >/dev/null
}
trap restore EXIT
mkdir -p evidence
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l 'app in (api,worker,postgres,valkey,s3,broker)' -o custom-columns=NAME:.metadata.name,UID:.metadata.uid > evidence/restart-before.txt
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/api deployment/worker --replicas=0
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" wait --for=delete pod -l 'app in (api,worker)' --timeout=60s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout restart statefulset/postgres statefulset/valkey statefulset/s3 statefulset/broker
for name in postgres valkey s3 broker; do
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status "statefulset/$name" --timeout=120s
done
restore
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status deployment/api --timeout=120s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status deployment/worker --timeout=120s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l 'app in (api,worker,postgres,valkey,s3,broker)' -o custom-columns=NAME:.metadata.name,UID:.metadata.uid > evidence/restart-after.txt
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
# Terminate the exact observed API pod; its replacement receives a different instance ID.
set -eu
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" "${TARGET_INSTANCE_ID:?Expected API pod name}"
case "$TARGET_INSTANCE_ID" in api-*) ;; *) echo 'Target is not a lab3 API pod' >&2; exit 2 ;; esac
label=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pod "$TARGET_INSTANCE_ID" -o jsonpath='{.metadata.labels.app}')
test "$label" = api
mkdir -p evidence
printf '%s target=%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$TARGET_INSTANCE_ID" >> evidence/failover-targets.txt
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" delete pod "$TARGET_INSTANCE_ID" --wait=false
+69
View File
@@ -0,0 +1,69 @@
#!/bin/sh
# Isolated telemetry backends with explicit state ownership; never touches shared monitoring.
set -eu
cd "$(dirname "$0")/.."
: "${LOCAL_TELEMETRY_STATE:?Set an explicit private state file path}"
if [ "${1:-up}" = down ]; then
. "$LOCAL_TELEMETRY_STATE"
for id in $TELEMETRY_CONTAINERS; do docker rm -f "$id" >/dev/null; done
docker network rm "$TELEMETRY_NETWORK" >/dev/null
rm -rf "$TELEMETRY_WORK"
rm "$LOCAL_TELEMETRY_STATE"
exit 0
fi
test ! -e "$LOCAL_TELEMETRY_STATE" || { echo 'State file already exists; stop its owned stack first' >&2; exit 1; }
umask 077
work=$(mktemp -d)
network="messenger-observe-$(openssl rand -hex 6)"
docker network create "$network" >/dev/null
containers=
success=false
cleanup() {
if ! $success; then
for id in $containers; do docker rm -f "$id" >/dev/null; done
docker network rm "$network" >/dev/null
rm -rf "$work"
fi
}
trap cleanup EXIT
config="$PWD/charts/messenger/files/observability"
start() {
name=$1
image=$2
port=$3
shift 3
mkdir "$work/$name"
chmod 777 "$work/$name"
id=$(docker run -d --network "$network" --network-alias "messenger-$name" -p "127.0.0.1::$port" -v "$config:/config:ro" -v "$work/$name:/data" "$image" "$@")
containers="$containers $id"
}
start tempo grafana/tempo:2.8.2 3200 -config.file=/config/tempo.yaml
tempo=$id
start loki grafana/loki:3.5.5 3100 -config.file=/config/loki.yaml
loki=$id
start prometheus prom/prometheus:v3.5.0 9090 --config.file=/config/prometheus.yaml --storage.tsdb.path=/data --storage.tsdb.retention.time=24h
prometheus=$id
start collector otel/opentelemetry-collector-contrib:0.135.0 4318 --config=/config/collector.yaml
collector=$id
mkdir "$work/grafana"
chmod 777 "$work/grafana"
printf 'GF_SECURITY_ADMIN_USER=admin\nGF_SECURITY_ADMIN_PASSWORD=%s\nGF_PATHS_DATA=/data\nGF_ANALYTICS_REPORTING_ENABLED=false\n' "$(openssl rand -hex 24)" > "$work/grafana.env"
grafana=$(docker run -d --network "$network" --network-alias messenger-grafana -p 127.0.0.1::3000 --env-file "$work/grafana.env" -v "$work/grafana:/data" -v "$config/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro" -v "$config/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro" -v "$config/messenger.json:/dashboards/messenger.json:ro" grafana/grafana:12.1.1)
containers="$containers $grafana"
cat > "$LOCAL_TELEMETRY_STATE" <<STATE
export TELEMETRY_NETWORK='$network'
export TELEMETRY_WORK='$work'
export TELEMETRY_CONTAINERS='$containers'
export TELEMETRY_TEMPO='$tempo'
export TELEMETRY_LOKI='$loki'
export TELEMETRY_PROMETHEUS='$prometheus'
export TELEMETRY_COLLECTOR='$collector'
export TELEMETRY_GRAFANA='$grafana'
export OTEL_EXPORTER_OTLP_ENDPOINT='http://$(docker port "$collector" 4318/tcp)'
export PROMETHEUS_URL='http://$(docker port "$prometheus" 9090/tcp)'
export LOKI_URL='http://$(docker port "$loki" 3100/tcp)'
export TEMPO_URL='http://$(docker port "$tempo" 3200/tcp)'
export GRAFANA_URL='http://$(docker port "$grafana" 3000/tcp)'
STATE
success=true
printf 'Telemetry stack created; state: %s\n' "$LOCAL_TELEMETRY_STATE"
+39
View File
@@ -0,0 +1,39 @@
#!/bin/sh
# Demonstrate accepted upload survival with worker stopped and all dependencies restarted.
set -eu
: "${LOCAL_TEST_WORK:?Run through test-local.sh}" "${BASE_URL:?}"
umask 077
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
LOCAL_ACTION=pause-worker scripts/local-action.sh
username="durable_$(openssl rand -hex 6)"
password=$(openssl rand -hex 16)
jq -n --arg username "$username" --arg password "$password" '{username:$username,display_name:"Durability test",password:$password}' > "$work/user.json"
curl -fsS "$BASE_URL/api/v1/users" -H 'Content-Type: application/json' --data-binary @"$work/user.json" > "$work/user-response.json"
# Temporary curl config avoids placing credentials or bearer tokens in process arguments.
printf 'user = "%s:%s"\n' "$username" "$password" > "$work/basic.conf"
curl -fsS --config "$work/basic.conf" -X POST "$BASE_URL/api/v1/auth/sessions" > "$work/session.json"
printf 'header = "Authorization: Bearer %s"\n' "$(jq -r .token "$work/session.json")" > "$work/auth.conf"
jq -n --arg id "$(jq -r .id "$work/user-response.json")" '{title:"Durability",member_ids:[$id]}' > "$work/chat.json"
curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/chats" -H 'Content-Type: application/json' --data-binary @"$work/chat.json" > "$work/chat-response.json"
chat=$(jq -r .id "$work/chat-response.json")
status=$(curl -sS --config "$work/auth.conf" -o "$work/attachment.json" -w '%{http_code}' "$BASE_URL/api/v1/chats/$chat/attachments?filename=sample.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png)
test "$status" = 202
jq -e '.status=="queued"' "$work/attachment.json" >/dev/null
id=$(jq -r .id "$work/attachment.json")
jq -n --arg id "$id" '{text:"",attachment_ids:[$id]}' > "$work/message.json"
curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/chats/$chat/messages" -H 'Content-Type: application/json' -H 'Idempotency-Key: durable-upload' --data-binary @"$work/message.json" > /dev/null
status=$(curl -sS --config "$work/auth.conf" -o /dev/null -w '%{http_code}' "$BASE_URL/api/v1/attachments/$id/content")
test "$status" = 409
printf 'worker_stopped=true upload_status=202 snapshot=queued message_created=true content_status=409\n'
LOCAL_ACTION=restart scripts/local-action.sh
ready=false
for attempt in $(seq 1 60); do
curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/attachments/$id" > "$work/status.json"
if jq -e '.status=="ready" and (.variants|length)==3' "$work/status.json" >/dev/null; then ready=true; break; fi
sleep 1
done
$ready
curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/attachments/$id/content" > "$work/original.png"
cmp tests/fixtures/sample.png "$work/original.png"
printf 'PASS: accepted intent survived API, worker, PostgreSQL, S3 and AOF broker restart; ready=1 variants=3 original_unchanged=true\n'
+112
View File
@@ -0,0 +1,112 @@
#!/bin/sh
# Disposable integration harness; Helm remains the deployment source of truth.
set -eu
cd "$(dirname "$0")/.."
umask 077
export LOCAL_TEST_WORK=$(mktemp -d)
containers=
cleanup() {
for label in local-a local-b worker; do
if [ -f "$LOCAL_TEST_WORK/$label.pid" ]; then kill -TERM "$(cat "$LOCAL_TEST_WORK/$label.pid")" 2>/dev/null || true; fi
done
for container in $containers; do docker rm -f "$container" >/dev/null; done
for label in local-a local-b worker; do if [ -f "$LOCAL_TEST_WORK/$label.log" ]; then cp "$LOCAL_TEST_WORK/$label.log" "evidence/local/$label.txt"; fi; done
rm -rf "$LOCAL_TEST_WORK"
}
trap cleanup EXIT
trap 'exit 130' INT TERM
mkdir -p evidence/local
pg=$(docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=messenger -p 127.0.0.1::5432 postgres:17.6-alpine)
containers="$pg"
vk=$(docker run -d -p 127.0.0.1::6379 valkey/valkey:8.1.3-alpine)
containers="$containers $vk"
broker=$(docker run -d -p 127.0.0.1::6379 valkey/valkey:8.1.3-alpine valkey-server --appendonly yes --appendfsync always --maxmemory 128mb --maxmemory-policy noeviction)
containers="$containers $broker"
admin=$(openssl rand -hex 24)
app=$(openssl rand -hex 24)
jq -n --arg admin "$admin" --arg app "$app" '{identities:[{name:"operator",credentials:[{accessKey:"operator",secretKey:$admin}],actions:["Admin","Read","Write","List"]},{name:"messenger",credentials:[{accessKey:"messenger",secretKey:$app}],actions:["Read:attachments","Write:attachments","List:attachments"]}]}' > "$LOCAL_TEST_WORK/identities.json"
s3=$(docker run -d -p 127.0.0.1::9000 -v "$LOCAL_TEST_WORK/identities.json:/identity.json:ro" chrislusf/seaweedfs:3.85 server -dir=/data -s3 -s3.port=9000 -s3.config=/identity.json -master.volumeSizeLimitMB=100 -volume.max=5)
containers="$containers $s3"
port_a=$(ruby -rsocket -e 's=TCPServer.new("127.0.0.1",0);puts s.addr[1];s.close')
port_b=$(ruby -rsocket -e 's=TCPServer.new("127.0.0.1",0);puts s.addr[1];s.close')
cat > "$LOCAL_TEST_WORK/environment" <<ENV
export DATABASE_URL='postgres://postgres@$(docker port "$pg" 5432/tcp)/messenger?sslmode=disable'
export VALKEY_SHARDS='shard0=$(docker port "$vk" 6379/tcp)'
export S3_ENDPOINT='$(docker port "$s3" 9000/tcp)'
export S3_TLS=false
export S3_BUCKET=attachments
export S3_ACCESS_KEY=messenger
export S3_SECRET_KEY='$app'
export CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
export REQUEST_TIMEOUT_SECONDS=15
export LOG_LEVEL=error
export BROKER_ADDRESS='$(docker port "$broker" 6379/tcp)'
export LOCAL_TEST_BROKER='$broker'
export LOCAL_TEST_PG='$pg'
export LOCAL_TEST_VK='$vk'
export LOCAL_TEST_S3='$s3'
export LOCAL_TEST_PORT_A='$port_a'
export LOCAL_TEST_PORT_B='$port_b'
ENV
. "$LOCAL_TEST_WORK/environment"
GOTOOLCHAIN=go1.26.1 go build -o "$LOCAL_TEST_WORK/server" ./cmd/server
for attempt in $(seq 1 60); do
if S3_ACCESS_KEY=operator S3_SECRET_KEY="$admin" "$LOCAL_TEST_WORK/server" s3-bootstrap >/dev/null 2>&1; then break; fi
sleep 1
done
"$LOCAL_TEST_WORK/server" migrate
if [ "${LOCAL_COVERAGE:-false}" = true ]; then
test_vk=$(docker run -d -p 127.0.0.1::6379 valkey/valkey:8.1.3-alpine)
containers="$containers $test_vk"
export TEST_DATABASE_URL="$DATABASE_URL"
export TEST_VALKEY_ADDRESS=$(docker port "$test_vk" 6379/tcp)
export S3_ADMIN_SECRET="$admin"
GOTOOLCHAIN=go1.26.1 scripts/coverage.sh
exit 0
fi
for label in local-a local-b worker; do
port=$port_a
if [ "$label" = local-b ]; then port=$port_b; fi
if [ "$label" = worker ]; then
"$LOCAL_TEST_WORK/server" worker > "$LOCAL_TEST_WORK/$label.log" 2>&1 &
else
INSTANCE_ID="$label" APP_PORT="$port" "$LOCAL_TEST_WORK/server" > "$LOCAL_TEST_WORK/$label.log" 2>&1 &
fi
echo "$!" > "$LOCAL_TEST_WORK/$label.pid"
done
cat > "$LOCAL_TEST_WORK/nginx.conf" <<CONF
events {}
http {
upstream api {
server host.docker.internal:$port_a max_fails=1 fail_timeout=1s;
server host.docker.internal:$port_b max_fails=1 fail_timeout=1s;
}
server {
listen 8080;
client_max_body_size 12m;
location / {
proxy_pass http://api;
proxy_connect_timeout 1s;
proxy_read_timeout 15s;
proxy_next_upstream error timeout http_502 http_503;
}
}
}
CONF
lb=$(docker run -d --add-host=host.docker.internal:host-gateway -p 127.0.0.1::8080 -v "$LOCAL_TEST_WORK/nginx.conf:/etc/nginx/nginx.conf:ro" nginx:1.27-alpine)
containers="$containers $lb"
export BASE_URL="http://$(docker port "$lb" 8080/tcp)"
for attempt in $(seq 1 60); do
if curl --fail --silent --max-time 1 "$BASE_URL/health/ready" >/dev/null; then break; fi
sleep 1
done
make test > evidence/local/smoke.txt 2>&1
LOCAL_ACTION=failover make test-failover ACTION_SCRIPT="$PWD/scripts/local-action.sh" > evidence/local/failover.txt 2>&1
make test-persistence ACTION_SCRIPT="$PWD/scripts/local-action.sh" > evidence/local/persistence.txt 2>&1
make test > evidence/local/smoke-after.txt 2>&1
scripts/test-durable.sh > evidence/local/durable.txt 2>&1
if [ "${OBSERVABILITY_TESTS:-false}" = true ]; then scripts/observability-incidents.sh > evidence/incidents-summary.txt 2>&1; fi
if [ "${RETRY_TRACE_TEST:-false}" = true ]; then scripts/trace-retry.sh > evidence/current/retry-result.txt 2>&1; fi
GOTOOLCHAIN=go1.26.1 go version > evidence/local/go-version.txt
shasum -a 256 "$LOCAL_TEST_WORK/server" > evidence/local/binary.sha256
echo 'PASS: isolated smoke, attachment failover and persistence'
+29
View File
@@ -0,0 +1,29 @@
#!/bin/sh
# Follow one controlled upload through API, broker, worker and all telemetry stores.
set -eu
: "${BASE_URL:?}" "${PROBE_WORK:?}" "${TEMPO_URL:?}" "${LOKI_URL:?}"
mkdir -p evidence/current
trace=$(openssl rand -hex 16)
parent=$(openssl rand -hex 8)
chat=$(cat "$PROBE_WORK/chat-id")
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=sample.png" -H 'Content-Type: image/png' -H "traceparent: 00-$trace-$parent-01" --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json"
id=$(jq -r .id "$PROBE_WORK/attachment.json")
ready=false
for attempt in $(seq 1 60); do
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/attachments/$id" > "$PROBE_WORK/status.json"
if jq -e '.status=="ready"' "$PROBE_WORK/status.json" >/dev/null; then ready=true;break;fi
sleep 1
done
$ready
jq -n --arg trace "$trace" --arg attachment "$id" '{trace_id:$trace,attachment_id:$attachment,ready:true}' > evidence/current/correlation.json
# Batch exporters flush asynchronously; query boundedly until the worker span is indexed.
found=false
for attempt in $(seq 1 30); do
curl -fsS -H 'Accept: application/json' "$TEMPO_URL/api/traces/$trace" > evidence/current/correlated-trace.json || true
if jq -e '..|objects|select(.name?=="attachment.process")' evidence/current/correlated-trace.json >/dev/null 2>&1; then found=true;break;fi
sleep 1
done
$found
curl -fsS --get "$LOKI_URL/loki/api/v1/query_range" --data-urlencode "query={service_name=~\"messenger-api|messenger-worker\"} | trace_id=\"$trace\"" --data-urlencode 'limit=100' > evidence/current/correlated-logs.json
jq -e '.data.result|length>0' evidence/current/correlated-logs.json >/dev/null
printf 'PASS: upload and worker share trace %s; correlated structured logs found\n' "$trace"
+46
View File
@@ -0,0 +1,46 @@
#!/bin/sh
# Exercise a real retry on an explicitly owned disposable stack.
set -eu
: "${LOCAL_TEST_WORK:?}" "${LOCAL_TEST_S3:?}" "${BASE_URL:?}" "${TEMPO_URL:?}"
export PROBE_WORK="$LOCAL_TEST_WORK/retry-probe"
scripts/prepare-probe.sh
worker=$(cat "$LOCAL_TEST_WORK/worker.pid")
paused=false
stopped=false
cleanup() {
if $paused; then docker unpause "$LOCAL_TEST_S3" >/dev/null; fi
if $stopped; then kill -CONT "$worker"; fi
}
trap cleanup EXIT
kill -STOP "$worker"
stopped=true
trace=$(openssl rand -hex 16)
parent=$(openssl rand -hex 8)
chat=$(cat "$PROBE_WORK/chat-id")
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=retry.png" -H 'Content-Type: image/png' -H "traceparent: 00-$trace-$parent-01" --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json"
id=$(jq -r .id "$PROBE_WORK/attachment.json")
docker pause "$LOCAL_TEST_S3" >/dev/null
paused=true
kill -CONT "$worker"
stopped=false
# S3 reads have a 15-second deadline; leave one attempt enough time to fail.
sleep 20
docker unpause "$LOCAL_TEST_S3" >/dev/null
paused=false
ready=false
for attempt in $(seq 1 90); do
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/attachments/$id" > "$PROBE_WORK/status.json"
if jq -e '.status=="ready"' "$PROBE_WORK/status.json" >/dev/null; then ready=true; break; fi
sleep 1
done
$ready
found=false
for attempt in $(seq 1 30); do
curl -fsS -H 'Accept: application/json' "$TEMPO_URL/api/traces/$trace" > evidence/current/retry-trace.json || true
if jq -e '[..|objects|select(.name?=="attachment.process")]|length>=2' evidence/current/retry-trace.json >/dev/null 2>&1; then found=true; break; fi
sleep 1
done
$found
jq -e '[..|objects|select(.name?=="attachment.process" and (.status.code?==2 or .status.code?=="STATUS_CODE_ERROR"))]|length>0' evidence/current/retry-trace.json >/dev/null
jq -n --arg trace "$trace" '{trace_id:$trace,ready:true,failed_attempt_and_retry:true}' > evidence/current/retry-result.json
printf 'PASS: S3 outage caused a failed consumer attempt; retry reached ready in the same trace\n'