forked from mtusi2026/mtusi2026lab6
feat: add bounded observability and verified SLO incident alerts
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
require 'json'
|
||||
require 'csv'
|
||||
root=ARGV.fetch(0)
|
||||
events=CSV.read("#{root}/events.csv").map{|t,p,e| [t.to_i,p,e]}
|
||||
snapshots=File.readlines("#{root}/alerts.jsonl").map{|l|JSON.parse(l)}
|
||||
rows=Dir["#{root}/http-*.csv"].flat_map{|p|CSV.read(p)}
|
||||
summary={phases: {}, alerts: {}}
|
||||
rows.group_by{|r|r[1]}.each do |phase,rs|
|
||||
vals=rs.map{|r|r[4].to_f*1000}.sort
|
||||
start=events.find{|t,p,e|p==phase && e=='start'}&.first
|
||||
finish=events.reverse.find{|t,p,e|p==phase && e=='end'}&.first
|
||||
summary[:phases][phase]={requests:rs.size,errors:rs.count{|r|r[3]!='200'},p50_ms:vals[(vals.size*0.5).floor],p95_ms:vals[(vals.size*0.95).floor],p99_ms:vals[(vals.size*0.99).floor],duration_seconds: start && finish ? finish-start : nil}
|
||||
end
|
||||
%w[MessengerAPIBurnRate MessengerReadyBurnRate MessengerQueueLag MessengerTelemetryDrops].each do |name|
|
||||
firing=snapshots.select{|s|s['alerts'].any?{|a|a['labels']['alertname']==name && a['state']=='firing'}}
|
||||
first=firing.first
|
||||
resolved=first && snapshots.find{|s|s['at']>firing.last['at'] && !s['alerts'].any?{|a|a['labels']['alertname']==name}}
|
||||
summary[:alerts][name]={first_firing:first && first['at'],last_firing:firing.last && firing.last['at'],first_resolved:resolved && resolved['at'],firing_samples:firing.size}
|
||||
end
|
||||
File.write("#{root}/summary.json",JSON.pretty_generate(summary)+"\n")
|
||||
@@ -0,0 +1,35 @@
|
||||
# Aggregate covered statements once per merged block; keep files with no statements as N/A.
|
||||
FNR == NR {
|
||||
split($0, fields, "\t")
|
||||
paths[fields[1]] = fields[2]
|
||||
hashes[fields[1]] = fields[3]
|
||||
executable[fields[1]] = fields[4]
|
||||
next
|
||||
}
|
||||
/^mode:/ { next }
|
||||
{
|
||||
file = $1
|
||||
sub(/:[^:]*$/, "", file)
|
||||
if (!(file in paths)) {
|
||||
print "unknown coverage source: " file > "/dev/stderr"
|
||||
invalid = 1
|
||||
}
|
||||
total[file] += $2
|
||||
if ($3 > 0) covered[file] += $2
|
||||
}
|
||||
END {
|
||||
print "{\"threshold_per_file\":90,\"files\":[" > summary
|
||||
separator = ""
|
||||
for (file in paths) {
|
||||
percent = total[file] ? sprintf("%.8f", covered[file] * 100 / total[file]) : "null"
|
||||
printf "%s{\"file\":\"%s\",\"statements\":%d,\"covered\":%d,\"percent\":%s,\"sha256\":\"%s\"}", separator, paths[file], total[file], covered[file], percent, hashes[file] > summary
|
||||
separator = ",\n"
|
||||
printf "%8s %s\n", (total[file] ? sprintf("%.2f%%", covered[file] * 100 / total[file]) : "N/A"), paths[file]
|
||||
if (covered[file] * 100 < total[file] * 90 || (executable[file] && !total[file])) invalid = 1
|
||||
}
|
||||
printf "\n],\"passed\":%s}\n", (invalid ? "false" : "true") > summary
|
||||
if (invalid) {
|
||||
print "FAIL: per-file coverage must reach 90%" > "/dev/stderr"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/bin/sh
|
||||
# Measure unit and real-process coverage, enforcing 90% separately for each Go source file.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
GO=${GO:-go}
|
||||
: "${DATABASE_URL:?Use an isolated test database}" "${VALKEY_SHARDS:?Use test shards}"
|
||||
: "${TEST_DATABASE_URL:?Use an isolated test database}" "${TEST_VALKEY_ADDRESS:?Use a test shard}"
|
||||
export APP_PORT=${COVERAGE_PORT:-18080}
|
||||
export LOG_LEVEL=error
|
||||
export CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
|
||||
worker_pid=
|
||||
server_pid=
|
||||
watchdog_pid=
|
||||
mkdir -p coverage
|
||||
run_dir=$(mktemp -d "$PWD/coverage/run.XXXXXX")
|
||||
cleanup() {
|
||||
if [ -n "$worker_pid" ]; then kill "$worker_pid" 2>/dev/null || true; wait "$worker_pid" 2>/dev/null || true; fi
|
||||
if [ -n "$server_pid" ]; then kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true; fi
|
||||
if [ -n "$watchdog_pid" ]; then kill "$watchdog_pid" 2>/dev/null || true; wait "$watchdog_pid" 2>/dev/null || true; fi
|
||||
rm -rf "$run_dir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
mkdir "$run_dir/process"
|
||||
"$GO" build -race -cover -covermode=atomic -coverpkg=./... -o "$run_dir/server" ./cmd/server
|
||||
"$run_dir/server" migrate
|
||||
"$GO" test -count=1 -race -coverpkg=./... -coverprofile="$run_dir/unit.out" ./...
|
||||
export GOCOVERDIR="$run_dir/process"
|
||||
if "$run_dir/server" unknown >"$run_dir/server-invalid.log" 2>&1; then
|
||||
echo 'server accepted an unknown command' >&2
|
||||
exit 1
|
||||
else
|
||||
test "$?" -eq 1
|
||||
fi
|
||||
"$run_dir/server" worker >"$run_dir/worker.log" 2>&1 &
|
||||
worker_pid=$!
|
||||
"$run_dir/server" >"$run_dir/server.log" 2>&1 &
|
||||
server_pid=$!
|
||||
ready=false
|
||||
for attempt in $(seq 1 60); do
|
||||
kill -0 "$server_pid" 2>/dev/null || { cat "$run_dir/server.log"; exit 1; }
|
||||
if curl --fail --silent --max-time 1 "http://localhost:$APP_PORT/health/ready" >/dev/null; then ready=true; break; fi
|
||||
sleep .25
|
||||
done
|
||||
$ready || { cat "$run_dir/server.log"; exit 1; }
|
||||
"$run_dir/server" healthcheck
|
||||
"$run_dir/server" migrate
|
||||
"${PYTHON:-python3}" tests/smoke.py --base-url "http://localhost:$APP_PORT"
|
||||
kill -TERM "$server_pid"
|
||||
# Bound the real-process shutdown check; a forced kill makes wait fail.
|
||||
(sleep 12; kill -KILL "$server_pid" 2>/dev/null || true) &
|
||||
watchdog_pid=$!
|
||||
wait "$server_pid"
|
||||
server_pid=
|
||||
kill "$watchdog_pid" 2>/dev/null || true
|
||||
wait "$watchdog_pid" 2>/dev/null || true
|
||||
watchdog_pid=
|
||||
kill -TERM "$worker_pid"
|
||||
wait "$worker_pid"
|
||||
worker_pid=
|
||||
unset GOCOVERDIR
|
||||
"$GO" tool covdata textfmt -i="$run_dir/process" -o="$run_dir/process.out"
|
||||
awk '
|
||||
/^mode:/ { if ($0 != "mode: atomic") exit 1; next }
|
||||
{
|
||||
if (($1 in statements) && statements[$1] != $2) exit 1;
|
||||
statements[$1] = $2; hits[$1] += $3
|
||||
}
|
||||
END {
|
||||
print "mode: atomic";
|
||||
for (block in statements) print block, statements[block], hits[block]
|
||||
}' "$run_dir/unit.out" "$run_dir/process.out" > coverage/coverage.out
|
||||
"$GO" list -f '{{.ImportPath}} {{.Dir}} {{join .GoFiles ","}}' ./... > "$run_dir/packages.txt"
|
||||
: > "$run_dir/inventory.tsv"
|
||||
while read -r import_path directory names; do
|
||||
old_ifs=$IFS
|
||||
IFS=,
|
||||
for name in $names; do
|
||||
checksum=$(shasum -a 256 "$directory/$name")
|
||||
checksum=${checksum%% *}
|
||||
relative=${directory#"$PWD/"}/$name
|
||||
executable=$(awk '/^func[ (]/ {found=1} END {print found+0}' "$directory/$name")
|
||||
printf '%s\t%s\t%s\t%s
|
||||
' "$import_path/$name" "$relative" "$checksum" "$executable" >> "$run_dir/inventory.tsv"
|
||||
done
|
||||
IFS=$old_ifs
|
||||
done < "$run_dir/packages.txt"
|
||||
awk -v summary=coverage/summary.json -f scripts/coverage.awk "$run_dir/inventory.tsv" coverage/coverage.out
|
||||
"$GO" version > coverage/go-version.txt
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/sh
|
||||
# Create lab credentials once, outside Helm values and release history.
|
||||
set -eu
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
|
||||
: "${REGISTRY:=asisti.registry.twcstorage.ru}" "${REGISTRY_USER:=token}"
|
||||
: "${REGISTRY_TOKEN_FILE:=$HOME/.secrets/twc-registry-token}"
|
||||
test -s "$REGISTRY_TOKEN_FILE" || { echo 'Registry token file is missing or empty' >&2; exit 1; }
|
||||
umask 077
|
||||
work=$(mktemp -d)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
kubectl --context "$KUBE_CONTEXT" create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" apply -f -
|
||||
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-secrets >/dev/null 2>&1; then
|
||||
openssl rand -hex 32 | tr -d '\n' > "$work/postgres-password"
|
||||
printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' "$(cat "$work/postgres-password")" > "$work/database-url"
|
||||
openssl rand -base64 32 | tr -d '\n' > "$work/cursor-key"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-secrets --from-file="$work/postgres-password" --from-file="$work/database-url" --from-file="$work/cursor-key"
|
||||
fi
|
||||
# Add the shared cursor key to an existing lab2 secret without replacing database credentials.
|
||||
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-secrets -o json | jq -e '.data["cursor-key"] | length > 0' >/dev/null; then
|
||||
openssl rand -base64 32 | tr -d '\n' > "$work/cursor-key"
|
||||
jq -n --rawfile key "$work/cursor-key" '{stringData:{"cursor-key":$key}}' > "$work/patch.json"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" patch secret messenger-secrets --type merge --patch-file "$work/patch.json"
|
||||
fi
|
||||
# Build a portable Docker config; macOS credential-helper references cannot be used by kubelet.
|
||||
jq -n --arg registry "$REGISTRY" --arg user "$REGISTRY_USER" --rawfile token "$REGISTRY_TOKEN_FILE" \
|
||||
'{auths:{($registry):{auth:(($user+":"+($token | sub("[\\r\\n]+$"; "")))|@base64)}}}' > "$work/config.json"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic registry-pull \
|
||||
--type=kubernetes.io/dockerconfigjson --from-file=".dockerconfigjson=$work/config.json" \
|
||||
--dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f -
|
||||
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-s3 >/dev/null 2>&1; then
|
||||
printf 'operator' > "$work/root-user"
|
||||
printf 'messenger' > "$work/access-key"
|
||||
openssl rand -hex 32 | tr -d '\n' > "$work/root-password"
|
||||
openssl rand -hex 32 | tr -d '\n' > "$work/secret-key"
|
||||
jq -n --rawfile admin "$work/root-user" --rawfile adminSecret "$work/root-password" --rawfile app "$work/access-key" --rawfile appSecret "$work/secret-key" '{identities:[{name:"operator",credentials:[{accessKey:$admin,secretKey:$adminSecret}],actions:["Admin","Read","Write","List"]},{name:"messenger",credentials:[{accessKey:$app,secretKey:$appSecret}],actions:["Read:attachments","Write:attachments","List:attachments"]}]}' > "$work/identities.json"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-s3 --from-file="$work/root-user" --from-file="$work/root-password" --from-file="$work/access-key" --from-file="$work/secret-key" --from-file="$work/identities.json"
|
||||
fi
|
||||
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-grafana >/dev/null 2>&1; then
|
||||
printf 'admin' > "$work/username"
|
||||
openssl rand -hex 24 | tr -d '\n' > "$work/password"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-grafana --from-file="$work/username" --from-file="$work/password"
|
||||
fi
|
||||
Executable
+59
@@ -0,0 +1,59 @@
|
||||
#!/bin/sh
|
||||
# Operate only on the isolated processes and containers recorded by test-local.sh.
|
||||
set -eu
|
||||
exec 2>> evidence/local/actions-errors.txt
|
||||
: "${LOCAL_TEST_WORK:?Run through scripts/test-local.sh}"
|
||||
. "$LOCAL_TEST_WORK/environment"
|
||||
stop_api() {
|
||||
label=$1
|
||||
pid=$(cat "$LOCAL_TEST_WORK/$label.pid")
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
state=$(ps -p "$pid" -o stat=)
|
||||
case "$state" in Z*) return ;; esac
|
||||
command=$(ps -p "$pid" -o command=)
|
||||
case "$command" in "$LOCAL_TEST_WORK/server"|"$LOCAL_TEST_WORK/server worker") ;; *) echo 'API PID ownership mismatch' >&2; exit 1 ;; esac
|
||||
kill -TERM "$pid"
|
||||
for attempt in $(seq 1 60); do kill -0 "$pid" 2>/dev/null || break; sleep .1; done
|
||||
fi
|
||||
}
|
||||
start_api() {
|
||||
label=$1
|
||||
port=$2
|
||||
INSTANCE_ID="$label" APP_PORT="$port" nohup "$LOCAL_TEST_WORK/server" > "$LOCAL_TEST_WORK/$label.log" 2>&1 < /dev/null &
|
||||
echo "$!" > "$LOCAL_TEST_WORK/$label.pid"
|
||||
}
|
||||
if [ "${LOCAL_ACTION:-restart}" = failover ]; then
|
||||
case "${TARGET_INSTANCE_ID:?Missing observed replica}" in local-a|local-b) ;; *) echo 'Unknown isolated replica' >&2; exit 1 ;; esac
|
||||
printf "%s target=%s action=stop\n" "$(date -u +%FT%TZ)" "$TARGET_INSTANCE_ID" >> evidence/local/actions.txt
|
||||
stop_api "$TARGET_INSTANCE_ID"
|
||||
exit 0
|
||||
fi
|
||||
if [ "${LOCAL_ACTION:-restart}" = pause-worker ]; then
|
||||
stop_api worker
|
||||
printf "%s action=stop-worker\n" "$(date -u +%FT%TZ)" >> evidence/local/actions.txt
|
||||
exit 0
|
||||
fi
|
||||
stop_api local-a
|
||||
stop_api local-b
|
||||
stop_api worker
|
||||
printf "%s action=restart-api-worker-postgres-valkey-s3-broker retained-container-filesystems\n" "$(date -u +%FT%TZ)" >> evidence/local/actions.txt
|
||||
docker restart "$LOCAL_TEST_PG" "$LOCAL_TEST_VK" "$LOCAL_TEST_S3" "$LOCAL_TEST_BROKER" >/dev/null
|
||||
# Docker may assign new host ports when restarting containers with ephemeral bindings.
|
||||
export DATABASE_URL="postgres://postgres@$(docker port "$LOCAL_TEST_PG" 5432/tcp)/messenger?sslmode=disable"
|
||||
export VALKEY_SHARDS="shard0=$(docker port "$LOCAL_TEST_VK" 6379/tcp)"
|
||||
export S3_ENDPOINT="$(docker port "$LOCAL_TEST_S3" 9000/tcp)"
|
||||
export BROKER_ADDRESS="$(docker port "$LOCAL_TEST_BROKER" 6379/tcp)"
|
||||
for attempt in $(seq 1 60); do
|
||||
if docker exec "$LOCAL_TEST_PG" pg_isready -U postgres >/dev/null 2>&1 && "$LOCAL_TEST_WORK/server" s3-bootstrap >/dev/null 2>&1; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
nohup "$LOCAL_TEST_WORK/server" worker > "$LOCAL_TEST_WORK/worker.log" 2>&1 < /dev/null &
|
||||
echo "$!" > "$LOCAL_TEST_WORK/worker.pid"
|
||||
start_api local-a "$LOCAL_TEST_PORT_A"
|
||||
start_api local-b "$LOCAL_TEST_PORT_B"
|
||||
for attempt in $(seq 1 60); do
|
||||
if curl --fail --silent --max-time 1 "http://127.0.0.1:$LOCAL_TEST_PORT_A/health/ready" >/dev/null && curl --fail --silent --max-time 1 "http://127.0.0.1:$LOCAL_TEST_PORT_B/health/ready" >/dev/null; then exit 0; fi
|
||||
sleep 1
|
||||
done
|
||||
echo 'Isolated restart readiness timed out' >&2
|
||||
exit 1
|
||||
Executable
+100
@@ -0,0 +1,100 @@
|
||||
#!/bin/sh
|
||||
# Bounded fault experiments on the disposable test-local.sh stack only.
|
||||
set -eu
|
||||
: "${LOCAL_TEST_WORK:?}" "${BASE_URL:?}" "${PROMETHEUS_URL:?}" "${TELEMETRY_TEMPO:?}"
|
||||
. "$LOCAL_TEST_WORK/environment"
|
||||
export PROBE_WORK="$LOCAL_TEST_WORK/probe"
|
||||
mkdir -p evidence/incidents
|
||||
scripts/prepare-probe.sh
|
||||
scripts/trace-probe.sh > evidence/current/correlation.txt 2>&1
|
||||
worker=$(cat "$LOCAL_TEST_WORK/worker.pid")
|
||||
api_a=$(cat "$LOCAL_TEST_WORK/local-a.pid")
|
||||
api_b=$(cat "$LOCAL_TEST_WORK/local-b.pid")
|
||||
chat=$(cat "$PROBE_WORK/chat-id")
|
||||
load_pids=
|
||||
paused_db=false
|
||||
paused_worker=false
|
||||
stopped_tempo=false
|
||||
cleanup() {
|
||||
for pid in $load_pids; do kill "$pid" 2>/dev/null || true; done
|
||||
if $paused_db; then docker unpause "$LOCAL_TEST_PG" >/dev/null; fi
|
||||
if $paused_worker; then kill -CONT "$worker"; fi
|
||||
if $stopped_tempo; then docker start "$TELEMETRY_TEMPO" >/dev/null; fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
printf 'baseline\n' > "$PROBE_WORK/phase"
|
||||
for client in 1 2; do
|
||||
(
|
||||
while :; do
|
||||
now=$(date +%s)
|
||||
phase=$(cat "$PROBE_WORK/phase")
|
||||
result=$(curl -sS --max-time 4 --config "$PROBE_WORK/auth.conf" -o /dev/null -w '%{http_code},%{time_total}' "$BASE_URL/api/v1/chats/$chat/messages" 2>/dev/null || true)
|
||||
printf '%s,%s,%s,%s\n' "$now" "$phase" "$client" "$result"
|
||||
sleep .25
|
||||
done
|
||||
) > "evidence/incidents/http-$client.csv" &
|
||||
load_pids="$load_pids $!"
|
||||
done
|
||||
snapshot() {
|
||||
phase=$1
|
||||
now=$(date +%s)
|
||||
curl -fsS "$PROMETHEUS_URL/api/v1/alerts" > "$PROBE_WORK/alerts.json"
|
||||
jq -c --arg phase "$phase" --argjson at "$now" '{at:$at,phase:$phase,alerts:.data.alerts}' "$PROBE_WORK/alerts.json" >> evidence/incidents/alerts.jsonl
|
||||
printf '%s %s\n' "$now" "$phase" >> evidence/incidents/rss.txt
|
||||
ps -p "$api_a,$api_b,$worker" -o pid=,rss=,time= >> evidence/incidents/rss.txt
|
||||
}
|
||||
phase() {
|
||||
name=$1
|
||||
seconds=$2
|
||||
printf '%s\n' "$name" > "$PROBE_WORK/phase"
|
||||
printf '%s,%s,start\n' "$(date +%s)" "$name" >> evidence/incidents/events.csv
|
||||
deadline=$(($(date +%s)+seconds))
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do snapshot "$name"; sleep 5; done
|
||||
printf '%s,%s,end\n' "$(date +%s)" "$name" >> evidence/incidents/events.csv
|
||||
}
|
||||
: > evidence/incidents/events.csv
|
||||
: > evidence/incidents/alerts.jsonl
|
||||
: > evidence/incidents/rss.txt
|
||||
phase baseline 120
|
||||
docker pause "$LOCAL_TEST_PG" >/dev/null
|
||||
paused_db=true
|
||||
phase short-db-spike 5
|
||||
docker unpause "$LOCAL_TEST_PG" >/dev/null
|
||||
paused_db=false
|
||||
phase spike-recovery 90
|
||||
docker pause "$LOCAL_TEST_PG" >/dev/null
|
||||
paused_db=true
|
||||
phase sustained-db-outage 180
|
||||
docker unpause "$LOCAL_TEST_PG" >/dev/null
|
||||
paused_db=false
|
||||
phase db-recovery 90
|
||||
kill -STOP "$worker"
|
||||
paused_worker=true
|
||||
printf 'worker-stopped\n' > "$PROBE_WORK/phase"
|
||||
printf '%s,worker-stopped,start\n' "$(date +%s)" >> evidence/incidents/events.csv
|
||||
for attempt in $(seq 1 20); do
|
||||
status=$(curl -sS --config "$PROBE_WORK/auth.conf" -o "$PROBE_WORK/queued.json" -w '%{http_code}' "$BASE_URL/api/v1/chats/$chat/attachments?filename=probe.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png)
|
||||
test "$status" = 202
|
||||
jq -e '.status=="queued"' "$PROBE_WORK/queued.json" >/dev/null
|
||||
sleep 1
|
||||
done
|
||||
phase worker-stopped 90
|
||||
kill -CONT "$worker"
|
||||
paused_worker=false
|
||||
phase worker-recovery 90
|
||||
docker stop --time 5 "$TELEMETRY_TEMPO" >/dev/null
|
||||
stopped_tempo=true
|
||||
phase trace-backend-outage 45
|
||||
make test > evidence/incidents/smoke-with-tempo-down.txt 2>&1
|
||||
docker start "$TELEMETRY_TEMPO" >/dev/null
|
||||
stopped_tempo=false
|
||||
phase telemetry-recovery 20
|
||||
for query in 'max(messenger_queue_oldest_seconds)' 'max(messenger_queue_depth)' 'sum(rate(messenger_http_requests_total{status_class="5xx"}[1m]))' 'sum(otelcol_exporter_send_failed_spans_total)' 'sum(otelcol_exporter_queue_size)'; do
|
||||
curl -fsS --get "$PROMETHEUS_URL/api/v1/query" --data-urlencode "query=$query" | jq -c --arg query "$query" '{query:$query,response:.}' >> evidence/incidents/final-metrics.jsonl
|
||||
done
|
||||
jq -s -e '[.[]|select(.phase=="sustained-db-outage" or .phase=="db-recovery")|.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null
|
||||
jq -s -e '[.[]|select(.phase=="worker-stopped")|.alerts[]|select(.labels.alertname=="MessengerQueueLag" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null
|
||||
jq -s -e '[.[]|select(.phase=="short-db-spike" or .phase=="spike-recovery")|.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" and .state=="firing")]|length==0' evidence/incidents/alerts.jsonl >/dev/null
|
||||
jq -s -e '[.[]|select(.phase=="worker-recovery")|.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" and .state=="firing")]|length>0' evidence/incidents/alerts.jsonl >/dev/null
|
||||
tail -n 1 evidence/incidents/alerts.jsonl | jq -e '[.alerts[]|select(.labels.alertname=="MessengerAPIBurnRate" or .labels.alertname=="MessengerQueueLag" or .labels.alertname=="MessengerReadyBurnRate")]|length==0' >/dev/null
|
||||
printf 'PASS: API, queue and ready alerts resolved\n'
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# Reconnect the loopback tunnel after the API pod is replaced during restart tests.
|
||||
set -eu
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
|
||||
LOCAL_PORT=${LOCAL_PORT:-8083}
|
||||
child=
|
||||
cleanup() {
|
||||
if [ -n "$child" ]; then
|
||||
kill "$child" 2>/dev/null || true
|
||||
wait "$child" 2>/dev/null || true
|
||||
fi
|
||||
exit 0
|
||||
}
|
||||
trap cleanup INT TERM
|
||||
while :; do
|
||||
kubectl --context "$KUBE_CONTEXT" -n traefik port-forward service/traefik "$LOCAL_PORT:80" --address 127.0.0.1 &
|
||||
child=$!
|
||||
wait "$child" || true
|
||||
child=
|
||||
sleep 1
|
||||
done
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
# Create synthetic probe credentials in an explicit private directory; print no secrets.
|
||||
set -eu
|
||||
: "${BASE_URL:?}" "${PROBE_WORK:?}"
|
||||
umask 077
|
||||
mkdir -p "$PROBE_WORK"
|
||||
username="observe_$(openssl rand -hex 6)"
|
||||
password=$(openssl rand -hex 16)
|
||||
jq -n --arg username "$username" --arg password "$password" '{username:$username,display_name:"Observability probe",password:$password}' > "$PROBE_WORK/user.json"
|
||||
curl -fsS "$BASE_URL/api/v1/users" -H 'Content-Type: application/json' --data-binary @"$PROBE_WORK/user.json" > "$PROBE_WORK/user-response.json"
|
||||
printf 'user = "%s:%s"\n' "$username" "$password" > "$PROBE_WORK/basic.conf"
|
||||
curl -fsS --config "$PROBE_WORK/basic.conf" -X POST "$BASE_URL/api/v1/auth/sessions" > "$PROBE_WORK/session.json"
|
||||
printf 'header = "Authorization: Bearer %s"\n' "$(jq -r .token "$PROBE_WORK/session.json")" > "$PROBE_WORK/auth.conf"
|
||||
jq -n --arg id "$(jq -r .id "$PROBE_WORK/user-response.json")" '{title:"Observability probe",member_ids:[$id]}' > "$PROBE_WORK/chat.json"
|
||||
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats" -H 'Content-Type: application/json' --data-binary @"$PROBE_WORK/chat.json" > "$PROBE_WORK/chat-response.json"
|
||||
printf '%s\n' "$(jq -r .id "$PROBE_WORK/chat-response.json")" > "$PROBE_WORK/chat-id"
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
# Verify time-to-ready burn against a real suspended worker on an owned test stack.
|
||||
set -eu
|
||||
: "${LOCAL_TEST_WORK:?}" "${BASE_URL:?}" "${PROMETHEUS_URL:?}"
|
||||
export PROBE_WORK="$LOCAL_TEST_WORK/ready-probe"
|
||||
scripts/prepare-probe.sh
|
||||
worker=$(cat "$LOCAL_TEST_WORK/worker.pid")
|
||||
trap 'kill -CONT "$worker" 2>/dev/null || true' EXIT
|
||||
mkdir -p evidence/ready-slo
|
||||
: > evidence/ready-slo/alerts.jsonl
|
||||
chat=$(cat "$PROBE_WORK/chat-id")
|
||||
# Warm the histogram before suspension so rate() observes a real counter delta.
|
||||
upload() {
|
||||
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=slo.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json"
|
||||
}
|
||||
upload
|
||||
sleep 15
|
||||
kill -STOP "$worker"
|
||||
printf '%s,worker-stopped\n' "$(date +%s)" > evidence/ready-slo/events.csv
|
||||
for i in $(seq 1 20); do upload; done
|
||||
observe() {
|
||||
phase=$1
|
||||
for i in $(seq 1 "$2"); do
|
||||
curl -fsS "$PROMETHEUS_URL/api/v1/alerts" | jq -c --arg phase "$phase" --argjson at "$(date +%s)" '{at:$at,phase:$phase,alerts:.data.alerts}' >> evidence/ready-slo/alerts.jsonl
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
observe worker-stopped 18
|
||||
kill -CONT "$worker"
|
||||
printf '%s,worker-resumed\n' "$(date +%s)" >> evidence/ready-slo/events.csv
|
||||
observe worker-recovery 24
|
||||
jq -s -e '[.[]|.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" and .state=="firing")]|length>0' evidence/ready-slo/alerts.jsonl >/dev/null
|
||||
tail -n 1 evidence/ready-slo/alerts.jsonl | jq -e '[.alerts[]|select(.labels.alertname=="MessengerReadyBurnRate" or .labels.alertname=="MessengerQueueLag")]|length==0' >/dev/null
|
||||
curl -fsS --get "$PROMETHEUS_URL/api/v1/query" --data-urlencode 'query=max(messenger_queue_depth)' > evidence/ready-slo/final-backlog.json
|
||||
printf 'PASS: ready burn and queue lag resolved after real worker recovery\n'
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/sh
|
||||
# Restart this lab's API, worker and persistent stores without removing PVCs.
|
||||
set -eu
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
|
||||
if kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get hpa api >/dev/null 2>&1; then
|
||||
echo 'Disable autoscaling in the lab release before this restart test, then restore it.' >&2
|
||||
exit 1
|
||||
fi
|
||||
replicas=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get deployment api -o jsonpath='{.spec.replicas}')
|
||||
worker_replicas=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get deployment worker -o jsonpath='{.spec.replicas}')
|
||||
restore() {
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/api --replicas="$replicas" >/dev/null
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/worker --replicas="$worker_replicas" >/dev/null
|
||||
}
|
||||
trap restore EXIT
|
||||
mkdir -p evidence
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l 'app in (api,worker,postgres,valkey,s3,broker)' -o custom-columns=NAME:.metadata.name,UID:.metadata.uid > evidence/restart-before.txt
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/api deployment/worker --replicas=0
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" wait --for=delete pod -l 'app in (api,worker)' --timeout=60s
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout restart statefulset/postgres statefulset/valkey statefulset/s3 statefulset/broker
|
||||
for name in postgres valkey s3 broker; do
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status "statefulset/$name" --timeout=120s
|
||||
done
|
||||
restore
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status deployment/api --timeout=120s
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status deployment/worker --timeout=120s
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l 'app in (api,worker,postgres,valkey,s3,broker)' -o custom-columns=NAME:.metadata.name,UID:.metadata.uid > evidence/restart-after.txt
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Terminate the exact observed API pod; its replacement receives a different instance ID.
|
||||
set -eu
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" "${TARGET_INSTANCE_ID:?Expected API pod name}"
|
||||
case "$TARGET_INSTANCE_ID" in api-*) ;; *) echo 'Target is not a lab3 API pod' >&2; exit 2 ;; esac
|
||||
label=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pod "$TARGET_INSTANCE_ID" -o jsonpath='{.metadata.labels.app}')
|
||||
test "$label" = api
|
||||
mkdir -p evidence
|
||||
printf '%s target=%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$TARGET_INSTANCE_ID" >> evidence/failover-targets.txt
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" delete pod "$TARGET_INSTANCE_ID" --wait=false
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/bin/sh
|
||||
# Isolated telemetry backends with explicit state ownership; never touches shared monitoring.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${LOCAL_TELEMETRY_STATE:?Set an explicit private state file path}"
|
||||
if [ "${1:-up}" = down ]; then
|
||||
. "$LOCAL_TELEMETRY_STATE"
|
||||
for id in $TELEMETRY_CONTAINERS; do docker rm -f "$id" >/dev/null; done
|
||||
docker network rm "$TELEMETRY_NETWORK" >/dev/null
|
||||
rm -rf "$TELEMETRY_WORK"
|
||||
rm "$LOCAL_TELEMETRY_STATE"
|
||||
exit 0
|
||||
fi
|
||||
test ! -e "$LOCAL_TELEMETRY_STATE" || { echo 'State file already exists; stop its owned stack first' >&2; exit 1; }
|
||||
umask 077
|
||||
work=$(mktemp -d)
|
||||
network="messenger-observe-$(openssl rand -hex 6)"
|
||||
docker network create "$network" >/dev/null
|
||||
containers=
|
||||
success=false
|
||||
cleanup() {
|
||||
if ! $success; then
|
||||
for id in $containers; do docker rm -f "$id" >/dev/null; done
|
||||
docker network rm "$network" >/dev/null
|
||||
rm -rf "$work"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
config="$PWD/charts/messenger/files/observability"
|
||||
start() {
|
||||
name=$1
|
||||
image=$2
|
||||
port=$3
|
||||
shift 3
|
||||
mkdir "$work/$name"
|
||||
chmod 777 "$work/$name"
|
||||
id=$(docker run -d --network "$network" --network-alias "messenger-$name" -p "127.0.0.1::$port" -v "$config:/config:ro" -v "$work/$name:/data" "$image" "$@")
|
||||
containers="$containers $id"
|
||||
}
|
||||
start tempo grafana/tempo:2.8.2 3200 -config.file=/config/tempo.yaml
|
||||
tempo=$id
|
||||
start loki grafana/loki:3.5.5 3100 -config.file=/config/loki.yaml
|
||||
loki=$id
|
||||
start prometheus prom/prometheus:v3.5.0 9090 --config.file=/config/prometheus.yaml --storage.tsdb.path=/data --storage.tsdb.retention.time=24h
|
||||
prometheus=$id
|
||||
start collector otel/opentelemetry-collector-contrib:0.135.0 4318 --config=/config/collector.yaml
|
||||
collector=$id
|
||||
mkdir "$work/grafana"
|
||||
chmod 777 "$work/grafana"
|
||||
printf 'GF_SECURITY_ADMIN_USER=admin\nGF_SECURITY_ADMIN_PASSWORD=%s\nGF_PATHS_DATA=/data\nGF_ANALYTICS_REPORTING_ENABLED=false\n' "$(openssl rand -hex 24)" > "$work/grafana.env"
|
||||
grafana=$(docker run -d --network "$network" --network-alias messenger-grafana -p 127.0.0.1::3000 --env-file "$work/grafana.env" -v "$work/grafana:/data" -v "$config/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro" -v "$config/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro" -v "$config/messenger.json:/dashboards/messenger.json:ro" grafana/grafana:12.1.1)
|
||||
containers="$containers $grafana"
|
||||
cat > "$LOCAL_TELEMETRY_STATE" <<STATE
|
||||
export TELEMETRY_NETWORK='$network'
|
||||
export TELEMETRY_WORK='$work'
|
||||
export TELEMETRY_CONTAINERS='$containers'
|
||||
export TELEMETRY_TEMPO='$tempo'
|
||||
export TELEMETRY_LOKI='$loki'
|
||||
export TELEMETRY_PROMETHEUS='$prometheus'
|
||||
export TELEMETRY_COLLECTOR='$collector'
|
||||
export TELEMETRY_GRAFANA='$grafana'
|
||||
export OTEL_EXPORTER_OTLP_ENDPOINT='http://$(docker port "$collector" 4318/tcp)'
|
||||
export PROMETHEUS_URL='http://$(docker port "$prometheus" 9090/tcp)'
|
||||
export LOKI_URL='http://$(docker port "$loki" 3100/tcp)'
|
||||
export TEMPO_URL='http://$(docker port "$tempo" 3200/tcp)'
|
||||
export GRAFANA_URL='http://$(docker port "$grafana" 3000/tcp)'
|
||||
STATE
|
||||
success=true
|
||||
printf 'Telemetry stack created; state: %s\n' "$LOCAL_TELEMETRY_STATE"
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/bin/sh
|
||||
# Demonstrate accepted upload survival with worker stopped and all dependencies restarted.
|
||||
set -eu
|
||||
: "${LOCAL_TEST_WORK:?Run through test-local.sh}" "${BASE_URL:?}"
|
||||
umask 077
|
||||
work=$(mktemp -d)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
LOCAL_ACTION=pause-worker scripts/local-action.sh
|
||||
username="durable_$(openssl rand -hex 6)"
|
||||
password=$(openssl rand -hex 16)
|
||||
jq -n --arg username "$username" --arg password "$password" '{username:$username,display_name:"Durability test",password:$password}' > "$work/user.json"
|
||||
curl -fsS "$BASE_URL/api/v1/users" -H 'Content-Type: application/json' --data-binary @"$work/user.json" > "$work/user-response.json"
|
||||
# Temporary curl config avoids placing credentials or bearer tokens in process arguments.
|
||||
printf 'user = "%s:%s"\n' "$username" "$password" > "$work/basic.conf"
|
||||
curl -fsS --config "$work/basic.conf" -X POST "$BASE_URL/api/v1/auth/sessions" > "$work/session.json"
|
||||
printf 'header = "Authorization: Bearer %s"\n' "$(jq -r .token "$work/session.json")" > "$work/auth.conf"
|
||||
jq -n --arg id "$(jq -r .id "$work/user-response.json")" '{title:"Durability",member_ids:[$id]}' > "$work/chat.json"
|
||||
curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/chats" -H 'Content-Type: application/json' --data-binary @"$work/chat.json" > "$work/chat-response.json"
|
||||
chat=$(jq -r .id "$work/chat-response.json")
|
||||
status=$(curl -sS --config "$work/auth.conf" -o "$work/attachment.json" -w '%{http_code}' "$BASE_URL/api/v1/chats/$chat/attachments?filename=sample.png" -H 'Content-Type: image/png' --data-binary @tests/fixtures/sample.png)
|
||||
test "$status" = 202
|
||||
jq -e '.status=="queued"' "$work/attachment.json" >/dev/null
|
||||
id=$(jq -r .id "$work/attachment.json")
|
||||
jq -n --arg id "$id" '{text:"",attachment_ids:[$id]}' > "$work/message.json"
|
||||
curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/chats/$chat/messages" -H 'Content-Type: application/json' -H 'Idempotency-Key: durable-upload' --data-binary @"$work/message.json" > /dev/null
|
||||
status=$(curl -sS --config "$work/auth.conf" -o /dev/null -w '%{http_code}' "$BASE_URL/api/v1/attachments/$id/content")
|
||||
test "$status" = 409
|
||||
printf 'worker_stopped=true upload_status=202 snapshot=queued message_created=true content_status=409\n'
|
||||
LOCAL_ACTION=restart scripts/local-action.sh
|
||||
ready=false
|
||||
for attempt in $(seq 1 60); do
|
||||
curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/attachments/$id" > "$work/status.json"
|
||||
if jq -e '.status=="ready" and (.variants|length)==3' "$work/status.json" >/dev/null; then ready=true; break; fi
|
||||
sleep 1
|
||||
done
|
||||
$ready
|
||||
curl -fsS --config "$work/auth.conf" "$BASE_URL/api/v1/attachments/$id/content" > "$work/original.png"
|
||||
cmp tests/fixtures/sample.png "$work/original.png"
|
||||
printf 'PASS: accepted intent survived API, worker, PostgreSQL, S3 and AOF broker restart; ready=1 variants=3 original_unchanged=true\n'
|
||||
Executable
+112
@@ -0,0 +1,112 @@
|
||||
#!/bin/sh
|
||||
# Disposable integration harness; Helm remains the deployment source of truth.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
export LOCAL_TEST_WORK=$(mktemp -d)
|
||||
containers=
|
||||
cleanup() {
|
||||
for label in local-a local-b worker; do
|
||||
if [ -f "$LOCAL_TEST_WORK/$label.pid" ]; then kill -TERM "$(cat "$LOCAL_TEST_WORK/$label.pid")" 2>/dev/null || true; fi
|
||||
done
|
||||
for container in $containers; do docker rm -f "$container" >/dev/null; done
|
||||
for label in local-a local-b worker; do if [ -f "$LOCAL_TEST_WORK/$label.log" ]; then cp "$LOCAL_TEST_WORK/$label.log" "evidence/local/$label.txt"; fi; done
|
||||
rm -rf "$LOCAL_TEST_WORK"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT TERM
|
||||
mkdir -p evidence/local
|
||||
pg=$(docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_DB=messenger -p 127.0.0.1::5432 postgres:17.6-alpine)
|
||||
containers="$pg"
|
||||
vk=$(docker run -d -p 127.0.0.1::6379 valkey/valkey:8.1.3-alpine)
|
||||
containers="$containers $vk"
|
||||
broker=$(docker run -d -p 127.0.0.1::6379 valkey/valkey:8.1.3-alpine valkey-server --appendonly yes --appendfsync always --maxmemory 128mb --maxmemory-policy noeviction)
|
||||
containers="$containers $broker"
|
||||
admin=$(openssl rand -hex 24)
|
||||
app=$(openssl rand -hex 24)
|
||||
jq -n --arg admin "$admin" --arg app "$app" '{identities:[{name:"operator",credentials:[{accessKey:"operator",secretKey:$admin}],actions:["Admin","Read","Write","List"]},{name:"messenger",credentials:[{accessKey:"messenger",secretKey:$app}],actions:["Read:attachments","Write:attachments","List:attachments"]}]}' > "$LOCAL_TEST_WORK/identities.json"
|
||||
s3=$(docker run -d -p 127.0.0.1::9000 -v "$LOCAL_TEST_WORK/identities.json:/identity.json:ro" chrislusf/seaweedfs:3.85 server -dir=/data -s3 -s3.port=9000 -s3.config=/identity.json -master.volumeSizeLimitMB=100 -volume.max=5)
|
||||
containers="$containers $s3"
|
||||
port_a=$(ruby -rsocket -e 's=TCPServer.new("127.0.0.1",0);puts s.addr[1];s.close')
|
||||
port_b=$(ruby -rsocket -e 's=TCPServer.new("127.0.0.1",0);puts s.addr[1];s.close')
|
||||
cat > "$LOCAL_TEST_WORK/environment" <<ENV
|
||||
export DATABASE_URL='postgres://postgres@$(docker port "$pg" 5432/tcp)/messenger?sslmode=disable'
|
||||
export VALKEY_SHARDS='shard0=$(docker port "$vk" 6379/tcp)'
|
||||
export S3_ENDPOINT='$(docker port "$s3" 9000/tcp)'
|
||||
export S3_TLS=false
|
||||
export S3_BUCKET=attachments
|
||||
export S3_ACCESS_KEY=messenger
|
||||
export S3_SECRET_KEY='$app'
|
||||
export CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
|
||||
export REQUEST_TIMEOUT_SECONDS=15
|
||||
export LOG_LEVEL=error
|
||||
export BROKER_ADDRESS='$(docker port "$broker" 6379/tcp)'
|
||||
export LOCAL_TEST_BROKER='$broker'
|
||||
export LOCAL_TEST_PG='$pg'
|
||||
export LOCAL_TEST_VK='$vk'
|
||||
export LOCAL_TEST_S3='$s3'
|
||||
export LOCAL_TEST_PORT_A='$port_a'
|
||||
export LOCAL_TEST_PORT_B='$port_b'
|
||||
ENV
|
||||
. "$LOCAL_TEST_WORK/environment"
|
||||
GOTOOLCHAIN=go1.26.1 go build -o "$LOCAL_TEST_WORK/server" ./cmd/server
|
||||
for attempt in $(seq 1 60); do
|
||||
if S3_ACCESS_KEY=operator S3_SECRET_KEY="$admin" "$LOCAL_TEST_WORK/server" s3-bootstrap >/dev/null 2>&1; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
"$LOCAL_TEST_WORK/server" migrate
|
||||
if [ "${LOCAL_COVERAGE:-false}" = true ]; then
|
||||
test_vk=$(docker run -d -p 127.0.0.1::6379 valkey/valkey:8.1.3-alpine)
|
||||
containers="$containers $test_vk"
|
||||
export TEST_DATABASE_URL="$DATABASE_URL"
|
||||
export TEST_VALKEY_ADDRESS=$(docker port "$test_vk" 6379/tcp)
|
||||
export S3_ADMIN_SECRET="$admin"
|
||||
GOTOOLCHAIN=go1.26.1 scripts/coverage.sh
|
||||
exit 0
|
||||
fi
|
||||
for label in local-a local-b worker; do
|
||||
port=$port_a
|
||||
if [ "$label" = local-b ]; then port=$port_b; fi
|
||||
if [ "$label" = worker ]; then
|
||||
"$LOCAL_TEST_WORK/server" worker > "$LOCAL_TEST_WORK/$label.log" 2>&1 &
|
||||
else
|
||||
INSTANCE_ID="$label" APP_PORT="$port" "$LOCAL_TEST_WORK/server" > "$LOCAL_TEST_WORK/$label.log" 2>&1 &
|
||||
fi
|
||||
echo "$!" > "$LOCAL_TEST_WORK/$label.pid"
|
||||
done
|
||||
cat > "$LOCAL_TEST_WORK/nginx.conf" <<CONF
|
||||
events {}
|
||||
http {
|
||||
upstream api {
|
||||
server host.docker.internal:$port_a max_fails=1 fail_timeout=1s;
|
||||
server host.docker.internal:$port_b max_fails=1 fail_timeout=1s;
|
||||
}
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 12m;
|
||||
location / {
|
||||
proxy_pass http://api;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 15s;
|
||||
proxy_next_upstream error timeout http_502 http_503;
|
||||
}
|
||||
}
|
||||
}
|
||||
CONF
|
||||
lb=$(docker run -d --add-host=host.docker.internal:host-gateway -p 127.0.0.1::8080 -v "$LOCAL_TEST_WORK/nginx.conf:/etc/nginx/nginx.conf:ro" nginx:1.27-alpine)
|
||||
containers="$containers $lb"
|
||||
export BASE_URL="http://$(docker port "$lb" 8080/tcp)"
|
||||
for attempt in $(seq 1 60); do
|
||||
if curl --fail --silent --max-time 1 "$BASE_URL/health/ready" >/dev/null; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
make test > evidence/local/smoke.txt 2>&1
|
||||
LOCAL_ACTION=failover make test-failover ACTION_SCRIPT="$PWD/scripts/local-action.sh" > evidence/local/failover.txt 2>&1
|
||||
make test-persistence ACTION_SCRIPT="$PWD/scripts/local-action.sh" > evidence/local/persistence.txt 2>&1
|
||||
make test > evidence/local/smoke-after.txt 2>&1
|
||||
scripts/test-durable.sh > evidence/local/durable.txt 2>&1
|
||||
if [ "${OBSERVABILITY_TESTS:-false}" = true ]; then scripts/observability-incidents.sh > evidence/incidents-summary.txt 2>&1; fi
|
||||
if [ "${RETRY_TRACE_TEST:-false}" = true ]; then scripts/trace-retry.sh > evidence/current/retry-result.txt 2>&1; fi
|
||||
GOTOOLCHAIN=go1.26.1 go version > evidence/local/go-version.txt
|
||||
shasum -a 256 "$LOCAL_TEST_WORK/server" > evidence/local/binary.sha256
|
||||
echo 'PASS: isolated smoke, attachment failover and persistence'
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
# Follow one controlled upload through API, broker, worker and all telemetry stores.
|
||||
set -eu
|
||||
: "${BASE_URL:?}" "${PROBE_WORK:?}" "${TEMPO_URL:?}" "${LOKI_URL:?}"
|
||||
mkdir -p evidence/current
|
||||
trace=$(openssl rand -hex 16)
|
||||
parent=$(openssl rand -hex 8)
|
||||
chat=$(cat "$PROBE_WORK/chat-id")
|
||||
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=sample.png" -H 'Content-Type: image/png' -H "traceparent: 00-$trace-$parent-01" --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json"
|
||||
id=$(jq -r .id "$PROBE_WORK/attachment.json")
|
||||
ready=false
|
||||
for attempt in $(seq 1 60); do
|
||||
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/attachments/$id" > "$PROBE_WORK/status.json"
|
||||
if jq -e '.status=="ready"' "$PROBE_WORK/status.json" >/dev/null; then ready=true;break;fi
|
||||
sleep 1
|
||||
done
|
||||
$ready
|
||||
jq -n --arg trace "$trace" --arg attachment "$id" '{trace_id:$trace,attachment_id:$attachment,ready:true}' > evidence/current/correlation.json
|
||||
# Batch exporters flush asynchronously; query boundedly until the worker span is indexed.
|
||||
found=false
|
||||
for attempt in $(seq 1 30); do
|
||||
curl -fsS -H 'Accept: application/json' "$TEMPO_URL/api/traces/$trace" > evidence/current/correlated-trace.json || true
|
||||
if jq -e '..|objects|select(.name?=="attachment.process")' evidence/current/correlated-trace.json >/dev/null 2>&1; then found=true;break;fi
|
||||
sleep 1
|
||||
done
|
||||
$found
|
||||
curl -fsS --get "$LOKI_URL/loki/api/v1/query_range" --data-urlencode "query={service_name=~\"messenger-api|messenger-worker\"} | trace_id=\"$trace\"" --data-urlencode 'limit=100' > evidence/current/correlated-logs.json
|
||||
jq -e '.data.result|length>0' evidence/current/correlated-logs.json >/dev/null
|
||||
printf 'PASS: upload and worker share trace %s; correlated structured logs found\n' "$trace"
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/bin/sh
|
||||
# Exercise a real retry on an explicitly owned disposable stack.
|
||||
set -eu
|
||||
: "${LOCAL_TEST_WORK:?}" "${LOCAL_TEST_S3:?}" "${BASE_URL:?}" "${TEMPO_URL:?}"
|
||||
export PROBE_WORK="$LOCAL_TEST_WORK/retry-probe"
|
||||
scripts/prepare-probe.sh
|
||||
worker=$(cat "$LOCAL_TEST_WORK/worker.pid")
|
||||
paused=false
|
||||
stopped=false
|
||||
cleanup() {
|
||||
if $paused; then docker unpause "$LOCAL_TEST_S3" >/dev/null; fi
|
||||
if $stopped; then kill -CONT "$worker"; fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
kill -STOP "$worker"
|
||||
stopped=true
|
||||
trace=$(openssl rand -hex 16)
|
||||
parent=$(openssl rand -hex 8)
|
||||
chat=$(cat "$PROBE_WORK/chat-id")
|
||||
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/chats/$chat/attachments?filename=retry.png" -H 'Content-Type: image/png' -H "traceparent: 00-$trace-$parent-01" --data-binary @tests/fixtures/sample.png > "$PROBE_WORK/attachment.json"
|
||||
id=$(jq -r .id "$PROBE_WORK/attachment.json")
|
||||
docker pause "$LOCAL_TEST_S3" >/dev/null
|
||||
paused=true
|
||||
kill -CONT "$worker"
|
||||
stopped=false
|
||||
# S3 reads have a 15-second deadline; leave one attempt enough time to fail.
|
||||
sleep 20
|
||||
docker unpause "$LOCAL_TEST_S3" >/dev/null
|
||||
paused=false
|
||||
ready=false
|
||||
for attempt in $(seq 1 90); do
|
||||
curl -fsS --config "$PROBE_WORK/auth.conf" "$BASE_URL/api/v1/attachments/$id" > "$PROBE_WORK/status.json"
|
||||
if jq -e '.status=="ready"' "$PROBE_WORK/status.json" >/dev/null; then ready=true; break; fi
|
||||
sleep 1
|
||||
done
|
||||
$ready
|
||||
found=false
|
||||
for attempt in $(seq 1 30); do
|
||||
curl -fsS -H 'Accept: application/json' "$TEMPO_URL/api/traces/$trace" > evidence/current/retry-trace.json || true
|
||||
if jq -e '[..|objects|select(.name?=="attachment.process")]|length>=2' evidence/current/retry-trace.json >/dev/null 2>&1; then found=true; break; fi
|
||||
sleep 1
|
||||
done
|
||||
$found
|
||||
jq -e '[..|objects|select(.name?=="attachment.process" and (.status.code?==2 or .status.code?=="STATUS_CODE_ERROR"))]|length>0' evidence/current/retry-trace.json >/dev/null
|
||||
jq -n --arg trace "$trace" '{trace_id:$trace,ready:true,failed_attempt_and_retry:true}' > evidence/current/retry-result.json
|
||||
printf 'PASS: S3 outage caused a failed consumer attempt; retry reached ready in the same trace\n'
|
||||
Reference in New Issue
Block a user