Files

113 lines
5.4 KiB
Bash
Executable File

#!/bin/sh
# Exercise real dependency outages on disposable containers; never target the dev cluster.
set -eu
cd "$(dirname "$0")/.."
umask 077
work=$(mktemp -d)
name="lab2-fault-$(basename "$work" | tr '[:upper:].' '[:lower:]-')"
containers=
cleanup() {
for container in $containers; do
docker unpause "$container" >/dev/null 2>&1 || true
docker rm -f "$container" >/dev/null 2>&1 || true
done
docker network rm "$name" >/dev/null 2>&1 || true
docker image rm "$name" >/dev/null 2>&1 || true
rm -rf "$work"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
printf 'UTC %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
git rev-parse HEAD
shasum -a 256 internal/store/postgres/*.go internal/service/errors.go internal/store/interface.go internal/controller/http/fiber/public_errors.go scripts/test-faults.sh
# Build the working tree, including uncommitted fixes, without publishing an image.
docker build -q -t "$name" . > "$work/build.txt"
docker network create "$name" >/dev/null
pg=$(docker run -d --network "$name" --network-alias postgres -e POSTGRES_PASSWORD=synthetic-fault-password -e POSTGRES_DB=messenger postgres:17.6-alpine)
containers="$pg"
cache=$(docker run -d --network "$name" --network-alias valkey valkey/valkey:8.1.3-alpine)
containers="$containers $cache"
for attempt in $(seq 1 60); do
if docker exec "$pg" pg_isready -U postgres >/dev/null 2>&1; then break; fi
sleep 1
done
api=$(docker run -d --network "$name" -p 127.0.0.1::8080 \
-e DATABASE_URL=postgres://postgres:synthetic-fault-password@postgres:5432/messenger?sslmode=disable \
-e VALKEY_SHARDS=shard0=valkey:6379 -e VALKEY_COORDINATOR=shard0 \
-e DEPENDENCY_TIMEOUT_MS=500 -e LOG_LEVEL=error "$name")
containers="$containers $api"
base="http://$(docker port "$api" 8080/tcp)"
: > "$work/anonymous"
wait_ready() {
for attempt in $(seq 1 60); do
if curl -fsS --max-time 2 "$base/health/ready" >/dev/null 2>&1; then return; fi
sleep 1
done
echo 'FAIL: API did not recover within 60 seconds' >&2
exit 1
}
# Credentials remain in private temporary curl config files, never in evidence or argv.
request() {
method=$1; path=$2; expected=$3; auth=$4
shift 4
result=$(curl -sS --max-time 3 --config "$work/$auth" -X "$method" \
-H 'Content-Type: application/json' -o "$work/response" \
-w '%{http_code} %{time_total}' "$base$path" "$@")
status=${result%% *}
elapsed=${result#* }
if [ "$status" != "$expected" ]; then
printf 'FAIL: %s %s expected=%s actual=%s\n' "$method" "$path" "$expected" "$status" >&2
exit 1
fi
}
assert_error() {
jq -e --arg code "$1" '.error.code == $code and (.error.request_id | type == "string") and (has("items") | not)' "$work/response" >/dev/null
if rg -qi 'postgres|synthetic-fault-password|dial tcp|SELECT |SQLSTATE|redis|valkey|token_hash' "$work/response"; then
echo 'FAIL: internal dependency details exposed' >&2
exit 1
fi
}
wait_ready
request POST /api/v1/users 201 anonymous --data '{"username":"fault_actor","display_name":"Synthetic","password":"synthetic-user-password"}'
user=$(jq -er .id "$work/response")
printf 'header = "Authorization: Basic %s"\n' "$(printf 'fault_actor:synthetic-user-password' | openssl base64 -A)" > "$work/basic"
request POST /api/v1/auth/sessions 201 basic
jq -er '"header = \"Authorization: Bearer " + .token + "\""' "$work/response" > "$work/bearer"
printf 'header = "X-User-Id: %s"\n' "$user" > "$work/spoofed"
printf 'header = "Authorization: Bearer malformed"\n' > "$work/malformed"
printf 'header = "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"\n' > "$work/unknown"
request POST /api/v1/chats 201 bearer --data "{\"title\":\"Synthetic fault chat\",\"member_ids\":[\"$user\"]}"
chat=$(jq -er .id "$work/response")
history="/api/v1/chats/$chat/messages"
request POST "$history" 201 bearer --data '{"text":"confirmed before outage"}'
cp "$work/response" "$work/message"
request GET "$history" 200 bearer
for dependency in "$pg" "$cache"; do
if [ "$dependency" = "$pg" ]; then label=postgres; else label=valkey; fi
for fault in stop pause; do
docker "$fault" "$dependency" >/dev/null
request GET "$history" 503 bearer
assert_error unavailable
printf 'PASS: %s %s authenticated history => 503 unavailable (%ss)\n' "$label" "$fault" "$elapsed"
request GET /health/ready 503 anonymous
assert_error unavailable
request GET /health/live 200 anonymous
for auth in anonymous spoofed malformed unknown; do
expected=503
if [ "$label" = postgres ] && [ "$auth" != unknown ]; then expected=401; fi
request GET "$history" "$expected" "$auth"
if [ "$expected" = 401 ]; then assert_error unauthorized; else assert_error unavailable; fi
done
printf 'PASS: %s %s rejects anonymous/spoofed/malformed/unknown identity; liveness=200 readiness=503\n' "$label" "$fault"
if [ "$fault" = stop ]; then docker start "$dependency" >/dev/null; else docker unpause "$dependency" >/dev/null; fi
wait_ready
request GET "$history" 200 bearer
jq -e --slurpfile message "$work/message" '.items == $message' "$work/response" >/dev/null
request GET "$history" 401 anonymous
assert_error unauthorized
printf 'PASS: %s %s recovery preserves the original session and message\n' "$label" "$fault"
done
done
printf 'PASS: real dependency fault scenarios completed\n'