forked from mtusi2026/mtusi2026lab3
114 lines
5.8 KiB
Bash
Executable File
114 lines
5.8 KiB
Bash
Executable File
#!/bin/sh
|
|
# Exercise real dependency outages on disposable containers; never target the dev cluster.
|
|
set -eu
|
|
cd "$(dirname "$0")/.."
|
|
umask 077
|
|
work=$(mktemp -d)
|
|
name="lab3-fault-$(basename "$work" | tr '[:upper:].' '[:lower:]-')"
|
|
containers=
|
|
cleanup() {
|
|
for container in $containers; do
|
|
docker unpause "$container" >/dev/null 2>&1 || true
|
|
docker rm -f "$container" >/dev/null 2>&1 || true
|
|
done
|
|
docker network rm "$name" >/dev/null 2>&1 || true
|
|
docker image rm "$name" >/dev/null 2>&1 || true
|
|
rm -rf "$work"
|
|
}
|
|
trap cleanup EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
printf 'UTC %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
|
git rev-parse HEAD
|
|
shasum -a 256 internal/store/postgres/*.go internal/service/errors.go internal/store/interface.go internal/controller/http/fiber/public_errors.go scripts/test-faults.sh
|
|
# Build the working tree, including uncommitted fixes, without publishing an image.
|
|
docker build -q -t "$name" . > "$work/build.txt"
|
|
docker network create "$name" >/dev/null
|
|
pg=$(docker run -d --network "$name" --network-alias postgres -e POSTGRES_PASSWORD=synthetic-fault-password -e POSTGRES_DB=messenger postgres:17.6-alpine)
|
|
containers="$pg"
|
|
cache=$(docker run -d --network "$name" --network-alias valkey valkey/valkey:8.1.3-alpine)
|
|
containers="$containers $cache"
|
|
for attempt in $(seq 1 60); do
|
|
if docker exec "$pg" pg_isready -U postgres >/dev/null 2>&1; then break; fi
|
|
sleep 1
|
|
done
|
|
docker run --rm --network "$name" -e DATABASE_URL=postgres://postgres:synthetic-fault-password@postgres:5432/messenger?sslmode=disable -e VALKEY_SHARDS=shard0=valkey:6379 -e CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= "$name" migrate
|
|
api=$(docker run -d --network "$name" -p 127.0.0.1::8080 \
|
|
-e DATABASE_URL=postgres://postgres:synthetic-fault-password@postgres:5432/messenger?sslmode=disable \
|
|
-e VALKEY_SHARDS=shard0=valkey:6379 -e VALKEY_COORDINATOR=shard0 \
|
|
-e CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= -e DEPENDENCY_TIMEOUT_MS=500 -e LOG_LEVEL=error "$name")
|
|
containers="$containers $api"
|
|
base="http://$(docker port "$api" 8080/tcp)"
|
|
: > "$work/anonymous"
|
|
wait_ready() {
|
|
for attempt in $(seq 1 60); do
|
|
if curl -fsS --max-time 2 "$base/health/ready" >/dev/null 2>&1; then return; fi
|
|
sleep 1
|
|
done
|
|
echo 'FAIL: API did not recover within 60 seconds' >&2
|
|
exit 1
|
|
}
|
|
# Credentials remain in private temporary curl config files, never in evidence or argv.
|
|
request() {
|
|
method=$1; path=$2; expected=$3; auth=$4
|
|
shift 4
|
|
result=$(curl -sS --max-time 3 --config "$work/$auth" -X "$method" \
|
|
-H 'Content-Type: application/json' -o "$work/response" \
|
|
-w '%{http_code} %{time_total}' "$base$path" "$@")
|
|
status=${result%% *}
|
|
elapsed=${result#* }
|
|
if [ "$status" != "$expected" ]; then
|
|
printf 'FAIL: %s %s expected=%s actual=%s\n' "$method" "$path" "$expected" "$status" >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
assert_error() {
|
|
jq -e --arg code "$1" '.error.code == $code and (.error.request_id | type == "string") and (has("items") | not)' "$work/response" >/dev/null
|
|
if rg -qi 'postgres|synthetic-fault-password|dial tcp|SELECT |SQLSTATE|redis|valkey|token_hash' "$work/response"; then
|
|
echo 'FAIL: internal dependency details exposed' >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
wait_ready
|
|
request POST /api/v1/users 201 anonymous --data '{"username":"fault_actor","display_name":"Synthetic","password":"synthetic-user-password"}'
|
|
user=$(jq -er .id "$work/response")
|
|
printf 'header = "Authorization: Basic %s"\n' "$(printf 'fault_actor:synthetic-user-password' | openssl base64 -A)" > "$work/basic"
|
|
request POST /api/v1/auth/sessions 201 basic
|
|
jq -er '"header = \"Authorization: Bearer " + .token + "\""' "$work/response" > "$work/bearer"
|
|
printf 'header = "X-User-Id: %s"\n' "$user" > "$work/spoofed"
|
|
printf 'header = "Authorization: Bearer malformed"\n' > "$work/malformed"
|
|
printf 'header = "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"\n' > "$work/unknown"
|
|
request POST /api/v1/chats 201 bearer --data "{\"title\":\"Synthetic fault chat\",\"member_ids\":[\"$user\"]}"
|
|
chat=$(jq -er .id "$work/response")
|
|
history="/api/v1/chats/$chat/messages"
|
|
request POST "$history" 201 bearer -H "Idempotency-Key: fault-message" --data '{"text":"confirmed before outage"}'
|
|
cp "$work/response" "$work/message"
|
|
request GET "$history" 200 bearer
|
|
for dependency in "$pg" "$cache"; do
|
|
if [ "$dependency" = "$pg" ]; then label=postgres; else label=valkey; fi
|
|
for fault in stop pause; do
|
|
docker "$fault" "$dependency" >/dev/null
|
|
request GET "$history" 503 bearer
|
|
assert_error unavailable
|
|
printf 'PASS: %s %s authenticated history => 503 unavailable (%ss)\n' "$label" "$fault" "$elapsed"
|
|
request GET /health/ready 503 anonymous
|
|
assert_error unavailable
|
|
request GET /health/live 200 anonymous
|
|
for auth in anonymous spoofed malformed unknown; do
|
|
expected=503
|
|
if [ "$label" = postgres ] && [ "$auth" != unknown ]; then expected=401; fi
|
|
request GET "$history" "$expected" "$auth"
|
|
if [ "$expected" = 401 ]; then assert_error unauthorized; else assert_error unavailable; fi
|
|
done
|
|
printf 'PASS: %s %s rejects anonymous/spoofed/malformed/unknown identity; liveness=200 readiness=503\n' "$label" "$fault"
|
|
if [ "$fault" = stop ]; then docker start "$dependency" >/dev/null; else docker unpause "$dependency" >/dev/null; fi
|
|
wait_ready
|
|
request GET "$history" 200 bearer
|
|
jq -e --slurpfile message "$work/message" '.items == $message' "$work/response" >/dev/null
|
|
request GET "$history" 401 anonymous
|
|
assert_error unauthorized
|
|
printf 'PASS: %s %s recovery preserves the original session and message\n' "$label" "$fault"
|
|
done
|
|
done
|
|
printf 'PASS: real dependency fault scenarios completed\n'
|