#!/bin/sh # Exercise real dependency outages on disposable containers; never target the dev cluster. set -eu cd "$(dirname "$0")/.." umask 077 work=$(mktemp -d) name="lab3-fault-$(basename "$work" | tr '[:upper:].' '[:lower:]-')" containers= cleanup() { for container in $containers; do docker unpause "$container" >/dev/null 2>&1 || true docker rm -f "$container" >/dev/null 2>&1 || true done docker network rm "$name" >/dev/null 2>&1 || true docker image rm "$name" >/dev/null 2>&1 || true rm -rf "$work" } trap cleanup EXIT trap 'exit 130' INT trap 'exit 143' TERM printf 'UTC %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" git rev-parse HEAD shasum -a 256 internal/store/postgres/*.go internal/service/errors.go internal/store/interface.go internal/controller/http/fiber/public_errors.go scripts/test-faults.sh # Build the working tree, including uncommitted fixes, without publishing an image. docker build -q -t "$name" . > "$work/build.txt" docker network create "$name" >/dev/null pg=$(docker run -d --network "$name" --network-alias postgres -e POSTGRES_PASSWORD=synthetic-fault-password -e POSTGRES_DB=messenger postgres:17.6-alpine) containers="$pg" cache=$(docker run -d --network "$name" --network-alias valkey valkey/valkey:8.1.3-alpine) containers="$containers $cache" for attempt in $(seq 1 60); do if docker exec "$pg" pg_isready -U postgres >/dev/null 2>&1; then break; fi sleep 1 done docker run --rm --network "$name" -e DATABASE_URL=postgres://postgres:synthetic-fault-password@postgres:5432/messenger?sslmode=disable -e VALKEY_SHARDS=shard0=valkey:6379 -e CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= "$name" migrate api=$(docker run -d --network "$name" -p 127.0.0.1::8080 \ -e DATABASE_URL=postgres://postgres:synthetic-fault-password@postgres:5432/messenger?sslmode=disable \ -e VALKEY_SHARDS=shard0=valkey:6379 -e VALKEY_COORDINATOR=shard0 \ -e CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= -e DEPENDENCY_TIMEOUT_MS=500 -e LOG_LEVEL=error "$name") containers="$containers $api" base="http://$(docker port "$api" 8080/tcp)" : > "$work/anonymous" wait_ready() { for attempt in $(seq 1 60); do if curl -fsS --max-time 2 "$base/health/ready" >/dev/null 2>&1; then return; fi sleep 1 done echo 'FAIL: API did not recover within 60 seconds' >&2 exit 1 } # Credentials remain in private temporary curl config files, never in evidence or argv. request() { method=$1; path=$2; expected=$3; auth=$4 shift 4 result=$(curl -sS --max-time 3 --config "$work/$auth" -X "$method" \ -H 'Content-Type: application/json' -o "$work/response" \ -w '%{http_code} %{time_total}' "$base$path" "$@") status=${result%% *} elapsed=${result#* } if [ "$status" != "$expected" ]; then printf 'FAIL: %s %s expected=%s actual=%s\n' "$method" "$path" "$expected" "$status" >&2 exit 1 fi } assert_error() { jq -e --arg code "$1" '.error.code == $code and (.error.request_id | type == "string") and (has("items") | not)' "$work/response" >/dev/null if rg -qi 'postgres|synthetic-fault-password|dial tcp|SELECT |SQLSTATE|redis|valkey|token_hash' "$work/response"; then echo 'FAIL: internal dependency details exposed' >&2 exit 1 fi } wait_ready request POST /api/v1/users 201 anonymous --data '{"username":"fault_actor","display_name":"Synthetic","password":"synthetic-user-password"}' user=$(jq -er .id "$work/response") printf 'header = "Authorization: Basic %s"\n' "$(printf 'fault_actor:synthetic-user-password' | openssl base64 -A)" > "$work/basic" request POST /api/v1/auth/sessions 201 basic jq -er '"header = \"Authorization: Bearer " + .token + "\""' "$work/response" > "$work/bearer" printf 'header = "X-User-Id: %s"\n' "$user" > "$work/spoofed" printf 'header = "Authorization: Bearer malformed"\n' > "$work/malformed" printf 'header = "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"\n' > "$work/unknown" request POST /api/v1/chats 201 bearer --data "{\"title\":\"Synthetic fault chat\",\"member_ids\":[\"$user\"]}" chat=$(jq -er .id "$work/response") history="/api/v1/chats/$chat/messages" request POST "$history" 201 bearer -H "Idempotency-Key: fault-message" --data '{"text":"confirmed before outage"}' cp "$work/response" "$work/message" request GET "$history" 200 bearer for dependency in "$pg" "$cache"; do if [ "$dependency" = "$pg" ]; then label=postgres; else label=valkey; fi for fault in stop pause; do docker "$fault" "$dependency" >/dev/null request GET "$history" 503 bearer assert_error unavailable printf 'PASS: %s %s authenticated history => 503 unavailable (%ss)\n' "$label" "$fault" "$elapsed" request GET /health/ready 503 anonymous assert_error unavailable request GET /health/live 200 anonymous for auth in anonymous spoofed malformed unknown; do expected=503 if [ "$label" = postgres ] && [ "$auth" != unknown ]; then expected=401; fi request GET "$history" "$expected" "$auth" if [ "$expected" = 401 ]; then assert_error unauthorized; else assert_error unavailable; fi done printf 'PASS: %s %s rejects anonymous/spoofed/malformed/unknown identity; liveness=200 readiness=503\n' "$label" "$fault" if [ "$fault" = stop ]; then docker start "$dependency" >/dev/null; else docker unpause "$dependency" >/dev/null; fi wait_ready request GET "$history" 200 bearer jq -e --slurpfile message "$work/message" '.items == $message' "$work/response" >/dev/null request GET "$history" 401 anonymous assert_error unauthorized printf 'PASS: %s %s recovery preserves the original session and message\n' "$label" "$fault" done done printf 'PASS: real dependency fault scenarios completed\n'