forked from mtusi2026/mtusi2026lab3
Перенести актуальную lab2 как базу lab3
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
# Aggregate covered statements once per merged block; keep files with no statements as N/A.
|
||||
FNR == NR {
|
||||
split($0, fields, "\t")
|
||||
paths[fields[1]] = fields[2]
|
||||
hashes[fields[1]] = fields[3]
|
||||
executable[fields[1]] = fields[4]
|
||||
next
|
||||
}
|
||||
/^mode:/ { next }
|
||||
{
|
||||
file = $1
|
||||
sub(/:[^:]*$/, "", file)
|
||||
if (!(file in paths)) {
|
||||
print "unknown coverage source: " file > "/dev/stderr"
|
||||
invalid = 1
|
||||
}
|
||||
total[file] += $2
|
||||
if ($3 > 0) covered[file] += $2
|
||||
}
|
||||
END {
|
||||
print "{\"threshold_per_file\":90,\"files\":[" > summary
|
||||
separator = ""
|
||||
for (file in paths) {
|
||||
percent = total[file] ? sprintf("%.8f", covered[file] * 100 / total[file]) : "null"
|
||||
printf "%s{\"file\":\"%s\",\"statements\":%d,\"covered\":%d,\"percent\":%s,\"sha256\":\"%s\"}", separator, paths[file], total[file], covered[file], percent, hashes[file] > summary
|
||||
separator = ",\n"
|
||||
printf "%8s %s\n", (total[file] ? sprintf("%.2f%%", covered[file] * 100 / total[file]) : "N/A"), paths[file]
|
||||
if (covered[file] * 100 < total[file] * 90 || (executable[file] && !total[file])) invalid = 1
|
||||
}
|
||||
printf "\n],\"passed\":%s}\n", (invalid ? "false" : "true") > summary
|
||||
if (invalid) {
|
||||
print "FAIL: per-file coverage must reach 90%" > "/dev/stderr"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/bin/sh
|
||||
# Measure unit and real-process coverage, enforcing 90% separately for each Go source file.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
GO=${GO:-go}
|
||||
: "${DATABASE_URL:?Use an isolated test database}" "${VALKEY_SHARDS:?Use test shards}"
|
||||
: "${TEST_DATABASE_URL:?Use an isolated test database}" "${TEST_VALKEY_ADDRESS:?Use a test shard}"
|
||||
export APP_PORT=${COVERAGE_PORT:-18080}
|
||||
export LOG_LEVEL=error
|
||||
server_pid=
|
||||
watchdog_pid=
|
||||
mkdir -p coverage
|
||||
run_dir=$(mktemp -d "$PWD/coverage/run.XXXXXX")
|
||||
cleanup() {
|
||||
if [ -n "$server_pid" ]; then kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true; fi
|
||||
if [ -n "$watchdog_pid" ]; then kill "$watchdog_pid" 2>/dev/null || true; wait "$watchdog_pid" 2>/dev/null || true; fi
|
||||
rm -rf "$run_dir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
mkdir "$run_dir/process"
|
||||
"$GO" build -race -cover -covermode=atomic -coverpkg=./... -o "$run_dir/server" ./cmd/server
|
||||
"$GO" test -race -coverpkg=./... -coverprofile="$run_dir/unit.out" ./...
|
||||
export GOCOVERDIR="$run_dir/process"
|
||||
if "$run_dir/server" unknown >"$run_dir/server-invalid.log" 2>&1; then
|
||||
echo 'server accepted an unknown command' >&2
|
||||
exit 1
|
||||
else
|
||||
test "$?" -eq 1
|
||||
fi
|
||||
"$run_dir/server" >"$run_dir/server.log" 2>&1 &
|
||||
server_pid=$!
|
||||
ready=false
|
||||
for attempt in $(seq 1 60); do
|
||||
kill -0 "$server_pid" 2>/dev/null || { cat "$run_dir/server.log"; exit 1; }
|
||||
if curl --fail --silent --max-time 1 "http://localhost:$APP_PORT/health/ready" >/dev/null; then ready=true; break; fi
|
||||
sleep .25
|
||||
done
|
||||
$ready || { cat "$run_dir/server.log"; exit 1; }
|
||||
"$run_dir/server" healthcheck
|
||||
"${PYTHON:-python3}" tests/smoke.py --base-url "http://localhost:$APP_PORT"
|
||||
kill -TERM "$server_pid"
|
||||
# Bound the real-process shutdown check; a forced kill makes wait fail.
|
||||
(sleep 12; kill -KILL "$server_pid" 2>/dev/null || true) &
|
||||
watchdog_pid=$!
|
||||
wait "$server_pid"
|
||||
server_pid=
|
||||
kill "$watchdog_pid" 2>/dev/null || true
|
||||
wait "$watchdog_pid" 2>/dev/null || true
|
||||
watchdog_pid=
|
||||
unset GOCOVERDIR
|
||||
"$GO" tool covdata textfmt -i="$run_dir/process" -o="$run_dir/process.out"
|
||||
awk '
|
||||
/^mode:/ { if ($0 != "mode: atomic") exit 1; next }
|
||||
{
|
||||
if (($1 in statements) && statements[$1] != $2) exit 1;
|
||||
statements[$1] = $2; hits[$1] += $3
|
||||
}
|
||||
END {
|
||||
print "mode: atomic";
|
||||
for (block in statements) print block, statements[block], hits[block]
|
||||
}' "$run_dir/unit.out" "$run_dir/process.out" > coverage/coverage.out
|
||||
"$GO" list -f '{{.ImportPath}} {{.Dir}} {{join .GoFiles ","}}' ./... > "$run_dir/packages.txt"
|
||||
: > "$run_dir/inventory.tsv"
|
||||
while read -r import_path directory names; do
|
||||
old_ifs=$IFS
|
||||
IFS=,
|
||||
for name in $names; do
|
||||
checksum=$(shasum -a 256 "$directory/$name")
|
||||
checksum=${checksum%% *}
|
||||
relative=${directory#"$PWD/"}/$name
|
||||
executable=$(awk '/^func[ (]/ {found=1} END {print found+0}' "$directory/$name")
|
||||
printf '%s\t%s\t%s\t%s\n' "$import_path/$name" "$relative" "$checksum" "$executable" >> "$run_dir/inventory.tsv"
|
||||
done
|
||||
IFS=$old_ifs
|
||||
done < "$run_dir/packages.txt"
|
||||
awk -v summary=coverage/summary.json -f scripts/coverage.awk "$run_dir/inventory.tsv" coverage/coverage.out
|
||||
"$GO" version > coverage/go-version.txt
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/sh
|
||||
# Compare controlled replica/resource budgets, then restore the normal HPA configuration.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" "${RELEASE:=lab3}"
|
||||
: "${DURATION:=30}" "${VUS:=32}"
|
||||
run_id=$(date -u +%Y%m%d%H%M%S)
|
||||
export KUBE_CONTEXT NAMESPACE DURATION VUS
|
||||
mkdir -p evidence
|
||||
restore() {
|
||||
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" -f charts/messenger/values.yaml --wait --wait-for-jobs --timeout 240s
|
||||
}
|
||||
trap restore EXIT
|
||||
printf 'profile,run,replicas,cpu_millicores_per_replica,memory_mib_per_replica,pool\n' > evidence/scaling-profiles.csv
|
||||
for profile in fixed1 fixed2 fixed4 growing1 growing2 growing4; do
|
||||
case "$profile" in
|
||||
fixed1) replicas=1; cpu=1000; memory=512 ;;
|
||||
fixed2) replicas=2; cpu=500; memory=256 ;;
|
||||
fixed4) replicas=4; cpu=250; memory=128 ;;
|
||||
growing1) replicas=1; cpu=250; memory=128 ;;
|
||||
growing2) replicas=2; cpu=250; memory=128 ;;
|
||||
growing4) replicas=4; cpu=250; memory=128 ;;
|
||||
esac
|
||||
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set autoscaling.enabled=false --set "api.replicas=$replicas" --set "api.resources.requests.cpu=${cpu}m,api.resources.limits.cpu=${cpu}m,api.resources.requests.memory=${memory}Mi,api.resources.limits.memory=${memory}Mi" --set-string api.env.GOMEMLIMIT=80MiB --wait --wait-for-jobs --timeout 240s > "evidence/rollout-$profile.txt"
|
||||
printf '%s,%s,%s,%s,%s,8\n' "$profile" "$profile-$run_id" "$replicas" "$cpu" "$memory" >> evidence/scaling-profiles.csv
|
||||
RUN="$profile-$run_id" scripts/run-load.sh
|
||||
done
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# Create lab credentials once, outside Helm values and release history.
|
||||
set -eu
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
|
||||
: "${REGISTRY:=asisti.registry.twcstorage.ru}" "${REGISTRY_USER:=token}"
|
||||
: "${REGISTRY_TOKEN_FILE:=$HOME/.secrets/twc-registry-token}"
|
||||
test -s "$REGISTRY_TOKEN_FILE" || { echo 'Registry token file is missing or empty' >&2; exit 1; }
|
||||
umask 077
|
||||
work=$(mktemp -d)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
kubectl --context "$KUBE_CONTEXT" create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" apply -f -
|
||||
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-secrets >/dev/null 2>&1; then
|
||||
openssl rand -hex 32 | tr -d '\n' > "$work/postgres-password"
|
||||
printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' "$(cat "$work/postgres-password")" > "$work/database-url"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-secrets --from-file="$work/postgres-password" --from-file="$work/database-url"
|
||||
fi
|
||||
jq -n --arg registry "$REGISTRY" --arg user "$REGISTRY_USER" --rawfile token "$REGISTRY_TOKEN_FILE" \
|
||||
'{auths:{($registry):{auth:(($user+":"+($token | sub("[\\r\\n]+$"; "")))|@base64)}}}' > "$work/config.json"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic registry-pull \
|
||||
--type=kubernetes.io/dockerconfigjson --from-file=".dockerconfigjson=$work/config.json" \
|
||||
--dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f -
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# Reconnect the loopback tunnel after the API pod is replaced during restart tests.
|
||||
set -eu
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
|
||||
LOCAL_PORT=${LOCAL_PORT:-8080}
|
||||
child=
|
||||
cleanup() {
|
||||
if [ -n "$child" ]; then
|
||||
kill "$child" 2>/dev/null || true
|
||||
wait "$child" 2>/dev/null || true
|
||||
fi
|
||||
exit 0
|
||||
}
|
||||
trap cleanup INT TERM
|
||||
while :; do
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" port-forward service/api "$LOCAL_PORT:8080" --address 127.0.0.1 &
|
||||
child=$!
|
||||
wait "$child" || true
|
||||
child=
|
||||
sleep 1
|
||||
done
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/bin/sh
|
||||
# Run the SQL experiment in a fresh disposable container, never the deployed DB.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p evidence
|
||||
work=$(mktemp -d)
|
||||
container=
|
||||
cleanup() {
|
||||
if [ -n "$container" ]; then docker rm -f "$container" >/dev/null; fi
|
||||
rm -rf "$work"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
container=$(docker run -d --label purpose=lab2-research --memory=512m --cpus=1 -e POSTGRES_HOST_AUTH_METHOD=trust postgres:17.6-alpine)
|
||||
for attempt in $(seq 1 60); do
|
||||
if docker exec "$container" pg_isready -U postgres >/dev/null 2>&1; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
docker exec "$container" createdb -U postgres messenger
|
||||
docker exec -i "$container" psql -X -v ON_ERROR_STOP=1 -U postgres -d messenger < internal/store/postgres/migrations/001_initial.sql > /dev/null
|
||||
docker cp student-tests/plans.sql "$container:/tmp/plans.sql" >/dev/null
|
||||
{
|
||||
date -u '+UTC: %Y-%m-%dT%H:%M:%SZ'
|
||||
echo 'Hypothesis: composite index improves late-page reads, with additional write cost.'
|
||||
echo '100000 rows, distribution 90/9/1%; three reads and 1000-row rollback writes per index.'
|
||||
echo 'Limits: 1 CPU, 512MiB; single local Docker instance; no concurrent writes or PITR.'
|
||||
docker inspect --format 'Image: {{.Image}}' "$container"
|
||||
git ls-files --cached --others --exclude-standard cmd internal student-tests/research.sql student-tests/plans.sql scripts/research.sh | sort | while read -r file; do shasum -a 256 "$file"; done
|
||||
docker exec -i "$container" psql -XAt -v ON_ERROR_STOP=1 -U postgres -d messenger < student-tests/research.sql
|
||||
} > evidence/sql-research-helm.txt
|
||||
# Use the actual schema, all table contents, and a fresh target database.
|
||||
query="SELECT count(*),md5(string_agg(id||':'||chat_id||':'||sender_id||':'||text||':'||created_at,',' ORDER BY id)) FROM messages"
|
||||
docker exec "$container" psql -XAt -U postgres -d messenger -c "$query" > "$work/before"
|
||||
{ time docker exec "$container" pg_dump -U postgres -d messenger --no-owner --no-acl > "$work/dump.sql"; } 2>> evidence/sql-research-helm.txt
|
||||
docker exec "$container" createdb -U postgres restored
|
||||
{ time docker exec -i "$container" psql -X -v ON_ERROR_STOP=1 -U postgres -d restored < "$work/dump.sql" > /dev/null; } 2>> evidence/sql-research-helm.txt
|
||||
docker exec "$container" psql -XAt -U postgres -d restored -c "$query" > "$work/after"
|
||||
diff -u "$work/before" "$work/after"
|
||||
{
|
||||
echo 'PASS: restore row count and checksum match'
|
||||
cat "$work/after"
|
||||
wc -c < "$work/dump.sql"
|
||||
docker exec "$container" psql -X -U postgres -d restored -c "SELECT id,text FROM messages WHERE id IN(md5('1')::uuid,md5('50000')::uuid,md5('100000')::uuid) ORDER BY id"
|
||||
} >> evidence/sql-research-helm.txt
|
||||
echo 'PASS: evidence/sql-research-helm.txt'
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
# Restart only this lab's API and persistent stores, retaining every PVC.
|
||||
set -eu
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
|
||||
replicas=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get deployment api -o jsonpath='{.spec.replicas}')
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/api --replicas=0
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" wait --for=delete pod -l app=api --timeout=60s
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout restart statefulset/postgres statefulset/valkey
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status statefulset/postgres --timeout=120s
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status statefulset/valkey --timeout=120s
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/api --replicas="$replicas"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status deployment/api --timeout=120s
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/sh
|
||||
# Run the pinned k6 workload inside the cluster and retain raw results and resource samples.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
|
||||
: "${RUN:=manual-$(date -u +%Y%m%d%H%M%S)}" "${DURATION:=30}" "${VUS:=32}" "${LOAD_SCRIPT:=scaling.js}"
|
||||
case "$RUN" in *[!a-z0-9-]*|'') echo 'RUN must be a DNS label' >&2; exit 2 ;; esac
|
||||
mkdir -p evidence
|
||||
manifest=$(mktemp)
|
||||
sampler=
|
||||
cleanup() {
|
||||
if [ -n "$sampler" ]; then kill "$sampler" 2>/dev/null || true; wait "$sampler" 2>/dev/null || true; fi
|
||||
rm -f "$manifest"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create configmap lab3-load-scripts --from-file=student-tests/load/ --dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f -
|
||||
helm template lab3 charts/messenger -n "$NAMESPACE" -s templates/load.yaml --set load.enabled=true --set-string "load.run=$RUN" --set-string "load.script=$LOAD_SCRIPT" --set "load.duration=$DURATION" --set "load.vus=$VUS" > "$manifest"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f "$manifest"
|
||||
(
|
||||
while :; do
|
||||
date -u '+UTC %Y-%m-%dT%H:%M:%SZ'
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" top pods -l 'app in (lab3-api,lab3-lb,lab3-postgres,lab3-valkey,lab3-load)' || true
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" exec lab3-postgres-0 -- psql -U messenger -d messenger -Atc "SELECT state,wait_event_type,wait_event,count(*) FROM pg_stat_activity WHERE datname='messenger' GROUP BY state,wait_event_type,wait_event" || true
|
||||
sleep 5
|
||||
done
|
||||
) > "evidence/resources-$RUN.txt" 2>&1 &
|
||||
sampler=$!
|
||||
complete=false
|
||||
for attempt in $(seq 1 140); do
|
||||
state=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get "job/lab3-load-$RUN" -o json)
|
||||
if printf '%s' "$state" | jq -e '.status.succeeded > 0' >/dev/null; then complete=true; break; fi
|
||||
if printf '%s' "$state" | jq -e '.status.failed > 0' >/dev/null; then break; fi
|
||||
sleep 2
|
||||
done
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" logs "job/lab3-load-$RUN" > "evidence/load-$RUN.txt"
|
||||
$complete || { echo "Load failed; inspect evidence/load-$RUN.txt" >&2; exit 1; }
|
||||
if [ "$LOAD_SCRIPT" = scaling.js ]; then
|
||||
jq -R 'fromjson? | select(has("profile"))' "evidence/load-$RUN.txt" > "evidence/load-$RUN.json"
|
||||
test -s "evidence/load-$RUN.json"
|
||||
fi
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
# Change the ring only after all API pods and their leases have drained.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" "${RELEASE:=lab2}"
|
||||
: "${SHARDS:?Set SHARDS=2 or SHARDS=3}"
|
||||
case "$SHARDS" in 2|3) ;; *) echo 'SHARDS must be 2 or 3' >&2; exit 2 ;; esac
|
||||
# Keep the desired replica count explicit so an interrupted run can resume from zero.
|
||||
: "${API_REPLICAS:=1}"
|
||||
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set api.replicas=0 --wait --timeout 180s
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" wait --for=delete pod -l app=api --timeout=60s
|
||||
# Maximum lease is 30 seconds; the rate window is one second.
|
||||
sleep 31
|
||||
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set api.replicas=0 --set "valkey.shards=$SHARDS" --wait --timeout 180s
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" exec valkey-0 -- valkey-cli DEL admission:topology
|
||||
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set "api.replicas=$API_REPLICAS" --wait --timeout 180s
|
||||
printf 'Ring updated. Set valkey.shards=%s in your values file before the next make deploy.\n' "$SHARDS"
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Terminate the exact observed API pod; its replacement receives a different instance ID.
|
||||
set -eu
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" "${TARGET_INSTANCE_ID:?Expected API pod name}"
|
||||
case "$TARGET_INSTANCE_ID" in lab3-api-*) ;; *) echo 'Target is not a lab3 API pod' >&2; exit 2 ;; esac
|
||||
label=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pod "$TARGET_INSTANCE_ID" -o jsonpath='{.metadata.labels.app}')
|
||||
test "$label" = lab3-api
|
||||
mkdir -p evidence
|
||||
printf '%s target=%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$TARGET_INSTANCE_ID" >> evidence/failover-targets.txt
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" delete pod "$TARGET_INSTANCE_ID" --wait=false
|
||||
Executable
+112
@@ -0,0 +1,112 @@
|
||||
#!/bin/sh
|
||||
# Exercise real dependency outages on disposable containers; never target the dev cluster.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
work=$(mktemp -d)
|
||||
name="lab2-fault-$(basename "$work" | tr '[:upper:].' '[:lower:]-')"
|
||||
containers=
|
||||
cleanup() {
|
||||
for container in $containers; do
|
||||
docker unpause "$container" >/dev/null 2>&1 || true
|
||||
docker rm -f "$container" >/dev/null 2>&1 || true
|
||||
done
|
||||
docker network rm "$name" >/dev/null 2>&1 || true
|
||||
docker image rm "$name" >/dev/null 2>&1 || true
|
||||
rm -rf "$work"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
printf 'UTC %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
git rev-parse HEAD
|
||||
shasum -a 256 internal/store/postgres/*.go internal/service/errors.go internal/store/interface.go internal/controller/http/fiber/public_errors.go scripts/test-faults.sh
|
||||
# Build the working tree, including uncommitted fixes, without publishing an image.
|
||||
docker build -q -t "$name" . > "$work/build.txt"
|
||||
docker network create "$name" >/dev/null
|
||||
pg=$(docker run -d --network "$name" --network-alias postgres -e POSTGRES_PASSWORD=synthetic-fault-password -e POSTGRES_DB=messenger postgres:17.6-alpine)
|
||||
containers="$pg"
|
||||
cache=$(docker run -d --network "$name" --network-alias valkey valkey/valkey:8.1.3-alpine)
|
||||
containers="$containers $cache"
|
||||
for attempt in $(seq 1 60); do
|
||||
if docker exec "$pg" pg_isready -U postgres >/dev/null 2>&1; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
api=$(docker run -d --network "$name" -p 127.0.0.1::8080 \
|
||||
-e DATABASE_URL=postgres://postgres:synthetic-fault-password@postgres:5432/messenger?sslmode=disable \
|
||||
-e VALKEY_SHARDS=shard0=valkey:6379 -e VALKEY_COORDINATOR=shard0 \
|
||||
-e DEPENDENCY_TIMEOUT_MS=500 -e LOG_LEVEL=error "$name")
|
||||
containers="$containers $api"
|
||||
base="http://$(docker port "$api" 8080/tcp)"
|
||||
: > "$work/anonymous"
|
||||
wait_ready() {
|
||||
for attempt in $(seq 1 60); do
|
||||
if curl -fsS --max-time 2 "$base/health/ready" >/dev/null 2>&1; then return; fi
|
||||
sleep 1
|
||||
done
|
||||
echo 'FAIL: API did not recover within 60 seconds' >&2
|
||||
exit 1
|
||||
}
|
||||
# Credentials remain in private temporary curl config files, never in evidence or argv.
|
||||
request() {
|
||||
method=$1; path=$2; expected=$3; auth=$4
|
||||
shift 4
|
||||
result=$(curl -sS --max-time 3 --config "$work/$auth" -X "$method" \
|
||||
-H 'Content-Type: application/json' -o "$work/response" \
|
||||
-w '%{http_code} %{time_total}' "$base$path" "$@")
|
||||
status=${result%% *}
|
||||
elapsed=${result#* }
|
||||
if [ "$status" != "$expected" ]; then
|
||||
printf 'FAIL: %s %s expected=%s actual=%s\n' "$method" "$path" "$expected" "$status" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
assert_error() {
|
||||
jq -e --arg code "$1" '.error.code == $code and (.error.request_id | type == "string") and (has("items") | not)' "$work/response" >/dev/null
|
||||
if rg -qi 'postgres|synthetic-fault-password|dial tcp|SELECT |SQLSTATE|redis|valkey|token_hash' "$work/response"; then
|
||||
echo 'FAIL: internal dependency details exposed' >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
wait_ready
|
||||
request POST /api/v1/users 201 anonymous --data '{"username":"fault_actor","display_name":"Synthetic","password":"synthetic-user-password"}'
|
||||
user=$(jq -er .id "$work/response")
|
||||
printf 'header = "Authorization: Basic %s"\n' "$(printf 'fault_actor:synthetic-user-password' | openssl base64 -A)" > "$work/basic"
|
||||
request POST /api/v1/auth/sessions 201 basic
|
||||
jq -er '"header = \"Authorization: Bearer " + .token + "\""' "$work/response" > "$work/bearer"
|
||||
printf 'header = "X-User-Id: %s"\n' "$user" > "$work/spoofed"
|
||||
printf 'header = "Authorization: Bearer malformed"\n' > "$work/malformed"
|
||||
printf 'header = "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"\n' > "$work/unknown"
|
||||
request POST /api/v1/chats 201 bearer --data "{\"title\":\"Synthetic fault chat\",\"member_ids\":[\"$user\"]}"
|
||||
chat=$(jq -er .id "$work/response")
|
||||
history="/api/v1/chats/$chat/messages"
|
||||
request POST "$history" 201 bearer --data '{"text":"confirmed before outage"}'
|
||||
cp "$work/response" "$work/message"
|
||||
request GET "$history" 200 bearer
|
||||
for dependency in "$pg" "$cache"; do
|
||||
if [ "$dependency" = "$pg" ]; then label=postgres; else label=valkey; fi
|
||||
for fault in stop pause; do
|
||||
docker "$fault" "$dependency" >/dev/null
|
||||
request GET "$history" 503 bearer
|
||||
assert_error unavailable
|
||||
printf 'PASS: %s %s authenticated history => 503 unavailable (%ss)\n' "$label" "$fault" "$elapsed"
|
||||
request GET /health/ready 503 anonymous
|
||||
assert_error unavailable
|
||||
request GET /health/live 200 anonymous
|
||||
for auth in anonymous spoofed malformed unknown; do
|
||||
expected=503
|
||||
if [ "$label" = postgres ] && [ "$auth" != unknown ]; then expected=401; fi
|
||||
request GET "$history" "$expected" "$auth"
|
||||
if [ "$expected" = 401 ]; then assert_error unauthorized; else assert_error unavailable; fi
|
||||
done
|
||||
printf 'PASS: %s %s rejects anonymous/spoofed/malformed/unknown identity; liveness=200 readiness=503\n' "$label" "$fault"
|
||||
if [ "$fault" = stop ]; then docker start "$dependency" >/dev/null; else docker unpause "$dependency" >/dev/null; fi
|
||||
wait_ready
|
||||
request GET "$history" 200 bearer
|
||||
jq -e --slurpfile message "$work/message" '.items == $message' "$work/response" >/dev/null
|
||||
request GET "$history" 401 anonymous
|
||||
assert_error unauthorized
|
||||
printf 'PASS: %s %s recovery preserves the original session and message\n' "$label" "$fault"
|
||||
done
|
||||
done
|
||||
printf 'PASS: real dependency fault scenarios completed\n'
|
||||
Reference in New Issue
Block a user