Реализовать lab3 в общем Helm-релизе с Traefik, TLS и HPA

This commit is contained in:
Vladislav Agapov
2026-10-16 11:51:20 +03:00
parent a57cbcf329
commit 9494547300
88 changed files with 1922 additions and 255 deletions
+3
View File
@@ -7,6 +7,7 @@ GO=${GO:-go}
: "${TEST_DATABASE_URL:?Use an isolated test database}" "${TEST_VALKEY_ADDRESS:?Use a test shard}"
export APP_PORT=${COVERAGE_PORT:-18080}
export LOG_LEVEL=error
export CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
server_pid=
watchdog_pid=
mkdir -p coverage
@@ -22,6 +23,7 @@ trap 'exit 130' INT
trap 'exit 143' TERM
mkdir "$run_dir/process"
"$GO" build -race -cover -covermode=atomic -coverpkg=./... -o "$run_dir/server" ./cmd/server
"$run_dir/server" migrate
"$GO" test -race -coverpkg=./... -coverprofile="$run_dir/unit.out" ./...
export GOCOVERDIR="$run_dir/process"
if "$run_dir/server" unknown >"$run_dir/server-invalid.log" 2>&1; then
@@ -40,6 +42,7 @@ for attempt in $(seq 1 60); do
done
$ready || { cat "$run_dir/server.log"; exit 1; }
"$run_dir/server" healthcheck
"$run_dir/server" migrate
"${PYTHON:-python3}" tests/smoke.py --base-url "http://localhost:$APP_PORT"
kill -TERM "$server_pid"
# Bound the real-process shutdown check; a forced kill makes wait fail.
+1 -1
View File
@@ -2,7 +2,7 @@
# Compare controlled replica/resource budgets, then restore the normal HPA configuration.
set -eu
cd "$(dirname "$0")/.."
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" "${RELEASE:=lab3}"
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" "${RELEASE:=lab2}"
: "${DURATION:=30}" "${VUS:=32}"
run_id=$(date -u +%Y%m%d%H%M%S)
export KUBE_CONTEXT NAMESPACE DURATION VUS
+9 -1
View File
@@ -12,8 +12,16 @@ kubectl --context "$KUBE_CONTEXT" create namespace "$NAMESPACE" --dry-run=client
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-secrets >/dev/null 2>&1; then
openssl rand -hex 32 | tr -d '\n' > "$work/postgres-password"
printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' "$(cat "$work/postgres-password")" > "$work/database-url"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-secrets --from-file="$work/postgres-password" --from-file="$work/database-url"
openssl rand -base64 32 | tr -d '\n' > "$work/cursor-key"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic messenger-secrets --from-file="$work/postgres-password" --from-file="$work/database-url" --from-file="$work/cursor-key"
fi
# Add the shared cursor key to an existing lab2 secret without replacing database credentials.
if ! kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get secret messenger-secrets -o json | jq -e '.data["cursor-key"] | length > 0' >/dev/null; then
openssl rand -base64 32 | tr -d '\n' > "$work/cursor-key"
jq -n --rawfile key "$work/cursor-key" '{stringData:{"cursor-key":$key}}' > "$work/patch.json"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" patch secret messenger-secrets --type merge --patch-file "$work/patch.json"
fi
# Build a portable Docker config; macOS credential-helper references cannot be used by kubelet.
jq -n --arg registry "$REGISTRY" --arg user "$REGISTRY_USER" --rawfile token "$REGISTRY_TOKEN_FILE" \
'{auths:{($registry):{auth:(($user+":"+($token | sub("[\\r\\n]+$"; "")))|@base64)}}}' > "$work/config.json"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create secret generic registry-pull \
+2 -2
View File
@@ -2,7 +2,7 @@
# Reconnect the loopback tunnel after the API pod is replaced during restart tests.
set -eu
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
LOCAL_PORT=${LOCAL_PORT:-8080}
LOCAL_PORT=${LOCAL_PORT:-8083}
child=
cleanup() {
if [ -n "$child" ]; then
@@ -13,7 +13,7 @@ cleanup() {
}
trap cleanup INT TERM
while :; do
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" port-forward service/api "$LOCAL_PORT:8080" --address 127.0.0.1 &
kubectl --context "$KUBE_CONTEXT" -n traefik port-forward service/traefik "$LOCAL_PORT:80" --address 127.0.0.1 &
child=$!
wait "$child" || true
child=
+3
View File
@@ -3,6 +3,8 @@
set -eu
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
replicas=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get deployment api -o jsonpath='{.spec.replicas}')
mkdir -p evidence
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l 'app in (api,postgres,valkey)' -o custom-columns=NAME:.metadata.name,UID:.metadata.uid > evidence/restart-before.txt
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/api --replicas=0
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" wait --for=delete pod -l app=api --timeout=60s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout restart statefulset/postgres statefulset/valkey
@@ -10,3 +12,4 @@ kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status statefulset/pos
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status statefulset/valkey --timeout=120s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" scale deployment/api --replicas="$replicas"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status deployment/api --timeout=120s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l 'app in (api,postgres,valkey)' -o custom-columns=NAME:.metadata.name,UID:.metadata.uid > evidence/restart-after.txt
+6 -6
View File
@@ -13,26 +13,26 @@ cleanup() {
rm -f "$manifest"
}
trap cleanup EXIT
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create configmap lab3-load-scripts --from-file=student-tests/load/ --dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f -
helm template lab3 charts/messenger -n "$NAMESPACE" -s templates/load.yaml --set load.enabled=true --set-string "load.run=$RUN" --set-string "load.script=$LOAD_SCRIPT" --set "load.duration=$DURATION" --set "load.vus=$VUS" > "$manifest"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create configmap load-scripts --from-file=student-tests/load/ --dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f -
helm template lab2 charts/messenger -n "$NAMESPACE" -s templates/load.yaml --set load.enabled=true --set-string "load.run=$RUN" --set-string "load.script=$LOAD_SCRIPT" --set "load.duration=$DURATION" --set "load.vus=$VUS" > "$manifest"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f "$manifest"
(
while :; do
date -u '+UTC %Y-%m-%dT%H:%M:%SZ'
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" top pods -l 'app in (lab3-api,lab3-lb,lab3-postgres,lab3-valkey,lab3-load)' || true
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" exec lab3-postgres-0 -- psql -U messenger -d messenger -Atc "SELECT state,wait_event_type,wait_event,count(*) FROM pg_stat_activity WHERE datname='messenger' GROUP BY state,wait_event_type,wait_event" || true
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" top pods -l 'app in (api,lb,postgres,valkey,load)' || true
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" exec postgres-0 -- psql -U messenger -d messenger -Atc "SELECT state,wait_event_type,wait_event,count(*) FROM pg_stat_activity WHERE datname='messenger' GROUP BY state,wait_event_type,wait_event" || true
sleep 5
done
) > "evidence/resources-$RUN.txt" 2>&1 &
sampler=$!
complete=false
for attempt in $(seq 1 140); do
state=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get "job/lab3-load-$RUN" -o json)
state=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get "job/load-$RUN" -o json)
if printf '%s' "$state" | jq -e '.status.succeeded > 0' >/dev/null; then complete=true; break; fi
if printf '%s' "$state" | jq -e '.status.failed > 0' >/dev/null; then break; fi
sleep 2
done
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" logs "job/lab3-load-$RUN" > "evidence/load-$RUN.txt"
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" logs "job/load-$RUN" > "evidence/load-$RUN.txt"
$complete || { echo "Load failed; inspect evidence/load-$RUN.txt" >&2; exit 1; }
if [ "$LOAD_SCRIPT" = scaling.js ]; then
jq -R 'fromjson? | select(has("profile"))' "evidence/load-$RUN.txt" > "evidence/load-$RUN.json"
+4 -4
View File
@@ -6,12 +6,12 @@ cd "$(dirname "$0")/.."
: "${SHARDS:?Set SHARDS=2 or SHARDS=3}"
case "$SHARDS" in 2|3) ;; *) echo 'SHARDS must be 2 or 3' >&2; exit 2 ;; esac
# Keep the desired replica count explicit so an interrupted run can resume from zero.
: "${API_REPLICAS:=1}"
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set api.replicas=0 --wait --timeout 180s
: "${API_REPLICAS:=2}"
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set autoscaling.enabled=false --set api.replicas=0 --wait --timeout 180s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" wait --for=delete pod -l app=api --timeout=60s
# Maximum lease is 30 seconds; the rate window is one second.
sleep 31
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set api.replicas=0 --set "valkey.shards=$SHARDS" --wait --timeout 180s
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set autoscaling.enabled=false --set api.replicas=0 --set "valkey.shards=$SHARDS" --wait --timeout 180s
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" exec valkey-0 -- valkey-cli DEL admission:topology
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set "api.replicas=$API_REPLICAS" --wait --timeout 180s
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set autoscaling.enabled=true --set "api.replicas=$API_REPLICAS" --wait --timeout 180s
printf 'Ring updated. Set valkey.shards=%s in your values file before the next make deploy.\n' "$SHARDS"
+2 -2
View File
@@ -2,9 +2,9 @@
# Terminate the exact observed API pod; its replacement receives a different instance ID.
set -eu
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" "${TARGET_INSTANCE_ID:?Expected API pod name}"
case "$TARGET_INSTANCE_ID" in lab3-api-*) ;; *) echo 'Target is not a lab3 API pod' >&2; exit 2 ;; esac
case "$TARGET_INSTANCE_ID" in api-*) ;; *) echo 'Target is not a lab3 API pod' >&2; exit 2 ;; esac
label=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pod "$TARGET_INSTANCE_ID" -o jsonpath='{.metadata.labels.app}')
test "$label" = lab3-api
test "$label" = api
mkdir -p evidence
printf '%s target=%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$TARGET_INSTANCE_ID" >> evidence/failover-targets.txt
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" delete pod "$TARGET_INSTANCE_ID" --wait=false
+4 -3
View File
@@ -4,7 +4,7 @@ set -eu
cd "$(dirname "$0")/.."
umask 077
work=$(mktemp -d)
name="lab2-fault-$(basename "$work" | tr '[:upper:].' '[:lower:]-')"
name="lab3-fault-$(basename "$work" | tr '[:upper:].' '[:lower:]-')"
containers=
cleanup() {
for container in $containers; do
@@ -32,10 +32,11 @@ for attempt in $(seq 1 60); do
if docker exec "$pg" pg_isready -U postgres >/dev/null 2>&1; then break; fi
sleep 1
done
docker run --rm --network "$name" -e DATABASE_URL=postgres://postgres:synthetic-fault-password@postgres:5432/messenger?sslmode=disable -e VALKEY_SHARDS=shard0=valkey:6379 -e CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= "$name" migrate
api=$(docker run -d --network "$name" -p 127.0.0.1::8080 \
-e DATABASE_URL=postgres://postgres:synthetic-fault-password@postgres:5432/messenger?sslmode=disable \
-e VALKEY_SHARDS=shard0=valkey:6379 -e VALKEY_COORDINATOR=shard0 \
-e DEPENDENCY_TIMEOUT_MS=500 -e LOG_LEVEL=error "$name")
-e CURSOR_SIGNING_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= -e DEPENDENCY_TIMEOUT_MS=500 -e LOG_LEVEL=error "$name")
containers="$containers $api"
base="http://$(docker port "$api" 8080/tcp)"
: > "$work/anonymous"
@@ -80,7 +81,7 @@ printf 'header = "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
request POST /api/v1/chats 201 bearer --data "{\"title\":\"Synthetic fault chat\",\"member_ids\":[\"$user\"]}"
chat=$(jq -er .id "$work/response")
history="/api/v1/chats/$chat/messages"
request POST "$history" 201 bearer --data '{"text":"confirmed before outage"}'
request POST "$history" 201 bearer -H "Idempotency-Key: fault-message" --data '{"text":"confirmed before outage"}'
cp "$work/response" "$work/message"
request GET "$history" 200 bearer
for dependency in "$pg" "$cache"; do