forked from mtusi2026/mtusi2026lab3
Исправить заголовок 413 и дополнить проверку отказов и масштабирования lab3
This commit is contained in:
+42
-13
@@ -1,27 +1,56 @@
|
||||
#!/bin/sh
|
||||
# Compare controlled replica/resource budgets, then restore the normal HPA configuration.
|
||||
# Measure replica budgets, load saturation and a pool-size A/B using the same workload.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}" "${RELEASE:=lab2}"
|
||||
: "${DURATION:=30}" "${VUS:=32}"
|
||||
run_id=$(date -u +%Y%m%d%H%M%S)
|
||||
export KUBE_CONTEXT NAMESPACE DURATION VUS
|
||||
mkdir -p evidence
|
||||
: "${EVIDENCE_DIR:=evidence/research-$(date -u +%Y%m%d%H%M%S)}"
|
||||
helm_apply_mode=
|
||||
if helm upgrade --help | rg -q -- '--server-side'; then helm_apply_mode=--server-side=false; fi
|
||||
export KUBE_CONTEXT NAMESPACE DURATION VUS EVIDENCE_DIR
|
||||
mkdir -p "$EVIDENCE_DIR"
|
||||
helm get values "$RELEASE" --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" -o yaml > "$EVIDENCE_DIR/restore-values.yaml"
|
||||
restore() {
|
||||
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" -f charts/messenger/values.yaml --wait --wait-for-jobs --timeout 240s
|
||||
helm upgrade $helm_apply_mode "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" -f "$EVIDENCE_DIR/restore-values.yaml" --wait --wait-for-jobs --timeout 240s > "$EVIDENCE_DIR/restore.txt" 2>&1
|
||||
}
|
||||
trap restore EXIT
|
||||
printf 'profile,run,replicas,cpu_millicores_per_replica,memory_mib_per_replica,pool\n' > evidence/scaling-profiles.csv
|
||||
for profile in fixed1 fixed2 fixed4 growing1 growing2 growing4; do
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
helm upgrade $helm_apply_mode "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set autoscaling.enabled=false --wait --wait-for-jobs --timeout 240s > "$EVIDENCE_DIR/disable-hpa.txt"
|
||||
# Allow an already-running HPA reconciliation to finish before changing replicas.
|
||||
sleep 16
|
||||
date -u '+%Y-%m-%dT%H:%M:%SZ' > "$EVIDENCE_DIR/date.txt"
|
||||
git rev-parse HEAD > "$EVIDENCE_DIR/base-revision.txt"
|
||||
shasum -a 256 student-tests/load/scaling.js charts/messenger/values.yaml internal/controller/http/fiber/errors.go > "$EVIDENCE_DIR/source.sha256"
|
||||
kubectl --context "$KUBE_CONTEXT" get nodes -o wide > "$EVIDENCE_DIR/nodes.txt"
|
||||
printf 'profile,run,replicas,cpu_millicores_per_replica,memory_mib_per_replica,pool,vus,duration\n' > "$EVIDENCE_DIR/profiles.csv"
|
||||
for profile in ${PROFILES:-fixed1 fixed2 fixed4 growing1 growing2 growing4 saturation8 saturation64 single32a single8b single32b}; do
|
||||
pool=8; vus=$VUS; probe_timeout=2; probe_failures=2
|
||||
case "$profile" in
|
||||
fixed1) replicas=1; cpu=1000; memory=512 ;;
|
||||
fixed2) replicas=2; cpu=500; memory=256 ;;
|
||||
fixed4) replicas=4; cpu=250; memory=128 ;;
|
||||
fixed4|growing4|pool8b) replicas=4; cpu=250; memory=128 ;;
|
||||
growing1) replicas=1; cpu=250; memory=128 ;;
|
||||
growing2) replicas=2; cpu=250; memory=128 ;;
|
||||
growing4) replicas=4; cpu=250; memory=128 ;;
|
||||
saturation8) replicas=1; cpu=1000; memory=512; vus=8 ;;
|
||||
saturation64) replicas=1; cpu=1000; memory=512; vus=64 ;;
|
||||
resource-low-a|resource-low-b) replicas=2; cpu=250; memory=128 ;;
|
||||
resource-high-a|resource-high-b) replicas=2; cpu=500; memory=256 ;;
|
||||
probe2a|probe2b) replicas=1; cpu=1000; memory=512 ;;
|
||||
probe5a|probe5b) replicas=1; cpu=1000; memory=512; probe_timeout=5; probe_failures=1 ;;
|
||||
single8a|single8b) replicas=1; cpu=1000; memory=512 ;;
|
||||
single32a|single32b) replicas=1; cpu=1000; memory=512; pool=32 ;;
|
||||
pool32a|pool32b) replicas=4; cpu=250; memory=128; pool=32 ;;
|
||||
*) echo "Unknown profile $profile" >&2; exit 2 ;;
|
||||
esac
|
||||
helm upgrade "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set autoscaling.enabled=false --set "api.replicas=$replicas" --set "api.resources.requests.cpu=${cpu}m,api.resources.limits.cpu=${cpu}m,api.resources.requests.memory=${memory}Mi,api.resources.limits.memory=${memory}Mi" --set-string api.env.GOMEMLIMIT=80MiB --wait --wait-for-jobs --timeout 240s > "evidence/rollout-$profile.txt"
|
||||
printf '%s,%s,%s,%s,%s,8\n' "$profile" "$profile-$run_id" "$replicas" "$cpu" "$memory" >> evidence/scaling-profiles.csv
|
||||
RUN="$profile-$run_id" scripts/run-load.sh
|
||||
done
|
||||
set --
|
||||
if [ -n "${BENCHMARK_NODE:-}" ]; then
|
||||
set -- --set-json "api.nodeSelector={\"kubernetes.io/hostname\":\"$BENCHMARK_NODE\"}"
|
||||
fi
|
||||
run="$profile-$(date -u +%Y%m%d%H%M%S)"
|
||||
helm upgrade $helm_apply_mode "$RELEASE" charts/messenger --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" --reuse-values --set autoscaling.enabled=false --set "api.replicas=$replicas" --set "api.resources.requests.cpu=${cpu}m,api.resources.limits.cpu=${cpu}m,api.resources.requests.memory=${memory}Mi,api.resources.limits.memory=${memory}Mi" --set-string api.env.GOMEMLIMIT=80MiB --set-string "api.env.DEPENDENCY_POOL_SIZE=$pool" --set "api.readiness.timeoutSeconds=$probe_timeout,api.readiness.failureThreshold=$probe_failures" "$@" --wait --wait-for-jobs --timeout 240s > "$EVIDENCE_DIR/rollout-$profile.txt"
|
||||
# Let old admission leases expire after rolling replacement.
|
||||
sleep 31
|
||||
printf '%s,%s,%s,%s,%s,%s,%s,%s\n' "$profile" "$run" "$replicas" "$cpu" "$memory" "$pool" "$vus" "$DURATION" >> "$EVIDENCE_DIR/profiles.csv"
|
||||
RUN="$run" VUS="$vus" scripts/run-load.sh
|
||||
done
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Summarize measured k6 counters and plot reproducible research figures."""
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
roots = [Path(arg) for arg in sys.argv[1:]]
|
||||
def resource_peaks(root, run):
|
||||
"""resource_peaks aggregates five-second samples, including setup and warmup."""
|
||||
path = root / ("resources-" + run + ".txt")
|
||||
lines = path.read_text().splitlines()
|
||||
pod_snapshot = root / ("pods-" + run + ".txt")
|
||||
if pod_snapshot.exists():
|
||||
api_pods = {line.split()[0] for line in pod_snapshot.read_text().splitlines() if line.startswith("api-") and "Running" in line}
|
||||
else:
|
||||
api_pods = set()
|
||||
block = set()
|
||||
for line in lines + ["UTC END"]:
|
||||
if line.startswith("UTC "):
|
||||
if block:
|
||||
api_pods = block
|
||||
block = set()
|
||||
elif line.startswith("api-"):
|
||||
block.add(line.split()[0])
|
||||
peaks = dict(api_cpu_m=0, postgres_cpu_m=0, valkey_cpu_m=0, load_cpu_m=0, api_memory_mib=0, postgres_memory_mib=0, pg_active=0, pg_idle_transaction=0, pg_lock_waiters=0)
|
||||
sample = dict.fromkeys(peaks, 0)
|
||||
for line in lines + ["UTC END"]:
|
||||
if line.startswith("UTC "):
|
||||
for key in peaks:
|
||||
peaks[key] = max(peaks[key], sample[key])
|
||||
sample = dict.fromkeys(peaks, 0)
|
||||
continue
|
||||
fields = line.split()
|
||||
if len(fields) == 3 and fields[0].startswith(("api-", "postgres-", "valkey-", "load-")):
|
||||
role = fields[0].split("-")[0]
|
||||
if role == "api" and fields[0] not in api_pods:
|
||||
continue
|
||||
if role == "load" and not fields[0].startswith("load-" + run + "-"):
|
||||
continue
|
||||
cpu = float(fields[1][:-1]) if fields[1].endswith("m") else float(fields[1]) * 1000
|
||||
memory = float(fields[2][:-2]) * (1024 if fields[2].endswith("Gi") else 1)
|
||||
sample[role + "_cpu_m"] += cpu
|
||||
if role in ("api", "postgres"):
|
||||
sample[role + "_memory_mib"] += memory
|
||||
if line.startswith("active|"):
|
||||
sample["pg_active"] += int(line.split("|")[-1])
|
||||
if line.startswith("idle in transaction|"):
|
||||
sample["pg_idle_transaction"] += int(line.split("|")[-1])
|
||||
if "|Lock|" in line:
|
||||
sample["pg_lock_waiters"] += int(line.split("|")[-1])
|
||||
return peaks
|
||||
|
||||
rows = []
|
||||
|
||||
excluded = []
|
||||
for root in roots:
|
||||
for profile in csv.DictReader((root / "profiles.csv").open()):
|
||||
deployment = root / ("deployment-" + profile["run"] + ".json")
|
||||
if deployment.exists() and json.loads(deployment.read_text())["spec"]["replicas"] != int(profile["replicas"]):
|
||||
excluded.append(str(deployment) + " (replica mismatch)")
|
||||
continue
|
||||
source = root / ("load-" + profile["run"] + ".json")
|
||||
if not source.exists():
|
||||
excluded.append(str(source))
|
||||
continue
|
||||
result = json.loads(source.read_text())
|
||||
metrics = result["summary"]["metrics"]
|
||||
if "message_latency_ms" not in metrics:
|
||||
excluded.append(str(source))
|
||||
continue
|
||||
def value(name, key="count"):
|
||||
"""value reads a measured metric, using zero for an absent error counter."""
|
||||
return metrics.get(name, {}).get("values", {}).get(key, 0)
|
||||
latency = metrics["message_latency_ms"]["values"]
|
||||
seconds = int(profile["duration"])
|
||||
operations = value("measured_operations")
|
||||
rows.append(dict(profile, source=str(root), confirmed=value("confirmed_messages"),
|
||||
rps=value("confirmed_messages") / seconds,
|
||||
errors=value("application_errors"),
|
||||
error_percent=100 * value("application_errors") / max(operations, 1),
|
||||
p50_ms=latency["p(50)"], p95_ms=latency["p(95)"],
|
||||
p99_ms=latency["p(99)"],
|
||||
rejected_429=value("application_errors{status:429}"),
|
||||
unavailable_503=value("application_errors{status:503}"),
|
||||
network_errors=value("application_errors{status:0}"), **resource_peaks(root, profile["run"])))
|
||||
output = roots[0]
|
||||
with (output / "summary.csv").open("w") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
(output / "excluded.json").write_text(json.dumps(excluded, indent=2)+"\n")
|
||||
fig, axes = plt.subplots(3, 1, figsize=(13, 11), constrained_layout=True)
|
||||
names = [row["profile"] for row in rows]
|
||||
for axis, field, label in zip(axes, ["rps", "p95_ms", "error_percent"],
|
||||
["Confirmed messages / second", "Write attempt p95, ms", "Failed operations, %"]):
|
||||
axis.bar(names, [row[field] for row in rows], color="#267c75")
|
||||
axis.set_ylabel(label)
|
||||
axis.grid(axis="y", alpha=0.25)
|
||||
axis.tick_params(axis="x", rotation=35)
|
||||
fig.suptitle("Lab 3: measured replica budgets and pool-size experiment")
|
||||
fig.savefig(output / "results.png", dpi=160)
|
||||
fig.savefig(output / "results.svg")
|
||||
plt.close(fig)
|
||||
by_name = {row["profile"]: row for row in rows}
|
||||
if all(f"{series}{n}" in by_name for series in ["fixed", "growing"] for n in [1, 2, 4]):
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4), constrained_layout=True)
|
||||
for series, label in [("fixed", "Fixed total CPU/RAM"), ("growing", "Growing total CPU/RAM")]:
|
||||
for axis, field, ylabel in zip(axes, ["rps", "error_percent"], ["Confirmed messages / second", "Failed operations, %"]):
|
||||
axis.plot([1, 2, 4], [by_name[f"{series}{n}"][field] for n in [1, 2, 4]], marker="o", label=label)
|
||||
axis.set(xlabel="API replicas", ylabel=ylabel, xticks=[1, 2, 4])
|
||||
axis.grid(alpha=0.25)
|
||||
axis.legend()
|
||||
fig.savefig(output / "matrix.png", dpi=160)
|
||||
fig.savefig(output / "matrix.svg")
|
||||
print(json.dumps(rows, indent=2))
|
||||
+26
-8
@@ -5,17 +5,32 @@ cd "$(dirname "$0")/.."
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
|
||||
: "${RUN:=manual-$(date -u +%Y%m%d%H%M%S)}" "${DURATION:=30}" "${VUS:=32}" "${LOAD_SCRIPT:=scaling.js}"
|
||||
case "$RUN" in *[!a-z0-9-]*|'') echo 'RUN must be a DNS label' >&2; exit 2 ;; esac
|
||||
mkdir -p evidence
|
||||
EVIDENCE_DIR=${EVIDENCE_DIR:-evidence}
|
||||
mkdir -p "$EVIDENCE_DIR"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get deployment api -o json > "$EVIDENCE_DIR/deployment-$RUN.json"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l app=api -o wide > "$EVIDENCE_DIR/pods-$RUN.txt"
|
||||
helm get values lab2 --kube-context "$KUBE_CONTEXT" -n "$NAMESPACE" -o json > "$EVIDENCE_DIR/values-$RUN.json"
|
||||
if jq -e '.autoscaling.enabled == false' "$EVIDENCE_DIR/values-$RUN.json" >/dev/null; then
|
||||
expected=$(jq -r .api.replicas "$EVIDENCE_DIR/values-$RUN.json")
|
||||
actual=$(jq -r .spec.replicas "$EVIDENCE_DIR/deployment-$RUN.json")
|
||||
test "$expected" = "$actual" || { echo "Replica mismatch: expected $expected, actual $actual" >&2; exit 1; }
|
||||
fi
|
||||
manifest=$(mktemp)
|
||||
complete=false
|
||||
created=false
|
||||
sampler=
|
||||
cleanup() {
|
||||
if [ -n "$sampler" ]; then kill "$sampler" 2>/dev/null || true; wait "$sampler" 2>/dev/null || true; fi
|
||||
if ! $complete && $created; then
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" delete "job/load-$RUN" --ignore-not-found --wait=true --timeout=60s >/dev/null || true
|
||||
fi
|
||||
rm -f "$manifest"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" create configmap load-scripts --from-file=student-tests/load/ --dry-run=client -o yaml | kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f -
|
||||
helm template lab2 charts/messenger -n "$NAMESPACE" -s templates/load.yaml --set load.enabled=true --set-string "load.run=$RUN" --set-string "load.script=$LOAD_SCRIPT" --set "load.duration=$DURATION" --set "load.vus=$VUS" > "$manifest"
|
||||
helm template lab2 charts/messenger -n "$NAMESPACE" -s templates/load.yaml --set load.enabled=true --set-string "load.run=$RUN" --set-string "load.script=$LOAD_SCRIPT" --set "load.duration=$DURATION" --set "load.vus=$VUS" --set-string "load.target=${TARGET_INSTANCE_ID:-}" > "$manifest"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f "$manifest"
|
||||
created=true
|
||||
(
|
||||
while :; do
|
||||
date -u '+UTC %Y-%m-%dT%H:%M:%SZ'
|
||||
@@ -23,18 +38,21 @@ kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" apply -f "$manifest"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" exec postgres-0 -- psql -U messenger -d messenger -Atc "SELECT state,wait_event_type,wait_event,count(*) FROM pg_stat_activity WHERE datname='messenger' GROUP BY state,wait_event_type,wait_event" || true
|
||||
sleep 5
|
||||
done
|
||||
) > "evidence/resources-$RUN.txt" 2>&1 &
|
||||
) > "$EVIDENCE_DIR/resources-$RUN.txt" 2>&1 &
|
||||
sampler=$!
|
||||
complete=false
|
||||
for attempt in $(seq 1 140); do
|
||||
state=$(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get "job/load-$RUN" -o json)
|
||||
if ! state=$(kubectl --context "$KUBE_CONTEXT" --request-timeout=10s -n "$NAMESPACE" get "job/load-$RUN" -o json); then
|
||||
sleep 2
|
||||
continue
|
||||
fi
|
||||
if printf '%s' "$state" | jq -e '.status.succeeded > 0' >/dev/null; then complete=true; break; fi
|
||||
if printf '%s' "$state" | jq -e '.status.failed > 0' >/dev/null; then break; fi
|
||||
sleep 2
|
||||
done
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" logs "job/load-$RUN" > "evidence/load-$RUN.txt"
|
||||
$complete || { echo "Load failed; inspect evidence/load-$RUN.txt" >&2; exit 1; }
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" logs "job/load-$RUN" > "$EVIDENCE_DIR/load-$RUN.txt"
|
||||
$complete || { echo "Load failed; inspect $EVIDENCE_DIR/load-$RUN.txt" >&2; exit 1; }
|
||||
if [ "$LOAD_SCRIPT" = scaling.js ]; then
|
||||
jq -R 'fromjson? | select(has("profile"))' "evidence/load-$RUN.txt" > "evidence/load-$RUN.json"
|
||||
test -s "evidence/load-$RUN.json"
|
||||
jq -R 'fromjson? | select(has("profile"))' "$EVIDENCE_DIR/load-$RUN.txt" > "$EVIDENCE_DIR/load-$RUN.json"
|
||||
test -s "$EVIDENCE_DIR/load-$RUN.json"
|
||||
fi
|
||||
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
trap 'exit 0' TERM INT
|
||||
for sample in $(seq 1 180); do
|
||||
date -u '+UTC %Y-%m-%dT%H:%M:%SZ'
|
||||
kubectl --context dev --request-timeout=10s -n highload-applications exec valkey-0 -- valkey-cli EVAL "local t=redis.call('TIME'); local n=t[1]*1000+math.floor(t[2]/1000); return {redis.call('ZCOUNT',KEYS[1],n,'+inf'),redis.call('GET',KEYS[2]) or '0'}" 2 admission:global:slots admission:global:rate || true
|
||||
sleep 5
|
||||
done
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env ruby
|
||||
# Verify recovery relative to the actual kubectl action, using only non-secret workload events.
|
||||
require 'json'
|
||||
directory, run = ARGV
|
||||
result = File.readlines("#{directory}/load-#{run}.txt").map do |line|
|
||||
match = line.match(/\bmsg=("(?:\\.|[^"])*")/)
|
||||
next unless match
|
||||
message = JSON.parse(match[1])
|
||||
next unless message.start_with?('{')
|
||||
JSON.parse(message)
|
||||
end.compact.find { |item| item['pass'] }
|
||||
abort 'Missing successful failover report' unless result
|
||||
stopped = Float(File.read("#{directory}/#{run}-stop-epoch.txt"))
|
||||
relative_stop = stopped - result.fetch('started_epoch_ms') / 1000.0
|
||||
event = result.fetch('events').find { |item| item['seconds'] >= relative_stop && item['instance'] != result['target'] }
|
||||
abort 'No recovered write' unless event
|
||||
recovery = event['seconds'] - relative_stop
|
||||
abort "Recovery exceeded 10 seconds: #{recovery}" unless recovery <= 10
|
||||
summary = result.reject { |key, _| key == 'events' }.merge('recovery_seconds' => recovery.round(3), 'confirmed_rps' => (result['confirmed'] - 1) / 25.0)
|
||||
File.write("#{directory}/#{run}-summary.json", JSON.pretty_generate(summary)+"\n")
|
||||
puts JSON.generate(summary)
|
||||
Executable
+33
@@ -0,0 +1,33 @@
|
||||
#!/bin/sh
|
||||
# Stop each original pod explicitly and preserve request and Kubernetes evidence.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${KUBE_CONTEXT:=dev}" "${NAMESPACE:=highload-applications}"
|
||||
: "${EVIDENCE_DIR:=evidence/review-20261029}"
|
||||
mkdir -p "$EVIDENCE_DIR"
|
||||
if [ "${RESUME_FROM:-1}" = 1 ]; then
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pods -l app=api -o json | jq ' .items |= map(select(.metadata.deletionTimestamp == null)) ' > "$EVIDENCE_DIR/initial-api.json"
|
||||
test "$(jq '.items|length' "$EVIDENCE_DIR/initial-api.json")" -eq 2
|
||||
jq -r '.items[].metadata.name' "$EVIDENCE_DIR/initial-api.json" > "$EVIDENCE_DIR/targets.txt"
|
||||
fi
|
||||
index=0
|
||||
while read -r target; do
|
||||
index=$((index+1))
|
||||
[ "$index" -ge "${RESUME_FROM:-1}" ] || continue
|
||||
run="failover-$index-$(date -u +%Y%m%d%H%M%S)"
|
||||
TARGET_INSTANCE_ID="$target" RUN="$run" LOAD_SCRIPT=failover.js EVIDENCE_DIR="$EVIDENCE_DIR" scripts/run-load.sh > "$EVIDENCE_DIR/$run-run.txt" 2>&1 &
|
||||
runner=$!
|
||||
ready=false
|
||||
for attempt in $(seq 1 90); do
|
||||
if kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" logs "job/load-$run" 2>/dev/null | rg -q READY_TO_STOP; then ready=true; break; fi
|
||||
kill -0 "$runner" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
if ! $ready; then wait "$runner" || true; echo 'Failover setup did not become ready' >&2; exit 1; fi
|
||||
ruby -e 'puts Time.now.to_f' > "$EVIDENCE_DIR/$run-stop-epoch.txt"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" get pod "$target" -o json > "$EVIDENCE_DIR/$run-target.json"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" delete pod "$target" --wait=false > "$EVIDENCE_DIR/$run-delete.txt"
|
||||
wait "$runner"
|
||||
ruby scripts/summarize-failover.rb "$EVIDENCE_DIR" "$run"
|
||||
kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" rollout status deployment/api --timeout=120s > "$EVIDENCE_DIR/$run-restored.txt"
|
||||
done < "$EVIDENCE_DIR/targets.txt"
|
||||
Reference in New Issue
Block a user