forked from mtusi2026/mtusi2026lab7
108 lines
4.4 KiB
Bash
Executable File
108 lines
4.4 KiB
Bash
Executable File
#!/bin/sh
|
|
|
|
set -eu
|
|
|
|
mode=${1:-short}
|
|
env_file=${ENV_FILE:-lab6.env.example}
|
|
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
|
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
|
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
|
|
require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1}
|
|
case "$mode" in
|
|
short)
|
|
incident_seconds=${SHORT_WORKER_OUTAGE_SECONDS:-75}
|
|
load_seconds=${SHORT_WORKER_LOAD_SECONDS:-60}
|
|
requests=${SHORT_WORKER_REQUESTS:-300}
|
|
default_alert=AttachmentWorkerUnavailable
|
|
;;
|
|
long)
|
|
incident_seconds=${LONG_WORKER_OUTAGE_SECONDS:-180}
|
|
load_seconds=${LONG_WORKER_LOAD_SECONDS:-120}
|
|
requests=${LONG_WORKER_REQUESTS:-1000}
|
|
default_alert=AttachmentQueueBacklogGrowing
|
|
;;
|
|
*)
|
|
echo "usage: $0 [short|long]" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
worker_alert_name=${WORKER_ALERT_NAME:-$default_alert}
|
|
|
|
run_id=$(date -u +%Y%m%dT%H%M%SZ)
|
|
output_dir="$evidence_root/worker-$mode-$run_id"
|
|
mkdir -p "$output_dir"
|
|
load_pid=
|
|
|
|
compose() {
|
|
docker compose --env-file "$env_file" "$@"
|
|
}
|
|
|
|
restore() {
|
|
if [ -n "$load_pid" ]; then
|
|
kill "$load_pid" >/dev/null 2>&1 || true
|
|
fi
|
|
compose up -d worker >/dev/null 2>&1 || true
|
|
}
|
|
|
|
trap restore EXIT INT TERM
|
|
|
|
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
|
|
started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
compose stop worker >/dev/null
|
|
python3 scripts/lab6-traffic.py \
|
|
--base-url "$base_url" \
|
|
--mode attachments \
|
|
--duration "$load_seconds" \
|
|
--requests "$requests" \
|
|
--concurrency 4 \
|
|
--rate "${WORKER_LOAD_RATE:-5}" \
|
|
--wait-ready \
|
|
--ready-timeout 480 \
|
|
--output "$output_dir/traffic.json" >/dev/null &
|
|
load_pid=$!
|
|
sleep "$incident_seconds"
|
|
if [ "$require_alerts" = "1" ]; then
|
|
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null
|
|
fi
|
|
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
|
|
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
compose up -d --wait --wait-timeout 90 worker >/dev/null
|
|
wait "$load_pid"
|
|
load_pid=
|
|
if [ "$require_alerts" = "1" ]; then
|
|
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$worker_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null
|
|
if [ "$mode" = "long" ]; then
|
|
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnFast --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-fast-firing.json" >/dev/null
|
|
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name AttachmentReadyHighErrorBudgetBurnSlow --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-slow-firing.json" >/dev/null
|
|
fi
|
|
fi
|
|
sleep "${RECOVERY_OBSERVATION_SECONDS:-20}"
|
|
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null
|
|
recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
python3 - "$output_dir/summary.json" "$mode" "$started_at" "$recovery_started_at" "$recovered_at" "$incident_seconds" <<'PY'
|
|
import datetime
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
output, mode, started, recovery_started, recovered, duration = sys.argv[1:]
|
|
parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
report = {
|
|
"schema_version": 1,
|
|
"incident": "worker_unavailable",
|
|
"profile": mode,
|
|
"started_at": started,
|
|
"recovery_started_at": recovery_started,
|
|
"recovered_at": recovered,
|
|
"planned_injection_seconds": int(duration),
|
|
"injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()),
|
|
"recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()),
|
|
"artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "alert-firing.json", "alert-resolved.json"],
|
|
}
|
|
if mode == "long":
|
|
report["artifacts"].extend(["burn-fast-firing.json", "burn-slow-firing.json"])
|
|
pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
PY
|
|
trap - EXIT INT TERM
|
|
printf '%s\n' "$output_dir"
|