Files
mtusi2026lab7/scripts/incident-database.sh
T
2026-11-01 17:59:39 +03:00

92 lines
4.0 KiB
Bash
Executable File

#!/bin/sh
set -eu
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
outage_seconds=${DB_OUTAGE_SECONDS:-75}
require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1}
api_alert_name=${API_ALERT_NAME:-MessengerAPIBusiness5xxHigh}
burn_alert_name=${API_BURN_ALERT_NAME:-MessengerAPIHighErrorBudgetBurnFast}
run_id=$(date -u +%Y%m%dT%H%M%SZ)
output_dir="$evidence_root/database-$run_id"
mkdir -p "$output_dir"
load_pid=
compose() {
docker compose --env-file "$env_file" "$@"
}
restore() {
if [ -n "$load_pid" ]; then
kill "$load_pid" >/dev/null 2>&1 || true
fi
compose up -d postgres migrate api nginx >/dev/null 2>&1 || true
}
trap restore EXIT INT TERM
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--mode messages \
--duration "$((outage_seconds + 20))" \
--concurrency 4 \
--rate "${DB_LOAD_RATE:-20}" \
--started-file "$output_dir/load-started" \
--output "$output_dir/traffic.json" >/dev/null &
load_pid=$!
remaining=60
while [ ! -f "$output_dir/load-started" ] && [ "$remaining" -gt 0 ]; do
sleep 0.5
remaining=$((remaining - 1))
done
test -f "$output_dir/load-started"
sleep 2
started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose stop postgres >/dev/null
sleep "$outage_seconds"
if [ "$require_alerts" = "1" ]; then
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$api_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/alert-firing.json" >/dev/null
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$burn_alert_name" --state firing --timeout "${ALERT_WAIT_SECONDS:-90}" --output "$output_dir/burn-alert-firing.json" >/dev/null
fi
python3 scripts/lab6-correlation.py --mode database --tempo-url "$tempo_url" --loki-url "$loki_url" --output "$output_dir/database-correlation.json" --require >/dev/null
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose up -d --wait --wait-timeout 90 postgres migrate api nginx >/dev/null
wait "$load_pid" || true
load_pid=
if [ "$require_alerts" = "1" ]; then
python3 scripts/lab6-alert.py --prometheus-url "$prometheus_url" --name "$api_alert_name" --state resolved --timeout "${ALERT_RESOLVE_SECONDS:-180}" --output "$output_dir/alert-resolved.json" >/dev/null
fi
sleep "${RECOVERY_OBSERVATION_SECONDS:-20}"
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase recovered --output "$output_dir/recovered.json" >/dev/null
recovered_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
python3 - "$output_dir/summary.json" "$started_at" "$recovery_started_at" "$recovered_at" "$outage_seconds" <<'PY'
import datetime
import json
import pathlib
import sys
output, started, recovery_started, recovered, duration = sys.argv[1:]
parse = lambda value: datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
report = {
"schema_version": 1,
"incident": "postgres_unavailable",
"started_at": started,
"recovery_started_at": recovery_started,
"recovered_at": recovered,
"planned_injection_seconds": int(duration),
"injected_duration_seconds": int((parse(recovery_started) - parse(started)).total_seconds()),
"recovery_observation_seconds": int((parse(recovered) - parse(recovery_started)).total_seconds()),
"artifacts": ["baseline.json", "degraded.json", "recovered.json", "traffic.json", "database-correlation.json", "alert-firing.json", "alert-resolved.json", "burn-alert-firing.json"],
}
pathlib.Path(output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
PY
trap - EXIT INT TERM
printf '%s\n' "$output_dir"