146 lines
7.9 KiB
Bash
146 lines
7.9 KiB
Bash
#!/bin/sh
|
||
set -eu
|
||
# Прогоняет проверки lab3–lab6 на запущенном Docker-стенде и сохраняет вывод в evidence/.
|
||
# Токены и секреты не печатаются (тесты держат их в памяти, mc читает их внутри контейнера).
|
||
#
|
||
# sh scripts/collect-evidence.sh base # уровни 3–4 lab3/lab4 + smoke/persistence/failover lab5
|
||
# sh scripts/collect-evidence.sh files # вложения lab4: негативный набор, отказы S3/БД, приватность, память
|
||
# sh scripts/collect-evidence.sh pipeline # lab5: демонстрация, outbox, падения worker, broker, retry, replay, лимиты
|
||
# sh scripts/collect-evidence.sh obs # lab6: сигналы, сквозная трасса, утечки, инциденты, отказ backend
|
||
# sh scripts/collect-evidence.sh track1 # lab6 трек 1: цена наблюдаемости (off / full / head10 / tail)
|
||
# sh scripts/collect-evidence.sh all
|
||
# Windows (Git Bash): PYTHON=python STOP=scripts/win/stop-one.cmd RESTART=scripts/win/restart.cmd sh scripts/collect-evidence.sh all
|
||
cd "$(dirname "$0")/.."
|
||
PY=${PYTHON:-python3}
|
||
STOP=${STOP:-scripts/stop-one.sh}
|
||
RESTART=${RESTART:-scripts/restart.sh}
|
||
BASE=${BASE_URL:-http://localhost:8080}
|
||
WHAT=${1:-all}
|
||
export PYTHONUTF8=1
|
||
mkdir -p evidence evidence/files evidence/pipeline evidence/obs evidence/track1
|
||
|
||
step() { echo "== $*"; }
|
||
|
||
base() {
|
||
step "состояние стенда и образы"
|
||
{ date -u +%Y-%m-%dT%H:%M:%SZ; docker compose version; docker compose ps; } > evidence/docker_ps.txt
|
||
for img in $(docker compose config --images | sort -u); do
|
||
printf '%s %s\n' "$img" "$(docker image inspect --format '{{index .RepoDigests 0}}' "$img" 2>/dev/null || echo local-build)"
|
||
done > evidence/images.txt
|
||
cat evidence/images.txt
|
||
|
||
step "X-Instance-Id через LB"
|
||
$PY - "$BASE" > evidence/instances.txt <<'EOF'
|
||
import sys, urllib.request, collections
|
||
seen = collections.Counter()
|
||
for _ in range(20):
|
||
with urllib.request.urlopen(sys.argv[1] + '/health/ready') as r:
|
||
seen[r.headers['X-Instance-Id']] += 1
|
||
for k, v in sorted(seen.items()):
|
||
print(k, v)
|
||
EOF
|
||
cat evidence/instances.txt
|
||
|
||
step "smoke (lab5: 202 → ready, thumbnail 256×160)"
|
||
$PY tests/smoke.py --base-url "$BASE" | tee evidence/smoke.txt
|
||
|
||
step "конкурентная идемпотентность"
|
||
$PY student-tests/idempotency_concurrent.py --base-url "$BASE" --output evidence/idempotency_concurrent.json | tail -1
|
||
|
||
step "регрессии lab3"
|
||
$PY student-tests/regressions.py --base-url "$BASE" > evidence/regressions.txt 2>&1
|
||
cat evidence/regressions.txt
|
||
|
||
step "публичный failover (с вложением), 2 прогона"
|
||
for n in 1 2; do
|
||
{ $PY tests/resilience.py --mode failover --action-script "$STOP" --base-url "$BASE" || echo "FAILED"; \
|
||
echo "-- контейнеры api после stop-one:"; docker compose ps -a api --format '{{.Name}} {{.ID}} {{.Status}}'; } \
|
||
> evidence/failover_$n.txt 2>&1
|
||
cat evidence/failover_$n.txt
|
||
sh scripts/restore.sh >> evidence/failover_$n.txt
|
||
done
|
||
|
||
step "persistence (restart всего стенда: PostgreSQL, Redis, S3, RabbitMQ, с volumes)"
|
||
{ echo "-- до:"; docker compose ps --format '{{.Name}} {{.Status}}'; \
|
||
$PY tests/resilience.py --mode persistence --action-script "$RESTART" --base-url "$BASE" || echo "FAILED"; \
|
||
echo "-- после:"; docker compose ps --format '{{.Name}} {{.Status}}'; } > evidence/persistence.txt 2>&1
|
||
cat evidence/persistence.txt
|
||
|
||
step "серия отказов каждой реплики под нагрузкой"
|
||
$PY student-tests/failover_series.py --base-url "$BASE" --output evidence/failover_series.json | tail -1
|
||
|
||
step "rolling update под нагрузкой"
|
||
docker compose build -q api
|
||
{ echo "-- до:"; docker compose ps api --format '{{.Name}} {{.ID}} {{.Status}}'; } > evidence/rolling_update.txt
|
||
$PY tests/load.py --base-url "$BASE" --duration 90 --concurrency 4 --output evidence/rolling_update_load.json > /dev/null &
|
||
LOAD=$!
|
||
sleep 5
|
||
start=$(date +%s)
|
||
sh scripts/rolling-update.sh >> evidence/rolling_update.txt 2>&1
|
||
echo "-- замена заняла $(( $(date +%s) - start )) s (внутри 90 s нагрузки)" >> evidence/rolling_update.txt
|
||
wait $LOAD
|
||
tail -4 evidence/rolling_update.txt; cat evidence/rolling_update_load.json
|
||
}
|
||
|
||
files() {
|
||
step "негативный набор вложений"
|
||
$PY student-tests/attachments_negative.py --base-url "$BASE" > evidence/files/attachments_negative.txt 2>&1 || echo "FAILED: attachments_negative"
|
||
tail -3 evidence/files/attachments_negative.txt
|
||
for s in failover s3-outage interrupt security db-gap memory; do
|
||
step "вложения: $s"
|
||
$PY student-tests/attachments_ops.py "$s" --base-url "$BASE" --output "evidence/files/files_$s.json" > "evidence/files/files_$s.log" 2>&1 || echo "FAILED: $s"
|
||
tail -1 "evidence/files/files_$s.log"
|
||
done
|
||
}
|
||
|
||
pipeline() {
|
||
for s in demo outbox-gap crash-effect crash-commit broker-restart transient failed-replay limits; do
|
||
step "pipeline: $s"
|
||
$PY student-tests/pipeline_ops.py "$s" --base-url "$BASE" --output "evidence/pipeline/$s.json" > "evidence/pipeline/$s.log" 2>&1 || echo "FAILED: $s"
|
||
tail -1 "evidence/pipeline/$s.log"
|
||
done
|
||
docker compose logs --no-color worker relay 2>&1 | grep -E 'FAULT|lease|duplicate|retry|dead-letter|failed' | tail -60 > evidence/pipeline/worker_relay_log.txt || true
|
||
}
|
||
|
||
obs() {
|
||
for s in signals retry-trace incident-worker incident-db backend-down; do
|
||
step "наблюдаемость: $s"
|
||
$PY student-tests/observability_ops.py "$s" --base-url "$BASE" --output "evidence/obs/$s.json" > "evidence/obs/$s.log" 2>&1 || echo "FAILED: $s"
|
||
tail -1 "evidence/obs/$s.log"
|
||
done
|
||
step "правила, alerts, имена метрик (из Prometheus через Grafana)"
|
||
(cd student-tests && $PY - <<'EOF' > ../evidence/obs/prometheus_state.json
|
||
import json, obsutil as o
|
||
names = o.prom_label_values('__name__')
|
||
keep = [n for n in names if n.startswith(('http_', 'messenger', 'pgxpool', 'process_', 'go_', 'otelcol_exporter', 'otelcol_receiver', 'rabbitmq_detailed_queue_messages', 'target_info'))]
|
||
rules = o.grafana('/api/datasources/proxy/uid/prometheus/api/v1/rules')['data']['groups']
|
||
print(json.dumps({'metric_names': keep,
|
||
'rules': [{'group': g['name'], 'name': r['name'], 'type': r['type'], 'health': r['health'],
|
||
'state': r.get('state'), 'query': r['query'], 'duration': r.get('duration')}
|
||
for g in rules for r in g['rules']],
|
||
'alerts_now': o.alerts()}, ensure_ascii=False, indent=1))
|
||
EOF
|
||
) || echo "FAILED: prometheus_state"
|
||
step "снимок dashboard (headless Chrome, анонимный Viewer только в CI)"
|
||
if command -v google-chrome >/dev/null 2>&1; then
|
||
google-chrome --headless=new --no-sandbox --disable-gpu --hide-scrollbars --window-size=1600,3000 --virtual-time-budget=25000 --screenshot=evidence/obs/dashboard.png "http://127.0.0.1:3000/d/messenger-overview/?orgId=1&from=now-45m&to=now&kiosk" >/dev/null 2>&1 || echo "screenshot failed"
|
||
fi
|
||
}
|
||
|
||
track1() {
|
||
step "трек 1: цена наблюдаемости (3 повтора × 4 конфигурации)"
|
||
$PY student-tests/telemetry_cost.py --rounds 3 --base-url "$BASE" --output evidence/track1/telemetry_cost.json > evidence/track1/telemetry_cost.log 2>&1 || echo "FAILED: telemetry_cost"
|
||
tail -60 evidence/track1/telemetry_cost.log
|
||
}
|
||
|
||
case "$WHAT" in
|
||
base) base ;;
|
||
files) files ;;
|
||
pipeline) pipeline ;;
|
||
obs) obs ;;
|
||
track1) track1 ;;
|
||
all) base; files; pipeline; obs; track1 ;;
|
||
*) echo "usage: $0 base|files|pipeline|obs|track1|all" >&2; exit 2 ;;
|
||
esac
|
||
docker stats --no-stream --format '{{.Name}} {{.CPUPerc}} {{.MemUsage}}' > "evidence/docker_stats_$WHAT.txt"
|