Files
mtusi2026lab6/scripts/collect-evidence.sh
T

146 lines
7.9 KiB
Bash
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/sh
set -eu
# Прогоняет проверки lab3–lab6 на запущенном Docker-стенде и сохраняет вывод в evidence/.
# Токены и секреты не печатаются (тесты держат их в памяти, mc читает их внутри контейнера).
#
# sh scripts/collect-evidence.sh base # уровни 3–4 lab3/lab4 + smoke/persistence/failover lab5
# sh scripts/collect-evidence.sh files # вложения lab4: негативный набор, отказы S3/БД, приватность, память
# sh scripts/collect-evidence.sh pipeline # lab5: демонстрация, outbox, падения worker, broker, retry, replay, лимиты
# sh scripts/collect-evidence.sh obs # lab6: сигналы, сквозная трасса, утечки, инциденты, отказ backend
# sh scripts/collect-evidence.sh track1 # lab6 трек 1: цена наблюдаемости (off / full / head10 / tail)
# sh scripts/collect-evidence.sh all
# Windows (Git Bash): PYTHON=python STOP=scripts/win/stop-one.cmd RESTART=scripts/win/restart.cmd sh scripts/collect-evidence.sh all
cd "$(dirname "$0")/.."
PY=${PYTHON:-python3}
STOP=${STOP:-scripts/stop-one.sh}
RESTART=${RESTART:-scripts/restart.sh}
BASE=${BASE_URL:-http://localhost:8080}
WHAT=${1:-all}
export PYTHONUTF8=1
mkdir -p evidence evidence/files evidence/pipeline evidence/obs evidence/track1
step() { echo "== $*"; }
base() {
step "состояние стенда и образы"
{ date -u +%Y-%m-%dT%H:%M:%SZ; docker compose version; docker compose ps; } > evidence/docker_ps.txt
for img in $(docker compose config --images | sort -u); do
printf '%s %s\n' "$img" "$(docker image inspect --format '{{index .RepoDigests 0}}' "$img" 2>/dev/null || echo local-build)"
done > evidence/images.txt
cat evidence/images.txt
step "X-Instance-Id через LB"
$PY - "$BASE" > evidence/instances.txt <<'EOF'
import sys, urllib.request, collections
seen = collections.Counter()
for _ in range(20):
with urllib.request.urlopen(sys.argv[1] + '/health/ready') as r:
seen[r.headers['X-Instance-Id']] += 1
for k, v in sorted(seen.items()):
print(k, v)
EOF
cat evidence/instances.txt
step "smoke (lab5: 202 → ready, thumbnail 256×160)"
$PY tests/smoke.py --base-url "$BASE" | tee evidence/smoke.txt
step "конкурентная идемпотентность"
$PY student-tests/idempotency_concurrent.py --base-url "$BASE" --output evidence/idempotency_concurrent.json | tail -1
step "регрессии lab3"
$PY student-tests/regressions.py --base-url "$BASE" > evidence/regressions.txt 2>&1
cat evidence/regressions.txt
step "публичный failover (с вложением), 2 прогона"
for n in 1 2; do
{ $PY tests/resilience.py --mode failover --action-script "$STOP" --base-url "$BASE" || echo "FAILED"; \
echo "-- контейнеры api после stop-one:"; docker compose ps -a api --format '{{.Name}} {{.ID}} {{.Status}}'; } \
> evidence/failover_$n.txt 2>&1
cat evidence/failover_$n.txt
sh scripts/restore.sh >> evidence/failover_$n.txt
done
step "persistence (restart всего стенда: PostgreSQL, Redis, S3, RabbitMQ, с volumes)"
{ echo "-- до:"; docker compose ps --format '{{.Name}} {{.Status}}'; \
$PY tests/resilience.py --mode persistence --action-script "$RESTART" --base-url "$BASE" || echo "FAILED"; \
echo "-- после:"; docker compose ps --format '{{.Name}} {{.Status}}'; } > evidence/persistence.txt 2>&1
cat evidence/persistence.txt
step "серия отказов каждой реплики под нагрузкой"
$PY student-tests/failover_series.py --base-url "$BASE" --output evidence/failover_series.json | tail -1
step "rolling update под нагрузкой"
docker compose build -q api
{ echo "-- до:"; docker compose ps api --format '{{.Name}} {{.ID}} {{.Status}}'; } > evidence/rolling_update.txt
$PY tests/load.py --base-url "$BASE" --duration 90 --concurrency 4 --output evidence/rolling_update_load.json > /dev/null &
LOAD=$!
sleep 5
start=$(date +%s)
sh scripts/rolling-update.sh >> evidence/rolling_update.txt 2>&1
echo "-- замена заняла $(( $(date +%s) - start )) s (внутри 90 s нагрузки)" >> evidence/rolling_update.txt
wait $LOAD
tail -4 evidence/rolling_update.txt; cat evidence/rolling_update_load.json
}
files() {
step "негативный набор вложений"
$PY student-tests/attachments_negative.py --base-url "$BASE" > evidence/files/attachments_negative.txt 2>&1 || echo "FAILED: attachments_negative"
tail -3 evidence/files/attachments_negative.txt
for s in failover s3-outage interrupt security db-gap memory; do
step "вложения: $s"
$PY student-tests/attachments_ops.py "$s" --base-url "$BASE" --output "evidence/files/files_$s.json" > "evidence/files/files_$s.log" 2>&1 || echo "FAILED: $s"
tail -1 "evidence/files/files_$s.log"
done
}
pipeline() {
for s in demo outbox-gap crash-effect crash-commit broker-restart transient failed-replay limits; do
step "pipeline: $s"
$PY student-tests/pipeline_ops.py "$s" --base-url "$BASE" --output "evidence/pipeline/$s.json" > "evidence/pipeline/$s.log" 2>&1 || echo "FAILED: $s"
tail -1 "evidence/pipeline/$s.log"
done
docker compose logs --no-color worker relay 2>&1 | grep -E 'FAULT|lease|duplicate|retry|dead-letter|failed' | tail -60 > evidence/pipeline/worker_relay_log.txt || true
}
obs() {
for s in signals retry-trace incident-worker incident-db backend-down; do
step "наблюдаемость: $s"
$PY student-tests/observability_ops.py "$s" --base-url "$BASE" --output "evidence/obs/$s.json" > "evidence/obs/$s.log" 2>&1 || echo "FAILED: $s"
tail -1 "evidence/obs/$s.log"
done
step "правила, alerts, имена метрик (из Prometheus через Grafana)"
(cd student-tests && $PY - <<'EOF' > ../evidence/obs/prometheus_state.json
import json, obsutil as o
names = o.prom_label_values('__name__')
keep = [n for n in names if n.startswith(('http_', 'messenger', 'pgxpool', 'process_', 'go_', 'otelcol_exporter', 'otelcol_receiver', 'rabbitmq_detailed_queue_messages', 'target_info'))]
rules = o.grafana('/api/datasources/proxy/uid/prometheus/api/v1/rules')['data']['groups']
print(json.dumps({'metric_names': keep,
'rules': [{'group': g['name'], 'name': r['name'], 'type': r['type'], 'health': r['health'],
'state': r.get('state'), 'query': r['query'], 'duration': r.get('duration')}
for g in rules for r in g['rules']],
'alerts_now': o.alerts()}, ensure_ascii=False, indent=1))
EOF
) || echo "FAILED: prometheus_state"
step "снимок dashboard (headless Chrome, анонимный Viewer только в CI)"
if command -v google-chrome >/dev/null 2>&1; then
google-chrome --headless=new --no-sandbox --disable-gpu --hide-scrollbars --window-size=1600,3000 --virtual-time-budget=25000 --screenshot=evidence/obs/dashboard.png "http://127.0.0.1:3000/d/messenger-overview/?orgId=1&from=now-45m&to=now&kiosk" >/dev/null 2>&1 || echo "screenshot failed"
fi
}
track1() {
step "трек 1: цена наблюдаемости (3 повтора × 4 конфигурации)"
$PY student-tests/telemetry_cost.py --rounds 3 --base-url "$BASE" --output evidence/track1/telemetry_cost.json > evidence/track1/telemetry_cost.log 2>&1 || echo "FAILED: telemetry_cost"
tail -60 evidence/track1/telemetry_cost.log
}
case "$WHAT" in
base) base ;;
files) files ;;
pipeline) pipeline ;;
obs) obs ;;
track1) track1 ;;
all) base; files; pipeline; obs; track1 ;;
*) echo "usage: $0 base|files|pipeline|obs|track1|all" >&2; exit 2 ;;
esac
docker stats --no-stream --format '{{.Name}} {{.CPUPerc}} {{.MemUsage}}' > "evidence/docker_stats_$WHAT.txt"