forked from mtusi2026/mtusi2026lab6
196 lines
7.4 KiB
Makefile
196 lines
7.4 KiB
Makefile
PYTHON ?= python3
|
||
BASE_URL ?= http://localhost:8080
|
||
CA_FILE ?=
|
||
ACTION_SCRIPT ?=
|
||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||
|
||
.PHONY: help check validate test load test-persistence test-failover \
|
||
secrets up wait ps down migrate restore rolling-update logs-lb \
|
||
test-student test-idempotency test-failover-series test-failover-series-kill \
|
||
test-rolling test-storage test-expiry test-failures test-revocation explain evidence \
|
||
test-attachments test-consistency test-s3-outage test-upload-memory s3-check wait-s3 reconcile reconcile-apply \
|
||
test-pipeline test-pipeline-faults test-worker-outage worker-stop worker-start jobs-stats jobs-failed replay \
|
||
obs-urls obs-probe incident-worker incident-db incident-telemetry alerts-watch destroy
|
||
|
||
help:
|
||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||
@echo "up/wait/ps/down/restore/rolling-update: stand; test-*: own checks (see REPORT.md)"
|
||
@echo "lab4: test-attachments test-consistency test-s3-outage test-upload-memory s3-check reconcile"
|
||
@echo "lab5: test-pipeline test-pipeline-faults test-worker-outage worker-stop worker-start jobs-stats jobs-failed replay ID=..."
|
||
@echo "lab6: obs-urls obs-probe incident-worker incident-db incident-telemetry alerts-watch"
|
||
|
||
check:
|
||
$(PYTHON) scripts/check_contract.py
|
||
|
||
validate:
|
||
$(PYTHON) scripts/validate_openapi.py
|
||
|
||
test:
|
||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||
|
||
load:
|
||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||
|
||
test-persistence:
|
||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||
|
||
test-failover:
|
||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||
|
||
# ---------------------------------------------------------------- Решение
|
||
COMPOSE ?= docker compose --env-file .env
|
||
|
||
secrets:
|
||
$(PYTHON) scripts/init_secrets.py
|
||
@test -f .env || cp .env.example .env
|
||
|
||
up: secrets
|
||
$(COMPOSE) config --quiet
|
||
$(COMPOSE) up -d --build
|
||
$(PYTHON) scripts/ops.py wait
|
||
$(PYTHON) scripts/ops.py wait-s3
|
||
$(PYTHON) scripts/ops.py worker-start
|
||
|
||
wait:
|
||
$(PYTHON) scripts/ops.py wait
|
||
|
||
ps:
|
||
$(PYTHON) scripts/ops.py ps
|
||
|
||
# Безопасная остановка: volumes (pgdata, s3data, brokerdata, promdata, lokidata, tempodata, grafanadata) сохраняются.
|
||
down:
|
||
$(COMPOSE) --profile ttl --profile faults down
|
||
|
||
# НАМЕРЕННОЕ удаление всех учебных данных (БД, S3, брокер, история телеметрии).
|
||
destroy:
|
||
$(COMPOSE) --profile ttl --profile faults down -v
|
||
|
||
migrate:
|
||
$(COMPOSE) run --rm -T migrate
|
||
|
||
restore:
|
||
$(PYTHON) scripts/ops.py restore
|
||
|
||
rolling-update:
|
||
$(PYTHON) scripts/ops.py rolling-update
|
||
|
||
logs-lb:
|
||
$(COMPOSE) logs --tail 50 lb
|
||
|
||
test-student:
|
||
$(PYTHON) student-tests/run_all.py --base-url "$(BASE_URL)"
|
||
|
||
test-idempotency:
|
||
BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p test_idempotency.py -v
|
||
|
||
test-failover-series:
|
||
$(PYTHON) student-tests/failover_series.py --base-url "$(BASE_URL)" --output evidence/failover_series.json
|
||
|
||
test-failover-series-kill:
|
||
$(PYTHON) student-tests/failover_series.py --base-url "$(BASE_URL)" --mode kill --output evidence/failover_series_kill.json
|
||
|
||
test-rolling:
|
||
$(PYTHON) student-tests/rolling_update_load.py --base-url "$(BASE_URL)" --output evidence/rolling_update.json
|
||
|
||
test-storage:
|
||
$(PYTHON) scripts/ops.py in-network storage
|
||
|
||
test-expiry:
|
||
$(PYTHON) scripts/ops.py in-network expiry
|
||
|
||
test-failures:
|
||
$(PYTHON) student-tests/dependency_failure.py --base-url "$(BASE_URL)" --output evidence/dependency_failure.json
|
||
|
||
test-revocation:
|
||
$(PYTHON) student-tests/revocation_partition.py --base-url "$(BASE_URL)" | tee evidence/revocation_partition.txt
|
||
|
||
explain:
|
||
$(COMPOSE) exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 < scripts/sql/explain_history.sql > evidence/explain_history.txt
|
||
@echo "saved evidence/explain_history.txt"
|
||
|
||
# Все проверки на compose-стенде с записью результатов в evidence/ (~6-8 минут).
|
||
evidence:
|
||
$(PYTHON) scripts/collect_evidence.py
|
||
|
||
# ---------------------------------------------------------------- lab4: вложения и S3
|
||
test-attachments:
|
||
BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p test_attachments.py -v
|
||
|
||
test-consistency:
|
||
$(PYTHON) scripts/ops.py in-network consistency
|
||
|
||
test-s3-outage:
|
||
$(PYTHON) student-tests/s3_outage.py --base-url "$(BASE_URL)" --output evidence/s3_outage.json
|
||
|
||
test-upload-memory:
|
||
$(PYTHON) student-tests/upload_memory.py --base-url "$(BASE_URL)" --concurrency 10 --rounds 3 --docker --output evidence/upload_memory_c10.json
|
||
|
||
s3-check:
|
||
$(PYTHON) scripts/ops.py s3-check
|
||
|
||
wait-s3:
|
||
$(PYTHON) scripts/ops.py wait-s3
|
||
|
||
reconcile:
|
||
$(PYTHON) scripts/ops.py reconcile
|
||
|
||
reconcile-apply:
|
||
$(PYTHON) scripts/ops.py reconcile --apply
|
||
|
||
# ---------------------------------------------------------------- lab5: очередь и worker
|
||
test-pipeline:
|
||
BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p test_pipeline.py -v
|
||
|
||
# Внедрённые отказы (outbox, crash до/после ack, retry/backoff, DLQ, replay, lease). ~2-3 мин.
|
||
test-pipeline-faults:
|
||
$(PYTHON) scripts/ops.py pipeline-faults
|
||
|
||
# Сценарий защиты уровня 3: worker остановлен -> 202 -> рестарт API и broker -> worker -> ready.
|
||
test-worker-outage:
|
||
$(PYTHON) student-tests/worker_outage.py --base-url "$(BASE_URL)" --output evidence/worker_outage.json
|
||
|
||
worker-stop:
|
||
$(PYTHON) scripts/ops.py worker-stop
|
||
|
||
worker-start:
|
||
$(PYTHON) scripts/ops.py worker-start
|
||
|
||
jobs-stats:
|
||
$(PYTHON) scripts/ops.py jobs stats
|
||
|
||
jobs-failed:
|
||
$(PYTHON) scripts/ops.py jobs failed
|
||
|
||
# make replay ID=<attachment_id> (или ID=--all-failed)
|
||
replay:
|
||
@test -n "$(ID)" || (echo "usage: make replay ID=<attachment_id>|--all-failed"; exit 2)
|
||
$(PYTHON) scripts/ops.py jobs replay $(ID)
|
||
|
||
# ---------------------------------------------------------------- lab6: наблюдаемость
|
||
obs-urls:
|
||
@echo "Grafana: http://127.0.0.1:3000 (логин admin, пароль: secrets/grafana_admin_password)"
|
||
@echo "Prometheus: http://127.0.0.1:9090/alerts"
|
||
|
||
# Контролируемый upload: печатает X-Request-Id и готовые запросы TraceQL/LogQL
|
||
obs-probe:
|
||
$(PYTHON) scripts/incident.py --base-url "$(BASE_URL)" probe
|
||
|
||
# Инцидент 1: worker остановлен ~150 с под upload'ами -> QueueLagGrowing firing -> resolved
|
||
incident-worker:
|
||
$(PYTHON) scripts/incident.py --base-url "$(BASE_URL)" worker-stop
|
||
|
||
# Инцидент 2: таблица messages заблокирована 180 с под нагрузкой -> ApiHighErrorRate firing -> resolved
|
||
incident-db:
|
||
$(PYTHON) scripts/incident.py --base-url "$(BASE_URL)" db-lock
|
||
|
||
# Инцидент 3: остановить telemetry (по умолчанию otel-collector; SERVICES=tempo,loki)
|
||
SERVICES ?= otel-collector
|
||
incident-telemetry:
|
||
$(PYTHON) scripts/incident.py --base-url "$(BASE_URL)" telemetry-down --services "$(SERVICES)"
|
||
|
||
# Журнал переходов alert'ов (запускать в отдельном терминале параллельно с инцидентом)
|
||
alerts-watch:
|
||
$(PYTHON) scripts/alerts_watch.py --duration 900
|