PYTHON ?= python3 BASE_URL ?= http://localhost:8080 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) .PHONY: help check validate test load test-persistence test-failover \ secrets up wait ps down migrate restore rolling-update logs-lb \ test-student test-idempotency test-failover-series test-failover-series-kill \ test-rolling test-storage test-expiry test-failures test-revocation explain evidence \ test-attachments test-consistency test-s3-outage test-upload-memory s3-check wait-s3 reconcile reconcile-apply \ test-pipeline test-pipeline-faults test-worker-outage worker-stop worker-start jobs-stats jobs-failed replay \ obs-urls obs-probe incident-worker incident-db incident-telemetry alerts-watch destroy help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @echo "up/wait/ps/down/restore/rolling-update: stand; test-*: own checks (see REPORT.md)" @echo "lab4: test-attachments test-consistency test-s3-outage test-upload-memory s3-check reconcile" @echo "lab5: test-pipeline test-pipeline-faults test-worker-outage worker-stop worker-start jobs-stats jobs-failed replay ID=..." @echo "lab6: obs-urls obs-probe incident-worker incident-db incident-telemetry alerts-watch" check: $(PYTHON) scripts/check_contract.py validate: $(PYTHON) scripts/validate_openapi.py test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) # ---------------------------------------------------------------- Решение COMPOSE ?= docker compose --env-file .env secrets: $(PYTHON) scripts/init_secrets.py @test -f .env || cp .env.example .env up: secrets $(COMPOSE) config --quiet $(COMPOSE) up -d --build $(PYTHON) scripts/ops.py wait $(PYTHON) scripts/ops.py wait-s3 $(PYTHON) scripts/ops.py worker-start wait: $(PYTHON) scripts/ops.py wait ps: $(PYTHON) scripts/ops.py ps # Безопасная остановка: volumes (pgdata, s3data, brokerdata, promdata, lokidata, tempodata, grafanadata) сохраняются. down: $(COMPOSE) --profile ttl --profile faults down # НАМЕРЕННОЕ удаление всех учебных данных (БД, S3, брокер, история телеметрии). destroy: $(COMPOSE) --profile ttl --profile faults down -v migrate: $(COMPOSE) run --rm -T migrate restore: $(PYTHON) scripts/ops.py restore rolling-update: $(PYTHON) scripts/ops.py rolling-update logs-lb: $(COMPOSE) logs --tail 50 lb test-student: $(PYTHON) student-tests/run_all.py --base-url "$(BASE_URL)" test-idempotency: BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p test_idempotency.py -v test-failover-series: $(PYTHON) student-tests/failover_series.py --base-url "$(BASE_URL)" --output evidence/failover_series.json test-failover-series-kill: $(PYTHON) student-tests/failover_series.py --base-url "$(BASE_URL)" --mode kill --output evidence/failover_series_kill.json test-rolling: $(PYTHON) student-tests/rolling_update_load.py --base-url "$(BASE_URL)" --output evidence/rolling_update.json test-storage: $(PYTHON) scripts/ops.py in-network storage test-expiry: $(PYTHON) scripts/ops.py in-network expiry test-failures: $(PYTHON) student-tests/dependency_failure.py --base-url "$(BASE_URL)" --output evidence/dependency_failure.json test-revocation: $(PYTHON) student-tests/revocation_partition.py --base-url "$(BASE_URL)" | tee evidence/revocation_partition.txt explain: $(COMPOSE) exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 < scripts/sql/explain_history.sql > evidence/explain_history.txt @echo "saved evidence/explain_history.txt" # Все проверки на compose-стенде с записью результатов в evidence/ (~6-8 минут). evidence: $(PYTHON) scripts/collect_evidence.py # ---------------------------------------------------------------- lab4: вложения и S3 test-attachments: BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p test_attachments.py -v test-consistency: $(PYTHON) scripts/ops.py in-network consistency test-s3-outage: $(PYTHON) student-tests/s3_outage.py --base-url "$(BASE_URL)" --output evidence/s3_outage.json test-upload-memory: $(PYTHON) student-tests/upload_memory.py --base-url "$(BASE_URL)" --concurrency 10 --rounds 3 --docker --output evidence/upload_memory_c10.json s3-check: $(PYTHON) scripts/ops.py s3-check wait-s3: $(PYTHON) scripts/ops.py wait-s3 reconcile: $(PYTHON) scripts/ops.py reconcile reconcile-apply: $(PYTHON) scripts/ops.py reconcile --apply # ---------------------------------------------------------------- lab5: очередь и worker test-pipeline: BASE_URL="$(BASE_URL)" $(PYTHON) -m unittest discover -s student-tests -p test_pipeline.py -v # Внедрённые отказы (outbox, crash до/после ack, retry/backoff, DLQ, replay, lease). ~2-3 мин. test-pipeline-faults: $(PYTHON) scripts/ops.py pipeline-faults # Сценарий защиты уровня 3: worker остановлен -> 202 -> рестарт API и broker -> worker -> ready. test-worker-outage: $(PYTHON) student-tests/worker_outage.py --base-url "$(BASE_URL)" --output evidence/worker_outage.json worker-stop: $(PYTHON) scripts/ops.py worker-stop worker-start: $(PYTHON) scripts/ops.py worker-start jobs-stats: $(PYTHON) scripts/ops.py jobs stats jobs-failed: $(PYTHON) scripts/ops.py jobs failed # make replay ID= (или ID=--all-failed) replay: @test -n "$(ID)" || (echo "usage: make replay ID=|--all-failed"; exit 2) $(PYTHON) scripts/ops.py jobs replay $(ID) # ---------------------------------------------------------------- lab6: наблюдаемость obs-urls: @echo "Grafana: http://127.0.0.1:3000 (логин admin, пароль: secrets/grafana_admin_password)" @echo "Prometheus: http://127.0.0.1:9090/alerts" # Контролируемый upload: печатает X-Request-Id и готовые запросы TraceQL/LogQL obs-probe: $(PYTHON) scripts/incident.py --base-url "$(BASE_URL)" probe # Инцидент 1: worker остановлен ~150 с под upload'ами -> QueueLagGrowing firing -> resolved incident-worker: $(PYTHON) scripts/incident.py --base-url "$(BASE_URL)" worker-stop # Инцидент 2: таблица messages заблокирована 180 с под нагрузкой -> ApiHighErrorRate firing -> resolved incident-db: $(PYTHON) scripts/incident.py --base-url "$(BASE_URL)" db-lock # Инцидент 3: остановить telemetry (по умолчанию otel-collector; SERVICES=tempo,loki) SERVICES ?= otel-collector incident-telemetry: $(PYTHON) scripts/incident.py --base-url "$(BASE_URL)" telemetry-down --services "$(SERVICES)" # Журнал переходов alert'ов (запускать в отдельном терминале параллельно с инцидентом) alerts-watch: $(PYTHON) scripts/alerts_watch.py --duration 900