From 5ac64f51faecefc09ebc795f62dc73d81410fd75 Mon Sep 17 00:00:00 2001 From: dizi Date: Fri, 6 Nov 2026 21:17:03 +0300 Subject: [PATCH] test: verify security boundaries and recovery --- .gitea/workflows/ci.yml | 56 +++++--- .gitea/workflows/deploy.yml | 64 +++++---- .gitea/workflows/incidents.yml | 20 ++- Makefile | 37 +++++- scripts/bao-docker.sh | 10 ++ scripts/bootstrap-local.sh | 56 ++++++++ scripts/incident-database.sh | 6 +- scripts/incident-telemetry.sh | 8 +- scripts/incident-worker.sh | 6 +- scripts/lab6-traffic.py | 11 +- scripts/lab7-worker-compromise.sh | 209 ++++++++++++++++++++++++++++++ scripts/openbao-bootstrap.sh | 93 +++++++++++++ scripts/openbao-common.sh | 91 +++++++++++++ scripts/openbao-init.sh | 46 +++++++ scripts/openbao-issue-approle.sh | 87 +++++++++++++ scripts/openbao-rotate.sh | 42 ++++++ scripts/openbao-seed.sh | 62 +++++++++ scripts/openbao-test-policy.sh | 140 ++++++++++++++++++++ scripts/openbao-unseal.sh | 51 ++++++++ scripts/rolling-update.sh | 13 +- scripts/scan-security.sh | 132 +++++++++++++++++++ scripts/test-lab6.sh | 14 +- scripts/test-leaf-rotation.sh | 160 +++++++++++++++++++++++ scripts/test-login-rate-limit.py | 66 ++++++++++ scripts/test-openbao-lifecycle.sh | 177 +++++++++++++++++++++++++ scripts/test-tls-matrix.sh | 163 +++++++++++++++++++++++ 26 files changed, 1752 insertions(+), 68 deletions(-) create mode 100755 scripts/bao-docker.sh create mode 100755 scripts/bootstrap-local.sh create mode 100755 scripts/lab7-worker-compromise.sh create mode 100755 scripts/openbao-bootstrap.sh create mode 100755 scripts/openbao-common.sh create mode 100755 scripts/openbao-init.sh create mode 100755 scripts/openbao-issue-approle.sh create mode 100755 scripts/openbao-rotate.sh create mode 100755 scripts/openbao-seed.sh create mode 100755 scripts/openbao-test-policy.sh create mode 100755 scripts/openbao-unseal.sh create mode 100755 scripts/scan-security.sh create mode 100755 scripts/test-leaf-rotation.sh create mode 100644 scripts/test-login-rate-limit.py create mode 100755 scripts/test-openbao-lifecycle.sh create mode 100755 scripts/test-tls-matrix.sh diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 33018c7..36db92a 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -14,15 +14,16 @@ concurrency: jobs: verify: runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 env: - COMPOSE_PROJECT_NAME: mtusi2026lab6-${{ github.run_id }} + COMPOSE_PROJECT_NAME: mtusi2026lab7-${{ github.run_id }} API_BIND_ADDRESS: 127.0.0.1:0 SWAGGER_BIND_ADDRESS: 127.0.0.1:0 PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 GRAFANA_BIND_ADDRESS: 127.0.0.1:0 LOKI_BIND_ADDRESS: 127.0.0.1:0 TEMPO_BIND_ADDRESS: 127.0.0.1:0 + OPENBAO_BIND_ADDRESS: 127.0.0.1:0 steps: - name: Checkout exact commit run: | @@ -48,8 +49,8 @@ jobs: - name: Check formatting and shell scripts run: | test -z "$(gofmt -l cmd internal migrations)" - sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh - python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py + for file in infra/minio/*.sh infra/pki/*.sh infra/postgres/*.sh infra/redis/*.sh infra/security/*.sh infra/openbao/*.sh scripts/*.sh; do sh -n "$file"; done + python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py scripts/test-login-rate-limit.py - name: Lint uses: golangci/golangci-lint-action@v7 @@ -65,28 +66,48 @@ jobs: - name: Prepare local stack run: | - cp lab6.env.example .env + cp lab7.env.example .env docker compose config --quiet - docker compose up -d --build --scale api=2 --wait --wait-timeout 150 + ./scripts/bootstrap-local.sh .env docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" - name: Integration checks run: | - set -a - . ./.env - set +a - test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403" - make test BASE_URL=http://nginx:8080 + test "$(curl --noproxy '*' --cacert .runtime/security/internal-ca/ca.crt -s -o /dev/null -w '%{http_code}' "https://s3:9000/messenger-attachments")" = "403" + make test BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt + make test-security BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt + make test-tls-matrix ENV_FILE=.env + ENV_FILE=.env BASE_URL=https://nginx:8443 ./scripts/test-leaf-rotation.sh + docker compose --profile security run --rm dependency-tests + OPENBAO_ADDRESS=$(docker compose port openbao 8200) + ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/openbao-test-policy.sh + ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/test-openbao-lifecycle.sh + make test-login-rate-limit BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt docker compose --profile tests run --rm --build tests - go run ./cmd/dependencycheck -base-url=http://nginx:8080 - make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 - make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + go run ./cmd/dependencycheck -base-url=https://nginx:8443 -ca-file=.runtime/security/public-ca/ca.crt + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt docker compose up -d --scale api=2 --wait --wait-timeout 90 make test-lab5-acceptance - make test-lab5 - ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080 + make test-worker-compromise ENV_FILE=.env + make scan-security + ENV_FILE=.env CA_FILE=.runtime/security/public-ca/ca.crt PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=https://nginx:8443 + + - name: Upload security evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: lab7-security-${{ github.sha }} + path: | + evidence/lab7-*.json + evidence/runtime/security-scan + if-no-files-found: warn + retention-days: 14 - name: Container logs if: failure() @@ -97,5 +118,8 @@ jobs: run: | docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" 2>/dev/null || true if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index b1b9a29..7fdf0aa 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -12,7 +12,7 @@ jobs: name: Verify selected commit if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 outputs: api_image: ${{ steps.publish.outputs.api_image }} nginx_image: ${{ steps.publish.outputs.nginx_image }} @@ -22,7 +22,7 @@ jobs: s3_image: ${{ steps.publish.outputs.s3_image }} s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }} env: - COMPOSE_PROJECT_NAME: mtusi2026lab6-deploy-${{ github.run_id }} + COMPOSE_PROJECT_NAME: mtusi2026lab7-deploy-${{ github.run_id }} COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml API_BIND_ADDRESS: 127.0.0.1:0 SWAGGER_BIND_ADDRESS: 127.0.0.1:0 @@ -30,13 +30,14 @@ jobs: GRAFANA_BIND_ADDRESS: 127.0.0.1:0 LOKI_BIND_ADDRESS: 127.0.0.1:0 TEMPO_BIND_ADDRESS: 127.0.0.1:0 - API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-api:${{ github.sha }} - NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-nginx:${{ github.sha }} - SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-swagger:${{ github.sha }} - PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-prometheus:${{ github.sha }} - GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-grafana:${{ github.sha }} - S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio:${{ github.sha }} - S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio-client:${{ github.sha }} + OPENBAO_BIND_ADDRESS: 127.0.0.1:0 + API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-api:${{ github.sha }} + NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-nginx:${{ github.sha }} + SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-swagger:${{ github.sha }} + PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-prometheus:${{ github.sha }} + GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-grafana:${{ github.sha }} + S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-minio:${{ github.sha }} + S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-minio-client:${{ github.sha }} steps: - name: Checkout exact commit run: | @@ -62,8 +63,8 @@ jobs: - name: Check formatting and shell scripts run: | test -z "$(gofmt -l cmd internal migrations)" - sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh - python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py + for file in infra/minio/*.sh infra/pki/*.sh infra/postgres/*.sh infra/redis/*.sh infra/security/*.sh infra/openbao/*.sh scripts/*.sh; do sh -n "$file"; done + python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py scripts/test-login-rate-limit.py - name: Lint uses: golangci/golangci-lint-action@v7 @@ -79,27 +80,35 @@ jobs: - name: Prepare local stack run: | - cp lab6.env.example .env + cp lab7.env.example .env docker compose config --quiet - docker compose up -d --build --scale api=2 --wait --wait-timeout 150 + ./scripts/bootstrap-local.sh .env docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" - name: Integration checks run: | - set -a - . ./.env - set +a - test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403" - make test BASE_URL=http://nginx:8080 + test "$(curl --noproxy '*' --cacert .runtime/security/internal-ca/ca.crt -s -o /dev/null -w '%{http_code}' "https://s3:9000/messenger-attachments")" = "403" + make test BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt + make test-security BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt + make test-tls-matrix ENV_FILE=.env + ENV_FILE=.env BASE_URL=https://nginx:8443 ./scripts/test-leaf-rotation.sh + docker compose --profile security run --rm dependency-tests + OPENBAO_ADDRESS=$(docker compose port openbao 8200) + ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/openbao-test-policy.sh + ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/test-openbao-lifecycle.sh + make test-login-rate-limit BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt docker compose --profile tests run --rm --build tests - go run ./cmd/dependencycheck -base-url=http://nginx:8080 - make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080 - make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080 + go run ./cmd/dependencycheck -base-url=https://nginx:8443 -ca-file=.runtime/security/public-ca/ca.crt + make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt make test-lab5-acceptance - make test-lab5 - ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080 + make test-worker-compromise ENV_FILE=.env + ENV_FILE=.env CA_FILE=.runtime/security/public-ca/ca.crt PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=https://nginx:8443 - name: Publish verified images id: publish @@ -137,7 +146,10 @@ jobs: run: | docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" 2>/dev/null || true if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi deploy: @@ -177,17 +189,17 @@ jobs: run: printf '%s' "$REGISTRY_PASSWORD" | ssh "$DEPLOY_USER@$DEPLOY_HOST" "docker login gitea.privcloud.tech --username '$REGISTRY_USERNAME' --password-stdin" - name: Upload image manifest - run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab6-${{ github.run_id }}.env" + run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab7-${{ github.run_id }}.env" - name: Deploy and verify run: | ssh "$DEPLOY_USER@$DEPLOY_HOST" " set -eu - cd ~/mtusi2026lab6 + cd ~/mtusi2026lab7 git fetch origin main git cat-file -e '$DEPLOY_SHA^{commit}' git reset --hard '$DEPLOY_SHA' - manifest='/tmp/mtusi2026lab6-${{ github.run_id }}.env' + manifest='/tmp/mtusi2026lab7-${{ github.run_id }}.env' if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then rm -f \"\$manifest\" else diff --git a/.gitea/workflows/incidents.yml b/.gitea/workflows/incidents.yml index 244f11e..57c4b52 100644 --- a/.gitea/workflows/incidents.yml +++ b/.gitea/workflows/incidents.yml @@ -1,10 +1,10 @@ -name: Lab 6 incidents +name: Lab 7 inherited incidents on: workflow_dispatch: concurrency: - group: lab6-incidents-${{ github.repository }}-${{ github.ref }} + group: lab7-incidents-${{ github.repository }}-${{ github.ref }} cancel-in-progress: false jobs: @@ -12,13 +12,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 35 env: - COMPOSE_PROJECT_NAME: mtusi2026lab6-incidents-${{ github.run_id }} + COMPOSE_PROJECT_NAME: mtusi2026lab7-incidents-${{ github.run_id }} API_BIND_ADDRESS: 127.0.0.1:0 SWAGGER_BIND_ADDRESS: 127.0.0.1:0 PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0 GRAFANA_BIND_ADDRESS: 127.0.0.1:0 LOKI_BIND_ADDRESS: 127.0.0.1:0 TEMPO_BIND_ADDRESS: 127.0.0.1:0 + OPENBAO_BIND_ADDRESS: 127.0.0.1:0 steps: - name: Checkout exact commit run: | @@ -34,17 +35,21 @@ jobs: - name: Prepare stack run: | - cp lab6.env.example .env + cp lab7.env.example .env docker compose config --quiet - docker compose up -d --build --scale api=2 --wait --wait-timeout 180 + ./scripts/bootstrap-local.sh .env docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" + docker network connect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" - name: Reproduce SLO incidents run: | ENV_FILE=.env \ - BASE_URL=http://nginx:8080 \ + BASE_URL=https://nginx:8443 \ + CA_FILE=.runtime/security/public-ca/ca.crt \ PROMETHEUS_URL=http://prometheus:9090 \ TEMPO_URL=http://tempo:3200 \ LOKI_URL=http://loki:3100 \ @@ -69,5 +74,8 @@ jobs: run: | docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" 2>/dev/null || true + docker network disconnect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" 2>/dev/null || true docker compose down -v --remove-orphans diff --git a/Makefile b/Makefile index 5ad43ad..fcbd5c7 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,10 @@ PYTHON ?= python3 BASE_URL ?= https://localhost:8443 CA_FILE ?= ACTION_SCRIPT ?= +ENV_FILE ?= $(if $(wildcard .env),.env,lab7.env.example) TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover test-security +.PHONY: help check validate test load test-persistence test-failover test-security test-tls-matrix test-leaf-rotation test-openbao-lifecycle test-login-rate-limit test-worker-compromise scan-security test-lab5 test-lab5-acceptance lab6-load test-lab6-smoke test-lab6 help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -31,3 +32,37 @@ test-failover: test-security: $(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-tls-matrix: + docker compose --env-file "$(ENV_FILE)" --profile security run --rm security-tests + +test-leaf-rotation: + ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" ./scripts/test-leaf-rotation.sh + +test-openbao-lifecycle: + ENV_FILE="$(ENV_FILE)" ./scripts/test-openbao-lifecycle.sh + +test-login-rate-limit: + @test -n "$(CA_FILE)" || (echo "Set CA_FILE"; exit 2) + $(PYTHON) scripts/test-login-rate-limit.py --base-url "$(BASE_URL)" --ca-file "$(CA_FILE)" + +test-worker-compromise: + ENV_FILE="$(ENV_FILE)" ./scripts/lab7-worker-compromise.sh + +scan-security: + ./scripts/scan-security.sh + +test-lab5: + ./scripts/test-lab5.sh + +test-lab5-acceptance: + ./scripts/test-lab5-acceptance.sh + +lab6-load: + $(PYTHON) scripts/lab6-traffic.py --base-url "$(BASE_URL)" $(TLS_ARGS) --mode mixed --duration 60 --concurrency 4 --wait-ready + +test-lab6-smoke: + ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" ./scripts/test-lab6.sh smoke + +test-lab6: + ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" ./scripts/test-lab6.sh full diff --git a/scripts/bao-docker.sh b/scripts/bao-docker.sh new file mode 100755 index 0000000..303b988 --- /dev/null +++ b/scripts/bao-docker.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +env_file=${ENV_FILE:-$repo_dir/lab7.env.example} + +exec docker compose --project-directory "$repo_dir" --env-file "$env_file" run --rm --no-deps -T \ + -v "$repo_dir:$repo_dir:ro" openbao-cli "$@" diff --git a/scripts/bootstrap-local.sh b/scripts/bootstrap-local.sh new file mode 100755 index 0000000..07c6233 --- /dev/null +++ b/scripts/bootstrap-local.sh @@ -0,0 +1,56 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +env_file=${1:-$repo_dir/lab7.env.example} + +if [ ! -f "$env_file" ]; then + echo "environment file not found: $env_file" >&2 + exit 2 +fi + +compose() { + docker compose --project-directory "$repo_dir" --env-file "$env_file" "$@" +} + +if [ ! -f "$repo_dir/.runtime/security/openbao/tls.crt" ]; then + "$repo_dir/infra/pki/bootstrap.sh" +fi + +"$script_dir/openbao-seed.sh" "$env_file" +compose up -d security-bootstrap openbao-permissions openbao +openbao_address=$(compose port openbao 8200) +BAO_ADDR="https://$openbao_address" +BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt" +export BAO_ADDR BAO_CACERT ENV_FILE="$env_file" + +remaining=60 +until curl --silent --show-error --cacert "$BAO_CACERT" \ + --output /dev/null "$BAO_ADDR/v1/sys/health"; do + remaining=$((remaining - 1)) + [ "$remaining" -gt 0 ] || { + echo "OpenBao did not start" >&2 + exit 1 + } + sleep 1 +done + +ENV_FILE="$env_file" "$script_dir/openbao-init.sh" +ENV_FILE="$env_file" "$script_dir/openbao-unseal.sh" +remaining=60 +until curl --fail --silent --show-error --cacert "$BAO_CACERT" \ + --output /dev/null "$BAO_ADDR/v1/sys/health"; do + remaining=$((remaining - 1)) + [ "$remaining" -gt 0 ] || { + echo "OpenBao did not become active after unseal" >&2 + exit 1 + } + sleep 1 +done +ENV_FILE="$env_file" "$script_dir/openbao-bootstrap.sh" \ + "$repo_dir/.runtime/openbao/seed/api.json" \ + "$repo_dir/.runtime/openbao/seed/worker.json" + +compose up -d --build --scale api=2 --wait --wait-timeout 240 diff --git a/scripts/incident-database.sh b/scripts/incident-database.sh index bc3c6e0..6bc3cc6 100755 --- a/scripts/incident-database.sh +++ b/scripts/incident-database.sh @@ -2,8 +2,9 @@ set -eu -env_file=${ENV_FILE:-lab6.env.example} -base_url=${BASE_URL:-http://127.0.0.1:8080} +env_file=${ENV_FILE:-lab7.env.example} +base_url=${BASE_URL:-https://localhost:8443} +ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt} prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} tempo_url=${TEMPO_URL:-http://127.0.0.1:3200} loki_url=${LOKI_URL:-http://127.0.0.1:3100} @@ -33,6 +34,7 @@ trap restore EXIT INT TERM python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null python3 scripts/lab6-traffic.py \ --base-url "$base_url" \ + --ca-file "$ca_file" \ --mode messages \ --duration "$((outage_seconds + 20))" \ --concurrency 4 \ diff --git a/scripts/incident-telemetry.sh b/scripts/incident-telemetry.sh index 1f0cc50..716414a 100755 --- a/scripts/incident-telemetry.sh +++ b/scripts/incident-telemetry.sh @@ -2,8 +2,9 @@ set -eu -env_file=${ENV_FILE:-lab6.env.example} -base_url=${BASE_URL:-http://127.0.0.1:8080} +env_file=${ENV_FILE:-lab7.env.example} +base_url=${BASE_URL:-https://localhost:8443} +ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt} prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} evidence_root=${EVIDENCE_DIR:-evidence/runtime} outage_seconds=${TELEMETRY_OUTAGE_SECONDS:-30} @@ -30,6 +31,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) compose stop tempo loki >/dev/null python3 scripts/lab6-traffic.py \ --base-url "$base_url" \ + --ca-file "$ca_file" \ --mode mixed \ --duration "$outage_seconds" \ --concurrency 4 \ @@ -41,7 +43,7 @@ load_pid=$! sleep "$outage_seconds" wait "$load_pid" load_pid= -health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' "$base_url/health/ready") +health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' --cacert "$ca_file" "$base_url/health/ready") python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) compose up -d --wait --wait-timeout 90 tempo loki >/dev/null diff --git a/scripts/incident-worker.sh b/scripts/incident-worker.sh index 016df05..f5477de 100755 --- a/scripts/incident-worker.sh +++ b/scripts/incident-worker.sh @@ -3,8 +3,9 @@ set -eu mode=${1:-short} -env_file=${ENV_FILE:-lab6.env.example} -base_url=${BASE_URL:-http://127.0.0.1:8080} +env_file=${ENV_FILE:-lab7.env.example} +base_url=${BASE_URL:-https://localhost:8443} +ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt} prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} evidence_root=${EVIDENCE_DIR:-evidence/runtime} require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1} @@ -51,6 +52,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) compose stop worker >/dev/null python3 scripts/lab6-traffic.py \ --base-url "$base_url" \ + --ca-file "$ca_file" \ --mode attachments \ --duration "$load_seconds" \ --requests "$requests" \ diff --git a/scripts/lab6-traffic.py b/scripts/lab6-traffic.py index 244d53b..e87ece1 100755 --- a/scripts/lab6-traffic.py +++ b/scripts/lab6-traffic.py @@ -7,6 +7,7 @@ import json import math from pathlib import Path import secrets +import ssl import struct import threading import time @@ -16,6 +17,7 @@ import urllib.request import zlib MAX_RESPONSE_BYTES = 1 << 20 +TLS_CONTEXT = None def png_fixture(width=96, height=64): @@ -37,7 +39,7 @@ def png_fixture(width=96, height=64): def request(method, url, body=None, headers=None, timeout=10): req = urllib.request.Request(url, data=body, method=method, headers=headers or {}) try: - with urllib.request.urlopen(req, timeout=timeout) as response: + with urllib.request.urlopen(req, timeout=timeout, context=TLS_CONTEXT) as response: payload = response.read(MAX_RESPONSE_BYTES) return response.status, payload except urllib.error.HTTPError as error: @@ -167,7 +169,8 @@ def wait_for_attachments(base_url, token, attachment_ids, timeout): def main(): parser = argparse.ArgumentParser(description="Bounded synthetic load for Lab 6 telemetry and incidents") - parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--base-url", default="https://localhost:8443") + parser.add_argument("--ca-file", default=".runtime/security/public-ca/ca.crt") parser.add_argument("--mode", choices=("messages", "attachments", "mixed"), default="mixed") parser.add_argument("--duration", type=float, default=30) parser.add_argument("--requests", type=int, default=0, help="Maximum requests; zero means duration only") @@ -179,6 +182,10 @@ def main(): parser.add_argument("--started-file", help="Touch after synthetic session and chat are ready") args = parser.parse_args() + global TLS_CONTEXT + if args.base_url.startswith("https://"): + TLS_CONTEXT = ssl.create_default_context(cafile=args.ca_file) + if args.duration <= 0 or args.duration > 1800: parser.error("duration must be in (0, 1800]") if args.requests < 0: diff --git a/scripts/lab7-worker-compromise.sh b/scripts/lab7-worker-compromise.sh new file mode 100755 index 0000000..f9d9294 --- /dev/null +++ b/scripts/lab7-worker-compromise.sh @@ -0,0 +1,209 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +env_file=${ENV_FILE:-$repo_dir/lab7.env.example} +runtime_dir="$repo_dir/.runtime/evidence/lab7-worker" +state_file="$runtime_dir/queued-attachment.json" +rotated_secret="$repo_dir/.runtime/openbao/seed/worker-rotated.json" +rotated_password="rotated-worker-$(date -u +%Y%m%d%H%M%S)" +rotated_database_password="rotated-db-$(date -u +%Y%m%d%H%M%S)" +rotated_s3_secret="rotated-s3-$(date -u +%Y%m%d%H%M%S)" +summary_file="$repo_dir/evidence/lab7-worker-compromise.json" +started_at=$(date +%s) + +compose() { + docker compose --project-directory "$repo_dir" --env-file "$env_file" "$@" +} + +wait_healthy() { + service=$1 + remaining=60 + while [ "$remaining" -gt 0 ]; do + container=$(compose ps -q "$service") + if [ -n "$container" ] && [ "$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")" = healthy ]; then + return 0 + fi + remaining=$((remaining - 1)) + sleep 1 + done + echo "$service did not become healthy" >&2 + return 1 +} + +rotate_worker_secret() { + secret_file=$1 + compose stop openbao-agent-worker >/dev/null + ENV_FILE="$env_file" BAO_ADDR="$BAO_ADDR" BAO_CACERT="$BAO_CACERT" \ + "$script_dir/openbao-rotate.sh" worker "$secret_file" >/dev/null + docker start "$(compose ps -aq openbao-agent-worker)" >/dev/null + wait_healthy openbao-agent-worker +} + +restore() { + compose stop worker openbao-agent-worker >/dev/null 2>&1 || true + compose run --rm --no-deps acl-bootstrap >/dev/null 2>&1 || true + compose restart redis-broker redis-gateway >/dev/null 2>&1 || true + compose run --rm --no-deps postgres-roles >/dev/null 2>&1 || true + compose run --rm --no-deps s3-bootstrap >/dev/null 2>&1 || true + rotate_worker_secret "$repo_dir/.runtime/openbao/seed/worker.json" >/dev/null 2>&1 || true + compose up -d --no-deps --force-recreate worker >/dev/null 2>&1 || true +} + +mkdir -p "$runtime_dir" +chmod 0700 "$repo_dir/.runtime/evidence" "$runtime_dir" +trap restore EXIT HUP INT TERM + +openbao_address=$(compose port openbao 8200) +BAO_ADDR="https://$openbao_address" +BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt" +export BAO_ADDR BAO_CACERT + +compose stop worker >/dev/null +compose --profile experiments run --rm --no-deps lab5-experiment /app/lab5experiment api-produce >"$state_file" +chmod 0600 "$state_file" + +old_database_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY' +import json +import sys +import urllib.parse + +with open(sys.argv[1], encoding="utf-8") as source: + secret = json.load(source) +print(urllib.parse.urlsplit(secret["database_url"]).password) +PY +) +old_broker_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY' +import json +import sys +import urllib.parse + +with open(sys.argv[1], encoding="utf-8") as source: + print(urllib.parse.urlsplit(json.load(source)["broker_url"]).password) +PY +) +old_s3_access_key=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + print(json.load(source)["s3_access_key"]) +PY +) +old_s3_secret=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + print(json.load(source)["s3_secret_key"]) +PY +) + +python3 - "$repo_dir/.runtime/openbao/seed/worker.json" "$rotated_secret" \ + "$rotated_password" "$rotated_database_password" "$rotated_s3_secret" <<'PY' +import json +import os +import sys +import urllib.parse + + +def replace_password(url, password): + parsed = urllib.parse.urlsplit(url) + username = urllib.parse.quote(parsed.username or "", safe="") + netloc = f"{username}:{urllib.parse.quote(password, safe='')}@{parsed.hostname}" + if parsed.port: + netloc += f":{parsed.port}" + return urllib.parse.urlunsplit((parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment)) + +with open(sys.argv[1], encoding="utf-8") as source: + secret = json.load(source) +secret["broker_url"] = replace_password(secret["broker_url"], sys.argv[3]) +secret["database_url"] = replace_password(secret["database_url"], sys.argv[4]) +secret["s3_secret_key"] = sys.argv[5] +with open(sys.argv[2], "w", encoding="utf-8") as destination: + json.dump(secret, destination, separators=(",", ":")) + destination.write("\n") +os.chmod(sys.argv[2], 0o600) +PY + +compose run --rm --no-deps -e REDIS_WORKER_PASSWORD="$rotated_password" acl-bootstrap >/dev/null +compose restart redis-broker redis-gateway >/dev/null +compose exec -T postgres sh -c \ + "psql --set=ON_ERROR_STOP=1 --username \"\$POSTGRES_USER\" --dbname \"\$POSTGRES_DB\" --command \"ALTER ROLE messenger_worker PASSWORD '$rotated_database_password'\"" >/dev/null +compose exec -T postgres sh -c \ + 'psql --set=ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" --command="SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = '\''messenger_worker'\'' AND pid <> pg_backend_pid()"' >/dev/null +compose run --rm --no-deps -e S3_WORKER_SECRET_KEY="$rotated_s3_secret" s3-bootstrap >/dev/null +compose --profile security run --rm --no-deps \ + -e REDISCLI_AUTH="$old_broker_password" \ + --entrypoint /bin/sh dependency-tests -c ' + output=$(redis-cli --tls --cacert /run/security-source/internal-ca/ca.crt \ + --cert /run/security-source/worker-client/tls.crt --key /run/security-source/worker-client/tls.key \ + --user messenger_worker -h redis-gateway -p 36379 PING 2>&1 || true) + case "$output" in + *WRONGPASS*|*invalid\ username-password*) exit 0 ;; + *) echo "expected old broker credential rejection" >&2; exit 1 ;; + esac + ' + +if compose --profile security run --rm --no-deps \ + -e OLD_DATABASE_PASSWORD="$old_database_password" \ + --entrypoint /bin/sh dependency-tests -c ' + psql "postgresql://messenger_worker:${OLD_DATABASE_PASSWORD}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/security-source/internal-ca/ca.crt&sslcert=/run/security-source/worker-client/tls.crt&sslkey=/run/security-source/worker-client/tls.key" \ + --set=ON_ERROR_STOP=1 --command="SELECT 1" >/dev/null 2>&1 + '; then + echo "old worker database credential was not rejected" >&2 + exit 1 +fi + +if compose --profile security run --rm --no-deps \ + -e OLD_S3_ACCESS_KEY="$old_s3_access_key" -e OLD_S3_SECRET="$old_s3_secret" \ + --entrypoint /bin/sh dependency-tests -c ' + export SSL_CERT_FILE=/run/security-source/internal-ca/ca.crt + export MC_CONFIG_DIR=/tmp/old-worker-mc + mc alias set old https://s3:9000 "$OLD_S3_ACCESS_KEY" "$OLD_S3_SECRET" >/dev/null 2>&1 && + mc ls old/messenger-attachments/attachments/ >/dev/null 2>&1 + '; then + echo "old worker S3 credential was not rejected" >&2 + exit 1 +fi + +rotate_worker_secret "$rotated_secret" +compose up -d --no-deps --force-recreate worker >/dev/null +wait_healthy worker + +compose --profile experiments run --rm --no-deps -T lab5-experiment \ + /app/lab5experiment api-verify <"$state_file" >/dev/null + +finished_at=$(date +%s) +python3 - "$summary_file" "$started_at" "$finished_at" <<'PY' +import datetime +import json +import sys + +started_at = int(sys.argv[2]) +finished_at = int(sys.argv[3]) +summary = { + "schema_version": 1, + "scenario": "worker credential compromise and rotation", + "generated_at": datetime.datetime.fromtimestamp( + finished_at, datetime.timezone.utc + ).replace(microsecond=0).isoformat(), + "recovery_seconds": finished_at - started_at, + "old_credential_rejected": True, + "old_database_credential_rejected": True, + "old_s3_credential_rejected": True, + "openbao_secret_rotated": True, + "queued_attachment_preserved": True, + "result": "PASS", +} +with open(sys.argv[1], "w", encoding="utf-8") as destination: + json.dump(summary, destination, ensure_ascii=False, indent=2) + destination.write("\n") +PY + +trap - EXIT HUP INT TERM +restore +rm -f "$rotated_secret" +echo "PASS: revoked worker credential failed; rotated OpenBao secret restored processing without losing the queued attachment" diff --git a/scripts/openbao-bootstrap.sh b/scripts/openbao-bootstrap.sh new file mode 100755 index 0000000..dc67ac5 --- /dev/null +++ b/scripts/openbao-bootstrap.sh @@ -0,0 +1,93 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +if [ "$#" -ne 2 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +api_secret_file=$1 +worker_secret_file=$2 + +require_command "$BAO_BIN" +require_command python3 +require_runtime_path "$api_secret_file" +require_runtime_path "$worker_secret_file" +require_private_file "$api_secret_file" +require_private_file "$worker_secret_file" +require_unsealed +load_operator_token + +python3 - "$api_secret_file" "$worker_secret_file" <<'PY' +import json +import sys + +for path in sys.argv[1:]: + with open(path, encoding="utf-8") as source: + value = json.load(source) + if not isinstance(value, dict) or not value: + raise SystemExit(f"secret input must be a non-empty JSON object: {path}") +PY + +mounts_file=$(mktemp "$openbao_runtime_dir/.mounts.XXXXXX") +auth_file=$(mktemp "$openbao_runtime_dir/.auth.XXXXXX") +chmod 0600 "$mounts_file" "$auth_file" + +cleanup() { + rm -f "$mounts_file" "$auth_file" +} + +trap cleanup EXIT HUP INT TERM + +"$BAO_BIN" secrets list -format=json >"$mounts_file" +if ! python3 - "$mounts_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + raise SystemExit(0 if "secret/" in json.load(source) else 1) +PY +then + "$BAO_BIN" secrets enable -path=secret -version=2 kv >/dev/null +fi + +"$BAO_BIN" auth list -format=json >"$auth_file" +if ! python3 - "$auth_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + raise SystemExit(0 if "approle/" in json.load(source) else 1) +PY +then + "$BAO_BIN" auth enable approle >/dev/null +fi + +"$BAO_BIN" policy write messenger-api "$repo_dir/infra/openbao/policies/messenger-api.hcl" >/dev/null +"$BAO_BIN" policy write messenger-worker "$repo_dir/infra/openbao/policies/messenger-worker.hcl" >/dev/null + +for service in api worker; do + "$BAO_BIN" delete "auth/approle/role/messenger-$service" >/dev/null 2>&1 || true + "$BAO_BIN" write "auth/approle/role/messenger-$service" \ + bind_secret_id=true \ + secret_id_ttl=1h \ + secret_id_num_uses=0 \ + token_policies="messenger-$service" \ + token_no_default_policy=true \ + token_ttl=15m \ + token_max_ttl=1h \ + token_num_uses=0 >/dev/null +done + +"$BAO_BIN" kv put -mount=secret messenger/api @"$api_secret_file" >/dev/null +"$BAO_BIN" kv put -mount=secret messenger/worker @"$worker_secret_file" >/dev/null + +"$script_dir/openbao-issue-approle.sh" api "$openbao_runtime_dir/approle/api" +"$script_dir/openbao-issue-approle.sh" worker "$openbao_runtime_dir/approle/worker" + +echo "OpenBao policies, AppRoles and KV v2 secrets configured" + diff --git a/scripts/openbao-common.sh b/scripts/openbao-common.sh new file mode 100755 index 0000000..9497ce6 --- /dev/null +++ b/scripts/openbao-common.sh @@ -0,0 +1,91 @@ +#!/bin/sh + +set -eu + +umask 077 + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +openbao_runtime_dir="$repo_dir/.runtime/openbao" +openbao_operator_file="$openbao_runtime_dir/operator/init.json" + +BAO_BIN=${BAO_BIN:-$repo_dir/scripts/bao-docker.sh} +BAO_ADDR=${BAO_ADDR:-https://localhost:8200} +BAO_CACERT=${BAO_CACERT:-$repo_dir/.runtime/security/openbao/ca.crt} + +export BAO_ADDR BAO_CACERT + +fail() { + echo "$*" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1" +} + +require_file() { + [ -f "$1" ] || fail "required file not found: $1" +} + +require_private_file() { + require_file "$1" + permissions=$(stat -f '%Lp' "$1" 2>/dev/null || stat -c '%a' "$1") + case "$permissions" in + 600|400) ;; + *) fail "file must have permissions 0600 or 0400: $1" ;; + esac +} + +require_runtime_path() { + case "$1" in + "$openbao_runtime_dir"/*) ;; + *) fail "runtime credentials must stay under $openbao_runtime_dir" ;; + esac +} + +json_field() { + file=$1 + expression=$2 + python3 - "$file" "$expression" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + value = json.load(source) + +for component in sys.argv[2].split("."): + value = value[component] + +if isinstance(value, bool): + print(str(value).lower()) +else: + print(value) +PY +} + +load_operator_token() { + require_private_file "$openbao_operator_file" + BAO_TOKEN=$(json_field "$openbao_operator_file" root_token) + export BAO_TOKEN +} + +openbao_status() { + status_file=$1 + "$BAO_BIN" status -format=json >"$status_file" 2>/dev/null || true + [ -s "$status_file" ] || fail "OpenBao is not reachable at $BAO_ADDR" +} + +require_unsealed() { + mkdir -p "$openbao_runtime_dir" + chmod 0700 "$openbao_runtime_dir" + status_file=$(mktemp "$openbao_runtime_dir/.status.XXXXXX") + chmod 0600 "$status_file" + openbao_status "$status_file" + initialized=$(json_field "$status_file" initialized) + sealed=$(json_field "$status_file" sealed) + rm -f "$status_file" + [ "$initialized" = "true" ] || fail "OpenBao is not initialized" + [ "$sealed" = "false" ] || fail "OpenBao is sealed" +} + diff --git a/scripts/openbao-init.sh b/scripts/openbao-init.sh new file mode 100755 index 0000000..9e36a33 --- /dev/null +++ b/scripts/openbao-init.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +require_command "$BAO_BIN" +require_command python3 +require_file "$BAO_CACERT" + +mkdir -p "$openbao_runtime_dir/operator" +chmod 0700 "$openbao_runtime_dir" "$openbao_runtime_dir/operator" + +status_file=$(mktemp "$openbao_runtime_dir/.status.XXXXXX") +init_file=$(mktemp "$openbao_runtime_dir/operator/.init.XXXXXX") +chmod 0600 "$status_file" "$init_file" + +cleanup() { + rm -f "$status_file" "$init_file" +} + +trap cleanup EXIT HUP INT TERM + +openbao_status "$status_file" + +if [ "$(json_field "$status_file" initialized)" = "true" ]; then + echo "OpenBao is already initialized" + exit 0 +fi + +if [ -e "$openbao_operator_file" ]; then + fail "operator material already exists for an uninitialized OpenBao instance" +fi + +"$BAO_BIN" operator init \ + -key-shares=3 \ + -key-threshold=2 \ + -format=json >"$init_file" + +chmod 0600 "$init_file" +mv "$init_file" "$openbao_operator_file" + +echo "OpenBao initialized; operator material saved to .runtime/openbao/operator/init.json" +echo "Keep this file offline and run scripts/openbao-unseal.sh" + diff --git a/scripts/openbao-issue-approle.sh b/scripts/openbao-issue-approle.sh new file mode 100755 index 0000000..4506ab9 --- /dev/null +++ b/scripts/openbao-issue-approle.sh @@ -0,0 +1,87 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +if [ "$#" -ne 2 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +service=$1 +output_dir=$2 + +case "$service" in + api|worker) ;; + *) fail "service must be api or worker" ;; +esac + +require_runtime_path "$output_dir" +require_command "$BAO_BIN" +require_command python3 +require_unsealed + +if [ -z "${BAO_TOKEN:-}" ]; then + load_operator_token +fi + +role="messenger-$service" +parent_dir=$(dirname "$output_dir") +staging_dir="$parent_dir/.${service}.$$" +response_file="$staging_dir/response.json" + +cleanup() { + rm -rf "$staging_dir" +} + +trap cleanup EXIT HUP INT TERM + +mkdir -p "$parent_dir" "$staging_dir" +chmod 0700 "$parent_dir" "$staging_dir" + +"$BAO_BIN" read -field=role_id "auth/approle/role/$role/role-id" >"$staging_dir/role-id" +"$BAO_BIN" write -f -format=json "auth/approle/role/$role/secret-id" >"$response_file" + +python3 - "$response_file" "$staging_dir" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + data = json.load(source)["data"] + +for name, field in (("secret-id", "secret_id"), ("secret-id-accessor", "secret_id_accessor")): + path = os.path.join(sys.argv[2], name) + with open(path, "w", encoding="utf-8") as destination: + destination.write(data[field] + "\n") + os.chmod(path, 0o600) +PY + +rm -f "$response_file" +chmod 0600 "$staging_dir/role-id" + +old_accessor= +if [ -f "$output_dir/secret-id-accessor" ]; then + old_accessor=$(cat "$output_dir/secret-id-accessor") +fi + +mkdir -p "$output_dir" +chmod 0700 "$output_dir" +for file in role-id secret-id secret-id-accessor; do + mv "$staging_dir/$file" "$output_dir/$file" +done + +if [ -n "$old_accessor" ]; then + "$BAO_BIN" write "auth/approle/role/$role/secret-id-accessor/destroy" \ + secret_id_accessor="$old_accessor" >/dev/null 2>&1 || true +fi + +if [ "$output_dir" = "$openbao_runtime_dir/approle/$service" ]; then + docker compose --project-directory "$repo_dir" --env-file "${ENV_FILE:-$repo_dir/lab7.env.example}" \ + run --rm --no-deps -e SERVICE="$service" openbao-identity-install +fi + +echo "issued short-lived AppRole credentials for $service in ${output_dir#"$repo_dir/"}" + diff --git a/scripts/openbao-rotate.sh b/scripts/openbao-rotate.sh new file mode 100755 index 0000000..7460197 --- /dev/null +++ b/scripts/openbao-rotate.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +if [ "$#" -ne 2 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +service=$1 +secret_file=$2 + +case "$service" in + api|worker) ;; + *) fail "service must be api or worker" ;; +esac + +require_command "$BAO_BIN" +require_command python3 +require_runtime_path "$secret_file" +require_private_file "$secret_file" +require_unsealed +load_operator_token + +python3 - "$secret_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + value = json.load(source) +if not isinstance(value, dict) or not value: + raise SystemExit("replacement secret must be a non-empty JSON object") +PY + +"$BAO_BIN" kv put -mount=secret "messenger/$service" @"$secret_file" >/dev/null +"$script_dir/openbao-issue-approle.sh" "$service" "$openbao_runtime_dir/approle/$service" + +echo "$service secret rotated; OpenBao Agent will render the new KV version" + diff --git a/scripts/openbao-seed.sh b/scripts/openbao-seed.sh new file mode 100755 index 0000000..c20f553 --- /dev/null +++ b/scripts/openbao-seed.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +set -eu + +if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +seed_dir="$repo_dir/.runtime/openbao/seed" +mkdir -p "$seed_dir" +chmod 0700 "$repo_dir/.runtime/openbao" "$seed_dir" + +python3 - "$1" "$seed_dir/api.json" "$seed_dir/worker.json" <<'PY' +import json +import os +import sys + +values = {} +with open(sys.argv[1], encoding="utf-8") as source: + for raw_line in source: + line = raw_line.strip() + if not line or line.startswith("#"): + continue + name, separator, value = line.partition("=") + if not separator: + raise SystemExit(f"invalid env line: {raw_line.rstrip()}") + values[name.strip()] = os.environ.get(name.strip(), value) + +required = ( + "DATABASE_URL", "DATABASE_WORKER_URL", "CACHE_URL", "BROKER_WORKER_URL", + "S3_API_ACCESS_KEY", "S3_API_SECRET_KEY", "S3_WORKER_ACCESS_KEY", "S3_WORKER_SECRET_KEY", +) +missing = [name for name in required if not values.get(name)] +if missing: + raise SystemExit("missing environment values: " + ", ".join(missing)) + +documents = ( + (sys.argv[2], { + "database_url": values["DATABASE_URL"], + "cache_url": values["CACHE_URL"], + "s3_access_key": values["S3_API_ACCESS_KEY"], + "s3_secret_key": values["S3_API_SECRET_KEY"], + }), + (sys.argv[3], { + "database_url": values["DATABASE_WORKER_URL"], + "broker_url": values["BROKER_WORKER_URL"], + "s3_access_key": values["S3_WORKER_ACCESS_KEY"], + "s3_secret_key": values["S3_WORKER_SECRET_KEY"], + }), +) + +for path, value in documents: + with open(path, "w", encoding="utf-8") as destination: + json.dump(value, destination, separators=(",", ":")) + destination.write("\n") + os.chmod(path, 0o600) +PY + +echo "OpenBao seed files created under .runtime/openbao/seed" diff --git a/scripts/openbao-test-policy.sh b/scripts/openbao-test-policy.sh new file mode 100755 index 0000000..7134e5c --- /dev/null +++ b/scripts/openbao-test-policy.sh @@ -0,0 +1,140 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +require_command curl +require_command python3 +require_command "$BAO_BIN" +require_file "$BAO_CACERT" +require_unsealed +load_operator_token + +test_dir="$openbao_runtime_dir/policy-test.$$" +mkdir -p "$test_dir" +chmod 0700 "$test_dir" + +cleanup() { + for service in api worker; do + headers="$test_dir/$service/token-header" + if [ -f "$headers" ]; then + curl --silent --show-error --cacert "$BAO_CACERT" \ + --header @"$headers" \ + --request POST \ + "$BAO_ADDR/v1/auth/token/revoke-self" >/dev/null 2>&1 || true + fi + done + rm -rf "$test_dir" +} + +trap cleanup EXIT HUP INT TERM + +login() { + service=$1 + identity_dir="$test_dir/$service/identity" + session_dir="$test_dir/$service" + response_file="$session_dir/login.json" + payload_file="$session_dir/login-payload.json" + + mkdir -p "$session_dir" + chmod 0700 "$session_dir" + "$script_dir/openbao-issue-approle.sh" "$service" "$identity_dir" >/dev/null + + python3 - "$identity_dir/role-id" "$identity_dir/secret-id" "$payload_file" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as role_source: + role_id = role_source.read().strip() +with open(sys.argv[2], encoding="utf-8") as secret_source: + secret_id = secret_source.read().strip() +with open(sys.argv[3], "w", encoding="utf-8") as destination: + json.dump({"role_id": role_id, "secret_id": secret_id}, destination) +os.chmod(sys.argv[3], 0o600) +PY + + status=$(curl --silent --show-error --cacert "$BAO_CACERT" \ + --header "Content-Type: application/json" \ + --request POST \ + --data-binary @"$payload_file" \ + --output "$response_file" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/auth/approle/login") + [ "$status" = "200" ] || fail "$service AppRole login returned HTTP $status" + chmod 0600 "$response_file" + + python3 - "$response_file" "$service" "$session_dir/token-header" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + auth = json.load(source)["auth"] + +expected_policy = f"messenger-{sys.argv[2]}" +if not auth["renewable"]: + raise SystemExit("AppRole token is not renewable") +if not 0 < auth["lease_duration"] <= 3600: + raise SystemExit("AppRole token TTL is outside the configured limit") +if auth["policies"] != [expected_policy]: + raise SystemExit(f"unexpected token policies: {auth['policies']}") + +with open(sys.argv[3], "w", encoding="utf-8") as destination: + destination.write(f"X-Vault-Token: {auth['client_token']}\n") +os.chmod(sys.argv[3], 0o600) +PY + + rm -f "$payload_file" "$response_file" "$identity_dir/secret-id" +} + +expect_status() { + service=$1 + method=$2 + path=$3 + expected=$4 + payload=${5:-} + headers="$test_dir/$service/token-header" + output="$test_dir/$service/response.json" + + if [ -n "$payload" ]; then + status=$(curl --silent --show-error --cacert "$BAO_CACERT" \ + --header @"$headers" \ + --header "Content-Type: application/json" \ + --request "$method" \ + --data-binary "$payload" \ + --output "$output" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/$path") + else + status=$(curl --silent --show-error --cacert "$BAO_CACERT" \ + --header @"$headers" \ + --request "$method" \ + --output "$output" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/$path") + fi + + [ "$status" = "$expected" ] || fail "$service $method $path: expected HTTP $expected, got $status" + echo "PASS $service $method $path -> $status" +} + +login api +login worker + +expect_status api GET secret/data/messenger/api 200 +expect_status api GET secret/data/messenger/worker 403 +expect_status api POST secret/data/messenger/api/probe 403 '{"data":{"probe":"denied"}}' +expect_status api GET sys/policies/acl 403 +expect_status api POST auth/token/renew-self 200 '{}' + +expect_status worker GET secret/data/messenger/worker 200 +expect_status worker GET secret/data/messenger/api 403 +expect_status worker POST secret/data/messenger/worker/probe 403 '{"data":{"probe":"denied"}}' +expect_status worker GET sys/policies/acl 403 +expect_status worker POST auth/token/renew-self 200 '{}' + +echo "OpenBao AppRole allow/deny and renewal checks passed" + diff --git a/scripts/openbao-unseal.sh b/scripts/openbao-unseal.sh new file mode 100755 index 0000000..ae8586d --- /dev/null +++ b/scripts/openbao-unseal.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +require_command "$BAO_BIN" +require_command python3 +require_file "$BAO_CACERT" +require_private_file "$openbao_operator_file" + +mkdir -p "$openbao_runtime_dir" +keys_file=$(mktemp "$openbao_runtime_dir/.unseal.XXXXXX") +status_file=$(mktemp "$openbao_runtime_dir/.status.XXXXXX") +chmod 0600 "$keys_file" "$status_file" + +cleanup() { + rm -f "$keys_file" "$status_file" +} + +trap cleanup EXIT HUP INT TERM + +openbao_status "$status_file" + +if [ "$(json_field "$status_file" sealed)" = "false" ]; then + echo "OpenBao is already unsealed" + exit 0 +fi + +python3 - "$openbao_operator_file" >"$keys_file" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + material = json.load(source) + +threshold = material["unseal_threshold"] +for key in material["unseal_keys_b64"][:threshold]: + print(key) +PY + +while IFS= read -r key; do + "$BAO_BIN" operator unseal "$key" /dev/null +done <"$keys_file" + +openbao_status "$status_file" +[ "$(json_field "$status_file" sealed)" = "false" ] || fail "OpenBao remains sealed" + +echo "OpenBao unsealed" + diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh index 695e3f8..d29cd0b 100755 --- a/scripts/rolling-update.sh +++ b/scripts/rolling-update.sh @@ -8,7 +8,7 @@ if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then fi manifest=$1 -deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab5"} +deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab7"} current_manifest=$deploy_dir/current-images.env active_containers=$deploy_dir/active-api-containers previous_containers=$deploy_dir/previous-api-containers @@ -16,7 +16,9 @@ upstream_file=$deploy_dir/upstream.conf upstream_backup=$deploy_dir/upstream.previous.conf candidate_file=$deploy_dir/candidate-api-containers replicas=${API_REPLICAS:-2} -project=${COMPOSE_PROJECT_NAME:-mtusi2026lab5} +project=${COMPOSE_PROJECT_NAME:-mtusi2026lab7} +public_base_url=${PUBLIC_BASE_URL:-https://localhost:8443} +public_ca_file=${PUBLIC_CA_FILE:-.runtime/security/public-ca/ca.crt} switched=false nginx_created=false upstream_changed=false @@ -52,7 +54,7 @@ write_upstream() { : > "$output" for container in $containers; do name=$(container_name "$container") - printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output" + printf 'server %s:8443 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output" done } @@ -81,7 +83,8 @@ wait_for_containers() { } smoke_check() { - compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready + curl --fail --silent --show-error --noproxy '*' \ + --cacert "$public_ca_file" "$public_base_url/health/ready" >/dev/null } remove_containers() { @@ -123,6 +126,7 @@ trap 'exit 143' TERM load_manifest "$manifest" new_api_image=$API_IMAGE +test -r "$public_ca_file" if [ -f "$previous_containers" ]; then remove_containers "$(cat "$previous_containers")" @@ -151,6 +155,7 @@ cp "$upstream_file" "$upstream_backup" compose pull api nginx swagger prometheus grafana s3 s3-bootstrap compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3 +compose run --rm security-bootstrap compose run --rm --no-deps s3-bootstrap compose run --rm --no-deps migrate diff --git a/scripts/scan-security.sh b/scripts/scan-security.sh new file mode 100755 index 0000000..ffd40ad --- /dev/null +++ b/scripts/scan-security.sh @@ -0,0 +1,132 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +runtime_dir="$repo_dir/evidence/runtime/security-scan" +cache_dir="$repo_dir/.runtime/trivy-cache" +raw_file="$runtime_dir/trivy-fs.json" +summary_file="$repo_dir/evidence/lab7-security-scan.json" +env_file=${ENV_FILE:-$repo_dir/lab7.env.example} +image_list=${SCAN_IMAGES:-} +if [ -z "$image_list" ]; then + containers=$(docker compose --project-directory "$repo_dir" --env-file "$env_file" ps -q) + [ -n "$containers" ] || { echo "start the stack or set SCAN_IMAGES" >&2; exit 1; } + image_list=$(docker inspect --format '{{.Config.Image}}' $containers | sort -u) +fi + +mkdir -p "$runtime_dir" "$cache_dir" + +docker run --rm \ + -v "$repo_dir:/src:ro" \ + -v "$runtime_dir:/out" \ + -v "$cache_dir:/root/.cache/trivy" \ + aquasec/trivy:0.76.2 fs \ + --scanners vuln,misconfig,secret \ + --format json \ + --output /out/trivy-fs.json \ + --skip-dirs /src/.git \ + --skip-dirs /src/.runtime \ + --skip-dirs /src/.venv \ + --skip-dirs /src/evidence/runtime \ + --skip-files /src/Dockerfile.example \ + /src + +image_reports= +index=0 +for image in $image_list; do + index=$((index + 1)) + report="$runtime_dir/trivy-image-$index.json" + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "$runtime_dir:/out" \ + -v "$cache_dir:/root/.cache/trivy" \ + aquasec/trivy:0.76.2 image \ + --scanners vuln,secret \ + --format json \ + --output "/out/trivy-image-$index.json" \ + "$image" + image_reports="$image_reports $image=$report" +done + +python3 - "$raw_file" "$summary_file" $image_reports <<'PY' +import datetime +import json +import sys +from collections import Counter + +with open(sys.argv[1], encoding="utf-8") as source: + report = json.load(source) + +vulnerabilities = Counter() +misconfigurations = Counter() +secrets = 0 +for result in report.get("Results") or []: + for vulnerability in result.get("Vulnerabilities") or []: + vulnerabilities[vulnerability.get("Severity", "UNKNOWN")] += 1 + for finding in result.get("Misconfigurations") or []: + misconfigurations[finding.get("Severity", "UNKNOWN")] += 1 + secrets += len(result.get("Secrets") or []) + +image_vulnerabilities = Counter() +image_secrets = 0 +scanned_images = [] +image_raw_reports = [] +findings = {} + + +def record_findings(report, scope): + for result in report.get("Results") or []: + for finding in result.get("Vulnerabilities") or []: + key = (finding["VulnerabilityID"], finding["PkgName"], finding.get("InstalledVersion", "")) + entry = findings.setdefault(key, { + "id": finding["VulnerabilityID"], + "package": finding["PkgName"], + "installed_version": finding.get("InstalledVersion", ""), + "fixed_version": finding.get("FixedVersion", ""), + "severity": finding.get("Severity", "UNKNOWN"), + "title": finding.get("Title", ""), + "scopes": [], + }) + if scope not in entry["scopes"]: + entry["scopes"].append(scope) + + +record_findings(report, "repository") +for item in sys.argv[3:]: + image, path = item.split("=", 1) + scanned_images.append(image) + with open(path, encoding="utf-8") as source: + image_report = json.load(source) + image_raw_reports.append("evidence/runtime/security-scan/" + path.rsplit("/", 1)[-1]) + record_findings(image_report, image) + for result in image_report.get("Results") or []: + for vulnerability in result.get("Vulnerabilities") or []: + image_vulnerabilities[vulnerability.get("Severity", "UNKNOWN")] += 1 + image_secrets += len(result.get("Secrets") or []) + +summary = { + "schema_version": 1, + "scanner": "Trivy 0.76.2", + "generated_at": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat(), + "scope": "repository filesystem: vulnerabilities, misconfigurations, secrets", + "vulnerabilities": dict(sorted(vulnerabilities.items())), + "misconfigurations": dict(sorted(misconfigurations.items())), + "secret_findings": secrets, + "scanned_images": scanned_images, + "image_vulnerabilities": dict(sorted(image_vulnerabilities.items())), + "image_secret_findings": image_secrets, + "findings": sorted((finding for finding in findings.values() + if "repository" in finding["scopes"] or finding["severity"] == "CRITICAL"), + key=lambda finding: (finding["package"], finding["id"])), + "findings_scope": "all repository findings and critical image findings; full findings in raw reports", + "raw_report": "evidence/runtime/security-scan/trivy-fs.json", + "image_raw_reports": image_raw_reports, +} +with open(sys.argv[2], "w", encoding="utf-8") as destination: + json.dump(summary, destination, ensure_ascii=False, indent=2) + destination.write("\n") +PY + +cat "$summary_file" diff --git a/scripts/test-lab6.sh b/scripts/test-lab6.sh index c2ffb45..4e26b5e 100755 --- a/scripts/test-lab6.sh +++ b/scripts/test-lab6.sh @@ -11,8 +11,9 @@ case "$profile" in ;; esac -env_file=${ENV_FILE:-lab6.env.example} -base_url=${BASE_URL:-http://127.0.0.1:8080} +env_file=${ENV_FILE:-lab7.env.example} +base_url=${BASE_URL:-https://localhost:8443} +ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt} prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090} tempo_url=${TEMPO_URL:-http://127.0.0.1:3200} loki_url=${LOKI_URL:-http://127.0.0.1:3100} @@ -35,6 +36,7 @@ compose config --quiet compose up -d --build --wait --wait-timeout 180 python3 scripts/lab6-traffic.py \ --base-url "$base_url" \ + --ca-file "$ca_file" \ --mode mixed \ --duration "${BASELINE_LOAD_SECONDS:-15}" \ --requests "${BASELINE_REQUESTS:-120}" \ @@ -54,10 +56,10 @@ python3 scripts/lab6-correlation.py \ --require >/dev/null if [ "$profile" = "full" ]; then - ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh - ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long - ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short - ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh + ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh + ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long + ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short + ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh fi arguments= diff --git a/scripts/test-leaf-rotation.sh b/scripts/test-leaf-rotation.sh new file mode 100755 index 0000000..c68e779 --- /dev/null +++ b/scripts/test-leaf-rotation.sh @@ -0,0 +1,160 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +env_file=${ENV_FILE:-$repo_dir/lab7.env.example} +security_dir=$repo_dir/.runtime/security +rotation_dir=$security_dir/rotation-test +old_dir=$rotation_dir/old-load-balancer +new_dir=$rotation_dir/new-load-balancer +expired_dir=$rotation_dir/expired-load-balancer +override_file=$repo_dir/.runtime/leaf-rotation.override.yaml +new_identity=spiffe://mtusi/lab7/load-balancer-v2 +rotation_active=0 +base_url=${BASE_URL:-https://localhost:8443} + +compose() { + docker compose --env-file "$env_file" -f "$repo_dir/compose.yaml" "$@" +} + +compose_rotated() { + docker compose --env-file "$env_file" \ + -f "$repo_dir/compose.yaml" -f "$override_file" "$@" +} + +install_security_material() { + compose run --rm --no-deps security-bootstrap >/dev/null +} + +wait_for_https() { + attempts=0 + while [ "$attempts" -lt 30 ]; do + if curl --silent --show-error --fail \ + --cacert "$security_dir/public-ca/ca.crt" \ + "$base_url/health/ready" >/dev/null 2>&1; then + return 0 + fi + attempts=$((attempts + 1)) + sleep 1 + done + return 1 +} + +restore() { + status=$? + trap - EXIT HUP INT TERM + + if [ "$rotation_active" -eq 1 ] && [ -f "$old_dir/tls.crt" ]; then + cp "$old_dir/tls.crt" "$security_dir/load-balancer/tls.crt" + cp "$old_dir/tls.key" "$security_dir/load-balancer/tls.key" + chmod 0600 "$security_dir/load-balancer/tls.crt" "$security_dir/load-balancer/tls.key" + install_security_material || status=1 + compose up -d --no-deps --no-build --force-recreate api nginx >/dev/null || status=1 + wait_for_https || status=1 + fi + + rm -rf "$rotation_dir" + rm -f "$override_file" + exit "$status" +} + +trap restore EXIT HUP INT TERM + +[ -f "$env_file" ] || { + echo "environment file not found: $env_file" >&2 + exit 1 +} +[ -f "$security_dir/internal-ca/ca.crt" ] || { + echo "run infra/pki/bootstrap.sh first" >&2 + exit 1 +} + +umask 077 +rm -rf "$rotation_dir" +mkdir -p "$old_dir" "$new_dir" "$expired_dir" +cp "$security_dir/load-balancer/tls.crt" "$old_dir/tls.crt" +cp "$security_dir/load-balancer/tls.key" "$old_dir/tls.key" + +cat >"$rotation_dir/extensions.cnf" </dev/null + rm "$directory/tls.csr" +} + +issue_leaf "$new_dir" 30 +issue_leaf "$expired_dir" 0 + +openssl verify -purpose sslclient \ + -CAfile "$security_dir/internal-ca/ca.crt" "$new_dir/tls.crt" >/dev/null +if openssl verify -purpose sslclient -attime "$(( $(date +%s) + 2 ))" \ + -CAfile "$security_dir/internal-ca/ca.crt" "$expired_dir/tls.crt" >/dev/null 2>&1; then + echo "zero-lifetime certificate unexpectedly passed expiry verification" >&2 + exit 1 +fi + +cat >"$override_file" </dev/null +wait_for_https + +compose_rotated --profile security run --rm --no-deps --entrypoint /bin/sh security-tests -c ' + set -eu + curl --silent --show-error --fail \ + --cacert /src/.runtime/security/internal-ca/ca.crt \ + --cert /src/.runtime/security/load-balancer/tls.crt \ + --key /src/.runtime/security/load-balancer/tls.key \ + https://api:8443/health/ready >/dev/null + if curl --silent --show-error --fail \ + --cacert /src/.runtime/security/internal-ca/ca.crt \ + --cert /src/.runtime/security/rotation-test/old-load-balancer/tls.crt \ + --key /src/.runtime/security/rotation-test/old-load-balancer/tls.key \ + https://api:8443/health/ready >/dev/null 2>&1; then + echo "retired load-balancer identity unexpectedly reached API" >&2 + exit 1 + fi + if curl --silent --show-error --fail \ + --cacert /src/.runtime/security/internal-ca/ca.crt \ + --cert /src/.runtime/security/rotation-test/expired-load-balancer/tls.crt \ + --key /src/.runtime/security/rotation-test/expired-load-balancer/tls.key \ + https://api:8443/health/ready >/dev/null 2>&1; then + echo "expired load-balancer leaf unexpectedly reached API" >&2 + exit 1 + fi +' + +echo "leaf rotation passed: new identity accepted, retired identity rejected, expired leaf rejected" diff --git a/scripts/test-login-rate-limit.py b/scripts/test-login-rate-limit.py new file mode 100644 index 0000000..3db9269 --- /dev/null +++ b/scripts/test-login-rate-limit.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +import argparse +import base64 +import json +import secrets +import ssl +import urllib.error +import urllib.request + + +def request(context, method, url, payload=None, headers=None): + data = None if payload is None else json.dumps(payload).encode("utf-8") + request_headers = dict(headers or {}) + if data is not None: + request_headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, data=data, method=method, headers=request_headers) + try: + with urllib.request.urlopen(request, context=context, timeout=10) as response: + return response.status, response.headers + except urllib.error.HTTPError as error: + return error.code, error.headers + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="https://localhost:8443") + parser.add_argument("--ca-file", required=True) + args = parser.parse_args() + + context = ssl.create_default_context(cafile=args.ca_file) + username = "limit_" + secrets.token_hex(6) + password = "Rate-limit-" + secrets.token_urlsafe(18) + status, _ = request(context, "POST", args.base_url + "/api/v1/users", { + "username": username, + "display_name": "Rate limit check", + "password": password, + }) + if status != 201: + raise SystemExit(f"registration returned {status}") + + authorization = "Basic " + base64.b64encode((username + ":wrong-password").encode()).decode() + instances = set() + statuses = [] + retry_after = None + for _ in range(6): + status, headers = request(context, "POST", args.base_url + "/api/v1/auth/sessions", headers={ + "Authorization": authorization, + }) + statuses.append(status) + instances.add(headers.get("X-Instance-Id")) + if status == 429: + retry_after = headers.get("Retry-After") + + if statuses != [401, 401, 401, 401, 401, 429]: + raise SystemExit(f"unexpected statuses: {statuses}") + if not retry_after or int(retry_after) < 1: + raise SystemExit("429 response has no positive Retry-After") + instances.discard(None) + if len(instances) < 2: + raise SystemExit(f"requests reached {len(instances)} API instance") + print("PASS: shared login rate limit returned 429 with Retry-After across two API replicas") + + +if __name__ == "__main__": + main() diff --git a/scripts/test-openbao-lifecycle.sh b/scripts/test-openbao-lifecycle.sh new file mode 100755 index 0000000..0ffd61f --- /dev/null +++ b/scripts/test-openbao-lifecycle.sh @@ -0,0 +1,177 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$script_dir/openbao-common.sh" + +require_command curl +require_command docker +require_command python3 +require_command "$BAO_BIN" +require_file "$BAO_CACERT" +require_unsealed +load_operator_token + +test_dir=$(mktemp -d "$openbao_runtime_dir/.lifecycle.XXXXXX") +chmod 0700 "$test_dir" +sealed=0 + +compose() { + docker compose --project-directory "$repo_dir" \ + --env-file "${ENV_FILE:-$repo_dir/lab7.env.example}" "$@" +} + +agent_lookup() { + compose exec -T openbao-agent-worker sh -c ' + BAO_ADDR=https://openbao:8200 \ + BAO_CACERT=/run/security/openbao/ca.crt \ + BAO_TOKEN=$(cat /run/openbao-agent/token) \ + bao token lookup >/dev/null 2>&1 + ' +} + +recover() { + status=$? + trap - EXIT HUP INT TERM + if [ "$sealed" -eq 1 ]; then + "$script_dir/openbao-unseal.sh" >/dev/null || status=1 + compose restart openbao-agent-worker >/dev/null || status=1 + attempts=0 + while [ "$attempts" -lt 30 ] && ! agent_lookup; do + attempts=$((attempts + 1)) + sleep 1 + done + agent_lookup || status=1 + fi + rm -rf "$test_dir" + exit "$status" +} + +trap recover EXIT HUP INT TERM + +http_status() { + header_file=$1 + method=$2 + path=$3 + payload=${4:-} + output=$test_dir/response.json + + if [ -n "$payload" ]; then + curl --silent --show-error --cacert "$BAO_CACERT" \ + --header @"$header_file" \ + --header "Content-Type: application/json" \ + --request "$method" \ + --data-binary "$payload" \ + --output "$output" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/$path" + else + curl --silent --show-error --cacert "$BAO_CACERT" \ + --header @"$header_file" \ + --request "$method" \ + --output "$output" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/$path" + fi +} + +write_token_header() { + response_file=$1 + header_file=$2 + python3 - "$response_file" "$header_file" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + document = json.load(source) +token = document.get("auth", document)["client_token"] +with open(sys.argv[2], "w", encoding="utf-8") as destination: + destination.write(f"X-Vault-Token: {token}\n") +os.chmod(sys.argv[2], 0o600) +PY +} + +issue_login_material() { + name=$1 + identity_dir=$test_dir/$name-identity + payload_file=$test_dir/$name-login.json + + "$script_dir/openbao-issue-approle.sh" worker "$identity_dir" >/dev/null + python3 - "$identity_dir/role-id" "$identity_dir/secret-id" "$payload_file" <<'PY' +import json +import os +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + role_id = source.read().strip() +with open(sys.argv[2], encoding="utf-8") as source: + secret_id = source.read().strip() +with open(sys.argv[3], "w", encoding="utf-8") as destination: + json.dump({"role_id": role_id, "secret_id": secret_id}, destination) +os.chmod(sys.argv[3], 0o600) +PY + printf '%s\n' "$payload_file" +} + +login_payload=$(issue_login_material renewable) +login_response=$test_dir/renewable-login-response.json +login_status=$(curl --silent --show-error --cacert "$BAO_CACERT" \ + --header "Content-Type: application/json" \ + --request POST \ + --data-binary @"$login_payload" \ + --output "$login_response" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/auth/approle/login") +[ "$login_status" = 200 ] || fail "AppRole login returned HTTP $login_status" +chmod 0600 "$login_response" +renewable_header=$test_dir/renewable-token-header +write_token_header "$login_response" "$renewable_header" + +[ "$(http_status "$renewable_header" GET secret/data/messenger/worker)" = 200 ] || fail "worker token cannot read own secret" +[ "$(http_status "$renewable_header" POST auth/token/renew-self '{}')" = 200 ] || fail "worker token renewal failed" +[ "$(http_status "$renewable_header" POST auth/token/revoke-self)" = 204 ] || fail "worker token self-revocation failed" +[ "$(http_status "$renewable_header" GET auth/token/lookup-self)" = 403 ] || fail "revoked token remained valid" + +short_response=$test_dir/short-token-response.json +"$BAO_BIN" token create \ + -policy=messenger-worker \ + -no-default-policy \ + -ttl=2s \ + -explicit-max-ttl=2s \ + -format=json >"$short_response" +chmod 0600 "$short_response" +short_header=$test_dir/short-token-header +write_token_header "$short_response" "$short_header" +[ "$(http_status "$short_header" GET secret/data/messenger/worker)" = 200 ] || fail "short-lived token cannot read own secret" +sleep 3 +[ "$(http_status "$short_header" GET secret/data/messenger/worker)" = 403 ] || fail "expired token remained valid" + +sealed_login_payload=$(issue_login_material sealed) +"$BAO_BIN" operator seal >/dev/null +sealed=1 + +if agent_lookup; then + fail "Agent token lookup succeeded while OpenBao was sealed" +fi +sealed_status=$(curl --silent --show-error --cacert "$BAO_CACERT" \ + --header "Content-Type: application/json" \ + --request POST \ + --data-binary @"$sealed_login_payload" \ + --output "$test_dir/sealed-login-response.json" \ + --write-out '%{http_code}' \ + "$BAO_ADDR/v1/auth/approle/login") +[ "$sealed_status" = 503 ] || fail "sealed AppRole login: expected HTTP 503, got $sealed_status" + +"$script_dir/openbao-unseal.sh" >/dev/null +compose restart openbao-agent-worker >/dev/null +attempts=0 +while [ "$attempts" -lt 30 ] && ! agent_lookup; do + attempts=$((attempts + 1)) + sleep 1 +done +agent_lookup || fail "Agent did not authenticate after unseal" +sealed=0 + +echo "OpenBao lifecycle passed: renew, revoke, expiry, seal denial and Agent re-authentication" diff --git a/scripts/test-tls-matrix.sh b/scripts/test-tls-matrix.sh new file mode 100755 index 0000000..a88a8bb --- /dev/null +++ b/scripts/test-tls-matrix.sh @@ -0,0 +1,163 @@ +#!/bin/sh + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +security_dir=${SECURITY_DIR:-$repo_dir/.runtime/security} +public_port=${PUBLIC_TLS_PORT:-8443} +public_host=${PUBLIC_TLS_HOST:-localhost} +api_address=${API_TLS_ADDRESS:-api:8443} +metrics_address=${METRICS_TLS_ADDRESS:-api:9091} +empty_ca="${TMPDIR:-/tmp}/lab7-empty-ca.$$" + +cleanup() { + rm -f "$empty_ca" +} + +trap cleanup EXIT HUP INT TERM + +require_file() { + if [ ! -r "$1" ]; then + echo "missing security material: $1" >&2 + exit 1 + fi +} + +expect_failure() { + name=$1 + shift + + if "$@" >/dev/null 2>&1; then + echo "FAIL: $name unexpectedly succeeded" >&2 + exit 1 + fi + echo "PASS: $name" +} + +expect_tls_success() { + name=$1 + certificate=$2 + key=$3 + + if curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/internal-ca/ca.crt" \ + --cert "$certificate" --key "$key" \ + "https://$api_address/health/ready" >/dev/null 2>&1; then + echo "PASS: $name" + return + fi + echo "FAIL: $name" >&2 + exit 1 +} + +expect_tls_failure() { + name=$1 + shift + + if curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/internal-ca/ca.crt" \ + "$@" "https://$api_address/health/ready" >/dev/null 2>&1; then + echo "FAIL: $name unexpectedly succeeded" >&2 + exit 1 + fi + echo "PASS: $name" +} + +expect_public_hostname_failure() { + if openssl s_client \ + -connect "$public_host:$public_port" \ + -servername wrong.local \ + -verify_hostname wrong.local \ + -verify_return_error \ + -CAfile "$security_dir/public-ca/ca.crt" /dev/null 2>&1; then + echo "FAIL: public TLS with wrong hostname unexpectedly succeeded" >&2 + exit 1 + fi + echo "PASS: public TLS with wrong hostname" +} + +for file in \ + "$security_dir/public-ca/ca.crt" \ + "$security_dir/internal-ca/ca.crt" \ + "$security_dir/foreign-ca/ca.crt" \ + "$security_dir/load-balancer/tls.crt" \ + "$security_dir/load-balancer/tls.key" \ + "$security_dir/prometheus/tls.crt" \ + "$security_dir/prometheus/tls.key" \ + "$security_dir/forbidden-client/tls.crt" \ + "$security_dir/forbidden-client/tls.key" \ + "$security_dir/foreign-client/tls.crt" \ + "$security_dir/foreign-client/tls.key"; do + require_file "$file" +done + +: >"$empty_ca" +chmod 0600 "$empty_ca" + +curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/public-ca/ca.crt" \ + "https://$public_host:$public_port/health/ready" >/dev/null +echo "PASS: public TLS with trusted CA and hostname" + +expect_failure "public TLS with empty CA" \ + curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$empty_ca" \ + "https://$public_host:$public_port/health/ready" + +expect_failure "public TLS with foreign CA" \ + curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/foreign-ca/ca.crt" \ + "https://$public_host:$public_port/health/ready" + +expect_public_hostname_failure + +expect_tls_failure "API mTLS without client certificate" +expect_tls_failure "API mTLS with forbidden trusted identity" \ + --cert "$security_dir/forbidden-client/tls.crt" \ + --key "$security_dir/forbidden-client/tls.key" +expect_tls_failure "API mTLS with client certificate from foreign CA" \ + --cert "$security_dir/foreign-client/tls.crt" \ + --key "$security_dir/foreign-client/tls.key" +expect_tls_success "API mTLS with load-balancer identity" \ + "$security_dir/load-balancer/tls.crt" \ + "$security_dir/load-balancer/tls.key" + +expect_tls_failure "business API with Prometheus identity" \ + --cert "$security_dir/prometheus/tls.crt" \ + --key "$security_dir/prometheus/tls.key" + +expect_failure "metrics mTLS without client certificate" \ + curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/internal-ca/ca.crt" \ + "https://$metrics_address/metrics" + +expect_failure "metrics mTLS with load-balancer identity" \ + curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/internal-ca/ca.crt" \ + --cert "$security_dir/load-balancer/tls.crt" \ + --key "$security_dir/load-balancer/tls.key" \ + "https://$metrics_address/metrics" + +expect_failure "metrics mTLS with client certificate from foreign CA" \ + curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/internal-ca/ca.crt" \ + --cert "$security_dir/foreign-client/tls.crt" \ + --key "$security_dir/foreign-client/tls.key" \ + "https://$metrics_address/metrics" + +curl --fail --silent --show-error --noproxy '*' \ + --connect-timeout 5 --max-time 10 \ + --cacert "$security_dir/internal-ca/ca.crt" \ + --cert "$security_dir/prometheus/tls.crt" \ + --key "$security_dir/prometheus/tls.key" \ + "https://$metrics_address/metrics" >/dev/null +echo "PASS: metrics mTLS with Prometheus identity"