forked from mtusi2026/mtusi2026lab7
test: verify security boundaries and recovery
This commit is contained in:
+40
-16
@@ -14,15 +14,16 @@ concurrency:
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab6-${{ github.run_id }}
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab7-${{ github.run_id }}
|
||||
API_BIND_ADDRESS: 127.0.0.1:0
|
||||
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
|
||||
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
|
||||
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
|
||||
LOKI_BIND_ADDRESS: 127.0.0.1:0
|
||||
TEMPO_BIND_ADDRESS: 127.0.0.1:0
|
||||
OPENBAO_BIND_ADDRESS: 127.0.0.1:0
|
||||
steps:
|
||||
- name: Checkout exact commit
|
||||
run: |
|
||||
@@ -48,8 +49,8 @@ jobs:
|
||||
- name: Check formatting and shell scripts
|
||||
run: |
|
||||
test -z "$(gofmt -l cmd internal migrations)"
|
||||
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh
|
||||
python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py
|
||||
for file in infra/minio/*.sh infra/pki/*.sh infra/postgres/*.sh infra/redis/*.sh infra/security/*.sh infra/openbao/*.sh scripts/*.sh; do sh -n "$file"; done
|
||||
python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py scripts/test-login-rate-limit.py
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v7
|
||||
@@ -65,28 +66,48 @@ jobs:
|
||||
|
||||
- name: Prepare local stack
|
||||
run: |
|
||||
cp lab6.env.example .env
|
||||
cp lab7.env.example .env
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
|
||||
./scripts/bootstrap-local.sh .env
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME"
|
||||
|
||||
- name: Integration checks
|
||||
run: |
|
||||
set -a
|
||||
. ./.env
|
||||
set +a
|
||||
test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403"
|
||||
make test BASE_URL=http://nginx:8080
|
||||
test "$(curl --noproxy '*' --cacert .runtime/security/internal-ca/ca.crt -s -o /dev/null -w '%{http_code}' "https://s3:9000/messenger-attachments")" = "403"
|
||||
make test BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
|
||||
make test-security BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
|
||||
make test-tls-matrix ENV_FILE=.env
|
||||
ENV_FILE=.env BASE_URL=https://nginx:8443 ./scripts/test-leaf-rotation.sh
|
||||
docker compose --profile security run --rm dependency-tests
|
||||
OPENBAO_ADDRESS=$(docker compose port openbao 8200)
|
||||
ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/openbao-test-policy.sh
|
||||
ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/test-openbao-lifecycle.sh
|
||||
make test-login-rate-limit BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
|
||||
docker compose --profile tests run --rm --build tests
|
||||
go run ./cmd/dependencycheck -base-url=http://nginx:8080
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
|
||||
go run ./cmd/dependencycheck -base-url=https://nginx:8443 -ca-file=.runtime/security/public-ca/ca.crt
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
|
||||
docker compose up -d --scale api=2 --wait --wait-timeout 90
|
||||
make test-lab5-acceptance
|
||||
make test-lab5
|
||||
ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080
|
||||
make test-worker-compromise ENV_FILE=.env
|
||||
make scan-security
|
||||
ENV_FILE=.env CA_FILE=.runtime/security/public-ca/ca.crt PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=https://nginx:8443
|
||||
|
||||
- name: Upload security evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: lab7-security-${{ github.sha }}
|
||||
path: |
|
||||
evidence/lab7-*.json
|
||||
evidence/runtime/security-scan
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
- name: Container logs
|
||||
if: failure()
|
||||
@@ -97,5 +118,8 @@ jobs:
|
||||
run: |
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" 2>/dev/null || true
|
||||
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
|
||||
|
||||
+38
-26
@@ -12,7 +12,7 @@ jobs:
|
||||
name: Verify selected commit
|
||||
if: github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
timeout-minutes: 60
|
||||
outputs:
|
||||
api_image: ${{ steps.publish.outputs.api_image }}
|
||||
nginx_image: ${{ steps.publish.outputs.nginx_image }}
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
s3_image: ${{ steps.publish.outputs.s3_image }}
|
||||
s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }}
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab6-deploy-${{ github.run_id }}
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab7-deploy-${{ github.run_id }}
|
||||
COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml
|
||||
API_BIND_ADDRESS: 127.0.0.1:0
|
||||
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
|
||||
@@ -30,13 +30,14 @@ jobs:
|
||||
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
|
||||
LOKI_BIND_ADDRESS: 127.0.0.1:0
|
||||
TEMPO_BIND_ADDRESS: 127.0.0.1:0
|
||||
API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-api:${{ github.sha }}
|
||||
NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-nginx:${{ github.sha }}
|
||||
SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-swagger:${{ github.sha }}
|
||||
PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-prometheus:${{ github.sha }}
|
||||
GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-grafana:${{ github.sha }}
|
||||
S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio:${{ github.sha }}
|
||||
S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio-client:${{ github.sha }}
|
||||
OPENBAO_BIND_ADDRESS: 127.0.0.1:0
|
||||
API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-api:${{ github.sha }}
|
||||
NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-nginx:${{ github.sha }}
|
||||
SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-swagger:${{ github.sha }}
|
||||
PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-prometheus:${{ github.sha }}
|
||||
GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-grafana:${{ github.sha }}
|
||||
S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-minio:${{ github.sha }}
|
||||
S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-minio-client:${{ github.sha }}
|
||||
steps:
|
||||
- name: Checkout exact commit
|
||||
run: |
|
||||
@@ -62,8 +63,8 @@ jobs:
|
||||
- name: Check formatting and shell scripts
|
||||
run: |
|
||||
test -z "$(gofmt -l cmd internal migrations)"
|
||||
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh
|
||||
python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py
|
||||
for file in infra/minio/*.sh infra/pki/*.sh infra/postgres/*.sh infra/redis/*.sh infra/security/*.sh infra/openbao/*.sh scripts/*.sh; do sh -n "$file"; done
|
||||
python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py scripts/test-login-rate-limit.py
|
||||
|
||||
- name: Lint
|
||||
uses: golangci/golangci-lint-action@v7
|
||||
@@ -79,27 +80,35 @@ jobs:
|
||||
|
||||
- name: Prepare local stack
|
||||
run: |
|
||||
cp lab6.env.example .env
|
||||
cp lab7.env.example .env
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
|
||||
./scripts/bootstrap-local.sh .env
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME"
|
||||
|
||||
- name: Integration checks
|
||||
run: |
|
||||
set -a
|
||||
. ./.env
|
||||
set +a
|
||||
test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403"
|
||||
make test BASE_URL=http://nginx:8080
|
||||
test "$(curl --noproxy '*' --cacert .runtime/security/internal-ca/ca.crt -s -o /dev/null -w '%{http_code}' "https://s3:9000/messenger-attachments")" = "403"
|
||||
make test BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
|
||||
make test-security BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
|
||||
make test-tls-matrix ENV_FILE=.env
|
||||
ENV_FILE=.env BASE_URL=https://nginx:8443 ./scripts/test-leaf-rotation.sh
|
||||
docker compose --profile security run --rm dependency-tests
|
||||
OPENBAO_ADDRESS=$(docker compose port openbao 8200)
|
||||
ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/openbao-test-policy.sh
|
||||
ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/test-openbao-lifecycle.sh
|
||||
make test-login-rate-limit BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
|
||||
docker compose --profile tests run --rm --build tests
|
||||
go run ./cmd/dependencycheck -base-url=http://nginx:8080
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
|
||||
go run ./cmd/dependencycheck -base-url=https://nginx:8443 -ca-file=.runtime/security/public-ca/ca.crt
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
|
||||
make test-lab5-acceptance
|
||||
make test-lab5
|
||||
ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080
|
||||
make test-worker-compromise ENV_FILE=.env
|
||||
ENV_FILE=.env CA_FILE=.runtime/security/public-ca/ca.crt PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=https://nginx:8443
|
||||
|
||||
- name: Publish verified images
|
||||
id: publish
|
||||
@@ -137,7 +146,10 @@ jobs:
|
||||
run: |
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" 2>/dev/null || true
|
||||
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
|
||||
|
||||
deploy:
|
||||
@@ -177,17 +189,17 @@ jobs:
|
||||
run: printf '%s' "$REGISTRY_PASSWORD" | ssh "$DEPLOY_USER@$DEPLOY_HOST" "docker login gitea.privcloud.tech --username '$REGISTRY_USERNAME' --password-stdin"
|
||||
|
||||
- name: Upload image manifest
|
||||
run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab6-${{ github.run_id }}.env"
|
||||
run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab7-${{ github.run_id }}.env"
|
||||
|
||||
- name: Deploy and verify
|
||||
run: |
|
||||
ssh "$DEPLOY_USER@$DEPLOY_HOST" "
|
||||
set -eu
|
||||
cd ~/mtusi2026lab6
|
||||
cd ~/mtusi2026lab7
|
||||
git fetch origin main
|
||||
git cat-file -e '$DEPLOY_SHA^{commit}'
|
||||
git reset --hard '$DEPLOY_SHA'
|
||||
manifest='/tmp/mtusi2026lab6-${{ github.run_id }}.env'
|
||||
manifest='/tmp/mtusi2026lab7-${{ github.run_id }}.env'
|
||||
if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then
|
||||
rm -f \"\$manifest\"
|
||||
else
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
name: Lab 6 incidents
|
||||
name: Lab 7 inherited incidents
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: lab6-incidents-${{ github.repository }}-${{ github.ref }}
|
||||
group: lab7-incidents-${{ github.repository }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
@@ -12,13 +12,14 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 35
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab6-incidents-${{ github.run_id }}
|
||||
COMPOSE_PROJECT_NAME: mtusi2026lab7-incidents-${{ github.run_id }}
|
||||
API_BIND_ADDRESS: 127.0.0.1:0
|
||||
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
|
||||
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
|
||||
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
|
||||
LOKI_BIND_ADDRESS: 127.0.0.1:0
|
||||
TEMPO_BIND_ADDRESS: 127.0.0.1:0
|
||||
OPENBAO_BIND_ADDRESS: 127.0.0.1:0
|
||||
steps:
|
||||
- name: Checkout exact commit
|
||||
run: |
|
||||
@@ -34,17 +35,21 @@ jobs:
|
||||
|
||||
- name: Prepare stack
|
||||
run: |
|
||||
cp lab6.env.example .env
|
||||
cp lab7.env.example .env
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --scale api=2 --wait --wait-timeout 180
|
||||
./scripts/bootstrap-local.sh .env
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME"
|
||||
docker network connect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME"
|
||||
|
||||
- name: Reproduce SLO incidents
|
||||
run: |
|
||||
ENV_FILE=.env \
|
||||
BASE_URL=http://nginx:8080 \
|
||||
BASE_URL=https://nginx:8443 \
|
||||
CA_FILE=.runtime/security/public-ca/ca.crt \
|
||||
PROMETHEUS_URL=http://prometheus:9090 \
|
||||
TEMPO_URL=http://tempo:3200 \
|
||||
LOKI_URL=http://loki:3100 \
|
||||
@@ -69,5 +74,8 @@ jobs:
|
||||
run: |
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" 2>/dev/null || true
|
||||
docker network disconnect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" 2>/dev/null || true
|
||||
docker compose down -v --remove-orphans
|
||||
|
||||
@@ -2,9 +2,10 @@ PYTHON ?= python3
|
||||
BASE_URL ?= https://localhost:8443
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
ENV_FILE ?= $(if $(wildcard .env),.env,lab7.env.example)
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover test-security
|
||||
.PHONY: help check validate test load test-persistence test-failover test-security test-tls-matrix test-leaf-rotation test-openbao-lifecycle test-login-rate-limit test-worker-compromise scan-security test-lab5 test-lab5-acceptance lab6-load test-lab6-smoke test-lab6
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -31,3 +32,37 @@ test-failover:
|
||||
|
||||
test-security:
|
||||
$(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-tls-matrix:
|
||||
docker compose --env-file "$(ENV_FILE)" --profile security run --rm security-tests
|
||||
|
||||
test-leaf-rotation:
|
||||
ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" ./scripts/test-leaf-rotation.sh
|
||||
|
||||
test-openbao-lifecycle:
|
||||
ENV_FILE="$(ENV_FILE)" ./scripts/test-openbao-lifecycle.sh
|
||||
|
||||
test-login-rate-limit:
|
||||
@test -n "$(CA_FILE)" || (echo "Set CA_FILE"; exit 2)
|
||||
$(PYTHON) scripts/test-login-rate-limit.py --base-url "$(BASE_URL)" --ca-file "$(CA_FILE)"
|
||||
|
||||
test-worker-compromise:
|
||||
ENV_FILE="$(ENV_FILE)" ./scripts/lab7-worker-compromise.sh
|
||||
|
||||
scan-security:
|
||||
./scripts/scan-security.sh
|
||||
|
||||
test-lab5:
|
||||
./scripts/test-lab5.sh
|
||||
|
||||
test-lab5-acceptance:
|
||||
./scripts/test-lab5-acceptance.sh
|
||||
|
||||
lab6-load:
|
||||
$(PYTHON) scripts/lab6-traffic.py --base-url "$(BASE_URL)" $(TLS_ARGS) --mode mixed --duration 60 --concurrency 4 --wait-ready
|
||||
|
||||
test-lab6-smoke:
|
||||
ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" ./scripts/test-lab6.sh smoke
|
||||
|
||||
test-lab6:
|
||||
ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" ./scripts/test-lab6.sh full
|
||||
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
|
||||
env_file=${ENV_FILE:-$repo_dir/lab7.env.example}
|
||||
|
||||
exec docker compose --project-directory "$repo_dir" --env-file "$env_file" run --rm --no-deps -T \
|
||||
-v "$repo_dir:$repo_dir:ro" openbao-cli "$@"
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
|
||||
env_file=${1:-$repo_dir/lab7.env.example}
|
||||
|
||||
if [ ! -f "$env_file" ]; then
|
||||
echo "environment file not found: $env_file" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
compose() {
|
||||
docker compose --project-directory "$repo_dir" --env-file "$env_file" "$@"
|
||||
}
|
||||
|
||||
if [ ! -f "$repo_dir/.runtime/security/openbao/tls.crt" ]; then
|
||||
"$repo_dir/infra/pki/bootstrap.sh"
|
||||
fi
|
||||
|
||||
"$script_dir/openbao-seed.sh" "$env_file"
|
||||
compose up -d security-bootstrap openbao-permissions openbao
|
||||
openbao_address=$(compose port openbao 8200)
|
||||
BAO_ADDR="https://$openbao_address"
|
||||
BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt"
|
||||
export BAO_ADDR BAO_CACERT ENV_FILE="$env_file"
|
||||
|
||||
remaining=60
|
||||
until curl --silent --show-error --cacert "$BAO_CACERT" \
|
||||
--output /dev/null "$BAO_ADDR/v1/sys/health"; do
|
||||
remaining=$((remaining - 1))
|
||||
[ "$remaining" -gt 0 ] || {
|
||||
echo "OpenBao did not start" >&2
|
||||
exit 1
|
||||
}
|
||||
sleep 1
|
||||
done
|
||||
|
||||
ENV_FILE="$env_file" "$script_dir/openbao-init.sh"
|
||||
ENV_FILE="$env_file" "$script_dir/openbao-unseal.sh"
|
||||
remaining=60
|
||||
until curl --fail --silent --show-error --cacert "$BAO_CACERT" \
|
||||
--output /dev/null "$BAO_ADDR/v1/sys/health"; do
|
||||
remaining=$((remaining - 1))
|
||||
[ "$remaining" -gt 0 ] || {
|
||||
echo "OpenBao did not become active after unseal" >&2
|
||||
exit 1
|
||||
}
|
||||
sleep 1
|
||||
done
|
||||
ENV_FILE="$env_file" "$script_dir/openbao-bootstrap.sh" \
|
||||
"$repo_dir/.runtime/openbao/seed/api.json" \
|
||||
"$repo_dir/.runtime/openbao/seed/worker.json"
|
||||
|
||||
compose up -d --build --scale api=2 --wait --wait-timeout 240
|
||||
@@ -2,8 +2,9 @@
|
||||
|
||||
set -eu
|
||||
|
||||
env_file=${ENV_FILE:-lab6.env.example}
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
env_file=${ENV_FILE:-lab7.env.example}
|
||||
base_url=${BASE_URL:-https://localhost:8443}
|
||||
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
|
||||
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
||||
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
|
||||
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
|
||||
@@ -33,6 +34,7 @@ trap restore EXIT INT TERM
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
|
||||
python3 scripts/lab6-traffic.py \
|
||||
--base-url "$base_url" \
|
||||
--ca-file "$ca_file" \
|
||||
--mode messages \
|
||||
--duration "$((outage_seconds + 20))" \
|
||||
--concurrency 4 \
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
|
||||
set -eu
|
||||
|
||||
env_file=${ENV_FILE:-lab6.env.example}
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
env_file=${ENV_FILE:-lab7.env.example}
|
||||
base_url=${BASE_URL:-https://localhost:8443}
|
||||
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
|
||||
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
||||
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
|
||||
outage_seconds=${TELEMETRY_OUTAGE_SECONDS:-30}
|
||||
@@ -30,6 +31,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
compose stop tempo loki >/dev/null
|
||||
python3 scripts/lab6-traffic.py \
|
||||
--base-url "$base_url" \
|
||||
--ca-file "$ca_file" \
|
||||
--mode mixed \
|
||||
--duration "$outage_seconds" \
|
||||
--concurrency 4 \
|
||||
@@ -41,7 +43,7 @@ load_pid=$!
|
||||
sleep "$outage_seconds"
|
||||
wait "$load_pid"
|
||||
load_pid=
|
||||
health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' "$base_url/health/ready")
|
||||
health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' --cacert "$ca_file" "$base_url/health/ready")
|
||||
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
|
||||
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
compose up -d --wait --wait-timeout 90 tempo loki >/dev/null
|
||||
|
||||
@@ -3,8 +3,9 @@
|
||||
set -eu
|
||||
|
||||
mode=${1:-short}
|
||||
env_file=${ENV_FILE:-lab6.env.example}
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
env_file=${ENV_FILE:-lab7.env.example}
|
||||
base_url=${BASE_URL:-https://localhost:8443}
|
||||
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
|
||||
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
||||
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
|
||||
require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1}
|
||||
@@ -51,6 +52,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
compose stop worker >/dev/null
|
||||
python3 scripts/lab6-traffic.py \
|
||||
--base-url "$base_url" \
|
||||
--ca-file "$ca_file" \
|
||||
--mode attachments \
|
||||
--duration "$load_seconds" \
|
||||
--requests "$requests" \
|
||||
|
||||
@@ -7,6 +7,7 @@ import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import ssl
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
@@ -16,6 +17,7 @@ import urllib.request
|
||||
import zlib
|
||||
|
||||
MAX_RESPONSE_BYTES = 1 << 20
|
||||
TLS_CONTEXT = None
|
||||
|
||||
|
||||
def png_fixture(width=96, height=64):
|
||||
@@ -37,7 +39,7 @@ def png_fixture(width=96, height=64):
|
||||
def request(method, url, body=None, headers=None, timeout=10):
|
||||
req = urllib.request.Request(url, data=body, method=method, headers=headers or {})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
with urllib.request.urlopen(req, timeout=timeout, context=TLS_CONTEXT) as response:
|
||||
payload = response.read(MAX_RESPONSE_BYTES)
|
||||
return response.status, payload
|
||||
except urllib.error.HTTPError as error:
|
||||
@@ -167,7 +169,8 @@ def wait_for_attachments(base_url, token, attachment_ids, timeout):
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Bounded synthetic load for Lab 6 telemetry and incidents")
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--base-url", default="https://localhost:8443")
|
||||
parser.add_argument("--ca-file", default=".runtime/security/public-ca/ca.crt")
|
||||
parser.add_argument("--mode", choices=("messages", "attachments", "mixed"), default="mixed")
|
||||
parser.add_argument("--duration", type=float, default=30)
|
||||
parser.add_argument("--requests", type=int, default=0, help="Maximum requests; zero means duration only")
|
||||
@@ -179,6 +182,10 @@ def main():
|
||||
parser.add_argument("--started-file", help="Touch after synthetic session and chat are ready")
|
||||
args = parser.parse_args()
|
||||
|
||||
global TLS_CONTEXT
|
||||
if args.base_url.startswith("https://"):
|
||||
TLS_CONTEXT = ssl.create_default_context(cafile=args.ca_file)
|
||||
|
||||
if args.duration <= 0 or args.duration > 1800:
|
||||
parser.error("duration must be in (0, 1800]")
|
||||
if args.requests < 0:
|
||||
|
||||
Executable
+209
@@ -0,0 +1,209 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
|
||||
env_file=${ENV_FILE:-$repo_dir/lab7.env.example}
|
||||
runtime_dir="$repo_dir/.runtime/evidence/lab7-worker"
|
||||
state_file="$runtime_dir/queued-attachment.json"
|
||||
rotated_secret="$repo_dir/.runtime/openbao/seed/worker-rotated.json"
|
||||
rotated_password="rotated-worker-$(date -u +%Y%m%d%H%M%S)"
|
||||
rotated_database_password="rotated-db-$(date -u +%Y%m%d%H%M%S)"
|
||||
rotated_s3_secret="rotated-s3-$(date -u +%Y%m%d%H%M%S)"
|
||||
summary_file="$repo_dir/evidence/lab7-worker-compromise.json"
|
||||
started_at=$(date +%s)
|
||||
|
||||
compose() {
|
||||
docker compose --project-directory "$repo_dir" --env-file "$env_file" "$@"
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
service=$1
|
||||
remaining=60
|
||||
while [ "$remaining" -gt 0 ]; do
|
||||
container=$(compose ps -q "$service")
|
||||
if [ -n "$container" ] && [ "$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")" = healthy ]; then
|
||||
return 0
|
||||
fi
|
||||
remaining=$((remaining - 1))
|
||||
sleep 1
|
||||
done
|
||||
echo "$service did not become healthy" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
rotate_worker_secret() {
|
||||
secret_file=$1
|
||||
compose stop openbao-agent-worker >/dev/null
|
||||
ENV_FILE="$env_file" BAO_ADDR="$BAO_ADDR" BAO_CACERT="$BAO_CACERT" \
|
||||
"$script_dir/openbao-rotate.sh" worker "$secret_file" >/dev/null
|
||||
docker start "$(compose ps -aq openbao-agent-worker)" >/dev/null
|
||||
wait_healthy openbao-agent-worker
|
||||
}
|
||||
|
||||
restore() {
|
||||
compose stop worker openbao-agent-worker >/dev/null 2>&1 || true
|
||||
compose run --rm --no-deps acl-bootstrap >/dev/null 2>&1 || true
|
||||
compose restart redis-broker redis-gateway >/dev/null 2>&1 || true
|
||||
compose run --rm --no-deps postgres-roles >/dev/null 2>&1 || true
|
||||
compose run --rm --no-deps s3-bootstrap >/dev/null 2>&1 || true
|
||||
rotate_worker_secret "$repo_dir/.runtime/openbao/seed/worker.json" >/dev/null 2>&1 || true
|
||||
compose up -d --no-deps --force-recreate worker >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
mkdir -p "$runtime_dir"
|
||||
chmod 0700 "$repo_dir/.runtime/evidence" "$runtime_dir"
|
||||
trap restore EXIT HUP INT TERM
|
||||
|
||||
openbao_address=$(compose port openbao 8200)
|
||||
BAO_ADDR="https://$openbao_address"
|
||||
BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt"
|
||||
export BAO_ADDR BAO_CACERT
|
||||
|
||||
compose stop worker >/dev/null
|
||||
compose --profile experiments run --rm --no-deps lab5-experiment /app/lab5experiment api-produce >"$state_file"
|
||||
chmod 0600 "$state_file"
|
||||
|
||||
old_database_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
import urllib.parse
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
secret = json.load(source)
|
||||
print(urllib.parse.urlsplit(secret["database_url"]).password)
|
||||
PY
|
||||
)
|
||||
old_broker_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
import urllib.parse
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
print(urllib.parse.urlsplit(json.load(source)["broker_url"]).password)
|
||||
PY
|
||||
)
|
||||
old_s3_access_key=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
print(json.load(source)["s3_access_key"])
|
||||
PY
|
||||
)
|
||||
old_s3_secret=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
print(json.load(source)["s3_secret_key"])
|
||||
PY
|
||||
)
|
||||
|
||||
python3 - "$repo_dir/.runtime/openbao/seed/worker.json" "$rotated_secret" \
|
||||
"$rotated_password" "$rotated_database_password" "$rotated_s3_secret" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.parse
|
||||
|
||||
|
||||
def replace_password(url, password):
|
||||
parsed = urllib.parse.urlsplit(url)
|
||||
username = urllib.parse.quote(parsed.username or "", safe="")
|
||||
netloc = f"{username}:{urllib.parse.quote(password, safe='')}@{parsed.hostname}"
|
||||
if parsed.port:
|
||||
netloc += f":{parsed.port}"
|
||||
return urllib.parse.urlunsplit((parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment))
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
secret = json.load(source)
|
||||
secret["broker_url"] = replace_password(secret["broker_url"], sys.argv[3])
|
||||
secret["database_url"] = replace_password(secret["database_url"], sys.argv[4])
|
||||
secret["s3_secret_key"] = sys.argv[5]
|
||||
with open(sys.argv[2], "w", encoding="utf-8") as destination:
|
||||
json.dump(secret, destination, separators=(",", ":"))
|
||||
destination.write("\n")
|
||||
os.chmod(sys.argv[2], 0o600)
|
||||
PY
|
||||
|
||||
compose run --rm --no-deps -e REDIS_WORKER_PASSWORD="$rotated_password" acl-bootstrap >/dev/null
|
||||
compose restart redis-broker redis-gateway >/dev/null
|
||||
compose exec -T postgres sh -c \
|
||||
"psql --set=ON_ERROR_STOP=1 --username \"\$POSTGRES_USER\" --dbname \"\$POSTGRES_DB\" --command \"ALTER ROLE messenger_worker PASSWORD '$rotated_database_password'\"" >/dev/null
|
||||
compose exec -T postgres sh -c \
|
||||
'psql --set=ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" --command="SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = '\''messenger_worker'\'' AND pid <> pg_backend_pid()"' >/dev/null
|
||||
compose run --rm --no-deps -e S3_WORKER_SECRET_KEY="$rotated_s3_secret" s3-bootstrap >/dev/null
|
||||
compose --profile security run --rm --no-deps \
|
||||
-e REDISCLI_AUTH="$old_broker_password" \
|
||||
--entrypoint /bin/sh dependency-tests -c '
|
||||
output=$(redis-cli --tls --cacert /run/security-source/internal-ca/ca.crt \
|
||||
--cert /run/security-source/worker-client/tls.crt --key /run/security-source/worker-client/tls.key \
|
||||
--user messenger_worker -h redis-gateway -p 36379 PING 2>&1 || true)
|
||||
case "$output" in
|
||||
*WRONGPASS*|*invalid\ username-password*) exit 0 ;;
|
||||
*) echo "expected old broker credential rejection" >&2; exit 1 ;;
|
||||
esac
|
||||
'
|
||||
|
||||
if compose --profile security run --rm --no-deps \
|
||||
-e OLD_DATABASE_PASSWORD="$old_database_password" \
|
||||
--entrypoint /bin/sh dependency-tests -c '
|
||||
psql "postgresql://messenger_worker:${OLD_DATABASE_PASSWORD}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/security-source/internal-ca/ca.crt&sslcert=/run/security-source/worker-client/tls.crt&sslkey=/run/security-source/worker-client/tls.key" \
|
||||
--set=ON_ERROR_STOP=1 --command="SELECT 1" >/dev/null 2>&1
|
||||
'; then
|
||||
echo "old worker database credential was not rejected" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if compose --profile security run --rm --no-deps \
|
||||
-e OLD_S3_ACCESS_KEY="$old_s3_access_key" -e OLD_S3_SECRET="$old_s3_secret" \
|
||||
--entrypoint /bin/sh dependency-tests -c '
|
||||
export SSL_CERT_FILE=/run/security-source/internal-ca/ca.crt
|
||||
export MC_CONFIG_DIR=/tmp/old-worker-mc
|
||||
mc alias set old https://s3:9000 "$OLD_S3_ACCESS_KEY" "$OLD_S3_SECRET" >/dev/null 2>&1 &&
|
||||
mc ls old/messenger-attachments/attachments/ >/dev/null 2>&1
|
||||
'; then
|
||||
echo "old worker S3 credential was not rejected" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rotate_worker_secret "$rotated_secret"
|
||||
compose up -d --no-deps --force-recreate worker >/dev/null
|
||||
wait_healthy worker
|
||||
|
||||
compose --profile experiments run --rm --no-deps -T lab5-experiment \
|
||||
/app/lab5experiment api-verify <"$state_file" >/dev/null
|
||||
|
||||
finished_at=$(date +%s)
|
||||
python3 - "$summary_file" "$started_at" "$finished_at" <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import sys
|
||||
|
||||
started_at = int(sys.argv[2])
|
||||
finished_at = int(sys.argv[3])
|
||||
summary = {
|
||||
"schema_version": 1,
|
||||
"scenario": "worker credential compromise and rotation",
|
||||
"generated_at": datetime.datetime.fromtimestamp(
|
||||
finished_at, datetime.timezone.utc
|
||||
).replace(microsecond=0).isoformat(),
|
||||
"recovery_seconds": finished_at - started_at,
|
||||
"old_credential_rejected": True,
|
||||
"old_database_credential_rejected": True,
|
||||
"old_s3_credential_rejected": True,
|
||||
"openbao_secret_rotated": True,
|
||||
"queued_attachment_preserved": True,
|
||||
"result": "PASS",
|
||||
}
|
||||
with open(sys.argv[1], "w", encoding="utf-8") as destination:
|
||||
json.dump(summary, destination, ensure_ascii=False, indent=2)
|
||||
destination.write("\n")
|
||||
PY
|
||||
|
||||
trap - EXIT HUP INT TERM
|
||||
restore
|
||||
rm -f "$rotated_secret"
|
||||
echo "PASS: revoked worker credential failed; rotated OpenBao secret restored processing without losing the queued attachment"
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
. "$script_dir/openbao-common.sh"
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "usage: $0 <api-secret-json> <worker-secret-json>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
api_secret_file=$1
|
||||
worker_secret_file=$2
|
||||
|
||||
require_command "$BAO_BIN"
|
||||
require_command python3
|
||||
require_runtime_path "$api_secret_file"
|
||||
require_runtime_path "$worker_secret_file"
|
||||
require_private_file "$api_secret_file"
|
||||
require_private_file "$worker_secret_file"
|
||||
require_unsealed
|
||||
load_operator_token
|
||||
|
||||
python3 - "$api_secret_file" "$worker_secret_file" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
for path in sys.argv[1:]:
|
||||
with open(path, encoding="utf-8") as source:
|
||||
value = json.load(source)
|
||||
if not isinstance(value, dict) or not value:
|
||||
raise SystemExit(f"secret input must be a non-empty JSON object: {path}")
|
||||
PY
|
||||
|
||||
mounts_file=$(mktemp "$openbao_runtime_dir/.mounts.XXXXXX")
|
||||
auth_file=$(mktemp "$openbao_runtime_dir/.auth.XXXXXX")
|
||||
chmod 0600 "$mounts_file" "$auth_file"
|
||||
|
||||
cleanup() {
|
||||
rm -f "$mounts_file" "$auth_file"
|
||||
}
|
||||
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
"$BAO_BIN" secrets list -format=json >"$mounts_file"
|
||||
if ! python3 - "$mounts_file" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
raise SystemExit(0 if "secret/" in json.load(source) else 1)
|
||||
PY
|
||||
then
|
||||
"$BAO_BIN" secrets enable -path=secret -version=2 kv >/dev/null
|
||||
fi
|
||||
|
||||
"$BAO_BIN" auth list -format=json >"$auth_file"
|
||||
if ! python3 - "$auth_file" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
raise SystemExit(0 if "approle/" in json.load(source) else 1)
|
||||
PY
|
||||
then
|
||||
"$BAO_BIN" auth enable approle >/dev/null
|
||||
fi
|
||||
|
||||
"$BAO_BIN" policy write messenger-api "$repo_dir/infra/openbao/policies/messenger-api.hcl" >/dev/null
|
||||
"$BAO_BIN" policy write messenger-worker "$repo_dir/infra/openbao/policies/messenger-worker.hcl" >/dev/null
|
||||
|
||||
for service in api worker; do
|
||||
"$BAO_BIN" delete "auth/approle/role/messenger-$service" >/dev/null 2>&1 || true
|
||||
"$BAO_BIN" write "auth/approle/role/messenger-$service" \
|
||||
bind_secret_id=true \
|
||||
secret_id_ttl=1h \
|
||||
secret_id_num_uses=0 \
|
||||
token_policies="messenger-$service" \
|
||||
token_no_default_policy=true \
|
||||
token_ttl=15m \
|
||||
token_max_ttl=1h \
|
||||
token_num_uses=0 >/dev/null
|
||||
done
|
||||
|
||||
"$BAO_BIN" kv put -mount=secret messenger/api @"$api_secret_file" >/dev/null
|
||||
"$BAO_BIN" kv put -mount=secret messenger/worker @"$worker_secret_file" >/dev/null
|
||||
|
||||
"$script_dir/openbao-issue-approle.sh" api "$openbao_runtime_dir/approle/api"
|
||||
"$script_dir/openbao-issue-approle.sh" worker "$openbao_runtime_dir/approle/worker"
|
||||
|
||||
echo "OpenBao policies, AppRoles and KV v2 secrets configured"
|
||||
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
umask 077
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
|
||||
openbao_runtime_dir="$repo_dir/.runtime/openbao"
|
||||
openbao_operator_file="$openbao_runtime_dir/operator/init.json"
|
||||
|
||||
BAO_BIN=${BAO_BIN:-$repo_dir/scripts/bao-docker.sh}
|
||||
BAO_ADDR=${BAO_ADDR:-https://localhost:8200}
|
||||
BAO_CACERT=${BAO_CACERT:-$repo_dir/.runtime/security/openbao/ca.crt}
|
||||
|
||||
export BAO_ADDR BAO_CACERT
|
||||
|
||||
fail() {
|
||||
echo "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1"
|
||||
}
|
||||
|
||||
require_file() {
|
||||
[ -f "$1" ] || fail "required file not found: $1"
|
||||
}
|
||||
|
||||
require_private_file() {
|
||||
require_file "$1"
|
||||
permissions=$(stat -f '%Lp' "$1" 2>/dev/null || stat -c '%a' "$1")
|
||||
case "$permissions" in
|
||||
600|400) ;;
|
||||
*) fail "file must have permissions 0600 or 0400: $1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
require_runtime_path() {
|
||||
case "$1" in
|
||||
"$openbao_runtime_dir"/*) ;;
|
||||
*) fail "runtime credentials must stay under $openbao_runtime_dir" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
json_field() {
|
||||
file=$1
|
||||
expression=$2
|
||||
python3 - "$file" "$expression" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
value = json.load(source)
|
||||
|
||||
for component in sys.argv[2].split("."):
|
||||
value = value[component]
|
||||
|
||||
if isinstance(value, bool):
|
||||
print(str(value).lower())
|
||||
else:
|
||||
print(value)
|
||||
PY
|
||||
}
|
||||
|
||||
load_operator_token() {
|
||||
require_private_file "$openbao_operator_file"
|
||||
BAO_TOKEN=$(json_field "$openbao_operator_file" root_token)
|
||||
export BAO_TOKEN
|
||||
}
|
||||
|
||||
openbao_status() {
|
||||
status_file=$1
|
||||
"$BAO_BIN" status -format=json >"$status_file" 2>/dev/null || true
|
||||
[ -s "$status_file" ] || fail "OpenBao is not reachable at $BAO_ADDR"
|
||||
}
|
||||
|
||||
require_unsealed() {
|
||||
mkdir -p "$openbao_runtime_dir"
|
||||
chmod 0700 "$openbao_runtime_dir"
|
||||
status_file=$(mktemp "$openbao_runtime_dir/.status.XXXXXX")
|
||||
chmod 0600 "$status_file"
|
||||
openbao_status "$status_file"
|
||||
initialized=$(json_field "$status_file" initialized)
|
||||
sealed=$(json_field "$status_file" sealed)
|
||||
rm -f "$status_file"
|
||||
[ "$initialized" = "true" ] || fail "OpenBao is not initialized"
|
||||
[ "$sealed" = "false" ] || fail "OpenBao is sealed"
|
||||
}
|
||||
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
. "$script_dir/openbao-common.sh"
|
||||
|
||||
require_command "$BAO_BIN"
|
||||
require_command python3
|
||||
require_file "$BAO_CACERT"
|
||||
|
||||
mkdir -p "$openbao_runtime_dir/operator"
|
||||
chmod 0700 "$openbao_runtime_dir" "$openbao_runtime_dir/operator"
|
||||
|
||||
status_file=$(mktemp "$openbao_runtime_dir/.status.XXXXXX")
|
||||
init_file=$(mktemp "$openbao_runtime_dir/operator/.init.XXXXXX")
|
||||
chmod 0600 "$status_file" "$init_file"
|
||||
|
||||
cleanup() {
|
||||
rm -f "$status_file" "$init_file"
|
||||
}
|
||||
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
openbao_status "$status_file"
|
||||
|
||||
if [ "$(json_field "$status_file" initialized)" = "true" ]; then
|
||||
echo "OpenBao is already initialized"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -e "$openbao_operator_file" ]; then
|
||||
fail "operator material already exists for an uninitialized OpenBao instance"
|
||||
fi
|
||||
|
||||
"$BAO_BIN" operator init \
|
||||
-key-shares=3 \
|
||||
-key-threshold=2 \
|
||||
-format=json >"$init_file"
|
||||
|
||||
chmod 0600 "$init_file"
|
||||
mv "$init_file" "$openbao_operator_file"
|
||||
|
||||
echo "OpenBao initialized; operator material saved to .runtime/openbao/operator/init.json"
|
||||
echo "Keep this file offline and run scripts/openbao-unseal.sh"
|
||||
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
. "$script_dir/openbao-common.sh"
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "usage: $0 <api|worker> <runtime-output-directory>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
service=$1
|
||||
output_dir=$2
|
||||
|
||||
case "$service" in
|
||||
api|worker) ;;
|
||||
*) fail "service must be api or worker" ;;
|
||||
esac
|
||||
|
||||
require_runtime_path "$output_dir"
|
||||
require_command "$BAO_BIN"
|
||||
require_command python3
|
||||
require_unsealed
|
||||
|
||||
if [ -z "${BAO_TOKEN:-}" ]; then
|
||||
load_operator_token
|
||||
fi
|
||||
|
||||
role="messenger-$service"
|
||||
parent_dir=$(dirname "$output_dir")
|
||||
staging_dir="$parent_dir/.${service}.$$"
|
||||
response_file="$staging_dir/response.json"
|
||||
|
||||
cleanup() {
|
||||
rm -rf "$staging_dir"
|
||||
}
|
||||
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
mkdir -p "$parent_dir" "$staging_dir"
|
||||
chmod 0700 "$parent_dir" "$staging_dir"
|
||||
|
||||
"$BAO_BIN" read -field=role_id "auth/approle/role/$role/role-id" >"$staging_dir/role-id"
|
||||
"$BAO_BIN" write -f -format=json "auth/approle/role/$role/secret-id" >"$response_file"
|
||||
|
||||
python3 - "$response_file" "$staging_dir" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
data = json.load(source)["data"]
|
||||
|
||||
for name, field in (("secret-id", "secret_id"), ("secret-id-accessor", "secret_id_accessor")):
|
||||
path = os.path.join(sys.argv[2], name)
|
||||
with open(path, "w", encoding="utf-8") as destination:
|
||||
destination.write(data[field] + "\n")
|
||||
os.chmod(path, 0o600)
|
||||
PY
|
||||
|
||||
rm -f "$response_file"
|
||||
chmod 0600 "$staging_dir/role-id"
|
||||
|
||||
old_accessor=
|
||||
if [ -f "$output_dir/secret-id-accessor" ]; then
|
||||
old_accessor=$(cat "$output_dir/secret-id-accessor")
|
||||
fi
|
||||
|
||||
mkdir -p "$output_dir"
|
||||
chmod 0700 "$output_dir"
|
||||
for file in role-id secret-id secret-id-accessor; do
|
||||
mv "$staging_dir/$file" "$output_dir/$file"
|
||||
done
|
||||
|
||||
if [ -n "$old_accessor" ]; then
|
||||
"$BAO_BIN" write "auth/approle/role/$role/secret-id-accessor/destroy" \
|
||||
secret_id_accessor="$old_accessor" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
if [ "$output_dir" = "$openbao_runtime_dir/approle/$service" ]; then
|
||||
docker compose --project-directory "$repo_dir" --env-file "${ENV_FILE:-$repo_dir/lab7.env.example}" \
|
||||
run --rm --no-deps -e SERVICE="$service" openbao-identity-install
|
||||
fi
|
||||
|
||||
echo "issued short-lived AppRole credentials for $service in ${output_dir#"$repo_dir/"}"
|
||||
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
. "$script_dir/openbao-common.sh"
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "usage: $0 <api|worker> <replacement-secret-json>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
service=$1
|
||||
secret_file=$2
|
||||
|
||||
case "$service" in
|
||||
api|worker) ;;
|
||||
*) fail "service must be api or worker" ;;
|
||||
esac
|
||||
|
||||
require_command "$BAO_BIN"
|
||||
require_command python3
|
||||
require_runtime_path "$secret_file"
|
||||
require_private_file "$secret_file"
|
||||
require_unsealed
|
||||
load_operator_token
|
||||
|
||||
python3 - "$secret_file" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
value = json.load(source)
|
||||
if not isinstance(value, dict) or not value:
|
||||
raise SystemExit("replacement secret must be a non-empty JSON object")
|
||||
PY
|
||||
|
||||
"$BAO_BIN" kv put -mount=secret "messenger/$service" @"$secret_file" >/dev/null
|
||||
"$script_dir/openbao-issue-approle.sh" "$service" "$openbao_runtime_dir/approle/$service"
|
||||
|
||||
echo "$service secret rotated; OpenBao Agent will render the new KV version"
|
||||
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
|
||||
echo "usage: $0 <env-file>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
|
||||
seed_dir="$repo_dir/.runtime/openbao/seed"
|
||||
mkdir -p "$seed_dir"
|
||||
chmod 0700 "$repo_dir/.runtime/openbao" "$seed_dir"
|
||||
|
||||
python3 - "$1" "$seed_dir/api.json" "$seed_dir/worker.json" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
values = {}
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
for raw_line in source:
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
name, separator, value = line.partition("=")
|
||||
if not separator:
|
||||
raise SystemExit(f"invalid env line: {raw_line.rstrip()}")
|
||||
values[name.strip()] = os.environ.get(name.strip(), value)
|
||||
|
||||
required = (
|
||||
"DATABASE_URL", "DATABASE_WORKER_URL", "CACHE_URL", "BROKER_WORKER_URL",
|
||||
"S3_API_ACCESS_KEY", "S3_API_SECRET_KEY", "S3_WORKER_ACCESS_KEY", "S3_WORKER_SECRET_KEY",
|
||||
)
|
||||
missing = [name for name in required if not values.get(name)]
|
||||
if missing:
|
||||
raise SystemExit("missing environment values: " + ", ".join(missing))
|
||||
|
||||
documents = (
|
||||
(sys.argv[2], {
|
||||
"database_url": values["DATABASE_URL"],
|
||||
"cache_url": values["CACHE_URL"],
|
||||
"s3_access_key": values["S3_API_ACCESS_KEY"],
|
||||
"s3_secret_key": values["S3_API_SECRET_KEY"],
|
||||
}),
|
||||
(sys.argv[3], {
|
||||
"database_url": values["DATABASE_WORKER_URL"],
|
||||
"broker_url": values["BROKER_WORKER_URL"],
|
||||
"s3_access_key": values["S3_WORKER_ACCESS_KEY"],
|
||||
"s3_secret_key": values["S3_WORKER_SECRET_KEY"],
|
||||
}),
|
||||
)
|
||||
|
||||
for path, value in documents:
|
||||
with open(path, "w", encoding="utf-8") as destination:
|
||||
json.dump(value, destination, separators=(",", ":"))
|
||||
destination.write("\n")
|
||||
os.chmod(path, 0o600)
|
||||
PY
|
||||
|
||||
echo "OpenBao seed files created under .runtime/openbao/seed"
|
||||
Executable
+140
@@ -0,0 +1,140 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
. "$script_dir/openbao-common.sh"
|
||||
|
||||
require_command curl
|
||||
require_command python3
|
||||
require_command "$BAO_BIN"
|
||||
require_file "$BAO_CACERT"
|
||||
require_unsealed
|
||||
load_operator_token
|
||||
|
||||
test_dir="$openbao_runtime_dir/policy-test.$$"
|
||||
mkdir -p "$test_dir"
|
||||
chmod 0700 "$test_dir"
|
||||
|
||||
cleanup() {
|
||||
for service in api worker; do
|
||||
headers="$test_dir/$service/token-header"
|
||||
if [ -f "$headers" ]; then
|
||||
curl --silent --show-error --cacert "$BAO_CACERT" \
|
||||
--header @"$headers" \
|
||||
--request POST \
|
||||
"$BAO_ADDR/v1/auth/token/revoke-self" >/dev/null 2>&1 || true
|
||||
fi
|
||||
done
|
||||
rm -rf "$test_dir"
|
||||
}
|
||||
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
login() {
|
||||
service=$1
|
||||
identity_dir="$test_dir/$service/identity"
|
||||
session_dir="$test_dir/$service"
|
||||
response_file="$session_dir/login.json"
|
||||
payload_file="$session_dir/login-payload.json"
|
||||
|
||||
mkdir -p "$session_dir"
|
||||
chmod 0700 "$session_dir"
|
||||
"$script_dir/openbao-issue-approle.sh" "$service" "$identity_dir" >/dev/null
|
||||
|
||||
python3 - "$identity_dir/role-id" "$identity_dir/secret-id" "$payload_file" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as role_source:
|
||||
role_id = role_source.read().strip()
|
||||
with open(sys.argv[2], encoding="utf-8") as secret_source:
|
||||
secret_id = secret_source.read().strip()
|
||||
with open(sys.argv[3], "w", encoding="utf-8") as destination:
|
||||
json.dump({"role_id": role_id, "secret_id": secret_id}, destination)
|
||||
os.chmod(sys.argv[3], 0o600)
|
||||
PY
|
||||
|
||||
status=$(curl --silent --show-error --cacert "$BAO_CACERT" \
|
||||
--header "Content-Type: application/json" \
|
||||
--request POST \
|
||||
--data-binary @"$payload_file" \
|
||||
--output "$response_file" \
|
||||
--write-out '%{http_code}' \
|
||||
"$BAO_ADDR/v1/auth/approle/login")
|
||||
[ "$status" = "200" ] || fail "$service AppRole login returned HTTP $status"
|
||||
chmod 0600 "$response_file"
|
||||
|
||||
python3 - "$response_file" "$service" "$session_dir/token-header" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
auth = json.load(source)["auth"]
|
||||
|
||||
expected_policy = f"messenger-{sys.argv[2]}"
|
||||
if not auth["renewable"]:
|
||||
raise SystemExit("AppRole token is not renewable")
|
||||
if not 0 < auth["lease_duration"] <= 3600:
|
||||
raise SystemExit("AppRole token TTL is outside the configured limit")
|
||||
if auth["policies"] != [expected_policy]:
|
||||
raise SystemExit(f"unexpected token policies: {auth['policies']}")
|
||||
|
||||
with open(sys.argv[3], "w", encoding="utf-8") as destination:
|
||||
destination.write(f"X-Vault-Token: {auth['client_token']}\n")
|
||||
os.chmod(sys.argv[3], 0o600)
|
||||
PY
|
||||
|
||||
rm -f "$payload_file" "$response_file" "$identity_dir/secret-id"
|
||||
}
|
||||
|
||||
expect_status() {
|
||||
service=$1
|
||||
method=$2
|
||||
path=$3
|
||||
expected=$4
|
||||
payload=${5:-}
|
||||
headers="$test_dir/$service/token-header"
|
||||
output="$test_dir/$service/response.json"
|
||||
|
||||
if [ -n "$payload" ]; then
|
||||
status=$(curl --silent --show-error --cacert "$BAO_CACERT" \
|
||||
--header @"$headers" \
|
||||
--header "Content-Type: application/json" \
|
||||
--request "$method" \
|
||||
--data-binary "$payload" \
|
||||
--output "$output" \
|
||||
--write-out '%{http_code}' \
|
||||
"$BAO_ADDR/v1/$path")
|
||||
else
|
||||
status=$(curl --silent --show-error --cacert "$BAO_CACERT" \
|
||||
--header @"$headers" \
|
||||
--request "$method" \
|
||||
--output "$output" \
|
||||
--write-out '%{http_code}' \
|
||||
"$BAO_ADDR/v1/$path")
|
||||
fi
|
||||
|
||||
[ "$status" = "$expected" ] || fail "$service $method $path: expected HTTP $expected, got $status"
|
||||
echo "PASS $service $method $path -> $status"
|
||||
}
|
||||
|
||||
login api
|
||||
login worker
|
||||
|
||||
expect_status api GET secret/data/messenger/api 200
|
||||
expect_status api GET secret/data/messenger/worker 403
|
||||
expect_status api POST secret/data/messenger/api/probe 403 '{"data":{"probe":"denied"}}'
|
||||
expect_status api GET sys/policies/acl 403
|
||||
expect_status api POST auth/token/renew-self 200 '{}'
|
||||
|
||||
expect_status worker GET secret/data/messenger/worker 200
|
||||
expect_status worker GET secret/data/messenger/api 403
|
||||
expect_status worker POST secret/data/messenger/worker/probe 403 '{"data":{"probe":"denied"}}'
|
||||
expect_status worker GET sys/policies/acl 403
|
||||
expect_status worker POST auth/token/renew-self 200 '{}'
|
||||
|
||||
echo "OpenBao AppRole allow/deny and renewal checks passed"
|
||||
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
. "$script_dir/openbao-common.sh"
|
||||
|
||||
require_command "$BAO_BIN"
|
||||
require_command python3
|
||||
require_file "$BAO_CACERT"
|
||||
require_private_file "$openbao_operator_file"
|
||||
|
||||
mkdir -p "$openbao_runtime_dir"
|
||||
keys_file=$(mktemp "$openbao_runtime_dir/.unseal.XXXXXX")
|
||||
status_file=$(mktemp "$openbao_runtime_dir/.status.XXXXXX")
|
||||
chmod 0600 "$keys_file" "$status_file"
|
||||
|
||||
cleanup() {
|
||||
rm -f "$keys_file" "$status_file"
|
||||
}
|
||||
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
openbao_status "$status_file"
|
||||
|
||||
if [ "$(json_field "$status_file" sealed)" = "false" ]; then
|
||||
echo "OpenBao is already unsealed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
python3 - "$openbao_operator_file" >"$keys_file" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
material = json.load(source)
|
||||
|
||||
threshold = material["unseal_threshold"]
|
||||
for key in material["unseal_keys_b64"][:threshold]:
|
||||
print(key)
|
||||
PY
|
||||
|
||||
while IFS= read -r key; do
|
||||
"$BAO_BIN" operator unseal "$key" </dev/null >/dev/null
|
||||
done <"$keys_file"
|
||||
|
||||
openbao_status "$status_file"
|
||||
[ "$(json_field "$status_file" sealed)" = "false" ] || fail "OpenBao remains sealed"
|
||||
|
||||
echo "OpenBao unsealed"
|
||||
|
||||
@@ -8,7 +8,7 @@ if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
|
||||
fi
|
||||
|
||||
manifest=$1
|
||||
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab5"}
|
||||
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab7"}
|
||||
current_manifest=$deploy_dir/current-images.env
|
||||
active_containers=$deploy_dir/active-api-containers
|
||||
previous_containers=$deploy_dir/previous-api-containers
|
||||
@@ -16,7 +16,9 @@ upstream_file=$deploy_dir/upstream.conf
|
||||
upstream_backup=$deploy_dir/upstream.previous.conf
|
||||
candidate_file=$deploy_dir/candidate-api-containers
|
||||
replicas=${API_REPLICAS:-2}
|
||||
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab5}
|
||||
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab7}
|
||||
public_base_url=${PUBLIC_BASE_URL:-https://localhost:8443}
|
||||
public_ca_file=${PUBLIC_CA_FILE:-.runtime/security/public-ca/ca.crt}
|
||||
switched=false
|
||||
nginx_created=false
|
||||
upstream_changed=false
|
||||
@@ -52,7 +54,7 @@ write_upstream() {
|
||||
: > "$output"
|
||||
for container in $containers; do
|
||||
name=$(container_name "$container")
|
||||
printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
|
||||
printf 'server %s:8443 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
|
||||
done
|
||||
}
|
||||
|
||||
@@ -81,7 +83,8 @@ wait_for_containers() {
|
||||
}
|
||||
|
||||
smoke_check() {
|
||||
compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--cacert "$public_ca_file" "$public_base_url/health/ready" >/dev/null
|
||||
}
|
||||
|
||||
remove_containers() {
|
||||
@@ -123,6 +126,7 @@ trap 'exit 143' TERM
|
||||
|
||||
load_manifest "$manifest"
|
||||
new_api_image=$API_IMAGE
|
||||
test -r "$public_ca_file"
|
||||
|
||||
if [ -f "$previous_containers" ]; then
|
||||
remove_containers "$(cat "$previous_containers")"
|
||||
@@ -151,6 +155,7 @@ cp "$upstream_file" "$upstream_backup"
|
||||
|
||||
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
|
||||
compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3
|
||||
compose run --rm security-bootstrap
|
||||
compose run --rm --no-deps s3-bootstrap
|
||||
compose run --rm --no-deps migrate
|
||||
|
||||
|
||||
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
|
||||
runtime_dir="$repo_dir/evidence/runtime/security-scan"
|
||||
cache_dir="$repo_dir/.runtime/trivy-cache"
|
||||
raw_file="$runtime_dir/trivy-fs.json"
|
||||
summary_file="$repo_dir/evidence/lab7-security-scan.json"
|
||||
env_file=${ENV_FILE:-$repo_dir/lab7.env.example}
|
||||
image_list=${SCAN_IMAGES:-}
|
||||
if [ -z "$image_list" ]; then
|
||||
containers=$(docker compose --project-directory "$repo_dir" --env-file "$env_file" ps -q)
|
||||
[ -n "$containers" ] || { echo "start the stack or set SCAN_IMAGES" >&2; exit 1; }
|
||||
image_list=$(docker inspect --format '{{.Config.Image}}' $containers | sort -u)
|
||||
fi
|
||||
|
||||
mkdir -p "$runtime_dir" "$cache_dir"
|
||||
|
||||
docker run --rm \
|
||||
-v "$repo_dir:/src:ro" \
|
||||
-v "$runtime_dir:/out" \
|
||||
-v "$cache_dir:/root/.cache/trivy" \
|
||||
aquasec/trivy:0.76.2 fs \
|
||||
--scanners vuln,misconfig,secret \
|
||||
--format json \
|
||||
--output /out/trivy-fs.json \
|
||||
--skip-dirs /src/.git \
|
||||
--skip-dirs /src/.runtime \
|
||||
--skip-dirs /src/.venv \
|
||||
--skip-dirs /src/evidence/runtime \
|
||||
--skip-files /src/Dockerfile.example \
|
||||
/src
|
||||
|
||||
image_reports=
|
||||
index=0
|
||||
for image in $image_list; do
|
||||
index=$((index + 1))
|
||||
report="$runtime_dir/trivy-image-$index.json"
|
||||
docker run --rm \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v "$runtime_dir:/out" \
|
||||
-v "$cache_dir:/root/.cache/trivy" \
|
||||
aquasec/trivy:0.76.2 image \
|
||||
--scanners vuln,secret \
|
||||
--format json \
|
||||
--output "/out/trivy-image-$index.json" \
|
||||
"$image"
|
||||
image_reports="$image_reports $image=$report"
|
||||
done
|
||||
|
||||
python3 - "$raw_file" "$summary_file" $image_reports <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
report = json.load(source)
|
||||
|
||||
vulnerabilities = Counter()
|
||||
misconfigurations = Counter()
|
||||
secrets = 0
|
||||
for result in report.get("Results") or []:
|
||||
for vulnerability in result.get("Vulnerabilities") or []:
|
||||
vulnerabilities[vulnerability.get("Severity", "UNKNOWN")] += 1
|
||||
for finding in result.get("Misconfigurations") or []:
|
||||
misconfigurations[finding.get("Severity", "UNKNOWN")] += 1
|
||||
secrets += len(result.get("Secrets") or [])
|
||||
|
||||
image_vulnerabilities = Counter()
|
||||
image_secrets = 0
|
||||
scanned_images = []
|
||||
image_raw_reports = []
|
||||
findings = {}
|
||||
|
||||
|
||||
def record_findings(report, scope):
|
||||
for result in report.get("Results") or []:
|
||||
for finding in result.get("Vulnerabilities") or []:
|
||||
key = (finding["VulnerabilityID"], finding["PkgName"], finding.get("InstalledVersion", ""))
|
||||
entry = findings.setdefault(key, {
|
||||
"id": finding["VulnerabilityID"],
|
||||
"package": finding["PkgName"],
|
||||
"installed_version": finding.get("InstalledVersion", ""),
|
||||
"fixed_version": finding.get("FixedVersion", ""),
|
||||
"severity": finding.get("Severity", "UNKNOWN"),
|
||||
"title": finding.get("Title", ""),
|
||||
"scopes": [],
|
||||
})
|
||||
if scope not in entry["scopes"]:
|
||||
entry["scopes"].append(scope)
|
||||
|
||||
|
||||
record_findings(report, "repository")
|
||||
for item in sys.argv[3:]:
|
||||
image, path = item.split("=", 1)
|
||||
scanned_images.append(image)
|
||||
with open(path, encoding="utf-8") as source:
|
||||
image_report = json.load(source)
|
||||
image_raw_reports.append("evidence/runtime/security-scan/" + path.rsplit("/", 1)[-1])
|
||||
record_findings(image_report, image)
|
||||
for result in image_report.get("Results") or []:
|
||||
for vulnerability in result.get("Vulnerabilities") or []:
|
||||
image_vulnerabilities[vulnerability.get("Severity", "UNKNOWN")] += 1
|
||||
image_secrets += len(result.get("Secrets") or [])
|
||||
|
||||
summary = {
|
||||
"schema_version": 1,
|
||||
"scanner": "Trivy 0.76.2",
|
||||
"generated_at": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat(),
|
||||
"scope": "repository filesystem: vulnerabilities, misconfigurations, secrets",
|
||||
"vulnerabilities": dict(sorted(vulnerabilities.items())),
|
||||
"misconfigurations": dict(sorted(misconfigurations.items())),
|
||||
"secret_findings": secrets,
|
||||
"scanned_images": scanned_images,
|
||||
"image_vulnerabilities": dict(sorted(image_vulnerabilities.items())),
|
||||
"image_secret_findings": image_secrets,
|
||||
"findings": sorted((finding for finding in findings.values()
|
||||
if "repository" in finding["scopes"] or finding["severity"] == "CRITICAL"),
|
||||
key=lambda finding: (finding["package"], finding["id"])),
|
||||
"findings_scope": "all repository findings and critical image findings; full findings in raw reports",
|
||||
"raw_report": "evidence/runtime/security-scan/trivy-fs.json",
|
||||
"image_raw_reports": image_raw_reports,
|
||||
}
|
||||
with open(sys.argv[2], "w", encoding="utf-8") as destination:
|
||||
json.dump(summary, destination, ensure_ascii=False, indent=2)
|
||||
destination.write("\n")
|
||||
PY
|
||||
|
||||
cat "$summary_file"
|
||||
@@ -11,8 +11,9 @@ case "$profile" in
|
||||
;;
|
||||
esac
|
||||
|
||||
env_file=${ENV_FILE:-lab6.env.example}
|
||||
base_url=${BASE_URL:-http://127.0.0.1:8080}
|
||||
env_file=${ENV_FILE:-lab7.env.example}
|
||||
base_url=${BASE_URL:-https://localhost:8443}
|
||||
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
|
||||
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
|
||||
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
|
||||
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
|
||||
@@ -35,6 +36,7 @@ compose config --quiet
|
||||
compose up -d --build --wait --wait-timeout 180
|
||||
python3 scripts/lab6-traffic.py \
|
||||
--base-url "$base_url" \
|
||||
--ca-file "$ca_file" \
|
||||
--mode mixed \
|
||||
--duration "${BASELINE_LOAD_SECONDS:-15}" \
|
||||
--requests "${BASELINE_REQUESTS:-120}" \
|
||||
@@ -54,10 +56,10 @@ python3 scripts/lab6-correlation.py \
|
||||
--require >/dev/null
|
||||
|
||||
if [ "$profile" = "full" ]; then
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short
|
||||
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh
|
||||
fi
|
||||
|
||||
arguments=
|
||||
|
||||
Executable
+160
@@ -0,0 +1,160 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
|
||||
env_file=${ENV_FILE:-$repo_dir/lab7.env.example}
|
||||
security_dir=$repo_dir/.runtime/security
|
||||
rotation_dir=$security_dir/rotation-test
|
||||
old_dir=$rotation_dir/old-load-balancer
|
||||
new_dir=$rotation_dir/new-load-balancer
|
||||
expired_dir=$rotation_dir/expired-load-balancer
|
||||
override_file=$repo_dir/.runtime/leaf-rotation.override.yaml
|
||||
new_identity=spiffe://mtusi/lab7/load-balancer-v2
|
||||
rotation_active=0
|
||||
base_url=${BASE_URL:-https://localhost:8443}
|
||||
|
||||
compose() {
|
||||
docker compose --env-file "$env_file" -f "$repo_dir/compose.yaml" "$@"
|
||||
}
|
||||
|
||||
compose_rotated() {
|
||||
docker compose --env-file "$env_file" \
|
||||
-f "$repo_dir/compose.yaml" -f "$override_file" "$@"
|
||||
}
|
||||
|
||||
install_security_material() {
|
||||
compose run --rm --no-deps security-bootstrap >/dev/null
|
||||
}
|
||||
|
||||
wait_for_https() {
|
||||
attempts=0
|
||||
while [ "$attempts" -lt 30 ]; do
|
||||
if curl --silent --show-error --fail \
|
||||
--cacert "$security_dir/public-ca/ca.crt" \
|
||||
"$base_url/health/ready" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
attempts=$((attempts + 1))
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
restore() {
|
||||
status=$?
|
||||
trap - EXIT HUP INT TERM
|
||||
|
||||
if [ "$rotation_active" -eq 1 ] && [ -f "$old_dir/tls.crt" ]; then
|
||||
cp "$old_dir/tls.crt" "$security_dir/load-balancer/tls.crt"
|
||||
cp "$old_dir/tls.key" "$security_dir/load-balancer/tls.key"
|
||||
chmod 0600 "$security_dir/load-balancer/tls.crt" "$security_dir/load-balancer/tls.key"
|
||||
install_security_material || status=1
|
||||
compose up -d --no-deps --no-build --force-recreate api nginx >/dev/null || status=1
|
||||
wait_for_https || status=1
|
||||
fi
|
||||
|
||||
rm -rf "$rotation_dir"
|
||||
rm -f "$override_file"
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap restore EXIT HUP INT TERM
|
||||
|
||||
[ -f "$env_file" ] || {
|
||||
echo "environment file not found: $env_file" >&2
|
||||
exit 1
|
||||
}
|
||||
[ -f "$security_dir/internal-ca/ca.crt" ] || {
|
||||
echo "run infra/pki/bootstrap.sh first" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
umask 077
|
||||
rm -rf "$rotation_dir"
|
||||
mkdir -p "$old_dir" "$new_dir" "$expired_dir"
|
||||
cp "$security_dir/load-balancer/tls.crt" "$old_dir/tls.crt"
|
||||
cp "$security_dir/load-balancer/tls.key" "$old_dir/tls.key"
|
||||
|
||||
cat >"$rotation_dir/extensions.cnf" <<EOF
|
||||
basicConstraints = critical,CA:FALSE
|
||||
keyUsage = critical,digitalSignature
|
||||
extendedKeyUsage = clientAuth
|
||||
subjectAltName = URI:$new_identity
|
||||
subjectKeyIdentifier = hash
|
||||
authorityKeyIdentifier = keyid,issuer
|
||||
EOF
|
||||
|
||||
issue_leaf() {
|
||||
directory=$1
|
||||
days=$2
|
||||
|
||||
openssl genpkey -quiet -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$directory/tls.key"
|
||||
openssl req -new -sha256 \
|
||||
-key "$directory/tls.key" \
|
||||
-out "$directory/tls.csr" \
|
||||
-subj "/CN=load-balancer-v2"
|
||||
openssl x509 -req -sha256 -days "$days" \
|
||||
-in "$directory/tls.csr" \
|
||||
-CA "$security_dir/internal-ca/ca.crt" \
|
||||
-CAkey "$security_dir/internal-ca/ca.key" \
|
||||
-CAcreateserial \
|
||||
-out "$directory/tls.crt" \
|
||||
-extfile "$rotation_dir/extensions.cnf" >/dev/null
|
||||
rm "$directory/tls.csr"
|
||||
}
|
||||
|
||||
issue_leaf "$new_dir" 30
|
||||
issue_leaf "$expired_dir" 0
|
||||
|
||||
openssl verify -purpose sslclient \
|
||||
-CAfile "$security_dir/internal-ca/ca.crt" "$new_dir/tls.crt" >/dev/null
|
||||
if openssl verify -purpose sslclient -attime "$(( $(date +%s) + 2 ))" \
|
||||
-CAfile "$security_dir/internal-ca/ca.crt" "$expired_dir/tls.crt" >/dev/null 2>&1; then
|
||||
echo "zero-lifetime certificate unexpectedly passed expiry verification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat >"$override_file" <<EOF
|
||||
services:
|
||||
api:
|
||||
environment:
|
||||
APP_TLS_ALLOWED_CLIENT_URI: $new_identity
|
||||
EOF
|
||||
|
||||
cp "$new_dir/tls.crt" "$security_dir/load-balancer/tls.crt"
|
||||
cp "$new_dir/tls.key" "$security_dir/load-balancer/tls.key"
|
||||
chmod 0600 "$security_dir/load-balancer/tls.crt" "$security_dir/load-balancer/tls.key"
|
||||
rotation_active=1
|
||||
|
||||
install_security_material
|
||||
compose_rotated up -d --no-deps --no-build --force-recreate api nginx >/dev/null
|
||||
wait_for_https
|
||||
|
||||
compose_rotated --profile security run --rm --no-deps --entrypoint /bin/sh security-tests -c '
|
||||
set -eu
|
||||
curl --silent --show-error --fail \
|
||||
--cacert /src/.runtime/security/internal-ca/ca.crt \
|
||||
--cert /src/.runtime/security/load-balancer/tls.crt \
|
||||
--key /src/.runtime/security/load-balancer/tls.key \
|
||||
https://api:8443/health/ready >/dev/null
|
||||
if curl --silent --show-error --fail \
|
||||
--cacert /src/.runtime/security/internal-ca/ca.crt \
|
||||
--cert /src/.runtime/security/rotation-test/old-load-balancer/tls.crt \
|
||||
--key /src/.runtime/security/rotation-test/old-load-balancer/tls.key \
|
||||
https://api:8443/health/ready >/dev/null 2>&1; then
|
||||
echo "retired load-balancer identity unexpectedly reached API" >&2
|
||||
exit 1
|
||||
fi
|
||||
if curl --silent --show-error --fail \
|
||||
--cacert /src/.runtime/security/internal-ca/ca.crt \
|
||||
--cert /src/.runtime/security/rotation-test/expired-load-balancer/tls.crt \
|
||||
--key /src/.runtime/security/rotation-test/expired-load-balancer/tls.key \
|
||||
https://api:8443/health/ready >/dev/null 2>&1; then
|
||||
echo "expired load-balancer leaf unexpectedly reached API" >&2
|
||||
exit 1
|
||||
fi
|
||||
'
|
||||
|
||||
echo "leaf rotation passed: new identity accepted, retired identity rejected, expired leaf rejected"
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import secrets
|
||||
import ssl
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
|
||||
def request(context, method, url, payload=None, headers=None):
|
||||
data = None if payload is None else json.dumps(payload).encode("utf-8")
|
||||
request_headers = dict(headers or {})
|
||||
if data is not None:
|
||||
request_headers["Content-Type"] = "application/json"
|
||||
request = urllib.request.Request(url, data=data, method=method, headers=request_headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, context=context, timeout=10) as response:
|
||||
return response.status, response.headers
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, error.headers
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", default="https://localhost:8443")
|
||||
parser.add_argument("--ca-file", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
context = ssl.create_default_context(cafile=args.ca_file)
|
||||
username = "limit_" + secrets.token_hex(6)
|
||||
password = "Rate-limit-" + secrets.token_urlsafe(18)
|
||||
status, _ = request(context, "POST", args.base_url + "/api/v1/users", {
|
||||
"username": username,
|
||||
"display_name": "Rate limit check",
|
||||
"password": password,
|
||||
})
|
||||
if status != 201:
|
||||
raise SystemExit(f"registration returned {status}")
|
||||
|
||||
authorization = "Basic " + base64.b64encode((username + ":wrong-password").encode()).decode()
|
||||
instances = set()
|
||||
statuses = []
|
||||
retry_after = None
|
||||
for _ in range(6):
|
||||
status, headers = request(context, "POST", args.base_url + "/api/v1/auth/sessions", headers={
|
||||
"Authorization": authorization,
|
||||
})
|
||||
statuses.append(status)
|
||||
instances.add(headers.get("X-Instance-Id"))
|
||||
if status == 429:
|
||||
retry_after = headers.get("Retry-After")
|
||||
|
||||
if statuses != [401, 401, 401, 401, 401, 429]:
|
||||
raise SystemExit(f"unexpected statuses: {statuses}")
|
||||
if not retry_after or int(retry_after) < 1:
|
||||
raise SystemExit("429 response has no positive Retry-After")
|
||||
instances.discard(None)
|
||||
if len(instances) < 2:
|
||||
raise SystemExit(f"requests reached {len(instances)} API instance")
|
||||
print("PASS: shared login rate limit returned 429 with Retry-After across two API replicas")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+177
@@ -0,0 +1,177 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
. "$script_dir/openbao-common.sh"
|
||||
|
||||
require_command curl
|
||||
require_command docker
|
||||
require_command python3
|
||||
require_command "$BAO_BIN"
|
||||
require_file "$BAO_CACERT"
|
||||
require_unsealed
|
||||
load_operator_token
|
||||
|
||||
test_dir=$(mktemp -d "$openbao_runtime_dir/.lifecycle.XXXXXX")
|
||||
chmod 0700 "$test_dir"
|
||||
sealed=0
|
||||
|
||||
compose() {
|
||||
docker compose --project-directory "$repo_dir" \
|
||||
--env-file "${ENV_FILE:-$repo_dir/lab7.env.example}" "$@"
|
||||
}
|
||||
|
||||
agent_lookup() {
|
||||
compose exec -T openbao-agent-worker sh -c '
|
||||
BAO_ADDR=https://openbao:8200 \
|
||||
BAO_CACERT=/run/security/openbao/ca.crt \
|
||||
BAO_TOKEN=$(cat /run/openbao-agent/token) \
|
||||
bao token lookup >/dev/null 2>&1
|
||||
'
|
||||
}
|
||||
|
||||
recover() {
|
||||
status=$?
|
||||
trap - EXIT HUP INT TERM
|
||||
if [ "$sealed" -eq 1 ]; then
|
||||
"$script_dir/openbao-unseal.sh" >/dev/null || status=1
|
||||
compose restart openbao-agent-worker >/dev/null || status=1
|
||||
attempts=0
|
||||
while [ "$attempts" -lt 30 ] && ! agent_lookup; do
|
||||
attempts=$((attempts + 1))
|
||||
sleep 1
|
||||
done
|
||||
agent_lookup || status=1
|
||||
fi
|
||||
rm -rf "$test_dir"
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap recover EXIT HUP INT TERM
|
||||
|
||||
http_status() {
|
||||
header_file=$1
|
||||
method=$2
|
||||
path=$3
|
||||
payload=${4:-}
|
||||
output=$test_dir/response.json
|
||||
|
||||
if [ -n "$payload" ]; then
|
||||
curl --silent --show-error --cacert "$BAO_CACERT" \
|
||||
--header @"$header_file" \
|
||||
--header "Content-Type: application/json" \
|
||||
--request "$method" \
|
||||
--data-binary "$payload" \
|
||||
--output "$output" \
|
||||
--write-out '%{http_code}' \
|
||||
"$BAO_ADDR/v1/$path"
|
||||
else
|
||||
curl --silent --show-error --cacert "$BAO_CACERT" \
|
||||
--header @"$header_file" \
|
||||
--request "$method" \
|
||||
--output "$output" \
|
||||
--write-out '%{http_code}' \
|
||||
"$BAO_ADDR/v1/$path"
|
||||
fi
|
||||
}
|
||||
|
||||
write_token_header() {
|
||||
response_file=$1
|
||||
header_file=$2
|
||||
python3 - "$response_file" "$header_file" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
document = json.load(source)
|
||||
token = document.get("auth", document)["client_token"]
|
||||
with open(sys.argv[2], "w", encoding="utf-8") as destination:
|
||||
destination.write(f"X-Vault-Token: {token}\n")
|
||||
os.chmod(sys.argv[2], 0o600)
|
||||
PY
|
||||
}
|
||||
|
||||
issue_login_material() {
|
||||
name=$1
|
||||
identity_dir=$test_dir/$name-identity
|
||||
payload_file=$test_dir/$name-login.json
|
||||
|
||||
"$script_dir/openbao-issue-approle.sh" worker "$identity_dir" >/dev/null
|
||||
python3 - "$identity_dir/role-id" "$identity_dir/secret-id" "$payload_file" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as source:
|
||||
role_id = source.read().strip()
|
||||
with open(sys.argv[2], encoding="utf-8") as source:
|
||||
secret_id = source.read().strip()
|
||||
with open(sys.argv[3], "w", encoding="utf-8") as destination:
|
||||
json.dump({"role_id": role_id, "secret_id": secret_id}, destination)
|
||||
os.chmod(sys.argv[3], 0o600)
|
||||
PY
|
||||
printf '%s\n' "$payload_file"
|
||||
}
|
||||
|
||||
login_payload=$(issue_login_material renewable)
|
||||
login_response=$test_dir/renewable-login-response.json
|
||||
login_status=$(curl --silent --show-error --cacert "$BAO_CACERT" \
|
||||
--header "Content-Type: application/json" \
|
||||
--request POST \
|
||||
--data-binary @"$login_payload" \
|
||||
--output "$login_response" \
|
||||
--write-out '%{http_code}' \
|
||||
"$BAO_ADDR/v1/auth/approle/login")
|
||||
[ "$login_status" = 200 ] || fail "AppRole login returned HTTP $login_status"
|
||||
chmod 0600 "$login_response"
|
||||
renewable_header=$test_dir/renewable-token-header
|
||||
write_token_header "$login_response" "$renewable_header"
|
||||
|
||||
[ "$(http_status "$renewable_header" GET secret/data/messenger/worker)" = 200 ] || fail "worker token cannot read own secret"
|
||||
[ "$(http_status "$renewable_header" POST auth/token/renew-self '{}')" = 200 ] || fail "worker token renewal failed"
|
||||
[ "$(http_status "$renewable_header" POST auth/token/revoke-self)" = 204 ] || fail "worker token self-revocation failed"
|
||||
[ "$(http_status "$renewable_header" GET auth/token/lookup-self)" = 403 ] || fail "revoked token remained valid"
|
||||
|
||||
short_response=$test_dir/short-token-response.json
|
||||
"$BAO_BIN" token create \
|
||||
-policy=messenger-worker \
|
||||
-no-default-policy \
|
||||
-ttl=2s \
|
||||
-explicit-max-ttl=2s \
|
||||
-format=json >"$short_response"
|
||||
chmod 0600 "$short_response"
|
||||
short_header=$test_dir/short-token-header
|
||||
write_token_header "$short_response" "$short_header"
|
||||
[ "$(http_status "$short_header" GET secret/data/messenger/worker)" = 200 ] || fail "short-lived token cannot read own secret"
|
||||
sleep 3
|
||||
[ "$(http_status "$short_header" GET secret/data/messenger/worker)" = 403 ] || fail "expired token remained valid"
|
||||
|
||||
sealed_login_payload=$(issue_login_material sealed)
|
||||
"$BAO_BIN" operator seal >/dev/null
|
||||
sealed=1
|
||||
|
||||
if agent_lookup; then
|
||||
fail "Agent token lookup succeeded while OpenBao was sealed"
|
||||
fi
|
||||
sealed_status=$(curl --silent --show-error --cacert "$BAO_CACERT" \
|
||||
--header "Content-Type: application/json" \
|
||||
--request POST \
|
||||
--data-binary @"$sealed_login_payload" \
|
||||
--output "$test_dir/sealed-login-response.json" \
|
||||
--write-out '%{http_code}' \
|
||||
"$BAO_ADDR/v1/auth/approle/login")
|
||||
[ "$sealed_status" = 503 ] || fail "sealed AppRole login: expected HTTP 503, got $sealed_status"
|
||||
|
||||
"$script_dir/openbao-unseal.sh" >/dev/null
|
||||
compose restart openbao-agent-worker >/dev/null
|
||||
attempts=0
|
||||
while [ "$attempts" -lt 30 ] && ! agent_lookup; do
|
||||
attempts=$((attempts + 1))
|
||||
sleep 1
|
||||
done
|
||||
agent_lookup || fail "Agent did not authenticate after unseal"
|
||||
sealed=0
|
||||
|
||||
echo "OpenBao lifecycle passed: renew, revoke, expiry, seal denial and Agent re-authentication"
|
||||
Executable
+163
@@ -0,0 +1,163 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
|
||||
security_dir=${SECURITY_DIR:-$repo_dir/.runtime/security}
|
||||
public_port=${PUBLIC_TLS_PORT:-8443}
|
||||
public_host=${PUBLIC_TLS_HOST:-localhost}
|
||||
api_address=${API_TLS_ADDRESS:-api:8443}
|
||||
metrics_address=${METRICS_TLS_ADDRESS:-api:9091}
|
||||
empty_ca="${TMPDIR:-/tmp}/lab7-empty-ca.$$"
|
||||
|
||||
cleanup() {
|
||||
rm -f "$empty_ca"
|
||||
}
|
||||
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
require_file() {
|
||||
if [ ! -r "$1" ]; then
|
||||
echo "missing security material: $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
expect_failure() {
|
||||
name=$1
|
||||
shift
|
||||
|
||||
if "$@" >/dev/null 2>&1; then
|
||||
echo "FAIL: $name unexpectedly succeeded" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "PASS: $name"
|
||||
}
|
||||
|
||||
expect_tls_success() {
|
||||
name=$1
|
||||
certificate=$2
|
||||
key=$3
|
||||
|
||||
if curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/internal-ca/ca.crt" \
|
||||
--cert "$certificate" --key "$key" \
|
||||
"https://$api_address/health/ready" >/dev/null 2>&1; then
|
||||
echo "PASS: $name"
|
||||
return
|
||||
fi
|
||||
echo "FAIL: $name" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
expect_tls_failure() {
|
||||
name=$1
|
||||
shift
|
||||
|
||||
if curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/internal-ca/ca.crt" \
|
||||
"$@" "https://$api_address/health/ready" >/dev/null 2>&1; then
|
||||
echo "FAIL: $name unexpectedly succeeded" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "PASS: $name"
|
||||
}
|
||||
|
||||
expect_public_hostname_failure() {
|
||||
if openssl s_client \
|
||||
-connect "$public_host:$public_port" \
|
||||
-servername wrong.local \
|
||||
-verify_hostname wrong.local \
|
||||
-verify_return_error \
|
||||
-CAfile "$security_dir/public-ca/ca.crt" </dev/null >/dev/null 2>&1; then
|
||||
echo "FAIL: public TLS with wrong hostname unexpectedly succeeded" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "PASS: public TLS with wrong hostname"
|
||||
}
|
||||
|
||||
for file in \
|
||||
"$security_dir/public-ca/ca.crt" \
|
||||
"$security_dir/internal-ca/ca.crt" \
|
||||
"$security_dir/foreign-ca/ca.crt" \
|
||||
"$security_dir/load-balancer/tls.crt" \
|
||||
"$security_dir/load-balancer/tls.key" \
|
||||
"$security_dir/prometheus/tls.crt" \
|
||||
"$security_dir/prometheus/tls.key" \
|
||||
"$security_dir/forbidden-client/tls.crt" \
|
||||
"$security_dir/forbidden-client/tls.key" \
|
||||
"$security_dir/foreign-client/tls.crt" \
|
||||
"$security_dir/foreign-client/tls.key"; do
|
||||
require_file "$file"
|
||||
done
|
||||
|
||||
: >"$empty_ca"
|
||||
chmod 0600 "$empty_ca"
|
||||
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/public-ca/ca.crt" \
|
||||
"https://$public_host:$public_port/health/ready" >/dev/null
|
||||
echo "PASS: public TLS with trusted CA and hostname"
|
||||
|
||||
expect_failure "public TLS with empty CA" \
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$empty_ca" \
|
||||
"https://$public_host:$public_port/health/ready"
|
||||
|
||||
expect_failure "public TLS with foreign CA" \
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/foreign-ca/ca.crt" \
|
||||
"https://$public_host:$public_port/health/ready"
|
||||
|
||||
expect_public_hostname_failure
|
||||
|
||||
expect_tls_failure "API mTLS without client certificate"
|
||||
expect_tls_failure "API mTLS with forbidden trusted identity" \
|
||||
--cert "$security_dir/forbidden-client/tls.crt" \
|
||||
--key "$security_dir/forbidden-client/tls.key"
|
||||
expect_tls_failure "API mTLS with client certificate from foreign CA" \
|
||||
--cert "$security_dir/foreign-client/tls.crt" \
|
||||
--key "$security_dir/foreign-client/tls.key"
|
||||
expect_tls_success "API mTLS with load-balancer identity" \
|
||||
"$security_dir/load-balancer/tls.crt" \
|
||||
"$security_dir/load-balancer/tls.key"
|
||||
|
||||
expect_tls_failure "business API with Prometheus identity" \
|
||||
--cert "$security_dir/prometheus/tls.crt" \
|
||||
--key "$security_dir/prometheus/tls.key"
|
||||
|
||||
expect_failure "metrics mTLS without client certificate" \
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/internal-ca/ca.crt" \
|
||||
"https://$metrics_address/metrics"
|
||||
|
||||
expect_failure "metrics mTLS with load-balancer identity" \
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/internal-ca/ca.crt" \
|
||||
--cert "$security_dir/load-balancer/tls.crt" \
|
||||
--key "$security_dir/load-balancer/tls.key" \
|
||||
"https://$metrics_address/metrics"
|
||||
|
||||
expect_failure "metrics mTLS with client certificate from foreign CA" \
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/internal-ca/ca.crt" \
|
||||
--cert "$security_dir/foreign-client/tls.crt" \
|
||||
--key "$security_dir/foreign-client/tls.key" \
|
||||
"https://$metrics_address/metrics"
|
||||
|
||||
curl --fail --silent --show-error --noproxy '*' \
|
||||
--connect-timeout 5 --max-time 10 \
|
||||
--cacert "$security_dir/internal-ca/ca.crt" \
|
||||
--cert "$security_dir/prometheus/tls.crt" \
|
||||
--key "$security_dir/prometheus/tls.key" \
|
||||
"https://$metrics_address/metrics" >/dev/null
|
||||
echo "PASS: metrics mTLS with Prometheus identity"
|
||||
Reference in New Issue
Block a user