test: verify security boundaries and recovery

This commit is contained in:
2026-11-06 21:17:03 +03:00
parent ee547e5523
commit 5ac64f51fa
26 changed files with 1752 additions and 68 deletions
+40 -16
View File
@@ -14,15 +14,16 @@ concurrency:
jobs:
verify:
runs-on: ubuntu-latest
timeout-minutes: 45
timeout-minutes: 60
env:
COMPOSE_PROJECT_NAME: mtusi2026lab6-${{ github.run_id }}
COMPOSE_PROJECT_NAME: mtusi2026lab7-${{ github.run_id }}
API_BIND_ADDRESS: 127.0.0.1:0
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
LOKI_BIND_ADDRESS: 127.0.0.1:0
TEMPO_BIND_ADDRESS: 127.0.0.1:0
OPENBAO_BIND_ADDRESS: 127.0.0.1:0
steps:
- name: Checkout exact commit
run: |
@@ -48,8 +49,8 @@ jobs:
- name: Check formatting and shell scripts
run: |
test -z "$(gofmt -l cmd internal migrations)"
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh
python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py
for file in infra/minio/*.sh infra/pki/*.sh infra/postgres/*.sh infra/redis/*.sh infra/security/*.sh infra/openbao/*.sh scripts/*.sh; do sh -n "$file"; done
python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py scripts/test-login-rate-limit.py
- name: Lint
uses: golangci/golangci-lint-action@v7
@@ -65,28 +66,48 @@ jobs:
- name: Prepare local stack
run: |
cp lab6.env.example .env
cp lab7.env.example .env
docker compose config --quiet
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
./scripts/bootstrap-local.sh .env
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME"
- name: Integration checks
run: |
set -a
. ./.env
set +a
test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403"
make test BASE_URL=http://nginx:8080
test "$(curl --noproxy '*' --cacert .runtime/security/internal-ca/ca.crt -s -o /dev/null -w '%{http_code}' "https://s3:9000/messenger-attachments")" = "403"
make test BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
make test-security BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
make test-tls-matrix ENV_FILE=.env
ENV_FILE=.env BASE_URL=https://nginx:8443 ./scripts/test-leaf-rotation.sh
docker compose --profile security run --rm dependency-tests
OPENBAO_ADDRESS=$(docker compose port openbao 8200)
ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/openbao-test-policy.sh
ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/test-openbao-lifecycle.sh
make test-login-rate-limit BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
docker compose --profile tests run --rm --build tests
go run ./cmd/dependencycheck -base-url=http://nginx:8080
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
go run ./cmd/dependencycheck -base-url=https://nginx:8443 -ca-file=.runtime/security/public-ca/ca.crt
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
docker compose up -d --scale api=2 --wait --wait-timeout 90
make test-lab5-acceptance
make test-lab5
ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080
make test-worker-compromise ENV_FILE=.env
make scan-security
ENV_FILE=.env CA_FILE=.runtime/security/public-ca/ca.crt PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=https://nginx:8443
- name: Upload security evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: lab7-security-${{ github.sha }}
path: |
evidence/lab7-*.json
evidence/runtime/security-scan
if-no-files-found: warn
retention-days: 14
- name: Container logs
if: failure()
@@ -97,5 +118,8 @@ jobs:
run: |
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" 2>/dev/null || true
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
+38 -26
View File
@@ -12,7 +12,7 @@ jobs:
name: Verify selected commit
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
timeout-minutes: 45
timeout-minutes: 60
outputs:
api_image: ${{ steps.publish.outputs.api_image }}
nginx_image: ${{ steps.publish.outputs.nginx_image }}
@@ -22,7 +22,7 @@ jobs:
s3_image: ${{ steps.publish.outputs.s3_image }}
s3_bootstrap_image: ${{ steps.publish.outputs.s3_bootstrap_image }}
env:
COMPOSE_PROJECT_NAME: mtusi2026lab6-deploy-${{ github.run_id }}
COMPOSE_PROJECT_NAME: mtusi2026lab7-deploy-${{ github.run_id }}
COMPOSE_FILE: compose.yaml:deploy/compose.images.yaml
API_BIND_ADDRESS: 127.0.0.1:0
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
@@ -30,13 +30,14 @@ jobs:
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
LOKI_BIND_ADDRESS: 127.0.0.1:0
TEMPO_BIND_ADDRESS: 127.0.0.1:0
API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-api:${{ github.sha }}
NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-nginx:${{ github.sha }}
SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-swagger:${{ github.sha }}
PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-prometheus:${{ github.sha }}
GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-grafana:${{ github.sha }}
S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio:${{ github.sha }}
S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab6-minio-client:${{ github.sha }}
OPENBAO_BIND_ADDRESS: 127.0.0.1:0
API_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-api:${{ github.sha }}
NGINX_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-nginx:${{ github.sha }}
SWAGGER_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-swagger:${{ github.sha }}
PROMETHEUS_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-prometheus:${{ github.sha }}
GRAFANA_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-grafana:${{ github.sha }}
S3_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-minio:${{ github.sha }}
S3_BOOTSTRAP_IMAGE: gitea.privcloud.tech/${{ github.repository_owner }}/mtusi2026lab7-minio-client:${{ github.sha }}
steps:
- name: Checkout exact commit
run: |
@@ -62,8 +63,8 @@ jobs:
- name: Check formatting and shell scripts
run: |
test -z "$(gofmt -l cmd internal migrations)"
sh -n infra/minio/bootstrap.sh scripts/restart.sh scripts/rolling-update.sh scripts/stop-one.sh scripts/test-lab5.sh scripts/test-lab5-acceptance.sh scripts/test-lab6.sh scripts/incident-worker.sh scripts/incident-database.sh scripts/incident-telemetry.sh
python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py
for file in infra/minio/*.sh infra/pki/*.sh infra/postgres/*.sh infra/redis/*.sh infra/security/*.sh infra/openbao/*.sh scripts/*.sh; do sh -n "$file"; done
python3 -m py_compile scripts/lab6-traffic.py scripts/lab6-observe.py scripts/lab6-alert.py scripts/lab6-correlation.py scripts/validate-lab6-evidence.py scripts/test-login-rate-limit.py
- name: Lint
uses: golangci/golangci-lint-action@v7
@@ -79,27 +80,35 @@ jobs:
- name: Prepare local stack
run: |
cp lab6.env.example .env
cp lab7.env.example .env
docker compose config --quiet
docker compose up -d --build --scale api=2 --wait --wait-timeout 150
./scripts/bootstrap-local.sh .env
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME"
- name: Integration checks
run: |
set -a
. ./.env
set +a
test "$(curl --noproxy '*' -s -o /dev/null -w '%{http_code}' "http://s3:9000/$S3_BUCKET")" = "403"
make test BASE_URL=http://nginx:8080
test "$(curl --noproxy '*' --cacert .runtime/security/internal-ca/ca.crt -s -o /dev/null -w '%{http_code}' "https://s3:9000/messenger-attachments")" = "403"
make test BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
make test-security BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
make test-tls-matrix ENV_FILE=.env
ENV_FILE=.env BASE_URL=https://nginx:8443 ./scripts/test-leaf-rotation.sh
docker compose --profile security run --rm dependency-tests
OPENBAO_ADDRESS=$(docker compose port openbao 8200)
ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/openbao-test-policy.sh
ENV_FILE=.env BAO_ADDR="https://$OPENBAO_ADDRESS" BAO_CACERT="$PWD/.runtime/security/openbao/ca.crt" ./scripts/test-openbao-lifecycle.sh
make test-login-rate-limit BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
docker compose --profile tests run --rm --build tests
go run ./cmd/dependencycheck -base-url=http://nginx:8080
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://nginx:8080
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://nginx:8080
go run ./cmd/dependencycheck -base-url=https://nginx:8443 -ca-file=.runtime/security/public-ca/ca.crt
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=https://nginx:8443 CA_FILE=.runtime/security/public-ca/ca.crt
make test-lab5-acceptance
make test-lab5
ENV_FILE=.env PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=http://nginx:8080
make test-worker-compromise ENV_FILE=.env
ENV_FILE=.env CA_FILE=.runtime/security/public-ca/ca.crt PROMETHEUS_URL=http://prometheus:9090 TEMPO_URL=http://tempo:3200 LOKI_URL=http://loki:3100 make test-lab6-smoke BASE_URL=https://nginx:8443
- name: Publish verified images
id: publish
@@ -137,7 +146,10 @@ jobs:
run: |
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" 2>/dev/null || true
if [ -f .env ]; then docker compose --profile tests down -v --remove-orphans; fi
deploy:
@@ -177,17 +189,17 @@ jobs:
run: printf '%s' "$REGISTRY_PASSWORD" | ssh "$DEPLOY_USER@$DEPLOY_HOST" "docker login gitea.privcloud.tech --username '$REGISTRY_USERNAME' --password-stdin"
- name: Upload image manifest
run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab6-${{ github.run_id }}.env"
run: scp deploy-images.env "$DEPLOY_USER@$DEPLOY_HOST:/tmp/mtusi2026lab7-${{ github.run_id }}.env"
- name: Deploy and verify
run: |
ssh "$DEPLOY_USER@$DEPLOY_HOST" "
set -eu
cd ~/mtusi2026lab6
cd ~/mtusi2026lab7
git fetch origin main
git cat-file -e '$DEPLOY_SHA^{commit}'
git reset --hard '$DEPLOY_SHA'
manifest='/tmp/mtusi2026lab6-${{ github.run_id }}.env'
manifest='/tmp/mtusi2026lab7-${{ github.run_id }}.env'
if DEPLOY_GENERATION='${{ github.run_id }}' scripts/rolling-update.sh \"\$manifest\"; then
rm -f \"\$manifest\"
else
+14 -6
View File
@@ -1,10 +1,10 @@
name: Lab 6 incidents
name: Lab 7 inherited incidents
on:
workflow_dispatch:
concurrency:
group: lab6-incidents-${{ github.repository }}-${{ github.ref }}
group: lab7-incidents-${{ github.repository }}-${{ github.ref }}
cancel-in-progress: false
jobs:
@@ -12,13 +12,14 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 35
env:
COMPOSE_PROJECT_NAME: mtusi2026lab6-incidents-${{ github.run_id }}
COMPOSE_PROJECT_NAME: mtusi2026lab7-incidents-${{ github.run_id }}
API_BIND_ADDRESS: 127.0.0.1:0
SWAGGER_BIND_ADDRESS: 127.0.0.1:0
PROMETHEUS_BIND_ADDRESS: 127.0.0.1:0
GRAFANA_BIND_ADDRESS: 127.0.0.1:0
LOKI_BIND_ADDRESS: 127.0.0.1:0
TEMPO_BIND_ADDRESS: 127.0.0.1:0
OPENBAO_BIND_ADDRESS: 127.0.0.1:0
steps:
- name: Checkout exact commit
run: |
@@ -34,17 +35,21 @@ jobs:
- name: Prepare stack
run: |
cp lab6.env.example .env
cp lab7.env.example .env
docker compose config --quiet
docker compose up -d --build --scale api=2 --wait --wait-timeout 180
./scripts/bootstrap-local.sh .env
docker network connect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME"
docker network connect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME"
- name: Reproduce SLO incidents
run: |
ENV_FILE=.env \
BASE_URL=http://nginx:8080 \
BASE_URL=https://nginx:8443 \
CA_FILE=.runtime/security/public-ca/ca.crt \
PROMETHEUS_URL=http://prometheus:9090 \
TEMPO_URL=http://tempo:3200 \
LOKI_URL=http://loki:3100 \
@@ -69,5 +74,8 @@ jobs:
run: |
docker network disconnect "${COMPOSE_PROJECT_NAME}_application" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_data" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_data_backend" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_monitoring" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_secure_clients" "$HOSTNAME" 2>/dev/null || true
docker network disconnect "${COMPOSE_PROJECT_NAME}_telemetry_storage" "$HOSTNAME" 2>/dev/null || true
docker compose down -v --remove-orphans
+36 -1
View File
@@ -2,9 +2,10 @@ PYTHON ?= python3
BASE_URL ?= https://localhost:8443
CA_FILE ?=
ACTION_SCRIPT ?=
ENV_FILE ?= $(if $(wildcard .env),.env,lab7.env.example)
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover test-security
.PHONY: help check validate test load test-persistence test-failover test-security test-tls-matrix test-leaf-rotation test-openbao-lifecycle test-login-rate-limit test-worker-compromise scan-security test-lab5 test-lab5-acceptance lab6-load test-lab6-smoke test-lab6
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -31,3 +32,37 @@ test-failover:
test-security:
$(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-tls-matrix:
docker compose --env-file "$(ENV_FILE)" --profile security run --rm security-tests
test-leaf-rotation:
ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" ./scripts/test-leaf-rotation.sh
test-openbao-lifecycle:
ENV_FILE="$(ENV_FILE)" ./scripts/test-openbao-lifecycle.sh
test-login-rate-limit:
@test -n "$(CA_FILE)" || (echo "Set CA_FILE"; exit 2)
$(PYTHON) scripts/test-login-rate-limit.py --base-url "$(BASE_URL)" --ca-file "$(CA_FILE)"
test-worker-compromise:
ENV_FILE="$(ENV_FILE)" ./scripts/lab7-worker-compromise.sh
scan-security:
./scripts/scan-security.sh
test-lab5:
./scripts/test-lab5.sh
test-lab5-acceptance:
./scripts/test-lab5-acceptance.sh
lab6-load:
$(PYTHON) scripts/lab6-traffic.py --base-url "$(BASE_URL)" $(TLS_ARGS) --mode mixed --duration 60 --concurrency 4 --wait-ready
test-lab6-smoke:
ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" ./scripts/test-lab6.sh smoke
test-lab6:
ENV_FILE="$(ENV_FILE)" BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" ./scripts/test-lab6.sh full
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
env_file=${ENV_FILE:-$repo_dir/lab7.env.example}
exec docker compose --project-directory "$repo_dir" --env-file "$env_file" run --rm --no-deps -T \
-v "$repo_dir:$repo_dir:ro" openbao-cli "$@"
+56
View File
@@ -0,0 +1,56 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
env_file=${1:-$repo_dir/lab7.env.example}
if [ ! -f "$env_file" ]; then
echo "environment file not found: $env_file" >&2
exit 2
fi
compose() {
docker compose --project-directory "$repo_dir" --env-file "$env_file" "$@"
}
if [ ! -f "$repo_dir/.runtime/security/openbao/tls.crt" ]; then
"$repo_dir/infra/pki/bootstrap.sh"
fi
"$script_dir/openbao-seed.sh" "$env_file"
compose up -d security-bootstrap openbao-permissions openbao
openbao_address=$(compose port openbao 8200)
BAO_ADDR="https://$openbao_address"
BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt"
export BAO_ADDR BAO_CACERT ENV_FILE="$env_file"
remaining=60
until curl --silent --show-error --cacert "$BAO_CACERT" \
--output /dev/null "$BAO_ADDR/v1/sys/health"; do
remaining=$((remaining - 1))
[ "$remaining" -gt 0 ] || {
echo "OpenBao did not start" >&2
exit 1
}
sleep 1
done
ENV_FILE="$env_file" "$script_dir/openbao-init.sh"
ENV_FILE="$env_file" "$script_dir/openbao-unseal.sh"
remaining=60
until curl --fail --silent --show-error --cacert "$BAO_CACERT" \
--output /dev/null "$BAO_ADDR/v1/sys/health"; do
remaining=$((remaining - 1))
[ "$remaining" -gt 0 ] || {
echo "OpenBao did not become active after unseal" >&2
exit 1
}
sleep 1
done
ENV_FILE="$env_file" "$script_dir/openbao-bootstrap.sh" \
"$repo_dir/.runtime/openbao/seed/api.json" \
"$repo_dir/.runtime/openbao/seed/worker.json"
compose up -d --build --scale api=2 --wait --wait-timeout 240
+4 -2
View File
@@ -2,8 +2,9 @@
set -eu
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
env_file=${ENV_FILE:-lab7.env.example}
base_url=${BASE_URL:-https://localhost:8443}
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
@@ -33,6 +34,7 @@ trap restore EXIT INT TERM
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase baseline --output "$output_dir/baseline.json" >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--ca-file "$ca_file" \
--mode messages \
--duration "$((outage_seconds + 20))" \
--concurrency 4 \
+5 -3
View File
@@ -2,8 +2,9 @@
set -eu
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
env_file=${ENV_FILE:-lab7.env.example}
base_url=${BASE_URL:-https://localhost:8443}
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
outage_seconds=${TELEMETRY_OUTAGE_SECONDS:-30}
@@ -30,6 +31,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose stop tempo loki >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--ca-file "$ca_file" \
--mode mixed \
--duration "$outage_seconds" \
--concurrency 4 \
@@ -41,7 +43,7 @@ load_pid=$!
sleep "$outage_seconds"
wait "$load_pid"
load_pid=
health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' "$base_url/health/ready")
health_code=$(curl --silent --output /dev/null --write-out '%{http_code}' --cacert "$ca_file" "$base_url/health/ready")
python3 scripts/lab6-observe.py --prometheus-url "$prometheus_url" --phase degraded --output "$output_dir/degraded.json" >/dev/null
recovery_started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose up -d --wait --wait-timeout 90 tempo loki >/dev/null
+4 -2
View File
@@ -3,8 +3,9 @@
set -eu
mode=${1:-short}
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
env_file=${ENV_FILE:-lab7.env.example}
base_url=${BASE_URL:-https://localhost:8443}
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
evidence_root=${EVIDENCE_DIR:-evidence/runtime}
require_alerts=${REQUIRE_ALERT_TRANSITIONS:-1}
@@ -51,6 +52,7 @@ started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)
compose stop worker >/dev/null
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--ca-file "$ca_file" \
--mode attachments \
--duration "$load_seconds" \
--requests "$requests" \
+9 -2
View File
@@ -7,6 +7,7 @@ import json
import math
from pathlib import Path
import secrets
import ssl
import struct
import threading
import time
@@ -16,6 +17,7 @@ import urllib.request
import zlib
MAX_RESPONSE_BYTES = 1 << 20
TLS_CONTEXT = None
def png_fixture(width=96, height=64):
@@ -37,7 +39,7 @@ def png_fixture(width=96, height=64):
def request(method, url, body=None, headers=None, timeout=10):
req = urllib.request.Request(url, data=body, method=method, headers=headers or {})
try:
with urllib.request.urlopen(req, timeout=timeout) as response:
with urllib.request.urlopen(req, timeout=timeout, context=TLS_CONTEXT) as response:
payload = response.read(MAX_RESPONSE_BYTES)
return response.status, payload
except urllib.error.HTTPError as error:
@@ -167,7 +169,8 @@ def wait_for_attachments(base_url, token, attachment_ids, timeout):
def main():
parser = argparse.ArgumentParser(description="Bounded synthetic load for Lab 6 telemetry and incidents")
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--base-url", default="https://localhost:8443")
parser.add_argument("--ca-file", default=".runtime/security/public-ca/ca.crt")
parser.add_argument("--mode", choices=("messages", "attachments", "mixed"), default="mixed")
parser.add_argument("--duration", type=float, default=30)
parser.add_argument("--requests", type=int, default=0, help="Maximum requests; zero means duration only")
@@ -179,6 +182,10 @@ def main():
parser.add_argument("--started-file", help="Touch after synthetic session and chat are ready")
args = parser.parse_args()
global TLS_CONTEXT
if args.base_url.startswith("https://"):
TLS_CONTEXT = ssl.create_default_context(cafile=args.ca_file)
if args.duration <= 0 or args.duration > 1800:
parser.error("duration must be in (0, 1800]")
if args.requests < 0:
+209
View File
@@ -0,0 +1,209 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
env_file=${ENV_FILE:-$repo_dir/lab7.env.example}
runtime_dir="$repo_dir/.runtime/evidence/lab7-worker"
state_file="$runtime_dir/queued-attachment.json"
rotated_secret="$repo_dir/.runtime/openbao/seed/worker-rotated.json"
rotated_password="rotated-worker-$(date -u +%Y%m%d%H%M%S)"
rotated_database_password="rotated-db-$(date -u +%Y%m%d%H%M%S)"
rotated_s3_secret="rotated-s3-$(date -u +%Y%m%d%H%M%S)"
summary_file="$repo_dir/evidence/lab7-worker-compromise.json"
started_at=$(date +%s)
compose() {
docker compose --project-directory "$repo_dir" --env-file "$env_file" "$@"
}
wait_healthy() {
service=$1
remaining=60
while [ "$remaining" -gt 0 ]; do
container=$(compose ps -q "$service")
if [ -n "$container" ] && [ "$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")" = healthy ]; then
return 0
fi
remaining=$((remaining - 1))
sleep 1
done
echo "$service did not become healthy" >&2
return 1
}
rotate_worker_secret() {
secret_file=$1
compose stop openbao-agent-worker >/dev/null
ENV_FILE="$env_file" BAO_ADDR="$BAO_ADDR" BAO_CACERT="$BAO_CACERT" \
"$script_dir/openbao-rotate.sh" worker "$secret_file" >/dev/null
docker start "$(compose ps -aq openbao-agent-worker)" >/dev/null
wait_healthy openbao-agent-worker
}
restore() {
compose stop worker openbao-agent-worker >/dev/null 2>&1 || true
compose run --rm --no-deps acl-bootstrap >/dev/null 2>&1 || true
compose restart redis-broker redis-gateway >/dev/null 2>&1 || true
compose run --rm --no-deps postgres-roles >/dev/null 2>&1 || true
compose run --rm --no-deps s3-bootstrap >/dev/null 2>&1 || true
rotate_worker_secret "$repo_dir/.runtime/openbao/seed/worker.json" >/dev/null 2>&1 || true
compose up -d --no-deps --force-recreate worker >/dev/null 2>&1 || true
}
mkdir -p "$runtime_dir"
chmod 0700 "$repo_dir/.runtime/evidence" "$runtime_dir"
trap restore EXIT HUP INT TERM
openbao_address=$(compose port openbao 8200)
BAO_ADDR="https://$openbao_address"
BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt"
export BAO_ADDR BAO_CACERT
compose stop worker >/dev/null
compose --profile experiments run --rm --no-deps lab5-experiment /app/lab5experiment api-produce >"$state_file"
chmod 0600 "$state_file"
old_database_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
import json
import sys
import urllib.parse
with open(sys.argv[1], encoding="utf-8") as source:
secret = json.load(source)
print(urllib.parse.urlsplit(secret["database_url"]).password)
PY
)
old_broker_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
import json
import sys
import urllib.parse
with open(sys.argv[1], encoding="utf-8") as source:
print(urllib.parse.urlsplit(json.load(source)["broker_url"]).password)
PY
)
old_s3_access_key=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as source:
print(json.load(source)["s3_access_key"])
PY
)
old_s3_secret=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as source:
print(json.load(source)["s3_secret_key"])
PY
)
python3 - "$repo_dir/.runtime/openbao/seed/worker.json" "$rotated_secret" \
"$rotated_password" "$rotated_database_password" "$rotated_s3_secret" <<'PY'
import json
import os
import sys
import urllib.parse
def replace_password(url, password):
parsed = urllib.parse.urlsplit(url)
username = urllib.parse.quote(parsed.username or "", safe="")
netloc = f"{username}:{urllib.parse.quote(password, safe='')}@{parsed.hostname}"
if parsed.port:
netloc += f":{parsed.port}"
return urllib.parse.urlunsplit((parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment))
with open(sys.argv[1], encoding="utf-8") as source:
secret = json.load(source)
secret["broker_url"] = replace_password(secret["broker_url"], sys.argv[3])
secret["database_url"] = replace_password(secret["database_url"], sys.argv[4])
secret["s3_secret_key"] = sys.argv[5]
with open(sys.argv[2], "w", encoding="utf-8") as destination:
json.dump(secret, destination, separators=(",", ":"))
destination.write("\n")
os.chmod(sys.argv[2], 0o600)
PY
compose run --rm --no-deps -e REDIS_WORKER_PASSWORD="$rotated_password" acl-bootstrap >/dev/null
compose restart redis-broker redis-gateway >/dev/null
compose exec -T postgres sh -c \
"psql --set=ON_ERROR_STOP=1 --username \"\$POSTGRES_USER\" --dbname \"\$POSTGRES_DB\" --command \"ALTER ROLE messenger_worker PASSWORD '$rotated_database_password'\"" >/dev/null
compose exec -T postgres sh -c \
'psql --set=ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" --command="SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = '\''messenger_worker'\'' AND pid <> pg_backend_pid()"' >/dev/null
compose run --rm --no-deps -e S3_WORKER_SECRET_KEY="$rotated_s3_secret" s3-bootstrap >/dev/null
compose --profile security run --rm --no-deps \
-e REDISCLI_AUTH="$old_broker_password" \
--entrypoint /bin/sh dependency-tests -c '
output=$(redis-cli --tls --cacert /run/security-source/internal-ca/ca.crt \
--cert /run/security-source/worker-client/tls.crt --key /run/security-source/worker-client/tls.key \
--user messenger_worker -h redis-gateway -p 36379 PING 2>&1 || true)
case "$output" in
*WRONGPASS*|*invalid\ username-password*) exit 0 ;;
*) echo "expected old broker credential rejection" >&2; exit 1 ;;
esac
'
if compose --profile security run --rm --no-deps \
-e OLD_DATABASE_PASSWORD="$old_database_password" \
--entrypoint /bin/sh dependency-tests -c '
psql "postgresql://messenger_worker:${OLD_DATABASE_PASSWORD}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/security-source/internal-ca/ca.crt&sslcert=/run/security-source/worker-client/tls.crt&sslkey=/run/security-source/worker-client/tls.key" \
--set=ON_ERROR_STOP=1 --command="SELECT 1" >/dev/null 2>&1
'; then
echo "old worker database credential was not rejected" >&2
exit 1
fi
if compose --profile security run --rm --no-deps \
-e OLD_S3_ACCESS_KEY="$old_s3_access_key" -e OLD_S3_SECRET="$old_s3_secret" \
--entrypoint /bin/sh dependency-tests -c '
export SSL_CERT_FILE=/run/security-source/internal-ca/ca.crt
export MC_CONFIG_DIR=/tmp/old-worker-mc
mc alias set old https://s3:9000 "$OLD_S3_ACCESS_KEY" "$OLD_S3_SECRET" >/dev/null 2>&1 &&
mc ls old/messenger-attachments/attachments/ >/dev/null 2>&1
'; then
echo "old worker S3 credential was not rejected" >&2
exit 1
fi
rotate_worker_secret "$rotated_secret"
compose up -d --no-deps --force-recreate worker >/dev/null
wait_healthy worker
compose --profile experiments run --rm --no-deps -T lab5-experiment \
/app/lab5experiment api-verify <"$state_file" >/dev/null
finished_at=$(date +%s)
python3 - "$summary_file" "$started_at" "$finished_at" <<'PY'
import datetime
import json
import sys
started_at = int(sys.argv[2])
finished_at = int(sys.argv[3])
summary = {
"schema_version": 1,
"scenario": "worker credential compromise and rotation",
"generated_at": datetime.datetime.fromtimestamp(
finished_at, datetime.timezone.utc
).replace(microsecond=0).isoformat(),
"recovery_seconds": finished_at - started_at,
"old_credential_rejected": True,
"old_database_credential_rejected": True,
"old_s3_credential_rejected": True,
"openbao_secret_rotated": True,
"queued_attachment_preserved": True,
"result": "PASS",
}
with open(sys.argv[1], "w", encoding="utf-8") as destination:
json.dump(summary, destination, ensure_ascii=False, indent=2)
destination.write("\n")
PY
trap - EXIT HUP INT TERM
restore
rm -f "$rotated_secret"
echo "PASS: revoked worker credential failed; rotated OpenBao secret restored processing without losing the queued attachment"
+93
View File
@@ -0,0 +1,93 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
. "$script_dir/openbao-common.sh"
if [ "$#" -ne 2 ]; then
echo "usage: $0 <api-secret-json> <worker-secret-json>" >&2
exit 2
fi
api_secret_file=$1
worker_secret_file=$2
require_command "$BAO_BIN"
require_command python3
require_runtime_path "$api_secret_file"
require_runtime_path "$worker_secret_file"
require_private_file "$api_secret_file"
require_private_file "$worker_secret_file"
require_unsealed
load_operator_token
python3 - "$api_secret_file" "$worker_secret_file" <<'PY'
import json
import sys
for path in sys.argv[1:]:
with open(path, encoding="utf-8") as source:
value = json.load(source)
if not isinstance(value, dict) or not value:
raise SystemExit(f"secret input must be a non-empty JSON object: {path}")
PY
mounts_file=$(mktemp "$openbao_runtime_dir/.mounts.XXXXXX")
auth_file=$(mktemp "$openbao_runtime_dir/.auth.XXXXXX")
chmod 0600 "$mounts_file" "$auth_file"
cleanup() {
rm -f "$mounts_file" "$auth_file"
}
trap cleanup EXIT HUP INT TERM
"$BAO_BIN" secrets list -format=json >"$mounts_file"
if ! python3 - "$mounts_file" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as source:
raise SystemExit(0 if "secret/" in json.load(source) else 1)
PY
then
"$BAO_BIN" secrets enable -path=secret -version=2 kv >/dev/null
fi
"$BAO_BIN" auth list -format=json >"$auth_file"
if ! python3 - "$auth_file" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as source:
raise SystemExit(0 if "approle/" in json.load(source) else 1)
PY
then
"$BAO_BIN" auth enable approle >/dev/null
fi
"$BAO_BIN" policy write messenger-api "$repo_dir/infra/openbao/policies/messenger-api.hcl" >/dev/null
"$BAO_BIN" policy write messenger-worker "$repo_dir/infra/openbao/policies/messenger-worker.hcl" >/dev/null
for service in api worker; do
"$BAO_BIN" delete "auth/approle/role/messenger-$service" >/dev/null 2>&1 || true
"$BAO_BIN" write "auth/approle/role/messenger-$service" \
bind_secret_id=true \
secret_id_ttl=1h \
secret_id_num_uses=0 \
token_policies="messenger-$service" \
token_no_default_policy=true \
token_ttl=15m \
token_max_ttl=1h \
token_num_uses=0 >/dev/null
done
"$BAO_BIN" kv put -mount=secret messenger/api @"$api_secret_file" >/dev/null
"$BAO_BIN" kv put -mount=secret messenger/worker @"$worker_secret_file" >/dev/null
"$script_dir/openbao-issue-approle.sh" api "$openbao_runtime_dir/approle/api"
"$script_dir/openbao-issue-approle.sh" worker "$openbao_runtime_dir/approle/worker"
echo "OpenBao policies, AppRoles and KV v2 secrets configured"
+91
View File
@@ -0,0 +1,91 @@
#!/bin/sh
set -eu
umask 077
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
openbao_runtime_dir="$repo_dir/.runtime/openbao"
openbao_operator_file="$openbao_runtime_dir/operator/init.json"
BAO_BIN=${BAO_BIN:-$repo_dir/scripts/bao-docker.sh}
BAO_ADDR=${BAO_ADDR:-https://localhost:8200}
BAO_CACERT=${BAO_CACERT:-$repo_dir/.runtime/security/openbao/ca.crt}
export BAO_ADDR BAO_CACERT
fail() {
echo "$*" >&2
exit 1
}
require_command() {
command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1"
}
require_file() {
[ -f "$1" ] || fail "required file not found: $1"
}
require_private_file() {
require_file "$1"
permissions=$(stat -f '%Lp' "$1" 2>/dev/null || stat -c '%a' "$1")
case "$permissions" in
600|400) ;;
*) fail "file must have permissions 0600 or 0400: $1" ;;
esac
}
require_runtime_path() {
case "$1" in
"$openbao_runtime_dir"/*) ;;
*) fail "runtime credentials must stay under $openbao_runtime_dir" ;;
esac
}
json_field() {
file=$1
expression=$2
python3 - "$file" "$expression" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as source:
value = json.load(source)
for component in sys.argv[2].split("."):
value = value[component]
if isinstance(value, bool):
print(str(value).lower())
else:
print(value)
PY
}
load_operator_token() {
require_private_file "$openbao_operator_file"
BAO_TOKEN=$(json_field "$openbao_operator_file" root_token)
export BAO_TOKEN
}
openbao_status() {
status_file=$1
"$BAO_BIN" status -format=json >"$status_file" 2>/dev/null || true
[ -s "$status_file" ] || fail "OpenBao is not reachable at $BAO_ADDR"
}
require_unsealed() {
mkdir -p "$openbao_runtime_dir"
chmod 0700 "$openbao_runtime_dir"
status_file=$(mktemp "$openbao_runtime_dir/.status.XXXXXX")
chmod 0600 "$status_file"
openbao_status "$status_file"
initialized=$(json_field "$status_file" initialized)
sealed=$(json_field "$status_file" sealed)
rm -f "$status_file"
[ "$initialized" = "true" ] || fail "OpenBao is not initialized"
[ "$sealed" = "false" ] || fail "OpenBao is sealed"
}
+46
View File
@@ -0,0 +1,46 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
. "$script_dir/openbao-common.sh"
require_command "$BAO_BIN"
require_command python3
require_file "$BAO_CACERT"
mkdir -p "$openbao_runtime_dir/operator"
chmod 0700 "$openbao_runtime_dir" "$openbao_runtime_dir/operator"
status_file=$(mktemp "$openbao_runtime_dir/.status.XXXXXX")
init_file=$(mktemp "$openbao_runtime_dir/operator/.init.XXXXXX")
chmod 0600 "$status_file" "$init_file"
cleanup() {
rm -f "$status_file" "$init_file"
}
trap cleanup EXIT HUP INT TERM
openbao_status "$status_file"
if [ "$(json_field "$status_file" initialized)" = "true" ]; then
echo "OpenBao is already initialized"
exit 0
fi
if [ -e "$openbao_operator_file" ]; then
fail "operator material already exists for an uninitialized OpenBao instance"
fi
"$BAO_BIN" operator init \
-key-shares=3 \
-key-threshold=2 \
-format=json >"$init_file"
chmod 0600 "$init_file"
mv "$init_file" "$openbao_operator_file"
echo "OpenBao initialized; operator material saved to .runtime/openbao/operator/init.json"
echo "Keep this file offline and run scripts/openbao-unseal.sh"
+87
View File
@@ -0,0 +1,87 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
. "$script_dir/openbao-common.sh"
if [ "$#" -ne 2 ]; then
echo "usage: $0 <api|worker> <runtime-output-directory>" >&2
exit 2
fi
service=$1
output_dir=$2
case "$service" in
api|worker) ;;
*) fail "service must be api or worker" ;;
esac
require_runtime_path "$output_dir"
require_command "$BAO_BIN"
require_command python3
require_unsealed
if [ -z "${BAO_TOKEN:-}" ]; then
load_operator_token
fi
role="messenger-$service"
parent_dir=$(dirname "$output_dir")
staging_dir="$parent_dir/.${service}.$$"
response_file="$staging_dir/response.json"
cleanup() {
rm -rf "$staging_dir"
}
trap cleanup EXIT HUP INT TERM
mkdir -p "$parent_dir" "$staging_dir"
chmod 0700 "$parent_dir" "$staging_dir"
"$BAO_BIN" read -field=role_id "auth/approle/role/$role/role-id" >"$staging_dir/role-id"
"$BAO_BIN" write -f -format=json "auth/approle/role/$role/secret-id" >"$response_file"
python3 - "$response_file" "$staging_dir" <<'PY'
import json
import os
import sys
with open(sys.argv[1], encoding="utf-8") as source:
data = json.load(source)["data"]
for name, field in (("secret-id", "secret_id"), ("secret-id-accessor", "secret_id_accessor")):
path = os.path.join(sys.argv[2], name)
with open(path, "w", encoding="utf-8") as destination:
destination.write(data[field] + "\n")
os.chmod(path, 0o600)
PY
rm -f "$response_file"
chmod 0600 "$staging_dir/role-id"
old_accessor=
if [ -f "$output_dir/secret-id-accessor" ]; then
old_accessor=$(cat "$output_dir/secret-id-accessor")
fi
mkdir -p "$output_dir"
chmod 0700 "$output_dir"
for file in role-id secret-id secret-id-accessor; do
mv "$staging_dir/$file" "$output_dir/$file"
done
if [ -n "$old_accessor" ]; then
"$BAO_BIN" write "auth/approle/role/$role/secret-id-accessor/destroy" \
secret_id_accessor="$old_accessor" >/dev/null 2>&1 || true
fi
if [ "$output_dir" = "$openbao_runtime_dir/approle/$service" ]; then
docker compose --project-directory "$repo_dir" --env-file "${ENV_FILE:-$repo_dir/lab7.env.example}" \
run --rm --no-deps -e SERVICE="$service" openbao-identity-install
fi
echo "issued short-lived AppRole credentials for $service in ${output_dir#"$repo_dir/"}"
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
. "$script_dir/openbao-common.sh"
if [ "$#" -ne 2 ]; then
echo "usage: $0 <api|worker> <replacement-secret-json>" >&2
exit 2
fi
service=$1
secret_file=$2
case "$service" in
api|worker) ;;
*) fail "service must be api or worker" ;;
esac
require_command "$BAO_BIN"
require_command python3
require_runtime_path "$secret_file"
require_private_file "$secret_file"
require_unsealed
load_operator_token
python3 - "$secret_file" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as source:
value = json.load(source)
if not isinstance(value, dict) or not value:
raise SystemExit("replacement secret must be a non-empty JSON object")
PY
"$BAO_BIN" kv put -mount=secret "messenger/$service" @"$secret_file" >/dev/null
"$script_dir/openbao-issue-approle.sh" "$service" "$openbao_runtime_dir/approle/$service"
echo "$service secret rotated; OpenBao Agent will render the new KV version"
+62
View File
@@ -0,0 +1,62 @@
#!/bin/sh
set -eu
if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
echo "usage: $0 <env-file>" >&2
exit 2
fi
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
seed_dir="$repo_dir/.runtime/openbao/seed"
mkdir -p "$seed_dir"
chmod 0700 "$repo_dir/.runtime/openbao" "$seed_dir"
python3 - "$1" "$seed_dir/api.json" "$seed_dir/worker.json" <<'PY'
import json
import os
import sys
values = {}
with open(sys.argv[1], encoding="utf-8") as source:
for raw_line in source:
line = raw_line.strip()
if not line or line.startswith("#"):
continue
name, separator, value = line.partition("=")
if not separator:
raise SystemExit(f"invalid env line: {raw_line.rstrip()}")
values[name.strip()] = os.environ.get(name.strip(), value)
required = (
"DATABASE_URL", "DATABASE_WORKER_URL", "CACHE_URL", "BROKER_WORKER_URL",
"S3_API_ACCESS_KEY", "S3_API_SECRET_KEY", "S3_WORKER_ACCESS_KEY", "S3_WORKER_SECRET_KEY",
)
missing = [name for name in required if not values.get(name)]
if missing:
raise SystemExit("missing environment values: " + ", ".join(missing))
documents = (
(sys.argv[2], {
"database_url": values["DATABASE_URL"],
"cache_url": values["CACHE_URL"],
"s3_access_key": values["S3_API_ACCESS_KEY"],
"s3_secret_key": values["S3_API_SECRET_KEY"],
}),
(sys.argv[3], {
"database_url": values["DATABASE_WORKER_URL"],
"broker_url": values["BROKER_WORKER_URL"],
"s3_access_key": values["S3_WORKER_ACCESS_KEY"],
"s3_secret_key": values["S3_WORKER_SECRET_KEY"],
}),
)
for path, value in documents:
with open(path, "w", encoding="utf-8") as destination:
json.dump(value, destination, separators=(",", ":"))
destination.write("\n")
os.chmod(path, 0o600)
PY
echo "OpenBao seed files created under .runtime/openbao/seed"
+140
View File
@@ -0,0 +1,140 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
. "$script_dir/openbao-common.sh"
require_command curl
require_command python3
require_command "$BAO_BIN"
require_file "$BAO_CACERT"
require_unsealed
load_operator_token
test_dir="$openbao_runtime_dir/policy-test.$$"
mkdir -p "$test_dir"
chmod 0700 "$test_dir"
cleanup() {
for service in api worker; do
headers="$test_dir/$service/token-header"
if [ -f "$headers" ]; then
curl --silent --show-error --cacert "$BAO_CACERT" \
--header @"$headers" \
--request POST \
"$BAO_ADDR/v1/auth/token/revoke-self" >/dev/null 2>&1 || true
fi
done
rm -rf "$test_dir"
}
trap cleanup EXIT HUP INT TERM
login() {
service=$1
identity_dir="$test_dir/$service/identity"
session_dir="$test_dir/$service"
response_file="$session_dir/login.json"
payload_file="$session_dir/login-payload.json"
mkdir -p "$session_dir"
chmod 0700 "$session_dir"
"$script_dir/openbao-issue-approle.sh" "$service" "$identity_dir" >/dev/null
python3 - "$identity_dir/role-id" "$identity_dir/secret-id" "$payload_file" <<'PY'
import json
import os
import sys
with open(sys.argv[1], encoding="utf-8") as role_source:
role_id = role_source.read().strip()
with open(sys.argv[2], encoding="utf-8") as secret_source:
secret_id = secret_source.read().strip()
with open(sys.argv[3], "w", encoding="utf-8") as destination:
json.dump({"role_id": role_id, "secret_id": secret_id}, destination)
os.chmod(sys.argv[3], 0o600)
PY
status=$(curl --silent --show-error --cacert "$BAO_CACERT" \
--header "Content-Type: application/json" \
--request POST \
--data-binary @"$payload_file" \
--output "$response_file" \
--write-out '%{http_code}' \
"$BAO_ADDR/v1/auth/approle/login")
[ "$status" = "200" ] || fail "$service AppRole login returned HTTP $status"
chmod 0600 "$response_file"
python3 - "$response_file" "$service" "$session_dir/token-header" <<'PY'
import json
import os
import sys
with open(sys.argv[1], encoding="utf-8") as source:
auth = json.load(source)["auth"]
expected_policy = f"messenger-{sys.argv[2]}"
if not auth["renewable"]:
raise SystemExit("AppRole token is not renewable")
if not 0 < auth["lease_duration"] <= 3600:
raise SystemExit("AppRole token TTL is outside the configured limit")
if auth["policies"] != [expected_policy]:
raise SystemExit(f"unexpected token policies: {auth['policies']}")
with open(sys.argv[3], "w", encoding="utf-8") as destination:
destination.write(f"X-Vault-Token: {auth['client_token']}\n")
os.chmod(sys.argv[3], 0o600)
PY
rm -f "$payload_file" "$response_file" "$identity_dir/secret-id"
}
expect_status() {
service=$1
method=$2
path=$3
expected=$4
payload=${5:-}
headers="$test_dir/$service/token-header"
output="$test_dir/$service/response.json"
if [ -n "$payload" ]; then
status=$(curl --silent --show-error --cacert "$BAO_CACERT" \
--header @"$headers" \
--header "Content-Type: application/json" \
--request "$method" \
--data-binary "$payload" \
--output "$output" \
--write-out '%{http_code}' \
"$BAO_ADDR/v1/$path")
else
status=$(curl --silent --show-error --cacert "$BAO_CACERT" \
--header @"$headers" \
--request "$method" \
--output "$output" \
--write-out '%{http_code}' \
"$BAO_ADDR/v1/$path")
fi
[ "$status" = "$expected" ] || fail "$service $method $path: expected HTTP $expected, got $status"
echo "PASS $service $method $path -> $status"
}
login api
login worker
expect_status api GET secret/data/messenger/api 200
expect_status api GET secret/data/messenger/worker 403
expect_status api POST secret/data/messenger/api/probe 403 '{"data":{"probe":"denied"}}'
expect_status api GET sys/policies/acl 403
expect_status api POST auth/token/renew-self 200 '{}'
expect_status worker GET secret/data/messenger/worker 200
expect_status worker GET secret/data/messenger/api 403
expect_status worker POST secret/data/messenger/worker/probe 403 '{"data":{"probe":"denied"}}'
expect_status worker GET sys/policies/acl 403
expect_status worker POST auth/token/renew-self 200 '{}'
echo "OpenBao AppRole allow/deny and renewal checks passed"
+51
View File
@@ -0,0 +1,51 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
. "$script_dir/openbao-common.sh"
require_command "$BAO_BIN"
require_command python3
require_file "$BAO_CACERT"
require_private_file "$openbao_operator_file"
mkdir -p "$openbao_runtime_dir"
keys_file=$(mktemp "$openbao_runtime_dir/.unseal.XXXXXX")
status_file=$(mktemp "$openbao_runtime_dir/.status.XXXXXX")
chmod 0600 "$keys_file" "$status_file"
cleanup() {
rm -f "$keys_file" "$status_file"
}
trap cleanup EXIT HUP INT TERM
openbao_status "$status_file"
if [ "$(json_field "$status_file" sealed)" = "false" ]; then
echo "OpenBao is already unsealed"
exit 0
fi
python3 - "$openbao_operator_file" >"$keys_file" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as source:
material = json.load(source)
threshold = material["unseal_threshold"]
for key in material["unseal_keys_b64"][:threshold]:
print(key)
PY
while IFS= read -r key; do
"$BAO_BIN" operator unseal "$key" </dev/null >/dev/null
done <"$keys_file"
openbao_status "$status_file"
[ "$(json_field "$status_file" sealed)" = "false" ] || fail "OpenBao remains sealed"
echo "OpenBao unsealed"
+9 -4
View File
@@ -8,7 +8,7 @@ if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
fi
manifest=$1
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab5"}
deploy_dir=${DEPLOY_STATE_DIR:-"$HOME/.local/state/mtusi2026lab7"}
current_manifest=$deploy_dir/current-images.env
active_containers=$deploy_dir/active-api-containers
previous_containers=$deploy_dir/previous-api-containers
@@ -16,7 +16,9 @@ upstream_file=$deploy_dir/upstream.conf
upstream_backup=$deploy_dir/upstream.previous.conf
candidate_file=$deploy_dir/candidate-api-containers
replicas=${API_REPLICAS:-2}
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab5}
project=${COMPOSE_PROJECT_NAME:-mtusi2026lab7}
public_base_url=${PUBLIC_BASE_URL:-https://localhost:8443}
public_ca_file=${PUBLIC_CA_FILE:-.runtime/security/public-ca/ca.crt}
switched=false
nginx_created=false
upstream_changed=false
@@ -52,7 +54,7 @@ write_upstream() {
: > "$output"
for container in $containers; do
name=$(container_name "$container")
printf 'server %s:8080 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
printf 'server %s:8443 resolve max_fails=1 fail_timeout=3s;\n' "$name" >> "$output"
done
}
@@ -81,7 +83,8 @@ wait_for_containers() {
}
smoke_check() {
compose exec -T nginx wget -q -O /dev/null http://127.0.0.1:8080/health/ready
curl --fail --silent --show-error --noproxy '*' \
--cacert "$public_ca_file" "$public_base_url/health/ready" >/dev/null
}
remove_containers() {
@@ -123,6 +126,7 @@ trap 'exit 143' TERM
load_manifest "$manifest"
new_api_image=$API_IMAGE
test -r "$public_ca_file"
if [ -f "$previous_containers" ]; then
remove_containers "$(cat "$previous_containers")"
@@ -151,6 +155,7 @@ cp "$upstream_file" "$upstream_backup"
compose pull api nginx swagger prometheus grafana s3 s3-bootstrap
compose up -d --no-build --wait --wait-timeout 120 postgres valkey redis-broker s3
compose run --rm security-bootstrap
compose run --rm --no-deps s3-bootstrap
compose run --rm --no-deps migrate
+132
View File
@@ -0,0 +1,132 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
runtime_dir="$repo_dir/evidence/runtime/security-scan"
cache_dir="$repo_dir/.runtime/trivy-cache"
raw_file="$runtime_dir/trivy-fs.json"
summary_file="$repo_dir/evidence/lab7-security-scan.json"
env_file=${ENV_FILE:-$repo_dir/lab7.env.example}
image_list=${SCAN_IMAGES:-}
if [ -z "$image_list" ]; then
containers=$(docker compose --project-directory "$repo_dir" --env-file "$env_file" ps -q)
[ -n "$containers" ] || { echo "start the stack or set SCAN_IMAGES" >&2; exit 1; }
image_list=$(docker inspect --format '{{.Config.Image}}' $containers | sort -u)
fi
mkdir -p "$runtime_dir" "$cache_dir"
docker run --rm \
-v "$repo_dir:/src:ro" \
-v "$runtime_dir:/out" \
-v "$cache_dir:/root/.cache/trivy" \
aquasec/trivy:0.76.2 fs \
--scanners vuln,misconfig,secret \
--format json \
--output /out/trivy-fs.json \
--skip-dirs /src/.git \
--skip-dirs /src/.runtime \
--skip-dirs /src/.venv \
--skip-dirs /src/evidence/runtime \
--skip-files /src/Dockerfile.example \
/src
image_reports=
index=0
for image in $image_list; do
index=$((index + 1))
report="$runtime_dir/trivy-image-$index.json"
docker run --rm \
-v /var/run/docker.sock:/var/run/docker.sock \
-v "$runtime_dir:/out" \
-v "$cache_dir:/root/.cache/trivy" \
aquasec/trivy:0.76.2 image \
--scanners vuln,secret \
--format json \
--output "/out/trivy-image-$index.json" \
"$image"
image_reports="$image_reports $image=$report"
done
python3 - "$raw_file" "$summary_file" $image_reports <<'PY'
import datetime
import json
import sys
from collections import Counter
with open(sys.argv[1], encoding="utf-8") as source:
report = json.load(source)
vulnerabilities = Counter()
misconfigurations = Counter()
secrets = 0
for result in report.get("Results") or []:
for vulnerability in result.get("Vulnerabilities") or []:
vulnerabilities[vulnerability.get("Severity", "UNKNOWN")] += 1
for finding in result.get("Misconfigurations") or []:
misconfigurations[finding.get("Severity", "UNKNOWN")] += 1
secrets += len(result.get("Secrets") or [])
image_vulnerabilities = Counter()
image_secrets = 0
scanned_images = []
image_raw_reports = []
findings = {}
def record_findings(report, scope):
for result in report.get("Results") or []:
for finding in result.get("Vulnerabilities") or []:
key = (finding["VulnerabilityID"], finding["PkgName"], finding.get("InstalledVersion", ""))
entry = findings.setdefault(key, {
"id": finding["VulnerabilityID"],
"package": finding["PkgName"],
"installed_version": finding.get("InstalledVersion", ""),
"fixed_version": finding.get("FixedVersion", ""),
"severity": finding.get("Severity", "UNKNOWN"),
"title": finding.get("Title", ""),
"scopes": [],
})
if scope not in entry["scopes"]:
entry["scopes"].append(scope)
record_findings(report, "repository")
for item in sys.argv[3:]:
image, path = item.split("=", 1)
scanned_images.append(image)
with open(path, encoding="utf-8") as source:
image_report = json.load(source)
image_raw_reports.append("evidence/runtime/security-scan/" + path.rsplit("/", 1)[-1])
record_findings(image_report, image)
for result in image_report.get("Results") or []:
for vulnerability in result.get("Vulnerabilities") or []:
image_vulnerabilities[vulnerability.get("Severity", "UNKNOWN")] += 1
image_secrets += len(result.get("Secrets") or [])
summary = {
"schema_version": 1,
"scanner": "Trivy 0.76.2",
"generated_at": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat(),
"scope": "repository filesystem: vulnerabilities, misconfigurations, secrets",
"vulnerabilities": dict(sorted(vulnerabilities.items())),
"misconfigurations": dict(sorted(misconfigurations.items())),
"secret_findings": secrets,
"scanned_images": scanned_images,
"image_vulnerabilities": dict(sorted(image_vulnerabilities.items())),
"image_secret_findings": image_secrets,
"findings": sorted((finding for finding in findings.values()
if "repository" in finding["scopes"] or finding["severity"] == "CRITICAL"),
key=lambda finding: (finding["package"], finding["id"])),
"findings_scope": "all repository findings and critical image findings; full findings in raw reports",
"raw_report": "evidence/runtime/security-scan/trivy-fs.json",
"image_raw_reports": image_raw_reports,
}
with open(sys.argv[2], "w", encoding="utf-8") as destination:
json.dump(summary, destination, ensure_ascii=False, indent=2)
destination.write("\n")
PY
cat "$summary_file"
+8 -6
View File
@@ -11,8 +11,9 @@ case "$profile" in
;;
esac
env_file=${ENV_FILE:-lab6.env.example}
base_url=${BASE_URL:-http://127.0.0.1:8080}
env_file=${ENV_FILE:-lab7.env.example}
base_url=${BASE_URL:-https://localhost:8443}
ca_file=${CA_FILE:-.runtime/security/public-ca/ca.crt}
prometheus_url=${PROMETHEUS_URL:-http://127.0.0.1:9090}
tempo_url=${TEMPO_URL:-http://127.0.0.1:3200}
loki_url=${LOKI_URL:-http://127.0.0.1:3100}
@@ -35,6 +36,7 @@ compose config --quiet
compose up -d --build --wait --wait-timeout 180
python3 scripts/lab6-traffic.py \
--base-url "$base_url" \
--ca-file "$ca_file" \
--mode mixed \
--duration "${BASELINE_LOAD_SECONDS:-15}" \
--requests "${BASELINE_REQUESTS:-120}" \
@@ -54,10 +56,10 @@ python3 scripts/lab6-correlation.py \
--require >/dev/null
if [ "$profile" = "full" ]; then
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short
ENV_FILE="$env_file" BASE_URL="$base_url" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" TEMPO_URL="$tempo_url" LOKI_URL="$loki_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-database.sh
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh long
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-worker.sh short
ENV_FILE="$env_file" BASE_URL="$base_url" CA_FILE="$ca_file" PROMETHEUS_URL="$prometheus_url" EVIDENCE_DIR="$output_dir/incidents" scripts/incident-telemetry.sh
fi
arguments=
+160
View File
@@ -0,0 +1,160 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
env_file=${ENV_FILE:-$repo_dir/lab7.env.example}
security_dir=$repo_dir/.runtime/security
rotation_dir=$security_dir/rotation-test
old_dir=$rotation_dir/old-load-balancer
new_dir=$rotation_dir/new-load-balancer
expired_dir=$rotation_dir/expired-load-balancer
override_file=$repo_dir/.runtime/leaf-rotation.override.yaml
new_identity=spiffe://mtusi/lab7/load-balancer-v2
rotation_active=0
base_url=${BASE_URL:-https://localhost:8443}
compose() {
docker compose --env-file "$env_file" -f "$repo_dir/compose.yaml" "$@"
}
compose_rotated() {
docker compose --env-file "$env_file" \
-f "$repo_dir/compose.yaml" -f "$override_file" "$@"
}
install_security_material() {
compose run --rm --no-deps security-bootstrap >/dev/null
}
wait_for_https() {
attempts=0
while [ "$attempts" -lt 30 ]; do
if curl --silent --show-error --fail \
--cacert "$security_dir/public-ca/ca.crt" \
"$base_url/health/ready" >/dev/null 2>&1; then
return 0
fi
attempts=$((attempts + 1))
sleep 1
done
return 1
}
restore() {
status=$?
trap - EXIT HUP INT TERM
if [ "$rotation_active" -eq 1 ] && [ -f "$old_dir/tls.crt" ]; then
cp "$old_dir/tls.crt" "$security_dir/load-balancer/tls.crt"
cp "$old_dir/tls.key" "$security_dir/load-balancer/tls.key"
chmod 0600 "$security_dir/load-balancer/tls.crt" "$security_dir/load-balancer/tls.key"
install_security_material || status=1
compose up -d --no-deps --no-build --force-recreate api nginx >/dev/null || status=1
wait_for_https || status=1
fi
rm -rf "$rotation_dir"
rm -f "$override_file"
exit "$status"
}
trap restore EXIT HUP INT TERM
[ -f "$env_file" ] || {
echo "environment file not found: $env_file" >&2
exit 1
}
[ -f "$security_dir/internal-ca/ca.crt" ] || {
echo "run infra/pki/bootstrap.sh first" >&2
exit 1
}
umask 077
rm -rf "$rotation_dir"
mkdir -p "$old_dir" "$new_dir" "$expired_dir"
cp "$security_dir/load-balancer/tls.crt" "$old_dir/tls.crt"
cp "$security_dir/load-balancer/tls.key" "$old_dir/tls.key"
cat >"$rotation_dir/extensions.cnf" <<EOF
basicConstraints = critical,CA:FALSE
keyUsage = critical,digitalSignature
extendedKeyUsage = clientAuth
subjectAltName = URI:$new_identity
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid,issuer
EOF
issue_leaf() {
directory=$1
days=$2
openssl genpkey -quiet -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$directory/tls.key"
openssl req -new -sha256 \
-key "$directory/tls.key" \
-out "$directory/tls.csr" \
-subj "/CN=load-balancer-v2"
openssl x509 -req -sha256 -days "$days" \
-in "$directory/tls.csr" \
-CA "$security_dir/internal-ca/ca.crt" \
-CAkey "$security_dir/internal-ca/ca.key" \
-CAcreateserial \
-out "$directory/tls.crt" \
-extfile "$rotation_dir/extensions.cnf" >/dev/null
rm "$directory/tls.csr"
}
issue_leaf "$new_dir" 30
issue_leaf "$expired_dir" 0
openssl verify -purpose sslclient \
-CAfile "$security_dir/internal-ca/ca.crt" "$new_dir/tls.crt" >/dev/null
if openssl verify -purpose sslclient -attime "$(( $(date +%s) + 2 ))" \
-CAfile "$security_dir/internal-ca/ca.crt" "$expired_dir/tls.crt" >/dev/null 2>&1; then
echo "zero-lifetime certificate unexpectedly passed expiry verification" >&2
exit 1
fi
cat >"$override_file" <<EOF
services:
api:
environment:
APP_TLS_ALLOWED_CLIENT_URI: $new_identity
EOF
cp "$new_dir/tls.crt" "$security_dir/load-balancer/tls.crt"
cp "$new_dir/tls.key" "$security_dir/load-balancer/tls.key"
chmod 0600 "$security_dir/load-balancer/tls.crt" "$security_dir/load-balancer/tls.key"
rotation_active=1
install_security_material
compose_rotated up -d --no-deps --no-build --force-recreate api nginx >/dev/null
wait_for_https
compose_rotated --profile security run --rm --no-deps --entrypoint /bin/sh security-tests -c '
set -eu
curl --silent --show-error --fail \
--cacert /src/.runtime/security/internal-ca/ca.crt \
--cert /src/.runtime/security/load-balancer/tls.crt \
--key /src/.runtime/security/load-balancer/tls.key \
https://api:8443/health/ready >/dev/null
if curl --silent --show-error --fail \
--cacert /src/.runtime/security/internal-ca/ca.crt \
--cert /src/.runtime/security/rotation-test/old-load-balancer/tls.crt \
--key /src/.runtime/security/rotation-test/old-load-balancer/tls.key \
https://api:8443/health/ready >/dev/null 2>&1; then
echo "retired load-balancer identity unexpectedly reached API" >&2
exit 1
fi
if curl --silent --show-error --fail \
--cacert /src/.runtime/security/internal-ca/ca.crt \
--cert /src/.runtime/security/rotation-test/expired-load-balancer/tls.crt \
--key /src/.runtime/security/rotation-test/expired-load-balancer/tls.key \
https://api:8443/health/ready >/dev/null 2>&1; then
echo "expired load-balancer leaf unexpectedly reached API" >&2
exit 1
fi
'
echo "leaf rotation passed: new identity accepted, retired identity rejected, expired leaf rejected"
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
import argparse
import base64
import json
import secrets
import ssl
import urllib.error
import urllib.request
def request(context, method, url, payload=None, headers=None):
data = None if payload is None else json.dumps(payload).encode("utf-8")
request_headers = dict(headers or {})
if data is not None:
request_headers["Content-Type"] = "application/json"
request = urllib.request.Request(url, data=data, method=method, headers=request_headers)
try:
with urllib.request.urlopen(request, context=context, timeout=10) as response:
return response.status, response.headers
except urllib.error.HTTPError as error:
return error.code, error.headers
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="https://localhost:8443")
parser.add_argument("--ca-file", required=True)
args = parser.parse_args()
context = ssl.create_default_context(cafile=args.ca_file)
username = "limit_" + secrets.token_hex(6)
password = "Rate-limit-" + secrets.token_urlsafe(18)
status, _ = request(context, "POST", args.base_url + "/api/v1/users", {
"username": username,
"display_name": "Rate limit check",
"password": password,
})
if status != 201:
raise SystemExit(f"registration returned {status}")
authorization = "Basic " + base64.b64encode((username + ":wrong-password").encode()).decode()
instances = set()
statuses = []
retry_after = None
for _ in range(6):
status, headers = request(context, "POST", args.base_url + "/api/v1/auth/sessions", headers={
"Authorization": authorization,
})
statuses.append(status)
instances.add(headers.get("X-Instance-Id"))
if status == 429:
retry_after = headers.get("Retry-After")
if statuses != [401, 401, 401, 401, 401, 429]:
raise SystemExit(f"unexpected statuses: {statuses}")
if not retry_after or int(retry_after) < 1:
raise SystemExit("429 response has no positive Retry-After")
instances.discard(None)
if len(instances) < 2:
raise SystemExit(f"requests reached {len(instances)} API instance")
print("PASS: shared login rate limit returned 429 with Retry-After across two API replicas")
if __name__ == "__main__":
main()
+177
View File
@@ -0,0 +1,177 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
. "$script_dir/openbao-common.sh"
require_command curl
require_command docker
require_command python3
require_command "$BAO_BIN"
require_file "$BAO_CACERT"
require_unsealed
load_operator_token
test_dir=$(mktemp -d "$openbao_runtime_dir/.lifecycle.XXXXXX")
chmod 0700 "$test_dir"
sealed=0
compose() {
docker compose --project-directory "$repo_dir" \
--env-file "${ENV_FILE:-$repo_dir/lab7.env.example}" "$@"
}
agent_lookup() {
compose exec -T openbao-agent-worker sh -c '
BAO_ADDR=https://openbao:8200 \
BAO_CACERT=/run/security/openbao/ca.crt \
BAO_TOKEN=$(cat /run/openbao-agent/token) \
bao token lookup >/dev/null 2>&1
'
}
recover() {
status=$?
trap - EXIT HUP INT TERM
if [ "$sealed" -eq 1 ]; then
"$script_dir/openbao-unseal.sh" >/dev/null || status=1
compose restart openbao-agent-worker >/dev/null || status=1
attempts=0
while [ "$attempts" -lt 30 ] && ! agent_lookup; do
attempts=$((attempts + 1))
sleep 1
done
agent_lookup || status=1
fi
rm -rf "$test_dir"
exit "$status"
}
trap recover EXIT HUP INT TERM
http_status() {
header_file=$1
method=$2
path=$3
payload=${4:-}
output=$test_dir/response.json
if [ -n "$payload" ]; then
curl --silent --show-error --cacert "$BAO_CACERT" \
--header @"$header_file" \
--header "Content-Type: application/json" \
--request "$method" \
--data-binary "$payload" \
--output "$output" \
--write-out '%{http_code}' \
"$BAO_ADDR/v1/$path"
else
curl --silent --show-error --cacert "$BAO_CACERT" \
--header @"$header_file" \
--request "$method" \
--output "$output" \
--write-out '%{http_code}' \
"$BAO_ADDR/v1/$path"
fi
}
write_token_header() {
response_file=$1
header_file=$2
python3 - "$response_file" "$header_file" <<'PY'
import json
import os
import sys
with open(sys.argv[1], encoding="utf-8") as source:
document = json.load(source)
token = document.get("auth", document)["client_token"]
with open(sys.argv[2], "w", encoding="utf-8") as destination:
destination.write(f"X-Vault-Token: {token}\n")
os.chmod(sys.argv[2], 0o600)
PY
}
issue_login_material() {
name=$1
identity_dir=$test_dir/$name-identity
payload_file=$test_dir/$name-login.json
"$script_dir/openbao-issue-approle.sh" worker "$identity_dir" >/dev/null
python3 - "$identity_dir/role-id" "$identity_dir/secret-id" "$payload_file" <<'PY'
import json
import os
import sys
with open(sys.argv[1], encoding="utf-8") as source:
role_id = source.read().strip()
with open(sys.argv[2], encoding="utf-8") as source:
secret_id = source.read().strip()
with open(sys.argv[3], "w", encoding="utf-8") as destination:
json.dump({"role_id": role_id, "secret_id": secret_id}, destination)
os.chmod(sys.argv[3], 0o600)
PY
printf '%s\n' "$payload_file"
}
login_payload=$(issue_login_material renewable)
login_response=$test_dir/renewable-login-response.json
login_status=$(curl --silent --show-error --cacert "$BAO_CACERT" \
--header "Content-Type: application/json" \
--request POST \
--data-binary @"$login_payload" \
--output "$login_response" \
--write-out '%{http_code}' \
"$BAO_ADDR/v1/auth/approle/login")
[ "$login_status" = 200 ] || fail "AppRole login returned HTTP $login_status"
chmod 0600 "$login_response"
renewable_header=$test_dir/renewable-token-header
write_token_header "$login_response" "$renewable_header"
[ "$(http_status "$renewable_header" GET secret/data/messenger/worker)" = 200 ] || fail "worker token cannot read own secret"
[ "$(http_status "$renewable_header" POST auth/token/renew-self '{}')" = 200 ] || fail "worker token renewal failed"
[ "$(http_status "$renewable_header" POST auth/token/revoke-self)" = 204 ] || fail "worker token self-revocation failed"
[ "$(http_status "$renewable_header" GET auth/token/lookup-self)" = 403 ] || fail "revoked token remained valid"
short_response=$test_dir/short-token-response.json
"$BAO_BIN" token create \
-policy=messenger-worker \
-no-default-policy \
-ttl=2s \
-explicit-max-ttl=2s \
-format=json >"$short_response"
chmod 0600 "$short_response"
short_header=$test_dir/short-token-header
write_token_header "$short_response" "$short_header"
[ "$(http_status "$short_header" GET secret/data/messenger/worker)" = 200 ] || fail "short-lived token cannot read own secret"
sleep 3
[ "$(http_status "$short_header" GET secret/data/messenger/worker)" = 403 ] || fail "expired token remained valid"
sealed_login_payload=$(issue_login_material sealed)
"$BAO_BIN" operator seal >/dev/null
sealed=1
if agent_lookup; then
fail "Agent token lookup succeeded while OpenBao was sealed"
fi
sealed_status=$(curl --silent --show-error --cacert "$BAO_CACERT" \
--header "Content-Type: application/json" \
--request POST \
--data-binary @"$sealed_login_payload" \
--output "$test_dir/sealed-login-response.json" \
--write-out '%{http_code}' \
"$BAO_ADDR/v1/auth/approle/login")
[ "$sealed_status" = 503 ] || fail "sealed AppRole login: expected HTTP 503, got $sealed_status"
"$script_dir/openbao-unseal.sh" >/dev/null
compose restart openbao-agent-worker >/dev/null
attempts=0
while [ "$attempts" -lt 30 ] && ! agent_lookup; do
attempts=$((attempts + 1))
sleep 1
done
agent_lookup || fail "Agent did not authenticate after unseal"
sealed=0
echo "OpenBao lifecycle passed: renew, revoke, expiry, seal denial and Agent re-authentication"
+163
View File
@@ -0,0 +1,163 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
security_dir=${SECURITY_DIR:-$repo_dir/.runtime/security}
public_port=${PUBLIC_TLS_PORT:-8443}
public_host=${PUBLIC_TLS_HOST:-localhost}
api_address=${API_TLS_ADDRESS:-api:8443}
metrics_address=${METRICS_TLS_ADDRESS:-api:9091}
empty_ca="${TMPDIR:-/tmp}/lab7-empty-ca.$$"
cleanup() {
rm -f "$empty_ca"
}
trap cleanup EXIT HUP INT TERM
require_file() {
if [ ! -r "$1" ]; then
echo "missing security material: $1" >&2
exit 1
fi
}
expect_failure() {
name=$1
shift
if "$@" >/dev/null 2>&1; then
echo "FAIL: $name unexpectedly succeeded" >&2
exit 1
fi
echo "PASS: $name"
}
expect_tls_success() {
name=$1
certificate=$2
key=$3
if curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
--cert "$certificate" --key "$key" \
"https://$api_address/health/ready" >/dev/null 2>&1; then
echo "PASS: $name"
return
fi
echo "FAIL: $name" >&2
exit 1
}
expect_tls_failure() {
name=$1
shift
if curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
"$@" "https://$api_address/health/ready" >/dev/null 2>&1; then
echo "FAIL: $name unexpectedly succeeded" >&2
exit 1
fi
echo "PASS: $name"
}
expect_public_hostname_failure() {
if openssl s_client \
-connect "$public_host:$public_port" \
-servername wrong.local \
-verify_hostname wrong.local \
-verify_return_error \
-CAfile "$security_dir/public-ca/ca.crt" </dev/null >/dev/null 2>&1; then
echo "FAIL: public TLS with wrong hostname unexpectedly succeeded" >&2
exit 1
fi
echo "PASS: public TLS with wrong hostname"
}
for file in \
"$security_dir/public-ca/ca.crt" \
"$security_dir/internal-ca/ca.crt" \
"$security_dir/foreign-ca/ca.crt" \
"$security_dir/load-balancer/tls.crt" \
"$security_dir/load-balancer/tls.key" \
"$security_dir/prometheus/tls.crt" \
"$security_dir/prometheus/tls.key" \
"$security_dir/forbidden-client/tls.crt" \
"$security_dir/forbidden-client/tls.key" \
"$security_dir/foreign-client/tls.crt" \
"$security_dir/foreign-client/tls.key"; do
require_file "$file"
done
: >"$empty_ca"
chmod 0600 "$empty_ca"
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/public-ca/ca.crt" \
"https://$public_host:$public_port/health/ready" >/dev/null
echo "PASS: public TLS with trusted CA and hostname"
expect_failure "public TLS with empty CA" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$empty_ca" \
"https://$public_host:$public_port/health/ready"
expect_failure "public TLS with foreign CA" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/foreign-ca/ca.crt" \
"https://$public_host:$public_port/health/ready"
expect_public_hostname_failure
expect_tls_failure "API mTLS without client certificate"
expect_tls_failure "API mTLS with forbidden trusted identity" \
--cert "$security_dir/forbidden-client/tls.crt" \
--key "$security_dir/forbidden-client/tls.key"
expect_tls_failure "API mTLS with client certificate from foreign CA" \
--cert "$security_dir/foreign-client/tls.crt" \
--key "$security_dir/foreign-client/tls.key"
expect_tls_success "API mTLS with load-balancer identity" \
"$security_dir/load-balancer/tls.crt" \
"$security_dir/load-balancer/tls.key"
expect_tls_failure "business API with Prometheus identity" \
--cert "$security_dir/prometheus/tls.crt" \
--key "$security_dir/prometheus/tls.key"
expect_failure "metrics mTLS without client certificate" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
"https://$metrics_address/metrics"
expect_failure "metrics mTLS with load-balancer identity" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
--cert "$security_dir/load-balancer/tls.crt" \
--key "$security_dir/load-balancer/tls.key" \
"https://$metrics_address/metrics"
expect_failure "metrics mTLS with client certificate from foreign CA" \
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
--cert "$security_dir/foreign-client/tls.crt" \
--key "$security_dir/foreign-client/tls.key" \
"https://$metrics_address/metrics"
curl --fail --silent --show-error --noproxy '*' \
--connect-timeout 5 --max-time 10 \
--cacert "$security_dir/internal-ca/ca.crt" \
--cert "$security_dir/prometheus/tls.crt" \
--key "$security_dir/prometheus/tls.key" \
"https://$metrics_address/metrics" >/dev/null
echo "PASS: metrics mTLS with Prometheus identity"