#!/bin/sh set -eu script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) env_file=${ENV_FILE:-$repo_dir/lab7.env.example} runtime_dir="$repo_dir/.runtime/evidence/lab7-worker" state_file="$runtime_dir/queued-attachment.json" rotated_secret="$repo_dir/.runtime/openbao/seed/worker-rotated.json" rotated_password="rotated-worker-$(date -u +%Y%m%d%H%M%S)" rotated_database_password="rotated-db-$(date -u +%Y%m%d%H%M%S)" rotated_s3_secret="rotated-s3-$(date -u +%Y%m%d%H%M%S)" summary_file="$repo_dir/evidence/lab7-worker-compromise.json" started_at=$(date +%s) compose() { docker compose --project-directory "$repo_dir" --env-file "$env_file" "$@" } wait_healthy() { service=$1 remaining=60 while [ "$remaining" -gt 0 ]; do container=$(compose ps -q "$service") if [ -n "$container" ] && [ "$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")" = healthy ]; then return 0 fi remaining=$((remaining - 1)) sleep 1 done echo "$service did not become healthy" >&2 return 1 } rotate_worker_secret() { secret_file=$1 compose stop openbao-agent-worker >/dev/null ENV_FILE="$env_file" BAO_ADDR="$BAO_ADDR" BAO_CACERT="$BAO_CACERT" \ "$script_dir/openbao-rotate.sh" worker "$secret_file" >/dev/null docker start "$(compose ps -aq openbao-agent-worker)" >/dev/null wait_healthy openbao-agent-worker } restore() { compose stop worker openbao-agent-worker >/dev/null 2>&1 || true compose run --rm --no-deps acl-bootstrap >/dev/null 2>&1 || true compose restart redis-broker redis-gateway >/dev/null 2>&1 || true compose run --rm --no-deps postgres-roles >/dev/null 2>&1 || true compose run --rm --no-deps s3-bootstrap >/dev/null 2>&1 || true rotate_worker_secret "$repo_dir/.runtime/openbao/seed/worker.json" >/dev/null 2>&1 || true compose up -d --no-deps --force-recreate worker >/dev/null 2>&1 || true } mkdir -p "$runtime_dir" chmod 0700 "$repo_dir/.runtime/evidence" "$runtime_dir" trap restore EXIT HUP INT TERM if [ -z "${BAO_ADDR:-}" ]; then openbao_address=$(compose port openbao 8200) BAO_ADDR="https://$openbao_address" fi BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt" export BAO_ADDR BAO_CACERT compose stop worker >/dev/null compose --profile experiments run --rm --no-deps lab5-experiment /app/lab5experiment api-produce >"$state_file" chmod 0600 "$state_file" old_database_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY' import json import sys import urllib.parse with open(sys.argv[1], encoding="utf-8") as source: secret = json.load(source) print(urllib.parse.urlsplit(secret["database_url"]).password) PY ) old_broker_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY' import json import sys import urllib.parse with open(sys.argv[1], encoding="utf-8") as source: print(urllib.parse.urlsplit(json.load(source)["broker_url"]).password) PY ) old_s3_access_key=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: print(json.load(source)["s3_access_key"]) PY ) old_s3_secret=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as source: print(json.load(source)["s3_secret_key"]) PY ) python3 - "$repo_dir/.runtime/openbao/seed/worker.json" "$rotated_secret" \ "$rotated_password" "$rotated_database_password" "$rotated_s3_secret" <<'PY' import json import os import sys import urllib.parse def replace_password(url, password): parsed = urllib.parse.urlsplit(url) username = urllib.parse.quote(parsed.username or "", safe="") netloc = f"{username}:{urllib.parse.quote(password, safe='')}@{parsed.hostname}" if parsed.port: netloc += f":{parsed.port}" return urllib.parse.urlunsplit((parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment)) with open(sys.argv[1], encoding="utf-8") as source: secret = json.load(source) secret["broker_url"] = replace_password(secret["broker_url"], sys.argv[3]) secret["database_url"] = replace_password(secret["database_url"], sys.argv[4]) secret["s3_secret_key"] = sys.argv[5] with open(sys.argv[2], "w", encoding="utf-8") as destination: json.dump(secret, destination, separators=(",", ":")) destination.write("\n") os.chmod(sys.argv[2], 0o600) PY compose run --rm --no-deps -e REDIS_WORKER_PASSWORD="$rotated_password" acl-bootstrap >/dev/null compose restart redis-broker redis-gateway >/dev/null compose exec -T postgres sh -c \ "psql --set=ON_ERROR_STOP=1 --username \"\$POSTGRES_USER\" --dbname \"\$POSTGRES_DB\" --command \"ALTER ROLE messenger_worker PASSWORD '$rotated_database_password'\"" >/dev/null compose exec -T postgres sh -c \ 'psql --set=ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" --command="SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = '\''messenger_worker'\'' AND pid <> pg_backend_pid()"' >/dev/null compose run --rm --no-deps -e S3_WORKER_SECRET_KEY="$rotated_s3_secret" s3-bootstrap >/dev/null compose --profile security run --rm --no-deps \ -e REDISCLI_AUTH="$old_broker_password" \ --entrypoint /bin/sh dependency-tests -c ' output=$(redis-cli --tls --cacert /run/security-source/internal-ca/ca.crt \ --cert /run/security-source/worker-client/tls.crt --key /run/security-source/worker-client/tls.key \ --user messenger_worker -h redis-gateway -p 36379 PING 2>&1 || true) case "$output" in *WRONGPASS*|*invalid\ username-password*) exit 0 ;; *) echo "expected old broker credential rejection" >&2; exit 1 ;; esac ' if compose --profile security run --rm --no-deps \ -e OLD_DATABASE_PASSWORD="$old_database_password" \ --entrypoint /bin/sh dependency-tests -c ' psql "postgresql://messenger_worker:${OLD_DATABASE_PASSWORD}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/security-source/internal-ca/ca.crt&sslcert=/run/security-source/worker-client/tls.crt&sslkey=/run/security-source/worker-client/tls.key" \ --set=ON_ERROR_STOP=1 --command="SELECT 1" >/dev/null 2>&1 '; then echo "old worker database credential was not rejected" >&2 exit 1 fi if compose --profile security run --rm --no-deps \ -e OLD_S3_ACCESS_KEY="$old_s3_access_key" -e OLD_S3_SECRET="$old_s3_secret" \ --entrypoint /bin/sh dependency-tests -c ' export SSL_CERT_FILE=/run/security-source/internal-ca/ca.crt export MC_CONFIG_DIR=/tmp/old-worker-mc mc alias set old https://s3:9000 "$OLD_S3_ACCESS_KEY" "$OLD_S3_SECRET" >/dev/null 2>&1 && mc ls old/messenger-attachments/attachments/ >/dev/null 2>&1 '; then echo "old worker S3 credential was not rejected" >&2 exit 1 fi rotate_worker_secret "$rotated_secret" compose up -d --no-deps --force-recreate worker >/dev/null wait_healthy worker compose --profile experiments run --rm --no-deps -T lab5-experiment \ /app/lab5experiment api-verify <"$state_file" >/dev/null finished_at=$(date +%s) python3 - "$summary_file" "$started_at" "$finished_at" <<'PY' import datetime import json import sys started_at = int(sys.argv[2]) finished_at = int(sys.argv[3]) summary = { "schema_version": 1, "scenario": "worker credential compromise and rotation", "generated_at": datetime.datetime.fromtimestamp( finished_at, datetime.timezone.utc ).replace(microsecond=0).isoformat(), "recovery_seconds": finished_at - started_at, "old_credential_rejected": True, "old_database_credential_rejected": True, "old_s3_credential_rejected": True, "openbao_secret_rotated": True, "queued_attachment_preserved": True, "result": "PASS", } with open(sys.argv[1], "w", encoding="utf-8") as destination: json.dump(summary, destination, ensure_ascii=False, indent=2) destination.write("\n") PY trap - EXIT HUP INT TERM restore rm -f "$rotated_secret" echo "PASS: revoked worker credential failed; rotated OpenBao secret restored processing without losing the queued attachment"