forked from mtusi2026/mtusi2026lab7
212 lines
8.1 KiB
Bash
Executable File
212 lines
8.1 KiB
Bash
Executable File
#!/bin/sh
|
|
|
|
set -eu
|
|
|
|
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
|
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
|
|
env_file=${ENV_FILE:-$repo_dir/lab7.env.example}
|
|
runtime_dir="$repo_dir/.runtime/evidence/lab7-worker"
|
|
state_file="$runtime_dir/queued-attachment.json"
|
|
rotated_secret="$repo_dir/.runtime/openbao/seed/worker-rotated.json"
|
|
rotated_password="rotated-worker-$(date -u +%Y%m%d%H%M%S)"
|
|
rotated_database_password="rotated-db-$(date -u +%Y%m%d%H%M%S)"
|
|
rotated_s3_secret="rotated-s3-$(date -u +%Y%m%d%H%M%S)"
|
|
summary_file="$repo_dir/evidence/lab7-worker-compromise.json"
|
|
started_at=$(date +%s)
|
|
|
|
compose() {
|
|
docker compose --project-directory "$repo_dir" --env-file "$env_file" "$@"
|
|
}
|
|
|
|
wait_healthy() {
|
|
service=$1
|
|
remaining=60
|
|
while [ "$remaining" -gt 0 ]; do
|
|
container=$(compose ps -q "$service")
|
|
if [ -n "$container" ] && [ "$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")" = healthy ]; then
|
|
return 0
|
|
fi
|
|
remaining=$((remaining - 1))
|
|
sleep 1
|
|
done
|
|
echo "$service did not become healthy" >&2
|
|
return 1
|
|
}
|
|
|
|
rotate_worker_secret() {
|
|
secret_file=$1
|
|
compose stop openbao-agent-worker >/dev/null
|
|
ENV_FILE="$env_file" BAO_ADDR="$BAO_ADDR" BAO_CACERT="$BAO_CACERT" \
|
|
"$script_dir/openbao-rotate.sh" worker "$secret_file" >/dev/null
|
|
docker start "$(compose ps -aq openbao-agent-worker)" >/dev/null
|
|
wait_healthy openbao-agent-worker
|
|
}
|
|
|
|
restore() {
|
|
compose stop worker openbao-agent-worker >/dev/null 2>&1 || true
|
|
compose run --rm --no-deps acl-bootstrap >/dev/null 2>&1 || true
|
|
compose restart redis-broker redis-gateway >/dev/null 2>&1 || true
|
|
compose run --rm --no-deps postgres-roles >/dev/null 2>&1 || true
|
|
compose run --rm --no-deps s3-bootstrap >/dev/null 2>&1 || true
|
|
rotate_worker_secret "$repo_dir/.runtime/openbao/seed/worker.json" >/dev/null 2>&1 || true
|
|
compose up -d --no-deps --force-recreate worker >/dev/null 2>&1 || true
|
|
}
|
|
|
|
mkdir -p "$runtime_dir"
|
|
chmod 0700 "$repo_dir/.runtime/evidence" "$runtime_dir"
|
|
trap restore EXIT HUP INT TERM
|
|
|
|
if [ -z "${BAO_ADDR:-}" ]; then
|
|
openbao_address=$(compose port openbao 8200)
|
|
BAO_ADDR="https://$openbao_address"
|
|
fi
|
|
BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt"
|
|
export BAO_ADDR BAO_CACERT
|
|
|
|
compose stop worker >/dev/null
|
|
compose --profile experiments run --rm --no-deps lab5-experiment /app/lab5experiment api-produce >"$state_file"
|
|
chmod 0600 "$state_file"
|
|
|
|
old_database_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
|
|
import json
|
|
import sys
|
|
import urllib.parse
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as source:
|
|
secret = json.load(source)
|
|
print(urllib.parse.urlsplit(secret["database_url"]).password)
|
|
PY
|
|
)
|
|
old_broker_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
|
|
import json
|
|
import sys
|
|
import urllib.parse
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as source:
|
|
print(urllib.parse.urlsplit(json.load(source)["broker_url"]).password)
|
|
PY
|
|
)
|
|
old_s3_access_key=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as source:
|
|
print(json.load(source)["s3_access_key"])
|
|
PY
|
|
)
|
|
old_s3_secret=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as source:
|
|
print(json.load(source)["s3_secret_key"])
|
|
PY
|
|
)
|
|
|
|
python3 - "$repo_dir/.runtime/openbao/seed/worker.json" "$rotated_secret" \
|
|
"$rotated_password" "$rotated_database_password" "$rotated_s3_secret" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.parse
|
|
|
|
|
|
def replace_password(url, password):
|
|
parsed = urllib.parse.urlsplit(url)
|
|
username = urllib.parse.quote(parsed.username or "", safe="")
|
|
netloc = f"{username}:{urllib.parse.quote(password, safe='')}@{parsed.hostname}"
|
|
if parsed.port:
|
|
netloc += f":{parsed.port}"
|
|
return urllib.parse.urlunsplit((parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment))
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as source:
|
|
secret = json.load(source)
|
|
secret["broker_url"] = replace_password(secret["broker_url"], sys.argv[3])
|
|
secret["database_url"] = replace_password(secret["database_url"], sys.argv[4])
|
|
secret["s3_secret_key"] = sys.argv[5]
|
|
with open(sys.argv[2], "w", encoding="utf-8") as destination:
|
|
json.dump(secret, destination, separators=(",", ":"))
|
|
destination.write("\n")
|
|
os.chmod(sys.argv[2], 0o600)
|
|
PY
|
|
|
|
compose run --rm --no-deps -e REDIS_WORKER_PASSWORD="$rotated_password" acl-bootstrap >/dev/null
|
|
compose restart redis-broker redis-gateway >/dev/null
|
|
compose exec -T postgres sh -c \
|
|
"psql --set=ON_ERROR_STOP=1 --username \"\$POSTGRES_USER\" --dbname \"\$POSTGRES_DB\" --command \"ALTER ROLE messenger_worker PASSWORD '$rotated_database_password'\"" >/dev/null
|
|
compose exec -T postgres sh -c \
|
|
'psql --set=ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" --command="SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = '\''messenger_worker'\'' AND pid <> pg_backend_pid()"' >/dev/null
|
|
compose run --rm --no-deps -e S3_WORKER_SECRET_KEY="$rotated_s3_secret" s3-bootstrap >/dev/null
|
|
compose --profile security run --rm --no-deps \
|
|
-e REDISCLI_AUTH="$old_broker_password" \
|
|
--entrypoint /bin/sh dependency-tests -c '
|
|
output=$(redis-cli --tls --cacert /run/security-source/internal-ca/ca.crt \
|
|
--cert /run/security-source/worker-client/tls.crt --key /run/security-source/worker-client/tls.key \
|
|
--user messenger_worker -h redis-gateway -p 36379 PING 2>&1 || true)
|
|
case "$output" in
|
|
*WRONGPASS*|*invalid\ username-password*) exit 0 ;;
|
|
*) echo "expected old broker credential rejection" >&2; exit 1 ;;
|
|
esac
|
|
'
|
|
|
|
if compose --profile security run --rm --no-deps \
|
|
-e OLD_DATABASE_PASSWORD="$old_database_password" \
|
|
--entrypoint /bin/sh dependency-tests -c '
|
|
psql "postgresql://messenger_worker:${OLD_DATABASE_PASSWORD}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/security-source/internal-ca/ca.crt&sslcert=/run/security-source/worker-client/tls.crt&sslkey=/run/security-source/worker-client/tls.key" \
|
|
--set=ON_ERROR_STOP=1 --command="SELECT 1" >/dev/null 2>&1
|
|
'; then
|
|
echo "old worker database credential was not rejected" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if compose --profile security run --rm --no-deps \
|
|
-e OLD_S3_ACCESS_KEY="$old_s3_access_key" -e OLD_S3_SECRET="$old_s3_secret" \
|
|
--entrypoint /bin/sh dependency-tests -c '
|
|
export SSL_CERT_FILE=/run/security-source/internal-ca/ca.crt
|
|
export MC_CONFIG_DIR=/tmp/old-worker-mc
|
|
mc alias set old https://s3:9000 "$OLD_S3_ACCESS_KEY" "$OLD_S3_SECRET" >/dev/null 2>&1 &&
|
|
mc ls old/messenger-attachments/attachments/ >/dev/null 2>&1
|
|
'; then
|
|
echo "old worker S3 credential was not rejected" >&2
|
|
exit 1
|
|
fi
|
|
|
|
rotate_worker_secret "$rotated_secret"
|
|
compose up -d --no-deps --force-recreate worker >/dev/null
|
|
wait_healthy worker
|
|
|
|
compose --profile experiments run --rm --no-deps -T lab5-experiment \
|
|
/app/lab5experiment api-verify <"$state_file" >/dev/null
|
|
|
|
finished_at=$(date +%s)
|
|
python3 - "$summary_file" "$started_at" "$finished_at" <<'PY'
|
|
import datetime
|
|
import json
|
|
import sys
|
|
|
|
started_at = int(sys.argv[2])
|
|
finished_at = int(sys.argv[3])
|
|
summary = {
|
|
"schema_version": 1,
|
|
"scenario": "worker credential compromise and rotation",
|
|
"generated_at": datetime.datetime.fromtimestamp(
|
|
finished_at, datetime.timezone.utc
|
|
).replace(microsecond=0).isoformat(),
|
|
"recovery_seconds": finished_at - started_at,
|
|
"old_credential_rejected": True,
|
|
"old_database_credential_rejected": True,
|
|
"old_s3_credential_rejected": True,
|
|
"openbao_secret_rotated": True,
|
|
"queued_attachment_preserved": True,
|
|
"result": "PASS",
|
|
}
|
|
with open(sys.argv[1], "w", encoding="utf-8") as destination:
|
|
json.dump(summary, destination, ensure_ascii=False, indent=2)
|
|
destination.write("\n")
|
|
PY
|
|
|
|
trap - EXIT HUP INT TERM
|
|
restore
|
|
rm -f "$rotated_secret"
|
|
echo "PASS: revoked worker credential failed; rotated OpenBao secret restored processing without losing the queued attachment"
|