Files
mtusi2026lab7/scripts/lab7-worker-compromise.sh
dizi a8edaf058d
CI / verify (pull_request) Successful in 14m53s
chore: finalize lab7 security verification
2026-11-06 22:00:42 +03:00

212 lines
8.1 KiB
Bash
Executable File

#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_dir=$(CDPATH= cd -- "$script_dir/.." && pwd)
env_file=${ENV_FILE:-$repo_dir/lab7.env.example}
runtime_dir="$repo_dir/.runtime/evidence/lab7-worker"
state_file="$runtime_dir/queued-attachment.json"
rotated_secret="$repo_dir/.runtime/openbao/seed/worker-rotated.json"
rotated_password="rotated-worker-$(date -u +%Y%m%d%H%M%S)"
rotated_database_password="rotated-db-$(date -u +%Y%m%d%H%M%S)"
rotated_s3_secret="rotated-s3-$(date -u +%Y%m%d%H%M%S)"
summary_file="$repo_dir/evidence/lab7-worker-compromise.json"
started_at=$(date +%s)
compose() {
docker compose --project-directory "$repo_dir" --env-file "$env_file" "$@"
}
wait_healthy() {
service=$1
remaining=60
while [ "$remaining" -gt 0 ]; do
container=$(compose ps -q "$service")
if [ -n "$container" ] && [ "$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")" = healthy ]; then
return 0
fi
remaining=$((remaining - 1))
sleep 1
done
echo "$service did not become healthy" >&2
return 1
}
rotate_worker_secret() {
secret_file=$1
compose stop openbao-agent-worker >/dev/null
ENV_FILE="$env_file" BAO_ADDR="$BAO_ADDR" BAO_CACERT="$BAO_CACERT" \
"$script_dir/openbao-rotate.sh" worker "$secret_file" >/dev/null
docker start "$(compose ps -aq openbao-agent-worker)" >/dev/null
wait_healthy openbao-agent-worker
}
restore() {
compose stop worker openbao-agent-worker >/dev/null 2>&1 || true
compose run --rm --no-deps acl-bootstrap >/dev/null 2>&1 || true
compose restart redis-broker redis-gateway >/dev/null 2>&1 || true
compose run --rm --no-deps postgres-roles >/dev/null 2>&1 || true
compose run --rm --no-deps s3-bootstrap >/dev/null 2>&1 || true
rotate_worker_secret "$repo_dir/.runtime/openbao/seed/worker.json" >/dev/null 2>&1 || true
compose up -d --no-deps --force-recreate worker >/dev/null 2>&1 || true
}
mkdir -p "$runtime_dir"
chmod 0700 "$repo_dir/.runtime/evidence" "$runtime_dir"
trap restore EXIT HUP INT TERM
if [ -z "${BAO_ADDR:-}" ]; then
openbao_address=$(compose port openbao 8200)
BAO_ADDR="https://$openbao_address"
fi
BAO_CACERT="$repo_dir/.runtime/security/openbao/ca.crt"
export BAO_ADDR BAO_CACERT
compose stop worker >/dev/null
compose --profile experiments run --rm --no-deps lab5-experiment /app/lab5experiment api-produce >"$state_file"
chmod 0600 "$state_file"
old_database_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
import json
import sys
import urllib.parse
with open(sys.argv[1], encoding="utf-8") as source:
secret = json.load(source)
print(urllib.parse.urlsplit(secret["database_url"]).password)
PY
)
old_broker_password=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
import json
import sys
import urllib.parse
with open(sys.argv[1], encoding="utf-8") as source:
print(urllib.parse.urlsplit(json.load(source)["broker_url"]).password)
PY
)
old_s3_access_key=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as source:
print(json.load(source)["s3_access_key"])
PY
)
old_s3_secret=$(python3 - "$repo_dir/.runtime/openbao/seed/worker.json" <<'PY'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as source:
print(json.load(source)["s3_secret_key"])
PY
)
python3 - "$repo_dir/.runtime/openbao/seed/worker.json" "$rotated_secret" \
"$rotated_password" "$rotated_database_password" "$rotated_s3_secret" <<'PY'
import json
import os
import sys
import urllib.parse
def replace_password(url, password):
parsed = urllib.parse.urlsplit(url)
username = urllib.parse.quote(parsed.username or "", safe="")
netloc = f"{username}:{urllib.parse.quote(password, safe='')}@{parsed.hostname}"
if parsed.port:
netloc += f":{parsed.port}"
return urllib.parse.urlunsplit((parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment))
with open(sys.argv[1], encoding="utf-8") as source:
secret = json.load(source)
secret["broker_url"] = replace_password(secret["broker_url"], sys.argv[3])
secret["database_url"] = replace_password(secret["database_url"], sys.argv[4])
secret["s3_secret_key"] = sys.argv[5]
with open(sys.argv[2], "w", encoding="utf-8") as destination:
json.dump(secret, destination, separators=(",", ":"))
destination.write("\n")
os.chmod(sys.argv[2], 0o600)
PY
compose run --rm --no-deps -e REDIS_WORKER_PASSWORD="$rotated_password" acl-bootstrap >/dev/null
compose restart redis-broker redis-gateway >/dev/null
compose exec -T postgres sh -c \
"psql --set=ON_ERROR_STOP=1 --username \"\$POSTGRES_USER\" --dbname \"\$POSTGRES_DB\" --command \"ALTER ROLE messenger_worker PASSWORD '$rotated_database_password'\"" >/dev/null
compose exec -T postgres sh -c \
'psql --set=ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" --command="SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename = '\''messenger_worker'\'' AND pid <> pg_backend_pid()"' >/dev/null
compose run --rm --no-deps -e S3_WORKER_SECRET_KEY="$rotated_s3_secret" s3-bootstrap >/dev/null
compose --profile security run --rm --no-deps \
-e REDISCLI_AUTH="$old_broker_password" \
--entrypoint /bin/sh dependency-tests -c '
output=$(redis-cli --tls --cacert /run/security-source/internal-ca/ca.crt \
--cert /run/security-source/worker-client/tls.crt --key /run/security-source/worker-client/tls.key \
--user messenger_worker -h redis-gateway -p 36379 PING 2>&1 || true)
case "$output" in
*WRONGPASS*|*invalid\ username-password*) exit 0 ;;
*) echo "expected old broker credential rejection" >&2; exit 1 ;;
esac
'
if compose --profile security run --rm --no-deps \
-e OLD_DATABASE_PASSWORD="$old_database_password" \
--entrypoint /bin/sh dependency-tests -c '
psql "postgresql://messenger_worker:${OLD_DATABASE_PASSWORD}@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/security-source/internal-ca/ca.crt&sslcert=/run/security-source/worker-client/tls.crt&sslkey=/run/security-source/worker-client/tls.key" \
--set=ON_ERROR_STOP=1 --command="SELECT 1" >/dev/null 2>&1
'; then
echo "old worker database credential was not rejected" >&2
exit 1
fi
if compose --profile security run --rm --no-deps \
-e OLD_S3_ACCESS_KEY="$old_s3_access_key" -e OLD_S3_SECRET="$old_s3_secret" \
--entrypoint /bin/sh dependency-tests -c '
export SSL_CERT_FILE=/run/security-source/internal-ca/ca.crt
export MC_CONFIG_DIR=/tmp/old-worker-mc
mc alias set old https://s3:9000 "$OLD_S3_ACCESS_KEY" "$OLD_S3_SECRET" >/dev/null 2>&1 &&
mc ls old/messenger-attachments/attachments/ >/dev/null 2>&1
'; then
echo "old worker S3 credential was not rejected" >&2
exit 1
fi
rotate_worker_secret "$rotated_secret"
compose up -d --no-deps --force-recreate worker >/dev/null
wait_healthy worker
compose --profile experiments run --rm --no-deps -T lab5-experiment \
/app/lab5experiment api-verify <"$state_file" >/dev/null
finished_at=$(date +%s)
python3 - "$summary_file" "$started_at" "$finished_at" <<'PY'
import datetime
import json
import sys
started_at = int(sys.argv[2])
finished_at = int(sys.argv[3])
summary = {
"schema_version": 1,
"scenario": "worker credential compromise and rotation",
"generated_at": datetime.datetime.fromtimestamp(
finished_at, datetime.timezone.utc
).replace(microsecond=0).isoformat(),
"recovery_seconds": finished_at - started_at,
"old_credential_rejected": True,
"old_database_credential_rejected": True,
"old_s3_credential_rejected": True,
"openbao_secret_rotated": True,
"queued_attachment_preserved": True,
"result": "PASS",
}
with open(sys.argv[1], "w", encoding="utf-8") as destination:
json.dump(summary, destination, ensure_ascii=False, indent=2)
destination.write("\n")
PY
trap - EXIT HUP INT TERM
restore
rm -f "$rotated_secret"
echo "PASS: revoked worker credential failed; rotated OpenBao secret restored processing without losing the queued attachment"