Source: mtusi2026lab5-safronov, solution/bvt2301-safronov, 046ec9e. Contracts, public tests and evidence of lab 5 are not carried over. The compose project, image and CI prefix are renamed to lab6; the first broker start defect is recorded in KNOWN_ISSUES.md; PLAN.md outlines the OpenSpec changes of lab 6. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
601 lines
27 KiB
Bash
Executable File
601 lines
27 KiB
Bash
Executable File
#!/bin/sh
|
||
# Bootstrap S3 стенда (SeaweedFS): приватный bucket, пользователь API с
|
||
# политикой «только объекты своего bucket», пользователь уборщика с
|
||
# политикой «чтение и удаление объектов своего bucket и их список только по
|
||
# префиксу вложений своей базы» (S3_CLEANER_LIST_PREFIX, <база>/attachments/:
|
||
# для сверки bucket и БД), пользователь worker с политикой «чтение и запись
|
||
# объектов своего bucket», проверки приватности вне API. Design change
|
||
# s3-storage, решения 3, 4 и 8, design change upload-consistency, решение 8,
|
||
# design change track2-reconciler, решение 2, design change
|
||
# async-attachments, решение 12; правила — docker.md.
|
||
#
|
||
# Запускается сервисом compose s3-bootstrap на образе сервера в сетевом
|
||
# пространстве сервиса s3 (network_mode: service:s3): master и filer
|
||
# слушают 127.0.0.1 сервера, из сети стенда их нет. Административные
|
||
# вызовы — weed shell с ключом подписи (WEED_JWT_FILER_SIGNING_KEY), а не
|
||
# административной парой S3: её хранит только сервер.
|
||
#
|
||
# Режимы:
|
||
# apply (по умолчанию) — свести состояние к окружению (.env) и проверить
|
||
# права; каждый шаг безопасно повторять;
|
||
# check — ничего не менять в настройке, проверить права и приватность
|
||
# шире, чем apply: чужой bucket (временный, создаётся и удаляется
|
||
# здесь же), административные запросы ключей API и уборщика, запись
|
||
# ключом уборщика, его список по своему префиксу (200), без префикса и
|
||
# с чужим (403), права ключа worker, анонимные запросы. Общий проверочный
|
||
# объект остаётся для проверок с хоста и из сети (make s3_check) до
|
||
# режима cleanup; объект проверок удаления ключом уборщика — свой;
|
||
# cleanup — удалить проверочные объекты и временные bucket-ы check.
|
||
#
|
||
# Секреты: в weed shell — через stdin (printf — встроенная команда sh), в
|
||
# curl — файлом -K на tmpfs; в argv процессов и в выводе их нет. Ошибка
|
||
# шага — «FAILED: <шаг>» и код 1.
|
||
set -eu
|
||
|
||
MODE=${1:-apply}
|
||
BUCKET=${S3_BUCKET:?S3_BUCKET is not set}
|
||
S3_PORT=${S3_PORT:?S3_PORT is not set}
|
||
: "${S3_API_ACCESS_KEY:?S3_API_ACCESS_KEY is empty, copy S3 lines from .env.example}"
|
||
: "${S3_API_SECRET_KEY:?S3_API_SECRET_KEY is empty, copy S3 lines from .env.example}"
|
||
: "${S3_CLEANER_ACCESS_KEY:?S3_CLEANER_ACCESS_KEY is empty, run make env_setup}"
|
||
: "${S3_CLEANER_SECRET_KEY:?S3_CLEANER_SECRET_KEY is empty, run make env_setup}"
|
||
: "${S3_WORKER_ACCESS_KEY:?S3_WORKER_ACCESS_KEY is empty, run make env_setup}"
|
||
: "${S3_WORKER_SECRET_KEY:?S3_WORKER_SECRET_KEY is empty, run make env_setup}"
|
||
: "${WEED_JWT_FILER_SIGNING_KEY:?S3_JWT_SIGNING_KEY is empty, copy S3 lines from .env.example}"
|
||
# префикс списка уборщика: имя базы и /attachments/; в JSON политики идёт
|
||
# только проверенное значение
|
||
LIST_PREFIX=${S3_CLEANER_LIST_PREFIX:-}
|
||
|
||
MASTER=127.0.0.1:9333
|
||
S3=http://127.0.0.1:${S3_PORT}
|
||
API_USER=api
|
||
API_POLICY=api-objects
|
||
CLEANER_USER=cleaner
|
||
CLEANER_POLICY=cleaner-objects
|
||
WORKER_USER=worker
|
||
WORKER_POLICY=worker-objects
|
||
# регион подписи; SeaweedFS берёт его из запроса
|
||
REGION=us-east-1
|
||
TAG="s3-bootstrap"
|
||
# проверочный объект make s3_check: один и тот же ключ, повтор
|
||
# перезаписывает остаток прерванного прогона
|
||
CHECK_KEY=.s3-check/probe
|
||
# объект проверок ключа уборщика: его он удаляет, общий нужен проверкам с
|
||
# хоста
|
||
CLEANER_CHECK_KEY=.s3-check/cleaner-probe
|
||
# объект проверок ключа worker: его он пишет; удаляет ключ API
|
||
WORKER_CHECK_KEY=.s3-check/worker-probe
|
||
CHECK_BUCKET_PREFIX="${BUCKET}-check-"
|
||
# сколько секунд ждать, пока изменения IAM дойдут до шлюза (подписка на
|
||
# filer)
|
||
IAM_WAIT_SECONDS=10
|
||
|
||
umask 077
|
||
WORK=$(mktemp -d)
|
||
API_CRED="$WORK/api.cred"
|
||
CLEANER_CRED="$WORK/cleaner.cred"
|
||
WORKER_CRED="$WORK/worker.cred"
|
||
BODY="$WORK/body"
|
||
TEMP_BUCKET=""
|
||
|
||
cleanup_work() {
|
||
if [ -n "$TEMP_BUCKET" ]; then
|
||
ws "s3.bucket.delete -name ${TEMP_BUCKET}" >/dev/null 2>&1 || true
|
||
fi
|
||
rm -rf "$WORK"
|
||
}
|
||
trap cleanup_work EXIT
|
||
|
||
say() {
|
||
echo "${TAG}: $*"
|
||
}
|
||
|
||
fail() {
|
||
echo "${TAG}: FAILED: $*" >&2
|
||
exit 1
|
||
}
|
||
|
||
# одна команда weed shell; вывод — в stdout, код — код shell
|
||
ws() {
|
||
printf '%s\n' "$1" | weed shell -master="$MASTER" 2>&1
|
||
}
|
||
|
||
# команда без секретов: при ошибке печатается вывод shell
|
||
ws_step() {
|
||
step=$1
|
||
shift
|
||
if ! out=$(ws "$*"); then
|
||
printf '%s\n' "$out" | sed "s/^/${TAG}: /" >&2
|
||
fail "$step"
|
||
fi
|
||
}
|
||
|
||
# команда с секретом: вывод не печатается никогда
|
||
ws_secret_step() {
|
||
step=$1
|
||
shift
|
||
ws "$*" >/dev/null || fail "$step"
|
||
}
|
||
|
||
# ключи API, уборщика и worker для curl: файлы на tmpfs, в argv только путь
|
||
printf 'user = "%s:%s"\n' "$S3_API_ACCESS_KEY" "$S3_API_SECRET_KEY" \
|
||
>"$API_CRED"
|
||
printf 'user = "%s:%s"\n' "$S3_CLEANER_ACCESS_KEY" "$S3_CLEANER_SECRET_KEY" \
|
||
>"$CLEANER_CRED"
|
||
printf 'user = "%s:%s"\n' "$S3_WORKER_ACCESS_KEY" "$S3_WORKER_SECRET_KEY" \
|
||
>"$WORKER_CRED"
|
||
|
||
# подписанный ключом из файла $1 запрос: печатает код ответа, тело — в
|
||
# $BODY
|
||
signed_by() {
|
||
cred=$1
|
||
shift
|
||
curl -s -o "$BODY" -w '%{http_code}' --max-time 5 -K "$cred" \
|
||
--aws-sigv4 "aws:amz:${REGION}:s3" "$@" || echo 000
|
||
}
|
||
|
||
# подписанный ключом API запрос
|
||
signed() {
|
||
signed_by "$API_CRED" "$@"
|
||
}
|
||
|
||
# подписанный ключом уборщика запрос
|
||
cleaner_signed() {
|
||
signed_by "$CLEANER_CRED" "$@"
|
||
}
|
||
|
||
# подписанный ключом worker запрос
|
||
worker_signed() {
|
||
signed_by "$WORKER_CRED" "$@"
|
||
}
|
||
|
||
# анонимный запрос: печатает код ответа
|
||
anon() {
|
||
curl -s -o "$BODY" -w '%{http_code}' --max-time 5 "$@" || echo 000
|
||
}
|
||
|
||
FAILED_CHECKS=0
|
||
|
||
# строка проверки: имя, ожидаемый и полученный код. Код выхода 1 — в
|
||
# конце, после всех строк
|
||
check() {
|
||
name=$1
|
||
expected=$2
|
||
got=$3
|
||
if [ "$expected" = "$got" ]; then
|
||
verdict=ok
|
||
else
|
||
verdict=FAIL
|
||
FAILED_CHECKS=$((FAILED_CHECKS + 1))
|
||
fi
|
||
printf '%s: check\t%s\t%s\t%s\t%s\n' "$TAG" "$name" "$expected" "$got" "$verdict"
|
||
}
|
||
|
||
bucket_exists() {
|
||
ws "s3.bucket.list" | grep -qE "^[[:space:]]*$1([[:space:]]|\$)"
|
||
}
|
||
|
||
# ждать, пока ключ примет шлюз: изменения IAM доходят подпиской на filer.
|
||
# $1 — ожидаемый код, остальное — запрос signed_by с файлом ключа.
|
||
# SignatureDoesNotMatch — ключ есть, секрет другой: возвращается 2
|
||
wait_key() {
|
||
expected=$1
|
||
shift
|
||
i=0
|
||
while [ "$i" -lt "$IAM_WAIT_SECONDS" ]; do
|
||
code=$(signed_by "$@")
|
||
if [ "$code" = "$expected" ]; then
|
||
return 0
|
||
fi
|
||
if [ "$code" = 403 ] && grep -q SignatureDoesNotMatch "$BODY"; then
|
||
return 2
|
||
fi
|
||
i=$((i + 1))
|
||
sleep 1
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# ключ API — записью проверочного объекта
|
||
wait_api_key() {
|
||
wait_key 200 "$API_CRED" -X PUT --data-binary "$TAG" "$1"
|
||
}
|
||
|
||
# ключ уборщика — удалением ключа, которого нет (запись ему запрещена):
|
||
# 204, как у удаления существующего
|
||
wait_cleaner_key() {
|
||
wait_key 204 "$CLEANER_CRED" -X DELETE "$1"
|
||
}
|
||
|
||
# ключ worker — записью проверочного объекта, как у API
|
||
wait_worker_key() {
|
||
wait_key 200 "$WORKER_CRED" -X PUT --data-binary "$TAG" "$1"
|
||
}
|
||
|
||
# ждать ключ пользователя $1 через функцию $2 с адресом $3; новый секрет
|
||
# (SignatureDoesNotMatch) — пересоздать ключ и ждать снова
|
||
wait_user_key() {
|
||
user=$1
|
||
waiter=$2
|
||
url=$3
|
||
status=0
|
||
"$waiter" "$url" || status=$?
|
||
if [ "$status" = 2 ]; then
|
||
recreate_key "$user"
|
||
status=0
|
||
"$waiter" "$url" || status=$?
|
||
fi
|
||
[ "$status" = 0 ] || fail "${user} key wait (code ${code})"
|
||
}
|
||
|
||
# ---------------------------------------------------------------- apply
|
||
|
||
apply_bucket() {
|
||
if bucket_exists "$BUCKET"; then
|
||
say "bucket ${BUCKET} exists"
|
||
else
|
||
ws_step "bucket create" "s3.bucket.create -name ${BUCKET}"
|
||
say "bucket ${BUCKET} created"
|
||
fi
|
||
# анонимный доступ — none; меняется, только если он открыт
|
||
if ! ws "s3.anonymous.get -bucket ${BUCKET}" | grep -q '^Access: none$'; then
|
||
ws_step "anonymous access none" \
|
||
"s3.anonymous.set -bucket ${BUCKET} -access none"
|
||
say "anonymous access to ${BUCKET} closed"
|
||
fi
|
||
}
|
||
|
||
# политика $1 с действиями $2 (JSON-список) над объектами своего bucket
|
||
put_policy() {
|
||
cat >"$WORK/policy.json" <<EOF
|
||
{"Version": "2012-10-17", "Statement": [{"Effect": "Allow",
|
||
"Action": $2,
|
||
"Resource": ["arn:aws:s3:::${BUCKET}/*"]}]}
|
||
EOF
|
||
ws_step "policy put $1" "s3.policy -put -name=$1 -file=$WORK/policy.json"
|
||
}
|
||
|
||
# префикс списка уборщика — только вида <база>/attachments/: иначе в
|
||
# условие политики попало бы что угодно
|
||
check_list_prefix() {
|
||
database=${LIST_PREFIX%/attachments/}
|
||
# ^[a-z0-9_]+/attachments/$: суффикс на месте, имя базы непустое и без
|
||
# других символов (в том числе перевода строки и «/»)
|
||
case "$database" in
|
||
"$LIST_PREFIX" | "" | *[!a-z0-9_]*)
|
||
fail "cleaner list prefix: S3_CLEANER_LIST_PREFIX must be <database>/attachments/"
|
||
;;
|
||
esac
|
||
}
|
||
|
||
# уборщику — чтение и удаление объектов своего bucket и их список только с
|
||
# префиксом вложений своей базы (условие s3:prefix соблюдает SeaweedFS 4.48:
|
||
# весь bucket, другой префикс и HEAD bucket — 403); без записи и прав на
|
||
# bucket (design change track2-reconciler, решение 2). С change
|
||
# pipeline-reliability (решение 9) — ещё префикс staging той же базы: выводится
|
||
# из уже проверенного имени
|
||
staging_list_prefix() {
|
||
printf '%s/staging/' "${LIST_PREFIX%/attachments/}"
|
||
}
|
||
|
||
put_cleaner_policy() {
|
||
cat >"$WORK/policy.json" <<EOF
|
||
{"Version": "2012-10-17", "Statement": [{"Effect": "Allow",
|
||
"Action": ["s3:GetObject", "s3:DeleteObject"],
|
||
"Resource": ["arn:aws:s3:::${BUCKET}/*"]},
|
||
{"Effect": "Allow", "Action": ["s3:ListBucket"],
|
||
"Resource": ["arn:aws:s3:::${BUCKET}"],
|
||
"Condition": {"StringLike": {"s3:prefix": ["${LIST_PREFIX}*", "$(staging_list_prefix)*"]}}}]}
|
||
EOF
|
||
ws_step "policy put ${CLEANER_POLICY}" \
|
||
"s3.policy -put -name=${CLEANER_POLICY} -file=$WORK/policy.json"
|
||
}
|
||
|
||
apply_policies() {
|
||
put_policy "$API_POLICY" \
|
||
'["s3:GetObject", "s3:PutObject", "s3:DeleteObject"]'
|
||
put_cleaner_policy
|
||
# worker: читает staging, пишет конечный префикс; удаления и списка нет.
|
||
# По <bucket>/*, а не по префиксу: make test_isolated работает под
|
||
# test_<db>/ с теми же ключами (design change async-attachments, решение 12)
|
||
put_policy "$WORKER_POLICY" '["s3:GetObject", "s3:PutObject"]'
|
||
}
|
||
|
||
# ключ и секрет пользователя $1 — из окружения (.env)
|
||
user_keys() {
|
||
case "$1" in
|
||
"$API_USER")
|
||
ACCESS_KEY=$S3_API_ACCESS_KEY
|
||
SECRET_KEY=$S3_API_SECRET_KEY
|
||
;;
|
||
"$CLEANER_USER")
|
||
ACCESS_KEY=$S3_CLEANER_ACCESS_KEY
|
||
SECRET_KEY=$S3_CLEANER_SECRET_KEY
|
||
;;
|
||
"$WORKER_USER")
|
||
ACCESS_KEY=$S3_WORKER_ACCESS_KEY
|
||
SECRET_KEY=$S3_WORKER_SECRET_KEY
|
||
;;
|
||
*) fail "unknown user $1" ;;
|
||
esac
|
||
}
|
||
|
||
# пользователь $1 с политикой $2 и ключом из окружения; повтор — no-op
|
||
apply_user() {
|
||
name=$1
|
||
policy=$2
|
||
user_keys "$name"
|
||
if ws "s3.user.show -name ${name}" >/dev/null; then
|
||
say "user ${name} exists"
|
||
else
|
||
ws_secret_step "user ${name} create" \
|
||
"s3.user.create -name ${name} -access_key ${ACCESS_KEY} -secret_key ${SECRET_KEY}"
|
||
say "user ${name} created"
|
||
fi
|
||
ws_step "policy attach ${name}" \
|
||
"s3.policy.attach -policy ${policy} -user ${name}"
|
||
# ключи пользователя: чужие удаляются, нужного нет — создаётся
|
||
if ! keys_out=$(ws "s3.accesskey.list -user ${name}"); then
|
||
fail "access key list ${name}"
|
||
fi
|
||
keys=$(printf '%s\n' "$keys_out" | awk 'NR > 1 && NF { print $1 }')
|
||
for key in $keys; do
|
||
if [ "$key" != "$ACCESS_KEY" ]; then
|
||
ws_secret_step "stale key delete ${name}" \
|
||
"s3.accesskey.delete -user ${name} -access_key ${key}"
|
||
say "stale key of ${name} removed"
|
||
fi
|
||
done
|
||
if ! printf '%s\n' "$keys" | grep -qxF "$ACCESS_KEY"; then
|
||
ws_secret_step "key create ${name}" \
|
||
"s3.accesskey.create -user ${name} -access_key ${ACCESS_KEY} -secret_key ${SECRET_KEY}"
|
||
say "key of ${name} created"
|
||
fi
|
||
}
|
||
|
||
recreate_key() {
|
||
user_keys "$1"
|
||
ws_secret_step "key delete $1" \
|
||
"s3.accesskey.delete -user $1 -access_key ${ACCESS_KEY}"
|
||
ws_secret_step "key create $1" \
|
||
"s3.accesskey.create -user $1 -access_key ${ACCESS_KEY} -secret_key ${SECRET_KEY}"
|
||
say "key of $1 recreated (secret changed)"
|
||
}
|
||
|
||
# проверки каждого запуска: ключ API пишет и читает проверочный объект,
|
||
# список и создание bucket — 403; ключ уборщика читает его, запись и
|
||
# список — 403, затем удаляет; после удаления ключ API получает 404; ключ
|
||
# worker пишет и читает свой объект, читает объект API (тут же записанный),
|
||
# удаление, список и создание bucket — 403, объект убирает ключ API;
|
||
# аноним не читает объект и не видит список
|
||
apply_checks() {
|
||
probe="${S3}/${BUCKET}/.s3-bootstrap/check-$$"
|
||
wait_user_key "$API_USER" wait_api_key "$probe"
|
||
[ "$(signed "$probe")" = 200 ] || fail "api key get"
|
||
[ "$(anon "$probe")" = 403 ] || fail "anonymous get is not denied"
|
||
[ "$(anon "${S3}/${BUCKET}?list-type=2")" = 403 ] \
|
||
|| fail "anonymous list is not denied"
|
||
[ "$(signed "${S3}/${BUCKET}?list-type=2")" = 403 ] \
|
||
|| fail "api key list is not denied"
|
||
[ "$(signed -X PUT "${S3}/${CHECK_BUCKET_PREFIX}forbidden-$$")" = 403 ] \
|
||
|| fail "api key bucket create is not denied"
|
||
wait_user_key "$CLEANER_USER" wait_cleaner_key \
|
||
"${S3}/${BUCKET}/.s3-bootstrap/missing-$$"
|
||
[ "$(cleaner_signed "$probe")" = 200 ] && [ "$(cat "$BODY")" = "$TAG" ] \
|
||
|| fail "cleaner key get"
|
||
[ "$(cleaner_signed -X PUT --data-binary "$TAG" "$probe")" = 403 ] \
|
||
|| fail "cleaner key put is not denied"
|
||
[ "$(cleaner_signed "${S3}/${BUCKET}?list-type=2")" = 403 ] \
|
||
|| fail "cleaner key list is not denied"
|
||
# список своего префикса: политика доходит до шлюза подпиской, как ключ
|
||
wait_key 200 "$CLEANER_CRED" "$(list_url "$LIST_PREFIX")" \
|
||
|| fail "cleaner key list ${LIST_PREFIX} (code ${code})"
|
||
wait_key 200 "$CLEANER_CRED" "$(list_url "$(staging_list_prefix)")" \
|
||
|| fail "cleaner key list $(staging_list_prefix) (code ${code})"
|
||
[ "$(cleaner_signed -X DELETE "$probe")" = 204 ] || fail "cleaner key delete"
|
||
[ "$(signed "$probe")" = 404 ] || fail "object is still there after delete"
|
||
worker_probe="${S3}/${BUCKET}/.s3-bootstrap/worker-check-$$"
|
||
wait_user_key "$WORKER_USER" wait_worker_key "$worker_probe"
|
||
[ "$(worker_signed "$worker_probe")" = 200 ] && [ "$(cat "$BODY")" = "$TAG" ] \
|
||
|| fail "worker key get"
|
||
[ "$(signed -X PUT --data-binary "$TAG api" "$probe")" = 200 ] \
|
||
|| fail "api key put for worker"
|
||
[ "$(worker_signed "$probe")" = 200 ] && [ "$(cat "$BODY")" = "$TAG api" ] \
|
||
|| fail "worker key get object of api key"
|
||
[ "$(worker_signed -X DELETE "$worker_probe")" = 403 ] \
|
||
|| fail "worker key delete is not denied"
|
||
[ "$(worker_signed "${S3}/${BUCKET}?list-type=2")" = 403 ] \
|
||
|| fail "worker key list is not denied"
|
||
[ "$(worker_signed "$(list_url "$LIST_PREFIX")")" = 403 ] \
|
||
|| fail "worker key list ${LIST_PREFIX} is not denied"
|
||
[ "$(worker_signed -X PUT "${S3}/${CHECK_BUCKET_PREFIX}forbidden-$$")" = 403 ] \
|
||
|| fail "worker key bucket create is not denied"
|
||
[ "$(signed -X DELETE "$probe")" = 204 ] || fail "api key delete probe"
|
||
[ "$(signed -X DELETE "$worker_probe")" = 204 ] \
|
||
|| fail "api key delete worker probe"
|
||
say "checks passed (api key put/get; cleaner key get/delete and list" \
|
||
"${LIST_PREFIX} and $(staging_list_prefix); worker key put/get; api key list and bucket create," \
|
||
"cleaner key put and list of the bucket, worker key delete, list and" \
|
||
"bucket create, anonymous get and list denied)"
|
||
}
|
||
|
||
# адрес ListObjectsV2 своего bucket с префиксом $1 (символы [a-z0-9_.-/],
|
||
# «/» — %2F в строке запроса подписи) и одним ключом
|
||
list_url() {
|
||
encoded=$(printf '%s' "$1" | sed 's#/#%2F#g')
|
||
printf '%s' "${S3}/${BUCKET}?list-type=2&max-keys=1&prefix=${encoded}"
|
||
}
|
||
|
||
# ---------------------------------------------------------------- check
|
||
|
||
run_checks() {
|
||
bucket_exists "$BUCKET" || fail "bucket ${BUCKET} does not exist, run make run_docker"
|
||
access=$(ws "s3.anonymous.get -bucket ${BUCKET}" \
|
||
| sed -n 's/^Access: //p')
|
||
check "anonymous access setting" none "${access:-unknown}"
|
||
|
||
obj="${S3}/${BUCKET}/${CHECK_KEY}"
|
||
missing="${S3}/${BUCKET}/.s3-check/missing-$$"
|
||
# ключ API: объекты своего bucket
|
||
check "api key put object" 200 \
|
||
"$(signed -X PUT --data-binary "$TAG check" "$obj")"
|
||
check "api key get object" 200 "$(signed "$obj")"
|
||
check "api key head object" 200 "$(signed -I "$obj")"
|
||
check "api key get missing object" 404 "$(signed "$missing")"
|
||
# ключ API: всё остальное запрещено
|
||
check "api key list objects" 403 "$(signed "${S3}/${BUCKET}?list-type=2")"
|
||
check "api key create bucket" 403 \
|
||
"$(signed -X PUT "${S3}/${CHECK_BUCKET_PREFIX}forbidden-$$")"
|
||
check "api key delete bucket" 403 "$(signed -X DELETE "${S3}/${BUCKET}")"
|
||
check "api key get bucket policy" 403 "$(signed "${S3}/${BUCKET}?policy")"
|
||
check "api key put bucket policy" 403 \
|
||
"$(signed -X PUT --data-binary '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::'"${BUCKET}"'/*"}]}' "${S3}/${BUCKET}?policy")"
|
||
check "api key get bucket acl" 403 "$(signed "${S3}/${BUCKET}?acl")"
|
||
check "api key put bucket acl public-read" 403 \
|
||
"$(signed -X PUT -H 'x-amz-acl: public-read' "${S3}/${BUCKET}?acl")"
|
||
check "api key put object acl public-read" 403 \
|
||
"$(signed -X PUT -H 'x-amz-acl: public-read' "${obj}?acl")"
|
||
# ключ уборщика: чтение и удаление объектов своего bucket; удаляет он
|
||
# свой проверочный объект, общий нужен проверкам с хоста
|
||
own="${S3}/${BUCKET}/${CLEANER_CHECK_KEY}"
|
||
check "api key put cleaner probe object" 200 \
|
||
"$(signed -X PUT --data-binary "$TAG cleaner" "$own")"
|
||
check "cleaner key get object" 200 "$(cleaner_signed "$own")"
|
||
check "cleaner key head object" 200 "$(cleaner_signed -I "$own")"
|
||
check "cleaner key delete object" 204 "$(cleaner_signed -X DELETE "$own")"
|
||
check "cleaner key delete object again" 204 \
|
||
"$(cleaner_signed -X DELETE "$own")"
|
||
check "api key get deleted object" 404 "$(signed "$own")"
|
||
check "cleaner key delete missing object" 204 \
|
||
"$(cleaner_signed -X DELETE "$missing")"
|
||
# ключ уборщика: запись, список и всё, что с bucket, запрещено
|
||
check "cleaner key put object" 403 \
|
||
"$(cleaner_signed -X PUT --data-binary "$TAG" "$own")"
|
||
check "cleaner key list objects" 403 \
|
||
"$(cleaner_signed "${S3}/${BUCKET}?list-type=2")"
|
||
check "cleaner key list attachments prefix" 200 \
|
||
"$(cleaner_signed "$(list_url "$LIST_PREFIX")")"
|
||
check "cleaner key list staging prefix" 200 \
|
||
"$(cleaner_signed "$(list_url "$(staging_list_prefix)")")"
|
||
check "cleaner key list database prefix" 403 \
|
||
"$(cleaner_signed "$(list_url "${LIST_PREFIX%attachments/}")")"
|
||
check "cleaner key list other prefix" 403 \
|
||
"$(cleaner_signed "$(list_url .s3-check/)")"
|
||
check "cleaner key create bucket" 403 \
|
||
"$(cleaner_signed -X PUT "${S3}/${CHECK_BUCKET_PREFIX}forbidden-$$")"
|
||
check "cleaner key delete bucket" 403 \
|
||
"$(cleaner_signed -X DELETE "${S3}/${BUCKET}")"
|
||
check "cleaner key get bucket policy" 403 \
|
||
"$(cleaner_signed "${S3}/${BUCKET}?policy")"
|
||
check "cleaner key put bucket policy" 403 \
|
||
"$(cleaner_signed -X PUT --data-binary '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::'"${BUCKET}"'/*"}]}' "${S3}/${BUCKET}?policy")"
|
||
check "cleaner key get bucket acl" 403 \
|
||
"$(cleaner_signed "${S3}/${BUCKET}?acl")"
|
||
check "cleaner key put bucket acl public-read" 403 \
|
||
"$(cleaner_signed -X PUT -H 'x-amz-acl: public-read' "${S3}/${BUCKET}?acl")"
|
||
check "cleaner key put object acl public-read" 403 \
|
||
"$(cleaner_signed -X PUT -H 'x-amz-acl: public-read' "${obj}?acl")"
|
||
# ключ worker: чтение и запись объектов своего bucket, в том числе объект,
|
||
# записанный ключом API
|
||
worker_own="${S3}/${BUCKET}/${WORKER_CHECK_KEY}"
|
||
check "worker key put object" 200 \
|
||
"$(worker_signed -X PUT --data-binary "$TAG worker" "$worker_own")"
|
||
check "worker key get object" 200 "$(worker_signed "$worker_own")"
|
||
check "worker key head object" 200 "$(worker_signed -I "$worker_own")"
|
||
check "worker key get object of api key" 200 "$(worker_signed "$obj")"
|
||
# ключ worker: удаление, список и всё, что с bucket, запрещено
|
||
check "worker key delete object" 403 "$(worker_signed -X DELETE "$worker_own")"
|
||
check "api key get object after worker delete" 200 "$(signed "$worker_own")"
|
||
check "worker key list objects" 403 \
|
||
"$(worker_signed "${S3}/${BUCKET}?list-type=2")"
|
||
check "worker key list attachments prefix" 403 \
|
||
"$(worker_signed "$(list_url "$LIST_PREFIX")")"
|
||
check "worker key create bucket" 403 \
|
||
"$(worker_signed -X PUT "${S3}/${CHECK_BUCKET_PREFIX}forbidden-$$")"
|
||
check "worker key delete bucket" 403 \
|
||
"$(worker_signed -X DELETE "${S3}/${BUCKET}")"
|
||
check "worker key get bucket policy" 403 \
|
||
"$(worker_signed "${S3}/${BUCKET}?policy")"
|
||
check "worker key put bucket policy" 403 \
|
||
"$(worker_signed -X PUT --data-binary '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::'"${BUCKET}"'/*"}]}' "${S3}/${BUCKET}?policy")"
|
||
check "worker key get bucket acl" 403 \
|
||
"$(worker_signed "${S3}/${BUCKET}?acl")"
|
||
check "worker key put bucket acl public-read" 403 \
|
||
"$(worker_signed -X PUT -H 'x-amz-acl: public-read' "${S3}/${BUCKET}?acl")"
|
||
check "worker key put object acl public-read" 403 \
|
||
"$(worker_signed -X PUT -H 'x-amz-acl: public-read' "${worker_own}?acl")"
|
||
# чужой bucket: временный, создаёт и удаляет weed shell
|
||
TEMP_BUCKET="${CHECK_BUCKET_PREFIX}$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')"
|
||
ws_step "temporary bucket create" "s3.bucket.create -name ${TEMP_BUCKET}"
|
||
other="${S3}/${TEMP_BUCKET}"
|
||
check "api key put object to other bucket" 403 \
|
||
"$(signed -X PUT --data-binary "$TAG" "${other}/probe")"
|
||
check "api key get object from other bucket" 403 "$(signed "${other}/probe")"
|
||
check "api key list other bucket" 403 "$(signed "${other}?list-type=2")"
|
||
check "api key delete other bucket" 403 "$(signed -X DELETE "$other")"
|
||
check "cleaner key get object from other bucket" 403 \
|
||
"$(cleaner_signed "${other}/probe")"
|
||
check "cleaner key delete object in other bucket" 403 \
|
||
"$(cleaner_signed -X DELETE "${other}/probe")"
|
||
check "cleaner key list other bucket" 403 \
|
||
"$(cleaner_signed "${other}?list-type=2")"
|
||
check "cleaner key delete other bucket" 403 \
|
||
"$(cleaner_signed -X DELETE "$other")"
|
||
check "worker key put object to other bucket" 403 \
|
||
"$(worker_signed -X PUT --data-binary "$TAG" "${other}/probe")"
|
||
check "worker key get object from other bucket" 403 \
|
||
"$(worker_signed "${other}/probe")"
|
||
check "worker key list other bucket" 403 \
|
||
"$(worker_signed "${other}?list-type=2")"
|
||
ws_step "temporary bucket delete" "s3.bucket.delete -name ${TEMP_BUCKET}"
|
||
TEMP_BUCKET=""
|
||
# аноним изнутри сервера; с хоста и из сети стенда — make s3_check
|
||
check "anonymous get object" 403 "$(anon "$obj")"
|
||
check "anonymous head object" 403 "$(anon -I "$obj")"
|
||
check "anonymous get missing object" 403 "$(anon "$missing")"
|
||
check "anonymous list objects" 403 "$(anon "${S3}/${BUCKET}?list-type=2")"
|
||
check "anonymous list buckets" 403 "$(anon "${S3}/")"
|
||
check "anonymous put object" 403 \
|
||
"$(anon -X PUT --data-binary "$TAG" "${S3}/${BUCKET}/.s3-check/anonymous")"
|
||
check "anonymous create bucket" 403 \
|
||
"$(anon -X PUT "${S3}/${CHECK_BUCKET_PREFIX}anonymous-$$")"
|
||
if [ "$FAILED_CHECKS" -gt 0 ]; then
|
||
fail "${FAILED_CHECKS} check(s) failed"
|
||
fi
|
||
say "checks passed; probe object ${BUCKET}/${CHECK_KEY} kept for host and network checks"
|
||
}
|
||
|
||
run_cleanup() {
|
||
for key in "$CHECK_KEY" "$CLEANER_CHECK_KEY" "$WORKER_CHECK_KEY"; do
|
||
code=$(signed -X DELETE "${S3}/${BUCKET}/${key}")
|
||
case "$code" in
|
||
204 | 404) say "probe object ${BUCKET}/${key} removed" ;;
|
||
*) fail "probe object delete (code ${code})" ;;
|
||
esac
|
||
done
|
||
# временные bucket-ы прерванных проверок
|
||
for name in $(ws "s3.bucket.list" | awk '{ print $1 }' \
|
||
| grep -E "^${CHECK_BUCKET_PREFIX}[0-9a-f]+\$" || true); do
|
||
ws_step "temporary bucket delete" "s3.bucket.delete -name ${name}"
|
||
say "temporary bucket ${name} removed"
|
||
done
|
||
}
|
||
|
||
case "$MODE" in
|
||
apply | check) check_list_prefix ;;
|
||
esac
|
||
|
||
case "$MODE" in
|
||
apply)
|
||
apply_bucket
|
||
apply_policies
|
||
apply_user "$API_USER" "$API_POLICY"
|
||
apply_user "$CLEANER_USER" "$CLEANER_POLICY"
|
||
apply_user "$WORKER_USER" "$WORKER_POLICY"
|
||
apply_checks
|
||
;;
|
||
check) run_checks ;;
|
||
cleanup) run_cleanup ;;
|
||
*) fail "unknown mode ${MODE}: apply, check or cleanup" ;;
|
||
esac
|