forked from mtusi2026/mtusi2026lab2
Split the S3 keys of the API and the reconciler, drop root from one-offs
The API key now has only GetObject, PutObject and DeleteObject on the attachment objects: the server never lists the bucket, never runs multipart uploads and has its region set. The reconciler gets its own user messenger-reconciler with ListBucket, GetObject and DeleteObject, created by s3-init from a secret of its own, so the API replicas can no longer list the bucket. The cost seeding writes objects and therefore runs in a one-off API container under the API key. s3-check proves the API key cannot list and checks the reconciler key; its snapshot drops dated fields with a JSON filter instead of grep, which left a dangling comma, and its first line carries the date and the commit. s3-init and s3-cli run as uid 65534, and migrate and reconciler get an init process, so SIGTERM reaches python and docker stop ends a pass in a fraction of a second instead of waiting for SIGKILL. limits.py and CI check the users of the one-off services. Stack tests move a 3 MiB file up and down through the balancer at 256 and 192 KiB/s, longer than the request timeout, and check 401 before the declared size. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
54099e1da3
commit
2891b37e75
+4
-1
@@ -50,9 +50,12 @@ UPLOAD_SWEEP_INTERVAL_SECONDS=30
|
||||
UPLOAD_SWEEP_BATCH=100
|
||||
# Fixed glibc mmap threshold for the API: buffers above 128 KiB are returned to the system on free.
|
||||
MALLOC_MMAP_THRESHOLD_=131072
|
||||
# Minimal client transfer rate, bytes/s: every body byte moved in either direction extends the
|
||||
# request deadline by 1/rate, so slow but steady uploads and downloads outlive REQUEST_TIMEOUT_SECONDS.
|
||||
MIN_TRANSFER_RATE_BYTES=131072
|
||||
# Solution-specific for lab4 track 2: limits of the one-off reconciler (profile reconcile), its S3
|
||||
# read timeout for listing pages, optional CPU pinning of the measured services (empty = no pinning)
|
||||
# and the CPU limit of RustFS (make reconcile-cost raises it with its memory for 100k objects).
|
||||
# and the CPU limit of RustFS (make reconcile-cost raises it with its memory for the 100k point).
|
||||
RECONCILE_CPUS=1
|
||||
RECONCILE_MEMORY=256m
|
||||
RECONCILE_S3_READ_TIMEOUT_SECONDS=10
|
||||
|
||||
@@ -79,6 +79,14 @@ jobs:
|
||||
docker top "$id" -eo pid,uid,comm | awk 'NR>1 && $2==0 {print; bad=1} END {exit bad}' || exit 1
|
||||
done
|
||||
|
||||
- name: Одноразовые сервисы не от root
|
||||
run: |
|
||||
for service in migrate s3-init; do
|
||||
user=$(docker inspect -f '{{.Config.User}}' "$(docker compose ps -aq "$service")")
|
||||
echo "$service: $user"
|
||||
case "${user%%:*}" in ""|0|root) exit 1 ;; esac
|
||||
done
|
||||
|
||||
- name: Корректная остановка реплик API по SIGTERM
|
||||
run: |
|
||||
start=$(date +%s)
|
||||
|
||||
+18
-2
@@ -26,8 +26,15 @@ x-s3-client: &s3-client
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
RC_CONFIG_DIR: /tmp/rc
|
||||
user: "65534:65534"
|
||||
volumes: ["./infra/s3:/s3:ro"]
|
||||
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
|
||||
secrets:
|
||||
- s3_root_user
|
||||
- s3_root_password
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- s3_reconciler_access_key_id
|
||||
- s3_reconciler_secret_access_key
|
||||
cap_drop: [ALL]
|
||||
security_opt: ["no-new-privileges:true"]
|
||||
restart: "no"
|
||||
@@ -78,6 +85,7 @@ services:
|
||||
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
|
||||
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
|
||||
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8}
|
||||
MIN_TRANSFER_RATE_BYTES: ${MIN_TRANSFER_RATE_BYTES:-131072}
|
||||
DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1}
|
||||
DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2}
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
|
||||
@@ -114,6 +122,7 @@ services:
|
||||
migrate:
|
||||
<<: *app
|
||||
pull_policy: never
|
||||
init: true
|
||||
command: ["python", "-m", "messenger.migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
@@ -230,14 +239,17 @@ services:
|
||||
<<: *app
|
||||
pull_policy: never
|
||||
profiles: [reconcile]
|
||||
init: true
|
||||
entrypoint: ["python", "-m", "messenger.attachments.entrypoints.reconcile"]
|
||||
command: ["--dry-run"]
|
||||
environment:
|
||||
<<: *s3-env
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key
|
||||
S3_READ_TIMEOUT_SECONDS: ${RECONCILE_S3_READ_TIMEOUT_SECONDS:-10}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
secrets: [database_url, s3_reconciler_access_key_id, s3_reconciler_secret_access_key]
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
@@ -294,6 +306,10 @@ secrets:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
s3_reconciler_access_key_id:
|
||||
file: ./secrets/s3_reconciler_access_key_id
|
||||
s3_reconciler_secret_access_key:
|
||||
file: ./secrets/s3_reconciler_secret_access_key
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
|
||||
@@ -4,20 +4,8 @@
|
||||
{
|
||||
"Sid": "MessengerApiObjects",
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts"
|
||||
],
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
|
||||
},
|
||||
{
|
||||
"Sid": "MessengerApiBucket",
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
+2
-1
@@ -4,7 +4,8 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; }
|
||||
endpoint="${S3_ENDPOINT:-http://s3:9000}"
|
||||
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
|
||||
RC_HOST_api="${endpoint%%://*}://$(secret s3_access_key_id):$(secret s3_secret_access_key)@${endpoint#*://}"
|
||||
export RC_HOST_root RC_HOST_api
|
||||
RC_HOST_reconciler="${endpoint%%://*}://$(secret s3_reconciler_access_key_id):$(secret s3_reconciler_secret_access_key)@${endpoint#*://}"
|
||||
export RC_HOST_root RC_HOST_api RC_HOST_reconciler
|
||||
rc alias set anon "$endpoint" --anonymous >/dev/null
|
||||
case "${1:-}" in
|
||||
sh|wget) exec "$@" ;;
|
||||
|
||||
+11
-7
@@ -4,17 +4,21 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; }
|
||||
bucket="${S3_BUCKET:-messenger-attachments}"
|
||||
endpoint="${S3_ENDPOINT:-http://s3:9000}"
|
||||
policy="${S3_API_POLICY:-messenger-api}"
|
||||
reconciler="${S3_RECONCILER_POLICY:-messenger-reconciler}"
|
||||
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
|
||||
export RC_HOST_root
|
||||
api_key=$(secret s3_access_key_id)
|
||||
grant() {
|
||||
sed "s/messenger-attachments/$bucket/g" "/s3/$2" > "/tmp/$2"
|
||||
rc admin policy create root "$1" "/tmp/$2" >/dev/null
|
||||
rc admin user add root "$(secret "$3")" "$(secret "$4")" >/dev/null
|
||||
rc admin policy attach root "$1" --user "$(secret "$3")" >/dev/null
|
||||
rc admin user info root "$(secret "$3")" --json | grep -q "\"$1\""
|
||||
}
|
||||
rc ready root --timeout 10 >/dev/null
|
||||
rc bucket create --ignore-existing "root/$bucket" >/dev/null
|
||||
rc bucket anonymous set private "root/$bucket" >/dev/null
|
||||
sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.json
|
||||
rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null
|
||||
rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null
|
||||
rc admin policy attach root "$policy" --user "$api_key" >/dev/null
|
||||
grant "$policy" api-policy.json s3_access_key_id s3_secret_access_key
|
||||
grant "$reconciler" reconciler-policy.json s3_reconciler_access_key_id s3_reconciler_secret_access_key
|
||||
rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null
|
||||
rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"'
|
||||
rc admin user info root "$api_key" --json | grep -q "\"$policy\""
|
||||
echo "s3-init: bucket=$bucket anonymous=private policy=$policy lifecycle=abort-incomplete-uploads"
|
||||
echo "s3-init: bucket=$bucket anonymous=private policies=$policy,$reconciler lifecycle=abort-incomplete-uploads"
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "MessengerReconcilerObjects",
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
|
||||
},
|
||||
{
|
||||
"Sid": "MessengerReconcilerListing",
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,18 +1,18 @@
|
||||
## 1. Платформа
|
||||
|
||||
- [ ] 1.1 Конвейер: guard раньше проверки объявленного `Content-Length`
|
||||
- [ ] 1.2 Конвейер: дедлайн с минимальной скоростью передачи для `receive`/`send` приложения; `MIN_TRANSFER_RATE_BYTES` в `TransportSettings`, `main.py`, `compose.yaml`, `.env.example`
|
||||
- [ ] 1.3 Проверка группы: `make lint`; API-тесты: upload без токена с `Content-Length` больше лимита → `401`; медленный идущий upload и скачивание дольше таймаута → `201`/`200`; тело ползёт медленнее порога → `503`
|
||||
- [x] 1.1 Конвейер: guard раньше проверки объявленного `Content-Length`
|
||||
- [x] 1.2 Конвейер: дедлайн с минимальной скоростью передачи для `receive`/`send` приложения; `MIN_TRANSFER_RATE_BYTES` в `TransportSettings`, `main.py`, `compose.yaml`, `.env.example`
|
||||
- [x] 1.3 Проверка группы: `make lint`; API-тесты: upload без токена с `Content-Length` больше лимита → `401`; медленный идущий upload и скачивание дольше таймаута → `201`/`200`; тело ползёт медленнее порога → `503`
|
||||
|
||||
## 2. Стенд
|
||||
|
||||
- [ ] 2.1 `infra/s3/api-policy.json` без multipart, `GetBucketLocation` и `ListBucket`; `infra/s3/reconciler-policy.json`; `init.sh` создаёт обоих пользователей; `cli.sh` — alias `reconciler`
|
||||
- [ ] 2.2 `scripts/secrets.sh`: ключ сверки; `compose.yaml`: secrets сверки у `reconciler`, `s3-init`, `s3-cli`; `user` у `x-s3-client`; `init: true` у `migrate` и `reconciler`
|
||||
- [ ] 2.3 `scripts/limits.py`: политика секретов; `scripts/s3-check.sh`: запрет листинга ключом API, права ключа сверки; `scripts/reconcile_track.py`: наполнение под ключом API
|
||||
- [ ] 2.4 CI: uid одноразовых сервисов
|
||||
- [ ] 2.5 Проверка группы: стенд из чистого состояния, `make test`, `make s3-check`, `make limits`, `make reconcile`, `docker stop` прохода сверки
|
||||
- [x] 2.1 `infra/s3/api-policy.json` без multipart, `GetBucketLocation` и `ListBucket`; `infra/s3/reconciler-policy.json`; `init.sh` создаёт обоих пользователей; `cli.sh` — alias `reconciler`
|
||||
- [x] 2.2 `scripts/secrets.sh`: ключ сверки; `compose.yaml`: secrets сверки у `reconciler`, `s3-init`, `s3-cli`; `user` у `x-s3-client`; `init: true` у `migrate` и `reconciler`
|
||||
- [x] 2.3 `scripts/limits.py`: политика секретов; `scripts/s3-check.sh`: запрет листинга ключом API, права ключа сверки; `scripts/reconcile_track.py`: наполнение под ключом API
|
||||
- [x] 2.4 CI: uid одноразовых сервисов
|
||||
- [x] 2.5 Проверка группы: стенд из чистого состояния, `make test`, `make s3-check`, `make limits`, `make reconcile`, `docker stop` прохода сверки
|
||||
|
||||
## 3. Стенд: тесты
|
||||
|
||||
- [ ] 3.1 Тесты стенда: медленный upload и медленное скачивание 3 MiB через LB, upload без токена с большим `Content-Length` → `401`
|
||||
- [ ] 3.2 Проверка группы: `make student-test`, `make test-faults`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, `make check`
|
||||
- [x] 3.1 Тесты стенда: медленный upload и медленное скачивание 3 MiB через LB, upload без токена с большим `Content-Length` → `401`
|
||||
- [x] 3.2 Проверка группы: `make student-test`, `make test-faults`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, `make check`
|
||||
|
||||
+23
-3
@@ -7,7 +7,20 @@ import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"]
|
||||
S3_CLIENT_SECRETS = [
|
||||
"s3_access_key_id",
|
||||
"s3_reconciler_access_key_id",
|
||||
"s3_reconciler_secret_access_key",
|
||||
"s3_root_password",
|
||||
"s3_root_user",
|
||||
"s3_secret_access_key",
|
||||
]
|
||||
RECONCILER_SECRETS = [
|
||||
"database_url",
|
||||
"s3_reconciler_access_key_id",
|
||||
"s3_reconciler_secret_access_key",
|
||||
]
|
||||
OWN_USER = ("postgres", "valkey", "s3-init", "s3-cli")
|
||||
POLICY = {
|
||||
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
|
||||
"api": {
|
||||
@@ -23,7 +36,7 @@ POLICY = {
|
||||
},
|
||||
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
|
||||
"reconciler": {
|
||||
"secrets": ["database_url", "s3_access_key_id", "s3_secret_access_key"],
|
||||
"secrets": RECONCILER_SECRETS,
|
||||
"volumes": [],
|
||||
"networks": ["backend"],
|
||||
},
|
||||
@@ -70,6 +83,9 @@ def declared(name, service, networks, problems):
|
||||
problems.append(f"{name}: publishes {ports} beyond loopback")
|
||||
if name == "api" and not service.get("read_only"):
|
||||
problems.append("api: root filesystem is writable")
|
||||
user = str(service.get("user", ""))
|
||||
if name in OWN_USER and user.split(":", maxsplit=1)[0] in ("", "0", "root"):
|
||||
problems.append(f"{name}: runs as root")
|
||||
internal = [n for n in nets if networks.get(n, {}).get("internal")]
|
||||
return [
|
||||
name,
|
||||
@@ -79,6 +95,7 @@ def declared(name, service, networks, problems):
|
||||
",".join(volumes) or "-",
|
||||
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
|
||||
str(service.get("read_only", False)).lower(),
|
||||
user or "image",
|
||||
]
|
||||
|
||||
|
||||
@@ -144,7 +161,10 @@ def main():
|
||||
f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}",
|
||||
"$ docker compose --profile bench --profile tools --profile reconcile config --format json"
|
||||
" (declared)",
|
||||
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
|
||||
*table(
|
||||
["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro", "user"],
|
||||
services,
|
||||
),
|
||||
"",
|
||||
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
|
||||
*table(["service", "hostname", "cpus", "memory", "published"], containers),
|
||||
|
||||
@@ -558,7 +558,7 @@ def measure(flags, env, args, pinned):
|
||||
def seed(count, env):
|
||||
if count <= 0:
|
||||
return {"added": 0}
|
||||
env = dict(env, RECONCILE_CPUSET="", RECONCILE_CPUS="4", RECONCILE_MEMORY="512m")
|
||||
env = dict(env, API_CPUS="4", API_MEMORY="512m")
|
||||
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
|
||||
out = compose(
|
||||
"run",
|
||||
@@ -569,7 +569,7 @@ def seed(count, env):
|
||||
tools,
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"reconciler",
|
||||
"api",
|
||||
"/tools/seed_storage.py",
|
||||
"--count",
|
||||
str(count),
|
||||
|
||||
+30
-7
@@ -2,17 +2,30 @@
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
|
||||
undated() {
|
||||
python3 -c 'import json, sys
|
||||
def clean(value):
|
||||
if isinstance(value, dict):
|
||||
return {k: clean(v) for k, v in value.items() if not k.endswith("_date")}
|
||||
return [clean(v) for v in value] if isinstance(value, list) else value
|
||||
text, decoder, at = sys.stdin.read(), json.JSONDecoder(), 0
|
||||
while text[at:].strip():
|
||||
at += len(text[at:]) - len(text[at:].lstrip())
|
||||
value, at = decoder.raw_decode(text, at)
|
||||
print(json.dumps(clean(value), indent=2, ensure_ascii=False))'
|
||||
}
|
||||
snapshot() {
|
||||
cli sh -s <<'EOF' | grep -v '_date'
|
||||
cli sh -s <<'EOF' | undated
|
||||
ak=$(cat /run/secrets/s3_access_key_id)
|
||||
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
|
||||
rk=$(cat /run/secrets/s3_reconciler_access_key_id)
|
||||
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin policy info root messenger-reconciler" "admin user info root $ak" "admin user info root $rk" "bucket lifecycle rule export root/$S3_BUCKET"; do
|
||||
rc $c --json
|
||||
done | sed "s/$ak/<api-key>/g"
|
||||
done | sed "s/$ak/<api-key>/g; s/$rk/<reconciler-key>/g"
|
||||
EOF
|
||||
}
|
||||
failed=0
|
||||
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
|
||||
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD) == S3 privacy and bootstrap checks"
|
||||
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
|
||||
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
|
||||
before=$(snapshot)
|
||||
@@ -25,22 +38,32 @@ verdict "s3-init: состояние IAM/bucket не изменилось" "same
|
||||
cli sh -s <<'EOF' || failed=1
|
||||
b=$S3_BUCKET
|
||||
key="attachments/s3-check-$(date +%s)-$$"
|
||||
spare="$key-reconciler"
|
||||
other="s3-check-$(date +%s)-$$"
|
||||
head -c 4096 /dev/urandom > /tmp/probe
|
||||
result=0
|
||||
expect() {
|
||||
want=$1; name=$2; shift 2
|
||||
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
|
||||
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error|check bucket: Network error: service error' /tmp/out; then got=deny; else got=error; fi
|
||||
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
|
||||
}
|
||||
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
|
||||
expect allow "api: HeadObject" rc object stat "api/$b/$key"
|
||||
expect allow "api: GetObject" rc object show "api/$b/$key"
|
||||
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
|
||||
expect deny "api: ListBucket" rc object list "api/$b/attachments/"
|
||||
rc put /tmp/probe "api/$b/$spare" >/dev/null
|
||||
expect allow "reconciler: ListBucket" rc object list "reconciler/$b/attachments/"
|
||||
expect allow "reconciler: HeadObject" rc object stat "reconciler/$b/$key"
|
||||
expect allow "reconciler: GetObject" rc object show "reconciler/$b/$key"
|
||||
expect deny "reconciler: PutObject" rc put /tmp/probe "reconciler/$b/$key-put"
|
||||
expect allow "reconciler: DeleteObject" rc object remove --force "reconciler/$b/$spare"
|
||||
expect deny "reconciler: CreateBucket" rc bucket create "reconciler/$other"
|
||||
expect deny "reconciler: PutBucketPolicy (public)" rc bucket anonymous set public "reconciler/$b"
|
||||
expect deny "reconciler: admin ListUsers" rc admin user list reconciler
|
||||
expect deny "api: CreateBucket" rc bucket create "api/$other"
|
||||
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
|
||||
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
|
||||
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
|
||||
expect deny "api: PutBucketLifecycle (HeadBucket уже 403)" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
|
||||
expect deny "api: admin ListUsers" rc admin user list api
|
||||
rc bucket create --ignore-existing "root/$other" >/dev/null
|
||||
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
|
||||
|
||||
@@ -21,4 +21,6 @@ put s3_root_user "root$(random | cut -c1-16)"
|
||||
put s3_root_password "$(random)"
|
||||
put s3_access_key_id "api$(random | cut -c1-16)"
|
||||
put s3_secret_access_key "$(random)"
|
||||
put s3_reconciler_access_key_id "rec$(random | cut -c1-16)"
|
||||
put s3_reconciler_secret_access_key "$(random)"
|
||||
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
|
||||
|
||||
@@ -134,14 +134,23 @@ def replica_at(address: str) -> str:
|
||||
|
||||
|
||||
def raw(
|
||||
path: str, headers: dict[str, str], body: bytes, *, hang_up: bool
|
||||
path: str,
|
||||
headers: dict[str, str],
|
||||
body: bytes,
|
||||
*,
|
||||
hang_up: bool,
|
||||
piece: int = 0,
|
||||
delay: float = 0.0,
|
||||
) -> tuple[int, dict[str, Any], float]:
|
||||
parts = urlsplit(BASE_URL)
|
||||
head = f"POST {path} HTTP/1.1\r\nHost: {parts.netloc}\r\nConnection: close\r\n"
|
||||
head += "".join(f"{name}: {value}\r\n" for name, value in headers.items()) + "\r\n"
|
||||
started = time.perf_counter()
|
||||
with socket.create_connection((parts.hostname, parts.port or 80), timeout=30) as sock:
|
||||
sock.sendall(head.encode("latin-1") + body)
|
||||
sock.sendall(head.encode("latin-1"))
|
||||
for start in range(0, len(body), piece or max(len(body), 1)):
|
||||
sock.sendall(body[start : start + (piece or len(body))])
|
||||
time.sleep(delay)
|
||||
if hang_up:
|
||||
return 0, {}, 0.0
|
||||
response = HTTPResponse(sock)
|
||||
@@ -149,6 +158,56 @@ def raw(
|
||||
return response.status, json.loads(response.read()), time.perf_counter() - started
|
||||
|
||||
|
||||
READER = """
|
||||
import hashlib, http.client, sys, time
|
||||
path, token, request_id, piece, delay = sys.argv[1:6]
|
||||
connection = http.client.HTTPConnection("lb", 8080, timeout=30)
|
||||
headers = {"Authorization": "Bearer " + token, "X-Request-Id": request_id}
|
||||
connection.request("GET", path, headers=headers)
|
||||
response, digest, started = connection.getresponse(), hashlib.sha256(), time.monotonic()
|
||||
while chunk := response.read(int(piece)):
|
||||
digest.update(chunk)
|
||||
time.sleep(float(delay))
|
||||
print(response.status, digest.hexdigest(), round(time.monotonic() - started, 2))
|
||||
"""
|
||||
|
||||
|
||||
def slow_read(path: str, token: str, request_id: str, piece: int, delay: float) -> list[str]:
|
||||
template = "{{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}"
|
||||
networks = subprocess.run(
|
||||
["docker", "inspect", "-f", template, compose("ps", "-q", "lb")],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.split()
|
||||
image = subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.Config.Image}}", compose("ps", "-q", "api").split()[0]],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
command = [
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--network",
|
||||
next(name for name in networks if name.endswith("_edge")),
|
||||
"--sysctl",
|
||||
"net.ipv4.tcp_rmem=4096 16384 65536",
|
||||
"--entrypoint",
|
||||
"python",
|
||||
image,
|
||||
"-c",
|
||||
READER,
|
||||
path,
|
||||
token,
|
||||
request_id,
|
||||
str(piece),
|
||||
str(delay),
|
||||
]
|
||||
return subprocess.run(command, check=True, capture_output=True, text=True).stdout.split()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def http() -> AsyncIterator[AsyncClient]:
|
||||
async with AsyncClient(base_url=BASE_URL, timeout=30, trust_env=False) as client:
|
||||
@@ -466,3 +525,48 @@ async def test_stalled_upload_is_cut_and_leaves_nothing(http: AsyncClient, fault
|
||||
assert elapsed <= REQUEST_TIMEOUT + 2
|
||||
assert seen(name) is None
|
||||
assert (await upload(http, s["alice"], s["chats"][0])).status_code == 201
|
||||
|
||||
|
||||
async def test_token_is_checked_before_the_declared_size(http: AsyncClient) -> None:
|
||||
s = await scene(http)
|
||||
headers = {"Content-Type": "image/png", "Content-Length": str(LIMIT + 1)}
|
||||
path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=a.png"
|
||||
status, body, _ = await asyncio.to_thread(raw, path, headers, SAMPLE[:1024], hang_up=False)
|
||||
print(f"без токена, Content-Length {LIMIT + 1}: {status} {body['error']['code']}")
|
||||
assert (status, body["error"]["code"]) == (401, "unauthorized")
|
||||
|
||||
|
||||
async def test_slow_but_steady_transfers_outlive_the_request_timeout(http: AsyncClient) -> None:
|
||||
since = moment()
|
||||
s = await scene(http)
|
||||
picture = png(16, 16, size=3 << 20)
|
||||
sent, read = f"slow-upload-{uuid4().hex}", f"slow-download-{uuid4().hex}"
|
||||
headers = {
|
||||
**bearer(s["alice"]),
|
||||
"Content-Type": "image/png",
|
||||
"Content-Length": str(len(picture)),
|
||||
"X-Request-Id": sent,
|
||||
}
|
||||
path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=slow.png"
|
||||
status, body, up = await asyncio.to_thread(
|
||||
raw, path, headers, picture, hang_up=False, piece=64 << 10, delay=0.25
|
||||
)
|
||||
assert status == 201, body
|
||||
assert body["sha256"] == sha256(picture).hexdigest()
|
||||
content = f"/api/v1/attachments/{body['id']}/content"
|
||||
code, digest, down = await asyncio.to_thread(slow_read, content, s["bob"], read, 64 << 10, 0.33)
|
||||
assert (code, digest) == ("200", body["sha256"])
|
||||
|
||||
def handled() -> dict[str, tuple[int, float]]:
|
||||
return {
|
||||
e["request_id"]: (e["status"], e["duration_ms"] / 1000)
|
||||
for e in logs(since)
|
||||
if e.get("event") == "request.handled" and e.get("request_id") in (sent, read)
|
||||
}
|
||||
|
||||
await waited(lambda: len(handled()) == 2, 15)
|
||||
replica = handled()
|
||||
print(f"upload 3 MiB ≈ 256 KiB/s: клиент {up:.1f} с, реплика {replica[sent]}")
|
||||
print(f"скачивание 3 MiB ≈ 192 KiB/s из сети edge: клиент {down} с, реплика {replica[read]}")
|
||||
assert replica[sent][0] == 201 and replica[sent][1] > REQUEST_TIMEOUT
|
||||
assert replica[read][0] == 200 and replica[read][1] > REQUEST_TIMEOUT
|
||||
|
||||
Reference in New Issue
Block a user