diff --git a/.env.example b/.env.example index 7fac2122..265b0195 100644 --- a/.env.example +++ b/.env.example @@ -50,9 +50,12 @@ UPLOAD_SWEEP_INTERVAL_SECONDS=30 UPLOAD_SWEEP_BATCH=100 # Fixed glibc mmap threshold for the API: buffers above 128 KiB are returned to the system on free. MALLOC_MMAP_THRESHOLD_=131072 +# Minimal client transfer rate, bytes/s: every body byte moved in either direction extends the +# request deadline by 1/rate, so slow but steady uploads and downloads outlive REQUEST_TIMEOUT_SECONDS. +MIN_TRANSFER_RATE_BYTES=131072 # Solution-specific for lab4 track 2: limits of the one-off reconciler (profile reconcile), its S3 # read timeout for listing pages, optional CPU pinning of the measured services (empty = no pinning) -# and the CPU limit of RustFS (make reconcile-cost raises it with its memory for 100k objects). +# and the CPU limit of RustFS (make reconcile-cost raises it with its memory for the 100k point). RECONCILE_CPUS=1 RECONCILE_MEMORY=256m RECONCILE_S3_READ_TIMEOUT_SECONDS=10 diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml index ea8ec793..d336586e 100644 --- a/.github/workflows/application.yml +++ b/.github/workflows/application.yml @@ -79,6 +79,14 @@ jobs: docker top "$id" -eo pid,uid,comm | awk 'NR>1 && $2==0 {print; bad=1} END {exit bad}' || exit 1 done + - name: Одноразовые сервисы не от root + run: | + for service in migrate s3-init; do + user=$(docker inspect -f '{{.Config.User}}' "$(docker compose ps -aq "$service")") + echo "$service: $user" + case "${user%%:*}" in ""|0|root) exit 1 ;; esac + done + - name: Корректная остановка реплик API по SIGTERM run: | start=$(date +%s) diff --git a/compose.yaml b/compose.yaml index 0c96b57c..86bfffd4 100644 --- a/compose.yaml +++ b/compose.yaml @@ -26,8 +26,15 @@ x-s3-client: &s3-client S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} RC_CONFIG_DIR: /tmp/rc + user: "65534:65534" volumes: ["./infra/s3:/s3:ro"] - secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + secrets: + - s3_root_user + - s3_root_password + - s3_access_key_id + - s3_secret_access_key + - s3_reconciler_access_key_id + - s3_reconciler_secret_access_key cap_drop: [ALL] security_opt: ["no-new-privileges:true"] restart: "no" @@ -78,6 +85,7 @@ services: WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2} DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8} REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8} + MIN_TRANSFER_RATE_BYTES: ${MIN_TRANSFER_RATE_BYTES:-131072} DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1} DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2} DRAIN_SECONDS: ${DRAIN_SECONDS:-2} @@ -114,6 +122,7 @@ services: migrate: <<: *app pull_policy: never + init: true command: ["python", "-m", "messenger.migrate"] environment: DATABASE_URL_FILE: /run/secrets/database_url @@ -230,14 +239,17 @@ services: <<: *app pull_policy: never profiles: [reconcile] + init: true entrypoint: ["python", "-m", "messenger.attachments.entrypoints.reconcile"] command: ["--dry-run"] environment: <<: *s3-env DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key S3_READ_TIMEOUT_SECONDS: ${RECONCILE_S3_READ_TIMEOUT_SECONDS:-10} LOG_LEVEL: ${LOG_LEVEL:-INFO} - secrets: [database_url, s3_access_key_id, s3_secret_access_key] + secrets: [database_url, s3_reconciler_access_key_id, s3_reconciler_secret_access_key] healthcheck: disable: true restart: "no" @@ -294,6 +306,10 @@ secrets: file: ./secrets/s3_access_key_id s3_secret_access_key: file: ./secrets/s3_secret_access_key + s3_reconciler_access_key_id: + file: ./secrets/s3_reconciler_access_key_id + s3_reconciler_secret_access_key: + file: ./secrets/s3_reconciler_secret_access_key volumes: pgdata: diff --git a/infra/s3/api-policy.json b/infra/s3/api-policy.json index 1e46aa4f..90635a60 100644 --- a/infra/s3/api-policy.json +++ b/infra/s3/api-policy.json @@ -4,20 +4,8 @@ { "Sid": "MessengerApiObjects", "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:PutObject", - "s3:DeleteObject", - "s3:AbortMultipartUpload", - "s3:ListMultipartUploadParts" - ], + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], "Resource": ["arn:aws:s3:::messenger-attachments/*"] - }, - { - "Sid": "MessengerApiBucket", - "Effect": "Allow", - "Action": ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"], - "Resource": ["arn:aws:s3:::messenger-attachments"] } ] } diff --git a/infra/s3/cli.sh b/infra/s3/cli.sh index 9e0937f9..54ff852e 100755 --- a/infra/s3/cli.sh +++ b/infra/s3/cli.sh @@ -4,7 +4,8 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; } endpoint="${S3_ENDPOINT:-http://s3:9000}" RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}" RC_HOST_api="${endpoint%%://*}://$(secret s3_access_key_id):$(secret s3_secret_access_key)@${endpoint#*://}" -export RC_HOST_root RC_HOST_api +RC_HOST_reconciler="${endpoint%%://*}://$(secret s3_reconciler_access_key_id):$(secret s3_reconciler_secret_access_key)@${endpoint#*://}" +export RC_HOST_root RC_HOST_api RC_HOST_reconciler rc alias set anon "$endpoint" --anonymous >/dev/null case "${1:-}" in sh|wget) exec "$@" ;; diff --git a/infra/s3/init.sh b/infra/s3/init.sh index 3686b1c6..07a6954c 100755 --- a/infra/s3/init.sh +++ b/infra/s3/init.sh @@ -4,17 +4,21 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; } bucket="${S3_BUCKET:-messenger-attachments}" endpoint="${S3_ENDPOINT:-http://s3:9000}" policy="${S3_API_POLICY:-messenger-api}" +reconciler="${S3_RECONCILER_POLICY:-messenger-reconciler}" RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}" export RC_HOST_root -api_key=$(secret s3_access_key_id) +grant() { + sed "s/messenger-attachments/$bucket/g" "/s3/$2" > "/tmp/$2" + rc admin policy create root "$1" "/tmp/$2" >/dev/null + rc admin user add root "$(secret "$3")" "$(secret "$4")" >/dev/null + rc admin policy attach root "$1" --user "$(secret "$3")" >/dev/null + rc admin user info root "$(secret "$3")" --json | grep -q "\"$1\"" +} rc ready root --timeout 10 >/dev/null rc bucket create --ignore-existing "root/$bucket" >/dev/null rc bucket anonymous set private "root/$bucket" >/dev/null -sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.json -rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null -rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null -rc admin policy attach root "$policy" --user "$api_key" >/dev/null +grant "$policy" api-policy.json s3_access_key_id s3_secret_access_key +grant "$reconciler" reconciler-policy.json s3_reconciler_access_key_id s3_reconciler_secret_access_key rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"' -rc admin user info root "$api_key" --json | grep -q "\"$policy\"" -echo "s3-init: bucket=$bucket anonymous=private policy=$policy lifecycle=abort-incomplete-uploads" +echo "s3-init: bucket=$bucket anonymous=private policies=$policy,$reconciler lifecycle=abort-incomplete-uploads" diff --git a/infra/s3/reconciler-policy.json b/infra/s3/reconciler-policy.json new file mode 100644 index 00000000..a28fa837 --- /dev/null +++ b/infra/s3/reconciler-policy.json @@ -0,0 +1,17 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "MessengerReconcilerObjects", + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + }, + { + "Sid": "MessengerReconcilerListing", + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/openspec/changes/tighten-lab4-attachment-boundaries/tasks.md b/openspec/changes/tighten-lab4-attachment-boundaries/tasks.md index 56f67224..a82e89b3 100644 --- a/openspec/changes/tighten-lab4-attachment-boundaries/tasks.md +++ b/openspec/changes/tighten-lab4-attachment-boundaries/tasks.md @@ -1,18 +1,18 @@ ## 1. Платформа -- [ ] 1.1 Конвейер: guard раньше проверки объявленного `Content-Length` -- [ ] 1.2 Конвейер: дедлайн с минимальной скоростью передачи для `receive`/`send` приложения; `MIN_TRANSFER_RATE_BYTES` в `TransportSettings`, `main.py`, `compose.yaml`, `.env.example` -- [ ] 1.3 Проверка группы: `make lint`; API-тесты: upload без токена с `Content-Length` больше лимита → `401`; медленный идущий upload и скачивание дольше таймаута → `201`/`200`; тело ползёт медленнее порога → `503` +- [x] 1.1 Конвейер: guard раньше проверки объявленного `Content-Length` +- [x] 1.2 Конвейер: дедлайн с минимальной скоростью передачи для `receive`/`send` приложения; `MIN_TRANSFER_RATE_BYTES` в `TransportSettings`, `main.py`, `compose.yaml`, `.env.example` +- [x] 1.3 Проверка группы: `make lint`; API-тесты: upload без токена с `Content-Length` больше лимита → `401`; медленный идущий upload и скачивание дольше таймаута → `201`/`200`; тело ползёт медленнее порога → `503` ## 2. Стенд -- [ ] 2.1 `infra/s3/api-policy.json` без multipart, `GetBucketLocation` и `ListBucket`; `infra/s3/reconciler-policy.json`; `init.sh` создаёт обоих пользователей; `cli.sh` — alias `reconciler` -- [ ] 2.2 `scripts/secrets.sh`: ключ сверки; `compose.yaml`: secrets сверки у `reconciler`, `s3-init`, `s3-cli`; `user` у `x-s3-client`; `init: true` у `migrate` и `reconciler` -- [ ] 2.3 `scripts/limits.py`: политика секретов; `scripts/s3-check.sh`: запрет листинга ключом API, права ключа сверки; `scripts/reconcile_track.py`: наполнение под ключом API -- [ ] 2.4 CI: uid одноразовых сервисов -- [ ] 2.5 Проверка группы: стенд из чистого состояния, `make test`, `make s3-check`, `make limits`, `make reconcile`, `docker stop` прохода сверки +- [x] 2.1 `infra/s3/api-policy.json` без multipart, `GetBucketLocation` и `ListBucket`; `infra/s3/reconciler-policy.json`; `init.sh` создаёт обоих пользователей; `cli.sh` — alias `reconciler` +- [x] 2.2 `scripts/secrets.sh`: ключ сверки; `compose.yaml`: secrets сверки у `reconciler`, `s3-init`, `s3-cli`; `user` у `x-s3-client`; `init: true` у `migrate` и `reconciler` +- [x] 2.3 `scripts/limits.py`: политика секретов; `scripts/s3-check.sh`: запрет листинга ключом API, права ключа сверки; `scripts/reconcile_track.py`: наполнение под ключом API +- [x] 2.4 CI: uid одноразовых сервисов +- [x] 2.5 Проверка группы: стенд из чистого состояния, `make test`, `make s3-check`, `make limits`, `make reconcile`, `docker stop` прохода сверки ## 3. Стенд: тесты -- [ ] 3.1 Тесты стенда: медленный upload и медленное скачивание 3 MiB через LB, upload без токена с большим `Content-Length` → `401` -- [ ] 3.2 Проверка группы: `make student-test`, `make test-faults`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, `make check` +- [x] 3.1 Тесты стенда: медленный upload и медленное скачивание 3 MiB через LB, upload без токена с большим `Content-Length` → `401` +- [x] 3.2 Проверка группы: `make student-test`, `make test-faults`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, `make check` diff --git a/scripts/limits.py b/scripts/limits.py index 5a857222..7437726f 100644 --- a/scripts/limits.py +++ b/scripts/limits.py @@ -7,7 +7,20 @@ import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"] +S3_CLIENT_SECRETS = [ + "s3_access_key_id", + "s3_reconciler_access_key_id", + "s3_reconciler_secret_access_key", + "s3_root_password", + "s3_root_user", + "s3_secret_access_key", +] +RECONCILER_SECRETS = [ + "database_url", + "s3_reconciler_access_key_id", + "s3_reconciler_secret_access_key", +] +OWN_USER = ("postgres", "valkey", "s3-init", "s3-cli") POLICY = { "lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True}, "api": { @@ -23,7 +36,7 @@ POLICY = { }, "migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]}, "reconciler": { - "secrets": ["database_url", "s3_access_key_id", "s3_secret_access_key"], + "secrets": RECONCILER_SECRETS, "volumes": [], "networks": ["backend"], }, @@ -70,6 +83,9 @@ def declared(name, service, networks, problems): problems.append(f"{name}: publishes {ports} beyond loopback") if name == "api" and not service.get("read_only"): problems.append("api: root filesystem is writable") + user = str(service.get("user", "")) + if name in OWN_USER and user.split(":", maxsplit=1)[0] in ("", "0", "root"): + problems.append(f"{name}: runs as root") internal = [n for n in nets if networks.get(n, {}).get("internal")] return [ name, @@ -79,6 +95,7 @@ def declared(name, service, networks, problems): ",".join(volumes) or "-", ",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets), str(service.get("read_only", False)).lower(), + user or "image", ] @@ -144,7 +161,10 @@ def main(): f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}", "$ docker compose --profile bench --profile tools --profile reconcile config --format json" " (declared)", - *table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services), + *table( + ["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro", "user"], + services, + ), "", "$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)", *table(["service", "hostname", "cpus", "memory", "published"], containers), diff --git a/scripts/reconcile_track.py b/scripts/reconcile_track.py index 89562d06..b3e705ac 100644 --- a/scripts/reconcile_track.py +++ b/scripts/reconcile_track.py @@ -558,7 +558,7 @@ def measure(flags, env, args, pinned): def seed(count, env): if count <= 0: return {"added": 0} - env = dict(env, RECONCILE_CPUSET="", RECONCILE_CPUS="4", RECONCILE_MEMORY="512m") + env = dict(env, API_CPUS="4", API_MEMORY="512m") tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro" out = compose( "run", @@ -569,7 +569,7 @@ def seed(count, env): tools, "--entrypoint", "python", - "reconciler", + "api", "/tools/seed_storage.py", "--count", str(count), diff --git a/scripts/s3-check.sh b/scripts/s3-check.sh index fa1bdaec..6f719f6f 100755 --- a/scripts/s3-check.sh +++ b/scripts/s3-check.sh @@ -2,17 +2,30 @@ set -eu cd "$(dirname "$0")/.." cli() { docker compose run --rm --no-deps -T s3-cli "$@"; } +undated() { + python3 -c 'import json, sys +def clean(value): + if isinstance(value, dict): + return {k: clean(v) for k, v in value.items() if not k.endswith("_date")} + return [clean(v) for v in value] if isinstance(value, list) else value +text, decoder, at = sys.stdin.read(), json.JSONDecoder(), 0 +while text[at:].strip(): + at += len(text[at:]) - len(text[at:].lstrip()) + value, at = decoder.raw_decode(text, at) + print(json.dumps(clean(value), indent=2, ensure_ascii=False))' +} snapshot() { - cli sh -s <<'EOF' | grep -v '_date' + cli sh -s <<'EOF' | undated ak=$(cat /run/secrets/s3_access_key_id) -for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do +rk=$(cat /run/secrets/s3_reconciler_access_key_id) +for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin policy info root messenger-reconciler" "admin user info root $ak" "admin user info root $rk" "bucket lifecycle rule export root/$S3_BUCKET"; do rc $c --json -done | sed "s/$ak//g" +done | sed "s/$ak//g; s/$rk//g" EOF } failed=0 verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; } -echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks" +echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD) == S3 privacy and bootstrap checks" verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)" verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)" before=$(snapshot) @@ -25,22 +38,32 @@ verdict "s3-init: состояние IAM/bucket не изменилось" "same cli sh -s <<'EOF' || failed=1 b=$S3_BUCKET key="attachments/s3-check-$(date +%s)-$$" +spare="$key-reconciler" other="s3-check-$(date +%s)-$$" head -c 4096 /dev/urandom > /tmp/probe result=0 expect() { want=$1; name=$2; shift 2 - if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi + if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error|check bucket: Network error: service error' /tmp/out; then got=deny; else got=error; fi if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi } expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key" expect allow "api: HeadObject" rc object stat "api/$b/$key" expect allow "api: GetObject" rc object show "api/$b/$key" -expect allow "api: ListBucket" rc object list "api/$b/attachments/" +expect deny "api: ListBucket" rc object list "api/$b/attachments/" +rc put /tmp/probe "api/$b/$spare" >/dev/null +expect allow "reconciler: ListBucket" rc object list "reconciler/$b/attachments/" +expect allow "reconciler: HeadObject" rc object stat "reconciler/$b/$key" +expect allow "reconciler: GetObject" rc object show "reconciler/$b/$key" +expect deny "reconciler: PutObject" rc put /tmp/probe "reconciler/$b/$key-put" +expect allow "reconciler: DeleteObject" rc object remove --force "reconciler/$b/$spare" +expect deny "reconciler: CreateBucket" rc bucket create "reconciler/$other" +expect deny "reconciler: PutBucketPolicy (public)" rc bucket anonymous set public "reconciler/$b" +expect deny "reconciler: admin ListUsers" rc admin user list reconciler expect deny "api: CreateBucket" rc bucket create "api/$other" expect deny "api: DeleteBucket" rc bucket remove "api/$b" expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b" -expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json +expect deny "api: PutBucketLifecycle (HeadBucket уже 403)" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json expect deny "api: admin ListUsers" rc admin user list api rc bucket create --ignore-existing "root/$other" >/dev/null expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x" diff --git a/scripts/secrets.sh b/scripts/secrets.sh index bdc67ec5..6e133aac 100755 --- a/scripts/secrets.sh +++ b/scripts/secrets.sh @@ -21,4 +21,6 @@ put s3_root_user "root$(random | cut -c1-16)" put s3_root_password "$(random)" put s3_access_key_id "api$(random | cut -c1-16)" put s3_secret_access_key "$(random)" +put s3_reconciler_access_key_id "rec$(random | cut -c1-16)" +put s3_reconciler_secret_access_key "$(random)" [ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; } diff --git a/student-tests/stack/test_attachment_faults.py b/student-tests/stack/test_attachment_faults.py index b4270ec0..66af9fe0 100644 --- a/student-tests/stack/test_attachment_faults.py +++ b/student-tests/stack/test_attachment_faults.py @@ -134,14 +134,23 @@ def replica_at(address: str) -> str: def raw( - path: str, headers: dict[str, str], body: bytes, *, hang_up: bool + path: str, + headers: dict[str, str], + body: bytes, + *, + hang_up: bool, + piece: int = 0, + delay: float = 0.0, ) -> tuple[int, dict[str, Any], float]: parts = urlsplit(BASE_URL) head = f"POST {path} HTTP/1.1\r\nHost: {parts.netloc}\r\nConnection: close\r\n" head += "".join(f"{name}: {value}\r\n" for name, value in headers.items()) + "\r\n" started = time.perf_counter() with socket.create_connection((parts.hostname, parts.port or 80), timeout=30) as sock: - sock.sendall(head.encode("latin-1") + body) + sock.sendall(head.encode("latin-1")) + for start in range(0, len(body), piece or max(len(body), 1)): + sock.sendall(body[start : start + (piece or len(body))]) + time.sleep(delay) if hang_up: return 0, {}, 0.0 response = HTTPResponse(sock) @@ -149,6 +158,56 @@ def raw( return response.status, json.loads(response.read()), time.perf_counter() - started +READER = """ +import hashlib, http.client, sys, time +path, token, request_id, piece, delay = sys.argv[1:6] +connection = http.client.HTTPConnection("lb", 8080, timeout=30) +headers = {"Authorization": "Bearer " + token, "X-Request-Id": request_id} +connection.request("GET", path, headers=headers) +response, digest, started = connection.getresponse(), hashlib.sha256(), time.monotonic() +while chunk := response.read(int(piece)): + digest.update(chunk) + time.sleep(float(delay)) +print(response.status, digest.hexdigest(), round(time.monotonic() - started, 2)) +""" + + +def slow_read(path: str, token: str, request_id: str, piece: int, delay: float) -> list[str]: + template = "{{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}" + networks = subprocess.run( + ["docker", "inspect", "-f", template, compose("ps", "-q", "lb")], + check=True, + capture_output=True, + text=True, + ).stdout.split() + image = subprocess.run( + ["docker", "inspect", "-f", "{{.Config.Image}}", compose("ps", "-q", "api").split()[0]], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + command = [ + "docker", + "run", + "--rm", + "--network", + next(name for name in networks if name.endswith("_edge")), + "--sysctl", + "net.ipv4.tcp_rmem=4096 16384 65536", + "--entrypoint", + "python", + image, + "-c", + READER, + path, + token, + request_id, + str(piece), + str(delay), + ] + return subprocess.run(command, check=True, capture_output=True, text=True).stdout.split() + + @pytest.fixture async def http() -> AsyncIterator[AsyncClient]: async with AsyncClient(base_url=BASE_URL, timeout=30, trust_env=False) as client: @@ -466,3 +525,48 @@ async def test_stalled_upload_is_cut_and_leaves_nothing(http: AsyncClient, fault assert elapsed <= REQUEST_TIMEOUT + 2 assert seen(name) is None assert (await upload(http, s["alice"], s["chats"][0])).status_code == 201 + + +async def test_token_is_checked_before_the_declared_size(http: AsyncClient) -> None: + s = await scene(http) + headers = {"Content-Type": "image/png", "Content-Length": str(LIMIT + 1)} + path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=a.png" + status, body, _ = await asyncio.to_thread(raw, path, headers, SAMPLE[:1024], hang_up=False) + print(f"без токена, Content-Length {LIMIT + 1}: {status} {body['error']['code']}") + assert (status, body["error"]["code"]) == (401, "unauthorized") + + +async def test_slow_but_steady_transfers_outlive_the_request_timeout(http: AsyncClient) -> None: + since = moment() + s = await scene(http) + picture = png(16, 16, size=3 << 20) + sent, read = f"slow-upload-{uuid4().hex}", f"slow-download-{uuid4().hex}" + headers = { + **bearer(s["alice"]), + "Content-Type": "image/png", + "Content-Length": str(len(picture)), + "X-Request-Id": sent, + } + path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=slow.png" + status, body, up = await asyncio.to_thread( + raw, path, headers, picture, hang_up=False, piece=64 << 10, delay=0.25 + ) + assert status == 201, body + assert body["sha256"] == sha256(picture).hexdigest() + content = f"/api/v1/attachments/{body['id']}/content" + code, digest, down = await asyncio.to_thread(slow_read, content, s["bob"], read, 64 << 10, 0.33) + assert (code, digest) == ("200", body["sha256"]) + + def handled() -> dict[str, tuple[int, float]]: + return { + e["request_id"]: (e["status"], e["duration_ms"] / 1000) + for e in logs(since) + if e.get("event") == "request.handled" and e.get("request_id") in (sent, read) + } + + await waited(lambda: len(handled()) == 2, 15) + replica = handled() + print(f"upload 3 MiB ≈ 256 KiB/s: клиент {up:.1f} с, реплика {replica[sent]}") + print(f"скачивание 3 MiB ≈ 192 KiB/s из сети edge: клиент {down} с, реплика {replica[read]}") + assert replica[sent][0] == 201 and replica[sent][1] > REQUEST_TIMEOUT + assert replica[read][0] == 200 and replica[read][1] > REQUEST_TIMEOUT