Split the S3 keys of the API and the reconciler, drop root from one-offs

The API key now has only GetObject, PutObject and DeleteObject on the
attachment objects: the server never lists the bucket, never runs
multipart uploads and has its region set. The reconciler gets its own
user messenger-reconciler with ListBucket, GetObject and DeleteObject,
created by s3-init from a secret of its own, so the API replicas can no
longer list the bucket. The cost seeding writes objects and therefore
runs in a one-off API container under the API key. s3-check proves the
API key cannot list and checks the reconciler key; its snapshot drops
dated fields with a JSON filter instead of grep, which left a dangling
comma, and its first line carries the date and the commit.

s3-init and s3-cli run as uid 65534, and migrate and reconciler get an
init process, so SIGTERM reaches python and docker stop ends a pass in
a fraction of a second instead of waiting for SIGKILL. limits.py and CI
check the users of the one-off services. Stack tests move a 3 MiB file
up and down through the balancer at 256 and 192 KiB/s, longer than the
request timeout, and check 401 before the declared size.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
This commit is contained in:
Artyom
2026-10-30 08:53:39 +03:00
co-authored by Claude Opus 5.5
parent 54099e1da3
commit 2891b37e75
13 changed files with 234 additions and 48 deletions
+4 -1
View File
@@ -50,9 +50,12 @@ UPLOAD_SWEEP_INTERVAL_SECONDS=30
UPLOAD_SWEEP_BATCH=100
# Fixed glibc mmap threshold for the API: buffers above 128 KiB are returned to the system on free.
MALLOC_MMAP_THRESHOLD_=131072
# Minimal client transfer rate, bytes/s: every body byte moved in either direction extends the
# request deadline by 1/rate, so slow but steady uploads and downloads outlive REQUEST_TIMEOUT_SECONDS.
MIN_TRANSFER_RATE_BYTES=131072
# Solution-specific for lab4 track 2: limits of the one-off reconciler (profile reconcile), its S3
# read timeout for listing pages, optional CPU pinning of the measured services (empty = no pinning)
# and the CPU limit of RustFS (make reconcile-cost raises it with its memory for 100k objects).
# and the CPU limit of RustFS (make reconcile-cost raises it with its memory for the 100k point).
RECONCILE_CPUS=1
RECONCILE_MEMORY=256m
RECONCILE_S3_READ_TIMEOUT_SECONDS=10
+8
View File
@@ -79,6 +79,14 @@ jobs:
docker top "$id" -eo pid,uid,comm | awk 'NR>1 && $2==0 {print; bad=1} END {exit bad}' || exit 1
done
- name: Одноразовые сервисы не от root
run: |
for service in migrate s3-init; do
user=$(docker inspect -f '{{.Config.User}}' "$(docker compose ps -aq "$service")")
echo "$service: $user"
case "${user%%:*}" in ""|0|root) exit 1 ;; esac
done
- name: Корректная остановка реплик API по SIGTERM
run: |
start=$(date +%s)
+18 -2
View File
@@ -26,8 +26,15 @@ x-s3-client: &s3-client
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
RC_CONFIG_DIR: /tmp/rc
user: "65534:65534"
volumes: ["./infra/s3:/s3:ro"]
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
secrets:
- s3_root_user
- s3_root_password
- s3_access_key_id
- s3_secret_access_key
- s3_reconciler_access_key_id
- s3_reconciler_secret_access_key
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
restart: "no"
@@ -78,6 +85,7 @@ services:
WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS:-8}
MIN_TRANSFER_RATE_BYTES: ${MIN_TRANSFER_RATE_BYTES:-131072}
DB_ACQUIRE_TIMEOUT_SECONDS: ${DB_ACQUIRE_TIMEOUT_SECONDS:-1}
DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-2}
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
@@ -114,6 +122,7 @@ services:
migrate:
<<: *app
pull_policy: never
init: true
command: ["python", "-m", "messenger.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
@@ -230,14 +239,17 @@ services:
<<: *app
pull_policy: never
profiles: [reconcile]
init: true
entrypoint: ["python", "-m", "messenger.attachments.entrypoints.reconcile"]
command: ["--dry-run"]
environment:
<<: *s3-env
DATABASE_URL_FILE: /run/secrets/database_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key
S3_READ_TIMEOUT_SECONDS: ${RECONCILE_S3_READ_TIMEOUT_SECONDS:-10}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
secrets: [database_url, s3_reconciler_access_key_id, s3_reconciler_secret_access_key]
healthcheck:
disable: true
restart: "no"
@@ -294,6 +306,10 @@ secrets:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
s3_reconciler_access_key_id:
file: ./secrets/s3_reconciler_access_key_id
s3_reconciler_secret_access_key:
file: ./secrets/s3_reconciler_secret_access_key
volumes:
pgdata:
+1 -13
View File
@@ -4,20 +4,8 @@
{
"Sid": "MessengerApiObjects",
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts"
],
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
},
{
"Sid": "MessengerApiBucket",
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}
]
}
+2 -1
View File
@@ -4,7 +4,8 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; }
endpoint="${S3_ENDPOINT:-http://s3:9000}"
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
RC_HOST_api="${endpoint%%://*}://$(secret s3_access_key_id):$(secret s3_secret_access_key)@${endpoint#*://}"
export RC_HOST_root RC_HOST_api
RC_HOST_reconciler="${endpoint%%://*}://$(secret s3_reconciler_access_key_id):$(secret s3_reconciler_secret_access_key)@${endpoint#*://}"
export RC_HOST_root RC_HOST_api RC_HOST_reconciler
rc alias set anon "$endpoint" --anonymous >/dev/null
case "${1:-}" in
sh|wget) exec "$@" ;;
+11 -7
View File
@@ -4,17 +4,21 @@ secret() { tr -d '\r\n' < "/run/secrets/$1"; }
bucket="${S3_BUCKET:-messenger-attachments}"
endpoint="${S3_ENDPOINT:-http://s3:9000}"
policy="${S3_API_POLICY:-messenger-api}"
reconciler="${S3_RECONCILER_POLICY:-messenger-reconciler}"
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
export RC_HOST_root
api_key=$(secret s3_access_key_id)
grant() {
sed "s/messenger-attachments/$bucket/g" "/s3/$2" > "/tmp/$2"
rc admin policy create root "$1" "/tmp/$2" >/dev/null
rc admin user add root "$(secret "$3")" "$(secret "$4")" >/dev/null
rc admin policy attach root "$1" --user "$(secret "$3")" >/dev/null
rc admin user info root "$(secret "$3")" --json | grep -q "\"$1\""
}
rc ready root --timeout 10 >/dev/null
rc bucket create --ignore-existing "root/$bucket" >/dev/null
rc bucket anonymous set private "root/$bucket" >/dev/null
sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.json
rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null
rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null
rc admin policy attach root "$policy" --user "$api_key" >/dev/null
grant "$policy" api-policy.json s3_access_key_id s3_secret_access_key
grant "$reconciler" reconciler-policy.json s3_reconciler_access_key_id s3_reconciler_secret_access_key
rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null
rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"'
rc admin user info root "$api_key" --json | grep -q "\"$policy\""
echo "s3-init: bucket=$bucket anonymous=private policy=$policy lifecycle=abort-incomplete-uploads"
echo "s3-init: bucket=$bucket anonymous=private policies=$policy,$reconciler lifecycle=abort-incomplete-uploads"
+17
View File
@@ -0,0 +1,17 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "MessengerReconcilerObjects",
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
},
{
"Sid": "MessengerReconcilerListing",
"Effect": "Allow",
"Action": ["s3:ListBucket"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}
]
}
@@ -1,18 +1,18 @@
## 1. Платформа
- [ ] 1.1 Конвейер: guard раньше проверки объявленного `Content-Length`
- [ ] 1.2 Конвейер: дедлайн с минимальной скоростью передачи для `receive`/`send` приложения; `MIN_TRANSFER_RATE_BYTES` в `TransportSettings`, `main.py`, `compose.yaml`, `.env.example`
- [ ] 1.3 Проверка группы: `make lint`; API-тесты: upload без токена с `Content-Length` больше лимита → `401`; медленный идущий upload и скачивание дольше таймаута → `201`/`200`; тело ползёт медленнее порога → `503`
- [x] 1.1 Конвейер: guard раньше проверки объявленного `Content-Length`
- [x] 1.2 Конвейер: дедлайн с минимальной скоростью передачи для `receive`/`send` приложения; `MIN_TRANSFER_RATE_BYTES` в `TransportSettings`, `main.py`, `compose.yaml`, `.env.example`
- [x] 1.3 Проверка группы: `make lint`; API-тесты: upload без токена с `Content-Length` больше лимита → `401`; медленный идущий upload и скачивание дольше таймаута → `201`/`200`; тело ползёт медленнее порога → `503`
## 2. Стенд
- [ ] 2.1 `infra/s3/api-policy.json` без multipart, `GetBucketLocation` и `ListBucket`; `infra/s3/reconciler-policy.json`; `init.sh` создаёт обоих пользователей; `cli.sh` — alias `reconciler`
- [ ] 2.2 `scripts/secrets.sh`: ключ сверки; `compose.yaml`: secrets сверки у `reconciler`, `s3-init`, `s3-cli`; `user` у `x-s3-client`; `init: true` у `migrate` и `reconciler`
- [ ] 2.3 `scripts/limits.py`: политика секретов; `scripts/s3-check.sh`: запрет листинга ключом API, права ключа сверки; `scripts/reconcile_track.py`: наполнение под ключом API
- [ ] 2.4 CI: uid одноразовых сервисов
- [ ] 2.5 Проверка группы: стенд из чистого состояния, `make test`, `make s3-check`, `make limits`, `make reconcile`, `docker stop` прохода сверки
- [x] 2.1 `infra/s3/api-policy.json` без multipart, `GetBucketLocation` и `ListBucket`; `infra/s3/reconciler-policy.json`; `init.sh` создаёт обоих пользователей; `cli.sh` — alias `reconciler`
- [x] 2.2 `scripts/secrets.sh`: ключ сверки; `compose.yaml`: secrets сверки у `reconciler`, `s3-init`, `s3-cli`; `user` у `x-s3-client`; `init: true` у `migrate` и `reconciler`
- [x] 2.3 `scripts/limits.py`: политика секретов; `scripts/s3-check.sh`: запрет листинга ключом API, права ключа сверки; `scripts/reconcile_track.py`: наполнение под ключом API
- [x] 2.4 CI: uid одноразовых сервисов
- [x] 2.5 Проверка группы: стенд из чистого состояния, `make test`, `make s3-check`, `make limits`, `make reconcile`, `docker stop` прохода сверки
## 3. Стенд: тесты
- [ ] 3.1 Тесты стенда: медленный upload и медленное скачивание 3 MiB через LB, upload без токена с большим `Content-Length` → `401`
- [ ] 3.2 Проверка группы: `make student-test`, `make test-faults`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, `make check`
- [x] 3.1 Тесты стенда: медленный upload и медленное скачивание 3 MiB через LB, upload без токена с большим `Content-Length` → `401`
- [x] 3.2 Проверка группы: `make student-test`, `make test-faults`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, `make check`
+23 -3
View File
@@ -7,7 +7,20 @@ import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"]
S3_CLIENT_SECRETS = [
"s3_access_key_id",
"s3_reconciler_access_key_id",
"s3_reconciler_secret_access_key",
"s3_root_password",
"s3_root_user",
"s3_secret_access_key",
]
RECONCILER_SECRETS = [
"database_url",
"s3_reconciler_access_key_id",
"s3_reconciler_secret_access_key",
]
OWN_USER = ("postgres", "valkey", "s3-init", "s3-cli")
POLICY = {
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
"api": {
@@ -23,7 +36,7 @@ POLICY = {
},
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
"reconciler": {
"secrets": ["database_url", "s3_access_key_id", "s3_secret_access_key"],
"secrets": RECONCILER_SECRETS,
"volumes": [],
"networks": ["backend"],
},
@@ -70,6 +83,9 @@ def declared(name, service, networks, problems):
problems.append(f"{name}: publishes {ports} beyond loopback")
if name == "api" and not service.get("read_only"):
problems.append("api: root filesystem is writable")
user = str(service.get("user", ""))
if name in OWN_USER and user.split(":", maxsplit=1)[0] in ("", "0", "root"):
problems.append(f"{name}: runs as root")
internal = [n for n in nets if networks.get(n, {}).get("internal")]
return [
name,
@@ -79,6 +95,7 @@ def declared(name, service, networks, problems):
",".join(volumes) or "-",
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
str(service.get("read_only", False)).lower(),
user or "image",
]
@@ -144,7 +161,10 @@ def main():
f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}",
"$ docker compose --profile bench --profile tools --profile reconcile config --format json"
" (declared)",
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
*table(
["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro", "user"],
services,
),
"",
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
*table(["service", "hostname", "cpus", "memory", "published"], containers),
+2 -2
View File
@@ -558,7 +558,7 @@ def measure(flags, env, args, pinned):
def seed(count, env):
if count <= 0:
return {"added": 0}
env = dict(env, RECONCILE_CPUSET="", RECONCILE_CPUS="4", RECONCILE_MEMORY="512m")
env = dict(env, API_CPUS="4", API_MEMORY="512m")
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
out = compose(
"run",
@@ -569,7 +569,7 @@ def seed(count, env):
tools,
"--entrypoint",
"python",
"reconciler",
"api",
"/tools/seed_storage.py",
"--count",
str(count),
+30 -7
View File
@@ -2,17 +2,30 @@
set -eu
cd "$(dirname "$0")/.."
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
undated() {
python3 -c 'import json, sys
def clean(value):
if isinstance(value, dict):
return {k: clean(v) for k, v in value.items() if not k.endswith("_date")}
return [clean(v) for v in value] if isinstance(value, list) else value
text, decoder, at = sys.stdin.read(), json.JSONDecoder(), 0
while text[at:].strip():
at += len(text[at:]) - len(text[at:].lstrip())
value, at = decoder.raw_decode(text, at)
print(json.dumps(clean(value), indent=2, ensure_ascii=False))'
}
snapshot() {
cli sh -s <<'EOF' | grep -v '_date'
cli sh -s <<'EOF' | undated
ak=$(cat /run/secrets/s3_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
rk=$(cat /run/secrets/s3_reconciler_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin policy info root messenger-reconciler" "admin user info root $ak" "admin user info root $rk" "bucket lifecycle rule export root/$S3_BUCKET"; do
rc $c --json
done | sed "s/$ak/<api-key>/g"
done | sed "s/$ak/<api-key>/g; s/$rk/<reconciler-key>/g"
EOF
}
failed=0
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD) == S3 privacy and bootstrap checks"
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
before=$(snapshot)
@@ -25,22 +38,32 @@ verdict "s3-init: состояние IAM/bucket не изменилось" "same
cli sh -s <<'EOF' || failed=1
b=$S3_BUCKET
key="attachments/s3-check-$(date +%s)-$$"
spare="$key-reconciler"
other="s3-check-$(date +%s)-$$"
head -c 4096 /dev/urandom > /tmp/probe
result=0
expect() {
want=$1; name=$2; shift 2
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error|check bucket: Network error: service error' /tmp/out; then got=deny; else got=error; fi
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
}
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
expect allow "api: HeadObject" rc object stat "api/$b/$key"
expect allow "api: GetObject" rc object show "api/$b/$key"
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
expect deny "api: ListBucket" rc object list "api/$b/attachments/"
rc put /tmp/probe "api/$b/$spare" >/dev/null
expect allow "reconciler: ListBucket" rc object list "reconciler/$b/attachments/"
expect allow "reconciler: HeadObject" rc object stat "reconciler/$b/$key"
expect allow "reconciler: GetObject" rc object show "reconciler/$b/$key"
expect deny "reconciler: PutObject" rc put /tmp/probe "reconciler/$b/$key-put"
expect allow "reconciler: DeleteObject" rc object remove --force "reconciler/$b/$spare"
expect deny "reconciler: CreateBucket" rc bucket create "reconciler/$other"
expect deny "reconciler: PutBucketPolicy (public)" rc bucket anonymous set public "reconciler/$b"
expect deny "reconciler: admin ListUsers" rc admin user list reconciler
expect deny "api: CreateBucket" rc bucket create "api/$other"
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: PutBucketLifecycle (HeadBucket уже 403)" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: admin ListUsers" rc admin user list api
rc bucket create --ignore-existing "root/$other" >/dev/null
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
+2
View File
@@ -21,4 +21,6 @@ put s3_root_user "root$(random | cut -c1-16)"
put s3_root_password "$(random)"
put s3_access_key_id "api$(random | cut -c1-16)"
put s3_secret_access_key "$(random)"
put s3_reconciler_access_key_id "rec$(random | cut -c1-16)"
put s3_reconciler_secret_access_key "$(random)"
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
+106 -2
View File
@@ -134,14 +134,23 @@ def replica_at(address: str) -> str:
def raw(
path: str, headers: dict[str, str], body: bytes, *, hang_up: bool
path: str,
headers: dict[str, str],
body: bytes,
*,
hang_up: bool,
piece: int = 0,
delay: float = 0.0,
) -> tuple[int, dict[str, Any], float]:
parts = urlsplit(BASE_URL)
head = f"POST {path} HTTP/1.1\r\nHost: {parts.netloc}\r\nConnection: close\r\n"
head += "".join(f"{name}: {value}\r\n" for name, value in headers.items()) + "\r\n"
started = time.perf_counter()
with socket.create_connection((parts.hostname, parts.port or 80), timeout=30) as sock:
sock.sendall(head.encode("latin-1") + body)
sock.sendall(head.encode("latin-1"))
for start in range(0, len(body), piece or max(len(body), 1)):
sock.sendall(body[start : start + (piece or len(body))])
time.sleep(delay)
if hang_up:
return 0, {}, 0.0
response = HTTPResponse(sock)
@@ -149,6 +158,56 @@ def raw(
return response.status, json.loads(response.read()), time.perf_counter() - started
READER = """
import hashlib, http.client, sys, time
path, token, request_id, piece, delay = sys.argv[1:6]
connection = http.client.HTTPConnection("lb", 8080, timeout=30)
headers = {"Authorization": "Bearer " + token, "X-Request-Id": request_id}
connection.request("GET", path, headers=headers)
response, digest, started = connection.getresponse(), hashlib.sha256(), time.monotonic()
while chunk := response.read(int(piece)):
digest.update(chunk)
time.sleep(float(delay))
print(response.status, digest.hexdigest(), round(time.monotonic() - started, 2))
"""
def slow_read(path: str, token: str, request_id: str, piece: int, delay: float) -> list[str]:
template = "{{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}"
networks = subprocess.run(
["docker", "inspect", "-f", template, compose("ps", "-q", "lb")],
check=True,
capture_output=True,
text=True,
).stdout.split()
image = subprocess.run(
["docker", "inspect", "-f", "{{.Config.Image}}", compose("ps", "-q", "api").split()[0]],
check=True,
capture_output=True,
text=True,
).stdout.strip()
command = [
"docker",
"run",
"--rm",
"--network",
next(name for name in networks if name.endswith("_edge")),
"--sysctl",
"net.ipv4.tcp_rmem=4096 16384 65536",
"--entrypoint",
"python",
image,
"-c",
READER,
path,
token,
request_id,
str(piece),
str(delay),
]
return subprocess.run(command, check=True, capture_output=True, text=True).stdout.split()
@pytest.fixture
async def http() -> AsyncIterator[AsyncClient]:
async with AsyncClient(base_url=BASE_URL, timeout=30, trust_env=False) as client:
@@ -466,3 +525,48 @@ async def test_stalled_upload_is_cut_and_leaves_nothing(http: AsyncClient, fault
assert elapsed <= REQUEST_TIMEOUT + 2
assert seen(name) is None
assert (await upload(http, s["alice"], s["chats"][0])).status_code == 201
async def test_token_is_checked_before_the_declared_size(http: AsyncClient) -> None:
s = await scene(http)
headers = {"Content-Type": "image/png", "Content-Length": str(LIMIT + 1)}
path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=a.png"
status, body, _ = await asyncio.to_thread(raw, path, headers, SAMPLE[:1024], hang_up=False)
print(f"без токена, Content-Length {LIMIT + 1}: {status} {body['error']['code']}")
assert (status, body["error"]["code"]) == (401, "unauthorized")
async def test_slow_but_steady_transfers_outlive_the_request_timeout(http: AsyncClient) -> None:
since = moment()
s = await scene(http)
picture = png(16, 16, size=3 << 20)
sent, read = f"slow-upload-{uuid4().hex}", f"slow-download-{uuid4().hex}"
headers = {
**bearer(s["alice"]),
"Content-Type": "image/png",
"Content-Length": str(len(picture)),
"X-Request-Id": sent,
}
path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=slow.png"
status, body, up = await asyncio.to_thread(
raw, path, headers, picture, hang_up=False, piece=64 << 10, delay=0.25
)
assert status == 201, body
assert body["sha256"] == sha256(picture).hexdigest()
content = f"/api/v1/attachments/{body['id']}/content"
code, digest, down = await asyncio.to_thread(slow_read, content, s["bob"], read, 64 << 10, 0.33)
assert (code, digest) == ("200", body["sha256"])
def handled() -> dict[str, tuple[int, float]]:
return {
e["request_id"]: (e["status"], e["duration_ms"] / 1000)
for e in logs(since)
if e.get("event") == "request.handled" and e.get("request_id") in (sent, read)
}
await waited(lambda: len(handled()) == 2, 15)
replica = handled()
print(f"upload 3 MiB ≈ 256 KiB/s: клиент {up:.1f} с, реплика {replica[sent]}")
print(f"скачивание 3 MiB ≈ 192 KiB/s из сети edge: клиент {down} с, реплика {replica[read]}")
assert replica[sent][0] == 201 and replica[sent][1] > REQUEST_TIMEOUT
assert replica[read][0] == 200 and replica[read][1] > REQUEST_TIMEOUT