From 5aa766acffbe9da3195409ec4d5a8e6a99750194 Mon Sep 17 00:00:00 2001 From: Artyom Date: Wed, 28 Oct 2026 20:50:03 +0300 Subject: [PATCH] Run RustFS with a private bucket in the stand MinIO community images can no longer be pulled on this arm64 host, so the stand runs RustFS 1.0.4 pinned by digest as non-root with a read only root, no published port, the console off and data on the s3data volume. A one-off s3-init with rc v0.1.36 creates the bucket, keeps it private for anonymous clients, creates the API user with a policy on the objects and listing of that one bucket, imports a lifecycle rule for incomplete multipart uploads and checks itself; the API replicas start only after it succeeded. Root keys reach only s3, s3-init and the operator s3-cli; the API gets its own key and the S3 settings through an anchor merged into its environment. secrets.sh creates hex S3 keys, the restart hook restarts S3 next to PostgreSQL and Valkey and waits until it is healthy, the limits policy covers the S3 services, and the stack evidence lists the new packages. The object storage adapter is tested against a throwaway RustFS of the same image: a 10 MiB round trip, a missing object, an idempotent delete and an unreachable endpoint failing within the request budget. Co-Authored-By: Claude Opus 5.5 (1M context) --- .env.example | 6 ++ compose.yaml | 85 ++++++++++++++++- infra/s3/api-policy.json | 23 +++++ infra/s3/cli.sh | 12 +++ infra/s3/init.sh | 20 ++++ infra/s3/lifecycle.json | 10 ++ .../add-lab4-image-attachments/tasks.md | 10 +- scripts/evidence-stack.sh | 3 +- scripts/limits.py | 30 +++++- scripts/restart.sh | 2 +- scripts/secrets.sh | 4 + student-tests/adapters/test_s3_storage.py | 95 +++++++++++++++++++ 12 files changed, 289 insertions(+), 11 deletions(-) create mode 100644 infra/s3/api-policy.json create mode 100755 infra/s3/cli.sh create mode 100755 infra/s3/init.sh create mode 100644 infra/s3/lifecycle.json create mode 100644 student-tests/adapters/test_s3_storage.py diff --git a/.env.example b/.env.example index b6d66c7b..7f565633 100644 --- a/.env.example +++ b/.env.example @@ -36,3 +36,9 @@ PG_MEMORY=1g K6_CPUS=2 K6_MEMORY=512m CURSOR_SECRET_FILE=/run/secrets/cursor_secret +# Solution-specific for lab4: S3 client timeouts (connect + read, two attempts stay below +# REQUEST_TIMEOUT_SECONDS=8), parallel image checks per worker, RustFS memory limit. +S3_CONNECT_TIMEOUT_SECONDS=1 +S3_READ_TIMEOUT_SECONDS=2 +IMAGE_CHECK_CONCURRENCY=2 +S3_MEMORY=512m diff --git a/compose.yaml b/compose.yaml index c47342dd..de8050d3 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,6 +2,36 @@ x-app: &app image: ${API_IMAGE:-mtusi-messenger:lab4} networks: [backend] +x-s3-env: &s3-env + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-1} + S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-2} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + IMAGE_CHECK_CONCURRENCY: ${IMAGE_CHECK_CONCURRENCY:-2} + +x-s3-client: &s3-client + image: rustfs/rc:v0.1.36@sha256:ab024bfebee49a750ce886b4c70963ccd9ddaa03f491704a90710641d7a26699 + read_only: true + tmpfs: ["/tmp:size=16m"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + RC_CONFIG_DIR: /tmp/rc + volumes: ["./infra/s3:/s3:ro"] + secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: "no" + networks: [backend] + deploy: + resources: + limits: {cpus: "0.5", memory: 64m} + services: lb: image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b @@ -34,6 +64,7 @@ services: <<: *app build: . environment: + <<: *s3-env DATABASE_URL_FILE: /run/secrets/database_url CACHE_URL_FILE: /run/secrets/cache_url CURSOR_SECRET_FILE: /run/secrets/cursor_secret @@ -47,7 +78,7 @@ services: DRAIN_SECONDS: ${DRAIN_SECONDS:-2} SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5} LOG_LEVEL: ${LOG_LEVEL:-INFO} - secrets: [database_url, cache_url, cursor_secret] + secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key] healthcheck: test: ["CMD", "python", "-c", "import sys,urllib.request;sys.exit(urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status!=200)"] interval: 5s @@ -64,6 +95,8 @@ services: condition: service_completed_successfully valkey: condition: service_started + s3-init: + condition: service_completed_successfully deploy: replicas: ${API_REPLICAS:-2} resources: @@ -152,6 +185,47 @@ services: resources: limits: {cpus: "0.5", memory: 128m} + s3: + image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2 + environment: + RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_root_user + RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_root_password + RUSTFS_ADDRESS: ":9000" + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_REGION: ${S3_REGION:-us-east-1} + RUSTFS_OBS_LOG_DIRECTORY: "" + RUSTFS_OBS_LOGGER_LEVEL: ${S3_LOG_LEVEL:-warn} + secrets: [s3_root_user, s3_root_password] + volumes: [s3data:/data] + read_only: true + tmpfs: ["/tmp:size=64m"] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + healthcheck: + test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:9000/health/ready"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 20s + stop_grace_period: 10s + restart: unless-stopped + networks: [backend] + deploy: + resources: + limits: {cpus: "1", memory: "${S3_MEMORY:-512m}"} + + s3-init: + <<: *s3-client + entrypoint: ["/bin/sh", "/s3/init.sh"] + depends_on: + s3: + condition: service_healthy + + s3-cli: + <<: *s3-client + profiles: [tools] + entrypoint: ["/bin/sh", "/s3/cli.sh"] + k6: image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8 profiles: [bench] @@ -174,10 +248,19 @@ secrets: file: ./secrets/cache_url cursor_secret: file: ./secrets/cursor_secret + s3_root_user: + file: ./secrets/s3_root_user + s3_root_password: + file: ./secrets/s3_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key volumes: pgdata: valkeydata: + s3data: networks: edge: diff --git a/infra/s3/api-policy.json b/infra/s3/api-policy.json new file mode 100644 index 00000000..1e46aa4f --- /dev/null +++ b/infra/s3/api-policy.json @@ -0,0 +1,23 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "MessengerApiObjects", + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + }, + { + "Sid": "MessengerApiBucket", + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/infra/s3/cli.sh b/infra/s3/cli.sh new file mode 100755 index 00000000..9e0937f9 --- /dev/null +++ b/infra/s3/cli.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +secret() { tr -d '\r\n' < "/run/secrets/$1"; } +endpoint="${S3_ENDPOINT:-http://s3:9000}" +RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}" +RC_HOST_api="${endpoint%%://*}://$(secret s3_access_key_id):$(secret s3_secret_access_key)@${endpoint#*://}" +export RC_HOST_root RC_HOST_api +rc alias set anon "$endpoint" --anonymous >/dev/null +case "${1:-}" in + sh|wget) exec "$@" ;; +esac +exec rc "$@" diff --git a/infra/s3/init.sh b/infra/s3/init.sh new file mode 100755 index 00000000..3686b1c6 --- /dev/null +++ b/infra/s3/init.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +secret() { tr -d '\r\n' < "/run/secrets/$1"; } +bucket="${S3_BUCKET:-messenger-attachments}" +endpoint="${S3_ENDPOINT:-http://s3:9000}" +policy="${S3_API_POLICY:-messenger-api}" +RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}" +export RC_HOST_root +api_key=$(secret s3_access_key_id) +rc ready root --timeout 10 >/dev/null +rc bucket create --ignore-existing "root/$bucket" >/dev/null +rc bucket anonymous set private "root/$bucket" >/dev/null +sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.json +rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null +rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null +rc admin policy attach root "$policy" --user "$api_key" >/dev/null +rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null +rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"' +rc admin user info root "$api_key" --json | grep -q "\"$policy\"" +echo "s3-init: bucket=$bucket anonymous=private policy=$policy lifecycle=abort-incomplete-uploads" diff --git a/infra/s3/lifecycle.json b/infra/s3/lifecycle.json new file mode 100644 index 00000000..ab206a07 --- /dev/null +++ b/infra/s3/lifecycle.json @@ -0,0 +1,10 @@ +{ + "rules": [ + { + "id": "abort-incomplete-uploads", + "status": "Enabled", + "prefix": "", + "abortIncompleteMultipartUploadDays": 1 + } + ] +} diff --git a/openspec/changes/add-lab4-image-attachments/tasks.md b/openspec/changes/add-lab4-image-attachments/tasks.md index c71a7ae8..42b21066 100644 --- a/openspec/changes/add-lab4-image-attachments/tasks.md +++ b/openspec/changes/add-lab4-image-attachments/tasks.md @@ -16,7 +16,7 @@ - [x] 3.2 Адаптер `PillowImageInspector`: открытие только PNG/JPEG, подмена MIME, пиксели по заголовку, PNG `verify`, JPEG `draft` + `load`, `to_thread` под семафором - [x] 3.3 Адаптер `S3ObjectStorage` на aiobotocore (path-style, SigV4, таймауты, checksum `when_required`), отображение ошибок в `ObjectMissing`/`Unavailable`; зависимости aiobotocore и Pillow в `uv.lock` - [x] 3.4 Настройки зоны (`S3_*`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS`, таймауты, параллелизм проверки), провайдер dishka, схемы `AttachmentOut`, роутер upload/метаданных/скачивания, реестр ошибок, журнал -- [ ] 3.5 Проверка группы: тест адаптера S3 против одноразового RustFS (put/open/delete, пропавший объект, недоступный endpoint → `Unavailable` за ограниченное время) +- [x] 3.5 Проверка группы: тест адаптера S3 против одноразового RustFS (put/open/delete, пропавший объект, недоступный endpoint → `Unavailable` за ограниченное время) ## 4. Связь зон и платформа @@ -38,10 +38,10 @@ ## 7. Стенд -- [ ] 7.1 `infra/s3/` (`init.sh`, `cli.sh`, `api-policy.json`, `lifecycle.json`) из заготовки -- [ ] 7.2 `compose.yaml`: `s3`, `s3-init`, `s3-cli`, volume `s3data`, секреты S3, якорь `x-s3-env` у `api`, зависимость от `s3-init`; `.env.example` (таймауты S3, параллелизм проверки) -- [ ] 7.3 `scripts/secrets.sh` (секреты S3), `scripts/restart.sh` (S3 в списке хранилищ), `scripts/evidence-stack.sh` (пакеты и образы лабы 4) -- [ ] 7.4 Проверка группы: `docker compose config --quiet`, у `api` только ключ API S3, `make up` из чистого состояния, все сервисы `healthy`, `s3-init` завершился кодом 0, опубликованы только порты `lb` +- [x] 7.1 `infra/s3/` (`init.sh`, `cli.sh`, `api-policy.json`, `lifecycle.json`) из заготовки +- [x] 7.2 `compose.yaml`: `s3`, `s3-init`, `s3-cli`, volume `s3data`, секреты S3, якорь `x-s3-env` у `api`, зависимость от `s3-init`; `.env.example` (таймауты S3, параллелизм проверки) +- [x] 7.3 `scripts/secrets.sh` (секреты S3), `scripts/restart.sh` (S3 в списке хранилищ), `scripts/evidence-stack.sh` (пакеты и образы лабы 4) +- [x] 7.4 Проверка группы: `docker compose config --quiet`, у `api` только ключ API S3, `make up` из чистого состояния, все сервисы `healthy`, `s3-init` завершился кодом 0, опубликованы только порты `lb` ## 8. Сценарии отказов и обвязка diff --git a/scripts/evidence-stack.sh b/scripts/evidence-stack.sh index 8d33f590..0e5575e2 100755 --- a/scripts/evidence-stack.sh +++ b/scripts/evidence-stack.sh @@ -35,8 +35,9 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()): grep -h -E 'image:|@sha256' compose.yaml Dockerfile | sed 's/^ */ /' echo "python in image: $(docker exec "$(docker compose ps -q api | head -1)" python -V)" echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)" + echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)" echo "packages (uv.lock):" - for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis; do + for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow; do printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)" done echo "process users (service uid command):" diff --git a/scripts/limits.py b/scripts/limits.py index 2547ea0c..c487ef6f 100644 --- a/scripts/limits.py +++ b/scripts/limits.py @@ -7,10 +7,17 @@ import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] +S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"] POLICY = { "lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True}, "api": { - "secrets": ["cache_url", "cursor_secret", "database_url"], + "secrets": [ + "cache_url", + "cursor_secret", + "database_url", + "s3_access_key_id", + "s3_secret_access_key", + ], "volumes": [], "networks": ["backend"], }, @@ -18,6 +25,13 @@ POLICY = { "postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]}, "valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]}, "k6": {"secrets": [], "volumes": [], "networks": ["edge"]}, + "s3": { + "secrets": ["s3_root_password", "s3_root_user"], + "volumes": ["s3data"], + "networks": ["backend"], + }, + "s3-init": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]}, + "s3-cli": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]}, } @@ -99,7 +113,17 @@ def main(): parser.add_argument("--out", help="text report, written only after a complete check") args = parser.parse_args() problems = [] - _, raw = run("docker", "compose", "--profile", "bench", "config", "--format", "json") + _, raw = run( + "docker", + "compose", + "--profile", + "bench", + "--profile", + "tools", + "config", + "--format", + "json", + ) config = json.loads(raw) services = [ declared(name, service, config.get("networks", {}), problems) @@ -110,7 +134,7 @@ def main(): problems.append("no running api replicas to inspect") lines = [ f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}", - "$ docker compose --profile bench config --format json (declared)", + "$ docker compose --profile bench --profile tools config --format json (declared)", *table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services), "", "$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)", diff --git a/scripts/restart.sh b/scripts/restart.sh index 47645b40..c78416db 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -3,7 +3,7 @@ set -eu cd "$(dirname "$0")/.." defined=$(docker compose config --services) stores="" -for service in ${STATE_SERVICES:-postgres valkey}; do +for service in ${STATE_SERVICES:-postgres valkey s3}; do if printf '%s\n' "$defined" | grep -qx "$service"; then stores="$stores $service"; fi done [ -n "$stores" ] || { echo "no state services found" >&2; exit 3; } diff --git a/scripts/secrets.sh b/scripts/secrets.sh index 53997e3c..bdc67ec5 100755 --- a/scripts/secrets.sh +++ b/scripts/secrets.sh @@ -17,4 +17,8 @@ put cursor_secret "$(random)" put valkey_password "$(random)" put valkey.conf "requirepass $(cat secrets/valkey_password)" put cache_url "redis://:$(cat secrets/valkey_password)@valkey:6379/0" +put s3_root_user "root$(random | cut -c1-16)" +put s3_root_password "$(random)" +put s3_access_key_id "api$(random | cut -c1-16)" +put s3_secret_access_key "$(random)" [ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; } diff --git a/student-tests/adapters/test_s3_storage.py b/student-tests/adapters/test_s3_storage.py new file mode 100644 index 00000000..e4e631c3 --- /dev/null +++ b/student-tests/adapters/test_s3_storage.py @@ -0,0 +1,95 @@ +import os +import re +import subprocess +import time +from collections.abc import AsyncIterator, Iterator +from contextlib import asynccontextmanager +from uuid import uuid4 + +import httpx +import pytest + +from conftest import STARTUP_SECONDS, container, image, published +from messenger.attachments.application.errors import ObjectMissingError +from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client +from messenger.shared.application.errors import Unavailable + +RUSTFS_IMAGE = re.compile(r"image: (rustfs/rustfs:\S+)") +ROOT_KEY, ROOT_SECRET = "testroot", "testroot-secret-key" +BUCKET = "messenger-attachments" + + +@pytest.fixture(scope="session") +def rustfs() -> Iterator[str]: + name = f"mtusi-lab4-test-rustfs-{uuid4().hex[:8]}" + keys = ["-e", f"RUSTFS_ACCESS_KEY={ROOT_KEY}", "-e", f"RUSTFS_SECRET_KEY={ROOT_SECRET}"] + container(name, *keys, "-p", "127.0.0.1::9000", image(RUSTFS_IMAGE)) + try: + endpoint = f"http://127.0.0.1:{published(name, 9000)}" + deadline = time.monotonic() + STARTUP_SECONDS + while True: + try: + if httpx.get(f"{endpoint}/health/ready", timeout=1).status_code == 200: + break + except httpx.TransportError: + pass + assert time.monotonic() < deadline, "RustFS не стал готов" + time.sleep(0.3) + yield endpoint + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + + +@asynccontextmanager +async def storage(endpoint: str) -> AsyncIterator[S3ObjectStorage]: + async with s3_client( + endpoint=endpoint, + region="us-east-1", + access_key_id=ROOT_KEY, + secret_access_key=ROOT_SECRET, + connect_timeout=1, + read_timeout=2, + max_attempts=2, + ) as client: + buckets = await client.list_buckets() + if BUCKET not in {bucket["Name"] for bucket in buckets.get("Buckets", [])}: + await client.create_bucket(Bucket=BUCKET) + yield S3ObjectStorage(client, BUCKET) + + +async def test_round_trip_of_the_largest_object(rustfs: str) -> None: + data = os.urandom(10_485_760) + key = f"attachments/{uuid4()}" + async with storage(rustfs) as objects: + await objects.put(key, data, "image/png") + content = await objects.open(key) + assert content.size == len(data) + assert b"".join([chunk async for chunk in content.chunks]) == data + await objects.delete(key) + with pytest.raises(ObjectMissingError): + await objects.open(key) + await objects.delete(key) + + +async def test_unreachable_endpoint_is_unavailable_in_bounded_time() -> None: + started = time.monotonic() + async with storage_without_bucket("http://127.0.0.1:1") as objects: + with pytest.raises(Unavailable): + await objects.put("attachments/x", b"x", "image/png") + with pytest.raises(Unavailable): + await objects.open("attachments/x") + assert time.monotonic() - started < 8 + + +@asynccontextmanager +async def storage_without_bucket(endpoint: str) -> AsyncIterator[S3ObjectStorage]: + async with s3_client( + endpoint=endpoint, + region="us-east-1", + access_key_id=ROOT_KEY, + secret_access_key=ROOT_SECRET, + connect_timeout=1, + read_timeout=2, + max_attempts=2, + ) as client: + yield S3ObjectStorage(client, BUCKET)