diff --git a/.env.example b/.env.example index b6d66c7b..7f565633 100644 --- a/.env.example +++ b/.env.example @@ -36,3 +36,9 @@ PG_MEMORY=1g K6_CPUS=2 K6_MEMORY=512m CURSOR_SECRET_FILE=/run/secrets/cursor_secret +# Solution-specific for lab4: S3 client timeouts (connect + read, two attempts stay below +# REQUEST_TIMEOUT_SECONDS=8), parallel image checks per worker, RustFS memory limit. +S3_CONNECT_TIMEOUT_SECONDS=1 +S3_READ_TIMEOUT_SECONDS=2 +IMAGE_CHECK_CONCURRENCY=2 +S3_MEMORY=512m diff --git a/compose.yaml b/compose.yaml index c47342dd..de8050d3 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,6 +2,36 @@ x-app: &app image: ${API_IMAGE:-mtusi-messenger:lab4} networks: [backend] +x-s3-env: &s3-env + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-1} + S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-2} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + IMAGE_CHECK_CONCURRENCY: ${IMAGE_CHECK_CONCURRENCY:-2} + +x-s3-client: &s3-client + image: rustfs/rc:v0.1.36@sha256:ab024bfebee49a750ce886b4c70963ccd9ddaa03f491704a90710641d7a26699 + read_only: true + tmpfs: ["/tmp:size=16m"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + RC_CONFIG_DIR: /tmp/rc + volumes: ["./infra/s3:/s3:ro"] + secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: "no" + networks: [backend] + deploy: + resources: + limits: {cpus: "0.5", memory: 64m} + services: lb: image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b @@ -34,6 +64,7 @@ services: <<: *app build: . environment: + <<: *s3-env DATABASE_URL_FILE: /run/secrets/database_url CACHE_URL_FILE: /run/secrets/cache_url CURSOR_SECRET_FILE: /run/secrets/cursor_secret @@ -47,7 +78,7 @@ services: DRAIN_SECONDS: ${DRAIN_SECONDS:-2} SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5} LOG_LEVEL: ${LOG_LEVEL:-INFO} - secrets: [database_url, cache_url, cursor_secret] + secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key] healthcheck: test: ["CMD", "python", "-c", "import sys,urllib.request;sys.exit(urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status!=200)"] interval: 5s @@ -64,6 +95,8 @@ services: condition: service_completed_successfully valkey: condition: service_started + s3-init: + condition: service_completed_successfully deploy: replicas: ${API_REPLICAS:-2} resources: @@ -152,6 +185,47 @@ services: resources: limits: {cpus: "0.5", memory: 128m} + s3: + image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2 + environment: + RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_root_user + RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_root_password + RUSTFS_ADDRESS: ":9000" + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_REGION: ${S3_REGION:-us-east-1} + RUSTFS_OBS_LOG_DIRECTORY: "" + RUSTFS_OBS_LOGGER_LEVEL: ${S3_LOG_LEVEL:-warn} + secrets: [s3_root_user, s3_root_password] + volumes: [s3data:/data] + read_only: true + tmpfs: ["/tmp:size=64m"] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + healthcheck: + test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:9000/health/ready"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 20s + stop_grace_period: 10s + restart: unless-stopped + networks: [backend] + deploy: + resources: + limits: {cpus: "1", memory: "${S3_MEMORY:-512m}"} + + s3-init: + <<: *s3-client + entrypoint: ["/bin/sh", "/s3/init.sh"] + depends_on: + s3: + condition: service_healthy + + s3-cli: + <<: *s3-client + profiles: [tools] + entrypoint: ["/bin/sh", "/s3/cli.sh"] + k6: image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8 profiles: [bench] @@ -174,10 +248,19 @@ secrets: file: ./secrets/cache_url cursor_secret: file: ./secrets/cursor_secret + s3_root_user: + file: ./secrets/s3_root_user + s3_root_password: + file: ./secrets/s3_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key volumes: pgdata: valkeydata: + s3data: networks: edge: diff --git a/infra/s3/api-policy.json b/infra/s3/api-policy.json new file mode 100644 index 00000000..1e46aa4f --- /dev/null +++ b/infra/s3/api-policy.json @@ -0,0 +1,23 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "MessengerApiObjects", + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + }, + { + "Sid": "MessengerApiBucket", + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/infra/s3/cli.sh b/infra/s3/cli.sh new file mode 100755 index 00000000..9e0937f9 --- /dev/null +++ b/infra/s3/cli.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu +secret() { tr -d '\r\n' < "/run/secrets/$1"; } +endpoint="${S3_ENDPOINT:-http://s3:9000}" +RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}" +RC_HOST_api="${endpoint%%://*}://$(secret s3_access_key_id):$(secret s3_secret_access_key)@${endpoint#*://}" +export RC_HOST_root RC_HOST_api +rc alias set anon "$endpoint" --anonymous >/dev/null +case "${1:-}" in + sh|wget) exec "$@" ;; +esac +exec rc "$@" diff --git a/infra/s3/init.sh b/infra/s3/init.sh new file mode 100755 index 00000000..3686b1c6 --- /dev/null +++ b/infra/s3/init.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +secret() { tr -d '\r\n' < "/run/secrets/$1"; } +bucket="${S3_BUCKET:-messenger-attachments}" +endpoint="${S3_ENDPOINT:-http://s3:9000}" +policy="${S3_API_POLICY:-messenger-api}" +RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}" +export RC_HOST_root +api_key=$(secret s3_access_key_id) +rc ready root --timeout 10 >/dev/null +rc bucket create --ignore-existing "root/$bucket" >/dev/null +rc bucket anonymous set private "root/$bucket" >/dev/null +sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.json +rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null +rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null +rc admin policy attach root "$policy" --user "$api_key" >/dev/null +rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null +rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"' +rc admin user info root "$api_key" --json | grep -q "\"$policy\"" +echo "s3-init: bucket=$bucket anonymous=private policy=$policy lifecycle=abort-incomplete-uploads" diff --git a/infra/s3/lifecycle.json b/infra/s3/lifecycle.json new file mode 100644 index 00000000..ab206a07 --- /dev/null +++ b/infra/s3/lifecycle.json @@ -0,0 +1,10 @@ +{ + "rules": [ + { + "id": "abort-incomplete-uploads", + "status": "Enabled", + "prefix": "", + "abortIncompleteMultipartUploadDays": 1 + } + ] +} diff --git a/openspec/changes/add-lab4-image-attachments/tasks.md b/openspec/changes/add-lab4-image-attachments/tasks.md index c71a7ae8..42b21066 100644 --- a/openspec/changes/add-lab4-image-attachments/tasks.md +++ b/openspec/changes/add-lab4-image-attachments/tasks.md @@ -16,7 +16,7 @@ - [x] 3.2 Адаптер `PillowImageInspector`: открытие только PNG/JPEG, подмена MIME, пиксели по заголовку, PNG `verify`, JPEG `draft` + `load`, `to_thread` под семафором - [x] 3.3 Адаптер `S3ObjectStorage` на aiobotocore (path-style, SigV4, таймауты, checksum `when_required`), отображение ошибок в `ObjectMissing`/`Unavailable`; зависимости aiobotocore и Pillow в `uv.lock` - [x] 3.4 Настройки зоны (`S3_*`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS`, таймауты, параллелизм проверки), провайдер dishka, схемы `AttachmentOut`, роутер upload/метаданных/скачивания, реестр ошибок, журнал -- [ ] 3.5 Проверка группы: тест адаптера S3 против одноразового RustFS (put/open/delete, пропавший объект, недоступный endpoint → `Unavailable` за ограниченное время) +- [x] 3.5 Проверка группы: тест адаптера S3 против одноразового RustFS (put/open/delete, пропавший объект, недоступный endpoint → `Unavailable` за ограниченное время) ## 4. Связь зон и платформа @@ -38,10 +38,10 @@ ## 7. Стенд -- [ ] 7.1 `infra/s3/` (`init.sh`, `cli.sh`, `api-policy.json`, `lifecycle.json`) из заготовки -- [ ] 7.2 `compose.yaml`: `s3`, `s3-init`, `s3-cli`, volume `s3data`, секреты S3, якорь `x-s3-env` у `api`, зависимость от `s3-init`; `.env.example` (таймауты S3, параллелизм проверки) -- [ ] 7.3 `scripts/secrets.sh` (секреты S3), `scripts/restart.sh` (S3 в списке хранилищ), `scripts/evidence-stack.sh` (пакеты и образы лабы 4) -- [ ] 7.4 Проверка группы: `docker compose config --quiet`, у `api` только ключ API S3, `make up` из чистого состояния, все сервисы `healthy`, `s3-init` завершился кодом 0, опубликованы только порты `lb` +- [x] 7.1 `infra/s3/` (`init.sh`, `cli.sh`, `api-policy.json`, `lifecycle.json`) из заготовки +- [x] 7.2 `compose.yaml`: `s3`, `s3-init`, `s3-cli`, volume `s3data`, секреты S3, якорь `x-s3-env` у `api`, зависимость от `s3-init`; `.env.example` (таймауты S3, параллелизм проверки) +- [x] 7.3 `scripts/secrets.sh` (секреты S3), `scripts/restart.sh` (S3 в списке хранилищ), `scripts/evidence-stack.sh` (пакеты и образы лабы 4) +- [x] 7.4 Проверка группы: `docker compose config --quiet`, у `api` только ключ API S3, `make up` из чистого состояния, все сервисы `healthy`, `s3-init` завершился кодом 0, опубликованы только порты `lb` ## 8. Сценарии отказов и обвязка diff --git a/scripts/evidence-stack.sh b/scripts/evidence-stack.sh index 8d33f590..0e5575e2 100755 --- a/scripts/evidence-stack.sh +++ b/scripts/evidence-stack.sh @@ -35,8 +35,9 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()): grep -h -E 'image:|@sha256' compose.yaml Dockerfile | sed 's/^ */ /' echo "python in image: $(docker exec "$(docker compose ps -q api | head -1)" python -V)" echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)" + echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)" echo "packages (uv.lock):" - for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis; do + for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow; do printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)" done echo "process users (service uid command):" diff --git a/scripts/limits.py b/scripts/limits.py index 2547ea0c..c487ef6f 100644 --- a/scripts/limits.py +++ b/scripts/limits.py @@ -7,10 +7,17 @@ import time from pathlib import Path ROOT = Path(__file__).resolve().parents[1] +S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"] POLICY = { "lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True}, "api": { - "secrets": ["cache_url", "cursor_secret", "database_url"], + "secrets": [ + "cache_url", + "cursor_secret", + "database_url", + "s3_access_key_id", + "s3_secret_access_key", + ], "volumes": [], "networks": ["backend"], }, @@ -18,6 +25,13 @@ POLICY = { "postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]}, "valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]}, "k6": {"secrets": [], "volumes": [], "networks": ["edge"]}, + "s3": { + "secrets": ["s3_root_password", "s3_root_user"], + "volumes": ["s3data"], + "networks": ["backend"], + }, + "s3-init": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]}, + "s3-cli": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]}, } @@ -99,7 +113,17 @@ def main(): parser.add_argument("--out", help="text report, written only after a complete check") args = parser.parse_args() problems = [] - _, raw = run("docker", "compose", "--profile", "bench", "config", "--format", "json") + _, raw = run( + "docker", + "compose", + "--profile", + "bench", + "--profile", + "tools", + "config", + "--format", + "json", + ) config = json.loads(raw) services = [ declared(name, service, config.get("networks", {}), problems) @@ -110,7 +134,7 @@ def main(): problems.append("no running api replicas to inspect") lines = [ f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}", - "$ docker compose --profile bench config --format json (declared)", + "$ docker compose --profile bench --profile tools config --format json (declared)", *table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services), "", "$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)", diff --git a/scripts/restart.sh b/scripts/restart.sh index 47645b40..c78416db 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -3,7 +3,7 @@ set -eu cd "$(dirname "$0")/.." defined=$(docker compose config --services) stores="" -for service in ${STATE_SERVICES:-postgres valkey}; do +for service in ${STATE_SERVICES:-postgres valkey s3}; do if printf '%s\n' "$defined" | grep -qx "$service"; then stores="$stores $service"; fi done [ -n "$stores" ] || { echo "no state services found" >&2; exit 3; } diff --git a/scripts/secrets.sh b/scripts/secrets.sh index 53997e3c..bdc67ec5 100755 --- a/scripts/secrets.sh +++ b/scripts/secrets.sh @@ -17,4 +17,8 @@ put cursor_secret "$(random)" put valkey_password "$(random)" put valkey.conf "requirepass $(cat secrets/valkey_password)" put cache_url "redis://:$(cat secrets/valkey_password)@valkey:6379/0" +put s3_root_user "root$(random | cut -c1-16)" +put s3_root_password "$(random)" +put s3_access_key_id "api$(random | cut -c1-16)" +put s3_secret_access_key "$(random)" [ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; } diff --git a/student-tests/adapters/test_s3_storage.py b/student-tests/adapters/test_s3_storage.py new file mode 100644 index 00000000..e4e631c3 --- /dev/null +++ b/student-tests/adapters/test_s3_storage.py @@ -0,0 +1,95 @@ +import os +import re +import subprocess +import time +from collections.abc import AsyncIterator, Iterator +from contextlib import asynccontextmanager +from uuid import uuid4 + +import httpx +import pytest + +from conftest import STARTUP_SECONDS, container, image, published +from messenger.attachments.application.errors import ObjectMissingError +from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client +from messenger.shared.application.errors import Unavailable + +RUSTFS_IMAGE = re.compile(r"image: (rustfs/rustfs:\S+)") +ROOT_KEY, ROOT_SECRET = "testroot", "testroot-secret-key" +BUCKET = "messenger-attachments" + + +@pytest.fixture(scope="session") +def rustfs() -> Iterator[str]: + name = f"mtusi-lab4-test-rustfs-{uuid4().hex[:8]}" + keys = ["-e", f"RUSTFS_ACCESS_KEY={ROOT_KEY}", "-e", f"RUSTFS_SECRET_KEY={ROOT_SECRET}"] + container(name, *keys, "-p", "127.0.0.1::9000", image(RUSTFS_IMAGE)) + try: + endpoint = f"http://127.0.0.1:{published(name, 9000)}" + deadline = time.monotonic() + STARTUP_SECONDS + while True: + try: + if httpx.get(f"{endpoint}/health/ready", timeout=1).status_code == 200: + break + except httpx.TransportError: + pass + assert time.monotonic() < deadline, "RustFS не стал готов" + time.sleep(0.3) + yield endpoint + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False) + + +@asynccontextmanager +async def storage(endpoint: str) -> AsyncIterator[S3ObjectStorage]: + async with s3_client( + endpoint=endpoint, + region="us-east-1", + access_key_id=ROOT_KEY, + secret_access_key=ROOT_SECRET, + connect_timeout=1, + read_timeout=2, + max_attempts=2, + ) as client: + buckets = await client.list_buckets() + if BUCKET not in {bucket["Name"] for bucket in buckets.get("Buckets", [])}: + await client.create_bucket(Bucket=BUCKET) + yield S3ObjectStorage(client, BUCKET) + + +async def test_round_trip_of_the_largest_object(rustfs: str) -> None: + data = os.urandom(10_485_760) + key = f"attachments/{uuid4()}" + async with storage(rustfs) as objects: + await objects.put(key, data, "image/png") + content = await objects.open(key) + assert content.size == len(data) + assert b"".join([chunk async for chunk in content.chunks]) == data + await objects.delete(key) + with pytest.raises(ObjectMissingError): + await objects.open(key) + await objects.delete(key) + + +async def test_unreachable_endpoint_is_unavailable_in_bounded_time() -> None: + started = time.monotonic() + async with storage_without_bucket("http://127.0.0.1:1") as objects: + with pytest.raises(Unavailable): + await objects.put("attachments/x", b"x", "image/png") + with pytest.raises(Unavailable): + await objects.open("attachments/x") + assert time.monotonic() - started < 8 + + +@asynccontextmanager +async def storage_without_bucket(endpoint: str) -> AsyncIterator[S3ObjectStorage]: + async with s3_client( + endpoint=endpoint, + region="us-east-1", + access_key_id=ROOT_KEY, + secret_access_key=ROOT_SECRET, + connect_timeout=1, + read_timeout=2, + max_attempts=2, + ) as client: + yield S3ObjectStorage(client, BUCKET)