Run RustFS with a private bucket in the stand

MinIO community images can no longer be pulled on this arm64 host, so
the stand runs RustFS 1.0.4 pinned by digest as non-root with a read
only root, no published port, the console off and data on the s3data
volume. A one-off s3-init with rc v0.1.36 creates the bucket, keeps it
private for anonymous clients, creates the API user with a policy on
the objects and listing of that one bucket, imports a lifecycle rule
for incomplete multipart uploads and checks itself; the API replicas
start only after it succeeded. Root keys reach only s3, s3-init and the
operator s3-cli; the API gets its own key and the S3 settings through
an anchor merged into its environment.

secrets.sh creates hex S3 keys, the restart hook restarts S3 next to
PostgreSQL and Valkey and waits until it is healthy, the limits policy
covers the S3 services, and the stack evidence lists the new packages.
The object storage adapter is tested against a throwaway RustFS of the
same image: a 10 MiB round trip, a missing object, an idempotent delete
and an unreachable endpoint failing within the request budget.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Artyom
2026-10-28 20:50:03 +03:00
co-authored by Claude Opus 5.5
parent f9760fc098
commit 5aa766acff
12 changed files with 289 additions and 11 deletions
+6
View File
@@ -36,3 +36,9 @@ PG_MEMORY=1g
K6_CPUS=2
K6_MEMORY=512m
CURSOR_SECRET_FILE=/run/secrets/cursor_secret
# Solution-specific for lab4: S3 client timeouts (connect + read, two attempts stay below
# REQUEST_TIMEOUT_SECONDS=8), parallel image checks per worker, RustFS memory limit.
S3_CONNECT_TIMEOUT_SECONDS=1
S3_READ_TIMEOUT_SECONDS=2
IMAGE_CHECK_CONCURRENCY=2
S3_MEMORY=512m
+84 -1
View File
@@ -2,6 +2,36 @@ x-app: &app
image: ${API_IMAGE:-mtusi-messenger:lab4}
networks: [backend]
x-s3-env: &s3-env
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-1}
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-2}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
IMAGE_CHECK_CONCURRENCY: ${IMAGE_CHECK_CONCURRENCY:-2}
x-s3-client: &s3-client
image: rustfs/rc:v0.1.36@sha256:ab024bfebee49a750ce886b4c70963ccd9ddaa03f491704a90710641d7a26699
read_only: true
tmpfs: ["/tmp:size=16m"]
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
RC_CONFIG_DIR: /tmp/rc
volumes: ["./infra/s3:/s3:ro"]
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
restart: "no"
networks: [backend]
deploy:
resources:
limits: {cpus: "0.5", memory: 64m}
services:
lb:
image: haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b
@@ -34,6 +64,7 @@ services:
<<: *app
build: .
environment:
<<: *s3-env
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
CURSOR_SECRET_FILE: /run/secrets/cursor_secret
@@ -47,7 +78,7 @@ services:
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
secrets: [database_url, cache_url, cursor_secret]
secrets: [database_url, cache_url, cursor_secret, s3_access_key_id, s3_secret_access_key]
healthcheck:
test: ["CMD", "python", "-c", "import sys,urllib.request;sys.exit(urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status!=200)"]
interval: 5s
@@ -64,6 +95,8 @@ services:
condition: service_completed_successfully
valkey:
condition: service_started
s3-init:
condition: service_completed_successfully
deploy:
replicas: ${API_REPLICAS:-2}
resources:
@@ -152,6 +185,47 @@ services:
resources:
limits: {cpus: "0.5", memory: 128m}
s3:
image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2
environment:
RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_root_user
RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_root_password
RUSTFS_ADDRESS: ":9000"
RUSTFS_CONSOLE_ENABLE: "false"
RUSTFS_REGION: ${S3_REGION:-us-east-1}
RUSTFS_OBS_LOG_DIRECTORY: ""
RUSTFS_OBS_LOGGER_LEVEL: ${S3_LOG_LEVEL:-warn}
secrets: [s3_root_user, s3_root_password]
volumes: [s3data:/data]
read_only: true
tmpfs: ["/tmp:size=64m"]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
healthcheck:
test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:9000/health/ready"]
interval: 2s
timeout: 3s
retries: 30
start_period: 20s
stop_grace_period: 10s
restart: unless-stopped
networks: [backend]
deploy:
resources:
limits: {cpus: "1", memory: "${S3_MEMORY:-512m}"}
s3-init:
<<: *s3-client
entrypoint: ["/bin/sh", "/s3/init.sh"]
depends_on:
s3:
condition: service_healthy
s3-cli:
<<: *s3-client
profiles: [tools]
entrypoint: ["/bin/sh", "/s3/cli.sh"]
k6:
image: grafana/k6:2.3.0@sha256:2dd4756ec6e14e62778916e1eb850108e4b499304c02df56760fa0bdb0fba0b8
profiles: [bench]
@@ -174,10 +248,19 @@ secrets:
file: ./secrets/cache_url
cursor_secret:
file: ./secrets/cursor_secret
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
volumes:
pgdata:
valkeydata:
s3data:
networks:
edge:
+23
View File
@@ -0,0 +1,23 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "MessengerApiObjects",
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts"
],
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
},
{
"Sid": "MessengerApiBucket",
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}
]
}
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
secret() { tr -d '\r\n' < "/run/secrets/$1"; }
endpoint="${S3_ENDPOINT:-http://s3:9000}"
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
RC_HOST_api="${endpoint%%://*}://$(secret s3_access_key_id):$(secret s3_secret_access_key)@${endpoint#*://}"
export RC_HOST_root RC_HOST_api
rc alias set anon "$endpoint" --anonymous >/dev/null
case "${1:-}" in
sh|wget) exec "$@" ;;
esac
exec rc "$@"
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
secret() { tr -d '\r\n' < "/run/secrets/$1"; }
bucket="${S3_BUCKET:-messenger-attachments}"
endpoint="${S3_ENDPOINT:-http://s3:9000}"
policy="${S3_API_POLICY:-messenger-api}"
RC_HOST_root="${endpoint%%://*}://$(secret s3_root_user):$(secret s3_root_password)@${endpoint#*://}"
export RC_HOST_root
api_key=$(secret s3_access_key_id)
rc ready root --timeout 10 >/dev/null
rc bucket create --ignore-existing "root/$bucket" >/dev/null
rc bucket anonymous set private "root/$bucket" >/dev/null
sed "s/messenger-attachments/$bucket/g" /s3/api-policy.json > /tmp/api-policy.json
rc admin policy create root "$policy" /tmp/api-policy.json >/dev/null
rc admin user add root "$api_key" "$(secret s3_secret_access_key)" >/dev/null
rc admin policy attach root "$policy" --user "$api_key" >/dev/null
rc bucket lifecycle rule import "root/$bucket" /s3/lifecycle.json >/dev/null
rc bucket anonymous get "root/$bucket" --json | grep -q '"permission": *"private"'
rc admin user info root "$api_key" --json | grep -q "\"$policy\""
echo "s3-init: bucket=$bucket anonymous=private policy=$policy lifecycle=abort-incomplete-uploads"
+10
View File
@@ -0,0 +1,10 @@
{
"rules": [
{
"id": "abort-incomplete-uploads",
"status": "Enabled",
"prefix": "",
"abortIncompleteMultipartUploadDays": 1
}
]
}
@@ -16,7 +16,7 @@
- [x] 3.2 Адаптер `PillowImageInspector`: открытие только PNG/JPEG, подмена MIME, пиксели по заголовку, PNG `verify`, JPEG `draft` + `load`, `to_thread` под семафором
- [x] 3.3 Адаптер `S3ObjectStorage` на aiobotocore (path-style, SigV4, таймауты, checksum `when_required`), отображение ошибок в `ObjectMissing`/`Unavailable`; зависимости aiobotocore и Pillow в `uv.lock`
- [x] 3.4 Настройки зоны (`S3_*`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS`, таймауты, параллелизм проверки), провайдер dishka, схемы `AttachmentOut`, роутер upload/метаданных/скачивания, реестр ошибок, журнал
- [ ] 3.5 Проверка группы: тест адаптера S3 против одноразового RustFS (put/open/delete, пропавший объект, недоступный endpoint → `Unavailable` за ограниченное время)
- [x] 3.5 Проверка группы: тест адаптера S3 против одноразового RustFS (put/open/delete, пропавший объект, недоступный endpoint → `Unavailable` за ограниченное время)
## 4. Связь зон и платформа
@@ -38,10 +38,10 @@
## 7. Стенд
- [ ] 7.1 `infra/s3/` (`init.sh`, `cli.sh`, `api-policy.json`, `lifecycle.json`) из заготовки
- [ ] 7.2 `compose.yaml`: `s3`, `s3-init`, `s3-cli`, volume `s3data`, секреты S3, якорь `x-s3-env` у `api`, зависимость от `s3-init`; `.env.example` (таймауты S3, параллелизм проверки)
- [ ] 7.3 `scripts/secrets.sh` (секреты S3), `scripts/restart.sh` (S3 в списке хранилищ), `scripts/evidence-stack.sh` (пакеты и образы лабы 4)
- [ ] 7.4 Проверка группы: `docker compose config --quiet`, у `api` только ключ API S3, `make up` из чистого состояния, все сервисы `healthy`, `s3-init` завершился кодом 0, опубликованы только порты `lb`
- [x] 7.1 `infra/s3/` (`init.sh`, `cli.sh`, `api-policy.json`, `lifecycle.json`) из заготовки
- [x] 7.2 `compose.yaml`: `s3`, `s3-init`, `s3-cli`, volume `s3data`, секреты S3, якорь `x-s3-env` у `api`, зависимость от `s3-init`; `.env.example` (таймауты S3, параллелизм проверки)
- [x] 7.3 `scripts/secrets.sh` (секреты S3), `scripts/restart.sh` (S3 в списке хранилищ), `scripts/evidence-stack.sh` (пакеты и образы лабы 4)
- [x] 7.4 Проверка группы: `docker compose config --quiet`, у `api` только ключ API S3, `make up` из чистого состояния, все сервисы `healthy`, `s3-init` завершился кодом 0, опубликованы только порты `lb`
## 8. Сценарии отказов и обвязка
+2 -1
View File
@@ -35,8 +35,9 @@ for r in csv.DictReader(sys.stdin.read().lstrip("# ").splitlines()):
grep -h -E 'image:|@sha256' compose.yaml Dockerfile | sed 's/^ */ /'
echo "python in image: $(docker exec "$(docker compose ps -q api | head -1)" python -V)"
echo "haproxy: $(docker compose exec -T lb haproxy -v | head -1)"
echo "rustfs: $(docker compose exec -T s3 rustfs --version 2>/dev/null | head -1)"
echo "packages (uv.lock):"
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis; do
for name in fastapi uvicorn uvicorn-worker gunicorn asyncpg argon2-cffi pydantic pydantic-settings dishka structlog redis aiobotocore botocore aiohttp pillow; do
printf ' %s %s\n' "$name" "$(sed -n "/^name = \"$name\"$/{n;s/version = \"\(.*\)\"/\1/p;}" uv.lock)"
done
echo "process users (service uid command):"
+27 -3
View File
@@ -7,10 +7,17 @@ import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"]
POLICY = {
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
"api": {
"secrets": ["cache_url", "cursor_secret", "database_url"],
"secrets": [
"cache_url",
"cursor_secret",
"database_url",
"s3_access_key_id",
"s3_secret_access_key",
],
"volumes": [],
"networks": ["backend"],
},
@@ -18,6 +25,13 @@ POLICY = {
"postgres": {"secrets": ["postgres_password"], "volumes": ["pgdata"], "networks": ["backend"]},
"valkey": {"secrets": ["valkey.conf"], "volumes": ["valkeydata"], "networks": ["backend"]},
"k6": {"secrets": [], "volumes": [], "networks": ["edge"]},
"s3": {
"secrets": ["s3_root_password", "s3_root_user"],
"volumes": ["s3data"],
"networks": ["backend"],
},
"s3-init": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]},
"s3-cli": {"secrets": S3_CLIENT_SECRETS, "volumes": [], "networks": ["backend"]},
}
@@ -99,7 +113,17 @@ def main():
parser.add_argument("--out", help="text report, written only after a complete check")
args = parser.parse_args()
problems = []
_, raw = run("docker", "compose", "--profile", "bench", "config", "--format", "json")
_, raw = run(
"docker",
"compose",
"--profile",
"bench",
"--profile",
"tools",
"config",
"--format",
"json",
)
config = json.loads(raw)
services = [
declared(name, service, config.get("networks", {}), problems)
@@ -110,7 +134,7 @@ def main():
problems.append("no running api replicas to inspect")
lines = [
f"date: {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
"$ docker compose --profile bench config --format json (declared)",
"$ docker compose --profile bench --profile tools config --format json (declared)",
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
"",
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
+1 -1
View File
@@ -3,7 +3,7 @@ set -eu
cd "$(dirname "$0")/.."
defined=$(docker compose config --services)
stores=""
for service in ${STATE_SERVICES:-postgres valkey}; do
for service in ${STATE_SERVICES:-postgres valkey s3}; do
if printf '%s\n' "$defined" | grep -qx "$service"; then stores="$stores $service"; fi
done
[ -n "$stores" ] || { echo "no state services found" >&2; exit 3; }
+4
View File
@@ -17,4 +17,8 @@ put cursor_secret "$(random)"
put valkey_password "$(random)"
put valkey.conf "requirepass $(cat secrets/valkey_password)"
put cache_url "redis://:$(cat secrets/valkey_password)@valkey:6379/0"
put s3_root_user "root$(random | cut -c1-16)"
put s3_root_password "$(random)"
put s3_access_key_id "api$(random | cut -c1-16)"
put s3_secret_access_key "$(random)"
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
+95
View File
@@ -0,0 +1,95 @@
import os
import re
import subprocess
import time
from collections.abc import AsyncIterator, Iterator
from contextlib import asynccontextmanager
from uuid import uuid4
import httpx
import pytest
from conftest import STARTUP_SECONDS, container, image, published
from messenger.attachments.application.errors import ObjectMissingError
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
from messenger.shared.application.errors import Unavailable
RUSTFS_IMAGE = re.compile(r"image: (rustfs/rustfs:\S+)")
ROOT_KEY, ROOT_SECRET = "testroot", "testroot-secret-key"
BUCKET = "messenger-attachments"
@pytest.fixture(scope="session")
def rustfs() -> Iterator[str]:
name = f"mtusi-lab4-test-rustfs-{uuid4().hex[:8]}"
keys = ["-e", f"RUSTFS_ACCESS_KEY={ROOT_KEY}", "-e", f"RUSTFS_SECRET_KEY={ROOT_SECRET}"]
container(name, *keys, "-p", "127.0.0.1::9000", image(RUSTFS_IMAGE))
try:
endpoint = f"http://127.0.0.1:{published(name, 9000)}"
deadline = time.monotonic() + STARTUP_SECONDS
while True:
try:
if httpx.get(f"{endpoint}/health/ready", timeout=1).status_code == 200:
break
except httpx.TransportError:
pass
assert time.monotonic() < deadline, "RustFS не стал готов"
time.sleep(0.3)
yield endpoint
finally:
subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False)
@asynccontextmanager
async def storage(endpoint: str) -> AsyncIterator[S3ObjectStorage]:
async with s3_client(
endpoint=endpoint,
region="us-east-1",
access_key_id=ROOT_KEY,
secret_access_key=ROOT_SECRET,
connect_timeout=1,
read_timeout=2,
max_attempts=2,
) as client:
buckets = await client.list_buckets()
if BUCKET not in {bucket["Name"] for bucket in buckets.get("Buckets", [])}:
await client.create_bucket(Bucket=BUCKET)
yield S3ObjectStorage(client, BUCKET)
async def test_round_trip_of_the_largest_object(rustfs: str) -> None:
data = os.urandom(10_485_760)
key = f"attachments/{uuid4()}"
async with storage(rustfs) as objects:
await objects.put(key, data, "image/png")
content = await objects.open(key)
assert content.size == len(data)
assert b"".join([chunk async for chunk in content.chunks]) == data
await objects.delete(key)
with pytest.raises(ObjectMissingError):
await objects.open(key)
await objects.delete(key)
async def test_unreachable_endpoint_is_unavailable_in_bounded_time() -> None:
started = time.monotonic()
async with storage_without_bucket("http://127.0.0.1:1") as objects:
with pytest.raises(Unavailable):
await objects.put("attachments/x", b"x", "image/png")
with pytest.raises(Unavailable):
await objects.open("attachments/x")
assert time.monotonic() - started < 8
@asynccontextmanager
async def storage_without_bucket(endpoint: str) -> AsyncIterator[S3ObjectStorage]:
async with s3_client(
endpoint=endpoint,
region="us-east-1",
access_key_id=ROOT_KEY,
secret_access_key=ROOT_SECRET,
connect_timeout=1,
read_timeout=2,
max_attempts=2,
) as client:
yield S3ObjectStorage(client, BUCKET)