From 51f4adfd58b122178e8b20df563b26635a803b07 Mon Sep 17 00:00:00 2001 From: aegor Date: Thu, 5 Nov 2026 13:38:18 +0300 Subject: [PATCH] feat: add private s3 storage with restricted api user --- compose.yaml | 77 +++++++++++++++++++++++++++++++---- scripts/check-s3-anonymous.py | 21 ++++++++++ scripts/check-s3.sh | 46 +++++++++++++++++++++ scripts/restart.sh | 4 +- 4 files changed, 139 insertions(+), 9 deletions(-) create mode 100755 scripts/check-s3-anonymous.py create mode 100755 scripts/check-s3.sh diff --git a/compose.yaml b/compose.yaml index fe461f8..58fdea3 100644 --- a/compose.yaml +++ b/compose.yaml @@ -10,17 +10,17 @@ services: - -ec - | cd /secrets - mkdir -p postgres redis api migrate - for name in pg_password:postgres redis_password:redis; do - file=$${name%%:*}; dir=$${name##*:} - [ ! -s "$$file" ] || mv "$$file" "$$dir/$$file" - [ -s "$$dir/$$file" ] || head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n' > "$$dir/$$file" + mkdir -p postgres redis api migrate s3 s3-init + for entry in pg_password:postgres:24 redis_password:redis:24 s3_root_user:s3:10 s3_root_password:s3:20 s3_access_key_id:api:10 s3_secret_access_key:api:20; do + file=$${entry%%:*}; rest=$${entry#*:}; dir=$${rest%%:*}; size=$${rest##*:} + [ -s "$$dir/$$file" ] || head -c "$$size" /dev/urandom | od -An -tx1 | tr -d ' \n' > "$$dir/$$file" done - rm -f database_url cache_url + rm -f s3-init/* + cp s3/s3_root_user s3/s3_root_password api/s3_access_key_id api/s3_secret_access_key s3-init/ printf 'postgresql://messenger:%s@postgres:5432/messenger?connection_limit=10&pool_timeout=5&connect_timeout=5&socket_timeout=10&application_name=api&options=-c%%20statement_timeout%%3D5000' "$$(cat postgres/pg_password)" > api/database_url printf 'redis://:%s@redis:6379/0' "$$(cat redis/redis_password)" > api/cache_url printf 'postgresql://messenger:%s@postgres:5432/messenger' "$$(cat postgres/pg_password)" > migrate/database_url - chmod 0444 postgres/* redis/* api/* migrate/* + chmod 0444 postgres/* redis/* api/* migrate/* s3/* s3-init/* deploy: resources: limits: { cpus: "0.1", memory: 32M } @@ -69,6 +69,59 @@ services: condition: service_completed_successfully restart: unless-stopped + s3: + image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372 + command: ["server", "/data", "--address", ":9000"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password + MINIO_BROWSER: "off" + volumes: + - s3data:/data + - { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: s3 } } + deploy: + resources: + limits: { cpus: "0.5", memory: 512M } + healthcheck: + test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://127.0.0.1:9000/minio/health/live"] + interval: 5s + timeout: 3s + retries: 10 + depends_on: + secrets-init: + condition: service_completed_successfully + restart: unless-stopped + + s3-init: + image: pgsty/mc:RELEASE.2026-10-26T00-00-00Z@sha256:ce62c9053c69ce12a527a322345b5ad9f44b5f8a7b8189f82e516f2e3f30f1d8 + entrypoint: ["sh", "-ec"] + command: + - | + cd /run/secrets + export MC_HOST_root="http://$$(cat s3_root_user):$$(cat s3_root_password)@s3:9000" + mc mb --ignore-existing "root/$$S3_BUCKET" + mc anonymous set none "root/$$S3_BUCKET" + cat > /tmp/policy.json < /dev/null + mc admin policy attach root messenger-api --user "$$(cat s3_access_key_id)" + echo "bucket $$S3_BUCKET ready" + environment: + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + volumes: + - { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: s3-init } } + deploy: + resources: + limits: { cpus: "0.25", memory: 256M } + depends_on: + s3: + condition: service_healthy + migrate: build: context: . @@ -99,6 +152,13 @@ services: SHUTDOWN_DELAY_MS: 3000 DATABASE_URL_FILE: /run/secrets/database_url CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} volumes: - { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: api } } deploy: @@ -111,6 +171,8 @@ services: condition: service_completed_successfully redis: condition: service_started + s3-init: + condition: service_completed_successfully restart: unless-stopped lb: @@ -146,4 +208,5 @@ services: volumes: pgdata: + s3data: secrets: diff --git a/scripts/check-s3-anonymous.py b/scripts/check-s3-anonymous.py new file mode 100755 index 0000000..636f406 --- /dev/null +++ b/scripts/check-s3-anonymous.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +import sys +import urllib.error +import urllib.request + +bucket, key = sys.argv[1:3] +base = 'http://s3:9000/' + bucket +attempts = ( + ('GET', '/', base + '/' + key, None), + ('GET', '/', base + '/', None), + ('PUT', '/anon.txt', base + '/anon.txt', b'x'), +) +failed = False +for method, label, url, data in attempts: + try: + code = urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5).status + except urllib.error.HTTPError as error: + code = error.code + print(method, bucket + label, '->', code) + failed = failed or code != 403 +sys.exit(1 if failed else 0) diff --git a/scripts/check-s3.sh b/scripts/check-s3.sh new file mode 100755 index 0000000..d6fd983 --- /dev/null +++ b/scripts/check-s3.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +bucket="${S3_BUCKET:-messenger-attachments}" +object="check/$(date +%s)-$$.txt" + +as_api() { + docker compose run --rm --no-deps -T -e BUCKET="$bucket" -e OBJECT="$object" --entrypoint sh s3-init -c "$1" +} + +echo "## S3 не опубликован наружу, консоль выключена" +docker compose ps s3 --format '{{.Service}} ports={{.Ports}}' +echo "MINIO_BROWSER=$(docker compose exec -T s3 printenv MINIO_BROWSER)" + +echo +echo "## Права пользователя API: только свой bucket" +as_api ' +set -e +cd /run/secrets +export MC_HOST_api="http://$(cat s3_access_key_id):$(cat s3_secret_access_key)@s3:9000" +export MC_HOST_root="http://$(cat s3_root_user):$(cat s3_root_password)@s3:9000" +expect() { want=$1; label=$2; shift 2; if "$@" >/dev/null 2>&1; then got=allowed; else got=denied; fi; echo "$label -> $got"; [ "$got" = "$want" ]; } +echo probe > /tmp/probe.txt +mc mb --ignore-existing root/check-other-bucket >/dev/null +mc cp /tmp/probe.txt root/check-other-bucket/probe.txt >/dev/null +trap "mc rb --force root/check-other-bucket >/dev/null" EXIT +expect allowed "PUT в свой bucket" mc cp /tmp/probe.txt "api/$BUCKET/$OBJECT" +expect allowed "GET из своего bucket" mc cat "api/$BUCKET/$OBJECT" +expect denied "создание другого bucket" mc mb api/forbidden-bucket +expect denied "GET из другого bucket" mc cat api/check-other-bucket/probe.txt +expect denied "PUT в другой bucket" mc cp /tmp/probe.txt api/check-other-bucket/new.txt +expect denied "изменение политики bucket" mc anonymous set public "api/$BUCKET" +expect denied "администрирование пользователей" mc admin user list api +' + +echo +echo "## Анонимный доступ напрямую к S3, мимо API" +python3 scripts/check-s3-anonymous.py "$bucket" "$object" + +as_api ' +cd /run/secrets +export MC_HOST_api="http://$(cat s3_access_key_id):$(cat s3_secret_access_key)@s3:9000" +mc rm "api/$BUCKET/$OBJECT" >/dev/null +' +echo +echo "PASS: bucket приватный, у API нет прав вне своего bucket" diff --git a/scripts/restart.sh b/scripts/restart.sh index 783157a..20e1da6 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -2,8 +2,8 @@ set -eu cd "$(dirname "$0")/.." started() { - docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis api lb) + docker inspect -f '{{.Name}} {{.State.Status}} started={{.State.StartedAt}}' $(docker compose ps -aq postgres redis s3 api lb) } started -docker compose restart postgres redis api lb +docker compose restart postgres redis s3 api lb started