Split the S3 keys of the API and the reconciler, drop root from one-offs

The API key now has only GetObject, PutObject and DeleteObject on the
attachment objects: the server never lists the bucket, never runs
multipart uploads and has its region set. The reconciler gets its own
user messenger-reconciler with ListBucket, GetObject and DeleteObject,
created by s3-init from a secret of its own, so the API replicas can no
longer list the bucket. The cost seeding writes objects and therefore
runs in a one-off API container under the API key. s3-check proves the
API key cannot list and checks the reconciler key; its snapshot drops
dated fields with a JSON filter instead of grep, which left a dangling
comma, and its first line carries the date and the commit.

s3-init and s3-cli run as uid 65534, and migrate and reconciler get an
init process, so SIGTERM reaches python and docker stop ends a pass in
a fraction of a second instead of waiting for SIGKILL. limits.py and CI
check the users of the one-off services. Stack tests move a 3 MiB file
up and down through the balancer at 256 and 192 KiB/s, longer than the
request timeout, and check 401 before the declared size.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
This commit is contained in:
Artyom
2026-10-30 08:53:39 +03:00
co-authored by Claude Opus 5.5
parent 54099e1da3
commit 2891b37e75
13 changed files with 234 additions and 48 deletions
+23 -3
View File
@@ -7,7 +7,20 @@ import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"]
S3_CLIENT_SECRETS = [
"s3_access_key_id",
"s3_reconciler_access_key_id",
"s3_reconciler_secret_access_key",
"s3_root_password",
"s3_root_user",
"s3_secret_access_key",
]
RECONCILER_SECRETS = [
"database_url",
"s3_reconciler_access_key_id",
"s3_reconciler_secret_access_key",
]
OWN_USER = ("postgres", "valkey", "s3-init", "s3-cli")
POLICY = {
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
"api": {
@@ -23,7 +36,7 @@ POLICY = {
},
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
"reconciler": {
"secrets": ["database_url", "s3_access_key_id", "s3_secret_access_key"],
"secrets": RECONCILER_SECRETS,
"volumes": [],
"networks": ["backend"],
},
@@ -70,6 +83,9 @@ def declared(name, service, networks, problems):
problems.append(f"{name}: publishes {ports} beyond loopback")
if name == "api" and not service.get("read_only"):
problems.append("api: root filesystem is writable")
user = str(service.get("user", ""))
if name in OWN_USER and user.split(":", maxsplit=1)[0] in ("", "0", "root"):
problems.append(f"{name}: runs as root")
internal = [n for n in nets if networks.get(n, {}).get("internal")]
return [
name,
@@ -79,6 +95,7 @@ def declared(name, service, networks, problems):
",".join(volumes) or "-",
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
str(service.get("read_only", False)).lower(),
user or "image",
]
@@ -144,7 +161,10 @@ def main():
f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}",
"$ docker compose --profile bench --profile tools --profile reconcile config --format json"
" (declared)",
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
*table(
["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro", "user"],
services,
),
"",
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
*table(["service", "hostname", "cpus", "memory", "published"], containers),
+2 -2
View File
@@ -558,7 +558,7 @@ def measure(flags, env, args, pinned):
def seed(count, env):
if count <= 0:
return {"added": 0}
env = dict(env, RECONCILE_CPUSET="", RECONCILE_CPUS="4", RECONCILE_MEMORY="512m")
env = dict(env, API_CPUS="4", API_MEMORY="512m")
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
out = compose(
"run",
@@ -569,7 +569,7 @@ def seed(count, env):
tools,
"--entrypoint",
"python",
"reconciler",
"api",
"/tools/seed_storage.py",
"--count",
str(count),
+30 -7
View File
@@ -2,17 +2,30 @@
set -eu
cd "$(dirname "$0")/.."
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
undated() {
python3 -c 'import json, sys
def clean(value):
if isinstance(value, dict):
return {k: clean(v) for k, v in value.items() if not k.endswith("_date")}
return [clean(v) for v in value] if isinstance(value, list) else value
text, decoder, at = sys.stdin.read(), json.JSONDecoder(), 0
while text[at:].strip():
at += len(text[at:]) - len(text[at:].lstrip())
value, at = decoder.raw_decode(text, at)
print(json.dumps(clean(value), indent=2, ensure_ascii=False))'
}
snapshot() {
cli sh -s <<'EOF' | grep -v '_date'
cli sh -s <<'EOF' | undated
ak=$(cat /run/secrets/s3_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
rk=$(cat /run/secrets/s3_reconciler_access_key_id)
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin policy info root messenger-reconciler" "admin user info root $ak" "admin user info root $rk" "bucket lifecycle rule export root/$S3_BUCKET"; do
rc $c --json
done | sed "s/$ak/<api-key>/g"
done | sed "s/$ak/<api-key>/g; s/$rk/<reconciler-key>/g"
EOF
}
failed=0
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD) == S3 privacy and bootstrap checks"
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
before=$(snapshot)
@@ -25,22 +38,32 @@ verdict "s3-init: состояние IAM/bucket не изменилось" "same
cli sh -s <<'EOF' || failed=1
b=$S3_BUCKET
key="attachments/s3-check-$(date +%s)-$$"
spare="$key-reconciler"
other="s3-check-$(date +%s)-$$"
head -c 4096 /dev/urandom > /tmp/probe
result=0
expect() {
want=$1; name=$2; shift 2
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error|check bucket: Network error: service error' /tmp/out; then got=deny; else got=error; fi
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
}
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
expect allow "api: HeadObject" rc object stat "api/$b/$key"
expect allow "api: GetObject" rc object show "api/$b/$key"
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
expect deny "api: ListBucket" rc object list "api/$b/attachments/"
rc put /tmp/probe "api/$b/$spare" >/dev/null
expect allow "reconciler: ListBucket" rc object list "reconciler/$b/attachments/"
expect allow "reconciler: HeadObject" rc object stat "reconciler/$b/$key"
expect allow "reconciler: GetObject" rc object show "reconciler/$b/$key"
expect deny "reconciler: PutObject" rc put /tmp/probe "reconciler/$b/$key-put"
expect allow "reconciler: DeleteObject" rc object remove --force "reconciler/$b/$spare"
expect deny "reconciler: CreateBucket" rc bucket create "reconciler/$other"
expect deny "reconciler: PutBucketPolicy (public)" rc bucket anonymous set public "reconciler/$b"
expect deny "reconciler: admin ListUsers" rc admin user list reconciler
expect deny "api: CreateBucket" rc bucket create "api/$other"
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: PutBucketLifecycle (HeadBucket уже 403)" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
expect deny "api: admin ListUsers" rc admin user list api
rc bucket create --ignore-existing "root/$other" >/dev/null
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
+2
View File
@@ -21,4 +21,6 @@ put s3_root_user "root$(random | cut -c1-16)"
put s3_root_password "$(random)"
put s3_access_key_id "api$(random | cut -c1-16)"
put s3_secret_access_key "$(random)"
put s3_reconciler_access_key_id "rec$(random | cut -c1-16)"
put s3_reconciler_secret_access_key "$(random)"
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }