Split the S3 keys of the API and the reconciler, drop root from one-offs
The API key now has only GetObject, PutObject and DeleteObject on the attachment objects: the server never lists the bucket, never runs multipart uploads and has its region set. The reconciler gets its own user messenger-reconciler with ListBucket, GetObject and DeleteObject, created by s3-init from a secret of its own, so the API replicas can no longer list the bucket. The cost seeding writes objects and therefore runs in a one-off API container under the API key. s3-check proves the API key cannot list and checks the reconciler key; its snapshot drops dated fields with a JSON filter instead of grep, which left a dangling comma, and its first line carries the date and the commit. s3-init and s3-cli run as uid 65534, and migrate and reconciler get an init process, so SIGTERM reaches python and docker stop ends a pass in a fraction of a second instead of waiting for SIGKILL. limits.py and CI check the users of the one-off services. Stack tests move a 3 MiB file up and down through the balancer at 256 and 192 KiB/s, longer than the request timeout, and check 401 before the declared size. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
54099e1da3
commit
2891b37e75
+23
-3
@@ -7,7 +7,20 @@ import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
S3_CLIENT_SECRETS = ["s3_access_key_id", "s3_root_password", "s3_root_user", "s3_secret_access_key"]
|
||||
S3_CLIENT_SECRETS = [
|
||||
"s3_access_key_id",
|
||||
"s3_reconciler_access_key_id",
|
||||
"s3_reconciler_secret_access_key",
|
||||
"s3_root_password",
|
||||
"s3_root_user",
|
||||
"s3_secret_access_key",
|
||||
]
|
||||
RECONCILER_SECRETS = [
|
||||
"database_url",
|
||||
"s3_reconciler_access_key_id",
|
||||
"s3_reconciler_secret_access_key",
|
||||
]
|
||||
OWN_USER = ("postgres", "valkey", "s3-init", "s3-cli")
|
||||
POLICY = {
|
||||
"lb": {"secrets": [], "volumes": [], "networks": ["backend", "edge"], "ports": True},
|
||||
"api": {
|
||||
@@ -23,7 +36,7 @@ POLICY = {
|
||||
},
|
||||
"migrate": {"secrets": ["database_url"], "volumes": [], "networks": ["backend"]},
|
||||
"reconciler": {
|
||||
"secrets": ["database_url", "s3_access_key_id", "s3_secret_access_key"],
|
||||
"secrets": RECONCILER_SECRETS,
|
||||
"volumes": [],
|
||||
"networks": ["backend"],
|
||||
},
|
||||
@@ -70,6 +83,9 @@ def declared(name, service, networks, problems):
|
||||
problems.append(f"{name}: publishes {ports} beyond loopback")
|
||||
if name == "api" and not service.get("read_only"):
|
||||
problems.append("api: root filesystem is writable")
|
||||
user = str(service.get("user", ""))
|
||||
if name in OWN_USER and user.split(":", maxsplit=1)[0] in ("", "0", "root"):
|
||||
problems.append(f"{name}: runs as root")
|
||||
internal = [n for n in nets if networks.get(n, {}).get("internal")]
|
||||
return [
|
||||
name,
|
||||
@@ -79,6 +95,7 @@ def declared(name, service, networks, problems):
|
||||
",".join(volumes) or "-",
|
||||
",".join(f"{n}{'(internal)' if n in internal else ''}" for n in nets),
|
||||
str(service.get("read_only", False)).lower(),
|
||||
user or "image",
|
||||
]
|
||||
|
||||
|
||||
@@ -144,7 +161,10 @@ def main():
|
||||
f"commit: {run('git', 'rev-parse', '--short', 'HEAD')[1]}",
|
||||
"$ docker compose --profile bench --profile tools --profile reconcile config --format json"
|
||||
" (declared)",
|
||||
*table(["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro"], services),
|
||||
*table(
|
||||
["service", "cpus/mem", "ports", "secrets", "volumes", "networks", "ro", "user"],
|
||||
services,
|
||||
),
|
||||
"",
|
||||
"$ docker inspect (running containers: hostname = X-Instance-Id, applied limits)",
|
||||
*table(["service", "hostname", "cpus", "memory", "published"], containers),
|
||||
|
||||
@@ -558,7 +558,7 @@ def measure(flags, env, args, pinned):
|
||||
def seed(count, env):
|
||||
if count <= 0:
|
||||
return {"added": 0}
|
||||
env = dict(env, RECONCILE_CPUSET="", RECONCILE_CPUS="4", RECONCILE_MEMORY="512m")
|
||||
env = dict(env, API_CPUS="4", API_MEMORY="512m")
|
||||
tools = f"{ROOT / 'student-tests' / 'tools'}:/tools:ro"
|
||||
out = compose(
|
||||
"run",
|
||||
@@ -569,7 +569,7 @@ def seed(count, env):
|
||||
tools,
|
||||
"--entrypoint",
|
||||
"python",
|
||||
"reconciler",
|
||||
"api",
|
||||
"/tools/seed_storage.py",
|
||||
"--count",
|
||||
str(count),
|
||||
|
||||
+30
-7
@@ -2,17 +2,30 @@
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
cli() { docker compose run --rm --no-deps -T s3-cli "$@"; }
|
||||
undated() {
|
||||
python3 -c 'import json, sys
|
||||
def clean(value):
|
||||
if isinstance(value, dict):
|
||||
return {k: clean(v) for k, v in value.items() if not k.endswith("_date")}
|
||||
return [clean(v) for v in value] if isinstance(value, list) else value
|
||||
text, decoder, at = sys.stdin.read(), json.JSONDecoder(), 0
|
||||
while text[at:].strip():
|
||||
at += len(text[at:]) - len(text[at:].lstrip())
|
||||
value, at = decoder.raw_decode(text, at)
|
||||
print(json.dumps(clean(value), indent=2, ensure_ascii=False))'
|
||||
}
|
||||
snapshot() {
|
||||
cli sh -s <<'EOF' | grep -v '_date'
|
||||
cli sh -s <<'EOF' | undated
|
||||
ak=$(cat /run/secrets/s3_access_key_id)
|
||||
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin user info root $ak" "bucket lifecycle rule export root/$S3_BUCKET"; do
|
||||
rk=$(cat /run/secrets/s3_reconciler_access_key_id)
|
||||
for c in "bucket list root" "bucket anonymous get root/$S3_BUCKET" "admin policy info root messenger-api" "admin policy info root messenger-reconciler" "admin user info root $ak" "admin user info root $rk" "bucket lifecycle rule export root/$S3_BUCKET"; do
|
||||
rc $c --json
|
||||
done | sed "s/$ak/<api-key>/g"
|
||||
done | sed "s/$ak/<api-key>/g; s/$rk/<reconciler-key>/g"
|
||||
EOF
|
||||
}
|
||||
failed=0
|
||||
verdict() { if [ "$2" = "$3" ]; then echo "PASS $1: $3"; else echo "FAIL $1: ожидалось $2, получено $3"; failed=1; fi; }
|
||||
echo "== $(date -u +%FT%TZ) S3 privacy and bootstrap checks"
|
||||
echo "date: $(date -u +%FT%TZ) commit: $(git rev-parse --short HEAD) == S3 privacy and bootstrap checks"
|
||||
verdict "s3: опубликованные host-порты" "нет" "$(docker inspect -f '{{json .NetworkSettings.Ports}}' "$(docker compose ps -q s3)" | grep -q HostPort && echo есть || echo нет)"
|
||||
verdict "s3: консоль :9001 внутри контейнера" "000" "$(docker compose exec -T s3 curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:9001/ || true)"
|
||||
before=$(snapshot)
|
||||
@@ -25,22 +38,32 @@ verdict "s3-init: состояние IAM/bucket не изменилось" "same
|
||||
cli sh -s <<'EOF' || failed=1
|
||||
b=$S3_BUCKET
|
||||
key="attachments/s3-check-$(date +%s)-$$"
|
||||
spare="$key-reconciler"
|
||||
other="s3-check-$(date +%s)-$$"
|
||||
head -c 4096 /dev/urandom > /tmp/probe
|
||||
result=0
|
||||
expect() {
|
||||
want=$1; name=$2; shift 2
|
||||
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error' /tmp/out; then got=deny; else got=error; fi
|
||||
if "$@" >/tmp/out 2>&1; then got=allow; elif grep -qE '403|AccessDenied|set_bucket_policy: service error|check bucket: Network error: service error' /tmp/out; then got=deny; else got=error; fi
|
||||
if [ "$got" = "$want" ]; then echo "PASS $name: $got"; else echo "FAIL $name: ожидалось $want, получено $got"; result=1; fi
|
||||
}
|
||||
expect allow "api: PutObject" rc put /tmp/probe "api/$b/$key"
|
||||
expect allow "api: HeadObject" rc object stat "api/$b/$key"
|
||||
expect allow "api: GetObject" rc object show "api/$b/$key"
|
||||
expect allow "api: ListBucket" rc object list "api/$b/attachments/"
|
||||
expect deny "api: ListBucket" rc object list "api/$b/attachments/"
|
||||
rc put /tmp/probe "api/$b/$spare" >/dev/null
|
||||
expect allow "reconciler: ListBucket" rc object list "reconciler/$b/attachments/"
|
||||
expect allow "reconciler: HeadObject" rc object stat "reconciler/$b/$key"
|
||||
expect allow "reconciler: GetObject" rc object show "reconciler/$b/$key"
|
||||
expect deny "reconciler: PutObject" rc put /tmp/probe "reconciler/$b/$key-put"
|
||||
expect allow "reconciler: DeleteObject" rc object remove --force "reconciler/$b/$spare"
|
||||
expect deny "reconciler: CreateBucket" rc bucket create "reconciler/$other"
|
||||
expect deny "reconciler: PutBucketPolicy (public)" rc bucket anonymous set public "reconciler/$b"
|
||||
expect deny "reconciler: admin ListUsers" rc admin user list reconciler
|
||||
expect deny "api: CreateBucket" rc bucket create "api/$other"
|
||||
expect deny "api: DeleteBucket" rc bucket remove "api/$b"
|
||||
expect deny "api: PutBucketPolicy (public)" rc bucket anonymous set public "api/$b"
|
||||
expect deny "api: PutBucketLifecycle" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
|
||||
expect deny "api: PutBucketLifecycle (HeadBucket уже 403)" rc bucket lifecycle rule import "api/$b" /s3/lifecycle.json
|
||||
expect deny "api: admin ListUsers" rc admin user list api
|
||||
rc bucket create --ignore-existing "root/$other" >/dev/null
|
||||
expect deny "api: PutObject в чужой bucket" rc put /tmp/probe "api/$other/x"
|
||||
|
||||
@@ -21,4 +21,6 @@ put s3_root_user "root$(random | cut -c1-16)"
|
||||
put s3_root_password "$(random)"
|
||||
put s3_access_key_id "api$(random | cut -c1-16)"
|
||||
put s3_secret_access_key "$(random)"
|
||||
put s3_reconciler_access_key_id "rec$(random | cut -c1-16)"
|
||||
put s3_reconciler_secret_access_key "$(random)"
|
||||
[ -f .env ] || { cp .env.example .env; echo "создан .env из .env.example"; }
|
||||
|
||||
Reference in New Issue
Block a user