Done Track 4
This commit is contained in:
@@ -0,0 +1,161 @@
|
||||
"""Idempotent bootstrap of the private attachments bucket (service s3-init).
|
||||
|
||||
Runs with the S3 ADMIN credentials (only this one-shot container gets them),
|
||||
before any API replica starts:
|
||||
|
||||
1. wait until the S3 gateway answers;
|
||||
2. create the bucket if it does not exist (a repeated run prints "keep");
|
||||
3. self-checks, the stand does not start if one fails:
|
||||
- anonymous (unsigned) GET/PUT/LIST of the bucket -> 403: bucket is private;
|
||||
- API credentials can PUT/GET/DELETE under the prefix;
|
||||
- API credentials can NOT create/delete buckets or touch other buckets.
|
||||
|
||||
The API identity and its rights are declared in infra/seaweedfs/s3.json.template.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import botocore.session
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
file_name = os.getenv(f"{name}_FILE")
|
||||
if file_name:
|
||||
return Path(file_name).read_text(encoding="utf-8").strip()
|
||||
value = os.getenv(name)
|
||||
if not value:
|
||||
raise RuntimeError(f"{name} or {name}_FILE is required")
|
||||
return value
|
||||
|
||||
|
||||
ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
|
||||
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
PREFIX = os.getenv("S3_OBJECT_PREFIX", "originals/")
|
||||
|
||||
|
||||
def client(access_key: str, secret_key: str):
|
||||
return botocore.session.get_session().create_client(
|
||||
"s3",
|
||||
endpoint_url=ENDPOINT,
|
||||
region_name=os.getenv("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=2,
|
||||
read_timeout=5,
|
||||
retries={"max_attempts": 1},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def error_code(exc: ClientError) -> str:
|
||||
return exc.response.get("Error", {}).get("Code", "")
|
||||
|
||||
|
||||
failures = 0
|
||||
|
||||
|
||||
def check(label: str, ok: bool) -> None:
|
||||
global failures
|
||||
failures += not ok
|
||||
print(("PASS " if ok else "FAIL ") + label, flush=True)
|
||||
|
||||
|
||||
def denied(call) -> bool:
|
||||
try:
|
||||
call()
|
||||
except ClientError as exc:
|
||||
return error_code(exc) in {"AccessDenied", "403", "Forbidden", "AllAccessDisabled"}
|
||||
return False
|
||||
|
||||
|
||||
def anonymous_status(method: str, path: str, data: bytes | None = None) -> int:
|
||||
request = urllib.request.Request(ENDPOINT + path, data=data, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
return response.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
|
||||
|
||||
def main() -> int:
|
||||
admin = client(secret("S3_ADMIN_ACCESS_KEY_ID"), secret("S3_ADMIN_SECRET_ACCESS_KEY"))
|
||||
api = client(secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"))
|
||||
|
||||
for attempt in range(1, 61):
|
||||
try:
|
||||
admin.list_buckets()
|
||||
break
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
if attempt == 60:
|
||||
raise
|
||||
print(f"WAIT s3 ({type(exc).__name__}), retry {attempt}", flush=True)
|
||||
time.sleep(1)
|
||||
|
||||
try:
|
||||
admin.head_bucket(Bucket=BUCKET)
|
||||
print(f"keep bucket {BUCKET}", flush=True)
|
||||
except ClientError as exc:
|
||||
if error_code(exc) not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
raise
|
||||
try:
|
||||
admin.create_bucket(Bucket=BUCKET)
|
||||
print(f"create bucket {BUCKET}", flush=True)
|
||||
except ClientError as create_exc:
|
||||
# Two bootstrap runs at the same time: the other one won.
|
||||
if error_code(create_exc) not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
|
||||
raise
|
||||
print(f"keep bucket {BUCKET}", flush=True)
|
||||
|
||||
probe = f"{PREFIX}bootstrap-probe-{uuid.uuid4().hex}"
|
||||
|
||||
check("api credentials: PUT object", not denied(
|
||||
lambda: api.put_object(Bucket=BUCKET, Key=probe, Body=b"probe")))
|
||||
check("api credentials: GET object",
|
||||
api.get_object(Bucket=BUCKET, Key=probe)["Body"].read() == b"probe")
|
||||
|
||||
check("anonymous GET object -> 403", anonymous_status("GET", f"/{BUCKET}/{probe}") == 403)
|
||||
check("anonymous LIST bucket -> 403", anonymous_status("GET", f"/{BUCKET}?list-type=2") == 403)
|
||||
check("anonymous PUT object -> 403",
|
||||
anonymous_status("PUT", f"/{BUCKET}/{PREFIX}anonymous-{uuid.uuid4().hex}", b"x") == 403)
|
||||
|
||||
other = f"forbidden-{uuid.uuid4().hex[:12]}"
|
||||
check("api credentials: CreateBucket denied",
|
||||
denied(lambda: api.create_bucket(Bucket=other)))
|
||||
check("api credentials: DeleteBucket denied",
|
||||
denied(lambda: api.delete_bucket(Bucket=BUCKET)))
|
||||
check("api credentials: DELETE object", not denied(
|
||||
lambda: api.delete_object(Bucket=BUCKET, Key=probe)))
|
||||
|
||||
def foreign_buckets_hidden() -> bool:
|
||||
# SeaweedFS denies ListBuckets to a bucket-scoped identity; other
|
||||
# servers may answer with only the allowed bucket. Both are fine.
|
||||
try:
|
||||
names = [b["Name"] for b in api.list_buckets().get("Buckets", [])]
|
||||
except ClientError as exc:
|
||||
return error_code(exc) in {"AccessDenied", "403", "Forbidden"}
|
||||
return all(name == BUCKET for name in names)
|
||||
|
||||
check("api credentials: ListBuckets shows no foreign buckets", foreign_buckets_hidden())
|
||||
|
||||
if failures:
|
||||
print(f"FAIL: {failures} bucket checks failed; the API must not start", flush=True)
|
||||
return 1
|
||||
|
||||
print("PASS: bucket bootstrap complete (idempotent, private, least privilege)", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user