Files
mtusi2026lab6/scripts/s3_bootstrap.py
T

162 lines
5.5 KiB
Python

"""Idempotent bootstrap of the private attachments bucket (service s3-init).
Runs with the S3 ADMIN credentials (only this one-shot container gets them),
before any API replica starts:
1. wait until the S3 gateway answers;
2. create the bucket if it does not exist (a repeated run prints "keep");
3. self-checks, the stand does not start if one fails:
- anonymous (unsigned) GET/PUT/LIST of the bucket -> 403: bucket is private;
- API credentials can PUT/GET/DELETE under the prefix;
- API credentials can NOT create/delete buckets or touch other buckets.
The API identity and its rights are declared in infra/seaweedfs/s3.json.template.
"""
from __future__ import annotations
import os
import sys
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
import botocore.session
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
def secret(name: str) -> str:
file_name = os.getenv(f"{name}_FILE")
if file_name:
return Path(file_name).read_text(encoding="utf-8").strip()
value = os.getenv(name)
if not value:
raise RuntimeError(f"{name} or {name}_FILE is required")
return value
ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
PREFIX = os.getenv("S3_OBJECT_PREFIX", "originals/")
def client(access_key: str, secret_key: str):
return botocore.session.get_session().create_client(
"s3",
endpoint_url=ENDPOINT,
region_name=os.getenv("S3_REGION", "us-east-1"),
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=2,
read_timeout=5,
retries={"max_attempts": 1},
),
)
def error_code(exc: ClientError) -> str:
return exc.response.get("Error", {}).get("Code", "")
failures = 0
def check(label: str, ok: bool) -> None:
global failures
failures += not ok
print(("PASS " if ok else "FAIL ") + label, flush=True)
def denied(call) -> bool:
try:
call()
except ClientError as exc:
return error_code(exc) in {"AccessDenied", "403", "Forbidden", "AllAccessDisabled"}
return False
def anonymous_status(method: str, path: str, data: bytes | None = None) -> int:
request = urllib.request.Request(ENDPOINT + path, data=data, method=method)
try:
with urllib.request.urlopen(request, timeout=5) as response:
return response.status
except urllib.error.HTTPError as exc:
return exc.code
def main() -> int:
admin = client(secret("S3_ADMIN_ACCESS_KEY_ID"), secret("S3_ADMIN_SECRET_ACCESS_KEY"))
api = client(secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"))
for attempt in range(1, 61):
try:
admin.list_buckets()
break
except (BotoCoreError, ClientError) as exc:
if attempt == 60:
raise
print(f"WAIT s3 ({type(exc).__name__}), retry {attempt}", flush=True)
time.sleep(1)
try:
admin.head_bucket(Bucket=BUCKET)
print(f"keep bucket {BUCKET}", flush=True)
except ClientError as exc:
if error_code(exc) not in {"404", "NoSuchBucket", "NotFound"}:
raise
try:
admin.create_bucket(Bucket=BUCKET)
print(f"create bucket {BUCKET}", flush=True)
except ClientError as create_exc:
# Two bootstrap runs at the same time: the other one won.
if error_code(create_exc) not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
raise
print(f"keep bucket {BUCKET}", flush=True)
probe = f"{PREFIX}bootstrap-probe-{uuid.uuid4().hex}"
check("api credentials: PUT object", not denied(
lambda: api.put_object(Bucket=BUCKET, Key=probe, Body=b"probe")))
check("api credentials: GET object",
api.get_object(Bucket=BUCKET, Key=probe)["Body"].read() == b"probe")
check("anonymous GET object -> 403", anonymous_status("GET", f"/{BUCKET}/{probe}") == 403)
check("anonymous LIST bucket -> 403", anonymous_status("GET", f"/{BUCKET}?list-type=2") == 403)
check("anonymous PUT object -> 403",
anonymous_status("PUT", f"/{BUCKET}/{PREFIX}anonymous-{uuid.uuid4().hex}", b"x") == 403)
other = f"forbidden-{uuid.uuid4().hex[:12]}"
check("api credentials: CreateBucket denied",
denied(lambda: api.create_bucket(Bucket=other)))
check("api credentials: DeleteBucket denied",
denied(lambda: api.delete_bucket(Bucket=BUCKET)))
check("api credentials: DELETE object", not denied(
lambda: api.delete_object(Bucket=BUCKET, Key=probe)))
def foreign_buckets_hidden() -> bool:
# SeaweedFS denies ListBuckets to a bucket-scoped identity; other
# servers may answer with only the allowed bucket. Both are fine.
try:
names = [b["Name"] for b in api.list_buckets().get("Buckets", [])]
except ClientError as exc:
return error_code(exc) in {"AccessDenied", "403", "Forbidden"}
return all(name == BUCKET for name in names)
check("api credentials: ListBuckets shows no foreign buckets", foreign_buckets_hidden())
if failures:
print(f"FAIL: {failures} bucket checks failed; the API must not start", flush=True)
return 1
print("PASS: bucket bootstrap complete (idempotent, private, least privilege)", flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())