162 lines
5.5 KiB
Python
162 lines
5.5 KiB
Python
"""Idempotent bootstrap of the private attachments bucket (service s3-init).
|
|
|
|
Runs with the S3 ADMIN credentials (only this one-shot container gets them),
|
|
before any API replica starts:
|
|
|
|
1. wait until the S3 gateway answers;
|
|
2. create the bucket if it does not exist (a repeated run prints "keep");
|
|
3. self-checks, the stand does not start if one fails:
|
|
- anonymous (unsigned) GET/PUT/LIST of the bucket -> 403: bucket is private;
|
|
- API credentials can PUT/GET/DELETE under the prefix;
|
|
- API credentials can NOT create/delete buckets or touch other buckets.
|
|
|
|
The API identity and its rights are declared in infra/seaweedfs/s3.json.template.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
import botocore.session
|
|
from botocore.config import Config
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
|
|
|
|
def secret(name: str) -> str:
|
|
file_name = os.getenv(f"{name}_FILE")
|
|
if file_name:
|
|
return Path(file_name).read_text(encoding="utf-8").strip()
|
|
value = os.getenv(name)
|
|
if not value:
|
|
raise RuntimeError(f"{name} or {name}_FILE is required")
|
|
return value
|
|
|
|
|
|
ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
|
|
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
|
PREFIX = os.getenv("S3_OBJECT_PREFIX", "originals/")
|
|
|
|
|
|
def client(access_key: str, secret_key: str):
|
|
return botocore.session.get_session().create_client(
|
|
"s3",
|
|
endpoint_url=ENDPOINT,
|
|
region_name=os.getenv("S3_REGION", "us-east-1"),
|
|
aws_access_key_id=access_key,
|
|
aws_secret_access_key=secret_key,
|
|
config=Config(
|
|
signature_version="s3v4",
|
|
s3={"addressing_style": "path"},
|
|
connect_timeout=2,
|
|
read_timeout=5,
|
|
retries={"max_attempts": 1},
|
|
),
|
|
)
|
|
|
|
|
|
def error_code(exc: ClientError) -> str:
|
|
return exc.response.get("Error", {}).get("Code", "")
|
|
|
|
|
|
failures = 0
|
|
|
|
|
|
def check(label: str, ok: bool) -> None:
|
|
global failures
|
|
failures += not ok
|
|
print(("PASS " if ok else "FAIL ") + label, flush=True)
|
|
|
|
|
|
def denied(call) -> bool:
|
|
try:
|
|
call()
|
|
except ClientError as exc:
|
|
return error_code(exc) in {"AccessDenied", "403", "Forbidden", "AllAccessDisabled"}
|
|
return False
|
|
|
|
|
|
def anonymous_status(method: str, path: str, data: bytes | None = None) -> int:
|
|
request = urllib.request.Request(ENDPOINT + path, data=data, method=method)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=5) as response:
|
|
return response.status
|
|
except urllib.error.HTTPError as exc:
|
|
return exc.code
|
|
|
|
|
|
def main() -> int:
|
|
admin = client(secret("S3_ADMIN_ACCESS_KEY_ID"), secret("S3_ADMIN_SECRET_ACCESS_KEY"))
|
|
api = client(secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"))
|
|
|
|
for attempt in range(1, 61):
|
|
try:
|
|
admin.list_buckets()
|
|
break
|
|
except (BotoCoreError, ClientError) as exc:
|
|
if attempt == 60:
|
|
raise
|
|
print(f"WAIT s3 ({type(exc).__name__}), retry {attempt}", flush=True)
|
|
time.sleep(1)
|
|
|
|
try:
|
|
admin.head_bucket(Bucket=BUCKET)
|
|
print(f"keep bucket {BUCKET}", flush=True)
|
|
except ClientError as exc:
|
|
if error_code(exc) not in {"404", "NoSuchBucket", "NotFound"}:
|
|
raise
|
|
try:
|
|
admin.create_bucket(Bucket=BUCKET)
|
|
print(f"create bucket {BUCKET}", flush=True)
|
|
except ClientError as create_exc:
|
|
# Two bootstrap runs at the same time: the other one won.
|
|
if error_code(create_exc) not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
|
|
raise
|
|
print(f"keep bucket {BUCKET}", flush=True)
|
|
|
|
probe = f"{PREFIX}bootstrap-probe-{uuid.uuid4().hex}"
|
|
|
|
check("api credentials: PUT object", not denied(
|
|
lambda: api.put_object(Bucket=BUCKET, Key=probe, Body=b"probe")))
|
|
check("api credentials: GET object",
|
|
api.get_object(Bucket=BUCKET, Key=probe)["Body"].read() == b"probe")
|
|
|
|
check("anonymous GET object -> 403", anonymous_status("GET", f"/{BUCKET}/{probe}") == 403)
|
|
check("anonymous LIST bucket -> 403", anonymous_status("GET", f"/{BUCKET}?list-type=2") == 403)
|
|
check("anonymous PUT object -> 403",
|
|
anonymous_status("PUT", f"/{BUCKET}/{PREFIX}anonymous-{uuid.uuid4().hex}", b"x") == 403)
|
|
|
|
other = f"forbidden-{uuid.uuid4().hex[:12]}"
|
|
check("api credentials: CreateBucket denied",
|
|
denied(lambda: api.create_bucket(Bucket=other)))
|
|
check("api credentials: DeleteBucket denied",
|
|
denied(lambda: api.delete_bucket(Bucket=BUCKET)))
|
|
check("api credentials: DELETE object", not denied(
|
|
lambda: api.delete_object(Bucket=BUCKET, Key=probe)))
|
|
|
|
def foreign_buckets_hidden() -> bool:
|
|
# SeaweedFS denies ListBuckets to a bucket-scoped identity; other
|
|
# servers may answer with only the allowed bucket. Both are fine.
|
|
try:
|
|
names = [b["Name"] for b in api.list_buckets().get("Buckets", [])]
|
|
except ClientError as exc:
|
|
return error_code(exc) in {"AccessDenied", "403", "Forbidden"}
|
|
return all(name == BUCKET for name in names)
|
|
|
|
check("api credentials: ListBuckets shows no foreign buckets", foreign_buckets_hidden())
|
|
|
|
if failures:
|
|
print(f"FAIL: {failures} bucket checks failed; the API must not start", flush=True)
|
|
return 1
|
|
|
|
print("PASS: bucket bootstrap complete (idempotent, private, least privilege)", flush=True)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|