forked from mtusi2026/mtusi2026lab4
- S3: SeaweedFS 4.51 (MinIO-образы недоступны), приватный bucket, отдельные ограниченные ключи API, служебные порты закрыты; идемпотентный s3-init с проверкой приватности и прогревом записи - вложения: потоковая загрузка PNG/JPEG, лимиты 10 MiB / 40 Мпикс, тип по содержимому, проверка filename, скачивание только участникам - согласованность PG/S3: pending -> PUT -> ready, компенсация, sweeper с grace и dry-run; attachment_ids в сообщениях, FK по чату - back-pressure на S3 PUT после найденного OOM SeaweedFS - тесты student-tests, evidence, REPORT.md, DEMO.md
77 lines
2.8 KiB
Python
77 lines
2.8 KiB
Python
"""Idempotent bucket bootstrap with the admin identity (the API identity cannot create buckets).
|
|
|
|
Checks afterwards that anonymous access to the bucket is denied, and warms up the write path: after a
|
|
SeaweedFS restart the first PUT waits ~2.5 s for the volume topology, which a client with a short timeout
|
|
would see as a failed upload. The API starts only after this one-shot service exits 0.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import logging
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
|
|
from app import config
|
|
from app.config import _secret
|
|
from app.objects import ObjectStore, S3Unavailable
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
|
|
logger = logging.getLogger("s3-bootstrap")
|
|
|
|
|
|
def anonymous_status(path: str) -> int:
|
|
try:
|
|
with urllib.request.urlopen(config.S3_ENDPOINT.rstrip("/") + path, timeout=3) as r:
|
|
return r.status
|
|
except urllib.error.HTTPError as exc:
|
|
return exc.code
|
|
|
|
|
|
def main() -> int:
|
|
admin = ObjectStore(_secret("S3_ADMIN_ACCESS_KEY_ID"), _secret("S3_ADMIN_SECRET_ACCESS_KEY"))
|
|
for attempt in range(1, 31):
|
|
try:
|
|
created = admin.create_bucket()
|
|
break
|
|
except (BotoCoreError, ClientError, S3Unavailable) as exc:
|
|
logger.warning("s3 not ready (attempt %d/30): %s", attempt, type(exc).__name__)
|
|
time.sleep(1)
|
|
else:
|
|
return 1
|
|
logger.info("bucket %s %s", config.S3_BUCKET, "created" if created else "already exists")
|
|
|
|
for path in ("/" + config.S3_BUCKET, "/" + config.S3_BUCKET + "/originals/probe"):
|
|
status = anonymous_status(path)
|
|
if status not in (401, 403):
|
|
logger.error("anonymous request %s returned %d: bucket is NOT private", path, status)
|
|
return 1
|
|
logger.info("anonymous list/get denied: bucket is private")
|
|
|
|
probe = "health/bootstrap-probe" # outside originals/, so the sweeper never looks at it
|
|
for attempt in range(1, 31):
|
|
started = time.monotonic()
|
|
try:
|
|
admin._s3.put_object(Bucket=admin.bucket, Key=probe, Body=io.BytesIO(b"probe"))
|
|
admin._s3.get_object(Bucket=admin.bucket, Key=probe)["Body"].read()
|
|
admin._s3.delete_object(Bucket=admin.bucket, Key=probe)
|
|
except (BotoCoreError, ClientError) as exc:
|
|
logger.warning("write probe failed (attempt %d/30): %s", attempt, type(exc).__name__)
|
|
time.sleep(1)
|
|
continue
|
|
elapsed = time.monotonic() - started
|
|
if elapsed < 0.5:
|
|
logger.info("write path warm: put/get/delete probe %.3fs", elapsed)
|
|
return 0
|
|
logger.info("write probe slow (%.2fs), repeating", elapsed)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
os.umask(0o077)
|
|
sys.exit(main())
|