Files
mtusi2026lab4/app/s3_bootstrap.py
Ilya a97c35fc12 выполнение ЛР4 на 4
- S3: SeaweedFS 4.51 (MinIO-образы недоступны), приватный bucket,
  отдельные ограниченные ключи API, служебные порты закрыты;
  идемпотентный s3-init с проверкой приватности и прогревом записи
- вложения: потоковая загрузка PNG/JPEG, лимиты 10 MiB / 40 Мпикс,
  тип по содержимому, проверка filename, скачивание только участникам
- согласованность PG/S3: pending -> PUT -> ready, компенсация, sweeper
  с grace и dry-run; attachment_ids в сообщениях, FK по чату
- back-pressure на S3 PUT после найденного OOM SeaweedFS
- тесты student-tests, evidence, REPORT.md, DEMO.md
2026-10-28 18:06:07 +03:00

77 lines
2.8 KiB
Python

"""Idempotent bucket bootstrap with the admin identity (the API identity cannot create buckets).
Checks afterwards that anonymous access to the bucket is denied, and warms up the write path: after a
SeaweedFS restart the first PUT waits ~2.5 s for the volume topology, which a client with a short timeout
would see as a failed upload. The API starts only after this one-shot service exits 0.
"""
from __future__ import annotations
import io
import logging
import os
import sys
import time
import urllib.error
import urllib.request
from botocore.exceptions import BotoCoreError, ClientError
from app import config
from app.config import _secret
from app.objects import ObjectStore, S3Unavailable
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
logger = logging.getLogger("s3-bootstrap")
def anonymous_status(path: str) -> int:
try:
with urllib.request.urlopen(config.S3_ENDPOINT.rstrip("/") + path, timeout=3) as r:
return r.status
except urllib.error.HTTPError as exc:
return exc.code
def main() -> int:
admin = ObjectStore(_secret("S3_ADMIN_ACCESS_KEY_ID"), _secret("S3_ADMIN_SECRET_ACCESS_KEY"))
for attempt in range(1, 31):
try:
created = admin.create_bucket()
break
except (BotoCoreError, ClientError, S3Unavailable) as exc:
logger.warning("s3 not ready (attempt %d/30): %s", attempt, type(exc).__name__)
time.sleep(1)
else:
return 1
logger.info("bucket %s %s", config.S3_BUCKET, "created" if created else "already exists")
for path in ("/" + config.S3_BUCKET, "/" + config.S3_BUCKET + "/originals/probe"):
status = anonymous_status(path)
if status not in (401, 403):
logger.error("anonymous request %s returned %d: bucket is NOT private", path, status)
return 1
logger.info("anonymous list/get denied: bucket is private")
probe = "health/bootstrap-probe" # outside originals/, so the sweeper never looks at it
for attempt in range(1, 31):
started = time.monotonic()
try:
admin._s3.put_object(Bucket=admin.bucket, Key=probe, Body=io.BytesIO(b"probe"))
admin._s3.get_object(Bucket=admin.bucket, Key=probe)["Body"].read()
admin._s3.delete_object(Bucket=admin.bucket, Key=probe)
except (BotoCoreError, ClientError) as exc:
logger.warning("write probe failed (attempt %d/30): %s", attempt, type(exc).__name__)
time.sleep(1)
continue
elapsed = time.monotonic() - started
if elapsed < 0.5:
logger.info("write path warm: put/get/delete probe %.3fs", elapsed)
return 0
logger.info("write probe slow (%.2fs), repeating", elapsed)
return 1
if __name__ == "__main__":
os.umask(0o077)
sys.exit(main())