more before level3 changes
This commit is contained in:
@@ -5,6 +5,15 @@ from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ObjectSettings:
|
||||
endpoint: str
|
||||
region: str
|
||||
bucket: str
|
||||
access_key: str
|
||||
secret_key: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
host: str
|
||||
@@ -59,6 +68,32 @@ def _database_url() -> str:
|
||||
)
|
||||
|
||||
|
||||
def load_object_settings() -> ObjectSettings:
|
||||
"""S3 credentials come from files. The values are not logged."""
|
||||
endpoint = os.getenv('S3_ENDPOINT', '').strip()
|
||||
region = os.getenv('S3_REGION', 'us-east-1').strip() or 'us-east-1'
|
||||
bucket = os.getenv('S3_BUCKET', '').strip()
|
||||
key_file = os.getenv('S3_ACCESS_KEY_ID_FILE', '').strip()
|
||||
secret_file = os.getenv('S3_SECRET_ACCESS_KEY_FILE', '').strip()
|
||||
missing = [
|
||||
name for name, value in (
|
||||
('S3_ENDPOINT', endpoint),
|
||||
('S3_BUCKET', bucket),
|
||||
('S3_ACCESS_KEY_ID_FILE', key_file),
|
||||
('S3_SECRET_ACCESS_KEY_FILE', secret_file),
|
||||
) if not value
|
||||
]
|
||||
if missing:
|
||||
raise RuntimeError('%s is required' % ', '.join(missing))
|
||||
return ObjectSettings(
|
||||
endpoint,
|
||||
region,
|
||||
bucket,
|
||||
_read_secret_file('S3_ACCESS_KEY_ID_FILE', key_file),
|
||||
_read_secret_file('S3_SECRET_ACCESS_KEY_FILE', secret_file),
|
||||
)
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
return Settings(
|
||||
host=os.getenv('APP_HOST', '0.0.0.0'),
|
||||
|
||||
+8
-2
@@ -10,9 +10,10 @@ from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from .config import load_settings
|
||||
from .config import load_object_settings, load_settings
|
||||
from .db import open_pool
|
||||
from .errors import ApiError
|
||||
from .objects import ObjectStore
|
||||
from .routes import router
|
||||
from .store import Store
|
||||
|
||||
@@ -59,7 +60,12 @@ async def lifespan(app: FastAPI):
|
||||
settings = load_settings()
|
||||
pool = await open_pool(settings.database_url)
|
||||
try:
|
||||
app.state.store = Store(pool, settings.session_ttl_seconds, settings.idempotency_ttl_seconds)
|
||||
app.state.store = Store(
|
||||
pool,
|
||||
settings.session_ttl_seconds,
|
||||
settings.idempotency_ttl_seconds,
|
||||
objects=ObjectStore(load_object_settings()),
|
||||
)
|
||||
logger.info('instance %s started', settings.instance_id)
|
||||
yield
|
||||
finally:
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Private S3 bucket. Callers pass a server-generated key; the display filename never becomes one."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import TimeoutError as FutureTimeout
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .config import ObjectSettings
|
||||
|
||||
# Docker DNS can spend several seconds on a stopped MinIO name. The HTTP handler
|
||||
# must still answer inside HAProxy's one-second health check.
|
||||
_POOL = ThreadPoolExecutor(max_workers=16, thread_name_prefix='s3')
|
||||
|
||||
|
||||
class MissingObject(Exception):
|
||||
"""The key is not in the bucket."""
|
||||
|
||||
|
||||
class StorageError(Exception):
|
||||
"""The object store could not complete the call. The message stays internal."""
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self, settings: ObjectSettings) -> None:
|
||||
self._bucket = settings.bucket
|
||||
self._client = boto3.client(
|
||||
's3',
|
||||
endpoint_url=settings.endpoint,
|
||||
region_name=settings.region,
|
||||
aws_access_key_id=settings.access_key,
|
||||
aws_secret_access_key=settings.secret_key,
|
||||
config=Config(
|
||||
signature_version='s3v4',
|
||||
s3={'addressing_style': 'path'},
|
||||
connect_timeout=1,
|
||||
read_timeout=2,
|
||||
retries={'max_attempts': 1},
|
||||
),
|
||||
)
|
||||
|
||||
def ready(self) -> None:
|
||||
self._bounded(self._head, 0.7)
|
||||
|
||||
def put(self, key: str, body: bytes, content_type: str) -> None:
|
||||
self._bounded(lambda: self._put(key, body, content_type), 8)
|
||||
|
||||
def get(self, key: str) -> bytes:
|
||||
return self._bounded(lambda: self._get(key), 8)
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self._bounded(lambda: self._delete(key), 8)
|
||||
|
||||
def _bounded(self, fn, timeout: float):
|
||||
future = _POOL.submit(fn)
|
||||
try:
|
||||
return future.result(timeout=timeout)
|
||||
except FutureTimeout:
|
||||
raise StorageError() from None
|
||||
|
||||
def _head(self) -> None:
|
||||
try:
|
||||
self._client.head_bucket(Bucket=self._bucket)
|
||||
except (BotoCoreError, ClientError):
|
||||
raise StorageError() from None
|
||||
|
||||
def _put(self, key: str, body: bytes, content_type: str) -> None:
|
||||
try:
|
||||
self._client.put_object(
|
||||
Bucket=self._bucket,
|
||||
Key=key,
|
||||
Body=body,
|
||||
ContentType=content_type,
|
||||
ContentLength=len(body),
|
||||
)
|
||||
except (BotoCoreError, ClientError):
|
||||
raise StorageError() from None
|
||||
|
||||
def _get(self, key: str) -> bytes:
|
||||
try:
|
||||
response = self._client.get_object(Bucket=self._bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
code = exc.response.get('Error', {}).get('Code', '')
|
||||
if code in ('NoSuchKey', '404', 'NotFound'):
|
||||
raise MissingObject() from None
|
||||
raise StorageError() from None
|
||||
except BotoCoreError:
|
||||
raise StorageError() from None
|
||||
payload = response['Body']
|
||||
try:
|
||||
return payload.read()
|
||||
finally:
|
||||
payload.close()
|
||||
|
||||
def _delete(self, key: str) -> None:
|
||||
try:
|
||||
self._client.delete_object(Bucket=self._bucket, Key=key)
|
||||
except (BotoCoreError, ClientError):
|
||||
raise StorageError() from None
|
||||
+69
-24
@@ -1,5 +1,7 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
import threading
|
||||
import uuid
|
||||
@@ -11,9 +13,12 @@ import asyncpg
|
||||
|
||||
from .errors import conflict, invalid_request, not_found, unavailable
|
||||
from .migrate import expected_versions
|
||||
from .objects import MissingObject, ObjectStore, StorageError
|
||||
from .pagination import Key
|
||||
from .passwords import hash_password, password_matches
|
||||
|
||||
logger = logging.getLogger('messenger')
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class User:
|
||||
@@ -132,10 +137,21 @@ def _db_down(exc: BaseException) -> bool:
|
||||
|
||||
|
||||
class Store:
|
||||
"""PostgreSQL is the only state shared by replicas: users, chats, sessions, messages and attachment bytes."""
|
||||
"""PostgreSQL holds users, chats, sessions, messages and attachment metadata.
|
||||
|
||||
def __init__(self, pool: asyncpg.Pool, session_ttl_seconds: int = 3600, idempotency_ttl_seconds: int = 86400) -> None:
|
||||
Original bytes are in the private bucket. Any replica can read them by the server-assigned key.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
pool: asyncpg.Pool,
|
||||
session_ttl_seconds: int = 3600,
|
||||
idempotency_ttl_seconds: int = 86400,
|
||||
*,
|
||||
objects: ObjectStore,
|
||||
) -> None:
|
||||
self._pool = pool
|
||||
self._objects = objects
|
||||
self._session_ttl = timedelta(seconds=session_ttl_seconds)
|
||||
self._idempotency_ttl = timedelta(seconds=idempotency_ttl_seconds)
|
||||
self._lock = threading.Lock()
|
||||
@@ -152,7 +168,7 @@ class Store:
|
||||
return moment
|
||||
|
||||
async def ready(self) -> None:
|
||||
"""Ready only when PostgreSQL answers and the separate migrator has applied this build's schema."""
|
||||
"""Ready when PostgreSQL has this build's schema and the private bucket answers."""
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
applied = await conn.fetchval(
|
||||
@@ -163,6 +179,10 @@ class Store:
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
if applied != len(self._migrations):
|
||||
raise unavailable('Схема БД ещё не применена')
|
||||
try:
|
||||
await asyncio.to_thread(self._objects.ready)
|
||||
except StorageError:
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
|
||||
async def create_user(self, username: str, display_name: str, password: str) -> User:
|
||||
password_hash = hash_password(password)
|
||||
@@ -362,27 +382,37 @@ class Store:
|
||||
chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id')
|
||||
owner_uuid = _as_uuid(owner_id, 'Некорректный user_id')
|
||||
attachment_id = uuid.uuid4()
|
||||
# The key is not derived from filename. A crash after PUT and before INSERT leaves an
|
||||
# unreferenced object; a failed INSERT deletes it. 201 is returned only after both succeed.
|
||||
object_key = 'originals/%s' % attachment_id
|
||||
created = self._now()
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
'''
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
object_key, body, created_at, updated_at
|
||||
try:
|
||||
await asyncio.to_thread(self._objects.put, object_key, body, content_type)
|
||||
except StorageError:
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
'''
|
||||
INSERT INTO attachments (
|
||||
id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
object_key, created_at, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $9)
|
||||
''',
|
||||
attachment_id,
|
||||
chat_uuid,
|
||||
owner_uuid,
|
||||
filename,
|
||||
content_type,
|
||||
len(body),
|
||||
sha256,
|
||||
object_key,
|
||||
created,
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $10)
|
||||
''',
|
||||
attachment_id,
|
||||
chat_uuid,
|
||||
owner_uuid,
|
||||
filename,
|
||||
content_type,
|
||||
len(body),
|
||||
sha256,
|
||||
uuid.uuid4().hex,
|
||||
body,
|
||||
created,
|
||||
)
|
||||
except Exception:
|
||||
await self._discard_object(object_key)
|
||||
raise
|
||||
stamp = _stamp(created)
|
||||
return Attachment(
|
||||
str(attachment_id),
|
||||
@@ -432,7 +462,7 @@ class Store:
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
'''
|
||||
SELECT a.content_type, a.body
|
||||
SELECT a.content_type, a.size_bytes, a.sha256, a.object_key
|
||||
FROM attachments a
|
||||
JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2
|
||||
WHERE a.id = $1
|
||||
@@ -442,7 +472,22 @@ class Store:
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
return row['content_type'], bytes(row['body'])
|
||||
try:
|
||||
body = await asyncio.to_thread(self._objects.get, row['object_key'])
|
||||
except MissingObject:
|
||||
return None
|
||||
except StorageError:
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
if len(body) != row['size_bytes'] or hashlib.sha256(body).hexdigest() != row['sha256']:
|
||||
logger.warning('object bytes do not match attachment metadata')
|
||||
raise unavailable('Хранилище недоступно')
|
||||
return row['content_type'], body
|
||||
|
||||
async def _discard_object(self, object_key: str) -> None:
|
||||
try:
|
||||
await asyncio.to_thread(self._objects.delete, object_key)
|
||||
except StorageError:
|
||||
logger.warning('object left in the bucket after a failed metadata write')
|
||||
|
||||
async def add_message(
|
||||
self,
|
||||
|
||||
@@ -158,11 +158,24 @@ services:
|
||||
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-3}
|
||||
DB_POOL_MIN: ${DB_POOL_MIN:-1}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-10}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:?set in .env}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:?set in .env}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets:
|
||||
- source: postgres_password
|
||||
uid: "10001"
|
||||
gid: "10001"
|
||||
mode: 0400
|
||||
- source: s3_access_key_id
|
||||
uid: "10001"
|
||||
gid: "10001"
|
||||
mode: 0400
|
||||
- source: s3_secret_access_key
|
||||
uid: "10001"
|
||||
gid: "10001"
|
||||
mode: 0400
|
||||
networks:
|
||||
- edge
|
||||
- data
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_len;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000);
|
||||
|
||||
-- filename is the display name. object_key is assigned by the server and is never taken from filename.
|
||||
-- The original bytes sit next to the metadata so any API replica can read them.
|
||||
-- filename is the display name only. object_key is assigned by the server and is never taken from filename.
|
||||
-- The original bytes live in the private S3 bucket under that key, not in this table.
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats (id),
|
||||
@@ -14,15 +14,13 @@ CREATE TABLE attachments (
|
||||
size_bytes INTEGER NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL,
|
||||
body BYTEA NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL,
|
||||
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_object_key_unique UNIQUE (object_key),
|
||||
CONSTRAINT attachments_body_len CHECK (octet_length(body) = size_bytes)
|
||||
CONSTRAINT attachments_object_key_unique UNIQUE (object_key)
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_idx ON attachments (chat_id);
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Databases that already applied the earlier 004 kept the original in a BYTEA column.
|
||||
-- Drop it. Fresh installs create the table without that column; both statements are then no-ops.
|
||||
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_body_len;
|
||||
ALTER TABLE attachments DROP COLUMN IF EXISTS body;
|
||||
+8
-1
@@ -1,4 +1,6 @@
|
||||
# Direct: fastapi, pydantic, uvicorn, argon2-cffi. The rest are their dependencies, pinned.
|
||||
# Direct: fastapi, pydantic, uvicorn, argon2-cffi, boto3. The rest are their dependencies, pinned.
|
||||
boto3==1.43.126
|
||||
botocore==1.43.126
|
||||
asyncpg==0.32.0
|
||||
annotated-doc==0.0.5
|
||||
annotated-types==0.8.0
|
||||
@@ -10,10 +12,15 @@ click==8.5.0
|
||||
fastapi==0.141.1
|
||||
h11==0.16.0
|
||||
idna==3.20
|
||||
jmespath==1.1.0
|
||||
pycparser==3.0
|
||||
python-dateutil==2.9.0.post0
|
||||
pydantic==2.13.5
|
||||
pydantic_core==2.46.5
|
||||
s3transfer==0.19.3
|
||||
six==1.17.0
|
||||
starlette==1.7.0
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
uvicorn==0.53.0
|
||||
|
||||
Reference in New Issue
Block a user