From 03469378478c32da07cb40f8d223f1cba7e8e9e3 Mon Sep 17 00:00:00 2001 From: Sunrise443 Date: Mon, 9 Nov 2026 18:25:54 +0300 Subject: [PATCH] more before level3 changes --- app/config.py | 35 ++++++++ app/main.py | 10 ++- app/objects.py | 99 +++++++++++++++++++++++ app/store.py | 93 +++++++++++++++------ compose.yaml | 13 +++ migrations/004_attachments.sql | 8 +- migrations/005_attachment_bytes_in_s3.sql | 4 + requirements.txt | 9 ++- 8 files changed, 239 insertions(+), 32 deletions(-) create mode 100644 app/objects.py create mode 100644 migrations/005_attachment_bytes_in_s3.sql diff --git a/app/config.py b/app/config.py index 07ac86e5..a71beb94 100644 --- a/app/config.py +++ b/app/config.py @@ -5,6 +5,15 @@ from pathlib import Path from urllib.parse import quote +@dataclass(frozen=True) +class ObjectSettings: + endpoint: str + region: str + bucket: str + access_key: str + secret_key: str + + @dataclass(frozen=True) class Settings: host: str @@ -59,6 +68,32 @@ def _database_url() -> str: ) +def load_object_settings() -> ObjectSettings: + """S3 credentials come from files. The values are not logged.""" + endpoint = os.getenv('S3_ENDPOINT', '').strip() + region = os.getenv('S3_REGION', 'us-east-1').strip() or 'us-east-1' + bucket = os.getenv('S3_BUCKET', '').strip() + key_file = os.getenv('S3_ACCESS_KEY_ID_FILE', '').strip() + secret_file = os.getenv('S3_SECRET_ACCESS_KEY_FILE', '').strip() + missing = [ + name for name, value in ( + ('S3_ENDPOINT', endpoint), + ('S3_BUCKET', bucket), + ('S3_ACCESS_KEY_ID_FILE', key_file), + ('S3_SECRET_ACCESS_KEY_FILE', secret_file), + ) if not value + ] + if missing: + raise RuntimeError('%s is required' % ', '.join(missing)) + return ObjectSettings( + endpoint, + region, + bucket, + _read_secret_file('S3_ACCESS_KEY_ID_FILE', key_file), + _read_secret_file('S3_SECRET_ACCESS_KEY_FILE', secret_file), + ) + + def load_settings() -> Settings: return Settings( host=os.getenv('APP_HOST', '0.0.0.0'), diff --git a/app/main.py b/app/main.py index b20713cc..63587886 100644 --- a/app/main.py +++ b/app/main.py @@ -10,9 +10,10 @@ from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse from starlette.exceptions import HTTPException as StarletteHTTPException -from .config import load_settings +from .config import load_object_settings, load_settings from .db import open_pool from .errors import ApiError +from .objects import ObjectStore from .routes import router from .store import Store @@ -59,7 +60,12 @@ async def lifespan(app: FastAPI): settings = load_settings() pool = await open_pool(settings.database_url) try: - app.state.store = Store(pool, settings.session_ttl_seconds, settings.idempotency_ttl_seconds) + app.state.store = Store( + pool, + settings.session_ttl_seconds, + settings.idempotency_ttl_seconds, + objects=ObjectStore(load_object_settings()), + ) logger.info('instance %s started', settings.instance_id) yield finally: diff --git a/app/objects.py b/app/objects.py new file mode 100644 index 00000000..76312c6c --- /dev/null +++ b/app/objects.py @@ -0,0 +1,99 @@ +"""Private S3 bucket. Callers pass a server-generated key; the display filename never becomes one.""" +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import TimeoutError as FutureTimeout + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from .config import ObjectSettings + +# Docker DNS can spend several seconds on a stopped MinIO name. The HTTP handler +# must still answer inside HAProxy's one-second health check. +_POOL = ThreadPoolExecutor(max_workers=16, thread_name_prefix='s3') + + +class MissingObject(Exception): + """The key is not in the bucket.""" + + +class StorageError(Exception): + """The object store could not complete the call. The message stays internal.""" + + +class ObjectStore: + def __init__(self, settings: ObjectSettings) -> None: + self._bucket = settings.bucket + self._client = boto3.client( + 's3', + endpoint_url=settings.endpoint, + region_name=settings.region, + aws_access_key_id=settings.access_key, + aws_secret_access_key=settings.secret_key, + config=Config( + signature_version='s3v4', + s3={'addressing_style': 'path'}, + connect_timeout=1, + read_timeout=2, + retries={'max_attempts': 1}, + ), + ) + + def ready(self) -> None: + self._bounded(self._head, 0.7) + + def put(self, key: str, body: bytes, content_type: str) -> None: + self._bounded(lambda: self._put(key, body, content_type), 8) + + def get(self, key: str) -> bytes: + return self._bounded(lambda: self._get(key), 8) + + def delete(self, key: str) -> None: + self._bounded(lambda: self._delete(key), 8) + + def _bounded(self, fn, timeout: float): + future = _POOL.submit(fn) + try: + return future.result(timeout=timeout) + except FutureTimeout: + raise StorageError() from None + + def _head(self) -> None: + try: + self._client.head_bucket(Bucket=self._bucket) + except (BotoCoreError, ClientError): + raise StorageError() from None + + def _put(self, key: str, body: bytes, content_type: str) -> None: + try: + self._client.put_object( + Bucket=self._bucket, + Key=key, + Body=body, + ContentType=content_type, + ContentLength=len(body), + ) + except (BotoCoreError, ClientError): + raise StorageError() from None + + def _get(self, key: str) -> bytes: + try: + response = self._client.get_object(Bucket=self._bucket, Key=key) + except ClientError as exc: + code = exc.response.get('Error', {}).get('Code', '') + if code in ('NoSuchKey', '404', 'NotFound'): + raise MissingObject() from None + raise StorageError() from None + except BotoCoreError: + raise StorageError() from None + payload = response['Body'] + try: + return payload.read() + finally: + payload.close() + + def _delete(self, key: str) -> None: + try: + self._client.delete_object(Bucket=self._bucket, Key=key) + except (BotoCoreError, ClientError): + raise StorageError() from None diff --git a/app/store.py b/app/store.py index 53e56846..60c4c6da 100644 --- a/app/store.py +++ b/app/store.py @@ -1,5 +1,7 @@ +import asyncio import hashlib import json +import logging import secrets import threading import uuid @@ -11,9 +13,12 @@ import asyncpg from .errors import conflict, invalid_request, not_found, unavailable from .migrate import expected_versions +from .objects import MissingObject, ObjectStore, StorageError from .pagination import Key from .passwords import hash_password, password_matches +logger = logging.getLogger('messenger') + @dataclass(frozen=True) class User: @@ -132,10 +137,21 @@ def _db_down(exc: BaseException) -> bool: class Store: - """PostgreSQL is the only state shared by replicas: users, chats, sessions, messages and attachment bytes.""" + """PostgreSQL holds users, chats, sessions, messages and attachment metadata. - def __init__(self, pool: asyncpg.Pool, session_ttl_seconds: int = 3600, idempotency_ttl_seconds: int = 86400) -> None: + Original bytes are in the private bucket. Any replica can read them by the server-assigned key. + """ + + def __init__( + self, + pool: asyncpg.Pool, + session_ttl_seconds: int = 3600, + idempotency_ttl_seconds: int = 86400, + *, + objects: ObjectStore, + ) -> None: self._pool = pool + self._objects = objects self._session_ttl = timedelta(seconds=session_ttl_seconds) self._idempotency_ttl = timedelta(seconds=idempotency_ttl_seconds) self._lock = threading.Lock() @@ -152,7 +168,7 @@ class Store: return moment async def ready(self) -> None: - """Ready only when PostgreSQL answers and the separate migrator has applied this build's schema.""" + """Ready when PostgreSQL has this build's schema and the private bucket answers.""" try: async with self._pool.acquire() as conn: applied = await conn.fetchval( @@ -163,6 +179,10 @@ class Store: raise unavailable('Хранилище недоступно') from None if applied != len(self._migrations): raise unavailable('Схема БД ещё не применена') + try: + await asyncio.to_thread(self._objects.ready) + except StorageError: + raise unavailable('Хранилище недоступно') from None async def create_user(self, username: str, display_name: str, password: str) -> User: password_hash = hash_password(password) @@ -362,27 +382,37 @@ class Store: chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id') owner_uuid = _as_uuid(owner_id, 'Некорректный user_id') attachment_id = uuid.uuid4() + # The key is not derived from filename. A crash after PUT and before INSERT leaves an + # unreferenced object; a failed INSERT deletes it. 201 is returned only after both succeed. + object_key = 'originals/%s' % attachment_id created = self._now() - async with self._pool.acquire() as conn: - await conn.execute( - ''' - INSERT INTO attachments ( - id, chat_id, owner_id, filename, content_type, size_bytes, sha256, - object_key, body, created_at, updated_at + try: + await asyncio.to_thread(self._objects.put, object_key, body, content_type) + except StorageError: + raise unavailable('Хранилище недоступно') from None + try: + async with self._pool.acquire() as conn: + await conn.execute( + ''' + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $9) + ''', + attachment_id, + chat_uuid, + owner_uuid, + filename, + content_type, + len(body), + sha256, + object_key, + created, ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $10) - ''', - attachment_id, - chat_uuid, - owner_uuid, - filename, - content_type, - len(body), - sha256, - uuid.uuid4().hex, - body, - created, - ) + except Exception: + await self._discard_object(object_key) + raise stamp = _stamp(created) return Attachment( str(attachment_id), @@ -432,7 +462,7 @@ class Store: async with self._pool.acquire() as conn: row = await conn.fetchrow( ''' - SELECT a.content_type, a.body + SELECT a.content_type, a.size_bytes, a.sha256, a.object_key FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 WHERE a.id = $1 @@ -442,7 +472,22 @@ class Store: ) if row is None: return None - return row['content_type'], bytes(row['body']) + try: + body = await asyncio.to_thread(self._objects.get, row['object_key']) + except MissingObject: + return None + except StorageError: + raise unavailable('Хранилище недоступно') from None + if len(body) != row['size_bytes'] or hashlib.sha256(body).hexdigest() != row['sha256']: + logger.warning('object bytes do not match attachment metadata') + raise unavailable('Хранилище недоступно') + return row['content_type'], body + + async def _discard_object(self, object_key: str) -> None: + try: + await asyncio.to_thread(self._objects.delete, object_key) + except StorageError: + logger.warning('object left in the bucket after a failed metadata write') async def add_message( self, diff --git a/compose.yaml b/compose.yaml index e818d0c3..d000899d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -158,11 +158,24 @@ services: SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-3} DB_POOL_MIN: ${DB_POOL_MIN:-1} DB_POOL_MAX: ${DB_POOL_MAX:-10} + S3_ENDPOINT: ${S3_ENDPOINT:?set in .env} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:?set in .env} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key secrets: - source: postgres_password uid: "10001" gid: "10001" mode: 0400 + - source: s3_access_key_id + uid: "10001" + gid: "10001" + mode: 0400 + - source: s3_secret_access_key + uid: "10001" + gid: "10001" + mode: 0400 networks: - edge - data diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql index 348f04c4..2f4673d4 100644 --- a/migrations/004_attachments.sql +++ b/migrations/004_attachments.sql @@ -3,8 +3,8 @@ ALTER TABLE messages DROP CONSTRAINT messages_text_len; ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000); --- filename is the display name. object_key is assigned by the server and is never taken from filename. --- The original bytes sit next to the metadata so any API replica can read them. +-- filename is the display name only. object_key is assigned by the server and is never taken from filename. +-- The original bytes live in the private S3 bucket under that key, not in this table. CREATE TABLE attachments ( id UUID PRIMARY KEY, chat_id UUID NOT NULL REFERENCES chats (id), @@ -14,15 +14,13 @@ CREATE TABLE attachments ( size_bytes INTEGER NOT NULL, sha256 TEXT NOT NULL, object_key TEXT NOT NULL, - body BYTEA NOT NULL, created_at TIMESTAMPTZ NOT NULL, updated_at TIMESTAMPTZ NOT NULL, CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), - CONSTRAINT attachments_object_key_unique UNIQUE (object_key), - CONSTRAINT attachments_body_len CHECK (octet_length(body) = size_bytes) + CONSTRAINT attachments_object_key_unique UNIQUE (object_key) ); CREATE INDEX attachments_chat_idx ON attachments (chat_id); diff --git a/migrations/005_attachment_bytes_in_s3.sql b/migrations/005_attachment_bytes_in_s3.sql new file mode 100644 index 00000000..81fdfdef --- /dev/null +++ b/migrations/005_attachment_bytes_in_s3.sql @@ -0,0 +1,4 @@ +-- Databases that already applied the earlier 004 kept the original in a BYTEA column. +-- Drop it. Fresh installs create the table without that column; both statements are then no-ops. +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_body_len; +ALTER TABLE attachments DROP COLUMN IF EXISTS body; diff --git a/requirements.txt b/requirements.txt index 0c7146cb..3298a46c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ -# Direct: fastapi, pydantic, uvicorn, argon2-cffi. The rest are their dependencies, pinned. +# Direct: fastapi, pydantic, uvicorn, argon2-cffi, boto3. The rest are their dependencies, pinned. +boto3==1.43.126 +botocore==1.43.126 asyncpg==0.32.0 annotated-doc==0.0.5 annotated-types==0.8.0 @@ -10,10 +12,15 @@ click==8.5.0 fastapi==0.141.1 h11==0.16.0 idna==3.20 +jmespath==1.1.0 pycparser==3.0 +python-dateutil==2.9.0.post0 pydantic==2.13.5 pydantic_core==2.46.5 +s3transfer==0.19.3 +six==1.17.0 starlette==1.7.0 typing-inspection==0.4.4 typing_extensions==4.16.0 +urllib3==2.8.0 uvicorn==0.53.0