more before level3 changes

This commit is contained in:
2026-11-09 18:25:54 +03:00
parent 1686d51bca
commit 0346937847
8 changed files with 239 additions and 32 deletions
+35
View File
@@ -5,6 +5,15 @@ from pathlib import Path
from urllib.parse import quote
@dataclass(frozen=True)
class ObjectSettings:
endpoint: str
region: str
bucket: str
access_key: str
secret_key: str
@dataclass(frozen=True)
class Settings:
host: str
@@ -59,6 +68,32 @@ def _database_url() -> str:
)
def load_object_settings() -> ObjectSettings:
"""S3 credentials come from files. The values are not logged."""
endpoint = os.getenv('S3_ENDPOINT', '').strip()
region = os.getenv('S3_REGION', 'us-east-1').strip() or 'us-east-1'
bucket = os.getenv('S3_BUCKET', '').strip()
key_file = os.getenv('S3_ACCESS_KEY_ID_FILE', '').strip()
secret_file = os.getenv('S3_SECRET_ACCESS_KEY_FILE', '').strip()
missing = [
name for name, value in (
('S3_ENDPOINT', endpoint),
('S3_BUCKET', bucket),
('S3_ACCESS_KEY_ID_FILE', key_file),
('S3_SECRET_ACCESS_KEY_FILE', secret_file),
) if not value
]
if missing:
raise RuntimeError('%s is required' % ', '.join(missing))
return ObjectSettings(
endpoint,
region,
bucket,
_read_secret_file('S3_ACCESS_KEY_ID_FILE', key_file),
_read_secret_file('S3_SECRET_ACCESS_KEY_FILE', secret_file),
)
def load_settings() -> Settings:
return Settings(
host=os.getenv('APP_HOST', '0.0.0.0'),
+8 -2
View File
@@ -10,9 +10,10 @@ from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from starlette.exceptions import HTTPException as StarletteHTTPException
from .config import load_settings
from .config import load_object_settings, load_settings
from .db import open_pool
from .errors import ApiError
from .objects import ObjectStore
from .routes import router
from .store import Store
@@ -59,7 +60,12 @@ async def lifespan(app: FastAPI):
settings = load_settings()
pool = await open_pool(settings.database_url)
try:
app.state.store = Store(pool, settings.session_ttl_seconds, settings.idempotency_ttl_seconds)
app.state.store = Store(
pool,
settings.session_ttl_seconds,
settings.idempotency_ttl_seconds,
objects=ObjectStore(load_object_settings()),
)
logger.info('instance %s started', settings.instance_id)
yield
finally:
+99
View File
@@ -0,0 +1,99 @@
"""Private S3 bucket. Callers pass a server-generated key; the display filename never becomes one."""
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FutureTimeout
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from .config import ObjectSettings
# Docker DNS can spend several seconds on a stopped MinIO name. The HTTP handler
# must still answer inside HAProxy's one-second health check.
_POOL = ThreadPoolExecutor(max_workers=16, thread_name_prefix='s3')
class MissingObject(Exception):
"""The key is not in the bucket."""
class StorageError(Exception):
"""The object store could not complete the call. The message stays internal."""
class ObjectStore:
def __init__(self, settings: ObjectSettings) -> None:
self._bucket = settings.bucket
self._client = boto3.client(
's3',
endpoint_url=settings.endpoint,
region_name=settings.region,
aws_access_key_id=settings.access_key,
aws_secret_access_key=settings.secret_key,
config=Config(
signature_version='s3v4',
s3={'addressing_style': 'path'},
connect_timeout=1,
read_timeout=2,
retries={'max_attempts': 1},
),
)
def ready(self) -> None:
self._bounded(self._head, 0.7)
def put(self, key: str, body: bytes, content_type: str) -> None:
self._bounded(lambda: self._put(key, body, content_type), 8)
def get(self, key: str) -> bytes:
return self._bounded(lambda: self._get(key), 8)
def delete(self, key: str) -> None:
self._bounded(lambda: self._delete(key), 8)
def _bounded(self, fn, timeout: float):
future = _POOL.submit(fn)
try:
return future.result(timeout=timeout)
except FutureTimeout:
raise StorageError() from None
def _head(self) -> None:
try:
self._client.head_bucket(Bucket=self._bucket)
except (BotoCoreError, ClientError):
raise StorageError() from None
def _put(self, key: str, body: bytes, content_type: str) -> None:
try:
self._client.put_object(
Bucket=self._bucket,
Key=key,
Body=body,
ContentType=content_type,
ContentLength=len(body),
)
except (BotoCoreError, ClientError):
raise StorageError() from None
def _get(self, key: str) -> bytes:
try:
response = self._client.get_object(Bucket=self._bucket, Key=key)
except ClientError as exc:
code = exc.response.get('Error', {}).get('Code', '')
if code in ('NoSuchKey', '404', 'NotFound'):
raise MissingObject() from None
raise StorageError() from None
except BotoCoreError:
raise StorageError() from None
payload = response['Body']
try:
return payload.read()
finally:
payload.close()
def _delete(self, key: str) -> None:
try:
self._client.delete_object(Bucket=self._bucket, Key=key)
except (BotoCoreError, ClientError):
raise StorageError() from None
+69 -24
View File
@@ -1,5 +1,7 @@
import asyncio
import hashlib
import json
import logging
import secrets
import threading
import uuid
@@ -11,9 +13,12 @@ import asyncpg
from .errors import conflict, invalid_request, not_found, unavailable
from .migrate import expected_versions
from .objects import MissingObject, ObjectStore, StorageError
from .pagination import Key
from .passwords import hash_password, password_matches
logger = logging.getLogger('messenger')
@dataclass(frozen=True)
class User:
@@ -132,10 +137,21 @@ def _db_down(exc: BaseException) -> bool:
class Store:
"""PostgreSQL is the only state shared by replicas: users, chats, sessions, messages and attachment bytes."""
"""PostgreSQL holds users, chats, sessions, messages and attachment metadata.
def __init__(self, pool: asyncpg.Pool, session_ttl_seconds: int = 3600, idempotency_ttl_seconds: int = 86400) -> None:
Original bytes are in the private bucket. Any replica can read them by the server-assigned key.
"""
def __init__(
self,
pool: asyncpg.Pool,
session_ttl_seconds: int = 3600,
idempotency_ttl_seconds: int = 86400,
*,
objects: ObjectStore,
) -> None:
self._pool = pool
self._objects = objects
self._session_ttl = timedelta(seconds=session_ttl_seconds)
self._idempotency_ttl = timedelta(seconds=idempotency_ttl_seconds)
self._lock = threading.Lock()
@@ -152,7 +168,7 @@ class Store:
return moment
async def ready(self) -> None:
"""Ready only when PostgreSQL answers and the separate migrator has applied this build's schema."""
"""Ready when PostgreSQL has this build's schema and the private bucket answers."""
try:
async with self._pool.acquire() as conn:
applied = await conn.fetchval(
@@ -163,6 +179,10 @@ class Store:
raise unavailable('Хранилище недоступно') from None
if applied != len(self._migrations):
raise unavailable('Схема БД ещё не применена')
try:
await asyncio.to_thread(self._objects.ready)
except StorageError:
raise unavailable('Хранилище недоступно') from None
async def create_user(self, username: str, display_name: str, password: str) -> User:
password_hash = hash_password(password)
@@ -362,27 +382,37 @@ class Store:
chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id')
owner_uuid = _as_uuid(owner_id, 'Некорректный user_id')
attachment_id = uuid.uuid4()
# The key is not derived from filename. A crash after PUT and before INSERT leaves an
# unreferenced object; a failed INSERT deletes it. 201 is returned only after both succeed.
object_key = 'originals/%s' % attachment_id
created = self._now()
async with self._pool.acquire() as conn:
await conn.execute(
'''
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, body, created_at, updated_at
try:
await asyncio.to_thread(self._objects.put, object_key, body, content_type)
except StorageError:
raise unavailable('Хранилище недоступно') from None
try:
async with self._pool.acquire() as conn:
await conn.execute(
'''
INSERT INTO attachments (
id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, created_at, updated_at
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $9)
''',
attachment_id,
chat_uuid,
owner_uuid,
filename,
content_type,
len(body),
sha256,
object_key,
created,
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $10)
''',
attachment_id,
chat_uuid,
owner_uuid,
filename,
content_type,
len(body),
sha256,
uuid.uuid4().hex,
body,
created,
)
except Exception:
await self._discard_object(object_key)
raise
stamp = _stamp(created)
return Attachment(
str(attachment_id),
@@ -432,7 +462,7 @@ class Store:
async with self._pool.acquire() as conn:
row = await conn.fetchrow(
'''
SELECT a.content_type, a.body
SELECT a.content_type, a.size_bytes, a.sha256, a.object_key
FROM attachments a
JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2
WHERE a.id = $1
@@ -442,7 +472,22 @@ class Store:
)
if row is None:
return None
return row['content_type'], bytes(row['body'])
try:
body = await asyncio.to_thread(self._objects.get, row['object_key'])
except MissingObject:
return None
except StorageError:
raise unavailable('Хранилище недоступно') from None
if len(body) != row['size_bytes'] or hashlib.sha256(body).hexdigest() != row['sha256']:
logger.warning('object bytes do not match attachment metadata')
raise unavailable('Хранилище недоступно')
return row['content_type'], body
async def _discard_object(self, object_key: str) -> None:
try:
await asyncio.to_thread(self._objects.delete, object_key)
except StorageError:
logger.warning('object left in the bucket after a failed metadata write')
async def add_message(
self,
+13
View File
@@ -158,11 +158,24 @@ services:
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-3}
DB_POOL_MIN: ${DB_POOL_MIN:-1}
DB_POOL_MAX: ${DB_POOL_MAX:-10}
S3_ENDPOINT: ${S3_ENDPOINT:?set in .env}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:?set in .env}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets:
- source: postgres_password
uid: "10001"
gid: "10001"
mode: 0400
- source: s3_access_key_id
uid: "10001"
gid: "10001"
mode: 0400
- source: s3_secret_access_key
uid: "10001"
gid: "10001"
mode: 0400
networks:
- edge
- data
+3 -5
View File
@@ -3,8 +3,8 @@
ALTER TABLE messages DROP CONSTRAINT messages_text_len;
ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000);
-- filename is the display name. object_key is assigned by the server and is never taken from filename.
-- The original bytes sit next to the metadata so any API replica can read them.
-- filename is the display name only. object_key is assigned by the server and is never taken from filename.
-- The original bytes live in the private S3 bucket under that key, not in this table.
CREATE TABLE attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats (id),
@@ -14,15 +14,13 @@ CREATE TABLE attachments (
size_bytes INTEGER NOT NULL,
sha256 TEXT NOT NULL,
object_key TEXT NOT NULL,
body BYTEA NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL,
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachments_object_key_unique UNIQUE (object_key),
CONSTRAINT attachments_body_len CHECK (octet_length(body) = size_bytes)
CONSTRAINT attachments_object_key_unique UNIQUE (object_key)
);
CREATE INDEX attachments_chat_idx ON attachments (chat_id);
@@ -0,0 +1,4 @@
-- Databases that already applied the earlier 004 kept the original in a BYTEA column.
-- Drop it. Fresh installs create the table without that column; both statements are then no-ops.
ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_body_len;
ALTER TABLE attachments DROP COLUMN IF EXISTS body;
+8 -1
View File
@@ -1,4 +1,6 @@
# Direct: fastapi, pydantic, uvicorn, argon2-cffi. The rest are their dependencies, pinned.
# Direct: fastapi, pydantic, uvicorn, argon2-cffi, boto3. The rest are their dependencies, pinned.
boto3==1.43.126
botocore==1.43.126
asyncpg==0.32.0
annotated-doc==0.0.5
annotated-types==0.8.0
@@ -10,10 +12,15 @@ click==8.5.0
fastapi==0.141.1
h11==0.16.0
idna==3.20
jmespath==1.1.0
pycparser==3.0
python-dateutil==2.9.0.post0
pydantic==2.13.5
pydantic_core==2.46.5
s3transfer==0.19.3
six==1.17.0
starlette==1.7.0
typing-inspection==0.4.4
typing_extensions==4.16.0
urllib3==2.8.0
uvicorn==0.53.0