From 5be0a6c0b49104eb4f1393b94ac7d6ac0d0ea01a Mon Sep 17 00:00:00 2001 From: thefankor Date: Fri, 30 Oct 2026 19:09:55 +0300 Subject: [PATCH 1/5] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=BD=D0=BE=D1=81?= =?UTF-8?q?=20=D1=80=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=B0=D1=86=D0=B8=D0=B8?= =?UTF-8?q?=20=D0=BB=D0=B0=D0=B1=D0=BE=D1=80=D0=B0=D1=82=D0=BE=D1=80=D0=BD?= =?UTF-8?q?=D0=BE=D0=B9=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 17 ++ Dockerfile.test | 14 ++ alembic.ini | 36 +++++ alembic/env.py | 53 +++++++ alembic/script.py.mako | 22 +++ .../versions/20261019_0001_initial_schema.py | 64 ++++++++ .../20261023_0002_message_idempotency.py | 35 +++++ app/__init__.py | 1 + app/api/__init__.py | 1 + app/api/cursors.py | 28 ++++ app/api/deps.py | 72 +++++++++ app/api/errors.py | 96 ++++++++++++ app/api/middleware.py | 43 +++++ app/api/routers/__init__.py | 3 + app/api/routers/auth.py | 71 +++++++++ app/api/routers/chats.py | 49 ++++++ app/api/routers/health.py | 28 ++++ app/api/routers/messages.py | 51 ++++++ app/api/routers/users.py | 18 +++ app/api/schemas.py | 147 ++++++++++++++++++ app/api/timefmt.py | 11 ++ app/auth/__init__.py | 4 + app/auth/passwords.py | 15 ++ app/auth/tokens.py | 11 ++ app/config/__init__.py | 3 + app/config/settings.py | 36 +++++ app/db/__init__.py | 4 + app/db/postgres.py | 23 +++ app/db/redis.py | 51 ++++++ app/domain/__init__.py | 0 app/domain/errors.py | 29 ++++ app/domain/models.py | 56 +++++++ app/main.py | 87 +++++++++++ app/models/__init__.py | 5 + app/models/chat.py | 26 ++++ app/models/message.py | 30 ++++ app/models/user.py | 20 +++ app/repositories/__init__.py | 5 + app/repositories/chats.py | 69 ++++++++ app/repositories/messages.py | 104 +++++++++++++ app/repositories/users.py | 53 +++++++ app/services/__init__.py | 0 app/services/chats.py | 35 +++++ app/services/messages.py | 36 +++++ app/services/users.py | 35 +++++ compose.tests.yaml | 62 ++++++++ compose.yaml | 90 +++++++++++ infra/haproxy.cfg | 31 ++++ requirements-dev.txt | 1 + requirements.txt | 7 + scripts/restart.sh | 12 ++ scripts/stop-one.sh | 24 +++ 52 files changed, 1824 insertions(+) create mode 100644 Dockerfile create mode 100644 Dockerfile.test create mode 100644 alembic.ini create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/20261019_0001_initial_schema.py create mode 100644 alembic/versions/20261023_0002_message_idempotency.py create mode 100644 app/__init__.py create mode 100644 app/api/__init__.py create mode 100644 app/api/cursors.py create mode 100644 app/api/deps.py create mode 100644 app/api/errors.py create mode 100644 app/api/middleware.py create mode 100644 app/api/routers/__init__.py create mode 100644 app/api/routers/auth.py create mode 100644 app/api/routers/chats.py create mode 100644 app/api/routers/health.py create mode 100644 app/api/routers/messages.py create mode 100644 app/api/routers/users.py create mode 100644 app/api/schemas.py create mode 100644 app/api/timefmt.py create mode 100644 app/auth/__init__.py create mode 100644 app/auth/passwords.py create mode 100644 app/auth/tokens.py create mode 100644 app/config/__init__.py create mode 100644 app/config/settings.py create mode 100644 app/db/__init__.py create mode 100644 app/db/postgres.py create mode 100644 app/db/redis.py create mode 100644 app/domain/__init__.py create mode 100644 app/domain/errors.py create mode 100644 app/domain/models.py create mode 100644 app/main.py create mode 100644 app/models/__init__.py create mode 100644 app/models/chat.py create mode 100644 app/models/message.py create mode 100644 app/models/user.py create mode 100644 app/repositories/__init__.py create mode 100644 app/repositories/chats.py create mode 100644 app/repositories/messages.py create mode 100644 app/repositories/users.py create mode 100644 app/services/__init__.py create mode 100644 app/services/chats.py create mode 100644 app/services/messages.py create mode 100644 app/services/users.py create mode 100644 compose.tests.yaml create mode 100644 compose.yaml create mode 100644 infra/haproxy.cfg create mode 100644 requirements.txt create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2997c107 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.13.15-slim + +WORKDIR /app +COPY requirements.txt . +RUN python -m pip install -r requirements.txt \ + && useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin app +COPY app ./app +COPY alembic ./alembic +COPY alembic.ini . + +USER 10001:10001 +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "-c", "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=3).read()"] + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5"] diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 00000000..51c08a89 --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,14 @@ +FROM python:3.13.15-slim + +WORKDIR /tests +COPY requirements-dev.txt . +RUN python -m pip install --no-cache-dir -r requirements-dev.txt \ + && useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin tests + +COPY lab.json . +COPY contracts ./contracts +COPY tests ./tests +RUN chown -R tests:tests /tests + +USER 10001:10001 +CMD ["pytest", "-q"] diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..5c6d9f28 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,36 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +sqlalchemy.url = placeholder://configured-at-runtime + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 00000000..24524a9f --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import asyncio +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.ext.asyncio import async_engine_from_config + +from app.config import load_settings +from app.db.postgres import Base +from app import models # noqa: F401 - imports ORM models into Base.metadata + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def _url() -> str: + return load_settings().database_url + + +def run_migrations_offline() -> None: + context.configure(url=_url(), target_metadata=target_metadata, literal_binds=True, dialect_opts={'paramstyle': 'named'}) + with context.begin_transaction(): + context.run_migrations() + + +def _run_migrations(connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def _run_migrations_async() -> None: + configuration = config.get_section(config.config_ini_section, {}) + configuration['sqlalchemy.url'] = _url() + connectable = async_engine_from_config(configuration, prefix='sqlalchemy.', poolclass=pool.NullPool) + async with connectable.connect() as connection: + await connection.run_sync(_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(_run_migrations_async()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 00000000..36cd8efe --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/20261019_0001_initial_schema.py b/alembic/versions/20261019_0001_initial_schema.py new file mode 100644 index 00000000..e72e2f09 --- /dev/null +++ b/alembic/versions/20261019_0001_initial_schema.py @@ -0,0 +1,64 @@ +"""Initial PostgreSQL schema for lab 2. + +Revision ID: 20261019_0001 +Revises: +Create Date: 2026-10-19 +""" +from alembic import op +import sqlalchemy as sa + + +revision = '20261019_0001' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + 'users', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('username', sa.String(length=32), nullable=False), + sa.Column('display_name', sa.String(length=100), nullable=False), + sa.Column('password_hash', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('username'), + ) + op.create_table( + 'chats', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('title', sa.String(length=100), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint('id'), + ) + op.create_table( + 'chat_members', + sa.Column('chat_id', sa.Uuid(), nullable=False), + sa.Column('user_id', sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('chat_id', 'user_id'), + ) + op.create_index('chat_members_user_chat_idx', 'chat_members', ['user_id', 'chat_id']) + op.create_table( + 'messages', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('chat_id', sa.Uuid(), nullable=False), + sa.Column('sender_id', sa.Uuid(), nullable=False), + sa.Column('text', sa.String(length=4000), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['sender_id'], ['users.id']), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index('messages_history_idx', 'messages', ['chat_id', 'created_at', 'id']) + + +def downgrade() -> None: + op.drop_index('messages_history_idx', table_name='messages') + op.drop_table('messages') + op.drop_index('chat_members_user_chat_idx', table_name='chat_members') + op.drop_table('chat_members') + op.drop_table('chats') + op.drop_table('users') diff --git a/alembic/versions/20261023_0002_message_idempotency.py b/alembic/versions/20261023_0002_message_idempotency.py new file mode 100644 index 00000000..60d5eebe --- /dev/null +++ b/alembic/versions/20261023_0002_message_idempotency.py @@ -0,0 +1,35 @@ +"""Add shared message idempotency for lab 3. + +Revision ID: 20261023_0002 +Revises: 20261019_0001 +Create Date: 2026-10-23 +""" +from alembic import op +import sqlalchemy as sa + + +revision = '20261023_0002' +down_revision = '20261019_0001' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + 'messages', + sa.Column('idempotency_key', sa.String(length=128), nullable=True), + ) + op.create_unique_constraint( + 'messages_sender_chat_idempotency_key', + 'messages', + ['sender_id', 'chat_id', 'idempotency_key'], + ) + + +def downgrade() -> None: + op.drop_constraint( + 'messages_sender_chat_idempotency_key', + 'messages', + type_='unique', + ) + op.drop_column('messages', 'idempotency_key') diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..af04c352 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Мессенджер на PostgreSQL и Redis, лабораторная 2.""" diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 00000000..97e2e0cc --- /dev/null +++ b/app/api/__init__.py @@ -0,0 +1 @@ +"""HTTP-граница: FastAPI, коды состояния, формат ошибок, курсоры.""" diff --git a/app/api/cursors.py b/app/api/cursors.py new file mode 100644 index 00000000..f0a93957 --- /dev/null +++ b/app/api/cursors.py @@ -0,0 +1,28 @@ +import base64 +import json +from uuid import UUID + +from ..domain.errors import Invalid +from ..domain.models import Keyset +from .timefmt import from_rfc3339, to_rfc3339 + +MAX_LENGTH = 512 + + +def encode(scope: str, position: Keyset) -> str: + payload = {'s': scope, 't': to_rfc3339(position.created_at), 'i': str(position.id)} + raw = json.dumps(payload, separators=(',', ':'), sort_keys=True).encode('utf-8') + return base64.urlsafe_b64encode(raw).decode('ascii').rstrip('=') + + +def decode(scope: str, value: str) -> Keyset: + if not value or len(value) > MAX_LENGTH: + raise Invalid('cursor недействителен для этого запроса') + try: + raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4)) + payload = json.loads(raw) + if not isinstance(payload, dict) or payload.get('s') != scope: + raise ValueError('scope mismatch') + return Keyset(created_at=from_rfc3339(payload['t']), id=UUID(payload['i'])) + except Exception: + raise Invalid('cursor недействителен для этого запроса') from None diff --git a/app/api/deps.py b/app/api/deps.py new file mode 100644 index 00000000..10d16267 --- /dev/null +++ b/app/api/deps.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from fastapi import Depends, Header, HTTPException, Request, status + +from ..db.redis import RedisSessionStore +from ..domain.errors import Unauthorized +from ..domain.models import User +from ..services.chats import ChatService +from ..services.messages import MessageService +from ..services.users import UserService + +_BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +@dataclass(frozen=True, slots=True) +class Services: + users: UserService + chats: ChatService + messages: MessageService + sessions: RedisSessionStore + + +def services(request: Request) -> Services: + return request.app.state.services + + +def user_service(container: Services = Depends(services)) -> UserService: + return container.users + + +def chat_service(container: Services = Depends(services)) -> ChatService: + return container.chats + + +def message_service(container: Services = Depends(services)) -> MessageService: + return container.messages + + +def session_store(container: Services = Depends(services)) -> RedisSessionStore: + return container.sessions + + +def _bearer_error() -> HTTPException: + return HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail='требуется действующая Bearer-сессия', + headers={'WWW-Authenticate': _BEARER_CHALLENGE}, + ) + + +async def bearer_token(authorization: str | None = Header(None)) -> str: + if not authorization: + raise _bearer_error() + scheme, separator, token = authorization.partition(' ') + if scheme.lower() != 'bearer' or not separator or not token or token.strip() != token: + raise _bearer_error() + return token + + +async def current_user( + token: str = Depends(bearer_token), + container: Services = Depends(services), +) -> User: + user_id = await container.sessions.get_user_id(token) + if user_id is None: + raise _bearer_error() + try: + return await container.users.identify(user_id) + except Unauthorized: + raise _bearer_error() from None diff --git a/app/api/errors.py b/app/api/errors.py new file mode 100644 index 00000000..85a7c8e4 --- /dev/null +++ b/app/api/errors.py @@ -0,0 +1,96 @@ +import logging + +from fastapi import FastAPI +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from sqlalchemy.exc import SQLAlchemyError +from starlette.exceptions import HTTPException as StarletteHTTPException + +from ..domain.errors import Conflict, DomainError, Invalid, NotFound, Unauthorized, Unavailable + +logger = logging.getLogger('app.error') + +CODE_BY_STATUS = { + 400: 'invalid_request', + 401: 'unauthorized', + 404: 'not_found', + 409: 'conflict', + 413: 'payload_too_large', + 415: 'unsupported_media_type', + 429: 'rate_limited', + 500: 'internal_error', + 503: 'unavailable', +} + +STATUS_BY_DOMAIN = { + Invalid: 400, + Unauthorized: 401, + NotFound: 404, + Conflict: 409, + Unavailable: 503, +} + + +def code_for(status: int) -> str: + if status in CODE_BY_STATUS: + return CODE_BY_STATUS[status] + return 'invalid_request' if status < 500 else 'internal_error' + + +def error_response(status: int, message: str, request_id: str, code: str = '') -> JSONResponse: + return JSONResponse( + status_code=status, + content={'error': {'code': code or code_for(status), 'message': message, 'request_id': request_id}}, + ) + + +def _domain_status(error: DomainError) -> int: + for kind, status in STATUS_BY_DOMAIN.items(): + if isinstance(error, kind): + return status + return 500 + + +def _describe(error: RequestValidationError) -> str: + """Краткое указание на пол, без stack trace и внутреннего устройства""" + for item in error.errors(): + if item.get('type') == 'json_invalid': + return 'тело запроса не является корректным JSON' + location = '.'.join(str(part) for part in item.get('loc', ()) if part != 'body') + return f'некорректное поле {location}' if location else 'некорректный запрос' + return 'некорректный запрос' + + +def request_id(request) -> str: + return getattr(request.state, 'request_id', 'unknown') + + +def install(app: FastAPI) -> None: + @app.exception_handler(DomainError) + async def _on_domain(request, error: DomainError) -> JSONResponse: + return error_response(_domain_status(error), error.message, request_id(request)) + + @app.exception_handler(RequestValidationError) + async def _on_validation(request, error: RequestValidationError) -> JSONResponse: + return error_response(400, _describe(error), request_id(request)) + + @app.exception_handler(SQLAlchemyError) + async def _on_database(request, error: SQLAlchemyError) -> JSONResponse: + logger.warning('database error request_id=%s type=%s', request_id(request), type(error).__name__) + return error_response(503, 'база данных недоступна', request_id(request)) + + @app.exception_handler(OSError) + async def _on_storage_socket(request, error: OSError) -> JSONResponse: + logger.warning('storage socket error request_id=%s type=%s', request_id(request), type(error).__name__) + return error_response(503, 'обязательное хранилище недоступно', request_id(request)) + + @app.exception_handler(StarletteHTTPException) + async def _on_http(request, error: StarletteHTTPException) -> JSONResponse: + message = error.detail if isinstance(error.detail, str) and error.detail else 'ошибка запроса' + response = error_response(error.status_code, message, request_id(request)) + for name, value in (error.headers or {}).items(): + response.headers[name] = value + return response + + +__all__ = ['code_for', 'error_response', 'install'] diff --git a/app/api/middleware.py b/app/api/middleware.py new file mode 100644 index 00000000..fb4e64f1 --- /dev/null +++ b/app/api/middleware.py @@ -0,0 +1,43 @@ +import logging +import os +import socket +from uuid import uuid4 + +from starlette.datastructures import MutableHeaders + +from .errors import error_response + +logger = logging.getLogger('app') + + +class RequestContext: + def __init__(self, app) -> None: + self.app = app + self.instance_id = os.getenv('INSTANCE_ID') or socket.gethostname() + + async def __call__(self, scope, receive, send) -> None: + if scope['type'] != 'http': + await self.app(scope, receive, send) + return + + request_id = uuid4().hex + scope.setdefault('state', {})['request_id'] = request_id + started = False + + async def send_wrapper(message) -> None: + nonlocal started + if message['type'] == 'http.response.start': + started = True + headers = MutableHeaders(scope=message) + headers['X-Request-Id'] = request_id + headers['X-Instance-Id'] = self.instance_id + await send(message) + + try: + await self.app(scope, receive, send_wrapper) + except Exception: + logger.exception(f'unhandled error request_id={request_id} path={scope.get("path")}') + if started: + raise + response = error_response(500, 'внутренняя ошибка', request_id) + await response(scope, receive, send_wrapper) diff --git a/app/api/routers/__init__.py b/app/api/routers/__init__.py new file mode 100644 index 00000000..dede845e --- /dev/null +++ b/app/api/routers/__init__.py @@ -0,0 +1,3 @@ +from . import auth, chats, health, messages, users + +__all__ = ['auth', 'chats', 'health', 'messages', 'users'] diff --git a/app/api/routers/auth.py b/app/api/routers/auth.py new file mode 100644 index 00000000..e41e8b1c --- /dev/null +++ b/app/api/routers/auth.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import base64 +from binascii import Error as Base64Error + +from fastapi import APIRouter, Depends, Header, HTTPException, status + +from ...auth.tokens import create_token +from ...db.redis import RedisSessionStore +from ...domain.models import User +from ...services.users import UserService +from ..deps import bearer_token, current_user, session_store, user_service +from ..schemas import SessionOut, UserOut +from ..timefmt import to_rfc3339 + +router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth']) + +_BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' + + +def _unauthorized_basic() -> HTTPException: + return HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail='неверные учётные данные', + headers={'WWW-Authenticate': _BASIC_CHALLENGE}, + ) + + +def _parse_basic(authorization: str | None) -> tuple[str, str]: + if not authorization: + raise _unauthorized_basic() + scheme, separator, encoded = authorization.partition(' ') + if scheme.lower() != 'basic' or not separator or not encoded: + raise _unauthorized_basic() + try: + decoded = base64.b64decode(encoded, validate=True).decode('utf-8') + except (Base64Error, UnicodeDecodeError): + raise _unauthorized_basic() from None + username, separator, password = decoded.partition(':') + if not separator or not username: + raise _unauthorized_basic() + return username, password + + +@router.post('', status_code=status.HTTP_201_CREATED, response_model=SessionOut) +async def create_session( + authorization: str | None = Header(None), + users: UserService = Depends(user_service), + sessions: RedisSessionStore = Depends(session_store), +) -> SessionOut: + username, password = _parse_basic(authorization) + user = await users.authenticate(username, password) + if user is None: + raise _unauthorized_basic() + token = create_token() + expires_at = await sessions.create(user.id, token) + return SessionOut( + token=token, + token_type='Bearer', + expires_at=to_rfc3339(expires_at), + user=UserOut.from_domain(user), + ) + + +@router.delete('/current', status_code=status.HTTP_204_NO_CONTENT, response_model=None) +async def revoke_session( + token: str = Depends(bearer_token), + _: User = Depends(current_user), + sessions: RedisSessionStore = Depends(session_store), +) -> None: + await sessions.revoke(token) diff --git a/app/api/routers/chats.py b/app/api/routers/chats.py new file mode 100644 index 00000000..1dd5126c --- /dev/null +++ b/app/api/routers/chats.py @@ -0,0 +1,49 @@ +from typing import Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, Query + +from ...domain.models import User +from ...services.chats import ChatService +from .. import cursors +from ..deps import chat_service, current_user +from ..schemas import ( + LIMIT_DEFAULT, + LIMIT_MAX, + LIMIT_MIN, + ChatOut, + ChatPageOut, + CreateChat, +) + +router = APIRouter(prefix='/api/v1/chats', tags=['chats']) + + +@router.post('', status_code=201, response_model=ChatOut) +async def create( + body: CreateChat, + actor: User = Depends(current_user), + chats: ChatService = Depends(chat_service), +) -> ChatOut: + return ChatOut.from_domain(await chats.create(actor, body.title, body.member_ids)) + + +@router.get('', response_model=ChatPageOut) +async def page( + actor: User = Depends(current_user), + chats: ChatService = Depends(chat_service), + limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX), + cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH), +) -> ChatPageOut: + scope = f'chats:{actor.id}' + after = cursors.decode(scope, cursor) if cursor is not None else None + return ChatPageOut.from_domain(await chats.page(actor, after, limit), scope) + + +@router.get('/{chat_id}', response_model=ChatOut) +async def get( + chat_id: UUID, + actor: User = Depends(current_user), + chats: ChatService = Depends(chat_service), +) -> ChatOut: + return ChatOut.from_domain(await chats.get(actor, chat_id)) diff --git a/app/api/routers/health.py b/app/api/routers/health.py new file mode 100644 index 00000000..a4e2029b --- /dev/null +++ b/app/api/routers/health.py @@ -0,0 +1,28 @@ +from fastapi import APIRouter, HTTPException, Request +from redis.exceptions import RedisError +from sqlalchemy import text +from sqlalchemy.exc import SQLAlchemyError + +from ...domain.errors import Unavailable + +from ..schemas import HealthOut + +router = APIRouter(tags=['health']) + + +@router.get('/health/live', response_model=HealthOut) +async def live() -> HealthOut: + return HealthOut(status='ok') + + +@router.get('/health/ready', response_model=HealthOut) +async def ready(request: Request) -> HealthOut: + if not request.app.state.ready: + raise HTTPException(status_code=503, detail='приложение не готово принимать трафик') + try: + async with request.app.state.engine.connect() as connection: + await connection.execute(text('SELECT 1')) + await request.app.state.redis.ping() + except (SQLAlchemyError, RedisError, Unavailable, OSError): + raise HTTPException(status_code=503, detail='обязательное хранилище недоступно') from None + return HealthOut(status='ok') diff --git a/app/api/routers/messages.py b/app/api/routers/messages.py new file mode 100644 index 00000000..b97f10f6 --- /dev/null +++ b/app/api/routers/messages.py @@ -0,0 +1,51 @@ +from typing import Annotated, Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, Header, Query + +from ...domain.models import User +from ...services.messages import MessageService +from .. import cursors +from ..deps import current_user, message_service +from ..schemas import ( + LIMIT_DEFAULT, + LIMIT_MAX, + LIMIT_MIN, + CreateMessage, + MessageOut, + MessagePageOut, +) + +router = APIRouter(prefix='/api/v1/chats/{chat_id}/messages', tags=['messages']) + + +@router.post('', status_code=201, response_model=MessageOut) +async def send( + chat_id: UUID, + body: CreateMessage, + idempotency_key: Annotated[ + str, + Header( + alias='Idempotency-Key', + min_length=1, + max_length=128, + pattern=r'^[A-Za-z0-9._:-]+$', + ), + ], + actor: User = Depends(current_user), + messages: MessageService = Depends(message_service), +) -> MessageOut: + return MessageOut.from_domain(await messages.send(actor, chat_id, body.text, idempotency_key)) + + +@router.get('', response_model=MessagePageOut) +async def page( + chat_id: UUID, + actor: User = Depends(current_user), + messages: MessageService = Depends(message_service), + limit: int = Query(LIMIT_DEFAULT, ge=LIMIT_MIN, le=LIMIT_MAX), + cursor: Optional[str] = Query(None, min_length=1, max_length=cursors.MAX_LENGTH), +) -> MessagePageOut: + scope = f'messages:{chat_id}' + after = cursors.decode(scope, cursor) if cursor is not None else None + return MessagePageOut.from_domain(await messages.history(actor, chat_id, after, limit), scope) diff --git a/app/api/routers/users.py b/app/api/routers/users.py new file mode 100644 index 00000000..7da40af4 --- /dev/null +++ b/app/api/routers/users.py @@ -0,0 +1,18 @@ +from fastapi import APIRouter, Depends + +from ...domain.models import User +from ...services.users import UserService +from ..deps import current_user, user_service +from ..schemas import CreateUser, UserOut + +router = APIRouter(prefix='/api/v1', tags=['users']) + + +@router.post('/users', status_code=201, response_model=UserOut) +async def register(body: CreateUser, users: UserService = Depends(user_service)) -> UserOut: + return UserOut.from_domain(await users.register(body.username, body.display_name, body.password)) + + +@router.get('/users/me', response_model=UserOut) +async def me(actor: User = Depends(current_user)) -> UserOut: + return UserOut.from_domain(actor) diff --git a/app/api/schemas.py b/app/api/schemas.py new file mode 100644 index 00000000..145b0c0c --- /dev/null +++ b/app/api/schemas.py @@ -0,0 +1,147 @@ +from typing import Optional +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from ..domain.models import Chat, Message, Page, User +from . import cursors +from .timefmt import to_rfc3339 + +NAME_MAX = 100 +TEXT_MAX = 4000 +MEMBERS_MAX = 100 +LIMIT_MIN = 1 +LIMIT_MAX = 100 +LIMIT_DEFAULT = 50 + + +def not_blank(value: str) -> str: + if not value.strip(): + raise ValueError('строка не может состоять только из пробельных символов') + return value + + +class Strict(BaseModel): + model_config = ConfigDict(extra='forbid') + + +class HealthOut(Strict): + status: str = 'ok' + + +class CreateUser(Strict): + username: str = Field(min_length=3, max_length=32, pattern=r'^[a-z0-9_]{3,32}$') + display_name: str = Field(min_length=1, max_length=NAME_MAX) + password: str = Field(min_length=12, max_length=128) + + @field_validator('display_name') + @classmethod + def _display_name(cls, value: str) -> str: + return not_blank(value) + + +class UserOut(Strict): + id: UUID + username: str + display_name: str + created_at: str + + @classmethod + def from_domain(cls, user: User) -> 'UserOut': + return cls( + id=user.id, + username=user.username, + display_name=user.display_name, + created_at=to_rfc3339(user.created_at), + ) + + +class SessionOut(Strict): + token: str = Field(min_length=32) + token_type: str = 'Bearer' + expires_at: str + user: UserOut + + +class CreateChat(Strict): + title: str = Field(min_length=1, max_length=NAME_MAX) + member_ids: list[UUID] = Field(min_length=1, max_length=MEMBERS_MAX) + + @field_validator('title') + @classmethod + def _title(cls, value: str) -> str: + return not_blank(value) + + @field_validator('member_ids') + @classmethod + def _members_unique(cls, value: list[UUID]) -> list[UUID]: + if len(set(value)) != len(value): + raise ValueError('member_ids не должен содержать дубликаты') + return value + + +class ChatOut(Strict): + id: UUID + title: str + member_ids: list[UUID] + created_at: str + + @classmethod + def from_domain(cls, chat: Chat) -> 'ChatOut': + return cls( + id=chat.id, + title=chat.title, + member_ids=list(chat.member_ids), + created_at=to_rfc3339(chat.created_at), + ) + + +class CreateMessage(Strict): + text: str = Field(min_length=1, max_length=TEXT_MAX) + + @field_validator('text') + @classmethod + def _text(cls, value: str) -> str: + return not_blank(value) + + +class MessageOut(Strict): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: str + + @classmethod + def from_domain(cls, message: Message) -> 'MessageOut': + return cls( + id=message.id, + chat_id=message.chat_id, + sender_id=message.sender_id, + text=message.text, + created_at=to_rfc3339(message.created_at), + ) + + +class ChatPageOut(Strict): + items: list[ChatOut] + next_cursor: Optional[str] + + @classmethod + def from_domain(cls, page: Page, scope: str) -> 'ChatPageOut': + return cls( + items=[ChatOut.from_domain(chat) for chat in page.items], + next_cursor=cursors.encode(scope, page.next) if page.next else None, + ) + + +class MessagePageOut(Strict): + items: list[MessageOut] + next_cursor: Optional[str] + + @classmethod + def from_domain(cls, page: Page, scope: str) -> 'MessagePageOut': + return cls( + items=[MessageOut.from_domain(message) for message in page.items], + next_cursor=cursors.encode(scope, page.next) if page.next else None, + ) diff --git a/app/api/timefmt.py b/app/api/timefmt.py new file mode 100644 index 00000000..b9d7b101 --- /dev/null +++ b/app/api/timefmt.py @@ -0,0 +1,11 @@ +from datetime import datetime, timezone + + +def to_rfc3339(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def from_rfc3339(value: str) -> datetime: + if not isinstance(value, str) or not value.endswith('Z'): + raise ValueError('timestamp must end with Z') + return datetime.fromisoformat(value[:-1]).replace(tzinfo=timezone.utc) diff --git a/app/auth/__init__.py b/app/auth/__init__.py new file mode 100644 index 00000000..9bd80fdb --- /dev/null +++ b/app/auth/__init__.py @@ -0,0 +1,4 @@ +from .passwords import hash_password, password_matches +from .tokens import create_token + +__all__ = ['create_token', 'hash_password', 'password_matches'] diff --git a/app/auth/passwords.py b/app/auth/passwords.py new file mode 100644 index 00000000..70576fc0 --- /dev/null +++ b/app/auth/passwords.py @@ -0,0 +1,15 @@ +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError, VerifyMismatchError + +_hasher = PasswordHasher() + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def password_matches(stored_hash: str, password: str) -> bool: + try: + return _hasher.verify(stored_hash, password) + except (VerifyMismatchError, VerificationError): + return False diff --git a/app/auth/tokens.py b/app/auth/tokens.py new file mode 100644 index 00000000..c60beff0 --- /dev/null +++ b/app/auth/tokens.py @@ -0,0 +1,11 @@ +import hashlib +import secrets + + +def create_token() -> str: + return secrets.token_urlsafe(32) + + +def session_key(token: str) -> str: + digest = hashlib.sha256(token.encode('utf-8')).hexdigest() + return f'session:{digest}' diff --git a/app/config/__init__.py b/app/config/__init__.py new file mode 100644 index 00000000..e6b731e5 --- /dev/null +++ b/app/config/__init__.py @@ -0,0 +1,3 @@ +from .settings import Settings, load_settings + +__all__ = ['Settings', 'load_settings'] diff --git a/app/config/settings.py b/app/config/settings.py new file mode 100644 index 00000000..886023e0 --- /dev/null +++ b/app/config/settings.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +def _read_value(name: str, file_name: str) -> str: + value = os.getenv(name) + if value: + return value + path = os.getenv(file_name) + if path: + return Path(path).read_text(encoding='utf-8').strip() + raise RuntimeError(f'не задана настройка {name} или {file_name}') + + +@dataclass(frozen=True, slots=True) +class Settings: + database_url: str + redis_url: str + session_ttl_seconds: int + + +def load_settings() -> Settings: + try: + ttl = int(os.getenv('SESSION_TTL_SECONDS', '3600')) + except ValueError as exc: + raise RuntimeError('SESSION_TTL_SECONDS должен быть целым числом') from exc + if ttl <= 0: + raise RuntimeError('SESSION_TTL_SECONDS должен быть больше нуля') + return Settings( + database_url=_read_value('DATABASE_URL', 'DATABASE_URL_FILE'), + redis_url=_read_value('REDIS_URL', 'CACHE_URL_FILE'), + session_ttl_seconds=ttl, + ) diff --git a/app/db/__init__.py b/app/db/__init__.py new file mode 100644 index 00000000..6e0b53d7 --- /dev/null +++ b/app/db/__init__.py @@ -0,0 +1,4 @@ +from .postgres import Base, create_engine, create_session_factory +from .redis import RedisSessionStore, create_redis + +__all__ = ['Base', 'RedisSessionStore', 'create_engine', 'create_redis', 'create_session_factory'] diff --git a/app/db/postgres.py b/app/db/postgres.py new file mode 100644 index 00000000..d3b31752 --- /dev/null +++ b/app/db/postgres.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.orm import DeclarativeBase + + +class Base(DeclarativeBase): + pass + + +def create_engine(url: str) -> AsyncEngine: + return create_async_engine( + url, + pool_size=5, + max_overflow=5, + pool_timeout=5, + pool_pre_ping=True, + connect_args={'server_settings': {'statement_timeout': '5000'}}, + ) + + +def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]: + return async_sessionmaker(engine, expire_on_commit=False) diff --git a/app/db/redis.py b/app/db/redis.py new file mode 100644 index 00000000..fc9e3570 --- /dev/null +++ b/app/db/redis.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from uuid import UUID + +from redis.asyncio import Redis +from redis.exceptions import RedisError + +from ..auth.tokens import session_key +from ..domain.errors import Unavailable + + +def create_redis(url: str) -> Redis: + return Redis.from_url(url, decode_responses=True, socket_connect_timeout=3, socket_timeout=3) + + +class RedisSessionStore: + def __init__(self, client: Redis, ttl_seconds: int) -> None: + self._client = client + self._ttl_seconds = ttl_seconds + + async def create(self, user_id: UUID, token: str) -> datetime: + try: + await self._client.set(session_key(token), str(user_id), ex=self._ttl_seconds) + except RedisError as exc: + raise Unavailable('хранилище сессий недоступно') from exc + return datetime.now(timezone.utc) + timedelta(seconds=self._ttl_seconds) + + async def get_user_id(self, token: str) -> UUID | None: + try: + value = await self._client.get(session_key(token)) + except RedisError as exc: + raise Unavailable('хранилище сессий недоступно') from exc + if value is None: + return None + try: + return UUID(value) + except ValueError: + return None + + async def revoke(self, token: str) -> None: + try: + await self._client.delete(session_key(token)) + except RedisError as exc: + raise Unavailable('хранилище сессий недоступно') from exc + + async def ping(self) -> None: + try: + await self._client.ping() + except RedisError as exc: + raise Unavailable('хранилище сессий недоступно') from exc diff --git a/app/domain/__init__.py b/app/domain/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/domain/errors.py b/app/domain/errors.py new file mode 100644 index 00000000..d15b362b --- /dev/null +++ b/app/domain/errors.py @@ -0,0 +1,29 @@ +from __future__ import annotations + + +class DomainError(Exception): + """Сообщение попадает клиенту, поэтому не содержит внутренних деталей.""" + + def __init__(self, message: str) -> None: + super().__init__(message) + self.message = message + + +class Invalid(DomainError): + """Запрос нарушает правила предметной области.""" + + +class Unauthorized(DomainError): + """Нет действительной идентификации.""" + + +class NotFound(DomainError): + """Объекта нет либо он недоступен вызывающему: снаружи это неразличимо.""" + + +class Conflict(DomainError): + """Состояние не позволяет выполнить операцию.""" + + +class Unavailable(DomainError): + """Обязательная внешняя зависимость временно недоступна.""" diff --git a/app/domain/models.py b/app/domain/models.py new file mode 100644 index 00000000..33668bbd --- /dev/null +++ b/app/domain/models.py @@ -0,0 +1,56 @@ +from dataclasses import dataclass +from datetime import datetime +from typing import Optional +from uuid import UUID + + +@dataclass(frozen=True, slots=True) +class User: + id: UUID + username: str + display_name: str + created_at: datetime + + +@dataclass(frozen=True, slots=True) +class Chat: + id: UUID + title: str + member_ids: tuple[UUID, ...] + created_at: datetime + + def has_member(self, user_id: UUID) -> bool: + return user_id in self.member_ids + + +@dataclass(frozen=True, slots=True) +class Message: + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + + +@dataclass(frozen=True, slots=True) +class Keyset: + created_at: datetime + id: UUID + + @property + def order(self) -> tuple[datetime, int]: + return (self.created_at, self.id.int) + + @classmethod + def from_item(cls, item) -> 'Keyset': + return cls(created_at=item.created_at, id=item.id) + + +def order_key(item) -> tuple[datetime, int]: + return (item.created_at, item.id.int) + + +@dataclass(frozen=True, slots=True) +class Page: + items: tuple + next: Optional[Keyset] diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..9426a9d1 --- /dev/null +++ b/app/main.py @@ -0,0 +1,87 @@ +"""Сборка FastAPI приложения и подключение обязательных хранилищ.""" +from __future__ import annotations + +import logging +import os +import sys +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from sqlalchemy import text + +from .api import errors +from .api.deps import Services +from .api.middleware import RequestContext +from .api.routers import auth, chats, health, messages, users +from .config import load_settings +from .db import RedisSessionStore, create_engine, create_redis, create_session_factory +from .repositories import ChatRepository, MessageRepository, UserRepository +from .services.chats import ChatService +from .services.messages import MessageService +from .services.users import UserService + +logger = logging.getLogger('app') + +DESCRIPTION = 'PostgreSQL хранит данные мессенджера; Redis хранит активные сессии.' + + +def configure_logging() -> None: + logging.basicConfig( + level=os.getenv('LOG_LEVEL', 'info').upper(), + stream=sys.stdout, + format='%(asctime)s %(levelname)s %(name)s %(message)s', + force=True, + ) + + +def create_app() -> FastAPI: + configure_logging() + + @asynccontextmanager + async def lifespan(application: FastAPI): + application.state.ready = False + settings = load_settings() + engine = create_engine(settings.database_url) + redis = create_redis(settings.redis_url) + try: + async with engine.connect() as connection: + await connection.execute(text('SELECT 1')) + session_store = RedisSessionStore(redis, settings.session_ttl_seconds) + await session_store.ping() + session_factory = create_session_factory(engine) + user_repository = UserRepository(session_factory) + chat_repository = ChatRepository(session_factory) + message_repository = MessageRepository(session_factory) + application.state.services = Services( + users=UserService(user_repository), + chats=ChatService(chat_repository), + messages=MessageService(message_repository), + sessions=session_store, + ) + application.state.engine = engine + application.state.redis = redis + application.state.ready = True + logger.info('startup storage=postgres redis_sessions=true') + yield + finally: + application.state.ready = False + await redis.aclose() + await engine.dispose() + logger.info('shutdown complete') + + api = FastAPI( + title='MTUSI Messenger. LR 3', + version='1.3.0', + description=DESCRIPTION, + lifespan=lifespan, + ) + api.router.redirect_slashes = False + api.state.ready = False + api.add_middleware(RequestContext) + errors.install(api) + for router in (health.router, users.router, auth.router, chats.router, messages.router): + api.include_router(router) + return api + + +app = create_app() diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 00000000..64b2f78f --- /dev/null +++ b/app/models/__init__.py @@ -0,0 +1,5 @@ +from .chat import ChatMemberRow, ChatRow +from .message import MessageRow +from .user import UserRow + +__all__ = ['ChatMemberRow', 'ChatRow', 'MessageRow', 'UserRow'] diff --git a/app/models/chat.py b/app/models/chat.py new file mode 100644 index 00000000..9c464ecd --- /dev/null +++ b/app/models/chat.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import DateTime, ForeignKey, Index, String +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import Uuid + +from ..db.postgres import Base + + +class ChatRow(Base): + __tablename__ = 'chats' + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + title: Mapped[str] = mapped_column(String(100), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class ChatMemberRow(Base): + __tablename__ = 'chat_members' + __table_args__ = (Index('chat_members_user_chat_idx', 'user_id', 'chat_id'),) + + chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True) + user_id: Mapped[UUID] = mapped_column(ForeignKey('users.id', ondelete='CASCADE'), primary_key=True) diff --git a/app/models/message.py b/app/models/message.py new file mode 100644 index 00000000..96a716f5 --- /dev/null +++ b/app/models/message.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import DateTime, ForeignKey, Index, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import Uuid + +from ..db.postgres import Base + + +class MessageRow(Base): + __tablename__ = 'messages' + __table_args__ = ( + Index('messages_history_idx', 'chat_id', 'created_at', 'id'), + UniqueConstraint( + 'sender_id', + 'chat_id', + 'idempotency_key', + name='messages_sender_chat_idempotency_key', + ), + ) + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False) + sender_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False) + text: Mapped[str] = mapped_column(String(4000), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True) diff --git a/app/models/user.py b/app/models/user.py new file mode 100644 index 00000000..f4aeefd0 --- /dev/null +++ b/app/models/user.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import DateTime, String, Text +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import Uuid + +from ..db.postgres import Base + + +class UserRow(Base): + __tablename__ = 'users' + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + username: Mapped[str] = mapped_column(String(32), unique=True, nullable=False) + display_name: Mapped[str] = mapped_column(String(100), nullable=False) + password_hash: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) diff --git a/app/repositories/__init__.py b/app/repositories/__init__.py new file mode 100644 index 00000000..4d82b99c --- /dev/null +++ b/app/repositories/__init__.py @@ -0,0 +1,5 @@ +from .chats import ChatRepository +from .messages import MessageRepository +from .users import UserRepository + +__all__ = ['ChatRepository', 'MessageRepository', 'UserRepository'] diff --git a/app/repositories/chats.py b/app/repositories/chats.py new file mode 100644 index 00000000..04d4261a --- /dev/null +++ b/app/repositories/chats.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Sequence +from uuid import UUID + +from sqlalchemy import select, tuple_ +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from ..domain.errors import Invalid +from ..domain.models import Chat, Keyset +from ..models.chat import ChatMemberRow, ChatRow +from ..models.user import UserRow + + +class ChatRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + @staticmethod + def _to_domain(row: ChatRow, member_ids: Sequence[UUID]) -> Chat: + return Chat(id=row.id, title=row.title, member_ids=tuple(member_ids), created_at=row.created_at) + + @staticmethod + async def _members(session: AsyncSession, chat_id: UUID) -> tuple[UUID, ...]: + result = await session.scalars( + select(ChatMemberRow.user_id) + .where(ChatMemberRow.chat_id == chat_id) + .order_by(ChatMemberRow.user_id) + ) + return tuple(result) + + async def create(self, chat_id: UUID, title: str, member_ids: Sequence[UUID], created_at: datetime) -> Chat: + async with self._sessions() as session: + async with session.begin(): + found = set(await session.scalars(select(UserRow.id).where(UserRow.id.in_(member_ids)))) + if len(found) != len(member_ids): + raise Invalid(f'member_ids содержит неизвестных участников: {len(member_ids) - len(found)}') + row = ChatRow(id=chat_id, title=title, created_at=created_at) + session.add(row) + await session.flush() + session.add_all(ChatMemberRow(chat_id=chat_id, user_id=user_id) for user_id in member_ids) + await session.flush() + return self._to_domain(row, member_ids) + + async def get_for_member(self, chat_id: UUID, user_id: UUID) -> Chat | None: + async with self._sessions() as session: + row = await session.scalar( + select(ChatRow) + .join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id) + .where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id) + ) + if row is None: + return None + return self._to_domain(row, await self._members(session, row.id)) + + async def page_for_member(self, user_id: UUID, after: Keyset | None, limit: int) -> list[Chat]: + async with self._sessions() as session: + query = ( + select(ChatRow) + .join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id) + .where(ChatMemberRow.user_id == user_id) + .order_by(ChatRow.created_at, ChatRow.id) + .limit(limit) + ) + if after is not None: + query = query.where(tuple_(ChatRow.created_at, ChatRow.id) > tuple_(after.created_at, after.id)) + rows = list((await session.scalars(query)).all()) + return [self._to_domain(row, await self._members(session, row.id)) for row in rows] diff --git a/app/repositories/messages.py b/app/repositories/messages.py new file mode 100644 index 00000000..f387e9a9 --- /dev/null +++ b/app/repositories/messages.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import select, tuple_ +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from ..domain.errors import Conflict +from ..domain.models import Keyset, Message +from ..models.chat import ChatMemberRow, ChatRow +from ..models.message import MessageRow + + +class MessageRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + @staticmethod + def _to_domain(row: MessageRow) -> Message: + return Message( + id=row.id, + chat_id=row.chat_id, + sender_id=row.sender_id, + text=row.text, + created_at=row.created_at, + ) + + @staticmethod + async def _accessible_chat_id(session: AsyncSession, chat_id: UUID, user_id: UUID) -> UUID | None: + return await session.scalar( + select(ChatRow.id) + .join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id) + .where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id) + ) + + async def create_for_member( + self, + message_id: UUID, + chat_id: UUID, + sender_id: UUID, + text: str, + created_at: datetime, + idempotency_key: str, + ) -> Message | None: + async with self._sessions() as session: + async with session.begin(): + accessible_chat_id = await self._accessible_chat_id(session, chat_id, sender_id) + if accessible_chat_id is None: + return None + + statement = ( + insert(MessageRow) + .values( + id=message_id, + chat_id=accessible_chat_id, + sender_id=sender_id, + text=text, + created_at=created_at, + idempotency_key=idempotency_key, + ) + .on_conflict_do_nothing( + index_elements=[ + MessageRow.sender_id, + MessageRow.chat_id, + MessageRow.idempotency_key, + ] + ) + .returning(MessageRow.id) + ) + created_id = await session.scalar(statement) + if created_id is not None: + row = await session.get(MessageRow, created_id) + else: + row = await session.scalar( + select(MessageRow).where( + MessageRow.sender_id == sender_id, + MessageRow.chat_id == accessible_chat_id, + MessageRow.idempotency_key == idempotency_key, + ) + ) + if row is None: + raise RuntimeError('результат идемпотентной операции не найден') + if row.text != text: + raise Conflict('Idempotency-Key уже использован с другим сообщением') + return self._to_domain(row) + + async def page_for_member( + self, chat_id: UUID, user_id: UUID, after: Keyset | None, limit: int + ) -> list[Message] | None: + async with self._sessions() as session: + accessible_chat_id = await self._accessible_chat_id(session, chat_id, user_id) + if accessible_chat_id is None: + return None + query = ( + select(MessageRow) + .where(MessageRow.chat_id == accessible_chat_id) + .order_by(MessageRow.created_at, MessageRow.id) + .limit(limit) + ) + if after is not None: + query = query.where(tuple_(MessageRow.created_at, MessageRow.id) > tuple_(after.created_at, after.id)) + return [self._to_domain(row) for row in (await session.scalars(query)).all()] diff --git a/app/repositories/users.py b/app/repositories/users.py new file mode 100644 index 00000000..d65b165b --- /dev/null +++ b/app/repositories/users.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from ..domain.errors import Conflict +from ..domain.models import User +from ..models.user import UserRow + + +class UserRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + @staticmethod + def _to_domain(row: UserRow) -> User: + return User(id=row.id, username=row.username, display_name=row.display_name, created_at=row.created_at) + + async def register( + self, user_id: UUID, username: str, display_name: str, password_hash: str, created_at: datetime + ) -> User: + row = UserRow( + id=user_id, + username=username, + display_name=display_name, + password_hash=password_hash, + created_at=created_at, + ) + try: + async with self._sessions() as session: + async with session.begin(): + session.add(row) + await session.flush() + except IntegrityError as exc: + raise Conflict('username уже занят') from exc + return self._to_domain(row) + + async def get_by_id(self, user_id: UUID) -> User | None: + async with self._sessions() as session: + row = await session.get(UserRow, user_id) + return self._to_domain(row) if row is not None else None + + async def get_credentials(self, username: str) -> UserRow | None: + async with self._sessions() as session: + return await session.scalar(select(UserRow).where(UserRow.username == username)) + + @classmethod + def to_domain(cls, row: UserRow) -> User: + return cls._to_domain(row) diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/services/chats.py b/app/services/chats.py new file mode 100644 index 00000000..9b55f715 --- /dev/null +++ b/app/services/chats.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Optional, Sequence +from uuid import UUID, uuid4 + +from ..domain.errors import Invalid, NotFound +from ..domain.models import Chat, Keyset, Page, User +from ..repositories import ChatRepository + + +class ChatService: + def __init__(self, repository: ChatRepository) -> None: + self._repository = repository + + async def create(self, actor: User, title: str, member_ids: Sequence[UUID]) -> Chat: + if actor.id not in member_ids: + raise Invalid('вызывающий должен входить в member_ids') + return await self._repository.create( + chat_id=uuid4(), + title=title, + member_ids=member_ids, + created_at=datetime.now(timezone.utc), + ) + + async def get(self, actor: User, chat_id: UUID) -> Chat: + chat = await self._repository.get_for_member(chat_id, actor.id) + if chat is None: + raise NotFound('чат не найден') + return chat + + async def page(self, actor: User, after: Optional[Keyset], limit: int) -> Page: + rows = await self._repository.page_for_member(actor.id, after, limit + 1) + items = tuple(rows[:limit]) + return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None) diff --git a/app/services/messages.py b/app/services/messages.py new file mode 100644 index 00000000..ec3de200 --- /dev/null +++ b/app/services/messages.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Optional +from uuid import UUID, uuid4 + +from ..domain.errors import NotFound +from ..domain.models import Keyset, Message, Page, User +from ..repositories import MessageRepository + + +class MessageService: + def __init__(self, repository: MessageRepository) -> None: + self._repository = repository + + async def send(self, actor: User, chat_id: UUID, text: str, idempotency_key: str) -> Message: + message = await self._repository.create_for_member( + message_id=uuid4(), + chat_id=chat_id, + sender_id=actor.id, + text=text, + created_at=datetime.now(timezone.utc), + idempotency_key=idempotency_key, + ) + if message is None: + raise NotFound('чат не найден') + return message + + async def history( + self, actor: User, chat_id: UUID, after: Optional[Keyset], limit: int + ) -> Page: + rows = await self._repository.page_for_member(chat_id, actor.id, after, limit + 1) + if rows is None: + raise NotFound('чат не найден') + items = tuple(rows[:limit]) + return Page(items=items, next=Keyset.from_item(items[-1]) if len(rows) > limit else None) diff --git a/app/services/users.py b/app/services/users.py new file mode 100644 index 00000000..9a516a86 --- /dev/null +++ b/app/services/users.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from ..auth.passwords import hash_password, password_matches +from ..domain.errors import Unauthorized +from ..domain.models import User +from ..repositories import UserRepository + + +class UserService: + def __init__(self, repository: UserRepository) -> None: + self._repository = repository + + async def register(self, username: str, display_name: str, password: str) -> User: + return await self._repository.register( + user_id=uuid4(), + username=username, + display_name=display_name, + password_hash=hash_password(password), + created_at=datetime.now(timezone.utc), + ) + + async def identify(self, user_id: UUID) -> User: + user = await self._repository.get_by_id(user_id) + if user is None: + raise Unauthorized('требуется действующая сессия') + return user + + async def authenticate(self, username: str, password: str) -> User | None: + row = await self._repository.get_credentials(username) + if row is None or not password_matches(row.password_hash, password): + return None + return self._repository.to_domain(row) diff --git a/compose.tests.yaml b/compose.tests.yaml new file mode 100644 index 00000000..65e4f825 --- /dev/null +++ b/compose.tests.yaml @@ -0,0 +1,62 @@ +services: + postgres: + image: postgres:17.4-alpine + environment: + POSTGRES_USER: messenger_test + POSTGRES_PASSWORD: messenger_test + POSTGRES_DB: messenger_test + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger_test -d messenger_test"] + interval: 2s + timeout: 2s + retries: 30 + + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 30 + + migrate: + build: . + command: ["alembic", "upgrade", "head"] + environment: + DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: 1 + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + build: . + environment: + DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: 1 + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"] + interval: 2s + timeout: 2s + retries: 30 + + tests: + build: + context: . + dockerfile: Dockerfile.test + environment: + BASE_URL: http://api:8080 + SESSION_TTL_SECONDS: 1 + depends_on: + api: + condition: service_healthy + restart: "no" diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..0511f041 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,90 @@ +services: + postgres: + image: postgres:17.4-alpine + cpus: "1.0" + mem_limit: 512m + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 3s + retries: 20 + + redis: + image: redis:7.4.2-alpine + cpus: "0.25" + mem_limit: 128m + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 20 + + migrate: + build: . + cpus: "0.5" + mem_limit: 256m + command: ["alembic", "upgrade", "head"] + environment: + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + build: . + cpus: "0.5" + mem_limit: 256m + environment: + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + LOG_LEVEL: ${LOG_LEVEL:-info} + expose: + - "8080" + deploy: + replicas: 2 + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"] + interval: 2s + timeout: 2s + retries: 5 + stop_grace_period: 10s + + lb: + image: haproxy:3.2.26-alpine + cpus: "0.25" + mem_limit: 128m + ports: + - "${APP_PORT:-8080}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/live"] + interval: 2s + timeout: 2s + retries: 5 + stop_grace_period: 5s + +volumes: + postgres_data: + redis_data: diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..5bd0c142 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,31 @@ +global + log stdout format raw local0 + +defaults + log global + mode http + option httplog + timeout connect 2s + timeout client 30s + timeout server 30s + timeout check 2s + +resolvers docker + nameserver docker_dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + +frontend public + bind :8080 + default_backend api_backend + +backend api_backend + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none + server-template api 2 api:8080 diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b103ddb..8d16cf19 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ # Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. openapi-spec-validator==0.7.2 +pytest==8.3.5 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..31148b51 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +SQLAlchemy[asyncio]==2.0.40 +asyncpg==0.30.0 +alembic==1.15.2 +redis==5.2.1 +argon2-cffi==25.1.0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..691598a3 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +project_dir=$(dirname -- "$script_dir") +cd "$project_dir" + +# Останавливаем публичный маршрут до перезапуска общих хранилищ. +docker compose --env-file .env stop lb api >/dev/null +docker compose --env-file .env restart postgres redis >/dev/null +docker compose --env-file .env up -d --wait --wait-timeout 60 postgres redis >/dev/null +docker compose --env-file .env up -d --wait --wait-timeout 60 api lb >/dev/null diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..3f770799 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -eu + +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +project_dir=$(dirname -- "$script_dir") +cd "$project_dir" + +target_container='' +for container_id in $(docker compose --env-file .env ps -q api); do + instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container_id") + if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then + target_container=$container_id + break + fi +done + +if [ -z "$target_container" ]; then + printf '%s\n' 'API instance from TARGET_INSTANCE_ID was not found' >&2 + exit 1 +fi + +docker stop --time 5 "$target_container" >/dev/null -- 2.54.0 From 0ed4b9cea5f2151aeca0537f400b580d966203fc Mon Sep 17 00:00:00 2001 From: thefankor Date: Thu, 5 Nov 2026 19:45:49 +0300 Subject: [PATCH 2/5] =?UTF-8?q?feat:=20=D0=9B=D0=A0=204=20=D1=81=20=D1=85?= =?UTF-8?q?=D1=80=D0=B0=D0=BD=D0=B5=D0=BD=D0=B8=D0=B5=D0=BC=20=D0=B2=D0=BB?= =?UTF-8?q?=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=B2=20MinIO?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 22 ++- Dockerfile.test | 7 +- Makefile | 12 +- REPORT.md | 177 ++++++++++++++++-- alembic/versions/20261030_0003_attachments.py | 69 +++++++ app/api/deps.py | 6 + app/api/errors.py | 13 +- app/api/routers/attachments.py | 66 +++++++ app/api/routers/health.py | 1 + app/api/routers/messages.py | 4 +- app/api/schemas.py | 55 +++++- app/config/settings.py | 32 +++- app/domain/errors.py | 8 + app/domain/models.py | 19 ++ app/main.py | 39 +++- app/models/__init__.py | 10 +- app/models/attachment.py | 49 +++++ app/repositories/__init__.py | 3 +- app/repositories/attachments.py | 93 +++++++++ app/repositories/messages.py | 51 ++++- app/services/attachments.py | 144 ++++++++++++++ app/services/messages.py | 12 +- app/storage/__init__.py | 3 + app/storage/s3.py | 121 ++++++++++++ app/tools/__init__.py | 1 + app/tools/cleanup_orphans.py | 61 ++++++ compose.tests.yaml | 57 +++++- compose.yaml | 58 ++++++ evidence/results.md | 103 ++++++++++ evidence/upload-memory.json | 20 ++ infra/s3-api-policy.json | 15 ++ requirements.txt | 2 + scripts/check-private-bucket.sh | 25 +++ scripts/check-s3-outage.sh | 29 +++ scripts/measure-upload-memory.py | 109 +++++++++++ scripts/restart.sh | 4 +- tests/attachment_failover.py | 67 +++++++ tests/test_attachment_compensation.py | 54 ++++++ tests/test_attachments.py | 144 ++++++++++++++ tests/upload_load.py | 63 +++++++ 40 files changed, 1763 insertions(+), 65 deletions(-) create mode 100644 alembic/versions/20261030_0003_attachments.py create mode 100644 app/api/routers/attachments.py create mode 100644 app/models/attachment.py create mode 100644 app/repositories/attachments.py create mode 100644 app/services/attachments.py create mode 100644 app/storage/__init__.py create mode 100644 app/storage/s3.py create mode 100644 app/tools/__init__.py create mode 100644 app/tools/cleanup_orphans.py create mode 100644 evidence/results.md create mode 100644 evidence/upload-memory.json create mode 100644 infra/s3-api-policy.json create mode 100755 scripts/check-private-bucket.sh create mode 100755 scripts/check-s3-outage.sh create mode 100755 scripts/measure-upload-memory.py create mode 100644 tests/attachment_failover.py create mode 100644 tests/test_attachment_compensation.py create mode 100644 tests/test_attachments.py create mode 100644 tests/upload_load.py diff --git a/.env.example b/.env.example index d99d26a2..1d735209 100644 --- a/.env.example +++ b/.env.example @@ -4,17 +4,21 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 + +# Synthetic local credentials: replace them in your untracked .env. +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-me-local +POSTGRES_DB=messenger +DATABASE_URL=postgresql+asyncpg://messenger:change-me-local@postgres:5432/messenger +REDIS_URL=redis://redis:6379/0 + +MINIO_ROOT_USER=local-minio-admin +MINIO_ROOT_PASSWORD=change-me-minio-root S3_ENDPOINT=http://s3:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=change-me-minio-api MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +LOG_LEVEL=info diff --git a/Dockerfile.test b/Dockerfile.test index 51c08a89..5780cf9e 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,14 +1,15 @@ FROM python:3.13.15-slim WORKDIR /tests -COPY requirements-dev.txt . -RUN python -m pip install --no-cache-dir -r requirements-dev.txt \ +COPY requirements.txt requirements-dev.txt ./ +RUN python -m pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt \ && useradd --uid 10001 --user-group --no-create-home --shell /usr/sbin/nologin tests COPY lab.json . +COPY app ./app COPY contracts ./contracts COPY tests ./tests RUN chown -R tests:tests /tests USER 10001:10001 -CMD ["pytest", "-q"] +CMD ["python", "-m", "pytest", "-q"] diff --git a/Makefile b/Makefile index 5d86d171..2b90fec8 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test test-attachments load load-attachments test-persistence test-failover test-attachment-failover help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -18,9 +18,15 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-attachments: + BASE_URL="$(BASE_URL)" $(PYTHON) -m pytest -q tests/test_attachments.py + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) +load-attachments: + $(PYTHON) tests/upload_load.py --base-url "$(BASE_URL)" --concurrency 10 --uploads 30 $(TLS_ARGS) + test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) @@ -28,3 +34,7 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-attachment-failover: + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) + $(PYTHON) tests/attachment_failover.py --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..212f1568 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,182 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Сергей Возжин, БПИ2301. +- Целевая оценка: 4; трек 5 не выбран. +- Commit/PR: TODO после публикации. +- Использованные источники: контракт репозитория, документация Docker Compose, Chainguard Images, MinIO SDK и Pillow. +- Библиотеки: FastAPI, Uvicorn, SQLAlchemy, asyncpg, Alembic, redis-py, MinIO Python SDK, Pillow, Argon2, pytest. +- Помощь ИИ: OpenAI Codex использовался для плана, реализации, тестов и оформления отчёта; результат проверен исполняемыми тестами. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Engine с Compose v2, Python 3.9+ для публичных тестов и свободный host-порт 8080. Проверенный стенд: Apple M4 Pro, 12 CPU cores, 24 GB RAM, Docker Desktop; API на Python 3.13.15. + +Закреплённые компоненты: + +- `python:3.13.15-slim`; +- `postgres:17.4-alpine`; +- `redis:7.4.2-alpine`; +- `haproxy:3.2.26-alpine`; +- Chainguard MinIO `RELEASE.2026-10-23T20-06-51Z`, OCI index digest `sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597`; +- Chainguard MinIO Client, вариант `latest-dev`, OCI index digest `sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805` (`mc --version` сообщает upstream build marker `DEVELOPMENT.GOGET`). + +Готовые образы MinIO и `mc` скачиваются из реестра Chainguard `cgr.dev`; в Compose оба образа закреплены по OCI digest. Сервис `s3` использует минимальный образ `minio` и в учебном стенде запускается с `user: "0"`, чтобы иметь доступ к Docker volume `/data`; его порты на host не публикуются. Для одноразового `s3-bootstrap` используется образ `minio-client` с shell и `sed`, необходимыми для создания bucket, отдельного пользователя API и bucket-scoped policy. + +Создать локальную конфигурацию и заменить синтетические пароли, не добавляя `.env` в Git: + +```sh +cp .env.example .env +``` + +`S3_BUCKET` допускает изменение: bootstrap подставляет его в ограниченную policy. `MINIO_ROOT_*` получает только MinIO и bootstrap; API получает отдельные `S3_ACCESS_KEY_ID` и `S3_SECRET_ACCESS_KEY`. + +Проверить конфигурацию, собрать образы, применить миграции и дождаться готовности: + +```sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +docker compose --env-file .env ps +``` + +`migrate` выполняет `alembic upgrade head` отдельно и однократно. `s3-bootstrap` идемпотентно создаёт приватный bucket, отдельного пользователя API и bucket-scoped policy. Две API-реплики стартуют только после миграции, Redis и успешного bootstrap. Единственная публичная точка входа — `http://localhost:8080`; API, PostgreSQL, Redis, S3 API и административная консоль MinIO наружу не публикуются. + +Основные проверки: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make check +make validate PYTHON=.venv/bin/python +make test +make test-attachments PYTHON=.venv/bin/python +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +docker compose --env-file .env up -d --wait --wait-timeout 60 api lb +make test-attachment-failover ACTION_SCRIPT=scripts/stop-one.sh +docker compose --env-file .env up -d --wait --wait-timeout 60 api lb +scripts/check-private-bucket.sh +scripts/check-s3-outage.sh +.venv/bin/python scripts/measure-upload-memory.py --concurrency 10 --uploads 30 +``` + +Каждый failover-тест намеренно оставляет выбранную API-реплику остановленной, поэтому между сценариями и после последнего сценария она явно восстанавливается указанной командой `docker compose up`. + +Безопасная остановка с сохранением данных: + +```sh +docker compose --env-file .env down +``` + +Намеренное необратимое удаление учебных данных выполняет только оператор: + +```sh +docker compose --env-file .env down -v +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +HTTP client + | + v +HAProxy :8080 + | + +-- round-robin --> API replica 1 --+--> PostgreSQL: metadata + | +--> Redis: sessions + +-- round-robin --> API replica 2 --+--> private MinIO: original bytes +``` + +Из lab3 сохранены две взаимозаменяемые API-реплики, HAProxy, общие PostgreSQL/Redis, Bearer-сессии, `X-Instance-Id`, общая идемпотентность сообщений, readiness и failover hooks. + +В lab4 добавлены: + +- MinIO и persistent `s3_data` volume; +- таблицы `attachments` и `message_attachments`; +- синхронный binary upload PNG/JPEG; +- авторизованные metadata/download endpoints; +- `attachment_ids` в сообщениях и в области сравнения идемпотентного body; +- компенсационное удаление объекта и безопасный cleanup старых orphan-объектов. + +PostgreSQL является источником истины для метаданных. В БД не хранятся bytes изображения. MinIO хранит оригинал по серверному ключу `originals/`. Пользовательский `filename` хранится только как метаданные и не используется в object key или пути файловой системы. + +### Upload и лимиты + +API сначала проверяет сессию, доступ к чату, filename и заявленный MIME. Тело читается чанками в `SpooledTemporaryFile`: до 1 MiB находится в памяти, затем временно сбрасывается на ephemeral disk API. Как только суммарный размер становится больше 10 MiB, запрос отклоняется с 413; превысивший лимит чанк в spool не записывается. Постоянной локальной копии нет. + +Pillow определяет реальный PNG/JPEG, проверяет совпадение MIME, размеры до декодирования пикселей и целостность изображения через `verify()`. Максимум — 10 485 760 bytes и 40 000 000 пикселей. Пустой, повреждённый или чужой формат возвращает 415; превышение лимита bytes/pixels — 413. + +S3-вызовы выполняются вне event loop. HTTP client S3 имеет ограниченные connect/read timeout и retry. Readiness проверяет PostgreSQL, Redis и существование bucket. + +### Транзакционная граница PostgreSQL/S3 + +Атомарной транзакции между S3 и PostgreSQL нет. Для синхронного `201 ready` выбран порядок: + +1. Полностью проверить ограниченный файл. +2. Выполнить S3 PUT. +3. Выполнить транзакцию INSERT метаданных. +4. Вернуть `201` только после успешного завершения обоих шагов. + +Если PUT не удался, метаданные не создаются. Если PUT прошёл, а транзакция БД упала, API выполняет best-effort S3 DELETE. Unit-тест с внедрённой ошибкой БД подтверждает компенсацию. + +Если компенсационное удаление также не удалось, остаётся orphan без метаданных. CLI находит такие объекты: + +```sh +docker compose --env-file .env run --rm --no-deps api \ + python -m app.tools.cleanup_orphans --dry-run --grace-seconds 3600 + +docker compose --env-file .env run --rm --no-deps api \ + python -m app.tools.cleanup_orphans --apply --grace-seconds 3600 +``` + +По умолчанию используется grace period один час; CLI запрещает значения меньше 60 секунд. Объекты с существующими метаданными и свежие orphan-объекты не удаляются. Сначала используется `--dry-run`, затем осознанный `--apply`. + +### Доступ и идемпотентность + +Метаданные и content доступны любому участнику чата. Неизвестный объект и запрос постороннего одинаково возвращают 404. Download идёт через API без redirect. Публичный ответ не раскрывает object key, endpoint или credentials. + +Сообщение принимает 0–10 уникальных `attachment_ids`. Все вложения должны быть `ready` и относиться к тому же чату. Пустой текст допустим только с вложением. Сравнение идемпотентного запроса включает упорядоченный список attachment ID; отсутствие поля и `[]` считаются одинаковыми. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий README | Команда/сценарий | Наблюдаемый результат | Доказательство | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Offline/OpenAPI | `make check`; `make validate PYTHON=.venv/bin/python` | оба PASS | `evidence/results.md` | +| Публичный smoke | `make test` | `PASS: lab4 public API scenario` | `evidence/results.md` | +| Негативные сценарии | `make test-attachments PYTHON=.venv/bin/python` | `12 passed` | `tests/test_attachments.py`, `evidence/results.md` | +| Компенсация S3/БД | unit-тест с ошибкой repository | S3 DELETE вызван для загруженного ключа; `1 passed` | `tests/test_attachment_compensation.py` | +| Private bucket | `scripts/check-private-bucket.sh` | anonymous GET получил 403 | `evidence/results.md` | +| Недоступность S3 | `scripts/check-s3-outage.sh` | readiness вернул структурированный 503; стенд восстановлен | `evidence/results.md` | +| Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | прежние token, metadata и bytes доступны после restart PostgreSQL/Redis/MinIO/API | `evidence/results.md` | +| Общий lab3 failover | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | 111 записей, 1 переходная ошибка, восстановление 2,333 с, файл сохранён | `evidence/results.md` | +| Точный attachment failover | `make test-attachment-failover ACTION_SCRIPT=scripts/stop-one.sh` | загрузившая реплика остановлена, download через другую за 3,912 с, SHA-256 совпал | `evidence/results.md` | +| Concurrency/RAM | `scripts/measure-upload-memory.py --concurrency 10 --uploads 30` | 30/30 upload, обе реплики, пики 68,50/68,33 MiB | `evidence/upload-memory.json` | +| Идемпотентный bootstrap | повторный запуск `docker compose up` после restart/failover | bootstrap повторно завершился с code 0 | `evidence/results.md` | +| Cleanup dry-run | `cleanup_orphans --dry-run --grace-seconds 60` | 164 известных объекта, удалено 0 | `evidence/results.md` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Проверяемые ожидания: + +- остановка загрузившей API-реплики не меняет файл и не лишает вторую реплику доступа; +- restart обязательных хранилищ с volumes сохраняет метаданные, bytes и сессию; +- недоступность S3 наблюдаема как 503, а не 500; +- 10 параллельных upload не приводят к неограниченному росту RAM. + +Профиль upload: 30 запросов, concurrency 10, fixture 111 817 bytes, closed-loop, без retry. Получено 30 успешных upload за 0,382 с; максимальная latency 0,300 с; задействованы обе API-реплики. Пиковая наблюдаемая RAM: 68,50 MiB и 68,33 MiB при лимите 256 MiB на реплику. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Restart API + PostgreSQL + Redis + MinIO | token, metadata и bytes сохраняются | persistence PASS | потерь нет | +| Остановка API в общем failover | вторая реплика продолжает чтение/запись | 111 записей, 1 краткая ошибка | 2,333 с; потерь и дублей нет | +| Остановка загрузившей API | файл скачивается через другую | SHA-256 совпал, разные instance ID | 3,912 с; потерь нет | +| Остановка MinIO | readiness сообщает отказ зависимости | структурированный 503, после запуска readiness восстановился | потерь нет | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Стенд выдерживает остановку одной API-реплики и штатный restart контейнеров с сохранёнными volumes. Он не выдерживает потерю Docker-host, единственного HAProxy, PostgreSQL, Redis или MinIO volume. Несколько API-процессов не делают единственные хранилища высокодоступными. + +Upload синхронный, поэтому клиент ждёт S3 и PostgreSQL. `SpooledTemporaryFile` ограничивает RAM одной загрузки, но временный disk API должен иметь достаточный объём и отдельную квоту в production. Cleanup делает полный список объектов и строк БД; для большого bucket нужны пагинация, метрики и плановый запуск. + +Chainguard-образы зафиксированы по digest, поэтому обновляются только явно. Стенд изолирован учебной сетью. Перед production-развёртыванием потребовались бы непривилегированный UID с корректными правами на volume, TLS, внешний secret manager, HA хранилищ, аудит и резервное копирование. + +Presigned upload, worker, thumbnail, варианты изображений и исследовательский reconciler для оценки 5 намеренно не реализованы. diff --git a/alembic/versions/20261030_0003_attachments.py b/alembic/versions/20261030_0003_attachments.py new file mode 100644 index 00000000..da724596 --- /dev/null +++ b/alembic/versions/20261030_0003_attachments.py @@ -0,0 +1,69 @@ +"""Add S3 attachment metadata and message links for lab 4. + +Revision ID: 20261030_0003 +Revises: 20261023_0002 +Create Date: 2026-10-30 +""" +from alembic import op +import sqlalchemy as sa + + +revision = '20261030_0003' +down_revision = '20261023_0002' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + 'attachments', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('chat_id', sa.Uuid(), nullable=False), + sa.Column('owner_id', sa.Uuid(), nullable=False), + sa.Column('filename', sa.String(length=255), nullable=False), + sa.Column('object_key', sa.String(length=255), nullable=False), + sa.Column('content_type', sa.String(length=32), nullable=False), + sa.Column('size_bytes', sa.BigInteger(), nullable=False), + sa.Column('sha256', sa.String(length=64), nullable=False), + sa.Column('width', sa.Integer(), nullable=False), + sa.Column('height', sa.Integer(), nullable=False), + sa.Column('status', sa.String(length=16), nullable=False), + sa.Column('error_code', sa.String(length=64), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['owner_id'], ['users.id']), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('object_key', name='attachments_object_key_key'), + ) + op.create_index( + 'attachments_chat_created_idx', + 'attachments', + ['chat_id', 'created_at', 'id'], + ) + op.create_table( + 'message_attachments', + sa.Column('message_id', sa.Uuid(), nullable=False), + sa.Column('attachment_id', sa.Uuid(), nullable=False), + sa.Column('position', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['attachment_id'], ['attachments.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['message_id'], ['messages.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('message_id', 'attachment_id'), + sa.UniqueConstraint( + 'message_id', + 'position', + name='message_attachments_message_position_key', + ), + ) + op.create_index( + 'message_attachments_attachment_idx', + 'message_attachments', + ['attachment_id'], + ) + + +def downgrade() -> None: + op.drop_index('message_attachments_attachment_idx', table_name='message_attachments') + op.drop_table('message_attachments') + op.drop_index('attachments_chat_created_idx', table_name='attachments') + op.drop_table('attachments') diff --git a/app/api/deps.py b/app/api/deps.py index 10d16267..4d3c634d 100644 --- a/app/api/deps.py +++ b/app/api/deps.py @@ -8,6 +8,7 @@ from ..db.redis import RedisSessionStore from ..domain.errors import Unauthorized from ..domain.models import User from ..services.chats import ChatService +from ..services.attachments import AttachmentService from ..services.messages import MessageService from ..services.users import UserService @@ -16,6 +17,7 @@ _BEARER_CHALLENGE = 'Bearer realm="messenger"' @dataclass(frozen=True, slots=True) class Services: + attachments: AttachmentService users: UserService chats: ChatService messages: MessageService @@ -38,6 +40,10 @@ def message_service(container: Services = Depends(services)) -> MessageService: return container.messages +def attachment_service(container: Services = Depends(services)) -> AttachmentService: + return container.attachments + + def session_store(container: Services = Depends(services)) -> RedisSessionStore: return container.sessions diff --git a/app/api/errors.py b/app/api/errors.py index 85a7c8e4..8208ad1d 100644 --- a/app/api/errors.py +++ b/app/api/errors.py @@ -6,7 +6,16 @@ from fastapi.responses import JSONResponse from sqlalchemy.exc import SQLAlchemyError from starlette.exceptions import HTTPException as StarletteHTTPException -from ..domain.errors import Conflict, DomainError, Invalid, NotFound, Unauthorized, Unavailable +from ..domain.errors import ( + Conflict, + DomainError, + Invalid, + NotFound, + PayloadTooLarge, + Unauthorized, + Unavailable, + UnsupportedMediaType, +) logger = logging.getLogger('app.error') @@ -27,6 +36,8 @@ STATUS_BY_DOMAIN = { Unauthorized: 401, NotFound: 404, Conflict: 409, + PayloadTooLarge: 413, + UnsupportedMediaType: 415, Unavailable: 503, } diff --git a/app/api/routers/attachments.py b/app/api/routers/attachments.py new file mode 100644 index 00000000..e799f5cf --- /dev/null +++ b/app/api/routers/attachments.py @@ -0,0 +1,66 @@ +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Depends, Header, Query, Request +from fastapi.responses import StreamingResponse + +from ...domain.models import User +from ...services.attachments import AttachmentService +from ..deps import attachment_service, current_user +from ..schemas import AttachmentOut + +router = APIRouter(tags=['attachments']) + + +@router.post( + '/api/v1/chats/{chat_id}/attachments', + status_code=201, + response_model=AttachmentOut, +) +async def upload( + chat_id: UUID, + request: Request, + filename: str = Query(..., min_length=1, max_length=255), + content_type: Annotated[str | None, Header(alias='Content-Type')] = None, + actor: User = Depends(current_user), + attachments: AttachmentService = Depends(attachment_service), +) -> AttachmentOut: + attachment = await attachments.upload( + actor, + chat_id, + filename, + content_type or '', + request.stream(), + ) + return AttachmentOut.from_domain(attachment) + + +@router.get('/api/v1/attachments/{attachment_id}', response_model=AttachmentOut) +async def metadata( + attachment_id: UUID, + actor: User = Depends(current_user), + attachments: AttachmentService = Depends(attachment_service), +) -> AttachmentOut: + return AttachmentOut.from_domain(await attachments.get(actor, attachment_id)) + + +@router.get('/api/v1/attachments/{attachment_id}/content') +async def content( + attachment_id: UUID, + variant: str = Query('original'), + actor: User = Depends(current_user), + attachments: AttachmentService = Depends(attachment_service), +) -> StreamingResponse: + if variant != 'original': + from ...domain.errors import NotFound + + raise NotFound('вариант вложения не найден') + attachment, stream = await attachments.content(actor, attachment_id) + return StreamingResponse( + stream.chunks(), + media_type=attachment.content_type, + headers={ + 'Content-Length': str(attachment.size_bytes), + 'Content-Disposition': f'inline; filename="attachment-{attachment.id}"', + }, + ) diff --git a/app/api/routers/health.py b/app/api/routers/health.py index a4e2029b..853fa39e 100644 --- a/app/api/routers/health.py +++ b/app/api/routers/health.py @@ -23,6 +23,7 @@ async def ready(request: Request) -> HealthOut: async with request.app.state.engine.connect() as connection: await connection.execute(text('SELECT 1')) await request.app.state.redis.ping() + await request.app.state.storage.ping() except (SQLAlchemyError, RedisError, Unavailable, OSError): raise HTTPException(status_code=503, detail='обязательное хранилище недоступно') from None return HealthOut(status='ok') diff --git a/app/api/routers/messages.py b/app/api/routers/messages.py index b97f10f6..7a86200d 100644 --- a/app/api/routers/messages.py +++ b/app/api/routers/messages.py @@ -35,7 +35,9 @@ async def send( actor: User = Depends(current_user), messages: MessageService = Depends(message_service), ) -> MessageOut: - return MessageOut.from_domain(await messages.send(actor, chat_id, body.text, idempotency_key)) + return MessageOut.from_domain( + await messages.send(actor, chat_id, body.text, body.attachment_ids, idempotency_key) + ) @router.get('', response_model=MessagePageOut) diff --git a/app/api/schemas.py b/app/api/schemas.py index 145b0c0c..acdc2214 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -1,9 +1,9 @@ from typing import Optional from uuid import UUID -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from ..domain.models import Chat, Message, Page, User +from ..domain.models import Attachment, Chat, Message, Page, User from . import cursors from .timefmt import to_rfc3339 @@ -97,12 +97,21 @@ class ChatOut(Strict): class CreateMessage(Strict): - text: str = Field(min_length=1, max_length=TEXT_MAX) + text: str = Field(max_length=TEXT_MAX) + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) - @field_validator('text') + @field_validator('attachment_ids') @classmethod - def _text(cls, value: str) -> str: - return not_blank(value) + def _attachments_unique(cls, value: list[UUID]) -> list[UUID]: + if len(set(value)) != len(value): + raise ValueError('attachment_ids не должен содержать дубликаты') + return value + + @model_validator(mode='after') + def _content_present(self) -> 'CreateMessage': + if not self.text.strip() and not self.attachment_ids: + raise ValueError('сообщение должно содержать текст или вложение') + return self class MessageOut(Strict): @@ -111,6 +120,7 @@ class MessageOut(Strict): sender_id: UUID text: str created_at: str + attachment_ids: list[UUID] @classmethod def from_domain(cls, message: Message) -> 'MessageOut': @@ -120,6 +130,39 @@ class MessageOut(Strict): sender_id=message.sender_id, text=message.text, created_at=to_rfc3339(message.created_at), + attachment_ids=list(message.attachment_ids), + ) + + +class AttachmentOut(Strict): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + variants: list = Field(default_factory=list) + error_code: str | None + created_at: str + updated_at: str + + @classmethod + def from_domain(cls, attachment: Attachment) -> 'AttachmentOut': + return cls( + id=attachment.id, + chat_id=attachment.chat_id, + owner_id=attachment.owner_id, + filename=attachment.filename, + content_type=attachment.content_type, + size_bytes=attachment.size_bytes, + sha256=attachment.sha256, + status=attachment.status, + variants=[], + error_code=attachment.error_code, + created_at=to_rfc3339(attachment.created_at), + updated_at=to_rfc3339(attachment.updated_at), ) diff --git a/app/config/settings.py b/app/config/settings.py index 886023e0..92aa9469 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -20,17 +20,35 @@ class Settings: database_url: str redis_url: str session_ttl_seconds: int + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key_id: str + s3_secret_access_key: str + max_upload_bytes: int + max_image_pixels: int + + +def _positive_int(name: str, default: str) -> int: + try: + value = int(os.getenv(name, default)) + except ValueError as exc: + raise RuntimeError(f'{name} должен быть целым числом') from exc + if value <= 0: + raise RuntimeError(f'{name} должен быть больше нуля') + return value def load_settings() -> Settings: - try: - ttl = int(os.getenv('SESSION_TTL_SECONDS', '3600')) - except ValueError as exc: - raise RuntimeError('SESSION_TTL_SECONDS должен быть целым числом') from exc - if ttl <= 0: - raise RuntimeError('SESSION_TTL_SECONDS должен быть больше нуля') return Settings( database_url=_read_value('DATABASE_URL', 'DATABASE_URL_FILE'), redis_url=_read_value('REDIS_URL', 'CACHE_URL_FILE'), - session_ttl_seconds=ttl, + session_ttl_seconds=_positive_int('SESSION_TTL_SECONDS', '3600'), + s3_endpoint=os.getenv('S3_ENDPOINT', 'http://s3:9000'), + s3_region=os.getenv('S3_REGION', 'us-east-1'), + s3_bucket=os.getenv('S3_BUCKET', 'messenger-attachments'), + s3_access_key_id=_read_value('S3_ACCESS_KEY_ID', 'S3_ACCESS_KEY_ID_FILE'), + s3_secret_access_key=_read_value('S3_SECRET_ACCESS_KEY', 'S3_SECRET_ACCESS_KEY_FILE'), + max_upload_bytes=_positive_int('MAX_UPLOAD_BYTES', '10485760'), + max_image_pixels=_positive_int('MAX_IMAGE_PIXELS', '40000000'), ) diff --git a/app/domain/errors.py b/app/domain/errors.py index d15b362b..c6c45560 100644 --- a/app/domain/errors.py +++ b/app/domain/errors.py @@ -27,3 +27,11 @@ class Conflict(DomainError): class Unavailable(DomainError): """Обязательная внешняя зависимость временно недоступна.""" + + +class PayloadTooLarge(DomainError): + """Тело запроса или декодированное содержимое превышает лимит.""" + + +class UnsupportedMediaType(DomainError): + """Формат содержимого не поддерживается или не соответствует MIME.""" diff --git a/app/domain/models.py b/app/domain/models.py index 33668bbd..41df1fdb 100644 --- a/app/domain/models.py +++ b/app/domain/models.py @@ -30,6 +30,25 @@ class Message: sender_id: UUID text: str created_at: datetime + attachment_ids: tuple[UUID, ...] = () + + +@dataclass(frozen=True, slots=True) +class Attachment: + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + object_key: str + content_type: str + size_bytes: int + sha256: str + width: int + height: int + status: str + error_code: str | None + created_at: datetime + updated_at: datetime @dataclass(frozen=True, slots=True) diff --git a/app/main.py b/app/main.py index 9426a9d1..66d9c98f 100644 --- a/app/main.py +++ b/app/main.py @@ -12,17 +12,19 @@ from sqlalchemy import text from .api import errors from .api.deps import Services from .api.middleware import RequestContext -from .api.routers import auth, chats, health, messages, users +from .api.routers import attachments, auth, chats, health, messages, users from .config import load_settings from .db import RedisSessionStore, create_engine, create_redis, create_session_factory -from .repositories import ChatRepository, MessageRepository, UserRepository +from .repositories import AttachmentRepository, ChatRepository, MessageRepository, UserRepository +from .services.attachments import AttachmentService from .services.chats import ChatService from .services.messages import MessageService from .services.users import UserService +from .storage import S3Storage logger = logging.getLogger('app') -DESCRIPTION = 'PostgreSQL хранит данные мессенджера; Redis хранит активные сессии.' +DESCRIPTION = 'PostgreSQL хранит метаданные, Redis — сессии, приватный S3 — оригиналы вложений.' def configure_logging() -> None: @@ -43,16 +45,31 @@ def create_app() -> FastAPI: settings = load_settings() engine = create_engine(settings.database_url) redis = create_redis(settings.redis_url) + storage = S3Storage( + endpoint=settings.s3_endpoint, + access_key=settings.s3_access_key_id, + secret_key=settings.s3_secret_access_key, + bucket=settings.s3_bucket, + region=settings.s3_region, + ) try: async with engine.connect() as connection: await connection.execute(text('SELECT 1')) session_store = RedisSessionStore(redis, settings.session_ttl_seconds) await session_store.ping() + await storage.ping() session_factory = create_session_factory(engine) user_repository = UserRepository(session_factory) chat_repository = ChatRepository(session_factory) message_repository = MessageRepository(session_factory) + attachment_repository = AttachmentRepository(session_factory) application.state.services = Services( + attachments=AttachmentService( + attachment_repository, + storage, + settings.max_upload_bytes, + settings.max_image_pixels, + ), users=UserService(user_repository), chats=ChatService(chat_repository), messages=MessageService(message_repository), @@ -60,8 +77,9 @@ def create_app() -> FastAPI: ) application.state.engine = engine application.state.redis = redis + application.state.storage = storage application.state.ready = True - logger.info('startup storage=postgres redis_sessions=true') + logger.info('startup storage=postgres+s3 redis_sessions=true') yield finally: application.state.ready = False @@ -70,8 +88,8 @@ def create_app() -> FastAPI: logger.info('shutdown complete') api = FastAPI( - title='MTUSI Messenger. LR 3', - version='1.3.0', + title='MTUSI Messenger. LR 4', + version='1.4.0', description=DESCRIPTION, lifespan=lifespan, ) @@ -79,7 +97,14 @@ def create_app() -> FastAPI: api.state.ready = False api.add_middleware(RequestContext) errors.install(api) - for router in (health.router, users.router, auth.router, chats.router, messages.router): + for router in ( + health.router, + users.router, + auth.router, + chats.router, + messages.router, + attachments.router, + ): api.include_router(router) return api diff --git a/app/models/__init__.py b/app/models/__init__.py index 64b2f78f..9d34b52b 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,5 +1,13 @@ +from .attachment import AttachmentRow, MessageAttachmentRow from .chat import ChatMemberRow, ChatRow from .message import MessageRow from .user import UserRow -__all__ = ['ChatMemberRow', 'ChatRow', 'MessageRow', 'UserRow'] +__all__ = [ + 'AttachmentRow', + 'ChatMemberRow', + 'ChatRow', + 'MessageAttachmentRow', + 'MessageRow', + 'UserRow', +] diff --git a/app/models/attachment.py b/app/models/attachment.py new file mode 100644 index 00000000..4f89286f --- /dev/null +++ b/app/models/attachment.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, String, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.types import Uuid + +from ..db.postgres import Base + + +class AttachmentRow(Base): + __tablename__ = 'attachments' + __table_args__ = ( + Index('attachments_chat_created_idx', 'chat_id', 'created_at', 'id'), + UniqueConstraint('object_key', name='attachments_object_key_key'), + ) + + id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + chat_id: Mapped[UUID] = mapped_column(ForeignKey('chats.id', ondelete='CASCADE'), nullable=False) + owner_id: Mapped[UUID] = mapped_column(ForeignKey('users.id'), nullable=False) + filename: Mapped[str] = mapped_column(String(255), nullable=False) + object_key: Mapped[str] = mapped_column(String(255), nullable=False) + content_type: Mapped[str] = mapped_column(String(32), nullable=False) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + sha256: Mapped[str] = mapped_column(String(64), nullable=False) + width: Mapped[int] = mapped_column(Integer, nullable=False) + height: Mapped[int] = mapped_column(Integer, nullable=False) + status: Mapped[str] = mapped_column(String(16), nullable=False) + error_code: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class MessageAttachmentRow(Base): + __tablename__ = 'message_attachments' + __table_args__ = ( + UniqueConstraint('message_id', 'position', name='message_attachments_message_position_key'), + Index('message_attachments_attachment_idx', 'attachment_id'), + ) + + message_id: Mapped[UUID] = mapped_column( + ForeignKey('messages.id', ondelete='CASCADE'), primary_key=True + ) + attachment_id: Mapped[UUID] = mapped_column( + ForeignKey('attachments.id', ondelete='CASCADE'), primary_key=True + ) + position: Mapped[int] = mapped_column(Integer, nullable=False) diff --git a/app/repositories/__init__.py b/app/repositories/__init__.py index 4d82b99c..a80de674 100644 --- a/app/repositories/__init__.py +++ b/app/repositories/__init__.py @@ -1,5 +1,6 @@ +from .attachments import AttachmentRepository from .chats import ChatRepository from .messages import MessageRepository from .users import UserRepository -__all__ = ['ChatRepository', 'MessageRepository', 'UserRepository'] +__all__ = ['AttachmentRepository', 'ChatRepository', 'MessageRepository', 'UserRepository'] diff --git a/app/repositories/attachments.py b/app/repositories/attachments.py new file mode 100644 index 00000000..d508f651 --- /dev/null +++ b/app/repositories/attachments.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from ..domain.models import Attachment +from ..models.attachment import AttachmentRow +from ..models.chat import ChatMemberRow, ChatRow + + +class AttachmentRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + @staticmethod + def _to_domain(row: AttachmentRow) -> Attachment: + return Attachment( + id=row.id, + chat_id=row.chat_id, + owner_id=row.owner_id, + filename=row.filename, + object_key=row.object_key, + content_type=row.content_type, + size_bytes=row.size_bytes, + sha256=row.sha256, + width=row.width, + height=row.height, + status=row.status, + error_code=row.error_code, + created_at=row.created_at, + updated_at=row.updated_at, + ) + + async def chat_exists_for_member(self, chat_id: UUID, user_id: UUID) -> bool: + async with self._sessions() as session: + found = await session.scalar( + select(ChatRow.id) + .join(ChatMemberRow, ChatMemberRow.chat_id == ChatRow.id) + .where(ChatRow.id == chat_id, ChatMemberRow.user_id == user_id) + ) + return found is not None + + async def create( + self, + *, + attachment_id: UUID, + chat_id: UUID, + owner_id: UUID, + filename: str, + object_key: str, + content_type: str, + size_bytes: int, + sha256: str, + width: int, + height: int, + now: datetime, + ) -> Attachment: + row = AttachmentRow( + id=attachment_id, + chat_id=chat_id, + owner_id=owner_id, + filename=filename, + object_key=object_key, + content_type=content_type, + size_bytes=size_bytes, + sha256=sha256, + width=width, + height=height, + status='ready', + error_code=None, + created_at=now, + updated_at=now, + ) + async with self._sessions() as session: + async with session.begin(): + session.add(row) + await session.flush() + return self._to_domain(row) + + async def get_for_member(self, attachment_id: UUID, user_id: UUID) -> Attachment | None: + async with self._sessions() as session: + row = await session.scalar( + select(AttachmentRow) + .join(ChatMemberRow, ChatMemberRow.chat_id == AttachmentRow.chat_id) + .where( + AttachmentRow.id == attachment_id, + ChatMemberRow.user_id == user_id, + ) + ) + return self._to_domain(row) if row is not None else None diff --git a/app/repositories/messages.py b/app/repositories/messages.py index f387e9a9..12a6459d 100644 --- a/app/repositories/messages.py +++ b/app/repositories/messages.py @@ -1,14 +1,16 @@ from __future__ import annotations from datetime import datetime +from typing import Sequence from uuid import UUID from sqlalchemy import select, tuple_ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker -from ..domain.errors import Conflict +from ..domain.errors import Conflict, NotFound from ..domain.models import Keyset, Message +from ..models.attachment import AttachmentRow, MessageAttachmentRow from ..models.chat import ChatMemberRow, ChatRow from ..models.message import MessageRow @@ -18,15 +20,25 @@ class MessageRepository: self._sessions = sessions @staticmethod - def _to_domain(row: MessageRow) -> Message: + def _to_domain(row: MessageRow, attachment_ids: Sequence[UUID]) -> Message: return Message( id=row.id, chat_id=row.chat_id, sender_id=row.sender_id, text=row.text, created_at=row.created_at, + attachment_ids=tuple(attachment_ids), ) + @staticmethod + async def _attachment_ids(session: AsyncSession, message_id: UUID) -> tuple[UUID, ...]: + result = await session.scalars( + select(MessageAttachmentRow.attachment_id) + .where(MessageAttachmentRow.message_id == message_id) + .order_by(MessageAttachmentRow.position) + ) + return tuple(result) + @staticmethod async def _accessible_chat_id(session: AsyncSession, chat_id: UUID, user_id: UUID) -> UUID | None: return await session.scalar( @@ -41,6 +53,7 @@ class MessageRepository: chat_id: UUID, sender_id: UUID, text: str, + attachment_ids: Sequence[UUID], created_at: datetime, idempotency_key: str, ) -> Message | None: @@ -50,6 +63,19 @@ class MessageRepository: if accessible_chat_id is None: return None + if attachment_ids: + found = set( + await session.scalars( + select(AttachmentRow.id).where( + AttachmentRow.id.in_(attachment_ids), + AttachmentRow.chat_id == accessible_chat_id, + AttachmentRow.status == 'ready', + ) + ) + ) + if found != set(attachment_ids): + raise NotFound('вложение не найдено') + statement = ( insert(MessageRow) .values( @@ -72,6 +98,16 @@ class MessageRepository: created_id = await session.scalar(statement) if created_id is not None: row = await session.get(MessageRow, created_id) + session.add_all( + MessageAttachmentRow( + message_id=created_id, + attachment_id=attachment_id, + position=position, + ) + for position, attachment_id in enumerate(attachment_ids) + ) + await session.flush() + result_attachment_ids = tuple(attachment_ids) else: row = await session.scalar( select(MessageRow).where( @@ -82,9 +118,10 @@ class MessageRepository: ) if row is None: raise RuntimeError('результат идемпотентной операции не найден') - if row.text != text: + result_attachment_ids = await self._attachment_ids(session, row.id) + if row.text != text or result_attachment_ids != tuple(attachment_ids): raise Conflict('Idempotency-Key уже использован с другим сообщением') - return self._to_domain(row) + return self._to_domain(row, result_attachment_ids) async def page_for_member( self, chat_id: UUID, user_id: UUID, after: Keyset | None, limit: int @@ -101,4 +138,8 @@ class MessageRepository: ) if after is not None: query = query.where(tuple_(MessageRow.created_at, MessageRow.id) > tuple_(after.created_at, after.id)) - return [self._to_domain(row) for row in (await session.scalars(query)).all()] + rows = list((await session.scalars(query)).all()) + return [ + self._to_domain(row, await self._attachment_ids(session, row.id)) + for row in rows + ] diff --git a/app/services/attachments.py b/app/services/attachments.py new file mode 100644 index 00000000..e7d5e253 --- /dev/null +++ b/app/services/attachments.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import asyncio +import hashlib +import logging +from datetime import datetime, timezone +from tempfile import SpooledTemporaryFile +from typing import AsyncIterator, BinaryIO +import unicodedata +from uuid import UUID, uuid4 + +from PIL import Image, UnidentifiedImageError + +from ..domain.errors import ( + Invalid, + NotFound, + PayloadTooLarge, + Unavailable, + UnsupportedMediaType, +) +from ..domain.models import Attachment, User +from ..repositories import AttachmentRepository +from ..storage import ObjectStream, S3Storage + +logger = logging.getLogger('app.attachments') + +MIME_BY_FORMAT = {'PNG': 'image/png', 'JPEG': 'image/jpeg'} + + +def _validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255: + raise Invalid('filename должен содержать от 1 до 255 символов') + if any( + character in {'/', '\\'} or unicodedata.category(character) == 'Cc' + for character in filename + ): + raise Invalid('filename содержит запрещённые символы') + + +def _inspect_image(source: BinaryIO, declared_type: str, max_pixels: int) -> tuple[str, int, int]: + try: + source.seek(0) + with Image.open(source) as image: + actual_type = MIME_BY_FORMAT.get(image.format or '') + if actual_type is None or actual_type != declared_type: + raise UnsupportedMediaType('MIME не соответствует содержимому PNG/JPEG') + width, height = image.size + if width <= 0 or height <= 0: + raise UnsupportedMediaType('изображение имеет неверные размеры') + if width * height > max_pixels: + raise PayloadTooLarge('изображение превышает лимит декодированных пикселей') + image.verify() + except Image.DecompressionBombError as error: + raise PayloadTooLarge('изображение превышает лимит декодированных пикселей') from error + except (UnidentifiedImageError, OSError, SyntaxError) as error: + raise UnsupportedMediaType('повреждённое или неподдерживаемое изображение') from error + finally: + source.seek(0) + return actual_type, width, height + + +class AttachmentService: + def __init__( + self, + repository: AttachmentRepository, + storage: S3Storage, + max_upload_bytes: int, + max_image_pixels: int, + ) -> None: + self._repository = repository + self._storage = storage + self._max_upload_bytes = max_upload_bytes + self._max_image_pixels = max_image_pixels + + async def upload( + self, + actor: User, + chat_id: UUID, + filename: str, + declared_type: str, + chunks: AsyncIterator[bytes], + ) -> Attachment: + _validate_filename(filename) + if declared_type not in MIME_BY_FORMAT.values(): + raise UnsupportedMediaType('поддерживаются только image/png и image/jpeg') + if not await self._repository.chat_exists_for_member(chat_id, actor.id): + raise NotFound('чат не найден') + + digest = hashlib.sha256() + size = 0 + with SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b') as source: + async for chunk in chunks: + size += len(chunk) + if size > self._max_upload_bytes: + raise PayloadTooLarge('файл превышает лимит 10 MiB') + digest.update(chunk) + source.write(chunk) + if size == 0: + raise UnsupportedMediaType('пустой файл не является изображением') + + actual_type, width, height = await asyncio.to_thread( + _inspect_image, + source, + declared_type, + self._max_image_pixels, + ) + attachment_id = uuid4() + object_key = f'originals/{attachment_id}' + await self._storage.put(object_key, source, size, actual_type) + now = datetime.now(timezone.utc) + try: + return await self._repository.create( + attachment_id=attachment_id, + chat_id=chat_id, + owner_id=actor.id, + filename=filename, + object_key=object_key, + content_type=actual_type, + size_bytes=size, + sha256=digest.hexdigest(), + width=width, + height=height, + now=now, + ) + except Exception: + try: + await self._storage.remove(object_key) + except Unavailable: + logger.warning('failed to compensate orphan object key=%s', object_key) + raise + + async def get(self, actor: User, attachment_id: UUID) -> Attachment: + attachment = await self._repository.get_for_member(attachment_id, actor.id) + if attachment is None: + raise NotFound('вложение не найдено') + return attachment + + async def content(self, actor: User, attachment_id: UUID) -> tuple[Attachment, ObjectStream]: + attachment = await self.get(actor, attachment_id) + try: + stream = await self._storage.get(attachment.object_key) + except FileNotFoundError as error: + raise Unavailable('оригинал вложения отсутствует в хранилище') from error + return attachment, stream diff --git a/app/services/messages.py b/app/services/messages.py index ec3de200..509c437f 100644 --- a/app/services/messages.py +++ b/app/services/messages.py @@ -1,7 +1,7 @@ from __future__ import annotations from datetime import datetime, timezone -from typing import Optional +from typing import Optional, Sequence from uuid import UUID, uuid4 from ..domain.errors import NotFound @@ -13,12 +13,20 @@ class MessageService: def __init__(self, repository: MessageRepository) -> None: self._repository = repository - async def send(self, actor: User, chat_id: UUID, text: str, idempotency_key: str) -> Message: + async def send( + self, + actor: User, + chat_id: UUID, + text: str, + attachment_ids: Sequence[UUID], + idempotency_key: str, + ) -> Message: message = await self._repository.create_for_member( message_id=uuid4(), chat_id=chat_id, sender_id=actor.id, text=text, + attachment_ids=attachment_ids, created_at=datetime.now(timezone.utc), idempotency_key=idempotency_key, ) diff --git a/app/storage/__init__.py b/app/storage/__init__.py new file mode 100644 index 00000000..17b31476 --- /dev/null +++ b/app/storage/__init__.py @@ -0,0 +1,3 @@ +from .s3 import ObjectStream, S3Storage, StoredObject + +__all__ = ['ObjectStream', 'S3Storage', 'StoredObject'] diff --git a/app/storage/s3.py b/app/storage/s3.py new file mode 100644 index 00000000..251951c8 --- /dev/null +++ b/app/storage/s3.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import asyncio +import certifi +from dataclasses import dataclass +from datetime import datetime +from typing import BinaryIO, Iterator +from urllib.parse import urlparse + +from minio import Minio +from minio.error import InvalidResponseError, S3Error +from urllib3 import PoolManager, Retry, Timeout +from urllib3.exceptions import HTTPError + +from ..domain.errors import Unavailable + + +def _unavailable(error: Exception) -> Unavailable: + return Unavailable('объектное хранилище недоступно') + + +@dataclass(slots=True) +class ObjectStream: + response: object + + def chunks(self, size: int = 64 * 1024) -> Iterator[bytes]: + try: + while chunk := self.response.read(size): + yield chunk + finally: + self.response.close() + self.response.release_conn() + + +@dataclass(frozen=True, slots=True) +class StoredObject: + key: str + last_modified: datetime + + +class S3Storage: + def __init__( + self, + endpoint: str, + access_key: str, + secret_key: str, + bucket: str, + region: str, + ) -> None: + parsed = urlparse(endpoint) + if parsed.scheme not in {'http', 'https'} or not parsed.netloc or parsed.path not in {'', '/'}: + raise RuntimeError('S3_ENDPOINT должен содержать только http(s)://host:port') + self._client = Minio( + parsed.netloc, + access_key=access_key, + secret_key=secret_key, + secure=parsed.scheme == 'https', + region=region, + http_client=PoolManager( + timeout=Timeout(connect=2, read=5), + retries=Retry(total=1, connect=1, read=1, backoff_factor=0.1), + cert_reqs='CERT_REQUIRED', + ca_certs=certifi.where(), + ), + ) + self.bucket = bucket + + @staticmethod + def _translate(error: Exception) -> None: + if isinstance(error, S3Error) and error.code in {'NoSuchKey', 'NoSuchObject'}: + raise FileNotFoundError from error + if isinstance(error, (S3Error, InvalidResponseError, HTTPError, OSError)): + raise _unavailable(error) from error + raise error + + async def ping(self) -> None: + try: + exists = await asyncio.to_thread(self._client.bucket_exists, self.bucket) + except Exception as error: + self._translate(error) + if not exists: + raise Unavailable('bucket объектного хранилища не создан') + + async def put(self, key: str, source: BinaryIO, length: int, content_type: str) -> None: + source.seek(0) + try: + await asyncio.to_thread( + self._client.put_object, + self.bucket, + key, + source, + length, + content_type, + ) + except Exception as error: + self._translate(error) + + async def get(self, key: str) -> ObjectStream: + try: + response = await asyncio.to_thread(self._client.get_object, self.bucket, key) + except Exception as error: + self._translate(error) + return ObjectStream(response) + + async def remove(self, key: str) -> None: + try: + await asyncio.to_thread(self._client.remove_object, self.bucket, key) + except Exception as error: + self._translate(error) + + async def list(self, prefix: str = '') -> tuple[StoredObject, ...]: + def collect() -> tuple[StoredObject, ...]: + return tuple( + StoredObject(item.object_name, item.last_modified) + for item in self._client.list_objects(self.bucket, prefix=prefix, recursive=True) + ) + + try: + return await asyncio.to_thread(collect) + except Exception as error: + self._translate(error) diff --git a/app/tools/__init__.py b/app/tools/__init__.py new file mode 100644 index 00000000..621302de --- /dev/null +++ b/app/tools/__init__.py @@ -0,0 +1 @@ +"""Операторские CLI-инструменты приложения.""" diff --git a/app/tools/cleanup_orphans.py b/app/tools/cleanup_orphans.py new file mode 100644 index 00000000..65740899 --- /dev/null +++ b/app/tools/cleanup_orphans.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import argparse +import asyncio +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select + +from ..config import load_settings +from ..db import create_engine, create_session_factory +from ..models.attachment import AttachmentRow +from ..storage import S3Storage + + +async def cleanup(apply: bool, grace_seconds: int) -> int: + settings = load_settings() + engine = create_engine(settings.database_url) + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + try: + sessions = create_session_factory(engine) + async with sessions() as session: + known = set(await session.scalars(select(AttachmentRow.object_key))) + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + removed = 0 + for item in await storage.list('originals/'): + if item.key in known or item.last_modified > cutoff: + continue + action = 'DELETE' if apply else 'WOULD_DELETE' + print(f'{action} key={item.key} modified={item.last_modified.isoformat()}') + if apply: + await storage.remove(item.key) + removed += 1 + print( + f'summary mode={"apply" if apply else "dry-run"} ' + f'known={len(known)} removed={removed} grace_seconds={grace_seconds}' + ) + return 0 + finally: + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description='Safely find old S3 objects without PostgreSQL metadata') + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument('--dry-run', action='store_true') + mode.add_argument('--apply', action='store_true') + parser.add_argument('--grace-seconds', type=int, default=3600) + args = parser.parse_args() + if args.grace_seconds < 60: + parser.error('--grace-seconds must be at least 60') + raise SystemExit(asyncio.run(cleanup(args.apply, args.grace_seconds))) + + +if __name__ == '__main__': + main() diff --git a/compose.tests.yaml b/compose.tests.yaml index 65e4f825..5845cfce 100644 --- a/compose.tests.yaml +++ b/compose.tests.yaml @@ -20,13 +20,55 @@ services: timeout: 2s retries: 30 + s3: + image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597 + user: "0" + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: test-minio-admin + MINIO_ROOT_PASSWORD: test-minio-root-password + S3_BUCKET: messenger-attachments + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 2s + timeout: 2s + retries: 30 + + s3-bootstrap: + image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805 + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" + mc mb --ignore-existing local/"$$S3_BUCKET" + mc anonymous set none local/"$$S3_BUCKET" + mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" + sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json + mc admin policy info local messenger-api >/dev/null 2>&1 || mc admin policy create local messenger-api /tmp/s3-api-policy.json + mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID" + environment: + MINIO_ROOT_USER: test-minio-admin + MINIO_ROOT_PASSWORD: test-minio-root-password + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: messenger-api + S3_SECRET_ACCESS_KEY: test-minio-api-password + volumes: + - ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro + depends_on: + s3: + condition: service_healthy + restart: "no" + migrate: build: . command: ["alembic", "upgrade", "head"] environment: DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test REDIS_URL: redis://redis:6379/0 - SESSION_TTL_SECONDS: 1 + SESSION_TTL_SECONDS: 3600 + S3_ACCESS_KEY_ID: messenger-api + S3_SECRET_ACCESS_KEY: test-minio-api-password depends_on: postgres: condition: service_healthy @@ -37,12 +79,21 @@ services: environment: DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test REDIS_URL: redis://redis:6379/0 - SESSION_TTL_SECONDS: 1 + SESSION_TTL_SECONDS: 3600 + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: messenger-api + S3_SECRET_ACCESS_KEY: test-minio-api-password + MAX_UPLOAD_BYTES: 10485760 + MAX_IMAGE_PIXELS: 40000000 depends_on: migrate: condition: service_completed_successfully redis: condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"] interval: 2s @@ -55,7 +106,7 @@ services: dockerfile: Dockerfile.test environment: BASE_URL: http://api:8080 - SESSION_TTL_SECONDS: 1 + SESSION_TTL_SECONDS: 3600 depends_on: api: condition: service_healthy diff --git a/compose.yaml b/compose.yaml index 0511f041..f3a74d12 100644 --- a/compose.yaml +++ b/compose.yaml @@ -28,6 +28,52 @@ services: timeout: 3s retries: 20 + s3: + image: cgr.dev/chainguard/minio@sha256:3c60a2a8c4d33409a9bc7a623a7fcbfb4605adc3d89422d2f53b3f673a6e5597 + user: "0" + cpus: "0.5" + mem_limit: 256m + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + volumes: + - s3_data:/data + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 3s + retries: 20 + + s3-bootstrap: + image: cgr.dev/chainguard/minio-client@sha256:39768db4c3fa811bc8f2e2a3e0add8ea391648b1c980de179985e1c353dc1805 + cpus: "0.25" + mem_limit: 128m + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + mc alias set local http://s3:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" + mc mb --ignore-existing local/"$$S3_BUCKET" + mc anonymous set none local/"$$S3_BUCKET" + mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" + sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json + mc admin policy info local messenger-api >/dev/null 2>&1 || mc admin policy create local messenger-api /tmp/s3-api-policy.json + mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-change-me-minio-root} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api} + volumes: + - ./infra/s3-api-policy.json:/policy/s3-api-policy.json:ro + depends_on: + s3: + condition: service_healthy + restart: "no" + migrate: build: . cpus: "0.5" @@ -37,6 +83,8 @@ services: DATABASE_URL: ${DATABASE_URL} REDIS_URL: ${REDIS_URL} SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api} depends_on: postgres: condition: service_healthy @@ -51,6 +99,13 @@ services: REDIS_URL: ${REDIS_URL} SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} LOG_LEVEL: ${LOG_LEVEL:-info} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-messenger-api} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-change-me-minio-api} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} expose: - "8080" deploy: @@ -60,6 +115,8 @@ services: condition: service_completed_successfully redis: condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3).read()"] interval: 2s @@ -88,3 +145,4 @@ services: volumes: postgres_data: redis_data: + s3_data: diff --git a/evidence/results.md b/evidence/results.md new file mode 100644 index 00000000..690f444f --- /dev/null +++ b/evidence/results.md @@ -0,0 +1,103 @@ +# Результаты проверки lab4 + +Дата последней перепроверки: 2026-11-04. Стенд: Apple M4 Pro, 12 CPU cores, 24 GB RAM, Docker Desktop. + +## Версии runtime + +```text +Python 3.13.15 +PostgreSQL 17.4 +Redis 7.4.2 +HAProxy 3.2.26 +MinIO RELEASE.2026-10-23T20-06-51Z (commit 76c06ff1d2cc844050fdeb1d54bf8fa70db72278) +MinIO Client DEVELOPMENT.GOGET +``` + +MinIO и MinIO Client взяты из Chainguard Images и закреплены в Compose по multi-arch OCI index digest. + +## Контракт и smoke + +```text +$ make check +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +$ make validate PYTHON=.venv/bin/python +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. + +$ make test +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +``` + +## Негативные сценарии вложений + +```text +$ make test-attachments PYTHON=.venv/bin/python +............ [100%] +12 passed in 1.86s +``` + +Проверены: превышение 10 MiB, повреждённое изображение, подмена MIME, превышение 40 млн пикселей до декодирования, опасные filename, доступ постороннего, attachment другого чата и семантика идемпотентности `attachment_ids`. + +```text +$ docker compose -f compose.tests.yaml run --rm --no-deps tests python -m pytest -q tests/test_attachment_compensation.py +. [100%] +1 passed +``` + +Unit-тест внедряет ошибку сохранения метаданных после успешного S3 PUT и проверяет компенсационный DELETE того же object key. + +Полный запуск в чистом изолированном Compose-проекте, включая миграции и bootstrap MinIO: + +```text +$ docker compose -p lab4-chainguard-tests -f compose.tests.yaml run --rm tests python -m pytest -q +............. [100%] +13 passed in 0.41s +``` + +## Приватность и bootstrap + +```text +$ scripts/check-private-bucket.sh +PASS: private bucket rejected anonymous request with HTTP 403 +``` + +MinIO root credentials получает только server/bootstrap. API использует отдельного пользователя с bucket-scoped policy. Host-порты S3 API и консоли не опубликованы. Bootstrap повторно завершался с code 0 при восстановлении API после failover и при restart стенда. + +## Persistence и отказы + +```text +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +``` + +Hook перезапустил API, PostgreSQL, Redis и MinIO с сохранением named volumes. + +```text +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +{"observed_instances_before": 2, "successful_writes": 111, "failed_attempts": 1, "first_recovery_seconds": 2.333} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +``` + +```text +$ make test-attachment-failover ACTION_SCRIPT=scripts/stop-one.sh +PASS: upload_instance=c18e79906d77 download_instance=662967386db9 recovery_seconds=3.912 failures=1 sha256=99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 +``` + +В последнем сценарии остановлена именно реплика из `X-Instance-Id` ответа upload. Download выполнен через другую реплику и совпал побайтно. + +```text +$ scripts/check-s3-outage.sh +PASS: API readiness returned structured 503 while S3 was unavailable +``` + +Скрипт автоматически восстановил MinIO и дождался готовности стенда. + +## Cleanup + +```text +$ python -m app.tools.cleanup_orphans --dry-run --grace-seconds 60 +summary mode=dry-run known=164 removed=0 grace_seconds=60 +``` + +Dry-run не менял данные. Применение требует явного `--apply`; значения grace period меньше 60 секунд запрещены. diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 00000000..bcf2d243 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,20 @@ +{ + "date": "2026-11-04", + "profile": { + "concurrency": 10, + "uploads": 30, + "file_size_bytes": 111817, + "api_memory_limit_mib_each": 256 + }, + "result": { + "successful_uploads": 30, + "observed_api_instances": 2, + "duration_seconds": 0.382, + "max_latency_seconds": 0.3, + "peak_memory_mib": { + "api_replica_1": 68.5, + "api_replica_2": 68.33 + } + }, + "limitations": "Closed-loop profile with a small PNG; docker stats is sampled and may miss a shorter transient peak." +} diff --git a/infra/s3-api-policy.json b/infra/s3-api-policy.json new file mode 100644 index 00000000..a2fccff0 --- /dev/null +++ b/infra/s3-api-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::BUCKET_NAME"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::BUCKET_NAME/*"] + } + ] +} diff --git a/requirements.txt b/requirements.txt index 31148b51..20bb81f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ asyncpg==0.30.0 alembic==1.15.2 redis==5.2.1 argon2-cffi==25.1.0 +minio==7.2.20 +Pillow==13.0.0 diff --git a/scripts/check-private-bucket.sh b/scripts/check-private-bucket.sh new file mode 100755 index 00000000..8865974d --- /dev/null +++ b/scripts/check-private-bucket.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +project_dir=$(dirname -- "$script_dir") +cd "$project_dir" + +status=$(docker compose --env-file .env exec -T api python -c ' +import os +import urllib.error +import urllib.request + +url = "http://s3:9000/" + os.environ["S3_BUCKET"] + "/" +try: + response = urllib.request.urlopen(url, timeout=5) +except urllib.error.HTTPError as error: + print(error.code) +else: + print(response.status) +') + +case "$status" in + 401|403) printf '%s\n' "PASS: private bucket rejected anonymous request with HTTP $status" ;; + *) printf '%s\n' "FAIL: anonymous bucket request returned HTTP $status" >&2; exit 1 ;; +esac diff --git a/scripts/check-s3-outage.sh b/scripts/check-s3-outage.sh new file mode 100755 index 00000000..ff8313fc --- /dev/null +++ b/scripts/check-s3-outage.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +project_dir=$(dirname -- "$script_dir") +cd "$project_dir" + +restore() { + docker compose --env-file .env up -d --wait --wait-timeout 60 s3 api lb >/dev/null +} +trap restore EXIT HUP INT TERM + +docker compose --env-file .env stop s3 >/dev/null +docker compose --env-file .env exec -T api python -c ' +import json +import urllib.error +import urllib.request + +try: + urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=25) +except urllib.error.HTTPError as error: + body = json.loads(error.read()) + assert error.code == 503, error.code + assert body["error"]["code"] == "unavailable", body +else: + raise SystemExit("readiness unexpectedly succeeded while S3 was stopped") +' + +printf '%s\n' 'PASS: API readiness returned structured 503 while S3 was unavailable' diff --git a/scripts/measure-upload-memory.py b/scripts/measure-upload-memory.py new file mode 100755 index 00000000..250d05a7 --- /dev/null +++ b/scripts/measure-upload-memory.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Run concurrent uploads while sampling API container memory with docker stats.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re +import subprocess +import sys +import threading + + +ROOT = Path(__file__).resolve().parents[1] +UNITS = { + 'B': 1, + 'kB': 1000, + 'KB': 1000, + 'KiB': 1024, + 'MB': 1000**2, + 'MiB': 1024**2, + 'GB': 1000**3, + 'GiB': 1024**3, +} + + +def bytes_value(value: str) -> int: + match = re.fullmatch(r'([0-9.]+)([A-Za-z]+)', value.strip()) + if match is None or match.group(2) not in UNITS: + raise ValueError(f'unsupported docker memory value: {value}') + return round(float(match.group(1)) * UNITS[match.group(2)]) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--concurrency', type=int, default=10) + parser.add_argument('--uploads', type=int, default=30) + args = parser.parse_args() + + container_ids = subprocess.check_output( + ['docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api'], + cwd=ROOT, + text=True, + ).split() + if len(container_ids) < 2: + raise SystemExit('expected at least two API containers') + + peaks: dict[str, int] = {} + stop = threading.Event() + ready = threading.Event() + + def collect() -> None: + while not stop.is_set(): + output = subprocess.check_output( + ['docker', 'stats', '--no-stream', '--format', '{{json .}}', *container_ids], + cwd=ROOT, + text=True, + ) + for line in output.splitlines(): + sample = json.loads(line) + name, usage = sample['Name'], sample['MemUsage'] + current = bytes_value(usage.split('/', 1)[0]) + peaks[name] = max(peaks.get(name, 0), current) + ready.set() + + reader = threading.Thread(target=collect, daemon=True) + reader.start() + if not ready.wait(timeout=15): + raise SystemExit('docker stats did not produce an initial sample') + try: + load = subprocess.run( + [ + sys.executable, + 'tests/upload_load.py', + '--base-url', + args.base_url, + '--concurrency', + str(args.concurrency), + '--uploads', + str(args.uploads), + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + finally: + stop.set() + reader.join(timeout=15) + + if not peaks: + raise SystemExit('docker stats produced no samples; increase --uploads') + print(load.stdout.strip()) + print( + json.dumps( + { + 'peak_memory_bytes': peaks, + 'peak_memory_mib': { + name: round(value / 1024 / 1024, 2) for name, value in peaks.items() + }, + }, + ensure_ascii=False, + ) + ) + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh index 691598a3..39e05f7a 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -7,6 +7,6 @@ cd "$project_dir" # Останавливаем публичный маршрут до перезапуска общих хранилищ. docker compose --env-file .env stop lb api >/dev/null -docker compose --env-file .env restart postgres redis >/dev/null -docker compose --env-file .env up -d --wait --wait-timeout 60 postgres redis >/dev/null +docker compose --env-file .env restart postgres redis s3 >/dev/null +docker compose --env-file .env up -d --wait --wait-timeout 60 postgres redis s3 >/dev/null docker compose --env-file .env up -d --wait --wait-timeout 60 api lb >/dev/null diff --git a/tests/attachment_failover.py b/tests/attachment_failover.py new file mode 100644 index 00000000..10e376d4 --- /dev/null +++ b/tests/attachment_failover.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Stop the API replica that accepted upload and download through another replica.""" +import hashlib +import os +from pathlib import Path +import subprocess +import time + +from client import Client, ROOT, arguments, check, run + + +def main(): + def configure(parser): + parser.add_argument('--action-script', required=True) + + args = arguments('Attachment API-replica failover', configure) + script = Path(args.action_script).expanduser().resolve() + check(script.is_file() and os.access(script, os.X_OK), 'action script must be executable') + client = Client(args.base_url, args.ca_file, timeout=3) + alice, bob, _eve, chat = client.bootstrap() + content = (ROOT / 'tests/fixtures/sample.png').read_bytes() + uploaded = client.call( + 'POST', + f'/api/v1/chats/{chat["id"]}/attachments?filename=failover.png', + 201, + actor=alice, + raw=content, + headers={'Content-Type': 'image/png'}, + schema='Attachment', + ) + target = uploaded['headers']['X-Instance-Id'] + env = dict(os.environ) + env['TARGET_INSTANCE_ID'] = target + started = time.monotonic() + subprocess.run([str(script)], env=env, check=True) + + deadline = time.monotonic() + 15 + failures = 0 + while True: + try: + downloaded = client.call( + 'GET', + f'/api/v1/attachments/{uploaded["json"]["id"]}/content', + actor=bob, + ) + if downloaded['headers']['X-Instance-Id'] != target: + break + except Exception: + failures += 1 + check(time.monotonic() < deadline, 'download did not recover through another replica') + time.sleep(0.2) + + check(downloaded['bytes'] == content, 'downloaded file differs after replica failure') + print( + 'PASS: upload_instance=%s download_instance=%s recovery_seconds=%.3f failures=%d sha256=%s' + % ( + target, + downloaded['headers']['X-Instance-Id'], + time.monotonic() - started, + failures, + hashlib.sha256(content).hexdigest(), + ) + ) + + +if __name__ == '__main__': + run(main) diff --git a/tests/test_attachment_compensation.py b/tests/test_attachment_compensation.py new file mode 100644 index 00000000..b7ef1e22 --- /dev/null +++ b/tests/test_attachment_compensation.py @@ -0,0 +1,54 @@ +"""Проверка компенсации объекта при ошибке commit метаданных.""" +import asyncio +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +import pytest + +from app.domain.models import User +from app.services.attachments import AttachmentService + + +class FailingRepository: + async def chat_exists_for_member(self, chat_id, user_id): + return True + + async def create(self, **kwargs): + raise RuntimeError('injected database commit failure') + + +class RecordingStorage: + def __init__(self): + self.put_key = None + self.removed_key = None + + async def put(self, key, source, length, content_type): + self.put_key = key + + async def remove(self, key): + self.removed_key = key + + +def test_uploaded_object_is_removed_when_metadata_commit_fails(): + storage = RecordingStorage() + service = AttachmentService(FailingRepository(), storage, 10 * 1024 * 1024, 40_000_000) + actor = User(uuid4(), 'tester', 'Tester', datetime.now(timezone.utc)) + content = (Path(__file__).parent / 'fixtures/sample.png').read_bytes() + + async def chunks(): + yield content + + with pytest.raises(RuntimeError, match='injected database commit failure'): + asyncio.run( + service.upload( + actor, + uuid4(), + 'sample.png', + 'image/png', + chunks(), + ) + ) + + assert storage.put_key is not None + assert storage.removed_key == storage.put_key diff --git a/tests/test_attachments.py b/tests/test_attachments.py new file mode 100644 index 00000000..4f3b14a5 --- /dev/null +++ b/tests/test_attachments.py @@ -0,0 +1,144 @@ +"""Негативные интеграционные проверки вложений для уровня 4.""" +import struct +from urllib.parse import quote +import uuid +import zlib + +import pytest + +from tests.client import Client, ROOT + + +@pytest.fixture(scope='module') +def scenario(): + client = Client() + alice, bob, eve, chat = client.bootstrap() + return client, alice, bob, eve, chat + + +def upload(client, actor, chat, content, *, filename='sample.png', content_type='image/png', expected=201): + return client.call( + 'POST', + f'/api/v1/chats/{chat["id"]}/attachments?filename={quote(filename, safe="")}', + expected, + actor=actor, + raw=content, + headers={'Content-Type': content_type}, + schema='Attachment' if expected == 201 else None, + ) + + +def oversized_dimensions_png(content: bytes) -> bytes: + data = bytearray(content) + data[16:20] = struct.pack('>I', 10_000) + data[20:24] = struct.pack('>I', 5_000) + data[29:33] = struct.pack('>I', zlib.crc32(data[12:29]) & 0xFFFFFFFF) + return bytes(data) + + +def test_file_larger_than_10_mib_is_rejected(scenario): + client, alice, _bob, _eve, chat = scenario + upload(client, alice, chat, b'x' * (10 * 1024 * 1024 + 1), expected=413) + + +def test_corrupted_image_is_rejected(scenario): + client, alice, _bob, _eve, chat = scenario + upload(client, alice, chat, b'not-a-png', expected=415) + + +def test_mime_mismatch_is_rejected(scenario): + client, alice, _bob, _eve, chat = scenario + content = (ROOT / 'tests/fixtures/sample.png').read_bytes() + upload(client, alice, chat, content, content_type='image/jpeg', expected=415) + + +def test_pixel_limit_is_checked_before_decode(scenario): + client, alice, _bob, _eve, chat = scenario + content = oversized_dimensions_png((ROOT / 'tests/fixtures/sample.png').read_bytes()) + upload(client, alice, chat, content, expected=413) + + +@pytest.mark.parametrize( + 'filename', + ['../escape.png', 'folder\\escape.png', 'bad\x00name.png', 'bad\x85name.png'], +) +def test_dangerous_filename_is_rejected(scenario, filename): + client, alice, _bob, _eve, chat = scenario + content = (ROOT / 'tests/fixtures/sample.png').read_bytes() + upload(client, alice, chat, content, filename=filename, expected=400) + + +def test_outsider_cannot_read_attachment(scenario): + client, alice, _bob, eve, chat = scenario + content = (ROOT / 'tests/fixtures/sample.png').read_bytes() + attachment = upload(client, alice, chat, content)['json'] + client.call('GET', f'/api/v1/attachments/{attachment["id"]}', 404, actor=eve) + client.call('GET', f'/api/v1/attachments/{attachment["id"]}/content', 404, actor=eve) + + +def test_attachment_from_another_chat_cannot_be_linked(scenario): + client, alice, bob, _eve, chat = scenario + other_chat = client.call( + 'POST', + '/api/v1/chats', + 201, + actor=alice, + body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]}, + schema='Chat', + )['json'] + content = (ROOT / 'tests/fixtures/sample.png').read_bytes() + attachment = upload(client, alice, other_chat, content)['json'] + client.call( + 'POST', + f'/api/v1/chats/{chat["id"]}/messages', + 404, + actor=alice, + body={'text': 'чужое вложение', 'attachment_ids': [attachment['id']]}, + headers={'Idempotency-Key': uuid.uuid4().hex}, + ) + + +def test_idempotency_distinguishes_attachment_list(scenario): + client, alice, _bob, _eve, chat = scenario + content = (ROOT / 'tests/fixtures/sample.png').read_bytes() + attachment = upload(client, alice, chat, content)['json'] + key = uuid.uuid4().hex + first = client.message( + alice, + chat, + 'идемпотентное вложение', + key=key, + attachments=[attachment['id']], + )['json'] + replay = client.message( + alice, + chat, + 'идемпотентное вложение', + key=key, + attachments=[attachment['id']], + )['json'] + assert replay == first + client.call( + 'POST', + f'/api/v1/chats/{chat["id"]}/messages', + 409, + actor=alice, + body={'text': 'идемпотентное вложение', 'attachment_ids': []}, + headers={'Idempotency-Key': key}, + ) + + +def test_missing_attachment_ids_equals_empty_list_for_idempotency(scenario): + client, alice, _bob, _eve, chat = scenario + key = uuid.uuid4().hex + first = client.message(alice, chat, 'обычный текст', key=key)['json'] + replay = client.call( + 'POST', + f'/api/v1/chats/{chat["id"]}/messages', + 201, + actor=alice, + body={'text': 'обычный текст', 'attachment_ids': []}, + headers={'Idempotency-Key': key}, + schema='Message', + )['json'] + assert replay == first diff --git a/tests/upload_load.py b/tests/upload_load.py new file mode 100644 index 00000000..b4689667 --- /dev/null +++ b/tests/upload_load.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Простой профиль параллельных загрузок для измерения RAM контейнеров API.""" +from concurrent.futures import ThreadPoolExecutor +import json +import time +import uuid + +from client import Client, ROOT, arguments, run + + +def main(): + def configure(parser): + parser.add_argument('--concurrency', type=int, default=10) + parser.add_argument('--uploads', type=int, default=30) + + args = arguments( + 'Parallel attachment uploads', + configure, + ) + if args.concurrency < 10: + raise ValueError('concurrency must be at least 10 for lab4 evidence') + if args.uploads < args.concurrency: + raise ValueError('uploads must be greater than or equal to concurrency') + client = Client(args.base_url, args.ca_file, timeout=30) + alice, _bob, _eve, chat = client.bootstrap() + content = (ROOT / 'tests/fixtures/sample.png').read_bytes() + + def upload(index): + local = Client(args.base_url, args.ca_file, timeout=30) + started = time.monotonic() + response = local.call( + 'POST', + f'/api/v1/chats/{chat["id"]}/attachments?filename=load-{index}-{uuid.uuid4().hex}.png', + 201, + actor=alice, + raw=content, + headers={'Content-Type': 'image/png'}, + schema='Attachment', + ) + return response['headers']['X-Instance-Id'], time.monotonic() - started + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + results = list(pool.map(upload, range(args.uploads))) + duration = time.monotonic() - started + latencies = sorted(item[1] for item in results) + print( + json.dumps( + { + 'concurrency': args.concurrency, + 'file_size_bytes': len(content), + 'successful_uploads': len(results), + 'instances': len({item[0] for item in results}), + 'duration_seconds': round(duration, 3), + 'max_latency_seconds': round(latencies[-1], 3), + }, + ensure_ascii=False, + ) + ) + + +if __name__ == '__main__': + run(main) -- 2.54.0 From a3bbdeef42c4c5e58b6f80d7677e522379d1b758 Mon Sep 17 00:00:00 2001 From: thefankor Date: Thu, 5 Nov 2026 21:21:43 +0300 Subject: [PATCH 3/5] =?UTF-8?q?feat:=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20reconciler=20PostgreSQL=20=D0=B8=20S3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 11 +- REPORT.md | 69 ++++++++- app/repositories/messages.py | 22 +-- app/services/attachments.py | 3 + app/tools/reconcile_attachments.py | 205 +++++++++++++++++++++++++ app/tools/seed_reconciler_benchmark.py | 136 ++++++++++++++++ compose.tests.yaml | 7 + evidence/results.md | 44 +++++- scripts/benchmark-reconciler.sh | 9 ++ tests/test_attachment_compensation.py | 42 +++++ tests/test_reconciler.py | 50 ++++++ tests/test_reconciler_integration.py | 168 ++++++++++++++++++++ 12 files changed, 751 insertions(+), 15 deletions(-) create mode 100644 app/tools/reconcile_attachments.py create mode 100644 app/tools/seed_reconciler_benchmark.py create mode 100755 scripts/benchmark-reconciler.sh create mode 100644 tests/test_reconciler.py create mode 100644 tests/test_reconciler_integration.py diff --git a/Makefile b/Makefile index 2b90fec8..96f252a8 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test test-attachments load load-attachments test-persistence test-failover test-attachment-failover +.PHONY: help check validate test test-attachments load load-attachments test-persistence test-failover test-attachment-failover reconcile-dry reconcile-apply benchmark-reconciler help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -38,3 +38,12 @@ test-failover: test-attachment-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/attachment_failover.py --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +reconcile-dry: + docker compose exec -T api python -m app.tools.reconcile_attachments --dry-run + +reconcile-apply: + docker compose exec -T api python -m app.tools.reconcile_attachments --apply + +benchmark-reconciler: + ./scripts/benchmark-reconciler.sh diff --git a/REPORT.md b/REPORT.md index 212f1568..a4f4ecdc 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 4 - Автор, группа: Сергей Возжин, БПИ2301. -- Целевая оценка: 4; трек 5 не выбран. +- Целевая оценка: 5; выбран трек 2 — согласованность PostgreSQL и S3. - Commit/PR: TODO после публикации. - Использованные источники: контракт репозитория, документация Docker Compose, Chainguard Images, MinIO SDK и Pillow. - Библиотеки: FastAPI, Uvicorn, SQLAlchemy, asyncpg, Alembic, redis-py, MinIO Python SDK, Pillow, Argon2, pytest. @@ -57,8 +57,15 @@ docker compose --env-file .env up -d --wait --wait-timeout 60 api lb scripts/check-private-bucket.sh scripts/check-s3-outage.sh .venv/bin/python scripts/measure-upload-memory.py --concurrency 10 --uploads 30 +make reconcile-dry +make benchmark-reconciler ``` +Исправления следует сначала просмотреть через `make reconcile-dry`. Изменяющий данные +`make reconcile-apply` запускается оператором только после проверки dry-run. +`make benchmark-reconciler` сам создаёт 1000 изолированных пар metadata/object, +выполняет dry-run и удаляет тестовые данные; объём меняется через `SIZE=...`. + Каждый failover-тест намеренно оставляет выбранную API-реплику остановленной, поэтому между сценариями и после последнего сценария она явно восстанавливается указанной командой `docker compose up`. Безопасная остановка с сохранением данных: @@ -95,7 +102,8 @@ HAProxy :8080 - синхронный binary upload PNG/JPEG; - авторизованные metadata/download endpoints; - `attachment_ids` в сообщениях и в области сравнения идемпотентного body; -- компенсационное удаление объекта и безопасный cleanup старых orphan-объектов. +- компенсационное удаление объекта и безопасный cleanup старых orphan-объектов; +- повторяемый reconciler метаданных PostgreSQL и оригиналов S3. PostgreSQL является источником истины для метаданных. В БД не хранятся bytes изображения. MinIO хранит оригинал по серверному ключу `originals/`. Пользовательский `filename` хранится только как метаданные и не используется в object key или пути файловой системы. @@ -130,6 +138,42 @@ docker compose --env-file .env run --rm --no-deps api \ По умолчанию используется grace period один час; CLI запрещает значения меньше 60 секунд. Объекты с существующими метаданными и свежие orphan-объекты не удаляются. Сначала используется `--dry-run`, затем осознанный `--apply`. +### Reconciler PostgreSQL/S3 — трек 2 + +Расширенный reconciler запускается внутри API-контейнера, где уже есть те же настройки и +ограниченные S3 credentials, что у приложения: + +```sh +docker compose exec -T api \ + python -m app.tools.reconcile_attachments --dry-run --grace-seconds 3600 +docker compose exec -T api \ + python -m app.tools.reconcile_attachments --apply --grace-seconds 3600 +``` + +Он читает снимок всех `attachments` и список `originals/`, после чего различает: + +- `orphan`: объект S3 есть, но ни одной строки с таким `object_key` нет; +- `missing`: строка в состоянии `ready` есть, а оригинала S3 нет; +- корректную пару: присутствуют обе стороны; +- свежее расхождение: возраст меньше grace period, изменять его нельзя. + +В `--dry-run` выводятся `WOULD_DELETE_ORPHAN` и `WOULD_MARK_MISSING`, но записи и объекты +не меняются. В `--apply` старый orphan удаляется, а missing-метаданные сохраняются и +переходят в `status=failed`, `error_code=original_missing`. Сообщение и связь с вложением +не удаляются. Повторный `--apply` ничего не делает: orphan уже отсутствует, а `failed` +больше не считается ожидающим оригинал состоянием. При этом строка `failed` по-прежнему +защищает внезапно появившийся объект с тем же ключом от удаления как orphan. + +Grace period защищает upload, который ещё выполняется, и запрещён ниже 60 секунд. +Используемые сообщением вложения не имеют особого исключения: любой известный PostgreSQL +ключ защищён одинаково, поэтому корректный оригинал связанного сообщения не удаляется. +Content для `failed` возвращает 409, а привязать такое вложение к новому сообщению нельзя. + +Операции между двумя хранилищами всё ещё не образуют общей транзакции. Если apply +прервётся, уже выполненное безопасное удаление orphan не требует отката, условный SQL +UPDATE меняет только всё ещё `ready` строки, а повторный запуск завершает оставшуюся +работу. + ### Доступ и идемпотентность Метаданные и content доступны любому участнику чата. Неизвестный объект и запрос постороннего одинаково возвращают 404. Download идёт через API без redirect. Публичный ответ не раскрывает object key, endpoint или credentials. @@ -152,6 +196,9 @@ docker compose --env-file .env run --rm --no-deps api \ | Concurrency/RAM | `scripts/measure-upload-memory.py --concurrency 10 --uploads 30` | 30/30 upload, обе реплики, пики 68,50/68,33 MiB | `evidence/upload-memory.json` | | Идемпотентный bootstrap | повторный запуск `docker compose up` после restart/failover | bootstrap повторно завершился с code 0 | `evidence/results.md` | | Cleanup dry-run | `cleanup_orphans --dry-run --grace-seconds 60` | 164 известных объекта, удалено 0 | `evidence/results.md` | +| Reconciler fault injection | реальный PostgreSQL + MinIO: orphan, missing и используемый control | dry-run без изменений; apply исправил 2 расхождения; повторный apply — 0 действий | `tests/test_reconciler_integration.py`, `evidence/results.md` | +| Полный тестовый стенд | изолированный Compose, все pytest | `17 passed` | `evidence/results.md` | +| Стоимость обхода | `SIZE=1000 make benchmark-reconciler` | 1000 строк и 1000 объектов: 229,657 мс, пик 4 202 814 bytes | `evidence/results.md` | ## Эксперименты и отказы @@ -164,6 +211,20 @@ docker compose --env-file .env run --rm --no-deps api \ Профиль upload: 30 запросов, concurrency 10, fixture 111 817 bytes, closed-loop, без retry. Получено 30 успешных upload за 0,382 с; максимальная latency 0,300 с; задействованы обе API-реплики. Пиковая наблюдаемая RAM: 68,50 MiB и 68,33 MiB при лимите 256 MiB на реплику. +Fault injection reconciler выполняется на настоящих PostgreSQL и MinIO. Тест создаёт +старую `ready` строку без объекта, объект без строки и корректное вложение, уже связанное +с сообщением. Управляемые часы имитируют истечение 60-секундного grace period без +ожидания в тесте. Dry-run сохраняет обе ошибки, apply удаляет только orphan и маркирует +missing, второй apply сообщает нулевое число действий. Корректный объект и связь сообщения +после обоих запусков остаются на месте. + +Воспроизводимый benchmark сначала удаляет прежнюю фикстуру, затем создаёт 1000 строк +`attachments` и 1000 объектов MinIO, выполняет полный dry-run и удаляет фикстуру. +Обход занял 229,657 мс, пик Python allocations составил 4 202 814 bytes. Измерение +включает запрос PostgreSQL, listing MinIO и сравнение снимков; время подготовки и очистки +в `elapsed_ms` reconciler не входит. Контрольный dry-run после cleanup показал 0 строк и +0 объектов benchmark-набора. + | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | | Restart API + PostgreSQL + Redis + MinIO | token, metadata и bytes сохраняются | persistence PASS | потерь нет | @@ -175,8 +236,8 @@ docker compose --env-file .env run --rm --no-deps api \ Стенд выдерживает остановку одной API-реплики и штатный restart контейнеров с сохранёнными volumes. Он не выдерживает потерю Docker-host, единственного HAProxy, PostgreSQL, Redis или MinIO volume. Несколько API-процессов не делают единственные хранилища высокодоступными. -Upload синхронный, поэтому клиент ждёт S3 и PostgreSQL. `SpooledTemporaryFile` ограничивает RAM одной загрузки, но временный disk API должен иметь достаточный объём и отдельную квоту в production. Cleanup делает полный список объектов и строк БД; для большого bucket нужны пагинация, метрики и плановый запуск. +Upload синхронный, поэтому клиент ждёт S3 и PostgreSQL. `SpooledTemporaryFile` ограничивает RAM одной загрузки, но временный disk API должен иметь достаточный объём и отдельную квоту в production. Reconciler загружает полный список объектов и строк БД в память, поэтому его стоимость O(N), а для большого bucket нужны страничный/потоковый обход, метрики, checkpoint и плановый запуск. Chainguard-образы зафиксированы по digest, поэтому обновляются только явно. Стенд изолирован учебной сетью. Перед production-развёртыванием потребовались бы непривилегированный UID с корректными правами на volume, TLS, внешний secret manager, HA хранилищ, аудит и резервное копирование. -Presigned upload, worker, thumbnail, варианты изображений и исследовательский reconciler для оценки 5 намеренно не реализованы. +Presigned upload, worker, thumbnail и варианты изображений намеренно не реализованы: для выбранного трека 2 они не требуются. diff --git a/app/repositories/messages.py b/app/repositories/messages.py index 12a6459d..93736b8e 100644 --- a/app/repositories/messages.py +++ b/app/repositories/messages.py @@ -64,17 +64,21 @@ class MessageRepository: return None if attachment_ids: - found = set( - await session.scalars( - select(AttachmentRow.id).where( - AttachmentRow.id.in_(attachment_ids), - AttachmentRow.chat_id == accessible_chat_id, - AttachmentRow.status == 'ready', + found = { + attachment_id: status + for attachment_id, status in ( + await session.execute( + select(AttachmentRow.id, AttachmentRow.status).where( + AttachmentRow.id.in_(attachment_ids), + AttachmentRow.chat_id == accessible_chat_id, + ) ) - ) - ) - if found != set(attachment_ids): + ).all() + } + if set(found) != set(attachment_ids): raise NotFound('вложение не найдено') + if any(status != 'ready' for status in found.values()): + raise Conflict('оригинал вложения недоступен') statement = ( insert(MessageRow) diff --git a/app/services/attachments.py b/app/services/attachments.py index e7d5e253..105c6bf6 100644 --- a/app/services/attachments.py +++ b/app/services/attachments.py @@ -12,6 +12,7 @@ from uuid import UUID, uuid4 from PIL import Image, UnidentifiedImageError from ..domain.errors import ( + Conflict, Invalid, NotFound, PayloadTooLarge, @@ -137,6 +138,8 @@ class AttachmentService: async def content(self, actor: User, attachment_id: UUID) -> tuple[Attachment, ObjectStream]: attachment = await self.get(actor, attachment_id) + if attachment.status != 'ready': + raise Conflict('оригинал вложения недоступен') try: stream = await self._storage.get(attachment.object_key) except FileNotFoundError as error: diff --git a/app/tools/reconcile_attachments.py b/app/tools/reconcile_attachments.py new file mode 100644 index 00000000..d4f885f6 --- /dev/null +++ b/app/tools/reconcile_attachments.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import time +import tracemalloc +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Callable, Sequence +from uuid import UUID + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from ..config import load_settings +from ..db import create_engine, create_session_factory +from ..models.attachment import AttachmentRow +from ..storage import S3Storage, StoredObject + +MISSING_ERROR_CODE = 'original_missing' + + +@dataclass(frozen=True, slots=True) +class AttachmentSnapshot: + id: UUID + object_key: str + status: str + updated_at: datetime + + +@dataclass(frozen=True, slots=True) +class ReconcilePlan: + orphan: tuple[StoredObject, ...] + fresh_orphan: tuple[StoredObject, ...] + missing: tuple[AttachmentSnapshot, ...] + fresh_missing: tuple[AttachmentSnapshot, ...] + correct: int + + +def build_plan( + attachments: Sequence[AttachmentSnapshot], + objects: Sequence[StoredObject], + cutoff: datetime, +) -> ReconcilePlan: + """Compare immutable PostgreSQL and S3 snapshots without changing either side.""" + known_keys = {attachment.object_key for attachment in attachments} + object_keys = {item.key for item in objects} + + unknown_objects = [item for item in objects if item.key not in known_keys] + orphan = tuple(item for item in unknown_objects if item.last_modified <= cutoff) + fresh_orphan = tuple(item for item in unknown_objects if item.last_modified > cutoff) + + ready_without_object = [ + attachment + for attachment in attachments + if attachment.status == 'ready' and attachment.object_key not in object_keys + ] + missing = tuple( + attachment for attachment in ready_without_object if attachment.updated_at <= cutoff + ) + fresh_missing = tuple( + attachment for attachment in ready_without_object if attachment.updated_at > cutoff + ) + correct = sum(attachment.object_key in object_keys for attachment in attachments) + return ReconcilePlan(orphan, fresh_orphan, missing, fresh_missing, correct) + + +async def _load_attachments( + sessions: async_sessionmaker[AsyncSession], +) -> tuple[AttachmentSnapshot, ...]: + async with sessions() as session: + rows = ( + await session.execute( + select( + AttachmentRow.id, + AttachmentRow.object_key, + AttachmentRow.status, + AttachmentRow.updated_at, + ) + ) + ).all() + return tuple(AttachmentSnapshot(*row) for row in rows) + + +async def reconcile( + sessions: async_sessionmaker[AsyncSession], + storage: S3Storage, + *, + apply: bool, + grace_seconds: int, + now: datetime | None = None, + output: Callable[[str], None] = print, +) -> dict[str, int | float | str]: + started = time.perf_counter() + tracemalloc.start() + current_time = now or datetime.now(timezone.utc) + cutoff = current_time - timedelta(seconds=grace_seconds) + + attachments = await _load_attachments(sessions) + objects = await storage.list('originals/') + plan = build_plan(attachments, objects, cutoff) + + for item in plan.fresh_orphan: + output( + f'SKIP_FRESH_ORPHAN key={item.key} modified={item.last_modified.isoformat()}' + ) + for attachment in plan.fresh_missing: + output( + 'SKIP_FRESH_MISSING ' + f'attachment_id={attachment.id} key={attachment.object_key} ' + f'updated={attachment.updated_at.isoformat()}' + ) + + removed = 0 + for item in plan.orphan: + action = 'DELETE_ORPHAN' if apply else 'WOULD_DELETE_ORPHAN' + output(f'{action} key={item.key} modified={item.last_modified.isoformat()}') + if apply: + await storage.remove(item.key) + removed += 1 + + marked_failed = 0 + for attachment in plan.missing: + action = 'MARK_MISSING' if apply else 'WOULD_MARK_MISSING' + output( + f'{action} attachment_id={attachment.id} key={attachment.object_key} ' + f'updated={attachment.updated_at.isoformat()}' + ) + + if apply and plan.missing: + async with sessions() as session: + async with session.begin(): + result = await session.execute( + update(AttachmentRow) + .where( + AttachmentRow.id.in_([item.id for item in plan.missing]), + AttachmentRow.status == 'ready', + ) + .values( + status='failed', + error_code=MISSING_ERROR_CODE, + updated_at=current_time, + ) + ) + marked_failed = result.rowcount or 0 + + _, peak_bytes = tracemalloc.get_traced_memory() + tracemalloc.stop() + summary: dict[str, int | float | str] = { + 'mode': 'apply' if apply else 'dry-run', + 'metadata': len(attachments), + 'objects': len(objects), + 'correct': plan.correct, + 'orphan': len(plan.orphan), + 'missing': len(plan.missing), + 'fresh_protected': len(plan.fresh_orphan) + len(plan.fresh_missing), + 'removed': removed, + 'marked_failed': marked_failed, + 'grace_seconds': grace_seconds, + 'elapsed_ms': round((time.perf_counter() - started) * 1000, 3), + 'peak_memory_bytes': peak_bytes, + } + output(f'summary {json.dumps(summary, sort_keys=True)}') + return summary + + +async def run(apply: bool, grace_seconds: int) -> int: + settings = load_settings() + engine = create_engine(settings.database_url) + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + try: + await reconcile( + create_session_factory(engine), + storage, + apply=apply, + grace_seconds=grace_seconds, + ) + return 0 + finally: + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser( + description='Reconcile attachment metadata in PostgreSQL with original objects in S3' + ) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument('--dry-run', action='store_true') + mode.add_argument('--apply', action='store_true') + parser.add_argument('--grace-seconds', type=int, default=3600) + args = parser.parse_args() + if args.grace_seconds < 60: + parser.error('--grace-seconds must be at least 60') + raise SystemExit(asyncio.run(run(args.apply, args.grace_seconds))) + + +if __name__ == '__main__': + main() diff --git a/app/tools/seed_reconciler_benchmark.py b/app/tools/seed_reconciler_benchmark.py new file mode 100644 index 00000000..7d98ed80 --- /dev/null +++ b/app/tools/seed_reconciler_benchmark.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import argparse +import asyncio +from datetime import datetime, timedelta, timezone +from io import BytesIO +from uuid import NAMESPACE_URL, uuid5 + +from sqlalchemy import delete, insert + +from ..config import load_settings +from ..db import create_engine, create_session_factory +from ..models.attachment import AttachmentRow +from ..models.chat import ChatMemberRow, ChatRow +from ..models.user import UserRow +from ..storage import S3Storage + +USER_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:user') +CHAT_ID = uuid5(NAMESPACE_URL, 'reconciler-benchmark:chat') +OBJECT_PREFIX = 'originals/reconciler-benchmark/' +WORKERS = 16 + + +async def _put_objects( + storage: S3Storage, + keys: list[str], +) -> None: + for offset in range(0, len(keys), WORKERS): + batch = keys[offset : offset + WORKERS] + await asyncio.gather( + *(storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream') for key in batch) + ) + + +async def prepare(size: int) -> None: + settings = load_settings() + engine = create_engine(settings.database_url) + sessions = create_session_factory(engine) + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + old = datetime.now(timezone.utc) - timedelta(hours=2) + rows = [ + { + 'id': uuid5(NAMESPACE_URL, f'reconciler-benchmark:attachment:{index}'), + 'chat_id': CHAT_ID, + 'owner_id': USER_ID, + 'filename': f'benchmark-{index}.png', + 'object_key': f'{OBJECT_PREFIX}{index}', + 'content_type': 'image/png', + 'size_bytes': 1, + 'sha256': '0' * 64, + 'width': 1, + 'height': 1, + 'status': 'ready', + 'error_code': None, + 'created_at': old, + 'updated_at': old, + } + for index in range(size) + ] + try: + async with sessions() as session: + async with session.begin(): + session.add(UserRow( + id=USER_ID, + username=f'bench-{USER_ID.hex[:16]}', + display_name='Reconciler benchmark', + password_hash='not-used', + created_at=old, + )) + session.add(ChatRow(id=CHAT_ID, title='Reconciler benchmark', created_at=old)) + await session.flush() + session.add(ChatMemberRow(chat_id=CHAT_ID, user_id=USER_ID)) + await session.flush() + for offset in range(0, size, 500): + await session.execute(insert(AttachmentRow), rows[offset : offset + 500]) + + await _put_objects(storage, [row['object_key'] for row in rows]) + print(f'prepared metadata={size} objects={size}') + finally: + await engine.dispose() + + +async def cleanup() -> None: + settings = load_settings() + engine = create_engine(settings.database_url) + sessions = create_session_factory(engine) + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + try: + objects = await storage.list(OBJECT_PREFIX) + keys = [item.key for item in objects] + for offset in range(0, len(keys), WORKERS): + await asyncio.gather( + *(storage.remove(key) for key in keys[offset : offset + WORKERS]) + ) + + async with sessions() as session: + async with session.begin(): + await session.execute(delete(ChatRow).where(ChatRow.id == CHAT_ID)) + await session.execute(delete(UserRow).where(UserRow.id == USER_ID)) + print(f'cleaned objects={len(keys)}') + finally: + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser( + description='Prepare and clean isolated PostgreSQL/S3 data for reconciler benchmarks' + ) + subparsers = parser.add_subparsers(dest='command', required=True) + prepare_parser = subparsers.add_parser('prepare') + prepare_parser.add_argument('--size', type=int, required=True) + subparsers.add_parser('cleanup') + args = parser.parse_args() + + if args.command == 'prepare': + if args.size < 1 or args.size > 10000: + parser.error('--size must be between 1 and 10000') + asyncio.run(prepare(args.size)) + else: + asyncio.run(cleanup()) + + +if __name__ == '__main__': + main() diff --git a/compose.tests.yaml b/compose.tests.yaml index 5845cfce..a9e229a8 100644 --- a/compose.tests.yaml +++ b/compose.tests.yaml @@ -106,7 +106,14 @@ services: dockerfile: Dockerfile.test environment: BASE_URL: http://api:8080 + DATABASE_URL: postgresql+asyncpg://messenger_test:messenger_test@postgres:5432/messenger_test + REDIS_URL: redis://redis:6379/0 SESSION_TTL_SECONDS: 3600 + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: messenger-api + S3_SECRET_ACCESS_KEY: test-minio-api-password depends_on: api: condition: service_healthy diff --git a/evidence/results.md b/evidence/results.md index 690f444f..fa2a0c71 100644 --- a/evidence/results.md +++ b/evidence/results.md @@ -1,6 +1,6 @@ # Результаты проверки lab4 -Дата последней перепроверки: 2026-11-04. Стенд: Apple M4 Pro, 12 CPU cores, 24 GB RAM, Docker Desktop. +Дата последней перепроверки: 2026-11-05. Стенд: Apple M4 Pro, 12 CPU cores, 24 GB RAM, Docker Desktop. ## Версии runtime @@ -54,6 +54,14 @@ $ docker compose -p lab4-chainguard-tests -f compose.tests.yaml run --rm tests p 13 passed in 0.41s ``` +После добавления reconciler трека 2 итоговый запуск в отдельном Compose-проекте: + +```text +$ docker compose -p lab4track2test -f compose.tests.yaml run --rm tests +................. [100%] +17 passed in 0.55s +``` + ## Приватность и bootstrap ```text @@ -101,3 +109,37 @@ summary mode=dry-run known=164 removed=0 grace_seconds=60 ``` Dry-run не менял данные. Применение требует явного `--apply`; значения grace period меньше 60 секунд запрещены. + +## Reconciler PostgreSQL/S3 — трек 2 + +`tests/test_reconciler_integration.py` внедряет ошибки на обеих сторонах настоящего +тестового стенда: старую metadata без объекта (`missing`) и объект без metadata +(`orphan`). Контрольное вложение связано с сообщением. + +```text +$ docker compose -p lab4track2test -f compose.tests.yaml run --build --rm tests \ + python -m pytest -q tests/test_reconciler.py tests/test_reconciler_integration.py +... [100%] +3 passed in 0.32s +``` + +Проверено: dry-run обнаруживает по одному расхождению и ничего не меняет; apply удаляет +ровно один orphan и переводит ровно один missing в `failed/original_missing`; повторный +apply выполняет 0 изменений. Объект контрольного вложения и связь с сообщением сохранены. + +Воспроизводимый end-to-end benchmark самостоятельно подготовил реальные данные в +PostgreSQL и MinIO, а после измерения удалил их: + +```text +$ SIZE=1000 ./scripts/benchmark-reconciler.sh +cleaned objects=0 +prepared metadata=1000 objects=1000 +summary {"correct": 1000, "elapsed_ms": 229.657, "fresh_protected": 0, +"grace_seconds": 3600, "marked_failed": 0, "metadata": 1000, "missing": 0, +"mode": "dry-run", "objects": 1000, "orphan": 0, +"peak_memory_bytes": 4202814, "removed": 0} +cleaned objects=1000 +``` + +Повторный dry-run после cleanup показал `metadata=0`, `objects=0`: benchmark-фикстура +не осталась в PostgreSQL или MinIO. diff --git a/scripts/benchmark-reconciler.sh b/scripts/benchmark-reconciler.sh new file mode 100755 index 00000000..498b3f80 --- /dev/null +++ b/scripts/benchmark-reconciler.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +size="${SIZE:-1000}" + +docker compose exec -T api python -m app.tools.seed_reconciler_benchmark cleanup +docker compose exec -T api python -m app.tools.seed_reconciler_benchmark prepare --size "$size" +docker compose exec -T api python -m app.tools.reconcile_attachments --dry-run +docker compose exec -T api python -m app.tools.seed_reconciler_benchmark cleanup diff --git a/tests/test_attachment_compensation.py b/tests/test_attachment_compensation.py index b7ef1e22..9116acc7 100644 --- a/tests/test_attachment_compensation.py +++ b/tests/test_attachment_compensation.py @@ -6,6 +6,8 @@ from uuid import uuid4 import pytest +from app.domain.errors import Conflict +from app.domain.models import Attachment from app.domain.models import User from app.services.attachments import AttachmentService @@ -30,6 +32,19 @@ class RecordingStorage: self.removed_key = key +class FailedAttachmentRepository: + def __init__(self, attachment): + self.attachment = attachment + + async def get_for_member(self, attachment_id, user_id): + return self.attachment + + +class StorageThatMustNotBeRead: + async def get(self, key): + raise AssertionError('failed attachment must not read S3') + + def test_uploaded_object_is_removed_when_metadata_commit_fails(): storage = RecordingStorage() service = AttachmentService(FailingRepository(), storage, 10 * 1024 * 1024, 40_000_000) @@ -52,3 +67,30 @@ def test_uploaded_object_is_removed_when_metadata_commit_fails(): assert storage.put_key is not None assert storage.removed_key == storage.put_key + + +def test_failed_attachment_content_returns_conflict_without_s3_read(): + now = datetime.now(timezone.utc) + attachment = Attachment( + id=uuid4(), + chat_id=uuid4(), + owner_id=uuid4(), + filename='missing.png', + object_key=f'originals/{uuid4()}', + content_type='image/png', + size_bytes=1, + sha256='0' * 64, + width=1, + height=1, + status='failed', + error_code='original_missing', + created_at=now, + updated_at=now, + ) + service = AttachmentService( + FailedAttachmentRepository(attachment), StorageThatMustNotBeRead(), 1, 1 + ) + actor = User(uuid4(), 'tester', 'Tester', now) + + with pytest.raises(Conflict, match='недоступен'): + asyncio.run(service.content(actor, attachment.id)) diff --git a/tests/test_reconciler.py b/tests/test_reconciler.py new file mode 100644 index 00000000..271dfdd2 --- /dev/null +++ b/tests/test_reconciler.py @@ -0,0 +1,50 @@ +from datetime import datetime, timedelta, timezone +from uuid import uuid4 + +from app.storage import StoredObject +from app.tools.reconcile_attachments import AttachmentSnapshot, build_plan + + +def test_plan_classifies_both_sides_and_protects_fresh_items(): + now = datetime.now(timezone.utc) + cutoff = now - timedelta(minutes=1) + correct_key = f'originals/{uuid4()}' + missing = AttachmentSnapshot(uuid4(), f'originals/{uuid4()}', 'ready', cutoff) + fresh_missing = AttachmentSnapshot(uuid4(), f'originals/{uuid4()}', 'ready', now) + already_failed = AttachmentSnapshot(uuid4(), f'originals/{uuid4()}', 'failed', cutoff) + attachments = ( + AttachmentSnapshot(uuid4(), correct_key, 'ready', cutoff), + missing, + fresh_missing, + already_failed, + ) + old_orphan = StoredObject(f'originals/{uuid4()}', cutoff) + fresh_orphan = StoredObject(f'originals/{uuid4()}', now) + + plan = build_plan( + attachments, + (StoredObject(correct_key, cutoff), old_orphan, fresh_orphan), + cutoff, + ) + + assert plan.correct == 1 + assert plan.orphan == (old_orphan,) + assert plan.fresh_orphan == (fresh_orphan,) + assert plan.missing == (missing,) + assert plan.fresh_missing == (fresh_missing,) + + +def test_failed_metadata_still_protects_its_object_from_orphan_cleanup(): + now = datetime.now(timezone.utc) + key = f'originals/{uuid4()}' + failed = AttachmentSnapshot(uuid4(), key, 'failed', now - timedelta(hours=2)) + + plan = build_plan( + (failed,), + (StoredObject(key, now - timedelta(hours=2)),), + now - timedelta(hours=1), + ) + + assert plan.correct == 1 + assert plan.orphan == () + assert plan.missing == () diff --git a/tests/test_reconciler_integration.py b/tests/test_reconciler_integration.py new file mode 100644 index 00000000..843966f6 --- /dev/null +++ b/tests/test_reconciler_integration.py @@ -0,0 +1,168 @@ +"""Real PostgreSQL + S3 fault injection for grade 5, track 2.""" +from __future__ import annotations + +import asyncio +import os +from datetime import datetime, timedelta, timezone +from io import BytesIO +from uuid import uuid4 + +import pytest +from sqlalchemy import select + +from app.config import load_settings +from app.db import create_engine, create_session_factory +from app.models.attachment import AttachmentRow, MessageAttachmentRow +from app.models.chat import ChatMemberRow, ChatRow +from app.models.message import MessageRow +from app.models.user import UserRow +from app.storage import S3Storage +from app.tools.reconcile_attachments import MISSING_ERROR_CODE, reconcile + +pytestmark = pytest.mark.skipif( + not os.getenv('DATABASE_URL'), reason='requires the compose PostgreSQL and S3 services' +) + + +def test_dry_run_apply_and_repeat_are_safe(): + asyncio.run(_check_dry_run_apply_and_repeat_are_safe()) + + +async def _check_dry_run_apply_and_repeat_are_safe(): + settings = load_settings() + engine = create_engine(settings.database_url) + sessions = create_session_factory(engine) + storage = S3Storage( + settings.s3_endpoint, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_bucket, + settings.s3_region, + ) + now = datetime.now(timezone.utc) + old = now - timedelta(hours=2) + simulated_after_grace = now + timedelta(seconds=61) + user_id, chat_id, message_id = uuid4(), uuid4(), uuid4() + correct_id, missing_id = uuid4(), uuid4() + correct_key = f'originals/reconcile-correct-{uuid4()}' + orphan_key = f'originals/reconcile-orphan-{uuid4()}' + + try: + async with sessions() as session: + async with session.begin(): + session.add(UserRow( + id=user_id, + username=f'reconcile-{user_id.hex[:16]}', + display_name='Reconciler test', + password_hash='not-used', + created_at=old, + )) + session.add(ChatRow(id=chat_id, title='Reconciler test', created_at=old)) + await session.flush() + session.add(ChatMemberRow(chat_id=chat_id, user_id=user_id)) + for attachment_id, key in ( + (correct_id, correct_key), + (missing_id, f'originals/reconcile-missing-{uuid4()}'), + ): + session.add(AttachmentRow( + id=attachment_id, + chat_id=chat_id, + owner_id=user_id, + filename='test.png', + object_key=key, + content_type='image/png', + size_bytes=1, + sha256='0' * 64, + width=1, + height=1, + status='ready', + error_code=None, + created_at=old, + updated_at=old, + )) + await session.flush() + session.add(MessageRow( + id=message_id, + chat_id=chat_id, + sender_id=user_id, + text='used attachment must survive', + created_at=old, + idempotency_key=f'reconcile-{message_id}', + )) + await session.flush() + session.add(MessageAttachmentRow( + message_id=message_id, + attachment_id=correct_id, + position=0, + )) + + for key in (correct_key, orphan_key): + await storage.put(key, BytesIO(b'x'), 1, 'application/octet-stream') + + dry_run = await reconcile( + sessions, + storage, + apply=False, + grace_seconds=60, + now=simulated_after_grace, + output=lambda _line: None, + ) + assert dry_run['orphan'] == 1 + assert dry_run['missing'] == 1 + assert dry_run['removed'] == 0 + assert dry_run['marked_failed'] == 0 + assert {item.key for item in await storage.list('originals/')} >= { + correct_key, + orphan_key, + } + + applied = await reconcile( + sessions, + storage, + apply=True, + grace_seconds=60, + now=simulated_after_grace, + output=lambda _line: None, + ) + assert applied['removed'] == 1 + assert applied['marked_failed'] == 1 + + repeated = await reconcile( + sessions, + storage, + apply=True, + grace_seconds=60, + now=simulated_after_grace, + output=lambda _line: None, + ) + assert repeated['orphan'] == 0 + assert repeated['missing'] == 0 + assert repeated['removed'] == 0 + assert repeated['marked_failed'] == 0 + + object_keys = {item.key for item in await storage.list('originals/')} + assert correct_key in object_keys + assert orphan_key not in object_keys + async with sessions() as session: + missing_row = await session.get(AttachmentRow, missing_id) + assert missing_row is not None + assert missing_row.status == 'failed' + assert missing_row.error_code == MISSING_ERROR_CODE + assert await session.scalar( + select(MessageAttachmentRow.attachment_id).where( + MessageAttachmentRow.message_id == message_id + ) + ) == correct_id + finally: + await storage.remove(correct_key) + await storage.remove(orphan_key) + async with sessions() as session: + async with session.begin(): + chat = await session.get(ChatRow, chat_id) + if chat is not None: + await session.delete(chat) + await session.flush() + user = await session.get(UserRow, user_id) + if user is not None: + await session.delete(user) + await engine.dispose() -- 2.54.0 From 1ad44408ba82451a746d4e277fec268ca220ef6c Mon Sep 17 00:00:00 2001 From: thefankor Date: Thu, 5 Nov 2026 23:30:24 +0300 Subject: [PATCH 4/5] =?UTF-8?q?feat:=20=D1=83=D1=81=D0=B8=D0=BB=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20reconciler=20=D0=B4=D0=BB=D1=8F=20lab4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 59 ++++++++++---- app/storage/s3.py | 11 +++ app/tools/reconcile_attachments.py | 67 ++++++++++++--- compose.tests.yaml | 2 +- compose.yaml | 2 +- evidence/results.md | 36 ++++---- scripts/benchmark-reconciler.sh | 27 ++++-- tests/test_attachment_compensation.py | 2 + tests/test_attachments.py | 9 ++ tests/test_reconciler.py | 2 + tests/test_reconciler_integration.py | 113 +++++++++++++++++++++++++- 11 files changed, 276 insertions(+), 54 deletions(-) diff --git a/REPORT.md b/REPORT.md index a4f4ecdc..163fe2b3 100644 --- a/REPORT.md +++ b/REPORT.md @@ -28,7 +28,11 @@ cp .env.example .env ``` -`S3_BUCKET` допускает изменение: bootstrap подставляет его в ограниченную policy. `MINIO_ROOT_*` получает только MinIO и bootstrap; API получает отдельные `S3_ACCESS_KEY_ID` и `S3_SECRET_ACCESS_KEY`. +`S3_BUCKET` допускает изменение: bootstrap при каждом запуске заново применяет ограниченную +policy с текущим именем bucket. `MINIO_ROOT_*` получает только MinIO и bootstrap; API +получает отдельные `S3_ACCESS_KEY_ID` и `S3_SECRET_ACCESS_KEY`. Access key и secret +считаются параметрами первоначальной инициализации пользователя MinIO; их ротация требует +явной операторской процедуры, а не изменения `.env` на работающем volume. Проверить конфигурацию, собрать образы, применить миграции и дождаться готовности: @@ -63,8 +67,10 @@ make benchmark-reconciler Исправления следует сначала просмотреть через `make reconcile-dry`. Изменяющий данные `make reconcile-apply` запускается оператором только после проверки dry-run. -`make benchmark-reconciler` сам создаёт 1000 изолированных пар metadata/object, -выполняет dry-run и удаляет тестовые данные; объём меняется через `SIZE=...`. +`make benchmark-reconciler` по умолчанию создаёт 100, 1000 и 10000 изолированных пар +metadata/object, выполняет по три dry-run для каждого размера и удаляет тестовые данные. +Набор и число повторов меняются через `SIZES="..."` и `RUNS=...`; старый одиночный +параметр `SIZE=...` также поддерживается. Каждый failover-тест намеренно оставляет выбранную API-реплику остановленной, поэтому между сценариями и после последнего сценария она явно восстанавливается указанной командой `docker compose up`. @@ -140,6 +146,11 @@ docker compose --env-file .env run --rm --no-deps api \ ### Reconciler PostgreSQL/S3 — трек 2 +Гипотеза эксперимента: полный снимок PostgreSQL и listing MinIO дадут линейную +асимптотику `O(N)` по времени и памяти. На диапазоне 100–10000 согласованных пар рост +времени не должен становиться сверхлинейным; абсолютные значения зависят от Docker-host +и прогрева, поэтому для каждой точки используются три последовательных запуска и медиана. + Расширенный reconciler запускается внутри API-контейнера, где уже есть те же настройки и ограниченные S3 credentials, что у приложения: @@ -169,6 +180,14 @@ Grace period защищает upload, который ещё выполняетс ключ защищён одинаково, поэтому корректный оригинал связанного сообщения не удаляется. Content для `failed` возвращает 409, а привязать такое вложение к новому сообщению нельзя. +Начальные снимки могут устареть во время apply. Поэтому перед удалением orphan reconciler +повторно проверяет отсутствие `object_key` в PostgreSQL, перед маркировкой missing выполняет +`HEAD/stat` объекта, а SQL `UPDATE` дополнительно сравнивает `id`, `object_key`, `status` и +исходный `updated_at`. Гонки логируются как `SKIP_*` и учитываются в `race_protected`. +Общей транзакции между PostgreSQL и S3 всё равно нет: остаётся узкое TOCTOU-окно между +последней проверкой и действием. Полностью закрыть его можно только протоколом состояний/ +lease или запретом параллельного восстановления на время apply. + Операции между двумя хранилищами всё ещё не образуют общей транзакции. Если apply прервётся, уже выполненное безопасное удаление orphan не требует отката, условный SQL UPDATE меняет только всё ещё `ready` строки, а повторный запуск завершает оставшуюся @@ -196,9 +215,9 @@ UPDATE меняет только всё ещё `ready` строки, а повт | Concurrency/RAM | `scripts/measure-upload-memory.py --concurrency 10 --uploads 30` | 30/30 upload, обе реплики, пики 68,50/68,33 MiB | `evidence/upload-memory.json` | | Идемпотентный bootstrap | повторный запуск `docker compose up` после restart/failover | bootstrap повторно завершился с code 0 | `evidence/results.md` | | Cleanup dry-run | `cleanup_orphans --dry-run --grace-seconds 60` | 164 известных объекта, удалено 0 | `evidence/results.md` | -| Reconciler fault injection | реальный PostgreSQL + MinIO: orphan, missing и используемый control | dry-run без изменений; apply исправил 2 расхождения; повторный apply — 0 действий | `tests/test_reconciler_integration.py`, `evidence/results.md` | +| Reconciler fault injection | реальный PostgreSQL + MinIO: orphan, missing, fresh и три конкурентных изменения | apply исправляет только подтверждённые расхождения; свежие/изменённые данные защищены; повторный apply — 0 действий | `tests/test_reconciler_integration.py`, `evidence/results.md` | | Полный тестовый стенд | изолированный Compose, все pytest | `17 passed` | `evidence/results.md` | -| Стоимость обхода | `SIZE=1000 make benchmark-reconciler` | 1000 строк и 1000 объектов: 229,657 мс, пик 4 202 814 bytes | `evidence/results.md` | +| Стоимость обхода | `SIZES="100 1000 10000" RUNS=3 make benchmark-reconciler` | медианы: 80,712 / 227,467 / 1733,545 мс; Python peak 2,62 / 4,21 / 8,08 MB | `evidence/results.md` | ## Эксперименты и отказы @@ -213,17 +232,27 @@ UPDATE меняет только всё ещё `ready` строки, а повт Fault injection reconciler выполняется на настоящих PostgreSQL и MinIO. Тест создаёт старую `ready` строку без объекта, объект без строки и корректное вложение, уже связанное -с сообщением. Управляемые часы имитируют истечение 60-секундного grace period без -ожидания в тесте. Dry-run сохраняет обе ошибки, apply удаляет только orphan и маркирует -missing, второй apply сообщает нулевое число действий. Корректный объект и связь сообщения -после обоих запусков остаются на месте. +с сообщением. Дополнительно он восстанавливает missing-объект, изменяет `updated_at` +другой missing-строки и создаёт metadata для orphan уже после получения начальных +снимков, а также проверяет настоящий свежий orphan. +Управляемые часы имитируют истечение grace period без минутного ожидания. Apply исправляет +только подтверждённые расхождения; свежий объект и все три конкурентно изменённые +сущности защищены. Повторный apply не повторяет исправления, связь сообщения сохраняется. -Воспроизводимый benchmark сначала удаляет прежнюю фикстуру, затем создаёт 1000 строк -`attachments` и 1000 объектов MinIO, выполняет полный dry-run и удаляет фикстуру. -Обход занял 229,657 мс, пик Python allocations составил 4 202 814 bytes. Измерение -включает запрос PostgreSQL, listing MinIO и сравнение снимков; время подготовки и очистки -в `elapsed_ms` reconciler не входит. Контрольный dry-run после cleanup показал 0 строк и -0 объектов benchmark-набора. +Воспроизводимый benchmark для каждого размера удаляет прежнюю фикстуру, создаёт одинаковое +число строк `attachments` и объектов MinIO, выполняет три полных dry-run и удаляет данные. + +| Пар metadata/object | Время, мс (три запуска) | Медиана, мс | Медианный peak Python allocations | +| ---: | --- | ---: | ---: | +| 100 | 78,847; 80,712; 82,781 | 80,712 | 2 620 657 bytes | +| 1 000 | 232,254; 227,467; 226,264 | 227,467 | 4 205 016 bytes | +| 10 000 | 1701,005; 1792,955; 1733,545 | 1733,545 | 8 075 853 bytes | + +Измерение включает запрос PostgreSQL, listing MinIO и сравнение снимков; подготовка и +очистка в `elapsed_ms` не входят. Результат согласуется с гипотезой `O(N)`: после заметной +фиксированной стоимости время и память монотонно растут, на исследованном диапазоне нет +признаков сверхлинейного взрыва. Это не доказательство для production-масштаба: измеряется +Python `tracemalloc`, а не RSS контейнера, и полный снимок всё равно требует памяти `O(N)`. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | diff --git a/app/storage/s3.py b/app/storage/s3.py index 251951c8..dcc97ab2 100644 --- a/app/storage/s3.py +++ b/app/storage/s3.py @@ -102,6 +102,17 @@ class S3Storage: self._translate(error) return ObjectStream(response) + async def exists(self, key: str) -> bool: + try: + await asyncio.to_thread(self._client.stat_object, self.bucket, key) + return True + except Exception as error: + try: + self._translate(error) + except FileNotFoundError: + return False + raise + async def remove(self, key: str) -> None: try: await asyncio.to_thread(self._client.remove_object, self.bucket, key) diff --git a/app/tools/reconcile_attachments.py b/app/tools/reconcile_attachments.py index d4f885f6..7553e7bc 100644 --- a/app/tools/reconcile_attachments.py +++ b/app/tools/reconcile_attachments.py @@ -83,6 +83,16 @@ async def _load_attachments( return tuple(AttachmentSnapshot(*row) for row in rows) +async def _metadata_key_exists( + sessions: async_sessionmaker[AsyncSession], object_key: str +) -> bool: + async with sessions() as session: + attachment_id = await session.scalar( + select(AttachmentRow.id).where(AttachmentRow.object_key == object_key).limit(1) + ) + return attachment_id is not None + + async def reconcile( sessions: async_sessionmaker[AsyncSession], storage: S3Storage, @@ -113,29 +123,51 @@ async def reconcile( ) removed = 0 + race_protected = 0 for item in plan.orphan: action = 'DELETE_ORPHAN' if apply else 'WOULD_DELETE_ORPHAN' + if not apply: + output(f'{action} key={item.key} modified={item.last_modified.isoformat()}') + continue + # PostgreSQL and S3 cannot share a transaction. Re-read the authoritative + # metadata immediately before a destructive action so a row created after + # the initial snapshot protects its object. + if await _metadata_key_exists(sessions, item.key): + output(f'SKIP_ORPHAN_METADATA_APPEARED key={item.key}') + race_protected += 1 + continue output(f'{action} key={item.key} modified={item.last_modified.isoformat()}') - if apply: - await storage.remove(item.key) - removed += 1 + await storage.remove(item.key) + removed += 1 marked_failed = 0 for attachment in plan.missing: - action = 'MARK_MISSING' if apply else 'WOULD_MARK_MISSING' - output( - f'{action} attachment_id={attachment.id} key={attachment.object_key} ' - f'updated={attachment.updated_at.isoformat()}' - ) + if not apply: + output( + f'WOULD_MARK_MISSING attachment_id={attachment.id} ' + f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}' + ) + continue + + # The object may have been restored after list_objects(). Do not turn + # healthy metadata into failed based only on the earlier snapshot. + if await storage.exists(attachment.object_key): + output( + f'SKIP_MISSING_OBJECT_RESTORED attachment_id={attachment.id} ' + f'key={attachment.object_key}' + ) + race_protected += 1 + continue - if apply and plan.missing: async with sessions() as session: async with session.begin(): result = await session.execute( update(AttachmentRow) .where( - AttachmentRow.id.in_([item.id for item in plan.missing]), + AttachmentRow.id == attachment.id, + AttachmentRow.object_key == attachment.object_key, AttachmentRow.status == 'ready', + AttachmentRow.updated_at == attachment.updated_at, ) .values( status='failed', @@ -143,7 +175,19 @@ async def reconcile( updated_at=current_time, ) ) - marked_failed = result.rowcount or 0 + changed = result.rowcount or 0 + if changed: + output( + f'MARK_MISSING attachment_id={attachment.id} ' + f'key={attachment.object_key} updated={attachment.updated_at.isoformat()}' + ) + marked_failed += changed + else: + output( + f'SKIP_MISSING_METADATA_CHANGED attachment_id={attachment.id} ' + f'key={attachment.object_key}' + ) + race_protected += 1 _, peak_bytes = tracemalloc.get_traced_memory() tracemalloc.stop() @@ -157,6 +201,7 @@ async def reconcile( 'fresh_protected': len(plan.fresh_orphan) + len(plan.fresh_missing), 'removed': removed, 'marked_failed': marked_failed, + 'race_protected': race_protected, 'grace_seconds': grace_seconds, 'elapsed_ms': round((time.perf_counter() - started) * 1000, 3), 'peak_memory_bytes': peak_bytes, diff --git a/compose.tests.yaml b/compose.tests.yaml index a9e229a8..839a5007 100644 --- a/compose.tests.yaml +++ b/compose.tests.yaml @@ -45,7 +45,7 @@ services: mc anonymous set none local/"$$S3_BUCKET" mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json - mc admin policy info local messenger-api >/dev/null 2>&1 || mc admin policy create local messenger-api /tmp/s3-api-policy.json + mc admin policy create local messenger-api /tmp/s3-api-policy.json mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID" environment: MINIO_ROOT_USER: test-minio-admin diff --git a/compose.yaml b/compose.yaml index f3a74d12..8e1f5878 100644 --- a/compose.yaml +++ b/compose.yaml @@ -59,7 +59,7 @@ services: mc anonymous set none local/"$$S3_BUCKET" mc admin user info local "$$S3_ACCESS_KEY_ID" >/dev/null 2>&1 || mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" sed "s/BUCKET_NAME/$$S3_BUCKET/g" /policy/s3-api-policy.json >/tmp/s3-api-policy.json - mc admin policy info local messenger-api >/dev/null 2>&1 || mc admin policy create local messenger-api /tmp/s3-api-policy.json + mc admin policy create local messenger-api /tmp/s3-api-policy.json mc admin policy attach local messenger-api --user "$$S3_ACCESS_KEY_ID" environment: MINIO_ROOT_USER: ${MINIO_ROOT_USER:-local-minio-admin} diff --git a/evidence/results.md b/evidence/results.md index fa2a0c71..d5073a4f 100644 --- a/evidence/results.md +++ b/evidence/results.md @@ -57,9 +57,9 @@ $ docker compose -p lab4-chainguard-tests -f compose.tests.yaml run --rm tests p После добавления reconciler трека 2 итоговый запуск в отдельном Compose-проекте: ```text -$ docker compose -p lab4track2test -f compose.tests.yaml run --rm tests +$ docker compose -p lab4-review3 -f compose.tests.yaml run --build --rm tests ................. [100%] -17 passed in 0.55s +17 passed in 0.60s ``` ## Приватность и bootstrap @@ -117,29 +117,31 @@ Dry-run не менял данные. Применение требует явн (`orphan`). Контрольное вложение связано с сообщением. ```text -$ docker compose -p lab4track2test -f compose.tests.yaml run --build --rm tests \ +$ docker compose -p lab4-review3 -f compose.tests.yaml run --rm tests \ python -m pytest -q tests/test_reconciler.py tests/test_reconciler_integration.py ... [100%] -3 passed in 0.32s +3 passed in 0.30s ``` -Проверено: dry-run обнаруживает по одному расхождению и ничего не меняет; apply удаляет -ровно один orphan и переводит ровно один missing в `failed/original_missing`; повторный -apply выполняет 0 изменений. Объект контрольного вложения и связь с сообщением сохранены. +Проверено: dry-run ничего не меняет; apply удаляет один подтверждённый orphan и переводит +один подтверждённый missing в `failed/original_missing`. Missing-объект, восстановленный +после listing, missing-строка с изменившимся `updated_at` и orphan, для которого metadata +появилась после снимка, получают `SKIP_*` и учитываются в `race_protected`. Отдельный +свежий orphan переживает настоящий apply. +Повторный apply выполняет 0 повторных исправлений. Объект используемого вложения и связь +с сообщением сохранены. Воспроизводимый end-to-end benchmark самостоятельно подготовил реальные данные в PostgreSQL и MinIO, а после измерения удалил их: ```text -$ SIZE=1000 ./scripts/benchmark-reconciler.sh -cleaned objects=0 -prepared metadata=1000 objects=1000 -summary {"correct": 1000, "elapsed_ms": 229.657, "fresh_protected": 0, -"grace_seconds": 3600, "marked_failed": 0, "metadata": 1000, "missing": 0, -"mode": "dry-run", "objects": 1000, "orphan": 0, -"peak_memory_bytes": 4202814, "removed": 0} -cleaned objects=1000 +$ SIZES="100 1000 10000" RUNS=3 ./scripts/benchmark-reconciler.sh +size=100: elapsed_ms=78.847,80.712,82.781; peak median=2620657 +size=1000: elapsed_ms=232.254,227.467,226.264; peak median=4205016 +size=10000: elapsed_ms=1701.005,1792.955,1733.545; peak median=8075853 ``` -Повторный dry-run после cleanup показал `metadata=0`, `objects=0`: benchmark-фикстура -не осталась в PostgreSQL или MinIO. +Медианы времени: 80,712; 227,467; 1733,545 мс. Скрипт готовит каждый размер один раз, +делает три dry-run, затем удаляет benchmark-фикстуру. Измерение выполнено в отдельном +Compose-проекте; фиксированные три пары, созданные интеграционными HTTP-проверками, +присутствовали во всех точках одинаково и не влияют на вывод о масштабе. diff --git a/scripts/benchmark-reconciler.sh b/scripts/benchmark-reconciler.sh index 498b3f80..eb3c20e5 100755 --- a/scripts/benchmark-reconciler.sh +++ b/scripts/benchmark-reconciler.sh @@ -1,9 +1,26 @@ #!/bin/sh set -eu -size="${SIZE:-1000}" +sizes="${SIZES:-${SIZE:-100 1000 10000}}" +runs="${RUNS:-3}" -docker compose exec -T api python -m app.tools.seed_reconciler_benchmark cleanup -docker compose exec -T api python -m app.tools.seed_reconciler_benchmark prepare --size "$size" -docker compose exec -T api python -m app.tools.reconcile_attachments --dry-run -docker compose exec -T api python -m app.tools.seed_reconciler_benchmark cleanup +case "$runs" in + ''|*[!0-9]*) printf '%s\n' 'RUNS must be a positive integer' >&2; exit 2 ;; + 0) printf '%s\n' 'RUNS must be a positive integer' >&2; exit 2 ;; +esac + +cleanup() { + docker compose exec -T api python -m app.tools.seed_reconciler_benchmark cleanup +} +trap cleanup EXIT HUP INT TERM + +for size in $sizes; do + cleanup + docker compose exec -T api python -m app.tools.seed_reconciler_benchmark prepare --size "$size" + run=1 + while [ "$run" -le "$runs" ]; do + printf 'benchmark size=%s run=%s/%s\n' "$size" "$run" "$runs" + docker compose exec -T api python -m app.tools.reconcile_attachments --dry-run + run=$((run + 1)) + done +done diff --git a/tests/test_attachment_compensation.py b/tests/test_attachment_compensation.py index 9116acc7..975c84c4 100644 --- a/tests/test_attachment_compensation.py +++ b/tests/test_attachment_compensation.py @@ -46,6 +46,7 @@ class StorageThatMustNotBeRead: def test_uploaded_object_is_removed_when_metadata_commit_fails(): + """Удаляет объект после ошибки сохранения метаданных.""" storage = RecordingStorage() service = AttachmentService(FailingRepository(), storage, 10 * 1024 * 1024, 40_000_000) actor = User(uuid4(), 'tester', 'Tester', datetime.now(timezone.utc)) @@ -70,6 +71,7 @@ def test_uploaded_object_is_removed_when_metadata_commit_fails(): def test_failed_attachment_content_returns_conflict_without_s3_read(): + """Возвращает конфликт для ошибочного вложения без чтения S3.""" now = datetime.now(timezone.utc) attachment = Attachment( id=uuid4(), diff --git a/tests/test_attachments.py b/tests/test_attachments.py index 4f3b14a5..0a34c667 100644 --- a/tests/test_attachments.py +++ b/tests/test_attachments.py @@ -37,22 +37,26 @@ def oversized_dimensions_png(content: bytes) -> bytes: def test_file_larger_than_10_mib_is_rejected(scenario): + """Отклоняет файл размером больше 10 MiB.""" client, alice, _bob, _eve, chat = scenario upload(client, alice, chat, b'x' * (10 * 1024 * 1024 + 1), expected=413) def test_corrupted_image_is_rejected(scenario): + """Отклоняет повреждённое изображение.""" client, alice, _bob, _eve, chat = scenario upload(client, alice, chat, b'not-a-png', expected=415) def test_mime_mismatch_is_rejected(scenario): + """Отклоняет MIME, не совпадающий с содержимым.""" client, alice, _bob, _eve, chat = scenario content = (ROOT / 'tests/fixtures/sample.png').read_bytes() upload(client, alice, chat, content, content_type='image/jpeg', expected=415) def test_pixel_limit_is_checked_before_decode(scenario): + """Проверяет лимит пикселей до полной декомпрессии.""" client, alice, _bob, _eve, chat = scenario content = oversized_dimensions_png((ROOT / 'tests/fixtures/sample.png').read_bytes()) upload(client, alice, chat, content, expected=413) @@ -63,12 +67,14 @@ def test_pixel_limit_is_checked_before_decode(scenario): ['../escape.png', 'folder\\escape.png', 'bad\x00name.png', 'bad\x85name.png'], ) def test_dangerous_filename_is_rejected(scenario, filename): + """Отклоняет опасное отображаемое имя файла.""" client, alice, _bob, _eve, chat = scenario content = (ROOT / 'tests/fixtures/sample.png').read_bytes() upload(client, alice, chat, content, filename=filename, expected=400) def test_outsider_cannot_read_attachment(scenario): + """Скрывает метаданные и содержимое от постороннего пользователя.""" client, alice, _bob, eve, chat = scenario content = (ROOT / 'tests/fixtures/sample.png').read_bytes() attachment = upload(client, alice, chat, content)['json'] @@ -77,6 +83,7 @@ def test_outsider_cannot_read_attachment(scenario): def test_attachment_from_another_chat_cannot_be_linked(scenario): + """Запрещает прикреплять вложение другого чата.""" client, alice, bob, _eve, chat = scenario other_chat = client.call( 'POST', @@ -99,6 +106,7 @@ def test_attachment_from_another_chat_cannot_be_linked(scenario): def test_idempotency_distinguishes_attachment_list(scenario): + """Включает список вложений в сравнение идемпотентного запроса.""" client, alice, _bob, _eve, chat = scenario content = (ROOT / 'tests/fixtures/sample.png').read_bytes() attachment = upload(client, alice, chat, content)['json'] @@ -129,6 +137,7 @@ def test_idempotency_distinguishes_attachment_list(scenario): def test_missing_attachment_ids_equals_empty_list_for_idempotency(scenario): + """Считает отсутствие attachment_ids эквивалентным пустому списку.""" client, alice, _bob, _eve, chat = scenario key = uuid.uuid4().hex first = client.message(alice, chat, 'обычный текст', key=key)['json'] diff --git a/tests/test_reconciler.py b/tests/test_reconciler.py index 271dfdd2..442c6ecf 100644 --- a/tests/test_reconciler.py +++ b/tests/test_reconciler.py @@ -6,6 +6,7 @@ from app.tools.reconcile_attachments import AttachmentSnapshot, build_plan def test_plan_classifies_both_sides_and_protects_fresh_items(): + """Классифицирует orphan/missing и защищает свежие расхождения.""" now = datetime.now(timezone.utc) cutoff = now - timedelta(minutes=1) correct_key = f'originals/{uuid4()}' @@ -35,6 +36,7 @@ def test_plan_classifies_both_sides_and_protects_fresh_items(): def test_failed_metadata_still_protects_its_object_from_orphan_cleanup(): + """Проверяет защиту объекта существующей failed-metadata.""" now = datetime.now(timezone.utc) key = f'originals/{uuid4()}' failed = AttachmentSnapshot(uuid4(), key, 'failed', now - timedelta(hours=2)) diff --git a/tests/test_reconciler_integration.py b/tests/test_reconciler_integration.py index 843966f6..f44fc0c5 100644 --- a/tests/test_reconciler_integration.py +++ b/tests/test_reconciler_integration.py @@ -8,7 +8,7 @@ from io import BytesIO from uuid import uuid4 import pytest -from sqlalchemy import select +from sqlalchemy import select, update from app.config import load_settings from app.db import create_engine, create_session_factory @@ -24,7 +24,24 @@ pytestmark = pytest.mark.skipif( ) +class AfterListStorage: + def __init__(self, storage, after_list): + self._storage = storage + self._after_list = after_list + + async def list(self, prefix=''): + snapshot = await self._storage.list(prefix) + if self._after_list is not None: + after_list, self._after_list = self._after_list, None + await after_list() + return snapshot + + def __getattr__(self, name): + return getattr(self._storage, name) + + def test_dry_run_apply_and_repeat_are_safe(): + """Проверяет пробный, изменяющий и повторный запуски.""" asyncio.run(_check_dry_run_apply_and_repeat_are_safe()) @@ -43,9 +60,14 @@ async def _check_dry_run_apply_and_repeat_are_safe(): old = now - timedelta(hours=2) simulated_after_grace = now + timedelta(seconds=61) user_id, chat_id, message_id = uuid4(), uuid4(), uuid4() - correct_id, missing_id = uuid4(), uuid4() + correct_id, missing_id, restored_id = uuid4(), uuid4(), uuid4() + changed_id, appeared_id = uuid4(), uuid4() correct_key = f'originals/reconcile-correct-{uuid4()}' orphan_key = f'originals/reconcile-orphan-{uuid4()}' + restored_key = f'originals/reconcile-restored-{uuid4()}' + changed_key = f'originals/reconcile-changed-{uuid4()}' + appeared_key = f'originals/reconcile-appeared-{uuid4()}' + fresh_key = f'originals/reconcile-fresh-{uuid4()}' try: async with sessions() as session: @@ -63,6 +85,8 @@ async def _check_dry_run_apply_and_repeat_are_safe(): for attachment_id, key in ( (correct_id, correct_key), (missing_id, f'originals/reconcile-missing-{uuid4()}'), + (restored_id, restored_key), + (changed_id, changed_key), ): session.add(AttachmentRow( id=attachment_id, @@ -108,7 +132,7 @@ async def _check_dry_run_apply_and_repeat_are_safe(): output=lambda _line: None, ) assert dry_run['orphan'] == 1 - assert dry_run['missing'] == 1 + assert dry_run['missing'] == 3 assert dry_run['removed'] == 0 assert dry_run['marked_failed'] == 0 assert {item.key for item in await storage.list('originals/')} >= { @@ -116,9 +140,19 @@ async def _check_dry_run_apply_and_repeat_are_safe(): orphan_key, } + async def restore_missing_after_snapshot(): + await storage.put(restored_key, BytesIO(b'x'), 1, 'application/octet-stream') + async with sessions() as session: + async with session.begin(): + await session.execute( + update(AttachmentRow) + .where(AttachmentRow.id == changed_id) + .values(updated_at=now) + ) + applied = await reconcile( sessions, - storage, + AfterListStorage(storage, restore_missing_after_snapshot), apply=True, grace_seconds=60, now=simulated_after_grace, @@ -126,6 +160,11 @@ async def _check_dry_run_apply_and_repeat_are_safe(): ) assert applied['removed'] == 1 assert applied['marked_failed'] == 1 + assert applied['race_protected'] == 2 + + # Make the concurrently changed metadata healthy before checking that + # the next complete run has no work left. + await storage.put(changed_key, BytesIO(b'x'), 1, 'application/octet-stream') repeated = await reconcile( sessions, @@ -142,6 +181,8 @@ async def _check_dry_run_apply_and_repeat_are_safe(): object_keys = {item.key for item in await storage.list('originals/')} assert correct_key in object_keys + assert restored_key in object_keys + assert changed_key in object_keys assert orphan_key not in object_keys async with sessions() as session: missing_row = await session.get(AttachmentRow, missing_id) @@ -153,9 +194,73 @@ async def _check_dry_run_apply_and_repeat_are_safe(): MessageAttachmentRow.message_id == message_id ) ) == correct_id + + restored_row = await session.get(AttachmentRow, restored_id) + assert restored_row is not None + assert restored_row.status == 'ready' + + changed_row = await session.get(AttachmentRow, changed_id) + assert changed_row is not None + assert changed_row.status == 'ready' + + # An object younger than the grace period survives a real apply run. + await storage.put(fresh_key, BytesIO(b'x'), 1, 'application/octet-stream') + fresh_apply = await reconcile( + sessions, + storage, + apply=True, + grace_seconds=60, + now=datetime.now(timezone.utc), + output=lambda _line: None, + ) + assert fresh_apply['fresh_protected'] == 1 + assert fresh_apply['removed'] == 0 + assert await storage.exists(fresh_key) + await storage.remove(fresh_key) + + # Metadata created after the initial snapshots also protects an old + # object from the destructive orphan action. + await storage.put(appeared_key, BytesIO(b'x'), 1, 'application/octet-stream') + + async def create_metadata_after_snapshot(): + async with sessions() as session: + async with session.begin(): + session.add(AttachmentRow( + id=appeared_id, + chat_id=chat_id, + owner_id=user_id, + filename='appeared.png', + object_key=appeared_key, + content_type='image/png', + size_bytes=1, + sha256='0' * 64, + width=1, + height=1, + status='ready', + error_code=None, + created_at=old, + updated_at=old, + )) + + appeared_apply = await reconcile( + sessions, + AfterListStorage(storage, create_metadata_after_snapshot), + apply=True, + grace_seconds=60, + now=datetime.now(timezone.utc) + timedelta(seconds=61), + output=lambda _line: None, + ) + assert appeared_apply['orphan'] == 1 + assert appeared_apply['removed'] == 0 + assert appeared_apply['race_protected'] == 1 + assert await storage.exists(appeared_key) finally: await storage.remove(correct_key) await storage.remove(orphan_key) + await storage.remove(restored_key) + await storage.remove(changed_key) + await storage.remove(appeared_key) + await storage.remove(fresh_key) async with sessions() as session: async with session.begin(): chat = await session.get(ChatRow, chat_id) -- 2.54.0 From cd542ea305587abec3c856dfc659aa904bca0d68 Mon Sep 17 00:00:00 2001 From: thefankor Date: Fri, 6 Nov 2026 11:45:12 +0300 Subject: [PATCH 5/5] =?UTF-8?q?docs:=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D1=81=D1=81=D1=8B=D0=BB=D0=BA=D1=83=20?= =?UTF-8?q?=D0=BD=D0=B0=20PR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPORT.md b/REPORT.md index 163fe2b3..fa13d006 100644 --- a/REPORT.md +++ b/REPORT.md @@ -2,7 +2,7 @@ - Автор, группа: Сергей Возжин, БПИ2301. - Целевая оценка: 5; выбран трек 2 — согласованность PostgreSQL и S3. -- Commit/PR: TODO после публикации. +- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/70 - Использованные источники: контракт репозитория, документация Docker Compose, Chainguard Images, MinIO SDK и Pillow. - Библиотеки: FastAPI, Uvicorn, SQLAlchemy, asyncpg, Alembic, redis-py, MinIO Python SDK, Pillow, Argon2, pytest. - Помощь ИИ: OpenAI Codex использовался для плана, реализации, тестов и оформления отчёта; результат проверен исполняемыми тестами. -- 2.54.0