before level 3 initial commit

This commit is contained in:
2026-11-05 17:03:42 +03:00
parent 0ab3a341aa
commit 8cd25e448f
32 changed files with 1616 additions and 4 deletions
+6
View File
@@ -11,3 +11,9 @@ SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
# compose.yaml builds DATABASE_URL from these; pick your own local password.
POSTGRES_USER=messenger
POSTGRES_PASSWORD=change-me-locally
POSTGRES_DB=messenger
SHUTDOWN_TIMEOUT_SECONDS=5
LB_PORT=8080
+3
View File
@@ -0,0 +1,3 @@
*.sh text eol=lf
*.conf text eol=lf
*.cmd text eol=crlf
+35
View File
@@ -0,0 +1,35 @@
# Закреплённая среда выполнения. Тег называет версию, digest — то, что Docker реально скачает.
FROM python:3.14.8-slim-bookworm@sha256:c8137f4c460908c8763f281c8f22c431eb5c538514ba9553fc3a89c06b7cfb88
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
# Значения по умолчанию, секретов нет. `docker run --env-file .env` перекрывает их при запуске.
ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab \
LOG_LEVEL=info \
SHUTDOWN_TIMEOUT_SECONDS=5 \
SESSION_TTL_SECONDS=3600
RUN groupadd --gid 10001 app \
&& useradd --uid 10001 --gid app --create-home --shell /usr/sbin/nologin app
WORKDIR /app
COPY requirements.txt .
RUN python -m pip install --no-cache-dir --disable-pip-version-check --root-user-action=ignore -r requirements.txt
COPY --chown=app:app app ./app
COPY --chown=app:app migrations ./migrations
COPY --chown=app:app contracts/openapi.json ./contracts/openapi.json
USER app
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 \
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ['APP_PORT'], timeout=2)"]
STOPSIGNAL SIGTERM
CMD ["python", "-m", "app"]
+4 -1
View File
@@ -4,7 +4,7 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover test-idempotency
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -28,3 +28,6 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-idempotency:
$(PYTHON) scripts/check_idempotency.py --base-url "$(BASE_URL)"
+38 -3
View File
@@ -7,11 +7,43 @@
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Вариант: **Docker Compose + Nginx**. Нужны Docker Engine с Compose v2 (проверено на Docker 29.4, Compose 5.1), Python 3.9+ для тестов; на Windows скрипты `.sh` выполняются через Git Bash (обёртки `scripts/*.cmd`).
Образы: `postgres:16.10-alpine`, `nginx:1.28.0-alpine` (закреплены digest в `compose.yaml`), приложение собирается из `Dockerfile` на `python:3.14.8-slim-bookworm@sha256:…`.
Одна последовательность запуска:
```sh
cp .env.example .env # задать свой POSTGRES_PASSWORD
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait
# postgres (healthy) -> migrate (один раз, exit 0) -> api x API_REPLICAS (healthy) -> lb (healthy)
make test # smoke через LB, http://localhost:8080
make test-idempotency # один ключ параллельно через разные реплики
make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd
docker compose --env-file .env up -d --wait # вернуть остановленную реплику
make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/restart.cmd
```
Остановка без потери данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v` (удаляет volume `pgdata`).
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
client -> lb (nginx :8080, только loopback) -> api-1, api-2, … (порт не публикуется) -> postgres (volume pgdata)
migrate (one-shot) ----------------------^
```
- **Источник истины — PostgreSQL**: пользователи, чаты, сообщения, сессии (`sessions`, токен хранится как SHA-256) и результаты идемпотентности (`idempotency_keys`). В API нет локального состояния, от которого зависит корректность. Redis из lab2 убран: он был только кэшем, а для общего хранилища сессий достаточно PostgreSQL (`SESSION_STORE=postgres`).
- **Миграции** выполняет отдельный сервис `migrate` (`python -m app.migrate`) до старта реплик (`depends_on: service_completed_successfully`). Реплики схему не меняют; `/health/ready` возвращает 503, пока в `schema_migrations` нет всех миграций этой сборки. Параллельный запуск мигратора сериализуется `pg_advisory_lock`.
- **X-Instance-Id** = hostname контейнера (короткий Docker ID), стабилен на время жизни процесса. `stop-one.sh` по нему находит ровно один running-контейнер сервиса `api` этого проекта и делает `docker stop` (SIGTERM, затем SIGKILL через 8 с). `restart: unless-stopped` не поднимает вручную остановленный контейнер, поэтому реплика остаётся остановленной до `docker compose up -d`.
- **Балансировщик**: round robin без `ip_hash`/sticky. `server api:8080 resolve` + `resolver 127.0.0.11 valid=2s`: Nginx раз в 2 с перечитывает DNS Docker, так что остановленные и добавленные (`--scale api=N`) реплики уходят из пула/входят в него без перезапуска LB. Пассивная проверка: `max_fails=1 fail_timeout=5s`, `proxy_connect_timeout 1s`, `proxy_next_upstream error timeout http_502 http_504`. Активных проверок в open-source Nginx нет; readiness реплик проверяет Docker healthcheck (`/health/ready` каждые 3 с), и LB стартует только после healthy API.
- **Повтор на другой реплике**: для неотправленного запроса Nginx всегда пробует следующую реплику. Уже отправленный POST повторяется только для `/api/v1/chats/{id}/messages` (`non_idempotent`), потому что там обязателен `Idempotency-Key`. POST пользователей/чатов/сессий LB не повторяет.
- **Graceful shutdown**: на SIGTERM uvicorn перестаёт принимать соединения и дорабатывает текущие запросы до `SHUTDOWN_TIMEOUT_SECONDS=5`; `stop_grace_period: 8s` больше этого. `keepalive_timeout 4s` в upstream меньше keep-alive uvicorn (5 с), чтобы LB не переиспользовал закрываемое API соединение.
- **Идемпотентность**: в одной транзакции удаляется истёкший (старше `IDEMPOTENCY_TTL_SECONDS`) ключ, затем `INSERT … ON CONFLICT DO NOTHING` в `idempotency_keys` (PK: пользователь + метод + путь + ключ) вместе с готовым ответом и SHA-256 канонического JSON тела, затем вставка сообщения. Конкурентный дубль на другой реплике ждёт на уникальном индексе до COMMIT первой транзакции и получает сохранённый ответ (тот же 201 и Message) либо 409 при другом теле. При ошибке транзакция откатывается целиком, ключ не фиксируется, повтор с тем же ключом безопасен. Сессия и членство в чате проверяются при каждом повторе.
- Время сообщений назначает приложение; порядок между репликами опирается на общие часы хоста (учебное упрощение, одна машина).
Соответствие `.env.example`: `DATABASE_URL` собирается в `compose.yaml` из `POSTGRES_*` (вариант `DATABASE_URL_FILE` тоже поддерживается кодом); `API_REPLICAS` → `deploy.replicas`; `IDEMPOTENCY_TTL_SECONDS`, `SESSION_TTL_SECONDS`, `SHUTDOWN_TIMEOUT_SECONDS` передаются в API; `CACHE_URL_FILE` не используется.
## Доказательства критериев
@@ -30,7 +62,10 @@ TODO: гипотеза, hardware/лимиты, версии, объём данн
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| `stop-one` реплики, обслужившей тест (api-1), под записью | запись продолжается ≤10 с, дублей нет | 126 успешных записей, 0 ошибок, первая запись после stop через 1,6 с (`evidence/failover.txt`) | ~1,6 с / нет |
| `stop-one` второй реплики (api-2) после восстановления первой | то же | 126 записей, 0 ошибок, 1,7 с | ~1,7 с / нет |
| `restart.sh`: lb, api, postgres | данные и старый токен сохранены | PASS (`evidence/persistence.txt`) | нет потерь |
| 5 ключей × 20 параллельных POST через LB | одно сообщение на ключ | по 10 ответов с каждой реплики, 5 сообщений (`evidence/idempotency.txt`) | — |
## Ограничения и следующие шаги
View File
+18
View File
@@ -0,0 +1,18 @@
import uvicorn
from .config import load_settings
def main() -> None:
settings = load_settings()
uvicorn.run(
'app.main:app',
host=settings.host,
port=settings.port,
log_level=settings.log_level,
timeout_graceful_shutdown=settings.shutdown_timeout,
)
if __name__ == '__main__':
main()
+30
View File
@@ -0,0 +1,30 @@
"""Basic is only for creating a session. Every other route accepts only Bearer."""
import base64
import binascii
from typing import Optional, Tuple
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
def parse_basic(header: str) -> Optional[Tuple[str, str]]:
"""username:password. The password may itself contain a colon."""
scheme, separator, param = header.partition(' ')
if separator != ' ' or scheme.lower() != 'basic' or not param or param != param.strip():
return None
try:
decoded = base64.b64decode(param, validate=True)
text = decoded.decode('utf-8')
except (binascii.Error, UnicodeDecodeError, ValueError):
return None
username, separator, password = text.partition(':')
if separator != ':':
return None
return username, password
def parse_bearer(header: str) -> Optional[str]:
scheme, separator, token = header.partition(' ')
if separator != ' ' or scheme.lower() != 'bearer' or not token or token != token.strip() or ' ' in token:
return None
return token
+49
View File
@@ -0,0 +1,49 @@
import os
import socket
from dataclasses import dataclass
from pathlib import Path
@dataclass(frozen=True)
class Settings:
host: str
port: int
env: str
log_level: str
shutdown_timeout: int
session_ttl_seconds: int
idempotency_ttl_seconds: int
database_url: str
instance_id: str
def _secret(file_var: str, value_var: str) -> str:
"""A *_FILE variable wins. The secret value is not logged."""
file_name = os.getenv(file_var, '').strip()
if file_name:
try:
value = Path(file_name).read_text(encoding='utf-8').strip()
except OSError:
raise RuntimeError('%s could not be read' % file_var) from None
if not value:
raise RuntimeError('%s is empty' % file_var)
return value
value = os.getenv(value_var, '').strip()
if not value:
raise RuntimeError('%s or %s is required' % (value_var, file_var))
return value
def load_settings() -> Settings:
return Settings(
host=os.getenv('APP_HOST', '0.0.0.0'),
port=int(os.getenv('APP_PORT', '8080')),
env=os.getenv('APP_ENV', 'lab'),
log_level=os.getenv('LOG_LEVEL', 'info').lower(),
shutdown_timeout=int(os.getenv('SHUTDOWN_TIMEOUT_SECONDS', '5')),
session_ttl_seconds=int(os.getenv('SESSION_TTL_SECONDS', '3600')),
idempotency_ttl_seconds=int(os.getenv('IDEMPOTENCY_TTL_SECONDS', '86400')),
database_url=_secret('DATABASE_URL_FILE', 'DATABASE_URL'),
# Docker sets the hostname to the short container ID, which stop-one.sh maps back to a container.
instance_id=os.getenv('INSTANCE_ID', '').strip() or socket.gethostname(),
)
+38
View File
@@ -0,0 +1,38 @@
"""Connection pool. The URL is never written to the log."""
import asyncio
import logging
import asyncpg
logger = logging.getLogger('messenger')
_ATTEMPTS = 40
async def open_pool(database_url: str) -> asyncpg.Pool:
delay = 0.5
last_type = 'error'
for attempt in range(1, _ATTEMPTS + 1):
pool = None
try:
pool = await asyncpg.create_pool(
database_url,
min_size=1,
max_size=10,
timeout=5,
command_timeout=5,
server_settings={'statement_timeout': '5000'},
)
async with pool.acquire() as conn:
await conn.execute('SELECT 1')
return pool
except Exception as exc:
last_type = type(exc).__name__
if pool is not None:
await pool.close()
if attempt == _ATTEMPTS:
break
logger.warning('database is not ready (%s), retrying', last_type)
await asyncio.sleep(delay)
delay = min(delay * 1.5, 2)
raise RuntimeError('database did not become reachable (%s)' % last_type)
+29
View File
@@ -0,0 +1,29 @@
class ApiError(Exception):
"""Expected failure: the message never contains internal details."""
def __init__(self, status: int, code: str, message: str, challenge: str | None = None) -> None:
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.challenge = challenge
def invalid_request(message: str) -> ApiError:
return ApiError(400, 'invalid_request', message)
def unauthorized(message: str, challenge: str) -> ApiError:
return ApiError(401, 'unauthorized', message, challenge)
def not_found(message: str) -> ApiError:
return ApiError(404, 'not_found', message)
def conflict(message: str) -> ApiError:
return ApiError(409, 'conflict', message)
def unavailable(message: str) -> ApiError:
return ApiError(503, 'unavailable', message)
+117
View File
@@ -0,0 +1,117 @@
import json
import logging
import re
import uuid
from contextlib import asynccontextmanager
from pathlib import Path
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from starlette.exceptions import HTTPException as StarletteHTTPException
from .config import load_settings
from .db import open_pool
from .errors import ApiError
from .routes import router
from .store import Store
logger = logging.getLogger('messenger')
_SAFE_REQUEST_ID = re.compile(r'^[A-Za-z0-9._:-]{1,128}$')
_CODES = {
400: 'invalid_request',
401: 'unauthorized',
404: 'not_found',
409: 'conflict',
413: 'payload_too_large',
415: 'unsupported_media_type',
429: 'rate_limited',
503: 'unavailable',
}
_MESSAGES = {
'invalid_request': 'Некорректный запрос',
'unauthorized': 'Требуется идентификация',
'not_found': 'Объект не найден',
'internal_error': 'Внутренняя ошибка',
}
_BEARER_CHALLENGE = 'Bearer realm="messenger"'
def _error(status: int, code: str, message: str, request_id: str, challenge: str | None = None) -> JSONResponse:
body = {'error': {'code': code, 'message': message, 'request_id': request_id}}
headers = {'X-Request-Id': request_id}
if challenge:
headers['WWW-Authenticate'] = challenge
return JSONResponse(body, status_code=status, headers=headers)
def _request_id(request: Request) -> str:
return getattr(request.state, 'request_id', None) or str(uuid.uuid4())
_OPENAPI_PATH = Path(__file__).resolve().parents[1] / 'contracts' / 'openapi.json'
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Migrations are not applied here: the one-shot migrate service runs them before replicas start."""
settings = load_settings()
pool = await open_pool(settings.database_url)
try:
app.state.store = Store(pool, settings.session_ttl_seconds, settings.idempotency_ttl_seconds)
logger.info('instance %s started', settings.instance_id)
yield
finally:
await pool.close()
def create_app() -> FastAPI:
app = FastAPI(
title='MTUSI Messenger lab3',
version='1.3.0',
lifespan=lifespan,
)
instance_id = load_settings().instance_id
def openapi():
if app.openapi_schema is None:
app.openapi_schema = json.loads(_OPENAPI_PATH.read_text(encoding='utf-8'))
return app.openapi_schema
app.openapi = openapi
@app.middleware('http')
async def request_context(request: Request, call_next):
"""Gives every response, including unexpected failures, an X-Request-Id and X-Instance-Id."""
incoming = request.headers.get('X-Request-Id')
request_id = incoming if incoming and _SAFE_REQUEST_ID.match(incoming) else str(uuid.uuid4())
request.state.request_id = request_id
try:
response = await call_next(request)
except Exception:
logger.exception('request %s failed: %s %s', request_id, request.method, request.url.path)
response = _error(500, 'internal_error', _MESSAGES['internal_error'], request_id)
response.headers['X-Request-Id'] = request_id
response.headers['X-Instance-Id'] = instance_id
return response
@app.exception_handler(ApiError)
async def handle_api_error(request: Request, exc: ApiError) -> JSONResponse:
return _error(exc.status, exc.code, exc.message, _request_id(request), exc.challenge)
@app.exception_handler(RequestValidationError)
async def handle_validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
return _error(400, 'invalid_request', _MESSAGES['invalid_request'], _request_id(request))
@app.exception_handler(StarletteHTTPException)
async def handle_http_error(request: Request, exc: StarletteHTTPException) -> JSONResponse:
code = _CODES.get(exc.status_code, 'internal_error' if exc.status_code >= 500 else 'invalid_request')
challenge = _BEARER_CHALLENGE if exc.status_code == 401 else None
return _error(exc.status_code, code, _MESSAGES.get(code, _MESSAGES['invalid_request']), _request_id(request), challenge)
app.include_router(router)
return app
app = create_app()
+64
View File
@@ -0,0 +1,64 @@
"""Apply numbered SQL files once. Run as a separate one-shot step: python -m app.migrate."""
import asyncio
import logging
from pathlib import Path
import asyncpg
from .config import load_settings
from .db import open_pool
logger = logging.getLogger('messenger')
_MIGRATIONS = Path(__file__).resolve().parents[1] / 'migrations'
# Any constant shared by all runners; serializes concurrent migrators across processes.
_LOCK_ID = 3_2026_0003
_BOOTSTRAP = """
CREATE TABLE IF NOT EXISTS schema_migrations (
version TEXT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
)
"""
def expected_versions() -> list[str]:
return sorted(path.name for path in _MIGRATIONS.glob('*.sql'))
async def apply_migrations(pool: asyncpg.Pool) -> None:
async with pool.acquire() as conn:
await conn.execute('SET statement_timeout = 0')
await conn.execute('SELECT pg_advisory_lock($1)', _LOCK_ID)
try:
await conn.execute(_BOOTSTRAP)
rows = await conn.fetch('SELECT version FROM schema_migrations')
applied = {row['version'] for row in rows}
for version in expected_versions():
if version in applied:
logger.info('migration %s already applied', version)
continue
sql = (_MIGRATIONS / version).read_text(encoding='utf-8')
async with conn.transaction():
await conn.execute(sql)
await conn.execute(
'INSERT INTO schema_migrations (version) VALUES ($1)',
version,
)
logger.info('applied migration %s', version)
finally:
await conn.execute('SELECT pg_advisory_unlock($1)', _LOCK_ID)
async def _main() -> None:
settings = load_settings()
pool = await open_pool(settings.database_url)
try:
await apply_migrations(pool)
finally:
await pool.close()
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s')
asyncio.run(_main())
+43
View File
@@ -0,0 +1,43 @@
import base64
import json
import re
from typing import Mapping, Optional, Tuple
from .errors import invalid_request
# Position of the last item returned to the client.
Key = Tuple[str, str]
DEFAULT_LIMIT = 50
MAX_LIMIT = 100
_LIMIT = re.compile(r'^\d{1,3}$')
def encode_cursor(scope: str, key: Key) -> str:
"""Scope binds the cursor to one resource, so a foreign cursor is detectable."""
payload = json.dumps({'s': scope, 't': key[0], 'i': key[1]}, separators=(',', ':'))
return base64.urlsafe_b64encode(payload.encode('utf-8')).decode('ascii').rstrip('=')
def decode_cursor(value: str, scope: str) -> Key:
try:
raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4))
payload = json.loads(raw)
created_at, item_id = payload['t'], payload['i']
belongs = payload['s'] == scope
except (ValueError, TypeError, KeyError):
raise invalid_request('Некорректный cursor') from None
if not belongs or not isinstance(created_at, str) or not isinstance(item_id, str):
raise invalid_request('Cursor относится к другому запросу') from None
return created_at, item_id
def parse_page(params: Mapping[str, str], scope: str) -> Tuple[int, Optional[Key]]:
raw_limit = params.get('limit')
limit = DEFAULT_LIMIT
if raw_limit is not None:
if not _LIMIT.match(raw_limit) or not 1 <= int(raw_limit) <= MAX_LIMIT:
raise invalid_request('limit должен быть целым числом от 1 до %d' % MAX_LIMIT)
limit = int(raw_limit)
raw_cursor = params.get('cursor')
return limit, decode_cursor(raw_cursor, scope) if raw_cursor is not None else None
+27
View File
@@ -0,0 +1,27 @@
"""Argon2id password hashes. The hash never leaves this module in an API response."""
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError
_hasher = PasswordHasher()
# Same cost as a real hash, so a missing user is not faster than a wrong password.
_DUMMY_HASH = _hasher.hash('not-a-user-password')
_DUMMY_PASSWORD = 'not-a-user-password'
def hash_password(password: str) -> str:
return _hasher.hash(password)
def verify_password(password_hash: str, password: str) -> bool:
try:
return _hasher.verify(password_hash, password)
except VerificationError:
return False
def password_matches(stored_hash: str | None, password: str) -> bool:
"""One Argon2 check either way. An out-of-range secret is never the stored password."""
if stored_hash is None or not 12 <= len(password) <= 128 or not password.strip():
verify_password(_DUMMY_HASH, _DUMMY_PASSWORD)
return False
return verify_password(stored_hash, password)
+193
View File
@@ -0,0 +1,193 @@
import hashlib
import json
import re
import uuid
from typing import Optional, Sequence, Type, TypeVar
from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel, ValidationError
from starlette.responses import Response
from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, parse_basic, parse_bearer
from .errors import invalid_request, not_found, unauthorized
from .pagination import encode_cursor, parse_page
from .schemas import CreateChat, CreateMessage, CreateUser
from .store import Chat, Store, User
router = APIRouter()
Body = TypeVar('Body', bound=BaseModel)
_IDEMPOTENCY_KEY = re.compile(r'[A-Za-z0-9._:-]{1,128}')
async def get_store(request: Request) -> Store:
return request.app.state.store
async def current_user(request: Request) -> User:
"""Bearer only. X-User-Id is ignored and does not identify anyone."""
token = parse_bearer(request.headers.get('Authorization', ''))
if token is None:
raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE)
user = await request.app.state.store.user_for_token(token)
if user is None:
raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE)
return user
async def read_body(request: Request, model: Type[Body]) -> Body:
try:
payload = json.loads(await request.body())
except (UnicodeDecodeError, ValueError):
raise invalid_request('Тело запроса не является корректным JSON') from None
if not isinstance(payload, dict):
raise invalid_request('Тело запроса должно быть JSON-объектом')
try:
return model.model_validate(payload)
except ValidationError:
raise invalid_request('Тело запроса не соответствует контракту') from None
def _request_hash(payload: BaseModel) -> str:
"""Key order in the client JSON does not matter: the validated body is re-serialized canonically."""
canonical = json.dumps(payload.model_dump(), sort_keys=True, separators=(',', ':'), ensure_ascii=False)
return hashlib.sha256(canonical.encode('utf-8')).hexdigest()
async def _member_chat(store: Store, chat_id: str, actor: User) -> Chat:
try:
canonical = str(uuid.UUID(chat_id))
except ValueError:
raise invalid_request('Некорректный chat_id') from None
chat = await store.find_chat(canonical, actor.id)
if chat is None:
raise not_found('Чат не найден')
return chat
def _page(items: Sequence, has_more: bool, scope: str) -> dict:
cursor: Optional[str] = None
if has_more and items:
cursor = encode_cursor(scope, (items[-1].created_at, items[-1].id))
return {'items': [item.to_dict() for item in items], 'next_cursor': cursor}
@router.get('/health/live')
async def liveness() -> JSONResponse:
return JSONResponse({'status': 'ok'})
@router.get('/health/ready')
async def readiness(store: Store = Depends(get_store)) -> JSONResponse:
await store.ready()
return JSONResponse({'status': 'ok'})
@router.post('/api/v1/users')
async def register_user(request: Request, store: Store = Depends(get_store)) -> JSONResponse:
payload = await read_body(request, CreateUser)
user = await store.create_user(payload.username, payload.display_name, payload.password)
return JSONResponse(user.to_dict(), status_code=201)
@router.post('/api/v1/auth/sessions')
async def create_session(request: Request, store: Store = Depends(get_store)) -> JSONResponse:
parsed = parse_basic(request.headers.get('Authorization', ''))
if parsed is None:
raise unauthorized('Неверные учётные данные', BASIC_CHALLENGE)
user = await store.authenticate(*parsed)
if user is None:
raise unauthorized('Неверные учётные данные', BASIC_CHALLENGE)
token, expires_at = await store.create_session(user.id)
body = {
'token': token,
'token_type': 'Bearer',
'expires_at': expires_at,
'user': user.to_dict(),
}
return JSONResponse(body, status_code=201)
@router.delete('/api/v1/auth/sessions/current')
async def revoke_session(request: Request, actor: User = Depends(current_user)) -> Response:
token = parse_bearer(request.headers.get('Authorization', ''))
if token is None or not await request.app.state.store.revoke_session(token):
raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE)
return Response(status_code=204)
@router.get('/api/v1/users/me')
async def show_current_user(actor: User = Depends(current_user)) -> JSONResponse:
return JSONResponse(actor.to_dict())
@router.post('/api/v1/chats')
async def create_chat(
request: Request,
actor: User = Depends(current_user),
store: Store = Depends(get_store),
) -> JSONResponse:
payload = await read_body(request, CreateChat)
if actor.id not in payload.member_ids:
raise invalid_request('Создатель чата должен быть указан в member_ids')
chat = await store.create_chat(payload.title, payload.member_ids)
return JSONResponse(chat.to_dict(), status_code=201)
@router.get('/api/v1/chats')
async def list_chats(
request: Request,
actor: User = Depends(current_user),
store: Store = Depends(get_store),
) -> JSONResponse:
scope = 'chats:' + actor.id
limit, after = parse_page(request.query_params, scope)
chats, has_more = await store.list_chats(actor.id, limit, after)
return JSONResponse(_page(chats, has_more, scope))
@router.get('/api/v1/chats/{chat_id}')
async def show_chat(
chat_id: str,
actor: User = Depends(current_user),
store: Store = Depends(get_store),
) -> JSONResponse:
return JSONResponse((await _member_chat(store, chat_id, actor)).to_dict())
@router.post('/api/v1/chats/{chat_id}/messages')
async def send_message(
chat_id: str,
request: Request,
actor: User = Depends(current_user),
store: Store = Depends(get_store),
) -> JSONResponse:
chat = await _member_chat(store, chat_id, actor)
payload = await read_body(request, CreateMessage)
key = request.headers.get('Idempotency-Key')
if key is None or not _IDEMPOTENCY_KEY.fullmatch(key):
raise invalid_request('Требуется корректный Idempotency-Key')
message = await store.add_message(
chat.id,
actor.id,
payload.text,
path='/api/v1/chats/%s/messages' % chat.id,
key=key,
request_hash=_request_hash(payload),
)
return JSONResponse(message.to_dict(), status_code=201)
@router.get('/api/v1/chats/{chat_id}/messages')
async def list_messages(
chat_id: str,
request: Request,
actor: User = Depends(current_user),
store: Store = Depends(get_store),
) -> JSONResponse:
chat = await _member_chat(store, chat_id, actor)
scope = 'messages:' + chat.id
limit, after = parse_page(request.query_params, scope)
messages, has_more = await store.list_messages(chat.id, limit, after)
return JSONResponse(_page(messages, has_more, scope))
+55
View File
@@ -0,0 +1,55 @@
"""Request bodies. The contract forbids unknown fields, wrong types and blank strings."""
import uuid
from typing import List
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator
from typing_extensions import Annotated
def _not_blank(value: str) -> str:
if not value.strip():
raise ValueError('value must not be whitespace only')
return value
def _canonical_uuid(value: str) -> str:
parsed = uuid.UUID(value)
if str(parsed) != value.lower():
raise ValueError('value must be a canonical UUID')
return str(parsed)
Username = Annotated[str, Field(pattern=r'^[a-z0-9_]{3,32}$')]
DisplayName = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(_not_blank)]
Title = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(_not_blank)]
Text = Annotated[str, Field(min_length=1, max_length=4000), AfterValidator(_not_blank)]
Password = Annotated[str, Field(min_length=12, max_length=128), AfterValidator(_not_blank)]
UserId = Annotated[str, AfterValidator(_canonical_uuid)]
class CreateUser(BaseModel):
model_config = ConfigDict(extra='forbid')
username: Username
display_name: DisplayName
password: Password
class CreateChat(BaseModel):
model_config = ConfigDict(extra='forbid')
title: Title
member_ids: Annotated[List[UserId], Field(min_length=1, max_length=100)]
@field_validator('member_ids')
@classmethod
def _unique(cls, value: List[str]) -> List[str]:
if len(set(value)) != len(value):
raise ValueError('member_ids must be unique')
return value
class CreateMessage(BaseModel):
model_config = ConfigDict(extra='forbid')
text: Text
+467
View File
@@ -0,0 +1,467 @@
import hashlib
import json
import secrets
import threading
import uuid
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Dict, List, Optional, Sequence, Tuple
import asyncpg
from .errors import conflict, invalid_request, not_found, unavailable
from .migrate import expected_versions
from .pagination import Key
from .passwords import hash_password, password_matches
@dataclass(frozen=True)
class User:
id: str
username: str
display_name: str
created_at: str
def to_dict(self) -> dict:
return {
'id': self.id,
'username': self.username,
'display_name': self.display_name,
'created_at': self.created_at,
}
@dataclass(frozen=True)
class Chat:
id: str
title: str
member_ids: Tuple[str, ...]
created_at: str
def to_dict(self) -> dict:
return {
'id': self.id,
'title': self.title,
'member_ids': list(self.member_ids),
'created_at': self.created_at,
}
@dataclass(frozen=True)
class Message:
id: str
chat_id: str
sender_id: str
text: str
created_at: str
def to_dict(self) -> dict:
return {
'id': self.id,
'chat_id': self.chat_id,
'sender_id': self.sender_id,
'text': self.text,
'created_at': self.created_at,
}
def _token_key(token: str) -> str:
return hashlib.sha256(token.encode('utf-8')).hexdigest()
def _stamp(moment: datetime) -> str:
return moment.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def _parse_stamp(value: str) -> datetime:
if value.endswith('Z'):
value = value[:-1] + '+00:00'
return datetime.fromisoformat(value)
def _user(row: asyncpg.Record) -> User:
return User(str(row['id']), row['username'], row['display_name'], _stamp(row['created_at']))
def _as_uuid(value: str, message: str) -> uuid.UUID:
try:
return uuid.UUID(value)
except ValueError:
raise invalid_request(message) from None
def _db_down(exc: BaseException) -> bool:
return isinstance(exc, (
asyncpg.PostgresConnectionError,
asyncpg.InterfaceError,
asyncpg.CannotConnectNowError,
OSError,
TimeoutError,
))
class Store:
"""PostgreSQL is the only state: users, chats, sessions and idempotency results are shared by all replicas."""
def __init__(self, pool: asyncpg.Pool, session_ttl_seconds: int = 3600, idempotency_ttl_seconds: int = 86400) -> None:
self._pool = pool
self._session_ttl = timedelta(seconds=session_ttl_seconds)
self._idempotency_ttl = timedelta(seconds=idempotency_ttl_seconds)
self._lock = threading.Lock()
self._last_moment: Optional[datetime] = None
self._migrations = expected_versions()
def _now(self) -> datetime:
"""Strictly increasing within one process; across replicas order relies on the shared host clock."""
with self._lock:
moment = datetime.now(timezone.utc)
if self._last_moment is not None and moment <= self._last_moment:
moment = self._last_moment + timedelta(microseconds=1)
self._last_moment = moment
return moment
async def ready(self) -> None:
"""Ready only when PostgreSQL answers and the separate migrator has applied this build's schema."""
try:
async with self._pool.acquire() as conn:
applied = await conn.fetchval(
'SELECT count(*) FROM schema_migrations WHERE version = ANY($1::text[])',
self._migrations,
)
except Exception:
raise unavailable('Хранилище недоступно') from None
if applied != len(self._migrations):
raise unavailable('Схема БД ещё не применена')
async def create_user(self, username: str, display_name: str, password: str) -> User:
password_hash = hash_password(password)
user_id = uuid.uuid4()
created = self._now()
try:
async with self._pool.acquire() as conn:
await conn.execute(
'''
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES ($1, $2, $3, $4, $5)
''',
user_id,
username,
display_name,
password_hash,
created,
)
except asyncpg.UniqueViolationError:
raise conflict('Username уже занят') from None
return User(str(user_id), username, display_name, _stamp(created))
async def authenticate(self, username: str, password: str) -> Optional[User]:
try:
async with self._pool.acquire() as conn:
row = await conn.fetchrow(
'''
SELECT id, username, display_name, created_at, password_hash
FROM users
WHERE username = $1
''',
username,
)
except Exception as exc:
if _db_down(exc):
raise unavailable('Хранилище недоступно') from None
raise
stored_hash = row['password_hash'] if row is not None else None
if not password_matches(stored_hash, password) or row is None:
return None
return _user(row)
async def create_session(self, user_id: str) -> Tuple[str, str]:
token = secrets.token_urlsafe(32)
expires = datetime.now(timezone.utc) + self._session_ttl
try:
async with self._pool.acquire() as conn:
await conn.execute(
'INSERT INTO sessions (token_hash, user_id, expires_at) VALUES ($1, $2, $3)',
_token_key(token),
_as_uuid(user_id, 'Некорректный user_id'),
expires,
)
except Exception as exc:
if _db_down(exc):
raise unavailable('Хранилище недоступно') from None
raise
return token, _stamp(expires)
async def user_for_token(self, token: str) -> Optional[User]:
now = datetime.now(timezone.utc)
key = _token_key(token)
try:
async with self._pool.acquire() as conn:
row = await conn.fetchrow(
'''
SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at
FROM sessions s
JOIN users u ON u.id = s.user_id
WHERE s.token_hash = $1
''',
key,
)
if row is None:
return None
if row['expires_at'] <= now:
await conn.execute('DELETE FROM sessions WHERE token_hash = $1', key)
return None
return _user(row)
except Exception as exc:
if _db_down(exc):
raise unavailable('Хранилище недоступно') from None
raise
async def revoke_session(self, token: str) -> bool:
try:
async with self._pool.acquire() as conn:
row = await conn.fetchrow(
'''
DELETE FROM sessions
WHERE token_hash = $1 AND expires_at > $2
RETURNING token_hash
''',
_token_key(token),
datetime.now(timezone.utc),
)
except Exception as exc:
if _db_down(exc):
raise unavailable('Хранилище недоступно') from None
raise
return row is not None
async def create_chat(self, title: str, member_ids: Sequence[str]) -> Chat:
ids = [_as_uuid(member_id, 'Некорректный user_id') for member_id in member_ids]
chat_id = uuid.uuid4()
created = self._now()
async with self._pool.acquire() as conn:
async with conn.transaction():
found = await conn.fetch('SELECT id FROM users WHERE id = ANY($1::uuid[])', ids)
if len(found) != len(ids):
raise invalid_request('Все участники должны быть существующими пользователями')
await conn.execute(
'INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)',
chat_id,
title,
created,
)
await conn.execute(
'''
INSERT INTO chat_members (chat_id, user_id, position)
SELECT $1, member_id, (ordinality - 1)::integer
FROM unnest($2::uuid[]) WITH ORDINALITY AS members(member_id, ordinality)
''',
chat_id,
ids,
)
return Chat(str(chat_id), title, tuple(member_ids), _stamp(created))
async def find_chat(self, chat_id: str, member_id: str) -> Optional[Chat]:
"""A chat of somebody else is indistinguishable from a missing one."""
chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id')
member_uuid = _as_uuid(member_id, 'Некорректный user_id')
async with self._pool.acquire() as conn:
row = await conn.fetchrow(
'''
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $2
WHERE c.id = $1
''',
chat_uuid,
member_uuid,
)
if row is None:
return None
members = await self._members(conn, [chat_uuid])
return Chat(str(row['id']), row['title'], members.get(chat_uuid, ()), _stamp(row['created_at']))
async def list_chats(self, member_id: str, limit: int, after: Optional[Key]) -> Tuple[List[Chat], bool]:
member_uuid = _as_uuid(member_id, 'Некорректный user_id')
after_at, after_id = self._cursor(after)
async with self._pool.acquire() as conn:
if after_at is None:
rows = await conn.fetch(
'''
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1
ORDER BY c.created_at, c.id
LIMIT $2
''',
member_uuid,
limit + 1,
)
else:
rows = await conn.fetch(
'''
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1
WHERE (c.created_at, c.id) > ($2::timestamptz, $3::uuid)
ORDER BY c.created_at, c.id
LIMIT $4
''',
member_uuid,
after_at,
after_id,
limit + 1,
)
page = rows[:limit]
members = await self._members(conn, [row['id'] for row in page])
chats = [
Chat(str(row['id']), row['title'], members.get(row['id'], ()), _stamp(row['created_at']))
for row in page
]
return chats, len(rows) > limit
async def add_message(
self,
chat_id: str,
sender_id: str,
text: str,
*,
path: str,
key: str,
request_hash: str,
) -> Message:
"""Exactly one message per (sender, POST, path, key), whichever replica receives the request.
Concurrent duplicates block on the primary key of idempotency_keys until the first
transaction commits or rolls back, so a rolled-back attempt leaves the key free.
"""
chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id')
sender_uuid = _as_uuid(sender_id, 'Некорректный user_id')
message = Message(str(uuid.uuid4()), str(chat_uuid), sender_id, text, _stamp(self._now()))
cutoff = datetime.now(timezone.utc) - self._idempotency_ttl
async with self._pool.acquire() as conn:
async with conn.transaction():
await conn.execute(
'''
DELETE FROM idempotency_keys
WHERE user_id = $1 AND method = 'POST' AND path = $2 AND key = $3 AND created_at < $4
''',
sender_uuid,
path,
key,
cutoff,
)
claimed = await conn.fetchval(
'''
INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response)
VALUES ($1, 'POST', $2, $3, $4, 201, $5::jsonb)
ON CONFLICT (user_id, method, path, key) DO NOTHING
RETURNING 1
''',
sender_uuid,
path,
key,
request_hash,
json.dumps(message.to_dict()),
)
if claimed is None:
return await self._replay(conn, sender_uuid, path, key, request_hash)
await conn.execute(
'''
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
VALUES ($1, $2, $3, $4, $5)
''',
uuid.UUID(message.id),
chat_uuid,
sender_uuid,
text,
_parse_stamp(message.created_at),
)
return message
async def _replay(
self,
conn: asyncpg.Connection,
sender_uuid: uuid.UUID,
path: str,
key: str,
request_hash: str,
) -> Message:
row = await conn.fetchrow(
'''
SELECT request_hash, response
FROM idempotency_keys
WHERE user_id = $1 AND method = 'POST' AND path = $2 AND key = $3
''',
sender_uuid,
path,
key,
)
if row is None:
raise unavailable('Повторите запрос с тем же Idempotency-Key')
if row['request_hash'] != request_hash:
raise conflict('Idempotency-Key уже использован с другим сообщением')
data = json.loads(row['response'])
return Message(data['id'], data['chat_id'], data['sender_id'], data['text'], data['created_at'])
async def list_messages(self, chat_id: str, limit: int, after: Optional[Key]) -> Tuple[List[Message], bool]:
chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id')
after_at, after_id = self._cursor(after)
async with self._pool.acquire() as conn:
if after_at is None:
rows = await conn.fetch(
'''
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1
ORDER BY created_at, id
LIMIT $2
''',
chat_uuid,
limit + 1,
)
else:
rows = await conn.fetch(
'''
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1
AND (created_at, id) > ($2::timestamptz, $3::uuid)
ORDER BY created_at, id
LIMIT $4
''',
chat_uuid,
after_at,
after_id,
limit + 1,
)
page = rows[:limit]
messages = [
Message(str(row['id']), str(row['chat_id']), str(row['sender_id']), row['text'], _stamp(row['created_at']))
for row in page
]
return messages, len(rows) > limit
async def _members(self, conn: asyncpg.Connection, chat_ids: Sequence[uuid.UUID]) -> Dict[uuid.UUID, Tuple[str, ...]]:
if not chat_ids:
return {}
rows = await conn.fetch(
'''
SELECT chat_id, user_id
FROM chat_members
WHERE chat_id = ANY($1::uuid[])
ORDER BY chat_id, position
''',
list(chat_ids),
)
grouped: Dict[uuid.UUID, List[str]] = {}
for row in rows:
grouped.setdefault(row['chat_id'], []).append(str(row['user_id']))
return {chat_id: tuple(member_ids) for chat_id, member_ids in grouped.items()}
def _cursor(self, after: Optional[Key]) -> Tuple[Optional[datetime], Optional[uuid.UUID]]:
if after is None:
return None, None
try:
return _parse_stamp(after[0]), uuid.UUID(after[1])
except ValueError:
raise invalid_request('Некорректный cursor') from None
+96
View File
@@ -0,0 +1,96 @@
# Client -> lb (nginx) -> api x API_REPLICAS -> postgres.
# Secrets come from .env, which is not committed. Only the LB publishes a port.
name: messenger-lab3
x-app-env: &app-env
APP_ENV: ${APP_ENV:-lab}
LOG_LEVEL: ${LOG_LEVEL:-info}
DATABASE_URL: postgresql://${POSTGRES_USER:?set in .env}:${POSTGRES_PASSWORD:?set in .env}@postgres:5432/${POSTGRES_DB:?set in .env}
services:
postgres:
image: postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB}
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""]
interval: 2s
timeout: 5s
retries: 30
restart: unless-stopped
# One-shot schema step. Replicas never migrate on startup, so there is no race between them;
# an advisory lock additionally serializes accidental parallel runs.
migrate:
image: messenger-lab3-app:local
build:
context: .
dockerfile: Dockerfile
command: ["python", "-m", "app.migrate"]
environment:
<<: *app-env
depends_on:
postgres:
condition: service_healthy
restart: "no"
api:
image: messenger-lab3-app:local
build:
context: .
dockerfile: Dockerfile
deploy:
replicas: ${API_REPLICAS:-2}
environment:
<<: *app-env
APP_HOST: 0.0.0.0
APP_PORT: "8080"
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5}
expose:
- "8080"
depends_on:
postgres:
condition: service_healthy
migrate:
condition: service_completed_successfully
healthcheck:
test:
- CMD
- python
- -c
- import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)
interval: 3s
timeout: 3s
retries: 3
start_period: 20s
# SIGTERM -> uvicorn stops accepting, finishes in-flight requests within SHUTDOWN_TIMEOUT_SECONDS.
stop_signal: SIGTERM
stop_grace_period: 8s
restart: unless-stopped
lb:
image: nginx:1.28.0-alpine@sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284
# Loopback only. Both families: on Windows "localhost" tries ::1 first and waits ~2 s if nothing listens.
ports:
- "127.0.0.1:${LB_PORT:-8080}:8080"
- "[::1]:${LB_PORT:-8080}:8080"
volumes:
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/lb/health"]
interval: 5s
timeout: 3s
retries: 3
restart: unless-stopped
volumes:
pgdata:
+34
View File
@@ -0,0 +1,34 @@
# failover run 2026-11-05T16:51:38.6795653+03:00
## run 1 — before
messenger-lab3-api-1 5dce519fa2a3 Up 22 seconds (healthy)
messenger-lab3-api-2 d0fe578a76ab Up 22 seconds (healthy)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 1.598}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
## run 1 — after
messenger-lab3-api-1 5dce519fa2a3 Exited (0) 13 seconds ago
messenger-lab3-api-2 d0fe578a76ab Up 39 seconds (healthy)
## run 2 — before
messenger-lab3-api-1 5dce519fa2a3 Up 6 seconds (healthy)
messenger-lab3-api-2 d0fe578a76ab Up 48 seconds (healthy)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 1.575}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
## run 2 — after
messenger-lab3-api-1 5dce519fa2a3 Exited (0) 13 seconds ago
messenger-lab3-api-2 d0fe578a76ab Up About a minute (healthy)
## restored
messenger-lab3-api-1 5dce519fa2a3 Up 6 seconds (healthy)
messenger-lab3-api-2 d0fe578a76ab Up About a minute (healthy)
## run 3 (target api-2) — before
messenger-lab3-api-1 5dce519fa2a3 Up 17 seconds (healthy)
messenger-lab3-api-2 d0fe578a76ab Up About a minute (healthy)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 1.707}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
## run 3 — after
messenger-lab3-api-1 5dce519fa2a3 Up 34 seconds (healthy)
messenger-lab3-api-2 d0fe578a76ab Exited (0) 13 seconds ago
## restored
messenger-lab3-api-1 5dce519fa2a3 Up 44 seconds (healthy)
messenger-lab3-api-2 d0fe578a76ab Up 6 seconds (healthy)
Binary file not shown.
+12
View File
@@ -0,0 +1,12 @@
# before
messenger-lab3-api-1 5dce519fa2a3 Up 52 seconds (healthy)
messenger-lab3-api-2 d0fe578a76ab Up 14 seconds (healthy)
messenger-lab3-lb-1 ece2ac882f2d Up About a minute (healthy)
messenger-lab3-postgres-1 0ed6d04de263 Up 7 minutes (healthy)
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
# after
messenger-lab3-api-1 5dce519fa2a3 Up 12 seconds (healthy)
messenger-lab3-api-2 d0fe578a76ab Up 12 seconds (healthy)
messenger-lab3-lb-1 ece2ac882f2d Up 5 seconds (healthy)
messenger-lab3-postgres-1 0ed6d04de263 Up 17 seconds (healthy)
Binary file not shown.
+66
View File
@@ -0,0 +1,66 @@
worker_processes auto;
pid /tmp/nginx.pid;
events {
worker_connections 1024;
}
http {
access_log /dev/stdout;
error_log /dev/stderr warn;
# Docker embedded DNS. "api" resolves to every running replica of the Compose service;
# with "resolve" below nginx re-reads the set every 2 s, so stopped/scaled replicas leave and join the pool.
resolver 127.0.0.11 valid=2s ipv6=off;
resolver_timeout 2s;
log_format upstream '$remote_addr "$request" $status upstream=$upstream_addr '
'upstream_status=$upstream_status rt=$request_time';
upstream api {
zone api_upstream 64k;
# Round robin, no ip_hash/sticky: any replica serves any session from PostgreSQL.
server api:8080 resolve max_fails=1 fail_timeout=5s;
keepalive 32;
# Shorter than uvicorn's keep-alive (5 s), so nginx never reuses a connection the API is closing.
keepalive_timeout 4s;
}
server {
listen 8080;
access_log /dev/stdout upstream;
client_max_body_size 1m;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 1s;
proxy_send_timeout 10s;
proxy_read_timeout 10s;
# Not-yet-sent requests and idempotent methods go to the next replica on a dead backend.
# Non-idempotent POSTs (users, chats, sessions) are never replayed by the LB.
proxy_next_upstream error timeout http_502 http_504;
proxy_next_upstream_tries 3;
proxy_next_upstream_timeout 5s;
location = /lb/health {
access_log off;
default_type application/json;
return 200 '{"status":"ok"}';
}
# Message POST carries a mandatory Idempotency-Key, so replaying it on another replica is safe.
location ~ ^/api/v1/chats/[^/]+/messages$ {
proxy_next_upstream error timeout http_502 http_504 non_idempotent;
proxy_pass http://api;
}
location / {
proxy_pass http://api;
}
}
}
+53
View File
@@ -0,0 +1,53 @@
-- Initial schema. The runner records this file in schema_migrations and skips it on the next start.
CREATE TABLE IF NOT EXISTS schema_migrations (
version TEXT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE users (
id UUID PRIMARY KEY,
username TEXT NOT NULL,
display_name TEXT NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
CONSTRAINT users_username_unique UNIQUE (username),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
);
CREATE TABLE sessions (
token_hash TEXT PRIMARY KEY,
user_id UUID NOT NULL REFERENCES users (id),
expires_at TIMESTAMPTZ NOT NULL
);
CREATE TABLE chats (
id UUID PRIMARY KEY,
title TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
);
CREATE TABLE chat_members (
chat_id UUID NOT NULL REFERENCES chats (id),
user_id UUID NOT NULL REFERENCES users (id),
position INTEGER NOT NULL,
PRIMARY KEY (chat_id, user_id),
CONSTRAINT chat_members_position_nonnegative CHECK (position >= 0)
);
CREATE TABLE messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats (id),
sender_id UUID NOT NULL REFERENCES users (id),
text TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL,
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000)
);
-- Keyset history: WHERE chat_id = ? AND (created_at, id) > cursor ORDER BY created_at, id.
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
-- Chat list is filtered by the caller, then ordered by the chat's own (created_at, id).
CREATE INDEX chat_members_user_idx ON chat_members (user_id);
+16
View File
@@ -0,0 +1,16 @@
-- Result of POST /chats/{id}/messages per (user, method, path, Idempotency-Key).
-- The row and the message are written in one transaction, so a failed attempt leaves no key behind.
CREATE TABLE idempotency_keys (
user_id UUID NOT NULL REFERENCES users (id),
method TEXT NOT NULL,
path TEXT NOT NULL,
key TEXT NOT NULL,
request_hash TEXT NOT NULL,
status_code INTEGER NOT NULL,
response JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (user_id, method, path, key),
CONSTRAINT idempotency_keys_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$')
);
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
+19
View File
@@ -0,0 +1,19 @@
# Direct: fastapi, pydantic, uvicorn, argon2-cffi. The rest are their dependencies, pinned.
asyncpg==0.32.0
annotated-doc==0.0.5
annotated-types==0.8.0
anyio==4.15.1
argon2-cffi==25.1.0
argon2-cffi-bindings==25.1.0
cffi==2.0.0
click==8.5.0
fastapi==0.141.1
h11==0.16.0
idna==3.20
pycparser==3.0
pydantic==2.13.5
pydantic_core==2.46.5
starlette==1.7.0
typing-inspection==0.4.4
typing_extensions==4.16.0
uvicorn==0.53.0
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""Same Idempotency-Key sent concurrently through the LB must hit several replicas and create one message."""
import argparse
import json
import sys
import threading
import uuid
from collections import Counter
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, Failure, check, run # noqa: E402
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--base-url', default=None)
p.add_argument('--parallel', type=int, default=20)
p.add_argument('--rounds', type=int, default=5)
args = p.parse_args()
c = Client(args.base_url)
alice, bob, _, chat = c.bootstrap()
path = '/api/v1/chats/' + chat['id'] + '/messages'
instances = Counter()
for round_no in range(args.rounds):
key = 'race-' + uuid.uuid4().hex
text = 'Конкурентный повтор %d %s' % (round_no, key)
barrier = threading.Barrier(args.parallel)
results, errors = [], []
def send():
barrier.wait()
try:
r = c.call('POST', path, 201, actor=alice, body={'text': text}, headers={'Idempotency-Key': key}, schema='Message')
results.append((r['headers']['X-Instance-Id'], r['json']))
except Failure as exc:
errors.append(str(exc))
threads = [threading.Thread(target=send) for _ in range(args.parallel)]
for t in threads:
t.start()
for t in threads:
t.join()
check(not errors, 'concurrent POST failed: ' + '; '.join(errors[:3]))
bodies = {json.dumps(body, sort_keys=True) for _, body in results}
check(len(bodies) == 1, 'replicas returned different messages for one key')
served = Counter(instance for instance, _ in results)
instances.update(served)
print('round %d: %d responses, 1 message, replicas %s' % (round_no, len(results), dict(served)))
c.call('POST', path, 409, actor=alice, body={'text': text + ' другой'}, headers={'Idempotency-Key': key})
page = c.call('GET', path + '?limit=100', actor=bob, schema='MessagePage')['json']
texts = [m['text'] for m in page['items']]
check(len(texts) == args.rounds and len(set(texts)) == args.rounds, 'expected exactly one message per key')
check(len(instances) >= 2, 'all requests hit one replica; LB does not balance')
print('PASS: %d keys x %d parallel requests -> %d messages via %d replicas %s'
% (args.rounds, args.parallel, len(texts), len(instances), dict(instances)))
if __name__ == '__main__':
run(main)
+3
View File
@@ -0,0 +1,3 @@
@echo off
rem Windows entry point for tests/resilience.py: runs restart.sh with Git Bash.
"%ProgramFiles%\Git\bin\sh.exe" "%~dp0restart.sh"
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Restart the LB, every API replica and PostgreSQL. The pgdata volume stays; nothing is re-seeded.
docker compose --env-file .env stop lb api
docker compose --env-file .env restart postgres
# Brings back all replicas (including one left stopped by stop-one.sh); migrate re-runs as a no-op.
docker compose --env-file .env up -d --wait
+3
View File
@@ -0,0 +1,3 @@
@echo off
rem Windows entry point for tests/resilience.py: runs stop-one.sh with Git Bash.
"%ProgramFiles%\Git\bin\sh.exe" "%~dp0stop-one.sh"
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Stop exactly ONE API replica and leave it stopped. DB, LB and volumes are untouched.
# X-Instance-Id is the container hostname, i.e. the short Docker container ID.
# Restore afterwards with: docker compose --env-file .env up -d --wait
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
*[!0-9a-f]*) valid=no ;;
*) valid=yes ;;
esac
if [ "$valid" = no ] || [ "${#TARGET_INSTANCE_ID}" -lt 12 ]; then
echo "TARGET_INSTANCE_ID is not a container ID" >&2
exit 2
fi
project="${COMPOSE_PROJECT_NAME:-messenger-lab3}"
matches=$(docker ps -q --no-trunc \
--filter "label=com.docker.compose.project=$project" \
--filter "label=com.docker.compose.service=api" | grep "^$TARGET_INSTANCE_ID" || true)
count=$(printf '%s' "$matches" | grep -c . || true)
if [ "$count" -ne 1 ]; then
echo "expected one running api container for $TARGET_INSTANCE_ID, found $count" >&2
exit 1
fi
name=$(docker inspect --format '{{.Name}}' "$matches")
# Graceful: SIGTERM, then SIGKILL after 8 s; returns before the test's 10 s limit.
docker stop --time 8 "$matches" >/dev/null
echo "stopped api replica ${name#/} ($TARGET_INSTANCE_ID)"