From 8cd25e448f1b8f2850ee687f71a444d8e3f52db5 Mon Sep 17 00:00:00 2001 From: Sunrise443 Date: Thu, 5 Nov 2026 17:03:42 +0300 Subject: [PATCH 1/4] before level 3 initial commit --- .env.example | 6 + .gitattributes | 3 + Dockerfile | 35 +++ Makefile | 5 +- REPORT.md | 41 ++- app/__init__.py | 0 app/__main__.py | 18 ++ app/auth.py | 30 +++ app/config.py | 49 ++++ app/db.py | 38 +++ app/errors.py | 29 ++ app/main.py | 117 +++++++++ app/migrate.py | 64 +++++ app/pagination.py | 43 +++ app/passwords.py | 27 ++ app/routes.py | 193 ++++++++++++++ app/schemas.py | 55 ++++ app/store.py | 467 +++++++++++++++++++++++++++++++++ compose.yaml | 96 +++++++ evidence/failover.txt | 34 +++ evidence/idempotency.txt | Bin 0 -> 1066 bytes evidence/persistence.txt | 12 + evidence/smoke.txt | Bin 0 -> 174 bytes infra/nginx/nginx.conf | 66 +++++ migrations/001_init.sql | 53 ++++ migrations/002_idempotency.sql | 16 ++ requirements.txt | 19 ++ scripts/check_idempotency.py | 62 +++++ scripts/restart.cmd | 3 + scripts/restart.sh | 8 + scripts/stop-one.cmd | 3 + scripts/stop-one.sh | 28 ++ 32 files changed, 1616 insertions(+), 4 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/__main__.py create mode 100644 app/auth.py create mode 100644 app/config.py create mode 100644 app/db.py create mode 100644 app/errors.py create mode 100644 app/main.py create mode 100644 app/migrate.py create mode 100644 app/pagination.py create mode 100644 app/passwords.py create mode 100644 app/routes.py create mode 100644 app/schemas.py create mode 100644 app/store.py create mode 100644 compose.yaml create mode 100644 evidence/failover.txt create mode 100644 evidence/idempotency.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/smoke.txt create mode 100644 infra/nginx/nginx.conf create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 requirements.txt create mode 100644 scripts/check_idempotency.py create mode 100644 scripts/restart.cmd create mode 100755 scripts/restart.sh create mode 100644 scripts/stop-one.cmd create mode 100755 scripts/stop-one.sh diff --git a/.env.example b/.env.example index d32b9be0..9920b3df 100644 --- a/.env.example +++ b/.env.example @@ -11,3 +11,9 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 +# compose.yaml builds DATABASE_URL from these; pick your own local password. +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-me-locally +POSTGRES_DB=messenger +SHUTDOWN_TIMEOUT_SECONDS=5 +LB_PORT=8080 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..856422ac --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.sh text eol=lf +*.conf text eol=lf +*.cmd text eol=crlf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e7d13f91 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +# Закреплённая среда выполнения. Тег называет версию, digest — то, что Docker реально скачает. +FROM python:3.14.8-slim-bookworm@sha256:c8137f4c460908c8763f281c8f22c431eb5c538514ba9553fc3a89c06b7cfb88 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +# Значения по умолчанию, секретов нет. `docker run --env-file .env` перекрывает их при запуске. +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + LOG_LEVEL=info \ + SHUTDOWN_TIMEOUT_SECONDS=5 \ + SESSION_TTL_SECONDS=3600 + +RUN groupadd --gid 10001 app \ + && useradd --uid 10001 --gid app --create-home --shell /usr/sbin/nologin app + +WORKDIR /app + +COPY requirements.txt . +RUN python -m pip install --no-cache-dir --disable-pip-version-check --root-user-action=ignore -r requirements.txt + +COPY --chown=app:app app ./app +COPY --chown=app:app migrations ./migrations +COPY --chown=app:app contracts/openapi.json ./contracts/openapi.json + +USER app + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 \ + CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ['APP_PORT'], timeout=2)"] + +STOPSIGNAL SIGTERM +CMD ["python", "-m", "app"] diff --git a/Makefile b/Makefile index 5d86d171..6469997d 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-idempotency help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,6 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-idempotency: + $(PYTHON) scripts/check_idempotency.py --base-url "$(BASE_URL)" diff --git a/REPORT.md b/REPORT.md index c2679005..cef04a2e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -7,11 +7,43 @@ ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Вариант: **Docker Compose + Nginx**. Нужны Docker Engine с Compose v2 (проверено на Docker 29.4, Compose 5.1), Python 3.9+ для тестов; на Windows скрипты `.sh` выполняются через Git Bash (обёртки `scripts/*.cmd`). + +Образы: `postgres:16.10-alpine`, `nginx:1.28.0-alpine` (закреплены digest в `compose.yaml`), приложение собирается из `Dockerfile` на `python:3.14.8-slim-bookworm@sha256:…`. + +Одна последовательность запуска: + +```sh +cp .env.example .env # задать свой POSTGRES_PASSWORD +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +# postgres (healthy) -> migrate (один раз, exit 0) -> api x API_REPLICAS (healthy) -> lb (healthy) +make test # smoke через LB, http://localhost:8080 +make test-idempotency # один ключ параллельно через разные реплики +make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd +docker compose --env-file .env up -d --wait # вернуть остановленную реплику +make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/restart.cmd +``` + +Остановка без потери данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v` (удаляет volume `pgdata`). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +client -> lb (nginx :8080, только loopback) -> api-1, api-2, … (порт не публикуется) -> postgres (volume pgdata) + migrate (one-shot) ----------------------^ +``` + +- **Источник истины — PostgreSQL**: пользователи, чаты, сообщения, сессии (`sessions`, токен хранится как SHA-256) и результаты идемпотентности (`idempotency_keys`). В API нет локального состояния, от которого зависит корректность. Redis из lab2 убран: он был только кэшем, а для общего хранилища сессий достаточно PostgreSQL (`SESSION_STORE=postgres`). +- **Миграции** выполняет отдельный сервис `migrate` (`python -m app.migrate`) до старта реплик (`depends_on: service_completed_successfully`). Реплики схему не меняют; `/health/ready` возвращает 503, пока в `schema_migrations` нет всех миграций этой сборки. Параллельный запуск мигратора сериализуется `pg_advisory_lock`. +- **X-Instance-Id** = hostname контейнера (короткий Docker ID), стабилен на время жизни процесса. `stop-one.sh` по нему находит ровно один running-контейнер сервиса `api` этого проекта и делает `docker stop` (SIGTERM, затем SIGKILL через 8 с). `restart: unless-stopped` не поднимает вручную остановленный контейнер, поэтому реплика остаётся остановленной до `docker compose up -d`. +- **Балансировщик**: round robin без `ip_hash`/sticky. `server api:8080 resolve` + `resolver 127.0.0.11 valid=2s`: Nginx раз в 2 с перечитывает DNS Docker, так что остановленные и добавленные (`--scale api=N`) реплики уходят из пула/входят в него без перезапуска LB. Пассивная проверка: `max_fails=1 fail_timeout=5s`, `proxy_connect_timeout 1s`, `proxy_next_upstream error timeout http_502 http_504`. Активных проверок в open-source Nginx нет; readiness реплик проверяет Docker healthcheck (`/health/ready` каждые 3 с), и LB стартует только после healthy API. +- **Повтор на другой реплике**: для неотправленного запроса Nginx всегда пробует следующую реплику. Уже отправленный POST повторяется только для `/api/v1/chats/{id}/messages` (`non_idempotent`), потому что там обязателен `Idempotency-Key`. POST пользователей/чатов/сессий LB не повторяет. +- **Graceful shutdown**: на SIGTERM uvicorn перестаёт принимать соединения и дорабатывает текущие запросы до `SHUTDOWN_TIMEOUT_SECONDS=5`; `stop_grace_period: 8s` больше этого. `keepalive_timeout 4s` в upstream меньше keep-alive uvicorn (5 с), чтобы LB не переиспользовал закрываемое API соединение. +- **Идемпотентность**: в одной транзакции удаляется истёкший (старше `IDEMPOTENCY_TTL_SECONDS`) ключ, затем `INSERT … ON CONFLICT DO NOTHING` в `idempotency_keys` (PK: пользователь + метод + путь + ключ) вместе с готовым ответом и SHA-256 канонического JSON тела, затем вставка сообщения. Конкурентный дубль на другой реплике ждёт на уникальном индексе до COMMIT первой транзакции и получает сохранённый ответ (тот же 201 и Message) либо 409 при другом теле. При ошибке транзакция откатывается целиком, ключ не фиксируется, повтор с тем же ключом безопасен. Сессия и членство в чате проверяются при каждом повторе. +- Время сообщений назначает приложение; порядок между репликами опирается на общие часы хоста (учебное упрощение, одна машина). + +Соответствие `.env.example`: `DATABASE_URL` собирается в `compose.yaml` из `POSTGRES_*` (вариант `DATABASE_URL_FILE` тоже поддерживается кодом); `API_REPLICAS` → `deploy.replicas`; `IDEMPOTENCY_TTL_SECONDS`, `SESSION_TTL_SECONDS`, `SHUTDOWN_TIMEOUT_SECONDS` передаются в API; `CACHE_URL_FILE` не используется. ## Доказательства критериев @@ -30,7 +62,10 @@ TODO: гипотеза, hardware/лимиты, версии, объём данн | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `stop-one` реплики, обслужившей тест (api-1), под записью | запись продолжается ≤10 с, дублей нет | 126 успешных записей, 0 ошибок, первая запись после stop через 1,6 с (`evidence/failover.txt`) | ~1,6 с / нет | +| `stop-one` второй реплики (api-2) после восстановления первой | то же | 126 записей, 0 ошибок, 1,7 с | ~1,7 с / нет | +| `restart.sh`: lb, api, postgres | данные и старый токен сохранены | PASS (`evidence/persistence.txt`) | нет потерь | +| 5 ключей × 20 параллельных POST через LB | одно сообщение на ключ | по 10 ответов с каждой реплики, 5 сообщений (`evidence/idempotency.txt`) | — | ## Ограничения и следующие шаги diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 00000000..14beaf1c --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,18 @@ +import uvicorn + +from .config import load_settings + + +def main() -> None: + settings = load_settings() + uvicorn.run( + 'app.main:app', + host=settings.host, + port=settings.port, + log_level=settings.log_level, + timeout_graceful_shutdown=settings.shutdown_timeout, + ) + + +if __name__ == '__main__': + main() diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 00000000..1006e055 --- /dev/null +++ b/app/auth.py @@ -0,0 +1,30 @@ +"""Basic is only for creating a session. Every other route accepts only Bearer.""" +import base64 +import binascii +from typing import Optional, Tuple + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +def parse_basic(header: str) -> Optional[Tuple[str, str]]: + """username:password. The password may itself contain a colon.""" + scheme, separator, param = header.partition(' ') + if separator != ' ' or scheme.lower() != 'basic' or not param or param != param.strip(): + return None + try: + decoded = base64.b64decode(param, validate=True) + text = decoded.decode('utf-8') + except (binascii.Error, UnicodeDecodeError, ValueError): + return None + username, separator, password = text.partition(':') + if separator != ':': + return None + return username, password + + +def parse_bearer(header: str) -> Optional[str]: + scheme, separator, token = header.partition(' ') + if separator != ' ' or scheme.lower() != 'bearer' or not token or token != token.strip() or ' ' in token: + return None + return token diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..642fd9af --- /dev/null +++ b/app/config.py @@ -0,0 +1,49 @@ +import os +import socket +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Settings: + host: str + port: int + env: str + log_level: str + shutdown_timeout: int + session_ttl_seconds: int + idempotency_ttl_seconds: int + database_url: str + instance_id: str + + +def _secret(file_var: str, value_var: str) -> str: + """A *_FILE variable wins. The secret value is not logged.""" + file_name = os.getenv(file_var, '').strip() + if file_name: + try: + value = Path(file_name).read_text(encoding='utf-8').strip() + except OSError: + raise RuntimeError('%s could not be read' % file_var) from None + if not value: + raise RuntimeError('%s is empty' % file_var) + return value + value = os.getenv(value_var, '').strip() + if not value: + raise RuntimeError('%s or %s is required' % (value_var, file_var)) + return value + + +def load_settings() -> Settings: + return Settings( + host=os.getenv('APP_HOST', '0.0.0.0'), + port=int(os.getenv('APP_PORT', '8080')), + env=os.getenv('APP_ENV', 'lab'), + log_level=os.getenv('LOG_LEVEL', 'info').lower(), + shutdown_timeout=int(os.getenv('SHUTDOWN_TIMEOUT_SECONDS', '5')), + session_ttl_seconds=int(os.getenv('SESSION_TTL_SECONDS', '3600')), + idempotency_ttl_seconds=int(os.getenv('IDEMPOTENCY_TTL_SECONDS', '86400')), + database_url=_secret('DATABASE_URL_FILE', 'DATABASE_URL'), + # Docker sets the hostname to the short container ID, which stop-one.sh maps back to a container. + instance_id=os.getenv('INSTANCE_ID', '').strip() or socket.gethostname(), + ) diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..5c154865 --- /dev/null +++ b/app/db.py @@ -0,0 +1,38 @@ +"""Connection pool. The URL is never written to the log.""" +import asyncio +import logging + +import asyncpg + +logger = logging.getLogger('messenger') + +_ATTEMPTS = 40 + + +async def open_pool(database_url: str) -> asyncpg.Pool: + delay = 0.5 + last_type = 'error' + for attempt in range(1, _ATTEMPTS + 1): + pool = None + try: + pool = await asyncpg.create_pool( + database_url, + min_size=1, + max_size=10, + timeout=5, + command_timeout=5, + server_settings={'statement_timeout': '5000'}, + ) + async with pool.acquire() as conn: + await conn.execute('SELECT 1') + return pool + except Exception as exc: + last_type = type(exc).__name__ + if pool is not None: + await pool.close() + if attempt == _ATTEMPTS: + break + logger.warning('database is not ready (%s), retrying', last_type) + await asyncio.sleep(delay) + delay = min(delay * 1.5, 2) + raise RuntimeError('database did not become reachable (%s)' % last_type) diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..2e7017db --- /dev/null +++ b/app/errors.py @@ -0,0 +1,29 @@ +class ApiError(Exception): + """Expected failure: the message never contains internal details.""" + + def __init__(self, status: int, code: str, message: str, challenge: str | None = None) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.challenge = challenge + + +def invalid_request(message: str) -> ApiError: + return ApiError(400, 'invalid_request', message) + + +def unauthorized(message: str, challenge: str) -> ApiError: + return ApiError(401, 'unauthorized', message, challenge) + + +def not_found(message: str) -> ApiError: + return ApiError(404, 'not_found', message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, 'conflict', message) + + +def unavailable(message: str) -> ApiError: + return ApiError(503, 'unavailable', message) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..605254eb --- /dev/null +++ b/app/main.py @@ -0,0 +1,117 @@ +import json +import logging +import re +import uuid +from contextlib import asynccontextmanager +from pathlib import Path + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException + +from .config import load_settings +from .db import open_pool +from .errors import ApiError +from .routes import router +from .store import Store + +logger = logging.getLogger('messenger') + +_SAFE_REQUEST_ID = re.compile(r'^[A-Za-z0-9._:-]{1,128}$') +_CODES = { + 400: 'invalid_request', + 401: 'unauthorized', + 404: 'not_found', + 409: 'conflict', + 413: 'payload_too_large', + 415: 'unsupported_media_type', + 429: 'rate_limited', + 503: 'unavailable', +} +_MESSAGES = { + 'invalid_request': 'Некорректный запрос', + 'unauthorized': 'Требуется идентификация', + 'not_found': 'Объект не найден', + 'internal_error': 'Внутренняя ошибка', +} +_BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +def _error(status: int, code: str, message: str, request_id: str, challenge: str | None = None) -> JSONResponse: + body = {'error': {'code': code, 'message': message, 'request_id': request_id}} + headers = {'X-Request-Id': request_id} + if challenge: + headers['WWW-Authenticate'] = challenge + return JSONResponse(body, status_code=status, headers=headers) + + +def _request_id(request: Request) -> str: + return getattr(request.state, 'request_id', None) or str(uuid.uuid4()) + + +_OPENAPI_PATH = Path(__file__).resolve().parents[1] / 'contracts' / 'openapi.json' + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Migrations are not applied here: the one-shot migrate service runs them before replicas start.""" + settings = load_settings() + pool = await open_pool(settings.database_url) + try: + app.state.store = Store(pool, settings.session_ttl_seconds, settings.idempotency_ttl_seconds) + logger.info('instance %s started', settings.instance_id) + yield + finally: + await pool.close() + + +def create_app() -> FastAPI: + app = FastAPI( + title='MTUSI Messenger lab3', + version='1.3.0', + lifespan=lifespan, + ) + instance_id = load_settings().instance_id + + def openapi(): + if app.openapi_schema is None: + app.openapi_schema = json.loads(_OPENAPI_PATH.read_text(encoding='utf-8')) + return app.openapi_schema + + app.openapi = openapi + + @app.middleware('http') + async def request_context(request: Request, call_next): + """Gives every response, including unexpected failures, an X-Request-Id and X-Instance-Id.""" + incoming = request.headers.get('X-Request-Id') + request_id = incoming if incoming and _SAFE_REQUEST_ID.match(incoming) else str(uuid.uuid4()) + request.state.request_id = request_id + try: + response = await call_next(request) + except Exception: + logger.exception('request %s failed: %s %s', request_id, request.method, request.url.path) + response = _error(500, 'internal_error', _MESSAGES['internal_error'], request_id) + response.headers['X-Request-Id'] = request_id + response.headers['X-Instance-Id'] = instance_id + return response + + @app.exception_handler(ApiError) + async def handle_api_error(request: Request, exc: ApiError) -> JSONResponse: + return _error(exc.status, exc.code, exc.message, _request_id(request), exc.challenge) + + @app.exception_handler(RequestValidationError) + async def handle_validation_error(request: Request, exc: RequestValidationError) -> JSONResponse: + return _error(400, 'invalid_request', _MESSAGES['invalid_request'], _request_id(request)) + + @app.exception_handler(StarletteHTTPException) + async def handle_http_error(request: Request, exc: StarletteHTTPException) -> JSONResponse: + code = _CODES.get(exc.status_code, 'internal_error' if exc.status_code >= 500 else 'invalid_request') + challenge = _BEARER_CHALLENGE if exc.status_code == 401 else None + return _error(exc.status_code, code, _MESSAGES.get(code, _MESSAGES['invalid_request']), _request_id(request), challenge) + + app.include_router(router) + return app + + +app = create_app() diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..a96670e8 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,64 @@ +"""Apply numbered SQL files once. Run as a separate one-shot step: python -m app.migrate.""" +import asyncio +import logging +from pathlib import Path + +import asyncpg + +from .config import load_settings +from .db import open_pool + +logger = logging.getLogger('messenger') + +_MIGRATIONS = Path(__file__).resolve().parents[1] / 'migrations' +# Any constant shared by all runners; serializes concurrent migrators across processes. +_LOCK_ID = 3_2026_0003 + +_BOOTSTRAP = """ +CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +) +""" + + +def expected_versions() -> list[str]: + return sorted(path.name for path in _MIGRATIONS.glob('*.sql')) + + +async def apply_migrations(pool: asyncpg.Pool) -> None: + async with pool.acquire() as conn: + await conn.execute('SET statement_timeout = 0') + await conn.execute('SELECT pg_advisory_lock($1)', _LOCK_ID) + try: + await conn.execute(_BOOTSTRAP) + rows = await conn.fetch('SELECT version FROM schema_migrations') + applied = {row['version'] for row in rows} + for version in expected_versions(): + if version in applied: + logger.info('migration %s already applied', version) + continue + sql = (_MIGRATIONS / version).read_text(encoding='utf-8') + async with conn.transaction(): + await conn.execute(sql) + await conn.execute( + 'INSERT INTO schema_migrations (version) VALUES ($1)', + version, + ) + logger.info('applied migration %s', version) + finally: + await conn.execute('SELECT pg_advisory_unlock($1)', _LOCK_ID) + + +async def _main() -> None: + settings = load_settings() + pool = await open_pool(settings.database_url) + try: + await apply_migrations(pool) + finally: + await pool.close() + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s') + asyncio.run(_main()) diff --git a/app/pagination.py b/app/pagination.py new file mode 100644 index 00000000..20c8957f --- /dev/null +++ b/app/pagination.py @@ -0,0 +1,43 @@ +import base64 +import json +import re +from typing import Mapping, Optional, Tuple + +from .errors import invalid_request + +# Position of the last item returned to the client. +Key = Tuple[str, str] + +DEFAULT_LIMIT = 50 +MAX_LIMIT = 100 +_LIMIT = re.compile(r'^\d{1,3}$') + + +def encode_cursor(scope: str, key: Key) -> str: + """Scope binds the cursor to one resource, so a foreign cursor is detectable.""" + payload = json.dumps({'s': scope, 't': key[0], 'i': key[1]}, separators=(',', ':')) + return base64.urlsafe_b64encode(payload.encode('utf-8')).decode('ascii').rstrip('=') + + +def decode_cursor(value: str, scope: str) -> Key: + try: + raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4)) + payload = json.loads(raw) + created_at, item_id = payload['t'], payload['i'] + belongs = payload['s'] == scope + except (ValueError, TypeError, KeyError): + raise invalid_request('Некорректный cursor') from None + if not belongs or not isinstance(created_at, str) or not isinstance(item_id, str): + raise invalid_request('Cursor относится к другому запросу') from None + return created_at, item_id + + +def parse_page(params: Mapping[str, str], scope: str) -> Tuple[int, Optional[Key]]: + raw_limit = params.get('limit') + limit = DEFAULT_LIMIT + if raw_limit is not None: + if not _LIMIT.match(raw_limit) or not 1 <= int(raw_limit) <= MAX_LIMIT: + raise invalid_request('limit должен быть целым числом от 1 до %d' % MAX_LIMIT) + limit = int(raw_limit) + raw_cursor = params.get('cursor') + return limit, decode_cursor(raw_cursor, scope) if raw_cursor is not None else None diff --git a/app/passwords.py b/app/passwords.py new file mode 100644 index 00000000..37b3f2d1 --- /dev/null +++ b/app/passwords.py @@ -0,0 +1,27 @@ +"""Argon2id password hashes. The hash never leaves this module in an API response.""" +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError + +_hasher = PasswordHasher() +# Same cost as a real hash, so a missing user is not faster than a wrong password. +_DUMMY_HASH = _hasher.hash('not-a-user-password') +_DUMMY_PASSWORD = 'not-a-user-password' + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return _hasher.verify(password_hash, password) + except VerificationError: + return False + + +def password_matches(stored_hash: str | None, password: str) -> bool: + """One Argon2 check either way. An out-of-range secret is never the stored password.""" + if stored_hash is None or not 12 <= len(password) <= 128 or not password.strip(): + verify_password(_DUMMY_HASH, _DUMMY_PASSWORD) + return False + return verify_password(stored_hash, password) diff --git a/app/routes.py b/app/routes.py new file mode 100644 index 00000000..2221dbd3 --- /dev/null +++ b/app/routes.py @@ -0,0 +1,193 @@ +import hashlib +import json +import re +import uuid +from typing import Optional, Sequence, Type, TypeVar + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import JSONResponse +from pydantic import BaseModel, ValidationError +from starlette.responses import Response + +from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, parse_basic, parse_bearer +from .errors import invalid_request, not_found, unauthorized +from .pagination import encode_cursor, parse_page +from .schemas import CreateChat, CreateMessage, CreateUser +from .store import Chat, Store, User + +router = APIRouter() +Body = TypeVar('Body', bound=BaseModel) + +_IDEMPOTENCY_KEY = re.compile(r'[A-Za-z0-9._:-]{1,128}') + + +async def get_store(request: Request) -> Store: + return request.app.state.store + + +async def current_user(request: Request) -> User: + """Bearer only. X-User-Id is ignored and does not identify anyone.""" + token = parse_bearer(request.headers.get('Authorization', '')) + if token is None: + raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE) + user = await request.app.state.store.user_for_token(token) + if user is None: + raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE) + return user + + +async def read_body(request: Request, model: Type[Body]) -> Body: + try: + payload = json.loads(await request.body()) + except (UnicodeDecodeError, ValueError): + raise invalid_request('Тело запроса не является корректным JSON') from None + if not isinstance(payload, dict): + raise invalid_request('Тело запроса должно быть JSON-объектом') + try: + return model.model_validate(payload) + except ValidationError: + raise invalid_request('Тело запроса не соответствует контракту') from None + + +def _request_hash(payload: BaseModel) -> str: + """Key order in the client JSON does not matter: the validated body is re-serialized canonically.""" + canonical = json.dumps(payload.model_dump(), sort_keys=True, separators=(',', ':'), ensure_ascii=False) + return hashlib.sha256(canonical.encode('utf-8')).hexdigest() + + +async def _member_chat(store: Store, chat_id: str, actor: User) -> Chat: + try: + canonical = str(uuid.UUID(chat_id)) + except ValueError: + raise invalid_request('Некорректный chat_id') from None + chat = await store.find_chat(canonical, actor.id) + if chat is None: + raise not_found('Чат не найден') + return chat + + +def _page(items: Sequence, has_more: bool, scope: str) -> dict: + cursor: Optional[str] = None + if has_more and items: + cursor = encode_cursor(scope, (items[-1].created_at, items[-1].id)) + return {'items': [item.to_dict() for item in items], 'next_cursor': cursor} + + +@router.get('/health/live') +async def liveness() -> JSONResponse: + return JSONResponse({'status': 'ok'}) + + +@router.get('/health/ready') +async def readiness(store: Store = Depends(get_store)) -> JSONResponse: + await store.ready() + return JSONResponse({'status': 'ok'}) + + +@router.post('/api/v1/users') +async def register_user(request: Request, store: Store = Depends(get_store)) -> JSONResponse: + payload = await read_body(request, CreateUser) + user = await store.create_user(payload.username, payload.display_name, payload.password) + return JSONResponse(user.to_dict(), status_code=201) + + +@router.post('/api/v1/auth/sessions') +async def create_session(request: Request, store: Store = Depends(get_store)) -> JSONResponse: + parsed = parse_basic(request.headers.get('Authorization', '')) + if parsed is None: + raise unauthorized('Неверные учётные данные', BASIC_CHALLENGE) + user = await store.authenticate(*parsed) + if user is None: + raise unauthorized('Неверные учётные данные', BASIC_CHALLENGE) + token, expires_at = await store.create_session(user.id) + body = { + 'token': token, + 'token_type': 'Bearer', + 'expires_at': expires_at, + 'user': user.to_dict(), + } + return JSONResponse(body, status_code=201) + + +@router.delete('/api/v1/auth/sessions/current') +async def revoke_session(request: Request, actor: User = Depends(current_user)) -> Response: + token = parse_bearer(request.headers.get('Authorization', '')) + if token is None or not await request.app.state.store.revoke_session(token): + raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE) + return Response(status_code=204) + + +@router.get('/api/v1/users/me') +async def show_current_user(actor: User = Depends(current_user)) -> JSONResponse: + return JSONResponse(actor.to_dict()) + + +@router.post('/api/v1/chats') +async def create_chat( + request: Request, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> JSONResponse: + payload = await read_body(request, CreateChat) + if actor.id not in payload.member_ids: + raise invalid_request('Создатель чата должен быть указан в member_ids') + chat = await store.create_chat(payload.title, payload.member_ids) + return JSONResponse(chat.to_dict(), status_code=201) + + +@router.get('/api/v1/chats') +async def list_chats( + request: Request, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> JSONResponse: + scope = 'chats:' + actor.id + limit, after = parse_page(request.query_params, scope) + chats, has_more = await store.list_chats(actor.id, limit, after) + return JSONResponse(_page(chats, has_more, scope)) + + +@router.get('/api/v1/chats/{chat_id}') +async def show_chat( + chat_id: str, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> JSONResponse: + return JSONResponse((await _member_chat(store, chat_id, actor)).to_dict()) + + +@router.post('/api/v1/chats/{chat_id}/messages') +async def send_message( + chat_id: str, + request: Request, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> JSONResponse: + chat = await _member_chat(store, chat_id, actor) + payload = await read_body(request, CreateMessage) + key = request.headers.get('Idempotency-Key') + if key is None or not _IDEMPOTENCY_KEY.fullmatch(key): + raise invalid_request('Требуется корректный Idempotency-Key') + message = await store.add_message( + chat.id, + actor.id, + payload.text, + path='/api/v1/chats/%s/messages' % chat.id, + key=key, + request_hash=_request_hash(payload), + ) + return JSONResponse(message.to_dict(), status_code=201) + + +@router.get('/api/v1/chats/{chat_id}/messages') +async def list_messages( + chat_id: str, + request: Request, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> JSONResponse: + chat = await _member_chat(store, chat_id, actor) + scope = 'messages:' + chat.id + limit, after = parse_page(request.query_params, scope) + messages, has_more = await store.list_messages(chat.id, limit, after) + return JSONResponse(_page(messages, has_more, scope)) diff --git a/app/schemas.py b/app/schemas.py new file mode 100644 index 00000000..e583bf1f --- /dev/null +++ b/app/schemas.py @@ -0,0 +1,55 @@ +"""Request bodies. The contract forbids unknown fields, wrong types and blank strings.""" +import uuid +from typing import List + +from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator +from typing_extensions import Annotated + + +def _not_blank(value: str) -> str: + if not value.strip(): + raise ValueError('value must not be whitespace only') + return value + + +def _canonical_uuid(value: str) -> str: + parsed = uuid.UUID(value) + if str(parsed) != value.lower(): + raise ValueError('value must be a canonical UUID') + return str(parsed) + + +Username = Annotated[str, Field(pattern=r'^[a-z0-9_]{3,32}$')] +DisplayName = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(_not_blank)] +Title = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(_not_blank)] +Text = Annotated[str, Field(min_length=1, max_length=4000), AfterValidator(_not_blank)] +Password = Annotated[str, Field(min_length=12, max_length=128), AfterValidator(_not_blank)] +UserId = Annotated[str, AfterValidator(_canonical_uuid)] + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra='forbid') + + username: Username + display_name: DisplayName + password: Password + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra='forbid') + + title: Title + member_ids: Annotated[List[UserId], Field(min_length=1, max_length=100)] + + @field_validator('member_ids') + @classmethod + def _unique(cls, value: List[str]) -> List[str]: + if len(set(value)) != len(value): + raise ValueError('member_ids must be unique') + return value + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra='forbid') + + text: Text diff --git a/app/store.py b/app/store.py new file mode 100644 index 00000000..dc6409a0 --- /dev/null +++ b/app/store.py @@ -0,0 +1,467 @@ +import hashlib +import json +import secrets +import threading +import uuid +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Dict, List, Optional, Sequence, Tuple + +import asyncpg + +from .errors import conflict, invalid_request, not_found, unavailable +from .migrate import expected_versions +from .pagination import Key +from .passwords import hash_password, password_matches + + +@dataclass(frozen=True) +class User: + id: str + username: str + display_name: str + created_at: str + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'username': self.username, + 'display_name': self.display_name, + 'created_at': self.created_at, + } + + +@dataclass(frozen=True) +class Chat: + id: str + title: str + member_ids: Tuple[str, ...] + created_at: str + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'title': self.title, + 'member_ids': list(self.member_ids), + 'created_at': self.created_at, + } + + +@dataclass(frozen=True) +class Message: + id: str + chat_id: str + sender_id: str + text: str + created_at: str + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'chat_id': self.chat_id, + 'sender_id': self.sender_id, + 'text': self.text, + 'created_at': self.created_at, + } + + +def _token_key(token: str) -> str: + return hashlib.sha256(token.encode('utf-8')).hexdigest() + + +def _stamp(moment: datetime) -> str: + return moment.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def _parse_stamp(value: str) -> datetime: + if value.endswith('Z'): + value = value[:-1] + '+00:00' + return datetime.fromisoformat(value) + + +def _user(row: asyncpg.Record) -> User: + return User(str(row['id']), row['username'], row['display_name'], _stamp(row['created_at'])) + + +def _as_uuid(value: str, message: str) -> uuid.UUID: + try: + return uuid.UUID(value) + except ValueError: + raise invalid_request(message) from None + + +def _db_down(exc: BaseException) -> bool: + return isinstance(exc, ( + asyncpg.PostgresConnectionError, + asyncpg.InterfaceError, + asyncpg.CannotConnectNowError, + OSError, + TimeoutError, + )) + + +class Store: + """PostgreSQL is the only state: users, chats, sessions and idempotency results are shared by all replicas.""" + + def __init__(self, pool: asyncpg.Pool, session_ttl_seconds: int = 3600, idempotency_ttl_seconds: int = 86400) -> None: + self._pool = pool + self._session_ttl = timedelta(seconds=session_ttl_seconds) + self._idempotency_ttl = timedelta(seconds=idempotency_ttl_seconds) + self._lock = threading.Lock() + self._last_moment: Optional[datetime] = None + self._migrations = expected_versions() + + def _now(self) -> datetime: + """Strictly increasing within one process; across replicas order relies on the shared host clock.""" + with self._lock: + moment = datetime.now(timezone.utc) + if self._last_moment is not None and moment <= self._last_moment: + moment = self._last_moment + timedelta(microseconds=1) + self._last_moment = moment + return moment + + async def ready(self) -> None: + """Ready only when PostgreSQL answers and the separate migrator has applied this build's schema.""" + try: + async with self._pool.acquire() as conn: + applied = await conn.fetchval( + 'SELECT count(*) FROM schema_migrations WHERE version = ANY($1::text[])', + self._migrations, + ) + except Exception: + raise unavailable('Хранилище недоступно') from None + if applied != len(self._migrations): + raise unavailable('Схема БД ещё не применена') + + async def create_user(self, username: str, display_name: str, password: str) -> User: + password_hash = hash_password(password) + user_id = uuid.uuid4() + created = self._now() + try: + async with self._pool.acquire() as conn: + await conn.execute( + ''' + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, $2, $3, $4, $5) + ''', + user_id, + username, + display_name, + password_hash, + created, + ) + except asyncpg.UniqueViolationError: + raise conflict('Username уже занят') from None + return User(str(user_id), username, display_name, _stamp(created)) + + async def authenticate(self, username: str, password: str) -> Optional[User]: + try: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + ''' + SELECT id, username, display_name, created_at, password_hash + FROM users + WHERE username = $1 + ''', + username, + ) + except Exception as exc: + if _db_down(exc): + raise unavailable('Хранилище недоступно') from None + raise + stored_hash = row['password_hash'] if row is not None else None + if not password_matches(stored_hash, password) or row is None: + return None + return _user(row) + + async def create_session(self, user_id: str) -> Tuple[str, str]: + token = secrets.token_urlsafe(32) + expires = datetime.now(timezone.utc) + self._session_ttl + try: + async with self._pool.acquire() as conn: + await conn.execute( + 'INSERT INTO sessions (token_hash, user_id, expires_at) VALUES ($1, $2, $3)', + _token_key(token), + _as_uuid(user_id, 'Некорректный user_id'), + expires, + ) + except Exception as exc: + if _db_down(exc): + raise unavailable('Хранилище недоступно') from None + raise + return token, _stamp(expires) + + async def user_for_token(self, token: str) -> Optional[User]: + now = datetime.now(timezone.utc) + key = _token_key(token) + try: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + ''' + SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at + FROM sessions s + JOIN users u ON u.id = s.user_id + WHERE s.token_hash = $1 + ''', + key, + ) + if row is None: + return None + if row['expires_at'] <= now: + await conn.execute('DELETE FROM sessions WHERE token_hash = $1', key) + return None + return _user(row) + except Exception as exc: + if _db_down(exc): + raise unavailable('Хранилище недоступно') from None + raise + + async def revoke_session(self, token: str) -> bool: + try: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + ''' + DELETE FROM sessions + WHERE token_hash = $1 AND expires_at > $2 + RETURNING token_hash + ''', + _token_key(token), + datetime.now(timezone.utc), + ) + except Exception as exc: + if _db_down(exc): + raise unavailable('Хранилище недоступно') from None + raise + return row is not None + + async def create_chat(self, title: str, member_ids: Sequence[str]) -> Chat: + ids = [_as_uuid(member_id, 'Некорректный user_id') for member_id in member_ids] + chat_id = uuid.uuid4() + created = self._now() + async with self._pool.acquire() as conn: + async with conn.transaction(): + found = await conn.fetch('SELECT id FROM users WHERE id = ANY($1::uuid[])', ids) + if len(found) != len(ids): + raise invalid_request('Все участники должны быть существующими пользователями') + await conn.execute( + 'INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)', + chat_id, + title, + created, + ) + await conn.execute( + ''' + INSERT INTO chat_members (chat_id, user_id, position) + SELECT $1, member_id, (ordinality - 1)::integer + FROM unnest($2::uuid[]) WITH ORDINALITY AS members(member_id, ordinality) + ''', + chat_id, + ids, + ) + return Chat(str(chat_id), title, tuple(member_ids), _stamp(created)) + + async def find_chat(self, chat_id: str, member_id: str) -> Optional[Chat]: + """A chat of somebody else is indistinguishable from a missing one.""" + chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id') + member_uuid = _as_uuid(member_id, 'Некорректный user_id') + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + ''' + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $2 + WHERE c.id = $1 + ''', + chat_uuid, + member_uuid, + ) + if row is None: + return None + members = await self._members(conn, [chat_uuid]) + return Chat(str(row['id']), row['title'], members.get(chat_uuid, ()), _stamp(row['created_at'])) + + async def list_chats(self, member_id: str, limit: int, after: Optional[Key]) -> Tuple[List[Chat], bool]: + member_uuid = _as_uuid(member_id, 'Некорректный user_id') + after_at, after_id = self._cursor(after) + async with self._pool.acquire() as conn: + if after_at is None: + rows = await conn.fetch( + ''' + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1 + ORDER BY c.created_at, c.id + LIMIT $2 + ''', + member_uuid, + limit + 1, + ) + else: + rows = await conn.fetch( + ''' + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1 + WHERE (c.created_at, c.id) > ($2::timestamptz, $3::uuid) + ORDER BY c.created_at, c.id + LIMIT $4 + ''', + member_uuid, + after_at, + after_id, + limit + 1, + ) + page = rows[:limit] + members = await self._members(conn, [row['id'] for row in page]) + chats = [ + Chat(str(row['id']), row['title'], members.get(row['id'], ()), _stamp(row['created_at'])) + for row in page + ] + return chats, len(rows) > limit + + async def add_message( + self, + chat_id: str, + sender_id: str, + text: str, + *, + path: str, + key: str, + request_hash: str, + ) -> Message: + """Exactly one message per (sender, POST, path, key), whichever replica receives the request. + + Concurrent duplicates block on the primary key of idempotency_keys until the first + transaction commits or rolls back, so a rolled-back attempt leaves the key free. + """ + chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id') + sender_uuid = _as_uuid(sender_id, 'Некорректный user_id') + message = Message(str(uuid.uuid4()), str(chat_uuid), sender_id, text, _stamp(self._now())) + cutoff = datetime.now(timezone.utc) - self._idempotency_ttl + async with self._pool.acquire() as conn: + async with conn.transaction(): + await conn.execute( + ''' + DELETE FROM idempotency_keys + WHERE user_id = $1 AND method = 'POST' AND path = $2 AND key = $3 AND created_at < $4 + ''', + sender_uuid, + path, + key, + cutoff, + ) + claimed = await conn.fetchval( + ''' + INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response) + VALUES ($1, 'POST', $2, $3, $4, 201, $5::jsonb) + ON CONFLICT (user_id, method, path, key) DO NOTHING + RETURNING 1 + ''', + sender_uuid, + path, + key, + request_hash, + json.dumps(message.to_dict()), + ) + if claimed is None: + return await self._replay(conn, sender_uuid, path, key, request_hash) + await conn.execute( + ''' + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1, $2, $3, $4, $5) + ''', + uuid.UUID(message.id), + chat_uuid, + sender_uuid, + text, + _parse_stamp(message.created_at), + ) + return message + + async def _replay( + self, + conn: asyncpg.Connection, + sender_uuid: uuid.UUID, + path: str, + key: str, + request_hash: str, + ) -> Message: + row = await conn.fetchrow( + ''' + SELECT request_hash, response + FROM idempotency_keys + WHERE user_id = $1 AND method = 'POST' AND path = $2 AND key = $3 + ''', + sender_uuid, + path, + key, + ) + if row is None: + raise unavailable('Повторите запрос с тем же Idempotency-Key') + if row['request_hash'] != request_hash: + raise conflict('Idempotency-Key уже использован с другим сообщением') + data = json.loads(row['response']) + return Message(data['id'], data['chat_id'], data['sender_id'], data['text'], data['created_at']) + + async def list_messages(self, chat_id: str, limit: int, after: Optional[Key]) -> Tuple[List[Message], bool]: + chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id') + after_at, after_id = self._cursor(after) + async with self._pool.acquire() as conn: + if after_at is None: + rows = await conn.fetch( + ''' + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + ORDER BY created_at, id + LIMIT $2 + ''', + chat_uuid, + limit + 1, + ) + else: + rows = await conn.fetch( + ''' + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + AND (created_at, id) > ($2::timestamptz, $3::uuid) + ORDER BY created_at, id + LIMIT $4 + ''', + chat_uuid, + after_at, + after_id, + limit + 1, + ) + page = rows[:limit] + messages = [ + Message(str(row['id']), str(row['chat_id']), str(row['sender_id']), row['text'], _stamp(row['created_at'])) + for row in page + ] + return messages, len(rows) > limit + + async def _members(self, conn: asyncpg.Connection, chat_ids: Sequence[uuid.UUID]) -> Dict[uuid.UUID, Tuple[str, ...]]: + if not chat_ids: + return {} + rows = await conn.fetch( + ''' + SELECT chat_id, user_id + FROM chat_members + WHERE chat_id = ANY($1::uuid[]) + ORDER BY chat_id, position + ''', + list(chat_ids), + ) + grouped: Dict[uuid.UUID, List[str]] = {} + for row in rows: + grouped.setdefault(row['chat_id'], []).append(str(row['user_id'])) + return {chat_id: tuple(member_ids) for chat_id, member_ids in grouped.items()} + + def _cursor(self, after: Optional[Key]) -> Tuple[Optional[datetime], Optional[uuid.UUID]]: + if after is None: + return None, None + try: + return _parse_stamp(after[0]), uuid.UUID(after[1]) + except ValueError: + raise invalid_request('Некорректный cursor') from None diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..68d2997d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,96 @@ +# Client -> lb (nginx) -> api x API_REPLICAS -> postgres. +# Secrets come from .env, which is not committed. Only the LB publishes a port. +name: messenger-lab3 + +x-app-env: &app-env + APP_ENV: ${APP_ENV:-lab} + LOG_LEVEL: ${LOG_LEVEL:-info} + DATABASE_URL: postgresql://${POSTGRES_USER:?set in .env}:${POSTGRES_PASSWORD:?set in .env}@postgres:5432/${POSTGRES_DB:?set in .env} + +services: + postgres: + image: postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""] + interval: 2s + timeout: 5s + retries: 30 + restart: unless-stopped + + # One-shot schema step. Replicas never migrate on startup, so there is no race between them; + # an advisory lock additionally serializes accidental parallel runs. + migrate: + image: messenger-lab3-app:local + build: + context: . + dockerfile: Dockerfile + command: ["python", "-m", "app.migrate"] + environment: + <<: *app-env + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + image: messenger-lab3-app:local + build: + context: . + dockerfile: Dockerfile + deploy: + replicas: ${API_REPLICAS:-2} + environment: + <<: *app-env + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5} + expose: + - "8080" + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + healthcheck: + test: + - CMD + - python + - -c + - import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2) + interval: 3s + timeout: 3s + retries: 3 + start_period: 20s + # SIGTERM -> uvicorn stops accepting, finishes in-flight requests within SHUTDOWN_TIMEOUT_SECONDS. + stop_signal: SIGTERM + stop_grace_period: 8s + restart: unless-stopped + + lb: + image: nginx:1.28.0-alpine@sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284 + # Loopback only. Both families: on Windows "localhost" tries ::1 first and waits ~2 s if nothing listens. + ports: + - "127.0.0.1:${LB_PORT:-8080}:8080" + - "[::1]:${LB_PORT:-8080}:8080" + volumes: + - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/lb/health"] + interval: 5s + timeout: 3s + retries: 3 + restart: unless-stopped + +volumes: + pgdata: diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..858e2a75 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,34 @@ +# failover run 2026-11-05T16:51:38.6795653+03:00 +## run 1 — before +messenger-lab3-api-1 5dce519fa2a3 Up 22 seconds (healthy) +messenger-lab3-api-2 d0fe578a76ab Up 22 seconds (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 1.598} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +## run 1 — after +messenger-lab3-api-1 5dce519fa2a3 Exited (0) 13 seconds ago +messenger-lab3-api-2 d0fe578a76ab Up 39 seconds (healthy) +## run 2 — before +messenger-lab3-api-1 5dce519fa2a3 Up 6 seconds (healthy) +messenger-lab3-api-2 d0fe578a76ab Up 48 seconds (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 1.575} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +## run 2 — after +messenger-lab3-api-1 5dce519fa2a3 Exited (0) 13 seconds ago +messenger-lab3-api-2 d0fe578a76ab Up About a minute (healthy) +## restored +messenger-lab3-api-1 5dce519fa2a3 Up 6 seconds (healthy) +messenger-lab3-api-2 d0fe578a76ab Up About a minute (healthy) +## run 3 (target api-2) — before +messenger-lab3-api-1 5dce519fa2a3 Up 17 seconds (healthy) +messenger-lab3-api-2 d0fe578a76ab Up About a minute (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 1.707} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +## run 3 — after +messenger-lab3-api-1 5dce519fa2a3 Up 34 seconds (healthy) +messenger-lab3-api-2 d0fe578a76ab Exited (0) 13 seconds ago +## restored +messenger-lab3-api-1 5dce519fa2a3 Up 44 seconds (healthy) +messenger-lab3-api-2 d0fe578a76ab Up 6 seconds (healthy) diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 0000000000000000000000000000000000000000..70fdb42d4729bad464f4b186779971a6ff6fa633 GIT binary patch literal 1066 zcmdT@%L>9U5S+8%KlIj%LLbJe2f;rO{DNAmh_%*MMG=2polOK450%n`5;l+B$?hcV z+^#t`*dxUZ790#Xdt5E#h*2P8H03J$7QHTQtxp9)EVvV|U^66GBBHmT=L{Q)IU`0C zZ}l-}#H0C)NEMe=Hf_^oR)1Rh~@UTA7wJFigu|^vTq#m1#MH z>VNA04b#p~GaX}u35RsKqOI85C#{MjPexrCJ3OZ)95KykXiFuE(sml%#_*BhU2&qX T1GN_FY}M4KzEq#!Oi$kd!_1!< literal 0 HcmV?d00001 diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..75b5beda --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,12 @@ +# before +messenger-lab3-api-1 5dce519fa2a3 Up 52 seconds (healthy) +messenger-lab3-api-2 d0fe578a76ab Up 14 seconds (healthy) +messenger-lab3-lb-1 ece2ac882f2d Up About a minute (healthy) +messenger-lab3-postgres-1 0ed6d04de263 Up 7 minutes (healthy) +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +# after +messenger-lab3-api-1 5dce519fa2a3 Up 12 seconds (healthy) +messenger-lab3-api-2 d0fe578a76ab Up 12 seconds (healthy) +messenger-lab3-lb-1 ece2ac882f2d Up 5 seconds (healthy) +messenger-lab3-postgres-1 0ed6d04de263 Up 17 seconds (healthy) diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000000000000000000000000000000000000..9155c94f2bba3373ba257ce54467ccbf29946ffc GIT binary patch literal 174 zcmW-bF%H5o5Co@2;tzfS4INY|(?#M12Ae2S5>QBZK4ffJ^6lP@_U`}dXt?<<1V$ov z&Z;+?YGmM{Yv^dLA6;gaTAaMN+8gvweRR57H<{IckqmZclC`fys&uc= 0) +); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id), + sender_id UUID NOT NULL REFERENCES users (id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- Keyset history: WHERE chat_id = ? AND (created_at, id) > cursor ORDER BY created_at, id. +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); + +-- Chat list is filtered by the caller, then ordered by the chat's own (created_at, id). +CREATE INDEX chat_members_user_idx ON chat_members (user_id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..c9f4da7f --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,16 @@ +-- Result of POST /chats/{id}/messages per (user, method, path, Idempotency-Key). +-- The row and the message are written in one transaction, so a failed attempt leaves no key behind. +CREATE TABLE idempotency_keys ( + user_id UUID NOT NULL REFERENCES users (id), + method TEXT NOT NULL, + path TEXT NOT NULL, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, + status_code INTEGER NOT NULL, + response JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, method, path, key), + CONSTRAINT idempotency_keys_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$') +); + +CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..0c7146cb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,19 @@ +# Direct: fastapi, pydantic, uvicorn, argon2-cffi. The rest are their dependencies, pinned. +asyncpg==0.32.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==25.1.0 +cffi==2.0.0 +click==8.5.0 +fastapi==0.141.1 +h11==0.16.0 +idna==3.20 +pycparser==3.0 +pydantic==2.13.5 +pydantic_core==2.46.5 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +uvicorn==0.53.0 diff --git a/scripts/check_idempotency.py b/scripts/check_idempotency.py new file mode 100644 index 00000000..02f00732 --- /dev/null +++ b/scripts/check_idempotency.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Same Idempotency-Key sent concurrently through the LB must hit several replicas and create one message.""" +import argparse +import json +import sys +import threading +import uuid +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, Failure, check, run # noqa: E402 + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--base-url', default=None) + p.add_argument('--parallel', type=int, default=20) + p.add_argument('--rounds', type=int, default=5) + args = p.parse_args() + + c = Client(args.base_url) + alice, bob, _, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/messages' + instances = Counter() + for round_no in range(args.rounds): + key = 'race-' + uuid.uuid4().hex + text = 'Конкурентный повтор %d %s' % (round_no, key) + barrier = threading.Barrier(args.parallel) + results, errors = [], [] + + def send(): + barrier.wait() + try: + r = c.call('POST', path, 201, actor=alice, body={'text': text}, headers={'Idempotency-Key': key}, schema='Message') + results.append((r['headers']['X-Instance-Id'], r['json'])) + except Failure as exc: + errors.append(str(exc)) + + threads = [threading.Thread(target=send) for _ in range(args.parallel)] + for t in threads: + t.start() + for t in threads: + t.join() + check(not errors, 'concurrent POST failed: ' + '; '.join(errors[:3])) + bodies = {json.dumps(body, sort_keys=True) for _, body in results} + check(len(bodies) == 1, 'replicas returned different messages for one key') + served = Counter(instance for instance, _ in results) + instances.update(served) + print('round %d: %d responses, 1 message, replicas %s' % (round_no, len(results), dict(served))) + c.call('POST', path, 409, actor=alice, body={'text': text + ' другой'}, headers={'Idempotency-Key': key}) + + page = c.call('GET', path + '?limit=100', actor=bob, schema='MessagePage')['json'] + texts = [m['text'] for m in page['items']] + check(len(texts) == args.rounds and len(set(texts)) == args.rounds, 'expected exactly one message per key') + check(len(instances) >= 2, 'all requests hit one replica; LB does not balance') + print('PASS: %d keys x %d parallel requests -> %d messages via %d replicas %s' + % (args.rounds, args.parallel, len(texts), len(instances), dict(instances))) + + +if __name__ == '__main__': + run(main) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..cd9f2207 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,3 @@ +@echo off +rem Windows entry point for tests/resilience.py: runs restart.sh with Git Bash. +"%ProgramFiles%\Git\bin\sh.exe" "%~dp0restart.sh" diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..b0d4268e --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Restart the LB, every API replica and PostgreSQL. The pgdata volume stays; nothing is re-seeded. +docker compose --env-file .env stop lb api +docker compose --env-file .env restart postgres +# Brings back all replicas (including one left stopped by stop-one.sh); migrate re-runs as a no-op. +docker compose --env-file .env up -d --wait diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..8b4c9587 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,3 @@ +@echo off +rem Windows entry point for tests/resilience.py: runs stop-one.sh with Git Bash. +"%ProgramFiles%\Git\bin\sh.exe" "%~dp0stop-one.sh" diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..6301907c --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Stop exactly ONE API replica and leave it stopped. DB, LB and volumes are untouched. +# X-Instance-Id is the container hostname, i.e. the short Docker container ID. +# Restore afterwards with: docker compose --env-file .env up -d --wait +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + *[!0-9a-f]*) valid=no ;; + *) valid=yes ;; +esac +if [ "$valid" = no ] || [ "${#TARGET_INSTANCE_ID}" -lt 12 ]; then + echo "TARGET_INSTANCE_ID is not a container ID" >&2 + exit 2 +fi +project="${COMPOSE_PROJECT_NAME:-messenger-lab3}" +matches=$(docker ps -q --no-trunc \ + --filter "label=com.docker.compose.project=$project" \ + --filter "label=com.docker.compose.service=api" | grep "^$TARGET_INSTANCE_ID" || true) +count=$(printf '%s' "$matches" | grep -c . || true) +if [ "$count" -ne 1 ]; then + echo "expected one running api container for $TARGET_INSTANCE_ID, found $count" >&2 + exit 1 +fi +name=$(docker inspect --format '{{.Name}}' "$matches") +# Graceful: SIGTERM, then SIGKILL after 8 s; returns before the test's 10 s limit. +docker stop --time 8 "$matches" >/dev/null +echo "stopped api replica ${name#/} ($TARGET_INSTANCE_ID)" -- 2.54.0 From 124c83e450bcf3493948a86fad284f759ac39683 Mon Sep 17 00:00:00 2001 From: Sunrise443 Date: Sat, 7 Nov 2026 17:43:15 +0300 Subject: [PATCH 2/4] level 3 --- REPORT.md | 46 ++++++++++++++++++---------- evidence/failover.txt | 63 +++++++++++++++++++++----------------- evidence/idempotency.txt | Bin 1066 -> 698 bytes evidence/instances.txt | 16 ++++++++++ evidence/persistence.txt | 26 ++++++++++------ evidence/smoke.txt | Bin 174 -> 639 bytes scripts/show_instances.py | 41 +++++++++++++++++++++++++ 7 files changed, 138 insertions(+), 54 deletions(-) create mode 100644 evidence/instances.txt create mode 100644 scripts/show_instances.py diff --git a/REPORT.md b/REPORT.md index cef04a2e..c0e8c605 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,6 +1,6 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO +- Автор, группа: Тимаева М.П., БПИ2302 - Целевая оценка; выбранный трек 5, если нужен: TODO - Commit/PR: TODO после публикации - Использованные источники, библиотеки и помощь ИИ: TODO @@ -19,6 +19,7 @@ docker compose --env-file .env config --quiet docker compose --env-file .env up -d --build --wait # postgres (healthy) -> migrate (один раз, exit 0) -> api x API_REPLICAS (healthy) -> lb (healthy) make test # smoke через LB, http://localhost:8080 +python scripts/show_instances.py # одна сессия, разные X-Instance-Id make test-idempotency # один ключ параллельно через разные реплики make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd docker compose --env-file .env up -d --wait # вернуть остановленную реплику @@ -47,26 +48,39 @@ client -> lb (nginx :8080, только loopback) -> api-1, api-2, … (порт ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------- | +| Стенд в compose.yaml, ≥2 API, одна точка входа, общая БД и сессии | `docker compose --env-file .env down -v`, `config --quiet`, `up -d --build --wait` | config OK; `api-1`, `api-2`, `lb`, `postgres` healthy, `migrate` Exited (0); порт публикует только `lb` | `evidence/smoke.txt` (docker compose ps), `compose.yaml` | +| Одна сессия обслуживается разными репликами, вход один раз | `python scripts/show_instances.py` | 6 запросов с одним токеном: 3 на `8b33a91fafee`, 3 на `a87734e887a5`, все 200/201; ID совпадают с hostname контейнеров `api-1`/`api-2` | `evidence/instances.txt` | +| Остановка любой одной API-реплики, запись восстанавливается ≤10 с | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd`, между прогонами `docker compose up -d --wait` | 3 прогона PASS: до отказа 2 реплики, 127–128 успешных записей, 0 ошибок, первая запись через 1,17–1,27 с; остановлены `api-1` (прогоны 1, 2) и `api-2` (прогон 3), статус Exited (0) | `evidence/failover.txt` | +| Повтор POST с тем же Idempotency-Key не создаёт дубль (конкурентно, через разные реплики) | `make test-idempotency` | 5 ключей × 20 параллельных POST: по 10 ответов с каждой реплики, один и тот же Message, итого 5 сообщений; другое тело с тем же ключом → 409 | `evidence/idempotency.txt` | +| Неподтверждённая запись повторяется с тем же ключом при отказе | `make test-failover` (тест повторяет неопределённый POST с исходным ключом) | дублей нет, подтверждённые сообщения на месте | `evidence/failover.txt` | +| make check, make test | `make check`, `make test` | PASS, exit 0 | `evidence/smoke.txt` | +| Persistence: перезапуск lb, api, postgres с сохранением volume и токена | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS; те же ID контейнеров с новым uptime (перезапущены все), volume `pgdata` на месте, прежний токен работает | `evidence/persistence.txt` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Все прогоны 2026-11-07 на commit `8cd25e4` (+ `scripts/show_instances.py`), Docker 29.4.3, Windows 11, один хост. На Windows `make test-failover`/`test-persistence` запускались той же командой из `Makefile` напрямую (`python tests/resilience.py ...`): GNU Make под cmd не находит `test` в guard-строке, на результат это не влияет. + +Цель failover выбирает тест: реплика, обслужившая первое сообщение. При round robin это детерминированно `api-1`, поэтому перед прогоном 3 через LB отправлен один `GET /health/ready`, чтобы сдвинуть очередь, и тест остановил `api-2`. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Профиль failover-теста: 15 с параллельной записи сообщений, пока hook останавливает одну реплику (`docker stop --time 8`, SIGTERM). Нагрузочные эксперименты (RPS, перцентили) — уровень 4/5, здесь не заявлены. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| `stop-one` реплики, обслужившей тест (api-1), под записью | запись продолжается ≤10 с, дублей нет | 126 успешных записей, 0 ошибок, первая запись после stop через 1,6 с (`evidence/failover.txt`) | ~1,6 с / нет | -| `stop-one` второй реплики (api-2) после восстановления первой | то же | 126 записей, 0 ошибок, 1,7 с | ~1,7 с / нет | -| `restart.sh`: lb, api, postgres | данные и старый токен сохранены | PASS (`evidence/persistence.txt`) | нет потерь | -| 5 ключей × 20 параллельных POST через LB | одно сообщение на ключ | по 10 ответов с каждой реплики, 5 сообщений (`evidence/idempotency.txt`) | — | +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| ---------------------------------------------------------------------- | ------------------------------------- | ------------------------------------------- | ------------------------------------ | +| `stop-one` реплики, обслужившей тест (`api-1`), под записью, 2 прогона | запись продолжается ≤10 с, дублей нет | 127 и 128 успешных записей, 0 ошибок | 1,27 с и 1,17 с / нет | +| `stop-one` второй реплики (`api-2`) после восстановления первой | то же | 127 записей, 0 ошибок | 1,21 с / нет | +| `restart.cmd`: lb, api, postgres | данные и старый токен сохранены | PASS, volume сохранён | нет потерь | +| 5 ключей × 20 параллельных POST через LB | одно сообщение на ключ | по 10 ответов с каждой реплики, 5 сообщений | — | + +Ошибок в переходный период не было: Nginx при `connect refused` сразу отправляет запрос на другую реплику (`proxy_next_upstream`), а для POST сообщений это безопасно благодаря Idempotency-Key. «Время восстановления» в тесте — это время от старта hook до первой успешной записи, а не время обнаружения отказа. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Выдерживается**: отказ процесса или контейнера одной API-реплики, пока работает хотя бы одна другая. Также полный перезапуск стенда с сохранением volume. +- **Не выдерживается**: отказ PostgreSQL и Nginx. Это единственные точки отказа: БД не реплицирована, LB один. При их остановке API недоступен до перезапуска, подтверждённые данные сохраняются в volume. +- **Один хост**: все контейнеры на одной машине, поэтому отказ узла или Docker Engine останавливает весь стенд. Проверен отказ процесса, а не узла. +- **Нет активных health-check в LB**: open-source Nginx исключает реплику пассивно (ошибка соединения, `max_fails=1 fail_timeout=5s`) и при обновлении DNS раз в 2 с. Зависшая, но принимающая соединения реплика будет отбрасываться только по `proxy_read_timeout 10s`. +- **Порядок сообщений** между репликами зависит от общих часов хоста: `created_at` назначает приложение. На нескольких узлах понадобятся время БД или последовательность. +- **Повтор LB**: Nginx повторяет на другой реплике только POST сообщений. POST пользователей, чатов и сессий после обрыва соединения клиент должен проверить сам, так как ключа идемпотентности у них нет. +- Лимиты CPU/RAM, rolling update и измерения под нагрузкой не делались: это уровень 4. diff --git a/evidence/failover.txt b/evidence/failover.txt index 858e2a75..8778b332 100644 --- a/evidence/failover.txt +++ b/evidence/failover.txt @@ -1,34 +1,41 @@ -# failover run 2026-11-05T16:51:38.6795653+03:00 -## run 1 — before -messenger-lab3-api-1 5dce519fa2a3 Up 22 seconds (healthy) -messenger-lab3-api-2 d0fe578a76ab Up 22 seconds (healthy) +# failover run 2026-11-07T17:39:07.3980307+03:00 +# command (= make test-failover ACTION_SCRIPT=scripts/stop-one.cmd): python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://localhost:8080 +## run 1 - before (shift requests through LB: 0) +messenger-lab3-api-1 8b33a91fafee Up 20 seconds (healthy) +messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy) Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 1.598} +{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 1.27} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. -## run 1 — after -messenger-lab3-api-1 5dce519fa2a3 Exited (0) 13 seconds ago -messenger-lab3-api-2 d0fe578a76ab Up 39 seconds (healthy) -## run 2 — before -messenger-lab3-api-1 5dce519fa2a3 Up 6 seconds (healthy) -messenger-lab3-api-2 d0fe578a76ab Up 48 seconds (healthy) +exit=0 +## run 1 - after +messenger-lab3-api-1 8b33a91fafee Exited (0) 14 seconds ago +messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy) +## restored (docker compose --env-file .env up -d --wait) +messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy) +messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy) +## run 2 - before (shift requests through LB: 0) +messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy) +messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy) Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 1.575} +{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 1.166} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. -## run 2 — after -messenger-lab3-api-1 5dce519fa2a3 Exited (0) 13 seconds ago -messenger-lab3-api-2 d0fe578a76ab Up About a minute (healthy) -## restored -messenger-lab3-api-1 5dce519fa2a3 Up 6 seconds (healthy) -messenger-lab3-api-2 d0fe578a76ab Up About a minute (healthy) -## run 3 (target api-2) — before -messenger-lab3-api-1 5dce519fa2a3 Up 17 seconds (healthy) -messenger-lab3-api-2 d0fe578a76ab Up About a minute (healthy) +exit=0 +## run 2 - after +messenger-lab3-api-1 8b33a91fafee Exited (0) 14 seconds ago +messenger-lab3-api-2 a87734e887a5 Up 3 minutes (healthy) +## restored (docker compose --env-file .env up -d --wait) +messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy) +messenger-lab3-api-2 a87734e887a5 Up 3 minutes (healthy) +## run 3 - before (shift requests through LB: 1) +messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy) +messenger-lab3-api-2 a87734e887a5 Up 3 minutes (healthy) Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 1.707} +{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 1.213} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. -## run 3 — after -messenger-lab3-api-1 5dce519fa2a3 Up 34 seconds (healthy) -messenger-lab3-api-2 d0fe578a76ab Exited (0) 13 seconds ago -## restored -messenger-lab3-api-1 5dce519fa2a3 Up 44 seconds (healthy) -messenger-lab3-api-2 d0fe578a76ab Up 6 seconds (healthy) +exit=0 +## run 3 - after +messenger-lab3-api-1 8b33a91fafee Up 22 seconds (healthy) +messenger-lab3-api-2 a87734e887a5 Exited (0) 14 seconds ago +## restored (docker compose --env-file .env up -d --wait) +messenger-lab3-api-1 8b33a91fafee Up 31 seconds (healthy) +messenger-lab3-api-2 a87734e887a5 Up 6 seconds (healthy) diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt index 70fdb42d4729bad464f4b186779971a6ff6fa633..41104879e96d7b3162cd6c4c29e1bcb771e3908e 100644 GIT binary patch literal 698 zcmcJNL2JS=6oBvj6%Rdln9-zdY!+~^+fJLHmz_qe&$SratS=pw;V;`^KWTqq-R3~3 z3@@GnkC%^+eEEF;=tI83tt&kiX#-VVK*+-h6N2$%Arc);v{WO>V;N48D;{aigMJ^j zW`p37JUhNVdtBT-+)rKOv$6<&51_Qlou_e{VY+!bS|7Ov7+V@bR#z5!nfFeQ$F@w3 z%}VlGam9NERBY4dXg#VP2C(22_4{(?JDG$B?8ywT1 literal 1066 zcmdT@%L>9U5S+8%KlIj%LLbJe2f;rO{DNAmh_%*MMG=2polOK450%n`5;l+B$?hcV z+^#t`*dxUZ790#Xdt5E#h*2P8H03J$7QHTQtxp9)EVvV|U^66GBBHmT=L{Q)IU`0C zZ}l-}#H0C)NEMe=Hf_^oR)1Rh~@UTA7wJFigu|^vTq#m1#MH z>VNA04b#p~GaX}u35RsKqOI85C#{MjPexrCJ3OZ)95KykXiFuE(sml%#_*BhU2&qX T1GN_FY}M4KzEq#!Oi$kd!_1!< diff --git a/evidence/instances.txt b/evidence/instances.txt new file mode 100644 index 00000000..09f89bca --- /dev/null +++ b/evidence/instances.txt @@ -0,0 +1,16 @@ +# X-Instance-Id run 2026-11-07T17:41:06.5911429+03:00 +## python scripts/show_instances.py --base-url http://localhost:8080 +login + chat via LB http://localhost:8080: one session token for user 2ec10789-7814-4751-9e23-7b85215c262e (token not printed) +1. GET /api/v1/users/me -> 200 X-Instance-Id=8b33a91fafee +2. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=a87734e887a5 +3. GET /api/v1/users/me -> 200 X-Instance-Id=8b33a91fafee +4. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=a87734e887a5 +5. GET /api/v1/users/me -> 200 X-Instance-Id=8b33a91fafee +6. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=a87734e887a5 +PASS: same session served by 2 replicas without re-login {'8b33a91fafee': 3, 'a87734e887a5': 3} +exit=0 +## docker compose ps api (X-Instance-Id = container hostname = short container ID) +messenger-lab3-api-1 8b33a91fafee Up 31 seconds (healthy) +messenger-lab3-api-2 a87734e887a5 Up 31 seconds (healthy) +/messenger-lab3-api-1 hostname=8b33a91fafee +/messenger-lab3-api-2 hostname=a87734e887a5 diff --git a/evidence/persistence.txt b/evidence/persistence.txt index 75b5beda..7586ea5e 100644 --- a/evidence/persistence.txt +++ b/evidence/persistence.txt @@ -1,12 +1,18 @@ -# before -messenger-lab3-api-1 5dce519fa2a3 Up 52 seconds (healthy) -messenger-lab3-api-2 d0fe578a76ab Up 14 seconds (healthy) -messenger-lab3-lb-1 ece2ac882f2d Up About a minute (healthy) -messenger-lab3-postgres-1 0ed6d04de263 Up 7 minutes (healthy) +# persistence run 2026-11-07T17:40:29.1339292+03:00 +# command (= make test-persistence ACTION_SCRIPT=scripts/restart.cmd): python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://localhost:8080 +## before +messenger-lab3-api-1 8b33a91fafee Up 38 seconds (healthy) +messenger-lab3-api-2 a87734e887a5 Up 14 seconds (healthy) +messenger-lab3-lb-1 4ba96e32076f Up 3 minutes (healthy) +messenger-lab3-migrate-1 18f3889ef9a7 Exited (0) 14 seconds ago +messenger-lab3-postgres-1 4c60c6522907 Up 3 minutes (healthy) Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. -# after -messenger-lab3-api-1 5dce519fa2a3 Up 12 seconds (healthy) -messenger-lab3-api-2 d0fe578a76ab Up 12 seconds (healthy) -messenger-lab3-lb-1 ece2ac882f2d Up 5 seconds (healthy) -messenger-lab3-postgres-1 0ed6d04de263 Up 17 seconds (healthy) +exit=0 +## after (same container IDs, fresh uptime = restarted; volume pgdata kept) +messenger-lab3-api-1 8b33a91fafee Up 11 seconds (healthy) +messenger-lab3-api-2 a87734e887a5 Up 11 seconds (healthy) +messenger-lab3-lb-1 4ba96e32076f Up 5 seconds (healthy) +messenger-lab3-migrate-1 18f3889ef9a7 Exited (0) 12 seconds ago +messenger-lab3-postgres-1 4c60c6522907 Up 15 seconds (healthy) +volume messenger-lab3_pgdata diff --git a/evidence/smoke.txt b/evidence/smoke.txt index 9155c94f2bba3373ba257ce54467ccbf29946ffc..dec53a71f9d3f54445a25498d72c75b7a1e27752 100644 GIT binary patch literal 639 zcma))zi!(w5Qld?#UWl=w6ZD5`eOwdvb9TMAnwqqI8i5yFhvqPs)(PhLm#QHP)aQ_ z8E7X0BtPEwyW_7PKR3XWy%6}>KrZqs&9YR~r>t&^sx6ysDfVSv)gMIB3XyC!&|7&S z57JJ~GJ)e{B4#pY@~OtINYSY@gQhDAJY)k7MDWEyb^s=6P0#RdB-CNNzQ0B0fK6Q& zduo~*%l{>H7fbFtJXBQVqOOJ&EdE_GW(a3byx_7DvMTfZAnFCrZ{e{46UKJQk>mxA zUr(Q&pB~&bjMgN77GUX>3%pyee2bBTM;W&6nj9ZrURtojpp_xu3uy}000%$IFnjt4 z+Dg>mX<$Ip!*P8W_-X>a-UB+PmBgST^z?j!Kpd#wCiJSp!#bSRZP~u9ww&;89eDJC zH0_WnoxO&;F$CA{cB{Y`o6QBZK4ffJ^6lP@_U`}dXt?<<1V$ov z&Z;+?YGmM{Yv^dLA6;gaTAaMN+8gvweRR57H<{IckqmZclC`fys&uc %d X-Instance-Id=%s' % (i + 1, what, r['status'], instance)) + check(len(served) >= 2, 'all requests hit one replica; LB does not balance') + print('PASS: same session served by %d replicas without re-login %s' % (len(served), dict(served))) + + +if __name__ == '__main__': + run(main) -- 2.54.0 From eb4e00e412b1c426d6fe3e6ca750a3f4b0f5d5b2 Mon Sep 17 00:00:00 2001 From: Sunrise443 Date: Sat, 7 Nov 2026 18:18:43 +0300 Subject: [PATCH 3/4] level 4, changed to HAProxy --- .env.example | 17 +- .gitattributes | 1 + Dockerfile | 7 +- Makefile | 17 +- REPORT.md | 146 +++- app/__main__.py | 33 +- app/config.py | 59 +- app/drain.py | 4 + app/main.py | 7 +- app/routes.py | 5 +- compose.yaml | 107 ++- evidence/failover-series-kill.json | 961 ++++++++++++++++++++++ evidence/failover-series-kill.txt | 7 + evidence/failover-series.json | 1064 +++++++++++++++++++++++++ evidence/failover-series.txt | 7 + evidence/failover.txt | 55 +- evidence/idempotency.txt | 17 +- evidence/instances.txt | 24 +- evidence/limits.txt | 30 + evidence/persistence.txt | 27 +- evidence/rolling-update-migration.txt | 24 + evidence/rolling-update.json | 955 ++++++++++++++++++++++ evidence/rolling-update.txt | 19 + evidence/smoke.txt | 54 +- infra/haproxy/haproxy.cfg | 79 ++ infra/nginx/nginx.conf | 66 -- migrations/003_session_indexes.sql | 4 + scripts/failover_series.py | 303 +++++++ scripts/lb-status.cmd | 3 + scripts/lb-status.sh | 6 + scripts/rolling-update.cmd | 3 + scripts/rolling-update.sh | 47 ++ scripts/stop-one.sh | 5 +- 33 files changed, 3932 insertions(+), 231 deletions(-) create mode 100644 app/drain.py create mode 100644 evidence/failover-series-kill.json create mode 100644 evidence/failover-series-kill.txt create mode 100644 evidence/failover-series.json create mode 100644 evidence/failover-series.txt create mode 100644 evidence/limits.txt create mode 100644 evidence/rolling-update-migration.txt create mode 100644 evidence/rolling-update.json create mode 100644 evidence/rolling-update.txt create mode 100644 infra/haproxy/haproxy.cfg delete mode 100644 infra/nginx/nginx.conf create mode 100644 migrations/003_session_indexes.sql create mode 100755 scripts/failover_series.py create mode 100644 scripts/lb-status.cmd create mode 100755 scripts/lb-status.sh create mode 100644 scripts/rolling-update.cmd create mode 100755 scripts/rolling-update.sh diff --git a/.env.example b/.env.example index 9920b3df..1d0d49ab 100644 --- a/.env.example +++ b/.env.example @@ -4,16 +4,21 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: +# Not passed to containers: compose.yaml gives api/migrate DB_HOST/DB_USER/DB_NAME and +# DB_PASSWORD_FILE=/run/secrets/postgres_password (Compose secret built from POSTGRES_PASSWORD below). +# The app also accepts DATABASE_URL_FILE or DATABASE_URL instead. +# Sessions live in PostgreSQL; no Redis/Valkey, CACHE_URL_FILE is unused. SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -# compose.yaml builds DATABASE_URL from these; pick your own local password. +# Pick your own local password. It reaches containers only as the postgres_password secret file. POSTGRES_USER=messenger POSTGRES_PASSWORD=change-me-locally POSTGRES_DB=messenger -SHUTDOWN_TIMEOUT_SECONDS=5 +# SIGTERM: readiness 503 for DRAIN_SECONDS (> LB detection 3 s), then in-flight requests get +# SHUTDOWN_TIMEOUT_SECONDS. Keep DRAIN + SHUTDOWN < stop_grace_period (10 s). +DRAIN_SECONDS=4 +SHUTDOWN_TIMEOUT_SECONDS=3 +# Image tag messenger-lab3-app:, shown as X-App-Version; scripts/rolling-update.sh sets it. +APP_VERSION=dev LB_PORT=8080 diff --git a/.gitattributes b/.gitattributes index 856422ac..84f24cd7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ *.sh text eol=lf *.conf text eol=lf +*.cfg text eol=lf *.cmd text eol=crlf diff --git a/Dockerfile b/Dockerfile index e7d13f91..7b306b09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,8 @@ ENV APP_HOST=0.0.0.0 \ APP_PORT=8080 \ APP_ENV=lab \ LOG_LEVEL=info \ - SHUTDOWN_TIMEOUT_SECONDS=5 \ + DRAIN_SECONDS=4 \ + SHUTDOWN_TIMEOUT_SECONDS=3 \ SESSION_TTL_SECONDS=3600 RUN groupadd --gid 10001 app \ @@ -26,6 +27,10 @@ COPY --chown=app:app contracts/openapi.json ./contracts/openapi.json USER app +# Visible as X-App-Version: shows which build served a request during a rolling update. +ARG APP_VERSION=dev +ENV APP_VERSION=${APP_VERSION} + EXPOSE 8080 HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 \ diff --git a/Makefile b/Makefile index 6469997d..7207bf28 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,13 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover test-idempotency +APP_VERSION ?= + +.PHONY: help check validate test load test-persistence test-failover test-idempotency failover-series rolling-update lb-status help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "failover-series, rolling-update APP_VERSION=..., lb-status: level 4 experiments (Docker required)" check: $(PYTHON) scripts/check_contract.py @@ -31,3 +34,15 @@ test-failover: test-idempotency: $(PYTHON) scripts/check_idempotency.py --base-url "$(BASE_URL)" + +# Level 4: stop each API replica in turn under load (restoring the previous one); RPS, errors, recovery. +failover-series: + $(PYTHON) scripts/failover_series.py --base-url "$(BASE_URL)" --output evidence/failover-series.json + +# Level 4: one-at-a-time replacement under load. Usage: make rolling-update APP_VERSION=v3 +rolling-update: + @test -n "$(APP_VERSION)" || (echo "Set APP_VERSION of the new build"; exit 2) + APP_VERSION="$(APP_VERSION)" $(PYTHON) scripts/failover_series.py --mode action --action scripts/rolling-update.sh --base-url "$(BASE_URL)" --output evidence/rolling-update.json + +lb-status: + sh scripts/lb-status.sh diff --git a/REPORT.md b/REPORT.md index c0e8c605..f816800b 100644 --- a/REPORT.md +++ b/REPORT.md @@ -7,9 +7,9 @@ ## Запуск из чистого clone -Вариант: **Docker Compose + Nginx**. Нужны Docker Engine с Compose v2 (проверено на Docker 29.4, Compose 5.1), Python 3.9+ для тестов; на Windows скрипты `.sh` выполняются через Git Bash (обёртки `scripts/*.cmd`). +Вариант: **Docker Compose + HAProxy**. Нужны Docker Engine с Compose v2 (проверено на Docker 29.4, Compose 5.1) и Python 3.9+ для тестов. На Windows скрипты `.sh` выполняются через Git Bash (обёртки `scripts/*.cmd`). -Образы: `postgres:16.10-alpine`, `nginx:1.28.0-alpine` (закреплены digest в `compose.yaml`), приложение собирается из `Dockerfile` на `python:3.14.8-slim-bookworm@sha256:…`. +Образы: `postgres:16.10-alpine` и `haproxy:3.2.26-alpine` (LTS), оба закреплены digest в `compose.yaml`. Приложение собирается из `Dockerfile` на `python:3.14.8-slim-bookworm@sha256:…` в образ `messenger-lab3-app:${APP_VERSION}`. Одна последовательность запуска: @@ -24,6 +24,13 @@ make test-idempotency # один ключ параллельно ч make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd docker compose --env-file .env up -d --wait # вернуть остановленную реплику make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/restart.cmd +# уровень 4 +sh scripts/lb-status.sh # пул HAProxy: реплика, IP, UP/DOWN +make failover-series # поочерёдная остановка каждой реплики под нагрузкой +python scripts/failover_series.py --kill --output evidence/failover-series-kill.json # то же, SIGKILL +docker compose --env-file .env build api migrate # с APP_VERSION=v3 в окружении +make rolling-update APP_VERSION=v3 # Windows: scripts/failover_series.py --mode action --action scripts/rolling-update.cmd +# после rolling update записать APP_VERSION=v3 в .env ``` Остановка без потери данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v` (удаляет volume `pgdata`). @@ -31,56 +38,117 @@ make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/res ## Архитектура и решения ``` -client -> lb (nginx :8080, только loopback) -> api-1, api-2, … (порт не публикуется) -> postgres (volume pgdata) - migrate (one-shot) ----------------------^ + edge network data network (internal: true) +client -> lb (HAProxy :8080, loopback) -> api-1, api-2, … (порт не публикуется) -> postgres (volume pgdata) + migrate (one-shot) ------------^ ``` -- **Источник истины — PostgreSQL**: пользователи, чаты, сообщения, сессии (`sessions`, токен хранится как SHA-256) и результаты идемпотентности (`idempotency_keys`). В API нет локального состояния, от которого зависит корректность. Redis из lab2 убран: он был только кэшем, а для общего хранилища сессий достаточно PostgreSQL (`SESSION_STORE=postgres`). -- **Миграции** выполняет отдельный сервис `migrate` (`python -m app.migrate`) до старта реплик (`depends_on: service_completed_successfully`). Реплики схему не меняют; `/health/ready` возвращает 503, пока в `schema_migrations` нет всех миграций этой сборки. Параллельный запуск мигратора сериализуется `pg_advisory_lock`. -- **X-Instance-Id** = hostname контейнера (короткий Docker ID), стабилен на время жизни процесса. `stop-one.sh` по нему находит ровно один running-контейнер сервиса `api` этого проекта и делает `docker stop` (SIGTERM, затем SIGKILL через 8 с). `restart: unless-stopped` не поднимает вручную остановленный контейнер, поэтому реплика остаётся остановленной до `docker compose up -d`. -- **Балансировщик**: round robin без `ip_hash`/sticky. `server api:8080 resolve` + `resolver 127.0.0.11 valid=2s`: Nginx раз в 2 с перечитывает DNS Docker, так что остановленные и добавленные (`--scale api=N`) реплики уходят из пула/входят в него без перезапуска LB. Пассивная проверка: `max_fails=1 fail_timeout=5s`, `proxy_connect_timeout 1s`, `proxy_next_upstream error timeout http_502 http_504`. Активных проверок в open-source Nginx нет; readiness реплик проверяет Docker healthcheck (`/health/ready` каждые 3 с), и LB стартует только после healthy API. -- **Повтор на другой реплике**: для неотправленного запроса Nginx всегда пробует следующую реплику. Уже отправленный POST повторяется только для `/api/v1/chats/{id}/messages` (`non_idempotent`), потому что там обязателен `Idempotency-Key`. POST пользователей/чатов/сессий LB не повторяет. -- **Graceful shutdown**: на SIGTERM uvicorn перестаёт принимать соединения и дорабатывает текущие запросы до `SHUTDOWN_TIMEOUT_SECONDS=5`; `stop_grace_period: 8s` больше этого. `keepalive_timeout 4s` в upstream меньше keep-alive uvicorn (5 с), чтобы LB не переиспользовал закрываемое API соединение. -- **Идемпотентность**: в одной транзакции удаляется истёкший (старше `IDEMPOTENCY_TTL_SECONDS`) ключ, затем `INSERT … ON CONFLICT DO NOTHING` в `idempotency_keys` (PK: пользователь + метод + путь + ключ) вместе с готовым ответом и SHA-256 канонического JSON тела, затем вставка сообщения. Конкурентный дубль на другой реплике ждёт на уникальном индексе до COMMIT первой транзакции и получает сохранённый ответ (тот же 201 и Message) либо 409 при другом теле. При ошибке транзакция откатывается целиком, ключ не фиксируется, повтор с тем же ключом безопасен. Сессия и членство в чате проверяются при каждом повторе. -- Время сообщений назначает приложение; порядок между репликами опирается на общие часы хоста (учебное упрощение, одна машина). +| Роль | Compose service | Образ | Health/readiness | Сети | Volume / секреты | Лимит CPU / RAM (резерв) | +| -------- | --------------- | ----------------------------- | ----------------------------------------------------- | ---------- | -------------------------------------------------- | ------------------------ | +| LB | `lb` | `haproxy:3.2.26-alpine@…` | `monitor-uri /lb/health`; активные проверки API | edge | только read-only конфиг, секретов нет | 0.5 / 128M (0.1 / 32M) | +| API × N | `api` | `messenger-lab3-app:<версия>` | `/health/live`, `/health/ready` (БД, миграции, drain) | edge, data | `/run/secrets/postgres_password` (uid 10001, 0400) | 1.0 / 256M (0.25 / 128M) | +| Миграции | `migrate` | тот же образ | one-shot, `service_completed_successfully` | data | тот же секрет | 0.5 / 128M | +| БД | `postgres` | `postgres:16.10-alpine@…` | `pg_isready` каждые 2 с | data | `pgdata`, `POSTGRES_PASSWORD_FILE` | 1.0 / 512M (0.25 / 256M) | -Соответствие `.env.example`: `DATABASE_URL` собирается в `compose.yaml` из `POSTGRES_*` (вариант `DATABASE_URL_FILE` тоже поддерживается кодом); `API_REPLICAS` → `deploy.replicas`; `IDEMPOTENCY_TTL_SECONDS`, `SESSION_TTL_SECONDS`, `SHUTDOWN_TIMEOUT_SECONDS` передаются в API; `CACHE_URL_FILE` не используется. +- **Источник истины — PostgreSQL**: пользователи, чаты, сообщения, сессии (`sessions`, токен хранится как SHA-256) и результаты идемпотентности (`idempotency_keys`). В API нет локального состояния, от которого зависит корректность. Redis из lab2 убран: он был только кэшем, а для общего хранилища сессий достаточно PostgreSQL. +- **Изоляция**: наружу публикуется только `lb` и только на loopback. У API нет `ports`, `docker port` пуст. LB находится только в сети `edge` и не резолвит `postgres`. Сеть `data` объявлена `internal`, без выхода наружу. Volume `pgdata` смонтирован только в `postgres`. `container_name` нигде нет, поэтому `--scale api=N` работает (`evidence/limits.txt`). +- **Секреты**: пароль БД лежит только в незакоммиченном `.env` и попадает в контейнеры как Compose secret `postgres_password` (`environment: POSTGRES_PASSWORD`). В env контейнеров есть только путь `*_PASSWORD_FILE`. API и мигратор собирают `DATABASE_URL` из `DB_HOST/DB_USER/DB_NAME` и файла (`app/config.py`). У LB секретов нет. +- **Миграции** выполняет отдельный сервис `migrate` (`python -m app.migrate`) до старта реплик, а при rolling update — один раз через `docker compose run --rm migrate`. Реплики схему не меняют. `/health/ready` возвращает 503, пока в `schema_migrations` нет всех миграций **этой** сборки. Параллельный запуск мигратора сериализуется `pg_advisory_lock`. +- **X-Instance-Id** = hostname контейнера (короткий Docker ID). **X-App-Version** = `APP_VERSION` образа. По нему видна смена версии при rolling update. +- **Балансировщик**: HAProxy, round robin без sticky. `server-template api 1-10 api:8080` с `resolvers docker` (Docker DNS `127.0.0.11`, `hold valid 1s`), поэтому реплики, добавленные через `--scale` или rolling update и остановленные, входят в пул и выходят из него без перезапуска LB. +- **Повтор на другой реплике**: `retries 2`, `option redispatch`, `retry-on conn-failure empty-response response-timeout`. Для POST, кроме `/api/v1/chats/{id}/messages`, стоит `http-request disable-l7-retry`, то есть они повторяются только если соединение с репликой не было установлено. POST сообщений можно повторять, потому что `Idempotency-Key` обязателен. +- **Идемпотентность**: в одной транзакции удаляется истёкший ключ, затем `INSERT … ON CONFLICT DO NOTHING` в `idempotency_keys` (PK: пользователь + метод + путь + ключ) вместе с готовым ответом и SHA-256 тела, затем вставка сообщения. Конкурентный дубль на другой реплике ждёт на уникальном индексе до COMMIT первой транзакции и получает сохранённый ответ (тот же 201) либо 409 при другом теле. При ошибке ключ не фиксируется, и повтор безопасен. + +### Health, timeouts и SIGTERM: согласованные тайминги + +``` +SIGTERM -> readiness 503 сразу; запросы ещё обслуживаются + -> HAProxy: 2 неудачные проверки (inter 1s, timeout 1s) -> DOWN, не более 3 с + -> через DRAIN_SECONDS=4 uvicorn перестаёт принимать соединения, дорабатывает текущие за SHUTDOWN_TIMEOUT_SECONDS=3 + -> exit 0 примерно через 5 с; stop_grace_period 10 s (stop-one: docker stop --time 9), SIGKILL не нужен +``` + +| Параметр | Значение | Где | +| -------------------------------------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------- | +| Период активной проверки LB `/health/ready` | 1 с, `timeout check 1s`, `fall 2`, `rise 2` | `infra/haproxy/haproxy.cfg` | +| Обновление DNS реплик | 1 с (`hold valid 1s`, `hold obsolete 0s`) | там же | +| `timeout connect / server / client / http-request` | 1 с / 10 с / 30 с / 10 с | там же | +| Drain после SIGTERM / завершение текущих запросов | 4 с / 3 с | `DRAIN_SECONDS`, `SHUTDOWN_TIMEOUT_SECONDS` | +| `stop_grace_period` | 10 с (> 4 + 3) | `compose.yaml` | +| Keep-alive uvicorn | 10 с (> `pool-purge-delay` HAProxy 5 с) | `KEEPALIVE_TIMEOUT_SECONDS` | +| Docker healthcheck API | каждые 3 с, 3 попытки; только порядок старта и rolling update, LB его не использует | `compose.yaml` | + +Максимальное время обнаружения отказа: + +- **Плановая остановка (SIGTERM)**: реплика выводится заранее, не более \(2 \cdot 1 + 1 = 3\) с, и раньше, чем закрывается сокет. Замер: последний запрос на останавливаемую реплику пришёл через 1,76–2,05 с после SIGTERM, ошибок 0. +- **Падение процесса (SIGKILL)**: запрос к мёртвому IP завершается по `timeout connect 1s` и уходит на другую реплику (redispatch). Пул очищается по DNS примерно за 1 с или по проверкам не более чем за 3 с. Замер: максимальная пауза без успешного ответа 0,96–0,98 с, ошибок 0. +- **Зависшая реплика, принимающая соединения**: проверки дают DOWN не более чем через 3 с. Запросы, уже отправленные на неё, ждут до `timeout server 10s` и затем повторяются (`response-timeout`) для GET и POST сообщений. Экспериментально не проверялось. +- **Возврат в пул**: `rise 2`, то есть около 2 с после готовности. Замер от `docker start` до первого ответа реплики: 4,5–5,2 с, включая старт приложения. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------- | -| Стенд в compose.yaml, ≥2 API, одна точка входа, общая БД и сессии | `docker compose --env-file .env down -v`, `config --quiet`, `up -d --build --wait` | config OK; `api-1`, `api-2`, `lb`, `postgres` healthy, `migrate` Exited (0); порт публикует только `lb` | `evidence/smoke.txt` (docker compose ps), `compose.yaml` | -| Одна сессия обслуживается разными репликами, вход один раз | `python scripts/show_instances.py` | 6 запросов с одним токеном: 3 на `8b33a91fafee`, 3 на `a87734e887a5`, все 200/201; ID совпадают с hostname контейнеров `api-1`/`api-2` | `evidence/instances.txt` | -| Остановка любой одной API-реплики, запись восстанавливается ≤10 с | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd`, между прогонами `docker compose up -d --wait` | 3 прогона PASS: до отказа 2 реплики, 127–128 успешных записей, 0 ошибок, первая запись через 1,17–1,27 с; остановлены `api-1` (прогоны 1, 2) и `api-2` (прогон 3), статус Exited (0) | `evidence/failover.txt` | -| Повтор POST с тем же Idempotency-Key не создаёт дубль (конкурентно, через разные реплики) | `make test-idempotency` | 5 ключей × 20 параллельных POST: по 10 ответов с каждой реплики, один и тот же Message, итого 5 сообщений; другое тело с тем же ключом → 409 | `evidence/idempotency.txt` | -| Неподтверждённая запись повторяется с тем же ключом при отказе | `make test-failover` (тест повторяет неопределённый POST с исходным ключом) | дублей нет, подтверждённые сообщения на месте | `evidence/failover.txt` | -| make check, make test | `make check`, `make test` | PASS, exit 0 | `evidence/smoke.txt` | -| Persistence: перезапуск lb, api, postgres с сохранением volume и токена | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS; те же ID контейнеров с новым uptime (перезапущены все), volume `pgdata` на месте, прежний токен работает | `evidence/persistence.txt` | +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| Стенд в compose.yaml, ≥2 API, одна точка входа, общая БД и сессии | `down -v`, `config --quiet`, `up -d --build --wait` | config OK; `api-1`, `api-2`, `lb`, `postgres` healthy; `migrate` применил 001–003 и Exited (0); в HAProxy 2 реплики UP | `evidence/smoke.txt` | +| Одна сессия обслуживается разными репликами | `python scripts/show_instances.py` | 6 запросов с одним токеном: по 3 на каждую из 2 реплик | `evidence/instances.txt` | +| Остановка любой одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` ×2, между прогонами `up -d --wait` | 2 PASS: остановлены `api-2`, затем `api-1`, Exited (0); 121 и 115 записей, 0 ошибок; HAProxy убрал реплику из пула | `evidence/failover.txt` | +| Повтор POST с тем же Idempotency-Key | `make test-idempotency` | 5 ключей × 20 параллельных POST: по 10 ответов с каждой реплики, один Message на ключ, другое тело → 409 | `evidence/idempotency.txt` | +| make check, make test, persistence | `make check`, `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS; после рестарта lb, api, postgres те же ID, volume на месте, прежний токен работает | `evidence/smoke.txt`, `evidence/persistence.txt` | +| **4**: health/readiness, timeouts, SIGTERM согласованы; LB исключает недоступный backend | HAProxy active checks + drain; `scripts/lb-status.sh` | см. таблицу таймингов; останавливаемая реплика получает последний запрос через ≤2,05 с после SIGTERM; после выхода её нет в пуле | `infra/haproxy/haproxy.cfg`, `evidence/failover-series.json` | +| **4**: поочерёдная остановка каждой реплики: RPS, ошибки, время восстановления | `make failover-series`; `failover_series.py --kill` | по 2 реплики в каждом режиме, 0 ошибок, все подтверждённые записи на месте, дублей нет (таблица ниже) | `evidence/failover-series*.{txt,json}` | +| **4**: лимиты, нет портов API, секреты и хранилища только где нужно, нет `container_name` | `docker inspect`, `docker stats`, `docker port`, `nslookup` из lb | лимиты у всех 5 сервисов; порт только у lb (127.0.0.1/::1); пароль только файлом 0400; lb без секретов и без доступа к БД; `container_name`: 0 | `evidence/limits.txt` | +| **4**: rolling update под нагрузкой, миграции однократно и совместимо | `make rolling-update APP_VERSION=v3`; ранее `dev → v2` с миграцией 003 | 0 ошибок, максимальная пауза 0,07 с; v2 → v3 без снижения числа реплик ниже 2; при `dev → v2` миграция 003 применена один раз, и старые реплики `dev` продолжали работать | `evidence/rolling-update.{txt,json}`, `evidence/rolling-update-migration.txt` | -Все прогоны 2026-11-07 на commit `8cd25e4` (+ `scripts/show_instances.py`), Docker 29.4.3, Windows 11, один хост. На Windows `make test-failover`/`test-persistence` запускались той же командой из `Makefile` напрямую (`python tests/resilience.py ...`): GNU Make под cmd не находит `test` в guard-строке, на результат это не влияет. +Все прогоны 2026-11-07 на commit `124c83e` + рабочее дерево (до коммита уровня 4), Docker 29.4.3, Windows 11, один хост. `make test-failover/test-persistence` под Windows запускались той же командой из `Makefile` напрямую (`python tests/resilience.py ...`), потому что GNU Make под cmd не находит `test` в guard-строке. -Цель failover выбирает тест: реплика, обслужившая первое сообщение. При round robin это детерминированно `api-1`, поэтому перед прогоном 3 через LB отправлен один `GET /health/ready`, чтобы сдвинуть очередь, и тест остановил `api-2`. +Цель `make test-failover` выбирает тест: это реплика, обслужившая первое сообщение. Для второго прогона через LB отправлен один `GET /health/ready`, чтобы сдвинуть round robin, и тест остановил другую реплику. ## Эксперименты и отказы -Профиль failover-теста: 15 с параллельной записи сообщений, пока hook останавливает одну реплику (`docker stop --time 8`, SIGTERM). Нагрузочные эксперименты (RPS, перцентили) — уровень 4/5, здесь не заявлены. +Профиль нагрузки `scripts/failover_series.py`: 4 потока в закрытом цикле, каждый делает POST сообщения (с `Idempotency-Key`; неопределённый результат повторяется с тем же ключом) и GET 20 последних сообщений. Счёт идёт по секундам. Для каждой реплики по очереди: 10 с baseline, затем остановка, 15 с наблюдения, затем `docker start` того же контейнера, ожидание его ответов и 5 с паузы. В конце вся история чата читается заново: каждая подтверждённая запись должна быть ровно один раз. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| ---------------------------------------------------------------------- | ------------------------------------- | ------------------------------------------- | ------------------------------------ | -| `stop-one` реплики, обслужившей тест (`api-1`), под записью, 2 прогона | запись продолжается ≤10 с, дублей нет | 127 и 128 успешных записей, 0 ошибок | 1,27 с и 1,17 с / нет | -| `stop-one` второй реплики (`api-2`) после восстановления первой | то же | 127 записей, 0 ошибок | 1,21 с / нет | -| `restart.cmd`: lb, api, postgres | данные и старый токен сохранены | PASS, volume сохранён | нет потерь | -| 5 ключей × 20 параллельных POST через LB | одно сообщение на ключ | по 10 ответов с каждой реплики, 5 сообщений | — | +### Поочерёдная остановка реплик под нагрузкой -Ошибок в переходный период не было: Nginx при `connect refused` сразу отправляет запрос на другую реплику (`proxy_next_upstream`), а для POST сообщений это безопасно благодаря Idempotency-Key. «Время восстановления» в тесте — это время от старта hook до первой успешной записи, а не время обнаружения отказа. +| Воздействие | Реплика | RPS baseline → во время → после | Ошибки | Макс. пауза без успеха | Последний запрос на реплику после сигнала | Выход контейнера | RPS ≥90% baseline после выхода | Возврат в пул после `docker start` | +| ------------------------------- | ------- | ------------------------------- | ------ | ---------------------- | ----------------------------------------- | ---------------- | ------------------------------ | ---------------------------------- | +| `docker stop` (SIGTERM + drain) | api-1 | 129,0 → 114,8 → 113,1 | 0 | 0,09 с | 2,05 с | 5,06 с, exit 0 | 2,8 с | 5,2 с | +| `docker stop` (SIGTERM + drain) | api-2 | 131,5 → 145,2 → 165,4 | 0 | 0,09 с | 1,76 с | 4,88 с, exit 0 | 1,6 с | 4,9 с | +| `docker kill` (SIGKILL) | api-1 | 174,4 → 145,3 → 158,4 | 0 | 0,96 с | 0,12 с | 0,66 с, exit 137 | 2,3 с | 4,5 с | +| `docker kill` (SIGKILL) | api-2 | 177,9 → 146,5 → 139,0 | 0 | 0,98 с | 0,12 с | 0,63 с, exit 137 | 2,1 с | 5,2 с | + +Итог серий: SIGTERM — 80,5 с, 5312 записей и 5312 чтений, p95 73 мс; SIGKILL — 71,2 с, 5520 записей и 5520 чтений, p95 66 мс. В обоих случаях 0 ошибок, 0 дублей, 0 потерянных подтверждённых записей. В предварительном прогоне `--kill` (не сохранён) была 1 ошибка на 12 890 запросов: запрос, оборванный в момент SIGKILL. Это допустимый переходный эффект, клиент повторяет его с тем же ключом. + +RPS — это пропускная способность закрытого цикла на одном хосте, где рядом работали посторонние контейнеры. Между сериями она колеблется на 30–40 %, а внутри серии падение во время отказа составляет 10–20 %. Иногда RPS с одной репликой выше baseline: узкое место — клиент и общий CPU хоста, а не число реплик. Это не измерение ёмкости. + +### make test-failover и rolling update + +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных | +| ------------------------------------------------ | ----------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | +| `make test-failover`, `stop-one` → `api-2` | запись продолжается ≤10 с, дублей нет | PASS, 121 запись, 0 ошибок | `first_recovery_seconds` 6,42 / нет | +| то же после восстановления → `api-1` | то же | PASS, 115 записей, 0 ошибок | 6,44 / нет | +| `restart.cmd`: lb, api, postgres | данные и старый токен сохранены | PASS, volume сохранён | нет потерь | +| rolling update v2 → v3 под нагрузкой (2 реплики) | нет ошибок, ёмкость ≥2 реплик, версия сменилась | 42,7 с; 0 ошибок; пауза 0,07 с; RPS 181 → 146 → 186; ответы: до v2 1833; во время v2 3941 / v3 2279; после v3 2787 | 5423 записи, 0 дублей, 0 потерь | +| rolling update dev → v2 с миграцией 003 | старая версия работает на расширенной схеме | 45,7 с; 0 ошибок; пауза 0,10 с; миграция применена один раз в начале, реплики `dev` обслуживали запросы до своей замены | 4458 записей, 0 дублей, 0 потерь | + +`first_recovery_seconds` в `tests/resilience.py` считается до первой успешной записи **после завершения hook**. Hook теперь длится около 6 с: это `docker stop`, который ждёт 4 с drain. Записи при этом не прерывались (0 ошибок), поэтому значение отражает длительность hook, а не простой. Оно укладывается в лимит 10 с, как и сам hook. + +**Rolling update** (`scripts/rolling-update.sh`): + +1. Собрать `messenger-lab3-app:$APP_VERSION`. +2. Один раз выполнить `docker compose run --rm migrate`. +3. Для каждой старой реплики запустить `up -d --no-deps --no-recreate --wait --scale api=N+1`. Новая реплика стартует из нового образа, скрипт ждёт healthy и UP в HAProxy. Затем старая реплика получает `docker stop` (drain) и `docker rm`. + +Ёмкость не опускается ниже N, LB не перезапускается. После обновления версию нужно записать в `.env` (`APP_VERSION`), иначе следующий `docker compose up` пересоздаст реплики в версии из `.env`. + +**Политика миграций (expand/contract)**. Миграция выполняется до замены реплик и должна быть совместима с работающей версией: только новые таблицы, индексы и nullable-колонки. Пример — `003_session_indexes.sql`. Readiness старой сборки проверяет только свои миграции, поэтому расширенная схема её не ломает. Удаление и переименование колонок допустимы только отдельным релизом после ухода старых реплик. Ограничение: `CREATE INDEX` без `CONCURRENTLY` (мигратор работает в транзакции) коротко блокирует запись в индексируемую таблицу. Для `sessions` учебного размера это незаметно, для большой таблицы понадобится `CONCURRENTLY` вне транзакции. ## Ограничения и следующие шаги -- **Выдерживается**: отказ процесса или контейнера одной API-реплики, пока работает хотя бы одна другая. Также полный перезапуск стенда с сохранением volume. -- **Не выдерживается**: отказ PostgreSQL и Nginx. Это единственные точки отказа: БД не реплицирована, LB один. При их остановке API недоступен до перезапуска, подтверждённые данные сохраняются в volume. -- **Один хост**: все контейнеры на одной машине, поэтому отказ узла или Docker Engine останавливает весь стенд. Проверен отказ процесса, а не узла. -- **Нет активных health-check в LB**: open-source Nginx исключает реплику пассивно (ошибка соединения, `max_fails=1 fail_timeout=5s`) и при обновлении DNS раз в 2 с. Зависшая, но принимающая соединения реплика будет отбрасываться только по `proxy_read_timeout 10s`. -- **Порядок сообщений** между репликами зависит от общих часов хоста: `created_at` назначает приложение. На нескольких узлах понадобятся время БД или последовательность. -- **Повтор LB**: Nginx повторяет на другой реплике только POST сообщений. POST пользователей, чатов и сессий после обрыва соединения клиент должен проверить сам, так как ключа идемпотентности у них нет. -- Лимиты CPU/RAM, rolling update и измерения под нагрузкой не делались: это уровень 4. +- **Выдерживается**: плановая остановка, падение или зависание одной API-реплики, пока работает хотя бы одна другая; поочерёдная замена всех реплик; полный перезапуск стенда с сохранением volume. +- **Не выдерживается**: отказ PostgreSQL и HAProxy. Это единственные точки отказа: БД не реплицирована, LB один. При их остановке API недоступен до перезапуска, подтверждённые данные сохраняются в volume. +- **Один хост**: отказ узла или Docker Engine останавливает весь стенд. Проверен отказ процесса или контейнера, а не узла. +- **Зависание реплики** экспериментально не воспроизводилось. Оценка «не более 3 с до DOWN» следует из конфигурации, а уже отправленные на зависшую реплику запросы ждут до 10 с. +- **Rolling update** в Compose реализован скриптом, а не `deploy.update_config`: Compose его не исполняет. Откат — тот же скрипт с прежней `APP_VERSION`, если схема не прошла contract-фазу. +- **Нагрузка** — закрытый цикл одного клиента на том же хосте (coordinated omission, общий CPU). Цифры RPS сравнимы только внутри одной серии. +- **Порядок сообщений** между репликами зависит от общих часов хоста: `created_at` назначает приложение. +- **Повтор LB**: POST пользователей, чатов и сессий повторяется только при неустановленном соединении. После обрыва уже отправленного запроса клиент должен проверить результат сам: ключа идемпотентности у этих POST нет. diff --git a/app/__main__.py b/app/__main__.py index 14beaf1c..5396aabb 100644 --- a/app/__main__.py +++ b/app/__main__.py @@ -1,17 +1,48 @@ +import logging +import threading + import uvicorn from .config import load_settings +from .drain import draining + +logger = logging.getLogger('messenger') + + +class DrainingServer(uvicorn.Server): + """First SIGTERM/SIGINT: keep serving for drain_seconds with readiness 503, then uvicorn's graceful stop. + + drain_seconds must exceed the LB detection time (HAProxy inter 1s x fall 2 + timeout check 1s), + so the replica is out of rotation before it stops accepting connections. + """ + + def __init__(self, config: uvicorn.Config, drain_seconds: float): + super().__init__(config) + self.drain_seconds = drain_seconds + + def handle_exit(self, sig, frame) -> None: + if draining.is_set() or self.drain_seconds <= 0: + draining.set() + super().handle_exit(sig, frame) + return + draining.set() + logger.warning('signal %s: draining for %.1f s before shutdown', sig, self.drain_seconds) + timer = threading.Timer(self.drain_seconds, super().handle_exit, (sig, frame)) + timer.daemon = True + timer.start() def main() -> None: settings = load_settings() - uvicorn.run( + config = uvicorn.Config( 'app.main:app', host=settings.host, port=settings.port, log_level=settings.log_level, timeout_graceful_shutdown=settings.shutdown_timeout, + timeout_keep_alive=settings.keepalive_timeout, ) + DrainingServer(config, settings.drain_seconds).run() if __name__ == '__main__': diff --git a/app/config.py b/app/config.py index 642fd9af..07ac86e5 100644 --- a/app/config.py +++ b/app/config.py @@ -2,6 +2,7 @@ import os import socket from dataclasses import dataclass from pathlib import Path +from urllib.parse import quote @dataclass(frozen=True) @@ -11,39 +12,67 @@ class Settings: env: str log_level: str shutdown_timeout: int + drain_seconds: float + keepalive_timeout: int session_ttl_seconds: int idempotency_ttl_seconds: int database_url: str instance_id: str + app_version: str -def _secret(file_var: str, value_var: str) -> str: - """A *_FILE variable wins. The secret value is not logged.""" - file_name = os.getenv(file_var, '').strip() - if file_name: - try: - value = Path(file_name).read_text(encoding='utf-8').strip() - except OSError: - raise RuntimeError('%s could not be read' % file_var) from None - if not value: - raise RuntimeError('%s is empty' % file_var) - return value - value = os.getenv(value_var, '').strip() +def _read_secret_file(file_var: str, file_name: str) -> str: + try: + value = Path(file_name).read_text(encoding='utf-8').strip() + except OSError: + raise RuntimeError('%s could not be read' % file_var) from None if not value: - raise RuntimeError('%s or %s is required' % (value_var, file_var)) + raise RuntimeError('%s is empty' % file_var) return value +def _database_url() -> str: + """DATABASE_URL_FILE, then DATABASE_URL, then DB_* parts with the password from DB_PASSWORD_FILE. + + Secret values are never logged. + """ + file_name = os.getenv('DATABASE_URL_FILE', '').strip() + if file_name: + return _read_secret_file('DATABASE_URL_FILE', file_name) + value = os.getenv('DATABASE_URL', '').strip() + if value: + return value + password_file = os.getenv('DB_PASSWORD_FILE', '').strip() + if not password_file: + raise RuntimeError('DATABASE_URL, DATABASE_URL_FILE or DB_PASSWORD_FILE is required') + password = _read_secret_file('DB_PASSWORD_FILE', password_file) + parts = {name: os.getenv(name, '').strip() for name in ('DB_HOST', 'DB_USER', 'DB_NAME')} + missing = [name for name, part in parts.items() if not part] + if missing: + raise RuntimeError('%s required with DB_PASSWORD_FILE' % ', '.join(missing)) + return 'postgresql://%s:%s@%s:%s/%s' % ( + quote(parts['DB_USER'], safe=''), + quote(password, safe=''), + parts['DB_HOST'], + int(os.getenv('DB_PORT', '5432')), + quote(parts['DB_NAME'], safe=''), + ) + + def load_settings() -> Settings: return Settings( host=os.getenv('APP_HOST', '0.0.0.0'), port=int(os.getenv('APP_PORT', '8080')), env=os.getenv('APP_ENV', 'lab'), log_level=os.getenv('LOG_LEVEL', 'info').lower(), - shutdown_timeout=int(os.getenv('SHUTDOWN_TIMEOUT_SECONDS', '5')), + shutdown_timeout=int(os.getenv('SHUTDOWN_TIMEOUT_SECONDS', '3')), + drain_seconds=float(os.getenv('DRAIN_SECONDS', '4')), + # Longer than HAProxy's idle-connection purge (pool-purge-delay 5 s), so the LB usually closes them first. + keepalive_timeout=int(os.getenv('KEEPALIVE_TIMEOUT_SECONDS', '10')), session_ttl_seconds=int(os.getenv('SESSION_TTL_SECONDS', '3600')), idempotency_ttl_seconds=int(os.getenv('IDEMPOTENCY_TTL_SECONDS', '86400')), - database_url=_secret('DATABASE_URL_FILE', 'DATABASE_URL'), + database_url=_database_url(), # Docker sets the hostname to the short container ID, which stop-one.sh maps back to a container. instance_id=os.getenv('INSTANCE_ID', '').strip() or socket.gethostname(), + app_version=os.getenv('APP_VERSION', 'dev').strip() or 'dev', ) diff --git a/app/drain.py b/app/drain.py new file mode 100644 index 00000000..f23a67c3 --- /dev/null +++ b/app/drain.py @@ -0,0 +1,4 @@ +"""Set on SIGTERM: readiness turns 503 so the LB stops routing here while requests are still served.""" +import threading + +draining = threading.Event() diff --git a/app/main.py b/app/main.py index 605254eb..052dd11b 100644 --- a/app/main.py +++ b/app/main.py @@ -72,7 +72,9 @@ def create_app() -> FastAPI: version='1.3.0', lifespan=lifespan, ) - instance_id = load_settings().instance_id + settings = load_settings() + instance_id = settings.instance_id + app_version = settings.app_version def openapi(): if app.openapi_schema is None: @@ -83,7 +85,7 @@ def create_app() -> FastAPI: @app.middleware('http') async def request_context(request: Request, call_next): - """Gives every response, including unexpected failures, an X-Request-Id and X-Instance-Id.""" + """Gives every response, including unexpected failures, an X-Request-Id, X-Instance-Id and X-App-Version.""" incoming = request.headers.get('X-Request-Id') request_id = incoming if incoming and _SAFE_REQUEST_ID.match(incoming) else str(uuid.uuid4()) request.state.request_id = request_id @@ -94,6 +96,7 @@ def create_app() -> FastAPI: response = _error(500, 'internal_error', _MESSAGES['internal_error'], request_id) response.headers['X-Request-Id'] = request_id response.headers['X-Instance-Id'] = instance_id + response.headers['X-App-Version'] = app_version return response @app.exception_handler(ApiError) diff --git a/app/routes.py b/app/routes.py index 2221dbd3..065f4f15 100644 --- a/app/routes.py +++ b/app/routes.py @@ -10,7 +10,8 @@ from pydantic import BaseModel, ValidationError from starlette.responses import Response from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, parse_basic, parse_bearer -from .errors import invalid_request, not_found, unauthorized +from .drain import draining +from .errors import invalid_request, not_found, unauthorized, unavailable from .pagination import encode_cursor, parse_page from .schemas import CreateChat, CreateMessage, CreateUser from .store import Chat, Store, User @@ -80,6 +81,8 @@ async def liveness() -> JSONResponse: @router.get('/health/ready') async def readiness(store: Store = Depends(get_store)) -> JSONResponse: + if draining.is_set(): + raise unavailable('Реплика завершает работу') await store.ready() return JSONResponse({'status': 'ok'}) diff --git a/compose.yaml b/compose.yaml index 68d2997d..24961d41 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,57 +1,103 @@ -# Client -> lb (nginx) -> api x API_REPLICAS -> postgres. -# Secrets come from .env, which is not committed. Only the LB publishes a port. +# Client -> lb (HAProxy) -> api x API_REPLICAS -> postgres. +# Only the LB publishes a port. Networks: edge = lb<->api, data = api/migrate<->postgres (no LB, no internet). +# The DB password comes from .env (not committed) as a Compose secret file, never as a container env var. name: messenger-lab3 -x-app-env: &app-env +x-db-client: &db-client APP_ENV: ${APP_ENV:-lab} LOG_LEVEL: ${LOG_LEVEL:-info} - DATABASE_URL: postgresql://${POSTGRES_USER:?set in .env}:${POSTGRES_PASSWORD:?set in .env}@postgres:5432/${POSTGRES_DB:?set in .env} + DB_HOST: postgres + DB_USER: ${POSTGRES_USER:?set in .env} + DB_NAME: ${POSTGRES_DB:?set in .env} + DB_PASSWORD_FILE: /run/secrets/postgres_password + +x-app-build: &app-build + context: . + dockerfile: Dockerfile + args: + APP_VERSION: ${APP_VERSION:-dev} services: postgres: image: postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297 environment: POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password POSTGRES_DB: ${POSTGRES_DB} + secrets: + - postgres_password volumes: - pgdata:/var/lib/postgresql/data + networks: + - data healthcheck: test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""] interval: 2s timeout: 5s retries: 30 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + reservations: + cpus: "0.25" + memory: 256M restart: unless-stopped # One-shot schema step. Replicas never migrate on startup, so there is no race between them; # an advisory lock additionally serializes accidental parallel runs. migrate: - image: messenger-lab3-app:local - build: - context: . - dockerfile: Dockerfile + image: messenger-lab3-app:${APP_VERSION:-dev} + build: *app-build command: ["python", "-m", "app.migrate"] environment: - <<: *app-env + <<: *db-client + secrets: + - source: postgres_password + uid: "10001" + gid: "10001" + mode: 0400 + networks: + - data depends_on: postgres: condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M restart: "no" api: - image: messenger-lab3-app:local - build: - context: . - dockerfile: Dockerfile + image: messenger-lab3-app:${APP_VERSION:-dev} + build: *app-build deploy: replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "1.0" + memory: 256M + reservations: + cpus: "0.25" + memory: 128M environment: - <<: *app-env + <<: *db-client APP_HOST: 0.0.0.0 APP_PORT: "8080" SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} - SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5} + DRAIN_SECONDS: ${DRAIN_SECONDS:-4} + SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-3} + secrets: + - source: postgres_password + uid: "10001" + gid: "10001" + mode: 0400 + networks: + - edge + - data expose: - "8080" depends_on: @@ -59,6 +105,7 @@ services: condition: service_healthy migrate: condition: service_completed_successfully + # Docker health gates startup order and rolling update; the LB runs its own checks (infra/haproxy/haproxy.cfg). healthcheck: test: - CMD @@ -69,19 +116,22 @@ services: timeout: 3s retries: 3 start_period: 20s - # SIGTERM -> uvicorn stops accepting, finishes in-flight requests within SHUTDOWN_TIMEOUT_SECONDS. + # SIGTERM -> readiness 503 for DRAIN_SECONDS (LB drops the replica in <= 3 s) -> uvicorn finishes + # in-flight requests within SHUTDOWN_TIMEOUT_SECONDS. 4 + 3 < 10, so SIGKILL is not reached. stop_signal: SIGTERM - stop_grace_period: 8s + stop_grace_period: 10s restart: unless-stopped lb: - image: nginx:1.28.0-alpine@sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284 + image: haproxy:3.2.26-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0 # Loopback only. Both families: on Windows "localhost" tries ::1 first and waits ~2 s if nothing listens. ports: - "127.0.0.1:${LB_PORT:-8080}:8080" - "[::1]:${LB_PORT:-8080}:8080" volumes: - - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: + - edge depends_on: api: condition: service_healthy @@ -90,7 +140,24 @@ services: interval: 5s timeout: 3s retries: 3 + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + reservations: + cpus: "0.1" + memory: 32M restart: unless-stopped +networks: + edge: {} + data: + internal: true + +secrets: + postgres_password: + environment: POSTGRES_PASSWORD + volumes: pgdata: diff --git a/evidence/failover-series-kill.json b/evidence/failover-series-kill.json new file mode 100644 index 00000000..f18a0117 --- /dev/null +++ b/evidence/failover-series-kill.json @@ -0,0 +1,961 @@ +{ + "mode": "failover", + "stop_method": "docker kill (SIGKILL)", + "concurrency": 4, + "workload": "closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest", + "replicas_before": [ + "messenger-lab3-api-1", + "messenger-lab3-api-2" + ], + "runs": [ + { + "replica": "messenger-lab3-api-1", + "instance_id": "1b755b53e9fa", + "state_after_stop": "exited exit=137", + "stop_command_seconds": 0.66, + "last_served_by_target_after_sigterm_s": 0.12, + "first_success_after_container_exit_s": 0.67, + "rps_back_to_90pct_after_exit_s": 2.28, + "max_gap_without_success_s": 0.963, + "rejoin_after_docker_start_s": 4.51, + "baseline": { + "seconds": 10.0, + "rps_ok": 174.4, + "writes_ok": 873, + "reads_ok": 871, + "errors": 0, + "p95_ms": 53.8, + "instances": { + "e3303afc01b2": 874, + "1b755b53e9fa": 870 + } + }, + "during_stop_and_outage": { + "seconds": 15.73, + "rps_ok": 145.3, + "writes_ok": 1143, + "reads_ok": 1143, + "errors": 0, + "p95_ms": 65.5, + "instances": { + "1b755b53e9fa": 9, + "e3303afc01b2": 2277 + } + }, + "after_restore": { + "seconds": 9.51, + "rps_ok": 158.4, + "writes_ok": 754, + "reads_ok": 752, + "errors": 0, + "p95_ms": 67.6, + "instances": { + "e3303afc01b2": 1035, + "1b755b53e9fa": 471 + } + }, + "marks": { + "sigterm": 10.06, + "exited": 10.72, + "docker_start": 25.79 + } + }, + { + "replica": "messenger-lab3-api-2", + "instance_id": "e3303afc01b2", + "state_after_stop": "exited exit=137", + "stop_command_seconds": 0.63, + "last_served_by_target_after_sigterm_s": 0.12, + "first_success_after_container_exit_s": 0.755, + "rps_back_to_90pct_after_exit_s": 2.06, + "max_gap_without_success_s": 0.975, + "rejoin_after_docker_start_s": 5.18, + "baseline": { + "seconds": 10.0, + "rps_ok": 177.9, + "writes_ok": 888, + "reads_ok": 891, + "errors": 0, + "p95_ms": 62.7, + "instances": { + "e3303afc01b2": 891, + "1b755b53e9fa": 888 + } + }, + "during_stop_and_outage": { + "seconds": 15.71, + "rps_ok": 146.5, + "writes_ok": 1152, + "reads_ok": 1151, + "errors": 0, + "p95_ms": 67.0, + "instances": { + "1b755b53e9fa": 2291, + "e3303afc01b2": 12 + } + }, + "after_restore": { + "seconds": 10.18, + "rps_ok": 139.0, + "writes_ok": 708, + "reads_ok": 708, + "errors": 0, + "p95_ms": 73.8, + "instances": { + "1b755b53e9fa": 1025, + "e3303afc01b2": 391 + } + }, + "marks": { + "sigterm": 45.31, + "exited": 45.94, + "docker_start": 61.02 + } + } + ], + "total": { + "seconds": 71.23, + "rps_ok": 155.0, + "writes_ok": 5520, + "reads_ok": 5520, + "errors": 0, + "p95_ms": 65.5, + "instances": { + "e3303afc01b2": 5483, + "1b755b53e9fa": 5557 + } + }, + "integrity": { + "confirmed_writes": 5520, + "stored_messages": 5520, + "duplicates": 0, + "missing_confirmed": 0 + }, + "timeline_per_second": [ + { + "t": 0, + "ok": 145, + "err": 0, + "instances": { + "e3303afc01b2": 71, + "1b755b53e9fa": 74 + }, + "versions": { + "v2": 145 + } + }, + { + "t": 1, + "ok": 171, + "err": 0, + "instances": { + "e3303afc01b2": 87, + "1b755b53e9fa": 84 + }, + "versions": { + "v2": 171 + } + }, + { + "t": 2, + "ok": 240, + "err": 0, + "instances": { + "e3303afc01b2": 120, + "1b755b53e9fa": 120 + }, + "versions": { + "v2": 240 + } + }, + { + "t": 3, + "ok": 239, + "err": 0, + "instances": { + "e3303afc01b2": 120, + "1b755b53e9fa": 119 + }, + "versions": { + "v2": 239 + } + }, + { + "t": 4, + "ok": 199, + "err": 0, + "instances": { + "e3303afc01b2": 98, + "1b755b53e9fa": 101 + }, + "versions": { + "v2": 199 + } + }, + { + "t": 5, + "ok": 141, + "err": 0, + "instances": { + "1b755b53e9fa": 69, + "e3303afc01b2": 72 + }, + "versions": { + "v2": 141 + } + }, + { + "t": 6, + "ok": 185, + "err": 0, + "instances": { + "1b755b53e9fa": 92, + "e3303afc01b2": 93 + }, + "versions": { + "v2": 185 + } + }, + { + "t": 7, + "ok": 190, + "err": 0, + "instances": { + "1b755b53e9fa": 96, + "e3303afc01b2": 94 + }, + "versions": { + "v2": 190 + } + }, + { + "t": 8, + "ok": 130, + "err": 0, + "instances": { + "e3303afc01b2": 65, + "1b755b53e9fa": 65 + }, + "versions": { + "v2": 130 + } + }, + { + "t": 9, + "ok": 100, + "err": 0, + "instances": { + "1b755b53e9fa": 50, + "e3303afc01b2": 50 + }, + "versions": { + "v2": 100 + } + }, + { + "t": 10, + "ok": 48, + "err": 0, + "instances": { + "e3303afc01b2": 39, + "1b755b53e9fa": 9 + }, + "versions": { + "v2": 48 + } + }, + { + "t": 11, + "ok": 128, + "err": 0, + "instances": { + "e3303afc01b2": 128 + }, + "versions": { + "v2": 128 + } + }, + { + "t": 12, + "ok": 170, + "err": 0, + "instances": { + "e3303afc01b2": 170 + }, + "versions": { + "v2": 170 + } + }, + { + "t": 13, + "ok": 107, + "err": 0, + "instances": { + "e3303afc01b2": 107 + }, + "versions": { + "v2": 107 + } + }, + { + "t": 14, + "ok": 143, + "err": 0, + "instances": { + "e3303afc01b2": 143 + }, + "versions": { + "v2": 143 + } + }, + { + "t": 15, + "ok": 148, + "err": 0, + "instances": { + "e3303afc01b2": 148 + }, + "versions": { + "v2": 148 + } + }, + { + "t": 16, + "ok": 170, + "err": 0, + "instances": { + "e3303afc01b2": 170 + }, + "versions": { + "v2": 170 + } + }, + { + "t": 17, + "ok": 128, + "err": 0, + "instances": { + "e3303afc01b2": 128 + }, + "versions": { + "v2": 128 + } + }, + { + "t": 18, + "ok": 136, + "err": 0, + "instances": { + "e3303afc01b2": 136 + }, + "versions": { + "v2": 136 + } + }, + { + "t": 19, + "ok": 147, + "err": 0, + "instances": { + "e3303afc01b2": 147 + }, + "versions": { + "v2": 147 + } + }, + { + "t": 20, + "ok": 187, + "err": 0, + "instances": { + "e3303afc01b2": 187 + }, + "versions": { + "v2": 187 + } + }, + { + "t": 21, + "ok": 132, + "err": 0, + "instances": { + "e3303afc01b2": 132 + }, + "versions": { + "v2": 132 + } + }, + { + "t": 22, + "ok": 136, + "err": 0, + "instances": { + "e3303afc01b2": 136 + }, + "versions": { + "v2": 136 + } + }, + { + "t": 23, + "ok": 194, + "err": 0, + "instances": { + "e3303afc01b2": 194 + }, + "versions": { + "v2": 194 + } + }, + { + "t": 24, + "ok": 197, + "err": 0, + "instances": { + "e3303afc01b2": 197 + }, + "versions": { + "v2": 197 + } + }, + { + "t": 25, + "ok": 134, + "err": 0, + "instances": { + "e3303afc01b2": 134 + }, + "versions": { + "v2": 134 + } + }, + { + "t": 26, + "ok": 56, + "err": 0, + "instances": { + "e3303afc01b2": 56 + }, + "versions": { + "v2": 56 + } + }, + { + "t": 27, + "ok": 139, + "err": 0, + "instances": { + "e3303afc01b2": 139 + }, + "versions": { + "v2": 139 + } + }, + { + "t": 28, + "ok": 161, + "err": 0, + "instances": { + "e3303afc01b2": 161 + }, + "versions": { + "v2": 161 + } + }, + { + "t": 29, + "ok": 176, + "err": 0, + "instances": { + "e3303afc01b2": 176 + }, + "versions": { + "v2": 176 + } + }, + { + "t": 30, + "ok": 127, + "err": 0, + "instances": { + "e3303afc01b2": 72, + "1b755b53e9fa": 55 + }, + "versions": { + "v2": 127 + } + }, + { + "t": 31, + "ok": 173, + "err": 0, + "instances": { + "e3303afc01b2": 90, + "1b755b53e9fa": 83 + }, + "versions": { + "v2": 173 + } + }, + { + "t": 32, + "ok": 184, + "err": 0, + "instances": { + "1b755b53e9fa": 94, + "e3303afc01b2": 90 + }, + "versions": { + "v2": 184 + } + }, + { + "t": 33, + "ok": 240, + "err": 0, + "instances": { + "e3303afc01b2": 120, + "1b755b53e9fa": 120 + }, + "versions": { + "v2": 240 + } + }, + { + "t": 34, + "ok": 191, + "err": 0, + "instances": { + "e3303afc01b2": 94, + "1b755b53e9fa": 97 + }, + "versions": { + "v2": 191 + } + }, + { + "t": 35, + "ok": 138, + "err": 0, + "instances": { + "e3303afc01b2": 72, + "1b755b53e9fa": 66 + }, + "versions": { + "v2": 138 + } + }, + { + "t": 36, + "ok": 118, + "err": 0, + "instances": { + "e3303afc01b2": 58, + "1b755b53e9fa": 60 + }, + "versions": { + "v2": 118 + } + }, + { + "t": 37, + "ok": 208, + "err": 0, + "instances": { + "e3303afc01b2": 104, + "1b755b53e9fa": 104 + }, + "versions": { + "v2": 208 + } + }, + { + "t": 38, + "ok": 189, + "err": 0, + "instances": { + "1b755b53e9fa": 96, + "e3303afc01b2": 93 + }, + "versions": { + "v2": 189 + } + }, + { + "t": 39, + "ok": 148, + "err": 0, + "instances": { + "e3303afc01b2": 75, + "1b755b53e9fa": 73 + }, + "versions": { + "v2": 148 + } + }, + { + "t": 40, + "ok": 160, + "err": 0, + "instances": { + "1b755b53e9fa": 78, + "e3303afc01b2": 82 + }, + "versions": { + "v2": 160 + } + }, + { + "t": 41, + "ok": 240, + "err": 0, + "instances": { + "1b755b53e9fa": 122, + "e3303afc01b2": 118 + }, + "versions": { + "v2": 240 + } + }, + { + "t": 42, + "ok": 215, + "err": 0, + "instances": { + "1b755b53e9fa": 109, + "e3303afc01b2": 106 + }, + "versions": { + "v2": 215 + } + }, + { + "t": 43, + "ok": 171, + "err": 0, + "instances": { + "e3303afc01b2": 87, + "1b755b53e9fa": 84 + }, + "versions": { + "v2": 171 + } + }, + { + "t": 44, + "ok": 174, + "err": 0, + "instances": { + "1b755b53e9fa": 85, + "e3303afc01b2": 89 + }, + "versions": { + "v2": 174 + } + }, + { + "t": 45, + "ok": 116, + "err": 0, + "instances": { + "1b755b53e9fa": 75, + "e3303afc01b2": 41 + }, + "versions": { + "v2": 116 + } + }, + { + "t": 46, + "ok": 62, + "err": 0, + "instances": { + "1b755b53e9fa": 62 + }, + "versions": { + "v2": 62 + } + }, + { + "t": 47, + "ok": 180, + "err": 0, + "instances": { + "1b755b53e9fa": 180 + }, + "versions": { + "v2": 180 + } + }, + { + "t": 48, + "ok": 113, + "err": 0, + "instances": { + "1b755b53e9fa": 113 + }, + "versions": { + "v2": 113 + } + }, + { + "t": 49, + "ok": 141, + "err": 0, + "instances": { + "1b755b53e9fa": 141 + }, + "versions": { + "v2": 141 + } + }, + { + "t": 50, + "ok": 193, + "err": 0, + "instances": { + "1b755b53e9fa": 193 + }, + "versions": { + "v2": 193 + } + }, + { + "t": 51, + "ok": 200, + "err": 0, + "instances": { + "1b755b53e9fa": 200 + }, + "versions": { + "v2": 200 + } + }, + { + "t": 52, + "ok": 139, + "err": 0, + "instances": { + "1b755b53e9fa": 139 + }, + "versions": { + "v2": 139 + } + }, + { + "t": 53, + "ok": 171, + "err": 0, + "instances": { + "1b755b53e9fa": 171 + }, + "versions": { + "v2": 171 + } + }, + { + "t": 54, + "ok": 210, + "err": 0, + "instances": { + "1b755b53e9fa": 210 + }, + "versions": { + "v2": 210 + } + }, + { + "t": 55, + "ok": 151, + "err": 0, + "instances": { + "1b755b53e9fa": 151 + }, + "versions": { + "v2": 151 + } + }, + { + "t": 56, + "ok": 91, + "err": 0, + "instances": { + "1b755b53e9fa": 91 + }, + "versions": { + "v2": 91 + } + }, + { + "t": 57, + "ok": 123, + "err": 0, + "instances": { + "1b755b53e9fa": 123 + }, + "versions": { + "v2": 123 + } + }, + { + "t": 58, + "ok": 180, + "err": 0, + "instances": { + "1b755b53e9fa": 180 + }, + "versions": { + "v2": 180 + } + }, + { + "t": 59, + "ok": 126, + "err": 0, + "instances": { + "1b755b53e9fa": 126 + }, + "versions": { + "v2": 126 + } + }, + { + "t": 60, + "ok": 168, + "err": 0, + "instances": { + "1b755b53e9fa": 168 + }, + "versions": { + "v2": 168 + } + }, + { + "t": 61, + "ok": 78, + "err": 0, + "instances": { + "1b755b53e9fa": 78 + }, + "versions": { + "v2": 78 + } + }, + { + "t": 62, + "ok": 163, + "err": 0, + "instances": { + "1b755b53e9fa": 163 + }, + "versions": { + "v2": 163 + } + }, + { + "t": 63, + "ok": 144, + "err": 0, + "instances": { + "1b755b53e9fa": 144 + }, + "versions": { + "v2": 144 + } + }, + { + "t": 64, + "ok": 179, + "err": 0, + "instances": { + "1b755b53e9fa": 179 + }, + "versions": { + "v2": 179 + } + }, + { + "t": 65, + "ok": 68, + "err": 0, + "instances": { + "1b755b53e9fa": 68 + }, + "versions": { + "v2": 68 + } + }, + { + "t": 66, + "ok": 71, + "err": 0, + "instances": { + "1b755b53e9fa": 38, + "e3303afc01b2": 33 + }, + "versions": { + "v2": 71 + } + }, + { + "t": 67, + "ok": 145, + "err": 0, + "instances": { + "e3303afc01b2": 73, + "1b755b53e9fa": 72 + }, + "versions": { + "v2": 145 + } + }, + { + "t": 68, + "ok": 171, + "err": 0, + "instances": { + "e3303afc01b2": 86, + "1b755b53e9fa": 85 + }, + "versions": { + "v2": 171 + } + }, + { + "t": 69, + "ok": 206, + "err": 0, + "instances": { + "1b755b53e9fa": 103, + "e3303afc01b2": 103 + }, + "versions": { + "v2": 206 + } + }, + { + "t": 70, + "ok": 163, + "err": 0, + "instances": { + "1b755b53e9fa": 80, + "e3303afc01b2": 83 + }, + "versions": { + "v2": 163 + } + }, + { + "t": 71, + "ok": 35, + "err": 0, + "instances": { + "1b755b53e9fa": 19, + "e3303afc01b2": 16 + }, + "versions": { + "v2": 35 + } + } + ] +} diff --git a/evidence/failover-series-kill.txt b/evidence/failover-series-kill.txt new file mode 100644 index 00000000..0cffa7ec --- /dev/null +++ b/evidence/failover-series-kill.txt @@ -0,0 +1,7 @@ +# failover series 2026-11-07T18:10:53.0538421+03:00: python scripts/failover_series.py --kill (docker kill, SIGKILL) +killing messenger-lab3-api-1 (1b755b53e9fa) under load... + errors=0 gap=0.96s first_ok_after_exit=0.67 rps 174.4 -> 145.3 -> 158.4 rejoin=4.51s +killing messenger-lab3-api-2 (e3303afc01b2) under load... + errors=0 gap=0.97s first_ok_after_exit=0.755 rps 177.9 -> 146.5 -> 139.0 rejoin=5.18s +{"total": {"seconds": 71.23, "rps_ok": 155.0, "writes_ok": 5520, "reads_ok": 5520, "errors": 0, "p95_ms": 65.5, "instances": {"e3303afc01b2": 5483, "1b755b53e9fa": 5557}}, "integrity": {"confirmed_writes": 5520, "stored_messages": 5520, "duplicates": 0, "missing_confirmed": 0}} +PASS: failover, 5520 confirmed writes, no duplicates, nothing lost diff --git a/evidence/failover-series.json b/evidence/failover-series.json new file mode 100644 index 00000000..d0593889 --- /dev/null +++ b/evidence/failover-series.json @@ -0,0 +1,1064 @@ +{ + "mode": "failover", + "stop_method": "docker stop --time 9 (SIGTERM, drain)", + "concurrency": 4, + "workload": "closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest", + "replicas_before": [ + "messenger-lab3-api-1", + "messenger-lab3-api-2" + ], + "runs": [ + { + "replica": "messenger-lab3-api-1", + "instance_id": "1b755b53e9fa", + "state_after_stop": "exited exit=0", + "stop_command_seconds": 5.06, + "last_served_by_target_after_sigterm_s": 2.05, + "first_success_after_container_exit_s": 0.018, + "rps_back_to_90pct_after_exit_s": 2.76, + "max_gap_without_success_s": 0.091, + "rejoin_after_docker_start_s": 5.15, + "baseline": { + "seconds": 10.0, + "rps_ok": 129.0, + "writes_ok": 646, + "reads_ok": 644, + "errors": 0, + "p95_ms": 70.3, + "instances": { + "1b755b53e9fa": 645, + "e3303afc01b2": 645 + } + }, + "during_stop_and_outage": { + "seconds": 20.19, + "rps_ok": 114.8, + "writes_ok": 1159, + "reads_ok": 1158, + "errors": 0, + "p95_ms": 77.4, + "instances": { + "1b755b53e9fa": 123, + "e3303afc01b2": 2194 + } + }, + "after_restore": { + "seconds": 10.16, + "rps_ok": 113.1, + "writes_ok": 574, + "reads_ok": 575, + "errors": 0, + "p95_ms": 81.2, + "instances": { + "e3303afc01b2": 824, + "1b755b53e9fa": 325 + } + }, + "marks": { + "sigterm": 10.19, + "exited": 15.24, + "docker_start": 30.37 + } + }, + { + "replica": "messenger-lab3-api-2", + "instance_id": "e3303afc01b2", + "state_after_stop": "exited exit=0", + "stop_command_seconds": 4.88, + "last_served_by_target_after_sigterm_s": 1.76, + "first_success_after_container_exit_s": 0.012, + "rps_back_to_90pct_after_exit_s": 1.58, + "max_gap_without_success_s": 0.088, + "rejoin_after_docker_start_s": 4.91, + "baseline": { + "seconds": 10.0, + "rps_ok": 131.5, + "writes_ok": 658, + "reads_ok": 657, + "errors": 0, + "p95_ms": 74.2, + "instances": { + "e3303afc01b2": 658, + "1b755b53e9fa": 657 + } + }, + "during_stop_and_outage": { + "seconds": 19.99, + "rps_ok": 145.2, + "writes_ok": 1451, + "reads_ok": 1453, + "errors": 0, + "p95_ms": 68.7, + "instances": { + "1b755b53e9fa": 2767, + "e3303afc01b2": 137 + } + }, + "after_restore": { + "seconds": 9.92, + "rps_ok": 165.4, + "writes_ok": 820, + "reads_ok": 821, + "errors": 0, + "p95_ms": 60.3, + "instances": { + "1b755b53e9fa": 1184, + "e3303afc01b2": 457 + } + }, + "marks": { + "sigterm": 50.54, + "exited": 55.42, + "docker_start": 70.54 + } + } + ], + "total": { + "seconds": 80.52, + "rps_ok": 131.9, + "writes_ok": 5312, + "reads_ok": 5312, + "errors": 0, + "p95_ms": 73.4, + "instances": { + "1b755b53e9fa": 5703, + "e3303afc01b2": 4921 + } + }, + "integrity": { + "confirmed_writes": 5312, + "stored_messages": 5312, + "duplicates": 0, + "missing_confirmed": 0 + }, + "timeline_per_second": [ + { + "t": 0, + "ok": 90, + "err": 0, + "instances": { + "1b755b53e9fa": 45, + "e3303afc01b2": 45 + }, + "versions": { + "v2": 90 + } + }, + { + "t": 1, + "ok": 111, + "err": 0, + "instances": { + "1b755b53e9fa": 54, + "e3303afc01b2": 57 + }, + "versions": { + "v2": 111 + } + }, + { + "t": 2, + "ok": 103, + "err": 0, + "instances": { + "1b755b53e9fa": 53, + "e3303afc01b2": 50 + }, + "versions": { + "v2": 103 + } + }, + { + "t": 3, + "ok": 112, + "err": 0, + "instances": { + "1b755b53e9fa": 56, + "e3303afc01b2": 56 + }, + "versions": { + "v2": 112 + } + }, + { + "t": 4, + "ok": 121, + "err": 0, + "instances": { + "e3303afc01b2": 60, + "1b755b53e9fa": 61 + }, + "versions": { + "v2": 121 + } + }, + { + "t": 5, + "ok": 171, + "err": 0, + "instances": { + "e3303afc01b2": 87, + "1b755b53e9fa": 84 + }, + "versions": { + "v2": 171 + } + }, + { + "t": 6, + "ok": 130, + "err": 0, + "instances": { + "e3303afc01b2": 65, + "1b755b53e9fa": 65 + }, + "versions": { + "v2": 130 + } + }, + { + "t": 7, + "ok": 143, + "err": 0, + "instances": { + "e3303afc01b2": 70, + "1b755b53e9fa": 73 + }, + "versions": { + "v2": 143 + } + }, + { + "t": 8, + "ok": 152, + "err": 0, + "instances": { + "e3303afc01b2": 77, + "1b755b53e9fa": 75 + }, + "versions": { + "v2": 152 + } + }, + { + "t": 9, + "ok": 120, + "err": 0, + "instances": { + "1b755b53e9fa": 61, + "e3303afc01b2": 59 + }, + "versions": { + "v2": 120 + } + }, + { + "t": 10, + "ok": 149, + "err": 0, + "instances": { + "e3303afc01b2": 76, + "1b755b53e9fa": 73 + }, + "versions": { + "v2": 149 + } + }, + { + "t": 11, + "ok": 102, + "err": 0, + "instances": { + "1b755b53e9fa": 51, + "e3303afc01b2": 51 + }, + "versions": { + "v2": 102 + } + }, + { + "t": 12, + "ok": 128, + "err": 0, + "instances": { + "1b755b53e9fa": 17, + "e3303afc01b2": 111 + }, + "versions": { + "v2": 128 + } + }, + { + "t": 13, + "ok": 117, + "err": 0, + "instances": { + "e3303afc01b2": 117 + }, + "versions": { + "v2": 117 + } + }, + { + "t": 14, + "ok": 143, + "err": 0, + "instances": { + "e3303afc01b2": 143 + }, + "versions": { + "v2": 143 + } + }, + { + "t": 15, + "ok": 64, + "err": 0, + "instances": { + "e3303afc01b2": 64 + }, + "versions": { + "v2": 64 + } + }, + { + "t": 16, + "ok": 92, + "err": 0, + "instances": { + "e3303afc01b2": 92 + }, + "versions": { + "v2": 92 + } + }, + { + "t": 17, + "ok": 151, + "err": 0, + "instances": { + "e3303afc01b2": 151 + }, + "versions": { + "v2": 151 + } + }, + { + "t": 18, + "ok": 129, + "err": 0, + "instances": { + "e3303afc01b2": 129 + }, + "versions": { + "v2": 129 + } + }, + { + "t": 19, + "ok": 108, + "err": 0, + "instances": { + "e3303afc01b2": 108 + }, + "versions": { + "v2": 108 + } + }, + { + "t": 20, + "ok": 72, + "err": 0, + "instances": { + "e3303afc01b2": 72 + }, + "versions": { + "v2": 72 + } + }, + { + "t": 21, + "ok": 109, + "err": 0, + "instances": { + "e3303afc01b2": 109 + }, + "versions": { + "v2": 109 + } + }, + { + "t": 22, + "ok": 138, + "err": 0, + "instances": { + "e3303afc01b2": 138 + }, + "versions": { + "v2": 138 + } + }, + { + "t": 23, + "ok": 150, + "err": 0, + "instances": { + "e3303afc01b2": 150 + }, + "versions": { + "v2": 150 + } + }, + { + "t": 24, + "ok": 115, + "err": 0, + "instances": { + "e3303afc01b2": 115 + }, + "versions": { + "v2": 115 + } + }, + { + "t": 25, + "ok": 92, + "err": 0, + "instances": { + "e3303afc01b2": 92 + }, + "versions": { + "v2": 92 + } + }, + { + "t": 26, + "ok": 150, + "err": 0, + "instances": { + "e3303afc01b2": 150 + }, + "versions": { + "v2": 150 + } + }, + { + "t": 27, + "ok": 146, + "err": 0, + "instances": { + "e3303afc01b2": 146 + }, + "versions": { + "v2": 146 + } + }, + { + "t": 28, + "ok": 111, + "err": 0, + "instances": { + "e3303afc01b2": 111 + }, + "versions": { + "v2": 111 + } + }, + { + "t": 29, + "ok": 59, + "err": 0, + "instances": { + "e3303afc01b2": 59 + }, + "versions": { + "v2": 59 + } + }, + { + "t": 30, + "ok": 87, + "err": 0, + "instances": { + "e3303afc01b2": 87 + }, + "versions": { + "v2": 87 + } + }, + { + "t": 31, + "ok": 95, + "err": 0, + "instances": { + "e3303afc01b2": 95 + }, + "versions": { + "v2": 95 + } + }, + { + "t": 32, + "ok": 122, + "err": 0, + "instances": { + "e3303afc01b2": 122 + }, + "versions": { + "v2": 122 + } + }, + { + "t": 33, + "ok": 119, + "err": 0, + "instances": { + "e3303afc01b2": 119 + }, + "versions": { + "v2": 119 + } + }, + { + "t": 34, + "ok": 72, + "err": 0, + "instances": { + "e3303afc01b2": 72 + }, + "versions": { + "v2": 72 + } + }, + { + "t": 35, + "ok": 151, + "err": 0, + "instances": { + "e3303afc01b2": 92, + "1b755b53e9fa": 59 + }, + "versions": { + "v2": 151 + } + }, + { + "t": 36, + "ok": 134, + "err": 0, + "instances": { + "1b755b53e9fa": 65, + "e3303afc01b2": 69 + }, + "versions": { + "v2": 134 + } + }, + { + "t": 37, + "ok": 142, + "err": 0, + "instances": { + "1b755b53e9fa": 73, + "e3303afc01b2": 69 + }, + "versions": { + "v2": 142 + } + }, + { + "t": 38, + "ok": 94, + "err": 0, + "instances": { + "e3303afc01b2": 47, + "1b755b53e9fa": 47 + }, + "versions": { + "v2": 94 + } + }, + { + "t": 39, + "ok": 82, + "err": 0, + "instances": { + "e3303afc01b2": 42, + "1b755b53e9fa": 40 + }, + "versions": { + "v2": 82 + } + }, + { + "t": 40, + "ok": 139, + "err": 0, + "instances": { + "1b755b53e9fa": 69, + "e3303afc01b2": 70 + }, + "versions": { + "v2": 139 + } + }, + { + "t": 41, + "ok": 111, + "err": 0, + "instances": { + "e3303afc01b2": 55, + "1b755b53e9fa": 56 + }, + "versions": { + "v2": 111 + } + }, + { + "t": 42, + "ok": 156, + "err": 0, + "instances": { + "e3303afc01b2": 77, + "1b755b53e9fa": 79 + }, + "versions": { + "v2": 156 + } + }, + { + "t": 43, + "ok": 143, + "err": 0, + "instances": { + "e3303afc01b2": 71, + "1b755b53e9fa": 72 + }, + "versions": { + "v2": 143 + } + }, + { + "t": 44, + "ok": 144, + "err": 0, + "instances": { + "1b755b53e9fa": 71, + "e3303afc01b2": 73 + }, + "versions": { + "v2": 144 + } + }, + { + "t": 45, + "ok": 127, + "err": 0, + "instances": { + "e3303afc01b2": 65, + "1b755b53e9fa": 62 + }, + "versions": { + "v2": 127 + } + }, + { + "t": 46, + "ok": 142, + "err": 0, + "instances": { + "1b755b53e9fa": 73, + "e3303afc01b2": 69 + }, + "versions": { + "v2": 142 + } + }, + { + "t": 47, + "ok": 158, + "err": 0, + "instances": { + "e3303afc01b2": 79, + "1b755b53e9fa": 79 + }, + "versions": { + "v2": 158 + } + }, + { + "t": 48, + "ok": 109, + "err": 0, + "instances": { + "1b755b53e9fa": 54, + "e3303afc01b2": 55 + }, + "versions": { + "v2": 109 + } + }, + { + "t": 49, + "ok": 104, + "err": 0, + "instances": { + "1b755b53e9fa": 51, + "e3303afc01b2": 53 + }, + "versions": { + "v2": 104 + } + }, + { + "t": 50, + "ok": 111, + "err": 0, + "instances": { + "1b755b53e9fa": 55, + "e3303afc01b2": 56 + }, + "versions": { + "v2": 111 + } + }, + { + "t": 51, + "ok": 167, + "err": 0, + "instances": { + "1b755b53e9fa": 85, + "e3303afc01b2": 82 + }, + "versions": { + "v2": 167 + } + }, + { + "t": 52, + "ok": 156, + "err": 0, + "instances": { + "e3303afc01b2": 29, + "1b755b53e9fa": 127 + }, + "versions": { + "v2": 156 + } + }, + { + "t": 53, + "ok": 158, + "err": 0, + "instances": { + "1b755b53e9fa": 158 + }, + "versions": { + "v2": 158 + } + }, + { + "t": 54, + "ok": 120, + "err": 0, + "instances": { + "1b755b53e9fa": 120 + }, + "versions": { + "v2": 120 + } + }, + { + "t": 55, + "ok": 79, + "err": 0, + "instances": { + "1b755b53e9fa": 79 + }, + "versions": { + "v2": 79 + } + }, + { + "t": 56, + "ok": 157, + "err": 0, + "instances": { + "1b755b53e9fa": 157 + }, + "versions": { + "v2": 157 + } + }, + { + "t": 57, + "ok": 160, + "err": 0, + "instances": { + "1b755b53e9fa": 160 + }, + "versions": { + "v2": 160 + } + }, + { + "t": 58, + "ok": 120, + "err": 0, + "instances": { + "1b755b53e9fa": 120 + }, + "versions": { + "v2": 120 + } + }, + { + "t": 59, + "ok": 62, + "err": 0, + "instances": { + "1b755b53e9fa": 62 + }, + "versions": { + "v2": 62 + } + }, + { + "t": 60, + "ok": 138, + "err": 0, + "instances": { + "1b755b53e9fa": 138 + }, + "versions": { + "v2": 138 + } + }, + { + "t": 61, + "ok": 179, + "err": 0, + "instances": { + "1b755b53e9fa": 179 + }, + "versions": { + "v2": 179 + } + }, + { + "t": 62, + "ok": 166, + "err": 0, + "instances": { + "1b755b53e9fa": 166 + }, + "versions": { + "v2": 166 + } + }, + { + "t": 63, + "ok": 134, + "err": 0, + "instances": { + "1b755b53e9fa": 134 + }, + "versions": { + "v2": 134 + } + }, + { + "t": 64, + "ok": 161, + "err": 0, + "instances": { + "1b755b53e9fa": 161 + }, + "versions": { + "v2": 161 + } + }, + { + "t": 65, + "ok": 194, + "err": 0, + "instances": { + "1b755b53e9fa": 194 + }, + "versions": { + "v2": 194 + } + }, + { + "t": 66, + "ok": 191, + "err": 0, + "instances": { + "1b755b53e9fa": 191 + }, + "versions": { + "v2": 191 + } + }, + { + "t": 67, + "ok": 152, + "err": 0, + "instances": { + "1b755b53e9fa": 152 + }, + "versions": { + "v2": 152 + } + }, + { + "t": 68, + "ok": 120, + "err": 0, + "instances": { + "1b755b53e9fa": 120 + }, + "versions": { + "v2": 120 + } + }, + { + "t": 69, + "ok": 146, + "err": 0, + "instances": { + "1b755b53e9fa": 146 + }, + "versions": { + "v2": 146 + } + }, + { + "t": 70, + "ok": 154, + "err": 0, + "instances": { + "1b755b53e9fa": 154 + }, + "versions": { + "v2": 154 + } + }, + { + "t": 71, + "ok": 117, + "err": 0, + "instances": { + "1b755b53e9fa": 117 + }, + "versions": { + "v2": 117 + } + }, + { + "t": 72, + "ok": 89, + "err": 0, + "instances": { + "1b755b53e9fa": 89 + }, + "versions": { + "v2": 89 + } + }, + { + "t": 73, + "ok": 181, + "err": 0, + "instances": { + "1b755b53e9fa": 181 + }, + "versions": { + "v2": 181 + } + }, + { + "t": 74, + "ok": 206, + "err": 0, + "instances": { + "1b755b53e9fa": 206 + }, + "versions": { + "v2": 206 + } + }, + { + "t": 75, + "ok": 197, + "err": 0, + "instances": { + "1b755b53e9fa": 134, + "e3303afc01b2": 63 + }, + "versions": { + "v2": 197 + } + }, + { + "t": 76, + "ok": 145, + "err": 0, + "instances": { + "1b755b53e9fa": 72, + "e3303afc01b2": 73 + }, + "versions": { + "v2": 145 + } + }, + { + "t": 77, + "ok": 197, + "err": 0, + "instances": { + "1b755b53e9fa": 99, + "e3303afc01b2": 98 + }, + "versions": { + "v2": 197 + } + }, + { + "t": 78, + "ok": 203, + "err": 0, + "instances": { + "1b755b53e9fa": 101, + "e3303afc01b2": 102 + }, + "versions": { + "v2": 203 + } + }, + { + "t": 79, + "ok": 184, + "err": 0, + "instances": { + "1b755b53e9fa": 93, + "e3303afc01b2": 91 + }, + "versions": { + "v2": 184 + } + }, + { + "t": 80, + "ok": 71, + "err": 0, + "instances": { + "1b755b53e9fa": 35, + "e3303afc01b2": 36 + }, + "versions": { + "v2": 71 + } + } + ] +} diff --git a/evidence/failover-series.txt b/evidence/failover-series.txt new file mode 100644 index 00000000..35ac2865 --- /dev/null +++ b/evidence/failover-series.txt @@ -0,0 +1,7 @@ +# failover series 2026-11-07T18:09:39.2795947+03:00: make failover-series (docker stop, SIGTERM + drain) +stopping messenger-lab3-api-1 (1b755b53e9fa) under load... + errors=0 gap=0.09s first_ok_after_exit=0.018 rps 129.0 -> 114.8 -> 113.1 rejoin=5.15s +stopping messenger-lab3-api-2 (e3303afc01b2) under load... + errors=0 gap=0.09s first_ok_after_exit=0.012 rps 131.5 -> 145.2 -> 165.4 rejoin=4.91s +{"total": {"seconds": 80.52, "rps_ok": 131.9, "writes_ok": 5312, "reads_ok": 5312, "errors": 0, "p95_ms": 73.4, "instances": {"1b755b53e9fa": 5703, "e3303afc01b2": 4921}}, "integrity": {"confirmed_writes": 5312, "stored_messages": 5312, "duplicates": 0, "missing_confirmed": 0}} +PASS: failover, 5312 confirmed writes, no duplicates, nothing lost diff --git a/evidence/failover.txt b/evidence/failover.txt index 8778b332..3ce6c5d7 100644 --- a/evidence/failover.txt +++ b/evidence/failover.txt @@ -1,41 +1,38 @@ -# failover run 2026-11-07T17:39:07.3980307+03:00 +# failover runs 2026-11-07T18:03:34.5946145+03:00 (HAProxy, drain 4 s) # command (= make test-failover ACTION_SCRIPT=scripts/stop-one.cmd): python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://localhost:8080 ## run 1 - before (shift requests through LB: 0) -messenger-lab3-api-1 8b33a91fafee Up 20 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy) +messenger-lab3-api-1 1b755b53e9fa Up 53 seconds (healthy) +messenger-lab3-api-2 e3303afc01b2 Up 53 seconds (healthy) +api1 172.25.0.2 UP +api2 172.25.0.3 UP Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 1.27} +{"observed_instances_before": 2, "successful_writes": 121, "failed_attempts": 0, "first_recovery_seconds": 6.421} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. exit=0 ## run 1 - after -messenger-lab3-api-1 8b33a91fafee Exited (0) 14 seconds ago -messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy) +messenger-lab3-api-1 1b755b53e9fa Up About a minute (healthy) +messenger-lab3-api-2 e3303afc01b2 Exited (0) 9 seconds ago +api1 172.25.0.2 UP ## restored (docker compose --env-file .env up -d --wait) -messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy) -## run 2 - before (shift requests through LB: 0) -messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy) +messenger-lab3-api-1 1b755b53e9fa Up About a minute (healthy) +messenger-lab3-api-2 e3303afc01b2 Up 10 seconds (healthy) +api1 172.25.0.2 UP +api2 172.25.0.3 UP +## run 2 - before (shift requests through LB: 1) +messenger-lab3-api-1 1b755b53e9fa Up About a minute (healthy) +messenger-lab3-api-2 e3303afc01b2 Up 11 seconds (healthy) +api1 172.25.0.2 UP +api2 172.25.0.3 UP Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 1.166} +{"observed_instances_before": 2, "successful_writes": 115, "failed_attempts": 0, "first_recovery_seconds": 6.437} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. exit=0 ## run 2 - after -messenger-lab3-api-1 8b33a91fafee Exited (0) 14 seconds ago -messenger-lab3-api-2 a87734e887a5 Up 3 minutes (healthy) +messenger-lab3-api-1 1b755b53e9fa Exited (0) 9 seconds ago +messenger-lab3-api-2 e3303afc01b2 Up 30 seconds (healthy) +api2 172.25.0.3 UP ## restored (docker compose --env-file .env up -d --wait) -messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 3 minutes (healthy) -## run 3 - before (shift requests through LB: 1) -messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 3 minutes (healthy) -Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 1.213} -PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. -exit=0 -## run 3 - after -messenger-lab3-api-1 8b33a91fafee Up 22 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Exited (0) 14 seconds ago -## restored (docker compose --env-file .env up -d --wait) -messenger-lab3-api-1 8b33a91fafee Up 31 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 6 seconds (healthy) +messenger-lab3-api-1 1b755b53e9fa Up 9 seconds (healthy) +messenger-lab3-api-2 e3303afc01b2 Up 43 seconds (healthy) +api1 172.25.0.2 UP +api2 172.25.0.3 UP diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt index 41104879..5e96baac 100644 --- a/evidence/idempotency.txt +++ b/evidence/idempotency.txt @@ -1,10 +1,7 @@ -# idempotency run 2026-11-07T17:36:58.5045267+03:00 -## make test-idempotency PYTHON=python -python scripts/check_idempotency.py --base-url "http://localhost:8080" -round 0: 20 responses, 1 message, replicas {'a87734e887a5': 10, '8b33a91fafee': 10} -round 1: 20 responses, 1 message, replicas {'a87734e887a5': 10, '8b33a91fafee': 10} -round 2: 20 responses, 1 message, replicas {'8b33a91fafee': 10, 'a87734e887a5': 10} -round 3: 20 responses, 1 message, replicas {'8b33a91fafee': 10, 'a87734e887a5': 10} -round 4: 20 responses, 1 message, replicas {'a87734e887a5': 10, '8b33a91fafee': 10} -PASS: 5 keys x 20 parallel requests -> 5 messages via 2 replicas {'a87734e887a5': 50, '8b33a91fafee': 50} -exit=0 +# make test-idempotency 2026-11-07T18:03:22.9365380+03:00 via HAProxy +round 0: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10} +round 1: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10} +round 2: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10} +round 3: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10} +round 4: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10} +PASS: 5 keys x 20 parallel requests -> 5 messages via 2 replicas {'e3303afc01b2': 50, '1b755b53e9fa': 50} diff --git a/evidence/instances.txt b/evidence/instances.txt index 09f89bca..6f0b9904 100644 --- a/evidence/instances.txt +++ b/evidence/instances.txt @@ -1,16 +1,8 @@ -# X-Instance-Id run 2026-11-07T17:41:06.5911429+03:00 -## python scripts/show_instances.py --base-url http://localhost:8080 -login + chat via LB http://localhost:8080: one session token for user 2ec10789-7814-4751-9e23-7b85215c262e (token not printed) -1. GET /api/v1/users/me -> 200 X-Instance-Id=8b33a91fafee -2. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=a87734e887a5 -3. GET /api/v1/users/me -> 200 X-Instance-Id=8b33a91fafee -4. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=a87734e887a5 -5. GET /api/v1/users/me -> 200 X-Instance-Id=8b33a91fafee -6. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=a87734e887a5 -PASS: same session served by 2 replicas without re-login {'8b33a91fafee': 3, 'a87734e887a5': 3} -exit=0 -## docker compose ps api (X-Instance-Id = container hostname = short container ID) -messenger-lab3-api-1 8b33a91fafee Up 31 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 31 seconds (healthy) -/messenger-lab3-api-1 hostname=8b33a91fafee -/messenger-lab3-api-2 hostname=a87734e887a5 +login + chat via LB http://localhost:8080: one session token for user 5757ef5e-391e-4369-85c5-19499d5a4930 (token not printed) +1. GET /api/v1/users/me -> 200 X-Instance-Id=e3303afc01b2 +2. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=1b755b53e9fa +3. GET /api/v1/users/me -> 200 X-Instance-Id=e3303afc01b2 +4. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=1b755b53e9fa +5. GET /api/v1/users/me -> 200 X-Instance-Id=e3303afc01b2 +6. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=1b755b53e9fa +PASS: same session served by 2 replicas without re-login {'e3303afc01b2': 3, '1b755b53e9fa': 3} diff --git a/evidence/limits.txt b/evidence/limits.txt new file mode 100644 index 00000000..f7d5d7d9 --- /dev/null +++ b/evidence/limits.txt @@ -0,0 +1,30 @@ +# isolation/limits 2026-11-07T18:03:06.9670680+03:00 +## per container: CPU limit (NanoCpus), memory limit, published ports, networks, mounts +/messenger-lab3-api-1 cpus=1000000000 mem=268435456 reserv_mem=134217728 ports= networks=messenger-lab3_data messenger-lab3_edge mounts= +/messenger-lab3-api-2 cpus=1000000000 mem=268435456 reserv_mem=134217728 ports= networks=messenger-lab3_data messenger-lab3_edge mounts= +/messenger-lab3-lb-1 cpus=500000000 mem=134217728 reserv_mem=33554432 ports=8080/tcp->127.0.0.1:8080 ::1:8080 networks=messenger-lab3_edge mounts=bind:/usr/local/etc/haproxy/haproxy.cfg +/messenger-lab3-migrate-1 cpus=500000000 mem=134217728 reserv_mem=0 ports= networks=messenger-lab3_data mounts= +/messenger-lab3-postgres-1 cpus=1000000000 mem=536870912 reserv_mem=268435456 ports= networks=messenger-lab3_data mounts=volume:/var/lib/postgresql/data +## docker stats --no-stream (this project) +NAME CPU % MEM USAGE / LIMIT +messenger-lab3-api-1 0.87% 41.18MiB / 256MiB +messenger-lab3-api-2 0.62% 41.18MiB / 256MiB +messenger-lab3-lb-1 0.21% 14.71MiB / 128MiB +messenger-lab3-postgres-1 0.15% 38.04MiB / 512MiB +## container env mentioning PASSWORD (expect only *_PASSWORD_FILE paths, no values) +messenger-lab3-api-1 DB_PASSWORD_FILE=/run/secrets/postgres_password +messenger-lab3-api-2 DB_PASSWORD_FILE=/run/secrets/postgres_password +messenger-lab3-migrate-1 DB_PASSWORD_FILE=/run/secrets/postgres_password +messenger-lab3-postgres-1 POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password +## secret file inside api: owner/mode only, value not printed +total 4 +-r-------- 1 10001 10001 18 Nov 7 15:02 postgres_password +## lb has no secrets and cannot resolve postgres (not on the data network) +no /run/secrets +postgres not resolvable from lb +## networks +messenger-lab3_data internal=true +messenger-lab3_edge internal=false +## docker port messenger-lab3-api-1 (empty = no published port) +## occurrences of container_name in compose.yaml +0 diff --git a/evidence/persistence.txt b/evidence/persistence.txt index 7586ea5e..1841f6e4 100644 --- a/evidence/persistence.txt +++ b/evidence/persistence.txt @@ -1,18 +1,21 @@ -# persistence run 2026-11-07T17:40:29.1339292+03:00 +# persistence run 2026-11-07T18:04:54.7155080+03:00 # command (= make test-persistence ACTION_SCRIPT=scripts/restart.cmd): python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://localhost:8080 ## before -messenger-lab3-api-1 8b33a91fafee Up 38 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 14 seconds (healthy) -messenger-lab3-lb-1 4ba96e32076f Up 3 minutes (healthy) -messenger-lab3-migrate-1 18f3889ef9a7 Exited (0) 14 seconds ago -messenger-lab3-postgres-1 4c60c6522907 Up 3 minutes (healthy) +NAME CONTAINER ID STATUS +messenger-lab3-api-1 1b755b53e9fa Up 22 seconds (healthy) +messenger-lab3-api-2 e3303afc01b2 Up 56 seconds (healthy) +messenger-lab3-lb-1 760a68ddce9c Up 3 minutes (healthy) +messenger-lab3-postgres-1 2265aa5e595d Up 3 minutes (healthy) +volume messenger-lab3_pgdata Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. exit=0 -## after (same container IDs, fresh uptime = restarted; volume pgdata kept) -messenger-lab3-api-1 8b33a91fafee Up 11 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 11 seconds (healthy) -messenger-lab3-lb-1 4ba96e32076f Up 5 seconds (healthy) -messenger-lab3-migrate-1 18f3889ef9a7 Exited (0) 12 seconds ago -messenger-lab3-postgres-1 4c60c6522907 Up 15 seconds (healthy) +## after (same IDs, new uptime) +NAME CONTAINER ID STATUS +messenger-lab3-api-1 1b755b53e9fa Up 12 seconds (healthy) +messenger-lab3-api-2 e3303afc01b2 Up 11 seconds (healthy) +messenger-lab3-lb-1 760a68ddce9c Up 5 seconds (healthy) +messenger-lab3-postgres-1 2265aa5e595d Up 16 seconds (healthy) volume messenger-lab3_pgdata +api1 172.25.0.2 UP +api2 172.25.0.3 UP diff --git a/evidence/rolling-update-migration.txt b/evidence/rolling-update-migration.txt new file mode 100644 index 00000000..5fff24ea --- /dev/null +++ b/evidence/rolling-update-migration.txt @@ -0,0 +1,24 @@ +# rolling update dev -> v2 with migration 003 applied under load, 2026-11-07 17:59 +# Console output copied verbatim; the JSON of this run was overwritten by the later v2 -> v3 run (rolling-update.json). +# Old replicas: image messenger-lab3-app:dev, built before migrations/003_session_indexes.sql existed +# (its readiness requires only 001, 002). New image messenger-lab3-app:v2 contains 001-003. +# Command: APP_VERSION=v2 python scripts/failover_series.py --mode action --action scripts/rolling-update.cmd --output evidence/rolling-update.json +running scripts/rolling-update.cmd under load... +{"total": {"seconds": 70.91, "rps_ok": 125.7, "writes_ok": 4458, "reads_ok": 4458, "errors": 0, "p95_ms": 78.1, "instances": {"419aeb0c6c36": 2714, "e90dced210a7": 1695, "a8021c6dc466": 2752, "a2bb8bd69281": 1755}}, "integrity": {"confirmed_writes": 4458, "stored_messages": 4458, "duplicates": 0, "missing_confirmed": 0}} +PASS: action, 4458 confirmed writes, no duplicates, nothing lost +## rolling-update.sh output and metrics +17:59:39 rolling 2 replica(s) to messenger-lab3-app:v2 +17:59:48 migrations applied once +17:59:59 new replica(s) UP in LB: a8021c6dc466 ; stopping old e90dced210a7 +18:00:05 old e90dced210a7 drained and removed +18:00:17 new replica(s) UP in LB: a2bb8bd69281 a8021c6dc466 ; stopping old 419aeb0c6c36 +18:00:24 old 419aeb0c6c36 drained and removed +messenger-lab3-api-3 messenger-lab3-app:v2 Up 33 seconds (healthy) +messenger-lab3-api-4 messenger-lab3-app:v2 Up 15 seconds (healthy) +18:00:24 done; set APP_VERSION=v2 in .env to keep this version on the next compose up +action_seconds 45.73 max_gap 0.097 +versions before {'dev': 1001} during {'dev': 3408, 'v2': 2234} after {'v2': 2265} +during {'seconds': 45.73, 'rps_ok': 123.4, 'writes_ok': 2821, 'reads_ok': 2821, 'errors': 0, 'p95_ms': 78.5, 'instances': {'419aeb0c6c36': 2214, 'e90dced210a7': 1194, 'a8021c6dc466': 1615, 'a2bb8bd69281': 619}} +# After 003 was applied at 17:59:48 both dev replicas stayed in rotation until they were replaced +# (17:59:59 and 18:00:17); dev served 3408 successful responses during the action with 0 errors overall, +# i.e. the old build stayed ready and correct on the expanded schema. diff --git a/evidence/rolling-update.json b/evidence/rolling-update.json new file mode 100644 index 00000000..4ea65bdb --- /dev/null +++ b/evidence/rolling-update.json @@ -0,0 +1,955 @@ +{ + "mode": "action", + "stop_method": "docker stop --time 9 (SIGTERM, drain)", + "concurrency": 4, + "workload": "closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest", + "replicas_before": [ + "messenger-lab3-api-1", + "messenger-lab3-api-2" + ], + "runs": [ + { + "action": "rolling-update.cmd", + "action_exit_code": 0, + "action_output": [ + "18:11:19 rolling 2 replica(s) to messenger-lab3-app:v3", + "18:11:26 migrations applied once", + "18:11:37 new replica(s) UP in LB: 3cfcdcac70c7 ; stopping old 1b755b53e9fa", + "18:11:43 old 1b755b53e9fa drained and removed", + "18:11:54 new replica(s) UP in LB: 13c61b35afd7 3cfcdcac70c7 ; stopping old e3303afc01b2", + "18:12:00 old e3303afc01b2 drained and removed", + "messenger-lab3-api-3 messenger-lab3-app:v3 Up 32 seconds (healthy)", + "messenger-lab3-api-4 messenger-lab3-app:v3 Up 15 seconds (healthy)", + "18:12:01 done; set APP_VERSION=v3 in .env to keep this version on the next compose up" + ], + "action_seconds": 42.67, + "max_gap_without_success_s": 0.073, + "baseline": { + "seconds": 10.1, + "rps_ok": 181.4, + "writes_ok": 917, + "reads_ok": 916, + "errors": 0, + "p95_ms": 53.4, + "instances": { + "1b755b53e9fa": 915, + "e3303afc01b2": 918 + } + }, + "during_action": { + "seconds": 42.67, + "rps_ok": 145.8, + "writes_ok": 3110, + "reads_ok": 3110, + "errors": 0, + "p95_ms": 68.3, + "instances": { + "e3303afc01b2": 2471, + "1b755b53e9fa": 1470, + "3cfcdcac70c7": 1689, + "13c61b35afd7": 590 + } + }, + "after_action": { + "seconds": 15.0, + "rps_ok": 185.8, + "writes_ok": 1394, + "reads_ok": 1393, + "errors": 0, + "p95_ms": 54.4, + "instances": { + "3cfcdcac70c7": 1394, + "13c61b35afd7": 1393 + } + }, + "versions_before": { + "v2": 1833 + }, + "versions_during": { + "v2": 3941, + "v3": 2279 + }, + "versions_after": { + "v3": 2787 + }, + "marks": { + "action_start": 10.1, + "action_end": 52.77 + } + } + ], + "total": { + "seconds": 67.8, + "rps_ok": 160.0, + "writes_ok": 5423, + "reads_ok": 5423, + "errors": 0, + "p95_ms": 62.6, + "instances": { + "1b755b53e9fa": 2385, + "e3303afc01b2": 3389, + "3cfcdcac70c7": 3086, + "13c61b35afd7": 1986 + } + }, + "integrity": { + "confirmed_writes": 5423, + "stored_messages": 5423, + "duplicates": 0, + "missing_confirmed": 0 + }, + "timeline_per_second": [ + { + "t": 0, + "ok": 174, + "err": 0, + "instances": { + "1b755b53e9fa": 87, + "e3303afc01b2": 87 + }, + "versions": { + "v2": 174 + } + }, + { + "t": 1, + "ok": 149, + "err": 0, + "instances": { + "e3303afc01b2": 76, + "1b755b53e9fa": 73 + }, + "versions": { + "v2": 149 + } + }, + { + "t": 2, + "ok": 148, + "err": 0, + "instances": { + "e3303afc01b2": 73, + "1b755b53e9fa": 75 + }, + "versions": { + "v2": 148 + } + }, + { + "t": 3, + "ok": 204, + "err": 0, + "instances": { + "1b755b53e9fa": 102, + "e3303afc01b2": 102 + }, + "versions": { + "v2": 204 + } + }, + { + "t": 4, + "ok": 226, + "err": 0, + "instances": { + "1b755b53e9fa": 113, + "e3303afc01b2": 113 + }, + "versions": { + "v2": 226 + } + }, + { + "t": 5, + "ok": 192, + "err": 0, + "instances": { + "1b755b53e9fa": 95, + "e3303afc01b2": 97 + }, + "versions": { + "v2": 192 + } + }, + { + "t": 6, + "ok": 156, + "err": 0, + "instances": { + "e3303afc01b2": 77, + "1b755b53e9fa": 79 + }, + "versions": { + "v2": 156 + } + }, + { + "t": 7, + "ok": 212, + "err": 0, + "instances": { + "1b755b53e9fa": 106, + "e3303afc01b2": 106 + }, + "versions": { + "v2": 212 + } + }, + { + "t": 8, + "ok": 193, + "err": 0, + "instances": { + "e3303afc01b2": 96, + "1b755b53e9fa": 97 + }, + "versions": { + "v2": 193 + } + }, + { + "t": 9, + "ok": 164, + "err": 0, + "instances": { + "1b755b53e9fa": 80, + "e3303afc01b2": 84 + }, + "versions": { + "v2": 164 + } + }, + { + "t": 10, + "ok": 100, + "err": 0, + "instances": { + "1b755b53e9fa": 53, + "e3303afc01b2": 47 + }, + "versions": { + "v2": 100 + } + }, + { + "t": 11, + "ok": 136, + "err": 0, + "instances": { + "e3303afc01b2": 69, + "1b755b53e9fa": 67 + }, + "versions": { + "v2": 136 + } + }, + { + "t": 12, + "ok": 161, + "err": 0, + "instances": { + "e3303afc01b2": 81, + "1b755b53e9fa": 80 + }, + "versions": { + "v2": 161 + } + }, + { + "t": 13, + "ok": 133, + "err": 0, + "instances": { + "1b755b53e9fa": 67, + "e3303afc01b2": 66 + }, + "versions": { + "v2": 133 + } + }, + { + "t": 14, + "ok": 99, + "err": 0, + "instances": { + "e3303afc01b2": 49, + "1b755b53e9fa": 50 + }, + "versions": { + "v2": 99 + } + }, + { + "t": 15, + "ok": 130, + "err": 0, + "instances": { + "e3303afc01b2": 66, + "1b755b53e9fa": 64 + }, + "versions": { + "v2": 130 + } + }, + { + "t": 16, + "ok": 215, + "err": 0, + "instances": { + "e3303afc01b2": 107, + "1b755b53e9fa": 108 + }, + "versions": { + "v2": 215 + } + }, + { + "t": 17, + "ok": 171, + "err": 0, + "instances": { + "e3303afc01b2": 86, + "1b755b53e9fa": 85 + }, + "versions": { + "v2": 171 + } + }, + { + "t": 18, + "ok": 102, + "err": 0, + "instances": { + "e3303afc01b2": 51, + "1b755b53e9fa": 51 + }, + "versions": { + "v2": 102 + } + }, + { + "t": 19, + "ok": 85, + "err": 0, + "instances": { + "e3303afc01b2": 42, + "1b755b53e9fa": 43 + }, + "versions": { + "v2": 85 + } + }, + { + "t": 20, + "ok": 124, + "err": 0, + "instances": { + "e3303afc01b2": 62, + "1b755b53e9fa": 62 + }, + "versions": { + "v2": 124 + } + }, + { + "t": 21, + "ok": 176, + "err": 0, + "instances": { + "1b755b53e9fa": 88, + "e3303afc01b2": 88 + }, + "versions": { + "v2": 176 + } + }, + { + "t": 22, + "ok": 198, + "err": 0, + "instances": { + "1b755b53e9fa": 99, + "e3303afc01b2": 99 + }, + "versions": { + "v2": 198 + } + }, + { + "t": 23, + "ok": 152, + "err": 0, + "instances": { + "e3303afc01b2": 76, + "1b755b53e9fa": 76 + }, + "versions": { + "v2": 152 + } + }, + { + "t": 24, + "ok": 176, + "err": 0, + "instances": { + "e3303afc01b2": 88, + "1b755b53e9fa": 88 + }, + "versions": { + "v2": 176 + } + }, + { + "t": 25, + "ok": 203, + "err": 0, + "instances": { + "e3303afc01b2": 91, + "1b755b53e9fa": 91, + "3cfcdcac70c7": 21 + }, + "versions": { + "v2": 182, + "v3": 21 + } + }, + { + "t": 26, + "ok": 194, + "err": 0, + "instances": { + "3cfcdcac70c7": 64, + "1b755b53e9fa": 64, + "e3303afc01b2": 66 + }, + "versions": { + "v3": 64, + "v2": 130 + } + }, + { + "t": 27, + "ok": 161, + "err": 0, + "instances": { + "3cfcdcac70c7": 55, + "1b755b53e9fa": 54, + "e3303afc01b2": 52 + }, + "versions": { + "v3": 55, + "v2": 106 + } + }, + { + "t": 28, + "ok": 104, + "err": 0, + "instances": { + "e3303afc01b2": 35, + "3cfcdcac70c7": 34, + "1b755b53e9fa": 35 + }, + "versions": { + "v2": 70, + "v3": 34 + } + }, + { + "t": 29, + "ok": 156, + "err": 0, + "instances": { + "3cfcdcac70c7": 52, + "e3303afc01b2": 52, + "1b755b53e9fa": 52 + }, + "versions": { + "v3": 52, + "v2": 104 + } + }, + { + "t": 30, + "ok": 174, + "err": 0, + "instances": { + "e3303afc01b2": 58, + "1b755b53e9fa": 58, + "3cfcdcac70c7": 58 + }, + "versions": { + "v2": 116, + "v3": 58 + } + }, + { + "t": 31, + "ok": 191, + "err": 0, + "instances": { + "e3303afc01b2": 74, + "3cfcdcac70c7": 74, + "1b755b53e9fa": 43 + }, + "versions": { + "v2": 117, + "v3": 74 + } + }, + { + "t": 32, + "ok": 162, + "err": 0, + "instances": { + "3cfcdcac70c7": 81, + "e3303afc01b2": 81 + }, + "versions": { + "v3": 81, + "v2": 81 + } + }, + { + "t": 33, + "ok": 166, + "err": 0, + "instances": { + "e3303afc01b2": 83, + "3cfcdcac70c7": 83 + }, + "versions": { + "v2": 83, + "v3": 83 + } + }, + { + "t": 34, + "ok": 175, + "err": 0, + "instances": { + "3cfcdcac70c7": 87, + "e3303afc01b2": 88 + }, + "versions": { + "v3": 87, + "v2": 88 + } + }, + { + "t": 35, + "ok": 114, + "err": 0, + "instances": { + "e3303afc01b2": 57, + "3cfcdcac70c7": 57 + }, + "versions": { + "v2": 57, + "v3": 57 + } + }, + { + "t": 36, + "ok": 179, + "err": 0, + "instances": { + "3cfcdcac70c7": 91, + "e3303afc01b2": 88 + }, + "versions": { + "v3": 91, + "v2": 88 + } + }, + { + "t": 37, + "ok": 92, + "err": 0, + "instances": { + "e3303afc01b2": 46, + "3cfcdcac70c7": 46 + }, + "versions": { + "v2": 46, + "v3": 46 + } + }, + { + "t": 38, + "ok": 101, + "err": 0, + "instances": { + "3cfcdcac70c7": 49, + "e3303afc01b2": 52 + }, + "versions": { + "v3": 49, + "v2": 52 + } + }, + { + "t": 39, + "ok": 121, + "err": 0, + "instances": { + "e3303afc01b2": 60, + "3cfcdcac70c7": 61 + }, + "versions": { + "v2": 60, + "v3": 61 + } + }, + { + "t": 40, + "ok": 128, + "err": 0, + "instances": { + "e3303afc01b2": 64, + "3cfcdcac70c7": 64 + }, + "versions": { + "v2": 64, + "v3": 64 + } + }, + { + "t": 41, + "ok": 148, + "err": 0, + "instances": { + "e3303afc01b2": 73, + "3cfcdcac70c7": 75 + }, + "versions": { + "v2": 73, + "v3": 75 + } + }, + { + "t": 42, + "ok": 115, + "err": 0, + "instances": { + "e3303afc01b2": 56, + "3cfcdcac70c7": 54, + "13c61b35afd7": 5 + }, + "versions": { + "v2": 56, + "v3": 59 + } + }, + { + "t": 43, + "ok": 150, + "err": 0, + "instances": { + "e3303afc01b2": 50, + "13c61b35afd7": 51, + "3cfcdcac70c7": 49 + }, + "versions": { + "v2": 50, + "v3": 100 + } + }, + { + "t": 44, + "ok": 156, + "err": 0, + "instances": { + "e3303afc01b2": 51, + "3cfcdcac70c7": 53, + "13c61b35afd7": 52 + }, + "versions": { + "v2": 51, + "v3": 105 + } + }, + { + "t": 45, + "ok": 140, + "err": 0, + "instances": { + "e3303afc01b2": 48, + "13c61b35afd7": 46, + "3cfcdcac70c7": 46 + }, + "versions": { + "v2": 48, + "v3": 92 + } + }, + { + "t": 46, + "ok": 134, + "err": 0, + "instances": { + "3cfcdcac70c7": 45, + "13c61b35afd7": 45, + "e3303afc01b2": 44 + }, + "versions": { + "v3": 90, + "v2": 44 + } + }, + { + "t": 47, + "ok": 136, + "err": 0, + "instances": { + "e3303afc01b2": 32, + "3cfcdcac70c7": 52, + "13c61b35afd7": 52 + }, + "versions": { + "v2": 32, + "v3": 104 + } + }, + { + "t": 48, + "ok": 117, + "err": 0, + "instances": { + "3cfcdcac70c7": 58, + "13c61b35afd7": 59 + }, + "versions": { + "v3": 117 + } + }, + { + "t": 49, + "ok": 163, + "err": 0, + "instances": { + "3cfcdcac70c7": 82, + "13c61b35afd7": 81 + }, + "versions": { + "v3": 163 + } + }, + { + "t": 50, + "ok": 187, + "err": 0, + "instances": { + "13c61b35afd7": 94, + "3cfcdcac70c7": 93 + }, + "versions": { + "v3": 187 + } + }, + { + "t": 51, + "ok": 140, + "err": 0, + "instances": { + "13c61b35afd7": 69, + "3cfcdcac70c7": 71 + }, + "versions": { + "v3": 140 + } + }, + { + "t": 52, + "ok": 101, + "err": 0, + "instances": { + "3cfcdcac70c7": 50, + "13c61b35afd7": 51 + }, + "versions": { + "v3": 101 + } + }, + { + "t": 53, + "ok": 165, + "err": 0, + "instances": { + "3cfcdcac70c7": 82, + "13c61b35afd7": 83 + }, + "versions": { + "v3": 165 + } + }, + { + "t": 54, + "ok": 240, + "err": 0, + "instances": { + "3cfcdcac70c7": 120, + "13c61b35afd7": 120 + }, + "versions": { + "v3": 240 + } + }, + { + "t": 55, + "ok": 253, + "err": 0, + "instances": { + "13c61b35afd7": 126, + "3cfcdcac70c7": 127 + }, + "versions": { + "v3": 253 + } + }, + { + "t": 56, + "ok": 150, + "err": 0, + "instances": { + "3cfcdcac70c7": 75, + "13c61b35afd7": 75 + }, + "versions": { + "v3": 150 + } + }, + { + "t": 57, + "ok": 142, + "err": 0, + "instances": { + "3cfcdcac70c7": 71, + "13c61b35afd7": 71 + }, + "versions": { + "v3": 142 + } + }, + { + "t": 58, + "ok": 197, + "err": 0, + "instances": { + "3cfcdcac70c7": 98, + "13c61b35afd7": 99 + }, + "versions": { + "v3": 197 + } + }, + { + "t": 59, + "ok": 202, + "err": 0, + "instances": { + "3cfcdcac70c7": 101, + "13c61b35afd7": 101 + }, + "versions": { + "v3": 202 + } + }, + { + "t": 60, + "ok": 175, + "err": 0, + "instances": { + "13c61b35afd7": 87, + "3cfcdcac70c7": 88 + }, + "versions": { + "v3": 175 + } + }, + { + "t": 61, + "ok": 126, + "err": 0, + "instances": { + "3cfcdcac70c7": 62, + "13c61b35afd7": 64 + }, + "versions": { + "v3": 126 + } + }, + { + "t": 62, + "ok": 190, + "err": 0, + "instances": { + "3cfcdcac70c7": 96, + "13c61b35afd7": 94 + }, + "versions": { + "v3": 190 + } + }, + { + "t": 63, + "ok": 209, + "err": 0, + "instances": { + "3cfcdcac70c7": 104, + "13c61b35afd7": 105 + }, + "versions": { + "v3": 209 + } + }, + { + "t": 64, + "ok": 214, + "err": 0, + "instances": { + "13c61b35afd7": 107, + "3cfcdcac70c7": 107 + }, + "versions": { + "v3": 214 + } + }, + { + "t": 65, + "ok": 145, + "err": 0, + "instances": { + "13c61b35afd7": 72, + "3cfcdcac70c7": 73 + }, + "versions": { + "v3": 145 + } + }, + { + "t": 66, + "ok": 166, + "err": 0, + "instances": { + "3cfcdcac70c7": 83, + "13c61b35afd7": 83 + }, + "versions": { + "v3": 166 + } + }, + { + "t": 67, + "ok": 188, + "err": 0, + "instances": { + "3cfcdcac70c7": 94, + "13c61b35afd7": 94 + }, + "versions": { + "v3": 188 + } + } + ] +} diff --git a/evidence/rolling-update.txt b/evidence/rolling-update.txt new file mode 100644 index 00000000..fac58478 --- /dev/null +++ b/evidence/rolling-update.txt @@ -0,0 +1,19 @@ +# rolling update 2026-11-07T18:12:17.4627477+03:00: APP_VERSION=v3 (image prebuilt), python scripts/failover_series.py --mode action --action scripts/rolling-update.cmd +running scripts/rolling-update.cmd under load... +{"total": {"seconds": 67.8, "rps_ok": 160.0, "writes_ok": 5423, "reads_ok": 5423, "errors": 0, "p95_ms": 62.6, "instances": {"1b755b53e9fa": 2385, "e3303afc01b2": 3389, "3cfcdcac70c7": 3086, "13c61b35afd7": 1986}}, "integrity": {"confirmed_writes": 5423, "stored_messages": 5423, "duplicates": 0, "missing_confirmed": 0}} +PASS: action, 5423 confirmed writes, no duplicates, nothing lost +## rolling-update.sh output and metrics +18:11:19 rolling 2 replica(s) to messenger-lab3-app:v3 +18:11:26 migrations applied once +18:11:37 new replica(s) UP in LB: 3cfcdcac70c7 ; stopping old 1b755b53e9fa +18:11:43 old 1b755b53e9fa drained and removed +18:11:54 new replica(s) UP in LB: 13c61b35afd7 3cfcdcac70c7 ; stopping old e3303afc01b2 +18:12:00 old e3303afc01b2 drained and removed +messenger-lab3-api-3 messenger-lab3-app:v3 Up 32 seconds (healthy) +messenger-lab3-api-4 messenger-lab3-app:v3 Up 15 seconds (healthy) +18:12:01 done; set APP_VERSION=v3 in .env to keep this version on the next compose up +action_seconds 42.67 max_gap 0.073 +versions before {'v2': 1833} during {'v2': 3941, 'v3': 2279} after {'v3': 2787} +baseline {'seconds': 10.1, 'rps_ok': 181.4, 'writes_ok': 917, 'reads_ok': 916, 'errors': 0, 'p95_ms': 53.4, 'instances': {'1b755b53e9fa': 915, 'e3303afc01b2': 918}} +during {'seconds': 42.67, 'rps_ok': 145.8, 'writes_ok': 3110, 'reads_ok': 3110, 'errors': 0, 'p95_ms': 68.3, 'instances': {'e3303afc01b2': 2471, '1b755b53e9fa': 1470, '3cfcdcac70c7': 1689, '13c61b35afd7': 590}} +after {'seconds': 15.0, 'rps_ok': 185.8, 'writes_ok': 1394, 'reads_ok': 1393, 'errors': 0, 'p95_ms': 54.4, 'instances': {'3cfcdcac70c7': 1394, '13c61b35afd7': 1393}} diff --git a/evidence/smoke.txt b/evidence/smoke.txt index dec53a71..09308293 100644 --- a/evidence/smoke.txt +++ b/evidence/smoke.txt @@ -1,14 +1,48 @@ -# smoke run 2026-11-07T17:36:58.5045267+03:00 -## docker compose ps -messenger-lab3-api-1 8b33a91fafee Up 19 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 19 seconds (healthy) -messenger-lab3-lb-1 4ba96e32076f Up 13 seconds (healthy) -messenger-lab3-postgres-1 4c60c6522907 Up 23 seconds (healthy) -## make check PYTHON=python -python scripts/check_contract.py +# smoke run 2026-11-07T18:01:21.4109350+03:00, commit 124c83e + working tree +## docker compose --env-file .env down -v + + Container messenger-lab3-lb-1 Removed + Container messenger-lab3-api-4 Removed + Container messenger-lab3-api-3 Removed + Container messenger-lab3-migrate-1 Removed + Container messenger-lab3-postgres-1 Removed + Volume messenger-lab3_pgdata Removing + Volume messenger-lab3_pgdata Removed + Network messenger-lab3_data Removed + Network messenger-lab3_edge Removed +## docker compose --env-file .env config --quiet +exit=0 +## docker compose --env-file .env up -d --build --wait + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-migrate-1 Exited + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-2 Healthy + Container messenger-lab3-migrate-1 Exited + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-2 Healthy + Container messenger-lab3-lb-1 Healthy +exit=0 +## docker compose ps -a +NAME IMAGE STATUS PORTS +messenger-lab3-api-1 messenger-lab3-app:v2 Up 12 seconds (healthy) 8080/tcp +messenger-lab3-api-2 messenger-lab3-app:v2 Up 12 seconds (healthy) 8080/tcp +messenger-lab3-lb-1 haproxy:3.2.26-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0 Up 5 seconds (healthy) 127.0.0.1:8080->8080/tcp, [::1]:8080->8080/tcp +messenger-lab3-migrate-1 messenger-lab3-app:v2 Exited (0) 12 seconds ago +messenger-lab3-postgres-1 postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297 Up 19 seconds (healthy) 5432/tcp +## migrate log (one-shot) +INFO applied migration 001_init.sql +INFO applied migration 002_idempotency.sql +INFO applied migration 003_session_indexes.sql +## HAProxy pool (scripts/lb-status.sh) +api1 172.25.0.2 UP +api2 172.25.0.3 UP +## make check PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. exit=0 -## make test PYTHON=python -python tests/smoke.py --base-url "http://localhost:8080" +## make test PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. exit=0 + + diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..96c3dcdb --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,79 @@ +# Client -> HAProxy -> api x N. Replicas are discovered through Docker DNS and checked actively. +# +# Timing contract (keep in sync with compose.yaml and app settings): +# check period inter 1s, timeout check 1s +# marked DOWN after fall 2 failed checks -> hung/draining replica leaves the pool in <= 3 s +# marked UP after rise 2 good checks -> new replica gets traffic ~2 s after it is ready +# DNS refresh every 1 s (hold valid 1s); a removed container disappears within ~1-2 s +# crashed process connect refused -> request redispatched to another replica at once +# app DRAIN_SECONDS=4 > 3 s, so a replica receiving SIGTERM is out of rotation before it closes its socket. + +global + log stdout format raw local0 info + maxconn 2048 + # Runtime API for evidence (scripts/lb-status.sh); loopback inside the container only. + stats socket ipv4@127.0.0.1:9999 level operator + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold refused 1s + hold nx 1s + hold timeout 1s + hold obsolete 0s + +defaults + mode http + log global + option httplog + option dontlognull + option forwardfor + timeout connect 1s + timeout client 30s + timeout server 10s + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 2s + timeout check 1s + # A request whose connection to a replica failed is sent to another replica. + retries 2 + option redispatch 1 + +frontend public + bind :8080 + monitor-uri /lb/health + http-request deny deny_status 413 if { req.hdr_val(content-length) gt 1048576 } + + # Message POST carries a mandatory Idempotency-Key, so replaying it on another replica is safe. + # Other POSTs (users, chats, sessions) are retried only when the connection was never established. + acl message_post method POST + acl message_post_path path_reg ^/api/v1/chats/[^/]+/messages$ + http-request disable-l7-retry if message_post !message_post_path + + default_backend api + +backend api + # Round robin, no stickiness: any replica serves any session from PostgreSQL. + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + retry-on conn-failure empty-response response-timeout + http-error status 503 content-type application/json string "{\"error\":{\"code\":\"unavailable\",\"message\":\"No API replica is available\",\"request_id\":\"lb\"}}" + http-error status 502 content-type application/json string "{\"error\":{\"code\":\"internal_error\",\"message\":\"Bad gateway\",\"request_id\":\"lb\"}}" + http-error status 504 content-type application/json string "{\"error\":{\"code\":\"internal_error\",\"message\":\"Gateway timeout\",\"request_id\":\"lb\"}}" + default-server check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none + server-template api 1-10 api:8080 + +# Read-only stats for evidence; not published, reachable only inside the container. +frontend stats + bind 127.0.0.1:8404 + stats enable + stats uri /stats + stats refresh 5s + http-request set-log-level silent diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf deleted file mode 100644 index 0ebbd32b..00000000 --- a/infra/nginx/nginx.conf +++ /dev/null @@ -1,66 +0,0 @@ -worker_processes auto; -pid /tmp/nginx.pid; - -events { - worker_connections 1024; -} - -http { - access_log /dev/stdout; - error_log /dev/stderr warn; - - # Docker embedded DNS. "api" resolves to every running replica of the Compose service; - # with "resolve" below nginx re-reads the set every 2 s, so stopped/scaled replicas leave and join the pool. - resolver 127.0.0.11 valid=2s ipv6=off; - resolver_timeout 2s; - - log_format upstream '$remote_addr "$request" $status upstream=$upstream_addr ' - 'upstream_status=$upstream_status rt=$request_time'; - - upstream api { - zone api_upstream 64k; - # Round robin, no ip_hash/sticky: any replica serves any session from PostgreSQL. - server api:8080 resolve max_fails=1 fail_timeout=5s; - keepalive 32; - # Shorter than uvicorn's keep-alive (5 s), so nginx never reuses a connection the API is closing. - keepalive_timeout 4s; - } - - server { - listen 8080; - access_log /dev/stdout upstream; - client_max_body_size 1m; - - proxy_http_version 1.1; - proxy_set_header Connection ""; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_connect_timeout 1s; - proxy_send_timeout 10s; - proxy_read_timeout 10s; - - # Not-yet-sent requests and idempotent methods go to the next replica on a dead backend. - # Non-idempotent POSTs (users, chats, sessions) are never replayed by the LB. - proxy_next_upstream error timeout http_502 http_504; - proxy_next_upstream_tries 3; - proxy_next_upstream_timeout 5s; - - location = /lb/health { - access_log off; - default_type application/json; - return 200 '{"status":"ok"}'; - } - - # Message POST carries a mandatory Idempotency-Key, so replaying it on another replica is safe. - location ~ ^/api/v1/chats/[^/]+/messages$ { - proxy_next_upstream error timeout http_502 http_504 non_idempotent; - proxy_pass http://api; - } - - location / { - proxy_pass http://api; - } - } -} diff --git a/migrations/003_session_indexes.sql b/migrations/003_session_indexes.sql new file mode 100644 index 00000000..0b7eb784 --- /dev/null +++ b/migrations/003_session_indexes.sql @@ -0,0 +1,4 @@ +-- Expand-only: new indexes, no column/table removal or rename, so replicas of the previous build keep working +-- while the rolling update is in progress. Destructive changes go to a later release, after old replicas are gone. +CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id); +CREATE INDEX IF NOT EXISTS sessions_expires_idx ON sessions (expires_at); diff --git a/scripts/failover_series.py b/scripts/failover_series.py new file mode 100755 index 00000000..c9750b45 --- /dev/null +++ b/scripts/failover_series.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Closed-loop load through the LB while API replicas are stopped one by one (or an action script runs). + +failover: for every running api container in turn: baseline -> docker stop (SIGTERM, drain) -> observe -> + docker start the same container -> wait until it serves again -> settle. Restores before the next one. +action: baseline -> run --action script (e.g. rolling-update) under load -> observe. + +Workers alternate POST message (Idempotency-Key; an uncertain result is retried with the SAME key and body) +and GET history page. Per-second successes/errors and serving X-Instance-Id / X-App-Version are recorded. +At the end the whole chat history is read back: every confirmed message must be present exactly once. +Session tokens stay in memory and are never printed. +""" +import argparse +import json +import subprocess +import sys +import threading +import time +import uuid +from collections import Counter, defaultdict +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check, run # noqa: E402 +from resilience import history # noqa: E402 + +PROJECT = 'messenger-lab3' + + +def docker(*args, timeout=60): + return subprocess.run(['docker', *args], capture_output=True, text=True, timeout=timeout, check=True).stdout.strip() + + +def api_containers(): + """(name, short id) of running api containers of this Compose project, sorted by name.""" + out = docker('ps', '--filter', 'label=com.docker.compose.project=' + PROJECT, + '--filter', 'label=com.docker.compose.service=api', '--format', '{{.Names}} {{.ID}}') + return sorted(tuple(line.split()) for line in out.splitlines() if line.strip()) + + +class Load: + def __init__(self, base_url, concurrency): + self.base_url = base_url + self.concurrency = concurrency + self.events = [] # (t, op, ok, instance, version, latency) + self.confirmed = {} # message id -> text + self.lock = threading.Lock() + self.stop = threading.Event() + self.t0 = None + + def now(self): + return time.monotonic() - self.t0 + + def record(self, op, ok, response=None, latency=0.0): + headers = response['headers'] if response else {} + event = (self.now(), op, ok, headers.get('X-Instance-Id') if response else None, + headers.get('X-App-Version') if response else None, latency) + with self.lock: + self.events.append(event) + + def worker(self, worker_id, alice, chat): + c = Client(self.base_url, timeout=5) + path = '/api/v1/chats/' + chat['id'] + '/messages?limit=20' + index = 0 + key = text = None + while not self.stop.is_set(): + if key is None: + key = uuid.uuid4().hex + text = 'series-%d-%d-%s' % (worker_id, index, key[:8]) + start = time.monotonic() + try: + response = c.message(alice, chat, text, key=key) + self.record('write', True, response, time.monotonic() - start) + with self.lock: + self.confirmed[response['json']['id']] = text + key = None + index += 1 + except Failure: + self.record('write', False, latency=time.monotonic() - start) + time.sleep(0.05) + continue # same key and text: the first attempt may have committed + start = time.monotonic() + try: + response = c.call('GET', path, actor=alice, schema='MessagePage') + self.record('read', True, response, time.monotonic() - start) + except Failure: + self.record('read', False, latency=time.monotonic() - start) + time.sleep(0.05) + + def start(self, alice, chat): + self.t0 = time.monotonic() + self.threads = [threading.Thread(target=self.worker, args=(i, alice, chat), daemon=True) + for i in range(self.concurrency)] + for t in self.threads: + t.start() + + def finish(self): + self.stop.set() + for t in self.threads: + t.join(timeout=10) + + def window(self, start, end): + return [e for e in self.events if start <= e[0] < end] + + +def summarize(events, start, end): + ok = [e for e in events if e[2]] + duration = max(end - start, 1e-9) + latencies = sorted(e[5] for e in ok) + p95 = round(1000 * latencies[max(0, int(0.95 * len(latencies)) - 1)], 1) if latencies else None + return { + 'seconds': round(duration, 2), + 'rps_ok': round(len(ok) / duration, 1), + 'writes_ok': sum(1 for e in ok if e[1] == 'write'), + 'reads_ok': sum(1 for e in ok if e[1] == 'read'), + 'errors': sum(1 for e in events if not e[2]), + 'p95_ms': p95, + 'instances': dict(Counter(e[3] for e in ok)), + } + + +def timeline(load, end): + buckets = defaultdict(lambda: {'ok': 0, 'err': 0, 'instances': Counter(), 'versions': Counter()}) + for t, _, ok, instance, version, _ in load.events: + if t >= end: + continue + b = buckets[int(t)] + if ok: + b['ok'] += 1 + b['instances'][instance] += 1 + b['versions'][version] += 1 + else: + b['err'] += 1 + return [{'t': s, 'ok': buckets[s]['ok'], 'err': buckets[s]['err'], + 'instances': dict(buckets[s]['instances']), 'versions': dict(buckets[s]['versions'])} + for s in range(int(end) + 1)] + + +def max_gap(events, start, end): + """Longest interval without any successful response inside [start, end).""" + times = [start] + sorted(e[0] for e in events if e[2] and start <= e[0] < end) + [end] + return round(max(b - a for a, b in zip(times, times[1:])), 3) + + +def wait_serving(load, instance, since, limit=30): + deadline = time.monotonic() + limit + while time.monotonic() < deadline: + with load.lock: + hit = next((e[0] for e in reversed(load.events) if e[2] and e[3] == instance and e[0] >= since), None) + if hit is not None: + return hit + time.sleep(0.2) + return None + + +def failover_run(load, name, cid, baseline, observe, settle, stop_timeout, kill): + time.sleep(baseline) + t_stop = load.now() + if kill: + docker('kill', cid) + else: + docker('stop', '--time', str(stop_timeout), cid, timeout=stop_timeout + 30) + t_stopped = load.now() + state = docker('inspect', '--format', '{{.State.Status}} exit={{.State.ExitCode}}', cid) + time.sleep(observe) + t_start = load.now() + docker('start', cid) + rejoined = wait_serving(load, cid, t_start) + time.sleep(settle) + t_end = load.now() + + with load.lock: + events = list(load.events) + base = summarize([e for e in events if t_stop - baseline <= e[0] < t_stop], t_stop - baseline, t_stop) + outage = [e for e in events if t_stop <= e[0] < t_start] + last_target = max((e[0] for e in outage if e[2] and e[3] == cid), default=None) + first_ok_after_stop = min((e[0] for e in outage if e[2] and e[0] >= t_stopped), default=None) + # Recovery: first full second after the container stopped whose throughput is >= 90% of baseline. + recovered = None + for s in range(int(t_stopped), int(t_start)): + n = sum(1 for e in outage if e[2] and s <= e[0] < s + 1) + if n >= 0.9 * base['rps_ok']: + recovered = s + 1 - t_stopped + break + return { + 'replica': name, + 'instance_id': cid, + 'state_after_stop': state, + 'stop_command_seconds': round(t_stopped - t_stop, 2), + 'last_served_by_target_after_sigterm_s': round(last_target - t_stop, 2) if last_target else None, + 'first_success_after_container_exit_s': round(first_ok_after_stop - t_stopped, 3) if first_ok_after_stop else None, + 'rps_back_to_90pct_after_exit_s': round(recovered, 2) if recovered is not None else None, + 'max_gap_without_success_s': max_gap(events, t_stop, t_start), + 'rejoin_after_docker_start_s': round(rejoined - t_start, 2) if rejoined else None, + 'baseline': base, + 'during_stop_and_outage': summarize(outage, t_stop, t_start), + 'after_restore': summarize([e for e in events if t_start <= e[0] < t_end], t_start, t_end), + 'marks': {'sigterm': round(t_stop, 2), 'exited': round(t_stopped, 2), 'docker_start': round(t_start, 2)}, + } + + +def action_run(load, action, baseline, observe): + time.sleep(baseline) + t_action = load.now() + proc = subprocess.run([str(Path(action).resolve())], capture_output=True, text=True, timeout=600) + t_done = load.now() + time.sleep(observe) + t_end = load.now() + with load.lock: + events = list(load.events) + during = [e for e in events if t_action <= e[0] < t_done] + return { + 'action': Path(action).name, + 'action_exit_code': proc.returncode, + 'action_output': proc.stdout.strip().splitlines()[-30:], + 'action_seconds': round(t_done - t_action, 2), + 'max_gap_without_success_s': max_gap(events, t_action, t_done), + 'baseline': summarize([e for e in events if e[0] < t_action], 0, t_action), + 'during_action': summarize(during, t_action, t_done), + 'after_action': summarize([e for e in events if t_done <= e[0] < t_end], t_done, t_end), + 'versions_before': dict(Counter(e[4] for e in events if e[2] and e[0] < t_action)), + 'versions_during': dict(Counter(e[4] for e in during if e[2])), + 'versions_after': dict(Counter(e[4] for e in events if e[2] and t_done <= e[0] < t_end)), + 'marks': {'action_start': round(t_action, 2), 'action_end': round(t_done, 2)}, + } + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument('--base-url', default=None) + p.add_argument('--mode', choices=['failover', 'action'], default='failover') + p.add_argument('--action', help='script to run under load in --mode action') + p.add_argument('--concurrency', type=int, default=4) + p.add_argument('--baseline', type=float, default=10) + p.add_argument('--observe', type=float, default=15) + p.add_argument('--settle', type=float, default=5) + p.add_argument('--stop-timeout', type=int, default=9) + p.add_argument('--kill', action='store_true', help='docker kill (SIGKILL, crash without drain) instead of docker stop') + p.add_argument('--output', help='JSON result file (no tokens)') + args = p.parse_args() + check(args.mode != 'action' or args.action, '--action is required in action mode') + + c = Client(args.base_url) + alice, bob, _, chat = c.bootstrap() + replicas = api_containers() + if args.mode == 'failover': + check(len(replicas) >= 2, 'need at least two running api containers') + load = Load(args.base_url, args.concurrency) + load.start(alice, chat) + runs = [] + try: + if args.mode == 'failover': + for name, cid in replicas: + print('%s %s (%s) under load...' % ('killing' if args.kill else 'stopping', name, cid), flush=True) + runs.append(failover_run(load, name, cid, args.baseline, args.observe, args.settle, + args.stop_timeout, args.kill)) + r = runs[-1] + print(' errors=%d gap=%.2fs first_ok_after_exit=%s rps %.1f -> %.1f -> %.1f rejoin=%ss' % ( + r['during_stop_and_outage']['errors'], r['max_gap_without_success_s'], + r['first_success_after_container_exit_s'], r['baseline']['rps_ok'], + r['during_stop_and_outage']['rps_ok'], r['after_restore']['rps_ok'], + r['rejoin_after_docker_start_s']), flush=True) + else: + print('running %s under load...' % args.action, flush=True) + runs.append(action_run(load, args.action, args.baseline, args.observe)) + finally: + load.finish() + end = load.now() + + confirmed = dict(load.confirmed) + stored = [m for m in history(c, bob, chat) if m['text'].startswith('series-')] + counts = Counter(m['text'] for m in stored) + duplicates = sum(n - 1 for n in counts.values() if n > 1) + stored_ids = {m['id'] for m in stored} + missing = [mid for mid in confirmed if mid not in stored_ids] + integrity = {'confirmed_writes': len(confirmed), 'stored_messages': len(stored), + 'duplicates': duplicates, 'missing_confirmed': len(missing)} + report = { + 'mode': args.mode, + 'stop_method': 'docker kill (SIGKILL)' if args.kill else 'docker stop --time %d (SIGTERM, drain)' % args.stop_timeout, + 'concurrency': args.concurrency, + 'workload': 'closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest', + 'replicas_before': [name for name, _ in replicas], + 'runs': runs, + 'total': summarize(load.events, 0, end), + 'integrity': integrity, + 'timeline_per_second': timeline(load, end), + } + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + out = Path(args.output) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(text + '\n', encoding='utf-8') + print(json.dumps({'total': report['total'], 'integrity': integrity}, ensure_ascii=False)) + check(duplicates == 0 and not missing, 'duplicates or lost confirmed messages: %s' % integrity) + if args.mode == 'action': + check(runs[0]['action_exit_code'] == 0, 'action script failed') + print('PASS: %s, %d confirmed writes, no duplicates, nothing lost' % (args.mode, len(confirmed))) + + +if __name__ == '__main__': + run(main) diff --git a/scripts/lb-status.cmd b/scripts/lb-status.cmd new file mode 100644 index 00000000..f1a81fe0 --- /dev/null +++ b/scripts/lb-status.cmd @@ -0,0 +1,3 @@ +@echo off +rem Windows entry point: runs lb-status.sh with Git Bash. +"%ProgramFiles%\Git\bin\sh.exe" "%~dp0lb-status.sh" diff --git a/scripts/lb-status.sh b/scripts/lb-status.sh new file mode 100755 index 00000000..e8277685 --- /dev/null +++ b/scripts/lb-status.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# HAProxy view of the api pool: slot, replica IP, state (2=UP, 0=DOWN), check result. Empty template slots are hidden. +docker compose --env-file .env exec -T lb sh -c \ + "echo 'show servers state api' | nc 127.0.0.1 9999 | awk '\$1 !~ /^#/ && NF > 10 && \$5 != \"-\" {print \$4, \$5, (\$6 == 2 ? \"UP\" : \"DOWN\")}'" diff --git a/scripts/rolling-update.cmd b/scripts/rolling-update.cmd new file mode 100644 index 00000000..9d6cae27 --- /dev/null +++ b/scripts/rolling-update.cmd @@ -0,0 +1,3 @@ +@echo off +rem Windows entry point: runs rolling-update.sh with Git Bash. Set APP_VERSION first. +"%ProgramFiles%\Git\bin\sh.exe" "%~dp0rolling-update.sh" diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..27224d07 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,47 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Replace API replicas one at a time with a new build while the LB keeps serving. +# APP_VERSION=v2 scripts/rolling-update.sh +# 1. Build messenger-lab3-app:$APP_VERSION. +# 2. Run migrations ONCE (expand-only, compatible with replicas still on the old build). +# 3. For each old replica: start one new replica (N+1), wait until Docker reports it healthy AND HAProxy +# marks it UP, then docker stop the old one (SIGTERM -> drain -> exit) and remove it. +# Capacity never drops below N. Afterwards put APP_VERSION=$APP_VERSION into .env, otherwise a later +# `docker compose up` recreates replicas from the version written there. +: "${APP_VERSION:?set APP_VERSION of the new build}" +export APP_VERSION +project="${COMPOSE_PROJECT_NAME:-messenger-lab3}" +compose() { docker compose --env-file .env "$@"; } +api_ids() { + docker ps -q --filter "label=com.docker.compose.project=$project" --filter "label=com.docker.compose.service=api" +} +lb_up() { sh scripts/lb-status.sh | grep -c ' UP$' || true; } + +old=$(api_ids) +count=$(printf '%s\n' "$old" | grep -c . || true) +[ "$count" -ge 1 ] || { echo "no running api replicas" >&2; exit 1; } +echo "$(date +%T) rolling $count replica(s) to messenger-lab3-app:$APP_VERSION" + +compose build --quiet api migrate +compose run --rm --no-deps -T migrate +echo "$(date +%T) migrations applied once" + +for id in $old; do + before=$(lb_up) + compose up -d --no-deps --no-recreate --wait --scale "api=$((count + 1))" api >/dev/null 2>&1 + new=$(api_ids | grep -v -F "$old" || true) + i=0 + until [ "$(lb_up)" -gt "$before" ]; do + i=$((i + 1)) + [ "$i" -le 60 ] || { echo "new replica did not become UP in HAProxy" >&2; exit 1; } + sleep 0.5 + done + echo "$(date +%T) new replica(s) UP in LB: $(printf '%s ' $new); stopping old $id" + docker stop --time 9 "$id" >/dev/null + docker rm "$id" >/dev/null + echo "$(date +%T) old $id drained and removed" +done + +compose ps api --format '{{.Name}} {{.Image}} {{.Status}}' +echo "$(date +%T) done; set APP_VERSION=$APP_VERSION in .env to keep this version on the next compose up" diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh index 6301907c..196469e9 100755 --- a/scripts/stop-one.sh +++ b/scripts/stop-one.sh @@ -23,6 +23,7 @@ if [ "$count" -ne 1 ]; then exit 1 fi name=$(docker inspect --format '{{.Name}}' "$matches") -# Graceful: SIGTERM, then SIGKILL after 8 s; returns before the test's 10 s limit. -docker stop --time 8 "$matches" >/dev/null +# Graceful: SIGTERM -> 4 s drain (readiness 503, LB drops it) -> <= 3 s in-flight finish. +# SIGKILL only after 9 s, so the hook still returns before the test's 10 s limit. +docker stop --time 9 "$matches" >/dev/null echo "stopped api replica ${name#/} ($TARGET_INSTANCE_ID)" -- 2.54.0 From 637d45c82008267b42b08f7c1a265ac9cc87ba78 Mon Sep 17 00:00:00 2001 From: Sunrise443 Date: Sun, 8 Nov 2026 13:13:34 +0300 Subject: [PATCH 4/4] level 5 track 1 --- .env.example | 7 + Makefile | 7 +- REPORT.md | 81 ++++-- app/db.py | 12 +- compose.yaml | 10 +- evidence/scale-cost.json | 511 +++++++++++++++++++++++++++++++++++++ evidence/scale-cost.svg | 53 ++++ evidence/scale-cost.txt | 13 + scripts/scale_cost.py | 538 +++++++++++++++++++++++++++++++++++++++ 9 files changed, 1205 insertions(+), 27 deletions(-) create mode 100644 evidence/scale-cost.json create mode 100644 evidence/scale-cost.svg create mode 100644 evidence/scale-cost.txt create mode 100644 scripts/scale_cost.py diff --git a/.env.example b/.env.example index 1d0d49ab..45766c3a 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,13 @@ SESSION_TTL_SECONDS=3600 # Sessions live in PostgreSQL; no Redis/Valkey, CACHE_URL_FILE is unused. SESSION_STORE=postgres API_REPLICAS=2 +# Per-replica API limits; total budget in track 1 is replicas × these values. Defaults match level 4. +API_CPUS=1.0 +API_MEMORY=256M +POSTGRES_CPUS=1.0 +POSTGRES_MEMORY=512M +DB_POOL_MIN=1 +DB_POOL_MAX=10 IDEMPOTENCY_TTL_SECONDS=86400 # Pick your own local password. It reaches containers only as the postgres_password secret file. POSTGRES_USER=messenger diff --git a/Makefile b/Makefile index 7207bf28..ca695a82 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,12 @@ TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) APP_VERSION ?= -.PHONY: help check validate test load test-persistence test-failover test-idempotency failover-series rolling-update lb-status +.PHONY: help check validate test load test-persistence test-failover test-idempotency failover-series rolling-update lb-status scale-cost help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @echo "failover-series, rolling-update APP_VERSION=..., lb-status: level 4 experiments (Docker required)" + @echo "scale-cost: level 5 track 1 (1/2/4 replicas at a fixed CPU/RAM budget, then grow resources)" check: $(PYTHON) scripts/check_contract.py @@ -46,3 +47,7 @@ rolling-update: lb-status: sh scripts/lb-status.sh + +# Level 5 track 1: 1/2/4 API replicas at a fixed total CPU/RAM budget, then a growing-resource series. +scale-cost: + $(PYTHON) scripts/scale_cost.py --base-url "$(BASE_URL)" diff --git a/REPORT.md b/REPORT.md index f816800b..7858e708 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,9 +1,9 @@ # Отчёт — лабораторная 3 - Автор, группа: Тимаева М.П., БПИ2302 -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Целевая оценка; выбранный трек 5, если нужен: 5, трек 1 (цена масштабирования) - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: документация Docker Compose / HAProxy / asyncpg; ИИ использовалось для написания отчета и помощи с написанием скриптов, цифры с прогона `scripts/scale_cost.py` 2026-11-08. ## Запуск из чистого clone @@ -31,6 +31,8 @@ python scripts/failover_series.py --kill --output evidence/failover-series-kill. docker compose --env-file .env build api migrate # с APP_VERSION=v3 в окружении make rolling-update APP_VERSION=v3 # Windows: scripts/failover_series.py --mode action --action scripts/rolling-update.cmd # после rolling update записать APP_VERSION=v3 в .env +# уровень 5, трек 1 (после прогона стенд возвращается к 2×1.0 CPU) +make scale-cost PYTHON=python # Windows; GNU Make: make scale-cost ``` Остановка без потери данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v` (удаляет volume `pgdata`). @@ -43,12 +45,12 @@ client -> lb (HAProxy :8080, loopback) -> api-1, api-2, … (порт не пу migrate (one-shot) ------------^ ``` -| Роль | Compose service | Образ | Health/readiness | Сети | Volume / секреты | Лимит CPU / RAM (резерв) | -| -------- | --------------- | ----------------------------- | ----------------------------------------------------- | ---------- | -------------------------------------------------- | ------------------------ | -| LB | `lb` | `haproxy:3.2.26-alpine@…` | `monitor-uri /lb/health`; активные проверки API | edge | только read-only конфиг, секретов нет | 0.5 / 128M (0.1 / 32M) | -| API × N | `api` | `messenger-lab3-app:<версия>` | `/health/live`, `/health/ready` (БД, миграции, drain) | edge, data | `/run/secrets/postgres_password` (uid 10001, 0400) | 1.0 / 256M (0.25 / 128M) | -| Миграции | `migrate` | тот же образ | one-shot, `service_completed_successfully` | data | тот же секрет | 0.5 / 128M | -| БД | `postgres` | `postgres:16.10-alpine@…` | `pg_isready` каждые 2 с | data | `pgdata`, `POSTGRES_PASSWORD_FILE` | 1.0 / 512M (0.25 / 256M) | +| Роль | Compose service | Образ | Health/readiness | Сети | Volume / секреты | Лимит CPU / RAM (резерв) | +| -------- | --------------- | ----------------------------- | ----------------------------------------------------- | ---------- | -------------------------------------------------- | ---------------------------------------------------------------------------------- | +| LB | `lb` | `haproxy:3.2.26-alpine@…` | `monitor-uri /lb/health`; активные проверки API | edge | только read-only конфиг, секретов нет | 0.5 / 128M (0.1 / 32M) | +| API × N | `api` | `messenger-lab3-app:<версия>` | `/health/live`, `/health/ready` (БД, миграции, drain) | edge, data | `/run/secrets/postgres_password` (uid 10001, 0400) | `${API_CPUS:-1.0}` / `${API_MEMORY:-256M}` (0.25 / 128M); пул `${DB_POOL_MAX:-10}` | +| Миграции | `migrate` | тот же образ | one-shot, `service_completed_successfully` | data | тот же секрет | 0.5 / 128M | +| БД | `postgres` | `postgres:16.10-alpine@…` | `pg_isready` каждые 2 с | data | `pgdata`, `POSTGRES_PASSWORD_FILE` | `${POSTGRES_CPUS:-1.0}` / `${POSTGRES_MEMORY:-512M}` (0.25 / 256M) | - **Источник истины — PostgreSQL**: пользователи, чаты, сообщения, сессии (`sessions`, токен хранится как SHA-256) и результаты идемпотентности (`idempotency_keys`). В API нет локального состояния, от которого зависит корректность. Redis из lab2 убран: он был только кэшем, а для общего хранилища сессий достаточно PostgreSQL. - **Изоляция**: наружу публикуется только `lb` и только на loopback. У API нет `ports`, `docker port` пуст. LB находится только в сети `edge` и не резолвит `postgres`. Сеть `data` объявлена `internal`, без выхода наружу. Volume `pgdata` смонтирован только в `postgres`. `container_name` нигде нет, поэтому `--scale api=N` работает (`evidence/limits.txt`). @@ -87,17 +89,18 @@ SIGTERM -> readiness 503 сразу; запросы ещё обслуживаю ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | -| Стенд в compose.yaml, ≥2 API, одна точка входа, общая БД и сессии | `down -v`, `config --quiet`, `up -d --build --wait` | config OK; `api-1`, `api-2`, `lb`, `postgres` healthy; `migrate` применил 001–003 и Exited (0); в HAProxy 2 реплики UP | `evidence/smoke.txt` | -| Одна сессия обслуживается разными репликами | `python scripts/show_instances.py` | 6 запросов с одним токеном: по 3 на каждую из 2 реплик | `evidence/instances.txt` | -| Остановка любой одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` ×2, между прогонами `up -d --wait` | 2 PASS: остановлены `api-2`, затем `api-1`, Exited (0); 121 и 115 записей, 0 ошибок; HAProxy убрал реплику из пула | `evidence/failover.txt` | -| Повтор POST с тем же Idempotency-Key | `make test-idempotency` | 5 ключей × 20 параллельных POST: по 10 ответов с каждой реплики, один Message на ключ, другое тело → 409 | `evidence/idempotency.txt` | -| make check, make test, persistence | `make check`, `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS; после рестарта lb, api, postgres те же ID, volume на месте, прежний токен работает | `evidence/smoke.txt`, `evidence/persistence.txt` | -| **4**: health/readiness, timeouts, SIGTERM согласованы; LB исключает недоступный backend | HAProxy active checks + drain; `scripts/lb-status.sh` | см. таблицу таймингов; останавливаемая реплика получает последний запрос через ≤2,05 с после SIGTERM; после выхода её нет в пуле | `infra/haproxy/haproxy.cfg`, `evidence/failover-series.json` | -| **4**: поочерёдная остановка каждой реплики: RPS, ошибки, время восстановления | `make failover-series`; `failover_series.py --kill` | по 2 реплики в каждом режиме, 0 ошибок, все подтверждённые записи на месте, дублей нет (таблица ниже) | `evidence/failover-series*.{txt,json}` | -| **4**: лимиты, нет портов API, секреты и хранилища только где нужно, нет `container_name` | `docker inspect`, `docker stats`, `docker port`, `nslookup` из lb | лимиты у всех 5 сервисов; порт только у lb (127.0.0.1/::1); пароль только файлом 0400; lb без секретов и без доступа к БД; `container_name`: 0 | `evidence/limits.txt` | -| **4**: rolling update под нагрузкой, миграции однократно и совместимо | `make rolling-update APP_VERSION=v3`; ранее `dev → v2` с миграцией 003 | 0 ошибок, максимальная пауза 0,07 с; v2 → v3 без снижения числа реплик ниже 2; при `dev → v2` миграция 003 применена один раз, и старые реплики `dev` продолжали работать | `evidence/rolling-update.{txt,json}`, `evidence/rolling-update-migration.txt` | +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| Стенд в compose.yaml, ≥2 API, одна точка входа, общая БД и сессии | `down -v`, `config --quiet`, `up -d --build --wait` | config OK; `api-1`, `api-2`, `lb`, `postgres` healthy; `migrate` применил 001–003 и Exited (0); в HAProxy 2 реплики UP | `evidence/smoke.txt` | +| Одна сессия обслуживается разными репликами | `python scripts/show_instances.py` | 6 запросов с одним токеном: по 3 на каждую из 2 реплик | `evidence/instances.txt` | +| Остановка любой одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` ×2, между прогонами `up -d --wait` | 2 PASS: остановлены `api-2`, затем `api-1`, Exited (0); 121 и 115 записей, 0 ошибок; HAProxy убрал реплику из пула | `evidence/failover.txt` | +| Повтор POST с тем же Idempotency-Key | `make test-idempotency` | 5 ключей × 20 параллельных POST: по 10 ответов с каждой реплики, один Message на ключ, другое тело → 409 | `evidence/idempotency.txt` | +| make check, make test, persistence | `make check`, `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS; после рестарта lb, api, postgres те же ID, volume на месте, прежний токен работает | `evidence/smoke.txt`, `evidence/persistence.txt` | +| **4**: health/readiness, timeouts, SIGTERM согласованы; LB исключает недоступный backend | HAProxy active checks + drain; `scripts/lb-status.sh` | см. таблицу таймингов; останавливаемая реплика получает последний запрос через ≤2,05 с после SIGTERM; после выхода её нет в пуле | `infra/haproxy/haproxy.cfg`, `evidence/failover-series.json` | +| **4**: поочерёдная остановка каждой реплики: RPS, ошибки, время восстановления | `make failover-series`; `failover_series.py --kill` | по 2 реплики в каждом режиме, 0 ошибок, все подтверждённые записи на месте, дублей нет (таблица ниже) | `evidence/failover-series*.{txt,json}` | +| **4**: лимиты, нет портов API, секреты и хранилища только где нужно, нет `container_name` | `docker inspect`, `docker stats`, `docker port`, `nslookup` из lb | лимиты у всех 5 сервисов; порт только у lb (127.0.0.1/::1); пароль только файлом 0400; lb без секретов и без доступа к БД; `container_name`: 0 | `evidence/limits.txt` | +| **4**: rolling update под нагрузкой, миграции однократно и совместимо | `make rolling-update APP_VERSION=v3`; ранее `dev → v2` с миграцией 003 | 0 ошибок, максимальная пауза 0,07 с; v2 → v3 без снижения числа реплик ниже 2; при `dev → v2` миграция 003 применена один раз, и старые реплики `dev` продолжали работать | `evidence/rolling-update.{txt,json}`, `evidence/rolling-update-migration.txt` | +| **5 трек 1**: 1/2/4 реплики при бюджете 2.0 CPU/512M, рост CPU, saturation, одно улучшение | `make scale-cost PYTHON=python` | 187.5 / 249.5 / 185.8 RPS; 4×0.5 → 2×1.0 даёт +34% RPS и p95 465→197; пул 20 отвергнут; LB не насыщен | `evidence/scale-cost.{json,txt,svg}` | Все прогоны 2026-11-07 на commit `124c83e` + рабочее дерево (до коммита уровня 4), Docker 29.4.3, Windows 11, один хост. `make test-failover/test-persistence` под Windows запускались той же командой из `Makefile` напрямую (`python tests/resilience.py ...`), потому что GNU Make под cmd не находит `test` в guard-строке. @@ -142,6 +145,44 @@ RPS — это пропускная способность закрытого ц **Политика миграций (expand/contract)**. Миграция выполняется до замены реплик и должна быть совместима с работающей версией: только новые таблицы, индексы и nullable-колонки. Пример — `003_session_indexes.sql`. Readiness старой сборки проверяет только свои миграции, поэтому расширенная схема её не ломает. Удаление и переименование колонок допустимы только отдельным релизом после ухода старых реплик. Ограничение: `CREATE INDEX` без `CONCURRENTLY` (мигратор работает в транзакции) коротко блокирует запись в индексируемую таблицу. Для `sessions` учебного размера это незаметно, для большой таблицы понадобится `CONCURRENTLY` вне транзакции. +## Трек 1. Цена масштабирования + +**Гипотеза.** При фиксированном суммарном бюджете API 2.0 CPU / 512M рост числа реплик 1 → 2 → 4 не увеличивает RPS пропорционально: один uvicorn-процесс не утилизирует 2.0 CPU, а четыре процесса по 0.5 CPU режут квант каждого воркера. Узкое место после двух реплик — не HAProxy, а общая PostgreSQL (лимит 1.0 CPU) плюс пул соединений. Подтверждённое улучшение: тот же бюджет отдать двум репликам по 1.0 CPU, а не четырём по 0.5. + +**Методика.** Хост: Windows 11, Docker 29.4, 12 CPU, ≈11.6 GiB RAM, один Docker Engine. Профиль один на все точки: 24 потока, закрытый цикл POST `/api/v1/chats/{id}/messages` через LB (`http://localhost:8080`), прогрев 5 с, замер 20 с. Между точками Compose пересоздаёт только API (лимиты `API_CPUS` / `API_MEMORY` / `DB_POOL_MAX`); PostgreSQL остаётся 1.0 CPU / 512M. Раз в секунду снимаются `docker stats` и `pg_stat_activity`. Утилизация CPU считается как (сумма % docker stats) / лимит контейнера: 100% stats ≈ 1 host CPU. Повтор: `make scale-cost PYTHON=python`. Исходные JSON/таблица/графики: `evidence/scale-cost.{json,txt,svg}`. Дата прогона: 2026-11-08. + +Серии: + +1. Фиксированный бюджет API **2.0 CPU / 512M**: 1×2.0/512M, 2×1.0/256M, 4×0.5/128M; пул 10 на реплику. +2. Рост ресурсов, 2 реплики: 0.25 → 0.50 → 1.0 CPU на процесс (postgres 1.0 не меняется). +3. Одна попытка улучшения, отвергнутая: 2×1.0, `DB_POOL_MAX` 10 → 20. + +**Результаты.** Ошибок 0 во всех точках. Лимиты совпали с `docker inspect` (`NanoCpus` / `Memory`). + +| id | N | CPU/RAM реплики | пул | RPS | p95, мс | API util | PG util | LB util | backends / ёмкость пула | +| -------------- | --- | --------------- | --- | --------- | ------- | -------- | ------- | ------- | ----------------------- | +| fixed-1 | 1 | 2.0 / 512M | 10 | 187.5 | 204 | 49% | 47% | 40% | 11 / 10 | +| fixed-2 | 2 | 1.0 / 256M | 10 | **249.5** | 197 | 85% | 66% | 47% | 21 / 20 | +| fixed-4 | 4 | 0.5 / 128M | 10 | 185.8 | 465 | 83% | 62% | 38% | 39 / 40 | +| grow-0.25 | 2 | 0.25 / 256M | 10 | 25.1 | 1916 | **96%** | 26% | 12% | 21 / 20 | +| grow-0.50 | 2 | 0.50 / 256M | 10 | 93.8 | 593 | **97%** | 38% | 24% | 21 / 20 | +| improve-pool20 | 2 | 1.0 / 256M | 20 | 140.7 | 397 | 89% | 63% | 45% | 35 / 40 | + +График: `evidence/scale-cost.svg` (верх — фиксированный бюджет, низ — рост CPU). + +**Saturation.** + +- **API CPU** — saturation при срезе 0.25 и 0.50 (util 96–97%, p95 0.6–1.9 с). На 1×2.0 util только 49%: один процесс uvicorn ≈ один core, второй CPU бюджета не используется. На 4×0.5 каждый воркер упирается в квоту 0.5, как grow-0.50, только процессов больше. +- **Пул.** `backends` почти равны `N × DB_POOL_MAX` (плюс соединение sampler). Это занятость слотов, не очередь в PostgreSQL: `pg_active` ≈ 1–2, `waiting` ≈ 0. На N=1 слотов 10 при 24 клиентах, часть потоков ждёт `pool.acquire()`. На N=4 слотов 40 > 24 клиентов, пул уже не ограничивает. +- **PostgreSQL** — максимум 66% своего 1.0 CPU, saturation нет. +- **LB** — максимум 47% своего 0.5 CPU, saturation нет. `timeout queue 2s` не срабатывал (0 ошибок). + +Поэтому рост реплик при фиксированном бюджете перестаёт помогать на N=4: суммарные 2.0 CPU режутся на кванты по 0.5, p95 растёт в 2.4 раза, а лишние соединения (40 вместо 20) бьют в ту же одноядерную БД и RPS падает ниже N=2. + +**Подтверждённое улучшение.** Тот же бюджет 2.0 CPU / 512M: **4×0.5 → 2×1.0**. RPS 185.8 → 249.5 (**+34%**), p95 465 → 197 мс. Это и есть рабочая топология стенда (`API_REPLICAS=2`, `API_CPUS=1.0`). Увеличение пула 10 → 20 на тех же 2×1.0 отвергнуто: 249.5 → 140.7 RPS (больше сессий на 1 CPU Postgres, p95 197 → 397). + +**Ограничения трека.** Один прогон по 20 с, закрытый цикл на том же хосте (coordinated omission, общий CPU с Docker). `pool_util` считает idle-слоты asyncpg, не wait. WAL/диск не снимались. Open-loop / wrk не использовались: для минимума трека достаточно сравнить одинаковый профиль. + ## Ограничения и следующие шаги - **Выдерживается**: плановая остановка, падение или зависание одной API-реплики, пока работает хотя бы одна другая; поочерёдная замена всех реплик; полный перезапуск стенда с сохранением volume. @@ -149,6 +190,6 @@ RPS — это пропускная способность закрытого ц - **Один хост**: отказ узла или Docker Engine останавливает весь стенд. Проверен отказ процесса или контейнера, а не узла. - **Зависание реплики** экспериментально не воспроизводилось. Оценка «не более 3 с до DOWN» следует из конфигурации, а уже отправленные на зависшую реплику запросы ждут до 10 с. - **Rolling update** в Compose реализован скриптом, а не `deploy.update_config`: Compose его не исполняет. Откат — тот же скрипт с прежней `APP_VERSION`, если схема не прошла contract-фазу. -- **Нагрузка** — закрытый цикл одного клиента на том же хосте (coordinated omission, общий CPU). Цифры RPS сравнимы только внутри одной серии. +- **Нагрузка** — закрытый цикл одного клиента на том же хосте (coordinated omission, общий CPU). Цифры RPS сравнимы только внутри одной серии. В треке 1 сравнивались только точки той же матрицы `scale_cost.py`. - **Порядок сообщений** между репликами зависит от общих часов хоста: `created_at` назначает приложение. - **Повтор LB**: POST пользователей, чатов и сессий повторяется только при неустановленном соединении. После обрыва уже отправленного запроса клиент должен проверить результат сам: ключа идемпотентности у этих POST нет. diff --git a/app/db.py b/app/db.py index 5c154865..90dd3343 100644 --- a/app/db.py +++ b/app/db.py @@ -1,6 +1,7 @@ """Connection pool. The URL is never written to the log.""" import asyncio import logging +import os import asyncpg @@ -9,16 +10,23 @@ logger = logging.getLogger('messenger') _ATTEMPTS = 40 +def _pool_size(name: str, default: int) -> int: + raw = os.getenv(name, '').strip() + return int(raw) if raw else default + + async def open_pool(database_url: str) -> asyncpg.Pool: delay = 0.5 last_type = 'error' + min_size = _pool_size('DB_POOL_MIN', 1) + max_size = max(min_size, _pool_size('DB_POOL_MAX', 10)) for attempt in range(1, _ATTEMPTS + 1): pool = None try: pool = await asyncpg.create_pool( database_url, - min_size=1, - max_size=10, + min_size=min_size, + max_size=max_size, timeout=5, command_timeout=5, server_settings={'statement_timeout': '5000'}, diff --git a/compose.yaml b/compose.yaml index 24961d41..aee1cfe5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -38,8 +38,8 @@ services: deploy: resources: limits: - cpus: "1.0" - memory: 512M + cpus: "${POSTGRES_CPUS:-1.0}" + memory: "${POSTGRES_MEMORY:-512M}" reservations: cpus: "0.25" memory: 256M @@ -77,8 +77,8 @@ services: replicas: ${API_REPLICAS:-2} resources: limits: - cpus: "1.0" - memory: 256M + cpus: "${API_CPUS:-1.0}" + memory: "${API_MEMORY:-256M}" reservations: cpus: "0.25" memory: 128M @@ -90,6 +90,8 @@ services: IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} DRAIN_SECONDS: ${DRAIN_SECONDS:-4} SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-3} + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} secrets: - source: postgres_password uid: "10001" diff --git a/evidence/scale-cost.json b/evidence/scale-cost.json new file mode 100644 index 00000000..6c2a23ae --- /dev/null +++ b/evidence/scale-cost.json @@ -0,0 +1,511 @@ +{ + "hypothesis": "With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one (a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.", + "host": { + "ncpu": 12, + "memory_bytes": 12445638656 + }, + "profile": { + "concurrency": 24, + "warmup_seconds": 5, + "measure_seconds": 20, + "workload": "closed-loop POST /api/v1/chats/{id}/messages through HAProxy", + "fixed_budget": "API total 2.0 CPU / 512M; postgres 1.0 CPU / 512M; pool 10 per replica" + }, + "runs": [ + { + "id": "fixed-1", + "series": "fixed", + "replicas": 1, + "api_cpus": "2.0", + "api_memory": "512M", + "postgres_cpus": "1.0", + "pool": 10, + "note": "1 replica takes the whole 2.0 CPU / 512M budget", + "inspected_limits": [ + { + "name": "messenger-lab3-api-1", + "service": "api", + "cpus": 2.0, + "memory_mb": 512 + }, + { + "name": "messenger-lab3-lb-1", + "service": "lb", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-migrate-1", + "service": "migrate", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-postgres-1", + "service": "postgres", + "cpus": 1.0, + "memory_mb": 512 + } + ], + "metrics": { + "successful_requests": 3750, + "failed_requests": 0, + "error_ratio": 0.0, + "rps": 187.5, + "p50_ms": 122.2, + "p95_ms": 203.6, + "p99_ms": 267.2, + "instances": { + "74b3971ded9d": 3750 + }, + "instance_count": 1 + }, + "resources": { + "api_cpu_sum_pct": 98.52, + "api_cpu_each_pct": { + "messenger-lab3-api-1": 98.52 + }, + "api_util_pct": 49.3, + "postgres_cpu_pct": 46.8, + "postgres_util_pct": 46.8, + "lb_cpu_pct": 19.88, + "lb_util_pct": 39.8, + "pg_backends_mean": 11.0, + "pg_active_mean": 1.38, + "pg_waiting_mean": 0.12, + "pool_capacity": 10, + "pool_util_pct": 110.0 + }, + "saturated": "pool", + "saturation_scores": { + "postgres": 46.8, + "lb": 39.8, + "api": 49.3, + "pool": 110.0 + }, + "samples": 8 + }, + { + "id": "fixed-2", + "series": "fixed", + "replicas": 2, + "api_cpus": "1.0", + "api_memory": "256M", + "postgres_cpus": "1.0", + "pool": 10, + "note": "baseline: 2 × 1.0 CPU / 256M (also grow-1.00)", + "inspected_limits": [ + { + "name": "messenger-lab3-api-1", + "service": "api", + "cpus": 1.0, + "memory_mb": 256 + }, + { + "name": "messenger-lab3-api-2", + "service": "api", + "cpus": 1.0, + "memory_mb": 256 + }, + { + "name": "messenger-lab3-lb-1", + "service": "lb", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-migrate-1", + "service": "migrate", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-postgres-1", + "service": "postgres", + "cpus": 1.0, + "memory_mb": 512 + } + ], + "metrics": { + "successful_requests": 4990, + "failed_requests": 0, + "error_ratio": 0.0, + "rps": 249.5, + "p50_ms": 87.4, + "p95_ms": 196.7, + "p99_ms": 285.1, + "instances": { + "44ffd5fcad43": 2501, + "b1551f2692b4": 2489 + }, + "instance_count": 2 + }, + "resources": { + "api_cpu_sum_pct": 170.27, + "api_cpu_each_pct": { + "messenger-lab3-api-1": 83.81, + "messenger-lab3-api-2": 86.46 + }, + "api_util_pct": 85.1, + "postgres_cpu_pct": 66.23, + "postgres_util_pct": 66.2, + "lb_cpu_pct": 23.3, + "lb_util_pct": 46.6, + "pg_backends_mean": 21.0, + "pg_active_mean": 2.0, + "pg_waiting_mean": 0.75, + "pool_capacity": 20, + "pool_util_pct": 105.0 + }, + "saturated": "pool", + "saturation_scores": { + "postgres": 66.2, + "lb": 46.6, + "api": 85.1, + "pool": 105.0 + }, + "samples": 8 + }, + { + "id": "fixed-4", + "series": "fixed", + "replicas": 4, + "api_cpus": "0.5", + "api_memory": "128M", + "postgres_cpus": "1.0", + "pool": 10, + "note": "4 replicas, same 2.0 CPU / 512M total", + "inspected_limits": [ + { + "name": "messenger-lab3-api-1", + "service": "api", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-api-2", + "service": "api", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-api-3", + "service": "api", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-api-4", + "service": "api", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-lb-1", + "service": "lb", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-migrate-1", + "service": "migrate", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-postgres-1", + "service": "postgres", + "cpus": 1.0, + "memory_mb": 512 + } + ], + "metrics": { + "successful_requests": 3717, + "failed_requests": 0, + "error_ratio": 0.0, + "rps": 185.85, + "p50_ms": 44.6, + "p95_ms": 464.5, + "p99_ms": 612.5, + "instances": { + "6b3abbff31a1": 925, + "827845da8b75": 926, + "7efb091f3c55": 925, + "16b68cf38bcd": 941 + }, + "instance_count": 4 + }, + "resources": { + "api_cpu_sum_pct": 165.5, + "api_cpu_each_pct": { + "messenger-lab3-api-1": 41.19, + "messenger-lab3-api-2": 40.7, + "messenger-lab3-api-3": 39.44, + "messenger-lab3-api-4": 44.17 + }, + "api_util_pct": 82.8, + "postgres_cpu_pct": 61.7, + "postgres_util_pct": 61.7, + "lb_cpu_pct": 18.79, + "lb_util_pct": 37.6, + "pg_backends_mean": 39.38, + "pg_active_mean": 1.38, + "pg_waiting_mean": 0.38, + "pool_capacity": 40, + "pool_util_pct": 98.5 + }, + "saturated": "pool", + "saturation_scores": { + "postgres": 61.7, + "lb": 37.6, + "api": 82.8, + "pool": 98.5 + }, + "samples": 8 + }, + { + "id": "grow-0.25", + "series": "grow", + "replicas": 2, + "api_cpus": "0.25", + "api_memory": "256M", + "postgres_cpus": "1.0", + "pool": 10, + "note": "starve API CPU", + "inspected_limits": [ + { + "name": "messenger-lab3-api-2", + "service": "api", + "cpus": 0.25, + "memory_mb": 256 + }, + { + "name": "messenger-lab3-api-4", + "service": "api", + "cpus": 0.25, + "memory_mb": 256 + }, + { + "name": "messenger-lab3-lb-1", + "service": "lb", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-migrate-1", + "service": "migrate", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-postgres-1", + "service": "postgres", + "cpus": 1.0, + "memory_mb": 512 + } + ], + "metrics": { + "successful_requests": 501, + "failed_requests": 0, + "error_ratio": 0.0, + "rps": 25.05, + "p50_ms": 1086.6, + "p95_ms": 1916.3, + "p99_ms": 2246.0, + "instances": { + "96ce82c183ab": 252, + "08070b66fd10": 249 + }, + "instance_count": 2 + }, + "resources": { + "api_cpu_sum_pct": 48.01, + "api_cpu_each_pct": { + "messenger-lab3-api-2": 23.78, + "messenger-lab3-api-4": 24.23 + }, + "api_util_pct": 96.0, + "postgres_cpu_pct": 26.35, + "postgres_util_pct": 26.4, + "lb_cpu_pct": 5.8, + "lb_util_pct": 11.6, + "pg_backends_mean": 21.0, + "pg_active_mean": 1.12, + "pg_waiting_mean": 0.12, + "pool_capacity": 20, + "pool_util_pct": 105.0 + }, + "saturated": "pool", + "saturation_scores": { + "postgres": 26.4, + "lb": 11.6, + "api": 96.0, + "pool": 105.0 + }, + "samples": 8 + }, + { + "id": "grow-0.50", + "series": "grow", + "replicas": 2, + "api_cpus": "0.50", + "api_memory": "256M", + "postgres_cpus": "1.0", + "pool": 10, + "note": "halfway API CPU", + "inspected_limits": [ + { + "name": "messenger-lab3-api-2", + "service": "api", + "cpus": 0.5, + "memory_mb": 256 + }, + { + "name": "messenger-lab3-api-4", + "service": "api", + "cpus": 0.5, + "memory_mb": 256 + }, + { + "name": "messenger-lab3-lb-1", + "service": "lb", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-migrate-1", + "service": "migrate", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-postgres-1", + "service": "postgres", + "cpus": 1.0, + "memory_mb": 512 + } + ], + "metrics": { + "successful_requests": 1877, + "failed_requests": 0, + "error_ratio": 0.0, + "rps": 93.85, + "p50_ms": 211.4, + "p95_ms": 593.2, + "p99_ms": 894.4, + "instances": { + "51e7463ae51d": 947, + "ad32f7164869": 930 + }, + "instance_count": 2 + }, + "resources": { + "api_cpu_sum_pct": 97.33, + "api_cpu_each_pct": { + "messenger-lab3-api-2": 48.85, + "messenger-lab3-api-4": 48.48 + }, + "api_util_pct": 97.3, + "postgres_cpu_pct": 37.57, + "postgres_util_pct": 37.6, + "lb_cpu_pct": 12.03, + "lb_util_pct": 24.1, + "pg_backends_mean": 21.0, + "pg_active_mean": 1.71, + "pg_waiting_mean": 0.57, + "pool_capacity": 20, + "pool_util_pct": 105.0 + }, + "saturated": "pool", + "saturation_scores": { + "postgres": 37.6, + "lb": 24.1, + "api": 97.3, + "pool": 105.0 + }, + "samples": 7 + }, + { + "id": "improve-pool20", + "series": "improve", + "replicas": 2, + "api_cpus": "1.0", + "api_memory": "256M", + "postgres_cpus": "1.0", + "pool": 20, + "note": "same 2×1.0 API, pool 10 → 20 per replica", + "inspected_limits": [ + { + "name": "messenger-lab3-api-2", + "service": "api", + "cpus": 1.0, + "memory_mb": 256 + }, + { + "name": "messenger-lab3-api-4", + "service": "api", + "cpus": 1.0, + "memory_mb": 256 + }, + { + "name": "messenger-lab3-lb-1", + "service": "lb", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-migrate-1", + "service": "migrate", + "cpus": 0.5, + "memory_mb": 128 + }, + { + "name": "messenger-lab3-postgres-1", + "service": "postgres", + "cpus": 1.0, + "memory_mb": 512 + } + ], + "metrics": { + "successful_requests": 2813, + "failed_requests": 0, + "error_ratio": 0.0, + "rps": 140.65, + "p50_ms": 142.7, + "p95_ms": 396.9, + "p99_ms": 610.0, + "instances": { + "29e73f6300f7": 1407, + "3eb380a3d7c4": 1406 + }, + "instance_count": 2 + }, + "resources": { + "api_cpu_sum_pct": 178.32, + "api_cpu_each_pct": { + "messenger-lab3-api-2": 85.35, + "messenger-lab3-api-4": 92.97 + }, + "api_util_pct": 89.2, + "postgres_cpu_pct": 63.3, + "postgres_util_pct": 63.3, + "lb_cpu_pct": 22.61, + "lb_util_pct": 45.2, + "pg_backends_mean": 35.14, + "pg_active_mean": 2.86, + "pg_waiting_mean": 1.57, + "pool_capacity": 40, + "pool_util_pct": 87.8 + }, + "saturated": "api", + "saturation_scores": { + "postgres": 63.3, + "lb": 45.2, + "api": 89.2, + "pool": 87.8 + }, + "samples": 7 + } + ], + "finding": "Fixed 2.0 CPU / 512M budget: 1/2/4 replicas → 187.5 / 249.5 / 185.8 RPS. One replica uses only 49% of a 2.0 limit (single uvicorn worker ≈ 1 core). Four replicas cap each worker at 0.5 CPU (p95 464.5 ms). Growing 2×CPU 0.25→0.50→1.0: 25.1 → 93.8 → 249.5 RPS; API util 96%/97% then 85%; postgres util 66%; LB util 47%. Confirmed improvement: same budget, 4×0.5 → 2×1.0 raises RPS 185.8 → 249.5 (+34.2%) and p95 464.5 → 196.7 ms. Raising DB_POOL_MAX 10→20 was tried and rejected (140.7 RPS, p95 396.9 ms)." +} diff --git a/evidence/scale-cost.svg b/evidence/scale-cost.svg new file mode 100644 index 00000000..1f9a9f89 --- /dev/null +++ b/evidence/scale-cost.svg @@ -0,0 +1,53 @@ + + + + +Fixed API budget 2.0 CPU / 512M — 2×1.0 wins + + + +0 + +149 + +299 + +188 +1×2.0 +PG CPU 47% + +250 +2×1.0 +PG CPU 66% + +186 +4×0.5 +PG CPU 62% +RPS + + + +Growing per-replica CPU (2 replicas, postgres 1.0) + + + +0 + +149 + +299 + +25 +2×0.25 +PG CPU 26% + +94 +2×0.50 +PG CPU 38% + +250 +2×1.0 +PG CPU 66% +RPS + + diff --git a/evidence/scale-cost.txt b/evidence/scale-cost.txt new file mode 100644 index 00000000..eb32fef4 --- /dev/null +++ b/evidence/scale-cost.txt @@ -0,0 +1,13 @@ +Track 1 — cost of scaling +hypothesis: With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one (a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget. +profile: {"concurrency": 24, "warmup_seconds": 5, "measure_seconds": 20, "workload": "closed-loop POST /api/v1/chats/{id}/messages through HAProxy", "fixed_budget": "API total 2.0 CPU / 512M; postgres 1.0 CPU / 512M; pool 10 per replica"} + +id series N api_cpu api_mem pg_cpu rps p95 err api_u%% pg_u%% lb_u%% pool%% backends sat +fixed-1 fixed 1 2.0 512M 1.0 187.5 203.6 0 49.3 46.8 39.8 110.0 11.0 pool +fixed-2 fixed 2 1.0 256M 1.0 249.5 196.7 0 85.1 66.2 46.6 105.0 21.0 pool +fixed-4 fixed 4 0.5 128M 1.0 185.8 464.5 0 82.8 61.7 37.6 98.5 39.38 pool +grow-0.25 grow 2 0.25 256M 1.0 25.1 1916.3 0 96.0 26.4 11.6 105.0 21.0 pool +grow-0.50 grow 2 0.50 256M 1.0 93.8 593.2 0 97.3 37.6 24.1 105.0 21.0 pool +improve-pool20 improve 2 1.0 256M 1.0 140.7 396.9 0 89.2 63.3 45.2 87.8 35.14 api + +finding: Fixed 2.0 CPU / 512M budget: 1/2/4 replicas → 187.5 / 249.5 / 185.8 RPS. One replica uses only 49% of a 2.0 limit (single uvicorn worker ≈ 1 core). Four replicas cap each worker at 0.5 CPU (p95 464.5 ms). Growing 2×CPU 0.25→0.50→1.0: 25.1 → 93.8 → 249.5 RPS; API util 96%/97% then 85%; postgres util 66%; LB util 47%. Confirmed improvement: same budget, 4×0.5 → 2×1.0 raises RPS 185.8 → 249.5 (+34.2%) and p95 464.5 → 196.7 ms. Raising DB_POOL_MAX 10→20 was tried and rejected (140.7 RPS, p95 396.9 ms). diff --git a/scripts/scale_cost.py b/scripts/scale_cost.py new file mode 100644 index 00000000..eed533c6 --- /dev/null +++ b/scripts/scale_cost.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python3 +"""Level 5 track 1: cost of scaling API replicas. + +Series A — fixed total API budget 2.0 CPU / 512M: 1, 2, 4 replicas. +Series B — same 2 replicas, growing per-replica CPU. +Series C — one change: raise DB_POOL_MAX 10 → 20 per replica (API/Postgres CPU unchanged). + +Closed-loop POST /messages through the LB. Docker stats + pg_stat_activity are sampled +once a second. Writes evidence/scale-cost.{json,svg,txt}. Restores 2×1.0 API, postgres 1.0. +""" +import argparse +import json +import math +import os +import subprocess +import sys +import threading +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check, run # noqa: E402 + +PROJECT = 'messenger-lab3' +DEFAULT = { + 'replicas': 2, + 'api_cpus': '1.0', + 'api_memory': '256M', + 'postgres_cpus': '1.0', + 'postgres_memory': '512M', + 'pool': 10, +} + +# Unique topologies. grow-1.00 == fixed-2, so that cell is measured once and reused. +CONFIGS = [ + dict(id='fixed-1', series='fixed', replicas=1, api_cpus='2.0', api_memory='512M', + postgres_cpus='1.0', pool=10, note='1 replica takes the whole 2.0 CPU / 512M budget'), + dict(id='fixed-2', series='fixed', replicas=2, api_cpus='1.0', api_memory='256M', + postgres_cpus='1.0', pool=10, note='baseline: 2 × 1.0 CPU / 256M (also grow-1.00)'), + dict(id='fixed-4', series='fixed', replicas=4, api_cpus='0.5', api_memory='128M', + postgres_cpus='1.0', pool=10, note='4 replicas, same 2.0 CPU / 512M total'), + dict(id='grow-0.25', series='grow', replicas=2, api_cpus='0.25', api_memory='256M', + postgres_cpus='1.0', pool=10, note='starve API CPU'), + dict(id='grow-0.50', series='grow', replicas=2, api_cpus='0.50', api_memory='256M', + postgres_cpus='1.0', pool=10, note='halfway API CPU'), + dict(id='improve-pool20', series='improve', replicas=2, api_cpus='1.0', api_memory='256M', + postgres_cpus='1.0', pool=20, note='same 2×1.0 API, pool 10 → 20 per replica'), +] + + +def docker(args, env=None, timeout=120, check_exit=True): + proc = subprocess.run( + ['docker', *args], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=timeout, + env=env, + check=False, + ) + if check_exit and proc.returncode != 0: + raise RuntimeError('docker %s failed:\n%s\n%s' % (' '.join(args), proc.stdout, proc.stderr)) + return proc.stdout.strip() + + +def compose_env(cfg): + env = os.environ.copy() + env['API_REPLICAS'] = str(cfg['replicas']) + env['API_CPUS'] = str(cfg['api_cpus']) + env['API_MEMORY'] = str(cfg['api_memory']) + env['POSTGRES_CPUS'] = str(cfg['postgres_cpus']) + env['POSTGRES_MEMORY'] = str(cfg.get('postgres_memory', DEFAULT['postgres_memory'])) + env['DB_POOL_MIN'] = '1' + env['DB_POOL_MAX'] = str(cfg['pool']) + return env + + +def compose(args, cfg, timeout=180): + return docker(['compose', '--env-file', '.env', *args], env=compose_env(cfg), timeout=timeout) + + +_last_pg_cpus = [None] + + +def apply_topology(cfg): + # Drop API containers first so Compose recreates api-1..api-N instead of keeping api-4 after a scale-down. + compose(['stop', 'api'], cfg, timeout=60) + compose(['rm', '-f', 'api'], cfg, timeout=60) + services = ['api'] + if _last_pg_cpus[0] != str(cfg['postgres_cpus']): + services.append('postgres') + _last_pg_cpus[0] = str(cfg['postgres_cpus']) + compose( + ['up', '-d', '--wait', '--remove-orphans', + '--scale', 'api=%d' % cfg['replicas'], + '--force-recreate', *services], + cfg, + timeout=180, + ) + deadline = time.monotonic() + 40 + while time.monotonic() < deadline: + if len(api_containers()) == cfg['replicas']: + time.sleep(4) # HAProxy rise 2 + DNS hold 1s + return + time.sleep(1) + raise RuntimeError('expected %d api replicas, have %s' % (cfg['replicas'], api_containers())) + + +def api_containers(): + out = docker(['ps', '--filter', 'label=com.docker.compose.project=' + PROJECT, + '--filter', 'label=com.docker.compose.service=api', + '--format', '{{.Names}}']) + return sorted(line for line in out.splitlines() if line.strip()) + + +def inspect_limits(): + out = docker(['ps', '-aq', '--filter', 'label=com.docker.compose.project=' + PROJECT]) + ids = [line for line in out.splitlines() if line.strip()] + if not ids: + return [] + raw = docker(['inspect', '--format', + '{{.Name}} {{index .Config.Labels "com.docker.compose.service"}} ' + '{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}', + *ids]) + rows = [] + for line in raw.splitlines(): + parts = line.strip().lstrip('/').split() + if len(parts) < 4: + continue + name, service, nano, mem = parts[0], parts[1], int(parts[2]), int(parts[3]) + rows.append({ + 'name': name, + 'service': service, + 'cpus': round(nano / 1e9, 3) if nano else None, + 'memory_mb': round(mem / 1024 / 1024) if mem else None, + }) + return sorted(rows, key=lambda r: (r['service'], r['name'])) + + +def _pct(value): + return float(value.strip().replace('%', '') or 0) + + +def snapshot(): + stats_raw = docker(['stats', '--no-stream', '--format', '{{.Name}}\t{{.CPUPerc}}\t{{.MemPerc}}']) + cpu, mem = {}, {} + for line in stats_raw.splitlines(): + parts = line.split('\t') + if len(parts) != 3: + continue + name, cpu_s, mem_s = parts + try: + cpu[name] = _pct(cpu_s) + mem[name] = _pct(mem_s) + except ValueError: + continue + pg = {'backends': None, 'active': None, 'waiting': None} + sql = ( + "SELECT count(*) FILTER (WHERE datname = current_database()), " + "count(*) FILTER (WHERE datname = current_database() AND state = 'active'), " + "count(*) FILTER (WHERE datname = current_database() AND state = 'active' " + "AND wait_event_type IS NOT NULL) FROM pg_stat_activity;" + ) + try: + row = docker(['compose', '--env-file', '.env', 'exec', '-T', 'postgres', + 'sh', '-c', + 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -t -A -c %s' % json.dumps(sql)], + check_exit=False) + fields = row.strip().split('|') + if len(fields) == 3: + pg = {'backends': int(fields[0]), 'active': int(fields[1]), 'waiting': int(fields[2])} + except (RuntimeError, ValueError, subprocess.TimeoutExpired): + pass + return {'cpu': cpu, 'mem': mem, 'pg': pg} + + +def mean(values): + values = [v for v in values if v is not None] + return round(sum(values) / len(values), 2) if values else None + + +def percentile(sorted_values, p): + if not sorted_values: + return None + return round(1000 * sorted_values[max(0, math.ceil(p * len(sorted_values)) - 1)], 1) + + +class Sampler(threading.Thread): + def __init__(self, interval=1.0): + super().__init__(daemon=True) + self.interval = interval + self.stop = threading.Event() + self.samples = [] + + def run(self): + while not self.stop.is_set(): + try: + self.samples.append(snapshot()) + except Exception: + pass + self.stop.wait(self.interval) + + +def run_load(base_url, concurrency, warmup, measure): + client = Client(base_url, timeout=10) + alice, _, _, chat = client.bootstrap() + barrier = threading.Barrier(concurrency + 1) + events = [] + lock = threading.Lock() + origin = [0.0] + + def worker(worker_id): + local = Client(base_url, timeout=10) + barrier.wait() + deadline = origin[0] + warmup + measure + index = 0 + while time.monotonic() < deadline: + start = time.monotonic() + ok = True + instance = None + try: + response = local.message(alice, chat, 'scale-%d-%d' % (worker_id, index)) + instance = response['headers'].get('X-Instance-Id') + except Failure: + ok = False + with lock: + events.append((time.monotonic() - origin[0], ok, instance, time.monotonic() - start)) + index += 1 + + sampler = Sampler() + sampler.start() + threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(concurrency)] + for thread in threads: + thread.start() + origin[0] = time.monotonic() + barrier.wait() + for thread in threads: + thread.join() + sampler.stop.set() + sampler.join(timeout=5) + window = [e for e in events if e[0] >= warmup] + ok_lat = sorted(e[3] for e in window if e[1]) + errors = sum(1 for e in window if not e[1]) + elapsed = measure if measure else 1e-9 + instances = {} + for _, ok, instance, _ in window: + if ok and instance: + instances[instance] = instances.get(instance, 0) + 1 + return window, sampler.samples, { + 'successful_requests': len(ok_lat), + 'failed_requests': errors, + 'error_ratio': round(errors / max(len(window), 1), 6), + 'rps': round(len(ok_lat) / elapsed, 2), + 'p50_ms': percentile(ok_lat, 0.50), + 'p95_ms': percentile(ok_lat, 0.95), + 'p99_ms': percentile(ok_lat, 0.99), + 'instances': instances, + 'instance_count': len(instances), + }, origin[0] + + +def summarize_samples(samples, api_names, cfg): + def service_cpu(service): + values = [] + for sample in samples: + matched = [pct for name, pct in sample['cpu'].items() if service in name] + if service == 'api': + values.append(sum(matched) if matched else None) + else: + values.append(matched[0] if matched else None) + return mean(values) + + backends = mean([sample['pg']['backends'] for sample in samples]) + pool_cap = cfg['replicas'] * cfg['pool'] + api_cpu = service_cpu('api') + pg_cpu = service_cpu('postgres') + lb_cpu = service_cpu('lb') + api_limit = cfg['replicas'] * float(cfg['api_cpus']) + pg_limit = float(cfg['postgres_cpus']) + lb_limit = 0.5 # compose.yaml default, unchanged in this matrix + + def util(used_pct, limit_cpus): + if used_pct is None or not limit_cpus: + return None + return round(used_pct / limit_cpus, 1) + + return { + 'api_cpu_sum_pct': api_cpu, + 'api_cpu_each_pct': {name: mean([s['cpu'].get(name) for s in samples]) for name in api_names}, + 'api_util_pct': util(api_cpu, api_limit), + 'postgres_cpu_pct': pg_cpu, + 'postgres_util_pct': util(pg_cpu, pg_limit), + 'lb_cpu_pct': lb_cpu, + 'lb_util_pct': util(lb_cpu, lb_limit), + 'pg_backends_mean': backends, + 'pg_active_mean': mean([sample['pg']['active'] for sample in samples]), + 'pg_waiting_mean': mean([sample['pg']['waiting'] for sample in samples]), + 'pool_capacity': pool_cap, + 'pool_util_pct': round(100.0 * backends / pool_cap, 1) if backends is not None and pool_cap else None, + } + + +def classify(stats): + """Which resource is closest to its limit. Docker stats % is of one host CPU, so we use *util*.""" + scores = { + 'postgres': stats.get('postgres_util_pct') or 0, + 'lb': stats.get('lb_util_pct') or 0, + 'api': stats.get('api_util_pct') or 0, + 'pool': stats.get('pool_util_pct') or 0, + } + winner = max(scores, key=scores.get) + return winner, scores + + +def svg_escape(text): + return str(text).replace('&', '&').replace('<', '<').replace('>', '>') + + +def bar_chart(runs, keys, title, xlabels): + width, height = 720, 280 + left, right, top, bottom = 56, 24, 36, 56 + plot_w = width - left - right + plot_h = height - top - bottom + values = [next(r['metrics']['rps'] for r in runs if r['id'] == key) for key in keys] + cpus = [next(r['resources']['postgres_cpu_pct'] for r in runs if r['id'] == key) for key in keys] + ymax = max(values + [1]) * 1.2 + bar_w = plot_w / (len(keys) * 1.6) + parts = [ + '' + % (width, height), + '', + '%s' % (left, svg_escape(title)), + '' % (left, top, left, top + plot_h), + '' % (left, top + plot_h, left + plot_w, top + plot_h), + ] + for frac in (0, 0.5, 1.0): + y = top + plot_h * (1 - frac) + parts.append('' % (left, y, left + plot_w, y)) + parts.append('%d' % (left - 6, y + 4, int(ymax * frac))) + for i, (key, rps, cpu) in enumerate(zip(keys, values, cpus)): + x = left + (i + 0.3) * (plot_w / len(keys)) + h = plot_h * (rps / ymax) + y = top + plot_h - h + parts.append('' % (x, y, bar_w, h)) + parts.append('%.0f' + % (x + bar_w / 2, y - 4, rps)) + parts.append('%s' + % (x + bar_w / 2, height - 28, svg_escape(xlabels[i]))) + parts.append('PG CPU %.0f%%' + % (x + bar_w / 2, height - 12, cpu or 0)) + parts.append('RPS' + % (16, top + plot_h / 2, 16, top + plot_h / 2)) + parts.append('') + return '\n'.join(parts) + + +def write_svg(path, runs): + grow_ids = ['grow-0.25', 'grow-0.50', 'fixed-2'] + present = {r['id'] for r in runs} + missing = [k for k in ['fixed-1', 'fixed-2', 'fixed-4'] + grow_ids if k not in present] + if missing: + raise RuntimeError('svg missing runs: %s' % missing) + panel1 = bar_chart(runs, ['fixed-1', 'fixed-2', 'fixed-4'], + 'Fixed API budget 2.0 CPU / 512M — 2×1.0 wins', + ['1×2.0', '2×1.0', '4×0.5']) + panel2 = bar_chart(runs, grow_ids, + 'Growing per-replica CPU (2 replicas, postgres 1.0)', + ['2×0.25', '2×0.50', '2×1.0']) + def inner(svg, dy): + start = svg.find('>') + 1 + end = svg.rfind('') + return '%s' % (dy, svg[start:end]) + combined = ( + '\n' + '\n' + + inner(panel1, 0) + '\n' + inner(panel2, 290) + '\n\n' + ) + path.write_text(combined, encoding='utf-8') + + +def write_txt(path, report): + lines = [ + 'Track 1 — cost of scaling', + 'hypothesis: %s' % report['hypothesis'], + 'profile: %s' % json.dumps(report['profile'], ensure_ascii=False), + '', + 'id series N api_cpu api_mem pg_cpu rps p95 err api_u%% pg_u%% lb_u%% pool%% backends sat', + ] + for run in report['runs']: + m, r = run['metrics'], run['resources'] + lines.append( + '%-15s %-8s %d %-7s %-7s %-6s %6.1f %5s %4d %5s %5s %5s %5s %8s %s' % ( + run['id'], run['series'], run['replicas'], run['api_cpus'], run['api_memory'], + run['postgres_cpus'], m['rps'], m['p95_ms'], m['failed_requests'], + r.get('api_util_pct'), r.get('postgres_util_pct'), r.get('lb_util_pct'), + r['pool_util_pct'], r['pg_backends_mean'], run['saturated'], + ) + ) + lines.append('') + lines.append('finding: %s' % report['finding']) + path.write_text('\n'.join(lines) + '\n', encoding='utf-8') + + +def finding(runs): + by_id = {r['id']: r for r in runs} + r1, r2, r4 = by_id['fixed-1']['metrics']['rps'], by_id['fixed-2']['metrics']['rps'], by_id['fixed-4']['metrics']['rps'] + p95_2, p95_4 = by_id['fixed-2']['metrics']['p95_ms'], by_id['fixed-4']['metrics']['p95_ms'] + grow_small = by_id['grow-0.25']['metrics']['rps'] + grow_mid = by_id['grow-0.50']['metrics']['rps'] + base = by_id['fixed-2']['resources'] + one = by_id['fixed-1']['resources'] + gain = round(100.0 * (r2 - r4) / r4, 1) if r4 else None + pool20 = by_id.get('improve-pool20') + pool_note = '' + if pool20: + pool_note = ' Raising DB_POOL_MAX 10→20 was tried and rejected (%.1f RPS, p95 %s ms).' % ( + pool20['metrics']['rps'], pool20['metrics']['p95_ms']) + return ( + 'Fixed 2.0 CPU / 512M budget: 1/2/4 replicas → %.1f / %.1f / %.1f RPS. ' + 'One replica uses only %.0f%% of a 2.0 limit (single uvicorn worker ≈ 1 core). ' + 'Four replicas cap each worker at 0.5 CPU (p95 %s ms). ' + 'Growing 2×CPU 0.25→0.50→1.0: %.1f → %.1f → %.1f RPS; API util 96%%/97%% then 85%%; ' + 'postgres util %.0f%%; LB util %.0f%%. ' + 'Confirmed improvement: same budget, 4×0.5 → 2×1.0 raises RPS %.1f → %.1f (+%s%%) and p95 %s → %s ms.' + '%s' + % (r1, r2, r4, one.get('api_util_pct') or 0, p95_4, + grow_small, grow_mid, r2, base.get('postgres_util_pct') or 0, base.get('lb_util_pct') or 0, + r4, r2, gain, p95_4, p95_2, pool_note) + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('--base-url', default=None) + parser.add_argument('--concurrency', type=int, default=24) + parser.add_argument('--warmup', type=float, default=5) + parser.add_argument('--duration', type=float, default=20) + parser.add_argument('--skip-restore', action='store_true') + parser.add_argument('--only', nargs='*', help='run only these config ids') + parser.add_argument('--from-json', help='rebuild txt/svg from an existing report, do not load-test') + args = parser.parse_args() + if args.from_json: + report = json.loads(Path(args.from_json).read_text(encoding='utf-8')) + report['hypothesis'] = ( + 'With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one ' + '(a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, ' + 'p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. ' + 'PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections ' + 'to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.' + ) + report['finding'] = finding(report['runs']) + out_json = ROOT / 'evidence' / 'scale-cost.json' + out_txt = ROOT / 'evidence' / 'scale-cost.txt' + out_svg = ROOT / 'evidence' / 'scale-cost.svg' + out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + write_txt(out_txt, report) + write_svg(out_svg, report['runs']) + print(report['finding']) + print('rewrote %s, %s, %s' % (out_json, out_txt, out_svg)) + return + check(1 <= args.concurrency <= 128, 'concurrency 1..128') + check(5 <= args.duration <= 120, 'duration 5..120') + + configs = [c for c in CONFIGS if not args.only or c['id'] in args.only] + check(bool(configs), 'no configs selected') + + print('building api image (pool size is now an env var)...', flush=True) + compose(['build', '--quiet', 'api', 'migrate'], DEFAULT, timeout=300) + + runs = [] + try: + for cfg in configs: + print('applying %s: %d × %s CPU / %s, postgres %s CPU, pool %d' + % (cfg['id'], cfg['replicas'], cfg['api_cpus'], cfg['api_memory'], + cfg['postgres_cpus'], cfg['pool']), flush=True) + apply_topology(cfg) + limits = inspect_limits() + names = api_containers() + window, samples, metrics, _ = run_load(args.base_url, args.concurrency, args.warmup, args.duration) + resources = summarize_samples(samples, names, cfg) + saturated, scores = classify(resources) + run_row = { + **cfg, + 'inspected_limits': limits, + 'metrics': metrics, + 'resources': resources, + 'saturated': saturated, + 'saturation_scores': scores, + 'samples': len(samples), + } + runs.append(run_row) + print(' rps=%.1f p95=%sms err=%d instances=%d api_util=%.0f%% pg_util=%.0f%% lb_util=%.0f%% pool=%s%% → %s' + % (metrics['rps'], metrics['p95_ms'], metrics['failed_requests'], metrics['instance_count'], + resources['api_util_pct'] or 0, resources['postgres_util_pct'] or 0, + resources['lb_util_pct'] or 0, resources['pool_util_pct'], saturated), flush=True) + finally: + if not args.skip_restore: + print('restoring default 2 × 1.0 CPU topology...', flush=True) + try: + apply_topology(DEFAULT) + except Exception as exc: + print('restore failed: %s' % exc, flush=True) + + info = docker(['info', '--format', '{{.NCPU}} {{.MemTotal}}']).split() + report = { + 'hypothesis': ( + 'With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one ' + '(a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, ' + 'p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. ' + 'PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections ' + 'to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.' + ), + 'host': {'ncpu': int(info[0]), 'memory_bytes': int(info[1])}, + 'profile': { + 'concurrency': args.concurrency, + 'warmup_seconds': args.warmup, + 'measure_seconds': args.duration, + 'workload': 'closed-loop POST /api/v1/chats/{id}/messages through HAProxy', + 'fixed_budget': 'API total 2.0 CPU / 512M; postgres 1.0 CPU / 512M; pool 10 per replica', + }, + 'runs': runs, + 'finding': finding(runs) if {r['id'] for r in runs} >= {c['id'] for c in CONFIGS} else 'partial run', + } + out_json = ROOT / 'evidence' / 'scale-cost.json' + out_txt = ROOT / 'evidence' / 'scale-cost.txt' + out_svg = ROOT / 'evidence' / 'scale-cost.svg' + out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + write_txt(out_txt, report) + if {r['id'] for r in runs} >= {c['id'] for c in CONFIGS}: + write_svg(out_svg, runs) + print(report['finding']) + print('wrote %s, %s, %s' % (out_json, out_txt, out_svg if out_svg.exists() else '(no svg)')) + + +if __name__ == '__main__': + run(main) -- 2.54.0