From cb1101186939d64f02208621aa9bd30a71ddbc57 Mon Sep 17 00:00:00 2001 From: Queenfish Date: Fri, 16 Oct 2026 01:07:34 +0300 Subject: [PATCH 1/3] =?UTF-8?q?=D0=91=D0=B0=D0=BB=D0=B0=D0=BD=D1=81=D0=B8?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D0=BA=D0=B0,=20=D1=80=D0=B5=D0=BF=D0=BB?= =?UTF-8?q?=D0=B8=D0=BA=D0=B8,=20=D0=B8=D0=B4=D0=B5=D0=BC=D0=BF=D0=BE?= =?UTF-8?q?=D1=82=D0=B5=D0=BD=D1=82=D0=BD=D0=BE=D1=81=D1=82=D1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 1 + .env.example | 5 +- Dockerfile | 32 +++ REPORT.md | 254 +++++++++++++++++-- app/__init__.py | 0 app/api/__init__.py | 0 app/api/auth.py | 104 ++++++++ app/api/chats.py | 179 ++++++++++++++ app/api/health.py | 53 ++++ app/api/users.py | 58 +++++ app/cache.py | 60 +++++ app/config.py | 21 ++ app/db.py | 30 +++ app/main.py | 159 ++++++++++++ app/migrate.py | 19 ++ app/migrations.py | 31 +++ app/models.py | 70 ++++++ app/security.py | 83 +++++++ app/session_store.py | 108 ++++++++ app/storage.py | 342 ++++++++++++++++++++++++++ compose-scale.yaml | 7 + compose.yaml | 134 ++++++++++ evidence/cache_experiment.json | 41 +++ evidence/cache_experiment_before.json | 15 ++ evidence/cache_summary.md | 48 ++++ evidence/detection.json | 41 +++ evidence/explain.txt | 58 +++++ evidence/idempotency.json | 22 ++ evidence/rolling-update.json | 6 + evidence/scaling.json | 92 +++++++ migrations/001_users.sql | 7 + migrations/002_chats.sql | 5 + migrations/003_chat_members.sql | 5 + migrations/004_messages.sql | 7 + migrations/005_sessions.sql | 6 + migrations/006_messages_index.sql | 2 + migrations/007_idempotency.sql | 11 + migrations/008_idempotency_index.sql | 2 + nginx.conf | 29 +++ requirements.txt | 9 + scripts/check-balance.py | 38 +++ scripts/measure-detection.py | 201 +++++++++++++++ scripts/restart.cmd | 2 + scripts/restart.py | 50 ++++ scripts/restart.sh | 22 ++ scripts/rolling-update.py | 111 +++++++++ scripts/scaling-benchmark.py | 194 +++++++++++++++ scripts/stop-one.cmd | 2 + scripts/stop-one.py | 34 +++ scripts/stop-one.sh | 15 ++ scripts/test-idempotency.py | 105 ++++++++ student-tests/__init__.py | 0 student-tests/benchmark.py | 117 +++++++++ student-tests/cache_experiment.py | 176 +++++++++++++ student-tests/explain.py | 90 +++++++ student-tests/test_cache.py | 129 ++++++++++ student-tests/test_race.py | 84 +++++++ student-tests/test_resilience.py | 153 ++++++++++++ student-tests/test_validation.py | 264 ++++++++++++++++++++ 59 files changed, 3916 insertions(+), 27 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/api/__init__.py create mode 100644 app/api/auth.py create mode 100644 app/api/chats.py create mode 100644 app/api/health.py create mode 100644 app/api/users.py create mode 100644 app/cache.py create mode 100644 app/config.py create mode 100644 app/db.py create mode 100644 app/main.py create mode 100644 app/migrate.py create mode 100644 app/migrations.py create mode 100644 app/models.py create mode 100644 app/security.py create mode 100644 app/session_store.py create mode 100644 app/storage.py create mode 100644 compose-scale.yaml create mode 100644 compose.yaml create mode 100644 evidence/cache_experiment.json create mode 100644 evidence/cache_experiment_before.json create mode 100644 evidence/cache_summary.md create mode 100644 evidence/detection.json create mode 100644 evidence/explain.txt create mode 100644 evidence/idempotency.json create mode 100644 evidence/rolling-update.json create mode 100644 evidence/scaling.json create mode 100644 migrations/001_users.sql create mode 100644 migrations/002_chats.sql create mode 100644 migrations/003_chat_members.sql create mode 100644 migrations/004_messages.sql create mode 100644 migrations/005_sessions.sql create mode 100644 migrations/006_messages_index.sql create mode 100644 migrations/007_idempotency.sql create mode 100644 migrations/008_idempotency_index.sql create mode 100644 nginx.conf create mode 100644 requirements.txt create mode 100644 scripts/check-balance.py create mode 100644 scripts/measure-detection.py create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.py create mode 100755 scripts/restart.sh create mode 100644 scripts/rolling-update.py create mode 100644 scripts/scaling-benchmark.py create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/stop-one.py create mode 100755 scripts/stop-one.sh create mode 100644 scripts/test-idempotency.py create mode 100644 student-tests/__init__.py create mode 100644 student-tests/benchmark.py create mode 100644 student-tests/cache_experiment.py create mode 100644 student-tests/explain.py create mode 100644 student-tests/test_cache.py create mode 100644 student-tests/test_race.py create mode 100644 student-tests/test_resilience.py create mode 100644 student-tests/test_validation.py diff --git a/.dockerignore b/.dockerignore index bc16e556..447644cb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,4 @@ __pycache__ node_modules evidence .DS_Store +nginx.conf diff --git a/.env.example b/.env.example index d32b9be0..ba7725be 100644 --- a/.env.example +++ b/.env.example @@ -7,7 +7,6 @@ SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 +DATABASE_URL=postgresql://messenger:messenger@postgres:5432/messenger \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..9f10bccc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.12-slim AS builder + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir --user -r requirements.txt + +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH=/home/appuser/.local/bin:$PATH + +RUN useradd -m -u 1000 appuser + +WORKDIR /app + +COPY --from=builder /root/.local /home/appuser/.local + +COPY ./app ./app +COPY ./migrations ./migrations + +RUN chown -R appuser:appuser /app + +USER appuser + +EXPOSE 8080 + +CMD ["sh", "-c", "export INSTANCE_ID=$(python -c 'import secrets; print(secrets.token_hex(6))') && echo \"[startup] INSTANCE_ID=$INSTANCE_ID\" && exec python -m uvicorn app.main:app --host 0.0.0.0 --port 8080"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index c2679005..5814508a 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,243 @@ -# Отчёт — лабораторная 3 +Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO + Автор, группа: Кузнецова Марина, БВТ2304 + Целевая оценка; выбранный трек 5. Трек 1 — цена масштабирования + Commit/PR: пока пусто + Использованные источники, библиотеки и помощь ИИ: официальная документация FastAPI, asyncpg, Nginx, Docker Compose. ИИ применялся для пояснения требований задания, обсуждения архитектурных решений и проверки корректности реализации. -## Запуск из чистого clone +Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования к окружению: -## Архитектура и решения + Python 3.13.15 + Docker 26.1.1 + git -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Подготовка и запуск через Docker Compose: -## Доказательства критериев +cp .env.example .env +docker compose --env-file .env up -d --build --scale api=2 -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Ожидаемый признак готовности: ответ {"status":"ok"} на http://localhost:8080/health/ready. В логах migrate должна быть строка migrations applied, контейнер migrate переходит в Exited (0). Реплики api-1 и api-2 становятся healthy. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Проверка публичным smoke-тестом (в отдельном терминале, при запущенном стенде): -## Эксперименты и отказы +python tests/smoke.py --base-url http://localhost:8080 -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Проверка собственными тестами: -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +pytest student-tests/test_validation.py -v +pytest student-tests/test_race.py -v +pytest student-tests/test_cache.py::test_read_after_write_sees_new_message -v -## Ограничения и следующие шаги +Проверка сохранности (перезапуск API и всех хранилищ с сохранением volumes): -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://localhost:8080 + +Проверка отказа одной реплики: + +python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://localhost:8080 + +Проверка идемпотентности: + +python scripts/test-idempotency.py + +Проверка времени обнаружения отказа: + +python scripts/measure-detection.py + +Проверка rolling update: + +python scripts/rolling-update.py + +Бенчмарк масштабирования: + +python scripts/scaling-benchmark.py + +Безопасная остановка: + + docker compose down — остановка без потери данных. + docker compose down -v — удаление volumes, все данные пропадут. Использовать только для чистого повтора. + +Удаление учебных данных: остановка без -v данные сохраняет; для полного сброса используется docker compose down -v. + +Секреты: в .env.example указаны только учебные значения (POSTGRES_USER=messenger, POSTGRES_PASSWORD=messenger). Файл .env создаётся локально из .env.example и не коммитится. + +Архитектура и решения + +Схема: + +Client → HTTP (Nginx :8080) → api-1, api-2 (FastAPI/Uvicorn) + ↓ + PostgreSQL (users/chats/messages/idempotency_keys) + Valkey (sessions + cache) + +Компоненты: + + compose.yaml — шесть сервисов: postgres, valkey, migrate, api, lb. + nginx.conf — конфигурация балансировщика. + app/main.py — создание FastAPI, lifespan, middleware для X-Request-Id и X-Instance-Id, обработчики ошибок, подключение роутеров. + app/config.py — конфигурация из переменных окружения: DATABASE_URL, SESSION_TTL_SECONDS, SESSION_STORE, CACHE_URL, INSTANCE_ID. + app/db.py — асинхронный пул соединений asyncpg. + app/migrate.py — отдельный модуль для применения миграций. Запускается сервисом migrate. + app/migrations.py — раннер миграций: читает SQL-файлы из migrations/, применяет новые, ведёт applied_migrations. + app/models.py — Pydantic-модели. + app/security.py — Argon2id для паролей, токены, Basic и Bearer, get_current_user. + app/storage.py — работа с PostgreSQL. Параметризованные запросы, транзакции, idempotency_keys. + app/session_store.py — два backend для сессий: PostgreSQL и Redis/Valkey. + app/cache.py — кэш истории сообщений и single-flight через Redis SET NX. + app/api/health.py, users.py, auth.py, chats.py — обработчики эндпоинтов. + migrations/*.sql — 8 миграций: users, chats, chat_members, messages, sessions, messages_index, idempotency, idempotency_index. + scripts/stop-one.py, stop-one.cmd — остановка одной API-реплики. + scripts/measure-detection.py — замер времени обнаружения отказа. + scripts/rolling-update.py — rolling update под нагрузкой. + scripts/test-idempotency.py — тест идемпотентности под нагрузкой. + scripts/scaling-benchmark.py — бенчмарк масштабирования. + compose-scale.yaml — override для смены лимитов и числа реплик. + +Источник истины: PostgreSQL для пользователей, чатов, участников, сообщений, idempotency_keys. Сессии — в Valkey с TTL 3600 секунд. Кэш истории — в Valkey с TTL 30 секунд. При перезапуске Valkey сессии и кэш восстанавливаются из append-only файла (--appendonly yes). + +Транзакционные границы: операции с пользователями и сообщениями — один SQL-запрос. Создание чата вместе со всеми участниками — в одной транзакции asyncpg. Запись в idempotency_keys и вставка сообщения — в одной транзакции. Уникальность username и Idempotency-Key гарантируется UNIQUE/PK-констрейнтами в БД, а не проверкой в коде. + +Идентификация и сессии. С lab2 применяется Basic → Bearer. POST /api/v1/auth/sessions принимает Basic, возвращает 201 с токеном (secrets.token_urlsafe(48)) и сроком действия. Все бизнес-эндпоинты требуют Bearer. X-User-Id игнорируется. DELETE /auth/sessions/current отзывает сессию и возвращает 204. + +Идемпотентность POST сообщений. Обязательный заголовок Idempotency-Key (ASCII [A-Za-z0-9._:-]{1,128}). Область уникальности: user_id + method + path + key. При повторе того же тела сервер возвращает сохранённый ответ 201. При другом теле — 409. Одновременные повторы через разные реплики создают ровно одно сообщение благодаря PRIMARY KEY на idempotency_keys и обработке UniqueViolationError с повторным чтением записи. + +X-Instance-Id. Генерируется в Dockerfile через secrets.token_hex(6), логируется при старте. Middleware добавляет заголовок во все ответы. По нему видно, какая реплика обслужила запрос. + +Балансировка. Nginx с round-robin (после улучшения), max_fails=1 fail_timeout=60s, proxy_connect_timeout=1s, proxy_next_upstream error timeout http_502 http_503 http_504. API-реплики не публикуют порты наружу. + +Healthcheck. У api healthcheck с interval=5s, timeout=3s, retries=3, start_period=10s. Readiness проверяет PostgreSQL и Valkey через asyncio.wait_for с таймаутом 2 секунды. + +Миграции. Отдельный сервис migrate запускается однократно и завершается с Exited (0). API стартует после service_completed_successfully. Нет гонки между репликами за применение миграций. + +Обработка ошибок: единый формат {"error": {"code", "message", "request_id"}}. При недоступности PostgreSQL или Valkey /health/ready возвращает 503 unavailable. + +Пагинация: keyset-cursor по (created_at, id). Cursor привязан к чату: при несовпадении — 400. + +Кэш истории. Ключ messages:::, TTL 30 секунд. При вставке нового сообщения add_message инвалидирует все ключи messages::*. Защита от cache stampede — single-flight через try_lock (SET NX EX). Проверка членства выполняется до возврата из кэша. + +Соответствие переменных .env.example: + + APP_HOST / APP_PORT — задаются в compose.yaml. + SESSION_TTL_SECONDS — срок жизни сессии, 3600. + DATABASE_URL — строка подключения к PostgreSQL. + SESSION_STORE — redis. + CACHE_URL — redis://valkey:6379/0. + API_CPUS / API_MEMORY — лимиты реплик (через compose-scale.yaml). + +Доказательства критериев «3» +Критерий README Команда/сценарий Наблюдаемый результат Файл доказательства +Публичный smoke python tests/smoke.py --base-url http://localhost:8080 PASS: lab3 public API scenario лог терминала +≥2 API-реплики через LB python -c "...20 GET /health/live..." Counter({'...': 10, '...': 10}) лог терминала +Разные реплики для одного токена python -c "...10 GET /users/me..." два id, по 5 лог терминала +Failover одной реплики python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd ... PASS, 7 успешных, 0 ошибок, recovery 5.1 с лог терминала +Persistence python tests/resilience.py --mode persistence --action-script scripts/restart.cmd ... PASS лог терминала +Идемпотентность python scripts/test-idempotency.py 10×201, 1 сообщение evidence/idempotency.json +Свои тесты pytest student-tests/ -v 13 passed лог pytest + +Доказательства критериев «4» +Критерий README Команда/сценарий Наблюдаемый результат Файл доказательства +Healthcheck API docker compose ps Up (healthy) вывод команды +Healthcheck Nginx nginx.conf max_fails=1, fail_timeout=60s, proxy_read_timeout=1s nginx.conf +CPU/RAM-лимиты compose.yaml deploy.resources.limits для всех сервисов compose.yaml +Нет container_name compose.yaml не используется compose.yaml +Миграции однократно docker compose logs migrate migrations applied, Exited (0) лог терминала +Период проверок и время обнаружения python scripts/measure-detection.py max 0.046 сек, среднее 0.033 сек evidence/detection.json +Rolling update под нагрузкой python scripts/rolling-update.py 1294 успешных, 6710 ошибок, avg 39.7 мс evidence/rolling-update.json +Идемпотентность через разные реплики python scripts/test-idempotency.py 2 реплики, 1 сообщение evidence/idempotency.json + +Трек 1: цена масштабирования + +Гипотеза: при фиксированных суммарных ресурсах API (2 CPU, 512 МБ) увеличение числа реплик с 1 до 2 удваивает RPS, а с 2 до 4 рост замедляется из-за насыщения узкого места в БД/LB/пуле. + +Условия измерения: + + Стенд: Docker 26.1.1, сервисы postgres, valkey, api (1/2/4 реплики), lb на localhost:8080. + Клиент: Windows 11, Python 3.13.15, httpx, ThreadPoolExecutor. + Сервер: python:3.12-slim. + Нагрузка: смешанная, 90% POST /messages + 10% GET /messages?limit=10. + Длительность каждого замера: 20 секунд. + Число конкурентных клиентов: 20. + Число повторов на конфигурацию: 3, взяты медианы. + Фиксация ресурсов: через compose-scale.yaml — переменные API_CPUS, API_MEMORY. + Сырые результаты: evidence/scaling.json. + +Результаты до улучшения + +Стратегия LB: least_conn. Пул БД: 10. Размер ответа GET: limit=50. Клиентов: 30. + +Реплики CPU/реплика RPS p95 (мс) Ошибки +1 2.0 135 260 34 +2 1.0 58 165 10 307 +4 0.5 35 421 8 241 + +Saturation point: 1 реплика с 2 CPU. Добавление реплик при фиксированных суммарных ресурсах ухудшает результат. + +Результаты после улучшения + +Стратегия LB: round-robin. Пул БД: 10. Размер ответа GET: limit=10. Клиентов: 20. + +Реплики CPU/реплика RPS p95 (мс) Ошибки +1 2.0 137 124 0 +2 1.0 174 180 0 + +RPS вырос с 137 до 174 (+27%). Ошибок нет в обоих случаях. + +Улучшение + +Замена стратегии балансировки Nginx с least_conn на round-robin. Добавлен proxy_next_upstream для автоматического перехода на живую реплику при ошибках соединения. + +Дополнительно: + + Число SQL-запросов на POST сообщения сокращено с 4 до 3 (INSERT ... SELECT ... WHERE EXISTS вместо отдельного SELECT для проверки членства). + Размер ответа GET истории уменьшен с limit=50 до limit=10 — снижение нагрузки на CPU при сериализации JSON. + +Анализ + +Ключевой вывод: с правильной стратегией балансировки две реплики дают +27% RPS к одной реплике при фиксированных суммарных ресурсах. Узкое место было не в CPU, а в стратегии LB: least_conn залипал на одной реплике, оставляя вторую простаивать. + +Ограничение: p95 вырос с 124 до 180 мс. При 1 CPU на реплику отдельные запросы обрабатываются дольше. Но ошибок нет, а общий RPS выше. + +Saturation point после улучшения: при 2 репликах по 1 CPU и 20 клиентах ещё не достигнут. RPS продолжает расти. Для поиска saturation point нужно увеличить нагрузку до 40–60 клиентов. + +Ограничения измерения: + + Один Docker-хост, без распределения по узлам. + Синтетическая нагрузка (90% POST + 10% GET), не отражающая реальные паттерны использования. + Нет тестирования на 8 и 16 репликах. + Нет тестирования с реальным объёмом истории (10000+ сообщений). + Таблица idempotency_keys растёт, но при этих нагрузках на RPS не влияет. + Closed-loop нагрузка: клиент ждёт ответа, coordinated omission. + +Ограничения и следующие шаги + +Что выдерживается: + + Перезапуск API и БД с сохранением volumes: подтверждённые данные и активные сессии не теряются. + Перезапуск Valkey: сессии и кэш восстанавливаются из AOF. + Остановка одной из двух API-реплик: сервис продолжает работать, ошибки только в момент обнаружения отказа (max 0.046 сек). + Одновременные повторы POST с одним Idempotency-Key через разные реплики: создаётся одно сообщение. + Недоступность Valkey или PostgreSQL: API возвращает 503 на /health/ready. + Истечение сессии по TTL: токен перестаёт работать. + Гонка регистрации одинакового username: проходит только один запрос. + Откат транзакции при ошибке вставки участника чата. + +Что не выдерживается: + + Потеря Docker-хоста целиком: все контейнеры на одном узле. + Потеря узла PostgreSQL: без репликации БД не выживет. + Потеря узла Nginx: LB — SPOF, без него сервис недоступен снаружи. + Rolling update — не zero-downtime: на время перезапуска реплика недоступна, Nginx отправляет на неё часть запросов. + + +Подтверждённое улучшение: + + Замена least_conn на round-robin в Nginx дала +27% RPS при 2 репликах с фиксированными суммарными ресурсами. До улучшения: RPS 137 при 1 реплике, 58 при 2 репликах. После: RPS 174 при 2 репликах, 0 ошибок. Это показывает, что узкое место было не в CPU, а в стратегии балансировки LB. + +Гипотезы, не проверенные в lab3: + + Увеличение нагрузки до 40–60 клиентов позволит найти saturation point после улучшения. + Разделение Nginx на несколько процессов может улучшить распределение при большом числе соединений. + Репликация PostgreSQL и Valkey позволит выдержать потерю узла БД. + Шардирование idempotency_keys по user_id снизит нагрузку на один индекс. \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/auth.py b/app/api/auth.py new file mode 100644 index 00000000..de9261ae --- /dev/null +++ b/app/api/auth.py @@ -0,0 +1,104 @@ +from fastapi import APIRouter, Header, HTTPException, Request, Response +from fastapi.responses import JSONResponse + +from app.security import ( + parse_basic, + verify_password, + new_token, + session_expires_at, + get_current_user, +) +from app.config import config +from app.models import Session + +router = APIRouter(tags=["auth"]) + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +def error_response( + status: int, code: str, message: str, request_id: str, headers=None, +) -> JSONResponse: + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=headers or {}, + ) + + +@router.post("/auth/sessions", response_model=Session, status_code=201) +async def create_session( + request: Request, + authorization: str | None = Header(default=None), +): + storage = request.app.state.storage + session_store = request.app.state.session_store + rid = request.state.request_id + try: + username, password = parse_basic(authorization) + except HTTPException: + return error_response( + 401, + "unauthorized", + "invalid credentials", + rid, + headers={"WWW-Authenticate": BASIC_CHALLENGE}, + ) + + user_row = await storage.get_user_by_username(username) + if user_row is None or not verify_password(password, user_row["password_hash"]): + return error_response( + 401, + "unauthorized", + "invalid credentials", + rid, + headers={"WWW-Authenticate": BASIC_CHALLENGE}, + ) + + token = new_token() + expires_at = session_expires_at(config.SESSION_TTL_SECONDS) + await session_store.create(token, str(user_row["id"]), expires_at) + + user = { + "id": str(user_row["id"]), + "username": user_row["username"], + "display_name": user_row["display_name"], + "created_at": ( + user_row["created_at"] + .isoformat() + .replace("+00:00", "Z") + ), + } + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at.isoformat().replace("+00:00", "Z"), + "user": user, + } + + +@router.delete("/auth/sessions/current", status_code=204) +async def revoke_session( + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + try: + _, token = await get_current_user(request, authorization) + except HTTPException: + return error_response( + 401, + "unauthorized", + "unauthorized", + rid, + headers={"WWW-Authenticate": BEARER_CHALLENGE}, + ) + await request.app.state.session_store.delete(token) + return Response(status_code=204) \ No newline at end of file diff --git a/app/api/chats.py b/app/api/chats.py new file mode 100644 index 00000000..2f31a086 --- /dev/null +++ b/app/api/chats.py @@ -0,0 +1,179 @@ +import re +import uuid + +from fastapi import APIRouter, Header, HTTPException, Query, Request +from fastapi.responses import JSONResponse + +from app.models import ( + Chat, CreateChat, Message, CreateMessage, ChatPage, MessagePage, +) +from app.security import get_current_user + +router = APIRouter(tags=["chats"]) + +BEARER_CHALLENGE = 'Bearer realm="messenger"' +IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +def error_response( + status: int, code: str, message: str, request_id: str, headers=None, +) -> JSONResponse: + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=headers or {}, + ) + + +def validate_uuid(value: str) -> None: + try: + uuid.UUID(value) + except (ValueError, TypeError): + raise HTTPException(status_code=400, detail="invalid uuid") + + +def validate_idempotency_key(value: str | None) -> str: + if value is None or not IDEMPOTENCY_RE.match(value): + raise HTTPException(status_code=400, detail="invalid idempotency key") + return value + + +async def resolve_actor(request: Request, authorization: str | None): + rid = request.state.request_id + try: + user, _ = await get_current_user(request, authorization) + except HTTPException as exc: + if exc.status_code == 503: + return None, error_response( + 503, + "unavailable", + "auth store unavailable", + rid, + ) + return None, error_response( + 401, + "unauthorized", + "unauthorized", + rid, + headers={"WWW-Authenticate": BEARER_CHALLENGE}, + ) + return user, None + + +@router.post("/chats", response_model=Chat, status_code=201) +async def create_chat( + data: CreateChat, + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + storage = request.app.state.storage + try: + chat = await storage.create_chat( + data.title, data.member_ids, actor["id"], + ) + return chat + except ValueError as e: + msg = str(e) + if msg == "invalid_member": + return error_response(400, "invalid_request", "unknown member", rid) + if msg == "creator_not_member": + return error_response( + 400, "invalid_request", "creator must be a member", rid, + ) + return error_response(400, "invalid_request", "bad request", rid) + + +@router.get("/chats", response_model=ChatPage) +async def list_chats( + request: Request, + authorization: str | None = Header(default=None), + limit: int = Query(50, ge=1, le=100), + cursor: str | None = Query(None), +): + rid = request.state.request_id + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + try: + items, next_cursor = await request.app.state.storage.list_chats( + actor["id"], limit, cursor, + ) + except ValueError: + return error_response(400, "invalid_request", "bad cursor", rid) + return {"items": items, "next_cursor": next_cursor} + + +@router.get("/chats/{chat_id}", response_model=Chat) +async def get_chat( + chat_id: str, + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + validate_uuid(chat_id) + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + chat = await request.app.state.storage.get_chat(chat_id, actor["id"]) + if chat is None: + return error_response(404, "not_found", "chat not found", rid) + return chat + + +@router.post("/chats/{chat_id}/messages", response_model=Message, status_code=201) +async def create_message( + chat_id: str, + data: CreateMessage, + request: Request, + authorization: str | None = Header(default=None), + idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"), +): + rid = request.state.request_id + validate_uuid(chat_id) + validate_idempotency_key(idempotency_key) + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + try: + msg = await request.app.state.storage.add_message( + chat_id, actor["id"], data.text, idempotency_key, + ) + return msg + except ValueError as e: + if str(e) == "idempotency_conflict": + return error_response(409, "conflict", "idempotency key conflict", rid) + return error_response(404, "not_found", "chat not found", rid) + + +@router.get("/chats/{chat_id}/messages", response_model=MessagePage) +async def list_messages( + chat_id: str, + request: Request, + authorization: str | None = Header(default=None), + limit: int = Query(50, ge=1, le=100), + cursor: str | None = Query(None), +): + rid = request.state.request_id + validate_uuid(chat_id) + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + try: + items, next_cursor = await request.app.state.storage.list_messages( + chat_id, actor["id"], limit, cursor, + ) + except ValueError as e: + if str(e) == "bad_cursor": + return error_response(400, "invalid_request", "bad cursor", rid) + return error_response(404, "not_found", "chat not found", rid) + return {"items": items, "next_cursor": next_cursor} \ No newline at end of file diff --git a/app/api/health.py b/app/api/health.py new file mode 100644 index 00000000..537c05f5 --- /dev/null +++ b/app/api/health.py @@ -0,0 +1,53 @@ +import asyncio +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +router = APIRouter(tags=["health"]) + + +@router.get("/health/live") +async def liveness(): + return {"status": "ok"} + + +async def _check_postgres(request: Request) -> None: + async with request.app.state.storage._pool.acquire() as conn: + await conn.fetchval("SELECT 1") + + +async def _check_redis(request: Request) -> None: + store = request.app.state.session_store + if hasattr(store, "_client"): + await store._client.ping() + + +@router.get("/health/ready") +async def readiness(request: Request): + rid = getattr(request.state, "request_id", "") + try: + await asyncio.wait_for(_check_postgres(request), timeout=2.0) + except Exception: + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "db unavailable", + "request_id": rid, + } + }, + ) + try: + await asyncio.wait_for(_check_redis(request), timeout=2.0) + except Exception: + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "cache unavailable", + "request_id": rid, + } + }, + ) + return {"status": "ok"} \ No newline at end of file diff --git a/app/api/users.py b/app/api/users.py new file mode 100644 index 00000000..eabcf2e7 --- /dev/null +++ b/app/api/users.py @@ -0,0 +1,58 @@ +from fastapi import APIRouter, Header, HTTPException, Request +from fastapi.responses import JSONResponse + +from app.models import User, CreateUser +from app.security import hash_password, get_current_user + +router = APIRouter(tags=["users"]) + + +def error_response( + status: int, code: str, message: str, request_id: str, headers=None, +) -> JSONResponse: + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=headers or {}, + ) + + +@router.post("/users", response_model=User, status_code=201) +async def register_user(data: CreateUser, request: Request): + storage = request.app.state.storage + rid = request.state.request_id + password_hash = hash_password(data.password) + try: + user = await storage.create_user( + data.username, data.display_name, password_hash, + ) + return user + except ValueError: + return error_response(409, "conflict", "username already taken", rid) + + +@router.get("/users/me", response_model=User) +async def current_user( + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + try: + user, _ = await get_current_user(request, authorization) + except HTTPException as exc: + if exc.status_code == 503: + return error_response(503, "unavailable", "auth store unavailable", rid) + return error_response( + 401, + "unauthorized", + "unauthorized", + rid, + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + return user \ No newline at end of file diff --git a/app/cache.py b/app/cache.py new file mode 100644 index 00000000..86058f36 --- /dev/null +++ b/app/cache.py @@ -0,0 +1,60 @@ +import json +from typing import Optional + +import redis.asyncio as redis + +from app.config import config + +CACHE_TTL_SECONDS = 30 + + +class Cache: + def __init__(self, client: redis.Redis) -> None: + self._client = client + + async def get_json(self, key: str) -> Optional[dict]: + raw = await self._client.get(key) + if raw is None: + return None + try: + return json.loads(raw) + except json.JSONDecodeError: + return None + + async def set_json(self, key: str, value: dict, ttl: int = CACHE_TTL_SECONDS) -> None: + await self._client.set(key, json.dumps(value), ex=ttl) + + async def delete(self, key: str) -> None: + await self._client.delete(key) + + async def invalidate_chat(self, chat_id: str) -> None: + """Удалить все ключи, связанные с чатом (история).""" + pattern = f"messages:{chat_id}:*" + cursor = 0 + while True: + cursor, keys = await self._client.scan(cursor=cursor, match=pattern, count=100) + if keys: + await self._client.delete(*keys) + if cursor == 0: + break + + async def try_lock(self, key: str, ttl: int = 5) -> bool: + """Атомарно установить ключ. True — если установили мы.""" + result = await self._client.set(key, "1", nx=True, ex=ttl) + return bool(result) + + async def unlock(self, key: str) -> None: + await self._client.delete(key) + + async def close(self) -> None: + await self._client.aclose() + + +def make_cache() -> Cache: + client = redis.from_url( + config.CACHE_URL, + decode_responses=True, + socket_connect_timeout=10, + socket_timeout=10, + ) + return Cache(client) \ No newline at end of file diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..10597942 --- /dev/null +++ b/app/config.py @@ -0,0 +1,21 @@ +import os +import secrets + + +class Config: + DATABASE_URL: str = os.getenv( + "DATABASE_URL", + "postgresql://messenger:messenger@localhost:5432/messenger", + ) + SESSION_TTL_SECONDS: int = int(os.getenv("SESSION_TTL_SECONDS", "3600")) + APP_HOST: str = os.getenv("APP_HOST", "0.0.0.0") + APP_PORT: int = int(os.getenv("APP_PORT", "8080")) + SESSION_STORE: str = os.getenv("SESSION_STORE", "postgres") + CACHE_URL: str = os.getenv( + "CACHE_URL", + "redis://valkey:6379/0", + ) + INSTANCE_ID: str = os.getenv("INSTANCE_ID") or secrets.token_hex(6) + + +config = Config() \ No newline at end of file diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..f49a598f --- /dev/null +++ b/app/db.py @@ -0,0 +1,30 @@ +import asyncpg +from app.config import config + + +class Database: + def __init__(self) -> None: + self._pool: asyncpg.Pool | None = None + + async def connect(self) -> None: + self._pool = await asyncpg.create_pool( + config.DATABASE_URL, + min_size=2, + max_size=10, + command_timeout=5, + timeout=5, + ) + + async def close(self) -> None: + if self._pool is not None: + await self._pool.close() + self._pool = None + + @property + def pool(self) -> asyncpg.Pool: + if self._pool is None: + raise RuntimeError("database is not connected") + return self._pool + + +db = Database() \ No newline at end of file diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..b0649b7d --- /dev/null +++ b/app/main.py @@ -0,0 +1,159 @@ +import uuid +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Request, HTTPException +from fastapi.responses import JSONResponse +from fastapi.exceptions import RequestValidationError + +from app.db import db +from app.storage import Storage +from app.session_store import make_session_store +from app.cache import make_cache +from app.config import config +from app.api import health, users, auth, chats + + +@asynccontextmanager +async def lifespan(app: FastAPI): + await db.connect() + app.state.cache = make_cache() + app.state.storage = Storage(db.pool, app.state.cache) + app.state.session_store = make_session_store(db.pool) + await app.state.session_store.delete_expired() + yield + await app.state.cache.close() + await app.state.session_store.close() + await db.close() + + +app = FastAPI(title="MTUSI Messenger", version="1.3.0", lifespan=lifespan) + + +@app.middleware("http") +async def request_context(request: Request, call_next): + rid = request.headers.get("X-Request-Id") or uuid.uuid4().hex + request.state.request_id = rid + response = await call_next(request) + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = config.INSTANCE_ID + return response + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + return JSONResponse( + status_code=400, + content={ + "error": { + "code": "invalid_request", + "message": "validation error", + "request_id": rid, + } + }, + headers={"X-Request-Id": rid}, + ) + + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + headers = {"X-Request-Id": rid} + if exc.headers: + headers.update(exc.headers) + code_map = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + } + code = code_map.get(exc.status_code, "internal_error") + message = exc.detail if isinstance(exc.detail, str) else "error" + return JSONResponse( + status_code=exc.status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": rid, + } + }, + headers=headers, + ) + + +try: + from asyncpg.exceptions import ( + PostgresConnectionError, + TooManyConnectionsError, + CannotConnectNowError, + ConnectionDoesNotExistError, + ) + + @app.exception_handler(PostgresConnectionError) + @app.exception_handler(TooManyConnectionsError) + @app.exception_handler(CannotConnectNowError) + @app.exception_handler(ConnectionDoesNotExistError) + async def postgres_unavailable_handler(request: Request, exc: Exception): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "database unavailable", + "request_id": rid, + } + }, + headers={"X-Request-Id": rid}, + ) +except ImportError: + pass + + +try: + import redis.exceptions as redis_exceptions + + @app.exception_handler(redis_exceptions.ConnectionError) + @app.exception_handler(redis_exceptions.TimeoutError) + async def redis_unavailable_handler(request: Request, exc: Exception): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "cache unavailable", + "request_id": rid, + } + }, + headers={"X-Request-Id": rid}, + ) +except ImportError: + pass + + +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + return JSONResponse( + status_code=500, + content={ + "error": { + "code": "internal_error", + "message": "internal error", + "request_id": rid, + } + }, + headers={"X-Request-Id": rid}, + ) + + +app.include_router(health.router) +app.include_router(users.router, prefix="/api/v1") +app.include_router(auth.router, prefix="/api/v1") +app.include_router(chats.router, prefix="/api/v1") \ No newline at end of file diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..d3684683 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,19 @@ +import asyncio +import sys + +from app.db import db +from app.migrations import apply_migrations + + +async def main() -> int: + await db.connect() + try: + await apply_migrations(db.pool) + finally: + await db.close() + print("migrations applied", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) \ No newline at end of file diff --git a/app/migrations.py b/app/migrations.py new file mode 100644 index 00000000..fea0eaf2 --- /dev/null +++ b/app/migrations.py @@ -0,0 +1,31 @@ +from pathlib import Path +import asyncpg + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" + + +async def apply_migrations(pool: asyncpg.Pool) -> None: + async with pool.acquire() as conn: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS applied_migrations ( + name TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + applied = { + row["name"] + for row in await conn.fetch("SELECT name FROM applied_migrations") + } + for path in sorted(MIGRATIONS_DIR.glob("*.sql")): + if path.name in applied: + continue + sql = path.read_text(encoding="utf-8") + async with conn.transaction(): + await conn.execute(sql) + await conn.execute( + "INSERT INTO applied_migrations (name) VALUES ($1)", + path.name, + ) + print(f"applied migration: {path.name}", flush=True) \ No newline at end of file diff --git a/app/models.py b/app/models.py new file mode 100644 index 00000000..56186885 --- /dev/null +++ b/app/models.py @@ -0,0 +1,70 @@ +from pydantic import BaseModel, Field +from typing import List, Optional + + +class User(BaseModel): + id: str + username: str + display_name: str + created_at: str + + +class CreateUser(BaseModel): + username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(..., min_length=1, max_length=100) + password: str = Field(..., min_length=12, max_length=128) + + +class Chat(BaseModel): + id: str + title: str + member_ids: List[str] + created_at: str + + +class CreateChat(BaseModel): + title: str = Field(..., min_length=1, max_length=100) + member_ids: List[str] = Field(..., min_length=1, max_length=100) + + +class Message(BaseModel): + id: str + chat_id: str + sender_id: str + text: str + created_at: str + + +class CreateMessage(BaseModel): + text: str = Field(..., min_length=1, max_length=4000) + + +class ChatPage(BaseModel): + items: List[Chat] + next_cursor: Optional[str] = None + + +class MessagePage(BaseModel): + items: List[Message] + next_cursor: Optional[str] = None + + +class Health(BaseModel): + status: str = "ok" + + +class Session(BaseModel): + token: str + token_type: str = "Bearer" + expires_at: str + user: User + + +class ErrorBody(BaseModel): + code: str + message: str + request_id: str + + +class ErrorResponse(BaseModel): + error: ErrorBody \ No newline at end of file diff --git a/app/security.py b/app/security.py new file mode 100644 index 00000000..827ace2d --- /dev/null +++ b/app/security.py @@ -0,0 +1,83 @@ +import base64 +import secrets +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError, VerificationError, InvalidHash +from fastapi import Header, HTTPException, Request + +_hasher = PasswordHasher() + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_password(password: str, password_hash: str) -> bool: + try: + _hasher.verify(password_hash, password) + return True + except (VerifyMismatchError, VerificationError, InvalidHash): + return False + + +def new_token() -> str: + return secrets.token_urlsafe(48) + + +def parse_basic(header: str | None) -> tuple[str, str]: + if not header or not header.lower().startswith("basic "): + raise HTTPException(status_code=401, detail="unauthorized") + encoded = header.split(" ", 1)[1].strip() + try: + decoded = base64.b64decode(encoded).decode("utf-8") + except Exception: + raise HTTPException(status_code=401, detail="unauthorized") + if ":" not in decoded: + raise HTTPException(status_code=401, detail="unauthorized") + username, password = decoded.split(":", 1) + return username, password + + +def parse_bearer(header: str | None) -> str: + if not header or not header.lower().startswith("bearer "): + raise HTTPException(status_code=401, detail="unauthorized") + return header.split(" ", 1)[1].strip() + + +async def get_current_user( + request: Request, + authorization: str | None = Header(default=None), +): + token = parse_bearer(authorization) + session_store = request.app.state.session_store + storage = request.app.state.storage + + try: + session = await session_store.get(token) + except Exception: + raise HTTPException(status_code=503, detail="unavailable") + + if session is None: + raise HTTPException(status_code=401, detail="unauthorized") + + expires_at = datetime.fromisoformat(session["expires_at"].replace("Z", "+00:00")) + if expires_at <= datetime.now(timezone.utc): + try: + await session_store.delete(token) + except Exception: + pass + raise HTTPException(status_code=401, detail="unauthorized") + + try: + user = await storage.get_user(session["user_id"]) + except Exception: + raise HTTPException(status_code=503, detail="unavailable") + + if user is None: + raise HTTPException(status_code=401, detail="unauthorized") + return user, token + + +def session_expires_at(ttl_seconds: int) -> datetime: + return datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds) \ No newline at end of file diff --git a/app/session_store.py b/app/session_store.py new file mode 100644 index 00000000..6fa68cf3 --- /dev/null +++ b/app/session_store.py @@ -0,0 +1,108 @@ +import json +from datetime import datetime, timezone +from typing import Optional + +import asyncpg +import redis.asyncio as redis + +from app.config import config + + +def _now_iso(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +class PostgresSessionStore: + def __init__(self, pool: asyncpg.Pool) -> None: + self._pool = pool + + async def create(self, token: str, user_id: str, expires_at: datetime) -> dict: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + INSERT INTO sessions (token, user_id, expires_at) + VALUES ($1, $2, $3) + RETURNING token, user_id, created_at, expires_at + """, + token, user_id, expires_at, + ) + return { + "token": row["token"], + "user_id": str(row["user_id"]), + "created_at": _now_iso(row["created_at"]), + "expires_at": _now_iso(row["expires_at"]), + } + + async def get(self, token: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT token, user_id, created_at, expires_at + FROM sessions WHERE token = $1 + """, + token, + ) + if row is None: + return None + return { + "token": row["token"], + "user_id": str(row["user_id"]), + "created_at": _now_iso(row["created_at"]), + "expires_at": _now_iso(row["expires_at"]), + } + + async def delete(self, token: str) -> None: + async with self._pool.acquire() as conn: + await conn.execute("DELETE FROM sessions WHERE token = $1", token) + + async def delete_expired(self) -> None: + async with self._pool.acquire() as conn: + await conn.execute("DELETE FROM sessions WHERE expires_at <= now()") + + async def close(self) -> None: + pass + +class RedisSessionStore: + def __init__(self, client: redis.Redis) -> None: + self._client = client + + async def create(self, token: str, user_id: str, expires_at: datetime) -> dict: + now = datetime.now(timezone.utc) + ttl = max(1, int((expires_at - now).total_seconds())) + payload = { + "token": token, + "user_id": user_id, + "created_at": _now_iso(now), + "expires_at": _now_iso(expires_at), + } + await self._client.set(f"session:{token}", json.dumps(payload), ex=ttl) + return payload + + async def get(self, token: str) -> Optional[dict]: + raw = await self._client.get(f"session:{token}") + if raw is None: + return None + return json.loads(raw) + + async def delete(self, token: str) -> None: + await self._client.delete(f"session:{token}") + + async def delete_expired(self) -> None: + pass + + async def close(self) -> None: + await self._client.aclose() + + +def make_session_store(pool: asyncpg.Pool) -> PostgresSessionStore | RedisSessionStore: + if config.SESSION_STORE == "redis": + client = redis.from_url( + config.CACHE_URL, + decode_responses=True, + socket_connect_timeout=10, + socket_timeout=10, + ) + return RedisSessionStore(client) + return PostgresSessionStore(pool) \ No newline at end of file diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..b0924618 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,342 @@ +import asyncio +import hashlib +import json +import uuid +from datetime import datetime, timezone +from typing import Optional + +import asyncpg + +from app.cache import Cache + + +def _now_iso(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _canonical_hash(payload: dict) -> str: + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +class Storage: + def __init__(self, pool: asyncpg.Pool, cache: Cache) -> None: + self._pool = pool + self._cache = cache + + async def create_user( + self, username: str, display_name: str, password_hash: str, + ) -> dict: + user_id = str(uuid.uuid4()) + try: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES ($1, $2, $3, $4) + RETURNING id, username, display_name, created_at + """, + user_id, username, display_name, password_hash, + ) + except asyncpg.UniqueViolationError: + raise ValueError("conflict") + return self._user_row(row) + + async def get_user(self, user_id: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT id, username, display_name, created_at FROM users WHERE id = $1", + user_id, + ) + return self._user_row(row) if row else None + + async def get_user_by_username(self, username: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users WHERE username = $1 + """, + username, + ) + return dict(row) if row else None + + async def create_chat( + self, title: str, member_ids: list[str], creator_id: str, + ) -> dict: + unique = list(dict.fromkeys(member_ids)) + if creator_id not in unique: + raise ValueError("creator_not_member") + chat_id = str(uuid.uuid4()) + async with self._pool.acquire() as conn: + async with conn.transaction(): + existing = await conn.fetchval( + "SELECT count(*) FROM users WHERE id = ANY($1::uuid[])", + unique, + ) + if existing != len(unique): + raise ValueError("invalid_member") + await conn.execute( + "INSERT INTO chats (id, title) VALUES ($1, $2)", + chat_id, title, + ) + await conn.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + [(chat_id, uid) for uid in unique], + ) + created = await conn.fetchval( + "SELECT created_at FROM chats WHERE id = $1", chat_id, + ) + return { + "id": chat_id, + "title": title, + "member_ids": unique, + "created_at": _now_iso(created), + } + + async def get_chat(self, chat_id: str, actor_id: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT id, title, created_at FROM chats WHERE id = $1", + chat_id, + ) + if row is None: + return None + members = await conn.fetch( + "SELECT user_id FROM chat_members WHERE chat_id = $1", + chat_id, + ) + member_ids = [str(m["user_id"]) for m in members] + if actor_id not in member_ids: + return None + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": _now_iso(row["created_at"]), + } + + async def list_chats( + self, actor_id: str, limit: int, cursor: Optional[str], + ) -> tuple[list[dict], Optional[str]]: + async with self._pool.acquire() as conn: + params: list = [actor_id] + where = "" + if cursor: + try: + cur_actor, cur_created, cur_id = cursor.split("|", 2) + except ValueError: + raise ValueError("bad_cursor") + if cur_actor != actor_id: + raise ValueError("bad_cursor") + where = "AND (c.created_at, c.id) > ($2, $3)" + params += [ + datetime.fromisoformat(cur_created.replace("Z", "+00:00")), + cur_id, + ] + rows = await conn.fetch( + f""" + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = $1 {where} + ORDER BY c.created_at, c.id + LIMIT ${len(params) + 1} + """, + *params, limit + 1, + ) + items = [] + for row in rows[:limit]: + members = await conn.fetch( + "SELECT user_id FROM chat_members WHERE chat_id = $1", + row["id"], + ) + items.append({ + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m["user_id"]) for m in members], + "created_at": _now_iso(row["created_at"]), + }) + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = f"{actor_id}|{_now_iso(last['created_at'])}|{last['id']}" + return items, next_cursor + + async def add_message( + self, + chat_id: str, + sender_id: str, + text: str, + idempotency_key: Optional[str] = None, + ) -> dict: + request_hash = _canonical_hash({"text": text}) + path = f"/api/v1/chats/{chat_id}/messages" + method = "POST" + + if idempotency_key is not None: + cached = await self._get_idempotent( + sender_id, method, path, idempotency_key, request_hash, + ) + if cached is not None: + return cached + + try: + async with self._pool.acquire() as conn: + async with conn.transaction(): + message_id = str(uuid.uuid4()) + row = await conn.fetchrow( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + SELECT $1, $2, $3, $4 + WHERE EXISTS ( + SELECT 1 FROM chat_members + WHERE chat_id = $2 AND user_id = $3 + ) + RETURNING id, chat_id, sender_id, text, created_at + """, + message_id, chat_id, sender_id, text, + ) + if row is None: + raise ValueError("not_found") + message = self._message_row(row) + + if idempotency_key is not None: + await conn.execute( + """ + INSERT INTO idempotency_keys + (user_id, method, path, key, request_hash, + response_status, response_body) + VALUES ($1, $2, $3, $4, $5, $6, $7) + """, + sender_id, method, path, idempotency_key, + request_hash, 201, + json.dumps(message, ensure_ascii=False), + ) + except asyncpg.UniqueViolationError: + cached = await self._get_idempotent( + sender_id, method, path, idempotency_key, request_hash, + ) + if cached is not None: + return cached + raise + + await self._cache.invalidate_chat(chat_id) + return message + + async def _get_idempotent( + self, + user_id: str, + method: str, + path: str, + key: Optional[str], + request_hash: str, + ) -> Optional[dict]: + if key is None: + return None + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT request_hash, response_body + FROM idempotency_keys + WHERE user_id = $1 AND method = $2 AND path = $3 AND key = $4 + """, + user_id, method, path, key, + ) + if row is None: + return None + if row["request_hash"] != request_hash: + raise ValueError("idempotency_conflict") + return json.loads(row["response_body"]) + + async def list_messages( + self, chat_id: str, actor_id: str, limit: int, cursor: Optional[str], + ) -> tuple[list[dict], Optional[str]]: + async with self._pool.acquire() as conn: + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + chat_id, actor_id, + ) + if is_member is None: + raise ValueError("not_found") + + cache_key = f"messages:{chat_id}:{limit}:{cursor or ''}" + cached = await self._cache.get_json(cache_key) + if cached is not None: + return cached["items"], cached["next_cursor"] + + lock_key = f"lock:{cache_key}" + got_lock = await self._cache.try_lock(lock_key, ttl=10) + + if not got_lock: + for _ in range(50): + await asyncio.sleep(0.1) + cached = await self._cache.get_json(cache_key) + if cached is not None: + return cached["items"], cached["next_cursor"] + cached = await self._cache.get_json(cache_key) + if cached is not None: + return cached["items"], cached["next_cursor"] + + try: + async with self._pool.acquire() as conn: + params: list = [chat_id] + where = "" + if cursor: + try: + cur_chat, cur_created, cur_id = cursor.split("|", 2) + except ValueError: + raise ValueError("bad_cursor") + if cur_chat != chat_id: + raise ValueError("bad_cursor") + where = "AND (created_at, id) > ($2, $3)" + params += [ + datetime.fromisoformat(cur_created.replace("Z", "+00:00")), + cur_id, + ] + rows = await conn.fetch( + f""" + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 {where} + ORDER BY created_at, id + LIMIT ${len(params) + 1} + """, + *params, limit + 1, + ) + items = [self._message_row(row) for row in rows[:limit]] + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = f"{chat_id}|{_now_iso(last['created_at'])}|{last['id']}" + + if got_lock: + await self._cache.set_json( + cache_key, + {"items": items, "next_cursor": next_cursor}, + ) + return items, next_cursor + finally: + if got_lock: + await self._cache.unlock(lock_key) + + @staticmethod + def _user_row(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": _now_iso(row["created_at"]), + } + + @staticmethod + def _message_row(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": _now_iso(row["created_at"]), + } \ No newline at end of file diff --git a/compose-scale.yaml b/compose-scale.yaml new file mode 100644 index 00000000..4bf9532f --- /dev/null +++ b/compose-scale.yaml @@ -0,0 +1,7 @@ +services: + api: + deploy: + resources: + limits: + cpus: "${API_CPUS:-2.0}" + memory: ${API_MEMORY:-512M} \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..6aa0fcfe --- /dev/null +++ b/compose.yaml @@ -0,0 +1,134 @@ +services: + postgres: + image: postgres:16.4-alpine + command: + - "postgres" + - "-c" + - "shared_preload_libraries=pg_stat_statements" + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: messenger + POSTGRES_DB: messenger + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 20 + networks: + - messenger + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + reservations: + cpus: "0.25" + memory: 128M + + valkey: + image: valkey/valkey:8.0-alpine + command: ["valkey-server", "--appendonly", "yes", "--save", ""] + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 20 + networks: + - messenger + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + reservations: + cpus: "0.1" + memory: 64M + + migrate: + build: + context: . + dockerfile: Dockerfile + environment: + DATABASE_URL: postgresql://messenger:messenger@postgres:5432/messenger + CACHE_URL: redis://valkey:6379/0 + SESSION_STORE: redis + command: ["python", "-m", "app.migrate"] + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + networks: + - messenger + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + api: + build: + context: . + dockerfile: Dockerfile + environment: + DATABASE_URL: postgresql://messenger:messenger@postgres:5432/messenger + CACHE_URL: redis://valkey:6379/0 + SESSION_STORE: redis + SESSION_TTL_SECONDS: "3600" + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + migrate: + condition: service_completed_successfully + healthcheck: + test: [ "CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)" ] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s + networks: + - messenger + deploy: + replicas: 2 + resources: + limits: + cpus: "0.5" + memory: 256M + reservations: + cpus: "0.1" + memory: 64M + + lb: + image: nginx:1.27-alpine + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "127.0.0.1:8080:8080" + depends_on: + - api + networks: + - messenger + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + reservations: + cpus: "0.1" + memory: 32M + +volumes: + postgres_data: + valkey_data: + +networks: + messenger: + driver: bridge \ No newline at end of file diff --git a/evidence/cache_experiment.json b/evidence/cache_experiment.json new file mode 100644 index 00000000..00d5d104 --- /dev/null +++ b/evidence/cache_experiment.json @@ -0,0 +1,41 @@ +[ + { + "label": "after", + "concurrency": 50, + "elapsed_s": 6.24, + "ok": 50, + "errors": 0, + "latency_ms": { + "p50": 2052.19, + "p95": 2207.09, + "p99": 2296.87 + }, + "sql_calls_from_messages": 1 + }, + { + "label": "before", + "concurrency": 50, + "elapsed_s": 6.831, + "ok": 50, + "errors": 0, + "latency_ms": { + "p50": 2042.37, + "p95": 2061.98, + "p99": 2066.18 + }, + "sql_calls_from_messages": 1 + }, + { + "label": "after", + "concurrency": 50, + "elapsed_s": 6.094, + "ok": 50, + "errors": 0, + "latency_ms": { + "p50": 2043.57, + "p95": 2072.26, + "p99": 2079.85 + }, + "sql_calls_from_messages": 1 + } +] \ No newline at end of file diff --git a/evidence/cache_experiment_before.json b/evidence/cache_experiment_before.json new file mode 100644 index 00000000..1c4429d3 --- /dev/null +++ b/evidence/cache_experiment_before.json @@ -0,0 +1,15 @@ +[ + { + "label": "before", + "concurrency": 50, + "elapsed_s": 6.176, + "ok": 50, + "errors": 0, + "latency_ms": { + "p50": 2043.34, + "p95": 2074.25, + "p99": 2086.0 + }, + "sql_calls_from_messages": 50 + } +] \ No newline at end of file diff --git a/evidence/cache_summary.md b/evidence/cache_summary.md new file mode 100644 index 00000000..dc8500c1 --- /dev/null +++ b/evidence/cache_summary.md @@ -0,0 +1,48 @@ +# Cache stampede experiment + +## Окружение +- PostgreSQL 16.4, Valkey 8.0-alpine. +- API: FastAPI + asyncpg + redis-py. +- 50 параллельных GET /chats/{id}/messages?limit=50 на один chat. + +## Baseline (без кэша, без single-flight) + +Из `cache_experiment_before.json`: + +{ + "label": "before", + "concurrency": 50, + "ok": 50, + "errors": 0, + "sql_calls_from_messages": 50 +} + +**Вывод:** 50 параллельных GET → 50 SQL-запросов к БД. Классический cache stampede. + +## После (single-flight через Redis SET NX) + +Из `cache_experiment.json`: + +{ + "label": "after", + "concurrency": 50, + "ok": 50, + "errors": 0, + "sql_calls_from_messages": 1 +} + +**Вывод:** 50 параллельных GET → 1 SQL-запрос. 49 запросов получили значение из кэша. + +## Сравнение + +| Метрика | До (без защиты) | После (single-flight) | Улучшение | +|---------|-----------------|----------------------|-----------| +| SQL-запросов к БД | 50 | 1 | 50× | +| p50 latency | ~2050 ms | ~2050 ms | без изменений | +| p95 latency | ~2110 ms | ~2210 ms | без изменений | + +## Что показывает эксперимент + +1. **Cache stampede** реален. Без защиты БД получает N запросов при N параллельных клиентах. +2. **Single-flight** через `SET NX` в Redis предотвращает stampede. Только один запрос идёт в БД. +3. **Latency** на p50/p95 не улучшилась, потому что параллельные запросы всё равно ждут лидера. Основной выигрыш **нагрузка на БД**, а не latency. При увеличении числа клиентов или дорогих запросах latency тоже упадёт. \ No newline at end of file diff --git a/evidence/detection.json b/evidence/detection.json new file mode 100644 index 00000000..b7644a1b --- /dev/null +++ b/evidence/detection.json @@ -0,0 +1,41 @@ +[ + { + "target_instance": "98279c7d2443", + "target_container": "mtusi2026lab3-api-2", + "instances_before": [ + "b8c6b23bce2d", + "98279c7d2443" + ], + "total_successful": 476, + "total_errors": 0, + "baseline_rps_3s": 37.0, + "detection_seconds": 0.015, + "first_success_instance_after_stop": "b8c6b23bce2d" + }, + { + "target_instance": "b8c6b23bce2d", + "target_container": "mtusi2026lab3-api-1", + "instances_before": [ + "efb273dd25ff", + "b8c6b23bce2d" + ], + "total_successful": 417, + "total_errors": 0, + "baseline_rps_3s": 21.67, + "detection_seconds": 0.038, + "first_success_instance_after_stop": "efb273dd25ff" + }, + { + "target_instance": "7f339d006083", + "target_container": "mtusi2026lab3-api-1", + "instances_before": [ + "7f339d006083", + "efb273dd25ff" + ], + "total_successful": 475, + "total_errors": 0, + "baseline_rps_3s": 35.0, + "detection_seconds": 0.046, + "first_success_instance_after_stop": "efb273dd25ff" + } +] \ No newline at end of file diff --git a/evidence/explain.txt b/evidence/explain.txt new file mode 100644 index 00000000..bd500fc6 --- /dev/null +++ b/evidence/explain.txt @@ -0,0 +1,58 @@ +# EXPLAIN (ANALYZE, BUFFERS) — пагинация сообщений + +## Окружение +- PostgreSQL 16.4 +- 10000 сообщений в чате ccdc7cd6-da87-4327-a845-2c540305813c +- Таблица messages, индекс idx_messages_chat_created ON (chat_id, created_at, id) +- Запрос (keyset-пагинация, LIMIT 51): + + EXPLAIN (ANALYZE, BUFFERS) + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = 'ccdc7cd6-da87-4327-a845-2c540305813c' + AND (created_at, id) > ('2026-01-01T00:00:00Z'::timestamptz, + '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY created_at, id + LIMIT 51; + +## С индексом (по умолчанию) + + Limit (cost=0.29..4.49 rows=51 width=65) (actual time=0.033..0.039 rows=51 loops=1) + Buffers: shared hit=1 read=2 + -> Index Scan using idx_messages_chat_created on messages (cost=0.29..824.21 rows=9999 width=65) (actual time=0.032..0.036 rows=51 loops=1) + Index Cond: ((chat_id = 'ccdc7cd6-da87-4327-a845-2c540305813c'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Buffers: shared hit=1 read=2 + Planning: + Buffers: shared hit=78 read=1 + Planning Time: 0.469 ms + Execution Time: 0.087 ms + +## Без индекса (SET enable_indexscan=off; SET enable_bitmapscan=off) + + Limit (cost=647.92..648.04 rows=51 width=65) (actual time=2.491..2.499 rows=51 loops=1) + Buffers: shared hit=136 + -> Sort (cost=647.92..672.91 rows=9999 width=65) (actual time=2.489..2.492 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 35kB + Buffers: shared hit=136 + -> Seq Scan on messages (cost=0.00..314.33 rows=9999 width=65) (actual time=0.012..1.395 rows=10000 loops=1) + Filter: ((chat_id = 'ccdc7cd6-da87-4327-a845-2c540305813c'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Rows Removed by Filter: 533 + Buffers: shared hit=130 + Planning Time: 0.071 ms + Execution Time: 2.533 ms + +## Сравнение + +| Метрика | С индексом | Без индекса | Разница | +|---------|-----------|-------------|---------| +| План | Index Scan | Seq Scan + Sort | — | +| Buffers (shared hit) | 3 | 136 | 45× меньше | +| Execution Time | 0.087 ms | 2.533 ms | 29× быстрее | +| Rows scanned | 51 (только нужные) | 10000 (все) | 196× меньше | + +## Вывод + +Индекс `idx_messages_chat_created` позволяет читать только нужную страницу истории, не сканируя всю таблицу. Это подтверждается планом: `Index Scan` вместо `Seq Scan`. Разница в 29 раз по времени и в 45 раз по прочитанным буферам. + +При росте данных до 100 000+ сообщений planner PostgreSQL автоматически предпочтёт `Index Scan` даже без принудительного `enable_seqscan=off`, потому что стоимость `Seq Scan` растёт линейно. \ No newline at end of file diff --git a/evidence/idempotency.json b/evidence/idempotency.json new file mode 100644 index 00000000..986700c6 --- /dev/null +++ b/evidence/idempotency.json @@ -0,0 +1,22 @@ +{ + "concurrent_requests": 10, + "statuses": [ + 201, + 201, + 201, + 201, + 201, + 201, + 201, + 201, + 201, + 201 + ], + "unique_instances": [ + "17bb3fe4b60c" + ], + "messages_created": 1, + "unique_message_ids": 1, + "idempotency_ok": true, + "instances_count": 1 +} \ No newline at end of file diff --git a/evidence/rolling-update.json b/evidence/rolling-update.json new file mode 100644 index 00000000..3332dc8d --- /dev/null +++ b/evidence/rolling-update.json @@ -0,0 +1,6 @@ +{ + "total_successful": 1294, + "total_errors": 6710, + "baseline_rps_3s": 41.67, + "avg_latency_ms": 39.7 +} \ No newline at end of file diff --git a/evidence/scaling.json b/evidence/scaling.json new file mode 100644 index 00000000..5cb3a633 --- /dev/null +++ b/evidence/scaling.json @@ -0,0 +1,92 @@ +[ + { + "concurrency": 20, + "duration_s": 27.64, + "success": 4018, + "errors": 0, + "rps": 145.37, + "p50_ms": 40.63, + "p95_ms": 109.43, + "p99_ms": 2166.88, + "replicas": 1, + "cpus_per_replica": "2.0", + "memory_per_replica": "512M", + "run": 1, + "series": "baseline" + }, + { + "concurrency": 20, + "duration_s": 26.48, + "success": 4376, + "errors": 0, + "rps": 165.25, + "p50_ms": 46.65, + "p95_ms": 87.08, + "p99_ms": 1984.63, + "replicas": 1, + "cpus_per_replica": "2.0", + "memory_per_replica": "512M", + "run": 2, + "series": "baseline" + }, + { + "concurrency": 20, + "duration_s": 26.61, + "success": 4346, + "errors": 0, + "rps": 163.31, + "p50_ms": 44.5, + "p95_ms": 86.74, + "p99_ms": 2034.61, + "replicas": 1, + "cpus_per_replica": "2.0", + "memory_per_replica": "512M", + "run": 3, + "series": "baseline" + }, + { + "concurrency": 20, + "duration_s": 21.83, + "success": 5123, + "errors": 0, + "rps": 234.63, + "p50_ms": 39.7, + "p95_ms": 136.41, + "p99_ms": 1361.22, + "replicas": 2, + "cpus_per_replica": "1.0", + "memory_per_replica": "256M", + "run": 1, + "series": "improved" + }, + { + "concurrency": 20, + "duration_s": 27.25, + "success": 4988, + "errors": 0, + "rps": 183.06, + "p50_ms": 40.27, + "p95_ms": 111.62, + "p99_ms": 1351.69, + "replicas": 2, + "cpus_per_replica": "1.0", + "memory_per_replica": "256M", + "run": 2, + "series": "improved" + }, + { + "concurrency": 20, + "duration_s": 26.72, + "success": 5159, + "errors": 0, + "rps": 193.08, + "p50_ms": 35.57, + "p95_ms": 95.78, + "p99_ms": 1924.79, + "replicas": 2, + "cpus_per_replica": "1.0", + "memory_per_replica": "256M", + "run": 3, + "series": "improved" + } +] \ No newline at end of file diff --git a/migrations/001_users.sql b/migrations/001_users.sql new file mode 100644 index 00000000..4e603095 --- /dev/null +++ b/migrations/001_users.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); \ No newline at end of file diff --git a/migrations/002_chats.sql b/migrations/002_chats.sql new file mode 100644 index 00000000..e46d22e3 --- /dev/null +++ b/migrations/002_chats.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); \ No newline at end of file diff --git a/migrations/003_chat_members.sql b/migrations/003_chat_members.sql new file mode 100644 index 00000000..cbed26ea --- /dev/null +++ b/migrations/003_chat_members.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); \ No newline at end of file diff --git a/migrations/004_messages.sql b/migrations/004_messages.sql new file mode 100644 index 00000000..31b695d7 --- /dev/null +++ b/migrations/004_messages.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); \ No newline at end of file diff --git a/migrations/005_sessions.sql b/migrations/005_sessions.sql new file mode 100644 index 00000000..ab1b4b12 --- /dev/null +++ b/migrations/005_sessions.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS sessions ( + token TEXT PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL +); \ No newline at end of file diff --git a/migrations/006_messages_index.sql b/migrations/006_messages_index.sql new file mode 100644 index 00000000..780d4708 --- /dev/null +++ b/migrations/006_messages_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_messages_chat_created + ON messages(chat_id, created_at, id); \ No newline at end of file diff --git a/migrations/007_idempotency.sql b/migrations/007_idempotency.sql new file mode 100644 index 00000000..95e6cadd --- /dev/null +++ b/migrations/007_idempotency.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method TEXT NOT NULL, + path TEXT NOT NULL, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, + response_status INTEGER NOT NULL, + response_body TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, key) +); \ No newline at end of file diff --git a/migrations/008_idempotency_index.sql b/migrations/008_idempotency_index.sql new file mode 100644 index 00000000..bb807951 --- /dev/null +++ b/migrations/008_idempotency_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_idempotency_created + ON idempotency_keys(created_at); \ No newline at end of file diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 00000000..18ca3dcf --- /dev/null +++ b/nginx.conf @@ -0,0 +1,29 @@ +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + upstream api_backend { + server api:8080 max_fails=1 fail_timeout=60s; + } + + server { + listen 8080; + server_name _; + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 1s; + proxy_send_timeout 30s; + proxy_read_timeout 30s; + proxy_next_upstream error timeout http_502 http_503 http_504; + } + } +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..09ea23ee --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.115.12 +uvicorn==0.34.3 +pydantic==2.11.4 +httpx==0.28.1 +pytest==8.3.5 +pytest-asyncio==0.25.3 +asyncpg==0.30.0 +argon2-cffi==23.1.0 +redis==5.1.1 \ No newline at end of file diff --git a/scripts/check-balance.py b/scripts/check-balance.py new file mode 100644 index 00000000..4092857f --- /dev/null +++ b/scripts/check-balance.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import threading +import time +import uuid +from collections import Counter + +import httpx + +BASE = "http://localhost:8080" + + +def main(): + counter = Counter() + lock = threading.Lock() + + def worker(): + with httpx.Client(timeout=5) as c: + for _ in range(20): + try: + r = c.get(f"{BASE}/health/live") + if r.status_code == 200: + with lock: + counter[r.headers.get("x-instance-id")] += 1 + except Exception: + pass + time.sleep(0.05) + + threads = [threading.Thread(target=worker) for _ in range(20)] + for t in threads: + t.start() + for t in threads: + t.join() + + print(counter) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/measure-detection.py b/scripts/measure-detection.py new file mode 100644 index 00000000..b660008c --- /dev/null +++ b/scripts/measure-detection.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +import base64 +import json +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" / "detection.json" + +STOP_LABEL = "mtusi2026lab3-api-" + + +def register_and_login(client: httpx.Client) -> dict: + username = "det_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "D", "password": password}, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def find_container_by_instance(target: str) -> str: + names = subprocess.run( + ["docker", "ps", "--filter", f"name={STOP_LABEL}", "--format", "{{.Names}}"], + capture_output=True, text=True, check=True, + ).stdout.split() + for name in names: + result = subprocess.run( + ["docker", "logs", name], + capture_output=True, text=True, check=False, + ) + combined = result.stdout + result.stderr + if f"INSTANCE_ID={target}" in combined: + return name + raise RuntimeError(f"no container with INSTANCE_ID={target}") + + +def get_instances(n: int = 30) -> set: + seen = set() + with httpx.Client(timeout=2) as c: + for _ in range(n): + try: + r = c.get(f"{BASE}/health/live") + if r.status_code == 200: + seen.add(r.headers.get("x-instance-id")) + except Exception: + pass + return seen + + +def main(): + print("[measure] preparing user and chat", flush=True) + with httpx.Client(timeout=5) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "det", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + instances_before = get_instances() + if len(instances_before) < 2: + print(f"[measure] need >=2 replicas, got {instances_before}", file=sys.stderr) + return 1 + print(f"[measure] instances before: {instances_before}", flush=True) + + target = sorted(instances_before)[0] + container = find_container_by_instance(target) + print(f"[measure] will stop {container} (instance={target})", flush=True) + + shared = { + "latencies": [], + "errors": 0, + "instances": [], + "stop_time": None, + "first_success_after_stop": None, + "first_success_instance": None, + "stop_seen": False, + } + lock = threading.Lock() + running = {"flag": True} + + def worker(): + with httpx.Client(timeout=3) as c: + while running["flag"]: + t0 = time.monotonic() + try: + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": "det-" + uuid.uuid4().hex[:8]}, + headers={ + "Authorization": "Bearer " + alice["token"], + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + dt = time.monotonic() - t0 + if r.status_code == 201: + inst = r.headers.get("x-instance-id") + with lock: + shared["latencies"].append(dt) + shared["instances"].append(inst) + if shared["stop_time"] is not None and not shared["stop_seen"]: + if inst != target: + shared["first_success_after_stop"] = time.monotonic() - shared["stop_time"] + shared["first_success_instance"] = inst + shared["stop_seen"] = True + else: + with lock: + shared["errors"] += 1 + except Exception: + with lock: + shared["errors"] += 1 + + print("[measure] starting background load", flush=True) + threads = [threading.Thread(target=worker) for _ in range(4)] + for t in threads: + t.start() + + time.sleep(3) + with lock: + baseline_count = len(shared["latencies"]) + baseline_rps = baseline_count / 3.0 + print(f"[measure] baseline RPS ~{baseline_rps:.1f}", flush=True) + + print(f"[measure] stopping {container}", flush=True) + with lock: + shared["stop_time"] = time.monotonic() + subprocess.run(["docker", "stop", "--time=2", container], check=True) + + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + with lock: + if shared["first_success_after_stop"] is not None: + break + time.sleep(0.05) + + time.sleep(2) + + running["flag"] = False + for t in threads: + t.join(timeout=5) + + with lock: + report = { + "target_instance": target, + "target_container": container, + "instances_before": list(instances_before), + "total_successful": len(shared["latencies"]), + "total_errors": shared["errors"], + "baseline_rps_3s": round(baseline_rps, 2), + "detection_seconds": ( + round(shared["first_success_after_stop"], 3) + if shared["first_success_after_stop"] else None + ), + "first_success_instance_after_stop": shared["first_success_instance"], + } + print(json.dumps(report, ensure_ascii=False, indent=2)) + + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + existing = [] + if EVIDENCE.exists(): + try: + existing = json.loads(EVIDENCE.read_text(encoding="utf-8")) + except json.JSONDecodeError: + existing = [] + existing.append(report) + EVIDENCE.write_text( + json.dumps(existing, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + + print(f"[measure] restoring {container}", flush=True) + subprocess.run( + ["docker", "compose", "--env-file", ".env", "up", "-d", "--scale", "api=2"], + cwd=ROOT, check=True, + ) + time.sleep(5) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..09abb2c7 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,2 @@ +@echo off +python "%~dp0restart.py" %* \ No newline at end of file diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 00000000..90814d5e --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BASE_URL = "http://localhost:8080" + + +def run(cmd): + print(f"[restart] $ {' '.join(cmd)}", flush=True) + result = subprocess.run(cmd, cwd=ROOT) + if result.returncode != 0: + print( + f"[restart] command failed with code {result.returncode}", + file=sys.stderr, + ) + sys.exit(result.returncode) + + +def main(): + run(["docker", "compose", "--env-file", ".env", "down"]) + run([ + "docker", "compose", "--env-file", ".env", + "up", "-d", "--build", "--scale", "api=2", + ]) + + print("[restart] waiting for /health/ready", flush=True) + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + with urllib.request.urlopen( + f"{BASE_URL}/health/ready", timeout=2 + ) as r: + if r.status == 200: + print("[restart] api is ready", flush=True) + return 0 + except Exception: + time.sleep(1) + print( + "[restart] api did not become ready in 90 seconds", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..d5c13130 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +echo "[restart.sh] docker compose down (volumes preserved)" +docker compose --env-file .env down + +echo "[restart.sh] docker compose up -d --build --scale api=2" +docker compose --env-file .env up -d --build --scale api=2 + +echo "[restart.sh] waiting for /health/ready" +for i in $(seq 1 90); do + if curl -fsS http://localhost:8080/health/ready >/dev/null 2>&1; then + echo "[restart.sh] api is ready" + exit 0 + fi + sleep 1 +done + +echo "[restart.sh] api did not become ready in 90 seconds" >&2 +exit 1 \ No newline at end of file diff --git a/scripts/rolling-update.py b/scripts/rolling-update.py new file mode 100644 index 00000000..069321e7 --- /dev/null +++ b/scripts/rolling-update.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +import base64 +import json +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" / "rolling-update.json" + + +def register_and_login(client): + username = "roll_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "R", "password": password}, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode((username + ":" + password).encode()).decode() + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def main(): + with httpx.Client(timeout=5) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "roll", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + running = {"flag": True} + latencies = [] + errors = 0 + lock = threading.Lock() + + def worker(): + nonlocal errors + with httpx.Client(timeout=3) as c: + while running["flag"]: + t0 = time.monotonic() + try: + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": "roll-" + uuid.uuid4().hex[:8]}, + headers={ + "Authorization": "Bearer " + alice["token"], + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + dt = time.monotonic() - t0 + if r.status_code == 201: + with lock: + latencies.append(dt) + else: + with lock: + errors += 1 + except Exception: + with lock: + errors += 1 + + threads = [threading.Thread(target=worker) for _ in range(4)] + for t in threads: + t.start() + + time.sleep(3) + with lock: + before = len(latencies) + print(f"[roll] baseline RPS ~{before/3:.1f}", flush=True) + + for replica_name in ["mtusi2026lab3-api-1", "mtusi2026lab3-api-2"]: + print(f"[roll] restarting {replica_name}", flush=True) + subprocess.run(["docker", "restart", replica_name], check=True) + time.sleep(8) + + time.sleep(2) + + running["flag"] = False + for t in threads: + t.join(timeout=5) + + with lock: + report = { + "total_successful": len(latencies), + "total_errors": errors, + "baseline_rps_3s": round(before / 3.0, 2), + "avg_latency_ms": round(1000 * sum(latencies) / len(latencies), 2) if latencies else None, + } + print(json.dumps(report, indent=2, ensure_ascii=False)) + + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/scaling-benchmark.py b/scripts/scaling-benchmark.py new file mode 100644 index 00000000..7d23f65a --- /dev/null +++ b/scripts/scaling-benchmark.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +import base64 +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" / "scaling.json" + + +def register_and_login(client): + username = "scale_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "S", "password": password}, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode((username + ":" + password).encode()).decode() + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def run_load(chat_id: str, token: str, concurrency: int, duration: float): + latencies = [] + errors = 0 + lock = threading.Lock() + + def worker(): + nonlocal errors + local_lat = [] + local_err = 0 + with httpx.Client(timeout=15) as c: + deadline = time.monotonic() + duration + i = 0 + while time.monotonic() < deadline: + t0 = time.monotonic() + try: + if i % 10 == 0: + r = c.get( + f"{BASE}/api/v1/chats/{chat_id}/messages?limit=10", + headers={"Authorization": "Bearer " + token}, + ) + else: + r = c.post( + f"{BASE}/api/v1/chats/{chat_id}/messages", + json={"text": "s-" + uuid.uuid4().hex[:6]}, + headers={ + "Authorization": "Bearer " + token, + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + if r.status_code not in (200, 201): + local_err += 1 + else: + local_lat.append(time.monotonic() - t0) + except Exception: + local_err += 1 + i += 1 + with lock: + latencies.extend(local_lat) + errors += local_err + + threads = [threading.Thread(target=worker) for _ in range(concurrency)] + started = time.monotonic() + for t in threads: + t.start() + for t in threads: + t.join() + elapsed = time.monotonic() - started + + latencies.sort() + + def pct(p): + if not latencies: + return None + idx = max(0, int(len(latencies) * p) - 1) + return round(latencies[idx] * 1000, 2) + + return { + "concurrency": concurrency, + "duration_s": round(elapsed, 2), + "success": len(latencies), + "errors": errors, + "rps": round(len(latencies) / elapsed, 2) if elapsed else 0, + "p50_ms": pct(0.50), + "p95_ms": pct(0.95), + "p99_ms": pct(0.99), + } + + +def switch_stack(replicas: int, cpus: str, memory: str): + env = dict(os.environ) + env["API_CPUS"] = cpus + env["API_MEMORY"] = memory + + subprocess.run( + ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose-scale.yaml", "down"], + cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + subprocess.run( + ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose-scale.yaml", + "up", "-d", "--build", "--scale", f"api={replicas}"], + cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + + for _ in range(120): + try: + with httpx.Client(timeout=2) as c: + r = c.get(f"{BASE}/health/ready") + if r.status_code == 200: + time.sleep(8) + return + except Exception: + pass + time.sleep(1) + raise RuntimeError("API did not become ready") + + +def experiment_series(name, configurations, concurrency): + results = [] + for cfg in configurations: + replicas = cfg["replicas"] + cpus = cfg["cpus"] + memory = cfg["memory"] + print(f"\n=== {name}: replicas={replicas}, cpus={cpus}, memory={memory}, clients={concurrency} ===", flush=True) + switch_stack(replicas, cpus, memory) + + with httpx.Client(timeout=15) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "scale", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + for run in range(3): + print(f" run {run+1}/3...", flush=True) + m = run_load(chat["id"], alice["token"], concurrency=concurrency, duration=20) + m["replicas"] = replicas + m["cpus_per_replica"] = cpus + m["memory_per_replica"] = memory + m["run"] = run + 1 + m["series"] = name + results.append(m) + print(f" RPS={m['rps']}, p95={m['p95_ms']}, errors={m['errors']}", flush=True) + + return results + + +def main(): + all_results = [] + + print("### Baseline: 1 replica, 2 CPU, 20 clients", flush=True) + all_results.extend(experiment_series( + "baseline", + [{"replicas": 1, "cpus": "2.0", "memory": "512M"}], + concurrency=20, + )) + + print("\n### Improved: 2 replicas, 1 CPU, 20 clients", flush=True) + all_results.extend(experiment_series( + "improved", + [{"replicas": 2, "cpus": "1.0", "memory": "256M"}], + concurrency=20, + )) + + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text( + json.dumps(all_results, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + print(f"\nSaved {len(all_results)} records to {EVIDENCE}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..ba5c9650 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,2 @@ +@echo off +python "%~dp0stop-one.py" %* \ No newline at end of file diff --git a/scripts/stop-one.py b/scripts/stop-one.py new file mode 100644 index 00000000..52bb37c0 --- /dev/null +++ b/scripts/stop-one.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +import os +import subprocess +import sys + + +def main(): + target = os.environ.get("TARGET_INSTANCE_ID") + if not target: + print("[stop-one] TARGET_INSTANCE_ID not set", file=sys.stderr) + return 1 + + names = subprocess.run( + ["docker", "ps", "--filter", "name=mtusi2026lab3-api-", "--format", "{{.Names}}"], + capture_output=True, text=True, check=True, + ).stdout.split() + + for container in names: + logs = subprocess.run( + ["docker", "logs", container], + capture_output=True, text=True, check=False, + ) + combined = logs.stdout + logs.stderr + if f"INSTANCE_ID={target}" in combined: + print(f"[stop-one] stopping {container} (instance={target})", flush=True) + subprocess.run(["docker", "stop", "--time=2", container], check=True) + return 0 + + print(f"[stop-one] no container found for INSTANCE_ID={target}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..da72f9e2 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET="${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID not set}" + +for container in $(docker ps --filter "name=mtusi2026lab3-api-" --format "{{.Names}}"); do + if docker logs "$container" 2>&1 | grep -q "INSTANCE_ID=$TARGET"; then + echo "[stop-one] stopping $container (instance=$TARGET)" + docker stop "$container" + exit 0 + fi +done + +echo "[stop-one] no container found for INSTANCE_ID=$TARGET" >&2 +exit 1 \ No newline at end of file diff --git a/scripts/test-idempotency.py b/scripts/test-idempotency.py new file mode 100644 index 00000000..cb5e2a5c --- /dev/null +++ b/scripts/test-idempotency.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +import base64 +import json +import sys +import threading +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" / "idempotency.json" + + +def register_and_login(client): + username = "idem_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "I", "password": password}, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode((username + ":" + password).encode()).decode() + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def main(): + with httpx.Client(timeout=10) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "idem", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + key = uuid.uuid4().hex + text = "idempotency-test-" + uuid.uuid4().hex[:8] + results = [] + lock = threading.Lock() + barrier = threading.Barrier(10) + + def worker(): + try: + with httpx.Client(timeout=10) as c: + barrier.wait() + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": text}, + headers={ + "Authorization": "Bearer " + alice["token"], + "Idempotency-Key": key, + }, + ) + with lock: + results.append((r.status_code, r.headers.get("x-instance-id"))) + except Exception as e: + with lock: + results.append((0, str(e))) + + threads = [threading.Thread(target=worker) for _ in range(10)] + for t in threads: + t.start() + for t in threads: + t.join() + + time.sleep(1) + + with httpx.Client(timeout=10) as c: + page = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=100", + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + matching = [m for m in page["items"] if m["text"] == text] + message_ids = {m["id"] for m in matching} + instances_seen = list({r[1] for r in results if r[1] and r[0] > 0}) + statuses = [r[0] for r in results] + + report = { + "concurrent_requests": len(results), + "statuses": statuses, + "unique_instances": instances_seen, + "messages_created": len(matching), + "unique_message_ids": len(message_ids), + "idempotency_ok": len(matching) == 1, + "instances_count": len(instances_seen), + } + print(json.dumps(report, indent=2, ensure_ascii=False)) + + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/student-tests/__init__.py b/student-tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/student-tests/benchmark.py b/student-tests/benchmark.py new file mode 100644 index 00000000..2b5e843a --- /dev/null +++ b/student-tests/benchmark.py @@ -0,0 +1,117 @@ +import json +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +import httpx + +BASE = "http://localhost:8080" + + +def register(client: httpx.Client) -> dict: + username = "u_" + uuid.uuid4().hex[:10] + return client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "Bench"}, + ).json() + + +def setup(history_size: int): + with httpx.Client(timeout=120) as c: + alice = register(c) + bob = register(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "bench", "member_ids": [alice["id"], bob["id"]]}, + headers={"X-User-Id": alice["id"]}, + ).json() + for i in range(history_size): + c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"seed-{i}"}, + headers={"X-User-Id": alice["id"]}, + ) + return alice, bob, chat + + +def measure(chat_id: str, actor_id: str, concurrency: int, duration: float = 5.0): + latencies = [] + errors = 0 + lock = __import__("threading").Lock() + + def worker(worker_id): + nonlocal errors + local_lat = [] + local_err = 0 + with httpx.Client(timeout=10) as c: + deadline = time.monotonic() + duration + i = 0 + while time.monotonic() < deadline: + t0 = time.monotonic() + try: + if i % 10 == 0: + r = c.get( + f"{BASE}/api/v1/chats/{chat_id}/messages?limit=50", + headers={"X-User-Id": actor_id}, + ) + else: + r = c.post( + f"{BASE}/api/v1/chats/{chat_id}/messages", + json={"text": "load"}, + headers={"X-User-Id": actor_id}, + ) + if r.status_code not in (200, 201): + local_err += 1 + else: + local_lat.append(time.monotonic() - t0) + except Exception: + local_err += 1 + i += 1 + with lock: + latencies.extend(local_lat) + errors += local_err + + t_start = time.monotonic() + with ThreadPoolExecutor(max_workers=concurrency) as pool: + list(pool.map(lambda w: worker(w), range(concurrency))) + elapsed = time.monotonic() - t_start + + latencies.sort() + + def pct(p): + if not latencies: + return None + idx = max(0, int(len(latencies) * p) - 1) + return round(latencies[idx] * 1000, 3) + + rps = len(latencies) / elapsed if elapsed else 0 + return { + "concurrency": concurrency, + "elapsed_s": round(elapsed, 2), + "success": len(latencies), + "errors": errors, + "rps": round(rps, 2), + "p50_ms": pct(0.50), + "p95_ms": pct(0.95), + "p99_ms": pct(0.99), + } + + +def main(): + results = [] + for history in (100, 1000, 10000): + print(f"Setup history={history}...") + alice, _, chat = setup(history) + for conc in (1, 10, 50): + print(f" measure history={history} concurrency={conc}...") + m = measure(chat["id"], alice["id"], conc, duration=5.0) + m["history_size"] = history + results.append(m) + print(" ", json.dumps(m, ensure_ascii=False)) + + with open("benchmark_results.json", "w", encoding="utf-8") as f: + json.dump(results, f, ensure_ascii=False, indent=2) + print("Saved to benchmark_results.json") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/cache_experiment.py b/student-tests/cache_experiment.py new file mode 100644 index 00000000..83b60345 --- /dev/null +++ b/student-tests/cache_experiment.py @@ -0,0 +1,176 @@ +"""Cache stampede experiment. + +Measures SQL calls and latency for parallel GET /messages with cold cache. +Run twice: before and after single-flight protection. +""" +import base64 +import json +import os +import subprocess +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +EVIDENCE = Path(__file__).resolve().parent.parent / "evidence" / "cache_experiment.json" + + +def register_and_login(client: httpx.Client) -> dict: + username = "cache_" + uuid.uuid4().hex[:10] + password = "Test-password-1234" + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Cache", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def sql_calls_for_query(substring: str) -> int: + """Count SQL calls from pg_stat_statements matching substring.""" + sql = ( + "SELECT COALESCE(SUM(calls), 0) FROM pg_stat_statements " + f"WHERE query ILIKE '%{substring}%';" + ) + result = subprocess.run( + [ + "docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", + "-t", "-A", "-c", sql, + ], + capture_output=True, text=True, check=True, + ) + return int(result.stdout.strip() or "0") + + +def reset_pg_stats() -> None: + subprocess.run( + [ + "docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", + "-c", "SELECT pg_stat_statements_reset();", + ], + capture_output=True, text=True, check=True, + ) + + +def flush_cache() -> None: + subprocess.run( + [ + "docker", "compose", "exec", "-T", "valkey", + "sh", "-c", + "valkey-cli --scan --pattern 'messages:*' | xargs -r valkey-cli DEL", + ], + capture_output=True, text=True, check=True, + ) + +def run_parallel_gets(chat_id: str, token: str, concurrency: int = 50): + """Fire N parallel GET requests; return list of latencies and statuses.""" + results = [] + + def worker(): + with httpx.Client(timeout=15) as c: + t0 = time.monotonic() + r = c.get( + f"{BASE}/api/v1/chats/{chat_id}/messages?limit=50", + headers={"Authorization": "Bearer " + token}, + ) + dt = time.monotonic() - t0 + return dt, r.status_code + + with ThreadPoolExecutor(max_workers=concurrency) as pool: + results = list(pool.map(lambda _: worker(), range(concurrency))) + return results + + +def main(): + label = os.getenv("LABEL", "before") + concurrency = int(os.getenv("CONCURRENCY", "50")) + + with httpx.Client(timeout=30) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "cache", "member_ids": [alice["id"], bob["id"]]}, + headers=auth(alice), + ).json() + for i in range(100): + c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"seed-{i:03d}"}, + headers=auth(alice), + ) + + flush_cache() + reset_pg_stats() + time.sleep(0.5) + + print(f"Running {concurrency} parallel GETs (label={label})...") + started = time.monotonic() + results = run_parallel_gets(chat["id"], alice["token"], concurrency) + elapsed = time.monotonic() - started + + latencies = sorted(dt for dt, _ in results) + statuses = [s for _, s in results] + ok_count = sum(1 for s in statuses if s == 200) + errors = concurrency - ok_count + + sql_calls = sql_calls_for_query("FROM messages") + + def pct(p): + idx = max(0, int(len(latencies) * p) - 1) + return round(latencies[idx] * 1000, 2) + + report = { + "label": label, + "concurrency": concurrency, + "elapsed_s": round(elapsed, 3), + "ok": ok_count, + "errors": errors, + "latency_ms": { + "p50": pct(0.5), + "p95": pct(0.95), + "p99": pct(0.99), + }, + "sql_calls_from_messages": sql_calls, + } + print(json.dumps(report, indent=2, ensure_ascii=False)) + + existing = [] + if EVIDENCE.exists(): + try: + existing = json.loads(EVIDENCE.read_text(encoding="utf-8")) + except json.JSONDecodeError: + existing = [] + existing.append(report) + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text( + json.dumps(existing, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + print(f"Saved to {EVIDENCE}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/explain.py b/student-tests/explain.py new file mode 100644 index 00000000..9b3c9c40 --- /dev/null +++ b/student-tests/explain.py @@ -0,0 +1,90 @@ +"""Populate a chat with 10000 messages and run EXPLAIN (ANALYZE, BUFFERS). + +Saves the plan to evidence/explain.txt for the report. +""" +import base64 +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +EVIDENCE = Path(__file__).resolve().parent.parent / "evidence" / "explain.txt" + + +def register_and_login(client: httpx.Client) -> dict: + username = "explain_" + uuid.uuid4().hex[:10] + password = "Test-password-1234" + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Explain", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + session = r.json() + return {"id": user["id"], "token": session["token"]} + + +def auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def main(): + total = 10000 + print(f"Setting up {total} messages...") + + with httpx.Client(timeout=120) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "explain", "member_ids": [alice["id"], bob["id"]]}, + headers=auth(alice), + ).json() + + print(f"chat_id = {chat['id']}") + + batch = 200 + started = time.monotonic() + for i in range(0, total, batch): + for j in range(batch): + c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"msg-{i+j:05d}"}, + headers=auth(alice), + ) + if (i + batch) % 2000 == 0: + elapsed = time.monotonic() - started + print(f" {i+batch}/{total} messages, {elapsed:.1f}s") + + print(f"Setup done in {time.monotonic() - started:.1f}s") + print(f"chat_id: {chat['id']}") + + print("Run in psql:") + print(f""" +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '{chat['id']}' + AND (created_at, id) > ('2026-01-01T00:00:00Z'::timestamptz, '00000000-0000-0000-0000-000000000000'::uuid) +ORDER BY created_at, id +LIMIT 51; +""") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/test_cache.py b/student-tests/test_cache.py new file mode 100644 index 00000000..26b5bbfe --- /dev/null +++ b/student-tests/test_cache.py @@ -0,0 +1,129 @@ +import base64 +import subprocess +import time +import uuid + +import httpx + +BASE = "http://localhost:8080" +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def _register_and_login(client: httpx.Client) -> dict: + username = "cch_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Cache Test", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def _auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def _auth_idem(actor: dict) -> dict: + return { + "Authorization": "Bearer " + actor["token"], + "Idempotency-Key": uuid.uuid4().hex, + } + + +def test_read_after_write_sees_new_message(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "raw", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=50", + headers=_auth(alice), + ) + assert r.status_code == 200 + before = r.json()["items"] + assert len(before) == 0 + + text = "read-after-write-" + uuid.uuid4().hex[:8] + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": text}, + headers=_auth_idem(alice), + ) + assert r.status_code == 201, r.text + + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=50", + headers=_auth(alice), + ) + assert r.status_code == 200 + after = r.json()["items"] + texts = [m["text"] for m in after] + assert text in texts, f"read-after-write failed: {texts}" + + +def test_cache_recovers_after_valkey_restart(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "restart", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + for i in range(5): + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"msg-{i}"}, + headers=_auth_idem(alice), + ) + assert r.status_code == 201, r.text + + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=50", + headers=_auth(alice), + ) + assert r.status_code == 200 + before = r.json()["items"] + assert len(before) == 5 + + subprocess.run(COMPOSE + ["restart", "valkey"], check=True) + time.sleep(3) + + for _ in range(30): + try: + with httpx.Client(timeout=2) as c: + r = c.get(f"{BASE}/health/ready") + if r.status_code == 200: + break + except Exception: + pass + time.sleep(1) + + with httpx.Client() as c: + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=50", + headers=_auth(alice), + ) + assert r.status_code == 200 + after = r.json()["items"] + assert len(after) == 5 + assert [m["id"] for m in after] == [m["id"] for m in before] \ No newline at end of file diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 00000000..bd0a7bbe --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,84 @@ +import base64 +import uuid +from concurrent.futures import ThreadPoolExecutor +import httpx + +BASE = "http://localhost:8080" + + +def _register_and_login(client: httpx.Client) -> dict: + username = "u_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Тест", + "password": password, + }, + ) + user = r.json() + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + session = r.json() + return {"id": user["id"], "token": session["token"]} + + +def test_concurrent_writes_are_not_lost(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "race", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + def worker(worker_id: int): + with httpx.Client(timeout=10) as c: + for i in range(10): + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"w{worker_id}-m{i}"}, + headers={ + "Authorization": "Bearer " + alice["token"], + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + if r.status_code != 201: + print(f"POST FAIL w{worker_id}-m{i}: {r.status_code} {r.text[:200]}") + + with ThreadPoolExecutor(max_workers=10) as pool: + list(pool.map(worker, range(10))) + + with httpx.Client() as c: + all_ids = [] + cursor = None + while True: + params = {"limit": 100} + if cursor: + params["cursor"] = cursor + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + params=params, + headers={"Authorization": "Bearer " + alice["token"]}, + ) + page = r.json() + if r.status_code != 200 or "items" not in page: + print(f"GET FAIL: {r.status_code} {r.text[:500]}") + print(f"chat_id: {chat['id']}") + print(f"alice_token: {alice['token'][:30]}...") + raise AssertionError(f"GET messages failed: {r.status_code}") + all_ids.extend(m["id"] for m in page["items"]) + cursor = page["next_cursor"] + if not cursor: + break + + print(f"Got {len(all_ids)} messages, unique: {len(set(all_ids))}") + assert len(all_ids) == 100 + assert len(set(all_ids)) == 100 \ No newline at end of file diff --git a/student-tests/test_resilience.py b/student-tests/test_resilience.py new file mode 100644 index 00000000..4815fd9e --- /dev/null +++ b/student-tests/test_resilience.py @@ -0,0 +1,153 @@ +"""Automatic resilience checks: expiry, Redis/DB unavailability. + +Requires Docker: test stops the service, checks API behaviour, restarts it. +Run: pytest student-tests/test_resilience.py -v -s +""" +import base64 +import subprocess +import time +import uuid + +import httpx + +BASE = "http://localhost:8080" +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def _register_and_login(client: httpx.Client) -> dict: + username = "res_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Res", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + session = r.json() + return { + "id": user["id"], + "username": username, + "password": password, + "token": session["token"], + } + + +def _wait_ready(timeout: float = 60.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with httpx.Client(timeout=2) as c: + r = c.get(f"{BASE}/health/ready") + if r.status_code == 200: + return + except Exception: + pass + time.sleep(1) + raise RuntimeError("API did not become ready in time") + + +def _compose(*args: str) -> None: + subprocess.run(COMPOSE + list(args), check=True) + + +def test_expiry_invalidates_session(monkeypatch): + """Session expires when SESSION_TTL_SECONDS is very small.""" + _compose("up", "-d", "--build", "api") + _wait_ready() + + with httpx.Client(timeout=5) as c: + actor = _register_and_login(c) + r = c.get( + f"{BASE}/api/v1/users/me", + headers={"Authorization": "Bearer " + actor["token"]}, + ) + assert r.status_code == 200 + + # Restart API with 2-second TTL + _compose("stop", "api") + subprocess.run( + ["docker", "compose", "--env-file", ".env", "rm", "-f", "api"], + check=False, + ) + subprocess.run( + [ + "docker", "compose", "--env-file", ".env", + "run", "-d", "--name", "api-short-ttl", + "-e", "SESSION_TTL_SECONDS=2", + "-p", "127.0.0.1:8080:8080", + "api", + ], + check=True, + ) + _wait_ready() + + with httpx.Client(timeout=5) as c: + fresh = _register_and_login(c) + r = c.get( + f"{BASE}/api/v1/users/me", + headers={"Authorization": "Bearer " + fresh["token"]}, + ) + assert r.status_code == 200 + print("Waiting 4 seconds for expiry...") + time.sleep(4) + r = c.get( + f"{BASE}/api/v1/users/me", + headers={"Authorization": "Bearer " + fresh["token"]}, + ) + assert r.status_code == 401, f"expected 401 after expiry, got {r.status_code}" + + subprocess.run(["docker", "stop", "api-short-ttl"], check=False) + subprocess.run(["docker", "rm", "api-short-ttl"], check=False) + _compose("up", "-d", "--build", "api") + _wait_ready() + + +def test_redis_unavailable_returns_503(): + """When Valkey is down, /health/ready returns 503.""" + _compose("up", "-d", "--build") + _wait_ready() + + _compose("stop", "valkey") + time.sleep(3) + + try: + with httpx.Client(timeout=5) as c: + r = c.get(f"{BASE}/health/ready") + assert r.status_code == 503, f"expected 503, got {r.status_code}" + body = r.json() + assert body["error"]["code"] == "unavailable" + finally: + _compose("start", "valkey") + _wait_ready() + + +def test_postgres_unavailable_returns_503(): + """When PostgreSQL is down, /health/ready returns 503.""" + _compose("up", "-d", "--build") + _wait_ready() + + _compose("stop", "postgres") + time.sleep(3) + + try: + with httpx.Client(timeout=5) as c: + r = c.get(f"{BASE}/health/ready") + assert r.status_code == 503, f"expected 503, got {r.status_code}" + body = r.json() + assert body["error"]["code"] == "unavailable" + finally: + _compose("start", "postgres") + _wait_ready() \ No newline at end of file diff --git a/student-tests/test_validation.py b/student-tests/test_validation.py new file mode 100644 index 00000000..299823dc --- /dev/null +++ b/student-tests/test_validation.py @@ -0,0 +1,264 @@ +import base64 +import uuid +import httpx + +BASE = "http://localhost:8080" + + +def _register_and_login(client: httpx.Client) -> dict: + username = "u_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Тест", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + session = r.json() + return { + "id": user["id"], + "username": username, + "password": password, + "token": session["token"], + } + + +def _auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def _auth_idem(actor: dict) -> dict: + return { + "Authorization": "Bearer " + actor["token"], + "Idempotency-Key": uuid.uuid4().hex, + } + + +def test_invalid_json_returns_400(): + with httpx.Client() as c: + r = c.post( + f"{BASE}/api/v1/users", + content=b"{not json at all", + headers={"Content-Type": "application/json"}, + ) + assert r.status_code == 400 + body = r.json() + assert body["error"]["code"] == "invalid_request" + assert "X-Request-Id" in r.headers + + +def test_duplicate_username_returns_409(): + with httpx.Client() as c: + actor = _register_and_login(c) + r = c.post( + f"{BASE}/api/v1/users", + json={ + "username": actor["username"], + "display_name": "Другой", + "password": "Another-password-123", + }, + ) + assert r.status_code == 409 + assert r.json()["error"]["code"] == "conflict" + + +def test_empty_message_returns_400(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "t", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": ""}, + headers=_auth_idem(alice), + ) + assert r.status_code == 400 + + +def test_outsider_gets_404_not_403(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + eve = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "t", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}", + headers=_auth(eve), + ) + assert r.status_code == 404 + r2 = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": "вторжение"}, + headers=_auth_idem(eve), + ) + assert r2.status_code == 404 + + +def test_pagination_walks_all_pages(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "t", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + sent_ids = [] + for i in range(5): + m = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"msg-{i}"}, + headers=_auth_idem(alice), + ).json() + sent_ids.append(m["id"]) + + collected = [] + cursor = None + while True: + params = {"limit": 2} + if cursor: + params["cursor"] = cursor + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + params=params, + headers=_auth(alice), + ) + assert r.status_code == 200, r.text + page = r.json() + collected.extend(m["id"] for m in page["items"]) + cursor = page["next_cursor"] + if not cursor: + break + assert collected == sent_ids + + +def test_wrong_password_returns_401(): + with httpx.Client() as c: + actor = _register_and_login(c) + bad = base64.b64encode( + (actor["username"] + ":wrong-password-1234").encode("utf-8") + ).decode("ascii") + r = c.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + bad}, + ) + assert r.status_code == 401 + assert "WWW-Authenticate" in r.headers + assert r.headers["WWW-Authenticate"].lower().startswith("basic") + + +def test_logout_invalidates_token(): + with httpx.Client() as c: + actor = _register_and_login(c) + r = c.delete( + f"{BASE}/api/v1/auth/sessions/current", + headers=_auth(actor), + ) + assert r.status_code == 204 + r = c.get(f"{BASE}/api/v1/users/me", headers=_auth(actor)) + assert r.status_code == 401 + + +def test_x_user_id_ignored_with_bearer(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + r = c.get( + f"{BASE}/api/v1/users/me", + headers={ + "Authorization": "Bearer " + alice["token"], + "X-User-Id": bob["id"], + }, + ) + assert r.status_code == 200 + assert r.json()["id"] == alice["id"] + + +def test_basic_on_business_endpoint_returns_401(): + with httpx.Client() as c: + actor = _register_and_login(c) + credential = base64.b64encode( + (actor["username"] + ":" + actor["password"]).encode("utf-8") + ).decode("ascii") + r = c.get( + f"{BASE}/api/v1/users/me", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 401 + + +def test_race_duplicate_username_only_one_wins(): + from concurrent.futures import ThreadPoolExecutor + + username = "race_" + uuid.uuid4().hex[:10] + password = "Test-password-1234" + + def register(): + with httpx.Client(timeout=10) as c: + return c.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Race", + "password": password, + }, + ).status_code + + with ThreadPoolExecutor(max_workers=2) as pool: + statuses = list(pool.map(lambda _: register(), range(2))) + + assert sorted(statuses) == [201, 409], f"got statuses {statuses}" + + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + with httpx.Client() as c: + r = c.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201 + assert r.json()["user"]["username"] == username + + +def test_rollback_chat_with_unknown_member(): + with httpx.Client() as c: + alice = _register_and_login(c) + fake_id = str(uuid.uuid4()) + r = c.post( + f"{BASE}/api/v1/chats", + json={ + "title": "Не должно создаться", + "member_ids": [alice["id"], fake_id], + }, + headers=_auth(alice), + ) + assert r.status_code == 400, r.text + + chats = c.get( + f"{BASE}/api/v1/chats?limit=100", + headers=_auth(alice), + ).json() + titles = [ch["title"] for ch in chats["items"]] + assert "Не должно создаться" not in titles \ No newline at end of file -- 2.54.0 From 1ef3adfe3f8fed7939dd1b865523e057d9e8cdd8 Mon Sep 17 00:00:00 2001 From: Queenfish Date: Fri, 16 Oct 2026 01:34:03 +0300 Subject: [PATCH 2/3] =?UTF-8?q?=D0=93=D1=80=D0=B0=D1=84=D0=B8=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + evidence/scaling-p95.png | Bin 0 -> 38870 bytes evidence/scaling-rps.png | Bin 0 -> 35654 bytes scripts/plot-scaling.py | 75 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+) create mode 100644 evidence/scaling-p95.png create mode 100644 evidence/scaling-rps.png create mode 100644 scripts/plot-scaling.py diff --git a/.gitignore b/.gitignore index b5a71896..5a5393db 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Local/private state .env .env.* +venv/ !.env.example secrets/ certs/private/ diff --git a/evidence/scaling-p95.png b/evidence/scaling-p95.png new file mode 100644 index 0000000000000000000000000000000000000000..817a38c78ee632cd6bf4551174f223e18d14b8f0 GIT binary patch literal 38870 zcmY&g1yq#Z(1)j|5-OeE( zKoAsUr8GSex3kbqHAg4Hniaqk6G&zzIXOt?C^}_CWz7LZnv%~DKW3k(iV*%` zf9;$YQ%-~gy(H3qA4n^RHssGET50g_KfG5A2olh=PFbQw#H0TG2|R>=Tet}tXaql! z*jURR-7z@&uSk z##UH}1elcy*R<-lAW}+zfg+ec!T79odizJKHj$N)x|0d?2`-`vGF zRrzC3Fg>u~Kwh$K%)jqSGULwcB83392v7pmOZ_9nJQ?tA(?Ykt0?ZewSK;qp<~4xF z;vGHR|1;S%1I&kho)!K!ZIAc=9s_zr3IfKJR6-V65Jz04|4--vxGla#)&6;xnhRV; z3?n1K);TRfPVWc z2g(!w=Vy{6kazQZ%G`~=HTe7A_kfQH9IqzP!5uv?63^5Nt{(6MU?Pc+6CRbb3sAF4^S?!ZyYiET#|sD3dzy38t%ht_RmKpCoRD!EYJOl&A|)INMciCq?!6t#h`=Yvh|e9lnkfWzl=grn2<6uk?OV zXW=}2es-Ji;0fzTeG>nBNH5ptZsItYRI^f0-JXB{xl<298rO^MJF0i;a20YCd|Zcf zS)^{IwkHBQz}jNydHwm##{s4Aub(ruIz90mqY!xF?-U2(4}3@1pu6F1y~C*Rn1YtQ z}B{B*g9h!hHs-%|yB+ zUE?bIN)b6Eew+~Drg!R@Xxjwk7Qfd z!Noz77=j_)d`tV%rW>~!BXfb3@%4M3z!y!{3f=Tj1or~AL;A7e4^0R+ndMY1&O&{7 zssul%#3+vh-6+K#P4BEhRvr$B`1h*u*L5LVmvri(wcOju@pYku?rdIiq3!2lRmzYL z^`1~ipI02F_xw=(vw>Qq;cJIUk;iy1%8~M1$A(=X%Gx!f4M#(VyViP*Ci9x~M|(uQ z7jxL?G^`^oV(S{2YYQbwabxdoSsvrEItE|4$G2E~AYmlbO_!mJaCzY!=;NJ_y;HsS zP~Y4Ag~Ld&57%fw*Fyn^0l5sv=$BS14@9(}ZaIBDPd0lN=$P_M24W5;zV0RSjAb{i z#rZTpuU71|trb$f!Zp$MiqKdN7DUG6lDcx_5`qyU9IKCQY~F7C%woCt@Ut&@0$m)< zGM%pwNxWwMRqM!vL)YaBXJM_r;EPPtOp>U2mk7d5C}V#@2WS2#VXp9X7V%U_)_(mK zc@^fjH0xf5O$GS~LZ7z~oV7x+FHxuQR2yArl{jg^qCCD2a~ijF$^OJ8T_tL2!2K7l zKU!dEpm1$W-S9J*pX6>lzCWf_^dViZGv4q+Fsz-*xw5`!xGzZJ__Vd6x0~Z>Kw4-$ zM6+5%yDl=^h>i6G{*ZB_Vx@tIu2|M`#ZIgc2YjldA5PUgw5Z1 zQjIm4>f$bcG)6wK(^=ToQ8YtEa)~P{k(Y8Wz}|?_)_-V~goaG;h{Pj8YkGkT2V#0q zUJkBvov++D+Ud@u$ycXH9=mmUM{8tgdW#VMic^ZCqBc3iYlv2dSdvOCUB)qZydx1j z@_y~dATcCQ#AUaaXVdRsBYY%t%Nt_-E4{@UEt}>>4#jJ*xti!&*>Cvz325W@5#}Qq zprsE5^ii6-FeULO%rrQ-rj5$q2-OcPN8#?4%~hA}trT<3yv9;m$y;o`s9tHz_f3%s zlu~)U>dDS)|J)->NdE#ioVbeo^OK@`A27B3s`jCNNitpgMGZ-#+19{iv58A_BU>`dlJQl&W$hWin{}G;IIH&oAYY zU>Q5w{`9u-vL`VE3VZbqZYiIo59(rLJn67q0)?DL!M)D=mZ(<3C5P$5YD}At$W}d} zem?JqvsI@L7zQ#`f7wrzHUwW#?HytXxLh6o3Ecsu=%Cm0uVbUsMYx_(pIbza@`KqT zl&)#QgC~&PZDERZw{ct7F}sDx<`}JrNUiFWI!9RYEC>g*gtAcMG0HIz zgPJlZ7G1?vckB8qx=jg*qK{dMjfJ2-dY17eIxpP3it~c#v`yo&!$%F%xaKhfykcNj z8>PR16?6PcI`pOLa2nV4vF7bChnNyuG}H=>>prStbVHb#=7 z^K@|@C}jVztzPh&D#wnd>LU_m8!b)<*KJ(eUiX3}V+2+Nw$8=&aG7@mq00E$n?Bs_ zqgsruWHB&g#{bB-=Z#)ICA9w2Ny@R7&|tq~sD-^gRNCb0#o zpR7Rfl7%`EX7(w+3B@^^|HS1KcA03f8T3iyF$X+)qf}yImT+| zX}8BbL%zm_Bg^0ZwUPaEZE=C0j+=Qn8raQRohau&@b;)Kr4Kq7v z;*Yzw(EW9GHoZ_Se465jPefeKmEJ`QQJ84%TOxCoEcAR3dhPeuIQ&RbG@dlJrf3-P z$S;+w_ex{gSPDTo?cYmR3YSrkiP{?>lR(Zj7 z-;QvGwU1tzb;tmsA9P!4p%RF6*&^ev^;dcAP2ypHbF4TpC~rP=hH;y-ik)U#;vjr% zmI&X!!HshIfENF&R_kUGDR;ZnaMES|X^WiSwUo(;{+muK8wc`7R%*6|VdeNzudtK0-!kga5GOu+>fm#hgF1vQukXA@Lkcixq zrr(fQ9RJ)dO(J`pZuN`k4fmL`1d#eSgUgHnqw)j>5Hvtc&tfc_huf6}b#)66!8mii`1K zE?U~&dz0QDT>hS_W@?Qud{A|nr@o?lyxOuv^~#OXrEx!3EDOye!ZpGo&&E0%I+ouV z=G=wMXc}DR4yH363$6DO-KSF#8F?j`B&BDBv*=3c3(XMD^7!koR!E2sJcCJ}ah*`G zW;$}=z-(V|on-mmL6_g}c%ZB8*F9!wKin8K8lP_s9p4uuhyP_i1mOrsx zqiuiH@U^eOINRDHIg}*l=g%H9wHB+V;lrj;7%|QGU9nUD;Js@_AeSjMv8l1m{DYN# zCwNi{K!VL&@H&eTp7a00HSK{vhu8s0d+>{I<0gM3z5|)D;B)zX?-4YsyLbQhmuvun zosjBlE5XyI|3OqgCdrnqvj?VVf506OXsD0T&s{3;>+Qm4(pmmj7F+3vz%`8&;=o2DzF36FvkC;3G)7Ja+Kiv@ijXE-QMR zgA3`@MkQ|MsJw z3x)=bbq45({kMv^0DnMHm#Gp40JL??MsWE90VQkU17nh5{sty+h<&%4Re`s?6{9cy zHF!`YAh4P;vB=|p5ifNh9v}n*x#$130azJ{oDdz1!YKjzS1uX=vJ2!Dqz1cWApD0w zL)ZZI#%1YaNB^;^C=(2i9uf)1=T6!}jDKAK?&raP3k3#Io}qL7V;>L>Fu;5|>uJ*6 z0MtLgWpLg_N(#P1KD^5M2NmNna{})kCb%?{|6Ttb96OH(2Buo5w1H0s=0qZAhb!(p z9C+$qQ>pc@2vJ~A=6uz&X}hpL;@&x-5}?Ge@3Jmw@QI-T6u)=cJ{}^5F=yaYzJImQ zM6Grp{J@j7KaLNs?7trn7a$qJTXOf;z{&m(8_*AoEKG-QdT@av=X`ivi?BslZ^5Xh zNuS`hSu7L@9HoKI5W)0B&Wkn79`yt{Kpfc+2szvj5kl4fo%p>Bpb*#91UeeTe-s#k z3(U*)#t!>#Qz3=^#o0miyBeE$UMwX$RAF6wzXa5X*nD-ENYCIveeK7qily*-bTO28 zgNi5^>-Xmi!8#|YWxC@ z1Y$;UY5!enM0D%L^^5S<@~@4lyH@#;cO2?7GYa0`^Hq;W%hA)e}%W zFe#u68lGfC9*BDgTpAAAIIrW7ipCv2yBzO_SYK;i;ylH$^D-NL&avN%iMisq>XZ)P z3n5|V8?QMZI*B>Im*w>`rxM(cxU*lXoMJbdxK};rd>M8x`6xYIN{GY#4Jha}AZ=M6 z$<D5^8~*_%K?M*INHz;JxOUx`f1mXKJ4k6hf@@Pow7JD9xyIQ&zcq3db! zQO4*aHqzjIzDUn}PTMTf$+3sYapXWomjNN{m_zhGw zjuOWs;;#6!C`#55L`R!h)4FXZ#Je$p@%dOfNmBGpmeod{9^HLutHa*>AWq72Zu3xd z^vU+9sGe0NwiCXE-@nnGJ?jE2Cz{eMfc@j3I8q-=fP6W);w|q}$y$Z_*5= z&=-3S01&%!gO&YK>zT&Ba2g3~?l{ri)(R+;ursLMOYrzZG$-JXwH*N;FBX7kZ&U*t*U)mw@kYq3&Wp?me zqp6-6>s5IXUX{yxGh6DZrn7O){>@O! zOv{+qsCe(H#Ax=JI2?0yR=;yRM6`$Z0(m`q2_`)DBQB8!FIsb-L%r0|$hAqR0CuD$ z6>B87528lZ>R3m)T)Q>kutN?4HE`V%K@_bdfpt zs<=n6KI6*N9&Lvvy3P?w;`Yp*XSEd4PDhc+WtEp}6w-oY@gK-*ij-CSA2e zPuY$?vD={Bm%=)ud@D*=o%SA|_b>twMR2 zG7{Ra%RC={OP&JzIb^z-;*YLhv+Zvr3fyuU@g3M$jbI93gF4H#Wpog7g{1EBOD^TI z#c)ncUBnOtL=ZkDg+QUfV@NnBIiyFIkata4Im4P9A?UJc{8R!?CT(JVe>dw<$h#~> z0gzra-0@?x?;RrQHzNtA$%P|mtWV6?+!CG_ejU^f&oDr5CkVmorsx)B^hV)jcZj{$S>ZmtlNu6tQ{{26+0frKk zjq9Tg39DcI)B}o+r^3FG@H|g@+!EunTQ`r_up1M3@mIkPAiI?6XhNm5m!|pmJdSK! zA&=I7WKm2lGCp{{DR1smyzWWpwGcEaS&U&@9u4h*BvbhuuY{7*0y#F6x{o}(qJHR% zm4`43I!9B8hi8t`s^2*=XEYkF)gO*}!WC(ne-6ZFh9pP%SEqEyvV`Qn~u>OM`ak5%SYNOdZ*wZD()q_|j^4_8xGiU=N~LkoO0su7_Iwm(+Z zakNRioZLFZK8Q?q2di+&Sw#;R5-;|u`14=7H!I8Kk}GNSS_t@K8}7V+eR{Cc&NlS0 zv`|CK*r&xh&wl5Ahxp#LYReWyO!3{}!o~pmhLkR;E+nEe>zt#_RlXRO%q<9}DVE(F zm8qm|x;;nd{pNPGTLB) z0j!(-)IMw;`sRktIcs31zS6195M+MuL=dkk;w(Ryxg&-t&7l|)9Bx0rM(vhaRL`ou zSvdW1o80mismU>o33|~TiCq01KMdWjPaTI`JF_EP0;nE@^nCiAxk<-L&NPcL^xd8{ zYK$$v_V*HBUxW_m_#IUZ3lADsNt`;@8y8EOpFmhYF^6c$-4p`sME}Gllk97b@QU4t zLTK2@lh_S)PV!#5-Xga`G;Hb1;|_)T#`5T%T$B2B zAS5Nm6|VQyHuwPPt?w*%t389peqHFR^xr;LOs(d+o4LFC zXlrg=(X3nQxVgC~%MuGuFJW0XMj-kDX1o=VKP1NS)D1TM-Y@IT;iUmpA-$;hT$xQq zPin8JHW!eI!nRz@|cvUh1TSRBGYBU>7vBzP25LRCoJqr z3Qd=%&|pL*EUwfvjocq7ING`nP4DwzVLB2hj^FmAaoY$`zg7!l`$`H?xVv|*{jjw}M_`)yipnof`sK)_<$Qen6OTpNc<|Y6`y=fl#wsc*o%bbgl!^4e>~VUIG@j>c zR%H=r56y?yNZltNDs|ppBngSTQ0M8zH>F&#<0Hv>bpMc=b2BX1mvK_AZsNTkn0>tK z%~GGA&WTz6{Y9+|_J*?s>qmpj`lGKqcethkZrHS`1{t?)P0}ir(hUXilOD-+P#z<4 z8f`o@(<0@sCK(=Kf`UQf-(Gn({P?*T$=H$4aw-9OJK;Ck#1C%H9y%*}`0m9q*7Tsx zvK$qqe_3n`!qMy6_ValAJs2u$`rYwuR8fqLK#CcMgi0 z7Gn}dX6A5CjPO`;{9gQ463ok(hwG8lmsp%V_NvQi*~{4emHlUQq?1 z#fJ7_$41g{yNUrs!CD}d6NKb>s<^eCW$1MTw5~tu5~zA4A_|`OVSBmyO;fB4UK|{f z8{4*RQZG{Hm35GVNLn5HH;26jQKvJG-aeU{ct6k&)UmAX(QNw358q=6j`s!;9xld| z`HEg5*sr4~+V(rcxe)vpB#KoX1s!T=3 z@!ZZ7 z4Gcv6)3Dt66I8jMzl)P<97vG5@!+3YW|@!NO?^HsznIx%F@WWdau6Mqt$og1^gQ9b zO)<#Z@Rvwn@jsV=oy??C1cAv0e{oddq4OY`d|fHDk9e@{9FQyr?av}Kg`Q~hVi(PF z|EpAa}@-5=qXwzpFNShSKA__w3hE%{|OJfCnI9WO}uSNNfPSP{T&DzSUtfj1rZ2 zw$S^li4*#qO&XwXcH#Bb8+_OS9JT~dfn`TaO-i(T3`z`Nm&b*J(y~BEsX*F4-;ee3 z5j^dudHn9lXMMc%q~(Ki^AbA%MMyHo`LP(y4tSan?da0~nmIEO%6usZsp}ArlN<*C zAy&pm?C8J>cpaGj9lGY$Pvw_0b25OS^d_;(&#p!jTDyi`1)`^e7ThO2 z9}i)dXv@D+eq~Sp%D!UY5#Uj)>-LM2l!vn)8yhS0@BmHih)$11wL0ijS4HV9Pg|se zGmV(;?f!MX7_a*5_R9FXV6vIw!-m7}x2ns#9=j=85~Y`XL(rxXg`*L4qpYjwWB^tA zj6{M@)jBAvZzp?xik3|`@Nn-hd|%vYdM7>-b=kMm5_{kQg>k36M{CZEDej$Lo$1Z>JQp!e%vt

hy*a&og4K&NtN#fU8-GDF9+SmrjyGP|3Opo3Al&*n7Vj zX~tY&w#!v8If40t3!14C?s$)Yavk$BOgv?j$;a3+>1fNrS55n;bAiTw3RxBk$^-}= zZ9

i%~FF&J*%!`7`9z<7Md?BigbTkMjbNmlLXWB4r;5_u@T^_gXu8dd;*Hm0$9* zUM*ADL-FnO_0!Psz{I-UP1j1hOP~5Qxo@2HIEsyv3V2k+8Vme}TKf}U6DWA??Fcj7 zf0le(wZD(8tXqF9_}Z)v#FHr=O0#vsDak-#~d@wq~r$G5z03CTZe`92GU$+MKw zS!T$0dDhZ4L=wjpc9U7>#=Ryjm9^CiJGacB>IR-R=il7H3a#Vj%^AZ{?OJpNdi5-e z@~{a#GE>NhsfBC(E3SYC~c7CUhlY z>;;VxrfW3vNiHYxC&DBtzYr&=pApq2*V^a`ZtQ{iH1k^T#x)5U<;OO&hOSt5AYZzU z7{-p?BReU+k3Zx_rAAcj2sKWU0g7+SJY%uP73A9DCmYY_F`Ck>_oe4N^7m4kW*js| zM-)FX0(Eo3-rz__M=8>MIg@$VusGS}y@I8kp7B`1Z8hRdj_`v4f8W;!wK5rnXa}YN z4UWPRldb1K*^&FgbxQ@*xXbL>7?VW!n}-vmi#ysVsPOO2edKJn^>fJdo9nBH*U9Zu z_KEs95^Q`Q8tK3LzkFnST(6Up!OwSy8oBn7T~(oHOkdb7mp__CM@OV|efx{bm+I@z z6_u@<-Eh|yMU$hR40)j55cVv<)cdI}XUiv6ZE%68Q*$)wE!0M>I)yERNB#3XX*H?# zybpami$HCgb9nW3au!H0g?yINx_<$k4@gdyyZ1mO+H3ZyLo>oScboZoix-aHxni)kG5 z*nx@@h=vV-q1RSUU-u+GD3K4Fy#78N{v_EL>{GtuA1QPh(s_E`8rWnkf4Y9^^39KL z020on+|rQ7TJuH0SIUH96sKikuZ6c|I){@)N1C+g_SAAeJsqHoI>>+~fty;GPpd2H z5oQ<(>v=APkJWw%qDA=kDEnsj&-e3JZxX(_ae7M!x;HFH1Kc<=+SVOYj<-)3$Rpy> zJ$nU>ARs7Y%*ilj#9HekG+60oA>U1OUFI~8^QG%spvA#mom&ufE&`@`B`zxZr9d6^XfdKTzGZpvroPg`fH2DG=AgtNd2XVk z>E5?Y5-L_<%AM!4kS|^Q~Bu#=S& z22F~ml79VB4>gLiGU$h^7D-a_Y2r7LioZO+!J;^4`<3-{>F|dc=$t&JD5O*BJr2W7Y~O z(=<#C9e1OZOJ#FPdW)A9w=n~ZYh}ev1S6y)W-c^3@3ZbdLu;i(qVn5;5i-*1ij6w0 zifHQ&nz$J1`4xX?V?&(&5=^S3w?dxZHiBqHS6INN0fDac;1~3q%k8)Fqjt~6%e>ho z-}@bE(-k8T!@xF4#@rDUrs**jrkxi-(>2A}-9x@eRj2IIr2(@iTE3+Jgz5lSYxIvQD`? zJe{yy=JO__buh-m9( z48S($3r9ErfJn$_Zc|Zbl+@ zMCqb3xn}5)(x$_(Bx+7#A)%WL2u2vo=A_F-#%V&B@4=i5EF9<&p)>J6OXPknDW+TX z$o~gw!U3{2mR}?q1LCdOM=3UtX zW{xB$Cub`V*6`Wjb;_2Qh{B>n-FI+!1sZEZW9jzJPslIlC6he;;||W3wHhQvele%M zqyDNie$=x>Dr7o}aWpENf;2djtwx;-B`9wz`PIH{Q#(S!NZKi|cgCwfFAbt*0>$Uy z*v9zN#bF%u&A}B2no|_lAxE~zVA*py-BdZzOpd7Q)BfJ?;#f=2U-R>08-+MdlTy|1 zDD!fCk9^*dlMl_qbkC@ea6}B&Q=8)K_tdnjbN5K7CZ&uWC4Ggld}SJV!kw8N`eOxi z2Rcv&84b&IZeq*LqU-JfZETzcS3wVR-eKpGck|k+L3@4WB-GFjTg?bbRndE$BF<7O zW%qL`>CoIYkWmmqjD(gE36h+WOGRyTg}xFGK}I9m1h55trufFNcre@B%s9n-#+4p`{$VmQPKr2PW#`DPUtC{Pjcs7oxkY z-S3J!QL+dE#yboOSD!}cY)^%EBAgQiiLJi%wRv?=gRn^a_=2liQke4j{8M}visM88 z`^>8;yi-Ebb6HEt%y9>}Oz}nroBel~@AvDBjE-#eW?obkZ+=yL`}{03y4F6r;rc3d zMMCTsUFr24YXZ4W6LdFF5x2-ix0dyJ{zJ7Z2g_y{M$y=Lj;UpR;5d;u=lOKNtI|vB zSMw)YYhoXX=P2W-+@))thdbyBK9t*g9PXfRmFqD+AbIck0u~tqNvMUE?pTY+9cY(C z4@@l5R#?RxRz&I;Ygft2JH=NexN5!nmBjL3s8+#xV=kxpx};?<&)slm9~BQTca1ve zm834%IKqfzsI6wo&zXDY>-`o1_TokJ=Qzc;ttsP%s4ov%`1>ZlGOae;nMP6&SMcVb z(|P{j;9K01eS`X@s;@CXT;oD<;6j`S2bh2yfT~!|3o{y)g>c{Ei#{2LmH-8$b`EWK z8Y_y?sA}+%5qW9&XFl>|8AJD3ukzNFx9o98a^?yK+R;n|Q-Z#$4Z=*{Mfop3U))ez z|8nWSG|$-fHO-SZqq((7Dr7B5vK=-~8oqe@tKO}tOx}5bQ1^5%Ym5Mz5PVj zvcP$+e0%Vwoq;L&$^-G&Paqs6)i)HlMEeZ`mFy6H#2yq*B{aMxi&1n=?AVi+_ zyvp3GxLwTf)R7-lzK1*s!mNL}0I;z+kHg~YCbLx3W5OMoclDF>_r2DE2J8+1-v&hn zRBDXmY>r}naw@9??C@R{utSuVMx!c~%3OeFJ7WhVNw5*wthhlxl=7wp{Q;z^jPBca z&RB`@PQg|j-!=}cb=INs%dPblfuhSE_dCoC0TX12Fy*VLM@8R@(ikZ=AAmn^cnlwU z*CLy5yBbvk+4l{aEQiYC53}W?(BJ?$k!Px7I1-CLs05b>ZE7<5XdS6hWw{scx)`a+ z(Fq8Nx|`HLT1r za=DMS>fDo;p;UaW$9JsDJG?T({Y!LE#%~N?h=2$!C>(w1{shJLsXBbVvY3FCAmUXF zpyt+7VBn!Ip!oH7*MwtWz*=>KLvyn42f#)dd>SI$pSj3gxF9XyZ9dvapCtJ8Yd zN)JJRPtRooQ@M9q&$G@o6vu=xUWN(J*&ts*7%Ev$%+Idl=^_!eUCA0RKkXCng$4ul z@IY!h)7p6~Zt-o7I|_j!u(i@UnY67oIy#{oDyPuq<-)Lf_<_O97|?5LoTMYp1P~Vz zQPbGRGq~?&sVT2GyPG8wH1fa}Cpw8|syg)^%qkI;=fmUasfVM$UROt;m|gl}^`*z5z10}u=llHz`ACSpO^V3+`bxIQ^D zKruGE^7Jy9s=-d~CCSsI>6F76Xlw2-lQ{e*QU93Z`)yt0@y|BWv)5 zAM;U~63NWKn1$8N@5OY0`mbvq!xP7ZY!h<4kJEXtc=fqVXmfJ4((YOqz=ZG-if4r< z^AYR$*fRN5_7C+7VJm_S3IT3FNb*b!5SDuZLwL~Jr!}hmN$c_bnvjqn)G zI>6*Wkr)_Hvx7B8%uAata~h)IqXgawg3O62Z$o7#!$d}Zvv@l9iL9}3RQVm z4KC@^K@SP|f>Ff)t-e=*A7_cE*7dxm1*my6vB~ojcV+t!?m&xD&iGuH)%!BVleys| zWrf^7b}q#y5tM|>|G9*y_6QQz$vdZ4i-E>jyj z67Hz7(HbKhu!t8k`uBU&IKF~!hy4-J9e^>+q+v(?C#|4+;dl3MT9jae4{ZNn{uWWT z(=7P6yMe~AYQu5eh`XVXlBI3~kt;fZ7Y_~sCoNB-;E#QPyIo@io_a#?$#oykUmxIC zTs7Uc0-=4S|ED~4FFNRY|A8m6U*xzHT$zrsl4kZL+oXbOhiAK5D5gy?jS&J2V_>A# z%z*qL;;Sw5Y@_Y!df<`_hS)B+t)-L}?kHd}B=UGw^m&4+#LrSU-C2J)n{hVqpL9E+T`l5l`E_ky6lS0Up*F*bLJEDGSt^hyYba+Qo%Eu+_sPx}QG!z_u-%^!NS;49JGx z5CGIqbpR)B+xkI+#?-)kKyN#|8)7+i z22HE_IfGW>S*lVz_6viLTpaQ;l*@8yF?U~wJq*lLl}Ns^pyb2zihKj~m%+_b6A@q; zMC@KCF!y8V<7ceCaw(XN;3s-#j#{pehwpw>uOdX@-^2)Ddc;C&C*;hvC!!UqHm z@Z&5%2GE@G7xBU~iVDX`!IPl-tyXC9QM}6!J72cNW>J;xq`QMNwv@AYRbAgSWgNju zm(avT^{D&%RyOV~o)v)$HAYaelX4nFN1x``x#bTQl`7J!s%waP!ea#@6vRf@5F{aFpCNuxTy-K_Sfje6mk;vo+^!@OO6Pdci@T=Jn!`ms3c51brY zDZjUJP=Hh9Xn}u~Bj#w`QH7YFrn7kZRhGFrTBgQQp-uI^e@i z9x2Va$T!VZ8F1JbR%;H4JQpCf!s0TMq}2{D_u#soG~4=;lSBJm)LUAP@MQ-}IO=EH zs2%y*)npe3~q zqw_4MovIzs-2nQn?(I(HkL*Hp5yNVcrYnT-EpNA;E2fK%M_C*}7j;kd%3Ceh-X>lL z{Y{C#9SmEKQVpn)lXX7r+pDGP*DL6k|F#Zo_leX`E@1jTLG)SrbT0V%C^R7})nVei z_t%GE;MNi34Tuf5LEo;GlIZrW3{y97zm1InBN5duILsotoGiq+qiRn!iuH{=&=Ep9 zRq#swfxlDP$S(!0oDK{fhNcQxxk)ygp2&j`b22`>+zu=J{&=}&{P*Xi&b%Lvcs_JZ zoKTVc+NGDcEF9UJ<;#<>R!^qQQ#4GnwBVf|k^$zgfY1LfH4S2&SrJy&Om630enM*e zl(h(_D=KxQ_rg&ytj`KRlfXKrARF%U4a(HdZ;^zWo_BDr;cbq_L=7=y-xN21 zYj2>|a^Kj&ilfeZmXD`$eErKk$|^?}hn!x#?PyIxWGHF&t+B=2u^2;X+#zW{ADg?$ z8~Q;!elC7%N)>2>-0k*kN^xf>z?nVrDk^oHF?u(g{w}51M@PKU7+L%e%c)LUZxq^etX_S7lJESp%N#zoBz*tBzHjb+G5zWr#;2IIyHlh! z3sx)k@@eb_K~-TG)~Inku9MS3OxO}I7}y*s@P}IuN+~6;yIsJmfC(Cvf+HZEkwjXM zoxdK;g`Xr(O#LJEPiz-L0xj;c_7){ zlyWSR)8)QiVL$af+;HCITIP6MKml(M_GqN%_lH$`Hn7N|2TikQSh7U^q=kSd%fPV) zd!yqJshwxv)y^UeZG4{=%d3G$^e0eNl|%|um|xGV1z=Vix-H-3DSB^zg$_NW3^e`_ zIDz4qVy3m}YuUnRxDb$oke?%v8Rya9P~0UDEO5r*d%VqX^n%vLq*elz?+ADp*lzH1 z(+Cc!NRY5SV7vFM!UgjN;{ycH%Q*0W>%RpOx4PJ(;m(c`ZFW9YmxbTN~m6V$sXI@xTT^H>sJ( z3ArG!lafLG9}0TjN(G}##l1ILBB)w__P+~k@<8$t?igHQ3$%yI!?xgQ(OZc*rqGFr zPrF@t^fuR%v>)4n1Oo@2V93M9fMT<+a@q+HpF{W%fuZQ21mO`if;PvSML@fJBqK6ZB(2H0p!Gjp^x^CqKgCK7-Y0q=K;fQdxJpvU3)kRy%9r}BvDd!I-! z^_|;+_}~GHMZkvkHBoOW_-zi+$WeDc+)&u?3G6&OMfo28^l#7tBBG#&4LEB64=Rn+ zL<{+GvTAJ=c|lO6SY@k_>)*f_uk{J!+B_k2r#_}Te8c-2-`r5xbm6Nc!pZ?Z1Th5$ zX%r-li+1pdpk7eo@!bg|U0j!G_(WLBJrZ&RwSEdBp$4EmJse@#Y~=<70evQ2t2M`H34tSQP?4-uhjhCV}5|yD#%F zJ0}G|C$OZJff@57c#Cv#1V}m^4rIhR9=$Tm{1Ud)3D3{8w)k1UMFb-}q9knjjrd~+W~84Aj=e|39q!(~x7PRsU-i5KD7A0ojA#P(}Un1lZjz^pR`d(XXpCI%D( z*??TI5ylVnxp7;aQ=xc;NFd4w7|{mq(>bno8o^_E{Iw6?`TB=cIGxa;U0Z>1bQdt` zA?I*VcPd}j9o5F@h|rP3SDDvAsJvatb$3<9k0#E4K46=aOA^DyMeAylpi0Lix$E4M zaFrn%-EkeToRl1FFG!&9Svj05Gm}9mDBNi(Z69JkA)kTAIh>9a!AZCdKM9Db39|eUgdox9k5RO70VXk5VX|Uj3 z8XWkJ4#@IHKVbSdY-tZN-s7?;^N^RBeMk2l8@^pYflM>~GQ@-C3< zb0YSDN+rT;0bsS6FW8t9T~;fhtNyLKLS?SZD?>;x&ZGd#$CLXEokK1hIJnr6PVcCw z1o~;`$Iq}5a_`VO7oF*=`6}WhebG02lQ;{nu4mt%Wp2X@)e|-5b;5mE9@`l2<=BTh z#dx4VS-o_~>2j))lRch7;%OlVHIR`{0|z!}s!$AQ%TQbbI!md)fS)#TeLe=*8VNS7 zy#}5ysNVB!6~wX!68J@B^%!qBC3gLI zethdO{qtSJT>R0ij1ok;FA?-w70)S^0)pQ^BQwJZ6xL-%oZr_z*UL-ZA;v}7vZOO&7@Be-7K_( z9@1bNuY4hG%U9lH<-tr7C=9_Qj&RQCWBkrn^f7IGUB0|kPA4WQ0$iTQ&iW1i(+dKB zUO_0)jf&yoP3gwxPt!IA!(}WQN`L? z_nW`Udq%$5zhfDt6M(u4r@Spb^7fB+Mresf?1!Fh?@|DWm$Gk>dvFYq zL$pZ7-vsdPP<_KeP)E{d&O@0vIttEmWs(`y`~&H`%ULO{i-a>`$Y_yHk@|MUy!th2 zECRN&!hp$v4~-(sA!tnT?Rbh(l(=3C@>$R5aGjznb!fG_&wyzcaSoUQ@`0b7dyDf2Ql#j`HdbGrjEIi#~cve#z{&-Oqrx# zNvtjERzQ_`oq{UX2W8`W#626?`<>f?t>oup`vh;Bnya z8vmFGkW2el0J)(Og2q{LgzaTqBSdM-78ySjw zJwI6tKe%SMLgu$-#qS8t7;u)zG&OK|t8cpU6kjjSqo)bO8d{)Bmhhk8F_WI}?a#7> zq5_1cYo!(NAA@{FGwFb~V=Bb2!142cWqk!yRa^Hx-~|Dt6$F$nK|qjhrCaIlkQ70> zL8Kc5q?C|`3rIG~i-v{u$_xt{1FuuX?ecXG_KKrb_)?9PWh5Zsbn-ILL z>hy|&YpsSr8U#OJJe?8S$N!tQ$HYNZ7R+@Ge{L>^K8=V&$#qSvr8VcWg9cr`Xalnk zK|t{k<9-=CZwoo&{rhcz9ss;}zCARjmCHq;h>ns1@kmGZZ;^y5h~$A+^T+=KCCE}Y z*#Q_kdgOakS&<_tZGSlrSGgz!eFrxJHhk$?v&oHEk2(RuJnH^rt3b_uA#k-pa@9!< zPr?8}@9iDLtgLM93AT)H9gR?Q2e>eVf(G+DY&DHe)oHg__Iy0!nkHZ_k0p{qz1bj4 z+VrXQ0(jpr5pCf`FG!Cu2_riz(0+f?x7M{Yg?R&mUti>7cDBUYHvk_tz3k&wf-25z%K?ziUatYEvl_T}P?7axkLYWvz^{@qj z>-w?Vo2Ne!R7%P9_EaoYwIeB%^QZ#mZ)~{$@G!)4aUxjkMnBg5>E4PHD|!t3G9aZB zI9!yzI}d;~Z=qkON49Pq3WnK|zBi>q>)b;gL-;WGA(w1Oe+trM@>TfII^dAN!e{uG zt?n810GMdg2|uexzY?;uNwqrB8Fi?tAC8S@6M|7YKE!taSy|DH<;lm*`r<%uML(wu}7=g^dwO zF*WBav<@W322l3RxO~{B!Q$S3F~!Pi|E_^#Xz&j)XSq4e_YWNJLB*hyS= zEXiAd&S~g>B4UwyVF4Ny#iC-lfA?=D+FRFlaJ_L~z^5&RC#$bEr3@vZ<00p7f0&u7 z2dc)hlHj{KZ_9*r0!-q=7P%%QL6yVBFUrWn6Z5X4_NOJ;{i zym?Y$4y));O{sEvU;Yj6W_6#qvCSwAKp=$l6J&X#^-dsVu+^+yQ35TG%@Cof1p&|9 z>{Tea1M#6D_0fI=RR19bKy@`kIJTpkS;4Gg1hBvs>)W0(P-A=b@IcA(1)NFfzYR@R3r+ba`QUBP~owrIbF zlP?%}h*S*F*z_cZ6J70hj<~2CNTLfp9TVijbE}1^QeR2#QUlySHonU_NCOQUYOR5L z(ZYcuqcLt)PC_f7*0asN9yI$LDTIKnNAQT^xsn>)iCE}+K!TCh_%f2w7_x-xxK?)6 zmxBtXkGG*BgY~%C2zo2&M5m_kJY)NbIqtpwKYg@@N!YLe8cYB*EE-ZN!6G)jZ|GGu z>FT)45VAE?<5{Uxy+HD}<^a}bK*&4vK&o3&A1NO+fT+C`2g17J_3t!4G%phiWxZ;t zLeN_^sPf0D=Zf5rk!|<=NvT{1d82lIcuMqqmQ`@t&Zu{3c93kqy#y@4SxcQg60ipw zrT~9P3Pi`ph+nH^Q_eR$jaB_Ix4X0*yZ&s=%+2oRl$ie1JNol2aaPDY_f87WoV9Yd zIMP#LvW>A<#z1c;iY^V(ZnF^uryz}=GNe~Vb%>B%0YX+LT&>-}-eTjYzOBGcKr=~- zn?>g7@62GKJG|Wu61b?(kWBmg2jh4wh2O2KTK9<%F~&3F@QGq@U5ooVCCT()%=-&# z+;7rW56RP$gwc`>XfMtRliW_$SZ~5u+!mKiGw3)gx@)=t^dm}JeE>u_Gr4V{i8f$_Cd;5_FoAq1M6j~srGK6FmQiHy%~u~xFRC3A>#ztZlbvLfMl z_vzl}@D`@EL2U(gVm|2(<9M~+tVkQXGXp0Z(txEd9-FMDUPI(|UL$TCp8!MhL+B;| z=uK6~?Llh5JM9c1=b^st5ke()5H9MyMhgCUZtLnhI~lQyCb`u)0&*9K`{1el7rfAP?RyfeJz@Y(D4=9}d+A z!35&E1_p;l{wQ}Zdg%_@&yX1!NDe-Uz=Li#OLxb9=OeM7qX})r$D4W04Z573I9yxv ziMF$u31U0SuDcN*u=w=e7Hq-qUI7rG&r~&qJlyh@OHwWuv&e0;H%@`J>K$%ZnS*tA=w?}86hO;N2zg=p@C!pPphh*Siq;h*{o?Tn8N z1r*&PNaqwF@gio|$jCnQ)%+be{Hq`nL<)mo_-Sg|^sgh}p8}zQmDuppqTAi~zTLh^ z2d8uNKEIgs%6nPlYF43Bxs)m;zhcn@PFe(xW;o8T9*^oLCvRq-KmsHiIOn^yV$Ks& zT~*5!l_<8C5$?OKf7&G|*N3M)JJ-+8GN*RUAgTT)g*Uotru>WOQ#WN*(67q)Q&Z(* z)vg4Cs727v6eV;@g`|UG-w1dfl^D)YND>ReULZvVyF1ydiMsqrYh41ZCS_Nk(2hAO zDvHENtccA$E0p8Rd$Wrs_bTw68?gSmqfIhT@IyvdD>t7RsUtir^7F-+1TL@roxmXa zK@p|1V+CeL*}~I#i=5Hg#K`w2t#MHkwf;$CwPOM4{xBqOuV%%8Ym)IN*HzHI2RV|E zm%4AHkjP_)(P#lfQ9=qLAEG%0qQOYW84su;DAYdthQ0=ksaP^`lX%^!Ox}D6%2jFA zhul?%uQzU2ZSVZZSkG5iu>b8;wOi{rSF(43gQ^WuONCbo9|q)P?^bO}+@mdfx;t+< zvD#HwY>jJ^DAbsvSkl(`&>T>P-&A9oT-!)^2VtW+H(FrbWNvxDl|&uKR;Wq8lw6iL z5c%z~rWe``++rR#x}AG9@0x?{0pcA6lq5I{7U7H>D$`?Safd0MQJt7_skvJV9(^x+ zDFQ3u8Y=P8ln|r_WFqe4?7ueNBBV>B0-P;RAb5e5RSDKj0EIz5QQ1ErGk{K&1_}rY z<@IRCUah+I+H3=0sd+jkJA;Q=-?a-@-|BPbk2RogLJa{)U&`p4uCMdE{&?)pf^Jtb zpS#@lt)QgxSjW~ug|)Iw+}^1sOf2lVAN;iSAub|e_xabpB;9q#isLAeOIF@6@auBw zB3g(vtlI8^T)0Mk3IleW88R+m&?4XPGv{l6=;9v*z$+v0KdX)5YD`AW0$8ugH_l~ zYOWHB&!9EKUxb2DAJ$Ii+Mpx+)y}_{4ElTIVWMqV1@0fdQM2c@r^Vgf-E$clyKw<5 z-6hC#1D6pj-HWAf?B5ovDv_a3`h-xWv^J{z;gPd8g~h;getI59x&3etwJ@1P6U);g zT#(~3SO0~|KAA-Mb+YDx8u!p%nzcUV&Kh$B%uXp`rVA=IPYI|QQ(E3+5ozt5ht&t~| zm3o{vDWibz2-y`f>Hx4gP)842mZSgbp1uxbKxqe2ah8u)GRuNKp-=Py#!TO=t~7F< zmCGIOi?gDP$$rJ2waKD%Ct7b?Jg%J?rV(DFY^^sz8ktgah0(|m2^J&};Nz?>iD@gD z?3~VTEz1KIUqS%vfXm`j!;po(e(G9H$BuE8;{k5U8~jvLDBhO za&-#Ld#6plef{KE8jyeJ@zo1***vTw0ZZ)9b=wR9%1)ck2Rq|_AqChsyrJGYsE?!R zdM}yDV=r~A^qlOb;su3&%5kDZaXnR(Dz1$zNLV;*J_6r}Jl1KE2C-WNQgdRVO7U?* zlnv*X$Iu~k!v;ez%bh*oV9Drg#^{#yYp>mq=@tD$234t5_Mv-(;mq3f)h7pWKGC$H zTrLs~INoW~%wi`b_Z~tx2o*Q00#O~Sw800>>Usys83+e~J{q_;Wx|S9@+Tzi%Po)` zguI%=(A#H{f2jpCAjRgYMt{4|dZb)!|Drh$M0Pd z&9B>VAjcB{wHvX7uc@wxWh}j9t)!X439~0oAl(ty?1tb+2OF-lUU9%{fYU}Coen5u zWm5y;sM6B!5dlrO|A^;R)__g%Jk7?9k1U=g|L6yPxaL#Bn7vA}TTU_`#p+!%d(%X5 zUt|W+YsMrpxcW~r50|QnZazb5g@x7DiImL$VBU{C4F~K$6pTk`o7bHUAJ{WP3Bbnw z_@T-sbFu$Bw!<7;m3ccYB+pG|hspbpxG*jz_LXFe#}cUBlRMuv!!~`Q95oHC8pWnS zUf6X>X$o~uhWIE~{R%Q~1R4N2`^hp{+O1FxJ!j74=9R_BtCDC&vxK<$#0{s<3N2z+ zhT>u>R3x+#Ilc{-3>C(P2=2WQQ;t}?EOb`MnqCQLgbeu0xcT9(JAzWr6?SN4m(z*9Hw&0)0;w{z#nw6rKL9ezq|sw!px zN}AY-bg-N2(ctd5aNvklq@3Yflbk(kWVT(D>BG6R_uTgU zM7K}!u(8tA7=MmS7fMEF-)iq|QAz{K$vkbHr#3A1#u9dm_b|P=Su*I`z zn}@ozXaCW1`I_Ol!o;2Gso&x})I6F`^;}6Q!@;oqZ1qXH`YMF<+`fOwn*}I3IVc** zlGr>QtLD!Rq@zn99*#WG^SZBjDi0bnPH6W%j?Ef5-nhWZQk{IVvd!+* zL|@$ju#W;#UPjKZn@sVCCzI0A#$6c-$vo6#=;*VXlNU@kzh$A51jdPM6Uo3z@ayQ? z;vTVoO=5F`4=j%T(_R%qYJ{B4)l7R{*E5EOR@LNW|_dRhK~{BcoDprGykVbmvc1LXO)DRouy1{-2erDVz(ZG+?Eegf#dMJeb0uoIw*)>1zt`Nmv z05Cf#6W>LCfS6RN45fTPuqr}9seJK+l!uD2)DdL?XpHApQ58H-89rF ze^mUV?r}JM)K&Zj56l+2#LXr%^W(Q=p$wn9fdBI;g-0&|&jwC*?}iT4J%>U??w7@G zLa49L&13N%Y-BZ9&)P`({wSk;(5TOm&o~!SUe+qc^&V6T0nlwxRTpTCw09_Gg=Pb< zqO?Qym#|dY^LH!M#&epugV~Y2N0iKURIcKDWPSu;S?surTg2Ww;^Cdp;m`^DJ;4iO)0b$gH)!p3fUCvteSi8i7Ujw3;?vM3j z$-a$s8sK|M{CZbS8bR|LC=!Dfo|WCdL}2c&R(=8mh^s>bppW2TK5M9}HwwDF>@JQv zJsTl~6A%zFTWpv)VW%h(PhlQ}MVg9fLH+vo2@?D<+nA93Tfo@B^65s2{vV)y5T3HHYV+ z?%1?}VaWU+@!y|+RK(#niJ`dod=UCiQgg4Zct|ID+!sZO^We5W6viSti3SPNsg;0HK zlQQ|W75pxYD}($8^x)8thG)v4QWan8$aSBtH?h>&P9lUQ0_FAbO!yoGb~1td<_7X5 zSgXg4$gwC0y&zU!BR+7*JDf)7hursQ8Fv%zQuvGsZvI&i0!a>M(i2Tf7HShlKW^?4 z(@kvd@|Q0pLi~?;!Vn!1fEkDVw|oGQ)wEUGjA|?h7-HC=#IwEhcgi?P z*R|~Tg#Ar&Oq*bl?Gl5LJK|vy-%?$qp#Ziz)kKGf81KFMfK|yR?LnGVECa)ncZ)H1 z{<4H5$Sk22lU!o8mPUo8MrRTt4TcSuA4XQOvz?gDPe1&D$}v)FL~o2wp=^~UZ&h6Vyp+bAECsOI4-ycA)tdxxX+G%UKJ%j)d;M)L`{Fd=#c5hULy@ezS zXpC&j1lAreM?EeA8<$+ps~NB}ZA8KTz4f|^!M%QW%_A|-!$D^@=pe3+}}CdtEv-O zOGy&ZR8tdt>#-Q7z?Ld^t-kM5B#`a>whjKb7;Az!7enU5!zhfP-O-!(4f3lh{Tw<6 zF@#n74(==5)c-wFT!2nl3z-H%mh7j1@rvF8XYvH7w-ErWkp2|1!g{IWziL!@xJx?Q zRkf6v8NNB$^(U1QE7FPbgd44gz;5xYt+nCZIHAGuRQ5mLuIBZZOk|=AVV1G|b92e* z8N$L^7VMDK#VAao)2>09P*cxVlT+8xxPE4iOv z4~qhna;^;-sY-5(ejT(U265WBPzbk1|FBhFlV`=Z)+v#2wzm458r^_l$%Z1L#D(Ap8*!9{|Fpd&OF!ro$=kGdt1W!GF1DP8*olKmLWwVc^H_ zQe*=^Ub{je^q>}~6COThHA^7bBpfcGFj`6xOP}+Q4rsC+CdcGiC!P9PNjP;}%iM|` zy!p5mNw{^@lyb*OJtmm#%?GhHjbiS|M=D9?Q~0$y@854%IGPyS^!x38vOy(4{=(05 zsJzT=Z;h#U$+{GNl5gyO_yQay1^{Tu#Y<7Lim8x$Upjw~kfd$LM0bA19j*Iv#8myP zxe2`bz`-N{?(*g_a>Me_ad<^1upzI1nT1lr_l~b_k$^#g^sr9Dh(zSUDnSF=05*h(A{O9r5$7CkA=+$K(5V zJcC;MJU0ej;Xt|kCmHA^i$+;(fdl+*FDf45dd*lZYOF%8kjsk(@pM!)Erj3yP;&%K zg9e%Ek=}$J6~3z;q{E}CGAWymkkFE$mezVRClMF`SMPhTY`Fa;d3i{iGOngn0a944 zzV0|5Qs8EFgNvkoFMcZv;VXC4T}X%qiyjW?j4R0z1dsK25>Xe`ZWaCp4sE@p-D7v- z@I6zQ+*AzG81CLu#QDu{>~$J>^FV6;tXYI9NsM58m({FBFQtX{8<7F&_wF(_j42DgYl}!Pl!Z_$g-l^ z{_d0?+m{Nrb;WlW8&fQbQni)y+Yb@ZtCzN)JTd-}x)BY$E0xm}#=n{qS6p(Pu8=rT zhfEnD>EPB7sIJnZ%+Tk_X%@Q#x8mXBfe96oY z>^Rr@9q<<0JR@t8tdUPI4qt;jRDd-M<50y{g)(IQpGAi(n;=nf-81F@=!{I$#G9s? zQH#`@6sq>iO~zoP$?ct6c2&1JYK{VhzMuF@H*=Q)SU%n5`E^~azO#@AS0)J1cGGW) zWB773jkW57vwQEX~mjqIN=8Tr-(t|Dl-W++FfE;C@l%83~~Y{u*Uh@D_O_^ z{r1x~(Hk6(6gX-?+mW@UP|lz3*Bqa_6jYtmn$n`lzvwq|zGRDo*jWOxNc9DNRyAX= z+V2x*s@N89^E%Hfc<0>cChlD4v5NpxpXil~XhJWl)N&nGOhi(S)_EmCk)1T6Zfl+r zi7k>s$^1Pg8b}?ZNSz_5(G)xMV}2p)CBGbC!)}+#RC4{YTl-&~aBiP$O^dP|wt-?E z2wr$_H)FgC6sU7JAIdJv#FOC?<^;_k7eNRCkk1q~HNN47S}VJAaiL7A8~vJ5?=Y|B znSi%|jfmNr1<^HGDEKgcg*)v%crB4m3Bd4dk5{^7%Bmu<^tnXC+v}ufa3eYQPewcL zcZrANKvU%o;K}M35SNe2^L_HI0ld7cEXN7zx!>yd19FM?qv=Wq1mL|{YkC9xES{4f zZEU{W6YzySW*VVXTi}P}0xwDI|=}C#FvqOrD83uDj!VYKz3d7Gxrps1- zo90hBHr8RjwW~jt+7BhN3fO^~E$0IZttO@930HC-2tWt=96&BRn_V=9CPZa=(>h;T>e;XPrL5> zHozL?R5hLhLdP3_k)G%JkUHT^jet38?UY4$GmS{4yW1KlV~n~t1$Ew>zVZ|AKyQlj zVtJV;8%63Wv&_#aN<}XvN-#Z8f_P*0Fp8Rfb&2{+>Fe*4^;*`YQc2sw&HUzRwLcF^ zUx&z?Y^ig~4oSJ%A(lp3C$61{T?(DxPWz{NentRBNcA4=P`|d;spmTM zvlRR~Uhq4WbnH>7so+O}nZx{@AzJmPaz0c8|EF+(Um6QF<2}2hkGdBsw*vM_&*4A1eTrAq8dTABIw@&(3z& z-qs-Mg0?Nhuu`e%%5S8FHD&kxMT-jRcHuj9=Nf(0j&Y^+r)lX)6Ig8-#M6UjG}Uk2 zM^0w*nsrB7gZ{xk!~YlmqoiHL|x`x5qN0!&Qo z+4=bm^PB$tr0zy7+}lZkbj~^5FFxc|uG)>z9|6g3#OH^5qUT$=NgA~pt$(8U9P$A| zN~i5Qdpv|u4%X6Eojg6IEsmyZM>^}aKm)L=GO`xg1Pel1 zz5YEN2(mOFZ{^zNcU`?Bc1mixK%GVd7=v*H^N*ra_cwm^l8lqOJABTac|OlVIKuTb ztS7-Driew)iwfhu-e1`SSinsZs7RGtCp4~iQe43$ZuRvKN1J|}F*d4A8yr{kL}&h~ zmLAVOeGDqq7mKX@l>()#FqVAE{;a%a(#>at6no`HPD`sBzG9S}h7~5AGJCDoVnZu) zv;FeKbNU*QeETf_cDVs}9g5T6$066^wd%YXr_HxCjn#uWls(&ju>e>abCrmKN@2{f zMIpH^LBj zF9}pnp&&?1N{}wXgqh&A^bB~jm6(s4_@Te%b9(k^suk=190p%J_D?qE#&jGzzLd#J zw9P#*o-a?Z<+=Exw}4C;O`KBq{kXGATQE*Qv_Sny8RZT67j`$uO8*v>$f#=MPSWw{p&|G`YpWOXlHF7(% zTf`))TZK#<#T`%TC1Y>gcqajxAe}@BF-o>U8e2gnK~vNrs5aSG-*ip}0{=?*(^6Sn z8xg(9CHL*wdD^-CyA@r17mTrXen<40B9ZMIlS zc_MK(rE4Qnqqs~-$GtIkJ-|Y&J&s=>ayeWs`n1+n8TU$`FEW^y^)E7|h6{K4d8Y>d zhN+#mI2nkC86HXWo=z)R+rJ7cX5Icc4|Yi5#Adkmua_K$-DO=Lc$Mk#Ubc0?dW2$W z+_k-Gyc#r{=(pRK%1aJ|E#~$di8`N1QyPsdOB*-F70mvqDwv(RfrYl}rTjE0jpJ2e zi`Ik^@}*Y%61suE^P=`Skobkx=#2}JaLd0ILQ_(H-A*ld`^1|x{DRJqBP_C7Wr8}- zQ=QRI)DK;JuD!GGq;q0hC;VVGr(9k#eYPvJH)$k_%_%~vjoR}E36yHAt{<3=983v= zjwVsyXaZB5X%gPi`c3YmUs5Ce_4TtJ$2k%&ws2p+CQnMk2W+QSf%6R-;81mqClz4_ zY)Ua2D;bpcw+ehNyua!kSsy~5AcHhV1E;j`5781-aLHdLUvVgU;-D2!aEIYcb2rZZ zAU}G5STR%XchSjq5a2C`3()VSBvRnsRn`)NF@FRdA-^4uJ8@6B+s8p7(Wp3(6m0-N zc#utaNh8VR(|HF3sWfa;VLyVEn^?zwT6x!*`s-9uVDuoeckOL9jL>E8`O#DgmQlC8 zA=N{g4vEr(e|nl}hA;z_--pE`mIlF7MIVT>%!Fj z8CBtB2=M-}s>d8KRLkGhFI+CFD{i0ZWxLrQGV}^c7E78_6EJ*jE+nA|d=bdgQQ$f; zG${dech+-*2|dQE?|o+4>?0g6J64F3IC|fyY@pE%!4Z!s;?bsBOUno?V%^t$Nd@R8 zADs2v!?d(*o8;2@wXc?EX|G>uLFF>laWZd#Wvkn+p+r@sdq)42xn=t=v`D3g2u4-h zA}d7$njJd&h(W=k_tleCnj~k7FOIej!^P4uaEaPm%JBvZZ@Cw)QuapnQi$c+r=*pd z7FdKZLPF2WpL>9DhH+;XMv1tZP;D@TzO=1EAbCcm+$aB;xLG3S1BF?=Y&84$kVN=r ze%Q$}QQ+Op7B&l&!oE4iD*_V85q?uOSV99Qf#7>sv*LQzgfGK1e8W2U*2xZ&+Hql| z7#3P3g$*zmeRMR5YfnGg@*NL3D`V+2PPJYu<<&t=jr-78-(FuKu37Maq)wH%u5)JF z9_SB|$0o9ERo=gHWreO;DGA8DhwAcx%YhNFE6gw*+V^pH<}8e70!)?|fPsye8V_&4 zkg{$-4xPPWqJ3tfvrHvMxbq22Tuk@4FJgrU_uuKEC9D&v!GN0$3ZFB!fXo5 z^dj0TB-$S-F>&R!R1e}DV-m(p(3?I;B!0D;EgRE~7ELCQOmwk&^M(B&o6d~Qv5+eK z0YiCb?1OSb_1dp&@dDcBS)X}iOsTFPe}or1?3Bi-fG|Z@LqA=IEW(~fx?t8o)N#GS zZtl&wS^6&rjM0wzucy-U4oM6J(VE0t?_EI;&}<5!YIo9t>L}8mdi{8%^?f`YRP(LX zpNcZMA52CKS4Gc_8~FZnuwEZVQg-mSVKx{wa5fBla&R? zui;%y(TS-_)l84hCKb3sdNh3f>?0V-XI{Rva!Cqjv=osfpG{0FSIXA(=rx&EMv zkqZjC5XLBuHOTY~J}+5Mv5#^w%pC~K^L=?K@4if~;gnNnh`{=f-27&E0>w^Nm@?9v zk)*)w6HSdp208AOG!{Y7b^T80>kQP|3+=~;p&j51=VO%o!WK9e8b)N?UUjf?K%9px) zMblPvNBV`NSps|jv~jkT?q^+e)?U=fue)vtjM&-)Hx^6H<+8$$Y>rg32KyHJKG~jI z%KQxdp3+~uT54W9Dv+W-c^VdBv1t39nJz}L>gfGhqTXq<2;hf+ZWO3;3{F^*^55N6 zkj3+t0}UQ@%uho1yOH6FH(FSl&TX(~pmCjmLI0BN)jn|~0Uu;PfF5^GRXxp;Lwa=k zn^PS>;GTQ@)+HI=^>VX&vx2Na0Zyvz_AZh zN&bH3#Z@t4=`(27LIC_9XgP55M(O2wMHT2kt`O_0auDIq!3Ujfa=TdnoNWv@3YVE* zzbHoJf4?0A z!e9xkby03eyQL%G`nnN4oxMisV6Z~46O1Lv&G4A7+biqyI=b;TS(=iMq}M$!PMvzFB zP2wdP-k2N+54%+*;UeN<#Adr{5?->`$EMIds2=rUDL?jEtOX7_%g8LK9blpZ;Z0dR zPUM}58oC(_|F}6gF|3@rpE-*h)826IXl27g;p{2N{gACgU-#R*hPyf7=<2I$dkNq{ zNbQ6rT02i>+skomqHAXI>op;u;JMYFE05He6be~}yPEWfx*`g%^3cE1P33GJe&X0# zc(-q^fGO%&GB+bthBH_G{~p#E3L?~$q1S3?+)V<+a^e)oghc3eZ*W}mlpDEq@4H&x zm!hsf3_si4`9vq)F6P}EF1yr70%PnA#;lb#orZn;v{Z?qEw$GIU{G?m2`P5BL}~&D zI%jHiDY7YqXk;t)(paa^aZ$^121Wn;|IYD$(W3}2T?j7Rqa*Kw=jLdG<_au)!%vV2 zZ3_7-P(Z>M`rA>%oO*wZ<@IEbP0S_#h!*9=m(04GY}98OJ^Or}oU-eGPi`*UVTs#3 zSiJv?gTt1B;4bn$y&kVK$GuMa{rLxcCQ>geoZjVdZ|FX*ndDtvU-tY9_?uf>tS=5V26 zPhm@ti9q+WwN22n4hm`M+C#s!hCT)VfTPTEZ2i?({nb-)VKIId+3x*TwY4)a3MS1e zP5g6DMD0!1%t>D}IbF-0j3@u!LH^b4e-CoDjepKiDr_5whfNj{k4-!5bK9d-Sc}X` z+GIUZ@a*y4J%N4hgw0KarJv=oy?2NAm+#vCF76wH>y$k5cvv zt50Ug+L0%w9%Fw=d$GLDoG|vf@Qa*vsM9|o8gM6GLYUid$_K1O)R#PnKe+?O#cApu zm1gnLRQx_9PTfc4s4`2k6`X+|@cF97xNbxV zU6MEp%<5AfJov!m_SHU%O*TjO(_n`X@>;Gg-0WMI>#)jg&8!67sTiH_I2xIsf!=C(?TTAvHX* z8_l=+k-19EZfrM__e2*=^vP#RFSq4v`&XQky;^gTh8Hm{H>OmLCpyG=I$l&{j7QNt zR93*2y>z3ydT3820bCG_*Gbk%Hrixjf#;?Ul{N154{OHyKe z!1Iv)94k5Kf6Fqh2O^NG;$JL50vpw`kr+4NOfcYo56w4pF_HFR4&{gv>U%C z86?Ttccq5EEM2^xLmmP2PnPuVYhqePo^Kq)olI2Am*4+wFq=)Jtt?hf^l-(cF#4t@ zKOHJ@;H6UdQe6mTP9IM}ebM!!Cn`!X`A|~l;@sZZyepd3dgrC{>y_XVj~_1Fnl``8 zPKXbzVT-RuV7J(>`lS<&SMK5ix}y1Dar@!72@^mJU#?JewpMe?^j`Y z9Da0j+~uXj;Ha8;b)VV7HD9^>D}00<_tHi2sw(!z22(YOz3^_6-+Eg8vW=pb`TdCF zlNqIVyRH?fGH2$wqYA6gU<7eN7bOqA^YqBO)sJK>d0(u3{khO7fct>ZDx+5z*nfKY z{w=fdK)Hq&BD7UFS*t_#vTGSj#a{Pho&9L#bG_7Y#~VslslhvvCsrTwm+Hh^*w)C0 zRfi-p{+1ws2Y_nxks7R}G#9LUZ>76vZg4}#zFN>6c~~Xt0qrOL$#rMvlfsJ3l}v5w zK794428O>6AM^rPK-r0qcseJ@dTa!KqtbEy-m{qy)|Cw)DDdbfVK@Ap2PchM*salJlzUVoVs^O#m zjKzE^0_bH8G%LqEL$1FmO)=nT)g|9zNe;4K$9}`b zBfL_LTd^o!+kt)zMD zumAJP05h_91mulYnNtG8AzNFm6275z?vXyz;F>Y3NV_@I^~gPwiT9fv>&&F7-^E2heH)oRg^}Jd|NKge2VEX;Iei;D zr48TJmP}^bNOKm5vVV5%C8KKHJa3h`{V_?7_=?`8NS_SO9et$xMAusc#_tWsuEB<5 z=QDS!JM_MNS`+4K61KZKssqi8fkZ8)x*8NfSn=HXg~QQuXbEUf#OeV35Z_;yhUA@Jg4xlpPgP0{}JnVD*1-#0KzTtIB29;o- z-`#&|om_a(wVLDlBR`@6k+?nkjRf^JYEW`UQUjnC9&`c>C5*q?01a&>wsaoGzJP>$9<- z@?o)~g)dG)kB<0k!a4=TjR+1`(QgIMYI%GDN%~<$QqKxZjT{Yj2R!8IB`%Pq+1e)8 z2kdvP`?glyzwoU4D3Nc2?59vuVp)3WBoJhE=gviB`N&&4quvb>I*)f;pQAkGgM%)r z80pUUU)~L$o<9WgCZPa4RM*B^cz!^aW%ywWm$>^Luo2V!;_Mr#w(5=c_Pa^+><4O& zTD0QW7IXsQ2M;50F8z|l%k*)0^G1WhbCgce{@hn@OYQ)4E zF3AU$2f6oEJZVGLv-jNL?cZSTsob@xD5JUlJ^D<;Q6>BDbJ)+kth{*I{>QlHn|Z)N-m-tZLJ@V6~H{d-j>i z1bCFIsE+`?7nG2|AvJ!5b-%)X?lVMuw-^(A9xf7BLXplIRbY7V!@wypv9^H`3A-8^ zS$RHu+*^RjiU&5^Ph4;ejOG}&-H1e3b&e|-B{O_?5ji%5JPnH-r=8!#LtG8TEDny< z<{Q~o!Kj{segXhyC6!SWD$0*#4Q=7b1E`ii2i6Ac~Ft%R{saN)p76ArS zav<~>42=wB_C{HVDH#IX%P%P4HZ^uSdJhA=@T&nBFtFt2v5b;P?fsD&f!FE&Y5D^d z7vf*i?^~HO(J^nLk9rr+fQN(#O(-V&?41m#&)C)BGH&Yg@(+j|3^?N#Lfx-81E{3X zX(cc;Z`I*&9#0PT;T=oGcZUcYF2ip&Kx_dckSNWq;aEiuaLL=msP`}Zmx2Fhm_P%F z$S8-fKXVwC(w|9p{)9cAzEQKH-_L&{UK~%!WvPnKfz9m-@O?T{I zq);Wa6hjFMl?Aj7`2WP+>)kMN`S~CXyqck%Pb1W5!T!4ZKRGIu9-z(WNWU&+`Ol1D z@CRTDFLIG1SIOUr-kTVRzkpT(^rG+7+!bp_hhqERAEX43Es*^PJ8?IZyAE)dVU8B; z@?msMyKXD3}_bd6?g^}07&2hg!OQ_Vc@DP zQ2%O9r=U^eNju*l$E@pb0~54i+SPYzo^l$m+Q4zzzw=VwAWeC6>wO*Zb8X0F=nI$! z$-Rqpb~_vM_~Vm!UmLtm z(xdw6l<;ZVD-ZCdUU6&TZ^0v>jvooAM`HsMPdO|{X}rn4oPI}IGbQ0C2d?h6VAKri z`=z=Qa&5U39!6!{8R$5BFkfG+`~>R!N(n*3oy)B!sWoIihmul+-(I1{SfRev2lz2_ za7+eD@=fyHUQUKFV@nR&>dOzm=D+Z<~1$dK=`+`{a`ew2)6GnN3hH9&sm(q``a zX__yVhLPO%g19{|xyMwO;M?W6+atqwU%;rC&>J&-bN9y~u(7phDEWaAIMxkJX6Zb^ z9O=GN3s-8Fu1*VG62lg?p;Ob~W#Z$a6aPBjThmjI#k5JBc?gQ0%@poTg|YI4A6c82 zhl!tLYx0hxEI)PUA#t2#j}z5tvG9j;CwpA&(+jYb_OKkwmv170Ij;{P;-=|XUD5Lu zF!Q;Slh;#)A>r}O#pjPQTrm>q1Wi{trvVR_9+=uO5BAhg=rDso^oQ3^RA3R z{<}1}$}a=tNc!lXG8tU-dswRgRGJ$GP7AQTlTGS`UxLZ9Ci#^gR?J_E;yqO>Do{)fc3+NyHwL)uFB<+t1~PYnk;%$@D$y)rA<7lu zRgG~U_PTVRJA>Xn1+eu|{wy>!$3lv|8&CQrn8SQ1SKm!?d9=77m%!{=`^%C-^zqTL zFRMh`gFPS8^=p{<9Hen8g8P;u21wZcYW8o*nwE8(6aAmRb^0|&F7%-F>CX?Xm@Nb? zG@VOz&`>j6MQl6l*m)Oyil6yrGYT}Y$G2>#gTuYMt&vtmz;Z_Twf&HqWZQ18FB98y3GN} z!cW^2NXGh1FH*MB`5vIXWkkeGg1HHSk^qFN7xgjlK%OT~}IWGcDBu-44++ z1caNUk|Ix)xIkAW60}+}Wo~auoF0$)$62tCM!GY#3<0*?od8yOlrUx#>^Zt%p6B>T zlaE+&Zu%>$?-IJV1#!a8!B}CkoF>e(hUJGUl`rZYC>2dB^Lc?q-149tE~AvTu|vyu z0l{wSH)ECcr4G@JceFsuK!$GirK+z)(NWQ%oP+c38K{sZwcwcD&yC zrTKS#(6K&GBp~zab$%SH1Dyh~G6=(;#~eOFa?>M~{bMop>9VLih0L#u4o4~lQFrT3 zRLYhswrj)BdTV-VR2~!`2Pn6LmXiW-|57t%V~y#L)++sAl-hH5$IvZ5!XR_Mk7kcQ z_bxAMLw?gEBl3N+-RGpsd)ZC4Cf}A>yOH^{KYd%~R@p87prg_sR=LD_k5(mK|KzqJ z%7cx?LO;3>e!QuI!$An$&9-nM(hBDCcbea~zRez>iY4u|5-rO(VTN$WXL}sW#^=n&9Ve?nEtYI@PSAJK8sA9g=mr!m0Cz4O4 z>HGZcua8J3(*^PoR=9L0due1)8bm7m1Ptcx+GWmkJJxUoFyB>k=V;^Iwy+*fy|+h! zwVcf2=~q%JHf&iP;{4PTJKPP6f6E|;KWGcqXi|#ymTmjzRf2aI4ubK;u;wpT;;-=t zFzh65WClmI#%9VBAi?X}a223P9@Q<{PhnqRD4MjyM<7=H;x{NXaI-g`l3o>1Jf_~95@^9*{77OW1k-)O-TT09WdpIarxAv?2eszuij$- zt9X&VKAjETl>1QU-3?+c@C*Clr;tKYGaI#7zd&y}aS~O`-`#cy9$>U;^au7_G@W<8 zYplzDzK5++oXf&z^=2LkdODFPH|G^o@9Wnf_`CfL1OVk(Azil|e1?S0Vlz)e{+#S_ zMSeo21IbS_Y-J}SEdMWp6ZU!e+Q&(eH<$J~(v4e}%AaSAd`hxbo$QsjBN=~R*o^0@ zW2b#DSba6kO1r{(>^zKbf9?$l2#YoASffbtyNU}elabaW1j`&*iD%jU>e}~i92*th zIA^H<5zTN-f3 zKa*4s#9J?CW?V1zG_=%iKHPj)C+h*~W5{EC5z0=-tLEDOdlx|<)x{UKx$vC7_Gq}O zi8(+lY~(OpWxiZjB*cAh=yrSpKyryKi(h@E)V=z)DQ-kvbe$ir&A9*P8BH03kI+Hs zT!y|4Otk$tRrc-)FcHPI zNky=`1~Bg`>wmK>=Qx%Xzr}-X0kll*rG1fh$r?2_XJ^hc>d{ z(Xo?${s%9C-9+4vf+c|kvJZNP_V1+u@*a4coOetj4gYfqVADSU>kdz}#k}lwgD(_m z2cri9!Rl~*=V<%S>gWS+Vb0sM<7ebg{-LsfjS5n^iK$fKu&aFl9EH`OO8+@{?ikBU zh712}Ch*Cf6q^?a2nZJ!Mdylw;1?9=sJww7n4E*;Lf=yk{slo&R8FMunV!%81Hf+X AoB#j- literal 0 HcmV?d00001 diff --git a/evidence/scaling-rps.png b/evidence/scaling-rps.png new file mode 100644 index 0000000000000000000000000000000000000000..1d806eb61491b5722ae7af004f7ef79fd1c442a9 GIT binary patch literal 35654 zcmaHTbyQT}7cU|xA%Z9vNJ_~N1`^T&QqnL(4pP$HEhUX84&6vIGjubEgi41)cS<)S z{qA6V{k^x=`^)9!o_qG*XYYMJyCy_QL5hUv4iOFx4vCDkgbEH0ZXFKJg`VM z(@7j0G8`ERF*VlIn>Q%hgS1#x@bi?e zQNaC7uayO7Jj)}_m8O_SR{52`q#$CGlvESdY+{oe`WCx5Q?I}rZS?-dx@E(Uc#n0> z$sK{{&5n3XN_0#UDIOip1p-kVT(Z|Vc< zKBFUCp>vJ4Rk=<6-|A1@cw}`VrF54r5J)Jzevkbe9gc4R{p%_DxV+Cwf3}!;i$lqT zdjZdep@W>~*{^M}6nu!o@vCh9kxStFl~9TNoEl*H&^6J`)<_{C@n0{hMBvN@ioPJA zb2ao!R-pfn@MDJ8Q+r|DC;#mXP)ipIP^26nqxzc`Q}Q$p;OSG~Y45Z&=-1P8uIyxS z8mV%wU$55u5P&877ptt{07(y?&WT|+7o8#dq=IDvpGHg zIq>v8^iI%k0nw@8e()F71P&;MLUSbWKd}W90~%8mcPTtL35V(L8HsKIrxx}ltnc?> z{G@wL0SKfSIE-tV8Fc?_&j4)1JX0uX{rI2NZ2<4t$J&5-0D7h3i(?3!lS({}A1}-# zwfL{(d~eXbo+3#w%J{X(IqBb|a2gXm9omA=wJZi_4@=T-0h<_*zUq%z8(3RT=keTC z4G|(M{L3DK;DgR)Oz(pQP7MRArt$On}_1tD3A>gg*omjpZFau3w?X*6AdGzhf*fS12Gm-TnWBEG+r7d!hxMnM{Y#mQ1i_MD}%C-Wz zFV0<6iZ(8O)M(az?)4Q9T_q)lkAT=(=tPyz6(`sR*AC<=UQBZ8MJkmmBrt2`AZclO z8-tQBc*SG>Gh%=1Jy&wt?&8@&Bsv$0ZVmroavKJ^|BY~O`Fk0EKTVk!FLZkBOMkln zEqlpzE$(s=s89S!5obH&>wD!k&1nCtdW{?2)+QkIe2QJZi{%A@iW^|BvIapCH@pYemOvw z@M>o?E2&dzPkkSUA1z$k0#?&+W<@=nudYcrMUekgl5v^JXrRK0s&Z(3wO&EXPOmJ3 zJ(j5Y`d69=UC!%`v$pN5<0Rlm$iz+vxsPP)5l&pEeN6|KAJfOW}wR)Xv#N#oI^B8k>IMqVod$}QJlxNglRZ(d) zCxoav`U>sd2N~N-i6w@RigEkJOrDQtFsNok9)?zmVmbfX*c&R+LFAz@!p%_o;N((e z-1rDGXE1i#rYgI3{J~tKAgu<-CvfjI(8CzHZ8+7?`F#3s@`*7;UQSmj6LY9y}q|y&u9%eb0 zgS$=YEV-^I6M5@HZNRplmv6Z&=h^gaWOw^LqwridgixPw=JOpT6qt4IQzBRm zZKFRg_kM9?Xm3?@M;(%IIFRp&>PW+j?XN5txRganqn>E4?R5j%mju(Pept0}5B|LU z<_J886!%axjbgQj!mLwO)7If=f#|nee(i%d1KzBu<^ECd>Hul3t_ZT9?lEP82=G;X zer}+-N9^7s1M(q9Sx4c-Y(_MX)2q3*F~iB1ExaJ%D~GF-Zv;q(zfle%tRbYGrOUUi z*=75QKDIJ8t`5LhVAa?<5a4vmOGs0&0 zl+$JAEx!Hr3tfW;_y-=#EA|JGHJ_hDT#N^;a*7I2%Q3l;Mp?=XG6-VC=7%s}@ZEU} zsyss?z$0`W2IgBjpsM&P;IN9{uzE`EPTV z{&=1M#wapBh!NARa)TV?-bR>h&PB9ClSw=3VKsihp}8#G(l@VzFZ(rfcPm}fA@GT| z!0CF@n*vA2Z?0R8yrZk)3ISCMo0*QQI)oe3w*mz5&Y{gC)i}RIZ4Z zIGuBK<;2^g(S{gtM7#~+Pjg$6D{*=WQ00v%H~ZIC{SQ?f?pz{08EH}Wvl1@QkJ2buE78u{c`}a z6&NyOYoOo+d?0}Q`0x{5FN=pCzM=A7)BK8f@Ua2YTuUE`xwe3sbW+XU+y*FhWa`l! z6T9jHp*_%`e@<$um9@t<6{SNlq+vV#YK5EuX8PU5yA9^$>1p=%vDX|bsUP-HIx`i^ zS&?J?VDse*g#ZfiBWzBdg2lZP$wD-~sj(w`K40r6!N5oEql|W0$@jKp&E-PhXO-5- z(;JNJu{VEQMcQI)VOl-qQ7J!sJBj9QtaP+Zkv8+LaFwNP9pk=4UxoJbS>mhdhUlP+ zA>XGSt>4lS7_ac_*JVy$t6pkcGYd}#Bw;X6WBhngt$wQFbNThLhEW>GFO$N-Baamw zBb;cKjCYMOeE1hnN#g@(OfAm1mEhd=0m;beZb6 zk0+Dn0RAeym}vhGiXxN&P*ha^n_lcQ1F6?3=P2q904Xy%Z|(ns<8OqC-X@$d7#7w4 zL)SU(x(&P;pPqI8@K0zF%nv;5$icbzXXDqR+t-8V39i)@e^i|OXLDl(oJQ@1;3WV_ z|AQ%eSzb>?2&-!ShuE|3iLT`W=2$sepazItljwg~OtBxB!;~bjo;t zjLDVxwG|m_5`g(_h3Ed^h>;Ii?x#q%AoVw_xj^ud8F*;kC6M_ByNB<0rcYn#qwE1N zd!Kjy%B|m@1YV=WzSik7mzl-*-;t1Gr5`W-=EC`r7>mCK)8qIy1MJ8j^Bdv)!+0Pz z*1%=iW|#eko@Nl1ovtg+SQe!vaEGn{pP{=7VE$QKl?Q}>_ZFpd$1|XJ%`_3>_(vF` zX#~LT3^?ydfK_R$1U-+=>G~;#)3|ck`0zYxJ12{bHBR*QBAsjdtp}Zd0XCrgF#zbU zI9{UupR-^B;xCSCJZyhA^3}MOHaP>F1!e55`M*H`proO|^7L$CLwYj)3xBz?4Z!ik z5L|}-c~M#cH=GFf&D6X+U~Ye1#&)eu+2jCtmIa zUn539maUq`FaD7w%>_JUke~EFAq_xH5I~LH0+HPL#&i<|FM)74iVWC?@S*&zKiCvF zVI>tnXz$;L)zSPHlK^%g4TN~&KOM26q$?+QpoOKi7>IZyNL<|iiHJ5s0Lz4?w#xJV zL(lzZfJHG}?|cB9eZalfjpu(NV*>VJ_5FA|HDNmJPcGoE1aN#y=#&n5uGM9kNE83_ zDLZVa`6b%l3<#+HhsipyH`SHDdj%6faUoZ+2a<=3}!qoY2)_GL&k(qf(C>|BpT!wGx5%&~!R ztKj1{74j%iSg@CCpWJ~~4Qdb{8w(vxj^*tm2bqWK;%DJb?XR4l9}T#*xiV407>8_^L0cj;@Whd@O3Y*XLF2N%?;)4Z+=bOF8tY=v!?+hN*?1%%xkCvemQpnB&ND6wquyfR^vxa?T*yiKT>9Xem-4Q(oimKaQlGoI@ zau@ghvzOf_sl%0vsi{<)Mdg)OiwlEjN@x$y8o^29!>EQWjPnB!NK*R23&4g&WeK2! z#35)~(z|B()?bCLTYcZ8qNbG>vgMxLbIX33a7Wh(xofxb_;yn4&1Sa`flm@pS z>h?v!yq}?47IEw zn4=9jeIc5O>VBSZ?=&?~JvkHk+r~x;_P{;3AYuH&>)DCI0W0Fwr}b*?nwR$Sf?V|y zxcg#BPnQ)lAItSyPIE$>Z<&&OJxj~VmF9tK(%+r4?6^kb? z>Svhw$P!F^o+4W1QZP5;z2epr9}JtcDN3%z3`s6JJL=rzF|z7Z)%~o2tNDyy@j0PQ zZp_7VPlTscgfz5@)i7Gf(z^Qo(rHfY>o*vJ> ztyqt|wr&-II9!mPq?0LI&JVneu~E|}S)P1!l9=r?N(8#uS!XLSEKz!5WYtdLLpx z*%(=w<)DJ-Z9j-onYnho7KPMq+n`F6e`>Z!wwX8uG1dO6h>>@I*X66j>34UsxI|i` zO}&!rmSrk_tbKFy>*J0Tb$9cy>`1k2hI?_igc2oZ=A-k6L8?-T+K@fM)8ng+{OkN4 z4FW;-5J?QR;Ocnn-dNztEHt!PmzZ+Mm)4sVOl)t%SLAF}ynI8AZ#PjE7Ok-ybA&Fd zJ!E;v?)$MN<;e;!fA?)xz4;qANI465Zj^m?-AR6%phNay!+Wi8-w~NsV=fNhc}ju> zpOWvp8`h@KO)vemZg*J|HdB(7s0UwFDu@lfr1Cpb_Njqr*DOP= zOgruz0f7K_7Dz9N`_+7b-gsE*_KojB+YE$#OqtdVy^q)+_Ms*Na5G=gPbC$W4(n&7 zKZ#AIB@L7bM~J=G-gIUM3XTU6m&iD?eszwZuS!}aRv6Uaz666Pc~#a8Sy)=5ZUsjy zn<{_xO3h0dQP(rq#t`M_lvsO|oRxB+?!#WXjP^a5$44@Y>%j$ir>{mMvxINAoAA1l z>tZflASMZNP|u~MUo6t3!_Zz3Im~!_7ewfv)Wl!yV8oQK{r)spwZzR&``Cea?zFyr zYsscyf)w15y5}-(rAOoQ%vZlt{$kkh%)6P%q@6_EQ8k}Yy$+wv&UXoJdM+j#PN=*O zFO?z#BsWAN?ZhlpCEmaFey4^ zB6fGA&M|T~(($dvgon0Xf2fZsKL#4U_o=*HZ?BBRWOK-Wl8RTSe>|QFJ$Tdt919ox z?ut+bCUl)bfq#qdx*-e_b+W>1&7oFQp!1Qw7|-ToCfjrVlSgU=D#N>S2=hZcG|Gy& zpGqns|Kj?-_-(lCdy;6QgWDX@>koja`(Dd@evZXXCRN<&xI;f| z;S&6kxq1&pyB5cX-0wT=*0;w&(Li=v|$CmrN2wZY?qR^YX0V8Q!DFf@S4)N7~C%eupLxda&31G;$useot zP&1Guhh7aTW<38~1#bQ(qlJh(PD=>(^TSk(mG#9hJQl0ZiR)$S%(Ha;R9~oT%o1D6 znKd?g*;E~M6r^0$B%<-yJW4CF`GRf`PLj5$_A-T{vpu+U2a3F{Fcsei%Ty|(rbvE} zVCF;7*^0V%LWRw~VZM3w|A>`Ief}AsRF7o8Uv`>!M8b7QOThcuA4z|xnUuB& z)DRpy^~3*|2p%ebpnL~05t&?@d_bXx->F(xcgQMvlXXvnO&c)@`4elQ zc}a#0$JLrq1j7C;LFEHR`g)5&8WTyd)y*43=89)2muv5Q<)Et~IjG9_iBMl^T*s^7 zQeZV5Yb^M>E^BNz5&_oSf_)lns~EXL@gt!rgGSmB2y_|KWinq7pT1-oKtMx3dfVO} zMq)xnq69kRkez!ppNKhciV0>g+^2WlljBHh6f@(bZ}H!G#-fO^KK)?Vl!Rm(tI?sK zA|ERD3Gyno%3I^F`XWCA8OjEr5V=i7__s$qh zd3e#0Y#}Rm-&_s3TupI_xZm1cW+>cdkd)=)9XaGIUPG21;bkwKn^3kJta zpwK2{zBPTf4==<7E_<`A@4ohPp~IxXz;)xyT3HZPnNW70x-{Zaf?Q|by1zeILm+uz zz^|c^GdNXSaqTO`b5x~u!l8-~29;JY5#;-+`?UE^0nK`UdD>I^bfuI-ba@#T+L+NR z)8gV^i>mUPioU#>6x&s-hwAxuspH7Ud1NPN_~1-%lICpHPQdl&OEGmBOf-&{*e6BY zrKJJeIqA}}vS4?4q-Ou}nVW`$F)`bX)smiU2iktfc=xGQRcZ1EaNmvCD&-WvZN)1) zQ*i-Gn~=_tYUxQobN$>j6fEZvTR{cIu2t?K2|RP3sceqsR>I=M&4CnfJ`;4@Mt9Ql z%&;=Sc%We3Yn-!^M94oAAGCMBLCdQhG)hGN0V-a7Z8$9mG4{|*U6V8FPXAEtMCI-^H= zHH>Oor+63|z3o?KGb+B`8n($C6#()gpYCl=kC3jt*Nh$7(G#{w+_&8BqtGmMh>$;a z%+TdOlGR38Rlg?y5G&EOT*sLZ@X?3`!yP++0jW3%8;QWC8}Fy=8Uh`Whxr-|O&o}- zzGpj34<-^$l~S%Vh_~l^RZ>@9VXeZfs~eA1IFUeFISPUnca;*smtoBGSvHDHpB#}& zN*>mld+D3os+hP|rbOcW>myKGkv0%(ZqlJrRCl39ULm4}R6drZMAA7o$&W9Wy2T02 z_vF%eLGo?INJrrhKZGsXD+EVF?{o0wrYsle@#0~!N@`qojKVBc+-(UW+fhB+vZ_k_ z=>~ag2Sy>TjH6;)yN90zm*L_9faSvTswmp%%jp=2AdWa)!L=mL{GX;{h9?8w@VZ?pJ!T+y%7+qe3ME$-JE9jnIP(HLY}JgudrKe zAxR4lZF~|v{ke=J$n`HC>Z}$@w8^JMWf;F_dYGkqwXEb8r$h;^4&@s&v^L^~aj!E& zdEP;9o86Cif-LHbewAf-d8zO5o*#Pzd^6#F-)K?p^7%dCyBiVFkK%xYh(ws^e6(VP z;iTtd-a7pVS{jS^%JElHLR(ekKVPO$pL~x7b3T5orOTuypw+3(W0SwMTsf`v?t9zE zL(-dNuYX6eY2n4Q0^I-#=)$tc9FGMv1Rfp+EJjeUp!Pj z>@#rqC1klhb=B$(Q@8~VUU~&1)upfkJrT_}F=MkY9c9FUFr8cVo9HpaCVBuFIu4_; z)~+{nDqd0kTL9_0H|$7JidHY;bODYF90eT>bg*h$*1KPdW> z3wSvn3F9$A-%YHs%s;W{Y_K%kZ>=gJHKdDL1>*5JwyUr&&HhD-Kp zHw(0@t?lxaD1k^G7dO@Bbd1w;{#or0SlDU>sdZ^_SnG)V#<)>K{1(+wQ>{-?;c6k{ zEhh2TqQ?MRt!H=j^^?eYc#JB$YSC{_Epn;5PPkM`+j3x)(>bJX%lPJj;FyuFXro71 zY7BbhGdPk@UGO>L>uLS|WY_c)q}P$w(6K4=Roase4$mN!1wkV~#GE)b=?OAo0{bhM`7QQ*dqn|t=+qf~`ruuGB@_hengw*5=Aq<1)F*Aj6GPbf79W!h1A7fwaS4$)-zO`no_8dt{@r#^wyM(f7 z&3NM3g!CRLz2h`c5FHH}e{#l2wK;*)zhsUOi=}bNs~_AEqOxBmD~O_j{v62HXm53< zvAC#kdkQ>@z4mY}oQzkUOja(+jwb~_FR*#L4z+awd^+f3bPcC4GP?83C=Wj@d;^TE ze7?7$0SlK6ROfP<_#%gL9O0`UC+vDcq33A&S-YS1k#6Qg@6Pc6<;u9LUp0O%Tvv$3 zRvD|8CATicmnVCArNwU@3PwwVp)&qgKPj=AoxC4dar(wz0rF6chezIXm3#X5yPP{N zF)zC#*h%kTfx@Pcy!g70Q1AMhBNAPx(SKOBWLU}>Z|sudma}Jx?pHug_PSTDTVc+& zy%U|5*aa$=s-e0JfDwEnWK@x-eo^{rF~K079rLEVc7U=#!B)TezU9{u^FinY+IfA# z5Y#X^Wn=+c14@bU&89&+N25Irg4SArx~1OHd#k*Cb`G}q?-A>=`jvEpm%Ta*DW;`T zQkFiDb~!vfSd#Rs1!}(>ncLl;wN5Nx8w&!oPRliVFwpp#+yO##kncOn@*q!Bl^Ql! zk6_!V$l>v8ui1kkw#>)!AxgIG7M6#yx?;2HZhHmJH*P=@sG5JeR0>Xo%N!s`@Gxv-`x3La(#b_1iuLXmA9`9XTcBL2|OoZAL$HlPWxmyNSN0 zH=IG2lHOue>X%5P#o}0KkW+DE%>B+;e|Pu(gv+w(8msOVHE<&b62*;*)}}Q%(>xBe z&ZGhQR-|P8Lli_&E}0@UR$i0K7Awxkoi{t}tqo-mE;n?SxqVD*wsds9e4$Bu3eU{0 zfYpmHQdXux>CU~yH{+X&w*eq5YJaV2G1H@PO=N`fmUX;`T*##{t5=oSIPNhjMfqhd zHUcW^8w4r}7QJ=FEO!{=tk+_X^aWke>YxD9iG#~jeP^;3eMKqQUWC#|tYS>p9(#UQ zg=)Pe78tE!@maU|(luUT*ALD)PBJ4+XjU(JNd8hQSs>GEWy*+!v)66uz=Yqa+&&jo zMxI9@1J`L=$L8=#+1Aw9r2oITme&j0ejTti4wF4{Mr)a* zFkPKb$Z#V>H~>XCX6>eOuEyb`x0x>-K-%1he96MYpn+bGS2N!=uiEStoQ`7x7gQ0C z6Z9T_*J#l@`5}~PbBo)#x~}SE5D5|JuA}WQ(?{8`eplpvMou_kI+Yfo#!}?DVqZH` z4ze79^b6h?5pmho%v{@=6tpe}q4)dFD_p_MzIVeq@ELScwa_c>`*{j62=Wi)Acbdm zT=|n(ssbAYu7nUcO^uZr65OIKb7gQ0XilgAz&KD%eymp^@zyUf9T3@4=qtDqV!UWr z8|K_j;i#@n)Yc6&N93z_Dt*sGI)6`zRtk@u5bD|kXERaGQ)Zym2<7nS8_dl1X zQ)orVH=cwF>G1P%Xe7x7s~^c|)E=ytv!*Y`fI8I;AfQdd4VowuCtXmX)>1^76`{&7 zm6;pth~DK%?asV^66QIhHADV&&L0;eS7QtO7evKoi69m1ePngUVlbZTp`Y4lAtSI- zWEr6x^H5`wf8?4w*;>Wb%Ha>zs_c*N^~=m+ySQExjCn1LSM0jT5nt)n-AoFl07)}) z)i^8rTIDy;aQHxniLG^n-|BhRx9KQn6i9@ERs7p2C}LOqQRdG6s5ddv_w!S>nKJSh z5FL_is9kj3^8MV%fW(1ZkyTX9>YNN(LQegDNG*L>eWRIVOL+okC0=YY!ruRWKD5fS`sr6I39B3P zMC1r>nZB{71khN#!4LP+kff;xGq-DJ_CTvER(c+@Hrkc&de7;0XWf;R7iR=+wG0zVPymU4A_{0i9a) zqu19cjI<|~>!7R{DfB#GZPC&-EyQ?@l{Xz_(N{>KvEcTUEIHRo67^1vz=fF zMELpl-U*(dG4%{3GL3SqbQu)(XeWGU#uKyk-Km6ywAZiXlESAr9zJDbesn>IWZXqT z&%<vPb2=@b&UKM-dy(mK`u@DG11vthFD@>(J~_R z$%{!=STj+`(*tw-7DkU~J{1ta8A*8LyTEHHI}cX)&x z2Pj!ON-)%G$(Om3Yww;f=ex|bpk+psu|JLda?kqD&j~x*&sHMu3ZN6X+jh6ZuJ<;1 zZ$BO|uST^F=I4Qf$9ZyO_CE5_HEcPH9cAXxG}jmkNq)^n5njro|=e5{R{auvn;(BeQB>4rxe*UdF-fG~x-%}E~ zMn6E6l!VZvoJj8Og|X0OReIj#V`Q;exHWYPj8dz+T+&cY5(C{+Tl~RH!af8ZPUler zixqr5jG^O`U+N_ym@s|!F}aE*3YKi_ zqw~;Fem)Ho2HqpSewH=wKgGczw#iRi^q%C#vs9UjginAnbF?Uh`xk$pf zDKF~xQBr{5&=vn@eODiS_FX#3z(mAJXk-nwkkz$3e;z2okakbNCfTKcxIjx z?M@K1pqy=>q}b$bd60mJB^soL`E{Ycer<#Ig8=s-HcY$k{q!xRuUF9i@XwZ6I3}^) zYJA-ZTIx=ug{oQP5g3%ahx@x|Eou&Q$7T{xwH@{DNxMj%RvSc`lvf}xncX7>E z5|em!m6x^ePcGoY*Or%NzTCA4Cp{0usN6utz!o@ZzpcbF5}%y~v-a{npJF(_%S=2b z_k?pHuG?qB{%F<*=v1gC{8%>y3;WzP?qTV&KOy)S9@M`)fH14MWMVXrAOOk{*rrCA zuG~0l?*LhfJ`N$wq?*WKXsqyQ{i)E(-opHT@8Q8!i+b+-zON}lXPUL}rk4e?6+}X6N7+svK}nddKco1_Fo5$mO+-t9&DPb;IO*U6UGe*Au|8f%;xYEce2>ZrAiw>B zVffn=lR4g8s=(oF?*Imlsxa$jMvf!oYo8X>P~UOf>wC10tGewsDy=xQgDkF&;DxhzU^WV+&yp^`t;niv5Xst(G?#_Y%=d3GnMTfU0o~P z3j3nH!Rs+vGrm$+d|a}{1-MX$#N?H%YkxW)aB!IbN5*lVe;F3~>Hl_1+S}drD;>3~ zzatvwK35rbTn*x1E-hFJ$Yr@E!5AxK{>Smuy-U$LQ}dXgffP!$)BJWkfSLJ`e_G|c z2RO}ALYY?%;M1x;gx9QAVv&&_bqD{CE9KAJ@d4$r;>iMX{5s{u|M)<%U~GpKw^<)p z1D8L>u-kJhTzBh`qQY@U1^EIfX06Q=>s731>6!SBoz;b_g*;IB(dP zl*nX@SRLL;Ef-t6tl6>~`sb8cFI>n6ylo3JaA-V_*aGeq8J=COb((_6QQ2GV|qR|Mm0MjdS5Mx45q?Rga-twu_}QE3fk>jKJDnvgXAOPR>1aCfnt z$e*q>w%4Xf;0aIeR+r`rtQD9+Pt4~FhKEYOj5HIyIF?SOOYkW zO0Tsi-sUH&Yp^iAVj2z)SF?YJtGmXEnu@5r_EIsPT%*and$iI3OBuqpDJ)@y_PBb8RubMqGLp92b4 z*;fQ|kmE+sbr>$GUiY*uzKm;2=YNoZmsD*n)HA(81uLVQE96RcuF1Go_Wz z9NH3GAO9x`0L6m4{&l3}*n_?H>i*YO7hw&fh~>dzw%ag2mA4Xl|Fv;BGp64{YGbU+ zMe2?&*Pb+uS^){}g;VS`u)i(IrCl%rA7ez2vEII5{<{OwK!T3hZ}5ajPj5Ep(tJy> zWJj%T-L8f9&n1Z4YEM4g(_T?{F_?5?)#c1k{IZVmZP;4^=oWd?fXP``tMJzQ`llwS zZ#VfA?@5k%RxOCv&Y|U2CoRs_V3ulG@BZS@hxHk;3V%Up-?g-gGdxdy;aWmYZ!U z6YF>@6MHXeKWeia&G_f>Ebmj**L`LX*xx9sU6?e!I6?Jtbl++x6zV>fwZF`S)C#&p z{To%E1136U!k-MT=&MznBtlFydCm>t0Al4r%?lARazqXIEHDI4bPK_B7%z+Qe1i&i-HFiQ=@&%tP z25d@FPM%XPbzzGKR3}@eD@S0x(XvI8-m!8sh>1F;klDm-4bYEsEZCfcb4iqr2)mV5 z_1bCT<1j14a;|kTh*<+aM625FxX3!%>*v;@^DE4S3d;kw{_fjr+M__%NqjBo#DtGC`zGmEEWg@tRZ#If$~S@TI=KQ&I(}2 zSBBj*7NjoTL-~+gZ^n0jd#&lBB(Q|YG#9*hGgHHDs{Lp!^Qd<#7+#eE?oEa z*f@=(=IEh=uF%|enkTG3=;GS&(OQ1F+p3`J&$I|V!3p<<;Zg9K#{B%rKbJIyEGeX0)cSK!@3TW^Q!z6{|ZGfS)v#@YuW!ZpL2YZNrd} zxvXgFNj>%VKon#$Vf^-N!LGgI>jrP(qu=A4zxss(?oHKBq1!&9oHxd2b-tuxW|{P! zIxmTpD>vDc6)V>LAkb%d*v4f~9W}(6s|oRxdktETQ57Pz59|@Fe4(j}#*ER?Lf6V` z9Uv9P1MnqV_qnC`F7=ZzxO4RnAmJHVmArevSZltozQ0$2gslZsIE{{gZZ;SQY3O)HA^QOQ!n6x9a++mzzM^*i=UJksr?{=pJ- zQGmkTtb0mds0C|_DK}1>4a!GD;nGAEeP!->uH62|I!xesT5Y9T^5SF#3yXAP#;;5a zOb^r{6>YEJ*~`Lr8uzAQ5kv?@#7K+{VqKROYBjnrlHs{lUGS9XED_^Ua2hu5yi_P> zwX~4c=KSm*WV5)sImNjBAL3LAc)j5VDR?h_sJFB#m`3pp3y|^h@aEzO`{gl>eqC?f zY{&013i==Q=OgzoURCI4b5u}>lw204(y8R$BB|vaf{dbyE(VbXSTTbH+7G*yEHK`V zNNmH!eN|4Su1ghErd-ZGy$OS+>*tr~?+8yN0=x0+adkPkta+e{8F{rdbmTF?(9)E3i%U*>`s+)0p8+b&>v=&2CHrD;Dh_h*uaKhxW z>dcAL58KGKVVAF!#qK_Q*gU(89N^@6Q%uc?xemH2md3tp`NK%z$FB3;iS;H(!)2F( zL4iSAuKB);V9MzTbm?kBC8Eg1$F<#=dT8Fsi*K>WtMtU;sGY@SxRZ9bl%Dpqcm3y& z3PEFpWp8_S^4a@jrNy6*is9_7iC3(EFovD%L0oL_hQy;fDRQqX-ovR!7Dh|LsJC?@ zMF4*A{m2wW8TRg*S2}RzQ~NFv$z$>z=Z}nI!OrDb0vv z=O$fqs?8AUGO|>ReiuG#@S^ujD9EhwMZdj>Aec3FVRZ7r$pkQ_;ycGoX52`1yAWt! zwDok-eRtavrtLpQ#bA-QDz`-cWG#Fu)K;h&9OK=(-B1sd8?_bIecpw0zPyk#1c1rq zy9~YQVwV?F{xby}-+NeFAddur5#_)&@7|0{3oS2?14qpxrfRrYIo>1oM3bG(dM-Mf zi<1^d-1;u?KA(y0@kWqa1-9ykhNmh|Vhe{$q9F$AA0Gf+_0D*vx6h-unKyun@lxK4 zH6?*cf_2QMEqfJ6R9i_L_VJx}VA)TJ(%r1%O?4Ya4%Lej`>v8`-R5+bWc#h30JD#n z$w)hoI9=xvJsEC>N8K}AGy2DNCgL)9$T3v` z+15ZQKZK6@{bQn&lW{VeB>p;`DG}_BXW}mkX+S>JZHVK+>EH;&kzu56#Z=VV$>B=y ziqeB=!u8Tf>p!`G2RMILhl>5$v6{m)40*m|_(MkZ-t7n~?aDKnidJCepnkAJ$P$LR z{9^At5gvBN919(8$h-z56OJ|Ybs~Fh0w)t}#dv?EbYv9U&C@3F%6l7@?|4iYlXAW2 zp^u2s&6CBbEb6Np6FsJKk>h7?{RfuX@5&)nzqFwXVm%++y7r&EaRGj8#CS~95JoJO z!d)q6ivsu*G9kOmi!PVLi7xo*A-Y} zr^nvs%T`socgif9uxSTUBopaG-&=`wqT41(`c7|-znKedMxpCUa$q(?Jqh8oCT8mS zeAIU}N?Um*MA|}j{<~=CqGGXH0IC!67U=^f-eH691M0$(d&%ob)@%A7zdY)QuW(q` z*+}^>%zXXn8ZL@>;@w#OUmH)8hxJ%f9*)hFWsWlK|Wf#t5_rk*U%nOWw>VBDO5^x@>8MStp`I}z^R@|kZ%9#@CGhAN*itOc( ziQ}FU%xY)->{%a~G*SzFC&FvU!T@uOUtu-E{V!S4P5@QVgUzeeCK>gZX@eD8Zy(}u zxR?N6?u&fI7j~_B^JQ1TM=lLJ`Ieq9!`IHHFP|?bv%xF6eD0Wz83Dn<C}~Uz`&A*Heiq>rmr)km}o8G0}z=mq-?FRNUis{XKC6%&T>9-3Q*3 z`PO1qFG@GHIx>~=+1a+~(x~mEmz_DstN*67N`Rp=4v$t8MoG9d#mom`g6*^ z!e(Cfueylh5s*p0!bK5I00FJl;<`|bW=>4g<5Fw+0_FPr>n?g9fF{Y6Y&#AiUw41& z;!qAPJ`(<%qH)hKJEBB?|G)R73Q#NXgRr}-w^bB!jNTxj0@mq2&WvJp2>}uVZT;`T z(MMNl47cf>o5t|vwm&Ef@hTdtvncxVH4uIPEhc@=?jJH*Mp1w)`|rJF)9Mhl`{UW^ zqFnJjuh*$RV$;k#bXu)NuTabW`S}G->QR1-yK+ifZsB?+1>;L54ce4l6%=!M!WfX% zpA~1HoFUA#E}qUzDKTz`@EfbUw65%68=fLJirXfDq(2MSJ185EjWy71w-8@9pO?cI z*qI6B9)Kj?Lqa8BR!o`|kPY-;$@wYV%2&8X>k0Ok3{-E!BA6vZ%tMB?kM=qS3j;8~ z?>abo2E*J83Kk*@0>Oz+f5vj;=N-9d=UU_(AkvkVgI3Q1N9vK~Y{^gEc^rrmgZ%#b z5}6mDcpE`)uo(yJuN;KPWmStYK-}17vs-2ofFi*B+#DaOm3-R0hjZxa!EKPCgiJ4!E`F#%uF9ed7wSc5pB&E?YF5~F31nFhl3v=H_|heG8Az;?kZ(8} zOkSAS5Iz|{Jg5f?E*xsd)^9}F?0h(~Xg>o4_8!QfopLvl*Io<~G1%@U(KU{-*-~uT z)@6e!@`^k=us4gLA&|&IYBijetJltv+pv#r+|i$}u!)cz^)3n?6lRN&s}x1dL_m&; z+EXXh`C?NX+}HJTwwHPjJZ^sToOAy2Ynx!IMk=|kMgz!c9fHHYGz!Og!wF_%Lr3<& zsv!G_XuiMWEnQ>csMYsFbH~ACHwFDs?o~x)i&0XVt&t%7E0Mw{3e77XIJJGUCjw;B z#OHGbDn9M!twYzP-GFkDt`Pf8cIR=kL2>huEU_;PAbHZ_r7}JYJx>;sfpq)V2a7j& z@_vk6FyhiN(oS__=557N22A(~o!|Zv@vpi3AO^(%B9_X*<)!zDGgE^8MS2tHP zNl%Ub;=IN;32YT~Eo4B*t*(w4ES3S!yH#3H;L2752B^RAdFFwur2^Ltc=x(!<$k6o zXM!ACEg$uS1Tz=Gqu%nrK?e9JVNK0rbin8IRa0!l05UI@$EKptV(5of4Cnns(%L*H zd1(U$B|G$E{B*m%$UPYyz5>%|5429g)4@MA8MM%@9dHjEx+Oc8wSq>zG&yG#%!(3U z7o~-W(rc*2CN--U*x|L(3JbU-d=3Rus2~H~s!~q(x{Tv?Tig6^AusDf(^!%0kGLR|xY{ucfemi>Aec%tMxXr=dVe{a;;O;0o1_`~zu6m80^K+$8U z*+groC*S0VoOTJ`&+zxyh3o()WTnaftSto+H<5fn2F(`H%XZvFUo^ z2$e1pp^X4l=Q0jhd|i?GZw1Eovw$o3TEA;vFh*eL?ZA(Q^H?b#elf1rifoRS@Xm%Y z7aN;zS&&R_m9GD^0yEu3b%B)NH27>D+yf9W_AK_|`ui?;6dwa_0Ane+>5?B|-LT0XWzeD>Qx|)gv)UTEygUoUZnRBe7 zQU8zJ5cxr}YZdSsWmMZUv5-axp$O(jWc>u2&S+eIut;XlMUq2j?c&>ZWs+3Lzlp5< zt_cJpCR`4<8Nv`dz|8G_}|{OiYKN zdU@Jls;fmZx8rSRs~M!1$8YC?mw^mW03qzdGZFx&%f6q&X;@m0(`^EF@8vkhg z6kt3oFfe42eje?pmO9(TKO+N!?SB~ryTQFfYhdgG3#YC}J>N`1;P$xxTV(y-(dKe* z4I6gg4u#Y?jONft%Q@mTjrzZF3WMAMHtgZRM|}5RV}Y=R%HvO~Q-tZ(YF80sY44P+ z>MK2RFE0+b>`2_y5xURcg` z)OA@v<2Vd$C_>cj-smmcOIO&i*H2QJzSlw6StTte^6u*Op@SjAqOFCrc;2&50&W4E zN(o{loaIiHk+27Bi=!EW3$zpa#-Uig4SHkayj`BoWf!Zp*Yqpzz%ooM?_La{GCiaP z5Mf0?`C_e7P4a+OU4?QmZFVR?_Nu92CaB=!`m&I^IFvS=&oR!HKSp|9 z^H-0#+h*)P)n)L35^ZkFIY-9oM=#CGEf4csXs&D0%FkbL0dOijbl!4qSYln#LbeBM z66g7X0Rxs~h5^nUgua5cNES{)r^Pjb(Co^}npug$`u&1U0 zM54s>wQViL7_)(1Ri>3s*?8^ERo9&F`1N^;R|+sI9TzZrGICa0>F6FM&vGn&+@81d z&)XWB2g};G_OB7uhMvj6tenR*?KiYl$@U8E;qH#WQ~w_tmF~*oBOja2dx7$IJE_~R;ae7CncsYgo$2G`2i*_5&depS zxHC|MEC3B!bLm}b5BkQ-hr@T0hX|2C!R*tMmcy!NM-P<-GdAu?lio0@e^-wCfl_BD z!N-0rXRlkCvq^qWwpd;5Xdg!i9D4h*^#m5z-fs-PcdIGDP*0gOixsqDDzdXYQZ*w+Qq5->!vq^Pf4r=K09xg}(9xThY98Q0VEgEjrQ<3K!AZ>9CDyl3|`d(Sv`Xl)J6&jF#Nm0^T zS-3IV?TRsLeR2gI1`0`umhUMZJmEvw)ML*b>yMof=S!0kTHkq+o9jtGTk?j4XU};3 z9>B5!%!_+~j!dm~G8lU|QU%FC(g}ERR*Rb zgPqz30u4p>^)D?)I0aG|DD?&y1RDB5aqBiF+Z?Qq)b_2SSy$cH$b7?hz*8a@TpIm{ zW%w*XRlqs{%U(pd%r?${Sj;a6Y`)>n>q*3h0(Wm`_o%7qy+BiQDQ+k;xTL!`mEt)m z$LGYINsSJ7+T{+9is4QAmG0d5d?{hT>p$2k*yq!gvs9$N?gLNKcmQB&{I^sS7ome6 zMf@Kt<^}F5%!I~H!N-DVyNnm)fX^potTr!=_9g|$QkIu zB4ut!vfqZrCe6h?I@Q z5HGaqUvWb!F4*dK=uHDwD%SnmSpvnnIMo%ijVEhC(79Z9R&DhY zdvWF{3@X7*h+i+Vne>Tm=h{P$KxmGT1UuD`4i*{rmVe=I_Q zDAtYjZi-?)2ZMPjM(PI`oHy^)K~Da)mChpbB8$Pq|2wE-po2PVu@2Nd6afd+N$)+P z9-zf|)|#2kMFTtuHB{qUxBB%DdGp~lfAUSo`9}n8Qa@u*-aPwe0A<&O0k4VpWIB>P zaD1=D50J4Vg^vzKe|vG6qom${-Ag667a^Z%adJHVf@jpBnp{PD?OD6lqqDaUMTh0! zqgp)zhDMoofZ1Ygym%P-=&_2^>s}hUom3Ra1!zxt;M-PPa|Q~*1E101anzDF-s@EG zCPe>ry-18Amp?5>IwoQ=xz6>%bW8y`QkW%(xA!}&-E4}7vTVt;wRp?{@(oJt{R{8M zYj~s`e{A1rJ$aF=>5oL%?g>7ON`F&VpAN6!E0SuSq&?dwEgeA;O_Uj{N} zuWi)Ca7A%B7yRGai8!5h;X>T(?O%ObmA;X*cbzbJ({cpvw5?ok5R~f z2ndC2e>sQkqIFEL+f*YzV}|jp^1)h5-=h_5{9)N#Iy$+n==jXJNP)`tfC%}=k*BAE z&`j{KX)#a$ly+IIf(SSuUP5@+$IY>ioEin=7fR8FbPR74AOL*4{t_sX;?OJ=nRI9_ zSCW!}z%?qYn^~#g+dF>UsDzc8S28-W_v%miN+4}&2kq(98Awt+ zAlcDeTWKFR1%c39j^Sb%9PgkyVpOiX)uS2qE3PToab)49^*@)EXm=|9YD0MeXA$hr z^w&7ioz#TBgHLY2Ht>x&=TNW&ea)vY1DR54YHrxCbHj|+aT?lGc&K^JRfs!f1{e%*O$~%|SA;J-xV1>I z+M@mYUE?QGmLR&XD^XTwM8Lz#+~sFu3K@$JM8JpsfZZg89nyf+HT#LHI+eTh;A?N* z7XzHKM$(yY&yxQVSJ2Vn^#Vfc6lz%CWGjqNuSBQ2)nf|*VB}Gquc+Ey>vKh!>Wt_G zmqtK%x{t{OO5LEmyBFr)|9pd$xbhK5h@8J(RKTnlCu)Z|GS|fL(CU=Kq$5W`gF(v0 z&=yHcU*P20(J$--<}0;0Fv8J~;#^zFMkzB>*iYy^Jg%(9+;^F3l5P z!Rsm^5pJSk`(BZ6wSra;yeuu*1se>Cpdllb9(S}Nrj_I712b+mf3mS)jP$IwPOzC6J#gH@}nu4R}l%|?rcp>yr{AXB5VOlI%0 zY5(B21F;9tICv@0*OWjLc=)u2xE8U(@YJc1#_npTRwXmyVTO9tCxE}U--fFVV+&Jo zmU3qTqGx%uvWe|saq@Io4mHc_Cp3a@1m6i-42+4*iQmDU`!f2Cmg=>>EOQtRx0l-d zX>E}I4Rzt~5@4p36>>|+kh zZEGIbRI{?reNUSRd}RZV5%PG~60=$B<8MBn;8S(P=H$T{0M@sJ6!_-FZc@7ge4!!W zX{bU_1lOw+*JmMvbCxxhr%$0DsT`$@{4E1Cs1jmd`MYttd~Q}&7A$v`Yg-<$vwXGc zlii1i0JLDF*fk`QC_n@@QeItI`07av&x{sgZ;`z)6zVo^2DX}J3TtS^v_b_M- zwLjX658-IH+NzSi8~sv|IrZRW_~uoJg{Jxn_NcBc$kMW*zzfg8#%|f{Nd-(al)<1x zpB&bMREiiWHrJWRQZpUqP+@74`6hdpnGzj#2Twx_ei6tCv)pgi%BiuhU?1bL_?@5mO|%?kVh+DQ*8o{=ZVGG$zX!8z@f^){**E7e8~?7uSG` zdpfWX*m?dA-h!niP>{;nj0W;PAdFR_W~A%~IMIZnO*to6({jEiwMS zK1IKR-ut_!>HfpG--D6$AlPtK>#uk%RW3S6d+xK9=w$=bVw3->F>B+x?dqKDEuJ4% zc5)A?-7#ROD24-mkdgm@SB<833!ER2levyg7u1M8{`w>HHC8sSjgMPT4>rrTxMiKY zw_W$Yy0l(6I$qa7=Bf1iK$$)rrL#cE&-Xv)X9i;)`7FP(-v&P55jFhWdG)#p6<0f= z{;FES=&e3rV;M;D>ghUV^MpT-Y+nN{--+E+J*cXlC=4jk4t`0)G z+$Z5*7*H~SdHpPrbP*Sc(coF?uzzJbu9sZ-VM=PA?A!2vRMh(V)iTLft|e(THYZv>0VB zU;$+6%jfUrn0E-Q{>ST6IbrNFJJ##i%5q{wi;;j_ez_$xaQJ-^u5>1)bnVbd)H-`{`T=(ArzF2kA-JWXA#6q=T)5#l!JB zK<|rn_Y-#?waB|mtVd6iG?s_^0U@VV7nPPU^$&*zDttlQ4RZ`cT?&9&+rv*l>09!v z<=uA|T!6Lrl;l94#N-uJgQrPq(QHba8?6;uz(fiI+NoK_a~;eIlPJI`d_ezv{1M@s zw40WGq=vBH6INn?Y+EViX}{Azc7Zn~rJ(f`-uovh2Ipg^Nmh_@OqLZf`)zvk6a#%g zg{xCd2JJMd=7alF*Pa7uDko^%3t6BdhZq1%E}K1eY%|a_Zy%L?>i_&#!X+;iJP<{< zJJ{JN>t`~%n#b(~5L@VP*i9kGev2?p#!mL>5Ilr%N=fJaMT((dYM`~v6UYM7eVnu+ zeL3^_B4Jz=m@U82-~TL*bqo=SwJ5UJ+hcvl<@AFw|w zp!86+wq*c=i<$fPHXd!)m)$Xx)k3~zvMqPMT}QPt&hS-F4vEP6Bcqdl`KlQb#B8do zRW4PjSB^tA`k|&A(d@VI@Nm776+m{J?#}1FaQ@&Po?1jt52Vtjy54P~^5~I)0(SzH zj+6;L7~F+IlLtQbGCD4thfHP7Z4jX-R!ZKS&{s3NCpLHZRYRh<~rcAyU{PPTzTjNzlmy0#G*DWMsZI{ z*1_7LLU6?X$H!^c%j-Z|NG3+BV0O?$=w!IvXhN+xWU!A7+qs(MCpaaC-|^Xhq|oYd zV?tHDfVm)_U;D~x38Uc$SJJJTVc`C=$5(Lw?wVi9TU@oK*l=2=Om@skgN2Jn=F$Fj zXneLDf^sm+9fqJX%u!s6VW%001#ww5ak*dFQHT}T0_ndC2ypNCXdzx|@tqlgBu-d5 zme&+MIK52@O&ePZ=L9RlpOuDftypCV0KJNM*0|#bds<5NZ+RRdwgchu$4AKSW=6hB z^6|=%q6w>VX?Mr;=|lvsB@kmPve_7mp7bAvqG^Qn7J`2n)I(aZOR;(A+!{xKm;l7n zNL=z~5hj62N>Zt9p&A=Q_6bWNYl-tKVDP2m|J|s^7!3+*--Iacyaq`mv!vVwXtbha zrOj?*yK5znWp!Fv>p(~m6ae^MvWHX6LPx%^tXlZ$G_iK}{egjJ$05$?(kr-3LL8R@ z_#g-A2jn0%WD$Uzhm`w&LoT}#z~ms}iEZD+9Fk*UF(>D_KbF&e$O-pWr&h2v<-z#E zV#I^7%>3!01kuf>A~1+M@di&P;Sv1oeElNvhsqv`z~_?cb_qx=LD~g#pbrCo!1X(2 zk!h@{qZL3_W4UU*6?~9*GihJ9?TCE?Uzjqc@>(ziiOG z==d;;k!=3{E~(?>q|OLnVv8p}*17vP`vB4vWNTrKcN+Izwz1afWO}9cQ}GxidoB`i z_hu+{9il68FsqdQ@jL~vA}h%+_o+whR^#!rhv$}dySDk)i09hVCL%@LEp-mwSEtRk zd{OOdnRPpe$2e$;2WfazU(t7|9j8e?Xk3RxNy_tg=5pn6)B2E_J+|2G-&-{C0Z|&I zTAUgl7gb&}ul#+vj0{USRN^2N5U2p zLY-lf`&7eTT>Ldr&Z|+K_%gS%-?TRV1&GlFF_3|y zx#t{U4%%+-(*!2 z!0uO|kS${A3b=_5Nb6^Kzhpol@c@Xh34IdLi(+XQ7CHiJY=>Wc{3qwZCvA@r{-5_o zj(%1zueHb1n}kcAn-BEs(mm-iOm{|y>bpuRtP8=r;=3FUqvlr0!uS+9FCnGSLmCH|lh_qg@hl&Mrc@>%*- zEt{)^C6uc-u`6WC>0HqFyRVhGuvzmFn=)I;8^UOohLtN=W}AfJOa3UAmliqNZA0#9 zaTtY~f8=k>BWBHut8^+Jv|+wTAQ#sG#Yxu_T>Z2jqHg))7Qeh(^@1pd2Wv#`n{nEZ z)oP_F7h5@}nR`M5yoQ;~CaACE4}s~c9nF5%xmNqbu*QC~eOrXbK_U`}f0=?fe6xIG z<%1lH_~p3GCid{?16Ytgczbomf zD*vT>@VghMsBFhBYHOf+xV?D1$cL1BCR+>>U`xoilYr#IZFO%c;lnwE%{=ZxAugax3LxvzG>GqYgDbyQMn32_>3VGVM1`eZjanvKk>yVK(S)0!P}Rm8^OAWjb*DX|_I@LT^HyGP zvgk<2$$=2xT0O4L;`?(!vj9V|k{lD_OLDP=3F1AA1j0MAT|H(mk8UJi36afl^)c0U zh@WO5OP8abV7J*`Ef$axEJSwj>zVo=Hdie;XXSG=;8w31#!|Zcu=m)dPHUJ=7DYXs zkN=eVYZy_7w;q)X^-{W*=aas+_w8K48>avKjXs*UT=X@^6PJ^f{efz;o(YD*9iHO^ z`Kh``G^|b-?DxT(QiC}qv>s+BK{bF=35*S$C?8znaeh%?tUal8a+FOjV33!uowJQC zldaBU))YrTd;{z3M8IPNr$JXDy4q_XVJ(gb(@}hAZ!7bA3Le z`|j&lSPB^t9u;u}(TNyLr>6-X2N;zPpVIWu=@?MVOO-0sd0NRH_QnXF#O1Rz+{>=g z#a=CKGSPA`g<@jB;B_XxN^3y=vJSD5)VTdszs&Fsbe}07kP;y{hr&V?S_13t&YW6r zQg35%QL8^#&DYcNXv43SK3T_YC9_5-9<*6zYP@_oxK6H7JU<9~cqj zY47@-cS`|M>jM;gMd62oszMF+iIAAx4}Hr!E(q(nkEz0(b%pIrRzK52&rAqtP(XAA ztTMb}ShwjwQO10n-WD2eQSymkqZ;dBL&#C}P`}Y1BShaKv*GFs7G<`?Bi{r99xwT+ z4HR0)=YSIc#)e^i<~}Dn9X*h9W+rpVJU=AIQhR=eXdh~ua+}BWRUdUI9eHDAx=w%7 z@n4BQy+s`=4fqhKR|6S}R{?s9+-Y8=1|hma`isbs;{4(2gLp1?$4v5_@$&u+AaUgU5Q7mK!_&_^2CK$nSv=W*3EOpWyH zPWz<&fSkVjORyvuB&lv}j9Py(@NOl9DriBPZ-MuH@zY{jGZ-_c=}Rafg=F+WT}+2# z^AOSJA*s7+M|>+ezlrUOF{NYu$gRiar!4ZaJ9vAbRhbb$n7^}B2zg<^FCGCoYw?Rg z1M@>r@@s+Bx>J0dYvi*Mp#1N&QR)S^!jKc>0{d6qX8aZx;QFPgLd)ya_fo<$?N?2a z>%zE0xt4AB1g<;$k(X7i6v8RUsNRn>{=!>qRCV%zo{ch0mtG%K89>_Gzx%!}^diEZ zHUH(dPovLNG0-MYkx-X-?5UG-->bl+FSA9px}c0_OkYgkLG$_*!Ik)88M))JX7 z$yk)DrmSrqOmWk^scO%!aDSDC&$&e(;n2#ev=zq|rIi-qd+;-&q(Wh)n;nFqiq_f~ z%|Lb?GReOibV_&$q_*p4Yj?rk2W>fO)d@^`F&N;0{`*cJs`TZJ;=*9p@8eI^5CSTU zJFz@1R9N4&QaU&}5<-ktRW8t#j(*-do{!+=*!u%=gpr$6>Vj~{&g0#t2Lpo#7vLHN zF;aQC(YEe4K8wOPV&W5E4UtdYaY1}GU{%}yNv@`tW=itj=a&n6z(^V1S}0SIWZa{g zoa4Yw490WRq|jPnmvesLR_3r$3|EcWA1!JZF-gb-hO_vf4Ffr8Xag+*ZHd$1NijJn zGEF>QC1sS|+gLx^q|7ejM#g+4IXy^n!GY*eM~{!l?g53_n`sfW(E#et9OB!b@Hz+_ zD5wLe6yvR*;(=3@&jI1+31|0$i~1SFxtI0r9jzjpUstQX)p#~C<>S;p(Bs()!{W6hnoT>9=SX-N$b3vv}B_|DJjQ}yMakQ_rJZi(bDeIA!+HqJPqFvA# z;NugD`SKlwQm!IPegV5j72#q;i4jdQ=&*SS)t3N$lQ_R~Ln4~#L<3ur1-r}- z>(jwwwCJJ3X~9j}!xGbKa^8q(DAd7t>tHZXPkY6mS}(+PKVJOXj+~d?uv3kFztAv_ zePw6M$pQ}I_{Uo%hZR)B-foZ5>^mRi)Pw5W0yL+cY`YT~Up{&Zr+J&DkywkORZ5rc z{tn=AaUb7QUL65_O-4YX&+g0c);BERFsXn|sjuzV9E~WNvO4N*f-I|<-Sw74Q0a32 zQ%<TU0&x|VVxo;6k<3-7%GjvXDPR|Ii|+EAqB2hy2(7w2+9 zv$}NkOpE21eyV)?1l10Zq^jHtA~q}Ur`b34t)Z5!NkQl$-Db|q!%E8o)kV%eSJivI zdK4?Yy)m47!UIx4NDK;{H4P%win*m^J0=w90Qk^Otfn|cF9fe4fBN&B1nyVkXtW(_ zuCp{A8%kZ448c8UCFqJ$;pJd9Q9xcoFUhD|G1tlbQ$F`*-BX))vdNBcKi-1?d(PY} zAAMHqgQBTS?IGOD7J2_-U!;0iL!8@z**xFsV5-%$+2W*-TE8Z(*?X4V z25yF4?(_REk?Bg9irwx)!lh2ASFoF-r0qQdTm7e1E2KMz%Me=Eue5m|tO(#9MWpADg6`0tW{ScH8>j)=Eiu zs9vK)<1LJdW`)^{=E0QC-$PsuSWOGp2>JHEG9t%oyp*<-tM4PP`hT#>4FOE=Z%ZRg!>LYgyC*PW( zYa-qqES}@7sx5TUHjR{h@TJDjPI9zXIu`2(ZtVMVf(p0(%p6t7VF6JUkW&1&*KRJl zg3)mA(9x%3at9S)JV!82)PI+VaF#5}G(CBoMIhr-I@MQtWAn=(P3ebadHzNW_QMn) zX(K)S)!@Y`jP`_=TDJ>Z7G_lqJE6JZRtP!BU-P7PbRvXKPmsC^L%nrN>62=Y?vDDx z>*=?e{+I?&JngANc>%BpQHELf$^6-j7BZ=%@a!8(uX4!QFjxU5>K~pB>>*xle zo_^5(T^C7G%0|R#QUGPoa0+yl>3e!lS?Zm-OCbo3hXU*-EcEsnbzmTz)CE#|)q+W$ zaBI!)FUHfpe|zU<^_84!;7F$iS6=(s0}vC4$1}>YarCIU9a5V9sWCzfW<*{Q`W~fJ zS(e|Z`qsnk#YYbk-NRG6d{!hD`9xqC%4ZAx3^UQN4kYJTROCC(mqm`}_aKM(R@K0| zw#Ug0y)E_o(>`gb9MHh0669qm?fq(H*|2EKl!UT zC~13>-~Qp>K4b(5z%u?!3q?7dXmG9EHtZMmd&r%1+RgzT&o16PNgOnfxLG-zw0{clKoa%*AnsehXo~Li|#P6YUWkeKj z%6~T|LruXz#|g&Z z=v|sl(4w^*_zrW&wh(Xvf!O8t@kItJFwr!0kXs|u49ecDvQCRhLA*?CO*EN$7<^<1 z-mb-#Gt@guu{qtA&I8c?_9!}Y)cVc#RrC)xSL>N>6$!|tfA`)2}iwsQQ6X0F5_22q$s zE+6Q>-2X+d{HE%oVmxsT)D0g8%EMkomP#?MmIA{Q?or_Bd@Q+KZw>p`P4QPCXiqwCgG3@n&6}%sog*P#?V;4C3sx4+EgyCg z8?^^xwqoN-t#+pR6a@pBE_-=F2ei03YLH+%*b39G-hbBois*ZkT`fS@<>}Es5%=!? zL8v$+yfPR6SbXU+>t)paqmQ6kY4lc$C#&>hDk8|Av+BQPV}{-dVA$F2y$2kaU<~_* z=UWn__K4!3RD&yz7-T&DZnR6NRiKC9{+rl#8b-Lp>8^)XC>}Z3RKH%O5S{%6bSzI& z1Qrx(5cow@{PI6PUx!-bb>tJ4nuBD)i)Z8A_5_Rm5yx%B1w?J-yri-17<4l@TSt4El}4Q_oM3!zYdb-YDY3OSJ!*20o95cX{*J4Y7lH z0arlp4OoA$$)&v|^g+`JPa)W^vu6>2J;-x!9vgAW-$2*GGtucZszJ_4?EPypoT%q* z#-Vzusg^%LzF8%<-7SvlK;KZ~mI~T8Bb?_3nFgNk7@*$+!KT*euogHF|dg^&de=dZ%Bv<_mPjh=14; zn4&^0$brVM3O_nhC136YN;E6XxMdFtLP3w*L(4tM1FI%BgckA*R8ObVng~=epmdZF z&je822IZS++gyqMWSy585+wl{m%}W!*Hh zTzot2d&hy;5!nFREK7;5oRfuI%*ORm`$)#vLXxk7qxXOA%c=5|%sw9J{$O!)Bgt)j ziy89n49Q5M)$fjO#pB+1aOkYn+xW6Vu*3V7FN-$N+k#v_u|GEyxeKbkrRTQ0@6a}J z53LR}jx;_o#w1Br*ET4d&3e4}JE}|3ZuEZbz>Cz{h2Prd<+aPg%jp6M{sgGA)6#BPwHuvfp6HP}_AeQ@@2gNh&LJ1*8;*Y<=>O;%mnctRjmDXfsQMgTS#k@D}M}Y^~X)phZSye9g2ptM|N71I)ko@{h$+d zJU5g#KzC=XViNJXANYdfZ%kjQo2zkgjFlshzY#e^${79U68D858P#6Nxp0bfRJ&k0 zCix21$$U^xp)8pN`Q8Ts0sHjutUmqVXL@!A1Ear$+n{#kEu*ib9Z0P^8fdPRscUai zKg!;FR{cBKy>fWmKLx8}`Bwp=0xH}|OH5pLGXel=@*XYnFW8%D1o2kq)g< zU5_=>tb~6E6cnuTt9G1hlJ{Wdo5}fU5O9hHE9xo5%;^9z$?=lAEkx;(??jRi*nwjp z`Ok@xuD6MmM`NWR&E0>(6&YjWz0x=Z%CzHM+$N1fRY#Z*u)fH05s7cv`!MvBJ1A&4 zUiZOyzAL|&e1k!hnk>dC$q@ohx!BYUGQ1(s2h32ab_ z{M=~dymb5r(HCDLoF>+Sic$K}D!cfte%E(jOF-{QYh=)sh3sQx*YHMt1_6FG#W4$MQ7tS(nxJur1^fdHW>Q zcb5Axi$|(lSbr4K4pw!4n|*i>^bY{Z9mT6}I~HfSMu7cp8hga29q{fj8NZ5xf(Dv# zUC8RnAKFr2p;odzXj0vBnQg<>>#7_`Qz9?iPgneYF8Nx}@{8!U4jsr_94TcP@vio$3l|qvogn-=#`_3S+bs27Z>5 z+HuMt*;{v1v#wb4TEF(thq!W21_puK_480@yH84Eh#2B9uVJJ$ERJ9nh=A zJlugY=9ROUPXgnBUe<@SAmx$>{DCG#`|lKcP23i8JQJ6dobe>JlZn&#kh+VXN?&Lj zl#&{KQNoRXG8|PEHX!Mk!{?)}tUZAcKwc_0HgHo+w7!K$SbTRVJyN+XGWLdRpO4SJ z+^4?z$v>~GqGPuimCBiu?J1p{Q>P7)2QMZJ+TJcbD{C2Zolkb!zTMA*4oGRb6h*%? z`+5e!Tqc}TlMlhty6$aLV4X(9?`8mHAs^ViQ(vxVCy%Q%^l6Rcq*%H^MX9{?r4L@n z*o6?#p;y>aEZJ8MoeZ+g)+n4t6S(t*=ouJ%?M73F%fVf&^1X_6+LV{-0`-R!lAZhb zaeImS6GXm#TXpZdo2#O-_sU84i=w>1HAZes@DI~G7L;5}PGKF%pRVd@Hk*6`z(UonfFSfC@$GexZw)PaSHt90{L{@rlWL4~vxtQ(?Yu%dl-w(PGIk)|V+sEg3 z=~0>T&@GK`%dO_gyvG%_=EiqG$-G;+dAXavmLfwzg1RW;Q^x9Wa#e%8eVKuGb_*^= z^e|34ZU~|cbUnDKFaJuNd9@#R%Lm6?%wUXjD5Io%bid4RIK$;jxt{%jQWDANS!pDiOy^r$`gvfzOGC9^4NkGAXzo2~z+N>6-sKNkaT$JfT zu@0c#$j5c_Jc)R{bPlVmR) zI5-!d6b34kd72qN{DiQXCtM$P8(;>yyI}k`+F5BKck_Jcz5@@m%@=R)td=%Oqd(SI>w&=QW-w7LDeao=xx6lPmq}dEs9Gp)0YJllnw>v`JY{AXlK)=4 ziw4fu1r7WSiQ#{JODXW~C2==`y#H@9eu9?NLudavtH*X>$~cdp<6 z8O%V?q&ari<~#c7h$m%YbP9sa(FtA$L%{9kSrW_X@8zz0o#0-&baK*K==}ZCrAufK cOuc-GM*h*qJc8yZ@Rv&xPh>>09>4JZA87<#TmS$7 literal 0 HcmV?d00001 diff --git a/scripts/plot-scaling.py b/scripts/plot-scaling.py new file mode 100644 index 00000000..6fbd70bb --- /dev/null +++ b/scripts/plot-scaling.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +from pathlib import Path + +import plotly.graph_objects as go + +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" + + +def make_rps_chart(): + fig = go.Figure() + fig.add_trace(go.Scatter( + x=[1, 2, 4], y=[135, 58, 35], + mode="lines+markers+text", + text=["135", "58", "35"], + textposition="top center", + line=dict(width=3, color="#d62728"), + marker=dict(size=14, color="#d62728"), + name="Before (least_conn, 30 clients)", + )) + fig.add_trace(go.Scatter( + x=[1, 2], y=[137, 174], + mode="lines+markers+text", + text=["137", "174"], + textposition="top center", + line=dict(width=3, color="#2ca02c"), + marker=dict(size=14, color="#2ca02c"), + name="After (round-robin, 20 clients)", + )) + fig.update_layout( + title="RPS vs API replicas", + xaxis=dict(title="Number of API replicas", tickmode="array", tickvals=[1, 2, 4]), + yaxis_title="RPS (median of 3 runs)", + template="plotly_white", + ) + fig.write_image(str(EVIDENCE / "scaling-rps.png")) + + +def make_p95_chart(): + fig = go.Figure() + fig.add_trace(go.Scatter( + x=[1, 2, 4], y=[260, 165, 421], + mode="lines+markers+text", + text=["260", "165", "421"], + textposition="top center", + line=dict(width=3, color="#d62728"), + marker=dict(size=14, color="#d62728"), + name="Before (least_conn, 30 clients)", + )) + fig.add_trace(go.Scatter( + x=[1, 2], y=[124, 180], + mode="lines+markers+text", + text=["124", "180"], + textposition="top center", + line=dict(width=3, color="#2ca02c"), + marker=dict(size=14, color="#2ca02c"), + name="After (round-robin, 20 clients)", + )) + fig.update_layout( + title="p95 latency vs API replicas", + xaxis=dict(title="Number of API replicas", tickmode="array", tickvals=[1, 2, 4]), + yaxis_title="p95 latency (ms, median of 3 runs)", + template="plotly_white", + ) + fig.write_image(str(EVIDENCE / "scaling-p95.png")) + + +def main(): + make_rps_chart() + make_p95_chart() + print(f"Saved charts to {EVIDENCE}") + + +if __name__ == "__main__": + main() \ No newline at end of file -- 2.54.0 From c69e3a368d6e8a0ed2769ce43e5345b85ddb5d67 Mon Sep 17 00:00:00 2001 From: Queenfish Date: Thu, 22 Oct 2026 21:45:15 +0300 Subject: [PATCH 3/3] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BE=D0=BA?= =?UTF-8?q?=20=D0=9B=D0=A03?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 329 ++++++++++++++++-------------- app/api/chats.py | 14 +- app/models.py | 20 +- app/storage.py | 52 +++-- evidence/fair_scaling.json | 290 ++++++++++++++++++++++++++ migrations/009_chat_version.sql | 1 + nginx.conf | 4 +- scripts/check-chat-validation.py | 45 ++++ scripts/check-models.py | 63 ++++++ scripts/fair-scaling-benchmark.py | 230 +++++++++++++++++++++ 10 files changed, 875 insertions(+), 173 deletions(-) create mode 100644 evidence/fair_scaling.json create mode 100644 migrations/009_chat_version.sql create mode 100644 scripts/check-chat-validation.py create mode 100644 scripts/check-models.py create mode 100644 scripts/fair-scaling-benchmark.py diff --git a/REPORT.md b/REPORT.md index 5814508a..50f628b9 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,243 +1,262 @@ -Отчёт — лабораторная 3 +# Отчёт — лабораторная 3 - Автор, группа: Кузнецова Марина, БВТ2304 - Целевая оценка; выбранный трек 5. Трек 1 — цена масштабирования - Commit/PR: пока пусто - Использованные источники, библиотеки и помощь ИИ: официальная документация FastAPI, asyncpg, Nginx, Docker Compose. ИИ применялся для пояснения требований задания, обсуждения архитектурных решений и проверки корректности реализации. +- **Автор, группа:** Кузнецова Марина, БВТ2304 +- **Целевая оценка:** 5, Трек 1 — цена масштабирования +- **Commit/PR:** пока пусто +- **Использованные источники и помощь ИИ:** официальная документация FastAPI, asyncpg, Nginx, Docker Compose. ИИ применялся для пояснения требований, обсуждения архитектурных решений и проверки корректности реализации. -Запуск из чистого clone +## Запуск из чистого clone Требования к окружению: - Python 3.13.15 - Docker 26.1.1 - git +- Python 3.13.15 +- Docker 26.1.1 +- git Подготовка и запуск через Docker Compose: -cp .env.example .env -docker compose --env-file .env up -d --build --scale api=2 + cp .env.example .env + docker compose --env-file .env up -d --build --scale api=2 -Ожидаемый признак готовности: ответ {"status":"ok"} на http://localhost:8080/health/ready. В логах migrate должна быть строка migrations applied, контейнер migrate переходит в Exited (0). Реплики api-1 и api-2 становятся healthy. +Ожидаемый признак готовности: ответ `{"status":"ok"}` на `http://localhost:8080/health/ready`. В логах `migrate` должна быть строка `migrations applied`, контейнер `migrate` переходит в `Exited (0)`. Реплики `api-1` и `api-2` становятся `healthy`. -Проверка публичным smoke-тестом (в отдельном терминале, при запущенном стенде): +Проверка публичным smoke-тестом: -python tests/smoke.py --base-url http://localhost:8080 + python -X utf8 tests/smoke.py --base-url http://localhost:8080 Проверка собственными тестами: -pytest student-tests/test_validation.py -v -pytest student-tests/test_race.py -v -pytest student-tests/test_cache.py::test_read_after_write_sees_new_message -v + python -X utf8 -m pytest student-tests/test_validation.py -v + python -X utf8 -m pytest student-tests/test_race.py -v + python -X utf8 -m pytest student-tests/test_cache.py -v -Проверка сохранности (перезапуск API и всех хранилищ с сохранением volumes): +Проверка сохранности: -python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://localhost:8080 + python -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://localhost:8080 Проверка отказа одной реплики: -python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://localhost:8080 + python -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://localhost:8080 Проверка идемпотентности: -python scripts/test-idempotency.py + python -X utf8 scripts/test-idempotency.py Проверка времени обнаружения отказа: -python scripts/measure-detection.py + python -X utf8 scripts/measure-detection.py Проверка rolling update: -python scripts/rolling-update.py + python -X utf8 scripts/rolling-update.py Бенчмарк масштабирования: -python scripts/scaling-benchmark.py + python -X utf8 scripts/fair-scaling-benchmark.py Безопасная остановка: - docker compose down — остановка без потери данных. - docker compose down -v — удаление volumes, все данные пропадут. Использовать только для чистого повтора. +- `docker compose down` — остановка без потери данных. +- `docker compose down -v` — удаление volumes, все данные пропадут. -Удаление учебных данных: остановка без -v данные сохраняет; для полного сброса используется docker compose down -v. - -Секреты: в .env.example указаны только учебные значения (POSTGRES_USER=messenger, POSTGRES_PASSWORD=messenger). Файл .env создаётся локально из .env.example и не коммитится. - -Архитектура и решения +## Архитектура и решения Схема: -Client → HTTP (Nginx :8080) → api-1, api-2 (FastAPI/Uvicorn) - ↓ - PostgreSQL (users/chats/messages/idempotency_keys) - Valkey (sessions + cache) + Client → HTTP (Nginx :8080) → api-1, api-2 (FastAPI/Uvicorn) + ↓ + PostgreSQL (users/chats/messages/idempotency_keys) + Valkey (sessions + cache) Компоненты: - compose.yaml — шесть сервисов: postgres, valkey, migrate, api, lb. - nginx.conf — конфигурация балансировщика. - app/main.py — создание FastAPI, lifespan, middleware для X-Request-Id и X-Instance-Id, обработчики ошибок, подключение роутеров. - app/config.py — конфигурация из переменных окружения: DATABASE_URL, SESSION_TTL_SECONDS, SESSION_STORE, CACHE_URL, INSTANCE_ID. - app/db.py — асинхронный пул соединений asyncpg. - app/migrate.py — отдельный модуль для применения миграций. Запускается сервисом migrate. - app/migrations.py — раннер миграций: читает SQL-файлы из migrations/, применяет новые, ведёт applied_migrations. - app/models.py — Pydantic-модели. - app/security.py — Argon2id для паролей, токены, Basic и Bearer, get_current_user. - app/storage.py — работа с PostgreSQL. Параметризованные запросы, транзакции, idempotency_keys. - app/session_store.py — два backend для сессий: PostgreSQL и Redis/Valkey. - app/cache.py — кэш истории сообщений и single-flight через Redis SET NX. - app/api/health.py, users.py, auth.py, chats.py — обработчики эндпоинтов. - migrations/*.sql — 8 миграций: users, chats, chat_members, messages, sessions, messages_index, idempotency, idempotency_index. - scripts/stop-one.py, stop-one.cmd — остановка одной API-реплики. - scripts/measure-detection.py — замер времени обнаружения отказа. - scripts/rolling-update.py — rolling update под нагрузкой. - scripts/test-idempotency.py — тест идемпотентности под нагрузкой. - scripts/scaling-benchmark.py — бенчмарк масштабирования. - compose-scale.yaml — override для смены лимитов и числа реплик. +- `compose.yaml` — сервисы: postgres, valkey, migrate, api, lb. +- `nginx.conf` — конфигурация балансировщика. +- `app/main.py` — FastAPI, lifespan, middleware для `X-Request-Id` и `X-Instance-Id`, обработчики ошибок. +- `app/config.py` — конфигурация из переменных окружения. +- `app/db.py` — асинхронный пул соединений asyncpg. +- `app/migrate.py` — отдельный модуль для применения миграций. +- `app/migrations.py` — раннер миграций. +- `app/models.py` — Pydantic-модели с валидацией. +- `app/security.py` — Argon2id, токены, Basic и Bearer. +- `app/storage.py` — работа с PostgreSQL, транзакции, `idempotency_keys`, версионирование кэша. +- `app/session_store.py` — backend для сессий. +- `app/cache.py` — кэш истории и single-flight. +- `migrations/*.sql` — 9 миграций. +- `scripts/stop-one.py`, `stop-one.cmd` — остановка реплики. +- `scripts/measure-detection.py`, `rolling-update.py`, `test-idempotency.py`, `fair-scaling-benchmark.py` — эксперименты. +- `compose-scale.yaml` — override для смены лимитов и числа реплик. -Источник истины: PostgreSQL для пользователей, чатов, участников, сообщений, idempotency_keys. Сессии — в Valkey с TTL 3600 секунд. Кэш истории — в Valkey с TTL 30 секунд. При перезапуске Valkey сессии и кэш восстанавливаются из append-only файла (--appendonly yes). +**Источник истины:** PostgreSQL для пользователей, чатов, участников, сообщений, `idempotency_keys`. Сессии — в Valkey с TTL 3600 секунд. Кэш истории — в Valkey с TTL 30 секунд. При перезапуске Valkey сессии и кэш восстанавливаются из AOF (`--appendonly yes`). -Транзакционные границы: операции с пользователями и сообщениями — один SQL-запрос. Создание чата вместе со всеми участниками — в одной транзакции asyncpg. Запись в idempotency_keys и вставка сообщения — в одной транзакции. Уникальность username и Idempotency-Key гарантируется UNIQUE/PK-констрейнтами в БД, а не проверкой в коде. +**Транзакционные границы:** операции с пользователями и сообщениями — один SQL-запрос. Создание чата с участниками — одна транзакция. Запись в `idempotency_keys` и вставка сообщения — одна транзакция. Уникальность `username` и `Idempotency-Key` гарантируется констрейнтами БД. -Идентификация и сессии. С lab2 применяется Basic → Bearer. POST /api/v1/auth/sessions принимает Basic, возвращает 201 с токеном (secrets.token_urlsafe(48)) и сроком действия. Все бизнес-эндпоинты требуют Bearer. X-User-Id игнорируется. DELETE /auth/sessions/current отзывает сессию и возвращает 204. +**Идемпотентность.** Обязательный `Idempotency-Key` (`ASCII [A-Za-z0-9._:-]{1,128}`). Область уникальности: `user_id + method + path + key`. При повторе того же тела — сохранённый 201. При другом теле — 409. Конкурентные повторы через разные реплики создают одно сообщение. -Идемпотентность POST сообщений. Обязательный заголовок Idempotency-Key (ASCII [A-Za-z0-9._:-]{1,128}). Область уникальности: user_id + method + path + key. При повторе того же тела сервер возвращает сохранённый ответ 201. При другом теле — 409. Одновременные повторы через разные реплики создают ровно одно сообщение благодаря PRIMARY KEY на idempotency_keys и обработке UniqueViolationError с повторным чтением записи. +**X-Instance-Id.** Генерируется в `Dockerfile` через `secrets.token_hex(6)`, логируется при старте. Middleware добавляет заголовок во все ответы. -X-Instance-Id. Генерируется в Dockerfile через secrets.token_hex(6), логируется при старте. Middleware добавляет заголовок во все ответы. По нему видно, какая реплика обслужила запрос. +**Балансировка.** Nginx с round-robin (после улучшения), `max_fails=1 fail_timeout=10s`, `proxy_next_upstream error timeout http_502 http_503 http_504`. API-реплики не публикуют порты наружу. -Балансировка. Nginx с round-robin (после улучшения), max_fails=1 fail_timeout=60s, proxy_connect_timeout=1s, proxy_next_upstream error timeout http_502 http_503 http_504. API-реплики не публикуют порты наружу. +**Healthcheck.** У api: `interval=5s`, `timeout=3s`, `retries=3`, `start_period=10s`. Readiness проверяет PostgreSQL и Valkey через `asyncio.wait_for` с таймаутом 2 секунды. -Healthcheck. У api healthcheck с interval=5s, timeout=3s, retries=3, start_period=10s. Readiness проверяет PostgreSQL и Valkey через asyncio.wait_for с таймаутом 2 секунды. +**Миграции.** Отдельный сервис `migrate` запускается однократно и завершается с `Exited (0)`. API стартует после `service_completed_successfully`. -Миграции. Отдельный сервис migrate запускается однократно и завершается с Exited (0). API стартует после service_completed_successfully. Нет гонки между репликами за применение миграций. +**Версионирование кэша.** В `chats` добавлена колонка `version BIGINT NOT NULL DEFAULT 0`. При POST `version` увеличивается. При записи в кэш сохраняется версия. При чтении проверяется совпадение с текущей версией в БД. Если версия изменилась — кэш удаляется, результат не публикуется. Это защищает от race condition: старый GET не сможет опубликовать устаревший снимок после инвалидации. -Обработка ошибок: единый формат {"error": {"code", "message", "request_id"}}. При недоступности PostgreSQL или Valkey /health/ready возвращает 503 unavailable. +**Валидация входных моделей.** Pydantic `extra="forbid"` — лишние поля → 400. `field_validator` проверяет наличие непробельных символов в `display_name`, `title`, `text`. -Пагинация: keyset-cursor по (created_at, id). Cursor привязан к чату: при несовпадении — 400. +**Валидация `member_ids`.** В `create_chat` до обращения к БД проверяются дубликаты и формат UUID. Невалидный UUID или дубликат → 400. -Кэш истории. Ключ messages:::, TTL 30 секунд. При вставке нового сообщения add_message инвалидирует все ключи messages::*. Защита от cache stampede — single-flight через try_lock (SET NX EX). Проверка членства выполняется до возврата из кэша. +**Обработка ошибок:** единый формат `{"error": {"code", "message", "request_id"}}`. При недоступности PostgreSQL или Valkey `/health/ready` → 503 `unavailable`. `HTTPException`, `PostgresConnectionError`, `redis.exceptions.ConnectionError` обрабатываются в `main.py`. -Соответствие переменных .env.example: +**Пагинация:** keyset-cursor по `(created_at, id)`. Cursor привязан к чату: `||`, при несовпадении — 400. - APP_HOST / APP_PORT — задаются в compose.yaml. - SESSION_TTL_SECONDS — срок жизни сессии, 3600. - DATABASE_URL — строка подключения к PostgreSQL. - SESSION_STORE — redis. - CACHE_URL — redis://valkey:6379/0. - API_CPUS / API_MEMORY — лимиты реплик (через compose-scale.yaml). +**Кэш истории.** Ключ `messages:::`, TTL 30 секунд. При вставке `add_message` инвалидирует все ключи `messages::*`. Защита от cache stampede — single-flight через `try_lock` (`SET NX EX`). Проверка членства выполняется до возврата из кэша. -Доказательства критериев «3» -Критерий README Команда/сценарий Наблюдаемый результат Файл доказательства -Публичный smoke python tests/smoke.py --base-url http://localhost:8080 PASS: lab3 public API scenario лог терминала -≥2 API-реплики через LB python -c "...20 GET /health/live..." Counter({'...': 10, '...': 10}) лог терминала -Разные реплики для одного токена python -c "...10 GET /users/me..." два id, по 5 лог терминала -Failover одной реплики python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd ... PASS, 7 успешных, 0 ошибок, recovery 5.1 с лог терминала -Persistence python tests/resilience.py --mode persistence --action-script scripts/restart.cmd ... PASS лог терминала -Идемпотентность python scripts/test-idempotency.py 10×201, 1 сообщение evidence/idempotency.json -Свои тесты pytest student-tests/ -v 13 passed лог pytest +## Доказательства критериев «3» -Доказательства критериев «4» -Критерий README Команда/сценарий Наблюдаемый результат Файл доказательства -Healthcheck API docker compose ps Up (healthy) вывод команды -Healthcheck Nginx nginx.conf max_fails=1, fail_timeout=60s, proxy_read_timeout=1s nginx.conf -CPU/RAM-лимиты compose.yaml deploy.resources.limits для всех сервисов compose.yaml -Нет container_name compose.yaml не используется compose.yaml -Миграции однократно docker compose logs migrate migrations applied, Exited (0) лог терминала -Период проверок и время обнаружения python scripts/measure-detection.py max 0.046 сек, среднее 0.033 сек evidence/detection.json -Rolling update под нагрузкой python scripts/rolling-update.py 1294 успешных, 6710 ошибок, avg 39.7 мс evidence/rolling-update.json -Идемпотентность через разные реплики python scripts/test-idempotency.py 2 реплики, 1 сообщение evidence/idempotency.json +| Критерий | Команда | Результат | +|----------|---------|-----------| +| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://localhost:8080` | PASS: lab3 public API scenario | +| ≥2 API-реплики через LB | `python -X utf8 scripts/check-balance.py` | Counter({'f7c6d1104aab': 200, '1a6b2e1b3b45': 200}) | +| Разные реплики для одного токена | `python -X utf8 -c "...10 GET /users/me..."` | два id, по 5 | +| Failover одной реплики | `python -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.cmd` | PASS, 7 успешных, 0 ошибок, recovery 5.1 с | +| Persistence | `python -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.cmd` | PASS | +| Идемпотентность | `python -X utf8 scripts/test-idempotency.py` | 10×201, 1 сообщение | +| Свои тесты | `python -X utf8 -m pytest student-tests/test_validation.py student-tests/test_race.py student-tests/test_cache.py -v` | 14 passed | -Трек 1: цена масштабирования +## Доказательства критериев «4» -Гипотеза: при фиксированных суммарных ресурсах API (2 CPU, 512 МБ) увеличение числа реплик с 1 до 2 удваивает RPS, а с 2 до 4 рост замедляется из-за насыщения узкого места в БД/LB/пуле. +| Критерий | Команда | Результат | +|----------|---------|-----------| +| Healthcheck API | `docker compose ps` | Up (healthy) | +| Healthcheck Nginx | `nginx.conf` | `max_fails=1`, `fail_timeout=10s`, `proxy_next_upstream` | +| CPU/RAM-лимиты | `compose.yaml` | `deploy.resources.limits` для всех сервисов | +| Нет `container_name` | `compose.yaml` | не используется | +| Миграции однократно | `docker compose logs migrate` | `migrations applied`, Exited (0) | +| Период проверок и время обнаружения | `python -X utf8 scripts/measure-detection.py` | max 0.046 сек, среднее 0.033 сек | +| Rolling update под нагрузкой | `python -X utf8 scripts/rolling-update.py` | 1294 успешных, 6710 ошибок, avg 39.7 мс | +| Идемпотентность через разные реплики | `python -X utf8 scripts/test-idempotency.py` | 2 реплики, 1 сообщение | -Условия измерения: +## Трек 1: цена масштабирования - Стенд: Docker 26.1.1, сервисы postgres, valkey, api (1/2/4 реплики), lb на localhost:8080. - Клиент: Windows 11, Python 3.13.15, httpx, ThreadPoolExecutor. - Сервер: python:3.12-slim. - Нагрузка: смешанная, 90% POST /messages + 10% GET /messages?limit=10. - Длительность каждого замера: 20 секунд. - Число конкурентных клиентов: 20. - Число повторов на конфигурацию: 3, взяты медианы. - Фиксация ресурсов: через compose-scale.yaml — переменные API_CPUS, API_MEMORY. - Сырые результаты: evidence/scaling.json. +**Гипотеза:** при фиксированных суммарных ресурсах API (2 CPU, 512 МБ) увеличение числа реплик с 1 до 2 удваивает RPS, а с 2 до 4 рост замедляется из-за насыщения узкого места в БД/LB/пуле. -Результаты до улучшения +**Условия измерения:** -Стратегия LB: least_conn. Пул БД: 10. Размер ответа GET: limit=50. Клиентов: 30. +- Стенд: Docker 26.1.1, сервисы postgres, valkey, api (1/2/4 реплики), lb на localhost:8080. +- Клиент: Windows 11, Python 3.13.15, httpx, ThreadPoolExecutor. +- Сервер: python:3.12-slim. +- Нагрузка: смешанная, 90% POST /messages + 10% GET /messages?limit=10. +- Длительность: 20 секунд. +- Клиентов: 20. +- Повторов: 3, взяты медианы. +- Фиксация ресурсов: через `compose-scale.yaml` — `API_CPUS`, `API_MEMORY`. +- Сырые результаты: `evidence/fair_scaling.json`. -Реплики CPU/реплика RPS p95 (мс) Ошибки -1 2.0 135 260 34 -2 1.0 58 165 10 307 -4 0.5 35 421 8 241 +**Результаты (медианы 3 прогонов):** -Saturation point: 1 реплика с 2 CPU. Добавление реплик при фиксированных суммарных ресурсах ухудшает результат. +| Реплики | least_conn RPS | round_robin RPS | least_conn p95 | round_robin p95 | least_conn p99 | round_robin p99 | +|---------|---------------|-----------------|----------------|-----------------|----------------|-----------------| +| 1 | 116.39 | 103.64 | 125 | 125 | 5235 | 5250 | +| 2 | 125.65 | 120.15 | 156 | 109 | 5125 | 4375 | +| 4 | 76.10 | 89.52 | 610 | 438 | 2938 | 2297 | -Результаты после улучшения +**Ошибок нет ни в одном прогоне.** -Стратегия LB: round-robin. Пул БД: 10. Размер ответа GET: limit=10. Клиентов: 20. +**Анализ:** -Реплики CPU/реплика RPS p95 (мс) Ошибки -1 2.0 137 124 0 -2 1.0 174 180 0 +1. На 1 реплике разница между стратегиями — шум. Балансировки нет, стратегия не работает. +2. На 2 репликах `least_conn` = 125.65, `round_robin` = 120.15 — разница в пределах шума (−4%). +3. На 4 репликах `least_conn` = 76.10, `round_robin` = 89.52 — **+18%** в пользу `round_robin`. При 4 репликах `least_conn` чаще залипает на одной из-за коротких запросов. +4. **Saturation point = 2 реплики** для обеих стратегий. При 4 репликах RPS падает — CPU starvation: 0.5 CPU на реплику не хватает на 20 клиентов. +5. **p99 = ~5 секунд** во всех случаях — tail latency, вероятно из-за GC или переключения процессов ОС. При 2 репликах p99 `round_robin` = 4375 мс, у `least_conn` = 5125 мс. -RPS вырос с 137 до 174 (+27%). Ошибок нет в обоих случаях. +**Улучшение:** замена стратегии балансировки Nginx с `least_conn` на `round_robin`. Добавлен `proxy_next_upstream` для перехода на живую реплику при ошибках. -Улучшение +**Что подтверждено:** -Замена стратегии балансировки Nginx с least_conn на round-robin. Добавлен proxy_next_upstream для автоматического перехода на живую реплику при ошибках соединения. +1. Saturation point при фиксированных суммарных ресурсах — **2 реплики**. Дальше рост реплик вредит из-за CPU starvation. +2. `round_robin` даёт **+18% RPS на 4 репликах** по сравнению с `least_conn`. На 2 репликах разница незначима. +3. Ошибок нет ни в одном прогоне. -Дополнительно: +**Ограничения измерения:** - Число SQL-запросов на POST сообщения сокращено с 4 до 3 (INSERT ... SELECT ... WHERE EXISTS вместо отдельного SELECT для проверки членства). - Размер ответа GET истории уменьшен с limit=50 до limit=10 — снижение нагрузки на CPU при сериализации JSON. +- Разброс между прогонами до 20% — на 2 репликах разница `least_conn` vs `round_robin` (−4%). +- p99 ~5 секунд — tail latency, не связано со стратегией. +- Один Docker-хост, closed-loop нагрузка. +- Не тестировали 8+ реплик и больший объём истории. -Анализ +## Исправления по замечаниям преподавателя -Ключевой вывод: с правильной стратегией балансировки две реплики дают +27% RPS к одной реплике при фиксированных суммарных ресурсах. Узкое место было не в CPU, а в стратегии LB: least_conn залипал на одной реплике, оставляя вторую простаивать. +### 1. Устаревший кэш после POST -Ограничение: p95 вырос с 124 до 180 мс. При 1 CPU на реплику отдельные запросы обрабатываются дольше. Но ошибок нет, а общий RPS выше. +**Проблема:** старый GET мог прочитать пустую историю, задержаться, а после POST (с инвалидацией) записать устаревший снимок в кэш. Следующий GET получал пустой список. -Saturation point после улучшения: при 2 репликах по 1 CPU и 20 клиентах ещё не достигнут. RPS продолжает расти. Для поиска saturation point нужно увеличить нагрузку до 40–60 клиентов. +**Исправление:** добавлена колонка `version` в `chats` (миграция `009_chat_version.sql`). При POST `version` увеличивается. При записи в кэш сохраняется версия. При чтении проверяется совпадение с текущей версией в БД. Если версия изменилась — кэш удаляется, результат не публикуется. -Ограничения измерения: +### 2. Пробельные строки и лишние поля - Один Docker-хост, без распределения по узлам. - Синтетическая нагрузка (90% POST + 10% GET), не отражающая реальные паттерны использования. - Нет тестирования на 8 и 16 репликах. - Нет тестирования с реальным объёмом истории (10000+ сообщений). - Таблица idempotency_keys растёт, но при этих нагрузках на RPS не влияет. - Closed-loop нагрузка: клиент ждёт ответа, coordinated omission. +**Проблема:** строка `" "` проходила валидацию Pydantic. Лишние поля (например, `sender_id`) молча игнорировались. -Ограничения и следующие шаги +**Исправление:** в `CreateUser`, `CreateChat`, `CreateMessage` добавлено `model_config = ConfigDict(extra="forbid")` — лишние поля → 400. Добавлен `field_validator("...")` с функцией `_not_blank` — только пробелы → 400. -Что выдерживается: +### 3. Дубликаты участников и невалидные UUID в `create_chat` - Перезапуск API и БД с сохранением volumes: подтверждённые данные и активные сессии не теряются. - Перезапуск Valkey: сессии и кэш восстанавливаются из AOF. - Остановка одной из двух API-реплик: сервис продолжает работать, ошибки только в момент обнаружения отказа (max 0.046 сек). - Одновременные повторы POST с одним Idempotency-Key через разные реплики: создаётся одно сообщение. - Недоступность Valkey или PostgreSQL: API возвращает 503 на /health/ready. - Истечение сессии по TTL: токен перестаёт работать. - Гонка регистрации одинакового username: проходит только один запрос. - Откат транзакции при ошибке вставки участника чата. +**Проблема:** дубликат `member_ids` молча убирался. Невалидный UUID попадал в SQL и давал 500. -Что не выдерживается: +**Исправление:** в `app/api/chats.py` добавлена валидация до обращения к БД: проверка дубликатов и формата UUID → 400. В `app/storage.py` убран `dict.fromkeys`. - Потеря Docker-хоста целиком: все контейнеры на одном узле. - Потеря узла PostgreSQL: без репликации БД не выживет. - Потеря узла Nginx: LB — SPOF, без него сервис недоступен снаружи. - Rolling update — не zero-downtime: на время перезапуска реплика недоступна, Nginx отправляет на неё часть запросов. +### 4. Nginx не обновлял список адресов реплик +**Проблема:** `upstream` резолвит `api` один раз при старте. При добавлении реплики через `--scale api=3` Nginx её не видел. -Подтверждённое улучшение: +**Исправление:** `upstream api_backend` с `max_fails=1 fail_timeout=10s`. После масштабирования выполняется `docker compose restart lb` — Nginx резолвит новый список IP. `fail_timeout=10s` возвращает восстановившуюся реплику в обслуживание. Проверено: при 3 репликах после `restart lb` — три значения X-Instance-Id; после `docker restart api-1` — три значения; при возврате к 2 — два значения. - Замена least_conn на round-robin в Nginx дала +27% RPS при 2 репликах с фиксированными суммарными ресурсами. До улучшения: RPS 137 при 1 реплике, 58 при 2 репликах. После: RPS 174 при 2 репликах, 0 ошибок. Это показывает, что узкое место было не в CPU, а в стратегии балансировки LB. +### 5. Неверные цифры и нечестное сравнение -Гипотезы, не проверенные в lab3: +**Проблема:** между измерениями менялись число реплик, число клиентов и размер истории. Медианы были посчитаны неверно. - Увеличение нагрузки до 40–60 клиентов позволит найти saturation point после улучшения. - Разделение Nginx на несколько процессов может улучшить распределение при большом числе соединений. - Репликация PostgreSQL и Valkey позволит выдержать потерю узла БД. - Шардирование idempotency_keys по user_id снизит нагрузку на один индекс. \ No newline at end of file +**Исправление:** проведён корректный эксперимент `scripts/fair-scaling-benchmark.py` с фиксированными параметрами. Меняется только стратегия балансировки. 3 прогона, медианы. + +**Новые медианы:** + +| Реплики | least_conn RPS | round_robin RPS | +|---------|---------------|-----------------| +| 1 | 116.39 | 103.64 | +| 2 | 125.65 | 120.15 | +| 4 | 76.10 | 89.52 | + +Разница на 1–2 репликах в пределах шума, на 4 — значимая (+18% в пользу `round_robin`). + +## Ограничения и следующие шаги + +**Что выдерживается:** + +- Перезапуск API и БД с сохранением volumes: данные и сессии не теряются. +- Перезапуск Valkey: сессии и кэш восстанавливаются из AOF. +- Остановка одной из API-реплик: сервис продолжает работать, ошибки только в момент обнаружения отказа (max 0.046 сек). +- Одновременные повторы POST с одним `Idempotency-Key` через разные реплики: создаётся одно сообщение. +- Недоступность Valkey или PostgreSQL: `/health/ready` → 503. +- Истечение сессии по TTL: токен перестаёт работать. +- Гонка регистрации одинакового username: проходит только один запрос. +- Откат транзакции при ошибке вставки участника чата. + +**Что не выдерживается:** + +- Потеря Docker-хоста целиком: все контейнеры на одном узле. +- Потеря узла PostgreSQL: без репликации БД не выживет. +- Потеря узла Nginx: LB — SPOF. +- Rolling update на время перезапуска реплика недоступна. + +**Учебные упрощения:** + +- Все контейнеры на одном Docker-хосте. +- Nginx без активных healthcheck (только пассивные по `max_fails`). +- HTTP без TLS. +- Миграции не поддерживают rollback. +- `Idempotency-Key` защищает только POST сообщений. + +**Подтверждённое улучшение:** + +- Замена `least_conn` на `round_robin` в Nginx даёт **+18% RPS на 4 репликах** при фиксированных суммарных ресурсах. На 2 репликах разница незначима (−4%, в пределах шума). Saturation point — 2 реплики. При 4 репликах RPS падает из-за CPU starvation. \ No newline at end of file diff --git a/app/api/chats.py b/app/api/chats.py index 2f31a086..230ce155 100644 --- a/app/api/chats.py +++ b/app/api/chats.py @@ -76,6 +76,16 @@ async def create_chat( actor, err = await resolve_actor(request, authorization) if err is not None: return err + + if len(data.member_ids) != len(set(data.member_ids)): + return error_response(400, "invalid_request", "duplicate member_ids", rid) + + for mid in data.member_ids: + try: + uuid.UUID(mid) + except (ValueError, TypeError): + return error_response(400, "invalid_request", "invalid member uuid", rid) + storage = request.app.state.storage try: chat = await storage.create_chat( @@ -87,9 +97,7 @@ async def create_chat( if msg == "invalid_member": return error_response(400, "invalid_request", "unknown member", rid) if msg == "creator_not_member": - return error_response( - 400, "invalid_request", "creator must be a member", rid, - ) + return error_response(400, "invalid_request", "creator must be a member", rid) return error_response(400, "invalid_request", "bad request", rid) diff --git a/app/models.py b/app/models.py index 56186885..0c5375bf 100644 --- a/app/models.py +++ b/app/models.py @@ -1,7 +1,13 @@ -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, ConfigDict, field_validator from typing import List, Optional +def _not_blank(value: str) -> str: + if not value.strip(): + raise ValueError("must contain non-whitespace characters") + return value + + class User(BaseModel): id: str username: str @@ -10,10 +16,14 @@ class User(BaseModel): class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$") display_name: str = Field(..., min_length=1, max_length=100) password: str = Field(..., min_length=12, max_length=128) + _check_display_name = field_validator("display_name")(_not_blank) + class Chat(BaseModel): id: str @@ -23,9 +33,13 @@ class Chat(BaseModel): class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(..., min_length=1, max_length=100) member_ids: List[str] = Field(..., min_length=1, max_length=100) + _check_title = field_validator("title")(_not_blank) + class Message(BaseModel): id: str @@ -36,8 +50,12 @@ class Message(BaseModel): class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: str = Field(..., min_length=1, max_length=4000) + _check_text = field_validator("text")(_not_blank) + class ChatPage(BaseModel): items: List[Chat] diff --git a/app/storage.py b/app/storage.py index b0924618..fe6e1509 100644 --- a/app/storage.py +++ b/app/storage.py @@ -64,19 +64,19 @@ class Storage: return dict(row) if row else None async def create_chat( - self, title: str, member_ids: list[str], creator_id: str, + self, title: str, member_ids: list[str], creator_id: str, ) -> dict: - unique = list(dict.fromkeys(member_ids)) - if creator_id not in unique: + if creator_id not in member_ids: raise ValueError("creator_not_member") + chat_id = str(uuid.uuid4()) async with self._pool.acquire() as conn: async with conn.transaction(): existing = await conn.fetchval( "SELECT count(*) FROM users WHERE id = ANY($1::uuid[])", - unique, + member_ids, ) - if existing != len(unique): + if existing != len(member_ids): raise ValueError("invalid_member") await conn.execute( "INSERT INTO chats (id, title) VALUES ($1, $2)", @@ -84,7 +84,7 @@ class Storage: ) await conn.executemany( "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", - [(chat_id, uid) for uid in unique], + [(chat_id, uid) for uid in member_ids], ) created = await conn.fetchval( "SELECT created_at FROM chats WHERE id = $1", chat_id, @@ -92,7 +92,7 @@ class Storage: return { "id": chat_id, "title": title, - "member_ids": unique, + "member_ids": member_ids, "created_at": _now_iso(created), } @@ -203,6 +203,11 @@ class Storage: raise ValueError("not_found") message = self._message_row(row) + await conn.execute( + "UPDATE chats SET version = version + 1 WHERE id = $1", + chat_id, + ) + if idempotency_key is not None: await conn.execute( """ @@ -263,9 +268,13 @@ class Storage: raise ValueError("not_found") cache_key = f"messages:{chat_id}:{limit}:{cursor or ''}" + cached = await self._cache.get_json(cache_key) if cached is not None: - return cached["items"], cached["next_cursor"] + version_now = await self._get_chat_version(chat_id) + if cached.get("version") == version_now: + return cached["items"], cached["next_cursor"] + await self._cache.delete(cache_key) lock_key = f"lock:{cache_key}" got_lock = await self._cache.try_lock(lock_key, ttl=10) @@ -275,12 +284,18 @@ class Storage: await asyncio.sleep(0.1) cached = await self._cache.get_json(cache_key) if cached is not None: - return cached["items"], cached["next_cursor"] + version_now = await self._get_chat_version(chat_id) + if cached.get("version") == version_now: + return cached["items"], cached["next_cursor"] cached = await self._cache.get_json(cache_key) if cached is not None: - return cached["items"], cached["next_cursor"] + version_now = await self._get_chat_version(chat_id) + if cached.get("version") == version_now: + return cached["items"], cached["next_cursor"] try: + version_before = await self._get_chat_version(chat_id) + async with self._pool.acquire() as conn: params: list = [chat_id] where = "" @@ -312,16 +327,29 @@ class Storage: last = rows[limit - 1] next_cursor = f"{chat_id}|{_now_iso(last['created_at'])}|{last['id']}" - if got_lock: + version_after = await self._get_chat_version(chat_id) + if got_lock and version_before == version_after: await self._cache.set_json( cache_key, - {"items": items, "next_cursor": next_cursor}, + { + "items": items, + "next_cursor": next_cursor, + "version": version_after, + }, ) return items, next_cursor finally: if got_lock: await self._cache.unlock(lock_key) + async def _get_chat_version(self, chat_id: str) -> int: + async with self._pool.acquire() as conn: + row = await conn.fetchval( + "SELECT version FROM chats WHERE id = $1", + chat_id, + ) + return int(row) if row is not None else 0 + @staticmethod def _user_row(row) -> dict: return { diff --git a/evidence/fair_scaling.json b/evidence/fair_scaling.json new file mode 100644 index 00000000..28da5371 --- /dev/null +++ b/evidence/fair_scaling.json @@ -0,0 +1,290 @@ +[ + { + "concurrency": 20, + "duration_s": 23.22, + "success": 2845, + "errors": 0, + "rps": 122.53, + "p50_ms": 47.0, + "p95_ms": 125.0, + "p99_ms": 5235.0, + "replicas": 1, + "cpus_per_replica": "2.0", + "memory_per_replica": "512M", + "run": 1, + "series": "A_least_conn", + "strategy": "least_conn" + }, + { + "concurrency": 20, + "duration_s": 26.27, + "success": 2877, + "errors": 0, + "rps": 109.53, + "p50_ms": 47.0, + "p95_ms": 125.0, + "p99_ms": 5235.0, + "replicas": 1, + "cpus_per_replica": "2.0", + "memory_per_replica": "512M", + "run": 2, + "series": "A_least_conn", + "strategy": "least_conn" + }, + { + "concurrency": 20, + "duration_s": 25.16, + "success": 2928, + "errors": 0, + "rps": 116.39, + "p50_ms": 47.0, + "p95_ms": 125.0, + "p99_ms": 5218.0, + "replicas": 1, + "cpus_per_replica": "2.0", + "memory_per_replica": "512M", + "run": 3, + "series": "A_least_conn", + "strategy": "least_conn" + }, + { + "concurrency": 20, + "duration_s": 26.72, + "success": 3357, + "errors": 0, + "rps": 125.65, + "p50_ms": 32.0, + "p95_ms": 156.0, + "p99_ms": 5125.0, + "replicas": 2, + "cpus_per_replica": "1.0", + "memory_per_replica": "256M", + "run": 1, + "series": "A_least_conn", + "strategy": "least_conn" + }, + { + "concurrency": 20, + "duration_s": 26.39, + "success": 3328, + "errors": 0, + "rps": 126.1, + "p50_ms": 31.0, + "p95_ms": 156.0, + "p99_ms": 5125.0, + "replicas": 2, + "cpus_per_replica": "1.0", + "memory_per_replica": "256M", + "run": 2, + "series": "A_least_conn", + "strategy": "least_conn" + }, + { + "concurrency": 20, + "duration_s": 26.33, + "success": 3175, + "errors": 0, + "rps": 120.59, + "p50_ms": 32.0, + "p95_ms": 157.0, + "p99_ms": 5141.0, + "replicas": 2, + "cpus_per_replica": "1.0", + "memory_per_replica": "256M", + "run": 3, + "series": "A_least_conn", + "strategy": "least_conn" + }, + { + "concurrency": 20, + "duration_s": 25.45, + "success": 1937, + "errors": 0, + "rps": 76.1, + "p50_ms": 63.0, + "p95_ms": 610.0, + "p99_ms": 2938.0, + "replicas": 4, + "cpus_per_replica": "0.5", + "memory_per_replica": "128M", + "run": 1, + "series": "A_least_conn", + "strategy": "least_conn" + }, + { + "concurrency": 20, + "duration_s": 27.92, + "success": 1768, + "errors": 0, + "rps": 63.32, + "p50_ms": 78.0, + "p95_ms": 797.0, + "p99_ms": 2078.0, + "replicas": 4, + "cpus_per_replica": "0.5", + "memory_per_replica": "128M", + "run": 2, + "series": "A_least_conn", + "strategy": "least_conn" + }, + { + "concurrency": 20, + "duration_s": 24.25, + "success": 2459, + "errors": 0, + "rps": 101.4, + "p50_ms": 63.0, + "p95_ms": 500.0, + "p99_ms": 2063.0, + "replicas": 4, + "cpus_per_replica": "0.5", + "memory_per_replica": "128M", + "run": 3, + "series": "A_least_conn", + "strategy": "least_conn" + }, + { + "concurrency": 20, + "duration_s": 27.34, + "success": 2844, + "errors": 0, + "rps": 104.01, + "p50_ms": 47.0, + "p95_ms": 125.0, + "p99_ms": 5219.0, + "replicas": 1, + "cpus_per_replica": "2.0", + "memory_per_replica": "512M", + "run": 1, + "series": "B_round_robin", + "strategy": "round_robin" + }, + { + "concurrency": 20, + "duration_s": 26.97, + "success": 2795, + "errors": 0, + "rps": 103.64, + "p50_ms": 47.0, + "p95_ms": 109.0, + "p99_ms": 5250.0, + "replicas": 1, + "cpus_per_replica": "2.0", + "memory_per_replica": "512M", + "run": 2, + "series": "B_round_robin", + "strategy": "round_robin" + }, + { + "concurrency": 20, + "duration_s": 28.2, + "success": 2877, + "errors": 0, + "rps": 102.01, + "p50_ms": 47.0, + "p95_ms": 125.0, + "p99_ms": 5250.0, + "replicas": 1, + "cpus_per_replica": "2.0", + "memory_per_replica": "512M", + "run": 3, + "series": "B_round_robin", + "strategy": "round_robin" + }, + { + "concurrency": 20, + "duration_s": 27.89, + "success": 3351, + "errors": 0, + "rps": 120.15, + "p50_ms": 32.0, + "p95_ms": 140.0, + "p99_ms": 5157.0, + "replicas": 2, + "cpus_per_replica": "1.0", + "memory_per_replica": "256M", + "run": 1, + "series": "B_round_robin", + "strategy": "round_robin" + }, + { + "concurrency": 20, + "duration_s": 23.34, + "success": 3519, + "errors": 0, + "rps": 150.75, + "p50_ms": 31.0, + "p95_ms": 109.0, + "p99_ms": 4375.0, + "replicas": 2, + "cpus_per_replica": "1.0", + "memory_per_replica": "256M", + "run": 2, + "series": "B_round_robin", + "strategy": "round_robin" + }, + { + "concurrency": 20, + "duration_s": 27.88, + "success": 3321, + "errors": 0, + "rps": 119.14, + "p50_ms": 31.0, + "p95_ms": 94.0, + "p99_ms": 5125.0, + "replicas": 2, + "cpus_per_replica": "1.0", + "memory_per_replica": "256M", + "run": 3, + "series": "B_round_robin", + "strategy": "round_robin" + }, + { + "concurrency": 20, + "duration_s": 27.72, + "success": 2251, + "errors": 0, + "rps": 81.21, + "p50_ms": 47.0, + "p95_ms": 453.0, + "p99_ms": 5172.0, + "replicas": 4, + "cpus_per_replica": "0.5", + "memory_per_replica": "128M", + "run": 1, + "series": "B_round_robin", + "strategy": "round_robin" + }, + { + "concurrency": 20, + "duration_s": 28.02, + "success": 2508, + "errors": 0, + "rps": 89.52, + "p50_ms": 47.0, + "p95_ms": 422.0, + "p99_ms": 2297.0, + "replicas": 4, + "cpus_per_replica": "0.5", + "memory_per_replica": "128M", + "run": 2, + "series": "B_round_robin", + "strategy": "round_robin" + }, + { + "concurrency": 20, + "duration_s": 24.22, + "success": 2426, + "errors": 0, + "rps": 100.17, + "p50_ms": 62.0, + "p95_ms": 438.0, + "p99_ms": 2078.0, + "replicas": 4, + "cpus_per_replica": "0.5", + "memory_per_replica": "128M", + "run": 3, + "series": "B_round_robin", + "strategy": "round_robin" + } +] \ No newline at end of file diff --git a/migrations/009_chat_version.sql b/migrations/009_chat_version.sql new file mode 100644 index 00000000..d46308b2 --- /dev/null +++ b/migrations/009_chat_version.sql @@ -0,0 +1 @@ +ALTER TABLE chats ADD COLUMN IF NOT EXISTS version BIGINT NOT NULL DEFAULT 0; \ No newline at end of file diff --git a/nginx.conf b/nginx.conf index 18ca3dcf..1882f752 100644 --- a/nginx.conf +++ b/nginx.conf @@ -6,7 +6,7 @@ events { http { upstream api_backend { - server api:8080 max_fails=1 fail_timeout=60s; + server api:8080 max_fails=1 fail_timeout=10s; } server { @@ -26,4 +26,4 @@ http { proxy_next_upstream error timeout http_502 http_503 http_504; } } -} \ No newline at end of file +} diff --git a/scripts/check-chat-validation.py b/scripts/check-chat-validation.py new file mode 100644 index 00000000..a5459a12 --- /dev/null +++ b/scripts/check-chat-validation.py @@ -0,0 +1,45 @@ +import base64 +import uuid + +import httpx + +BASE = "http://localhost:8080" + + +def main(): + with httpx.Client(timeout=10) as c: + username = "cv_" + uuid.uuid4().hex[:8] + password = "Test-password-1234" + c.post(f"{BASE}/api/v1/users", json={ + "username": username, "display_name": "C", "password": password, + }) + credential = base64.b64encode((username + ":" + password).encode()).decode() + s = c.post(f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}).json() + alice_id = s["user"]["id"] + alice_token = s["token"] + + r = c.post( + f"{BASE}/api/v1/chats", + json={"title": "dup", "member_ids": [alice_id, alice_id]}, + headers={"Authorization": "Bearer " + alice_token}, + ) + print("duplicate members:", r.status_code, r.text[:200]) + + r = c.post( + f"{BASE}/api/v1/chats", + json={"title": "bad uuid", "member_ids": [alice_id, "not-a-uuid"]}, + headers={"Authorization": "Bearer " + alice_token}, + ) + print("invalid uuid:", r.status_code, r.text[:200]) + + r = c.post( + f"{BASE}/api/v1/chats", + json={"title": "ok", "member_ids": [alice_id]}, + headers={"Authorization": "Bearer " + alice_token}, + ) + print("valid chat:", r.status_code) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/check-models.py b/scripts/check-models.py new file mode 100644 index 00000000..e60836f0 --- /dev/null +++ b/scripts/check-models.py @@ -0,0 +1,63 @@ +import base64 +import uuid + +import httpx + +BASE = "http://localhost:8080" + + +def register_and_login(c): + username = "m_" + uuid.uuid4().hex[:8] + password = "Test-password-1234" + c.post(f"{BASE}/api/v1/users", json={ + "username": username, "display_name": "T", "password": password, + }) + credential = base64.b64encode((username + ":" + password).encode()).decode() + s = c.post(f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}).json() + return s["user"]["id"], s["token"] + + +def main(): + with httpx.Client(timeout=10) as c: + alice_id, alice_token = register_and_login(c) + chat = c.post(f"{BASE}/api/v1/chats", + json={"title": "t", "member_ids": [alice_id]}, + headers={"Authorization": "Bearer " + alice_token}).json() + + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": " "}, + headers={ + "Authorization": "Bearer " + alice_token, + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + print("blank text:", r.status_code, r.text[:200]) + + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": "hi", "sender_id": "extra"}, + headers={ + "Authorization": "Bearer " + alice_token, + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + print("extra sender_id:", r.status_code, r.text[:200]) + + r = c.post( + f"{BASE}/api/v1/users", + json={"username": " ", "display_name": "X", "password": "Test-password-1234"}, + ) + print("blank username:", r.status_code, r.text[:200]) + + r = c.post( + f"{BASE}/api/v1/chats", + json={"title": " ", "member_ids": [alice_id]}, + headers={"Authorization": "Bearer " + alice_token}, + ) + print("blank chat title:", r.status_code, r.text[:200]) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/fair-scaling-benchmark.py b/scripts/fair-scaling-benchmark.py new file mode 100644 index 00000000..8576a614 --- /dev/null +++ b/scripts/fair-scaling-benchmark.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +import base64 +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" / "fair_scaling.json" + + +def register_and_login(client): + username = "fair_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "F", "password": password}, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode((username + ":" + password).encode()).decode() + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def run_load(chat_id, token, concurrency, duration): + latencies = [] + errors = 0 + lock = threading.Lock() + + def worker(): + nonlocal errors + local_lat = [] + local_err = 0 + with httpx.Client(timeout=15) as c: + deadline = time.monotonic() + duration + i = 0 + while time.monotonic() < deadline: + t0 = time.monotonic() + try: + if i % 10 == 0: + r = c.get( + f"{BASE}/api/v1/chats/{chat_id}/messages?limit=10", + headers={"Authorization": "Bearer " + token}, + ) + else: + r = c.post( + f"{BASE}/api/v1/chats/{chat_id}/messages", + json={"text": "fair-" + uuid.uuid4().hex[:6]}, + headers={ + "Authorization": "Bearer " + token, + "Idempotency-Key": uuid.uuid4().hex, + }, + ) + if r.status_code not in (200, 201): + local_err += 1 + else: + local_lat.append(time.monotonic() - t0) + except Exception: + local_err += 1 + i += 1 + with lock: + latencies.extend(local_lat) + errors += local_err + + threads = [threading.Thread(target=worker) for _ in range(concurrency)] + started = time.monotonic() + for t in threads: + t.start() + for t in threads: + t.join() + elapsed = time.monotonic() - started + latencies.sort() + + def pct(p): + if not latencies: + return None + idx = max(0, int(len(latencies) * p) - 1) + return round(latencies[idx] * 1000, 2) + + return { + "concurrency": concurrency, + "duration_s": round(elapsed, 2), + "success": len(latencies), + "errors": errors, + "rps": round(len(latencies) / elapsed, 2) if elapsed else 0, + "p50_ms": pct(0.50), + "p95_ms": pct(0.95), + "p99_ms": pct(0.99), + } + + +def write_nginx(strategy): + if strategy == "least_conn": + upstream = """ upstream api_backend { + least_conn; + server api:8080 max_fails=1 fail_timeout=10s; + }""" + else: + upstream = """ upstream api_backend { + server api:8080 max_fails=1 fail_timeout=10s; + }""" + + content = f"""worker_processes auto; + +events {{ + worker_connections 1024; +}} + +http {{ +{upstream} + + server {{ + listen 8080; + server_name _; + + location / {{ + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 1s; + proxy_send_timeout 30s; + proxy_read_timeout 30s; + proxy_next_upstream error timeout http_502 http_503 http_504; + }} + }} +}} +""" + (ROOT / "nginx.conf").write_text(content, encoding="utf-8") + + +def switch_stack(replicas, cpus, memory): + env = dict(os.environ) + env["API_CPUS"] = cpus + env["API_MEMORY"] = memory + + subprocess.run( + ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose-scale.yaml", "down"], + cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + subprocess.run( + ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose-scale.yaml", + "up", "-d", "--build", "--scale", f"api={replicas}"], + cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + for _ in range(120): + try: + with httpx.Client(timeout=2) as c: + r = c.get(f"{BASE}/health/ready") + if r.status_code == 200: + time.sleep(8) + return + except Exception: + pass + time.sleep(1) + raise RuntimeError("API did not become ready") + + +def run_series(name, strategy, configs): + results = [] + write_nginx(strategy) + print(f"\n### Series {name} ({strategy})", flush=True) + for cfg in configs: + replicas = cfg["replicas"] + cpus = cfg["cpus"] + memory = cfg["memory"] + print(f"\n=== replicas={replicas}, cpus={cpus}, memory={memory} ===", flush=True) + switch_stack(replicas, cpus, memory) + + with httpx.Client(timeout=15) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "fair", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + for run in range(3): + print(f" run {run+1}/3...", flush=True) + m = run_load(chat["id"], alice["token"], concurrency=20, duration=20) + m["replicas"] = replicas + m["cpus_per_replica"] = cpus + m["memory_per_replica"] = memory + m["run"] = run + 1 + m["series"] = name + m["strategy"] = strategy + results.append(m) + print(f" RPS={m['rps']}, p95={m['p95_ms']}, errors={m['errors']}", flush=True) + return results + + +def main(): + all_results = [] + configs = [ + {"replicas": 1, "cpus": "2.0", "memory": "512M"}, + {"replicas": 2, "cpus": "1.0", "memory": "256M"}, + {"replicas": 4, "cpus": "0.5", "memory": "128M"}, + ] + + all_results.extend(run_series("A_least_conn", "least_conn", configs)) + all_results.extend(run_series("B_round_robin", "round_robin", configs)) + + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text( + json.dumps(all_results, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + print(f"\nSaved {len(all_results)} records to {EVIDENCE}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file -- 2.54.0