forked from mtusi2026/mtusi2026lab3
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
04a74118a1 |
+3
-9
@@ -1,13 +1,7 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
# Per-replica limits; adjust for the fixed-budget track 1 experiment.
|
||||
API_CPU_LIMIT=1.0
|
||||
API_MEMORY_LIMIT=512M
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
name: Application checks
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
compose-tests:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: python -m pip install -r requirements-dev.txt
|
||||
- name: Create local test credentials
|
||||
run: |
|
||||
cp .env.example .env
|
||||
./scripts/bootstrap-secrets.sh
|
||||
- name: Validate and start the application
|
||||
run: |
|
||||
make check validate PYTHON=python
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180
|
||||
- name: Run API and concurrency tests
|
||||
run: |
|
||||
make test
|
||||
make test-idempotency
|
||||
- name: Verify persistence and failover
|
||||
run: |
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
- name: Stop test stack
|
||||
if: always()
|
||||
run: docker compose --env-file .env down -v
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
FROM python:3.12.7-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY . /app
|
||||
RUN chown -R app:app /app
|
||||
|
||||
USER app
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).read()" || exit 1
|
||||
|
||||
CMD ["python3", "src/server.py"]
|
||||
@@ -1,10 +1,10 @@
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
BASE_URL ?= http://127.0.0.1:8080
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover test-idempotency track1
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -28,3 +28,9 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-idempotency:
|
||||
$(PYTHON) student-tests/test_idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
track1:
|
||||
$(PYTHON) student-tests/track1_benchmark.py
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Запустить несколько взаимозаменяемых экземпляров API за единой точкой входа и сохранить корректность при остановке одного из них.
|
||||
|
||||
Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен.
|
||||
В репозитории реализован стенд лабораторной: контракт и публичные проверки дополнены приложением, Docker-инфраструктурой, сценариями отказов и исследовательским экспериментом. Критерии задания и выбранные решения описаны ниже и в [отчёте](REPORT.md).
|
||||
|
||||
[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md)
|
||||
|
||||
@@ -12,6 +12,34 @@
|
||||
|
||||
**Архитектура:** Клиент → LB → API × 2+ → общие PostgreSQL и хранилище сессий. Compose или Swarm — на выбор.
|
||||
|
||||
## Реализация этого решения
|
||||
|
||||
Используется Docker Compose: HAProxy с активной проверкой `GET /health/ready` и DNS-resolver Docker, два и более одинаковых API-контейнера, одна PostgreSQL. Публичный порт публикуется только у HAProxy на loopback. Сессии и идемпотентность находятся в PostgreSQL; локального состояния сессий у реплик нет. Миграции выполняет отдельный одноразовый сервис до старта API.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
./scripts/bootstrap-secrets.sh
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 120
|
||||
make check
|
||||
make test
|
||||
make test-idempotency
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
# Полная матрица трека 1: 1/2/4 реплики при фиксированном бюджете и 1/2/4/8 при росте.
|
||||
make track1
|
||||
```
|
||||
|
||||
Локальные секреты хранятся в игнорируемой папке `secrets/`; скрипт их не выводит. API использует имя контейнера как стабильный на время жизни процесса `X-Instance-Id`. Для намеренного удаления учебной БД после проверки выполните `docker compose --env-file .env down -v`; эта команда необратимо удаляет volume и не используется сценариями restart/failover.
|
||||
|
||||
Перед завершением работы восстановите остановленную реплику командой `docker compose --env-file .env up -d --scale api=2 --wait`. Подробные методика, ограничения и доказательства указаны в `REPORT.md`.
|
||||
|
||||
Для проверки поочерёдной замены экземпляров под нагрузкой запустите в одном терминале `python3 tests/load.py --duration 30 --concurrency 8 --output evidence/rolling-restart-load.json`, а пока она работает — в другом `scripts/rolling-restart.sh`. Скрипт перезапускает по одному контейнеру, не останавливая вторую реплику, и ждёт readiness обоих экземпляров.
|
||||
|
||||
Перед `make track1` запустите Compose-стенд и не допускайте параллельных записей в БД: скрипт останавливает API, создаёт временный custom-format snapshot PostgreSQL, а затем восстанавливает его перед каждым прогревом и каждым измерением. В `finally` скрипт возвращает исходную БД и 2 API-реплики, поэтому все benchmark-записи удаляются, а данные, бывшие в volume до эксперимента, сохраняются. Все семь случаев (1/2/4 фиксированный бюджет; 1/2/4/8 рост) используют один и тот же snapshot.
|
||||
|
||||
CI workflow `.github/workflows/application.yml` отдельно проверяет сборку Compose-стенда, публичный smoke, конкурентную идемпотентность, persistence и отказ одной реплики. CI использует только временные синтетические credentials и удаляет собственный тестовый volume после завершения job.
|
||||
|
||||
## Что сделать
|
||||
|
||||
1. Выберите Compose + Nginx/HAProxy/Traefik либо Docker Swarm. Для выбранного варианта дайте одну последовательность запуска.
|
||||
|
||||
@@ -1,37 +1,113 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Аккуратов Тимофей, бвт2304
|
||||
- Целевая оценка: 5; выбран трек 1 — цена масштабирования
|
||||
- Commit/PR: ещё не созданы; изменения находятся в локальном рабочем каталоге
|
||||
- Источники, библиотеки и помощь ИИ: перенесена собственная реализация из mtusi2026lab2; Python stdlib, psycopg 3.2.10 и psycopg-pool 3.3.3; официальные образы PostgreSQL и HAProxy. При разработке использовалась помощь Copilot в VS Code; код и результаты проверены локально. Чужие исходники не переносились.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker Desktop с Compose v2, `openssl`, `make` и Python 3.9+ для локальных проверок. API работает в образе Python 3.12.7; клиентские тесты используют только стандартную библиотеку.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
./scripts/bootstrap-secrets.sh
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 120
|
||||
|
||||
make check
|
||||
make test
|
||||
make test-idempotency
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
```
|
||||
|
||||
Для полной OpenAPI-валидации:
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
```
|
||||
|
||||
`bootstrap-secrets.sh` создаёт локальные `secrets/postgres_password` и `secrets/database_url`, не печатая их. `.env` и `secrets/` игнорируются Git. Миграции применяются одноразовым сервисом `migrate`; дождитесь состояния `healthy` у PostgreSQL и API. По умолчанию тесты обращаются к `http://127.0.0.1:8080`; при изменении `APP_PORT` задайте, например, `make test BASE_URL=http://127.0.0.1:8081`.
|
||||
|
||||
Безопасная остановка с сохранением данных: `docker compose --env-file .env down`. **Намеренное удаление всех учебных данных** — только отдельной командой `docker compose --env-file .env down -v`; эта команда не используется проверками restart/failover.
|
||||
|
||||
Исследовательский эксперимент целиком повторяется одной командой после запуска Compose-стенда:
|
||||
|
||||
```sh
|
||||
make track1
|
||||
```
|
||||
|
||||
Команда запускает все семь случаев матрицы: 1/2/4 реплики при фиксированном бюджете и 1/2/4/8 при росте ресурсов. Для сопоставимости используйте тот же Docker host, порт, число клиентов и длительность. Экспериментальные записи удаляются при восстановлении сохранённого baseline.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
**Клиент → HAProxy → API × 2+ → общая PostgreSQL.** В Compose нет `container_name`; API-порты не опубликованы на хост. Единственный внешний порт — `127.0.0.1:${APP_PORT}:8080` у HAProxy. Реплики не используют sticky sessions. Сессии, сообщения и результаты идемпотентности хранятся в PostgreSQL; это единственный источник истины. Redis из предыдущей реализации остаётся необязательной возможностью в коде, но в этом стенде не развёрнут и для корректности не нужен.
|
||||
|
||||
Из lab2 перенесены `src/`, `migrations/001_initial.sql`, Dockerfile и зависимости. Изменения этой работы:
|
||||
|
||||
- API больше не запускает миграции при старте. Сервис `migrate` применяет версионированные SQL-файлы до запуска реплик; таблица `schema_migrations` и advisory transaction lock защищают от повторного применения.
|
||||
- `migrations/002_idempotency.sql` добавляет общую таблицу ключей. Ключ уникален в области user + POST + chat + key; хранится SHA-256 канонического JSON и исходный ответ. Проверка повтора, создание сообщения и сохранение ответа выполняются в одной транзакции PostgreSQL. Transaction-level advisory lock сериализует конкурентные запросы с тем же ключом на разных репликах. Конфликтующий body возвращает 409, успешный повтор — исходное сообщение; срок хранения не меньше 24 часов.
|
||||
- `X-Instance-Id` берётся из hostname контейнера и неизменен в пределах процесса. `scripts/stop-one.sh` сопоставляет его с конкретным контейнером.
|
||||
- HAProxy обнаруживает backend через DNS Docker (`127.0.0.11`, `hold valid 2s`) и проверяет `GET /health/ready` каждую секунду; backend считается недоступным после двух неудач. С учётом DNS TTL и таймаута проверки консервативный бюджет обнаружения — до 5s, что ниже лимита 10s; connect timeout — 2s, client/server timeout — 15s. У API есть readiness healthcheck, лимиты БД: statement 5s, lock 2s, pool wait 3s. При SIGTERM сервер прекращает приём и завершает активные запросы; `stop_grace_period` — 20s.
|
||||
- Все контейнерные образы закреплены тегами: Python 3.12.7-slim, PostgreSQL 16.4-alpine, HAProxy 3.0.7-alpine. PostgreSQL использует именованный `postgres_data`. API по умолчанию ограничен 1 CPU/512 MiB на реплику; PostgreSQL — 1 CPU/1 GiB, HAProxy — 0.5 CPU/128 MiB. В переменных `.env.example` настраиваются host port, TTL и per-replica лимиты API.
|
||||
|
||||
Миграция 002 аддитивна и не меняет существующие таблицы lab2, поэтому старый API не использует новую таблицу. CI workflow `.github/workflows/application.yml` собирает Compose-стенд, запускает публичные и конкурентные тесты, persistence и отказ API-реплики; он не тестирует обновление приложения на новую версию бизнес-кода. Локально проверен поочерёдный restart одинакового образа под нагрузкой.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Контракт и OpenAPI | `make check`; `make validate PYTHON=.venv/bin/python` | PASS для структуры, синтаксиса и полной OpenAPI 3.1/JSON Schema валидации; контракт и публичные проверки не менялись | `evidence/openapi-validation.txt` |
|
||||
| Стенд и миграции | `docker compose ... up --scale api=2`; SQL `schema_migrations` | 2 API healthy, PostgreSQL и HAProxy healthy; применены ровно `001_initial.sql`, `002_idempotency.sql`; API опубликованных host ports не имеет | `evidence/deployment-checks.txt` |
|
||||
| Smoke и общая сессия | `make test` через HAProxy | PASS: полный публичный сценарий lab3 | `evidence/public-tests.txt` |
|
||||
| Общая идемпотентность | `make test-idempotency` | 12 конкурентных запросов с одинаковым ключом прошли через 2 наблюдаемые реплики, вернули одно сообщение; в истории ровно одна запись. Публичный smoke также проверяет повтор и 409 при другом body | `evidence/public-tests.txt` |
|
||||
| Persistence lab2 | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS: после перезапуска PostgreSQL, API и LB подтверждённые данные доступны, прежняя сессия действительна; volumes сохранены | `evidence/persistence.txt` |
|
||||
| Failover первой и второй API | Дважды `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, восстановление между прогонами | Проверки увидели ≥2 экземпляра. Для первого: 108 успешных записей, 2 неуспешные попытки, восстановление за 2.684s. Для второго: 109 успешных, 1 неуспешная, восстановление за 2.566s. Подтверждённые сообщения и сессия сохранились | `evidence/failover-1.txt`, `evidence/failover-2.txt`, `evidence/failover-targets.txt` |
|
||||
| Readiness, timeouts, SIGTERM | HAProxy health checks и `scripts/rolling-restart.sh` | Backend исключается после 2 проверок с интервалом 1s; отказная проверка восстановила записи за 2.566–2.684s. Поочерёдный restart обоих контейнеров завершился `healthy` | `haproxy.cfg`, `compose.yaml`, `evidence/failover-1.txt`, `evidence/failover-2.txt`, `evidence/rolling-restart-actions.txt` |
|
||||
| Лимиты и отсутствие обходного API-порта | `docker compose config`; `docker inspect` | Реально установлены API 1 CPU/512 MiB, PostgreSQL 1 CPU/1 GiB, HAProxy 0.5 CPU/128 MiB. На хост опубликован только loopback-порт LB; в `ps` у API показан только внутренний `8080/tcp` | `evidence/deployment-checks.txt` |
|
||||
| Rolling restart под нагрузкой | `tests/load.py --duration 35 --concurrency 8` одновременно с `scripts/rolling-restart.sh` | 19,968 успешных POST, 6 ошибок переходного периода; successful RPS 570.303, p50/p95/p99 8.385/45.880/73.085 ms. Оба API контейнера были поочерёдно перезапущены и стали healthy | `evidence/rolling-restart-load.json`, `evidence/rolling-restart-actions.txt` |
|
||||
| Трек 5 — цена масштабирования | `make track1` | Матрица 1/2/4 при фиксированных ресурсах, затем рост до 1/2/4/8; одна исходная custom-format копия PostgreSQL восстанавливается до каждого прогрева и измерения; три повтора на случай; исходная БД восстанавливается в `finally` | `evidence/track1/results.json`, `evidence/track1/throughput.svg` |
|
||||
| CI сборка и проверка приложения | `.github/workflows/application.yml` (`push`, `pull_request`) | Отдельный job устанавливает requirements-dev, собирает/ожидает Compose, выполняет `make check validate`, smoke, параллельную идемпотентность, persistence и failover. Workflow добавлен; выполнение на GitHub Actions ожидает push/PR | `.github/workflows/application.yml` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Трек 1: цена масштабирования
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
**Гипотеза:** при фиксированном суммарном бюджете API увеличение числа реплик само по себе не даст прироста; при добавлении ресурсов оно ускорит запись до насыщения общей PostgreSQL, после чего новые API-процессы почти не помогут.
|
||||
|
||||
Одна машина macOS 27.0.1 arm64, физически 10 CPU и 16 GiB RAM; Docker Desktop 28.0.4 / Compose 2.34.0-desktop.1 предоставлял контейнерам 10 vCPU и около 7.65 GiB. API: Python 3.12.7; фиксированные ресурсы PostgreSQL 1 CPU/1 GiB и HAProxy 0.5 CPU/128 MiB. На каждый случай — прогрев 5s и три измеряемых прогона по 30s, 16 параллельных клиентов. Генератор — штатный closed-loop `tests/load.py`, только POST сообщений через LB; прозрачных retry нет. Перед каждым прогревом и измеряемым прогоном API остановлен, PostgreSQL восстановлена из одного custom-format snapshot, проверены контрольные количества строк, затем запускается выбранное число реплик. После всей матрицы в `finally` возвращается исходная БД и две реплики; синтетические benchmark-записи не остаются в volume. Во время запуска исключены внешние записи в БД. В таблице latency — среднее p50/p95/p99 по успешным запросам, RPS — среднее ± стандартное отклонение между тремя прогонами. CPU — максимальная 1-секундная выборка Docker stats; 100% соответствует примерно одному CPU.
|
||||
|
||||
| Профиль | API | Суммарный лимит API | Успешные RPS, среднее ± σ | p50 / p95 / p99, ms | Ошибки | Peak PostgreSQL CPU |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| Фиксированный бюджет | 1 | 1 CPU / 512 MiB | 212.54 ± 1.34 | 79.36 / 115.01 / 181.42 | 0 | 28.22% |
|
||||
| Фиксированный бюджет | 2 | 1 CPU / 512 MiB | 212.71 ± 10.81 | 83.45 / 166.94 / 224.07 | 0 | 40.58% |
|
||||
| Фиксированный бюджет | 4 | 1 CPU / 512 MiB | 156.34 ± 17.60 | 92.69 / 260.44 / 663.89 | 0 | 46.22% |
|
||||
| Рост ресурсов | 1 | 1 CPU / 512 MiB | 211.67 ± 1.56 | 79.53 / 115.35 / 182.46 | 0 | 26.54% |
|
||||
| Рост ресурсов | 2 | 2 CPU / 1 GiB | 604.95 ± 6.29 | 19.72 / 60.32 / 79.17 | 0 | 62.93% |
|
||||
| Рост ресурсов | 4 | 4 CPU / 2 GiB | 1,099.73 ± 11.87 | 11.01 / 29.91 / 51.29 | 0 | 108.47% |
|
||||
| Рост ресурсов | 8 | 8 CPU / 4 GiB | 1,159.95 ± 16.88 | 11.75 / 27.90 / 37.07 | 0 | 105.96% |
|
||||
|
||||
При фиксированном бюджете переход к 4 маленьким репликам снизил успешный throughput на 26.4% относительно одной реплики и увеличил p95 с 115.01 до 260.44 ms: дробление CPU между процессами не дало выигрыша. При росте бюджета до четырёх API CPU/2 GiB throughput вырос в 5.19 раза относительно одной реплики при том же per-instance лимите, а p95 уменьшился примерно в 3.86 раза — подтверждённое улучшение для этого стенда. Дальнейшее удвоение API до 8, CPU до 8 и памяти до 4 GiB увеличило throughput на 5.48%; одновременно PostgreSQL достигла выборки 105.96–108.47% одного CPU. Это указывает на насыщение общей БД и уменьшающуюся отдачу от добавления API-ресурсов; это наблюдение относится только к данному профилю, а не универсальная характеристика PostgreSQL.
|
||||
|
||||
Все данные синтетические. До матрицы был зафиксирован baseline snapshot; каждое измерение стартовало с одного и того же состояния БД и создавало свой тестовый набор в новом уникальном чате. Row counts после завершения совпали с baseline: users=156, chats=52, messages=445522, sessions=154, idempotency_records=445522. Пиковая выборка памяти за benchmark: PostgreSQL 252.6 MiB, API 73.1 MiB на контейнер, HAProxy 21.1 MiB. Подробные исходные метрики и выборки CPU/RAM, единый SHA-256 snapshot и контрольные количества строк сохранены в JSON.
|
||||
|
||||
### Отказы
|
||||
|
||||
| Воздействие | Ожидание | Наблюдение | Восстановление / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Остановка первой API-реплики | LB продолжает читать/писать через оставшуюся реплику; повтор сообщения не создаёт дубль | 108 успешных записей, 2 переходные ошибки; readiness восстановилась за 2.684s | Данные и сессия сохранены; потеря подтверждённых сообщений — 0 |
|
||||
| Остановка второй API-реплики | Те же гарантии для другой реплики | 109 успешных записей, 1 переходная ошибка; readiness восстановилась за 2.566s | Данные и сессия сохранены; потеря подтверждённых сообщений — 0 |
|
||||
| Поочерёдный SIGTERM/restart двух API под нагрузкой | Вторая реплика остаётся доступной, остановленная возвращается healthy | 19,968 успешных POST, 6 ошибок за 35s; оба экземпляра перезапущены последовательно | Контейнеры вернулись в healthy; подтверждённые записи остаются в PostgreSQL |
|
||||
| Restart PostgreSQL + API + LB | Сохранить существующую сессию и сообщения на named volume | Persistence-тест PASS с прежним токеном | Потеря данных — 0; volumes не удалялись |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Эксперимент закрытый (closed-loop), проводится на одном Mac и одним клиентским процессом; он подвержен coordinated omission и не задаёт независимый arrival rate. Процентили считаются только по успешным ответам, а выборка CPU/RAM снимается раз в секунду.
|
||||
- Сценарий проверяет отказ API-процесса, не отказ Docker-host, HAProxy, PostgreSQL или диска. PostgreSQL и единственный LB — SPOF; отказ узла не исследовался, multi-node failover не заявляется.
|
||||
- Порт HTTP открыт только на loopback для синтетического локального стенда. Для удалённой демонстрации необходим TLS. Секреты Compose — локальные bind-mounted файлы, не production secret manager.
|
||||
- Идемпотентность ограничена текстовым контрактом lab3 и 24-часовым минимумом; транзакция держит PostgreSQL advisory lock. HAProxy `server-template` настроен максимум на 10 адресов — при масштабировании выше нужно расширить диапазон и повторить проверки.
|
||||
- Rolling-сценарий проверяет поочерёдный restart текущего образа, не обновление на новую версию приложения. Миграции аддитивны, но совместимость произвольной смены версий приложения не проверялась.
|
||||
- Результаты относятся только к указанным лимитам, размеру данных и клиентской интенсивности. Snapshot теперь одинаков перед каждым прогоном, но нагрузка остаётся closed-loop, на одном ноутбуке, без открытого arrival-rate профиля.
|
||||
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
restart: unless-stopped
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python3", "src/migrate.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
DB_POOL_MAX_SIZE: "2"
|
||||
secrets:
|
||||
- database_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DB_POOL_MAX_SIZE: "8"
|
||||
DB_POOL_TIMEOUT_SECONDS: "3"
|
||||
DB_STATEMENT_TIMEOUT_MS: "5000"
|
||||
DB_LOCK_TIMEOUT_MS: "2000"
|
||||
secrets:
|
||||
- database_url
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python3
|
||||
- -c
|
||||
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()"
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPU_LIMIT:-1.0}"
|
||||
memory: "${API_MEMORY_LIMIT:-512M}"
|
||||
stop_grace_period: 20s
|
||||
restart: unless-stopped
|
||||
|
||||
load_balancer:
|
||||
image: haproxy:3.0.7-alpine
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
restart: unless-stopped
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
@@ -0,0 +1,16 @@
|
||||
Capture time: 2026-10-21T05:57:02Z
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi2026lab3-api-5 mtusi2026lab3-api "python3 src/server.…" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab3-api-6 mtusi2026lab3-api "python3 src/server.…" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
mtusi2026lab3-load_balancer-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" load_balancer 29 minutes ago Up 17 minutes (healthy) 127.0.0.1:8080->8080/tcp
|
||||
mtusi2026lab3-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 29 minutes ago Up 17 minutes (healthy) 5432/tcp
|
||||
|
||||
Migration versions:
|
||||
001_initial.sql
|
||||
002_idempotency.sql
|
||||
|
||||
Container limits:
|
||||
/mtusi2026lab3-api-5 NanoCPUs=1000000000 MemoryBytes=536870912
|
||||
/mtusi2026lab3-api-6 NanoCPUs=1000000000 MemoryBytes=536870912
|
||||
/mtusi2026lab3-postgres-1 NanoCPUs=1000000000 MemoryBytes=1073741824
|
||||
/mtusi2026lab3-load_balancer-1 NanoCPUs=500000000 MemoryBytes=134217728
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 108, "failed_attempts": 2, "first_recovery_seconds": 2.684}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 109, "failed_attempts": 1, "first_recovery_seconds": 2.566}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,2 @@
|
||||
2026-10-21T05:39:45Z target_instance=498a08cd2b5c container=498a08cd2b5cf94be02873ea9eb0c9c5f6b79133ddfd50b44aff134f86222b71
|
||||
2026-10-21T05:40:39Z target_instance=137c7f092d63 container=137c7f092d63cd2e0d8d2b71ceecd67860c35ba79709df56831bba3a967629ba
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,6 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://127.0.0.1:8080"
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
python3 student-tests/test_idempotency_race.py --base-url "http://127.0.0.1:8080"
|
||||
PASS: 12 concurrent same-key requests on 2 replicas returned one message
|
||||
@@ -0,0 +1,29 @@
|
||||
Container mtusi2026lab3-postgres-1 Running
|
||||
Container mtusi2026lab3-migrate-1 Created
|
||||
Container mtusi2026lab3-api-3 Running
|
||||
Container mtusi2026lab3-api-4 Created
|
||||
Container mtusi2026lab3-load_balancer-1 Running
|
||||
Container mtusi2026lab3-postgres-1 Waiting
|
||||
Container mtusi2026lab3-postgres-1 Healthy
|
||||
Container mtusi2026lab3-migrate-1 Starting
|
||||
Container mtusi2026lab3-migrate-1 Started
|
||||
Container mtusi2026lab3-migrate-1 Waiting
|
||||
Container mtusi2026lab3-migrate-1 Exited
|
||||
Container mtusi2026lab3-api-4 Starting
|
||||
Container mtusi2026lab3-api-4 Started
|
||||
Container mtusi2026lab3-api-4 Waiting
|
||||
Container mtusi2026lab3-api-3 Waiting
|
||||
Container mtusi2026lab3-api-4 Healthy
|
||||
Container mtusi2026lab3-api-3 Healthy
|
||||
Container mtusi2026lab3-api-4 Waiting
|
||||
Container mtusi2026lab3-api-3 Waiting
|
||||
Container mtusi2026lab3-load_balancer-1 Waiting
|
||||
Container mtusi2026lab3-postgres-1 Waiting
|
||||
Container mtusi2026lab3-migrate-1 Waiting
|
||||
Container mtusi2026lab3-load_balancer-1 Healthy
|
||||
Container mtusi2026lab3-migrate-1 Exited
|
||||
Container mtusi2026lab3-postgres-1 Healthy
|
||||
Container mtusi2026lab3-api-4 Healthy
|
||||
Container mtusi2026lab3-api-3 Healthy
|
||||
PASS: API container 498a08cd2b5cf94be02873ea9eb0c9c5f6b79133ddfd50b44aff134f86222b71 restarted and is healthy
|
||||
PASS: API container 137c7f092d63cd2e0d8d2b71ceecd67860c35ba79709df56831bba3a967629ba restarted and is healthy
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 35.0,
|
||||
"elapsed_seconds": 35.013,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 19968,
|
||||
"failed_requests": 6,
|
||||
"error_ratio": 0.0003,
|
||||
"successful_rps": 570.303,
|
||||
"successful_latency_ms": {
|
||||
"p50": 8.385,
|
||||
"p95": 45.88,
|
||||
"p99": 73.085
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.046,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 6346,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 211.212,
|
||||
"successful_latency_ms": {
|
||||
"p50": 80.264,
|
||||
"p95": 114.896,
|
||||
"p99": 182.694
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.064,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 6445,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 214.379,
|
||||
"successful_latency_ms": {
|
||||
"p50": 78.662,
|
||||
"p95": 115.143,
|
||||
"p99": 181.9
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.056,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 6373,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 212.038,
|
||||
"successful_latency_ms": {
|
||||
"p50": 79.158,
|
||||
"p95": 114.991,
|
||||
"p99": 179.677
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 5.0,
|
||||
"elapsed_seconds": 5.065,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 1036,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 204.529,
|
||||
"successful_latency_ms": {
|
||||
"p50": 81.556,
|
||||
"p95": 109.862,
|
||||
"p99": 172.61
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.098,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 5968,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 198.285,
|
||||
"successful_latency_ms": {
|
||||
"p50": 86.571,
|
||||
"p95": 190.715,
|
||||
"p99": 278.78
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.03,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 6736,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 224.312,
|
||||
"successful_latency_ms": {
|
||||
"p50": 80.141,
|
||||
"p95": 150.189,
|
||||
"p99": 197.092
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.031,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 6473,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 215.547,
|
||||
"successful_latency_ms": {
|
||||
"p50": 83.643,
|
||||
"p95": 159.92,
|
||||
"p99": 196.349
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 5.0,
|
||||
"elapsed_seconds": 5.017,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 990,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 197.311,
|
||||
"successful_latency_ms": {
|
||||
"p50": 82.99,
|
||||
"p95": 199.803,
|
||||
"p99": 294.885
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.155,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 5464,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 181.196,
|
||||
"successful_latency_ms": {
|
||||
"p50": 85.443,
|
||||
"p95": 200.223,
|
||||
"p99": 467.896
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.201,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 4377,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 144.927,
|
||||
"successful_latency_ms": {
|
||||
"p50": 98.238,
|
||||
"p95": 287.157,
|
||||
"p99": 494.409
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.261,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 4324,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 142.888,
|
||||
"successful_latency_ms": {
|
||||
"p50": 94.377,
|
||||
"p95": 293.944,
|
||||
"p99": 1029.361
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 5.0,
|
||||
"elapsed_seconds": 5.173,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 1140,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 220.377,
|
||||
"successful_latency_ms": {
|
||||
"p50": 64.745,
|
||||
"p95": 197.524,
|
||||
"p99": 398.349
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.039,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 6333,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 210.824,
|
||||
"successful_latency_ms": {
|
||||
"p50": 80.103,
|
||||
"p95": 114.112,
|
||||
"p99": 182.734
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.073,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 6325,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 210.32,
|
||||
"successful_latency_ms": {
|
||||
"p50": 79.313,
|
||||
"p95": 117.188,
|
||||
"p99": 183.607
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.03,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 6422,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 213.855,
|
||||
"successful_latency_ms": {
|
||||
"p50": 79.164,
|
||||
"p95": 114.738,
|
||||
"p99": 181.052
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 5.0,
|
||||
"elapsed_seconds": 5.03,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 1054,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 209.531,
|
||||
"successful_latency_ms": {
|
||||
"p50": 77.196,
|
||||
"p95": 116.562,
|
||||
"p99": 184.922
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.017,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 18059,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 601.627,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.831,
|
||||
"p95": 60.095,
|
||||
"p99": 81.757
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.494,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 18280,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 599.463,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.535,
|
||||
"p95": 59.859,
|
||||
"p99": 78.355
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.029,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 18431,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 613.766,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.791,
|
||||
"p95": 61.005,
|
||||
"p99": 77.388
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 5.0,
|
||||
"elapsed_seconds": 5.037,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 3029,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 601.302,
|
||||
"successful_latency_ms": {
|
||||
"p50": 20.938,
|
||||
"p95": 59.639,
|
||||
"p99": 76.915
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.012,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 32846,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1094.415,
|
||||
"successful_latency_ms": {
|
||||
"p50": 10.732,
|
||||
"p95": 30.846,
|
||||
"p99": 51.188
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.013,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 32672,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1088.609,
|
||||
"successful_latency_ms": {
|
||||
"p50": 11.145,
|
||||
"p95": 30.074,
|
||||
"p99": 53.423
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.015,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 33502,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1116.177,
|
||||
"successful_latency_ms": {
|
||||
"p50": 11.142,
|
||||
"p95": 28.815,
|
||||
"p99": 49.26
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 5.0,
|
||||
"elapsed_seconds": 5.014,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 5472,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1091.286,
|
||||
"successful_latency_ms": {
|
||||
"p50": 11.367,
|
||||
"p95": 33.413,
|
||||
"p99": 54.849
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.033,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 34199,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1138.726,
|
||||
"successful_latency_ms": {
|
||||
"p50": 11.784,
|
||||
"p95": 29.272,
|
||||
"p99": 38.508
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.015,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 35418,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1180.016,
|
||||
"successful_latency_ms": {
|
||||
"p50": 11.711,
|
||||
"p95": 26.361,
|
||||
"p99": 36.002
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.012,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 34848,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1161.122,
|
||||
"successful_latency_ms": {
|
||||
"p50": 11.745,
|
||||
"p95": 28.074,
|
||||
"p99": 36.688
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 5.0,
|
||||
"elapsed_seconds": 5.014,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 5457,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1088.404,
|
||||
"successful_latency_ms": {
|
||||
"p50": 11.999,
|
||||
"p95": 31.015,
|
||||
"p99": 44.748
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,26 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="900" height="420" viewBox="0 0 900 420">
|
||||
<rect width="100%" height="100%" fill="white"/>
|
||||
<text x="450" y="28" text-anchor="middle" font-family="sans-serif" font-size="18">API scaling: average successful RPS</text>
|
||||
<line x1="60" y1="360" x2="840" y2="360" stroke="#333"/>
|
||||
<rect x="60.0" y="305.0" width="78" height="55.0" fill="#3572A5"/>
|
||||
<text x="99.0" y="297.0" text-anchor="middle" font-family="sans-serif" font-size="12">212.5</text>
|
||||
<text x="99.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">fixed / 1</text>
|
||||
<rect x="177.0" y="305.0" width="78" height="55.0" fill="#3572A5"/>
|
||||
<text x="216.0" y="297.0" text-anchor="middle" font-family="sans-serif" font-size="12">212.7</text>
|
||||
<text x="216.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">fixed / 2</text>
|
||||
<rect x="294.0" y="319.6" width="78" height="40.4" fill="#3572A5"/>
|
||||
<text x="333.0" y="311.6" text-anchor="middle" font-family="sans-serif" font-size="12">156.3</text>
|
||||
<text x="333.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">fixed / 4</text>
|
||||
<rect x="411.0" y="305.3" width="78" height="54.7" fill="#E38B29"/>
|
||||
<text x="450.0" y="297.3" text-anchor="middle" font-family="sans-serif" font-size="12">211.7</text>
|
||||
<text x="450.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">growth / 1</text>
|
||||
<rect x="528.0" y="203.5" width="78" height="156.5" fill="#E38B29"/>
|
||||
<text x="567.0" y="195.5" text-anchor="middle" font-family="sans-serif" font-size="12">605.0</text>
|
||||
<text x="567.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">growth / 2</text>
|
||||
<rect x="645.0" y="75.6" width="78" height="284.4" fill="#E38B29"/>
|
||||
<text x="684.0" y="67.6" text-anchor="middle" font-family="sans-serif" font-size="12">1099.7</text>
|
||||
<text x="684.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">growth / 4</text>
|
||||
<rect x="762.0" y="60.0" width="78" height="300.0" fill="#E38B29"/>
|
||||
<text x="801.0" y="52.0" text-anchor="middle" font-family="sans-serif" font-size="12">1160.0</text>
|
||||
<text x="801.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">growth / 8</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.2 KiB |
+35
@@ -0,0 +1,35 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 1024
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
option http-keep-alive
|
||||
timeout connect 2s
|
||||
timeout client 15s
|
||||
timeout server 15s
|
||||
timeout http-request 5s
|
||||
timeout check 1s
|
||||
retries 2
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold nx 2s
|
||||
|
||||
frontend messenger
|
||||
bind :8080
|
||||
default_backend api_replicas
|
||||
|
||||
backend api_replicas
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
option redispatch
|
||||
default-server inter 1s fall 2 rise 2
|
||||
server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr libc,none
|
||||
@@ -0,0 +1,43 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
username varchar(32) NOT NULL UNIQUE,
|
||||
display_name varchar(100) NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
title varchar(100) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
joined_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
text varchar(4000) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE TABLE idempotency_records (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method varchar(8) NOT NULL,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
idempotency_key varchar(128) NOT NULL,
|
||||
request_hash bytea NOT NULL,
|
||||
response jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, method, chat_id, idempotency_key),
|
||||
CONSTRAINT idempotency_method_post CHECK (method = 'POST'),
|
||||
CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32),
|
||||
CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at);
|
||||
@@ -0,0 +1,3 @@
|
||||
psycopg[binary]==3.2.10
|
||||
psycopg-pool==3.3.3
|
||||
redis==5.2.1
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
SECRETS_DIR="$ROOT/secrets"
|
||||
PASSWORD_FILE="$SECRETS_DIR/postgres_password"
|
||||
DATABASE_URL_FILE="$SECRETS_DIR/database_url"
|
||||
|
||||
if [ -e "$PASSWORD_FILE" ] || [ -e "$DATABASE_URL_FILE" ]; then
|
||||
printf '%s\n' 'Secret files already exist; refusing to overwrite them.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
command -v openssl >/dev/null 2>&1 || {
|
||||
printf '%s\n' 'openssl is required to generate local credentials.' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
mkdir -p "$SECRETS_DIR"
|
||||
chmod 700 "$SECRETS_DIR"
|
||||
umask 077
|
||||
password=$(openssl rand -hex 32)
|
||||
printf '%s' "$password" > "$PASSWORD_FILE"
|
||||
printf 'postgresql://messenger:%s@postgres:5432/messenger' "$password" > "$DATABASE_URL_FILE"
|
||||
chmod 644 "$PASSWORD_FILE" "$DATABASE_URL_FILE"
|
||||
unset password
|
||||
printf '%s\n' 'Local PostgreSQL secret files created.'
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
|
||||
docker compose --env-file .env restart postgres
|
||||
docker compose --env-file .env up -d --wait --wait-timeout 60 postgres
|
||||
docker compose --env-file .env restart api load_balancer
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
|
||||
docker compose --env-file .env up -d --scale api=2 --wait --wait-timeout 60
|
||||
containers=$(docker compose --env-file .env ps -q api)
|
||||
if [ -z "$containers" ]; then
|
||||
printf '%s\n' 'No running API containers found.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for container in $containers; do
|
||||
docker stop --time 15 "$container" >/dev/null
|
||||
docker start "$container" >/dev/null
|
||||
attempts=0
|
||||
while [ "$attempts" -lt 40 ]; do
|
||||
state=$(docker inspect --format '{{.State.Health.Status}}' "$container")
|
||||
if [ "$state" = healthy ]; then
|
||||
printf 'PASS: API container %s restarted and is healthy\n' "$container"
|
||||
break
|
||||
fi
|
||||
if [ "$state" = unhealthy ] || [ "$state" = none ]; then
|
||||
printf 'API container %s failed health check (state: %s)\n' "$container" "$state" >&2
|
||||
exit 1
|
||||
fi
|
||||
attempts=$((attempts + 1))
|
||||
sleep 1
|
||||
done
|
||||
if [ "$attempts" -eq 40 ]; then
|
||||
printf 'API container %s did not become healthy in 40 seconds\n' "$container" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
|
||||
target_container=
|
||||
for container in $(docker compose --env-file .env ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
target_container=$container
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$target_container" ]; then
|
||||
printf '%s\n' 'No running API container matches TARGET_INSTANCE_ID.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "${FAILOVER_EVIDENCE_FILE:-}" ]; then
|
||||
case "$FAILOVER_EVIDENCE_FILE" in
|
||||
/*) evidence_file=$FAILOVER_EVIDENCE_FILE ;;
|
||||
*) evidence_file="$ROOT/$FAILOVER_EVIDENCE_FILE" ;;
|
||||
esac
|
||||
printf '%s target_instance=%s container=%s\n' \
|
||||
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$TARGET_INSTANCE_ID" "$target_container" \
|
||||
>> "$evidence_file"
|
||||
fi
|
||||
|
||||
docker stop --time 2 "$target_container" >/dev/null
|
||||
@@ -0,0 +1 @@
|
||||
"""Messenger API application package."""
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import threading
|
||||
from functools import partial
|
||||
from http.server import ThreadingHTTPServer
|
||||
|
||||
from .config import APP_HOST, APP_PORT
|
||||
from .handlers import MessengerHandler
|
||||
from .store import Store
|
||||
|
||||
LOGGER = logging.getLogger("messenger")
|
||||
|
||||
|
||||
def run_server():
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
handler = partial(MessengerHandler, store=store)
|
||||
server = ThreadingHTTPServer((APP_HOST, APP_PORT), handler)
|
||||
|
||||
def handle_shutdown(signum, frame):
|
||||
LOGGER.info("received signal %s, shutting down gracefully", signum)
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, handle_shutdown)
|
||||
signal.signal(signal.SIGINT, handle_shutdown)
|
||||
LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT)
|
||||
try:
|
||||
server.serve_forever()
|
||||
finally:
|
||||
server.server_close()
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
|
||||
run_server()
|
||||
@@ -0,0 +1,29 @@
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import uuid
|
||||
|
||||
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = int(os.getenv("APP_PORT", "8080"))
|
||||
INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12])
|
||||
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000"))
|
||||
DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000"))
|
||||
DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16"))
|
||||
DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5"))
|
||||
MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128"))
|
||||
MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576"))
|
||||
USER_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
|
||||
if SESSION_TTL_SECONDS < 1:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS must be positive")
|
||||
if IDEMPOTENCY_TTL_SECONDS < 86400:
|
||||
raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400")
|
||||
if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1:
|
||||
raise RuntimeError("Database timeouts must be positive")
|
||||
if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0:
|
||||
raise RuntimeError("Database pool settings must be positive")
|
||||
if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1:
|
||||
raise RuntimeError("HTTP limits must be positive")
|
||||
@@ -0,0 +1,18 @@
|
||||
from typing import Dict, Optional
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
class CacheUnavailableError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class IdempotencyConflictError(Exception):
|
||||
pass
|
||||
+350
@@ -0,0 +1,350 @@
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import uuid
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
import psycopg
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from .config import (
|
||||
IDEMPOTENCY_KEY_RE,
|
||||
INSTANCE_ID,
|
||||
MAX_INFLIGHT_REQUESTS,
|
||||
MAX_REQUEST_BODY_BYTES,
|
||||
USER_RE,
|
||||
)
|
||||
from .errors import ApiError, CacheUnavailableError, IdempotencyConflictError
|
||||
from .security import hash_password, isoformat, valid_uuid
|
||||
from .store import Store
|
||||
|
||||
LOGGER = logging.getLogger("messenger.http")
|
||||
|
||||
|
||||
class RequestGuard:
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.inflight = 0
|
||||
|
||||
def acquire(self):
|
||||
with self.lock:
|
||||
if self.inflight >= MAX_INFLIGHT_REQUESTS:
|
||||
raise ApiError(503, "unavailable", "Server is busy")
|
||||
self.inflight += 1
|
||||
|
||||
def release(self):
|
||||
with self.lock:
|
||||
self.inflight = max(0, self.inflight - 1)
|
||||
|
||||
|
||||
REQUEST_GUARD = RequestGuard()
|
||||
|
||||
|
||||
class MessengerHandler(BaseHTTPRequestHandler):
|
||||
def __init__(self, request, client_address, server, store: Store):
|
||||
self.store = store
|
||||
super().__init__(request, client_address, server)
|
||||
|
||||
def log_message(self, format: str, *args):
|
||||
return
|
||||
|
||||
def do_GET(self):
|
||||
self._dispatch("GET")
|
||||
|
||||
def do_POST(self):
|
||||
self._dispatch("POST")
|
||||
|
||||
def do_DELETE(self):
|
||||
self._dispatch("DELETE")
|
||||
|
||||
def _dispatch(self, method: str):
|
||||
request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4())
|
||||
acquired = False
|
||||
try:
|
||||
REQUEST_GUARD.acquire()
|
||||
acquired = True
|
||||
parsed = urlsplit(self.path)
|
||||
path = parsed.path
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
if path == "/health/live" and method == "GET":
|
||||
self._send_json(200, {"status": "ok"}, request_id)
|
||||
elif path == "/health/ready" and method == "GET":
|
||||
try:
|
||||
self.store.ready()
|
||||
except (psycopg.Error, PoolTimeout, OSError):
|
||||
raise ApiError(503, "unavailable", "Database is unavailable")
|
||||
self._send_json(200, {"status": "ok"}, request_id)
|
||||
elif path == "/api/v1/users" and method == "POST":
|
||||
self._handle_register(request_id)
|
||||
elif path == "/api/v1/auth/sessions" and method == "POST":
|
||||
self._handle_create_session(request_id)
|
||||
elif path == "/api/v1/auth/sessions/current" and method == "DELETE":
|
||||
self._handle_revoke_session(request_id)
|
||||
elif path == "/api/v1/users/me" and method == "GET":
|
||||
self._send_json(200, self._get_actor(request_id), request_id)
|
||||
elif path == "/api/v1/chats" and method == "POST":
|
||||
self._handle_create_chat(request_id)
|
||||
elif path == "/api/v1/chats" and method == "GET":
|
||||
self._handle_list_chats(query, request_id)
|
||||
elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"):
|
||||
chat_id = path.rsplit("/", 2)[1]
|
||||
if method == "POST":
|
||||
self._handle_create_message(chat_id, request_id)
|
||||
else:
|
||||
self._handle_list_messages(chat_id, query, request_id)
|
||||
elif path.startswith("/api/v1/chats/") and method == "GET":
|
||||
self._handle_get_chat(path.rsplit("/", 1)[1], request_id)
|
||||
else:
|
||||
raise ApiError(404, "not_found", "Object not found")
|
||||
except ApiError as exc:
|
||||
self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers)
|
||||
except CacheUnavailableError:
|
||||
self._send_error(503, "unavailable", "Session cache is unavailable", request_id)
|
||||
except IdempotencyConflictError:
|
||||
self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id)
|
||||
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError):
|
||||
self._send_error(503, "unavailable", "Database is unavailable", request_id)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
self._send_error(409, "conflict", "Username already exists", request_id)
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
self._send_error(400, "invalid_request", "Unknown member", request_id)
|
||||
except Exception as exc:
|
||||
LOGGER.error("request %s failed with %s", request_id, type(exc).__name__)
|
||||
self._send_error(500, "internal_error", "Internal server error", request_id)
|
||||
finally:
|
||||
if acquired:
|
||||
REQUEST_GUARD.release()
|
||||
|
||||
def _read_json(self, request_id: str) -> Any:
|
||||
if self.headers.get_content_type() != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json")
|
||||
length = self.headers.get("Content-Length")
|
||||
if length is None:
|
||||
raise ApiError(400, "invalid_request", "Missing Content-Length")
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid Content-Length")
|
||||
if size < 0:
|
||||
raise ApiError(400, "invalid_request", "Invalid Content-Length")
|
||||
if size > MAX_REQUEST_BODY_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "Request body too large")
|
||||
raw = self.rfile.read(size)
|
||||
if not raw:
|
||||
raise ApiError(400, "invalid_request", "Empty body")
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
raise ApiError(400, "invalid_request", "Malformed JSON")
|
||||
|
||||
def _get_actor(self, request_id: str) -> Dict[str, Any]:
|
||||
header = self.headers.get("Authorization", "")
|
||||
scheme, separator, token = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not separator or not token or " " in token:
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
user = self.store.session_user(token)
|
||||
if user is None:
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
return user
|
||||
|
||||
def _handle_register(self, request_id: str):
|
||||
body = self._read_json(request_id)
|
||||
if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}:
|
||||
raise ApiError(400, "invalid_request", "Expected username, display_name and password")
|
||||
username = body["username"]
|
||||
display_name = body["display_name"]
|
||||
password = body["password"]
|
||||
if not isinstance(username, str) or not USER_RE.fullmatch(username):
|
||||
raise ApiError(400, "invalid_request", "Invalid username")
|
||||
if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid display_name")
|
||||
if not isinstance(password, str) or not (12 <= len(password) <= 128):
|
||||
raise ApiError(400, "invalid_request", "Invalid password")
|
||||
user = self.store.create_user(username, display_name, hash_password(password))
|
||||
self._send_json(201, user, request_id)
|
||||
|
||||
def _basic_credentials(self, request_id: str) -> Tuple[str, str]:
|
||||
value = self.headers.get("Authorization", "")
|
||||
scheme, separator, encoded = value.partition(" ")
|
||||
challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
if scheme.lower() != "basic" or not separator or not encoded:
|
||||
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
|
||||
try:
|
||||
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
except (ValueError, UnicodeError):
|
||||
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
|
||||
return username, password
|
||||
|
||||
def _handle_create_session(self, request_id: str):
|
||||
username, password = self._basic_credentials(request_id)
|
||||
user = self.store.authenticate(username, password)
|
||||
if user is None:
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Invalid credentials",
|
||||
{"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
|
||||
)
|
||||
token, expires_at = self.store.create_session(user["id"])
|
||||
self._send_json(
|
||||
201,
|
||||
{"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user},
|
||||
request_id,
|
||||
)
|
||||
|
||||
def _handle_revoke_session(self, request_id: str):
|
||||
self._get_actor(request_id)
|
||||
token = self.headers["Authorization"].partition(" ")[2]
|
||||
if not self.store.revoke_session(token):
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
self.send_response(204)
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
self.end_headers()
|
||||
|
||||
def _handle_create_chat(self, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
body = self._read_json(request_id)
|
||||
if not isinstance(body, dict) or set(body) != {"title", "member_ids"}:
|
||||
raise ApiError(400, "invalid_request", "Expected title and member_ids")
|
||||
title, member_ids = body["title"], body["member_ids"]
|
||||
if not isinstance(title, str) or not title.strip() or len(title) > 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid title")
|
||||
if (
|
||||
not isinstance(member_ids, list)
|
||||
or not 1 <= len(member_ids) <= 100
|
||||
or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids)
|
||||
or len(member_ids) != len(set(member_ids))
|
||||
or actor["id"] not in member_ids
|
||||
):
|
||||
raise ApiError(400, "invalid_request", "Invalid member_ids")
|
||||
chat = self.store.create_chat(title, member_ids, actor["id"])
|
||||
self._send_json(201, chat, request_id)
|
||||
|
||||
def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
try:
|
||||
limit = int(query.get("limit", ["50"])[0])
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
if not 1 <= limit <= 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
cursor = query.get("cursor", [None])[0]
|
||||
try:
|
||||
chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid cursor")
|
||||
self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id)
|
||||
|
||||
def _handle_get_chat(self, chat_id: str, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
chat = self.store.get_chat(chat_id, actor["id"])
|
||||
if chat is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
self._send_json(200, chat, request_id)
|
||||
|
||||
def _handle_create_message(self, chat_id: str, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
chat_id = str(uuid.UUID(chat_id))
|
||||
idempotency_key = self.headers.get("Idempotency-Key", "")
|
||||
if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
|
||||
raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key")
|
||||
body = self._read_json(request_id)
|
||||
if not isinstance(body, dict) or set(body) != {"text"}:
|
||||
raise ApiError(400, "invalid_request", "Expected text")
|
||||
text = body["text"]
|
||||
if not isinstance(text, str) or not text.strip() or len(text) > 4000:
|
||||
raise ApiError(400, "invalid_request", "Invalid message text")
|
||||
canonical_body = json.dumps(
|
||||
{"text": text},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
message = self.store.create_idempotent_message(
|
||||
chat_id,
|
||||
actor["id"],
|
||||
text,
|
||||
idempotency_key,
|
||||
canonical_body,
|
||||
)
|
||||
if message is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
self._send_json(201, message, request_id)
|
||||
|
||||
def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
try:
|
||||
limit = int(query.get("limit", ["50"])[0])
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
if not 1 <= limit <= 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
cursor = query.get("cursor", [None])[0]
|
||||
try:
|
||||
result = self.store.list_messages(chat_id, actor["id"], limit, cursor)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid cursor")
|
||||
if result is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
items, next_cursor = result
|
||||
self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id)
|
||||
|
||||
def _send_json(
|
||||
self,
|
||||
status: int,
|
||||
payload: Dict[str, Any],
|
||||
request_id: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
if headers:
|
||||
for key, value in headers.items():
|
||||
self.send_header(key, value)
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_error(
|
||||
self,
|
||||
status: int,
|
||||
code: str,
|
||||
message: str,
|
||||
request_id: str,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
self._send_json(
|
||||
status,
|
||||
{"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
request_id,
|
||||
extra_headers,
|
||||
)
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from src.store import Store
|
||||
else:
|
||||
from .store import Store
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level="INFO")
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
store.migrate()
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,53 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def isoformat(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def valid_uuid(value: str) -> bool:
|
||||
try:
|
||||
uuid.UUID(value)
|
||||
return True
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
return False
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
salt = secrets.token_bytes(16)
|
||||
derived = hashlib.scrypt(
|
||||
password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024
|
||||
)
|
||||
salt_text = base64.urlsafe_b64encode(salt).decode("ascii")
|
||||
hash_text = base64.urlsafe_b64encode(derived).decode("ascii")
|
||||
return f"scrypt$32768$8$1${salt_text}${hash_text}"
|
||||
|
||||
|
||||
def verify_password(password: str, stored: str) -> bool:
|
||||
try:
|
||||
scheme, n, r, p, salt_text, hash_text = stored.split("$")
|
||||
if scheme != "scrypt":
|
||||
return False
|
||||
salt = base64.urlsafe_b64decode(salt_text.encode("ascii"))
|
||||
expected = base64.urlsafe_b64decode(hash_text.encode("ascii"))
|
||||
actual = hashlib.scrypt(
|
||||
password.encode("utf-8"),
|
||||
salt=salt,
|
||||
n=int(n),
|
||||
r=int(r),
|
||||
p=int(p),
|
||||
dklen=len(expected),
|
||||
maxmem=64 * 1024 * 1024,
|
||||
)
|
||||
return hmac.compare_digest(actual, expected)
|
||||
except (ValueError, TypeError, UnicodeError):
|
||||
return False
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from src.app import main
|
||||
else:
|
||||
from .app import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+394
@@ -0,0 +1,394 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import secrets
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import redis
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from .config import (
|
||||
DB_LOCK_TIMEOUT_MS,
|
||||
DB_POOL_MAX_SIZE,
|
||||
DB_POOL_TIMEOUT_SECONDS,
|
||||
DB_STATEMENT_TIMEOUT_MS,
|
||||
IDEMPOTENCY_TTL_SECONDS,
|
||||
SESSION_TTL_SECONDS,
|
||||
)
|
||||
from .errors import CacheUnavailableError, IdempotencyConflictError
|
||||
from .security import isoformat, utc_now, valid_uuid, verify_password
|
||||
|
||||
LOGGER = logging.getLogger("messenger.store")
|
||||
|
||||
|
||||
def database_url() -> str:
|
||||
value = os.getenv("DATABASE_URL")
|
||||
if value:
|
||||
return value
|
||||
file_path = os.getenv("DATABASE_URL_FILE")
|
||||
if file_path:
|
||||
return Path(file_path).read_text(encoding="utf-8").strip()
|
||||
raise RuntimeError("Set DATABASE_URL or DATABASE_URL_FILE")
|
||||
|
||||
|
||||
class Store:
|
||||
"""PostgreSQL source of truth and optional Redis session lookup cache."""
|
||||
|
||||
def __init__(self):
|
||||
self.pool = ConnectionPool(
|
||||
conninfo=database_url(),
|
||||
min_size=1,
|
||||
max_size=DB_POOL_MAX_SIZE,
|
||||
timeout=DB_POOL_TIMEOUT_SECONDS,
|
||||
kwargs={
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": 5,
|
||||
"options": (
|
||||
f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} "
|
||||
f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} "
|
||||
"-c idle_in_transaction_session_timeout=10000"
|
||||
),
|
||||
},
|
||||
open=False,
|
||||
)
|
||||
cache_url = os.getenv("CACHE_URL")
|
||||
self.cache = (
|
||||
redis.Redis.from_url(
|
||||
cache_url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=0.25,
|
||||
socket_timeout=0.25,
|
||||
retry_on_timeout=False,
|
||||
max_connections=16,
|
||||
)
|
||||
if cache_url
|
||||
else None
|
||||
)
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=10)
|
||||
|
||||
def close(self):
|
||||
if self.cache is not None:
|
||||
self.cache.close()
|
||||
self.pool.close()
|
||||
|
||||
def ready(self) -> bool:
|
||||
with self.pool.connection(timeout=2) as conn:
|
||||
conn.execute("SELECT 1")
|
||||
return True
|
||||
|
||||
def migrate(self):
|
||||
migrations = Path(__file__).resolve().parents[1] / "migrations"
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"
|
||||
)
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2741662026)")
|
||||
applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
for migration in sorted(migrations.glob("*.sql")):
|
||||
if migration.name in applied:
|
||||
continue
|
||||
conn.execute(migration.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,))
|
||||
|
||||
@staticmethod
|
||||
def user(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": row["title"],
|
||||
"member_ids": member_ids,
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def message(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
}
|
||||
|
||||
def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO users (username, display_name, password_hash) "
|
||||
"VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at",
|
||||
(username, display_name, password_hash),
|
||||
).fetchone()
|
||||
return self.user(row)
|
||||
|
||||
def get_user(self, user_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
|
||||
(user_id,),
|
||||
).fetchone()
|
||||
return self.user(row) if row else None
|
||||
|
||||
def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash "
|
||||
"FROM users WHERE username = %s",
|
||||
(username,),
|
||||
).fetchone()
|
||||
if not row or not verify_password(password, row["password_hash"]):
|
||||
return None
|
||||
return self.user(row)
|
||||
|
||||
def create_session(self, user_id: str) -> Tuple[str, datetime]:
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
|
||||
token_hash = hashlib.sha256(token.encode("ascii")).digest()
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
|
||||
(token_hash, user_id, expires_at),
|
||||
)
|
||||
return token, expires_at
|
||||
|
||||
@staticmethod
|
||||
def _session_lock_key(token_hash: bytes) -> int:
|
||||
return int.from_bytes(token_hash[:8], byteorder="big", signed=True)
|
||||
|
||||
def session_user(self, token: str) -> Optional[Dict[str, Any]]:
|
||||
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
|
||||
cache_key = "messenger:session:" + token_hash.hex()
|
||||
if self.cache is not None:
|
||||
try:
|
||||
cached = self.cache.get(cache_key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
except (redis.RedisError, ValueError, TypeError):
|
||||
LOGGER.warning("session cache read failed; falling back to PostgreSQL")
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
|
||||
row = conn.execute(
|
||||
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at "
|
||||
"FROM sessions s JOIN users u ON u.id = s.user_id "
|
||||
"WHERE s.token_hash = %s AND s.expires_at > now()",
|
||||
(token_hash,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
user = self.user(row)
|
||||
if self.cache is not None:
|
||||
ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000)
|
||||
if ttl_ms > 0:
|
||||
try:
|
||||
self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms)
|
||||
except redis.RedisError:
|
||||
LOGGER.warning("session cache write failed; PostgreSQL remains authoritative")
|
||||
return user
|
||||
|
||||
def revoke_session(self, token: str) -> bool:
|
||||
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
|
||||
if self.cache is not None:
|
||||
try:
|
||||
self.cache.delete("messenger:session:" + token_hash.hex())
|
||||
except redis.RedisError as exc:
|
||||
raise CacheUnavailableError from exc
|
||||
result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,))
|
||||
return result.rowcount == 1
|
||||
|
||||
def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]:
|
||||
if actor_id not in member_ids or len(member_ids) != len(set(member_ids)):
|
||||
raise ValueError("invalid members")
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at",
|
||||
(title,),
|
||||
).fetchone()
|
||||
for member_id in member_ids:
|
||||
conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
|
||||
(row["id"], member_id),
|
||||
)
|
||||
return self.chat(row, member_ids)
|
||||
|
||||
def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT c.id, c.title, c.created_at FROM chats c "
|
||||
"JOIN chat_members m ON m.chat_id = c.id "
|
||||
"WHERE c.id = %s AND m.user_id = %s",
|
||||
(chat_id, user_id),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
|
||||
(chat_id,),
|
||||
).fetchall()
|
||||
return self.chat(row, [str(item["user_id"]) for item in members])
|
||||
|
||||
def list_chats(
|
||||
self, user_id: str, limit: int, cursor: Optional[str]
|
||||
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
|
||||
where = "own.user_id = %s"
|
||||
params: List[Any] = [user_id]
|
||||
if cursor is not None:
|
||||
try:
|
||||
raw = base64.b64decode(
|
||||
cursor.encode("ascii") + b"=" * (-len(cursor) % 4),
|
||||
altchars=b"-_",
|
||||
validate=True,
|
||||
)
|
||||
cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2)
|
||||
if cursor_user != user_id or not valid_uuid(chat_id):
|
||||
raise ValueError("cursor scope")
|
||||
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
if created_at.tzinfo is None or created_at.utcoffset() is None:
|
||||
raise ValueError("cursor timestamp must include a timezone")
|
||||
where += " AND (c.created_at, c.id) > (%s, %s)"
|
||||
params.extend((created_at, chat_id))
|
||||
except (ValueError, UnicodeError, TypeError) as exc:
|
||||
raise ValueError("invalid cursor") from exc
|
||||
params.append(limit + 1)
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT c.id, c.title, c.created_at FROM chats c "
|
||||
"JOIN chat_members own ON own.chat_id = c.id "
|
||||
f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
result = []
|
||||
for row in page:
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
result.append(self.chat(row, [str(item["user_id"]) for item in members]))
|
||||
next_cursor = None
|
||||
if has_more and page:
|
||||
last = page[-1]
|
||||
cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
|
||||
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
|
||||
return result, next_cursor
|
||||
|
||||
def create_idempotent_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
idempotency_key: str,
|
||||
canonical_body: str,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest()
|
||||
lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8")
|
||||
lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True)
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, sender_id),
|
||||
).fetchone()
|
||||
if not member:
|
||||
return None
|
||||
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,))
|
||||
existing = conn.execute(
|
||||
"SELECT request_hash, response, expires_at "
|
||||
"FROM idempotency_records "
|
||||
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s "
|
||||
"FOR UPDATE",
|
||||
(sender_id, chat_id, idempotency_key),
|
||||
).fetchone()
|
||||
if existing and existing["expires_at"] > utc_now():
|
||||
if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash):
|
||||
raise IdempotencyConflictError
|
||||
return existing["response"]
|
||||
if existing:
|
||||
conn.execute(
|
||||
"DELETE FROM idempotency_records "
|
||||
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s",
|
||||
(sender_id, chat_id, idempotency_key),
|
||||
)
|
||||
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages (chat_id, sender_id, text) "
|
||||
"VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at",
|
||||
(chat_id, sender_id, text),
|
||||
).fetchone()
|
||||
message = self.message(row)
|
||||
conn.execute(
|
||||
"INSERT INTO idempotency_records "
|
||||
"(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) "
|
||||
"VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)",
|
||||
(
|
||||
sender_id,
|
||||
chat_id,
|
||||
idempotency_key,
|
||||
request_hash,
|
||||
json.dumps(message, ensure_ascii=False, separators=(",", ":")),
|
||||
utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS),
|
||||
),
|
||||
)
|
||||
return message
|
||||
|
||||
def list_messages(
|
||||
self, chat_id: str, user_id: str, limit: int, cursor: Optional[str]
|
||||
) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]:
|
||||
with self.pool.connection() as conn:
|
||||
is_member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, user_id),
|
||||
).fetchone()
|
||||
if not is_member:
|
||||
return None
|
||||
where = "chat_id = %s"
|
||||
params: List[Any] = [chat_id]
|
||||
if cursor:
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4))
|
||||
decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2)
|
||||
if decoded_chat != chat_id or not valid_uuid(message_id):
|
||||
raise ValueError("cursor scope")
|
||||
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
where += " AND (created_at, id) > (%s, %s)"
|
||||
params.extend((created_at, message_id))
|
||||
except (ValueError, UnicodeError, TypeError) as exc:
|
||||
raise ValueError("invalid cursor") from exc
|
||||
params.append(limit + 1)
|
||||
rows = conn.execute(
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
|
||||
f"WHERE {where} ORDER BY created_at, id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
next_cursor = None
|
||||
if has_more and page:
|
||||
last = page[-1]
|
||||
cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
|
||||
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
|
||||
return [self.message(row) for row in page], next_cursor
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove a concurrent idempotency-key race through the load balancer."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import threading
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Concurrent shared-idempotency integration check")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
key = "race-" + uuid.uuid4().hex
|
||||
text = "concurrent-" + uuid.uuid4().hex
|
||||
count = 12
|
||||
barrier = threading.Barrier(count)
|
||||
|
||||
def send_once(_):
|
||||
worker = Client(args.base_url, args.ca_file)
|
||||
barrier.wait(timeout=10)
|
||||
return worker.message(alice, chat, text, key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=count) as pool:
|
||||
responses = list(pool.map(send_once, range(count)))
|
||||
|
||||
instance_ids = {response["headers"]["X-Instance-Id"] for response in responses}
|
||||
messages = [response["json"] for response in responses]
|
||||
check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas")
|
||||
check(all(message == messages[0] for message in messages), "same-key requests returned different messages")
|
||||
page = client.call(
|
||||
"GET",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages?limit=100",
|
||||
actor=alice,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
matches = [message for message in page["items"] if message["text"] == text]
|
||||
check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages")
|
||||
print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,385 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Repeatable 1/2/4 replica throughput experiment with Docker resource samples."""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import platform
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import hashlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LOAD_TEST = ROOT / "tests" / "load.py"
|
||||
OUTPUT_DIR = ROOT / "evidence" / "track1"
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def run(command, *, env=None, capture=True):
|
||||
return subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
check=True,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE if capture else None,
|
||||
stderr=subprocess.PIPE if capture else None,
|
||||
)
|
||||
|
||||
|
||||
def compose_environment(cpu_limit, memory_limit):
|
||||
import os
|
||||
|
||||
env = os.environ.copy()
|
||||
env["API_CPU_LIMIT"] = cpu_limit
|
||||
env["API_MEMORY_LIMIT"] = memory_limit
|
||||
return env
|
||||
|
||||
|
||||
def database_counts():
|
||||
query = (
|
||||
"SELECT json_build_object("
|
||||
"'users', (SELECT count(*) FROM users), "
|
||||
"'chats', (SELECT count(*) FROM chats), "
|
||||
"'messages', (SELECT count(*) FROM messages), "
|
||||
"'sessions', (SELECT count(*) FROM sessions), "
|
||||
"'idempotency_records', (SELECT count(*) FROM idempotency_records)"
|
||||
")::text"
|
||||
)
|
||||
result = run(
|
||||
COMPOSE
|
||||
+ [
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-X",
|
||||
"-qAt",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger",
|
||||
"-c",
|
||||
query,
|
||||
]
|
||||
)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def create_database_snapshot(destination):
|
||||
with destination.open("wb") as dump:
|
||||
result = subprocess.run(
|
||||
COMPOSE + ["exec", "-T", "postgres", "pg_dump", "-Fc", "-U", "messenger", "-d", "messenger"],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
stdout=dump,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError("failed to create PostgreSQL baseline snapshot: " + result.stderr.decode(errors="replace"))
|
||||
digest_builder = hashlib.sha256()
|
||||
with destination.open("rb") as dump:
|
||||
for chunk in iter(lambda: dump.read(1024 * 1024), b""):
|
||||
digest_builder.update(chunk)
|
||||
digest = digest_builder.hexdigest()
|
||||
return digest, database_counts()
|
||||
|
||||
|
||||
def restore_database_snapshot(snapshot, baseline_counts, api_replicas, cpu_limit, memory_limit):
|
||||
env = compose_environment(cpu_limit, memory_limit)
|
||||
run(COMPOSE + ["stop", "api"])
|
||||
with snapshot.open("rb") as dump:
|
||||
result = subprocess.run(
|
||||
COMPOSE
|
||||
+ [
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"pg_restore",
|
||||
"-U",
|
||||
"messenger",
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
"--exit-on-error",
|
||||
"--dbname=messenger",
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=False,
|
||||
stdin=dump,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError("failed to restore PostgreSQL baseline snapshot: " + result.stderr.decode(errors="replace"))
|
||||
restored_counts = database_counts()
|
||||
if restored_counts != baseline_counts:
|
||||
raise RuntimeError(
|
||||
"restored PostgreSQL baseline has different row counts: "
|
||||
+ json.dumps(restored_counts, sort_keys=True)
|
||||
)
|
||||
run(
|
||||
COMPOSE
|
||||
+ [
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--build",
|
||||
"--force-recreate",
|
||||
"--scale",
|
||||
f"api={api_replicas}",
|
||||
"--wait",
|
||||
"--wait-timeout",
|
||||
"120",
|
||||
"api",
|
||||
],
|
||||
env=env,
|
||||
)
|
||||
|
||||
|
||||
def docker_stats():
|
||||
containers = run(
|
||||
["docker", "compose", "--env-file", ".env", "ps", "-q"],
|
||||
).stdout.splitlines()
|
||||
if not containers:
|
||||
return []
|
||||
result = run(
|
||||
[
|
||||
"docker",
|
||||
"stats",
|
||||
"--no-stream",
|
||||
"--format",
|
||||
"{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}",
|
||||
*containers,
|
||||
]
|
||||
)
|
||||
samples = []
|
||||
for line in result.stdout.splitlines():
|
||||
name, cpu, memory = line.split("|", 2)
|
||||
used = memory.split("/", 1)[0].strip()
|
||||
samples.append(
|
||||
{
|
||||
"container": name,
|
||||
"service": "api" if "-api-" in name else "postgres" if "-postgres-" in name else "load_balancer",
|
||||
"cpu_percent": float(cpu.rstrip("%")),
|
||||
"memory_bytes": parse_bytes(used),
|
||||
}
|
||||
)
|
||||
return samples
|
||||
|
||||
|
||||
def parse_bytes(value):
|
||||
match = re.fullmatch(r"([0-9.]+)\s*([KMG]i?B|B)", value)
|
||||
if not match:
|
||||
return None
|
||||
amount = float(match.group(1))
|
||||
unit = match.group(2)
|
||||
power = {"B": 0, "KB": 1, "KiB": 1, "MB": 2, "MiB": 2, "GB": 3, "GiB": 3}[unit]
|
||||
return round(amount * 1024**power)
|
||||
|
||||
|
||||
def run_load(name, duration, concurrency, repetition, base_url):
|
||||
output = OUTPUT_DIR / f"{name}-run-{repetition}.json"
|
||||
command = [
|
||||
sys.executable,
|
||||
str(LOAD_TEST),
|
||||
"--base-url",
|
||||
base_url,
|
||||
"--duration",
|
||||
str(duration),
|
||||
"--concurrency",
|
||||
str(concurrency),
|
||||
"--output",
|
||||
str(output.relative_to(ROOT)),
|
||||
]
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
samples = []
|
||||
while process.poll() is None:
|
||||
samples.extend(docker_stats())
|
||||
time.sleep(1)
|
||||
stdout, stderr = process.communicate()
|
||||
if process.returncode:
|
||||
raise RuntimeError(f"load run failed:\n{stdout}\n{stderr}")
|
||||
metrics = json.loads(output.read_text(encoding="utf-8"))
|
||||
metrics["resource_samples"] = samples
|
||||
return metrics
|
||||
|
||||
|
||||
def system_info():
|
||||
def command_output(command):
|
||||
try:
|
||||
return run(command).stdout.strip()
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return "unavailable"
|
||||
|
||||
return {
|
||||
"platform": platform.platform(),
|
||||
"processor": platform.processor() or "unavailable",
|
||||
"cpu_count": __import__("os").cpu_count(),
|
||||
"host_memory_bytes": command_output(["sysctl", "-n", "hw.memsize"]),
|
||||
"docker_server": command_output(["docker", "version", "--format", "{{.Server.Version}}"]),
|
||||
"docker_compose": command_output(["docker", "compose", "version", "--short"]),
|
||||
}
|
||||
|
||||
|
||||
def configure_limits(count, phase):
|
||||
if phase == "fixed":
|
||||
return f"{1 / count:.4f}", f"{512 // count}M"
|
||||
return "1.0", "512M"
|
||||
|
||||
|
||||
def experiment_case(count, phase, args, snapshot, snapshot_hash, baseline_counts):
|
||||
cpu_limit, memory_limit = configure_limits(count, phase)
|
||||
restore_database_snapshot(snapshot, baseline_counts, count, cpu_limit, memory_limit)
|
||||
run_load(f"{phase}-{count}-warmup", args.warmup, args.concurrency, 0, args.base_url)
|
||||
warmup_counts = database_counts()
|
||||
if warmup_counts == baseline_counts:
|
||||
raise RuntimeError("warm-up did not create synthetic workload data")
|
||||
|
||||
repetitions = []
|
||||
for repetition in range(1, args.repetitions + 1):
|
||||
restore_database_snapshot(snapshot, baseline_counts, count, cpu_limit, memory_limit)
|
||||
repetitions.append(run_load(f"{phase}-{count}", args.duration, args.concurrency, repetition, args.base_url))
|
||||
if database_counts() == baseline_counts:
|
||||
raise RuntimeError(f"{phase}-{count} repetition {repetition} did not write benchmark data")
|
||||
return {
|
||||
"phase": phase,
|
||||
"api_replicas": count,
|
||||
"per_replica_cpu_limit": cpu_limit,
|
||||
"per_replica_memory_limit": memory_limit,
|
||||
"total_api_cpu_limit": round(float(cpu_limit) * count, 4),
|
||||
"total_api_memory_limit_mib": int(memory_limit[:-1]) * count,
|
||||
"warmup_seconds": args.warmup,
|
||||
"baseline_row_counts": baseline_counts,
|
||||
"baseline_snapshot_sha256": snapshot_hash,
|
||||
"each_repetition_restored_same_baseline": True,
|
||||
"repetitions": repetitions,
|
||||
}
|
||||
|
||||
|
||||
def make_graph(cases):
|
||||
width, height = 900, 420
|
||||
margin = 60
|
||||
values = [
|
||||
(
|
||||
f"{case['phase']} / {case['api_replicas']}",
|
||||
sum(run["successful_rps"] for run in case["repetitions"]) / len(case["repetitions"]),
|
||||
"#3572A5" if case["phase"] == "fixed" else "#E38B29",
|
||||
)
|
||||
for case in cases
|
||||
]
|
||||
top = max((value for _, value, _ in values), default=1) or 1
|
||||
graph_height = height - 2 * margin
|
||||
bar_width = 78
|
||||
gap = (width - 2 * margin - bar_width * len(values)) / max(1, len(values) - 1)
|
||||
parts = [
|
||||
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}">',
|
||||
'<rect width="100%" height="100%" fill="white"/>',
|
||||
'<text x="450" y="28" text-anchor="middle" font-family="sans-serif" font-size="18">API scaling: average successful RPS</text>',
|
||||
f'<line x1="{margin}" y1="{height-margin}" x2="{width-margin}" y2="{height-margin}" stroke="#333"/>',
|
||||
]
|
||||
for index, (label, value, color) in enumerate(values):
|
||||
x = margin + index * (bar_width + gap)
|
||||
bar_height = value / top * graph_height
|
||||
y = height - margin - bar_height
|
||||
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width}" height="{bar_height:.1f}" fill="{color}"/>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{max(48,y-8):.1f}" text-anchor="middle" font-family="sans-serif" font-size="12">{value:.1f}</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{height-margin+20}" text-anchor="middle" font-family="sans-serif" font-size="12">{label}</text>')
|
||||
parts.append("</svg>")
|
||||
(OUTPUT_DIR / "throughput.svg").write_text("\n".join(parts) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--duration", type=int, default=30)
|
||||
parser.add_argument("--warmup", type=int, default=5)
|
||||
parser.add_argument("--concurrency", type=int, default=16)
|
||||
parser.add_argument("--repetitions", type=int, default=3)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.duration <= 3600 or not 1 <= args.warmup <= 600:
|
||||
parser.error("duration and warmup must be positive and bounded")
|
||||
if not 1 <= args.concurrency <= 128 or not 1 <= args.repetitions <= 10:
|
||||
parser.error("concurrency or repetitions out of range")
|
||||
|
||||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
results_file = OUTPUT_DIR / "results.json"
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="lab3-track1-") as temporary_directory:
|
||||
snapshot = Path(temporary_directory) / "baseline.dump"
|
||||
run(COMPOSE + ["stop", "api"])
|
||||
baseline_counts = None
|
||||
try:
|
||||
snapshot_hash, baseline_counts = create_database_snapshot(snapshot)
|
||||
cases = []
|
||||
for phase, replica_counts in (("fixed", (1, 2, 4)), ("growth", (1, 2, 4, 8))):
|
||||
for count in replica_counts:
|
||||
cases.append(
|
||||
experiment_case(count, phase, args, snapshot, snapshot_hash, baseline_counts)
|
||||
)
|
||||
summary = {
|
||||
"method": (
|
||||
"closed-loop POST message workload via HAProxy; same concurrency and per-run "
|
||||
"restored PostgreSQL custom-format baseline snapshot"
|
||||
),
|
||||
"database_baseline": {
|
||||
"snapshot_sha256": snapshot_hash,
|
||||
"row_counts": baseline_counts,
|
||||
"restore_before_each_warmup_and_measured_repetition": True,
|
||||
"state_after_experiment": "baseline restored; all synthetic benchmark writes removed",
|
||||
},
|
||||
"system": system_info(),
|
||||
"database_and_lb_limits": {"postgres": "1 CPU / 1 GiB", "HAProxy": "0.5 CPU / 128 MiB"},
|
||||
"client_concurrency": args.concurrency,
|
||||
"requested_duration_seconds": args.duration,
|
||||
"repetitions_per_case": args.repetitions,
|
||||
"cases": cases,
|
||||
}
|
||||
results_file.write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
make_graph(cases)
|
||||
for case in cases:
|
||||
values = [run["successful_rps"] for run in case["repetitions"]]
|
||||
deviation = math.sqrt(sum((value - sum(values) / len(values)) ** 2 for value in values) / len(values))
|
||||
print(
|
||||
f"{case['phase']:6} replicas={case['api_replicas']} "
|
||||
f"mean_rps={sum(values) / len(values):.2f} stdev={deviation:.2f}"
|
||||
)
|
||||
print(f"Evidence saved under {OUTPUT_DIR.relative_to(ROOT)}")
|
||||
finally:
|
||||
if baseline_counts is not None:
|
||||
restore_database_snapshot(snapshot, baseline_counts, 2, "1.0", "512M")
|
||||
else:
|
||||
run(
|
||||
COMPOSE
|
||||
+ [
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--build",
|
||||
"--force-recreate",
|
||||
"--scale",
|
||||
"api=2",
|
||||
"--wait",
|
||||
"--wait-timeout",
|
||||
"120",
|
||||
"api",
|
||||
],
|
||||
env=compose_environment("1.0", "512M"),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user