ЛР 3 — Группа БВТ2304 — Аккуратов — на 5 #27

Closed
kaktuz2000 wants to merge 1 commits from kaktuz2000/mtusi2026lab3:solution/bvt2304-akkuratov into main
65 changed files with 8965 additions and 28 deletions
+3 -9
View File
@@ -1,13 +1,7 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
# Per-replica limits; adjust for the fixed-budget track 1 experiment.
API_CPU_LIMIT=1.0
API_MEMORY_LIMIT=512M
+34
View File
@@ -0,0 +1,34 @@
name: Application checks
on: [push, pull_request]
permissions:
contents: read
jobs:
compose-tests:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- run: python -m pip install -r requirements-dev.txt
- name: Create local test credentials
run: |
cp .env.example .env
./scripts/bootstrap-secrets.sh
- name: Validate and start the application
run: |
make check validate PYTHON=python
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180
- name: Run API and concurrency tests
run: |
make test
make test-idempotency
- name: Verify persistence and failover
run: |
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
- name: Stop test stack
if: always()
run: docker compose --env-file .env down -v
+25
View File
@@ -0,0 +1,25 @@
FROM python:3.12.7-slim
WORKDIR /app
ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
COPY . /app
RUN chown -R app:app /app
USER app
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).read()" || exit 1
CMD ["python3", "src/server.py"]
+8 -2
View File
@@ -1,10 +1,10 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://127.0.0.1:8080
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover test-idempotency track1
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -28,3 +28,9 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-idempotency:
$(PYTHON) student-tests/test_idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS)
track1:
$(PYTHON) student-tests/track1_benchmark.py
+29 -1
View File
@@ -2,7 +2,7 @@
Запустить несколько взаимозаменяемых экземпляров API за единой точкой входа и сохранить корректность при остановке одного из них.
Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен.
В репозитории реализован стенд лабораторной: контракт и публичные проверки дополнены приложением, Docker-инфраструктурой, сценариями отказов и исследовательским экспериментом. Критерии задания и выбранные решения описаны ниже и в [отчёте](REPORT.md).
[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md)
@@ -12,6 +12,34 @@
**Архитектура:** Клиент → LB → API × 2+ → общие PostgreSQL и хранилище сессий. Compose или Swarm — на выбор.
## Реализация этого решения
Используется Docker Compose: HAProxy с активной проверкой `GET /health/ready` и DNS-resolver Docker, два и более одинаковых API-контейнера, одна PostgreSQL. Публичный порт публикуется только у HAProxy на loopback. Сессии и идемпотентность находятся в PostgreSQL; локального состояния сессий у реплик нет. Миграции выполняет отдельный одноразовый сервис до старта API.
```sh
cp .env.example .env
./scripts/bootstrap-secrets.sh
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 120
make check
make test
make test-idempotency
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
# Полная матрица трека 1: 1/2/4 реплики при фиксированном бюджете и 1/2/4/8 при росте.
make track1
```
Локальные секреты хранятся в игнорируемой папке `secrets/`; скрипт их не выводит. API использует имя контейнера как стабильный на время жизни процесса `X-Instance-Id`. Для намеренного удаления учебной БД после проверки выполните `docker compose --env-file .env down -v`; эта команда необратимо удаляет volume и не используется сценариями restart/failover.
Перед завершением работы восстановите остановленную реплику командой `docker compose --env-file .env up -d --scale api=2 --wait`. Подробные методика, ограничения и доказательства указаны в `REPORT.md`.
Для проверки поочерёдной замены экземпляров под нагрузкой запустите в одном терминале `python3 tests/load.py --duration 30 --concurrency 8 --output evidence/rolling-restart-load.json`, а пока она работает — в другом `scripts/rolling-restart.sh`. Скрипт перезапускает по одному контейнеру, не останавливая вторую реплику, и ждёт readiness обоих экземпляров.
Перед `make track1` запустите Compose-стенд и не допускайте параллельных записей в БД: скрипт останавливает API, создаёт временный custom-format snapshot PostgreSQL, а затем восстанавливает его перед каждым прогревом и каждым измерением. В `finally` скрипт возвращает исходную БД и 2 API-реплики, поэтому все benchmark-записи удаляются, а данные, бывшие в volume до эксперимента, сохраняются. Все семь случаев (1/2/4 фиксированный бюджет; 1/2/4/8 рост) используют один и тот же snapshot.
CI workflow `.github/workflows/application.yml` отдельно проверяет сборку Compose-стенда, публичный smoke, конкурентную идемпотентность, persistence и отказ одной реплики. CI использует только временные синтетические credentials и удаляет собственный тестовый volume после завершения job.
## Что сделать
1. Выберите Compose + Nginx/HAProxy/Traefik либо Docker Swarm. Для выбранного варианта дайте одну последовательность запуска.
+92 -16
View File
@@ -1,37 +1,113 @@
# Отчёт — лабораторная 3
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Аккуратов Тимофей, бвт2304
- Целевая оценка: 5; выбран трек 1 — цена масштабирования
- Commit/PR: ещё не созданы; изменения находятся в локальном рабочем каталоге
- Источники, библиотеки и помощь ИИ: перенесена собственная реализация из mtusi2026lab2; Python stdlib, psycopg 3.2.10 и psycopg-pool 3.3.3; официальные образы PostgreSQL и HAProxy. При разработке использовалась помощь Copilot в VS Code; код и результаты проверены локально. Чужие исходники не переносились.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker Desktop с Compose v2, `openssl`, `make` и Python 3.9+ для локальных проверок. API работает в образе Python 3.12.7; клиентские тесты используют только стандартную библиотеку.
```sh
cp .env.example .env
./scripts/bootstrap-secrets.sh
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 120
make check
make test
make test-idempotency
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
```
Для полной OpenAPI-валидации:
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
make validate PYTHON=.venv/bin/python
```
`bootstrap-secrets.sh` создаёт локальные `secrets/postgres_password` и `secrets/database_url`, не печатая их. `.env` и `secrets/` игнорируются Git. Миграции применяются одноразовым сервисом `migrate`; дождитесь состояния `healthy` у PostgreSQL и API. По умолчанию тесты обращаются к `http://127.0.0.1:8080`; при изменении `APP_PORT` задайте, например, `make test BASE_URL=http://127.0.0.1:8081`.
Безопасная остановка с сохранением данных: `docker compose --env-file .env down`. **Намеренное удаление всех учебных данных** — только отдельной командой `docker compose --env-file .env down -v`; эта команда не используется проверками restart/failover.
Исследовательский эксперимент целиком повторяется одной командой после запуска Compose-стенда:
```sh
make track1
```
Команда запускает все семь случаев матрицы: 1/2/4 реплики при фиксированном бюджете и 1/2/4/8 при росте ресурсов. Для сопоставимости используйте тот же Docker host, порт, число клиентов и длительность. Экспериментальные записи удаляются при восстановлении сохранённого baseline.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
**Клиент → HAProxy → API × 2+ → общая PostgreSQL.** В Compose нет `container_name`; API-порты не опубликованы на хост. Единственный внешний порт — `127.0.0.1:${APP_PORT}:8080` у HAProxy. Реплики не используют sticky sessions. Сессии, сообщения и результаты идемпотентности хранятся в PostgreSQL; это единственный источник истины. Redis из предыдущей реализации остаётся необязательной возможностью в коде, но в этом стенде не развёрнут и для корректности не нужен.
Из lab2 перенесены `src/`, `migrations/001_initial.sql`, Dockerfile и зависимости. Изменения этой работы:
- API больше не запускает миграции при старте. Сервис `migrate` применяет версионированные SQL-файлы до запуска реплик; таблица `schema_migrations` и advisory transaction lock защищают от повторного применения.
- `migrations/002_idempotency.sql` добавляет общую таблицу ключей. Ключ уникален в области user + POST + chat + key; хранится SHA-256 канонического JSON и исходный ответ. Проверка повтора, создание сообщения и сохранение ответа выполняются в одной транзакции PostgreSQL. Transaction-level advisory lock сериализует конкурентные запросы с тем же ключом на разных репликах. Конфликтующий body возвращает 409, успешный повтор — исходное сообщение; срок хранения не меньше 24 часов.
- `X-Instance-Id` берётся из hostname контейнера и неизменен в пределах процесса. `scripts/stop-one.sh` сопоставляет его с конкретным контейнером.
- HAProxy обнаруживает backend через DNS Docker (`127.0.0.11`, `hold valid 2s`) и проверяет `GET /health/ready` каждую секунду; backend считается недоступным после двух неудач. С учётом DNS TTL и таймаута проверки консервативный бюджет обнаружения — до 5s, что ниже лимита 10s; connect timeout — 2s, client/server timeout — 15s. У API есть readiness healthcheck, лимиты БД: statement 5s, lock 2s, pool wait 3s. При SIGTERM сервер прекращает приём и завершает активные запросы; `stop_grace_period` — 20s.
- Все контейнерные образы закреплены тегами: Python 3.12.7-slim, PostgreSQL 16.4-alpine, HAProxy 3.0.7-alpine. PostgreSQL использует именованный `postgres_data`. API по умолчанию ограничен 1 CPU/512 MiB на реплику; PostgreSQL — 1 CPU/1 GiB, HAProxy — 0.5 CPU/128 MiB. В переменных `.env.example` настраиваются host port, TTL и per-replica лимиты API.
Миграция 002 аддитивна и не меняет существующие таблицы lab2, поэтому старый API не использует новую таблицу. CI workflow `.github/workflows/application.yml` собирает Compose-стенд, запускает публичные и конкурентные тесты, persistence и отказ API-реплики; он не тестирует обновление приложения на новую версию бизнес-кода. Локально проверен поочерёдный restart одинакового образа под нагрузкой.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Контракт и OpenAPI | `make check`; `make validate PYTHON=.venv/bin/python` | PASS для структуры, синтаксиса и полной OpenAPI 3.1/JSON Schema валидации; контракт и публичные проверки не менялись | `evidence/openapi-validation.txt` |
| Стенд и миграции | `docker compose ... up --scale api=2`; SQL `schema_migrations` | 2 API healthy, PostgreSQL и HAProxy healthy; применены ровно `001_initial.sql`, `002_idempotency.sql`; API опубликованных host ports не имеет | `evidence/deployment-checks.txt` |
| Smoke и общая сессия | `make test` через HAProxy | PASS: полный публичный сценарий lab3 | `evidence/public-tests.txt` |
| Общая идемпотентность | `make test-idempotency` | 12 конкурентных запросов с одинаковым ключом прошли через 2 наблюдаемые реплики, вернули одно сообщение; в истории ровно одна запись. Публичный smoke также проверяет повтор и 409 при другом body | `evidence/public-tests.txt` |
| Persistence lab2 | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS: после перезапуска PostgreSQL, API и LB подтверждённые данные доступны, прежняя сессия действительна; volumes сохранены | `evidence/persistence.txt` |
| Failover первой и второй API | Дважды `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, восстановление между прогонами | Проверки увидели ≥2 экземпляра. Для первого: 108 успешных записей, 2 неуспешные попытки, восстановление за 2.684s. Для второго: 109 успешных, 1 неуспешная, восстановление за 2.566s. Подтверждённые сообщения и сессия сохранились | `evidence/failover-1.txt`, `evidence/failover-2.txt`, `evidence/failover-targets.txt` |
| Readiness, timeouts, SIGTERM | HAProxy health checks и `scripts/rolling-restart.sh` | Backend исключается после 2 проверок с интервалом 1s; отказная проверка восстановила записи за 2.566–2.684s. Поочерёдный restart обоих контейнеров завершился `healthy` | `haproxy.cfg`, `compose.yaml`, `evidence/failover-1.txt`, `evidence/failover-2.txt`, `evidence/rolling-restart-actions.txt` |
| Лимиты и отсутствие обходного API-порта | `docker compose config`; `docker inspect` | Реально установлены API 1 CPU/512 MiB, PostgreSQL 1 CPU/1 GiB, HAProxy 0.5 CPU/128 MiB. На хост опубликован только loopback-порт LB; в `ps` у API показан только внутренний `8080/tcp` | `evidence/deployment-checks.txt` |
| Rolling restart под нагрузкой | `tests/load.py --duration 35 --concurrency 8` одновременно с `scripts/rolling-restart.sh` | 19,968 успешных POST, 6 ошибок переходного периода; successful RPS 570.303, p50/p95/p99 8.385/45.880/73.085 ms. Оба API контейнера были поочерёдно перезапущены и стали healthy | `evidence/rolling-restart-load.json`, `evidence/rolling-restart-actions.txt` |
| Трек 5 — цена масштабирования | `make track1` | Матрица 1/2/4 при фиксированных ресурсах, затем рост до 1/2/4/8; одна исходная custom-format копия PostgreSQL восстанавливается до каждого прогрева и измерения; три повтора на случай; исходная БД восстанавливается в `finally` | `evidence/track1/results.json`, `evidence/track1/throughput.svg` |
| CI сборка и проверка приложения | `.github/workflows/application.yml` (`push`, `pull_request`) | Отдельный job устанавливает requirements-dev, собирает/ожидает Compose, выполняет `make check validate`, smoke, параллельную идемпотентность, persistence и failover. Workflow добавлен; выполнение на GitHub Actions ожидает push/PR | `.github/workflows/application.yml` |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
### Трек 1: цена масштабирования
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
**Гипотеза:** при фиксированном суммарном бюджете API увеличение числа реплик само по себе не даст прироста; при добавлении ресурсов оно ускорит запись до насыщения общей PostgreSQL, после чего новые API-процессы почти не помогут.
Одна машина macOS 27.0.1 arm64, физически 10 CPU и 16 GiB RAM; Docker Desktop 28.0.4 / Compose 2.34.0-desktop.1 предоставлял контейнерам 10 vCPU и около 7.65 GiB. API: Python 3.12.7; фиксированные ресурсы PostgreSQL 1 CPU/1 GiB и HAProxy 0.5 CPU/128 MiB. На каждый случай — прогрев 5s и три измеряемых прогона по 30s, 16 параллельных клиентов. Генератор — штатный closed-loop `tests/load.py`, только POST сообщений через LB; прозрачных retry нет. Перед каждым прогревом и измеряемым прогоном API остановлен, PostgreSQL восстановлена из одного custom-format snapshot, проверены контрольные количества строк, затем запускается выбранное число реплик. После всей матрицы в `finally` возвращается исходная БД и две реплики; синтетические benchmark-записи не остаются в volume. Во время запуска исключены внешние записи в БД. В таблице latency — среднее p50/p95/p99 по успешным запросам, RPS — среднее ± стандартное отклонение между тремя прогонами. CPU — максимальная 1-секундная выборка Docker stats; 100% соответствует примерно одному CPU.
| Профиль | API | Суммарный лимит API | Успешные RPS, среднее ± σ | p50 / p95 / p99, ms | Ошибки | Peak PostgreSQL CPU |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| Фиксированный бюджет | 1 | 1 CPU / 512 MiB | 212.54 ± 1.34 | 79.36 / 115.01 / 181.42 | 0 | 28.22% |
| Фиксированный бюджет | 2 | 1 CPU / 512 MiB | 212.71 ± 10.81 | 83.45 / 166.94 / 224.07 | 0 | 40.58% |
| Фиксированный бюджет | 4 | 1 CPU / 512 MiB | 156.34 ± 17.60 | 92.69 / 260.44 / 663.89 | 0 | 46.22% |
| Рост ресурсов | 1 | 1 CPU / 512 MiB | 211.67 ± 1.56 | 79.53 / 115.35 / 182.46 | 0 | 26.54% |
| Рост ресурсов | 2 | 2 CPU / 1 GiB | 604.95 ± 6.29 | 19.72 / 60.32 / 79.17 | 0 | 62.93% |
| Рост ресурсов | 4 | 4 CPU / 2 GiB | 1,099.73 ± 11.87 | 11.01 / 29.91 / 51.29 | 0 | 108.47% |
| Рост ресурсов | 8 | 8 CPU / 4 GiB | 1,159.95 ± 16.88 | 11.75 / 27.90 / 37.07 | 0 | 105.96% |
При фиксированном бюджете переход к 4 маленьким репликам снизил успешный throughput на 26.4% относительно одной реплики и увеличил p95 с 115.01 до 260.44 ms: дробление CPU между процессами не дало выигрыша. При росте бюджета до четырёх API CPU/2 GiB throughput вырос в 5.19 раза относительно одной реплики при том же per-instance лимите, а p95 уменьшился примерно в 3.86 раза — подтверждённое улучшение для этого стенда. Дальнейшее удвоение API до 8, CPU до 8 и памяти до 4 GiB увеличило throughput на 5.48%; одновременно PostgreSQL достигла выборки 105.96–108.47% одного CPU. Это указывает на насыщение общей БД и уменьшающуюся отдачу от добавления API-ресурсов; это наблюдение относится только к данному профилю, а не универсальная характеристика PostgreSQL.
Все данные синтетические. До матрицы был зафиксирован baseline snapshot; каждое измерение стартовало с одного и того же состояния БД и создавало свой тестовый набор в новом уникальном чате. Row counts после завершения совпали с baseline: users=156, chats=52, messages=445522, sessions=154, idempotency_records=445522. Пиковая выборка памяти за benchmark: PostgreSQL 252.6 MiB, API 73.1 MiB на контейнер, HAProxy 21.1 MiB. Подробные исходные метрики и выборки CPU/RAM, единый SHA-256 snapshot и контрольные количества строк сохранены в JSON.
### Отказы
| Воздействие | Ожидание | Наблюдение | Восстановление / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Остановка первой API-реплики | LB продолжает читать/писать через оставшуюся реплику; повтор сообщения не создаёт дубль | 108 успешных записей, 2 переходные ошибки; readiness восстановилась за 2.684s | Данные и сессия сохранены; потеря подтверждённых сообщений — 0 |
| Остановка второй API-реплики | Те же гарантии для другой реплики | 109 успешных записей, 1 переходная ошибка; readiness восстановилась за 2.566s | Данные и сессия сохранены; потеря подтверждённых сообщений — 0 |
| Поочерёдный SIGTERM/restart двух API под нагрузкой | Вторая реплика остаётся доступной, остановленная возвращается healthy | 19,968 успешных POST, 6 ошибок за 35s; оба экземпляра перезапущены последовательно | Контейнеры вернулись в healthy; подтверждённые записи остаются в PostgreSQL |
| Restart PostgreSQL + API + LB | Сохранить существующую сессию и сообщения на named volume | Persistence-тест PASS с прежним токеном | Потеря данных — 0; volumes не удалялись |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Эксперимент закрытый (closed-loop), проводится на одном Mac и одним клиентским процессом; он подвержен coordinated omission и не задаёт независимый arrival rate. Процентили считаются только по успешным ответам, а выборка CPU/RAM снимается раз в секунду.
- Сценарий проверяет отказ API-процесса, не отказ Docker-host, HAProxy, PostgreSQL или диска. PostgreSQL и единственный LB — SPOF; отказ узла не исследовался, multi-node failover не заявляется.
- Порт HTTP открыт только на loopback для синтетического локального стенда. Для удалённой демонстрации необходим TLS. Секреты Compose — локальные bind-mounted файлы, не production secret manager.
- Идемпотентность ограничена текстовым контрактом lab3 и 24-часовым минимумом; транзакция держит PostgreSQL advisory lock. HAProxy `server-template` настроен максимум на 10 адресов — при масштабировании выше нужно расширить диапазон и повторить проверки.
- Rolling-сценарий проверяет поочерёдный restart текущего образа, не обновление на новую версию приложения. Миграции аддитивны, но совместимость произвольной смены версий приложения не проверялась.
- Результаты относятся только к указанным лимитам, размеру данных и клиентской интенсивности. Snapshot теперь одинаков перед каждым прогоном, но нагрузка остаётся closed-loop, на одном ноутбуке, без открытого arrival-rate профиля.
+105
View File
@@ -0,0 +1,105 @@
services:
postgres:
image: postgres:16.4-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets:
- postgres_password
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 20
start_period: 5s
deploy:
resources:
limits:
cpus: "1.0"
memory: 1G
restart: unless-stopped
migrate:
build:
context: .
dockerfile: Dockerfile
command: ["python3", "src/migrate.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
DB_POOL_MAX_SIZE: "2"
secrets:
- database_url
depends_on:
postgres:
condition: service_healthy
restart: "no"
api:
build:
context: .
dockerfile: Dockerfile
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL_FILE: /run/secrets/database_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DB_POOL_MAX_SIZE: "8"
DB_POOL_TIMEOUT_SECONDS: "3"
DB_STATEMENT_TIMEOUT_MS: "5000"
DB_LOCK_TIMEOUT_MS: "2000"
secrets:
- database_url
depends_on:
migrate:
condition: service_completed_successfully
healthcheck:
test:
- CMD
- python3
- -c
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()"
interval: 2s
timeout: 2s
retries: 3
start_period: 10s
deploy:
resources:
limits:
cpus: "${API_CPU_LIMIT:-1.0}"
memory: "${API_MEMORY_LIMIT:-512M}"
stop_grace_period: 20s
restart: unless-stopped
load_balancer:
image: haproxy:3.0.7-alpine
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
volumes:
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"]
interval: 10s
timeout: 3s
retries: 3
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
restart: unless-stopped
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
volumes:
postgres_data:
+16
View File
@@ -0,0 +1,16 @@
Capture time: 2026-10-21T05:57:02Z
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab3-api-5 mtusi2026lab3-api "python3 src/server.…" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp
mtusi2026lab3-api-6 mtusi2026lab3-api "python3 src/server.…" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp
mtusi2026lab3-load_balancer-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" load_balancer 29 minutes ago Up 17 minutes (healthy) 127.0.0.1:8080->8080/tcp
mtusi2026lab3-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 29 minutes ago Up 17 minutes (healthy) 5432/tcp
Migration versions:
001_initial.sql
002_idempotency.sql
Container limits:
/mtusi2026lab3-api-5 NanoCPUs=1000000000 MemoryBytes=536870912
/mtusi2026lab3-api-6 NanoCPUs=1000000000 MemoryBytes=536870912
/mtusi2026lab3-postgres-1 NanoCPUs=1000000000 MemoryBytes=1073741824
/mtusi2026lab3-load_balancer-1 NanoCPUs=500000000 MemoryBytes=134217728
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 108, "failed_attempts": 2, "first_recovery_seconds": 2.684}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 109, "failed_attempts": 1, "first_recovery_seconds": 2.566}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+2
View File
@@ -0,0 +1,2 @@
2026-10-21T05:39:45Z target_instance=498a08cd2b5c container=498a08cd2b5cf94be02873ea9eb0c9c5f6b79133ddfd50b44aff134f86222b71
2026-10-21T05:40:39Z target_instance=137c7f092d63 container=137c7f092d63cd2e0d8d2b71ceecd67860c35ba79709df56831bba3a967629ba
+4
View File
@@ -0,0 +1,4 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+6
View File
@@ -0,0 +1,6 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
python3 tests/smoke.py --base-url "http://127.0.0.1:8080"
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
python3 student-tests/test_idempotency_race.py --base-url "http://127.0.0.1:8080"
PASS: 12 concurrent same-key requests on 2 replicas returned one message
+29
View File
@@ -0,0 +1,29 @@
Container mtusi2026lab3-postgres-1 Running
Container mtusi2026lab3-migrate-1 Created
Container mtusi2026lab3-api-3 Running
Container mtusi2026lab3-api-4 Created
Container mtusi2026lab3-load_balancer-1 Running
Container mtusi2026lab3-postgres-1 Waiting
Container mtusi2026lab3-postgres-1 Healthy
Container mtusi2026lab3-migrate-1 Starting
Container mtusi2026lab3-migrate-1 Started
Container mtusi2026lab3-migrate-1 Waiting
Container mtusi2026lab3-migrate-1 Exited
Container mtusi2026lab3-api-4 Starting
Container mtusi2026lab3-api-4 Started
Container mtusi2026lab3-api-4 Waiting
Container mtusi2026lab3-api-3 Waiting
Container mtusi2026lab3-api-4 Healthy
Container mtusi2026lab3-api-3 Healthy
Container mtusi2026lab3-api-4 Waiting
Container mtusi2026lab3-api-3 Waiting
Container mtusi2026lab3-load_balancer-1 Waiting
Container mtusi2026lab3-postgres-1 Waiting
Container mtusi2026lab3-migrate-1 Waiting
Container mtusi2026lab3-load_balancer-1 Healthy
Container mtusi2026lab3-migrate-1 Exited
Container mtusi2026lab3-postgres-1 Healthy
Container mtusi2026lab3-api-4 Healthy
Container mtusi2026lab3-api-3 Healthy
PASS: API container 498a08cd2b5cf94be02873ea9eb0c9c5f6b79133ddfd50b44aff134f86222b71 restarted and is healthy
PASS: API container 137c7f092d63cd2e0d8d2b71ceecd67860c35ba79709df56831bba3a967629ba restarted and is healthy
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 35.0,
"elapsed_seconds": 35.013,
"concurrency": 8,
"successful_requests": 19968,
"failed_requests": 6,
"error_ratio": 0.0003,
"successful_rps": 570.303,
"successful_latency_ms": {
"p50": 8.385,
"p95": 45.88,
"p99": 73.085
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.046,
"concurrency": 16,
"successful_requests": 6346,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 211.212,
"successful_latency_ms": {
"p50": 80.264,
"p95": 114.896,
"p99": 182.694
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.064,
"concurrency": 16,
"successful_requests": 6445,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 214.379,
"successful_latency_ms": {
"p50": 78.662,
"p95": 115.143,
"p99": 181.9
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.056,
"concurrency": 16,
"successful_requests": 6373,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 212.038,
"successful_latency_ms": {
"p50": 79.158,
"p95": 114.991,
"p99": 179.677
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 5.0,
"elapsed_seconds": 5.065,
"concurrency": 16,
"successful_requests": 1036,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 204.529,
"successful_latency_ms": {
"p50": 81.556,
"p95": 109.862,
"p99": 172.61
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.098,
"concurrency": 16,
"successful_requests": 5968,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 198.285,
"successful_latency_ms": {
"p50": 86.571,
"p95": 190.715,
"p99": 278.78
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.03,
"concurrency": 16,
"successful_requests": 6736,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 224.312,
"successful_latency_ms": {
"p50": 80.141,
"p95": 150.189,
"p99": 197.092
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.031,
"concurrency": 16,
"successful_requests": 6473,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 215.547,
"successful_latency_ms": {
"p50": 83.643,
"p95": 159.92,
"p99": 196.349
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 5.0,
"elapsed_seconds": 5.017,
"concurrency": 16,
"successful_requests": 990,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 197.311,
"successful_latency_ms": {
"p50": 82.99,
"p95": 199.803,
"p99": 294.885
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.155,
"concurrency": 16,
"successful_requests": 5464,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 181.196,
"successful_latency_ms": {
"p50": 85.443,
"p95": 200.223,
"p99": 467.896
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.201,
"concurrency": 16,
"successful_requests": 4377,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 144.927,
"successful_latency_ms": {
"p50": 98.238,
"p95": 287.157,
"p99": 494.409
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.261,
"concurrency": 16,
"successful_requests": 4324,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 142.888,
"successful_latency_ms": {
"p50": 94.377,
"p95": 293.944,
"p99": 1029.361
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 5.0,
"elapsed_seconds": 5.173,
"concurrency": 16,
"successful_requests": 1140,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 220.377,
"successful_latency_ms": {
"p50": 64.745,
"p95": 197.524,
"p99": 398.349
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.039,
"concurrency": 16,
"successful_requests": 6333,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 210.824,
"successful_latency_ms": {
"p50": 80.103,
"p95": 114.112,
"p99": 182.734
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.073,
"concurrency": 16,
"successful_requests": 6325,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 210.32,
"successful_latency_ms": {
"p50": 79.313,
"p95": 117.188,
"p99": 183.607
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.03,
"concurrency": 16,
"successful_requests": 6422,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 213.855,
"successful_latency_ms": {
"p50": 79.164,
"p95": 114.738,
"p99": 181.052
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 5.0,
"elapsed_seconds": 5.03,
"concurrency": 16,
"successful_requests": 1054,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 209.531,
"successful_latency_ms": {
"p50": 77.196,
"p95": 116.562,
"p99": 184.922
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.017,
"concurrency": 16,
"successful_requests": 18059,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 601.627,
"successful_latency_ms": {
"p50": 19.831,
"p95": 60.095,
"p99": 81.757
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.494,
"concurrency": 16,
"successful_requests": 18280,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 599.463,
"successful_latency_ms": {
"p50": 19.535,
"p95": 59.859,
"p99": 78.355
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.029,
"concurrency": 16,
"successful_requests": 18431,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 613.766,
"successful_latency_ms": {
"p50": 19.791,
"p95": 61.005,
"p99": 77.388
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 5.0,
"elapsed_seconds": 5.037,
"concurrency": 16,
"successful_requests": 3029,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 601.302,
"successful_latency_ms": {
"p50": 20.938,
"p95": 59.639,
"p99": 76.915
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.012,
"concurrency": 16,
"successful_requests": 32846,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 1094.415,
"successful_latency_ms": {
"p50": 10.732,
"p95": 30.846,
"p99": 51.188
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.013,
"concurrency": 16,
"successful_requests": 32672,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 1088.609,
"successful_latency_ms": {
"p50": 11.145,
"p95": 30.074,
"p99": 53.423
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.015,
"concurrency": 16,
"successful_requests": 33502,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 1116.177,
"successful_latency_ms": {
"p50": 11.142,
"p95": 28.815,
"p99": 49.26
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 5.0,
"elapsed_seconds": 5.014,
"concurrency": 16,
"successful_requests": 5472,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 1091.286,
"successful_latency_ms": {
"p50": 11.367,
"p95": 33.413,
"p99": 54.849
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.033,
"concurrency": 16,
"successful_requests": 34199,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 1138.726,
"successful_latency_ms": {
"p50": 11.784,
"p95": 29.272,
"p99": 38.508
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.015,
"concurrency": 16,
"successful_requests": 35418,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 1180.016,
"successful_latency_ms": {
"p50": 11.711,
"p95": 26.361,
"p99": 36.002
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.012,
"concurrency": 16,
"successful_requests": 34848,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 1161.122,
"successful_latency_ms": {
"p50": 11.745,
"p95": 28.074,
"p99": 36.688
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 5.0,
"elapsed_seconds": 5.014,
"concurrency": 16,
"successful_requests": 5457,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 1088.404,
"successful_latency_ms": {
"p50": 11.999,
"p95": 31.015,
"p99": 44.748
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
File diff suppressed because it is too large Load Diff
+26
View File
@@ -0,0 +1,26 @@
<svg xmlns="http://www.w3.org/2000/svg" width="900" height="420" viewBox="0 0 900 420">
<rect width="100%" height="100%" fill="white"/>
<text x="450" y="28" text-anchor="middle" font-family="sans-serif" font-size="18">API scaling: average successful RPS</text>
<line x1="60" y1="360" x2="840" y2="360" stroke="#333"/>
<rect x="60.0" y="305.0" width="78" height="55.0" fill="#3572A5"/>
<text x="99.0" y="297.0" text-anchor="middle" font-family="sans-serif" font-size="12">212.5</text>
<text x="99.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">fixed / 1</text>
<rect x="177.0" y="305.0" width="78" height="55.0" fill="#3572A5"/>
<text x="216.0" y="297.0" text-anchor="middle" font-family="sans-serif" font-size="12">212.7</text>
<text x="216.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">fixed / 2</text>
<rect x="294.0" y="319.6" width="78" height="40.4" fill="#3572A5"/>
<text x="333.0" y="311.6" text-anchor="middle" font-family="sans-serif" font-size="12">156.3</text>
<text x="333.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">fixed / 4</text>
<rect x="411.0" y="305.3" width="78" height="54.7" fill="#E38B29"/>
<text x="450.0" y="297.3" text-anchor="middle" font-family="sans-serif" font-size="12">211.7</text>
<text x="450.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">growth / 1</text>
<rect x="528.0" y="203.5" width="78" height="156.5" fill="#E38B29"/>
<text x="567.0" y="195.5" text-anchor="middle" font-family="sans-serif" font-size="12">605.0</text>
<text x="567.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">growth / 2</text>
<rect x="645.0" y="75.6" width="78" height="284.4" fill="#E38B29"/>
<text x="684.0" y="67.6" text-anchor="middle" font-family="sans-serif" font-size="12">1099.7</text>
<text x="684.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">growth / 4</text>
<rect x="762.0" y="60.0" width="78" height="300.0" fill="#E38B29"/>
<text x="801.0" y="52.0" text-anchor="middle" font-family="sans-serif" font-size="12">1160.0</text>
<text x="801.0" y="380" text-anchor="middle" font-family="sans-serif" font-size="12">growth / 8</text>
</svg>

After

Width:  |  Height:  |  Size: 2.2 KiB

+35
View File
@@ -0,0 +1,35 @@
global
log stdout format raw local0
maxconn 1024
defaults
log global
mode http
option httplog
option http-keep-alive
timeout connect 2s
timeout client 15s
timeout server 15s
timeout http-request 5s
timeout check 1s
retries 2
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s
hold nx 2s
frontend messenger
bind :8080
default_backend api_replicas
backend api_replicas
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
option redispatch
default-server inter 1s fall 2 rise 2
server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr libc,none
+43
View File
@@ -0,0 +1,43 @@
CREATE TABLE users (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
username varchar(32) NOT NULL UNIQUE,
display_name varchar(100) NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
title varchar(100) NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
joined_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
text varchar(4000) NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0)
);
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
+16
View File
@@ -0,0 +1,16 @@
CREATE TABLE idempotency_records (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method varchar(8) NOT NULL,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
idempotency_key varchar(128) NOT NULL,
request_hash bytea NOT NULL,
response jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, method, chat_id, idempotency_key),
CONSTRAINT idempotency_method_post CHECK (method = 'POST'),
CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32),
CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at)
);
CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at);
+3
View File
@@ -0,0 +1,3 @@
psycopg[binary]==3.2.10
psycopg-pool==3.3.3
redis==5.2.1
+27
View File
@@ -0,0 +1,27 @@
#!/bin/sh
set -eu
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
SECRETS_DIR="$ROOT/secrets"
PASSWORD_FILE="$SECRETS_DIR/postgres_password"
DATABASE_URL_FILE="$SECRETS_DIR/database_url"
if [ -e "$PASSWORD_FILE" ] || [ -e "$DATABASE_URL_FILE" ]; then
printf '%s\n' 'Secret files already exist; refusing to overwrite them.' >&2
exit 1
fi
command -v openssl >/dev/null 2>&1 || {
printf '%s\n' 'openssl is required to generate local credentials.' >&2
exit 1
}
mkdir -p "$SECRETS_DIR"
chmod 700 "$SECRETS_DIR"
umask 077
password=$(openssl rand -hex 32)
printf '%s' "$password" > "$PASSWORD_FILE"
printf 'postgresql://messenger:%s@postgres:5432/messenger' "$password" > "$DATABASE_URL_FILE"
chmod 644 "$PASSWORD_FILE" "$DATABASE_URL_FILE"
unset password
printf '%s\n' 'Local PostgreSQL secret files created.'
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"
docker compose --env-file .env restart postgres
docker compose --env-file .env up -d --wait --wait-timeout 60 postgres
docker compose --env-file .env restart api load_balancer
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
set -eu
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"
docker compose --env-file .env up -d --scale api=2 --wait --wait-timeout 60
containers=$(docker compose --env-file .env ps -q api)
if [ -z "$containers" ]; then
printf '%s\n' 'No running API containers found.' >&2
exit 1
fi
for container in $containers; do
docker stop --time 15 "$container" >/dev/null
docker start "$container" >/dev/null
attempts=0
while [ "$attempts" -lt 40 ]; do
state=$(docker inspect --format '{{.State.Health.Status}}' "$container")
if [ "$state" = healthy ]; then
printf 'PASS: API container %s restarted and is healthy\n' "$container"
break
fi
if [ "$state" = unhealthy ] || [ "$state" = none ]; then
printf 'API container %s failed health check (state: %s)\n' "$container" "$state" >&2
exit 1
fi
attempts=$((attempts + 1))
sleep 1
done
if [ "$attempts" -eq 40 ]; then
printf 'API container %s did not become healthy in 40 seconds\n' "$container" >&2
exit 1
fi
done
+32
View File
@@ -0,0 +1,32 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"
target_container=
for container in $(docker compose --env-file .env ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
target_container=$container
break
fi
done
if [ -z "$target_container" ]; then
printf '%s\n' 'No running API container matches TARGET_INSTANCE_ID.' >&2
exit 1
fi
if [ -n "${FAILOVER_EVIDENCE_FILE:-}" ]; then
case "$FAILOVER_EVIDENCE_FILE" in
/*) evidence_file=$FAILOVER_EVIDENCE_FILE ;;
*) evidence_file="$ROOT/$FAILOVER_EVIDENCE_FILE" ;;
esac
printf '%s target_instance=%s container=%s\n' \
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$TARGET_INSTANCE_ID" "$target_container" \
>> "$evidence_file"
fi
docker stop --time 2 "$target_container" >/dev/null
+1
View File
@@ -0,0 +1 @@
"""Messenger API application package."""
+39
View File
@@ -0,0 +1,39 @@
import logging
import os
import signal
import threading
from functools import partial
from http.server import ThreadingHTTPServer
from .config import APP_HOST, APP_PORT
from .handlers import MessengerHandler
from .store import Store
LOGGER = logging.getLogger("messenger")
def run_server():
store = Store()
store.open()
try:
handler = partial(MessengerHandler, store=store)
server = ThreadingHTTPServer((APP_HOST, APP_PORT), handler)
def handle_shutdown(signum, frame):
LOGGER.info("received signal %s, shutting down gracefully", signum)
threading.Thread(target=server.shutdown, daemon=True).start()
signal.signal(signal.SIGTERM, handle_shutdown)
signal.signal(signal.SIGINT, handle_shutdown)
LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT)
try:
server.serve_forever()
finally:
server.server_close()
finally:
store.close()
def main():
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
run_server()
+29
View File
@@ -0,0 +1,29 @@
import os
import re
import socket
import uuid
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
APP_PORT = int(os.getenv("APP_PORT", "8080"))
INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12])
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000"))
DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000"))
DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16"))
DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5"))
MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128"))
MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576"))
USER_RE = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
if SESSION_TTL_SECONDS < 1:
raise RuntimeError("SESSION_TTL_SECONDS must be positive")
if IDEMPOTENCY_TTL_SECONDS < 86400:
raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400")
if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1:
raise RuntimeError("Database timeouts must be positive")
if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0:
raise RuntimeError("Database pool settings must be positive")
if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1:
raise RuntimeError("HTTP limits must be positive")
+18
View File
@@ -0,0 +1,18 @@
from typing import Dict, Optional
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None):
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
class CacheUnavailableError(Exception):
pass
class IdempotencyConflictError(Exception):
pass
+350
View File
@@ -0,0 +1,350 @@
import base64
import json
import logging
import threading
import uuid
from http.server import BaseHTTPRequestHandler
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import parse_qs, urlsplit
import psycopg
from psycopg_pool import PoolTimeout
from .config import (
IDEMPOTENCY_KEY_RE,
INSTANCE_ID,
MAX_INFLIGHT_REQUESTS,
MAX_REQUEST_BODY_BYTES,
USER_RE,
)
from .errors import ApiError, CacheUnavailableError, IdempotencyConflictError
from .security import hash_password, isoformat, valid_uuid
from .store import Store
LOGGER = logging.getLogger("messenger.http")
class RequestGuard:
def __init__(self):
self.lock = threading.Lock()
self.inflight = 0
def acquire(self):
with self.lock:
if self.inflight >= MAX_INFLIGHT_REQUESTS:
raise ApiError(503, "unavailable", "Server is busy")
self.inflight += 1
def release(self):
with self.lock:
self.inflight = max(0, self.inflight - 1)
REQUEST_GUARD = RequestGuard()
class MessengerHandler(BaseHTTPRequestHandler):
def __init__(self, request, client_address, server, store: Store):
self.store = store
super().__init__(request, client_address, server)
def log_message(self, format: str, *args):
return
def do_GET(self):
self._dispatch("GET")
def do_POST(self):
self._dispatch("POST")
def do_DELETE(self):
self._dispatch("DELETE")
def _dispatch(self, method: str):
request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4())
acquired = False
try:
REQUEST_GUARD.acquire()
acquired = True
parsed = urlsplit(self.path)
path = parsed.path
query = parse_qs(parsed.query, keep_blank_values=True)
if path == "/health/live" and method == "GET":
self._send_json(200, {"status": "ok"}, request_id)
elif path == "/health/ready" and method == "GET":
try:
self.store.ready()
except (psycopg.Error, PoolTimeout, OSError):
raise ApiError(503, "unavailable", "Database is unavailable")
self._send_json(200, {"status": "ok"}, request_id)
elif path == "/api/v1/users" and method == "POST":
self._handle_register(request_id)
elif path == "/api/v1/auth/sessions" and method == "POST":
self._handle_create_session(request_id)
elif path == "/api/v1/auth/sessions/current" and method == "DELETE":
self._handle_revoke_session(request_id)
elif path == "/api/v1/users/me" and method == "GET":
self._send_json(200, self._get_actor(request_id), request_id)
elif path == "/api/v1/chats" and method == "POST":
self._handle_create_chat(request_id)
elif path == "/api/v1/chats" and method == "GET":
self._handle_list_chats(query, request_id)
elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"):
chat_id = path.rsplit("/", 2)[1]
if method == "POST":
self._handle_create_message(chat_id, request_id)
else:
self._handle_list_messages(chat_id, query, request_id)
elif path.startswith("/api/v1/chats/") and method == "GET":
self._handle_get_chat(path.rsplit("/", 1)[1], request_id)
else:
raise ApiError(404, "not_found", "Object not found")
except ApiError as exc:
self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers)
except CacheUnavailableError:
self._send_error(503, "unavailable", "Session cache is unavailable", request_id)
except IdempotencyConflictError:
self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id)
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError):
self._send_error(503, "unavailable", "Database is unavailable", request_id)
except psycopg.errors.UniqueViolation:
self._send_error(409, "conflict", "Username already exists", request_id)
except psycopg.errors.ForeignKeyViolation:
self._send_error(400, "invalid_request", "Unknown member", request_id)
except Exception as exc:
LOGGER.error("request %s failed with %s", request_id, type(exc).__name__)
self._send_error(500, "internal_error", "Internal server error", request_id)
finally:
if acquired:
REQUEST_GUARD.release()
def _read_json(self, request_id: str) -> Any:
if self.headers.get_content_type() != "application/json":
raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json")
length = self.headers.get("Content-Length")
if length is None:
raise ApiError(400, "invalid_request", "Missing Content-Length")
try:
size = int(length)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size < 0:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size > MAX_REQUEST_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Request body too large")
raw = self.rfile.read(size)
if not raw:
raise ApiError(400, "invalid_request", "Empty body")
try:
return json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
raise ApiError(400, "invalid_request", "Malformed JSON")
def _get_actor(self, request_id: str) -> Dict[str, Any]:
header = self.headers.get("Authorization", "")
scheme, separator, token = header.partition(" ")
if scheme.lower() != "bearer" or not separator or not token or " " in token:
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
user = self.store.session_user(token)
if user is None:
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
return user
def _handle_register(self, request_id: str):
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}:
raise ApiError(400, "invalid_request", "Expected username, display_name and password")
username = body["username"]
display_name = body["display_name"]
password = body["password"]
if not isinstance(username, str) or not USER_RE.fullmatch(username):
raise ApiError(400, "invalid_request", "Invalid username")
if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100:
raise ApiError(400, "invalid_request", "Invalid display_name")
if not isinstance(password, str) or not (12 <= len(password) <= 128):
raise ApiError(400, "invalid_request", "Invalid password")
user = self.store.create_user(username, display_name, hash_password(password))
self._send_json(201, user, request_id)
def _basic_credentials(self, request_id: str) -> Tuple[str, str]:
value = self.headers.get("Authorization", "")
scheme, separator, encoded = value.partition(" ")
challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
if scheme.lower() != "basic" or not separator or not encoded:
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
try:
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
except (ValueError, UnicodeError):
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
return username, password
def _handle_create_session(self, request_id: str):
username, password = self._basic_credentials(request_id)
user = self.store.authenticate(username, password)
if user is None:
raise ApiError(
401,
"unauthorized",
"Invalid credentials",
{"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
)
token, expires_at = self.store.create_session(user["id"])
self._send_json(
201,
{"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user},
request_id,
)
def _handle_revoke_session(self, request_id: str):
self._get_actor(request_id)
token = self.headers["Authorization"].partition(" ")[2]
if not self.store.revoke_session(token):
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
self.send_response(204)
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
self.end_headers()
def _handle_create_chat(self, request_id: str):
actor = self._get_actor(request_id)
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body) != {"title", "member_ids"}:
raise ApiError(400, "invalid_request", "Expected title and member_ids")
title, member_ids = body["title"], body["member_ids"]
if not isinstance(title, str) or not title.strip() or len(title) > 100:
raise ApiError(400, "invalid_request", "Invalid title")
if (
not isinstance(member_ids, list)
or not 1 <= len(member_ids) <= 100
or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids)
or len(member_ids) != len(set(member_ids))
or actor["id"] not in member_ids
):
raise ApiError(400, "invalid_request", "Invalid member_ids")
chat = self.store.create_chat(title, member_ids, actor["id"])
self._send_json(201, chat, request_id)
def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str):
actor = self._get_actor(request_id)
try:
limit = int(query.get("limit", ["50"])[0])
except ValueError:
raise ApiError(400, "invalid_request", "Invalid limit")
if not 1 <= limit <= 100:
raise ApiError(400, "invalid_request", "Invalid limit")
cursor = query.get("cursor", [None])[0]
try:
chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid cursor")
self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id)
def _handle_get_chat(self, chat_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
chat = self.store.get_chat(chat_id, actor["id"])
if chat is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(200, chat, request_id)
def _handle_create_message(self, chat_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
chat_id = str(uuid.UUID(chat_id))
idempotency_key = self.headers.get("Idempotency-Key", "")
if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key")
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body) != {"text"}:
raise ApiError(400, "invalid_request", "Expected text")
text = body["text"]
if not isinstance(text, str) or not text.strip() or len(text) > 4000:
raise ApiError(400, "invalid_request", "Invalid message text")
canonical_body = json.dumps(
{"text": text},
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
)
message = self.store.create_idempotent_message(
chat_id,
actor["id"],
text,
idempotency_key,
canonical_body,
)
if message is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(201, message, request_id)
def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
try:
limit = int(query.get("limit", ["50"])[0])
except ValueError:
raise ApiError(400, "invalid_request", "Invalid limit")
if not 1 <= limit <= 100:
raise ApiError(400, "invalid_request", "Invalid limit")
cursor = query.get("cursor", [None])[0]
try:
result = self.store.list_messages(chat_id, actor["id"], limit, cursor)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid cursor")
if result is None:
raise ApiError(404, "not_found", "Chat not found")
items, next_cursor = result
self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id)
def _send_json(
self,
status: int,
payload: Dict[str, Any],
request_id: str,
headers: Optional[Dict[str, str]] = None,
):
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
if headers:
for key, value in headers.items():
self.send_header(key, value)
self.end_headers()
self.wfile.write(body)
def _send_error(
self,
status: int,
code: str,
message: str,
request_id: str,
extra_headers: Optional[Dict[str, str]] = None,
):
self._send_json(
status,
{"error": {"code": code, "message": message, "request_id": request_id}},
request_id,
extra_headers,
)
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env python3
import logging
import sys
from pathlib import Path
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.store import Store
else:
from .store import Store
def main():
logging.basicConfig(level="INFO")
store = Store()
store.open()
try:
store.migrate()
finally:
store.close()
if __name__ == "__main__":
main()
+53
View File
@@ -0,0 +1,53 @@
import base64
import hashlib
import hmac
import secrets
import uuid
from datetime import datetime, timezone
def isoformat(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def valid_uuid(value: str) -> bool:
try:
uuid.UUID(value)
return True
except (ValueError, TypeError, AttributeError):
return False
def hash_password(password: str) -> str:
salt = secrets.token_bytes(16)
derived = hashlib.scrypt(
password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024
)
salt_text = base64.urlsafe_b64encode(salt).decode("ascii")
hash_text = base64.urlsafe_b64encode(derived).decode("ascii")
return f"scrypt$32768$8$1${salt_text}${hash_text}"
def verify_password(password: str, stored: str) -> bool:
try:
scheme, n, r, p, salt_text, hash_text = stored.split("$")
if scheme != "scrypt":
return False
salt = base64.urlsafe_b64decode(salt_text.encode("ascii"))
expected = base64.urlsafe_b64decode(hash_text.encode("ascii"))
actual = hashlib.scrypt(
password.encode("utf-8"),
salt=salt,
n=int(n),
r=int(r),
p=int(p),
dklen=len(expected),
maxmem=64 * 1024 * 1024,
)
return hmac.compare_digest(actual, expected)
except (ValueError, TypeError, UnicodeError):
return False
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.app import main
else:
from .app import main
if __name__ == "__main__":
main()
+394
View File
@@ -0,0 +1,394 @@
import base64
import hashlib
import hmac
import json
import logging
import math
import os
import secrets
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import redis
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from .config import (
DB_LOCK_TIMEOUT_MS,
DB_POOL_MAX_SIZE,
DB_POOL_TIMEOUT_SECONDS,
DB_STATEMENT_TIMEOUT_MS,
IDEMPOTENCY_TTL_SECONDS,
SESSION_TTL_SECONDS,
)
from .errors import CacheUnavailableError, IdempotencyConflictError
from .security import isoformat, utc_now, valid_uuid, verify_password
LOGGER = logging.getLogger("messenger.store")
def database_url() -> str:
value = os.getenv("DATABASE_URL")
if value:
return value
file_path = os.getenv("DATABASE_URL_FILE")
if file_path:
return Path(file_path).read_text(encoding="utf-8").strip()
raise RuntimeError("Set DATABASE_URL or DATABASE_URL_FILE")
class Store:
"""PostgreSQL source of truth and optional Redis session lookup cache."""
def __init__(self):
self.pool = ConnectionPool(
conninfo=database_url(),
min_size=1,
max_size=DB_POOL_MAX_SIZE,
timeout=DB_POOL_TIMEOUT_SECONDS,
kwargs={
"row_factory": dict_row,
"connect_timeout": 5,
"options": (
f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} "
f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} "
"-c idle_in_transaction_session_timeout=10000"
),
},
open=False,
)
cache_url = os.getenv("CACHE_URL")
self.cache = (
redis.Redis.from_url(
cache_url,
decode_responses=True,
socket_connect_timeout=0.25,
socket_timeout=0.25,
retry_on_timeout=False,
max_connections=16,
)
if cache_url
else None
)
def open(self):
self.pool.open(wait=True, timeout=10)
def close(self):
if self.cache is not None:
self.cache.close()
self.pool.close()
def ready(self) -> bool:
with self.pool.connection(timeout=2) as conn:
conn.execute("SELECT 1")
return True
def migrate(self):
migrations = Path(__file__).resolve().parents[1] / "migrations"
with self.pool.connection() as conn:
with conn.transaction():
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"
)
conn.execute("SELECT pg_advisory_xact_lock(2741662026)")
applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")}
for migration in sorted(migrations.glob("*.sql")):
if migration.name in applied:
continue
conn.execute(migration.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,))
@staticmethod
def user(row: Dict[str, Any]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": isoformat(row["created_at"]),
}
@staticmethod
def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": member_ids,
"created_at": isoformat(row["created_at"]),
}
@staticmethod
def message(row: Dict[str, Any]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": isoformat(row["created_at"]),
}
def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO users (username, display_name, password_hash) "
"VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at",
(username, display_name, password_hash),
).fetchone()
return self.user(row)
def get_user(self, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(user_id,),
).fetchone()
return self.user(row) if row else None
def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at, password_hash "
"FROM users WHERE username = %s",
(username,),
).fetchone()
if not row or not verify_password(password, row["password_hash"]):
return None
return self.user(row)
def create_session(self, user_id: str) -> Tuple[str, datetime]:
token = secrets.token_urlsafe(32)
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
token_hash = hashlib.sha256(token.encode("ascii")).digest()
with self.pool.connection() as conn:
conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(token_hash, user_id, expires_at),
)
return token, expires_at
@staticmethod
def _session_lock_key(token_hash: bytes) -> int:
return int.from_bytes(token_hash[:8], byteorder="big", signed=True)
def session_user(self, token: str) -> Optional[Dict[str, Any]]:
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
cache_key = "messenger:session:" + token_hash.hex()
if self.cache is not None:
try:
cached = self.cache.get(cache_key)
if cached:
return json.loads(cached)
except (redis.RedisError, ValueError, TypeError):
LOGGER.warning("session cache read failed; falling back to PostgreSQL")
with self.pool.connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
row = conn.execute(
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at "
"FROM sessions s JOIN users u ON u.id = s.user_id "
"WHERE s.token_hash = %s AND s.expires_at > now()",
(token_hash,),
).fetchone()
if not row:
return None
user = self.user(row)
if self.cache is not None:
ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000)
if ttl_ms > 0:
try:
self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms)
except redis.RedisError:
LOGGER.warning("session cache write failed; PostgreSQL remains authoritative")
return user
def revoke_session(self, token: str) -> bool:
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
with self.pool.connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
if self.cache is not None:
try:
self.cache.delete("messenger:session:" + token_hash.hex())
except redis.RedisError as exc:
raise CacheUnavailableError from exc
result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,))
return result.rowcount == 1
def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]:
if actor_id not in member_ids or len(member_ids) != len(set(member_ids)):
raise ValueError("invalid members")
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at",
(title,),
).fetchone()
for member_id in member_ids:
conn.execute(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
(row["id"], member_id),
)
return self.chat(row, member_ids)
def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members m ON m.chat_id = c.id "
"WHERE c.id = %s AND m.user_id = %s",
(chat_id, user_id),
).fetchone()
if not row:
return None
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
(chat_id,),
).fetchall()
return self.chat(row, [str(item["user_id"]) for item in members])
def list_chats(
self, user_id: str, limit: int, cursor: Optional[str]
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
where = "own.user_id = %s"
params: List[Any] = [user_id]
if cursor is not None:
try:
raw = base64.b64decode(
cursor.encode("ascii") + b"=" * (-len(cursor) % 4),
altchars=b"-_",
validate=True,
)
cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2)
if cursor_user != user_id or not valid_uuid(chat_id):
raise ValueError("cursor scope")
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
if created_at.tzinfo is None or created_at.utcoffset() is None:
raise ValueError("cursor timestamp must include a timezone")
where += " AND (c.created_at, c.id) > (%s, %s)"
params.extend((created_at, chat_id))
except (ValueError, UnicodeError, TypeError) as exc:
raise ValueError("invalid cursor") from exc
params.append(limit + 1)
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members own ON own.chat_id = c.id "
f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s",
params,
).fetchall()
has_more = len(rows) > limit
page = rows[:limit]
result = []
for row in page:
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
(row["id"],),
).fetchall()
result.append(self.chat(row, [str(item["user_id"]) for item in members]))
next_cursor = None
if has_more and page:
last = page[-1]
cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
return result, next_cursor
def create_idempotent_message(
self,
chat_id: str,
sender_id: str,
text: str,
idempotency_key: str,
canonical_body: str,
) -> Optional[Dict[str, Any]]:
request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest()
lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8")
lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True)
with self.pool.connection() as conn:
with conn.transaction():
member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, sender_id),
).fetchone()
if not member:
return None
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,))
existing = conn.execute(
"SELECT request_hash, response, expires_at "
"FROM idempotency_records "
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s "
"FOR UPDATE",
(sender_id, chat_id, idempotency_key),
).fetchone()
if existing and existing["expires_at"] > utc_now():
if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash):
raise IdempotencyConflictError
return existing["response"]
if existing:
conn.execute(
"DELETE FROM idempotency_records "
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s",
(sender_id, chat_id, idempotency_key),
)
row = conn.execute(
"INSERT INTO messages (chat_id, sender_id, text) "
"VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at",
(chat_id, sender_id, text),
).fetchone()
message = self.message(row)
conn.execute(
"INSERT INTO idempotency_records "
"(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) "
"VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)",
(
sender_id,
chat_id,
idempotency_key,
request_hash,
json.dumps(message, ensure_ascii=False, separators=(",", ":")),
utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS),
),
)
return message
def list_messages(
self, chat_id: str, user_id: str, limit: int, cursor: Optional[str]
) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]:
with self.pool.connection() as conn:
is_member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone()
if not is_member:
return None
where = "chat_id = %s"
params: List[Any] = [chat_id]
if cursor:
try:
raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4))
decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2)
if decoded_chat != chat_id or not valid_uuid(message_id):
raise ValueError("cursor scope")
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
where += " AND (created_at, id) > (%s, %s)"
params.extend((created_at, message_id))
except (ValueError, UnicodeError, TypeError) as exc:
raise ValueError("invalid cursor") from exc
params.append(limit + 1)
rows = conn.execute(
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
f"WHERE {where} ORDER BY created_at, id LIMIT %s",
params,
).fetchall()
has_more = len(rows) > limit
page = rows[:limit]
next_cursor = None
if has_more and page:
last = page[-1]
cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
return [self.message(row) for row in page], next_cursor
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Prove a concurrent idempotency-key race through the load balancer."""
from concurrent.futures import ThreadPoolExecutor
import threading
import sys
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, arguments, check, run
def main():
args = arguments("Concurrent shared-idempotency integration check")
client = Client(args.base_url, args.ca_file)
alice, _, _, chat = client.bootstrap()
key = "race-" + uuid.uuid4().hex
text = "concurrent-" + uuid.uuid4().hex
count = 12
barrier = threading.Barrier(count)
def send_once(_):
worker = Client(args.base_url, args.ca_file)
barrier.wait(timeout=10)
return worker.message(alice, chat, text, key=key)
with ThreadPoolExecutor(max_workers=count) as pool:
responses = list(pool.map(send_once, range(count)))
instance_ids = {response["headers"]["X-Instance-Id"] for response in responses}
messages = [response["json"] for response in responses]
check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas")
check(all(message == messages[0] for message in messages), "same-key requests returned different messages")
page = client.call(
"GET",
"/api/v1/chats/" + chat["id"] + "/messages?limit=100",
actor=alice,
schema="MessagePage",
)["json"]
matches = [message for message in page["items"] if message["text"] == text]
check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages")
print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message")
if __name__ == "__main__":
run(main)
+385
View File
@@ -0,0 +1,385 @@
#!/usr/bin/env python3
"""Repeatable 1/2/4 replica throughput experiment with Docker resource samples."""
import argparse
import json
import math
import platform
from pathlib import Path
import re
import subprocess
import sys
import tempfile
import time
import hashlib
ROOT = Path(__file__).resolve().parents[1]
LOAD_TEST = ROOT / "tests" / "load.py"
OUTPUT_DIR = ROOT / "evidence" / "track1"
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def run(command, *, env=None, capture=True):
return subprocess.run(
command,
cwd=ROOT,
env=env,
check=True,
text=True,
stdout=subprocess.PIPE if capture else None,
stderr=subprocess.PIPE if capture else None,
)
def compose_environment(cpu_limit, memory_limit):
import os
env = os.environ.copy()
env["API_CPU_LIMIT"] = cpu_limit
env["API_MEMORY_LIMIT"] = memory_limit
return env
def database_counts():
query = (
"SELECT json_build_object("
"'users', (SELECT count(*) FROM users), "
"'chats', (SELECT count(*) FROM chats), "
"'messages', (SELECT count(*) FROM messages), "
"'sessions', (SELECT count(*) FROM sessions), "
"'idempotency_records', (SELECT count(*) FROM idempotency_records)"
")::text"
)
result = run(
COMPOSE
+ [
"exec",
"-T",
"postgres",
"psql",
"-X",
"-qAt",
"-U",
"messenger",
"-d",
"messenger",
"-c",
query,
]
)
return json.loads(result.stdout)
def create_database_snapshot(destination):
with destination.open("wb") as dump:
result = subprocess.run(
COMPOSE + ["exec", "-T", "postgres", "pg_dump", "-Fc", "-U", "messenger", "-d", "messenger"],
cwd=ROOT,
check=False,
stdout=dump,
stderr=subprocess.PIPE,
)
if result.returncode:
raise RuntimeError("failed to create PostgreSQL baseline snapshot: " + result.stderr.decode(errors="replace"))
digest_builder = hashlib.sha256()
with destination.open("rb") as dump:
for chunk in iter(lambda: dump.read(1024 * 1024), b""):
digest_builder.update(chunk)
digest = digest_builder.hexdigest()
return digest, database_counts()
def restore_database_snapshot(snapshot, baseline_counts, api_replicas, cpu_limit, memory_limit):
env = compose_environment(cpu_limit, memory_limit)
run(COMPOSE + ["stop", "api"])
with snapshot.open("rb") as dump:
result = subprocess.run(
COMPOSE
+ [
"exec",
"-T",
"postgres",
"pg_restore",
"-U",
"messenger",
"--clean",
"--if-exists",
"--no-owner",
"--no-acl",
"--exit-on-error",
"--dbname=messenger",
],
cwd=ROOT,
check=False,
stdin=dump,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
if result.returncode:
raise RuntimeError("failed to restore PostgreSQL baseline snapshot: " + result.stderr.decode(errors="replace"))
restored_counts = database_counts()
if restored_counts != baseline_counts:
raise RuntimeError(
"restored PostgreSQL baseline has different row counts: "
+ json.dumps(restored_counts, sort_keys=True)
)
run(
COMPOSE
+ [
"up",
"-d",
"--no-deps",
"--build",
"--force-recreate",
"--scale",
f"api={api_replicas}",
"--wait",
"--wait-timeout",
"120",
"api",
],
env=env,
)
def docker_stats():
containers = run(
["docker", "compose", "--env-file", ".env", "ps", "-q"],
).stdout.splitlines()
if not containers:
return []
result = run(
[
"docker",
"stats",
"--no-stream",
"--format",
"{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}",
*containers,
]
)
samples = []
for line in result.stdout.splitlines():
name, cpu, memory = line.split("|", 2)
used = memory.split("/", 1)[0].strip()
samples.append(
{
"container": name,
"service": "api" if "-api-" in name else "postgres" if "-postgres-" in name else "load_balancer",
"cpu_percent": float(cpu.rstrip("%")),
"memory_bytes": parse_bytes(used),
}
)
return samples
def parse_bytes(value):
match = re.fullmatch(r"([0-9.]+)\s*([KMG]i?B|B)", value)
if not match:
return None
amount = float(match.group(1))
unit = match.group(2)
power = {"B": 0, "KB": 1, "KiB": 1, "MB": 2, "MiB": 2, "GB": 3, "GiB": 3}[unit]
return round(amount * 1024**power)
def run_load(name, duration, concurrency, repetition, base_url):
output = OUTPUT_DIR / f"{name}-run-{repetition}.json"
command = [
sys.executable,
str(LOAD_TEST),
"--base-url",
base_url,
"--duration",
str(duration),
"--concurrency",
str(concurrency),
"--output",
str(output.relative_to(ROOT)),
]
process = subprocess.Popen(
command,
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
samples = []
while process.poll() is None:
samples.extend(docker_stats())
time.sleep(1)
stdout, stderr = process.communicate()
if process.returncode:
raise RuntimeError(f"load run failed:\n{stdout}\n{stderr}")
metrics = json.loads(output.read_text(encoding="utf-8"))
metrics["resource_samples"] = samples
return metrics
def system_info():
def command_output(command):
try:
return run(command).stdout.strip()
except (OSError, subprocess.CalledProcessError):
return "unavailable"
return {
"platform": platform.platform(),
"processor": platform.processor() or "unavailable",
"cpu_count": __import__("os").cpu_count(),
"host_memory_bytes": command_output(["sysctl", "-n", "hw.memsize"]),
"docker_server": command_output(["docker", "version", "--format", "{{.Server.Version}}"]),
"docker_compose": command_output(["docker", "compose", "version", "--short"]),
}
def configure_limits(count, phase):
if phase == "fixed":
return f"{1 / count:.4f}", f"{512 // count}M"
return "1.0", "512M"
def experiment_case(count, phase, args, snapshot, snapshot_hash, baseline_counts):
cpu_limit, memory_limit = configure_limits(count, phase)
restore_database_snapshot(snapshot, baseline_counts, count, cpu_limit, memory_limit)
run_load(f"{phase}-{count}-warmup", args.warmup, args.concurrency, 0, args.base_url)
warmup_counts = database_counts()
if warmup_counts == baseline_counts:
raise RuntimeError("warm-up did not create synthetic workload data")
repetitions = []
for repetition in range(1, args.repetitions + 1):
restore_database_snapshot(snapshot, baseline_counts, count, cpu_limit, memory_limit)
repetitions.append(run_load(f"{phase}-{count}", args.duration, args.concurrency, repetition, args.base_url))
if database_counts() == baseline_counts:
raise RuntimeError(f"{phase}-{count} repetition {repetition} did not write benchmark data")
return {
"phase": phase,
"api_replicas": count,
"per_replica_cpu_limit": cpu_limit,
"per_replica_memory_limit": memory_limit,
"total_api_cpu_limit": round(float(cpu_limit) * count, 4),
"total_api_memory_limit_mib": int(memory_limit[:-1]) * count,
"warmup_seconds": args.warmup,
"baseline_row_counts": baseline_counts,
"baseline_snapshot_sha256": snapshot_hash,
"each_repetition_restored_same_baseline": True,
"repetitions": repetitions,
}
def make_graph(cases):
width, height = 900, 420
margin = 60
values = [
(
f"{case['phase']} / {case['api_replicas']}",
sum(run["successful_rps"] for run in case["repetitions"]) / len(case["repetitions"]),
"#3572A5" if case["phase"] == "fixed" else "#E38B29",
)
for case in cases
]
top = max((value for _, value, _ in values), default=1) or 1
graph_height = height - 2 * margin
bar_width = 78
gap = (width - 2 * margin - bar_width * len(values)) / max(1, len(values) - 1)
parts = [
f'<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}">',
'<rect width="100%" height="100%" fill="white"/>',
'<text x="450" y="28" text-anchor="middle" font-family="sans-serif" font-size="18">API scaling: average successful RPS</text>',
f'<line x1="{margin}" y1="{height-margin}" x2="{width-margin}" y2="{height-margin}" stroke="#333"/>',
]
for index, (label, value, color) in enumerate(values):
x = margin + index * (bar_width + gap)
bar_height = value / top * graph_height
y = height - margin - bar_height
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width}" height="{bar_height:.1f}" fill="{color}"/>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{max(48,y-8):.1f}" text-anchor="middle" font-family="sans-serif" font-size="12">{value:.1f}</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{height-margin+20}" text-anchor="middle" font-family="sans-serif" font-size="12">{label}</text>')
parts.append("</svg>")
(OUTPUT_DIR / "throughput.svg").write_text("\n".join(parts) + "\n", encoding="utf-8")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--duration", type=int, default=30)
parser.add_argument("--warmup", type=int, default=5)
parser.add_argument("--concurrency", type=int, default=16)
parser.add_argument("--repetitions", type=int, default=3)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
args = parser.parse_args()
if not 1 <= args.duration <= 3600 or not 1 <= args.warmup <= 600:
parser.error("duration and warmup must be positive and bounded")
if not 1 <= args.concurrency <= 128 or not 1 <= args.repetitions <= 10:
parser.error("concurrency or repetitions out of range")
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
results_file = OUTPUT_DIR / "results.json"
with tempfile.TemporaryDirectory(prefix="lab3-track1-") as temporary_directory:
snapshot = Path(temporary_directory) / "baseline.dump"
run(COMPOSE + ["stop", "api"])
baseline_counts = None
try:
snapshot_hash, baseline_counts = create_database_snapshot(snapshot)
cases = []
for phase, replica_counts in (("fixed", (1, 2, 4)), ("growth", (1, 2, 4, 8))):
for count in replica_counts:
cases.append(
experiment_case(count, phase, args, snapshot, snapshot_hash, baseline_counts)
)
summary = {
"method": (
"closed-loop POST message workload via HAProxy; same concurrency and per-run "
"restored PostgreSQL custom-format baseline snapshot"
),
"database_baseline": {
"snapshot_sha256": snapshot_hash,
"row_counts": baseline_counts,
"restore_before_each_warmup_and_measured_repetition": True,
"state_after_experiment": "baseline restored; all synthetic benchmark writes removed",
},
"system": system_info(),
"database_and_lb_limits": {"postgres": "1 CPU / 1 GiB", "HAProxy": "0.5 CPU / 128 MiB"},
"client_concurrency": args.concurrency,
"requested_duration_seconds": args.duration,
"repetitions_per_case": args.repetitions,
"cases": cases,
}
results_file.write_text(
json.dumps(summary, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
make_graph(cases)
for case in cases:
values = [run["successful_rps"] for run in case["repetitions"]]
deviation = math.sqrt(sum((value - sum(values) / len(values)) ** 2 for value in values) / len(values))
print(
f"{case['phase']:6} replicas={case['api_replicas']} "
f"mean_rps={sum(values) / len(values):.2f} stdev={deviation:.2f}"
)
print(f"Evidence saved under {OUTPUT_DIR.relative_to(ROOT)}")
finally:
if baseline_counts is not None:
restore_database_snapshot(snapshot, baseline_counts, 2, "1.0", "512M")
else:
run(
COMPOSE
+ [
"up",
"-d",
"--no-deps",
"--build",
"--force-recreate",
"--scale",
"api=2",
"--wait",
"--wait-timeout",
"120",
"api",
],
env=compose_environment("1.0", "512M"),
)
if __name__ == "__main__":
main()