diff --git a/.env.example b/.env.example
index d32b9be0..f6739e98 100644
--- a/.env.example
+++ b/.env.example
@@ -3,11 +3,25 @@
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
+LOG_LEVEL=info
+SHUTDOWN_GRACE_SECONDS=10
SESSION_TTL_SECONDS=3600
+CACHE_TTL_SECONDS=60
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
-SESSION_STORE=postgres
+SESSION_STORE=redis
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
+API_CPUS=0.50
+API_MEM=256m
+POSTGRES_CPUS=1.0
+POSTGRES_MEM=512m
+DB_POOL_MAX=4
+LB_DRAIN_SECONDS=2
IDEMPOTENCY_TTL_SECONDS=86400
+
+# Compose interpolates these. Synthetic local values only; replace after copy.
+POSTGRES_PASSWORD=lab2-dev-only-change-me
+DATABASE_URL=postgresql://messenger:lab2-dev-only-change-me@postgres:5432/messenger
+REDIS_URL=redis://redis:6379/0
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index e6e63eb2..b93e3208 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -1,18 +1,18 @@
## Лабораторная 3
-Автор / группа: TODO
-Целевая оценка (3/4/5): TODO
-Трек 5, если заявлен: TODO
+Автор / группа: Булацев / БПИ2301
+Целевая оценка (3/4/5): 5
+Трек 5, если заявлен: 1, цена масштабирования
-Что реализовано и что перенесено: TODO
-Команды запуска из чистого clone: TODO
-Результаты проверок и ссылка на REPORT.md: TODO
-Ограничения / известные ошибки: TODO
-Источники и использование ИИ: TODO
+Что реализовано и что перенесено: API lab2 (PostgreSQL, Redis-сессии, кэш). Добавлены HAProxy, масштабируемый сервис API без внешнего порта, одноразовые миграции, общий Idempotency-Key и X-Instance-Id.
+Команды запуска из чистого clone: `cp .env.example .env`, затем `docker compose --env-file .env up -d --build --scale api=2`, дождаться `http://127.0.0.1:8080/health/ready`. Подробности в REPORT.md.
+Результаты проверок и ссылка на REPORT.md: smoke PASS; persistence PASS; failover — 2 реплики, 1 ошибка, восстановление записи 2.465 с; трек 1 в `evidence/track1/`. См. REPORT.md.
+Ограничения / известные ошибки: один хост; LB, Postgres и Redis остаются единой точкой отказа. На Windows `.sh` не запускается напрямую (`WinError 193`); hook рассчитан на POSIX.
+Источники и использование ИИ: README и контракт курса, документация HAProxy и Compose. Сборку решения вёл агент Cursor (Grok).
-- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
-- [ ] Контракт и публичные тесты не ослаблены.
-- [ ] make check и make test выполнены; приложен реальный вывод.
-- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
-- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
-- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
+- [x] Контракт и публичные тесты не ослаблены.
+- [x] make check и make test выполнены; приложен реальный вывод.
+- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
+- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
+- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..65e21fb0
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,29 @@
+FROM python:3.12.8-slim-bookworm AS runtime
+
+WORKDIR /app
+
+RUN groupadd --system app && useradd --system --gid app --home /app --shell /usr/sbin/nologin app \
+ && apt-get update \
+ && apt-get install -y --no-install-recommends curl \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY app ./app
+COPY migrations ./migrations
+COPY contracts/openapi.json ./contracts/openapi.json
+
+USER app
+
+ENV APP_HOST=0.0.0.0 \
+ APP_PORT=8080 \
+ APP_ENV=lab \
+ PYTHONUNBUFFERED=1
+
+EXPOSE 8080
+
+HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=12 \
+ CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"
+
+CMD ["python", "-m", "app.main"]
diff --git a/Makefile b/Makefile
index 5d86d171..27e00482 100644
--- a/Makefile
+++ b/Makefile
@@ -4,27 +4,45 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
-.PHONY: help check validate test load test-persistence test-failover
+.PHONY: help check validate test load test-persistence test-failover test-student test-infra test-expiry explain evidence-grade4 evidence-track1
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
check:
- $(PYTHON) scripts/check_contract.py
+ $(PYTHON) -X utf8 scripts/check_contract.py
validate:
- $(PYTHON) scripts/validate_openapi.py
+ $(PYTHON) -X utf8 scripts/validate_openapi.py
test:
- $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
+ $(PYTHON) -X utf8 tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
load:
- $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
+ $(PYTHON) -X utf8 tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
- $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
+ $(PYTHON) -X utf8 tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
- $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
+ $(PYTHON) -X utf8 tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
+
+test-student:
+ $(PYTHON) -m pytest student_tests -q -m "not infra and not expiry"
+
+test-infra:
+ RUN_INFRA=1 $(PYTHON) -m pytest student_tests/test_infra.py -q -m infra
+
+test-expiry:
+ $(PYTHON) scripts/run_expiry.py
+
+explain:
+ docker compose --env-file .env exec -T api python -m app.tools.explain
+
+evidence-grade4:
+ $(PYTHON) -X utf8 scripts/grade4_evidence.py
+
+evidence-track1:
+ $(PYTHON) -X utf8 scripts/track1_experiment.py
diff --git a/REPORT.md b/REPORT.md
index c2679005..c8f2f39e 100644
--- a/REPORT.md
+++ b/REPORT.md
@@ -1,37 +1,109 @@
# Отчёт — лабораторная 3
-- Автор, группа: TODO
-- Целевая оценка; выбранный трек 5, если нужен: TODO
-- Commit/PR: TODO после публикации
-- Использованные источники, библиотеки и помощь ИИ: TODO
+- Автор, группа: **Булацев**, **БПИ2301** (ветка `solution/bpi2301-Bulatsev`)
+- Целевая оценка; выбранный трек 5: **5, трек 1 «Цена масштабирования»**
+- Перенос: API lab2 (PostgreSQL, Redis-сессии, кэш истории, миграции). В этой работе добавлены HAProxy, две и более реплики API без внешнего порта, общий `Idempotency-Key` в PostgreSQL и `X-Instance-Id`.
+- Commit/PR: ещё не опубликован
+- Источники и помощь ИИ: контракт и README курса; образы Postgres 16.6, Redis 7.4.2, HAProxy 2.8. Документация HAProxy `server-template` и Docker Compose. Реализацию и прогоны помог собрать агент Cursor (Grok). На защите нужно объяснять схему своими словами.
## Запуск из чистого clone
-TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
+Стенд: Windows 11, CPU 12th Gen Intel Core i5-12450H, Docker Desktop 29.5.3 / Compose 5.1.4, лимит Docker 12 vCPU и 7.62 ГиБ. Образы зафиксированы тегами: `python:3.12.8-slim-bookworm` (сборка `mtusi-messenger:lab3`), `postgres:16.6-alpine`, `redis:7.4.2-alpine`, `haproxy:2.8.14-alpine`.
+
+```sh
+cp .env.example .env
+docker compose --env-file .env config --quiet
+docker compose --env-file .env up -d --build --scale api=2
+```
+
+Дождаться `200` от `http://127.0.0.1:8080/health/ready`. Этот адрес — HAProxy. Реплики API наружу порты не публикуют.
+
+Миграции выполняет одноразовый сервис `migrate` (`python -m app.migrate`) после healthy Postgres и до старта API. Повторный запуск пропускает уже записанные файлы. Дополнительно `pg_advisory_lock` не даёт двум миграторам применить один файл. У API `MIGRATIONS_ON_START=0`.
+
+Проверки:
+
+```sh
+python -X utf8 scripts/check_contract.py
+python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
+python -X utf8 -m pytest student_tests -q -m "not infra and not expiry"
+```
+
+`scripts/restart.sh` и `scripts/stop-one.sh` — POSIX. На Linux: `chmod +x` и `make test-persistence` / `make test-failover`. На этом Windows `CreateProcess` не запускает `.sh` (`WinError 193`); те же команды прогнаны интерпретатором `python3` в контейнере `docker:27-cli` с сокетом того же Docker. Скрипты при этом управляли контейнерами стенда.
+
+Остановка без удаления данных: `docker compose --env-file .env stop`. Учебные тома удаляет только явная команда `docker compose --env-file .env down -v`.
+
+Чтобы освободить `127.0.0.1:8080`, перед запуском был остановлен контейнер `mtusi2026lab2-api-1`. Postgres и Redis lab2 оставались запущены; в срезах `docker stats` их CPU был около 0–4%.
## Архитектура и решения
-TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
+Клиент ходит только в `lb` (HAProxy, `127.0.0.1:8080`). Backend — DNS-имя `api` сервиса Compose: `server-template` на 8 слотов, `resolvers docker` (`127.0.0.11`), `hold valid 1s`. Балансировка `roundrobin`, без sticky session. Сессии лежат в Redis (AOF + RDB), пользователи, чаты, сообщения и результаты идемпотентности — в PostgreSQL. Локальная память реплики не нужна для корректности следующего запроса.
+
+`X-Instance-Id` — hostname контейнера, он не меняется, пока жив процесс. `scripts/stop-one.sh` сравнивает `TARGET_INSTANCE_ID` с `/tmp/instance-id` или hostname и делает `docker stop -t 5` одного контейнера. `restart: unless-stopped` не поднимает его обратно.
+
+`POST /messages` требует `Idempotency-Key` (`[A-Za-z0-9._:-]{1,128}`). Область — пользователь + `POST` + путь чата + ключ. Канонический JSON (`text` и `attachment_ids`, пустой список равен отсутствию поля) хешируется SHA-256. Тот же хеш в течение `IDEMPOTENCY_TTL_SECONDS` (86400) возвращает сохранённый `201` и то же сообщение. Другой текст — `409`. Вставка сообщения и строки результата — одна транзакция; ошибка откатывает обе. Одновременные повторы сериализует `pg_advisory_xact_lock` в Postgres, поэтому две реплики не создают два сообщения. Неуспешный запрос строку не пишет, повтор с тем же ключом безопасен.
+
+Проверки готовности. `GET /health/live` — только процесс. `GET /health/ready` — Postgres и Redis; во время SIGTERM сразу `503`. HAProxy: `inter 1s`, `fall 2`, `rise 1`, `timeout check 2s`. Неготовая реплика выпадает примерно за 2 с. Если контейнер уже исчез, DNS-кэш HAProxy живёт ещё до 1 с. На SIGTERM API перестаёт принимать бизнес-запросы, ready становится 503, процесс завершается через `LB_DRAIN_SECONDS` (2). `stop_grace_period` сервиса — 8 с. Таймауты прокси: connect 2 с, client/server 10 с. HAProxy не повторяет POST.
+
+Соответствие `.env.example`: `DATABASE_URL` и `REDIS_URL` собираются в Compose из `POSTGRES_PASSWORD`; `API_CPUS`, `API_MEM`, `POSTGRES_CPUS`, `DB_POOL_MAX`, `LB_DRAIN_SECONDS`, `IDEMPOTENCY_TTL_SECONDS` интерполируются в `compose.yaml`. Рабочая точка стенда — 2 реплики по 0.50 CPU и 256 МиБ, Postgres 1.0 CPU / 512 МиБ, пул по 4 соединения. `container_name` нет.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
-| Публичный smoke | make test | TODO | TODO |
-| Остальные пункты уровня 3 | TODO | TODO | TODO |
-| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
-| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
+| Контракт offline | `python -X utf8 scripts/check_contract.py` | PASS | `evidence/public-checks.txt` |
+| Smoke через LB | `tests/smoke.py` | PASS, lab3 | `evidence/public-checks.txt` |
+| Две реплики, одна сессия | 12× `GET /health/ready` | Чередование `03c3a0545628` и `b555f8327eee` | наблюдение до отказов; после экспериментов новые id, см. ниже |
+| Общая БД и сессии | persistence hook | PASS, старый токен жив | `evidence/public-checks.txt` |
+| Отказ одной реплики ≤10 с | `tests/resilience.py --mode failover` | 2 реплики до отказа; 109 успешных записей; 1 ошибка; первое подтверждение после hook 2.465 с; дублей нет | `evidence/public-checks.txt` |
+| Остановлен именно целевой контейнер | `docker inspect` после failover | `mtusi2026lab3-api-1` Exited (0), hostname `b555f8327eee`; вторая реплика Up | `evidence/public-checks.txt` |
+| Идемпотентность 16 параллельных запросов | `scripts/grade4_evidence.py` | 16×201, 1 id, в истории 1 копия | `evidence/grade4/summary.json` |
+| Поочерёдный отказ каждой реплики | тот же скрипт | hook 2.373 с и 1.744 с; в последних ответах только живая реплика; 2 и 1 ошибка | `evidence/grade4/summary.json` |
+| Rolling update под нагрузкой | `scripts/rolling-update.sh` во время `tests/load.py` 50 с, concurrency 4 | 8361 успех, 18 ошибок (0.21%), RPS 166.9, p50/p95/p99 13.5/57.8/96.4 мс, замена 5.9 с | `evidence/grade4/summary.json`, `evidence/grade4/rolling-load.json` |
+| Лимиты, нет обхода LB | `docker ps` | API `8080/tcp` без host-порта; наружу только `127.0.0.1:8080` у lb | `compose.yaml` |
+| Трек 1 | `scripts/track1_experiment.py` | серии A/B и проверка двух правок, см. ниже | `evidence/track1/summary.json`, `evidence/track1/rps.svg` |
-TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
+Студенческие тесты lab2 (гонки, откат чата, кэш, конкурентные сообщения): 15 passed. `RUN_INFRA` и expiry в этот прогон не входили.
+
+После трека 1 стенд возвращён к 2 репликам по 0.50 CPU. Новые hostname на ready: `0868e9d2ba45` и `14cb087ebb10`. Старые id исчезли вместе с пересозданными контейнерами; это не sticky session.
## Эксперименты и отказы
-TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
+Гипотеза. При фиксированном бюджете API 1.0 CPU и 512 МиБ дробление на 2 и 4 реплики не поднимет RPS: каждая упрётся в свой cgroup, а Postgres ещё свободен. Если давать каждой новой реплике те же 0.50 CPU, RPS вырастет, пока реплика упирается в квоту, и почти остановится, когда клиентский closed-loop и единственный Postgres перестанут получать пользу от лишних процессов.
+
+Методика. Генератор `tests/load.py`: closed-loop POST сообщений, concurrency 8, 5 с прогрев (результат выбрасывается), 15 с замер. Ошибок автоматических retry нет. На каждую точку один прогон. В середине замера один срез `docker stats` и `pg_stat_activity`. В `docker stats` 100% — одно ядро. Профиль A держит сумму API равной 1.0 CPU / 512 МиБ и сумму пулов равной 8. Профиль B добавляет реплики по 0.50 CPU / 256 МиБ при Postgres 1.0 CPU.
+
+| Точка | Реплики | CPU API суммарно | Успешный RPS | Ошибки | p50 / p95 / p99, мс | CPU API по stats | CPU Postgres | Сессии Postgres | CPU LB |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| A1 | 1 × 1.00 | 1.0 | 344.0 | 0 | 19.7 / 42.1 / 81.0 | 98% | 33% | 9 | 21% |
+| A2 | 2 × 0.50 | 1.0 | 196.3 | 0 | 26.6 / 89.9 / 123.3 | 49% и 48% | 28% | 9 | 20% |
+| A4 | 4 × 0.25 | 1.0 | 124.5 | 0 | 39.1 / 185.7 / 267.6 | 20–25% | 16% | 9 | 9% |
+| B1 | 1 × 0.50 | 0.5 | 98.1 | 0 | 85.1 / 167.2 / 201.9 | 48% | 17% | 5 | 8% |
+| B2 | 2 × 0.50 | 1.0 | 223.8 | 0 | 24.1 / 81.0 / 142.4 | 49% и 48% | 29% | 9 | 16% |
+| B4 | 4 × 0.50 | 2.0 | 236.6 | 0 | 22.6 / 80.2 / 131.0 | 35–46% | 48% | 17 | 21% |
+| C4-pg2 | как B4, Postgres 2.0 CPU | 2.0 | 251.2 | 0 | 19.7 / 77.2 / 101.2 | 40–43% | 43% | 17 | 22% |
+| D4-pool2 | как B4, пул 2 на реплику | 2.0 | 239.0 | 0 | 22.7 / 74.9 / 98.1 | 45–48% | 49% | 9 | 22% |
+
+График: `evidence/track1/rps.svg`. Сырые точки: `evidence/track1/*.point.json`.
+
+Что из этого следует. В серии A Postgres не насыщен (16–33%), память API далеко от лимита. Падает сам API: одна реплика с целым ядром даёт 344 RPS, те же ресурсы четырьмя квотами по 0.25 CPU — 124 RPS, p95 с 42 до 186 мс. Рост числа процессов при фиксированном CPU вреден из-за квот CFS и накладных расходов, а не из-за нехватки БД.
+
+В серии B первое ядро API ещё нужно: B1 упирается в квоту 0.50 CPU (stats около 50%, p95 167 мс). Вторая такая реплика — подтверждённое улучшение: RPS 98.1 → 223.8, p95 167 → 81 мс, Postgres только с 17% до 29%. Четыре реплики против двух — уже площадка: +1.0 CPU API даёт 223.8 → 236.6 RPS (+6%), зато сессий Postgres 9 → 17 и CPU Postgres 29% → 48%, LB 16% → 21%. Восемь замкнутых воркеров не заполняют четыре процесса, а каждый INSERT всё равно идёт в один Postgres. Пул здесь не узкое место: D4 снижает сессии с 17 до 9 и почти не меняет RPS (239 против 237). Дополнительное ядро Postgres тоже не подтвердилось: 251 против 237 на одном 15-секундном прогоне, срез CPU Postgres так и не упёрся в 100%.
+
+Ограничение измерения. Один прогон на точку, closed-loop и coordinated omission, перцентили только успешных запросов, таблица `messages` растёт от точки к точке, `docker stats` — один срез. Это не паспорт максимальной пропускной способности машины.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
-| TODO | TODO | TODO | TODO |
+| `scripts/restart.sh` (api, postgres, redis), тома на месте | те же сообщения и тот же токен | PASS публичного persistence | готовность вернулась внутри бюджета теста 90 с; данных не теряли |
+| `scripts/stop-one.sh` по `X-Instance-Id` | живая реплика принимает запись ≤10 с, дубля нет | 2 id до отказа; 1 ошибка; 109 записей; целевой контейнер Exited (0) | 2.465 с до первого подтверждения после возврата hook |
+| Та же остановка второй реплики после старта первой | хвост ответов только от оставшейся | оба прогона: target отсутствует в последних 5 id | hook 1.7–2.4 с; ошибок 1 и 2 |
+| Поочерёдная замена обеих реплик под нагрузкой | короткие ошибки, запись продолжается | 18 ошибок на 8379 попыток | скрипт замены 5.9 с |
## Ограничения и следующие шаги
-TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
+Стенд переживает остановку одного процесса API и перезапуск контейнеров с сохранёнными томами. Он не переживает потерю единственного хоста, HAProxy, Postgres или Redis: это оставшиеся SPOF. Трек 2 не делался, выживание узла БД не заявляется.
+
+Sticky session нет и не нужен: сессия в Redis, идемпотентность в Postgres. Повторно безопасно отправлять POST сообщения с тем же ключом. Повтор POST пользователя или чата по-прежнему может создать второй объект; прокси такие POST не повторяет.
+
+Миграции совместимы с уже работающей схемой lab2 (новый файл `002_idempotency.sql`, старые таблицы не переписываются). Одновременный запуск двух версий кода с разной схемой отдельно не гонялся.
+
+Подтверждённое улучшение — вторая реплика, когда первая исчерпала квоту 0.50 CPU (B1→B2). Четвёртая реплика, удвоение CPU Postgres и урезание пула на этом профиле прироста не показали. Следующий осмысленный шаг — поднять concurrency клиента выше 8 и повторить B2/B4 несколькими прогонами, прежде чем добавлять реплики Postgres.
diff --git a/app/__init__.py b/app/__init__.py
new file mode 100644
index 00000000..eeb0eb82
--- /dev/null
+++ b/app/__init__.py
@@ -0,0 +1 @@
+"""Messenger API (lab 2): PostgreSQL source of truth, Redis sessions/cache."""
diff --git a/app/cache.py b/app/cache.py
new file mode 100644
index 00000000..ed9261df
--- /dev/null
+++ b/app/cache.py
@@ -0,0 +1,151 @@
+import asyncio
+import json
+from collections.abc import Awaitable, Callable
+from typing import TypeVar
+
+from app import config, rediscli
+from app.errors import AppError
+
+T = TypeVar("T")
+CFG_SINGLEFLIGHT = "cfg:singleflight"
+CFG_INVALIDATE = "cfg:invalidate"
+CFG_DELAY_MS = "cfg:delay_ms"
+SQL_HISTORY = "metrics:sql:messages_select"
+
+
+class SingleFlight:
+ def __init__(self) -> None:
+ self._lock = asyncio.Lock()
+ self._inflight: dict[str, asyncio.Task] = {}
+
+ async def do(self, key: str, factory: Callable[[], Awaitable[T]]) -> T:
+ async with self._lock:
+ task = self._inflight.get(key)
+ if task is None:
+ task = asyncio.create_task(factory())
+ self._inflight[key] = task
+ owner = True
+ else:
+ owner = False
+ try:
+ return await asyncio.shield(task)
+ finally:
+ if owner:
+ async with self._lock:
+ self._inflight.pop(key, None)
+
+
+_sf = SingleFlight()
+
+
+async def _flag(name: str, default: bool = True) -> bool:
+ if not config.lab_mode():
+ return default
+ try:
+ raw = await rediscli.require().get(name)
+ except Exception:
+ return default
+ if raw is None:
+ return default
+ return raw not in {"0", "false", "off"}
+
+
+async def singleflight_enabled() -> bool:
+ return await _flag(CFG_SINGLEFLIGHT, True)
+
+
+async def invalidate_enabled() -> bool:
+ return await _flag(CFG_INVALIDATE, True)
+
+
+async def incr_history_sql() -> None:
+ try:
+ await rediscli.require().incr(SQL_HISTORY)
+ except Exception:
+ pass
+
+
+async def history_sql_count() -> int:
+ try:
+ return int(await rediscli.require().get(SQL_HISTORY) or 0)
+ except Exception:
+ return 0
+
+
+async def reset_history_sql_count() -> None:
+ try:
+ await rediscli.require().set(SQL_HISTORY, 0)
+ except Exception:
+ pass
+
+
+async def load_delay_seconds() -> float:
+ if not config.lab_mode():
+ return 0.0
+ try:
+ raw = await rediscli.require().get(CFG_DELAY_MS)
+ except Exception:
+ return 0.0
+ try:
+ return max(0.0, float(raw or 0) / 1000.0)
+ except (TypeError, ValueError):
+ return 0.0
+
+
+async def set_flags(
+ *,
+ singleflight: bool | None = None,
+ invalidate: bool | None = None,
+ delay_ms: int | None = None,
+) -> dict:
+ r = rediscli.require()
+ if singleflight is not None:
+ await r.set(CFG_SINGLEFLIGHT, "1" if singleflight else "0")
+ if invalidate is not None:
+ await r.set(CFG_INVALIDATE, "1" if invalidate else "0")
+ if delay_ms is not None:
+ await r.set(CFG_DELAY_MS, str(int(delay_ms)))
+ return {
+ "singleflight": await singleflight_enabled(),
+ "invalidate": await invalidate_enabled(),
+ "history_sql": await history_sql_count(),
+ "delay_ms": int((await load_delay_seconds()) * 1000),
+ }
+
+
+async def get_history_page(chat_id: str, cursor: str, limit: int, loader: Callable[[], Awaitable[dict]]) -> dict:
+ r = rediscli.require()
+ try:
+ version = await r.get("histver:" + chat_id) or "0"
+ key = f"hist:{chat_id}:{version}:{cursor}:{limit}"
+ cached = await r.get(key)
+ if cached:
+ return json.loads(cached)
+
+ async def load_and_store() -> dict:
+ delay = await load_delay_seconds()
+ if delay:
+ await asyncio.sleep(delay)
+ page = await loader()
+ try:
+ await r.set(key, json.dumps(page, ensure_ascii=False), ex=config.CACHE_TTL_SECONDS)
+ except Exception:
+ pass
+ return page
+
+ if await singleflight_enabled():
+ return await _sf.do(key, load_and_store)
+ return await load_and_store()
+ except AppError:
+ raise
+ except Exception:
+ return await loader()
+
+
+async def invalidate_history(chat_id: str) -> None:
+ if not await invalidate_enabled():
+ return
+ try:
+ await rediscli.require().incr("histver:" + chat_id)
+ except Exception:
+ pass
diff --git a/app/config.py b/app/config.py
new file mode 100644
index 00000000..b6d1f3b1
--- /dev/null
+++ b/app/config.py
@@ -0,0 +1,42 @@
+import os
+import socket
+from pathlib import Path
+
+
+def _env(name: str, default: str = "") -> str:
+ value = os.environ.get(name)
+ return default if value is None or value == "" else value
+
+
+def _file_or_env(name: str, default: str = "") -> str:
+ path = os.environ.get(name + "_FILE")
+ if path:
+ return Path(path).read_text(encoding="utf-8").strip()
+ return _env(name, default)
+
+
+HOST = _env("APP_HOST", "0.0.0.0")
+PORT = int(_env("APP_PORT", "8080"))
+ENV = _env("APP_ENV", "lab")
+LOG_LEVEL = _env("LOG_LEVEL", "info").lower()
+SHUTDOWN_GRACE_SECONDS = float(_env("SHUTDOWN_GRACE_SECONDS", "10"))
+SESSION_TTL_SECONDS = int(_env("SESSION_TTL_SECONDS", "3600"))
+CACHE_TTL_SECONDS = int(_env("CACHE_TTL_SECONDS", "60"))
+DATABASE_URL = _file_or_env("DATABASE_URL")
+REDIS_URL = _file_or_env("REDIS_URL", "redis://redis:6379/0")
+SESSION_STORE = _env("SESSION_STORE", "redis")
+DB_POOL_MIN = int(_env("DB_POOL_MIN", "1"))
+DB_POOL_MAX = int(_env("DB_POOL_MAX", "10"))
+DB_COMMAND_TIMEOUT = float(_env("DB_COMMAND_TIMEOUT", "5"))
+DB_CONNECT_TIMEOUT = float(_env("DB_CONNECT_TIMEOUT", "5"))
+REDIS_SOCKET_TIMEOUT = float(_env("REDIS_SOCKET_TIMEOUT", "2"))
+IDEMPOTENCY_TTL_SECONDS = int(_env("IDEMPOTENCY_TTL_SECONDS", "86400"))
+MIGRATIONS_ON_START = _env("MIGRATIONS_ON_START", "1") == "1"
+LB_DRAIN_SECONDS = float(_env("LB_DRAIN_SECONDS", "2"))
+# Stable for the life of this process. Compose leaves it unset so each
+# container uses its own hostname; stop-one.sh maps that value to one replica.
+INSTANCE_ID = _env("INSTANCE_ID", socket.gethostname())
+
+
+def lab_mode() -> bool:
+ return ENV == "lab"
diff --git a/app/cursor_codec.py b/app/cursor_codec.py
new file mode 100644
index 00000000..40e6ce20
--- /dev/null
+++ b/app/cursor_codec.py
@@ -0,0 +1,112 @@
+import base64
+
+import json
+
+from datetime import datetime
+
+from typing import Any
+
+from uuid import UUID
+
+
+
+
+
+def encode_cursor(chat_id: str, created_at: str, entity_id: str) -> str:
+
+ payload = json.dumps({"c": chat_id, "t": created_at, "i": entity_id}, separators=(",", ":"))
+
+ return base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=")
+
+
+
+
+
+def _validate_cursor_timestamp(value: str) -> None:
+
+ if not value.endswith("Z"):
+
+ raise ValueError("invalid cursor timestamp")
+
+ try:
+
+ datetime.fromisoformat(value[:-1] + "+00:00")
+
+ except ValueError as exc:
+
+ raise ValueError("invalid cursor timestamp") from exc
+
+
+
+
+
+def _require_str_field(data: dict, key: str) -> str:
+
+ if key not in data:
+
+ raise ValueError("invalid cursor")
+
+ value = data[key]
+
+ if not isinstance(value, str):
+
+ raise ValueError("invalid cursor")
+
+ return value
+
+
+
+
+
+def decode_cursor(cursor: str, expected_chat_id: str) -> tuple[str, str]:
+
+ pad = "=" * (-len(cursor) % 4)
+
+ try:
+
+ raw = base64.urlsafe_b64decode((cursor + pad).encode("ascii"))
+
+ data: Any = json.loads(raw.decode("utf-8"))
+
+ except (ValueError, json.JSONDecodeError, UnicodeDecodeError) as exc:
+
+ raise ValueError("invalid cursor") from exc
+
+ if not isinstance(data, dict):
+
+ raise ValueError("invalid cursor")
+
+ try:
+
+ chat_id = _require_str_field(data, "c")
+
+ created_at = _require_str_field(data, "t")
+
+ entity_id = _require_str_field(data, "i")
+
+ except ValueError as exc:
+
+ raise ValueError("invalid cursor") from exc
+
+ if chat_id != expected_chat_id:
+
+ raise ValueError("cursor chat mismatch")
+
+ try:
+
+ UUID(entity_id)
+
+ except ValueError as exc:
+
+ raise ValueError("invalid cursor") from exc
+
+ try:
+
+ _validate_cursor_timestamp(created_at)
+
+ except ValueError as exc:
+
+ raise ValueError("invalid cursor") from exc
+
+ return created_at, entity_id
+
diff --git a/app/db.py b/app/db.py
new file mode 100644
index 00000000..f98fd341
--- /dev/null
+++ b/app/db.py
@@ -0,0 +1,87 @@
+from pathlib import Path
+
+import asyncpg
+
+from app import config
+from app.errors import AppError
+
+pool: asyncpg.Pool | None = None
+INTEGRITY = (
+ asyncpg.UniqueViolationError,
+ asyncpg.ForeignKeyViolationError,
+ asyncpg.CheckViolationError,
+)
+
+
+def _migrations_dir() -> Path:
+ here = Path(__file__).resolve().parent
+ for candidate in (here.parent / "migrations", Path("/app/migrations")):
+ if candidate.is_dir():
+ return candidate
+ raise RuntimeError("migrations directory not found")
+
+
+async def connect() -> asyncpg.Pool:
+ global pool
+ if not config.DATABASE_URL:
+ raise RuntimeError("DATABASE_URL is not set")
+ pool = await asyncpg.create_pool(
+ dsn=config.DATABASE_URL,
+ min_size=config.DB_POOL_MIN,
+ max_size=config.DB_POOL_MAX,
+ command_timeout=config.DB_COMMAND_TIMEOUT,
+ timeout=config.DB_CONNECT_TIMEOUT,
+ )
+ return pool
+
+
+async def close() -> None:
+ global pool
+ if pool is not None:
+ await pool.close()
+ pool = None
+
+
+def require() -> asyncpg.Pool:
+ if pool is None:
+ raise AppError(503, "service unavailable")
+ return pool
+
+
+async def ping() -> None:
+ try:
+ async with require().acquire() as conn:
+ await conn.execute("SELECT 1")
+ except AppError:
+ raise
+ except Exception:
+ raise AppError(503, "service unavailable") from None
+
+
+_MIGRATION_LOCK = 87133103
+
+
+async def apply_migrations() -> None:
+ """Apply pending SQL files once. A session advisory lock stops replica races."""
+ directory = _migrations_dir()
+ async with require().acquire() as conn:
+ await conn.execute("SELECT pg_advisory_lock($1)", _MIGRATION_LOCK)
+ try:
+ await conn.execute(
+ """
+ CREATE TABLE IF NOT EXISTS schema_migrations (
+ filename TEXT PRIMARY KEY,
+ applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
+ )
+ """
+ )
+ applied = {row["filename"] for row in await conn.fetch("SELECT filename FROM schema_migrations")}
+ for path in sorted(p for p in directory.glob("*.sql") if p.is_file()):
+ if path.name in applied:
+ continue
+ sql = path.read_text(encoding="utf-8")
+ async with conn.transaction():
+ await conn.execute(sql)
+ await conn.execute("INSERT INTO schema_migrations(filename) VALUES($1)", path.name)
+ finally:
+ await conn.execute("SELECT pg_advisory_unlock($1)", _MIGRATION_LOCK)
diff --git a/app/docs.py b/app/docs.py
new file mode 100644
index 00000000..7f68e1d9
--- /dev/null
+++ b/app/docs.py
@@ -0,0 +1,52 @@
+import json
+from functools import lru_cache
+from pathlib import Path
+
+from starlette.requests import Request
+from starlette.responses import HTMLResponse, JSONResponse, Response
+
+_SPEC_CANDIDATES = (
+ Path(__file__).resolve().parents[1] / "contracts" / "openapi.json",
+ Path("/app/contracts/openapi.json"),
+)
+
+_SWAGGER_HTML = """
+
+
+
+ MTUSI Messenger — Swagger (lab2)
+
+
+
+
+
+
+
+
+"""
+
+
+@lru_cache(maxsize=1)
+def _load_spec() -> dict:
+ for path in _SPEC_CANDIDATES:
+ if path.is_file():
+ return json.loads(path.read_text(encoding="utf-8"))
+ return {"openapi": "3.1.0", "info": {"title": "MTUSI Messenger", "version": "1.2.0"}, "paths": {}}
+
+
+async def openapi_json(_: Request) -> Response:
+ return JSONResponse(_load_spec())
+
+
+async def swagger_ui(_: Request) -> Response:
+ return HTMLResponse(_SWAGGER_HTML)
diff --git a/app/errors.py b/app/errors.py
new file mode 100644
index 00000000..105e8ca3
--- /dev/null
+++ b/app/errors.py
@@ -0,0 +1,49 @@
+from typing import Dict, Optional
+
+from starlette.responses import JSONResponse
+
+
+ERROR_CODES = {
+ 400: "invalid_request",
+ 401: "unauthorized",
+ 404: "not_found",
+ 409: "conflict",
+ 413: "payload_too_large",
+ 415: "unsupported_media_type",
+ 429: "rate_limited",
+ 503: "unavailable",
+ 500: "internal_error",
+}
+
+BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
+BEARER_CHALLENGE = 'Bearer realm="messenger"'
+
+
+class AppError(Exception):
+ def __init__(self, status: int, message: str, www_authenticate: str | None = None) -> None:
+ super().__init__(message)
+ self.status = status
+ self.message = message
+ self.www_authenticate = www_authenticate
+
+
+def error_response(
+ status: int,
+ message: str,
+ request_id: str,
+ extra_headers: Optional[Dict[str, str]] = None,
+) -> JSONResponse:
+ code = ERROR_CODES.get(status, "internal_error")
+ headers = {"X-Request-Id": request_id, "Content-Type": "application/json"}
+ if extra_headers:
+ headers.update(extra_headers)
+ return JSONResponse(
+ status_code=status,
+ headers=headers,
+ content={"error": {"code": code, "message": message, "request_id": request_id}},
+ )
+
+
+def from_app_error(exc: AppError, request_id: str) -> JSONResponse:
+ extra = {"WWW-Authenticate": exc.www_authenticate} if exc.www_authenticate else None
+ return error_response(exc.status, exc.message, request_id, extra)
diff --git a/app/main.py b/app/main.py
new file mode 100644
index 00000000..c59ca752
--- /dev/null
+++ b/app/main.py
@@ -0,0 +1,146 @@
+import asyncio
+import logging
+import signal
+import sys
+import uuid
+from pathlib import Path
+
+import uvicorn
+from starlette.applications import Starlette
+from starlette.exceptions import HTTPException
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.routing import Route
+
+from app import config, db, rediscli
+from app.docs import openapi_json, swagger_ui
+from app.errors import AppError, error_response, from_app_error
+from app.middleware import RequestContextMiddleware
+from app.routes import (
+ create_chat,
+ create_message,
+ create_session,
+ current_user,
+ get_chat,
+ health_live,
+ health_ready,
+ lab_flags,
+ lab_metrics,
+ list_chats,
+ list_messages,
+ register_user,
+ revoke_session,
+)
+
+
+def _request_id(request: Request) -> str:
+ existing = getattr(request.state, "request_id", None)
+ if existing:
+ return existing
+ return request.headers.get("X-Request-Id") or str(uuid.uuid4())
+
+
+async def not_found_handler(request: Request, exc: HTTPException) -> Response:
+ return error_response(404, "not found", _request_id(request))
+
+
+async def app_error_handler(request: Request, exc: AppError) -> Response:
+ return from_app_error(exc, _request_id(request))
+
+
+def publish_instance_id() -> None:
+ path = Path(config._env("INSTANCE_ID_FILE", "/tmp/instance-id"))
+ try:
+ path.write_text(config.INSTANCE_ID, encoding="utf-8")
+ except OSError:
+ logging.getLogger("messenger").warning("instance id file was not written")
+
+
+async def startup() -> None:
+ publish_instance_id()
+ logging.getLogger("messenger").info("instance_id=%s", config.INSTANCE_ID)
+ await db.connect()
+ if config.MIGRATIONS_ON_START:
+ await db.apply_migrations()
+ await rediscli.connect()
+
+
+async def shutdown() -> None:
+ await rediscli.close()
+ await db.close()
+
+
+def build_app() -> Starlette:
+ app = Starlette(
+ routes=[
+ Route("/docs", swagger_ui, methods=["GET"]),
+ Route("/openapi.json", openapi_json, methods=["GET"]),
+ Route("/health/live", health_live, methods=["GET"]),
+ Route("/health/ready", health_ready, methods=["GET"]),
+ Route("/api/v1/users", register_user, methods=["POST"]),
+ Route("/api/v1/users/me", current_user, methods=["GET"]),
+ Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
+ Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]),
+ Route("/api/v1/chats", create_chat, methods=["POST"]),
+ Route("/api/v1/chats", list_chats, methods=["GET"]),
+ Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
+ Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
+ Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
+ Route("/api/v1/_lab/metrics", lab_metrics, methods=["GET"]),
+ Route("/api/v1/_lab/flags", lab_flags, methods=["POST"]),
+ ],
+ exception_handlers={404: not_found_handler, AppError: app_error_handler},
+ on_startup=[startup],
+ on_shutdown=[shutdown],
+ )
+ app.state.shutting_down = False
+ app.add_middleware(RequestContextMiddleware)
+ return app
+
+
+def configure_logging() -> None:
+ logging.basicConfig(
+ level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO),
+ format="%(asctime)s %(levelname)s %(name)s %(message)s",
+ stream=sys.stdout,
+ )
+
+
+def main() -> None:
+ configure_logging()
+ app = build_app()
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+
+ server = uvicorn.Server(
+ uvicorn.Config(
+ app,
+ host=config.HOST,
+ port=config.PORT,
+ log_level=config.LOG_LEVEL,
+ access_log=False,
+ timeout_graceful_shutdown=int(config.SHUTDOWN_GRACE_SECONDS),
+ )
+ )
+
+ def _begin_shutdown() -> None:
+ if app.state.shutting_down:
+ return
+ logging.getLogger("messenger").info(
+ "shutdown signal received; readiness fails now, exit in %.1fs",
+ config.LB_DRAIN_SECONDS,
+ )
+ app.state.shutting_down = True
+ loop.call_later(config.LB_DRAIN_SECONDS, lambda: setattr(server, "should_exit", True))
+
+ for sig in (signal.SIGTERM, signal.SIGINT):
+ try:
+ loop.add_signal_handler(sig, _begin_shutdown)
+ except NotImplementedError:
+ signal.signal(sig, lambda *_: _begin_shutdown())
+
+ loop.run_until_complete(server.serve())
+
+
+if __name__ == "__main__":
+ main()
diff --git a/app/middleware.py b/app/middleware.py
new file mode 100644
index 00000000..4b05cca3
--- /dev/null
+++ b/app/middleware.py
@@ -0,0 +1,48 @@
+import logging
+import time
+import uuid
+from typing import Callable
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
+
+from app.config import INSTANCE_ID
+from app.errors import AppError, error_response, from_app_error
+
+logger = logging.getLogger("messenger.access")
+
+
+def _stamp(response: Response, request_id: str) -> Response:
+ response.headers["X-Request-Id"] = request_id
+ response.headers["X-Instance-Id"] = INSTANCE_ID
+ return response
+
+
+class RequestContextMiddleware(BaseHTTPMiddleware):
+ async def dispatch(self, request: Request, call_next: Callable) -> Response:
+ request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4())
+ request.state.request_id = request_id
+
+ if request.app.state.shutting_down and request.url.path not in ("/health/live", "/health/ready"):
+ return _stamp(error_response(503, "service is shutting down", request_id), request_id)
+
+ started = time.perf_counter()
+ try:
+ response = await call_next(request)
+ except AppError as exc:
+ return _stamp(from_app_error(exc, request_id), request_id)
+ except Exception:
+ logger.exception("request_id=%s unhandled error path=%s", request_id, request.url.path)
+ return _stamp(error_response(500, "internal error", request_id), request_id)
+ elapsed_ms = (time.perf_counter() - started) * 1000
+ _stamp(response, request_id)
+ logger.info(
+ "request_id=%s method=%s path=%s status=%s duration_ms=%.2f",
+ request_id,
+ request.method,
+ request.url.path,
+ response.status_code,
+ elapsed_ms,
+ )
+ return response
diff --git a/app/migrate.py b/app/migrate.py
new file mode 100644
index 00000000..03bc8db8
--- /dev/null
+++ b/app/migrate.py
@@ -0,0 +1,29 @@
+"""One-shot schema apply. API replicas do not run this on startup."""
+import asyncio
+import logging
+import sys
+
+from app import config, db
+
+
+async def _main() -> None:
+ if not config.DATABASE_URL:
+ raise SystemExit("DATABASE_URL is not set")
+ await db.connect()
+ try:
+ await db.apply_migrations()
+ finally:
+ await db.close()
+
+
+def main() -> None:
+ logging.basicConfig(
+ level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO),
+ format="%(asctime)s %(levelname)s %(name)s %(message)s",
+ stream=sys.stdout,
+ )
+ asyncio.run(_main())
+
+
+if __name__ == "__main__":
+ main()
diff --git a/app/passwords.py b/app/passwords.py
new file mode 100644
index 00000000..8a640cd0
--- /dev/null
+++ b/app/passwords.py
@@ -0,0 +1,23 @@
+from argon2 import PasswordHasher
+from argon2.exceptions import InvalidHashError, VerifyMismatchError
+
+hasher = PasswordHasher(time_cost=2, memory_cost=8192, parallelism=1)
+_dummy = None
+
+
+def hash_password(password: str) -> str:
+ return hasher.hash(password)
+
+
+def dummy_hash() -> str:
+ global _dummy
+ if _dummy is None:
+ _dummy = hash_password("timing-dummy-password")
+ return _dummy
+
+
+def verify_password(password_hash: str, password: str) -> bool:
+ try:
+ return hasher.verify(password_hash, password)
+ except (VerifyMismatchError, InvalidHashError, TypeError, ValueError):
+ return False
diff --git a/app/rediscli.py b/app/rediscli.py
new file mode 100644
index 00000000..de173117
--- /dev/null
+++ b/app/rediscli.py
@@ -0,0 +1,40 @@
+import redis.asyncio as redis
+
+from app import config
+from app.errors import AppError
+
+client: redis.Redis | None = None
+
+
+async def connect() -> redis.Redis:
+ global client
+ client = redis.from_url(
+ config.REDIS_URL,
+ socket_connect_timeout=config.REDIS_SOCKET_TIMEOUT,
+ socket_timeout=config.REDIS_SOCKET_TIMEOUT,
+ decode_responses=True,
+ )
+ await client.ping()
+ return client
+
+
+async def close() -> None:
+ global client
+ if client is not None:
+ await client.close()
+ client = None
+
+
+def require() -> redis.Redis:
+ if client is None:
+ raise AppError(503, "service unavailable")
+ return client
+
+
+async def ping() -> None:
+ try:
+ await require().ping()
+ except AppError:
+ raise
+ except Exception:
+ raise AppError(503, "service unavailable") from None
diff --git a/app/routes.py b/app/routes.py
new file mode 100644
index 00000000..0b7bb307
--- /dev/null
+++ b/app/routes.py
@@ -0,0 +1,284 @@
+import base64
+import binascii
+
+from starlette.requests import Request
+from starlette.responses import JSONResponse, Response
+
+from app import cache, config, db, rediscli, sessions, store
+from app.cursor_codec import decode_cursor, encode_cursor
+from app.errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError, error_response
+from app.passwords import dummy_hash, hash_password, verify_password
+from app.validation import (
+ parse_json_body,
+ require_exact_keys,
+ validate_display_name,
+ validate_limit,
+ validate_idempotency_key,
+ validate_message_text,
+ validate_password,
+ validate_title,
+ validate_username,
+ validate_uuid,
+)
+
+
+def _json(data: dict, request_id: str, status: int = 200) -> JSONResponse:
+ return JSONResponse(status_code=status, headers={"X-Request-Id": request_id}, content=data)
+
+
+def _empty(request_id: str) -> Response:
+ return Response(status_code=204, headers={"X-Request-Id": request_id})
+
+
+async def _bearer_user(request: Request) -> dict:
+ header = request.headers.get("Authorization") or ""
+ scheme, _, rest = header.partition(" ")
+ if scheme.lower() != "bearer" or not rest.strip():
+ raise AppError(401, "authentication required", BEARER_CHALLENGE)
+ user_id = str(await sessions.resolve(rest.strip()))
+ user = await store.get_user(user_id)
+ if user is None:
+ raise AppError(401, "authentication required", BEARER_CHALLENGE)
+ return user
+
+
+async def health_live(request: Request) -> Response:
+ return _json({"status": "ok"}, request.state.request_id)
+
+
+async def health_ready(request: Request) -> Response:
+ rid = request.state.request_id
+ if request.app.state.shutting_down:
+ return error_response(503, "service is shutting down", rid)
+ await db.ping()
+ await rediscli.ping()
+ return _json({"status": "ok"}, rid)
+
+
+async def register_user(request: Request) -> Response:
+ rid = request.state.request_id
+ if request.headers.get("content-type", "").split(";")[0].strip().lower() not in ("application/json", ""):
+ if request.headers.get("content-type"):
+ return error_response(415, "unsupported media type", rid)
+ raw = await request.body()
+ body, err = parse_json_body(raw)
+ if err:
+ return error_response(400, err, rid)
+ assert body is not None
+ if require_exact_keys(body, {"username", "display_name", "password"}, {"username", "display_name", "password"}):
+ return error_response(400, "invalid user payload", rid)
+ if validate_username(body["username"]) or validate_display_name(body["display_name"]) or validate_password(body["password"]):
+ return error_response(400, "invalid user fields", rid)
+ user, conflict = await store.create_user(body["username"], body["display_name"], hash_password(body["password"]))
+ if conflict:
+ return error_response(409, "username already taken", rid)
+ assert user is not None
+ return _json(user, rid, 201)
+
+
+async def current_user(request: Request) -> Response:
+ user = await _bearer_user(request)
+ return _json(user, request.state.request_id)
+
+
+async def create_session(request: Request) -> Response:
+ rid = request.state.request_id
+ header = request.headers.get("Authorization") or ""
+ scheme, _, rest = header.partition(" ")
+ if scheme.lower() != "basic" or not rest.strip():
+ return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
+ try:
+ decoded = base64.b64decode(rest.strip(), validate=True).decode("utf-8")
+ except (binascii.Error, UnicodeError, ValueError):
+ return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
+ username, sep, password = decoded.partition(":")
+ if not sep:
+ return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
+ row = await store.get_user_by_username(username)
+ password_hash = row["password_hash"] if row else dummy_hash()
+ if row is None or not verify_password(password_hash, password):
+ return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
+ token, expires_at = await sessions.create(row["id"])
+ return _json(
+ {
+ "token": token,
+ "token_type": "Bearer",
+ "expires_at": expires_at,
+ "user": store.user_dict(row),
+ },
+ rid,
+ 201,
+ )
+
+
+async def revoke_session(request: Request) -> Response:
+ header = request.headers.get("Authorization") or ""
+ scheme, _, rest = header.partition(" ")
+ if scheme.lower() != "bearer" or not rest.strip():
+ raise AppError(401, "authentication required", BEARER_CHALLENGE)
+ await sessions.revoke(rest.strip())
+ return _empty(request.state.request_id)
+
+
+async def create_chat(request: Request) -> Response:
+ rid = request.state.request_id
+ user = await _bearer_user(request)
+ ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
+ if ct and ct != "application/json":
+ return error_response(415, "unsupported media type", rid)
+ raw = await request.body()
+ body, err = parse_json_body(raw)
+ if err:
+ return error_response(400, err, rid)
+ assert body is not None
+ if require_exact_keys(body, {"title", "member_ids"}, {"title", "member_ids"}):
+ return error_response(400, "invalid chat payload", rid)
+ if validate_title(body["title"]):
+ return error_response(400, "invalid title", rid)
+ members = body["member_ids"]
+ if not isinstance(members, list) or not members:
+ return error_response(400, "invalid member_ids", rid)
+ if len(members) > 100:
+ return error_response(400, "too many members", rid)
+ for mid in members:
+ if validate_uuid(mid):
+ return error_response(400, "invalid member id", rid)
+ chat, create_err = await store.create_chat(body["title"], members, user["id"])
+ if create_err == "caller not in members":
+ return error_response(400, "caller must be a member", rid)
+ if create_err in ("duplicate members", "unknown member"):
+ return error_response(400, "invalid members", rid)
+ assert chat is not None
+ return _json(chat, rid, 201)
+
+
+async def list_chats(request: Request) -> Response:
+ rid = request.state.request_id
+ user = await _bearer_user(request)
+ limit, limit_err = validate_limit(request.query_params.get("limit"))
+ if limit_err:
+ return error_response(400, limit_err, rid)
+ assert limit is not None
+ cursor_raw = request.query_params.get("cursor")
+ after = None
+ if cursor_raw is not None:
+ if not cursor_raw.strip():
+ return error_response(400, "invalid cursor", rid)
+ try:
+ created_at, entity_id = decode_cursor(cursor_raw, user["id"])
+ except ValueError:
+ return error_response(400, "invalid cursor", rid)
+ after = (created_at, entity_id)
+ items, next_key = await store.list_chats(user["id"], limit, after)
+ next_cursor = None
+ if next_key is not None:
+ next_cursor = encode_cursor(user["id"], next_key[0], next_key[1])
+ return _json({"items": items, "next_cursor": next_cursor}, rid)
+
+
+async def get_chat(request: Request) -> Response:
+ rid = request.state.request_id
+ user = await _bearer_user(request)
+ chat_id = request.path_params["chat_id"]
+ if validate_uuid(chat_id):
+ return error_response(400, "invalid chat id", rid)
+ chat = await store.get_chat(chat_id, user["id"])
+ if chat is None:
+ return error_response(404, "chat not found", rid)
+ return _json(chat, rid)
+
+
+async def create_message(request: Request) -> Response:
+ rid = request.state.request_id
+ user = await _bearer_user(request)
+ chat_id = request.path_params["chat_id"]
+ if validate_uuid(chat_id):
+ return error_response(400, "invalid chat id", rid)
+ ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
+ if ct and ct != "application/json":
+ return error_response(415, "unsupported media type", rid)
+ raw = await request.body()
+ body, err = parse_json_body(raw)
+ if err:
+ return error_response(400, err, rid)
+ assert body is not None
+ if require_exact_keys(body, {"text"}, {"text"}):
+ return error_response(400, "invalid message payload", rid)
+ if validate_message_text(body["text"]):
+ return error_response(400, "invalid message text", rid)
+ if validate_idempotency_key(request.headers.get("Idempotency-Key")):
+ return error_response(400, "invalid idempotency key", rid)
+ message, msg_err = await store.add_message(
+ chat_id,
+ user["id"],
+ body["text"],
+ request.headers.get("Idempotency-Key") or "",
+ request.url.path,
+ )
+ if msg_err == "not_found":
+ return error_response(404, "chat not found", rid)
+ if msg_err == "conflict":
+ return error_response(409, "idempotency key reused with a different body", rid)
+ assert message is not None
+ return _json(message, rid, 201)
+
+
+async def list_messages(request: Request) -> Response:
+ rid = request.state.request_id
+ user = await _bearer_user(request)
+ chat_id = request.path_params["chat_id"]
+ if validate_uuid(chat_id):
+ return error_response(400, "invalid chat id", rid)
+ limit, limit_err = validate_limit(request.query_params.get("limit"))
+ if limit_err:
+ return error_response(400, limit_err, rid)
+ assert limit is not None
+ cursor_raw = request.query_params.get("cursor")
+ after = None
+ if cursor_raw is not None:
+ if not cursor_raw.strip():
+ return error_response(400, "invalid cursor", rid)
+ try:
+ created_at, entity_id = decode_cursor(cursor_raw, chat_id)
+ except ValueError:
+ return error_response(400, "invalid cursor", rid)
+ after = (created_at, entity_id)
+ page, next_key, list_err = await store.list_messages(chat_id, user["id"], limit, after)
+ if list_err == "not_found":
+ return error_response(404, "chat not found", rid)
+ assert page is not None
+ next_cursor = None
+ if next_key is not None:
+ next_cursor = encode_cursor(chat_id, next_key[0], next_key[1])
+ return _json({"items": page, "next_cursor": next_cursor}, rid)
+
+
+async def lab_metrics(request: Request) -> Response:
+ if not config.lab_mode():
+ return error_response(404, "not found", request.state.request_id)
+ return _json(
+ {
+ "singleflight": await cache.singleflight_enabled(),
+ "invalidate": await cache.invalidate_enabled(),
+ "history_sql": await cache.history_sql_count(),
+ },
+ request.state.request_id,
+ )
+
+
+async def lab_flags(request: Request) -> Response:
+ if not config.lab_mode():
+ return error_response(404, "not found", request.state.request_id)
+ raw = await request.body()
+ body, err = parse_json_body(raw)
+ if err:
+ return error_response(400, err, request.state.request_id)
+ assert body is not None
+ if body.get("reset_sql"):
+ await cache.reset_history_sql_count()
+ flags = await cache.set_flags(
+ singleflight=body.get("singleflight"),
+ invalidate=body.get("invalidate"),
+ delay_ms=body.get("delay_ms"),
+ )
+ return _json(flags, request.state.request_id)
diff --git a/app/sessions.py b/app/sessions.py
new file mode 100644
index 00000000..b0e30b11
--- /dev/null
+++ b/app/sessions.py
@@ -0,0 +1,56 @@
+import hashlib
+import json
+import secrets
+from datetime import datetime, timedelta, timezone
+from uuid import UUID
+
+from app import config, rediscli
+from app.errors import AppError, BEARER_CHALLENGE
+
+
+def new_token() -> str:
+ return secrets.token_urlsafe(32)
+
+
+def _key(token: str) -> str:
+ return "sess:" + hashlib.sha256(token.encode("utf-8")).hexdigest()
+
+
+def _fmt(dt: datetime) -> str:
+ return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
+
+
+async def create(user_id: UUID) -> tuple[str, str]:
+ token = new_token()
+ expires = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
+ payload = json.dumps({"user_id": str(user_id), "expires_at": _fmt(expires)})
+ try:
+ await rediscli.require().set(_key(token), payload, ex=config.SESSION_TTL_SECONDS)
+ except AppError:
+ raise
+ except Exception:
+ raise AppError(503, "service unavailable") from None
+ return token, _fmt(expires)
+
+
+async def resolve(token: str) -> UUID:
+ try:
+ raw = await rediscli.require().get(_key(token))
+ except AppError:
+ raise
+ except Exception:
+ raise AppError(503, "service unavailable") from None
+ if not raw:
+ raise AppError(401, "authentication required", BEARER_CHALLENGE)
+ return UUID(json.loads(raw)["user_id"])
+
+
+async def revoke(token: str) -> None:
+ try:
+ deleted = await rediscli.require().delete(_key(token))
+ except AppError:
+ raise
+ except Exception:
+ raise AppError(503, "service unavailable") from None
+ if not deleted:
+ raise AppError(401, "authentication required", BEARER_CHALLENGE)
diff --git a/app/store.py b/app/store.py
new file mode 100644
index 00000000..04e37555
--- /dev/null
+++ b/app/store.py
@@ -0,0 +1,361 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timezone
+from typing import List, Optional, Tuple
+from uuid import UUID, uuid4
+
+import asyncpg
+
+from app import cache, config, db
+from app.errors import AppError
+
+
+def format_ts(dt: datetime) -> str:
+ if dt.tzinfo is None:
+ dt = dt.replace(tzinfo=timezone.utc)
+ return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
+
+
+def user_dict(row) -> dict:
+ return {
+ "id": str(row["id"]),
+ "username": row["username"],
+ "display_name": row["display_name"],
+ "created_at": format_ts(row["created_at"]),
+ }
+
+
+def chat_dict(row, member_ids: List[str]) -> dict:
+ return {
+ "id": str(row["id"]),
+ "title": row["title"],
+ "member_ids": member_ids,
+ "created_at": format_ts(row["created_at"]),
+ }
+
+
+def message_dict(row) -> dict:
+ return {
+ "id": str(row["id"]),
+ "chat_id": str(row["chat_id"]),
+ "sender_id": str(row["sender_id"]),
+ "text": row["text"],
+ "created_at": format_ts(row["created_at"]),
+ }
+
+
+async def _run(coro):
+ try:
+ return await coro
+ except AppError:
+ raise
+ except db.INTEGRITY:
+ raise
+ except Exception:
+ raise AppError(503, "service unavailable") from None
+
+
+async def create_user(username: str, display_name: str, password_hash: str) -> Tuple[Optional[dict], Optional[str]]:
+ user_id = uuid4()
+ created = datetime.now(timezone.utc)
+ try:
+ async with db.require().acquire() as conn:
+ await conn.execute(
+ """
+ INSERT INTO users (id, username, display_name, password_hash, created_at)
+ VALUES ($1, $2, $3, $4, $5)
+ """,
+ user_id,
+ username,
+ display_name,
+ password_hash,
+ created,
+ )
+ except asyncpg.UniqueViolationError:
+ return None, "conflict"
+ except Exception:
+ raise AppError(503, "service unavailable") from None
+ return user_dict({"id": user_id, "username": username, "display_name": display_name, "created_at": created}), None
+
+
+async def get_user(user_id: str) -> Optional[dict]:
+ row = await _run(
+ _fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", UUID(user_id))
+ )
+ return user_dict(row) if row else None
+
+
+async def get_user_by_username(username: str):
+ return await _run(
+ _fetchrow(
+ "SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1",
+ username,
+ )
+ )
+
+
+async def _fetchrow(sql: str, *args):
+ async with db.require().acquire() as conn:
+ return await conn.fetchrow(sql, *args)
+
+
+async def _fetch(sql: str, *args):
+ async with db.require().acquire() as conn:
+ return await conn.fetch(sql, *args)
+
+
+async def member_ids_of(chat_id: UUID) -> List[str]:
+ rows = await _run(_fetch("SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id", chat_id))
+ return [str(r["user_id"]) for r in rows]
+
+
+async def create_chat(title: str, member_ids: List[str], caller_id: str) -> Tuple[Optional[dict], Optional[str]]:
+ if caller_id not in member_ids:
+ return None, "caller not in members"
+ if len(set(member_ids)) != len(member_ids):
+ return None, "duplicate members"
+ ids = [UUID(x) for x in member_ids]
+ chat_id = uuid4()
+ created = datetime.now(timezone.utc)
+ try:
+ async with db.require().acquire() as conn:
+ async with conn.transaction():
+ existing = await conn.fetch("SELECT id FROM users WHERE id = ANY($1::uuid[])", ids)
+ if len(existing) != len(ids):
+ return None, "unknown member"
+ await conn.execute(
+ "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)",
+ chat_id,
+ title,
+ created,
+ )
+ await conn.executemany(
+ "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)",
+ [(chat_id, mid) for mid in ids],
+ )
+ except AppError:
+ raise
+ except Exception:
+ raise AppError(503, "service unavailable") from None
+ return chat_dict({"id": chat_id, "title": title, "created_at": created}, member_ids), None
+
+
+async def get_chat(chat_id: str, user_id: str) -> Optional[dict]:
+ row = await load_chat_for_member(chat_id, user_id)
+ if row is None:
+ return None
+ return chat_dict(row, await member_ids_of(UUID(chat_id)))
+
+
+async def load_chat_for_member(chat_id: str, user_id: str):
+ return await _run(
+ _fetchrow(
+ """
+ SELECT c.id, c.title, c.created_at
+ FROM chats c
+ JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2
+ WHERE c.id = $1
+ """,
+ UUID(chat_id),
+ UUID(user_id),
+ )
+ )
+
+
+async def list_chats(user_id: str, limit: int, after: Optional[Tuple[str, str]] = None):
+ args: list = [UUID(user_id)]
+ clause = ""
+ if after is not None:
+ clause = "AND (c.created_at, c.id) > ($2, $3)"
+ args.extend([datetime.fromisoformat(after[0].replace("Z", "+00:00")), UUID(after[1])])
+ rows = await _run(
+ _fetch(
+ f"""
+ SELECT c.id, c.title, c.created_at
+ FROM chats c
+ JOIN chat_members me ON me.chat_id = c.id AND me.user_id = $1
+ WHERE 1=1 {clause}
+ ORDER BY c.created_at, c.id
+ LIMIT {int(limit) + 1}
+ """,
+ *args,
+ )
+ )
+ page = rows[:limit]
+ items = [chat_dict(row, await member_ids_of(row["id"])) for row in page]
+ next_key = None
+ if len(rows) > limit:
+ last = page[-1]
+ next_key = (format_ts(last["created_at"]), str(last["id"]))
+ return items, next_key
+
+
+def _canonical_message(text: str) -> str:
+ # Missing attachment_ids and [] are the same message. Key order does not matter.
+ return json.dumps(
+ {"attachment_ids": [], "text": text},
+ sort_keys=True,
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+
+
+async def add_message(
+ chat_id: str,
+ sender_id: str,
+ text: str,
+ idem_key: str,
+ path: str,
+) -> Tuple[Optional[dict], Optional[str]]:
+ """Insert one message or replay the stored 201. The row and the result commit together.
+
+ Same user+method+path+key is serialized with a Postgres advisory lock so two
+ replicas cannot insert two messages. A failed transaction stores nothing.
+ """
+ digest = hashlib.sha256(_canonical_message(text).encode("utf-8")).hexdigest()
+ lock_key = f"{sender_id}|POST|{path}|{idem_key}"
+ payload: Optional[dict] = None
+ created_new = False
+ try:
+ async with db.require().acquire() as conn:
+ async with conn.transaction():
+ await conn.execute("SELECT pg_advisory_xact_lock(hashtextextended($1::text, 0))", lock_key)
+ member = await conn.fetchrow(
+ """
+ SELECT c.id
+ FROM chats c
+ JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2
+ WHERE c.id = $1
+ """,
+ UUID(chat_id),
+ UUID(sender_id),
+ )
+ if member is None:
+ return None, "not_found"
+ row = await conn.fetchrow(
+ """
+ SELECT request_hash, response_json, created_at
+ FROM idempotency_keys
+ WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3
+ """,
+ UUID(sender_id),
+ path,
+ idem_key,
+ )
+ if row is not None:
+ created_at = row["created_at"]
+ if created_at.tzinfo is None:
+ created_at = created_at.replace(tzinfo=timezone.utc)
+ age = (datetime.now(timezone.utc) - created_at).total_seconds()
+ if age <= config.IDEMPOTENCY_TTL_SECONDS:
+ if row["request_hash"] != digest:
+ return None, "conflict"
+ return json.loads(row["response_json"]), None
+ await conn.execute(
+ """
+ DELETE FROM idempotency_keys
+ WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3
+ """,
+ UUID(sender_id),
+ path,
+ idem_key,
+ )
+ message_id = uuid4()
+ created = datetime.now(timezone.utc)
+ await conn.execute(
+ """
+ INSERT INTO messages (id, chat_id, sender_id, text, created_at)
+ VALUES ($1, $2, $3, $4, $5)
+ """,
+ message_id,
+ UUID(chat_id),
+ UUID(sender_id),
+ text,
+ created,
+ )
+ payload = message_dict(
+ {
+ "id": message_id,
+ "chat_id": UUID(chat_id),
+ "sender_id": UUID(sender_id),
+ "text": text,
+ "created_at": created,
+ }
+ )
+ await conn.execute(
+ """
+ INSERT INTO idempotency_keys
+ (user_id, http_method, path, idem_key, request_hash, response_json, created_at)
+ VALUES ($1, 'POST', $2, $3, $4, $5, $6)
+ """,
+ UUID(sender_id),
+ path,
+ idem_key,
+ digest,
+ json.dumps(payload, ensure_ascii=False, separators=(",", ":")),
+ created,
+ )
+ created_new = True
+ except AppError:
+ raise
+ except Exception:
+ raise AppError(503, "service unavailable") from None
+ if created_new:
+ await cache.invalidate_history(chat_id)
+ return payload, None
+
+
+async def list_messages(chat_id: str, user_id: str, limit: int, after: Optional[Tuple[str, str]] = None):
+ if await load_chat_for_member(chat_id, user_id) is None:
+ return None, None, "not_found"
+ cursor = ""
+ after_ts = after_id = None
+ if after is not None:
+ cursor = after[0] + ":" + after[1]
+ after_ts = datetime.fromisoformat(after[0].replace("Z", "+00:00"))
+ after_id = UUID(after[1])
+
+ async def loader() -> dict:
+ await cache.incr_history_sql()
+ if after_ts is None:
+ rows = await _run(
+ _fetch(
+ """
+ SELECT id, chat_id, sender_id, text, created_at
+ FROM messages
+ WHERE chat_id = $1
+ ORDER BY created_at, id
+ LIMIT $2
+ """,
+ UUID(chat_id),
+ limit + 1,
+ )
+ )
+ else:
+ rows = await _run(
+ _fetch(
+ """
+ SELECT id, chat_id, sender_id, text, created_at
+ FROM messages
+ WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
+ ORDER BY created_at, id
+ LIMIT $4
+ """,
+ UUID(chat_id),
+ after_ts,
+ after_id,
+ limit + 1,
+ )
+ )
+ page = rows[:limit]
+ items = [message_dict(r) for r in page]
+ next_key = None
+ if len(rows) > limit:
+ last = page[-1]
+ next_key = (format_ts(last["created_at"]), str(last["id"]))
+ return {"items": items, "next_key": next_key}
+
+ page = await cache.get_history_page(chat_id, cursor, limit, loader)
+ return page["items"], page.get("next_key"), None
diff --git a/app/tools/__init__.py b/app/tools/__init__.py
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/app/tools/__init__.py
@@ -0,0 +1 @@
+
diff --git a/app/tools/explain.py b/app/tools/explain.py
new file mode 100644
index 00000000..e649ad14
--- /dev/null
+++ b/app/tools/explain.py
@@ -0,0 +1,73 @@
+"""Seed ≥10 000 messages and print EXPLAIN (ANALYZE, BUFFERS) for keyset pagination."""
+import asyncio
+import os
+from uuid import uuid4
+
+import asyncpg
+
+from app.passwords import hash_password
+
+
+async def main() -> None:
+ conn = await asyncpg.connect(dsn=os.environ["DATABASE_URL"], timeout=10, command_timeout=120)
+ try:
+ alice = uuid4()
+ bob = uuid4()
+ chat = uuid4()
+ pwd = hash_password("Example-only-not-a-real-secret-123")
+ await conn.execute(
+ "INSERT INTO users (id, username, display_name, password_hash) VALUES ($1,$2,$3,$4),($5,$6,$7,$4)",
+ alice,
+ "explain_a_" + alice.hex[:8],
+ "Alice",
+ pwd,
+ bob,
+ "explain_b_" + bob.hex[:8],
+ "Bob",
+ )
+ await conn.execute("INSERT INTO chats (id, title) VALUES ($1,$2)", chat, "Explain chat")
+ await conn.execute(
+ "INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2),($1,$3)",
+ chat,
+ alice,
+ bob,
+ )
+ n = int(os.environ.get("EXPLAIN_MESSAGES", "10000"))
+ await conn.execute(
+ """
+ INSERT INTO messages (id, chat_id, sender_id, text, created_at)
+ SELECT gen_random_uuid(), $1, $2, 'seed-' || g, now() - (g || ' milliseconds')::interval
+ FROM generate_series(1, $3) AS g
+ """,
+ chat,
+ alice,
+ n,
+ )
+ count = await conn.fetchval("SELECT count(*) FROM messages WHERE chat_id = $1", chat)
+ print("seeded_messages", count, "chat_id", chat)
+ first = await conn.fetchrow(
+ "SELECT created_at, id FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT 1",
+ chat,
+ )
+ query = """
+ SELECT id, chat_id, sender_id, text, created_at
+ FROM messages
+ WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
+ ORDER BY created_at, id
+ LIMIT 50
+ """
+ print("\n=== WITH index messages_chat_created_id_idx ===")
+ plan = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"])
+ print("\n".join(row[0] for row in plan))
+ await conn.execute("SET enable_indexscan = off")
+ await conn.execute("SET enable_bitmapscan = off")
+ await conn.execute("SET enable_indexonlyscan = off")
+ print("\n=== WITHOUT index scans (seq scan baseline) ===")
+ plan2 = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"])
+ print("\n".join(row[0] for row in plan2))
+ finally:
+ await conn.close()
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/app/validation.py b/app/validation.py
new file mode 100644
index 00000000..9723267b
--- /dev/null
+++ b/app/validation.py
@@ -0,0 +1,102 @@
+import json
+import re
+from typing import Any, Dict, Optional, Tuple
+from uuid import UUID
+
+USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
+IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
+
+
+def parse_json_body(raw: bytes) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
+ if not raw:
+ return None, "empty body"
+ try:
+ data = json.loads(raw.decode("utf-8"))
+ except (UnicodeDecodeError, json.JSONDecodeError):
+ return None, "malformed JSON"
+ if not isinstance(data, dict):
+ return None, "JSON body must be an object"
+ return data, None
+
+
+def require_exact_keys(data: Dict[str, Any], allowed: set, required: set) -> Optional[str]:
+ keys = set(data.keys())
+ if keys - allowed:
+ return "unexpected fields"
+ if required - keys:
+ return "missing required fields"
+ return None
+
+
+def validate_username(value: Any) -> Optional[str]:
+ if not isinstance(value, str) or not USERNAME_RE.fullmatch(value):
+ return "invalid username"
+ return None
+
+
+def validate_display_name(value: Any) -> Optional[str]:
+ if not isinstance(value, str):
+ return "invalid display_name"
+ if not value.strip():
+ return "display_name must not be blank"
+ if len(value) > 100:
+ return "display_name too long"
+ return None
+
+
+def validate_password(value: Any) -> Optional[str]:
+ if not isinstance(value, str):
+ return "invalid password"
+ if not value.strip():
+ return "password must not be blank"
+ if len(value) < 12 or len(value) > 128:
+ return "password length"
+ return None
+
+
+def validate_title(value: Any) -> Optional[str]:
+ if not isinstance(value, str):
+ return "invalid title"
+ if not value.strip():
+ return "title must not be blank"
+ if len(value) > 100:
+ return "title too long"
+ return None
+
+
+def validate_uuid(value: Any) -> Optional[str]:
+ if not isinstance(value, str):
+ return "invalid uuid"
+ try:
+ UUID(value)
+ except ValueError:
+ return "invalid uuid"
+ return None
+
+
+def validate_idempotency_key(value: Optional[str]) -> Optional[str]:
+ if value is None or IDEMPOTENCY_RE.fullmatch(value) is None:
+ return "invalid idempotency key"
+ return None
+
+
+def validate_message_text(value: Any) -> Optional[str]:
+ if not isinstance(value, str):
+ return "invalid text"
+ if not value.strip():
+ return "text must not be blank"
+ if len(value) > 4000:
+ return "text too long"
+ return None
+
+
+def validate_limit(value: Optional[str], default: int = 50) -> Tuple[Optional[int], Optional[str]]:
+ if value is None:
+ return default, None
+ try:
+ limit = int(value)
+ except ValueError:
+ return None, "invalid limit"
+ if limit < 1 or limit > 100:
+ return None, "limit out of range"
+ return limit, None
diff --git a/compose.yaml b/compose.yaml
new file mode 100644
index 00000000..a1ee4f8c
--- /dev/null
+++ b/compose.yaml
@@ -0,0 +1,131 @@
+# Client -> lb:8080 -> api (scale) -> shared Postgres and Redis.
+# API replicas do not publish host ports. Migrations run once in `migrate`.
+# Check period: HAProxy inter 1s, fall 2, rise 1. Worst detection of a dead
+# or not-ready replica is about 2s, plus the 1s DNS hold. API on SIGTERM
+# fails /health/ready immediately and exits after LB_DRAIN_SECONDS (default 2).
+
+services:
+ migrate:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ image: mtusi-messenger:lab3
+ command: ["python", "-m", "app.migrate"]
+ environment:
+ APP_ENV: lab
+ LOG_LEVEL: info
+ DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
+ DB_POOL_MIN: "1"
+ DB_POOL_MAX: "2"
+ MIGRATIONS_ON_START: "1"
+ depends_on:
+ postgres:
+ condition: service_healthy
+ restart: "no"
+ mem_limit: 256m
+ cpus: "0.50"
+
+ api:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ image: mtusi-messenger:lab3
+ environment:
+ APP_HOST: 0.0.0.0
+ APP_PORT: "8080"
+ APP_ENV: lab
+ LOG_LEVEL: info
+ SHUTDOWN_GRACE_SECONDS: "3"
+ LB_DRAIN_SECONDS: ${LB_DRAIN_SECONDS:-2}
+ SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
+ CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-60}
+ IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
+ SESSION_STORE: redis
+ DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
+ REDIS_URL: redis://redis:6379/0
+ DB_POOL_MIN: "1"
+ DB_POOL_MAX: ${DB_POOL_MAX:-4}
+ DB_COMMAND_TIMEOUT: "5"
+ DB_CONNECT_TIMEOUT: "5"
+ MIGRATIONS_ON_START: "0"
+ depends_on:
+ migrate:
+ condition: service_completed_successfully
+ postgres:
+ condition: service_healthy
+ redis:
+ condition: service_healthy
+ expose:
+ - "8080"
+ restart: unless-stopped
+ mem_limit: ${API_MEM:-256m}
+ cpus: ${API_CPUS:-0.50}
+ stop_grace_period: 8s
+ healthcheck:
+ test:
+ [
+ "CMD",
+ "python",
+ "-c",
+ "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)",
+ ]
+ interval: 5s
+ timeout: 3s
+ retries: 6
+ start_period: 5s
+
+ lb:
+ image: haproxy:2.8.14-alpine
+ volumes:
+ - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+ ports:
+ - "127.0.0.1:8080:8080"
+ depends_on:
+ api:
+ condition: service_started
+ restart: unless-stopped
+ mem_limit: 128m
+ cpus: "0.25"
+
+ postgres:
+ image: postgres:16.6-alpine
+ environment:
+ POSTGRES_USER: messenger
+ POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+ POSTGRES_DB: messenger
+ volumes:
+ - pg_data:/var/lib/postgresql/data
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
+ interval: 3s
+ timeout: 3s
+ retries: 20
+ restart: unless-stopped
+ mem_limit: ${POSTGRES_MEM:-512m}
+ cpus: ${POSTGRES_CPUS:-1.0}
+
+ redis:
+ image: redis:7.4.2-alpine
+ command:
+ - redis-server
+ - --appendonly
+ - "yes"
+ - --appendfsync
+ - everysec
+ - --save
+ - "60"
+ - "1"
+ volumes:
+ - redis_data:/data
+ healthcheck:
+ test: ["CMD", "redis-cli", "ping"]
+ interval: 3s
+ timeout: 3s
+ retries: 20
+ restart: unless-stopped
+ mem_limit: 128m
+ cpus: "0.50"
+
+volumes:
+ pg_data:
+ redis_data:
diff --git a/evidence/grade4/rolling-load.json b/evidence/grade4/rolling-load.json
new file mode 100644
index 00000000..2e6941a8
--- /dev/null
+++ b/evidence/grade4/rolling-load.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 50.0,
+ "elapsed_seconds": 50.101,
+ "concurrency": 4,
+ "successful_requests": 8361,
+ "failed_requests": 18,
+ "error_ratio": 0.002148,
+ "successful_rps": 166.884,
+ "successful_latency_ms": {
+ "p50": 13.481,
+ "p95": 57.817,
+ "p99": 96.441
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/grade4/summary.json b/evidence/grade4/summary.json
new file mode 100644
index 00000000..7893768b
--- /dev/null
+++ b/evidence/grade4/summary.json
@@ -0,0 +1,70 @@
+{
+ "instances_before": [
+ "03c3a0545628",
+ "b555f8327eee"
+ ],
+ "parallel_idempotency": {
+ "workers": 16,
+ "successes": 16,
+ "errors": 0,
+ "distinct_message_ids": 1,
+ "history_copies": 1
+ },
+ "failovers": [
+ {
+ "target": "03c3a0545628",
+ "hook_exit": 0,
+ "hook_seconds": 2.373,
+ "hook_stderr": "",
+ "failed_attempts": 2,
+ "successes_after_hook_start": 130,
+ "first_recovery_seconds": 0.034,
+ "tail_instance_ids": [
+ "b555f8327eee",
+ "b555f8327eee",
+ "b555f8327eee",
+ "b555f8327eee",
+ "b555f8327eee"
+ ],
+ "target_still_in_tail": false
+ },
+ {
+ "target": "b555f8327eee",
+ "hook_exit": 0,
+ "hook_seconds": 1.744,
+ "hook_stderr": "",
+ "failed_attempts": 1,
+ "successes_after_hook_start": 132,
+ "first_recovery_seconds": 0.038,
+ "tail_instance_ids": [
+ "03c3a0545628",
+ "03c3a0545628",
+ "03c3a0545628",
+ "03c3a0545628",
+ "03c3a0545628"
+ ],
+ "target_still_in_tail": false
+ }
+ ],
+ "rolling": {
+ "rolling_seconds": 5.929,
+ "rolling_stdout": "replacing b555f8327eee\nreplacing 03c3a0545628\nrolling update finished; replicas=2\n",
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 50.0,
+ "elapsed_seconds": 50.101,
+ "concurrency": 4,
+ "successful_requests": 8361,
+ "failed_requests": 18,
+ "error_ratio": 0.002148,
+ "successful_rps": 166.884,
+ "successful_latency_ms": {
+ "p50": 13.481,
+ "p95": 57.817,
+ "p99": 96.441
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ }
+ }
+}
diff --git a/evidence/public-checks.txt b/evidence/public-checks.txt
new file mode 100644
index 00000000..711ae329
--- /dev/null
+++ b/evidence/public-checks.txt
@@ -0,0 +1,29 @@
+Date: 2026-11-07
+Host: 12th Gen Intel Core i5-12450H, Docker Desktop ncpu=12, MemTotal=8181665792
+Engine: Docker 29.5.3, Compose 5.1.4
+Stack: haproxy:2.8.14-alpine, mtusi-messenger:lab3 (python:3.12.8-slim-bookworm), postgres:16.6-alpine, redis:7.4.2-alpine
+
+python -X utf8 scripts/check_contract.py
+PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+
+python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
+PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
+
+python -X utf8 -m pytest student_tests -q -m "not infra and not expiry"
+15 passed, 3 deselected
+
+Official persistence and failover were executed by Linux python3 in docker:27-cli
+with the host Docker socket and BASE_URL=http://host.docker.internal:8080,
+because Windows CreateProcess cannot run scripts/restart.sh (WinError 193).
+The hooks and the API containers are the host stack.
+
+python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh
+PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+
+python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh
+{"observed_instances_before": 2, "successful_writes": 109, "failed_attempts": 1, "first_recovery_seconds": 2.465}
+PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+
+After that failover, docker inspect showed mtusi2026lab3-api-1 Exited (0),
+hostname b555f8327eee. mtusi2026lab3-api-2 stayed up, hostname 03c3a0545628.
+Exit code 0 means the process left on SIGTERM before the stop timeout.
diff --git a/evidence/track1/A1.json b/evidence/track1/A1.json
new file mode 100644
index 00000000..81e232fe
--- /dev/null
+++ b/evidence/track1/A1.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.109,
+ "concurrency": 8,
+ "successful_requests": 5197,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 343.964,
+ "successful_latency_ms": {
+ "p50": 19.732,
+ "p95": 42.061,
+ "p99": 81.001
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/A1.point.json b/evidence/track1/A1.point.json
new file mode 100644
index 00000000..3a986495
--- /dev/null
+++ b/evidence/track1/A1.point.json
@@ -0,0 +1,38 @@
+{
+ "tag": "A1",
+ "replicas": 1,
+ "api_cpus_each": "1.0",
+ "api_mem_each": "512m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 8,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.109,
+ "concurrency": 8,
+ "successful_requests": 5197,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 343.964,
+ "successful_latency_ms": {
+ "p50": 19.732,
+ "p95": 42.061,
+ "p99": 81.001
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB",
+ "mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+}
diff --git a/evidence/track1/A2.json b/evidence/track1/A2.json
new file mode 100644
index 00000000..f228ca31
--- /dev/null
+++ b/evidence/track1/A2.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.206,
+ "concurrency": 8,
+ "successful_requests": 2985,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 196.3,
+ "successful_latency_ms": {
+ "p50": 26.613,
+ "p95": 89.893,
+ "p99": 123.339
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/A2.point.json b/evidence/track1/A2.point.json
new file mode 100644
index 00000000..bb805c1c
--- /dev/null
+++ b/evidence/track1/A2.point.json
@@ -0,0 +1,39 @@
+{
+ "tag": "A2",
+ "replicas": 2,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.206,
+ "concurrency": 8,
+ "successful_requests": 2985,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 196.3,
+ "successful_latency_ms": {
+ "p50": 26.613,
+ "p95": 89.893,
+ "p99": 123.339
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB",
+ "mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+}
diff --git a/evidence/track1/A4.json b/evidence/track1/A4.json
new file mode 100644
index 00000000..7eefffc6
--- /dev/null
+++ b/evidence/track1/A4.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.224,
+ "concurrency": 8,
+ "successful_requests": 1895,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 124.476,
+ "successful_latency_ms": {
+ "p50": 39.136,
+ "p95": 185.654,
+ "p99": 267.62
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/A4.point.json b/evidence/track1/A4.point.json
new file mode 100644
index 00000000..87e0fcf0
--- /dev/null
+++ b/evidence/track1/A4.point.json
@@ -0,0 +1,41 @@
+{
+ "tag": "A4",
+ "replicas": 4,
+ "api_cpus_each": "0.25",
+ "api_mem_each": "128m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 2,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.224,
+ "concurrency": 8,
+ "successful_requests": 1895,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 124.476,
+ "successful_latency_ms": {
+ "p50": 39.136,
+ "p95": 185.654,
+ "p99": 267.62
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB",
+ "mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB",
+ "mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB",
+ "mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB",
+ "mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+}
diff --git a/evidence/track1/B1.json b/evidence/track1/B1.json
new file mode 100644
index 00000000..af7b2ad4
--- /dev/null
+++ b/evidence/track1/B1.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.139,
+ "concurrency": 8,
+ "successful_requests": 1485,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 98.094,
+ "successful_latency_ms": {
+ "p50": 85.074,
+ "p95": 167.209,
+ "p99": 201.857
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/B1.point.json b/evidence/track1/B1.point.json
new file mode 100644
index 00000000..20b66ffc
--- /dev/null
+++ b/evidence/track1/B1.point.json
@@ -0,0 +1,38 @@
+{
+ "tag": "B1",
+ "replicas": 1,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 0.5,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 4,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.139,
+ "concurrency": 8,
+ "successful_requests": 1485,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 98.094,
+ "successful_latency_ms": {
+ "p50": 85.074,
+ "p95": 167.209,
+ "p99": 201.857
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB"
+ ],
+ "postgres_sessions": "5"
+ }
+}
diff --git a/evidence/track1/B2.json b/evidence/track1/B2.json
new file mode 100644
index 00000000..393fe4c9
--- /dev/null
+++ b/evidence/track1/B2.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.136,
+ "concurrency": 8,
+ "successful_requests": 3388,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 223.842,
+ "successful_latency_ms": {
+ "p50": 24.1,
+ "p95": 81.009,
+ "p99": 142.445
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/B2.point.json b/evidence/track1/B2.point.json
new file mode 100644
index 00000000..10cd6d82
--- /dev/null
+++ b/evidence/track1/B2.point.json
@@ -0,0 +1,39 @@
+{
+ "tag": "B2",
+ "replicas": 2,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.136,
+ "concurrency": 8,
+ "successful_requests": 3388,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 223.842,
+ "successful_latency_ms": {
+ "p50": 24.1,
+ "p95": 81.009,
+ "p99": 142.445
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB",
+ "mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+}
diff --git a/evidence/track1/B4.json b/evidence/track1/B4.json
new file mode 100644
index 00000000..fb3cd66c
--- /dev/null
+++ b/evidence/track1/B4.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.181,
+ "concurrency": 8,
+ "successful_requests": 3592,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 236.616,
+ "successful_latency_ms": {
+ "p50": 22.559,
+ "p95": 80.196,
+ "p99": 131.013
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/B4.point.json b/evidence/track1/B4.point.json
new file mode 100644
index 00000000..7bed65df
--- /dev/null
+++ b/evidence/track1/B4.point.json
@@ -0,0 +1,41 @@
+{
+ "tag": "B4",
+ "replicas": 4,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 2.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 16,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.181,
+ "concurrency": 8,
+ "successful_requests": 3592,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 236.616,
+ "successful_latency_ms": {
+ "p50": 22.559,
+ "p95": 80.196,
+ "p99": 131.013
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB",
+ "mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB",
+ "mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB",
+ "mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB"
+ ],
+ "postgres_sessions": "17"
+ }
+}
diff --git a/evidence/track1/C4-pg2.json b/evidence/track1/C4-pg2.json
new file mode 100644
index 00000000..53f3215b
--- /dev/null
+++ b/evidence/track1/C4-pg2.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.128,
+ "concurrency": 8,
+ "successful_requests": 3800,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 251.189,
+ "successful_latency_ms": {
+ "p50": 19.749,
+ "p95": 77.161,
+ "p99": 101.17
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/C4-pg2.point.json b/evidence/track1/C4-pg2.point.json
new file mode 100644
index 00000000..7a43b388
--- /dev/null
+++ b/evidence/track1/C4-pg2.point.json
@@ -0,0 +1,41 @@
+{
+ "tag": "C4-pg2",
+ "replicas": 4,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 2.0,
+ "postgres_cpus": "2.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 16,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.128,
+ "concurrency": 8,
+ "successful_requests": 3800,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 251.189,
+ "successful_latency_ms": {
+ "p50": 19.749,
+ "p95": 77.161,
+ "p99": 101.17
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB",
+ "mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB",
+ "mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB",
+ "mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB"
+ ],
+ "postgres_sessions": "17"
+ }
+}
diff --git a/evidence/track1/D4-pool2.json b/evidence/track1/D4-pool2.json
new file mode 100644
index 00000000..c9e683e4
--- /dev/null
+++ b/evidence/track1/D4-pool2.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.156,
+ "concurrency": 8,
+ "successful_requests": 3622,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 238.98,
+ "successful_latency_ms": {
+ "p50": 22.721,
+ "p95": 74.859,
+ "p99": 98.063
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/track1/D4-pool2.point.json b/evidence/track1/D4-pool2.point.json
new file mode 100644
index 00000000..80c5703c
--- /dev/null
+++ b/evidence/track1/D4-pool2.point.json
@@ -0,0 +1,41 @@
+{
+ "tag": "D4-pool2",
+ "replicas": 4,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 2.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 2,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.156,
+ "concurrency": 8,
+ "successful_requests": 3622,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 238.98,
+ "successful_latency_ms": {
+ "p50": 22.721,
+ "p95": 74.859,
+ "p99": 98.063
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB",
+ "mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB",
+ "mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB",
+ "mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+}
diff --git a/evidence/track1/rps.svg b/evidence/track1/rps.svg
new file mode 100644
index 00000000..d97a031a
--- /dev/null
+++ b/evidence/track1/rps.svg
@@ -0,0 +1,29 @@
+
+
diff --git a/evidence/track1/summary.json b/evidence/track1/summary.json
new file mode 100644
index 00000000..f4916581
--- /dev/null
+++ b/evidence/track1/summary.json
@@ -0,0 +1,320 @@
+[
+ {
+ "tag": "A1",
+ "replicas": 1,
+ "api_cpus_each": "1.0",
+ "api_mem_each": "512m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 8,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.109,
+ "concurrency": 8,
+ "successful_requests": 5197,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 343.964,
+ "successful_latency_ms": {
+ "p50": 19.732,
+ "p95": 42.061,
+ "p99": 81.001
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB",
+ "mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+ },
+ {
+ "tag": "A2",
+ "replicas": 2,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.206,
+ "concurrency": 8,
+ "successful_requests": 2985,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 196.3,
+ "successful_latency_ms": {
+ "p50": 26.613,
+ "p95": 89.893,
+ "p99": 123.339
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB",
+ "mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+ },
+ {
+ "tag": "A4",
+ "replicas": 4,
+ "api_cpus_each": "0.25",
+ "api_mem_each": "128m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 2,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.224,
+ "concurrency": 8,
+ "successful_requests": 1895,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 124.476,
+ "successful_latency_ms": {
+ "p50": 39.136,
+ "p95": 185.654,
+ "p99": 267.62
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB",
+ "mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB",
+ "mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB",
+ "mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB",
+ "mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+ },
+ {
+ "tag": "B1",
+ "replicas": 1,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 0.5,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 4,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.139,
+ "concurrency": 8,
+ "successful_requests": 1485,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 98.094,
+ "successful_latency_ms": {
+ "p50": 85.074,
+ "p95": 167.209,
+ "p99": 201.857
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB"
+ ],
+ "postgres_sessions": "5"
+ }
+ },
+ {
+ "tag": "B2",
+ "replicas": 2,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 1.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.136,
+ "concurrency": 8,
+ "successful_requests": 3388,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 223.842,
+ "successful_latency_ms": {
+ "p50": 24.1,
+ "p95": 81.009,
+ "p99": 142.445
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB",
+ "mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+ },
+ {
+ "tag": "B4",
+ "replicas": 4,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 2.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 16,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.181,
+ "concurrency": 8,
+ "successful_requests": 3592,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 236.616,
+ "successful_latency_ms": {
+ "p50": 22.559,
+ "p95": 80.196,
+ "p99": 131.013
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB",
+ "mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB",
+ "mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB",
+ "mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB"
+ ],
+ "postgres_sessions": "17"
+ }
+ },
+ {
+ "tag": "C4-pg2",
+ "replicas": 4,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 2.0,
+ "postgres_cpus": "2.0",
+ "db_pool_max_each": 4,
+ "db_pool_max_total": 16,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.128,
+ "concurrency": 8,
+ "successful_requests": 3800,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 251.189,
+ "successful_latency_ms": {
+ "p50": 19.749,
+ "p95": 77.161,
+ "p99": 101.17
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB",
+ "mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB",
+ "mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB",
+ "mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB"
+ ],
+ "postgres_sessions": "17"
+ }
+ },
+ {
+ "tag": "D4-pool2",
+ "replicas": 4,
+ "api_cpus_each": "0.50",
+ "api_mem_each": "256m",
+ "api_cpu_total": 2.0,
+ "postgres_cpus": "1.0",
+ "db_pool_max_each": 2,
+ "db_pool_max_total": 8,
+ "load": {
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 15.0,
+ "elapsed_seconds": 15.156,
+ "concurrency": 8,
+ "successful_requests": 3622,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 238.98,
+ "successful_latency_ms": {
+ "p50": 22.721,
+ "p95": 74.859,
+ "p99": 98.063
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+ },
+ "resources": {
+ "docker_stats": [
+ "mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB",
+ "mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB",
+ "mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB",
+ "mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB",
+ "mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB",
+ "mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB",
+ "mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB",
+ "mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB",
+ "mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB"
+ ],
+ "postgres_sessions": "9"
+ }
+ }
+]
diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg
new file mode 100644
index 00000000..24c99b80
--- /dev/null
+++ b/infra/haproxy.cfg
@@ -0,0 +1,35 @@
+global
+ log stdout format raw local0 info
+ maxconn 200
+
+defaults
+ log global
+ mode http
+ option httplog
+ timeout connect 2s
+ timeout client 10s
+ timeout server 10s
+ timeout http-request 5s
+ timeout check 2s
+ timeout queue 2s
+ retries 1
+
+resolvers docker
+ nameserver dns 127.0.0.11:53
+ resolve_retries 3
+ timeout resolve 1s
+ timeout retry 1s
+ hold valid 1s
+ accepted_payload_size 8192
+
+frontend fe_http
+ bind *:8080
+ default_backend be_api
+
+backend be_api
+ balance roundrobin
+ option httpchk
+ http-check send meth GET uri /health/ready ver HTTP/1.1 hdr host api
+ http-check expect status 200
+ default-server inter 1s fall 2 rise 1
+ server-template srv 8 api:8080 check resolvers docker init-addr none resolve-prefer ipv4
diff --git a/migrations/001_init.sql b/migrations/001_init.sql
new file mode 100644
index 00000000..2501deea
--- /dev/null
+++ b/migrations/001_init.sql
@@ -0,0 +1,49 @@
+-- Lab2 schema (written before application queries).
+-- Source of truth: PostgreSQL for users, chats, membership, messages.
+-- Transaction boundary: INSERT chat + INSERT members in one transaction (rollback together).
+-- Sessions are stored in Redis with AOF, not in this database.
+
+CREATE TABLE IF NOT EXISTS schema_migrations (
+ filename TEXT PRIMARY KEY,
+ applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE TABLE users (
+ id UUID PRIMARY KEY,
+ username TEXT NOT NULL,
+ display_name TEXT NOT NULL,
+ password_hash TEXT NOT NULL,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
+ CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
+);
+
+CREATE UNIQUE INDEX users_username_key ON users (username);
+
+CREATE TABLE chats (
+ id UUID PRIMARY KEY,
+ title TEXT NOT NULL,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
+);
+
+CREATE TABLE chat_members (
+ chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
+ user_id UUID NOT NULL REFERENCES users (id),
+ PRIMARY KEY (chat_id, user_id)
+);
+
+CREATE INDEX chat_members_user_id_idx ON chat_members (user_id);
+CREATE INDEX chats_created_at_id_idx ON chats (created_at, id);
+
+CREATE TABLE messages (
+ id UUID PRIMARY KEY,
+ chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
+ sender_id UUID NOT NULL REFERENCES users (id),
+ text TEXT NOT NULL,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000)
+);
+
+-- Keyset history: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n
+CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id);
diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql
new file mode 100644
index 00000000..acc555c6
--- /dev/null
+++ b/migrations/002_idempotency.sql
@@ -0,0 +1,14 @@
+-- Shared idempotency for POST /messages. Scope: user + method + path + key.
+-- The message row and this result are committed in one transaction.
+-- A rolled-back request leaves no row, so a later retry with the same key can insert.
+
+CREATE TABLE IF NOT EXISTS idempotency_keys (
+ user_id UUID NOT NULL REFERENCES users (id),
+ http_method TEXT NOT NULL,
+ path TEXT NOT NULL,
+ idem_key TEXT NOT NULL,
+ request_hash TEXT NOT NULL,
+ response_json TEXT NOT NULL,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ PRIMARY KEY (user_id, http_method, path, idem_key)
+);
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 00000000..ce023a9a
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,6 @@
+[pytest]
+testpaths = student_tests
+pythonpath = .
+markers =
+ infra: compose stop/restart of redis or postgres
+ expiry: requires SESSION_TTL_SECONDS<=3
diff --git a/requirements-test.txt b/requirements-test.txt
new file mode 100644
index 00000000..f7f80320
--- /dev/null
+++ b/requirements-test.txt
@@ -0,0 +1,2 @@
+pytest==8.3.4
+httpx==0.28.1
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 00000000..4a028ebb
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,5 @@
+starlette==0.41.3
+uvicorn[standard]==0.32.1
+asyncpg==0.30.0
+redis==5.2.1
+argon2-cffi==23.1.0
diff --git a/scripts/grade4_evidence.py b/scripts/grade4_evidence.py
new file mode 100644
index 00000000..4ea1029a
--- /dev/null
+++ b/scripts/grade4_evidence.py
@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+"""Grade 4: parallel idempotency, stop each API replica, rolling replacement under load."""
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+import uuid
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "tests"))
+
+from client import Client, Failure # noqa: E402
+
+OUT = ROOT / "evidence" / "grade4"
+BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
+
+
+def compose(*args, check=True):
+ cmd = ["docker", "compose"]
+ if (ROOT / ".env").is_file():
+ cmd.extend(["--env-file", str(ROOT / ".env")])
+ cmd.extend(args)
+ proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
+ if check and proc.returncode != 0:
+ raise SystemExit("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or proc.stdout)[-800:]))
+ return proc
+
+
+def bash_argv(script: Path):
+ if os.name != "nt":
+ return [str(script)]
+ git_bash = Path(r"C:\Program Files\Git\bin\bash.exe")
+ if git_bash.is_file():
+ return [str(git_bash), str(script)]
+ return ["bash", str(script)]
+
+
+def hook_cmd():
+ return bash_argv(ROOT / "scripts" / "stop-one.sh")
+
+
+def wait_ready(seconds=60):
+ deadline = time.monotonic() + seconds
+ last = None
+ while time.monotonic() < deadline:
+ try:
+ Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health")
+ return
+ except Failure as exc:
+ last = exc
+ time.sleep(0.4)
+ raise SystemExit("API did not become ready: %s" % last)
+
+
+def instance_ids(actor, samples=24):
+ found = []
+ client = Client(BASE, timeout=3)
+ for _ in range(samples):
+ found.append(client.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"])
+ return sorted(set(found))
+
+
+def parallel_idempotency():
+ client = Client(BASE, timeout=20)
+ alice, _bob, _eve, chat = client.bootstrap()
+ key = uuid.uuid4().hex
+ text = "parallel-" + key
+ ids = []
+ errors = []
+ barrier = threading.Barrier(16)
+
+ def one():
+ local = Client(BASE, timeout=20)
+ barrier.wait()
+ try:
+ ids.append(local.message(alice, chat, text, key=key)["json"]["id"])
+ except Exception as exc:
+ errors.append(type(exc).__name__)
+
+ threads = [threading.Thread(target=one) for _ in range(16)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+ unique = sorted(set(ids))
+ report = {
+ "workers": 16,
+ "successes": len(ids),
+ "errors": len(errors),
+ "distinct_message_ids": len(unique),
+ }
+ if len(unique) != 1 or errors:
+ raise SystemExit("parallel idempotency failed: %s" % report)
+ page = client.call(
+ "GET",
+ "/api/v1/chats/%s/messages?limit=100" % chat["id"],
+ actor=alice,
+ schema="MessagePage",
+ )["json"]
+ copies = [item for item in page["items"] if item["text"] == text]
+ report["history_copies"] = len(copies)
+ if len(copies) != 1:
+ raise SystemExit("history has %s copies" % len(copies))
+ return report
+
+
+def failover_one(actor, chat, instance_id):
+ key = "failover-" + uuid.uuid4().hex
+ text = "during-" + key
+ successes = []
+ failures = 0
+ started_box = {}
+
+ def traffic():
+ nonlocal failures
+ local = Client(BASE, timeout=2)
+ while "t0" not in started_box:
+ time.sleep(0.01)
+ deadline = started_box["t0"] + 12
+ recovered = False
+ while time.monotonic() < deadline:
+ try:
+ if not recovered:
+ response = local.message(actor, chat, text, key=key)
+ recovered = True
+ else:
+ response = local.message(actor, chat, "after-" + uuid.uuid4().hex)
+ successes.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"]))
+ except Failure:
+ failures += 1
+ time.sleep(0.05)
+
+ worker = threading.Thread(target=traffic)
+ worker.start()
+ time.sleep(0.2)
+ started = time.monotonic()
+ started_box["t0"] = started
+ env = os.environ.copy()
+ env["TARGET_INSTANCE_ID"] = instance_id
+ proc = subprocess.run(hook_cmd(), cwd=ROOT, env=env, capture_output=True, text=True)
+ hook_s = time.monotonic() - started
+ worker.join()
+ after = [item for item in successes if item[0] >= started]
+ recovery = None
+ if after:
+ recovery = round(after[0][0] - started, 3)
+ tail = [item[1] for item in after[-5:]]
+ return {
+ "target": instance_id,
+ "hook_exit": proc.returncode,
+ "hook_seconds": round(hook_s, 3),
+ "hook_stderr": (proc.stderr or "")[-300:],
+ "failed_attempts": failures,
+ "successes_after_hook_start": len(after),
+ "first_recovery_seconds": recovery,
+ "tail_instance_ids": tail,
+ "target_still_in_tail": instance_id in tail,
+ }
+
+
+def restore_and_wait(actor):
+ compose("start", "api")
+ deadline = time.monotonic() + 40
+ while time.monotonic() < deadline:
+ try:
+ if len(instance_ids(actor, samples=16)) >= 2:
+ return
+ except Failure:
+ pass
+ time.sleep(0.5)
+ raise SystemExit("second replica did not return")
+
+
+def rolling_under_load():
+ out = OUT / "rolling-load.json"
+ load = subprocess.Popen(
+ [sys.executable, "-X", "utf8", str(ROOT / "tests" / "load.py"),
+ "--base-url", BASE, "--duration", "50", "--concurrency", "4", "--output", str(out)],
+ cwd=ROOT,
+ )
+ time.sleep(3)
+ cmd = bash_argv(ROOT / "scripts" / "rolling-update.sh")
+ started = time.monotonic()
+ proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
+ elapsed = round(time.monotonic() - started, 3)
+ load.wait(timeout=90)
+ metrics = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
+ if proc.returncode != 0:
+ raise SystemExit("rolling update failed: %s" % (proc.stderr or proc.stdout)[-800:])
+ return {
+ "rolling_seconds": elapsed,
+ "rolling_stdout": (proc.stdout or "")[-400:],
+ "load": metrics,
+ }
+
+
+def main():
+ OUT.mkdir(parents=True, exist_ok=True)
+ wait_ready()
+ client = Client(BASE, timeout=10)
+ alice, _bob, _eve, chat = client.bootstrap()
+ before = instance_ids(alice)
+ if len(before) < 2:
+ raise SystemExit("need 2 instance ids before grade-4 runs, saw %s" % before)
+ idem = parallel_idempotency()
+ failovers = []
+ for instance_id in before:
+ item = failover_one(alice, chat, instance_id)
+ failovers.append(item)
+ recovery = item["first_recovery_seconds"]
+ if item["hook_exit"] != 0 or item["target_still_in_tail"] or recovery is None or recovery > 10 or item["successes_after_hook_start"] < 5:
+ raise SystemExit("failover criteria failed: %s" % item)
+ restore_and_wait(alice)
+ rolling = rolling_under_load()
+ report = {
+ "instances_before": before,
+ "parallel_idempotency": idem,
+ "failovers": failovers,
+ "rolling": rolling,
+ }
+ (OUT / "summary.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ print(json.dumps(report, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/restart.bat b/scripts/restart.bat
new file mode 100644
index 00000000..d96ae9d7
--- /dev/null
+++ b/scripts/restart.bat
@@ -0,0 +1,3 @@
+@echo off
+python "%~dp0restart.py"
+exit /b %ERRORLEVEL%
diff --git a/scripts/restart.py b/scripts/restart.py
new file mode 100644
index 00000000..639651e8
--- /dev/null
+++ b/scripts/restart.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+"""Restart API, PostgreSQL and Redis without removing named volumes."""
+from pathlib import Path
+import subprocess
+import sys
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def main() -> int:
+ env_file = ROOT / ".env"
+ cmd = ["docker", "compose"]
+ if env_file.is_file():
+ cmd.extend(["--env-file", str(env_file)])
+ cmd.extend(["restart", "postgres", "redis", "api"])
+ return subprocess.run(cmd, cwd=ROOT).returncode
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/restart.sh b/scripts/restart.sh
new file mode 100755
index 00000000..6a3be3ad
--- /dev/null
+++ b/scripts/restart.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+# Restart API replicas and every store they use. Volumes are kept.
+set -eu
+ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+cd "$ROOT"
+if [ -f .env ]; then
+ docker compose --env-file .env restart postgres redis api
+else
+ docker compose restart postgres redis api
+fi
diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh
new file mode 100755
index 00000000..d22d4a10
--- /dev/null
+++ b/scripts/rolling-update.sh
@@ -0,0 +1,55 @@
+#!/bin/sh
+# Replace API replicas one at a time. The other replica keeps serving.
+set -eu
+ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+cd "$ROOT"
+
+compose() {
+ if [ -f .env ]; then
+ docker compose --env-file .env "$@"
+ else
+ docker compose "$@"
+ fi
+}
+
+if command -v python3 >/dev/null 2>&1; then
+ PY=python3
+else
+ PY=python
+fi
+
+ids=$(compose ps -q api)
+count=0
+for id in $ids; do
+ count=$((count + 1))
+done
+if [ "$count" -lt 2 ]; then
+ echo "rolling update needs at least 2 running API replicas" >&2
+ exit 1
+fi
+
+for id in $ids; do
+ echo "replacing $(docker inspect -f '{{.Config.Hostname}}' "$id")"
+ docker stop -t 8 "$id" >/dev/null
+ docker rm "$id" >/dev/null
+ compose up -d --scale "api=$count" --no-deps --no-recreate api >/dev/null
+ ready=0
+ i=0
+ while [ "$i" -lt 40 ]; do
+ running=0
+ for cid in $(compose ps -q api); do
+ running=$((running + 1))
+ done
+ if [ "$running" -ge "$count" ] && "$PY" -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" >/dev/null 2>&1; then
+ ready=1
+ break
+ fi
+ i=$((i + 1))
+ sleep 1
+ done
+ if [ "$ready" -ne 1 ]; then
+ echo "replacement did not become ready" >&2
+ exit 1
+ fi
+done
+echo "rolling update finished; replicas=$count"
diff --git a/scripts/run_expiry.py b/scripts/run_expiry.py
new file mode 100644
index 00000000..42d87200
--- /dev/null
+++ b/scripts/run_expiry.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""Recreate API with SESSION_TTL_SECONDS=2, test expiry, restore TTL."""
+import os
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def compose(args, extra_env=None):
+ env = os.environ.copy()
+ if extra_env:
+ env.update(extra_env)
+ cmd = ["docker", "compose"]
+ if (ROOT / ".env").is_file():
+ cmd.extend(["--env-file", str(ROOT / ".env")])
+ cmd.extend(args)
+ proc = subprocess.run(cmd, cwd=ROOT, env=env)
+ if proc.returncode != 0:
+ raise SystemExit(proc.returncode)
+
+
+def wait_ready():
+ import urllib.request
+
+ deadline = time.monotonic() + 90
+ while time.monotonic() < deadline:
+ try:
+ urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2)
+ return
+ except Exception:
+ time.sleep(0.5)
+ raise SystemExit("API did not become ready")
+
+
+def main():
+ compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "2"})
+ wait_ready()
+ env = os.environ.copy()
+ env["SESSION_TTL_SECONDS"] = "2"
+ env["RUN_EXPIRY"] = "1"
+ proc = subprocess.run(
+ [sys.executable, "-m", "pytest", "student_tests/test_expiry.py", "-q"],
+ cwd=ROOT,
+ env=env,
+ )
+ compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "3600"})
+ wait_ready()
+ raise SystemExit(proc.returncode)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh
new file mode 100755
index 00000000..707e422d
--- /dev/null
+++ b/scripts/stop-one.sh
@@ -0,0 +1,32 @@
+#!/bin/sh
+# Stop the single API replica whose X-Instance-Id is TARGET_INSTANCE_ID.
+# Leaves it stopped. Does not touch the database, Redis, the load balancer, or volumes.
+set -eu
+: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
+
+ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+cd "$ROOT"
+
+compose() {
+ if [ -f .env ]; then
+ docker compose --env-file .env "$@"
+ else
+ docker compose "$@"
+ fi
+}
+
+found=0
+for id in $(compose ps -q api); do
+ iid=$(docker exec "$id" cat /tmp/instance-id 2>/dev/null | tr -d '\r\n' || true)
+ host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
+ if [ "$iid" = "$TARGET_INSTANCE_ID" ] || [ "$host" = "$TARGET_INSTANCE_ID" ]; then
+ docker stop -t 5 "$id" >/dev/null
+ found=1
+ break
+ fi
+done
+
+if [ "$found" -ne 1 ]; then
+ echo "no running API replica matched TARGET_INSTANCE_ID" >&2
+ exit 1
+fi
diff --git a/scripts/track1_experiment.py b/scripts/track1_experiment.py
new file mode 100644
index 00000000..dcf96bf3
--- /dev/null
+++ b/scripts/track1_experiment.py
@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+"""Track 1: fixed API budget, then more replicas with more CPU, then one DB change.
+
+The load profile is the public closed-loop POST generator. Same duration and
+concurrency on every point. Raw JSON is written under evidence/track1/.
+"""
+import json
+import os
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+OUT = ROOT / "evidence" / "track1"
+BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
+DURATION = float(os.environ.get("TRACK_DURATION", "15"))
+WARMUP = float(os.environ.get("TRACK_WARMUP", "5"))
+CONCURRENCY = int(os.environ.get("TRACK_CONCURRENCY", "8"))
+
+
+def compose(*args):
+ cmd = ["docker", "compose"]
+ if (ROOT / ".env").is_file():
+ cmd.extend(["--env-file", str(ROOT / ".env")])
+ cmd.extend(args)
+ proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, env=os.environ.copy())
+ if proc.returncode != 0:
+ raise SystemExit((proc.stderr or proc.stdout)[-1000:])
+ return proc
+
+
+def wait_ready(seconds=90):
+ import urllib.request
+
+ deadline = time.monotonic() + seconds
+ while time.monotonic() < deadline:
+ try:
+ with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as response:
+ if response.status == 200:
+ return
+ except Exception:
+ time.sleep(0.5)
+ raise SystemExit("ready timeout")
+
+
+def apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool):
+ os.environ["API_CPUS"] = api_cpus
+ os.environ["API_MEM"] = api_mem
+ os.environ["POSTGRES_CPUS"] = postgres_cpus
+ os.environ["DB_POOL_MAX"] = str(pool)
+ compose("up", "-d", "--no-deps", "--force-recreate", "postgres")
+ deadline = time.monotonic() + 40
+ while time.monotonic() < deadline:
+ probe = subprocess.run(
+ ["docker", "compose", "--env-file", str(ROOT / ".env"), "exec", "-T", "postgres", "pg_isready", "-U", "messenger", "-d", "messenger"],
+ cwd=ROOT,
+ capture_output=True,
+ text=True,
+ )
+ if probe.returncode == 0:
+ break
+ time.sleep(0.5)
+ else:
+ raise SystemExit("postgres did not become ready")
+ compose(
+ "up",
+ "-d",
+ "--no-deps",
+ "--force-recreate",
+ "--scale",
+ "api=%s" % replicas,
+ "api",
+ )
+ wait_ready()
+ running = [line for line in compose("ps", "-q", "api").stdout.splitlines() if line.strip()]
+ if len(running) != replicas:
+ raise SystemExit("expected %s api containers, saw %s" % (replicas, len(running)))
+
+
+def run_load(tag, warmup=False):
+ args = [
+ sys.executable,
+ "-X",
+ "utf8",
+ str(ROOT / "tests" / "load.py"),
+ "--base-url",
+ BASE,
+ "--duration",
+ str(WARMUP if warmup else DURATION),
+ "--concurrency",
+ str(CONCURRENCY),
+ ]
+ dest = None
+ if not warmup:
+ dest = OUT / ("%s.json" % tag)
+ args.extend(["--output", str(dest)])
+ proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True)
+ if proc.returncode != 0:
+ raise SystemExit("load %s failed: %s" % (tag, (proc.stderr or proc.stdout)[-800:]))
+ if dest is None:
+ return None
+ return json.loads(dest.read_text(encoding="utf-8"))
+
+
+def sample_resources():
+ stats = subprocess.run(
+ ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
+ capture_output=True,
+ text=True,
+ )
+ activity = compose(
+ "exec",
+ "-T",
+ "postgres",
+ "psql",
+ "-U",
+ "messenger",
+ "-d",
+ "messenger",
+ "-tAc",
+ "SELECT count(*) FROM pg_stat_activity WHERE datname='messenger'",
+ )
+ return {
+ "docker_stats": [line for line in stats.stdout.splitlines() if line.strip()],
+ "postgres_sessions": activity.stdout.strip(),
+ }
+
+
+def measure(tag, replicas, api_cpus, api_mem, postgres_cpus, pool):
+ print("profile %s" % tag, flush=True)
+ apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool)
+ run_load(tag, warmup=True)
+ box = {}
+
+ def snapshot():
+ time.sleep(DURATION / 2)
+ box["resources"] = sample_resources()
+
+ thread = threading.Thread(target=snapshot)
+ thread.start()
+ metrics = run_load(tag)
+ thread.join()
+ point = {
+ "tag": tag,
+ "replicas": replicas,
+ "api_cpus_each": api_cpus,
+ "api_mem_each": api_mem,
+ "api_cpu_total": round(replicas * float(api_cpus), 3),
+ "postgres_cpus": postgres_cpus,
+ "db_pool_max_each": pool,
+ "db_pool_max_total": replicas * pool,
+ "load": metrics,
+ "resources": box.get("resources"),
+ }
+ (OUT / ("%s.point.json" % tag)).write_text(json.dumps(point, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ print(json.dumps({"tag": tag, "rps": metrics["successful_rps"], "errors": metrics["failed_requests"], "p95": metrics["successful_latency_ms"]["p95"]}), flush=True)
+ return point
+
+
+def svg(points, path):
+ width, height = 720, 320
+ labels = [p["tag"] for p in points]
+ values = [p["load"]["successful_rps"] for p in points]
+ peak = max(values) or 1
+ left, top, plot_w, plot_h = 48, 20, width - 70, height - 70
+ bar_w = plot_w / len(points) * 0.62
+ gap = plot_w / len(points)
+ parts = [
+ '',
+ '")
+ path.write_text("\n".join(parts) + "\n", encoding="utf-8")
+
+
+def main():
+ OUT.mkdir(parents=True, exist_ok=True)
+ points = []
+ # Series A: total API budget fixed at 1.0 CPU and 512 MiB. Pool total stays 8.
+ points.append(measure("A1", 1, "1.0", "512m", "1.0", 8))
+ points.append(measure("A2", 2, "0.50", "256m", "1.0", 4))
+ points.append(measure("A4", 4, "0.25", "128m", "1.0", 2))
+ # Series B: each new replica brings the same 0.50 CPU / 256 MiB. Postgres stays at 1 CPU.
+ points.append(measure("B1", 1, "0.50", "256m", "1.0", 4))
+ points.append(measure("B2", 2, "0.50", "256m", "1.0", 4))
+ points.append(measure("B4", 4, "0.50", "256m", "1.0", 4))
+ # Candidate improvement: give the database the extra CPU the API side already has.
+ points.append(measure("C4-pg2", 4, "0.50", "256m", "2.0", 4))
+ # Candidate improvement: stop the 4 pools from opening 16 server connections.
+ points.append(measure("D4-pool2", 4, "0.50", "256m", "1.0", 2))
+ (OUT / "summary.json").write_text(json.dumps(points, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ svg(points, OUT / "rps.svg")
+ # Return the stand to the documented 2 x 0.50 CPU layout.
+ apply_profile(2, "0.50", "256m", "1.0", 4)
+ print("wrote %s" % (OUT / "summary.json"))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/solve.md b/solve.md
new file mode 100644
index 00000000..ca0a1603
--- /dev/null
+++ b/solve.md
@@ -0,0 +1,71 @@
+# Разбор лабораторной 3
+
+Автор: Булацев, БПИ2301. Целевая оценка: 5, трек «Цена масштабирования».
+
+## Суть
+
+Нужно запустить несколько одинаковых процессов API за одной точкой входа и не потерять корректность, когда один из них останавливают.
+
+Клиент больше не обращается к конкретному процессу. Он ходит в балансировщик, а тот выбирает живую реплику. Любая реплика должна узнать ту же сессию, увидеть уже подтверждённое сообщение и не создать второе сообщение, если клиент повторил запрос с тем же ключом.
+
+Источник истины для пользователей, чатов и сообщений — PostgreSQL. Сессии — Redis с AOF. Память процесса для этого не используется.
+
+## Какие проблемы решались
+
+### Один процесс нельзя просто скопировать
+
+Если сессия или кэш лежат в памяти процесса, второй экземпляр пользователя не узнает. Повторный запрос попадёт на другую реплику и получит 401 либо устаревшую историю.
+
+Решение: сессия хранится в общем Redis, история читается из общего Postgres. Кэш истории тоже в Redis и сбрасывается после успешной записи, поэтому следующая реплика не отдаёт старую страницу. Клиента не приклеиваем к одной реплике.
+
+### Клиент не должен выбирать процесс сам
+
+Нужна одна публичная точка входа и при этом несколько процессов. `depends_on` только задаёт порядок запуска контейнеров. Healthcheck контейнера говорит Docker, жив ли он, но не перекладывает HTTP-запрос на другой адрес.
+
+Решение: HAProxy публикует `127.0.0.1:8080` и сам ходит в DNS-имя `api`. У сервиса Compose нет `container_name`, поэтому реплик может быть сколько угодно. Порты API наружу не опубликованы. HAProxy держит до 8 слотов через `server-template` и обновляет адреса из DNS Docker раз в секунду. Проверка — `GET /health/ready` каждые 1 с, реплика выпадает после двух неудач. Это около 2 с плюс до 1 с устаревшего DNS, если контейнер уже исчез.
+
+### Остановка процесса не должна обрывать стенд надолго
+
+Docker посылает SIGTERM. Если процесс сразу умирает, балансировщик ещё считает его живым и шлёт туда запросы.
+
+Решение согласовано с проверкой готовности. По SIGTERM реплика сразу отвечает 503 на `/health/ready` и на новые бизнес-запросы, а завершается через 2 с (`LB_DRAIN_SECONDS`). За это HAProxy успевает увидеть неготовый backend. `scripts/stop-one.sh` останавливает только контейнер с нужным `X-Instance-Id` и оставляет его остановленным. `X-Instance-Id` равен hostname и не меняется, пока жив процесс. На публичной проверке запись восстановилась за 2.465 с, подтверждённые сообщения и сессия сохранились.
+
+### Повтор POST после таймаута может создать второе сообщение
+
+Клиент не знает, успел ли сервер записать сообщение, если ответ потерялся. Повтор без защиты создаёт дубль. Две реплики могут принять один и тот же повтор одновременно.
+
+Решение: для `POST /messages` обязателен `Idempotency-Key`. Область ключа — пользователь, метод, путь чата и сам ключ. Текст приводится к одному JSON (порядок полей не важен, пустой список вложений равен отсутствию поля) и хешируется. Сообщение и сохранённый ответ 201 пишутся одной транзакцией. Пока транзакция не закоммичена, строки результата нет, поэтому неуспешный запрос можно повторить. Одновременные повторы сериализует advisory lock в Postgres: побеждает одна вставка, остальные получают то же сообщение. Другой текст с тем же ключом — 409. Проверено 16 параллельными запросами: в истории одна копия.
+
+### Миграции нельзя применять гонкой всех реплик
+
+Несколько процессов при старте могут одновременно выполнить один и тот же SQL.
+
+Решение: схему применяет отдельный одноразовый сервис `migrate` до старта API. У реплик `MIGRATIONS_ON_START=0`. Повторный запуск пропускает файлы, уже записанные в `schema_migrations`. На время применения берётся `pg_advisory_lock`.
+
+### Больше реплик не означает больше пропускной способности
+
+Гипотеза трека 1: при фиксированном CPU API дробление на процессы не поможет, а при росте числа реплик выигрыш кончится, когда упрёмся не в CPU отдельного API.
+
+Один и тот же closed-loop, 8 параллельных POST, 15 с. При бюджете 1 CPU / 512 МиБ результат 344, 196 и 124 RPS для 1, 2 и 4 реплик. Postgres был занят на 16–33%: мешала квота CPU на каждом процессе. Когда одна реплика упиралась в 0.50 CPU (98 RPS, p95 167 мс), вторая такая же реплика подняла результат до 224 RPS и p95 81 мс. Это подтверждённое улучшение. Четвёртая реплика дала только 237 RPS, зато сессий Postgres стало 17 вместо 9. Удвоение CPU Postgres и уменьшение пула на этом профиле заметного прироста не дали. Подробные таблицы — в `REPORT.md`.
+
+## Ответы на вопросы README
+
+### Чем отказ процесса отличается от отказа узла?
+
+Отказ процесса — это остановка одного API-контейнера. Балансировщик перестаёт слать ему запросы, вторая реплика на том же хосте продолжает читать и писать в общие Postgres и Redis. Данные не пропадают.
+
+Отказ узла — это потеря всей машины. Вместе с ней пропадают все реплики API, HAProxy, Postgres и Redis, потому что они запущены на одном Docker-хосте. Несколько процессов API такую потерю не переживают. Отдельно живая реплика базы в этой работе не поднималась, поэтому выживание узла БД не заявляется.
+
+### Почему depends_on и healthcheck сами по себе не являются балансировщиком?
+
+`depends_on` лишь откладывает старт контейнера, пока другой не стал healthy. Он не принимает соединения клиентов и не выбирает, какому процессу отдать конкретный HTTP-запрос. Healthcheck Docker перезапускает или помечает свой контейнер, но не знает о соседних репликах и не проксирует трафик. Пока снаружи опубликован сам API, клиент попадает в один процесс в обход остальных. Балансировщик — отдельный процесс: он слушает единственный адрес, смотрит готовность backend-ов и перекладывает запрос на живой.
+
+### Какие POST можно повторять после timeout и что защищает Idempotency-Key?
+
+После таймаута можно повторять POST сообщения с тем же ключом и тем же телом. Если запись успела закоммититься, сервер вернёт тот же `201` и то же сообщение. Если не успела, повтор создаст его один раз. Другое тело с тем же ключом получит 409.
+
+Защищает общая строка в Postgres на пару «пользователь + метод + путь + ключ» вместе с advisory lock и одной транзакцией на сообщение и сохранённый ответ. Ключ не защищает создание пользователя и создание чата: их повтор может создать второй объект, поэтому балансировщик такие POST сам не повторяет.
+
+### Что сломается при увеличении числа API, если сессии находятся в локальной памяти?
+
+Вход создаст сессию только в памяти той реплики, которая обработала логин. Следующий запрос балансировщик отдаст другой реплике, и она ответит 401, потому что токена у неё нет. Пользователю придётся входить заново, а при sticky session он останется привязан к одному процессу: отказ этого процесса снова потеряет сессию. Общие сообщения в Postgres это не чинят. Поэтому сессия лежит в Redis, который видят все реплики.
diff --git a/student_tests/__init__.py b/student_tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/student_tests/conftest.py b/student_tests/conftest.py
new file mode 100644
index 00000000..ae6a3291
--- /dev/null
+++ b/student_tests/conftest.py
@@ -0,0 +1,24 @@
+import os
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "tests"))
+from client import Client, Failure # noqa: E402
+
+
+@pytest.fixture(scope="session")
+def base_url():
+ return os.environ.get("BASE_URL", "http://127.0.0.1:8080")
+
+
+@pytest.fixture(scope="session")
+def live(base_url):
+ c = Client(base_url, timeout=15)
+ try:
+ c.call("GET", "/health/ready", schema="Health")
+ except Failure as exc:
+ pytest.skip("API is not ready: %s" % exc)
+ return c
diff --git a/student_tests/test_api.py b/student_tests/test_api.py
new file mode 100644
index 00000000..60971ea1
--- /dev/null
+++ b/student_tests/test_api.py
@@ -0,0 +1,195 @@
+"""Negative, pagination and auth tests for lab 2."""
+import uuid
+from urllib.parse import quote
+
+from client import Client
+
+
+PASSWORD = "Example-only-not-a-real-secret-123"
+
+
+def register(c: Client, username=None):
+ username = username or ("u_" + uuid.uuid4().hex[:10])
+ user = c.call(
+ "POST",
+ "/api/v1/users",
+ 201,
+ body={"username": username, "display_name": "Test User", "password": PASSWORD},
+ schema="User",
+ )["json"]
+ return user
+
+
+def login(c: Client, username, password=PASSWORD):
+ import base64
+
+ cred = base64.b64encode((username + ":" + password).encode()).decode()
+ session = c.call(
+ "POST",
+ "/api/v1/auth/sessions",
+ 201,
+ headers={"Authorization": "Basic " + cred},
+ schema="Session",
+ )["json"]
+ return {"id": session["user"]["id"], "user": session["user"], "token": session["token"], "registration": {"username": username, "password": password}}
+
+
+def actor_of(c: Client, username=None):
+ user = register(c, username)
+ return login(c, user["username"])
+
+
+def test_malformed_json(live: Client):
+ response = live.call(
+ "POST",
+ "/api/v1/users",
+ 400,
+ raw=b"{not-json",
+ headers={"Content-Type": "application/json"},
+ )
+ assert response["json"]["error"]["code"] == "invalid_request"
+ assert response["headers"].get("X-Request-Id")
+
+
+def test_duplicate_username(live: Client):
+ user = register(live, username="dup_" + uuid.uuid4().hex[:8])
+ again = live.call(
+ "POST",
+ "/api/v1/users",
+ 409,
+ body={"username": user["username"], "display_name": "Other", "password": PASSWORD},
+ )
+ assert again["json"]["error"]["code"] == "conflict"
+
+
+def test_empty_message_rejected(live: Client):
+ alice = actor_of(live)
+ bob = actor_of(live)
+ chat = live.call(
+ "POST",
+ "/api/v1/chats",
+ 201,
+ actor=alice,
+ body={"title": "t", "member_ids": [alice["id"], bob["id"]]},
+ schema="Chat",
+ )["json"]
+ response = live.call(
+ "POST",
+ "/api/v1/chats/" + chat["id"] + "/messages",
+ 400,
+ actor=alice,
+ body={"text": " "},
+ )
+ assert response["status"] == 400
+
+
+def test_outsider_chat_is_404(live: Client):
+ alice = actor_of(live)
+ bob = actor_of(live)
+ eve = actor_of(live)
+ chat = live.call(
+ "POST",
+ "/api/v1/chats",
+ 201,
+ actor=alice,
+ body={"title": "secret", "member_ids": [alice["id"], bob["id"]]},
+ schema="Chat",
+ )["json"]
+ response = live.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve)
+ assert response["json"]["error"]["code"] == "not_found"
+
+
+def test_message_pagination_multiple_pages(live: Client):
+ alice = actor_of(live)
+ bob = actor_of(live)
+ chat = live.call(
+ "POST",
+ "/api/v1/chats",
+ 201,
+ actor=alice,
+ body={"title": "pages", "member_ids": [alice["id"], bob["id"]]},
+ schema="Chat",
+ )["json"]
+ created = []
+ for idx in range(5):
+ created.append(live.message(alice, chat, "m%s" % idx)["json"])
+ collected = []
+ cursor = None
+ for _ in range(10):
+ path = "/api/v1/chats/%s/messages?limit=2" % chat["id"]
+ if cursor:
+ path += "&cursor=" + quote(cursor, safe="")
+ page = live.call("GET", path, actor=bob, schema="MessagePage")["json"]
+ collected.extend(page["items"])
+ cursor = page["next_cursor"]
+ if cursor is None:
+ break
+ assert len(collected) == 5
+ ordered = sorted(created, key=lambda x: (x["created_at"], x["id"]))
+ assert collected == ordered
+
+
+def test_cursor_from_other_chat_rejected(live: Client):
+ alice = actor_of(live)
+ bob = actor_of(live)
+ chat_a = live.call(
+ "POST",
+ "/api/v1/chats",
+ 201,
+ actor=alice,
+ body={"title": "a", "member_ids": [alice["id"], bob["id"]]},
+ schema="Chat",
+ )["json"]
+ chat_b = live.call(
+ "POST",
+ "/api/v1/chats",
+ 201,
+ actor=alice,
+ body={"title": "b", "member_ids": [alice["id"], bob["id"]]},
+ schema="Chat",
+ )["json"]
+ msg = live.message(alice, chat_a, "only in a")["json"]
+ page = live.call(
+ "GET",
+ "/api/v1/chats/%s/messages?limit=1" % chat_a["id"],
+ actor=alice,
+ schema="MessagePage",
+ )["json"]
+ assert page["items"][0]["id"] == msg["id"]
+ cursor = page["next_cursor"]
+ # If history is a single item, next_cursor is null; encode a foreign cursor via first page of A after a dummy extra message.
+ if cursor is None:
+ live.message(alice, chat_a, "second")
+ page = live.call(
+ "GET",
+ "/api/v1/chats/%s/messages?limit=1" % chat_a["id"],
+ actor=alice,
+ schema="MessagePage",
+ )["json"]
+ cursor = page["next_cursor"]
+ response = live.call(
+ "GET",
+ "/api/v1/chats/%s/messages?limit=10&cursor=%s" % (chat_b["id"], quote(cursor, safe="")),
+ 400,
+ actor=alice,
+ )
+ assert response["status"] == 400
+
+
+def test_x_user_id_does_not_authenticate(live: Client):
+ alice = actor_of(live)
+ live.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]})
+
+
+def test_wrong_password(live: Client):
+ user = register(live)
+ import base64
+
+ bad = base64.b64encode((user["username"] + ":wrong-password-xx").encode()).decode()
+ live.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + bad})
+
+
+def test_logout(live: Client):
+ alice = actor_of(live)
+ live.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice)
+ live.call("GET", "/api/v1/users/me", 401, actor=alice)
diff --git a/student_tests/test_concurrent.py b/student_tests/test_concurrent.py
new file mode 100644
index 00000000..5265031e
--- /dev/null
+++ b/student_tests/test_concurrent.py
@@ -0,0 +1,48 @@
+"""10 clients × 10 messages = 100 unique messages."""
+import threading
+import uuid
+
+from client import Client
+
+from student_tests.test_api import actor_of
+
+
+def test_concurrent_messages(live: Client):
+ users = [actor_of(live) for _ in range(10)]
+ member_ids = [u["id"] for u in users]
+ chat = live.call(
+ "POST",
+ "/api/v1/chats",
+ 201,
+ actor=users[0],
+ body={"title": "load", "member_ids": member_ids},
+ schema="Chat",
+ )["json"]
+
+ errors = []
+ lock = threading.Lock()
+
+ def worker(user):
+ local = Client(live.base, timeout=15)
+ for _ in range(10):
+ try:
+ local.message(user, chat, uuid.uuid4().hex)
+ except Exception as exc:
+ with lock:
+ errors.append(str(exc))
+
+ threads = [threading.Thread(target=worker, args=(u,)) for u in users]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join(timeout=60)
+
+ assert not errors, "unexpected failures: %s" % errors[:5]
+ page = live.call(
+ "GET",
+ "/api/v1/chats/%s/messages?limit=100" % chat["id"],
+ actor=users[0],
+ schema="MessagePage",
+ )["json"]
+ ids = [item["id"] for item in page["items"]]
+ assert len(ids) == len(set(ids)) == 100
diff --git a/student_tests/test_expiry.py b/student_tests/test_expiry.py
new file mode 100644
index 00000000..da31f3c9
--- /dev/null
+++ b/student_tests/test_expiry.py
@@ -0,0 +1,17 @@
+import os
+import time
+
+import pytest
+from client import Client
+
+
+@pytest.mark.expiry
+def test_session_expires(live: Client):
+ ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
+ if os.environ.get("RUN_EXPIRY") != "1":
+ pytest.skip("set RUN_EXPIRY=1 and SESSION_TTL_SECONDS<=3")
+ assert ttl <= 3
+ actor = live.user("ttl")
+ live.call("GET", "/api/v1/users/me", actor=actor, schema="User")
+ time.sleep(ttl + 1.5)
+ live.call("GET", "/api/v1/users/me", 401, actor=actor)
diff --git a/student_tests/test_grade4.py b/student_tests/test_grade4.py
new file mode 100644
index 00000000..92ceb895
--- /dev/null
+++ b/student_tests/test_grade4.py
@@ -0,0 +1,44 @@
+"""Grade-4: username race and atomic chat rollback."""
+import threading
+import uuid
+
+from client import Client
+
+from student_tests.test_api import PASSWORD, actor_of
+
+
+def test_register_race(live: Client):
+ username = "r_" + uuid.uuid4().hex[:12]
+ body = {"username": username, "display_name": "Race", "password": PASSWORD}
+ results = []
+
+ def once():
+ local = Client(live.base, timeout=20)
+ try:
+ results.append(local.call("POST", "/api/v1/users", [201, 409], body=body)["status"])
+ except Exception as exc:
+ results.append(str(exc))
+
+ threads = [threading.Thread(target=once) for _ in range(16)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ assert results.count(201) == 1, results
+ assert results.count(409) == 15, results
+
+
+def test_chat_rollback(live: Client):
+ alice = actor_of(live)
+ before = live.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"]
+ missing = str(uuid.uuid4())
+ live.call(
+ "POST",
+ "/api/v1/chats",
+ 400,
+ actor=alice,
+ body={"title": "rollback-should-not-exist", "member_ids": [alice["id"], missing]},
+ )
+ after = live.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"]
+ assert len(after) == len(before)
+ assert all(x["title"] != "rollback-should-not-exist" for x in after)
diff --git a/student_tests/test_infra.py b/student_tests/test_infra.py
new file mode 100644
index 00000000..fe1a022e
--- /dev/null
+++ b/student_tests/test_infra.py
@@ -0,0 +1,89 @@
+"""Dependency failure and Redis restart (AOF + cache rebuild)."""
+import os
+import subprocess
+import time
+from pathlib import Path
+
+import pytest
+from client import Client, Failure
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def compose(*args):
+ env_file = ROOT / ".env"
+ cmd = ["docker", "compose"]
+ if env_file.is_file():
+ cmd.extend(["--env-file", str(env_file)])
+ cmd.extend(args)
+ proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
+ if proc.returncode != 0:
+ raise AssertionError("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or "")[-500:]))
+ return proc
+
+
+def wait_ready(c: Client, seconds=90):
+ deadline = time.monotonic() + seconds
+ last = None
+ while time.monotonic() < deadline:
+ try:
+ c.call("GET", "/health/ready", schema="Health")
+ return
+ except Failure as exc:
+ last = exc
+ time.sleep(0.5)
+ raise AssertionError("ready not restored: %s" % last)
+
+
+def wait_status(c: Client, path, expected, seconds=30, actor=None):
+ deadline = time.monotonic() + seconds
+ last = None
+ while time.monotonic() < deadline:
+ try:
+ schema = "Health" if expected == 200 and path.startswith("/health") else None
+ c.call("GET", path, expected, actor=actor, schema=schema)
+ return
+ except Failure as exc:
+ last = exc
+ time.sleep(0.4)
+ raise AssertionError("did not observe %s for %s: %s" % (expected, path, last))
+
+
+@pytest.mark.infra
+def test_redis_and_postgres_unavailable(live: Client):
+ if os.environ.get("RUN_INFRA") != "1":
+ pytest.skip("set RUN_INFRA=1 to stop compose services")
+ alice = live.user("infra")
+ compose("stop", "redis")
+ wait_status(live, "/health/live", 200)
+ wait_status(live, "/health/ready", 503)
+ wait_status(live, "/api/v1/users/me", 503, actor=alice)
+ compose("start", "redis")
+ wait_ready(live)
+ live.call("GET", "/api/v1/users/me", actor=alice, schema="User")
+
+ compose("stop", "postgres")
+ wait_status(live, "/health/live", 200)
+ wait_status(live, "/health/ready", 503)
+ compose("start", "postgres")
+ wait_ready(live)
+ live.call("GET", "/api/v1/users/me", actor=alice, schema="User")
+
+
+@pytest.mark.infra
+def test_redis_restart_keeps_session_and_history(live: Client):
+ if os.environ.get("RUN_INFRA") != "1":
+ pytest.skip("set RUN_INFRA=1 to restart redis")
+ alice, bob, _eve, chat = live.bootstrap()
+ msg = live.message(alice, chat, "before-redis-restart")["json"]
+ compose("restart", "redis")
+ wait_ready(live)
+ me = live.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"]
+ assert me["id"] == alice["id"]
+ page = live.call(
+ "GET",
+ "/api/v1/chats/%s/messages?limit=100" % chat["id"],
+ actor=bob,
+ schema="MessagePage",
+ )["json"]
+ assert any(m["id"] == msg["id"] for m in page["items"])
diff --git a/student_tests/test_track1.py b/student_tests/test_track1.py
new file mode 100644
index 00000000..d71c3192
--- /dev/null
+++ b/student_tests/test_track1.py
@@ -0,0 +1,83 @@
+"""Track 1: cache consistency — stampede, stale read, read-after-write."""
+import json
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+
+from client import Client
+
+
+def _lab_metrics(c: Client):
+ return json.loads(c.call("GET", "/api/v1/_lab/metrics")["bytes"].decode())
+
+
+def _lab_flags(c: Client, body):
+ return json.loads(c.call("POST", "/api/v1/_lab/flags", 200, body=body)["bytes"].decode())
+
+
+def test_read_after_write(live: Client):
+ alice, bob, _eve, chat = live.bootstrap()
+ _lab_flags(live, {"invalidate": True, "singleflight": True, "reset_sql": True})
+ first = live.message(alice, chat, "raw-visible-" + uuid.uuid4().hex)["json"]
+ page = live.call(
+ "GET",
+ "/api/v1/chats/%s/messages?limit=100" % chat["id"],
+ actor=bob,
+ schema="MessagePage",
+ )["json"]
+ assert any(m["id"] == first["id"] for m in page["items"])
+
+
+def test_stale_without_invalidation_then_fixed(live: Client):
+ alice, bob, _eve, chat = live.bootstrap()
+ path = "/api/v1/chats/%s/messages?limit=50" % chat["id"]
+ live.call("GET", path, actor=alice, schema="MessagePage")
+ _lab_flags(live, {"invalidate": False, "singleflight": True})
+ try:
+ stale_text = "stale-probe-" + uuid.uuid4().hex
+ live.message(alice, chat, stale_text)
+ page = live.call("GET", path, actor=bob, schema="MessagePage")["json"]
+ assert all(m["text"] != stale_text for m in page["items"])
+ finally:
+ _lab_flags(live, {"invalidate": True})
+ fresh_text = "fresh-probe-" + uuid.uuid4().hex
+ live.message(alice, chat, fresh_text)
+ page2 = live.call("GET", path, actor=bob, schema="MessagePage")["json"]
+ assert any(m["text"] == fresh_text for m in page2["items"])
+
+
+def test_stampede_singleflight_reduces_sql(live: Client):
+ alice, _bob, _eve, chat = live.bootstrap()
+ live.message(alice, chat, "stampede-seed")
+ path = "/api/v1/chats/%s/messages?limit=50" % chat["id"]
+
+ def burst(n=24):
+ barrier = threading.Barrier(n)
+
+ def one(_i):
+ local = Client(live.base)
+ barrier.wait()
+ return local.call("GET", path, actor=alice, schema="MessagePage")
+
+ with ThreadPoolExecutor(max_workers=n) as pool:
+ list(pool.map(one, range(n)))
+
+ try:
+ _lab_flags(live, {"singleflight": False, "invalidate": True, "reset_sql": True, "delay_ms": 250})
+ live.message(alice, chat, "miss-" + uuid.uuid4().hex)
+ _lab_flags(live, {"reset_sql": True, "singleflight": False, "delay_ms": 250})
+ burst()
+ unprotected = _lab_metrics(live)["history_sql"]
+
+ _lab_flags(live, {"singleflight": True, "reset_sql": True, "delay_ms": 250})
+ live.message(alice, chat, "miss2-" + uuid.uuid4().hex)
+ _lab_flags(live, {"reset_sql": True, "singleflight": True, "delay_ms": 250})
+ burst()
+ protected = _lab_metrics(live)["history_sql"]
+ finally:
+ _lab_flags(live, {"invalidate": True, "singleflight": True, "delay_ms": 0})
+
+ print(json.dumps({"stampede_sql_without_singleflight": unprotected, "stampede_sql_with_singleflight": protected}))
+ assert unprotected >= 8, unprotected
+ assert protected < unprotected, (protected, unprotected)
+ assert protected <= 4, protected