Author SHA1 Message Date
dawuduk 2038551342 Описать решение на 5 и ответы на вопросы защиты. 2026-11-07 12:46:37 +03:00
dawuduk 99ce1b1391 Измерить отказ каждой реплики и цену добавления процессов.
Один и тот же профиль нагрузки сравнивает 1, 2 и 4 реплики при фиксированном и растущем бюджете CPU.
2026-11-07 12:46:37 +03:00
dawuduk fd54119d3f Поставить реплики API за HAProxy и описать остановку одной из них.
Внешний порт остаётся только у балансировщика, миграции применяются один раз, а stop-one оставляет выбранную реплику остановленной.
2026-11-07 12:46:37 +03:00
dawuduk 3601ca5443 Перенести API с общими сессиями и идемпотентной отправкой сообщения.
Реплика больше не держит корректность в своей памяти: сессия в Redis, а повтор POST сообщения фиксируется в Postgres вместе с X-Instance-Id.
2026-11-07 12:46:37 +03:00
68 changed files with 4179 additions and 36 deletions
+15 -1
View File
@@ -3,11 +3,25 @@
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
LOG_LEVEL=info
SHUTDOWN_GRACE_SECONDS=10
SESSION_TTL_SECONDS=3600
CACHE_TTL_SECONDS=60
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
SESSION_STORE=redis
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
API_CPUS=0.50
API_MEM=256m
POSTGRES_CPUS=1.0
POSTGRES_MEM=512m
DB_POOL_MAX=4
LB_DRAIN_SECONDS=2
IDEMPOTENCY_TTL_SECONDS=86400
# Compose interpolates these. Synthetic local values only; replace after copy.
POSTGRES_PASSWORD=lab2-dev-only-change-me
DATABASE_URL=postgresql://messenger:lab2-dev-only-change-me@postgres:5432/messenger
REDIS_URL=redis://redis:6379/0
+14 -14
View File
@@ -1,18 +1,18 @@
## Лабораторная 3
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Булацев / БПИ2301
Целевая оценка (3/4/5): 5
Трек 5, если заявлен: 1, цена масштабирования
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Что реализовано и что перенесено: API lab2 (PostgreSQL, Redis-сессии, кэш). Добавлены HAProxy, масштабируемый сервис API без внешнего порта, одноразовые миграции, общий Idempotency-Key и X-Instance-Id.
Команды запуска из чистого clone: `cp .env.example .env`, затем `docker compose --env-file .env up -d --build --scale api=2`, дождаться `http://127.0.0.1:8080/health/ready`. Подробности в REPORT.md.
Результаты проверок и ссылка на REPORT.md: smoke PASS; persistence PASS; failover — 2 реплики, 1 ошибка, восстановление записи 2.465 с; трек 1 в `evidence/track1/`. См. REPORT.md.
Ограничения / известные ошибки: один хост; LB, Postgres и Redis остаются единой точкой отказа. На Windows `.sh` не запускается напрямую (`WinError 193`); hook рассчитан на POSIX.
Источники и использование ИИ: README и контракт курса, документация HAProxy и Compose. Сборку решения вёл агент Cursor (Grok).
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены; приложен реальный вывод.
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+29
View File
@@ -0,0 +1,29 @@
FROM python:3.12.8-slim-bookworm AS runtime
WORKDIR /app
RUN groupadd --system app && useradd --system --gid app --home /app --shell /usr/sbin/nologin app \
&& apt-get update \
&& apt-get install -y --no-install-recommends curl \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app ./app
COPY migrations ./migrations
COPY contracts/openapi.json ./contracts/openapi.json
USER app
ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab \
PYTHONUNBUFFERED=1
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=12 \
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"
CMD ["python", "-m", "app.main"]
+25 -7
View File
@@ -4,27 +4,45 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover test-student test-infra test-expiry explain evidence-grade4 evidence-track1
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
check:
$(PYTHON) scripts/check_contract.py
$(PYTHON) -X utf8 scripts/check_contract.py
validate:
$(PYTHON) scripts/validate_openapi.py
$(PYTHON) -X utf8 scripts/validate_openapi.py
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
$(PYTHON) -X utf8 tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
$(PYTHON) -X utf8 tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
$(PYTHON) -X utf8 tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
$(PYTHON) -X utf8 tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-student:
$(PYTHON) -m pytest student_tests -q -m "not infra and not expiry"
test-infra:
RUN_INFRA=1 $(PYTHON) -m pytest student_tests/test_infra.py -q -m infra
test-expiry:
$(PYTHON) scripts/run_expiry.py
explain:
docker compose --env-file .env exec -T api python -m app.tools.explain
evidence-grade4:
$(PYTHON) -X utf8 scripts/grade4_evidence.py
evidence-track1:
$(PYTHON) -X utf8 scripts/track1_experiment.py
+86 -14
View File
@@ -1,37 +1,109 @@
# Отчёт — лабораторная 3
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: **Булацев**, **БПИ2301** (ветка `solution/bpi2301-Bulatsev`)
- Целевая оценка; выбранный трек 5: **5, трек 1 «Цена масштабирования»**
- Перенос: API lab2 (PostgreSQL, Redis-сессии, кэш истории, миграции). В этой работе добавлены HAProxy, две и более реплики API без внешнего порта, общий `Idempotency-Key` в PostgreSQL и `X-Instance-Id`.
- Commit/PR: ещё не опубликован
- Источники и помощь ИИ: контракт и README курса; образы Postgres 16.6, Redis 7.4.2, HAProxy 2.8. Документация HAProxy `server-template` и Docker Compose. Реализацию и прогоны помог собрать агент Cursor (Grok). На защите нужно объяснять схему своими словами.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Стенд: Windows 11, CPU 12th Gen Intel Core i5-12450H, Docker Desktop 29.5.3 / Compose 5.1.4, лимит Docker 12 vCPU и 7.62 ГиБ. Образы зафиксированы тегами: `python:3.12.8-slim-bookworm` (сборка `mtusi-messenger:lab3`), `postgres:16.6-alpine`, `redis:7.4.2-alpine`, `haproxy:2.8.14-alpine`.
```sh
cp .env.example .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --scale api=2
```
Дождаться `200` от `http://127.0.0.1:8080/health/ready`. Этот адрес — HAProxy. Реплики API наружу порты не публикуют.
Миграции выполняет одноразовый сервис `migrate` (`python -m app.migrate`) после healthy Postgres и до старта API. Повторный запуск пропускает уже записанные файлы. Дополнительно `pg_advisory_lock` не даёт двум миграторам применить один файл. У API `MIGRATIONS_ON_START=0`.
Проверки:
```sh
python -X utf8 scripts/check_contract.py
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
python -X utf8 -m pytest student_tests -q -m "not infra and not expiry"
```
`scripts/restart.sh` и `scripts/stop-one.sh` — POSIX. На Linux: `chmod +x` и `make test-persistence` / `make test-failover`. На этом Windows `CreateProcess` не запускает `.sh` (`WinError 193`); те же команды прогнаны интерпретатором `python3` в контейнере `docker:27-cli` с сокетом того же Docker. Скрипты при этом управляли контейнерами стенда.
Остановка без удаления данных: `docker compose --env-file .env stop`. Учебные тома удаляет только явная команда `docker compose --env-file .env down -v`.
Чтобы освободить `127.0.0.1:8080`, перед запуском был остановлен контейнер `mtusi2026lab2-api-1`. Postgres и Redis lab2 оставались запущены; в срезах `docker stats` их CPU был около 0–4%.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Клиент ходит только в `lb` (HAProxy, `127.0.0.1:8080`). Backend — DNS-имя `api` сервиса Compose: `server-template` на 8 слотов, `resolvers docker` (`127.0.0.11`), `hold valid 1s`. Балансировка `roundrobin`, без sticky session. Сессии лежат в Redis (AOF + RDB), пользователи, чаты, сообщения и результаты идемпотентности — в PostgreSQL. Локальная память реплики не нужна для корректности следующего запроса.
`X-Instance-Id` — hostname контейнера, он не меняется, пока жив процесс. `scripts/stop-one.sh` сравнивает `TARGET_INSTANCE_ID` с `/tmp/instance-id` или hostname и делает `docker stop -t 5` одного контейнера. `restart: unless-stopped` не поднимает его обратно.
`POST /messages` требует `Idempotency-Key` (`[A-Za-z0-9._:-]{1,128}`). Область — пользователь + `POST` + путь чата + ключ. Канонический JSON (`text` и `attachment_ids`, пустой список равен отсутствию поля) хешируется SHA-256. Тот же хеш в течение `IDEMPOTENCY_TTL_SECONDS` (86400) возвращает сохранённый `201` и то же сообщение. Другой текст — `409`. Вставка сообщения и строки результата — одна транзакция; ошибка откатывает обе. Одновременные повторы сериализует `pg_advisory_xact_lock` в Postgres, поэтому две реплики не создают два сообщения. Неуспешный запрос строку не пишет, повтор с тем же ключом безопасен.
Проверки готовности. `GET /health/live` — только процесс. `GET /health/ready` — Postgres и Redis; во время SIGTERM сразу `503`. HAProxy: `inter 1s`, `fall 2`, `rise 1`, `timeout check 2s`. Неготовая реплика выпадает примерно за 2 с. Если контейнер уже исчез, DNS-кэш HAProxy живёт ещё до 1 с. На SIGTERM API перестаёт принимать бизнес-запросы, ready становится 503, процесс завершается через `LB_DRAIN_SECONDS` (2). `stop_grace_period` сервиса — 8 с. Таймауты прокси: connect 2 с, client/server 10 с. HAProxy не повторяет POST.
Соответствие `.env.example`: `DATABASE_URL` и `REDIS_URL` собираются в Compose из `POSTGRES_PASSWORD`; `API_CPUS`, `API_MEM`, `POSTGRES_CPUS`, `DB_POOL_MAX`, `LB_DRAIN_SECONDS`, `IDEMPOTENCY_TTL_SECONDS` интерполируются в `compose.yaml`. Рабочая точка стенда — 2 реплики по 0.50 CPU и 256 МиБ, Postgres 1.0 CPU / 512 МиБ, пул по 4 соединения. `container_name` нет.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Контракт offline | `python -X utf8 scripts/check_contract.py` | PASS | `evidence/public-checks.txt` |
| Smoke через LB | `tests/smoke.py` | PASS, lab3 | `evidence/public-checks.txt` |
| Две реплики, одна сессия | 12× `GET /health/ready` | Чередование `03c3a0545628` и `b555f8327eee` | наблюдение до отказов; после экспериментов новые id, см. ниже |
| Общая БД и сессии | persistence hook | PASS, старый токен жив | `evidence/public-checks.txt` |
| Отказ одной реплики ≤10 с | `tests/resilience.py --mode failover` | 2 реплики до отказа; 109 успешных записей; 1 ошибка; первое подтверждение после hook 2.465 с; дублей нет | `evidence/public-checks.txt` |
| Остановлен именно целевой контейнер | `docker inspect` после failover | `mtusi2026lab3-api-1` Exited (0), hostname `b555f8327eee`; вторая реплика Up | `evidence/public-checks.txt` |
| Идемпотентность 16 параллельных запросов | `scripts/grade4_evidence.py` | 16×201, 1 id, в истории 1 копия | `evidence/grade4/summary.json` |
| Поочерёдный отказ каждой реплики | тот же скрипт | hook 2.373 с и 1.744 с; в последних ответах только живая реплика; 2 и 1 ошибка | `evidence/grade4/summary.json` |
| Rolling update под нагрузкой | `scripts/rolling-update.sh` во время `tests/load.py` 50 с, concurrency 4 | 8361 успех, 18 ошибок (0.21%), RPS 166.9, p50/p95/p99 13.5/57.8/96.4 мс, замена 5.9 с | `evidence/grade4/summary.json`, `evidence/grade4/rolling-load.json` |
| Лимиты, нет обхода LB | `docker ps` | API `8080/tcp` без host-порта; наружу только `127.0.0.1:8080` у lb | `compose.yaml` |
| Трек 1 | `scripts/track1_experiment.py` | серии A/B и проверка двух правок, см. ниже | `evidence/track1/summary.json`, `evidence/track1/rps.svg` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Студенческие тесты lab2 (гонки, откат чата, кэш, конкурентные сообщения): 15 passed. `RUN_INFRA` и expiry в этот прогон не входили.
После трека 1 стенд возвращён к 2 репликам по 0.50 CPU. Новые hostname на ready: `0868e9d2ba45` и `14cb087ebb10`. Старые id исчезли вместе с пересозданными контейнерами; это не sticky session.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Гипотеза. При фиксированном бюджете API 1.0 CPU и 512 МиБ дробление на 2 и 4 реплики не поднимет RPS: каждая упрётся в свой cgroup, а Postgres ещё свободен. Если давать каждой новой реплике те же 0.50 CPU, RPS вырастет, пока реплика упирается в квоту, и почти остановится, когда клиентский closed-loop и единственный Postgres перестанут получать пользу от лишних процессов.
Методика. Генератор `tests/load.py`: closed-loop POST сообщений, concurrency 8, 5 с прогрев (результат выбрасывается), 15 с замер. Ошибок автоматических retry нет. На каждую точку один прогон. В середине замера один срез `docker stats` и `pg_stat_activity`. В `docker stats` 100% — одно ядро. Профиль A держит сумму API равной 1.0 CPU / 512 МиБ и сумму пулов равной 8. Профиль B добавляет реплики по 0.50 CPU / 256 МиБ при Postgres 1.0 CPU.
| Точка | Реплики | CPU API суммарно | Успешный RPS | Ошибки | p50 / p95 / p99, мс | CPU API по stats | CPU Postgres | Сессии Postgres | CPU LB |
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
| A1 | 1 × 1.00 | 1.0 | 344.0 | 0 | 19.7 / 42.1 / 81.0 | 98% | 33% | 9 | 21% |
| A2 | 2 × 0.50 | 1.0 | 196.3 | 0 | 26.6 / 89.9 / 123.3 | 49% и 48% | 28% | 9 | 20% |
| A4 | 4 × 0.25 | 1.0 | 124.5 | 0 | 39.1 / 185.7 / 267.6 | 20–25% | 16% | 9 | 9% |
| B1 | 1 × 0.50 | 0.5 | 98.1 | 0 | 85.1 / 167.2 / 201.9 | 48% | 17% | 5 | 8% |
| B2 | 2 × 0.50 | 1.0 | 223.8 | 0 | 24.1 / 81.0 / 142.4 | 49% и 48% | 29% | 9 | 16% |
| B4 | 4 × 0.50 | 2.0 | 236.6 | 0 | 22.6 / 80.2 / 131.0 | 35–46% | 48% | 17 | 21% |
| C4-pg2 | как B4, Postgres 2.0 CPU | 2.0 | 251.2 | 0 | 19.7 / 77.2 / 101.2 | 40–43% | 43% | 17 | 22% |
| D4-pool2 | как B4, пул 2 на реплику | 2.0 | 239.0 | 0 | 22.7 / 74.9 / 98.1 | 45–48% | 49% | 9 | 22% |
График: `evidence/track1/rps.svg`. Сырые точки: `evidence/track1/*.point.json`.
Что из этого следует. В серии A Postgres не насыщен (16–33%), память API далеко от лимита. Падает сам API: одна реплика с целым ядром даёт 344 RPS, те же ресурсы четырьмя квотами по 0.25 CPU — 124 RPS, p95 с 42 до 186 мс. Рост числа процессов при фиксированном CPU вреден из-за квот CFS и накладных расходов, а не из-за нехватки БД.
В серии B первое ядро API ещё нужно: B1 упирается в квоту 0.50 CPU (stats около 50%, p95 167 мс). Вторая такая реплика — подтверждённое улучшение: RPS 98.1 → 223.8, p95 167 → 81 мс, Postgres только с 17% до 29%. Четыре реплики против двух — уже площадка: +1.0 CPU API даёт 223.8 → 236.6 RPS (+6%), зато сессий Postgres 9 → 17 и CPU Postgres 29% → 48%, LB 16% → 21%. Восемь замкнутых воркеров не заполняют четыре процесса, а каждый INSERT всё равно идёт в один Postgres. Пул здесь не узкое место: D4 снижает сессии с 17 до 9 и почти не меняет RPS (239 против 237). Дополнительное ядро Postgres тоже не подтвердилось: 251 против 237 на одном 15-секундном прогоне, срез CPU Postgres так и не упёрся в 100%.
Ограничение измерения. Один прогон на точку, closed-loop и coordinated omission, перцентили только успешных запросов, таблица `messages` растёт от точки к точке, `docker stats` — один срез. Это не паспорт максимальной пропускной способности машины.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| `scripts/restart.sh` (api, postgres, redis), тома на месте | те же сообщения и тот же токен | PASS публичного persistence | готовность вернулась внутри бюджета теста 90 с; данных не теряли |
| `scripts/stop-one.sh` по `X-Instance-Id` | живая реплика принимает запись ≤10 с, дубля нет | 2 id до отказа; 1 ошибка; 109 записей; целевой контейнер Exited (0) | 2.465 с до первого подтверждения после возврата hook |
| Та же остановка второй реплики после старта первой | хвост ответов только от оставшейся | оба прогона: target отсутствует в последних 5 id | hook 1.7–2.4 с; ошибок 1 и 2 |
| Поочерёдная замена обеих реплик под нагрузкой | короткие ошибки, запись продолжается | 18 ошибок на 8379 попыток | скрипт замены 5.9 с |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Стенд переживает остановку одного процесса API и перезапуск контейнеров с сохранёнными томами. Он не переживает потерю единственного хоста, HAProxy, Postgres или Redis: это оставшиеся SPOF. Трек 2 не делался, выживание узла БД не заявляется.
Sticky session нет и не нужен: сессия в Redis, идемпотентность в Postgres. Повторно безопасно отправлять POST сообщения с тем же ключом. Повтор POST пользователя или чата по-прежнему может создать второй объект; прокси такие POST не повторяет.
Миграции совместимы с уже работающей схемой lab2 (новый файл `002_idempotency.sql`, старые таблицы не переписываются). Одновременный запуск двух версий кода с разной схемой отдельно не гонялся.
Подтверждённое улучшение — вторая реплика, когда первая исчерпала квоту 0.50 CPU (B1→B2). Четвёртая реплика, удвоение CPU Postgres и урезание пула на этом профиле прироста не показали. Следующий осмысленный шаг — поднять concurrency клиента выше 8 и повторить B2/B4 несколькими прогонами, прежде чем добавлять реплики Postgres.
+1
View File
@@ -0,0 +1 @@
"""Messenger API (lab 2): PostgreSQL source of truth, Redis sessions/cache."""
+151
View File
@@ -0,0 +1,151 @@
import asyncio
import json
from collections.abc import Awaitable, Callable
from typing import TypeVar
from app import config, rediscli
from app.errors import AppError
T = TypeVar("T")
CFG_SINGLEFLIGHT = "cfg:singleflight"
CFG_INVALIDATE = "cfg:invalidate"
CFG_DELAY_MS = "cfg:delay_ms"
SQL_HISTORY = "metrics:sql:messages_select"
class SingleFlight:
def __init__(self) -> None:
self._lock = asyncio.Lock()
self._inflight: dict[str, asyncio.Task] = {}
async def do(self, key: str, factory: Callable[[], Awaitable[T]]) -> T:
async with self._lock:
task = self._inflight.get(key)
if task is None:
task = asyncio.create_task(factory())
self._inflight[key] = task
owner = True
else:
owner = False
try:
return await asyncio.shield(task)
finally:
if owner:
async with self._lock:
self._inflight.pop(key, None)
_sf = SingleFlight()
async def _flag(name: str, default: bool = True) -> bool:
if not config.lab_mode():
return default
try:
raw = await rediscli.require().get(name)
except Exception:
return default
if raw is None:
return default
return raw not in {"0", "false", "off"}
async def singleflight_enabled() -> bool:
return await _flag(CFG_SINGLEFLIGHT, True)
async def invalidate_enabled() -> bool:
return await _flag(CFG_INVALIDATE, True)
async def incr_history_sql() -> None:
try:
await rediscli.require().incr(SQL_HISTORY)
except Exception:
pass
async def history_sql_count() -> int:
try:
return int(await rediscli.require().get(SQL_HISTORY) or 0)
except Exception:
return 0
async def reset_history_sql_count() -> None:
try:
await rediscli.require().set(SQL_HISTORY, 0)
except Exception:
pass
async def load_delay_seconds() -> float:
if not config.lab_mode():
return 0.0
try:
raw = await rediscli.require().get(CFG_DELAY_MS)
except Exception:
return 0.0
try:
return max(0.0, float(raw or 0) / 1000.0)
except (TypeError, ValueError):
return 0.0
async def set_flags(
*,
singleflight: bool | None = None,
invalidate: bool | None = None,
delay_ms: int | None = None,
) -> dict:
r = rediscli.require()
if singleflight is not None:
await r.set(CFG_SINGLEFLIGHT, "1" if singleflight else "0")
if invalidate is not None:
await r.set(CFG_INVALIDATE, "1" if invalidate else "0")
if delay_ms is not None:
await r.set(CFG_DELAY_MS, str(int(delay_ms)))
return {
"singleflight": await singleflight_enabled(),
"invalidate": await invalidate_enabled(),
"history_sql": await history_sql_count(),
"delay_ms": int((await load_delay_seconds()) * 1000),
}
async def get_history_page(chat_id: str, cursor: str, limit: int, loader: Callable[[], Awaitable[dict]]) -> dict:
r = rediscli.require()
try:
version = await r.get("histver:" + chat_id) or "0"
key = f"hist:{chat_id}:{version}:{cursor}:{limit}"
cached = await r.get(key)
if cached:
return json.loads(cached)
async def load_and_store() -> dict:
delay = await load_delay_seconds()
if delay:
await asyncio.sleep(delay)
page = await loader()
try:
await r.set(key, json.dumps(page, ensure_ascii=False), ex=config.CACHE_TTL_SECONDS)
except Exception:
pass
return page
if await singleflight_enabled():
return await _sf.do(key, load_and_store)
return await load_and_store()
except AppError:
raise
except Exception:
return await loader()
async def invalidate_history(chat_id: str) -> None:
if not await invalidate_enabled():
return
try:
await rediscli.require().incr("histver:" + chat_id)
except Exception:
pass
+42
View File
@@ -0,0 +1,42 @@
import os
import socket
from pathlib import Path
def _env(name: str, default: str = "") -> str:
value = os.environ.get(name)
return default if value is None or value == "" else value
def _file_or_env(name: str, default: str = "") -> str:
path = os.environ.get(name + "_FILE")
if path:
return Path(path).read_text(encoding="utf-8").strip()
return _env(name, default)
HOST = _env("APP_HOST", "0.0.0.0")
PORT = int(_env("APP_PORT", "8080"))
ENV = _env("APP_ENV", "lab")
LOG_LEVEL = _env("LOG_LEVEL", "info").lower()
SHUTDOWN_GRACE_SECONDS = float(_env("SHUTDOWN_GRACE_SECONDS", "10"))
SESSION_TTL_SECONDS = int(_env("SESSION_TTL_SECONDS", "3600"))
CACHE_TTL_SECONDS = int(_env("CACHE_TTL_SECONDS", "60"))
DATABASE_URL = _file_or_env("DATABASE_URL")
REDIS_URL = _file_or_env("REDIS_URL", "redis://redis:6379/0")
SESSION_STORE = _env("SESSION_STORE", "redis")
DB_POOL_MIN = int(_env("DB_POOL_MIN", "1"))
DB_POOL_MAX = int(_env("DB_POOL_MAX", "10"))
DB_COMMAND_TIMEOUT = float(_env("DB_COMMAND_TIMEOUT", "5"))
DB_CONNECT_TIMEOUT = float(_env("DB_CONNECT_TIMEOUT", "5"))
REDIS_SOCKET_TIMEOUT = float(_env("REDIS_SOCKET_TIMEOUT", "2"))
IDEMPOTENCY_TTL_SECONDS = int(_env("IDEMPOTENCY_TTL_SECONDS", "86400"))
MIGRATIONS_ON_START = _env("MIGRATIONS_ON_START", "1") == "1"
LB_DRAIN_SECONDS = float(_env("LB_DRAIN_SECONDS", "2"))
# Stable for the life of this process. Compose leaves it unset so each
# container uses its own hostname; stop-one.sh maps that value to one replica.
INSTANCE_ID = _env("INSTANCE_ID", socket.gethostname())
def lab_mode() -> bool:
return ENV == "lab"
+112
View File
@@ -0,0 +1,112 @@
import base64
import json
from datetime import datetime
from typing import Any
from uuid import UUID
def encode_cursor(chat_id: str, created_at: str, entity_id: str) -> str:
payload = json.dumps({"c": chat_id, "t": created_at, "i": entity_id}, separators=(",", ":"))
return base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=")
def _validate_cursor_timestamp(value: str) -> None:
if not value.endswith("Z"):
raise ValueError("invalid cursor timestamp")
try:
datetime.fromisoformat(value[:-1] + "+00:00")
except ValueError as exc:
raise ValueError("invalid cursor timestamp") from exc
def _require_str_field(data: dict, key: str) -> str:
if key not in data:
raise ValueError("invalid cursor")
value = data[key]
if not isinstance(value, str):
raise ValueError("invalid cursor")
return value
def decode_cursor(cursor: str, expected_chat_id: str) -> tuple[str, str]:
pad = "=" * (-len(cursor) % 4)
try:
raw = base64.urlsafe_b64decode((cursor + pad).encode("ascii"))
data: Any = json.loads(raw.decode("utf-8"))
except (ValueError, json.JSONDecodeError, UnicodeDecodeError) as exc:
raise ValueError("invalid cursor") from exc
if not isinstance(data, dict):
raise ValueError("invalid cursor")
try:
chat_id = _require_str_field(data, "c")
created_at = _require_str_field(data, "t")
entity_id = _require_str_field(data, "i")
except ValueError as exc:
raise ValueError("invalid cursor") from exc
if chat_id != expected_chat_id:
raise ValueError("cursor chat mismatch")
try:
UUID(entity_id)
except ValueError as exc:
raise ValueError("invalid cursor") from exc
try:
_validate_cursor_timestamp(created_at)
except ValueError as exc:
raise ValueError("invalid cursor") from exc
return created_at, entity_id
+87
View File
@@ -0,0 +1,87 @@
from pathlib import Path
import asyncpg
from app import config
from app.errors import AppError
pool: asyncpg.Pool | None = None
INTEGRITY = (
asyncpg.UniqueViolationError,
asyncpg.ForeignKeyViolationError,
asyncpg.CheckViolationError,
)
def _migrations_dir() -> Path:
here = Path(__file__).resolve().parent
for candidate in (here.parent / "migrations", Path("/app/migrations")):
if candidate.is_dir():
return candidate
raise RuntimeError("migrations directory not found")
async def connect() -> asyncpg.Pool:
global pool
if not config.DATABASE_URL:
raise RuntimeError("DATABASE_URL is not set")
pool = await asyncpg.create_pool(
dsn=config.DATABASE_URL,
min_size=config.DB_POOL_MIN,
max_size=config.DB_POOL_MAX,
command_timeout=config.DB_COMMAND_TIMEOUT,
timeout=config.DB_CONNECT_TIMEOUT,
)
return pool
async def close() -> None:
global pool
if pool is not None:
await pool.close()
pool = None
def require() -> asyncpg.Pool:
if pool is None:
raise AppError(503, "service unavailable")
return pool
async def ping() -> None:
try:
async with require().acquire() as conn:
await conn.execute("SELECT 1")
except AppError:
raise
except Exception:
raise AppError(503, "service unavailable") from None
_MIGRATION_LOCK = 87133103
async def apply_migrations() -> None:
"""Apply pending SQL files once. A session advisory lock stops replica races."""
directory = _migrations_dir()
async with require().acquire() as conn:
await conn.execute("SELECT pg_advisory_lock($1)", _MIGRATION_LOCK)
try:
await conn.execute(
"""
CREATE TABLE IF NOT EXISTS schema_migrations (
filename TEXT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""
)
applied = {row["filename"] for row in await conn.fetch("SELECT filename FROM schema_migrations")}
for path in sorted(p for p in directory.glob("*.sql") if p.is_file()):
if path.name in applied:
continue
sql = path.read_text(encoding="utf-8")
async with conn.transaction():
await conn.execute(sql)
await conn.execute("INSERT INTO schema_migrations(filename) VALUES($1)", path.name)
finally:
await conn.execute("SELECT pg_advisory_unlock($1)", _MIGRATION_LOCK)
+52
View File
@@ -0,0 +1,52 @@
import json
from functools import lru_cache
from pathlib import Path
from starlette.requests import Request
from starlette.responses import HTMLResponse, JSONResponse, Response
_SPEC_CANDIDATES = (
Path(__file__).resolve().parents[1] / "contracts" / "openapi.json",
Path("/app/contracts/openapi.json"),
)
_SWAGGER_HTML = """<!DOCTYPE html>
<html lang="ru">
<head>
<meta charset="utf-8"/>
<title>MTUSI Messenger — Swagger (lab2)</title>
<link rel="stylesheet" href="https://unpkg.com/swagger-ui-dist@5.11.0/swagger-ui.css"/>
</head>
<body>
<div id="swagger-ui"></div>
<script src="https://unpkg.com/swagger-ui-dist@5.11.0/swagger-ui-bundle.js"></script>
<script>
window.onload = function () {
SwaggerUIBundle({
url: "/openapi.json",
dom_id: "#swagger-ui",
deepLinking: true,
persistAuthorization: true,
tryItOutEnabled: true
});
};
</script>
</body>
</html>
"""
@lru_cache(maxsize=1)
def _load_spec() -> dict:
for path in _SPEC_CANDIDATES:
if path.is_file():
return json.loads(path.read_text(encoding="utf-8"))
return {"openapi": "3.1.0", "info": {"title": "MTUSI Messenger", "version": "1.2.0"}, "paths": {}}
async def openapi_json(_: Request) -> Response:
return JSONResponse(_load_spec())
async def swagger_ui(_: Request) -> Response:
return HTMLResponse(_SWAGGER_HTML)
+49
View File
@@ -0,0 +1,49 @@
from typing import Dict, Optional
from starlette.responses import JSONResponse
ERROR_CODES = {
400: "invalid_request",
401: "unauthorized",
404: "not_found",
409: "conflict",
413: "payload_too_large",
415: "unsupported_media_type",
429: "rate_limited",
503: "unavailable",
500: "internal_error",
}
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
class AppError(Exception):
def __init__(self, status: int, message: str, www_authenticate: str | None = None) -> None:
super().__init__(message)
self.status = status
self.message = message
self.www_authenticate = www_authenticate
def error_response(
status: int,
message: str,
request_id: str,
extra_headers: Optional[Dict[str, str]] = None,
) -> JSONResponse:
code = ERROR_CODES.get(status, "internal_error")
headers = {"X-Request-Id": request_id, "Content-Type": "application/json"}
if extra_headers:
headers.update(extra_headers)
return JSONResponse(
status_code=status,
headers=headers,
content={"error": {"code": code, "message": message, "request_id": request_id}},
)
def from_app_error(exc: AppError, request_id: str) -> JSONResponse:
extra = {"WWW-Authenticate": exc.www_authenticate} if exc.www_authenticate else None
return error_response(exc.status, exc.message, request_id, extra)
+146
View File
@@ -0,0 +1,146 @@
import asyncio
import logging
import signal
import sys
import uuid
from pathlib import Path
import uvicorn
from starlette.applications import Starlette
from starlette.exceptions import HTTPException
from starlette.requests import Request
from starlette.responses import Response
from starlette.routing import Route
from app import config, db, rediscli
from app.docs import openapi_json, swagger_ui
from app.errors import AppError, error_response, from_app_error
from app.middleware import RequestContextMiddleware
from app.routes import (
create_chat,
create_message,
create_session,
current_user,
get_chat,
health_live,
health_ready,
lab_flags,
lab_metrics,
list_chats,
list_messages,
register_user,
revoke_session,
)
def _request_id(request: Request) -> str:
existing = getattr(request.state, "request_id", None)
if existing:
return existing
return request.headers.get("X-Request-Id") or str(uuid.uuid4())
async def not_found_handler(request: Request, exc: HTTPException) -> Response:
return error_response(404, "not found", _request_id(request))
async def app_error_handler(request: Request, exc: AppError) -> Response:
return from_app_error(exc, _request_id(request))
def publish_instance_id() -> None:
path = Path(config._env("INSTANCE_ID_FILE", "/tmp/instance-id"))
try:
path.write_text(config.INSTANCE_ID, encoding="utf-8")
except OSError:
logging.getLogger("messenger").warning("instance id file was not written")
async def startup() -> None:
publish_instance_id()
logging.getLogger("messenger").info("instance_id=%s", config.INSTANCE_ID)
await db.connect()
if config.MIGRATIONS_ON_START:
await db.apply_migrations()
await rediscli.connect()
async def shutdown() -> None:
await rediscli.close()
await db.close()
def build_app() -> Starlette:
app = Starlette(
routes=[
Route("/docs", swagger_ui, methods=["GET"]),
Route("/openapi.json", openapi_json, methods=["GET"]),
Route("/health/live", health_live, methods=["GET"]),
Route("/health/ready", health_ready, methods=["GET"]),
Route("/api/v1/users", register_user, methods=["POST"]),
Route("/api/v1/users/me", current_user, methods=["GET"]),
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]),
Route("/api/v1/chats", create_chat, methods=["POST"]),
Route("/api/v1/chats", list_chats, methods=["GET"]),
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
Route("/api/v1/_lab/metrics", lab_metrics, methods=["GET"]),
Route("/api/v1/_lab/flags", lab_flags, methods=["POST"]),
],
exception_handlers={404: not_found_handler, AppError: app_error_handler},
on_startup=[startup],
on_shutdown=[shutdown],
)
app.state.shutting_down = False
app.add_middleware(RequestContextMiddleware)
return app
def configure_logging() -> None:
logging.basicConfig(
level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO),
format="%(asctime)s %(levelname)s %(name)s %(message)s",
stream=sys.stdout,
)
def main() -> None:
configure_logging()
app = build_app()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
server = uvicorn.Server(
uvicorn.Config(
app,
host=config.HOST,
port=config.PORT,
log_level=config.LOG_LEVEL,
access_log=False,
timeout_graceful_shutdown=int(config.SHUTDOWN_GRACE_SECONDS),
)
)
def _begin_shutdown() -> None:
if app.state.shutting_down:
return
logging.getLogger("messenger").info(
"shutdown signal received; readiness fails now, exit in %.1fs",
config.LB_DRAIN_SECONDS,
)
app.state.shutting_down = True
loop.call_later(config.LB_DRAIN_SECONDS, lambda: setattr(server, "should_exit", True))
for sig in (signal.SIGTERM, signal.SIGINT):
try:
loop.add_signal_handler(sig, _begin_shutdown)
except NotImplementedError:
signal.signal(sig, lambda *_: _begin_shutdown())
loop.run_until_complete(server.serve())
if __name__ == "__main__":
main()
+48
View File
@@ -0,0 +1,48 @@
import logging
import time
import uuid
from typing import Callable
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from starlette.responses import Response
from app.config import INSTANCE_ID
from app.errors import AppError, error_response, from_app_error
logger = logging.getLogger("messenger.access")
def _stamp(response: Response, request_id: str) -> Response:
response.headers["X-Request-Id"] = request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
class RequestContextMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next: Callable) -> Response:
request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4())
request.state.request_id = request_id
if request.app.state.shutting_down and request.url.path not in ("/health/live", "/health/ready"):
return _stamp(error_response(503, "service is shutting down", request_id), request_id)
started = time.perf_counter()
try:
response = await call_next(request)
except AppError as exc:
return _stamp(from_app_error(exc, request_id), request_id)
except Exception:
logger.exception("request_id=%s unhandled error path=%s", request_id, request.url.path)
return _stamp(error_response(500, "internal error", request_id), request_id)
elapsed_ms = (time.perf_counter() - started) * 1000
_stamp(response, request_id)
logger.info(
"request_id=%s method=%s path=%s status=%s duration_ms=%.2f",
request_id,
request.method,
request.url.path,
response.status_code,
elapsed_ms,
)
return response
+29
View File
@@ -0,0 +1,29 @@
"""One-shot schema apply. API replicas do not run this on startup."""
import asyncio
import logging
import sys
from app import config, db
async def _main() -> None:
if not config.DATABASE_URL:
raise SystemExit("DATABASE_URL is not set")
await db.connect()
try:
await db.apply_migrations()
finally:
await db.close()
def main() -> None:
logging.basicConfig(
level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO),
format="%(asctime)s %(levelname)s %(name)s %(message)s",
stream=sys.stdout,
)
asyncio.run(_main())
if __name__ == "__main__":
main()
+23
View File
@@ -0,0 +1,23 @@
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerifyMismatchError
hasher = PasswordHasher(time_cost=2, memory_cost=8192, parallelism=1)
_dummy = None
def hash_password(password: str) -> str:
return hasher.hash(password)
def dummy_hash() -> str:
global _dummy
if _dummy is None:
_dummy = hash_password("timing-dummy-password")
return _dummy
def verify_password(password_hash: str, password: str) -> bool:
try:
return hasher.verify(password_hash, password)
except (VerifyMismatchError, InvalidHashError, TypeError, ValueError):
return False
+40
View File
@@ -0,0 +1,40 @@
import redis.asyncio as redis
from app import config
from app.errors import AppError
client: redis.Redis | None = None
async def connect() -> redis.Redis:
global client
client = redis.from_url(
config.REDIS_URL,
socket_connect_timeout=config.REDIS_SOCKET_TIMEOUT,
socket_timeout=config.REDIS_SOCKET_TIMEOUT,
decode_responses=True,
)
await client.ping()
return client
async def close() -> None:
global client
if client is not None:
await client.close()
client = None
def require() -> redis.Redis:
if client is None:
raise AppError(503, "service unavailable")
return client
async def ping() -> None:
try:
await require().ping()
except AppError:
raise
except Exception:
raise AppError(503, "service unavailable") from None
+284
View File
@@ -0,0 +1,284 @@
import base64
import binascii
from starlette.requests import Request
from starlette.responses import JSONResponse, Response
from app import cache, config, db, rediscli, sessions, store
from app.cursor_codec import decode_cursor, encode_cursor
from app.errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError, error_response
from app.passwords import dummy_hash, hash_password, verify_password
from app.validation import (
parse_json_body,
require_exact_keys,
validate_display_name,
validate_limit,
validate_idempotency_key,
validate_message_text,
validate_password,
validate_title,
validate_username,
validate_uuid,
)
def _json(data: dict, request_id: str, status: int = 200) -> JSONResponse:
return JSONResponse(status_code=status, headers={"X-Request-Id": request_id}, content=data)
def _empty(request_id: str) -> Response:
return Response(status_code=204, headers={"X-Request-Id": request_id})
async def _bearer_user(request: Request) -> dict:
header = request.headers.get("Authorization") or ""
scheme, _, rest = header.partition(" ")
if scheme.lower() != "bearer" or not rest.strip():
raise AppError(401, "authentication required", BEARER_CHALLENGE)
user_id = str(await sessions.resolve(rest.strip()))
user = await store.get_user(user_id)
if user is None:
raise AppError(401, "authentication required", BEARER_CHALLENGE)
return user
async def health_live(request: Request) -> Response:
return _json({"status": "ok"}, request.state.request_id)
async def health_ready(request: Request) -> Response:
rid = request.state.request_id
if request.app.state.shutting_down:
return error_response(503, "service is shutting down", rid)
await db.ping()
await rediscli.ping()
return _json({"status": "ok"}, rid)
async def register_user(request: Request) -> Response:
rid = request.state.request_id
if request.headers.get("content-type", "").split(";")[0].strip().lower() not in ("application/json", ""):
if request.headers.get("content-type"):
return error_response(415, "unsupported media type", rid)
raw = await request.body()
body, err = parse_json_body(raw)
if err:
return error_response(400, err, rid)
assert body is not None
if require_exact_keys(body, {"username", "display_name", "password"}, {"username", "display_name", "password"}):
return error_response(400, "invalid user payload", rid)
if validate_username(body["username"]) or validate_display_name(body["display_name"]) or validate_password(body["password"]):
return error_response(400, "invalid user fields", rid)
user, conflict = await store.create_user(body["username"], body["display_name"], hash_password(body["password"]))
if conflict:
return error_response(409, "username already taken", rid)
assert user is not None
return _json(user, rid, 201)
async def current_user(request: Request) -> Response:
user = await _bearer_user(request)
return _json(user, request.state.request_id)
async def create_session(request: Request) -> Response:
rid = request.state.request_id
header = request.headers.get("Authorization") or ""
scheme, _, rest = header.partition(" ")
if scheme.lower() != "basic" or not rest.strip():
return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
try:
decoded = base64.b64decode(rest.strip(), validate=True).decode("utf-8")
except (binascii.Error, UnicodeError, ValueError):
return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
username, sep, password = decoded.partition(":")
if not sep:
return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
row = await store.get_user_by_username(username)
password_hash = row["password_hash"] if row else dummy_hash()
if row is None or not verify_password(password_hash, password):
return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
token, expires_at = await sessions.create(row["id"])
return _json(
{
"token": token,
"token_type": "Bearer",
"expires_at": expires_at,
"user": store.user_dict(row),
},
rid,
201,
)
async def revoke_session(request: Request) -> Response:
header = request.headers.get("Authorization") or ""
scheme, _, rest = header.partition(" ")
if scheme.lower() != "bearer" or not rest.strip():
raise AppError(401, "authentication required", BEARER_CHALLENGE)
await sessions.revoke(rest.strip())
return _empty(request.state.request_id)
async def create_chat(request: Request) -> Response:
rid = request.state.request_id
user = await _bearer_user(request)
ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
if ct and ct != "application/json":
return error_response(415, "unsupported media type", rid)
raw = await request.body()
body, err = parse_json_body(raw)
if err:
return error_response(400, err, rid)
assert body is not None
if require_exact_keys(body, {"title", "member_ids"}, {"title", "member_ids"}):
return error_response(400, "invalid chat payload", rid)
if validate_title(body["title"]):
return error_response(400, "invalid title", rid)
members = body["member_ids"]
if not isinstance(members, list) or not members:
return error_response(400, "invalid member_ids", rid)
if len(members) > 100:
return error_response(400, "too many members", rid)
for mid in members:
if validate_uuid(mid):
return error_response(400, "invalid member id", rid)
chat, create_err = await store.create_chat(body["title"], members, user["id"])
if create_err == "caller not in members":
return error_response(400, "caller must be a member", rid)
if create_err in ("duplicate members", "unknown member"):
return error_response(400, "invalid members", rid)
assert chat is not None
return _json(chat, rid, 201)
async def list_chats(request: Request) -> Response:
rid = request.state.request_id
user = await _bearer_user(request)
limit, limit_err = validate_limit(request.query_params.get("limit"))
if limit_err:
return error_response(400, limit_err, rid)
assert limit is not None
cursor_raw = request.query_params.get("cursor")
after = None
if cursor_raw is not None:
if not cursor_raw.strip():
return error_response(400, "invalid cursor", rid)
try:
created_at, entity_id = decode_cursor(cursor_raw, user["id"])
except ValueError:
return error_response(400, "invalid cursor", rid)
after = (created_at, entity_id)
items, next_key = await store.list_chats(user["id"], limit, after)
next_cursor = None
if next_key is not None:
next_cursor = encode_cursor(user["id"], next_key[0], next_key[1])
return _json({"items": items, "next_cursor": next_cursor}, rid)
async def get_chat(request: Request) -> Response:
rid = request.state.request_id
user = await _bearer_user(request)
chat_id = request.path_params["chat_id"]
if validate_uuid(chat_id):
return error_response(400, "invalid chat id", rid)
chat = await store.get_chat(chat_id, user["id"])
if chat is None:
return error_response(404, "chat not found", rid)
return _json(chat, rid)
async def create_message(request: Request) -> Response:
rid = request.state.request_id
user = await _bearer_user(request)
chat_id = request.path_params["chat_id"]
if validate_uuid(chat_id):
return error_response(400, "invalid chat id", rid)
ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
if ct and ct != "application/json":
return error_response(415, "unsupported media type", rid)
raw = await request.body()
body, err = parse_json_body(raw)
if err:
return error_response(400, err, rid)
assert body is not None
if require_exact_keys(body, {"text"}, {"text"}):
return error_response(400, "invalid message payload", rid)
if validate_message_text(body["text"]):
return error_response(400, "invalid message text", rid)
if validate_idempotency_key(request.headers.get("Idempotency-Key")):
return error_response(400, "invalid idempotency key", rid)
message, msg_err = await store.add_message(
chat_id,
user["id"],
body["text"],
request.headers.get("Idempotency-Key") or "",
request.url.path,
)
if msg_err == "not_found":
return error_response(404, "chat not found", rid)
if msg_err == "conflict":
return error_response(409, "idempotency key reused with a different body", rid)
assert message is not None
return _json(message, rid, 201)
async def list_messages(request: Request) -> Response:
rid = request.state.request_id
user = await _bearer_user(request)
chat_id = request.path_params["chat_id"]
if validate_uuid(chat_id):
return error_response(400, "invalid chat id", rid)
limit, limit_err = validate_limit(request.query_params.get("limit"))
if limit_err:
return error_response(400, limit_err, rid)
assert limit is not None
cursor_raw = request.query_params.get("cursor")
after = None
if cursor_raw is not None:
if not cursor_raw.strip():
return error_response(400, "invalid cursor", rid)
try:
created_at, entity_id = decode_cursor(cursor_raw, chat_id)
except ValueError:
return error_response(400, "invalid cursor", rid)
after = (created_at, entity_id)
page, next_key, list_err = await store.list_messages(chat_id, user["id"], limit, after)
if list_err == "not_found":
return error_response(404, "chat not found", rid)
assert page is not None
next_cursor = None
if next_key is not None:
next_cursor = encode_cursor(chat_id, next_key[0], next_key[1])
return _json({"items": page, "next_cursor": next_cursor}, rid)
async def lab_metrics(request: Request) -> Response:
if not config.lab_mode():
return error_response(404, "not found", request.state.request_id)
return _json(
{
"singleflight": await cache.singleflight_enabled(),
"invalidate": await cache.invalidate_enabled(),
"history_sql": await cache.history_sql_count(),
},
request.state.request_id,
)
async def lab_flags(request: Request) -> Response:
if not config.lab_mode():
return error_response(404, "not found", request.state.request_id)
raw = await request.body()
body, err = parse_json_body(raw)
if err:
return error_response(400, err, request.state.request_id)
assert body is not None
if body.get("reset_sql"):
await cache.reset_history_sql_count()
flags = await cache.set_flags(
singleflight=body.get("singleflight"),
invalidate=body.get("invalidate"),
delay_ms=body.get("delay_ms"),
)
return _json(flags, request.state.request_id)
+56
View File
@@ -0,0 +1,56 @@
import hashlib
import json
import secrets
from datetime import datetime, timedelta, timezone
from uuid import UUID
from app import config, rediscli
from app.errors import AppError, BEARER_CHALLENGE
def new_token() -> str:
return secrets.token_urlsafe(32)
def _key(token: str) -> str:
return "sess:" + hashlib.sha256(token.encode("utf-8")).hexdigest()
def _fmt(dt: datetime) -> str:
return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
async def create(user_id: UUID) -> tuple[str, str]:
token = new_token()
expires = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
payload = json.dumps({"user_id": str(user_id), "expires_at": _fmt(expires)})
try:
await rediscli.require().set(_key(token), payload, ex=config.SESSION_TTL_SECONDS)
except AppError:
raise
except Exception:
raise AppError(503, "service unavailable") from None
return token, _fmt(expires)
async def resolve(token: str) -> UUID:
try:
raw = await rediscli.require().get(_key(token))
except AppError:
raise
except Exception:
raise AppError(503, "service unavailable") from None
if not raw:
raise AppError(401, "authentication required", BEARER_CHALLENGE)
return UUID(json.loads(raw)["user_id"])
async def revoke(token: str) -> None:
try:
deleted = await rediscli.require().delete(_key(token))
except AppError:
raise
except Exception:
raise AppError(503, "service unavailable") from None
if not deleted:
raise AppError(401, "authentication required", BEARER_CHALLENGE)
+361
View File
@@ -0,0 +1,361 @@
from __future__ import annotations
import hashlib
import json
from datetime import datetime, timezone
from typing import List, Optional, Tuple
from uuid import UUID, uuid4
import asyncpg
from app import cache, config, db
from app.errors import AppError
def format_ts(dt: datetime) -> str:
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def user_dict(row) -> dict:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": format_ts(row["created_at"]),
}
def chat_dict(row, member_ids: List[str]) -> dict:
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": member_ids,
"created_at": format_ts(row["created_at"]),
}
def message_dict(row) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": format_ts(row["created_at"]),
}
async def _run(coro):
try:
return await coro
except AppError:
raise
except db.INTEGRITY:
raise
except Exception:
raise AppError(503, "service unavailable") from None
async def create_user(username: str, display_name: str, password_hash: str) -> Tuple[Optional[dict], Optional[str]]:
user_id = uuid4()
created = datetime.now(timezone.utc)
try:
async with db.require().acquire() as conn:
await conn.execute(
"""
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES ($1, $2, $3, $4, $5)
""",
user_id,
username,
display_name,
password_hash,
created,
)
except asyncpg.UniqueViolationError:
return None, "conflict"
except Exception:
raise AppError(503, "service unavailable") from None
return user_dict({"id": user_id, "username": username, "display_name": display_name, "created_at": created}), None
async def get_user(user_id: str) -> Optional[dict]:
row = await _run(
_fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", UUID(user_id))
)
return user_dict(row) if row else None
async def get_user_by_username(username: str):
return await _run(
_fetchrow(
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1",
username,
)
)
async def _fetchrow(sql: str, *args):
async with db.require().acquire() as conn:
return await conn.fetchrow(sql, *args)
async def _fetch(sql: str, *args):
async with db.require().acquire() as conn:
return await conn.fetch(sql, *args)
async def member_ids_of(chat_id: UUID) -> List[str]:
rows = await _run(_fetch("SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id", chat_id))
return [str(r["user_id"]) for r in rows]
async def create_chat(title: str, member_ids: List[str], caller_id: str) -> Tuple[Optional[dict], Optional[str]]:
if caller_id not in member_ids:
return None, "caller not in members"
if len(set(member_ids)) != len(member_ids):
return None, "duplicate members"
ids = [UUID(x) for x in member_ids]
chat_id = uuid4()
created = datetime.now(timezone.utc)
try:
async with db.require().acquire() as conn:
async with conn.transaction():
existing = await conn.fetch("SELECT id FROM users WHERE id = ANY($1::uuid[])", ids)
if len(existing) != len(ids):
return None, "unknown member"
await conn.execute(
"INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)",
chat_id,
title,
created,
)
await conn.executemany(
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)",
[(chat_id, mid) for mid in ids],
)
except AppError:
raise
except Exception:
raise AppError(503, "service unavailable") from None
return chat_dict({"id": chat_id, "title": title, "created_at": created}, member_ids), None
async def get_chat(chat_id: str, user_id: str) -> Optional[dict]:
row = await load_chat_for_member(chat_id, user_id)
if row is None:
return None
return chat_dict(row, await member_ids_of(UUID(chat_id)))
async def load_chat_for_member(chat_id: str, user_id: str):
return await _run(
_fetchrow(
"""
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2
WHERE c.id = $1
""",
UUID(chat_id),
UUID(user_id),
)
)
async def list_chats(user_id: str, limit: int, after: Optional[Tuple[str, str]] = None):
args: list = [UUID(user_id)]
clause = ""
if after is not None:
clause = "AND (c.created_at, c.id) > ($2, $3)"
args.extend([datetime.fromisoformat(after[0].replace("Z", "+00:00")), UUID(after[1])])
rows = await _run(
_fetch(
f"""
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = $1
WHERE 1=1 {clause}
ORDER BY c.created_at, c.id
LIMIT {int(limit) + 1}
""",
*args,
)
)
page = rows[:limit]
items = [chat_dict(row, await member_ids_of(row["id"])) for row in page]
next_key = None
if len(rows) > limit:
last = page[-1]
next_key = (format_ts(last["created_at"]), str(last["id"]))
return items, next_key
def _canonical_message(text: str) -> str:
# Missing attachment_ids and [] are the same message. Key order does not matter.
return json.dumps(
{"attachment_ids": [], "text": text},
sort_keys=True,
ensure_ascii=False,
separators=(",", ":"),
)
async def add_message(
chat_id: str,
sender_id: str,
text: str,
idem_key: str,
path: str,
) -> Tuple[Optional[dict], Optional[str]]:
"""Insert one message or replay the stored 201. The row and the result commit together.
Same user+method+path+key is serialized with a Postgres advisory lock so two
replicas cannot insert two messages. A failed transaction stores nothing.
"""
digest = hashlib.sha256(_canonical_message(text).encode("utf-8")).hexdigest()
lock_key = f"{sender_id}|POST|{path}|{idem_key}"
payload: Optional[dict] = None
created_new = False
try:
async with db.require().acquire() as conn:
async with conn.transaction():
await conn.execute("SELECT pg_advisory_xact_lock(hashtextextended($1::text, 0))", lock_key)
member = await conn.fetchrow(
"""
SELECT c.id
FROM chats c
JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2
WHERE c.id = $1
""",
UUID(chat_id),
UUID(sender_id),
)
if member is None:
return None, "not_found"
row = await conn.fetchrow(
"""
SELECT request_hash, response_json, created_at
FROM idempotency_keys
WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3
""",
UUID(sender_id),
path,
idem_key,
)
if row is not None:
created_at = row["created_at"]
if created_at.tzinfo is None:
created_at = created_at.replace(tzinfo=timezone.utc)
age = (datetime.now(timezone.utc) - created_at).total_seconds()
if age <= config.IDEMPOTENCY_TTL_SECONDS:
if row["request_hash"] != digest:
return None, "conflict"
return json.loads(row["response_json"]), None
await conn.execute(
"""
DELETE FROM idempotency_keys
WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3
""",
UUID(sender_id),
path,
idem_key,
)
message_id = uuid4()
created = datetime.now(timezone.utc)
await conn.execute(
"""
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
VALUES ($1, $2, $3, $4, $5)
""",
message_id,
UUID(chat_id),
UUID(sender_id),
text,
created,
)
payload = message_dict(
{
"id": message_id,
"chat_id": UUID(chat_id),
"sender_id": UUID(sender_id),
"text": text,
"created_at": created,
}
)
await conn.execute(
"""
INSERT INTO idempotency_keys
(user_id, http_method, path, idem_key, request_hash, response_json, created_at)
VALUES ($1, 'POST', $2, $3, $4, $5, $6)
""",
UUID(sender_id),
path,
idem_key,
digest,
json.dumps(payload, ensure_ascii=False, separators=(",", ":")),
created,
)
created_new = True
except AppError:
raise
except Exception:
raise AppError(503, "service unavailable") from None
if created_new:
await cache.invalidate_history(chat_id)
return payload, None
async def list_messages(chat_id: str, user_id: str, limit: int, after: Optional[Tuple[str, str]] = None):
if await load_chat_for_member(chat_id, user_id) is None:
return None, None, "not_found"
cursor = ""
after_ts = after_id = None
if after is not None:
cursor = after[0] + ":" + after[1]
after_ts = datetime.fromisoformat(after[0].replace("Z", "+00:00"))
after_id = UUID(after[1])
async def loader() -> dict:
await cache.incr_history_sql()
if after_ts is None:
rows = await _run(
_fetch(
"""
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1
ORDER BY created_at, id
LIMIT $2
""",
UUID(chat_id),
limit + 1,
)
)
else:
rows = await _run(
_fetch(
"""
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
ORDER BY created_at, id
LIMIT $4
""",
UUID(chat_id),
after_ts,
after_id,
limit + 1,
)
)
page = rows[:limit]
items = [message_dict(r) for r in page]
next_key = None
if len(rows) > limit:
last = page[-1]
next_key = (format_ts(last["created_at"]), str(last["id"]))
return {"items": items, "next_key": next_key}
page = await cache.get_history_page(chat_id, cursor, limit, loader)
return page["items"], page.get("next_key"), None
+1
View File
@@ -0,0 +1 @@
+73
View File
@@ -0,0 +1,73 @@
"""Seed ≥10 000 messages and print EXPLAIN (ANALYZE, BUFFERS) for keyset pagination."""
import asyncio
import os
from uuid import uuid4
import asyncpg
from app.passwords import hash_password
async def main() -> None:
conn = await asyncpg.connect(dsn=os.environ["DATABASE_URL"], timeout=10, command_timeout=120)
try:
alice = uuid4()
bob = uuid4()
chat = uuid4()
pwd = hash_password("Example-only-not-a-real-secret-123")
await conn.execute(
"INSERT INTO users (id, username, display_name, password_hash) VALUES ($1,$2,$3,$4),($5,$6,$7,$4)",
alice,
"explain_a_" + alice.hex[:8],
"Alice",
pwd,
bob,
"explain_b_" + bob.hex[:8],
"Bob",
)
await conn.execute("INSERT INTO chats (id, title) VALUES ($1,$2)", chat, "Explain chat")
await conn.execute(
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2),($1,$3)",
chat,
alice,
bob,
)
n = int(os.environ.get("EXPLAIN_MESSAGES", "10000"))
await conn.execute(
"""
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), $1, $2, 'seed-' || g, now() - (g || ' milliseconds')::interval
FROM generate_series(1, $3) AS g
""",
chat,
alice,
n,
)
count = await conn.fetchval("SELECT count(*) FROM messages WHERE chat_id = $1", chat)
print("seeded_messages", count, "chat_id", chat)
first = await conn.fetchrow(
"SELECT created_at, id FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT 1",
chat,
)
query = """
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
ORDER BY created_at, id
LIMIT 50
"""
print("\n=== WITH index messages_chat_created_id_idx ===")
plan = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"])
print("\n".join(row[0] for row in plan))
await conn.execute("SET enable_indexscan = off")
await conn.execute("SET enable_bitmapscan = off")
await conn.execute("SET enable_indexonlyscan = off")
print("\n=== WITHOUT index scans (seq scan baseline) ===")
plan2 = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"])
print("\n".join(row[0] for row in plan2))
finally:
await conn.close()
if __name__ == "__main__":
asyncio.run(main())
+102
View File
@@ -0,0 +1,102 @@
import json
import re
from typing import Any, Dict, Optional, Tuple
from uuid import UUID
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
def parse_json_body(raw: bytes) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
if not raw:
return None, "empty body"
try:
data = json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
return None, "malformed JSON"
if not isinstance(data, dict):
return None, "JSON body must be an object"
return data, None
def require_exact_keys(data: Dict[str, Any], allowed: set, required: set) -> Optional[str]:
keys = set(data.keys())
if keys - allowed:
return "unexpected fields"
if required - keys:
return "missing required fields"
return None
def validate_username(value: Any) -> Optional[str]:
if not isinstance(value, str) or not USERNAME_RE.fullmatch(value):
return "invalid username"
return None
def validate_display_name(value: Any) -> Optional[str]:
if not isinstance(value, str):
return "invalid display_name"
if not value.strip():
return "display_name must not be blank"
if len(value) > 100:
return "display_name too long"
return None
def validate_password(value: Any) -> Optional[str]:
if not isinstance(value, str):
return "invalid password"
if not value.strip():
return "password must not be blank"
if len(value) < 12 or len(value) > 128:
return "password length"
return None
def validate_title(value: Any) -> Optional[str]:
if not isinstance(value, str):
return "invalid title"
if not value.strip():
return "title must not be blank"
if len(value) > 100:
return "title too long"
return None
def validate_uuid(value: Any) -> Optional[str]:
if not isinstance(value, str):
return "invalid uuid"
try:
UUID(value)
except ValueError:
return "invalid uuid"
return None
def validate_idempotency_key(value: Optional[str]) -> Optional[str]:
if value is None or IDEMPOTENCY_RE.fullmatch(value) is None:
return "invalid idempotency key"
return None
def validate_message_text(value: Any) -> Optional[str]:
if not isinstance(value, str):
return "invalid text"
if not value.strip():
return "text must not be blank"
if len(value) > 4000:
return "text too long"
return None
def validate_limit(value: Optional[str], default: int = 50) -> Tuple[Optional[int], Optional[str]]:
if value is None:
return default, None
try:
limit = int(value)
except ValueError:
return None, "invalid limit"
if limit < 1 or limit > 100:
return None, "limit out of range"
return limit, None
+131
View File
@@ -0,0 +1,131 @@
# Client -> lb:8080 -> api (scale) -> shared Postgres and Redis.
# API replicas do not publish host ports. Migrations run once in `migrate`.
# Check period: HAProxy inter 1s, fall 2, rise 1. Worst detection of a dead
# or not-ready replica is about 2s, plus the 1s DNS hold. API on SIGTERM
# fails /health/ready immediately and exits after LB_DRAIN_SECONDS (default 2).
services:
migrate:
build:
context: .
dockerfile: Dockerfile
image: mtusi-messenger:lab3
command: ["python", "-m", "app.migrate"]
environment:
APP_ENV: lab
LOG_LEVEL: info
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
DB_POOL_MIN: "1"
DB_POOL_MAX: "2"
MIGRATIONS_ON_START: "1"
depends_on:
postgres:
condition: service_healthy
restart: "no"
mem_limit: 256m
cpus: "0.50"
api:
build:
context: .
dockerfile: Dockerfile
image: mtusi-messenger:lab3
environment:
APP_HOST: 0.0.0.0
APP_PORT: "8080"
APP_ENV: lab
LOG_LEVEL: info
SHUTDOWN_GRACE_SECONDS: "3"
LB_DRAIN_SECONDS: ${LB_DRAIN_SECONDS:-2}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-60}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
SESSION_STORE: redis
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
REDIS_URL: redis://redis:6379/0
DB_POOL_MIN: "1"
DB_POOL_MAX: ${DB_POOL_MAX:-4}
DB_COMMAND_TIMEOUT: "5"
DB_CONNECT_TIMEOUT: "5"
MIGRATIONS_ON_START: "0"
depends_on:
migrate:
condition: service_completed_successfully
postgres:
condition: service_healthy
redis:
condition: service_healthy
expose:
- "8080"
restart: unless-stopped
mem_limit: ${API_MEM:-256m}
cpus: ${API_CPUS:-0.50}
stop_grace_period: 8s
healthcheck:
test:
[
"CMD",
"python",
"-c",
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)",
]
interval: 5s
timeout: 3s
retries: 6
start_period: 5s
lb:
image: haproxy:2.8.14-alpine
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "127.0.0.1:8080:8080"
depends_on:
api:
condition: service_started
restart: unless-stopped
mem_limit: 128m
cpus: "0.25"
postgres:
image: postgres:16.6-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: messenger
volumes:
- pg_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 3s
timeout: 3s
retries: 20
restart: unless-stopped
mem_limit: ${POSTGRES_MEM:-512m}
cpus: ${POSTGRES_CPUS:-1.0}
redis:
image: redis:7.4.2-alpine
command:
- redis-server
- --appendonly
- "yes"
- --appendfsync
- everysec
- --save
- "60"
- "1"
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 3s
timeout: 3s
retries: 20
restart: unless-stopped
mem_limit: 128m
cpus: "0.50"
volumes:
pg_data:
redis_data:
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 50.0,
"elapsed_seconds": 50.101,
"concurrency": 4,
"successful_requests": 8361,
"failed_requests": 18,
"error_ratio": 0.002148,
"successful_rps": 166.884,
"successful_latency_ms": {
"p50": 13.481,
"p95": 57.817,
"p99": 96.441
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+70
View File
@@ -0,0 +1,70 @@
{
"instances_before": [
"03c3a0545628",
"b555f8327eee"
],
"parallel_idempotency": {
"workers": 16,
"successes": 16,
"errors": 0,
"distinct_message_ids": 1,
"history_copies": 1
},
"failovers": [
{
"target": "03c3a0545628",
"hook_exit": 0,
"hook_seconds": 2.373,
"hook_stderr": "",
"failed_attempts": 2,
"successes_after_hook_start": 130,
"first_recovery_seconds": 0.034,
"tail_instance_ids": [
"b555f8327eee",
"b555f8327eee",
"b555f8327eee",
"b555f8327eee",
"b555f8327eee"
],
"target_still_in_tail": false
},
{
"target": "b555f8327eee",
"hook_exit": 0,
"hook_seconds": 1.744,
"hook_stderr": "",
"failed_attempts": 1,
"successes_after_hook_start": 132,
"first_recovery_seconds": 0.038,
"tail_instance_ids": [
"03c3a0545628",
"03c3a0545628",
"03c3a0545628",
"03c3a0545628",
"03c3a0545628"
],
"target_still_in_tail": false
}
],
"rolling": {
"rolling_seconds": 5.929,
"rolling_stdout": "replacing b555f8327eee\nreplacing 03c3a0545628\nrolling update finished; replicas=2\n",
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 50.0,
"elapsed_seconds": 50.101,
"concurrency": 4,
"successful_requests": 8361,
"failed_requests": 18,
"error_ratio": 0.002148,
"successful_rps": 166.884,
"successful_latency_ms": {
"p50": 13.481,
"p95": 57.817,
"p99": 96.441
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
}
}
+29
View File
@@ -0,0 +1,29 @@
Date: 2026-11-07
Host: 12th Gen Intel Core i5-12450H, Docker Desktop ncpu=12, MemTotal=8181665792
Engine: Docker 29.5.3, Compose 5.1.4
Stack: haproxy:2.8.14-alpine, mtusi-messenger:lab3 (python:3.12.8-slim-bookworm), postgres:16.6-alpine, redis:7.4.2-alpine
python -X utf8 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
python -X utf8 -m pytest student_tests -q -m "not infra and not expiry"
15 passed, 3 deselected
Official persistence and failover were executed by Linux python3 in docker:27-cli
with the host Docker socket and BASE_URL=http://host.docker.internal:8080,
because Windows CreateProcess cannot run scripts/restart.sh (WinError 193).
The hooks and the API containers are the host stack.
python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh
{"observed_instances_before": 2, "successful_writes": 109, "failed_attempts": 1, "first_recovery_seconds": 2.465}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
After that failover, docker inspect showed mtusi2026lab3-api-1 Exited (0),
hostname b555f8327eee. mtusi2026lab3-api-2 stayed up, hostname 03c3a0545628.
Exit code 0 means the process left on SIGTERM before the stop timeout.
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.109,
"concurrency": 8,
"successful_requests": 5197,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 343.964,
"successful_latency_ms": {
"p50": 19.732,
"p95": 42.061,
"p99": 81.001
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+38
View File
@@ -0,0 +1,38 @@
{
"tag": "A1",
"replicas": 1,
"api_cpus_each": "1.0",
"api_mem_each": "512m",
"api_cpu_total": 1.0,
"postgres_cpus": "1.0",
"db_pool_max_each": 8,
"db_pool_max_total": 8,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.109,
"concurrency": 8,
"successful_requests": 5197,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 343.964,
"successful_latency_ms": {
"p50": 19.732,
"p95": 42.061,
"p99": 81.001
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB",
"mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB",
"mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB",
"mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB",
"mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB",
"mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB"
],
"postgres_sessions": "9"
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.206,
"concurrency": 8,
"successful_requests": 2985,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 196.3,
"successful_latency_ms": {
"p50": 26.613,
"p95": 89.893,
"p99": 123.339
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+39
View File
@@ -0,0 +1,39 @@
{
"tag": "A2",
"replicas": 2,
"api_cpus_each": "0.50",
"api_mem_each": "256m",
"api_cpu_total": 1.0,
"postgres_cpus": "1.0",
"db_pool_max_each": 4,
"db_pool_max_total": 8,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.206,
"concurrency": 8,
"successful_requests": 2985,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 196.3,
"successful_latency_ms": {
"p50": 26.613,
"p95": 89.893,
"p99": 123.339
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB",
"mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB",
"mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB",
"mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB",
"mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB",
"mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB",
"mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB"
],
"postgres_sessions": "9"
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.224,
"concurrency": 8,
"successful_requests": 1895,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 124.476,
"successful_latency_ms": {
"p50": 39.136,
"p95": 185.654,
"p99": 267.62
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+41
View File
@@ -0,0 +1,41 @@
{
"tag": "A4",
"replicas": 4,
"api_cpus_each": "0.25",
"api_mem_each": "128m",
"api_cpu_total": 1.0,
"postgres_cpus": "1.0",
"db_pool_max_each": 2,
"db_pool_max_total": 8,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.224,
"concurrency": 8,
"successful_requests": 1895,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 124.476,
"successful_latency_ms": {
"p50": 39.136,
"p95": 185.654,
"p99": 267.62
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB",
"mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB",
"mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB",
"mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB",
"mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB",
"mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB",
"mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB",
"mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB",
"mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB"
],
"postgres_sessions": "9"
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.139,
"concurrency": 8,
"successful_requests": 1485,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 98.094,
"successful_latency_ms": {
"p50": 85.074,
"p95": 167.209,
"p99": 201.857
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+38
View File
@@ -0,0 +1,38 @@
{
"tag": "B1",
"replicas": 1,
"api_cpus_each": "0.50",
"api_mem_each": "256m",
"api_cpu_total": 0.5,
"postgres_cpus": "1.0",
"db_pool_max_each": 4,
"db_pool_max_total": 4,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.139,
"concurrency": 8,
"successful_requests": 1485,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 98.094,
"successful_latency_ms": {
"p50": 85.074,
"p95": 167.209,
"p99": 201.857
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB",
"mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB",
"mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB",
"mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB",
"mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB",
"mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB"
],
"postgres_sessions": "5"
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.136,
"concurrency": 8,
"successful_requests": 3388,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 223.842,
"successful_latency_ms": {
"p50": 24.1,
"p95": 81.009,
"p99": 142.445
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+39
View File
@@ -0,0 +1,39 @@
{
"tag": "B2",
"replicas": 2,
"api_cpus_each": "0.50",
"api_mem_each": "256m",
"api_cpu_total": 1.0,
"postgres_cpus": "1.0",
"db_pool_max_each": 4,
"db_pool_max_total": 8,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.136,
"concurrency": 8,
"successful_requests": 3388,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 223.842,
"successful_latency_ms": {
"p50": 24.1,
"p95": 81.009,
"p99": 142.445
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB",
"mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB",
"mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB",
"mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB",
"mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB",
"mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB",
"mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB"
],
"postgres_sessions": "9"
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.181,
"concurrency": 8,
"successful_requests": 3592,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 236.616,
"successful_latency_ms": {
"p50": 22.559,
"p95": 80.196,
"p99": 131.013
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+41
View File
@@ -0,0 +1,41 @@
{
"tag": "B4",
"replicas": 4,
"api_cpus_each": "0.50",
"api_mem_each": "256m",
"api_cpu_total": 2.0,
"postgres_cpus": "1.0",
"db_pool_max_each": 4,
"db_pool_max_total": 16,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.181,
"concurrency": 8,
"successful_requests": 3592,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 236.616,
"successful_latency_ms": {
"p50": 22.559,
"p95": 80.196,
"p99": 131.013
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB",
"mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB",
"mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB",
"mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB",
"mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB",
"mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB",
"mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB",
"mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
"mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB"
],
"postgres_sessions": "17"
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.128,
"concurrency": 8,
"successful_requests": 3800,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 251.189,
"successful_latency_ms": {
"p50": 19.749,
"p95": 77.161,
"p99": 101.17
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+41
View File
@@ -0,0 +1,41 @@
{
"tag": "C4-pg2",
"replicas": 4,
"api_cpus_each": "0.50",
"api_mem_each": "256m",
"api_cpu_total": 2.0,
"postgres_cpus": "2.0",
"db_pool_max_each": 4,
"db_pool_max_total": 16,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.128,
"concurrency": 8,
"successful_requests": 3800,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 251.189,
"successful_latency_ms": {
"p50": 19.749,
"p95": 77.161,
"p99": 101.17
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB",
"mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB",
"mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB",
"mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB",
"mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB",
"mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB",
"mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB",
"mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
"mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB"
],
"postgres_sessions": "17"
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.156,
"concurrency": 8,
"successful_requests": 3622,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 238.98,
"successful_latency_ms": {
"p50": 22.721,
"p95": 74.859,
"p99": 98.063
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+41
View File
@@ -0,0 +1,41 @@
{
"tag": "D4-pool2",
"replicas": 4,
"api_cpus_each": "0.50",
"api_mem_each": "256m",
"api_cpu_total": 2.0,
"postgres_cpus": "1.0",
"db_pool_max_each": 2,
"db_pool_max_total": 8,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.156,
"concurrency": 8,
"successful_requests": 3622,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 238.98,
"successful_latency_ms": {
"p50": 22.721,
"p95": 74.859,
"p99": 98.063
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB",
"mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB",
"mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB",
"mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB",
"mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB",
"mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB",
"mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB",
"mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB",
"mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB"
],
"postgres_sessions": "9"
}
}
+29
View File
@@ -0,0 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg" width="720" height="320">
<rect width="100%" height="100%" fill="white"/>
<text x="48" y="16" font-family="sans-serif" font-size="14">Successful RPS, closed-loop POST, concurrency 8, 15s</text>
<rect x="63.4" y="32.0" width="50.4" height="250.0" fill="#2f6f4e"/>
<text x="88.6" y="28.0" font-family="sans-serif" font-size="12" text-anchor="middle">344.0</text>
<text x="88.6" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">A1</text>
<rect x="144.7" y="139.3" width="50.4" height="142.7" fill="#2f6f4e"/>
<text x="169.9" y="135.3" font-family="sans-serif" font-size="12" text-anchor="middle">196.3</text>
<text x="169.9" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">A2</text>
<rect x="225.9" y="191.5" width="50.4" height="90.5" fill="#2f6f4e"/>
<text x="251.1" y="187.5" font-family="sans-serif" font-size="12" text-anchor="middle">124.5</text>
<text x="251.1" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">A4</text>
<rect x="307.2" y="210.7" width="50.4" height="71.3" fill="#2f6f4e"/>
<text x="332.4" y="206.7" font-family="sans-serif" font-size="12" text-anchor="middle">98.1</text>
<text x="332.4" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">B1</text>
<rect x="388.4" y="119.3" width="50.4" height="162.7" fill="#2f6f4e"/>
<text x="413.6" y="115.3" font-family="sans-serif" font-size="12" text-anchor="middle">223.8</text>
<text x="413.6" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">B2</text>
<rect x="469.7" y="110.0" width="50.4" height="172.0" fill="#2f6f4e"/>
<text x="494.9" y="106.0" font-family="sans-serif" font-size="12" text-anchor="middle">236.6</text>
<text x="494.9" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">B4</text>
<rect x="550.9" y="99.4" width="50.4" height="182.6" fill="#2f6f4e"/>
<text x="576.1" y="95.4" font-family="sans-serif" font-size="12" text-anchor="middle">251.2</text>
<text x="576.1" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">C4-pg2</text>
<rect x="632.2" y="108.3" width="50.4" height="173.7" fill="#2f6f4e"/>
<text x="657.4" y="104.3" font-family="sans-serif" font-size="12" text-anchor="middle">239.0</text>
<text x="657.4" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">D4-pool2</text>
</svg>

After

Width:  |  Height:  |  Size: 2.4 KiB

+320
View File
@@ -0,0 +1,320 @@
[
{
"tag": "A1",
"replicas": 1,
"api_cpus_each": "1.0",
"api_mem_each": "512m",
"api_cpu_total": 1.0,
"postgres_cpus": "1.0",
"db_pool_max_each": 8,
"db_pool_max_total": 8,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.109,
"concurrency": 8,
"successful_requests": 5197,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 343.964,
"successful_latency_ms": {
"p50": 19.732,
"p95": 42.061,
"p99": 81.001
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB",
"mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB",
"mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB",
"mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB",
"mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB",
"mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB"
],
"postgres_sessions": "9"
}
},
{
"tag": "A2",
"replicas": 2,
"api_cpus_each": "0.50",
"api_mem_each": "256m",
"api_cpu_total": 1.0,
"postgres_cpus": "1.0",
"db_pool_max_each": 4,
"db_pool_max_total": 8,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.206,
"concurrency": 8,
"successful_requests": 2985,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 196.3,
"successful_latency_ms": {
"p50": 26.613,
"p95": 89.893,
"p99": 123.339
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB",
"mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB",
"mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB",
"mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB",
"mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB",
"mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB",
"mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB"
],
"postgres_sessions": "9"
}
},
{
"tag": "A4",
"replicas": 4,
"api_cpus_each": "0.25",
"api_mem_each": "128m",
"api_cpu_total": 1.0,
"postgres_cpus": "1.0",
"db_pool_max_each": 2,
"db_pool_max_total": 8,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.224,
"concurrency": 8,
"successful_requests": 1895,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 124.476,
"successful_latency_ms": {
"p50": 39.136,
"p95": 185.654,
"p99": 267.62
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB",
"mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB",
"mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB",
"mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB",
"mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB",
"mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB",
"mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB",
"mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB",
"mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB"
],
"postgres_sessions": "9"
}
},
{
"tag": "B1",
"replicas": 1,
"api_cpus_each": "0.50",
"api_mem_each": "256m",
"api_cpu_total": 0.5,
"postgres_cpus": "1.0",
"db_pool_max_each": 4,
"db_pool_max_total": 4,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.139,
"concurrency": 8,
"successful_requests": 1485,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 98.094,
"successful_latency_ms": {
"p50": 85.074,
"p95": 167.209,
"p99": 201.857
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB",
"mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB",
"mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB",
"mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB",
"mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB",
"mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB"
],
"postgres_sessions": "5"
}
},
{
"tag": "B2",
"replicas": 2,
"api_cpus_each": "0.50",
"api_mem_each": "256m",
"api_cpu_total": 1.0,
"postgres_cpus": "1.0",
"db_pool_max_each": 4,
"db_pool_max_total": 8,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.136,
"concurrency": 8,
"successful_requests": 3388,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 223.842,
"successful_latency_ms": {
"p50": 24.1,
"p95": 81.009,
"p99": 142.445
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB",
"mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB",
"mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB",
"mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB",
"mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB",
"mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB",
"mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB"
],
"postgres_sessions": "9"
}
},
{
"tag": "B4",
"replicas": 4,
"api_cpus_each": "0.50",
"api_mem_each": "256m",
"api_cpu_total": 2.0,
"postgres_cpus": "1.0",
"db_pool_max_each": 4,
"db_pool_max_total": 16,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.181,
"concurrency": 8,
"successful_requests": 3592,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 236.616,
"successful_latency_ms": {
"p50": 22.559,
"p95": 80.196,
"p99": 131.013
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB",
"mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB",
"mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB",
"mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB",
"mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB",
"mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB",
"mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB",
"mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
"mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB"
],
"postgres_sessions": "17"
}
},
{
"tag": "C4-pg2",
"replicas": 4,
"api_cpus_each": "0.50",
"api_mem_each": "256m",
"api_cpu_total": 2.0,
"postgres_cpus": "2.0",
"db_pool_max_each": 4,
"db_pool_max_total": 16,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.128,
"concurrency": 8,
"successful_requests": 3800,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 251.189,
"successful_latency_ms": {
"p50": 19.749,
"p95": 77.161,
"p99": 101.17
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB",
"mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB",
"mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB",
"mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB",
"mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB",
"mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB",
"mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB",
"mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
"mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB"
],
"postgres_sessions": "17"
}
},
{
"tag": "D4-pool2",
"replicas": 4,
"api_cpus_each": "0.50",
"api_mem_each": "256m",
"api_cpu_total": 2.0,
"postgres_cpus": "1.0",
"db_pool_max_each": 2,
"db_pool_max_total": 8,
"load": {
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 15.0,
"elapsed_seconds": 15.156,
"concurrency": 8,
"successful_requests": 3622,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 238.98,
"successful_latency_ms": {
"p50": 22.721,
"p95": 74.859,
"p99": 98.063
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
},
"resources": {
"docker_stats": [
"mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB",
"mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB",
"mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB",
"mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB",
"mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB",
"mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB",
"mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB",
"mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB",
"mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB"
],
"postgres_sessions": "9"
}
}
]
+35
View File
@@ -0,0 +1,35 @@
global
log stdout format raw local0 info
maxconn 200
defaults
log global
mode http
option httplog
timeout connect 2s
timeout client 10s
timeout server 10s
timeout http-request 5s
timeout check 2s
timeout queue 2s
retries 1
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
accepted_payload_size 8192
frontend fe_http
bind *:8080
default_backend be_api
backend be_api
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr host api
http-check expect status 200
default-server inter 1s fall 2 rise 1
server-template srv 8 api:8080 check resolvers docker init-addr none resolve-prefer ipv4
+49
View File
@@ -0,0 +1,49 @@
-- Lab2 schema (written before application queries).
-- Source of truth: PostgreSQL for users, chats, membership, messages.
-- Transaction boundary: INSERT chat + INSERT members in one transaction (rollback together).
-- Sessions are stored in Redis with AOF, not in this database.
CREATE TABLE IF NOT EXISTS schema_migrations (
filename TEXT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE users (
id UUID PRIMARY KEY,
username TEXT NOT NULL,
display_name TEXT NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
);
CREATE UNIQUE INDEX users_username_key ON users (username);
CREATE TABLE chats (
id UUID PRIMARY KEY,
title TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
);
CREATE TABLE chat_members (
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users (id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_id_idx ON chat_members (user_id);
CREATE INDEX chats_created_at_id_idx ON chats (created_at, id);
CREATE TABLE messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
sender_id UUID NOT NULL REFERENCES users (id),
text TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000)
);
-- Keyset history: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n
CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id);
+14
View File
@@ -0,0 +1,14 @@
-- Shared idempotency for POST /messages. Scope: user + method + path + key.
-- The message row and this result are committed in one transaction.
-- A rolled-back request leaves no row, so a later retry with the same key can insert.
CREATE TABLE IF NOT EXISTS idempotency_keys (
user_id UUID NOT NULL REFERENCES users (id),
http_method TEXT NOT NULL,
path TEXT NOT NULL,
idem_key TEXT NOT NULL,
request_hash TEXT NOT NULL,
response_json TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, http_method, path, idem_key)
);
+6
View File
@@ -0,0 +1,6 @@
[pytest]
testpaths = student_tests
pythonpath = .
markers =
infra: compose stop/restart of redis or postgres
expiry: requires SESSION_TTL_SECONDS<=3
+2
View File
@@ -0,0 +1,2 @@
pytest==8.3.4
httpx==0.28.1
+5
View File
@@ -0,0 +1,5 @@
starlette==0.41.3
uvicorn[standard]==0.32.1
asyncpg==0.30.0
redis==5.2.1
argon2-cffi==23.1.0
+229
View File
@@ -0,0 +1,229 @@
#!/usr/bin/env python3
"""Grade 4: parallel idempotency, stop each API replica, rolling replacement under load."""
import json
import os
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
OUT = ROOT / "evidence" / "grade4"
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
def compose(*args, check=True):
cmd = ["docker", "compose"]
if (ROOT / ".env").is_file():
cmd.extend(["--env-file", str(ROOT / ".env")])
cmd.extend(args)
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
if check and proc.returncode != 0:
raise SystemExit("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or proc.stdout)[-800:]))
return proc
def bash_argv(script: Path):
if os.name != "nt":
return [str(script)]
git_bash = Path(r"C:\Program Files\Git\bin\bash.exe")
if git_bash.is_file():
return [str(git_bash), str(script)]
return ["bash", str(script)]
def hook_cmd():
return bash_argv(ROOT / "scripts" / "stop-one.sh")
def wait_ready(seconds=60):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health")
return
except Failure as exc:
last = exc
time.sleep(0.4)
raise SystemExit("API did not become ready: %s" % last)
def instance_ids(actor, samples=24):
found = []
client = Client(BASE, timeout=3)
for _ in range(samples):
found.append(client.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"])
return sorted(set(found))
def parallel_idempotency():
client = Client(BASE, timeout=20)
alice, _bob, _eve, chat = client.bootstrap()
key = uuid.uuid4().hex
text = "parallel-" + key
ids = []
errors = []
barrier = threading.Barrier(16)
def one():
local = Client(BASE, timeout=20)
barrier.wait()
try:
ids.append(local.message(alice, chat, text, key=key)["json"]["id"])
except Exception as exc:
errors.append(type(exc).__name__)
threads = [threading.Thread(target=one) for _ in range(16)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
unique = sorted(set(ids))
report = {
"workers": 16,
"successes": len(ids),
"errors": len(errors),
"distinct_message_ids": len(unique),
}
if len(unique) != 1 or errors:
raise SystemExit("parallel idempotency failed: %s" % report)
page = client.call(
"GET",
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
actor=alice,
schema="MessagePage",
)["json"]
copies = [item for item in page["items"] if item["text"] == text]
report["history_copies"] = len(copies)
if len(copies) != 1:
raise SystemExit("history has %s copies" % len(copies))
return report
def failover_one(actor, chat, instance_id):
key = "failover-" + uuid.uuid4().hex
text = "during-" + key
successes = []
failures = 0
started_box = {}
def traffic():
nonlocal failures
local = Client(BASE, timeout=2)
while "t0" not in started_box:
time.sleep(0.01)
deadline = started_box["t0"] + 12
recovered = False
while time.monotonic() < deadline:
try:
if not recovered:
response = local.message(actor, chat, text, key=key)
recovered = True
else:
response = local.message(actor, chat, "after-" + uuid.uuid4().hex)
successes.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"]))
except Failure:
failures += 1
time.sleep(0.05)
worker = threading.Thread(target=traffic)
worker.start()
time.sleep(0.2)
started = time.monotonic()
started_box["t0"] = started
env = os.environ.copy()
env["TARGET_INSTANCE_ID"] = instance_id
proc = subprocess.run(hook_cmd(), cwd=ROOT, env=env, capture_output=True, text=True)
hook_s = time.monotonic() - started
worker.join()
after = [item for item in successes if item[0] >= started]
recovery = None
if after:
recovery = round(after[0][0] - started, 3)
tail = [item[1] for item in after[-5:]]
return {
"target": instance_id,
"hook_exit": proc.returncode,
"hook_seconds": round(hook_s, 3),
"hook_stderr": (proc.stderr or "")[-300:],
"failed_attempts": failures,
"successes_after_hook_start": len(after),
"first_recovery_seconds": recovery,
"tail_instance_ids": tail,
"target_still_in_tail": instance_id in tail,
}
def restore_and_wait(actor):
compose("start", "api")
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
try:
if len(instance_ids(actor, samples=16)) >= 2:
return
except Failure:
pass
time.sleep(0.5)
raise SystemExit("second replica did not return")
def rolling_under_load():
out = OUT / "rolling-load.json"
load = subprocess.Popen(
[sys.executable, "-X", "utf8", str(ROOT / "tests" / "load.py"),
"--base-url", BASE, "--duration", "50", "--concurrency", "4", "--output", str(out)],
cwd=ROOT,
)
time.sleep(3)
cmd = bash_argv(ROOT / "scripts" / "rolling-update.sh")
started = time.monotonic()
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
elapsed = round(time.monotonic() - started, 3)
load.wait(timeout=90)
metrics = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
if proc.returncode != 0:
raise SystemExit("rolling update failed: %s" % (proc.stderr or proc.stdout)[-800:])
return {
"rolling_seconds": elapsed,
"rolling_stdout": (proc.stdout or "")[-400:],
"load": metrics,
}
def main():
OUT.mkdir(parents=True, exist_ok=True)
wait_ready()
client = Client(BASE, timeout=10)
alice, _bob, _eve, chat = client.bootstrap()
before = instance_ids(alice)
if len(before) < 2:
raise SystemExit("need 2 instance ids before grade-4 runs, saw %s" % before)
idem = parallel_idempotency()
failovers = []
for instance_id in before:
item = failover_one(alice, chat, instance_id)
failovers.append(item)
recovery = item["first_recovery_seconds"]
if item["hook_exit"] != 0 or item["target_still_in_tail"] or recovery is None or recovery > 10 or item["successes_after_hook_start"] < 5:
raise SystemExit("failover criteria failed: %s" % item)
restore_and_wait(alice)
rolling = rolling_under_load()
report = {
"instances_before": before,
"parallel_idempotency": idem,
"failovers": failovers,
"rolling": rolling,
}
(OUT / "summary.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+3
View File
@@ -0,0 +1,3 @@
@echo off
python "%~dp0restart.py"
exit /b %ERRORLEVEL%
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/env python3
"""Restart API, PostgreSQL and Redis without removing named volumes."""
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
def main() -> int:
env_file = ROOT / ".env"
cmd = ["docker", "compose"]
if env_file.is_file():
cmd.extend(["--env-file", str(env_file)])
cmd.extend(["restart", "postgres", "redis", "api"])
return subprocess.run(cmd, cwd=ROOT).returncode
if __name__ == "__main__":
sys.exit(main())
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
# Restart API replicas and every store they use. Volumes are kept.
set -eu
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"
if [ -f .env ]; then
docker compose --env-file .env restart postgres redis api
else
docker compose restart postgres redis api
fi
+55
View File
@@ -0,0 +1,55 @@
#!/bin/sh
# Replace API replicas one at a time. The other replica keeps serving.
set -eu
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"
compose() {
if [ -f .env ]; then
docker compose --env-file .env "$@"
else
docker compose "$@"
fi
}
if command -v python3 >/dev/null 2>&1; then
PY=python3
else
PY=python
fi
ids=$(compose ps -q api)
count=0
for id in $ids; do
count=$((count + 1))
done
if [ "$count" -lt 2 ]; then
echo "rolling update needs at least 2 running API replicas" >&2
exit 1
fi
for id in $ids; do
echo "replacing $(docker inspect -f '{{.Config.Hostname}}' "$id")"
docker stop -t 8 "$id" >/dev/null
docker rm "$id" >/dev/null
compose up -d --scale "api=$count" --no-deps --no-recreate api >/dev/null
ready=0
i=0
while [ "$i" -lt 40 ]; do
running=0
for cid in $(compose ps -q api); do
running=$((running + 1))
done
if [ "$running" -ge "$count" ] && "$PY" -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" >/dev/null 2>&1; then
ready=1
break
fi
i=$((i + 1))
sleep 1
done
if [ "$ready" -ne 1 ]; then
echo "replacement did not become ready" >&2
exit 1
fi
done
echo "rolling update finished; replicas=$count"
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env python3
"""Recreate API with SESSION_TTL_SECONDS=2, test expiry, restore TTL."""
import os
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def compose(args, extra_env=None):
env = os.environ.copy()
if extra_env:
env.update(extra_env)
cmd = ["docker", "compose"]
if (ROOT / ".env").is_file():
cmd.extend(["--env-file", str(ROOT / ".env")])
cmd.extend(args)
proc = subprocess.run(cmd, cwd=ROOT, env=env)
if proc.returncode != 0:
raise SystemExit(proc.returncode)
def wait_ready():
import urllib.request
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2)
return
except Exception:
time.sleep(0.5)
raise SystemExit("API did not become ready")
def main():
compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "2"})
wait_ready()
env = os.environ.copy()
env["SESSION_TTL_SECONDS"] = "2"
env["RUN_EXPIRY"] = "1"
proc = subprocess.run(
[sys.executable, "-m", "pytest", "student_tests/test_expiry.py", "-q"],
cwd=ROOT,
env=env,
)
compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "3600"})
wait_ready()
raise SystemExit(proc.returncode)
if __name__ == "__main__":
main()
+32
View File
@@ -0,0 +1,32 @@
#!/bin/sh
# Stop the single API replica whose X-Instance-Id is TARGET_INSTANCE_ID.
# Leaves it stopped. Does not touch the database, Redis, the load balancer, or volumes.
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"
compose() {
if [ -f .env ]; then
docker compose --env-file .env "$@"
else
docker compose "$@"
fi
}
found=0
for id in $(compose ps -q api); do
iid=$(docker exec "$id" cat /tmp/instance-id 2>/dev/null | tr -d '\r\n' || true)
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
if [ "$iid" = "$TARGET_INSTANCE_ID" ] || [ "$host" = "$TARGET_INSTANCE_ID" ]; then
docker stop -t 5 "$id" >/dev/null
found=1
break
fi
done
if [ "$found" -ne 1 ]; then
echo "no running API replica matched TARGET_INSTANCE_ID" >&2
exit 1
fi
+217
View File
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
"""Track 1: fixed API budget, then more replicas with more CPU, then one DB change.
The load profile is the public closed-loop POST generator. Same duration and
concurrency on every point. Raw JSON is written under evidence/track1/.
"""
import json
import os
import subprocess
import sys
import threading
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "evidence" / "track1"
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
DURATION = float(os.environ.get("TRACK_DURATION", "15"))
WARMUP = float(os.environ.get("TRACK_WARMUP", "5"))
CONCURRENCY = int(os.environ.get("TRACK_CONCURRENCY", "8"))
def compose(*args):
cmd = ["docker", "compose"]
if (ROOT / ".env").is_file():
cmd.extend(["--env-file", str(ROOT / ".env")])
cmd.extend(args)
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, env=os.environ.copy())
if proc.returncode != 0:
raise SystemExit((proc.stderr or proc.stdout)[-1000:])
return proc
def wait_ready(seconds=90):
import urllib.request
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as response:
if response.status == 200:
return
except Exception:
time.sleep(0.5)
raise SystemExit("ready timeout")
def apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool):
os.environ["API_CPUS"] = api_cpus
os.environ["API_MEM"] = api_mem
os.environ["POSTGRES_CPUS"] = postgres_cpus
os.environ["DB_POOL_MAX"] = str(pool)
compose("up", "-d", "--no-deps", "--force-recreate", "postgres")
deadline = time.monotonic() + 40
while time.monotonic() < deadline:
probe = subprocess.run(
["docker", "compose", "--env-file", str(ROOT / ".env"), "exec", "-T", "postgres", "pg_isready", "-U", "messenger", "-d", "messenger"],
cwd=ROOT,
capture_output=True,
text=True,
)
if probe.returncode == 0:
break
time.sleep(0.5)
else:
raise SystemExit("postgres did not become ready")
compose(
"up",
"-d",
"--no-deps",
"--force-recreate",
"--scale",
"api=%s" % replicas,
"api",
)
wait_ready()
running = [line for line in compose("ps", "-q", "api").stdout.splitlines() if line.strip()]
if len(running) != replicas:
raise SystemExit("expected %s api containers, saw %s" % (replicas, len(running)))
def run_load(tag, warmup=False):
args = [
sys.executable,
"-X",
"utf8",
str(ROOT / "tests" / "load.py"),
"--base-url",
BASE,
"--duration",
str(WARMUP if warmup else DURATION),
"--concurrency",
str(CONCURRENCY),
]
dest = None
if not warmup:
dest = OUT / ("%s.json" % tag)
args.extend(["--output", str(dest)])
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True)
if proc.returncode != 0:
raise SystemExit("load %s failed: %s" % (tag, (proc.stderr or proc.stdout)[-800:]))
if dest is None:
return None
return json.loads(dest.read_text(encoding="utf-8"))
def sample_resources():
stats = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
capture_output=True,
text=True,
)
activity = compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-tAc",
"SELECT count(*) FROM pg_stat_activity WHERE datname='messenger'",
)
return {
"docker_stats": [line for line in stats.stdout.splitlines() if line.strip()],
"postgres_sessions": activity.stdout.strip(),
}
def measure(tag, replicas, api_cpus, api_mem, postgres_cpus, pool):
print("profile %s" % tag, flush=True)
apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool)
run_load(tag, warmup=True)
box = {}
def snapshot():
time.sleep(DURATION / 2)
box["resources"] = sample_resources()
thread = threading.Thread(target=snapshot)
thread.start()
metrics = run_load(tag)
thread.join()
point = {
"tag": tag,
"replicas": replicas,
"api_cpus_each": api_cpus,
"api_mem_each": api_mem,
"api_cpu_total": round(replicas * float(api_cpus), 3),
"postgres_cpus": postgres_cpus,
"db_pool_max_each": pool,
"db_pool_max_total": replicas * pool,
"load": metrics,
"resources": box.get("resources"),
}
(OUT / ("%s.point.json" % tag)).write_text(json.dumps(point, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps({"tag": tag, "rps": metrics["successful_rps"], "errors": metrics["failed_requests"], "p95": metrics["successful_latency_ms"]["p95"]}), flush=True)
return point
def svg(points, path):
width, height = 720, 320
labels = [p["tag"] for p in points]
values = [p["load"]["successful_rps"] for p in points]
peak = max(values) or 1
left, top, plot_w, plot_h = 48, 20, width - 70, height - 70
bar_w = plot_w / len(points) * 0.62
gap = plot_w / len(points)
parts = [
'<?xml version="1.0" encoding="UTF-8"?>',
'<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d">' % (width, height),
'<rect width="100%" height="100%" fill="white"/>',
'<text x="48" y="16" font-family="sans-serif" font-size="14">Successful RPS, closed-loop POST, concurrency %d, %ss</text>'
% (CONCURRENCY, int(DURATION)),
]
for index, (label, value) in enumerate(zip(labels, values)):
h = plot_h * value / peak
x = left + index * gap + (gap - bar_w) / 2
y = top + 12 + (plot_h - h)
parts.append('<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" fill="#2f6f4e"/>' % (x, y, bar_w, h))
parts.append(
'<text x="%.1f" y="%.1f" font-family="sans-serif" font-size="12" text-anchor="middle">%.1f</text>'
% (x + bar_w / 2, y - 4, value)
)
parts.append(
'<text x="%.1f" y="%d" font-family="sans-serif" font-size="11" text-anchor="middle">%s</text>'
% (x + bar_w / 2, height - 18, label)
)
parts.append("</svg>")
path.write_text("\n".join(parts) + "\n", encoding="utf-8")
def main():
OUT.mkdir(parents=True, exist_ok=True)
points = []
# Series A: total API budget fixed at 1.0 CPU and 512 MiB. Pool total stays 8.
points.append(measure("A1", 1, "1.0", "512m", "1.0", 8))
points.append(measure("A2", 2, "0.50", "256m", "1.0", 4))
points.append(measure("A4", 4, "0.25", "128m", "1.0", 2))
# Series B: each new replica brings the same 0.50 CPU / 256 MiB. Postgres stays at 1 CPU.
points.append(measure("B1", 1, "0.50", "256m", "1.0", 4))
points.append(measure("B2", 2, "0.50", "256m", "1.0", 4))
points.append(measure("B4", 4, "0.50", "256m", "1.0", 4))
# Candidate improvement: give the database the extra CPU the API side already has.
points.append(measure("C4-pg2", 4, "0.50", "256m", "2.0", 4))
# Candidate improvement: stop the 4 pools from opening 16 server connections.
points.append(measure("D4-pool2", 4, "0.50", "256m", "1.0", 2))
(OUT / "summary.json").write_text(json.dumps(points, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
svg(points, OUT / "rps.svg")
# Return the stand to the documented 2 x 0.50 CPU layout.
apply_profile(2, "0.50", "256m", "1.0", 4)
print("wrote %s" % (OUT / "summary.json"))
if __name__ == "__main__":
main()
+71
View File
@@ -0,0 +1,71 @@
# Разбор лабораторной 3
Автор: Булацев, БПИ2301. Целевая оценка: 5, трек «Цена масштабирования».
## Суть
Нужно запустить несколько одинаковых процессов API за одной точкой входа и не потерять корректность, когда один из них останавливают.
Клиент больше не обращается к конкретному процессу. Он ходит в балансировщик, а тот выбирает живую реплику. Любая реплика должна узнать ту же сессию, увидеть уже подтверждённое сообщение и не создать второе сообщение, если клиент повторил запрос с тем же ключом.
Источник истины для пользователей, чатов и сообщений — PostgreSQL. Сессии — Redis с AOF. Память процесса для этого не используется.
## Какие проблемы решались
### Один процесс нельзя просто скопировать
Если сессия или кэш лежат в памяти процесса, второй экземпляр пользователя не узнает. Повторный запрос попадёт на другую реплику и получит 401 либо устаревшую историю.
Решение: сессия хранится в общем Redis, история читается из общего Postgres. Кэш истории тоже в Redis и сбрасывается после успешной записи, поэтому следующая реплика не отдаёт старую страницу. Клиента не приклеиваем к одной реплике.
### Клиент не должен выбирать процесс сам
Нужна одна публичная точка входа и при этом несколько процессов. `depends_on` только задаёт порядок запуска контейнеров. Healthcheck контейнера говорит Docker, жив ли он, но не перекладывает HTTP-запрос на другой адрес.
Решение: HAProxy публикует `127.0.0.1:8080` и сам ходит в DNS-имя `api`. У сервиса Compose нет `container_name`, поэтому реплик может быть сколько угодно. Порты API наружу не опубликованы. HAProxy держит до 8 слотов через `server-template` и обновляет адреса из DNS Docker раз в секунду. Проверка — `GET /health/ready` каждые 1 с, реплика выпадает после двух неудач. Это около 2 с плюс до 1 с устаревшего DNS, если контейнер уже исчез.
### Остановка процесса не должна обрывать стенд надолго
Docker посылает SIGTERM. Если процесс сразу умирает, балансировщик ещё считает его живым и шлёт туда запросы.
Решение согласовано с проверкой готовности. По SIGTERM реплика сразу отвечает 503 на `/health/ready` и на новые бизнес-запросы, а завершается через 2 с (`LB_DRAIN_SECONDS`). За это HAProxy успевает увидеть неготовый backend. `scripts/stop-one.sh` останавливает только контейнер с нужным `X-Instance-Id` и оставляет его остановленным. `X-Instance-Id` равен hostname и не меняется, пока жив процесс. На публичной проверке запись восстановилась за 2.465 с, подтверждённые сообщения и сессия сохранились.
### Повтор POST после таймаута может создать второе сообщение
Клиент не знает, успел ли сервер записать сообщение, если ответ потерялся. Повтор без защиты создаёт дубль. Две реплики могут принять один и тот же повтор одновременно.
Решение: для `POST /messages` обязателен `Idempotency-Key`. Область ключа — пользователь, метод, путь чата и сам ключ. Текст приводится к одному JSON (порядок полей не важен, пустой список вложений равен отсутствию поля) и хешируется. Сообщение и сохранённый ответ 201 пишутся одной транзакцией. Пока транзакция не закоммичена, строки результата нет, поэтому неуспешный запрос можно повторить. Одновременные повторы сериализует advisory lock в Postgres: побеждает одна вставка, остальные получают то же сообщение. Другой текст с тем же ключом — 409. Проверено 16 параллельными запросами: в истории одна копия.
### Миграции нельзя применять гонкой всех реплик
Несколько процессов при старте могут одновременно выполнить один и тот же SQL.
Решение: схему применяет отдельный одноразовый сервис `migrate` до старта API. У реплик `MIGRATIONS_ON_START=0`. Повторный запуск пропускает файлы, уже записанные в `schema_migrations`. На время применения берётся `pg_advisory_lock`.
### Больше реплик не означает больше пропускной способности
Гипотеза трека 1: при фиксированном CPU API дробление на процессы не поможет, а при росте числа реплик выигрыш кончится, когда упрёмся не в CPU отдельного API.
Один и тот же closed-loop, 8 параллельных POST, 15 с. При бюджете 1 CPU / 512 МиБ результат 344, 196 и 124 RPS для 1, 2 и 4 реплик. Postgres был занят на 16–33%: мешала квота CPU на каждом процессе. Когда одна реплика упиралась в 0.50 CPU (98 RPS, p95 167 мс), вторая такая же реплика подняла результат до 224 RPS и p95 81 мс. Это подтверждённое улучшение. Четвёртая реплика дала только 237 RPS, зато сессий Postgres стало 17 вместо 9. Удвоение CPU Postgres и уменьшение пула на этом профиле заметного прироста не дали. Подробные таблицы — в `REPORT.md`.
## Ответы на вопросы README
### Чем отказ процесса отличается от отказа узла?
Отказ процесса — это остановка одного API-контейнера. Балансировщик перестаёт слать ему запросы, вторая реплика на том же хосте продолжает читать и писать в общие Postgres и Redis. Данные не пропадают.
Отказ узла — это потеря всей машины. Вместе с ней пропадают все реплики API, HAProxy, Postgres и Redis, потому что они запущены на одном Docker-хосте. Несколько процессов API такую потерю не переживают. Отдельно живая реплика базы в этой работе не поднималась, поэтому выживание узла БД не заявляется.
### Почему depends_on и healthcheck сами по себе не являются балансировщиком?
`depends_on` лишь откладывает старт контейнера, пока другой не стал healthy. Он не принимает соединения клиентов и не выбирает, какому процессу отдать конкретный HTTP-запрос. Healthcheck Docker перезапускает или помечает свой контейнер, но не знает о соседних репликах и не проксирует трафик. Пока снаружи опубликован сам API, клиент попадает в один процесс в обход остальных. Балансировщик — отдельный процесс: он слушает единственный адрес, смотрит готовность backend-ов и перекладывает запрос на живой.
### Какие POST можно повторять после timeout и что защищает Idempotency-Key?
После таймаута можно повторять POST сообщения с тем же ключом и тем же телом. Если запись успела закоммититься, сервер вернёт тот же `201` и то же сообщение. Если не успела, повтор создаст его один раз. Другое тело с тем же ключом получит 409.
Защищает общая строка в Postgres на пару «пользователь + метод + путь + ключ» вместе с advisory lock и одной транзакцией на сообщение и сохранённый ответ. Ключ не защищает создание пользователя и создание чата: их повтор может создать второй объект, поэтому балансировщик такие POST сам не повторяет.
### Что сломается при увеличении числа API, если сессии находятся в локальной памяти?
Вход создаст сессию только в памяти той реплики, которая обработала логин. Следующий запрос балансировщик отдаст другой реплике, и она ответит 401, потому что токена у неё нет. Пользователю придётся входить заново, а при sticky session он останется привязан к одному процессу: отказ этого процесса снова потеряет сессию. Общие сообщения в Postgres это не чинят. Поэтому сессия лежит в Redis, который видят все реплики.
View File
+24
View File
@@ -0,0 +1,24 @@
import os
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
@pytest.fixture(scope="session")
def base_url():
return os.environ.get("BASE_URL", "http://127.0.0.1:8080")
@pytest.fixture(scope="session")
def live(base_url):
c = Client(base_url, timeout=15)
try:
c.call("GET", "/health/ready", schema="Health")
except Failure as exc:
pytest.skip("API is not ready: %s" % exc)
return c
+195
View File
@@ -0,0 +1,195 @@
"""Negative, pagination and auth tests for lab 2."""
import uuid
from urllib.parse import quote
from client import Client
PASSWORD = "Example-only-not-a-real-secret-123"
def register(c: Client, username=None):
username = username or ("u_" + uuid.uuid4().hex[:10])
user = c.call(
"POST",
"/api/v1/users",
201,
body={"username": username, "display_name": "Test User", "password": PASSWORD},
schema="User",
)["json"]
return user
def login(c: Client, username, password=PASSWORD):
import base64
cred = base64.b64encode((username + ":" + password).encode()).decode()
session = c.call(
"POST",
"/api/v1/auth/sessions",
201,
headers={"Authorization": "Basic " + cred},
schema="Session",
)["json"]
return {"id": session["user"]["id"], "user": session["user"], "token": session["token"], "registration": {"username": username, "password": password}}
def actor_of(c: Client, username=None):
user = register(c, username)
return login(c, user["username"])
def test_malformed_json(live: Client):
response = live.call(
"POST",
"/api/v1/users",
400,
raw=b"{not-json",
headers={"Content-Type": "application/json"},
)
assert response["json"]["error"]["code"] == "invalid_request"
assert response["headers"].get("X-Request-Id")
def test_duplicate_username(live: Client):
user = register(live, username="dup_" + uuid.uuid4().hex[:8])
again = live.call(
"POST",
"/api/v1/users",
409,
body={"username": user["username"], "display_name": "Other", "password": PASSWORD},
)
assert again["json"]["error"]["code"] == "conflict"
def test_empty_message_rejected(live: Client):
alice = actor_of(live)
bob = actor_of(live)
chat = live.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "t", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
response = live.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/messages",
400,
actor=alice,
body={"text": " "},
)
assert response["status"] == 400
def test_outsider_chat_is_404(live: Client):
alice = actor_of(live)
bob = actor_of(live)
eve = actor_of(live)
chat = live.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "secret", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
response = live.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve)
assert response["json"]["error"]["code"] == "not_found"
def test_message_pagination_multiple_pages(live: Client):
alice = actor_of(live)
bob = actor_of(live)
chat = live.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "pages", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
created = []
for idx in range(5):
created.append(live.message(alice, chat, "m%s" % idx)["json"])
collected = []
cursor = None
for _ in range(10):
path = "/api/v1/chats/%s/messages?limit=2" % chat["id"]
if cursor:
path += "&cursor=" + quote(cursor, safe="")
page = live.call("GET", path, actor=bob, schema="MessagePage")["json"]
collected.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
break
assert len(collected) == 5
ordered = sorted(created, key=lambda x: (x["created_at"], x["id"]))
assert collected == ordered
def test_cursor_from_other_chat_rejected(live: Client):
alice = actor_of(live)
bob = actor_of(live)
chat_a = live.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "a", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
chat_b = live.call(
"POST",
"/api/v1/chats",
201,
actor=alice,
body={"title": "b", "member_ids": [alice["id"], bob["id"]]},
schema="Chat",
)["json"]
msg = live.message(alice, chat_a, "only in a")["json"]
page = live.call(
"GET",
"/api/v1/chats/%s/messages?limit=1" % chat_a["id"],
actor=alice,
schema="MessagePage",
)["json"]
assert page["items"][0]["id"] == msg["id"]
cursor = page["next_cursor"]
# If history is a single item, next_cursor is null; encode a foreign cursor via first page of A after a dummy extra message.
if cursor is None:
live.message(alice, chat_a, "second")
page = live.call(
"GET",
"/api/v1/chats/%s/messages?limit=1" % chat_a["id"],
actor=alice,
schema="MessagePage",
)["json"]
cursor = page["next_cursor"]
response = live.call(
"GET",
"/api/v1/chats/%s/messages?limit=10&cursor=%s" % (chat_b["id"], quote(cursor, safe="")),
400,
actor=alice,
)
assert response["status"] == 400
def test_x_user_id_does_not_authenticate(live: Client):
alice = actor_of(live)
live.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]})
def test_wrong_password(live: Client):
user = register(live)
import base64
bad = base64.b64encode((user["username"] + ":wrong-password-xx").encode()).decode()
live.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + bad})
def test_logout(live: Client):
alice = actor_of(live)
live.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice)
live.call("GET", "/api/v1/users/me", 401, actor=alice)
+48
View File
@@ -0,0 +1,48 @@
"""10 clients × 10 messages = 100 unique messages."""
import threading
import uuid
from client import Client
from student_tests.test_api import actor_of
def test_concurrent_messages(live: Client):
users = [actor_of(live) for _ in range(10)]
member_ids = [u["id"] for u in users]
chat = live.call(
"POST",
"/api/v1/chats",
201,
actor=users[0],
body={"title": "load", "member_ids": member_ids},
schema="Chat",
)["json"]
errors = []
lock = threading.Lock()
def worker(user):
local = Client(live.base, timeout=15)
for _ in range(10):
try:
local.message(user, chat, uuid.uuid4().hex)
except Exception as exc:
with lock:
errors.append(str(exc))
threads = [threading.Thread(target=worker, args=(u,)) for u in users]
for thread in threads:
thread.start()
for thread in threads:
thread.join(timeout=60)
assert not errors, "unexpected failures: %s" % errors[:5]
page = live.call(
"GET",
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
actor=users[0],
schema="MessagePage",
)["json"]
ids = [item["id"] for item in page["items"]]
assert len(ids) == len(set(ids)) == 100
+17
View File
@@ -0,0 +1,17 @@
import os
import time
import pytest
from client import Client
@pytest.mark.expiry
def test_session_expires(live: Client):
ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
if os.environ.get("RUN_EXPIRY") != "1":
pytest.skip("set RUN_EXPIRY=1 and SESSION_TTL_SECONDS<=3")
assert ttl <= 3
actor = live.user("ttl")
live.call("GET", "/api/v1/users/me", actor=actor, schema="User")
time.sleep(ttl + 1.5)
live.call("GET", "/api/v1/users/me", 401, actor=actor)
+44
View File
@@ -0,0 +1,44 @@
"""Grade-4: username race and atomic chat rollback."""
import threading
import uuid
from client import Client
from student_tests.test_api import PASSWORD, actor_of
def test_register_race(live: Client):
username = "r_" + uuid.uuid4().hex[:12]
body = {"username": username, "display_name": "Race", "password": PASSWORD}
results = []
def once():
local = Client(live.base, timeout=20)
try:
results.append(local.call("POST", "/api/v1/users", [201, 409], body=body)["status"])
except Exception as exc:
results.append(str(exc))
threads = [threading.Thread(target=once) for _ in range(16)]
for t in threads:
t.start()
for t in threads:
t.join()
assert results.count(201) == 1, results
assert results.count(409) == 15, results
def test_chat_rollback(live: Client):
alice = actor_of(live)
before = live.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"]
missing = str(uuid.uuid4())
live.call(
"POST",
"/api/v1/chats",
400,
actor=alice,
body={"title": "rollback-should-not-exist", "member_ids": [alice["id"], missing]},
)
after = live.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"]
assert len(after) == len(before)
assert all(x["title"] != "rollback-should-not-exist" for x in after)
+89
View File
@@ -0,0 +1,89 @@
"""Dependency failure and Redis restart (AOF + cache rebuild)."""
import os
import subprocess
import time
from pathlib import Path
import pytest
from client import Client, Failure
ROOT = Path(__file__).resolve().parents[1]
def compose(*args):
env_file = ROOT / ".env"
cmd = ["docker", "compose"]
if env_file.is_file():
cmd.extend(["--env-file", str(env_file)])
cmd.extend(args)
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
if proc.returncode != 0:
raise AssertionError("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or "")[-500:]))
return proc
def wait_ready(c: Client, seconds=90):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
c.call("GET", "/health/ready", schema="Health")
return
except Failure as exc:
last = exc
time.sleep(0.5)
raise AssertionError("ready not restored: %s" % last)
def wait_status(c: Client, path, expected, seconds=30, actor=None):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
schema = "Health" if expected == 200 and path.startswith("/health") else None
c.call("GET", path, expected, actor=actor, schema=schema)
return
except Failure as exc:
last = exc
time.sleep(0.4)
raise AssertionError("did not observe %s for %s: %s" % (expected, path, last))
@pytest.mark.infra
def test_redis_and_postgres_unavailable(live: Client):
if os.environ.get("RUN_INFRA") != "1":
pytest.skip("set RUN_INFRA=1 to stop compose services")
alice = live.user("infra")
compose("stop", "redis")
wait_status(live, "/health/live", 200)
wait_status(live, "/health/ready", 503)
wait_status(live, "/api/v1/users/me", 503, actor=alice)
compose("start", "redis")
wait_ready(live)
live.call("GET", "/api/v1/users/me", actor=alice, schema="User")
compose("stop", "postgres")
wait_status(live, "/health/live", 200)
wait_status(live, "/health/ready", 503)
compose("start", "postgres")
wait_ready(live)
live.call("GET", "/api/v1/users/me", actor=alice, schema="User")
@pytest.mark.infra
def test_redis_restart_keeps_session_and_history(live: Client):
if os.environ.get("RUN_INFRA") != "1":
pytest.skip("set RUN_INFRA=1 to restart redis")
alice, bob, _eve, chat = live.bootstrap()
msg = live.message(alice, chat, "before-redis-restart")["json"]
compose("restart", "redis")
wait_ready(live)
me = live.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"]
assert me["id"] == alice["id"]
page = live.call(
"GET",
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
actor=bob,
schema="MessagePage",
)["json"]
assert any(m["id"] == msg["id"] for m in page["items"])
+83
View File
@@ -0,0 +1,83 @@
"""Track 1: cache consistency — stampede, stale read, read-after-write."""
import json
import threading
import uuid
from concurrent.futures import ThreadPoolExecutor
from client import Client
def _lab_metrics(c: Client):
return json.loads(c.call("GET", "/api/v1/_lab/metrics")["bytes"].decode())
def _lab_flags(c: Client, body):
return json.loads(c.call("POST", "/api/v1/_lab/flags", 200, body=body)["bytes"].decode())
def test_read_after_write(live: Client):
alice, bob, _eve, chat = live.bootstrap()
_lab_flags(live, {"invalidate": True, "singleflight": True, "reset_sql": True})
first = live.message(alice, chat, "raw-visible-" + uuid.uuid4().hex)["json"]
page = live.call(
"GET",
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
actor=bob,
schema="MessagePage",
)["json"]
assert any(m["id"] == first["id"] for m in page["items"])
def test_stale_without_invalidation_then_fixed(live: Client):
alice, bob, _eve, chat = live.bootstrap()
path = "/api/v1/chats/%s/messages?limit=50" % chat["id"]
live.call("GET", path, actor=alice, schema="MessagePage")
_lab_flags(live, {"invalidate": False, "singleflight": True})
try:
stale_text = "stale-probe-" + uuid.uuid4().hex
live.message(alice, chat, stale_text)
page = live.call("GET", path, actor=bob, schema="MessagePage")["json"]
assert all(m["text"] != stale_text for m in page["items"])
finally:
_lab_flags(live, {"invalidate": True})
fresh_text = "fresh-probe-" + uuid.uuid4().hex
live.message(alice, chat, fresh_text)
page2 = live.call("GET", path, actor=bob, schema="MessagePage")["json"]
assert any(m["text"] == fresh_text for m in page2["items"])
def test_stampede_singleflight_reduces_sql(live: Client):
alice, _bob, _eve, chat = live.bootstrap()
live.message(alice, chat, "stampede-seed")
path = "/api/v1/chats/%s/messages?limit=50" % chat["id"]
def burst(n=24):
barrier = threading.Barrier(n)
def one(_i):
local = Client(live.base)
barrier.wait()
return local.call("GET", path, actor=alice, schema="MessagePage")
with ThreadPoolExecutor(max_workers=n) as pool:
list(pool.map(one, range(n)))
try:
_lab_flags(live, {"singleflight": False, "invalidate": True, "reset_sql": True, "delay_ms": 250})
live.message(alice, chat, "miss-" + uuid.uuid4().hex)
_lab_flags(live, {"reset_sql": True, "singleflight": False, "delay_ms": 250})
burst()
unprotected = _lab_metrics(live)["history_sql"]
_lab_flags(live, {"singleflight": True, "reset_sql": True, "delay_ms": 250})
live.message(alice, chat, "miss2-" + uuid.uuid4().hex)
_lab_flags(live, {"reset_sql": True, "singleflight": True, "delay_ms": 250})
burst()
protected = _lab_metrics(live)["history_sql"]
finally:
_lab_flags(live, {"invalidate": True, "singleflight": True, "delay_ms": 0})
print(json.dumps({"stampede_sql_without_singleflight": unprotected, "stampede_sql_with_singleflight": protected}))
assert unprotected >= 8, unprotected
assert protected < unprotected, (protected, unprotected)
assert protected <= 4, protected