forked from mtusi2026/mtusi2026lab3
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2038551342 | ||
|
|
99ce1b1391 | ||
|
|
fd54119d3f | ||
|
|
3601ca5443 |
+15
-1
@@ -3,11 +3,25 @@
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
LOG_LEVEL=info
|
||||
SHUTDOWN_GRACE_SECONDS=10
|
||||
SESSION_TTL_SECONDS=3600
|
||||
CACHE_TTL_SECONDS=60
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=redis
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
API_CPUS=0.50
|
||||
API_MEM=256m
|
||||
POSTGRES_CPUS=1.0
|
||||
POSTGRES_MEM=512m
|
||||
DB_POOL_MAX=4
|
||||
LB_DRAIN_SECONDS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
|
||||
# Compose interpolates these. Synthetic local values only; replace after copy.
|
||||
POSTGRES_PASSWORD=lab2-dev-only-change-me
|
||||
DATABASE_URL=postgresql://messenger:lab2-dev-only-change-me@postgres:5432/messenger
|
||||
REDIS_URL=redis://redis:6379/0
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
## Лабораторная 3
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Булацев / БПИ2301
|
||||
Целевая оценка (3/4/5): 5
|
||||
Трек 5, если заявлен: 1, цена масштабирования
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено: API lab2 (PostgreSQL, Redis-сессии, кэш). Добавлены HAProxy, масштабируемый сервис API без внешнего порта, одноразовые миграции, общий Idempotency-Key и X-Instance-Id.
|
||||
Команды запуска из чистого clone: `cp .env.example .env`, затем `docker compose --env-file .env up -d --build --scale api=2`, дождаться `http://127.0.0.1:8080/health/ready`. Подробности в REPORT.md.
|
||||
Результаты проверок и ссылка на REPORT.md: smoke PASS; persistence PASS; failover — 2 реплики, 1 ошибка, восстановление записи 2.465 с; трек 1 в `evidence/track1/`. См. REPORT.md.
|
||||
Ограничения / известные ошибки: один хост; LB, Postgres и Redis остаются единой точкой отказа. На Windows `.sh` не запускается напрямую (`WinError 193`); hook рассчитан на POSIX.
|
||||
Источники и использование ИИ: README и контракт курса, документация HAProxy и Compose. Сборку решения вёл агент Cursor (Grok).
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
FROM python:3.12.8-slim-bookworm AS runtime
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN groupadd --system app && useradd --system --gid app --home /app --shell /usr/sbin/nologin app \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app ./app
|
||||
COPY migrations ./migrations
|
||||
COPY contracts/openapi.json ./contracts/openapi.json
|
||||
|
||||
USER app
|
||||
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=12 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"
|
||||
|
||||
CMD ["python", "-m", "app.main"]
|
||||
@@ -4,27 +4,45 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover test-student test-infra test-expiry explain evidence-grade4 evidence-track1
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
$(PYTHON) -X utf8 scripts/check_contract.py
|
||||
|
||||
validate:
|
||||
$(PYTHON) scripts/validate_openapi.py
|
||||
$(PYTHON) -X utf8 scripts/validate_openapi.py
|
||||
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
$(PYTHON) -X utf8 tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
$(PYTHON) -X utf8 tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
$(PYTHON) -X utf8 tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
$(PYTHON) -X utf8 tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-student:
|
||||
$(PYTHON) -m pytest student_tests -q -m "not infra and not expiry"
|
||||
|
||||
test-infra:
|
||||
RUN_INFRA=1 $(PYTHON) -m pytest student_tests/test_infra.py -q -m infra
|
||||
|
||||
test-expiry:
|
||||
$(PYTHON) scripts/run_expiry.py
|
||||
|
||||
explain:
|
||||
docker compose --env-file .env exec -T api python -m app.tools.explain
|
||||
|
||||
evidence-grade4:
|
||||
$(PYTHON) -X utf8 scripts/grade4_evidence.py
|
||||
|
||||
evidence-track1:
|
||||
$(PYTHON) -X utf8 scripts/track1_experiment.py
|
||||
|
||||
@@ -1,37 +1,109 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: **Булацев**, **БПИ2301** (ветка `solution/bpi2301-Bulatsev`)
|
||||
- Целевая оценка; выбранный трек 5: **5, трек 1 «Цена масштабирования»**
|
||||
- Перенос: API lab2 (PostgreSQL, Redis-сессии, кэш истории, миграции). В этой работе добавлены HAProxy, две и более реплики API без внешнего порта, общий `Idempotency-Key` в PostgreSQL и `X-Instance-Id`.
|
||||
- Commit/PR: ещё не опубликован
|
||||
- Источники и помощь ИИ: контракт и README курса; образы Postgres 16.6, Redis 7.4.2, HAProxy 2.8. Документация HAProxy `server-template` и Docker Compose. Реализацию и прогоны помог собрать агент Cursor (Grok). На защите нужно объяснять схему своими словами.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Стенд: Windows 11, CPU 12th Gen Intel Core i5-12450H, Docker Desktop 29.5.3 / Compose 5.1.4, лимит Docker 12 vCPU и 7.62 ГиБ. Образы зафиксированы тегами: `python:3.12.8-slim-bookworm` (сборка `mtusi-messenger:lab3`), `postgres:16.6-alpine`, `redis:7.4.2-alpine`, `haproxy:2.8.14-alpine`.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --scale api=2
|
||||
```
|
||||
|
||||
Дождаться `200` от `http://127.0.0.1:8080/health/ready`. Этот адрес — HAProxy. Реплики API наружу порты не публикуют.
|
||||
|
||||
Миграции выполняет одноразовый сервис `migrate` (`python -m app.migrate`) после healthy Postgres и до старта API. Повторный запуск пропускает уже записанные файлы. Дополнительно `pg_advisory_lock` не даёт двум миграторам применить один файл. У API `MIGRATIONS_ON_START=0`.
|
||||
|
||||
Проверки:
|
||||
|
||||
```sh
|
||||
python -X utf8 scripts/check_contract.py
|
||||
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
python -X utf8 -m pytest student_tests -q -m "not infra and not expiry"
|
||||
```
|
||||
|
||||
`scripts/restart.sh` и `scripts/stop-one.sh` — POSIX. На Linux: `chmod +x` и `make test-persistence` / `make test-failover`. На этом Windows `CreateProcess` не запускает `.sh` (`WinError 193`); те же команды прогнаны интерпретатором `python3` в контейнере `docker:27-cli` с сокетом того же Docker. Скрипты при этом управляли контейнерами стенда.
|
||||
|
||||
Остановка без удаления данных: `docker compose --env-file .env stop`. Учебные тома удаляет только явная команда `docker compose --env-file .env down -v`.
|
||||
|
||||
Чтобы освободить `127.0.0.1:8080`, перед запуском был остановлен контейнер `mtusi2026lab2-api-1`. Postgres и Redis lab2 оставались запущены; в срезах `docker stats` их CPU был около 0–4%.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Клиент ходит только в `lb` (HAProxy, `127.0.0.1:8080`). Backend — DNS-имя `api` сервиса Compose: `server-template` на 8 слотов, `resolvers docker` (`127.0.0.11`), `hold valid 1s`. Балансировка `roundrobin`, без sticky session. Сессии лежат в Redis (AOF + RDB), пользователи, чаты, сообщения и результаты идемпотентности — в PostgreSQL. Локальная память реплики не нужна для корректности следующего запроса.
|
||||
|
||||
`X-Instance-Id` — hostname контейнера, он не меняется, пока жив процесс. `scripts/stop-one.sh` сравнивает `TARGET_INSTANCE_ID` с `/tmp/instance-id` или hostname и делает `docker stop -t 5` одного контейнера. `restart: unless-stopped` не поднимает его обратно.
|
||||
|
||||
`POST /messages` требует `Idempotency-Key` (`[A-Za-z0-9._:-]{1,128}`). Область — пользователь + `POST` + путь чата + ключ. Канонический JSON (`text` и `attachment_ids`, пустой список равен отсутствию поля) хешируется SHA-256. Тот же хеш в течение `IDEMPOTENCY_TTL_SECONDS` (86400) возвращает сохранённый `201` и то же сообщение. Другой текст — `409`. Вставка сообщения и строки результата — одна транзакция; ошибка откатывает обе. Одновременные повторы сериализует `pg_advisory_xact_lock` в Postgres, поэтому две реплики не создают два сообщения. Неуспешный запрос строку не пишет, повтор с тем же ключом безопасен.
|
||||
|
||||
Проверки готовности. `GET /health/live` — только процесс. `GET /health/ready` — Postgres и Redis; во время SIGTERM сразу `503`. HAProxy: `inter 1s`, `fall 2`, `rise 1`, `timeout check 2s`. Неготовая реплика выпадает примерно за 2 с. Если контейнер уже исчез, DNS-кэш HAProxy живёт ещё до 1 с. На SIGTERM API перестаёт принимать бизнес-запросы, ready становится 503, процесс завершается через `LB_DRAIN_SECONDS` (2). `stop_grace_period` сервиса — 8 с. Таймауты прокси: connect 2 с, client/server 10 с. HAProxy не повторяет POST.
|
||||
|
||||
Соответствие `.env.example`: `DATABASE_URL` и `REDIS_URL` собираются в Compose из `POSTGRES_PASSWORD`; `API_CPUS`, `API_MEM`, `POSTGRES_CPUS`, `DB_POOL_MAX`, `LB_DRAIN_SECONDS`, `IDEMPOTENCY_TTL_SECONDS` интерполируются в `compose.yaml`. Рабочая точка стенда — 2 реплики по 0.50 CPU и 256 МиБ, Postgres 1.0 CPU / 512 МиБ, пул по 4 соединения. `container_name` нет.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Контракт offline | `python -X utf8 scripts/check_contract.py` | PASS | `evidence/public-checks.txt` |
|
||||
| Smoke через LB | `tests/smoke.py` | PASS, lab3 | `evidence/public-checks.txt` |
|
||||
| Две реплики, одна сессия | 12× `GET /health/ready` | Чередование `03c3a0545628` и `b555f8327eee` | наблюдение до отказов; после экспериментов новые id, см. ниже |
|
||||
| Общая БД и сессии | persistence hook | PASS, старый токен жив | `evidence/public-checks.txt` |
|
||||
| Отказ одной реплики ≤10 с | `tests/resilience.py --mode failover` | 2 реплики до отказа; 109 успешных записей; 1 ошибка; первое подтверждение после hook 2.465 с; дублей нет | `evidence/public-checks.txt` |
|
||||
| Остановлен именно целевой контейнер | `docker inspect` после failover | `mtusi2026lab3-api-1` Exited (0), hostname `b555f8327eee`; вторая реплика Up | `evidence/public-checks.txt` |
|
||||
| Идемпотентность 16 параллельных запросов | `scripts/grade4_evidence.py` | 16×201, 1 id, в истории 1 копия | `evidence/grade4/summary.json` |
|
||||
| Поочерёдный отказ каждой реплики | тот же скрипт | hook 2.373 с и 1.744 с; в последних ответах только живая реплика; 2 и 1 ошибка | `evidence/grade4/summary.json` |
|
||||
| Rolling update под нагрузкой | `scripts/rolling-update.sh` во время `tests/load.py` 50 с, concurrency 4 | 8361 успех, 18 ошибок (0.21%), RPS 166.9, p50/p95/p99 13.5/57.8/96.4 мс, замена 5.9 с | `evidence/grade4/summary.json`, `evidence/grade4/rolling-load.json` |
|
||||
| Лимиты, нет обхода LB | `docker ps` | API `8080/tcp` без host-порта; наружу только `127.0.0.1:8080` у lb | `compose.yaml` |
|
||||
| Трек 1 | `scripts/track1_experiment.py` | серии A/B и проверка двух правок, см. ниже | `evidence/track1/summary.json`, `evidence/track1/rps.svg` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Студенческие тесты lab2 (гонки, откат чата, кэш, конкурентные сообщения): 15 passed. `RUN_INFRA` и expiry в этот прогон не входили.
|
||||
|
||||
После трека 1 стенд возвращён к 2 репликам по 0.50 CPU. Новые hostname на ready: `0868e9d2ba45` и `14cb087ebb10`. Старые id исчезли вместе с пересозданными контейнерами; это не sticky session.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Гипотеза. При фиксированном бюджете API 1.0 CPU и 512 МиБ дробление на 2 и 4 реплики не поднимет RPS: каждая упрётся в свой cgroup, а Postgres ещё свободен. Если давать каждой новой реплике те же 0.50 CPU, RPS вырастет, пока реплика упирается в квоту, и почти остановится, когда клиентский closed-loop и единственный Postgres перестанут получать пользу от лишних процессов.
|
||||
|
||||
Методика. Генератор `tests/load.py`: closed-loop POST сообщений, concurrency 8, 5 с прогрев (результат выбрасывается), 15 с замер. Ошибок автоматических retry нет. На каждую точку один прогон. В середине замера один срез `docker stats` и `pg_stat_activity`. В `docker stats` 100% — одно ядро. Профиль A держит сумму API равной 1.0 CPU / 512 МиБ и сумму пулов равной 8. Профиль B добавляет реплики по 0.50 CPU / 256 МиБ при Postgres 1.0 CPU.
|
||||
|
||||
| Точка | Реплики | CPU API суммарно | Успешный RPS | Ошибки | p50 / p95 / p99, мс | CPU API по stats | CPU Postgres | Сессии Postgres | CPU LB |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| A1 | 1 × 1.00 | 1.0 | 344.0 | 0 | 19.7 / 42.1 / 81.0 | 98% | 33% | 9 | 21% |
|
||||
| A2 | 2 × 0.50 | 1.0 | 196.3 | 0 | 26.6 / 89.9 / 123.3 | 49% и 48% | 28% | 9 | 20% |
|
||||
| A4 | 4 × 0.25 | 1.0 | 124.5 | 0 | 39.1 / 185.7 / 267.6 | 20–25% | 16% | 9 | 9% |
|
||||
| B1 | 1 × 0.50 | 0.5 | 98.1 | 0 | 85.1 / 167.2 / 201.9 | 48% | 17% | 5 | 8% |
|
||||
| B2 | 2 × 0.50 | 1.0 | 223.8 | 0 | 24.1 / 81.0 / 142.4 | 49% и 48% | 29% | 9 | 16% |
|
||||
| B4 | 4 × 0.50 | 2.0 | 236.6 | 0 | 22.6 / 80.2 / 131.0 | 35–46% | 48% | 17 | 21% |
|
||||
| C4-pg2 | как B4, Postgres 2.0 CPU | 2.0 | 251.2 | 0 | 19.7 / 77.2 / 101.2 | 40–43% | 43% | 17 | 22% |
|
||||
| D4-pool2 | как B4, пул 2 на реплику | 2.0 | 239.0 | 0 | 22.7 / 74.9 / 98.1 | 45–48% | 49% | 9 | 22% |
|
||||
|
||||
График: `evidence/track1/rps.svg`. Сырые точки: `evidence/track1/*.point.json`.
|
||||
|
||||
Что из этого следует. В серии A Postgres не насыщен (16–33%), память API далеко от лимита. Падает сам API: одна реплика с целым ядром даёт 344 RPS, те же ресурсы четырьмя квотами по 0.25 CPU — 124 RPS, p95 с 42 до 186 мс. Рост числа процессов при фиксированном CPU вреден из-за квот CFS и накладных расходов, а не из-за нехватки БД.
|
||||
|
||||
В серии B первое ядро API ещё нужно: B1 упирается в квоту 0.50 CPU (stats около 50%, p95 167 мс). Вторая такая реплика — подтверждённое улучшение: RPS 98.1 → 223.8, p95 167 → 81 мс, Postgres только с 17% до 29%. Четыре реплики против двух — уже площадка: +1.0 CPU API даёт 223.8 → 236.6 RPS (+6%), зато сессий Postgres 9 → 17 и CPU Postgres 29% → 48%, LB 16% → 21%. Восемь замкнутых воркеров не заполняют четыре процесса, а каждый INSERT всё равно идёт в один Postgres. Пул здесь не узкое место: D4 снижает сессии с 17 до 9 и почти не меняет RPS (239 против 237). Дополнительное ядро Postgres тоже не подтвердилось: 251 против 237 на одном 15-секундном прогоне, срез CPU Postgres так и не упёрся в 100%.
|
||||
|
||||
Ограничение измерения. Один прогон на точку, closed-loop и coordinated omission, перцентили только успешных запросов, таблица `messages` растёт от точки к точке, `docker stats` — один срез. Это не паспорт максимальной пропускной способности машины.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| `scripts/restart.sh` (api, postgres, redis), тома на месте | те же сообщения и тот же токен | PASS публичного persistence | готовность вернулась внутри бюджета теста 90 с; данных не теряли |
|
||||
| `scripts/stop-one.sh` по `X-Instance-Id` | живая реплика принимает запись ≤10 с, дубля нет | 2 id до отказа; 1 ошибка; 109 записей; целевой контейнер Exited (0) | 2.465 с до первого подтверждения после возврата hook |
|
||||
| Та же остановка второй реплики после старта первой | хвост ответов только от оставшейся | оба прогона: target отсутствует в последних 5 id | hook 1.7–2.4 с; ошибок 1 и 2 |
|
||||
| Поочерёдная замена обеих реплик под нагрузкой | короткие ошибки, запись продолжается | 18 ошибок на 8379 попыток | скрипт замены 5.9 с |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Стенд переживает остановку одного процесса API и перезапуск контейнеров с сохранёнными томами. Он не переживает потерю единственного хоста, HAProxy, Postgres или Redis: это оставшиеся SPOF. Трек 2 не делался, выживание узла БД не заявляется.
|
||||
|
||||
Sticky session нет и не нужен: сессия в Redis, идемпотентность в Postgres. Повторно безопасно отправлять POST сообщения с тем же ключом. Повтор POST пользователя или чата по-прежнему может создать второй объект; прокси такие POST не повторяет.
|
||||
|
||||
Миграции совместимы с уже работающей схемой lab2 (новый файл `002_idempotency.sql`, старые таблицы не переписываются). Одновременный запуск двух версий кода с разной схемой отдельно не гонялся.
|
||||
|
||||
Подтверждённое улучшение — вторая реплика, когда первая исчерпала квоту 0.50 CPU (B1→B2). Четвёртая реплика, удвоение CPU Postgres и урезание пула на этом профиле прироста не показали. Следующий осмысленный шаг — поднять concurrency клиента выше 8 и повторить B2/B4 несколькими прогонами, прежде чем добавлять реплики Postgres.
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Messenger API (lab 2): PostgreSQL source of truth, Redis sessions/cache."""
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import TypeVar
|
||||
|
||||
from app import config, rediscli
|
||||
from app.errors import AppError
|
||||
|
||||
T = TypeVar("T")
|
||||
CFG_SINGLEFLIGHT = "cfg:singleflight"
|
||||
CFG_INVALIDATE = "cfg:invalidate"
|
||||
CFG_DELAY_MS = "cfg:delay_ms"
|
||||
SQL_HISTORY = "metrics:sql:messages_select"
|
||||
|
||||
|
||||
class SingleFlight:
|
||||
def __init__(self) -> None:
|
||||
self._lock = asyncio.Lock()
|
||||
self._inflight: dict[str, asyncio.Task] = {}
|
||||
|
||||
async def do(self, key: str, factory: Callable[[], Awaitable[T]]) -> T:
|
||||
async with self._lock:
|
||||
task = self._inflight.get(key)
|
||||
if task is None:
|
||||
task = asyncio.create_task(factory())
|
||||
self._inflight[key] = task
|
||||
owner = True
|
||||
else:
|
||||
owner = False
|
||||
try:
|
||||
return await asyncio.shield(task)
|
||||
finally:
|
||||
if owner:
|
||||
async with self._lock:
|
||||
self._inflight.pop(key, None)
|
||||
|
||||
|
||||
_sf = SingleFlight()
|
||||
|
||||
|
||||
async def _flag(name: str, default: bool = True) -> bool:
|
||||
if not config.lab_mode():
|
||||
return default
|
||||
try:
|
||||
raw = await rediscli.require().get(name)
|
||||
except Exception:
|
||||
return default
|
||||
if raw is None:
|
||||
return default
|
||||
return raw not in {"0", "false", "off"}
|
||||
|
||||
|
||||
async def singleflight_enabled() -> bool:
|
||||
return await _flag(CFG_SINGLEFLIGHT, True)
|
||||
|
||||
|
||||
async def invalidate_enabled() -> bool:
|
||||
return await _flag(CFG_INVALIDATE, True)
|
||||
|
||||
|
||||
async def incr_history_sql() -> None:
|
||||
try:
|
||||
await rediscli.require().incr(SQL_HISTORY)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def history_sql_count() -> int:
|
||||
try:
|
||||
return int(await rediscli.require().get(SQL_HISTORY) or 0)
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
async def reset_history_sql_count() -> None:
|
||||
try:
|
||||
await rediscli.require().set(SQL_HISTORY, 0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def load_delay_seconds() -> float:
|
||||
if not config.lab_mode():
|
||||
return 0.0
|
||||
try:
|
||||
raw = await rediscli.require().get(CFG_DELAY_MS)
|
||||
except Exception:
|
||||
return 0.0
|
||||
try:
|
||||
return max(0.0, float(raw or 0) / 1000.0)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
async def set_flags(
|
||||
*,
|
||||
singleflight: bool | None = None,
|
||||
invalidate: bool | None = None,
|
||||
delay_ms: int | None = None,
|
||||
) -> dict:
|
||||
r = rediscli.require()
|
||||
if singleflight is not None:
|
||||
await r.set(CFG_SINGLEFLIGHT, "1" if singleflight else "0")
|
||||
if invalidate is not None:
|
||||
await r.set(CFG_INVALIDATE, "1" if invalidate else "0")
|
||||
if delay_ms is not None:
|
||||
await r.set(CFG_DELAY_MS, str(int(delay_ms)))
|
||||
return {
|
||||
"singleflight": await singleflight_enabled(),
|
||||
"invalidate": await invalidate_enabled(),
|
||||
"history_sql": await history_sql_count(),
|
||||
"delay_ms": int((await load_delay_seconds()) * 1000),
|
||||
}
|
||||
|
||||
|
||||
async def get_history_page(chat_id: str, cursor: str, limit: int, loader: Callable[[], Awaitable[dict]]) -> dict:
|
||||
r = rediscli.require()
|
||||
try:
|
||||
version = await r.get("histver:" + chat_id) or "0"
|
||||
key = f"hist:{chat_id}:{version}:{cursor}:{limit}"
|
||||
cached = await r.get(key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
|
||||
async def load_and_store() -> dict:
|
||||
delay = await load_delay_seconds()
|
||||
if delay:
|
||||
await asyncio.sleep(delay)
|
||||
page = await loader()
|
||||
try:
|
||||
await r.set(key, json.dumps(page, ensure_ascii=False), ex=config.CACHE_TTL_SECONDS)
|
||||
except Exception:
|
||||
pass
|
||||
return page
|
||||
|
||||
if await singleflight_enabled():
|
||||
return await _sf.do(key, load_and_store)
|
||||
return await load_and_store()
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
return await loader()
|
||||
|
||||
|
||||
async def invalidate_history(chat_id: str) -> None:
|
||||
if not await invalidate_enabled():
|
||||
return
|
||||
try:
|
||||
await rediscli.require().incr("histver:" + chat_id)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,42 @@
|
||||
import os
|
||||
import socket
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _env(name: str, default: str = "") -> str:
|
||||
value = os.environ.get(name)
|
||||
return default if value is None or value == "" else value
|
||||
|
||||
|
||||
def _file_or_env(name: str, default: str = "") -> str:
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path:
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
return _env(name, default)
|
||||
|
||||
|
||||
HOST = _env("APP_HOST", "0.0.0.0")
|
||||
PORT = int(_env("APP_PORT", "8080"))
|
||||
ENV = _env("APP_ENV", "lab")
|
||||
LOG_LEVEL = _env("LOG_LEVEL", "info").lower()
|
||||
SHUTDOWN_GRACE_SECONDS = float(_env("SHUTDOWN_GRACE_SECONDS", "10"))
|
||||
SESSION_TTL_SECONDS = int(_env("SESSION_TTL_SECONDS", "3600"))
|
||||
CACHE_TTL_SECONDS = int(_env("CACHE_TTL_SECONDS", "60"))
|
||||
DATABASE_URL = _file_or_env("DATABASE_URL")
|
||||
REDIS_URL = _file_or_env("REDIS_URL", "redis://redis:6379/0")
|
||||
SESSION_STORE = _env("SESSION_STORE", "redis")
|
||||
DB_POOL_MIN = int(_env("DB_POOL_MIN", "1"))
|
||||
DB_POOL_MAX = int(_env("DB_POOL_MAX", "10"))
|
||||
DB_COMMAND_TIMEOUT = float(_env("DB_COMMAND_TIMEOUT", "5"))
|
||||
DB_CONNECT_TIMEOUT = float(_env("DB_CONNECT_TIMEOUT", "5"))
|
||||
REDIS_SOCKET_TIMEOUT = float(_env("REDIS_SOCKET_TIMEOUT", "2"))
|
||||
IDEMPOTENCY_TTL_SECONDS = int(_env("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
MIGRATIONS_ON_START = _env("MIGRATIONS_ON_START", "1") == "1"
|
||||
LB_DRAIN_SECONDS = float(_env("LB_DRAIN_SECONDS", "2"))
|
||||
# Stable for the life of this process. Compose leaves it unset so each
|
||||
# container uses its own hostname; stop-one.sh maps that value to one replica.
|
||||
INSTANCE_ID = _env("INSTANCE_ID", socket.gethostname())
|
||||
|
||||
|
||||
def lab_mode() -> bool:
|
||||
return ENV == "lab"
|
||||
@@ -0,0 +1,112 @@
|
||||
import base64
|
||||
|
||||
import json
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from typing import Any
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def encode_cursor(chat_id: str, created_at: str, entity_id: str) -> str:
|
||||
|
||||
payload = json.dumps({"c": chat_id, "t": created_at, "i": entity_id}, separators=(",", ":"))
|
||||
|
||||
return base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def _validate_cursor_timestamp(value: str) -> None:
|
||||
|
||||
if not value.endswith("Z"):
|
||||
|
||||
raise ValueError("invalid cursor timestamp")
|
||||
|
||||
try:
|
||||
|
||||
datetime.fromisoformat(value[:-1] + "+00:00")
|
||||
|
||||
except ValueError as exc:
|
||||
|
||||
raise ValueError("invalid cursor timestamp") from exc
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def _require_str_field(data: dict, key: str) -> str:
|
||||
|
||||
if key not in data:
|
||||
|
||||
raise ValueError("invalid cursor")
|
||||
|
||||
value = data[key]
|
||||
|
||||
if not isinstance(value, str):
|
||||
|
||||
raise ValueError("invalid cursor")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def decode_cursor(cursor: str, expected_chat_id: str) -> tuple[str, str]:
|
||||
|
||||
pad = "=" * (-len(cursor) % 4)
|
||||
|
||||
try:
|
||||
|
||||
raw = base64.urlsafe_b64decode((cursor + pad).encode("ascii"))
|
||||
|
||||
data: Any = json.loads(raw.decode("utf-8"))
|
||||
|
||||
except (ValueError, json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
|
||||
raise ValueError("invalid cursor") from exc
|
||||
|
||||
if not isinstance(data, dict):
|
||||
|
||||
raise ValueError("invalid cursor")
|
||||
|
||||
try:
|
||||
|
||||
chat_id = _require_str_field(data, "c")
|
||||
|
||||
created_at = _require_str_field(data, "t")
|
||||
|
||||
entity_id = _require_str_field(data, "i")
|
||||
|
||||
except ValueError as exc:
|
||||
|
||||
raise ValueError("invalid cursor") from exc
|
||||
|
||||
if chat_id != expected_chat_id:
|
||||
|
||||
raise ValueError("cursor chat mismatch")
|
||||
|
||||
try:
|
||||
|
||||
UUID(entity_id)
|
||||
|
||||
except ValueError as exc:
|
||||
|
||||
raise ValueError("invalid cursor") from exc
|
||||
|
||||
try:
|
||||
|
||||
_validate_cursor_timestamp(created_at)
|
||||
|
||||
except ValueError as exc:
|
||||
|
||||
raise ValueError("invalid cursor") from exc
|
||||
|
||||
return created_at, entity_id
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app import config
|
||||
from app.errors import AppError
|
||||
|
||||
pool: asyncpg.Pool | None = None
|
||||
INTEGRITY = (
|
||||
asyncpg.UniqueViolationError,
|
||||
asyncpg.ForeignKeyViolationError,
|
||||
asyncpg.CheckViolationError,
|
||||
)
|
||||
|
||||
|
||||
def _migrations_dir() -> Path:
|
||||
here = Path(__file__).resolve().parent
|
||||
for candidate in (here.parent / "migrations", Path("/app/migrations")):
|
||||
if candidate.is_dir():
|
||||
return candidate
|
||||
raise RuntimeError("migrations directory not found")
|
||||
|
||||
|
||||
async def connect() -> asyncpg.Pool:
|
||||
global pool
|
||||
if not config.DATABASE_URL:
|
||||
raise RuntimeError("DATABASE_URL is not set")
|
||||
pool = await asyncpg.create_pool(
|
||||
dsn=config.DATABASE_URL,
|
||||
min_size=config.DB_POOL_MIN,
|
||||
max_size=config.DB_POOL_MAX,
|
||||
command_timeout=config.DB_COMMAND_TIMEOUT,
|
||||
timeout=config.DB_CONNECT_TIMEOUT,
|
||||
)
|
||||
return pool
|
||||
|
||||
|
||||
async def close() -> None:
|
||||
global pool
|
||||
if pool is not None:
|
||||
await pool.close()
|
||||
pool = None
|
||||
|
||||
|
||||
def require() -> asyncpg.Pool:
|
||||
if pool is None:
|
||||
raise AppError(503, "service unavailable")
|
||||
return pool
|
||||
|
||||
|
||||
async def ping() -> None:
|
||||
try:
|
||||
async with require().acquire() as conn:
|
||||
await conn.execute("SELECT 1")
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
|
||||
|
||||
_MIGRATION_LOCK = 87133103
|
||||
|
||||
|
||||
async def apply_migrations() -> None:
|
||||
"""Apply pending SQL files once. A session advisory lock stops replica races."""
|
||||
directory = _migrations_dir()
|
||||
async with require().acquire() as conn:
|
||||
await conn.execute("SELECT pg_advisory_lock($1)", _MIGRATION_LOCK)
|
||||
try:
|
||||
await conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
filename TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
)
|
||||
"""
|
||||
)
|
||||
applied = {row["filename"] for row in await conn.fetch("SELECT filename FROM schema_migrations")}
|
||||
for path in sorted(p for p in directory.glob("*.sql") if p.is_file()):
|
||||
if path.name in applied:
|
||||
continue
|
||||
sql = path.read_text(encoding="utf-8")
|
||||
async with conn.transaction():
|
||||
await conn.execute(sql)
|
||||
await conn.execute("INSERT INTO schema_migrations(filename) VALUES($1)", path.name)
|
||||
finally:
|
||||
await conn.execute("SELECT pg_advisory_unlock($1)", _MIGRATION_LOCK)
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
import json
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import HTMLResponse, JSONResponse, Response
|
||||
|
||||
_SPEC_CANDIDATES = (
|
||||
Path(__file__).resolve().parents[1] / "contracts" / "openapi.json",
|
||||
Path("/app/contracts/openapi.json"),
|
||||
)
|
||||
|
||||
_SWAGGER_HTML = """<!DOCTYPE html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<title>MTUSI Messenger — Swagger (lab2)</title>
|
||||
<link rel="stylesheet" href="https://unpkg.com/swagger-ui-dist@5.11.0/swagger-ui.css"/>
|
||||
</head>
|
||||
<body>
|
||||
<div id="swagger-ui"></div>
|
||||
<script src="https://unpkg.com/swagger-ui-dist@5.11.0/swagger-ui-bundle.js"></script>
|
||||
<script>
|
||||
window.onload = function () {
|
||||
SwaggerUIBundle({
|
||||
url: "/openapi.json",
|
||||
dom_id: "#swagger-ui",
|
||||
deepLinking: true,
|
||||
persistAuthorization: true,
|
||||
tryItOutEnabled: true
|
||||
});
|
||||
};
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_spec() -> dict:
|
||||
for path in _SPEC_CANDIDATES:
|
||||
if path.is_file():
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
return {"openapi": "3.1.0", "info": {"title": "MTUSI Messenger", "version": "1.2.0"}, "paths": {}}
|
||||
|
||||
|
||||
async def openapi_json(_: Request) -> Response:
|
||||
return JSONResponse(_load_spec())
|
||||
|
||||
|
||||
async def swagger_ui(_: Request) -> Response:
|
||||
return HTMLResponse(_SWAGGER_HTML)
|
||||
@@ -0,0 +1,49 @@
|
||||
from typing import Dict, Optional
|
||||
|
||||
from starlette.responses import JSONResponse
|
||||
|
||||
|
||||
ERROR_CODES = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
503: "unavailable",
|
||||
500: "internal_error",
|
||||
}
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
class AppError(Exception):
|
||||
def __init__(self, status: int, message: str, www_authenticate: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.message = message
|
||||
self.www_authenticate = www_authenticate
|
||||
|
||||
|
||||
def error_response(
|
||||
status: int,
|
||||
message: str,
|
||||
request_id: str,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
) -> JSONResponse:
|
||||
code = ERROR_CODES.get(status, "internal_error")
|
||||
headers = {"X-Request-Id": request_id, "Content-Type": "application/json"}
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
headers=headers,
|
||||
content={"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
)
|
||||
|
||||
|
||||
def from_app_error(exc: AppError, request_id: str) -> JSONResponse:
|
||||
extra = {"WWW-Authenticate": exc.www_authenticate} if exc.www_authenticate else None
|
||||
return error_response(exc.status, exc.message, request_id, extra)
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import uvicorn
|
||||
from starlette.applications import Starlette
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
from starlette.routing import Route
|
||||
|
||||
from app import config, db, rediscli
|
||||
from app.docs import openapi_json, swagger_ui
|
||||
from app.errors import AppError, error_response, from_app_error
|
||||
from app.middleware import RequestContextMiddleware
|
||||
from app.routes import (
|
||||
create_chat,
|
||||
create_message,
|
||||
create_session,
|
||||
current_user,
|
||||
get_chat,
|
||||
health_live,
|
||||
health_ready,
|
||||
lab_flags,
|
||||
lab_metrics,
|
||||
list_chats,
|
||||
list_messages,
|
||||
register_user,
|
||||
revoke_session,
|
||||
)
|
||||
|
||||
|
||||
def _request_id(request: Request) -> str:
|
||||
existing = getattr(request.state, "request_id", None)
|
||||
if existing:
|
||||
return existing
|
||||
return request.headers.get("X-Request-Id") or str(uuid.uuid4())
|
||||
|
||||
|
||||
async def not_found_handler(request: Request, exc: HTTPException) -> Response:
|
||||
return error_response(404, "not found", _request_id(request))
|
||||
|
||||
|
||||
async def app_error_handler(request: Request, exc: AppError) -> Response:
|
||||
return from_app_error(exc, _request_id(request))
|
||||
|
||||
|
||||
def publish_instance_id() -> None:
|
||||
path = Path(config._env("INSTANCE_ID_FILE", "/tmp/instance-id"))
|
||||
try:
|
||||
path.write_text(config.INSTANCE_ID, encoding="utf-8")
|
||||
except OSError:
|
||||
logging.getLogger("messenger").warning("instance id file was not written")
|
||||
|
||||
|
||||
async def startup() -> None:
|
||||
publish_instance_id()
|
||||
logging.getLogger("messenger").info("instance_id=%s", config.INSTANCE_ID)
|
||||
await db.connect()
|
||||
if config.MIGRATIONS_ON_START:
|
||||
await db.apply_migrations()
|
||||
await rediscli.connect()
|
||||
|
||||
|
||||
async def shutdown() -> None:
|
||||
await rediscli.close()
|
||||
await db.close()
|
||||
|
||||
|
||||
def build_app() -> Starlette:
|
||||
app = Starlette(
|
||||
routes=[
|
||||
Route("/docs", swagger_ui, methods=["GET"]),
|
||||
Route("/openapi.json", openapi_json, methods=["GET"]),
|
||||
Route("/health/live", health_live, methods=["GET"]),
|
||||
Route("/health/ready", health_ready, methods=["GET"]),
|
||||
Route("/api/v1/users", register_user, methods=["POST"]),
|
||||
Route("/api/v1/users/me", current_user, methods=["GET"]),
|
||||
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
|
||||
Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]),
|
||||
Route("/api/v1/chats", create_chat, methods=["POST"]),
|
||||
Route("/api/v1/chats", list_chats, methods=["GET"]),
|
||||
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
|
||||
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
|
||||
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
|
||||
Route("/api/v1/_lab/metrics", lab_metrics, methods=["GET"]),
|
||||
Route("/api/v1/_lab/flags", lab_flags, methods=["POST"]),
|
||||
],
|
||||
exception_handlers={404: not_found_handler, AppError: app_error_handler},
|
||||
on_startup=[startup],
|
||||
on_shutdown=[shutdown],
|
||||
)
|
||||
app.state.shutting_down = False
|
||||
app.add_middleware(RequestContextMiddleware)
|
||||
return app
|
||||
|
||||
|
||||
def configure_logging() -> None:
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO),
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
configure_logging()
|
||||
app = build_app()
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
server = uvicorn.Server(
|
||||
uvicorn.Config(
|
||||
app,
|
||||
host=config.HOST,
|
||||
port=config.PORT,
|
||||
log_level=config.LOG_LEVEL,
|
||||
access_log=False,
|
||||
timeout_graceful_shutdown=int(config.SHUTDOWN_GRACE_SECONDS),
|
||||
)
|
||||
)
|
||||
|
||||
def _begin_shutdown() -> None:
|
||||
if app.state.shutting_down:
|
||||
return
|
||||
logging.getLogger("messenger").info(
|
||||
"shutdown signal received; readiness fails now, exit in %.1fs",
|
||||
config.LB_DRAIN_SECONDS,
|
||||
)
|
||||
app.state.shutting_down = True
|
||||
loop.call_later(config.LB_DRAIN_SECONDS, lambda: setattr(server, "should_exit", True))
|
||||
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
try:
|
||||
loop.add_signal_handler(sig, _begin_shutdown)
|
||||
except NotImplementedError:
|
||||
signal.signal(sig, lambda *_: _begin_shutdown())
|
||||
|
||||
loop.run_until_complete(server.serve())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,48 @@
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from typing import Callable
|
||||
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
from app.config import INSTANCE_ID
|
||||
from app.errors import AppError, error_response, from_app_error
|
||||
|
||||
logger = logging.getLogger("messenger.access")
|
||||
|
||||
|
||||
def _stamp(response: Response, request_id: str) -> Response:
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
class RequestContextMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request: Request, call_next: Callable) -> Response:
|
||||
request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4())
|
||||
request.state.request_id = request_id
|
||||
|
||||
if request.app.state.shutting_down and request.url.path not in ("/health/live", "/health/ready"):
|
||||
return _stamp(error_response(503, "service is shutting down", request_id), request_id)
|
||||
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except AppError as exc:
|
||||
return _stamp(from_app_error(exc, request_id), request_id)
|
||||
except Exception:
|
||||
logger.exception("request_id=%s unhandled error path=%s", request_id, request.url.path)
|
||||
return _stamp(error_response(500, "internal error", request_id), request_id)
|
||||
elapsed_ms = (time.perf_counter() - started) * 1000
|
||||
_stamp(response, request_id)
|
||||
logger.info(
|
||||
"request_id=%s method=%s path=%s status=%s duration_ms=%.2f",
|
||||
request_id,
|
||||
request.method,
|
||||
request.url.path,
|
||||
response.status_code,
|
||||
elapsed_ms,
|
||||
)
|
||||
return response
|
||||
@@ -0,0 +1,29 @@
|
||||
"""One-shot schema apply. API replicas do not run this on startup."""
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from app import config, db
|
||||
|
||||
|
||||
async def _main() -> None:
|
||||
if not config.DATABASE_URL:
|
||||
raise SystemExit("DATABASE_URL is not set")
|
||||
await db.connect()
|
||||
try:
|
||||
await db.apply_migrations()
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO),
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
stream=sys.stdout,
|
||||
)
|
||||
asyncio.run(_main())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,23 @@
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerifyMismatchError
|
||||
|
||||
hasher = PasswordHasher(time_cost=2, memory_cost=8192, parallelism=1)
|
||||
_dummy = None
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return hasher.hash(password)
|
||||
|
||||
|
||||
def dummy_hash() -> str:
|
||||
global _dummy
|
||||
if _dummy is None:
|
||||
_dummy = hash_password("timing-dummy-password")
|
||||
return _dummy
|
||||
|
||||
|
||||
def verify_password(password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return hasher.verify(password_hash, password)
|
||||
except (VerifyMismatchError, InvalidHashError, TypeError, ValueError):
|
||||
return False
|
||||
@@ -0,0 +1,40 @@
|
||||
import redis.asyncio as redis
|
||||
|
||||
from app import config
|
||||
from app.errors import AppError
|
||||
|
||||
client: redis.Redis | None = None
|
||||
|
||||
|
||||
async def connect() -> redis.Redis:
|
||||
global client
|
||||
client = redis.from_url(
|
||||
config.REDIS_URL,
|
||||
socket_connect_timeout=config.REDIS_SOCKET_TIMEOUT,
|
||||
socket_timeout=config.REDIS_SOCKET_TIMEOUT,
|
||||
decode_responses=True,
|
||||
)
|
||||
await client.ping()
|
||||
return client
|
||||
|
||||
|
||||
async def close() -> None:
|
||||
global client
|
||||
if client is not None:
|
||||
await client.close()
|
||||
client = None
|
||||
|
||||
|
||||
def require() -> redis.Redis:
|
||||
if client is None:
|
||||
raise AppError(503, "service unavailable")
|
||||
return client
|
||||
|
||||
|
||||
async def ping() -> None:
|
||||
try:
|
||||
await require().ping()
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
+284
@@ -0,0 +1,284 @@
|
||||
import base64
|
||||
import binascii
|
||||
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse, Response
|
||||
|
||||
from app import cache, config, db, rediscli, sessions, store
|
||||
from app.cursor_codec import decode_cursor, encode_cursor
|
||||
from app.errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError, error_response
|
||||
from app.passwords import dummy_hash, hash_password, verify_password
|
||||
from app.validation import (
|
||||
parse_json_body,
|
||||
require_exact_keys,
|
||||
validate_display_name,
|
||||
validate_limit,
|
||||
validate_idempotency_key,
|
||||
validate_message_text,
|
||||
validate_password,
|
||||
validate_title,
|
||||
validate_username,
|
||||
validate_uuid,
|
||||
)
|
||||
|
||||
|
||||
def _json(data: dict, request_id: str, status: int = 200) -> JSONResponse:
|
||||
return JSONResponse(status_code=status, headers={"X-Request-Id": request_id}, content=data)
|
||||
|
||||
|
||||
def _empty(request_id: str) -> Response:
|
||||
return Response(status_code=204, headers={"X-Request-Id": request_id})
|
||||
|
||||
|
||||
async def _bearer_user(request: Request) -> dict:
|
||||
header = request.headers.get("Authorization") or ""
|
||||
scheme, _, rest = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not rest.strip():
|
||||
raise AppError(401, "authentication required", BEARER_CHALLENGE)
|
||||
user_id = str(await sessions.resolve(rest.strip()))
|
||||
user = await store.get_user(user_id)
|
||||
if user is None:
|
||||
raise AppError(401, "authentication required", BEARER_CHALLENGE)
|
||||
return user
|
||||
|
||||
|
||||
async def health_live(request: Request) -> Response:
|
||||
return _json({"status": "ok"}, request.state.request_id)
|
||||
|
||||
|
||||
async def health_ready(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
if request.app.state.shutting_down:
|
||||
return error_response(503, "service is shutting down", rid)
|
||||
await db.ping()
|
||||
await rediscli.ping()
|
||||
return _json({"status": "ok"}, rid)
|
||||
|
||||
|
||||
async def register_user(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
if request.headers.get("content-type", "").split(";")[0].strip().lower() not in ("application/json", ""):
|
||||
if request.headers.get("content-type"):
|
||||
return error_response(415, "unsupported media type", rid)
|
||||
raw = await request.body()
|
||||
body, err = parse_json_body(raw)
|
||||
if err:
|
||||
return error_response(400, err, rid)
|
||||
assert body is not None
|
||||
if require_exact_keys(body, {"username", "display_name", "password"}, {"username", "display_name", "password"}):
|
||||
return error_response(400, "invalid user payload", rid)
|
||||
if validate_username(body["username"]) or validate_display_name(body["display_name"]) or validate_password(body["password"]):
|
||||
return error_response(400, "invalid user fields", rid)
|
||||
user, conflict = await store.create_user(body["username"], body["display_name"], hash_password(body["password"]))
|
||||
if conflict:
|
||||
return error_response(409, "username already taken", rid)
|
||||
assert user is not None
|
||||
return _json(user, rid, 201)
|
||||
|
||||
|
||||
async def current_user(request: Request) -> Response:
|
||||
user = await _bearer_user(request)
|
||||
return _json(user, request.state.request_id)
|
||||
|
||||
|
||||
async def create_session(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
header = request.headers.get("Authorization") or ""
|
||||
scheme, _, rest = header.partition(" ")
|
||||
if scheme.lower() != "basic" or not rest.strip():
|
||||
return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
|
||||
try:
|
||||
decoded = base64.b64decode(rest.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, UnicodeError, ValueError):
|
||||
return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
|
||||
username, sep, password = decoded.partition(":")
|
||||
if not sep:
|
||||
return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
|
||||
row = await store.get_user_by_username(username)
|
||||
password_hash = row["password_hash"] if row else dummy_hash()
|
||||
if row is None or not verify_password(password_hash, password):
|
||||
return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE})
|
||||
token, expires_at = await sessions.create(row["id"])
|
||||
return _json(
|
||||
{
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": expires_at,
|
||||
"user": store.user_dict(row),
|
||||
},
|
||||
rid,
|
||||
201,
|
||||
)
|
||||
|
||||
|
||||
async def revoke_session(request: Request) -> Response:
|
||||
header = request.headers.get("Authorization") or ""
|
||||
scheme, _, rest = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not rest.strip():
|
||||
raise AppError(401, "authentication required", BEARER_CHALLENGE)
|
||||
await sessions.revoke(rest.strip())
|
||||
return _empty(request.state.request_id)
|
||||
|
||||
|
||||
async def create_chat(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if ct and ct != "application/json":
|
||||
return error_response(415, "unsupported media type", rid)
|
||||
raw = await request.body()
|
||||
body, err = parse_json_body(raw)
|
||||
if err:
|
||||
return error_response(400, err, rid)
|
||||
assert body is not None
|
||||
if require_exact_keys(body, {"title", "member_ids"}, {"title", "member_ids"}):
|
||||
return error_response(400, "invalid chat payload", rid)
|
||||
if validate_title(body["title"]):
|
||||
return error_response(400, "invalid title", rid)
|
||||
members = body["member_ids"]
|
||||
if not isinstance(members, list) or not members:
|
||||
return error_response(400, "invalid member_ids", rid)
|
||||
if len(members) > 100:
|
||||
return error_response(400, "too many members", rid)
|
||||
for mid in members:
|
||||
if validate_uuid(mid):
|
||||
return error_response(400, "invalid member id", rid)
|
||||
chat, create_err = await store.create_chat(body["title"], members, user["id"])
|
||||
if create_err == "caller not in members":
|
||||
return error_response(400, "caller must be a member", rid)
|
||||
if create_err in ("duplicate members", "unknown member"):
|
||||
return error_response(400, "invalid members", rid)
|
||||
assert chat is not None
|
||||
return _json(chat, rid, 201)
|
||||
|
||||
|
||||
async def list_chats(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
limit, limit_err = validate_limit(request.query_params.get("limit"))
|
||||
if limit_err:
|
||||
return error_response(400, limit_err, rid)
|
||||
assert limit is not None
|
||||
cursor_raw = request.query_params.get("cursor")
|
||||
after = None
|
||||
if cursor_raw is not None:
|
||||
if not cursor_raw.strip():
|
||||
return error_response(400, "invalid cursor", rid)
|
||||
try:
|
||||
created_at, entity_id = decode_cursor(cursor_raw, user["id"])
|
||||
except ValueError:
|
||||
return error_response(400, "invalid cursor", rid)
|
||||
after = (created_at, entity_id)
|
||||
items, next_key = await store.list_chats(user["id"], limit, after)
|
||||
next_cursor = None
|
||||
if next_key is not None:
|
||||
next_cursor = encode_cursor(user["id"], next_key[0], next_key[1])
|
||||
return _json({"items": items, "next_cursor": next_cursor}, rid)
|
||||
|
||||
|
||||
async def get_chat(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
chat_id = request.path_params["chat_id"]
|
||||
if validate_uuid(chat_id):
|
||||
return error_response(400, "invalid chat id", rid)
|
||||
chat = await store.get_chat(chat_id, user["id"])
|
||||
if chat is None:
|
||||
return error_response(404, "chat not found", rid)
|
||||
return _json(chat, rid)
|
||||
|
||||
|
||||
async def create_message(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
chat_id = request.path_params["chat_id"]
|
||||
if validate_uuid(chat_id):
|
||||
return error_response(400, "invalid chat id", rid)
|
||||
ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if ct and ct != "application/json":
|
||||
return error_response(415, "unsupported media type", rid)
|
||||
raw = await request.body()
|
||||
body, err = parse_json_body(raw)
|
||||
if err:
|
||||
return error_response(400, err, rid)
|
||||
assert body is not None
|
||||
if require_exact_keys(body, {"text"}, {"text"}):
|
||||
return error_response(400, "invalid message payload", rid)
|
||||
if validate_message_text(body["text"]):
|
||||
return error_response(400, "invalid message text", rid)
|
||||
if validate_idempotency_key(request.headers.get("Idempotency-Key")):
|
||||
return error_response(400, "invalid idempotency key", rid)
|
||||
message, msg_err = await store.add_message(
|
||||
chat_id,
|
||||
user["id"],
|
||||
body["text"],
|
||||
request.headers.get("Idempotency-Key") or "",
|
||||
request.url.path,
|
||||
)
|
||||
if msg_err == "not_found":
|
||||
return error_response(404, "chat not found", rid)
|
||||
if msg_err == "conflict":
|
||||
return error_response(409, "idempotency key reused with a different body", rid)
|
||||
assert message is not None
|
||||
return _json(message, rid, 201)
|
||||
|
||||
|
||||
async def list_messages(request: Request) -> Response:
|
||||
rid = request.state.request_id
|
||||
user = await _bearer_user(request)
|
||||
chat_id = request.path_params["chat_id"]
|
||||
if validate_uuid(chat_id):
|
||||
return error_response(400, "invalid chat id", rid)
|
||||
limit, limit_err = validate_limit(request.query_params.get("limit"))
|
||||
if limit_err:
|
||||
return error_response(400, limit_err, rid)
|
||||
assert limit is not None
|
||||
cursor_raw = request.query_params.get("cursor")
|
||||
after = None
|
||||
if cursor_raw is not None:
|
||||
if not cursor_raw.strip():
|
||||
return error_response(400, "invalid cursor", rid)
|
||||
try:
|
||||
created_at, entity_id = decode_cursor(cursor_raw, chat_id)
|
||||
except ValueError:
|
||||
return error_response(400, "invalid cursor", rid)
|
||||
after = (created_at, entity_id)
|
||||
page, next_key, list_err = await store.list_messages(chat_id, user["id"], limit, after)
|
||||
if list_err == "not_found":
|
||||
return error_response(404, "chat not found", rid)
|
||||
assert page is not None
|
||||
next_cursor = None
|
||||
if next_key is not None:
|
||||
next_cursor = encode_cursor(chat_id, next_key[0], next_key[1])
|
||||
return _json({"items": page, "next_cursor": next_cursor}, rid)
|
||||
|
||||
|
||||
async def lab_metrics(request: Request) -> Response:
|
||||
if not config.lab_mode():
|
||||
return error_response(404, "not found", request.state.request_id)
|
||||
return _json(
|
||||
{
|
||||
"singleflight": await cache.singleflight_enabled(),
|
||||
"invalidate": await cache.invalidate_enabled(),
|
||||
"history_sql": await cache.history_sql_count(),
|
||||
},
|
||||
request.state.request_id,
|
||||
)
|
||||
|
||||
|
||||
async def lab_flags(request: Request) -> Response:
|
||||
if not config.lab_mode():
|
||||
return error_response(404, "not found", request.state.request_id)
|
||||
raw = await request.body()
|
||||
body, err = parse_json_body(raw)
|
||||
if err:
|
||||
return error_response(400, err, request.state.request_id)
|
||||
assert body is not None
|
||||
if body.get("reset_sql"):
|
||||
await cache.reset_history_sql_count()
|
||||
flags = await cache.set_flags(
|
||||
singleflight=body.get("singleflight"),
|
||||
invalidate=body.get("invalidate"),
|
||||
delay_ms=body.get("delay_ms"),
|
||||
)
|
||||
return _json(flags, request.state.request_id)
|
||||
@@ -0,0 +1,56 @@
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
|
||||
from app import config, rediscli
|
||||
from app.errors import AppError, BEARER_CHALLENGE
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def _key(token: str) -> str:
|
||||
return "sess:" + hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _fmt(dt: datetime) -> str:
|
||||
return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
async def create(user_id: UUID) -> tuple[str, str]:
|
||||
token = new_token()
|
||||
expires = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
|
||||
payload = json.dumps({"user_id": str(user_id), "expires_at": _fmt(expires)})
|
||||
try:
|
||||
await rediscli.require().set(_key(token), payload, ex=config.SESSION_TTL_SECONDS)
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
return token, _fmt(expires)
|
||||
|
||||
|
||||
async def resolve(token: str) -> UUID:
|
||||
try:
|
||||
raw = await rediscli.require().get(_key(token))
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
if not raw:
|
||||
raise AppError(401, "authentication required", BEARER_CHALLENGE)
|
||||
return UUID(json.loads(raw)["user_id"])
|
||||
|
||||
|
||||
async def revoke(token: str) -> None:
|
||||
try:
|
||||
deleted = await rediscli.require().delete(_key(token))
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
if not deleted:
|
||||
raise AppError(401, "authentication required", BEARER_CHALLENGE)
|
||||
+361
@@ -0,0 +1,361 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import List, Optional, Tuple
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app import cache, config, db
|
||||
from app.errors import AppError
|
||||
|
||||
|
||||
def format_ts(dt: datetime) -> str:
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def user_dict(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": format_ts(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def chat_dict(row, member_ids: List[str]) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": row["title"],
|
||||
"member_ids": member_ids,
|
||||
"created_at": format_ts(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def message_dict(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"created_at": format_ts(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
async def _run(coro):
|
||||
try:
|
||||
return await coro
|
||||
except AppError:
|
||||
raise
|
||||
except db.INTEGRITY:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
|
||||
|
||||
async def create_user(username: str, display_name: str, password_hash: str) -> Tuple[Optional[dict], Optional[str]]:
|
||||
user_id = uuid4()
|
||||
created = datetime.now(timezone.utc)
|
||||
try:
|
||||
async with db.require().acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
""",
|
||||
user_id,
|
||||
username,
|
||||
display_name,
|
||||
password_hash,
|
||||
created,
|
||||
)
|
||||
except asyncpg.UniqueViolationError:
|
||||
return None, "conflict"
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
return user_dict({"id": user_id, "username": username, "display_name": display_name, "created_at": created}), None
|
||||
|
||||
|
||||
async def get_user(user_id: str) -> Optional[dict]:
|
||||
row = await _run(
|
||||
_fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", UUID(user_id))
|
||||
)
|
||||
return user_dict(row) if row else None
|
||||
|
||||
|
||||
async def get_user_by_username(username: str):
|
||||
return await _run(
|
||||
_fetchrow(
|
||||
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1",
|
||||
username,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _fetchrow(sql: str, *args):
|
||||
async with db.require().acquire() as conn:
|
||||
return await conn.fetchrow(sql, *args)
|
||||
|
||||
|
||||
async def _fetch(sql: str, *args):
|
||||
async with db.require().acquire() as conn:
|
||||
return await conn.fetch(sql, *args)
|
||||
|
||||
|
||||
async def member_ids_of(chat_id: UUID) -> List[str]:
|
||||
rows = await _run(_fetch("SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id", chat_id))
|
||||
return [str(r["user_id"]) for r in rows]
|
||||
|
||||
|
||||
async def create_chat(title: str, member_ids: List[str], caller_id: str) -> Tuple[Optional[dict], Optional[str]]:
|
||||
if caller_id not in member_ids:
|
||||
return None, "caller not in members"
|
||||
if len(set(member_ids)) != len(member_ids):
|
||||
return None, "duplicate members"
|
||||
ids = [UUID(x) for x in member_ids]
|
||||
chat_id = uuid4()
|
||||
created = datetime.now(timezone.utc)
|
||||
try:
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
existing = await conn.fetch("SELECT id FROM users WHERE id = ANY($1::uuid[])", ids)
|
||||
if len(existing) != len(ids):
|
||||
return None, "unknown member"
|
||||
await conn.execute(
|
||||
"INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)",
|
||||
chat_id,
|
||||
title,
|
||||
created,
|
||||
)
|
||||
await conn.executemany(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)",
|
||||
[(chat_id, mid) for mid in ids],
|
||||
)
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
return chat_dict({"id": chat_id, "title": title, "created_at": created}, member_ids), None
|
||||
|
||||
|
||||
async def get_chat(chat_id: str, user_id: str) -> Optional[dict]:
|
||||
row = await load_chat_for_member(chat_id, user_id)
|
||||
if row is None:
|
||||
return None
|
||||
return chat_dict(row, await member_ids_of(UUID(chat_id)))
|
||||
|
||||
|
||||
async def load_chat_for_member(chat_id: str, user_id: str):
|
||||
return await _run(
|
||||
_fetchrow(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2
|
||||
WHERE c.id = $1
|
||||
""",
|
||||
UUID(chat_id),
|
||||
UUID(user_id),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def list_chats(user_id: str, limit: int, after: Optional[Tuple[str, str]] = None):
|
||||
args: list = [UUID(user_id)]
|
||||
clause = ""
|
||||
if after is not None:
|
||||
clause = "AND (c.created_at, c.id) > ($2, $3)"
|
||||
args.extend([datetime.fromisoformat(after[0].replace("Z", "+00:00")), UUID(after[1])])
|
||||
rows = await _run(
|
||||
_fetch(
|
||||
f"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = $1
|
||||
WHERE 1=1 {clause}
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT {int(limit) + 1}
|
||||
""",
|
||||
*args,
|
||||
)
|
||||
)
|
||||
page = rows[:limit]
|
||||
items = [chat_dict(row, await member_ids_of(row["id"])) for row in page]
|
||||
next_key = None
|
||||
if len(rows) > limit:
|
||||
last = page[-1]
|
||||
next_key = (format_ts(last["created_at"]), str(last["id"]))
|
||||
return items, next_key
|
||||
|
||||
|
||||
def _canonical_message(text: str) -> str:
|
||||
# Missing attachment_ids and [] are the same message. Key order does not matter.
|
||||
return json.dumps(
|
||||
{"attachment_ids": [], "text": text},
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
|
||||
async def add_message(
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
idem_key: str,
|
||||
path: str,
|
||||
) -> Tuple[Optional[dict], Optional[str]]:
|
||||
"""Insert one message or replay the stored 201. The row and the result commit together.
|
||||
|
||||
Same user+method+path+key is serialized with a Postgres advisory lock so two
|
||||
replicas cannot insert two messages. A failed transaction stores nothing.
|
||||
"""
|
||||
digest = hashlib.sha256(_canonical_message(text).encode("utf-8")).hexdigest()
|
||||
lock_key = f"{sender_id}|POST|{path}|{idem_key}"
|
||||
payload: Optional[dict] = None
|
||||
created_new = False
|
||||
try:
|
||||
async with db.require().acquire() as conn:
|
||||
async with conn.transaction():
|
||||
await conn.execute("SELECT pg_advisory_xact_lock(hashtextextended($1::text, 0))", lock_key)
|
||||
member = await conn.fetchrow(
|
||||
"""
|
||||
SELECT c.id
|
||||
FROM chats c
|
||||
JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2
|
||||
WHERE c.id = $1
|
||||
""",
|
||||
UUID(chat_id),
|
||||
UUID(sender_id),
|
||||
)
|
||||
if member is None:
|
||||
return None, "not_found"
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT request_hash, response_json, created_at
|
||||
FROM idempotency_keys
|
||||
WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3
|
||||
""",
|
||||
UUID(sender_id),
|
||||
path,
|
||||
idem_key,
|
||||
)
|
||||
if row is not None:
|
||||
created_at = row["created_at"]
|
||||
if created_at.tzinfo is None:
|
||||
created_at = created_at.replace(tzinfo=timezone.utc)
|
||||
age = (datetime.now(timezone.utc) - created_at).total_seconds()
|
||||
if age <= config.IDEMPOTENCY_TTL_SECONDS:
|
||||
if row["request_hash"] != digest:
|
||||
return None, "conflict"
|
||||
return json.loads(row["response_json"]), None
|
||||
await conn.execute(
|
||||
"""
|
||||
DELETE FROM idempotency_keys
|
||||
WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3
|
||||
""",
|
||||
UUID(sender_id),
|
||||
path,
|
||||
idem_key,
|
||||
)
|
||||
message_id = uuid4()
|
||||
created = datetime.now(timezone.utc)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
""",
|
||||
message_id,
|
||||
UUID(chat_id),
|
||||
UUID(sender_id),
|
||||
text,
|
||||
created,
|
||||
)
|
||||
payload = message_dict(
|
||||
{
|
||||
"id": message_id,
|
||||
"chat_id": UUID(chat_id),
|
||||
"sender_id": UUID(sender_id),
|
||||
"text": text,
|
||||
"created_at": created,
|
||||
}
|
||||
)
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO idempotency_keys
|
||||
(user_id, http_method, path, idem_key, request_hash, response_json, created_at)
|
||||
VALUES ($1, 'POST', $2, $3, $4, $5, $6)
|
||||
""",
|
||||
UUID(sender_id),
|
||||
path,
|
||||
idem_key,
|
||||
digest,
|
||||
json.dumps(payload, ensure_ascii=False, separators=(",", ":")),
|
||||
created,
|
||||
)
|
||||
created_new = True
|
||||
except AppError:
|
||||
raise
|
||||
except Exception:
|
||||
raise AppError(503, "service unavailable") from None
|
||||
if created_new:
|
||||
await cache.invalidate_history(chat_id)
|
||||
return payload, None
|
||||
|
||||
|
||||
async def list_messages(chat_id: str, user_id: str, limit: int, after: Optional[Tuple[str, str]] = None):
|
||||
if await load_chat_for_member(chat_id, user_id) is None:
|
||||
return None, None, "not_found"
|
||||
cursor = ""
|
||||
after_ts = after_id = None
|
||||
if after is not None:
|
||||
cursor = after[0] + ":" + after[1]
|
||||
after_ts = datetime.fromisoformat(after[0].replace("Z", "+00:00"))
|
||||
after_id = UUID(after[1])
|
||||
|
||||
async def loader() -> dict:
|
||||
await cache.incr_history_sql()
|
||||
if after_ts is None:
|
||||
rows = await _run(
|
||||
_fetch(
|
||||
"""
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
ORDER BY created_at, id
|
||||
LIMIT $2
|
||||
""",
|
||||
UUID(chat_id),
|
||||
limit + 1,
|
||||
)
|
||||
)
|
||||
else:
|
||||
rows = await _run(
|
||||
_fetch(
|
||||
"""
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
|
||||
ORDER BY created_at, id
|
||||
LIMIT $4
|
||||
""",
|
||||
UUID(chat_id),
|
||||
after_ts,
|
||||
after_id,
|
||||
limit + 1,
|
||||
)
|
||||
)
|
||||
page = rows[:limit]
|
||||
items = [message_dict(r) for r in page]
|
||||
next_key = None
|
||||
if len(rows) > limit:
|
||||
last = page[-1]
|
||||
next_key = (format_ts(last["created_at"]), str(last["id"]))
|
||||
return {"items": items, "next_key": next_key}
|
||||
|
||||
page = await cache.get_history_page(chat_id, cursor, limit, loader)
|
||||
return page["items"], page.get("next_key"), None
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Seed ≥10 000 messages and print EXPLAIN (ANALYZE, BUFFERS) for keyset pagination."""
|
||||
import asyncio
|
||||
import os
|
||||
from uuid import uuid4
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app.passwords import hash_password
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
conn = await asyncpg.connect(dsn=os.environ["DATABASE_URL"], timeout=10, command_timeout=120)
|
||||
try:
|
||||
alice = uuid4()
|
||||
bob = uuid4()
|
||||
chat = uuid4()
|
||||
pwd = hash_password("Example-only-not-a-real-secret-123")
|
||||
await conn.execute(
|
||||
"INSERT INTO users (id, username, display_name, password_hash) VALUES ($1,$2,$3,$4),($5,$6,$7,$4)",
|
||||
alice,
|
||||
"explain_a_" + alice.hex[:8],
|
||||
"Alice",
|
||||
pwd,
|
||||
bob,
|
||||
"explain_b_" + bob.hex[:8],
|
||||
"Bob",
|
||||
)
|
||||
await conn.execute("INSERT INTO chats (id, title) VALUES ($1,$2)", chat, "Explain chat")
|
||||
await conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2),($1,$3)",
|
||||
chat,
|
||||
alice,
|
||||
bob,
|
||||
)
|
||||
n = int(os.environ.get("EXPLAIN_MESSAGES", "10000"))
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), $1, $2, 'seed-' || g, now() - (g || ' milliseconds')::interval
|
||||
FROM generate_series(1, $3) AS g
|
||||
""",
|
||||
chat,
|
||||
alice,
|
||||
n,
|
||||
)
|
||||
count = await conn.fetchval("SELECT count(*) FROM messages WHERE chat_id = $1", chat)
|
||||
print("seeded_messages", count, "chat_id", chat)
|
||||
first = await conn.fetchrow(
|
||||
"SELECT created_at, id FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT 1",
|
||||
chat,
|
||||
)
|
||||
query = """
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
|
||||
ORDER BY created_at, id
|
||||
LIMIT 50
|
||||
"""
|
||||
print("\n=== WITH index messages_chat_created_id_idx ===")
|
||||
plan = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"])
|
||||
print("\n".join(row[0] for row in plan))
|
||||
await conn.execute("SET enable_indexscan = off")
|
||||
await conn.execute("SET enable_bitmapscan = off")
|
||||
await conn.execute("SET enable_indexonlyscan = off")
|
||||
print("\n=== WITHOUT index scans (seq scan baseline) ===")
|
||||
plan2 = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"])
|
||||
print("\n".join(row[0] for row in plan2))
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,102 @@
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from uuid import UUID
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
|
||||
|
||||
def parse_json_body(raw: bytes) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
if not raw:
|
||||
return None, "empty body"
|
||||
try:
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return None, "malformed JSON"
|
||||
if not isinstance(data, dict):
|
||||
return None, "JSON body must be an object"
|
||||
return data, None
|
||||
|
||||
|
||||
def require_exact_keys(data: Dict[str, Any], allowed: set, required: set) -> Optional[str]:
|
||||
keys = set(data.keys())
|
||||
if keys - allowed:
|
||||
return "unexpected fields"
|
||||
if required - keys:
|
||||
return "missing required fields"
|
||||
return None
|
||||
|
||||
|
||||
def validate_username(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str) or not USERNAME_RE.fullmatch(value):
|
||||
return "invalid username"
|
||||
return None
|
||||
|
||||
|
||||
def validate_display_name(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return "invalid display_name"
|
||||
if not value.strip():
|
||||
return "display_name must not be blank"
|
||||
if len(value) > 100:
|
||||
return "display_name too long"
|
||||
return None
|
||||
|
||||
|
||||
def validate_password(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return "invalid password"
|
||||
if not value.strip():
|
||||
return "password must not be blank"
|
||||
if len(value) < 12 or len(value) > 128:
|
||||
return "password length"
|
||||
return None
|
||||
|
||||
|
||||
def validate_title(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return "invalid title"
|
||||
if not value.strip():
|
||||
return "title must not be blank"
|
||||
if len(value) > 100:
|
||||
return "title too long"
|
||||
return None
|
||||
|
||||
|
||||
def validate_uuid(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return "invalid uuid"
|
||||
try:
|
||||
UUID(value)
|
||||
except ValueError:
|
||||
return "invalid uuid"
|
||||
return None
|
||||
|
||||
|
||||
def validate_idempotency_key(value: Optional[str]) -> Optional[str]:
|
||||
if value is None or IDEMPOTENCY_RE.fullmatch(value) is None:
|
||||
return "invalid idempotency key"
|
||||
return None
|
||||
|
||||
|
||||
def validate_message_text(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return "invalid text"
|
||||
if not value.strip():
|
||||
return "text must not be blank"
|
||||
if len(value) > 4000:
|
||||
return "text too long"
|
||||
return None
|
||||
|
||||
|
||||
def validate_limit(value: Optional[str], default: int = 50) -> Tuple[Optional[int], Optional[str]]:
|
||||
if value is None:
|
||||
return default, None
|
||||
try:
|
||||
limit = int(value)
|
||||
except ValueError:
|
||||
return None, "invalid limit"
|
||||
if limit < 1 or limit > 100:
|
||||
return None, "limit out of range"
|
||||
return limit, None
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
# Client -> lb:8080 -> api (scale) -> shared Postgres and Redis.
|
||||
# API replicas do not publish host ports. Migrations run once in `migrate`.
|
||||
# Check period: HAProxy inter 1s, fall 2, rise 1. Worst detection of a dead
|
||||
# or not-ready replica is about 2s, plus the 1s DNS hold. API on SIGTERM
|
||||
# fails /health/ready immediately and exits after LB_DRAIN_SECONDS (default 2).
|
||||
|
||||
services:
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: mtusi-messenger:lab3
|
||||
command: ["python", "-m", "app.migrate"]
|
||||
environment:
|
||||
APP_ENV: lab
|
||||
LOG_LEVEL: info
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
|
||||
DB_POOL_MIN: "1"
|
||||
DB_POOL_MAX: "2"
|
||||
MIGRATIONS_ON_START: "1"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
mem_limit: 256m
|
||||
cpus: "0.50"
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: mtusi-messenger:lab3
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: lab
|
||||
LOG_LEVEL: info
|
||||
SHUTDOWN_GRACE_SECONDS: "3"
|
||||
LB_DRAIN_SECONDS: ${LB_DRAIN_SECONDS:-2}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-60}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
SESSION_STORE: redis
|
||||
DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
DB_POOL_MIN: "1"
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-4}
|
||||
DB_COMMAND_TIMEOUT: "5"
|
||||
DB_CONNECT_TIMEOUT: "5"
|
||||
MIGRATIONS_ON_START: "0"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
expose:
|
||||
- "8080"
|
||||
restart: unless-stopped
|
||||
mem_limit: ${API_MEM:-256m}
|
||||
cpus: ${API_CPUS:-0.50}
|
||||
stop_grace_period: 8s
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"python",
|
||||
"-c",
|
||||
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)",
|
||||
]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 6
|
||||
start_period: 5s
|
||||
|
||||
lb:
|
||||
image: haproxy:2.8.14-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
mem_limit: 128m
|
||||
cpus: "0.25"
|
||||
|
||||
postgres:
|
||||
image: postgres:16.6-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_DB: messenger
|
||||
volumes:
|
||||
- pg_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
mem_limit: ${POSTGRES_MEM:-512m}
|
||||
cpus: ${POSTGRES_CPUS:-1.0}
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command:
|
||||
- redis-server
|
||||
- --appendonly
|
||||
- "yes"
|
||||
- --appendfsync
|
||||
- everysec
|
||||
- --save
|
||||
- "60"
|
||||
- "1"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
mem_limit: 128m
|
||||
cpus: "0.50"
|
||||
|
||||
volumes:
|
||||
pg_data:
|
||||
redis_data:
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 50.0,
|
||||
"elapsed_seconds": 50.101,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 8361,
|
||||
"failed_requests": 18,
|
||||
"error_ratio": 0.002148,
|
||||
"successful_rps": 166.884,
|
||||
"successful_latency_ms": {
|
||||
"p50": 13.481,
|
||||
"p95": 57.817,
|
||||
"p99": 96.441
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
{
|
||||
"instances_before": [
|
||||
"03c3a0545628",
|
||||
"b555f8327eee"
|
||||
],
|
||||
"parallel_idempotency": {
|
||||
"workers": 16,
|
||||
"successes": 16,
|
||||
"errors": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"history_copies": 1
|
||||
},
|
||||
"failovers": [
|
||||
{
|
||||
"target": "03c3a0545628",
|
||||
"hook_exit": 0,
|
||||
"hook_seconds": 2.373,
|
||||
"hook_stderr": "",
|
||||
"failed_attempts": 2,
|
||||
"successes_after_hook_start": 130,
|
||||
"first_recovery_seconds": 0.034,
|
||||
"tail_instance_ids": [
|
||||
"b555f8327eee",
|
||||
"b555f8327eee",
|
||||
"b555f8327eee",
|
||||
"b555f8327eee",
|
||||
"b555f8327eee"
|
||||
],
|
||||
"target_still_in_tail": false
|
||||
},
|
||||
{
|
||||
"target": "b555f8327eee",
|
||||
"hook_exit": 0,
|
||||
"hook_seconds": 1.744,
|
||||
"hook_stderr": "",
|
||||
"failed_attempts": 1,
|
||||
"successes_after_hook_start": 132,
|
||||
"first_recovery_seconds": 0.038,
|
||||
"tail_instance_ids": [
|
||||
"03c3a0545628",
|
||||
"03c3a0545628",
|
||||
"03c3a0545628",
|
||||
"03c3a0545628",
|
||||
"03c3a0545628"
|
||||
],
|
||||
"target_still_in_tail": false
|
||||
}
|
||||
],
|
||||
"rolling": {
|
||||
"rolling_seconds": 5.929,
|
||||
"rolling_stdout": "replacing b555f8327eee\nreplacing 03c3a0545628\nrolling update finished; replicas=2\n",
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 50.0,
|
||||
"elapsed_seconds": 50.101,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 8361,
|
||||
"failed_requests": 18,
|
||||
"error_ratio": 0.002148,
|
||||
"successful_rps": 166.884,
|
||||
"successful_latency_ms": {
|
||||
"p50": 13.481,
|
||||
"p95": 57.817,
|
||||
"p99": 96.441
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
Date: 2026-11-07
|
||||
Host: 12th Gen Intel Core i5-12450H, Docker Desktop ncpu=12, MemTotal=8181665792
|
||||
Engine: Docker 29.5.3, Compose 5.1.4
|
||||
Stack: haproxy:2.8.14-alpine, mtusi-messenger:lab3 (python:3.12.8-slim-bookworm), postgres:16.6-alpine, redis:7.4.2-alpine
|
||||
|
||||
python -X utf8 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
python -X utf8 -m pytest student_tests -q -m "not infra and not expiry"
|
||||
15 passed, 3 deselected
|
||||
|
||||
Official persistence and failover were executed by Linux python3 in docker:27-cli
|
||||
with the host Docker socket and BASE_URL=http://host.docker.internal:8080,
|
||||
because Windows CreateProcess cannot run scripts/restart.sh (WinError 193).
|
||||
The hooks and the API containers are the host stack.
|
||||
|
||||
python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh
|
||||
{"observed_instances_before": 2, "successful_writes": 109, "failed_attempts": 1, "first_recovery_seconds": 2.465}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
After that failover, docker inspect showed mtusi2026lab3-api-1 Exited (0),
|
||||
hostname b555f8327eee. mtusi2026lab3-api-2 stayed up, hostname 03c3a0545628.
|
||||
Exit code 0 means the process left on SIGTERM before the stop timeout.
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.109,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 5197,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 343.964,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.732,
|
||||
"p95": 42.061,
|
||||
"p99": 81.001
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"tag": "A1",
|
||||
"replicas": 1,
|
||||
"api_cpus_each": "1.0",
|
||||
"api_mem_each": "512m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 8,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.109,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 5197,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 343.964,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.732,
|
||||
"p95": 42.061,
|
||||
"p99": 81.001
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB",
|
||||
"mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.206,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 2985,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 196.3,
|
||||
"successful_latency_ms": {
|
||||
"p50": 26.613,
|
||||
"p95": 89.893,
|
||||
"p99": 123.339
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"tag": "A2",
|
||||
"replicas": 2,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.206,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 2985,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 196.3,
|
||||
"successful_latency_ms": {
|
||||
"p50": 26.613,
|
||||
"p95": 89.893,
|
||||
"p99": 123.339
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB",
|
||||
"mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.224,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 1895,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 124.476,
|
||||
"successful_latency_ms": {
|
||||
"p50": 39.136,
|
||||
"p95": 185.654,
|
||||
"p99": 267.62
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"tag": "A4",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.25",
|
||||
"api_mem_each": "128m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 2,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.224,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 1895,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 124.476,
|
||||
"successful_latency_ms": {
|
||||
"p50": 39.136,
|
||||
"p95": 185.654,
|
||||
"p99": 267.62
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB",
|
||||
"mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB",
|
||||
"mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB",
|
||||
"mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB",
|
||||
"mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.139,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 1485,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 98.094,
|
||||
"successful_latency_ms": {
|
||||
"p50": 85.074,
|
||||
"p95": 167.209,
|
||||
"p99": 201.857
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"tag": "B1",
|
||||
"replicas": 1,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 0.5,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 4,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.139,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 1485,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 98.094,
|
||||
"successful_latency_ms": {
|
||||
"p50": 85.074,
|
||||
"p95": 167.209,
|
||||
"p99": 201.857
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "5"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.136,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3388,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 223.842,
|
||||
"successful_latency_ms": {
|
||||
"p50": 24.1,
|
||||
"p95": 81.009,
|
||||
"p99": 142.445
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"tag": "B2",
|
||||
"replicas": 2,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.136,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3388,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 223.842,
|
||||
"successful_latency_ms": {
|
||||
"p50": 24.1,
|
||||
"p95": 81.009,
|
||||
"p99": 142.445
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.181,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3592,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 236.616,
|
||||
"successful_latency_ms": {
|
||||
"p50": 22.559,
|
||||
"p95": 80.196,
|
||||
"p99": 131.013
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"tag": "B4",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 2.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 16,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.181,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3592,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 236.616,
|
||||
"successful_latency_ms": {
|
||||
"p50": 22.559,
|
||||
"p95": 80.196,
|
||||
"p99": 131.013
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB",
|
||||
"mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB",
|
||||
"mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "17"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.128,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3800,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 251.189,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.749,
|
||||
"p95": 77.161,
|
||||
"p99": 101.17
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"tag": "C4-pg2",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 2.0,
|
||||
"postgres_cpus": "2.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 16,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.128,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3800,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 251.189,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.749,
|
||||
"p95": 77.161,
|
||||
"p99": 101.17
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB",
|
||||
"mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB",
|
||||
"mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "17"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.156,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3622,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 238.98,
|
||||
"successful_latency_ms": {
|
||||
"p50": 22.721,
|
||||
"p95": 74.859,
|
||||
"p99": 98.063
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"tag": "D4-pool2",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 2.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 2,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.156,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3622,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 238.98,
|
||||
"successful_latency_ms": {
|
||||
"p50": 22.721,
|
||||
"p95": 74.859,
|
||||
"p99": 98.063
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB",
|
||||
"mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB",
|
||||
"mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="720" height="320">
|
||||
<rect width="100%" height="100%" fill="white"/>
|
||||
<text x="48" y="16" font-family="sans-serif" font-size="14">Successful RPS, closed-loop POST, concurrency 8, 15s</text>
|
||||
<rect x="63.4" y="32.0" width="50.4" height="250.0" fill="#2f6f4e"/>
|
||||
<text x="88.6" y="28.0" font-family="sans-serif" font-size="12" text-anchor="middle">344.0</text>
|
||||
<text x="88.6" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">A1</text>
|
||||
<rect x="144.7" y="139.3" width="50.4" height="142.7" fill="#2f6f4e"/>
|
||||
<text x="169.9" y="135.3" font-family="sans-serif" font-size="12" text-anchor="middle">196.3</text>
|
||||
<text x="169.9" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">A2</text>
|
||||
<rect x="225.9" y="191.5" width="50.4" height="90.5" fill="#2f6f4e"/>
|
||||
<text x="251.1" y="187.5" font-family="sans-serif" font-size="12" text-anchor="middle">124.5</text>
|
||||
<text x="251.1" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">A4</text>
|
||||
<rect x="307.2" y="210.7" width="50.4" height="71.3" fill="#2f6f4e"/>
|
||||
<text x="332.4" y="206.7" font-family="sans-serif" font-size="12" text-anchor="middle">98.1</text>
|
||||
<text x="332.4" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">B1</text>
|
||||
<rect x="388.4" y="119.3" width="50.4" height="162.7" fill="#2f6f4e"/>
|
||||
<text x="413.6" y="115.3" font-family="sans-serif" font-size="12" text-anchor="middle">223.8</text>
|
||||
<text x="413.6" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">B2</text>
|
||||
<rect x="469.7" y="110.0" width="50.4" height="172.0" fill="#2f6f4e"/>
|
||||
<text x="494.9" y="106.0" font-family="sans-serif" font-size="12" text-anchor="middle">236.6</text>
|
||||
<text x="494.9" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">B4</text>
|
||||
<rect x="550.9" y="99.4" width="50.4" height="182.6" fill="#2f6f4e"/>
|
||||
<text x="576.1" y="95.4" font-family="sans-serif" font-size="12" text-anchor="middle">251.2</text>
|
||||
<text x="576.1" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">C4-pg2</text>
|
||||
<rect x="632.2" y="108.3" width="50.4" height="173.7" fill="#2f6f4e"/>
|
||||
<text x="657.4" y="104.3" font-family="sans-serif" font-size="12" text-anchor="middle">239.0</text>
|
||||
<text x="657.4" y="302" font-family="sans-serif" font-size="11" text-anchor="middle">D4-pool2</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.4 KiB |
@@ -0,0 +1,320 @@
|
||||
[
|
||||
{
|
||||
"tag": "A1",
|
||||
"replicas": 1,
|
||||
"api_cpus_each": "1.0",
|
||||
"api_mem_each": "512m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 8,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.109,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 5197,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 343.964,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.732,
|
||||
"p95": 42.061,
|
||||
"p99": 81.001
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB",
|
||||
"mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "A2",
|
||||
"replicas": 2,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.206,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 2985,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 196.3,
|
||||
"successful_latency_ms": {
|
||||
"p50": 26.613,
|
||||
"p95": 89.893,
|
||||
"p99": 123.339
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB",
|
||||
"mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "A4",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.25",
|
||||
"api_mem_each": "128m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 2,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.224,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 1895,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 124.476,
|
||||
"successful_latency_ms": {
|
||||
"p50": 39.136,
|
||||
"p95": 185.654,
|
||||
"p99": 267.62
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB",
|
||||
"mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB",
|
||||
"mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB",
|
||||
"mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB",
|
||||
"mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "B1",
|
||||
"replicas": 1,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 0.5,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 4,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.139,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 1485,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 98.094,
|
||||
"successful_latency_ms": {
|
||||
"p50": 85.074,
|
||||
"p95": 167.209,
|
||||
"p99": 201.857
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "5"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "B2",
|
||||
"replicas": 2,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 1.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.136,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3388,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 223.842,
|
||||
"successful_latency_ms": {
|
||||
"p50": 24.1,
|
||||
"p95": 81.009,
|
||||
"p99": 142.445
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "B4",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 2.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 16,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.181,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3592,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 236.616,
|
||||
"successful_latency_ms": {
|
||||
"p50": 22.559,
|
||||
"p95": 80.196,
|
||||
"p99": 131.013
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB",
|
||||
"mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB",
|
||||
"mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "17"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "C4-pg2",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 2.0,
|
||||
"postgres_cpus": "2.0",
|
||||
"db_pool_max_each": 4,
|
||||
"db_pool_max_total": 16,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.128,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3800,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 251.189,
|
||||
"successful_latency_ms": {
|
||||
"p50": 19.749,
|
||||
"p95": 77.161,
|
||||
"p99": 101.17
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB",
|
||||
"mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB",
|
||||
"mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "17"
|
||||
}
|
||||
},
|
||||
{
|
||||
"tag": "D4-pool2",
|
||||
"replicas": 4,
|
||||
"api_cpus_each": "0.50",
|
||||
"api_mem_each": "256m",
|
||||
"api_cpu_total": 2.0,
|
||||
"postgres_cpus": "1.0",
|
||||
"db_pool_max_each": 2,
|
||||
"db_pool_max_total": 8,
|
||||
"load": {
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 15.0,
|
||||
"elapsed_seconds": 15.156,
|
||||
"concurrency": 8,
|
||||
"successful_requests": 3622,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 238.98,
|
||||
"successful_latency_ms": {
|
||||
"p50": 22.721,
|
||||
"p95": 74.859,
|
||||
"p99": 98.063
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
},
|
||||
"resources": {
|
||||
"docker_stats": [
|
||||
"mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB",
|
||||
"mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB",
|
||||
"mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB",
|
||||
"mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB",
|
||||
"mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB",
|
||||
"mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB",
|
||||
"mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB",
|
||||
"mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB",
|
||||
"mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB"
|
||||
],
|
||||
"postgres_sessions": "9"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,35 @@
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 200
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
timeout connect 2s
|
||||
timeout client 10s
|
||||
timeout server 10s
|
||||
timeout http-request 5s
|
||||
timeout check 2s
|
||||
timeout queue 2s
|
||||
retries 1
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
accepted_payload_size 8192
|
||||
|
||||
frontend fe_http
|
||||
bind *:8080
|
||||
default_backend be_api
|
||||
|
||||
backend be_api
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr host api
|
||||
http-check expect status 200
|
||||
default-server inter 1s fall 2 rise 1
|
||||
server-template srv 8 api:8080 check resolvers docker init-addr none resolve-prefer ipv4
|
||||
@@ -0,0 +1,49 @@
|
||||
-- Lab2 schema (written before application queries).
|
||||
-- Source of truth: PostgreSQL for users, chats, membership, messages.
|
||||
-- Transaction boundary: INSERT chat + INSERT members in one transaction (rollback together).
|
||||
-- Sessions are stored in Redis with AOF, not in this database.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
filename TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY,
|
||||
username TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX users_username_key ON users (username);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_id_idx ON chat_members (user_id);
|
||||
CREATE INDEX chats_created_at_id_idx ON chats (created_at, id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users (id),
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000)
|
||||
);
|
||||
|
||||
-- Keyset history: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n
|
||||
CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id);
|
||||
@@ -0,0 +1,14 @@
|
||||
-- Shared idempotency for POST /messages. Scope: user + method + path + key.
|
||||
-- The message row and this result are committed in one transaction.
|
||||
-- A rolled-back request leaves no row, so a later retry with the same key can insert.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS idempotency_keys (
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
http_method TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
idem_key TEXT NOT NULL,
|
||||
request_hash TEXT NOT NULL,
|
||||
response_json TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, http_method, path, idem_key)
|
||||
);
|
||||
@@ -0,0 +1,6 @@
|
||||
[pytest]
|
||||
testpaths = student_tests
|
||||
pythonpath = .
|
||||
markers =
|
||||
infra: compose stop/restart of redis or postgres
|
||||
expiry: requires SESSION_TTL_SECONDS<=3
|
||||
@@ -0,0 +1,2 @@
|
||||
pytest==8.3.4
|
||||
httpx==0.28.1
|
||||
@@ -0,0 +1,5 @@
|
||||
starlette==0.41.3
|
||||
uvicorn[standard]==0.32.1
|
||||
asyncpg==0.30.0
|
||||
redis==5.2.1
|
||||
argon2-cffi==23.1.0
|
||||
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade 4: parallel idempotency, stop each API replica, rolling replacement under load."""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
OUT = ROOT / "evidence" / "grade4"
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
|
||||
|
||||
def compose(*args, check=True):
|
||||
cmd = ["docker", "compose"]
|
||||
if (ROOT / ".env").is_file():
|
||||
cmd.extend(["--env-file", str(ROOT / ".env")])
|
||||
cmd.extend(args)
|
||||
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
|
||||
if check and proc.returncode != 0:
|
||||
raise SystemExit("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or proc.stdout)[-800:]))
|
||||
return proc
|
||||
|
||||
|
||||
def bash_argv(script: Path):
|
||||
if os.name != "nt":
|
||||
return [str(script)]
|
||||
git_bash = Path(r"C:\Program Files\Git\bin\bash.exe")
|
||||
if git_bash.is_file():
|
||||
return [str(git_bash), str(script)]
|
||||
return ["bash", str(script)]
|
||||
|
||||
|
||||
def hook_cmd():
|
||||
return bash_argv(ROOT / "scripts" / "stop-one.sh")
|
||||
|
||||
|
||||
def wait_ready(seconds=60):
|
||||
deadline = time.monotonic() + seconds
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health")
|
||||
return
|
||||
except Failure as exc:
|
||||
last = exc
|
||||
time.sleep(0.4)
|
||||
raise SystemExit("API did not become ready: %s" % last)
|
||||
|
||||
|
||||
def instance_ids(actor, samples=24):
|
||||
found = []
|
||||
client = Client(BASE, timeout=3)
|
||||
for _ in range(samples):
|
||||
found.append(client.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"])
|
||||
return sorted(set(found))
|
||||
|
||||
|
||||
def parallel_idempotency():
|
||||
client = Client(BASE, timeout=20)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
text = "parallel-" + key
|
||||
ids = []
|
||||
errors = []
|
||||
barrier = threading.Barrier(16)
|
||||
|
||||
def one():
|
||||
local = Client(BASE, timeout=20)
|
||||
barrier.wait()
|
||||
try:
|
||||
ids.append(local.message(alice, chat, text, key=key)["json"]["id"])
|
||||
except Exception as exc:
|
||||
errors.append(type(exc).__name__)
|
||||
|
||||
threads = [threading.Thread(target=one) for _ in range(16)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
unique = sorted(set(ids))
|
||||
report = {
|
||||
"workers": 16,
|
||||
"successes": len(ids),
|
||||
"errors": len(errors),
|
||||
"distinct_message_ids": len(unique),
|
||||
}
|
||||
if len(unique) != 1 or errors:
|
||||
raise SystemExit("parallel idempotency failed: %s" % report)
|
||||
page = client.call(
|
||||
"GET",
|
||||
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
|
||||
actor=alice,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
copies = [item for item in page["items"] if item["text"] == text]
|
||||
report["history_copies"] = len(copies)
|
||||
if len(copies) != 1:
|
||||
raise SystemExit("history has %s copies" % len(copies))
|
||||
return report
|
||||
|
||||
|
||||
def failover_one(actor, chat, instance_id):
|
||||
key = "failover-" + uuid.uuid4().hex
|
||||
text = "during-" + key
|
||||
successes = []
|
||||
failures = 0
|
||||
started_box = {}
|
||||
|
||||
def traffic():
|
||||
nonlocal failures
|
||||
local = Client(BASE, timeout=2)
|
||||
while "t0" not in started_box:
|
||||
time.sleep(0.01)
|
||||
deadline = started_box["t0"] + 12
|
||||
recovered = False
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if not recovered:
|
||||
response = local.message(actor, chat, text, key=key)
|
||||
recovered = True
|
||||
else:
|
||||
response = local.message(actor, chat, "after-" + uuid.uuid4().hex)
|
||||
successes.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"]))
|
||||
except Failure:
|
||||
failures += 1
|
||||
time.sleep(0.05)
|
||||
|
||||
worker = threading.Thread(target=traffic)
|
||||
worker.start()
|
||||
time.sleep(0.2)
|
||||
started = time.monotonic()
|
||||
started_box["t0"] = started
|
||||
env = os.environ.copy()
|
||||
env["TARGET_INSTANCE_ID"] = instance_id
|
||||
proc = subprocess.run(hook_cmd(), cwd=ROOT, env=env, capture_output=True, text=True)
|
||||
hook_s = time.monotonic() - started
|
||||
worker.join()
|
||||
after = [item for item in successes if item[0] >= started]
|
||||
recovery = None
|
||||
if after:
|
||||
recovery = round(after[0][0] - started, 3)
|
||||
tail = [item[1] for item in after[-5:]]
|
||||
return {
|
||||
"target": instance_id,
|
||||
"hook_exit": proc.returncode,
|
||||
"hook_seconds": round(hook_s, 3),
|
||||
"hook_stderr": (proc.stderr or "")[-300:],
|
||||
"failed_attempts": failures,
|
||||
"successes_after_hook_start": len(after),
|
||||
"first_recovery_seconds": recovery,
|
||||
"tail_instance_ids": tail,
|
||||
"target_still_in_tail": instance_id in tail,
|
||||
}
|
||||
|
||||
|
||||
def restore_and_wait(actor):
|
||||
compose("start", "api")
|
||||
deadline = time.monotonic() + 40
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if len(instance_ids(actor, samples=16)) >= 2:
|
||||
return
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise SystemExit("second replica did not return")
|
||||
|
||||
|
||||
def rolling_under_load():
|
||||
out = OUT / "rolling-load.json"
|
||||
load = subprocess.Popen(
|
||||
[sys.executable, "-X", "utf8", str(ROOT / "tests" / "load.py"),
|
||||
"--base-url", BASE, "--duration", "50", "--concurrency", "4", "--output", str(out)],
|
||||
cwd=ROOT,
|
||||
)
|
||||
time.sleep(3)
|
||||
cmd = bash_argv(ROOT / "scripts" / "rolling-update.sh")
|
||||
started = time.monotonic()
|
||||
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
|
||||
elapsed = round(time.monotonic() - started, 3)
|
||||
load.wait(timeout=90)
|
||||
metrics = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {}
|
||||
if proc.returncode != 0:
|
||||
raise SystemExit("rolling update failed: %s" % (proc.stderr or proc.stdout)[-800:])
|
||||
return {
|
||||
"rolling_seconds": elapsed,
|
||||
"rolling_stdout": (proc.stdout or "")[-400:],
|
||||
"load": metrics,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
wait_ready()
|
||||
client = Client(BASE, timeout=10)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
before = instance_ids(alice)
|
||||
if len(before) < 2:
|
||||
raise SystemExit("need 2 instance ids before grade-4 runs, saw %s" % before)
|
||||
idem = parallel_idempotency()
|
||||
failovers = []
|
||||
for instance_id in before:
|
||||
item = failover_one(alice, chat, instance_id)
|
||||
failovers.append(item)
|
||||
recovery = item["first_recovery_seconds"]
|
||||
if item["hook_exit"] != 0 or item["target_still_in_tail"] or recovery is None or recovery > 10 or item["successes_after_hook_start"] < 5:
|
||||
raise SystemExit("failover criteria failed: %s" % item)
|
||||
restore_and_wait(alice)
|
||||
rolling = rolling_under_load()
|
||||
report = {
|
||||
"instances_before": before,
|
||||
"parallel_idempotency": idem,
|
||||
"failovers": failovers,
|
||||
"rolling": rolling,
|
||||
}
|
||||
(OUT / "summary.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
python "%~dp0restart.py"
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Restart API, PostgreSQL and Redis without removing named volumes."""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
env_file = ROOT / ".env"
|
||||
cmd = ["docker", "compose"]
|
||||
if env_file.is_file():
|
||||
cmd.extend(["--env-file", str(env_file)])
|
||||
cmd.extend(["restart", "postgres", "redis", "api"])
|
||||
return subprocess.run(cmd, cwd=ROOT).returncode
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Restart API replicas and every store they use. Volumes are kept.
|
||||
set -eu
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
if [ -f .env ]; then
|
||||
docker compose --env-file .env restart postgres redis api
|
||||
else
|
||||
docker compose restart postgres redis api
|
||||
fi
|
||||
Executable
+55
@@ -0,0 +1,55 @@
|
||||
#!/bin/sh
|
||||
# Replace API replicas one at a time. The other replica keeps serving.
|
||||
set -eu
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
|
||||
compose() {
|
||||
if [ -f .env ]; then
|
||||
docker compose --env-file .env "$@"
|
||||
else
|
||||
docker compose "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
PY=python3
|
||||
else
|
||||
PY=python
|
||||
fi
|
||||
|
||||
ids=$(compose ps -q api)
|
||||
count=0
|
||||
for id in $ids; do
|
||||
count=$((count + 1))
|
||||
done
|
||||
if [ "$count" -lt 2 ]; then
|
||||
echo "rolling update needs at least 2 running API replicas" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for id in $ids; do
|
||||
echo "replacing $(docker inspect -f '{{.Config.Hostname}}' "$id")"
|
||||
docker stop -t 8 "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
compose up -d --scale "api=$count" --no-deps --no-recreate api >/dev/null
|
||||
ready=0
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
running=0
|
||||
for cid in $(compose ps -q api); do
|
||||
running=$((running + 1))
|
||||
done
|
||||
if [ "$running" -ge "$count" ] && "$PY" -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" >/dev/null 2>&1; then
|
||||
ready=1
|
||||
break
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 1
|
||||
done
|
||||
if [ "$ready" -ne 1 ]; then
|
||||
echo "replacement did not become ready" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "rolling update finished; replicas=$count"
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Recreate API with SESSION_TTL_SECONDS=2, test expiry, restore TTL."""
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(args, extra_env=None):
|
||||
env = os.environ.copy()
|
||||
if extra_env:
|
||||
env.update(extra_env)
|
||||
cmd = ["docker", "compose"]
|
||||
if (ROOT / ".env").is_file():
|
||||
cmd.extend(["--env-file", str(ROOT / ".env")])
|
||||
cmd.extend(args)
|
||||
proc = subprocess.run(cmd, cwd=ROOT, env=env)
|
||||
if proc.returncode != 0:
|
||||
raise SystemExit(proc.returncode)
|
||||
|
||||
|
||||
def wait_ready():
|
||||
import urllib.request
|
||||
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2)
|
||||
return
|
||||
except Exception:
|
||||
time.sleep(0.5)
|
||||
raise SystemExit("API did not become ready")
|
||||
|
||||
|
||||
def main():
|
||||
compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "2"})
|
||||
wait_ready()
|
||||
env = os.environ.copy()
|
||||
env["SESSION_TTL_SECONDS"] = "2"
|
||||
env["RUN_EXPIRY"] = "1"
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-m", "pytest", "student_tests/test_expiry.py", "-q"],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
)
|
||||
compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "3600"})
|
||||
wait_ready()
|
||||
raise SystemExit(proc.returncode)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
# Stop the single API replica whose X-Instance-Id is TARGET_INSTANCE_ID.
|
||||
# Leaves it stopped. Does not touch the database, Redis, the load balancer, or volumes.
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
|
||||
compose() {
|
||||
if [ -f .env ]; then
|
||||
docker compose --env-file .env "$@"
|
||||
else
|
||||
docker compose "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
found=0
|
||||
for id in $(compose ps -q api); do
|
||||
iid=$(docker exec "$id" cat /tmp/instance-id 2>/dev/null | tr -d '\r\n' || true)
|
||||
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
|
||||
if [ "$iid" = "$TARGET_INSTANCE_ID" ] || [ "$host" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop -t 5 "$id" >/dev/null
|
||||
found=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$found" -ne 1 ]; then
|
||||
echo "no running API replica matched TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 1: fixed API budget, then more replicas with more CPU, then one DB change.
|
||||
|
||||
The load profile is the public closed-loop POST generator. Same duration and
|
||||
concurrency on every point. Raw JSON is written under evidence/track1/.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "evidence" / "track1"
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
DURATION = float(os.environ.get("TRACK_DURATION", "15"))
|
||||
WARMUP = float(os.environ.get("TRACK_WARMUP", "5"))
|
||||
CONCURRENCY = int(os.environ.get("TRACK_CONCURRENCY", "8"))
|
||||
|
||||
|
||||
def compose(*args):
|
||||
cmd = ["docker", "compose"]
|
||||
if (ROOT / ".env").is_file():
|
||||
cmd.extend(["--env-file", str(ROOT / ".env")])
|
||||
cmd.extend(args)
|
||||
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, env=os.environ.copy())
|
||||
if proc.returncode != 0:
|
||||
raise SystemExit((proc.stderr or proc.stdout)[-1000:])
|
||||
return proc
|
||||
|
||||
|
||||
def wait_ready(seconds=90):
|
||||
import urllib.request
|
||||
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
except Exception:
|
||||
time.sleep(0.5)
|
||||
raise SystemExit("ready timeout")
|
||||
|
||||
|
||||
def apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool):
|
||||
os.environ["API_CPUS"] = api_cpus
|
||||
os.environ["API_MEM"] = api_mem
|
||||
os.environ["POSTGRES_CPUS"] = postgres_cpus
|
||||
os.environ["DB_POOL_MAX"] = str(pool)
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "postgres")
|
||||
deadline = time.monotonic() + 40
|
||||
while time.monotonic() < deadline:
|
||||
probe = subprocess.run(
|
||||
["docker", "compose", "--env-file", str(ROOT / ".env"), "exec", "-T", "postgres", "pg_isready", "-U", "messenger", "-d", "messenger"],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if probe.returncode == 0:
|
||||
break
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
raise SystemExit("postgres did not become ready")
|
||||
compose(
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--force-recreate",
|
||||
"--scale",
|
||||
"api=%s" % replicas,
|
||||
"api",
|
||||
)
|
||||
wait_ready()
|
||||
running = [line for line in compose("ps", "-q", "api").stdout.splitlines() if line.strip()]
|
||||
if len(running) != replicas:
|
||||
raise SystemExit("expected %s api containers, saw %s" % (replicas, len(running)))
|
||||
|
||||
|
||||
def run_load(tag, warmup=False):
|
||||
args = [
|
||||
sys.executable,
|
||||
"-X",
|
||||
"utf8",
|
||||
str(ROOT / "tests" / "load.py"),
|
||||
"--base-url",
|
||||
BASE,
|
||||
"--duration",
|
||||
str(WARMUP if warmup else DURATION),
|
||||
"--concurrency",
|
||||
str(CONCURRENCY),
|
||||
]
|
||||
dest = None
|
||||
if not warmup:
|
||||
dest = OUT / ("%s.json" % tag)
|
||||
args.extend(["--output", str(dest)])
|
||||
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True)
|
||||
if proc.returncode != 0:
|
||||
raise SystemExit("load %s failed: %s" % (tag, (proc.stderr or proc.stdout)[-800:]))
|
||||
if dest is None:
|
||||
return None
|
||||
return json.loads(dest.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def sample_resources():
|
||||
stats = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
activity = compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"postgres",
|
||||
"psql",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger",
|
||||
"-tAc",
|
||||
"SELECT count(*) FROM pg_stat_activity WHERE datname='messenger'",
|
||||
)
|
||||
return {
|
||||
"docker_stats": [line for line in stats.stdout.splitlines() if line.strip()],
|
||||
"postgres_sessions": activity.stdout.strip(),
|
||||
}
|
||||
|
||||
|
||||
def measure(tag, replicas, api_cpus, api_mem, postgres_cpus, pool):
|
||||
print("profile %s" % tag, flush=True)
|
||||
apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool)
|
||||
run_load(tag, warmup=True)
|
||||
box = {}
|
||||
|
||||
def snapshot():
|
||||
time.sleep(DURATION / 2)
|
||||
box["resources"] = sample_resources()
|
||||
|
||||
thread = threading.Thread(target=snapshot)
|
||||
thread.start()
|
||||
metrics = run_load(tag)
|
||||
thread.join()
|
||||
point = {
|
||||
"tag": tag,
|
||||
"replicas": replicas,
|
||||
"api_cpus_each": api_cpus,
|
||||
"api_mem_each": api_mem,
|
||||
"api_cpu_total": round(replicas * float(api_cpus), 3),
|
||||
"postgres_cpus": postgres_cpus,
|
||||
"db_pool_max_each": pool,
|
||||
"db_pool_max_total": replicas * pool,
|
||||
"load": metrics,
|
||||
"resources": box.get("resources"),
|
||||
}
|
||||
(OUT / ("%s.point.json" % tag)).write_text(json.dumps(point, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"tag": tag, "rps": metrics["successful_rps"], "errors": metrics["failed_requests"], "p95": metrics["successful_latency_ms"]["p95"]}), flush=True)
|
||||
return point
|
||||
|
||||
|
||||
def svg(points, path):
|
||||
width, height = 720, 320
|
||||
labels = [p["tag"] for p in points]
|
||||
values = [p["load"]["successful_rps"] for p in points]
|
||||
peak = max(values) or 1
|
||||
left, top, plot_w, plot_h = 48, 20, width - 70, height - 70
|
||||
bar_w = plot_w / len(points) * 0.62
|
||||
gap = plot_w / len(points)
|
||||
parts = [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d">' % (width, height),
|
||||
'<rect width="100%" height="100%" fill="white"/>',
|
||||
'<text x="48" y="16" font-family="sans-serif" font-size="14">Successful RPS, closed-loop POST, concurrency %d, %ss</text>'
|
||||
% (CONCURRENCY, int(DURATION)),
|
||||
]
|
||||
for index, (label, value) in enumerate(zip(labels, values)):
|
||||
h = plot_h * value / peak
|
||||
x = left + index * gap + (gap - bar_w) / 2
|
||||
y = top + 12 + (plot_h - h)
|
||||
parts.append('<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" fill="#2f6f4e"/>' % (x, y, bar_w, h))
|
||||
parts.append(
|
||||
'<text x="%.1f" y="%.1f" font-family="sans-serif" font-size="12" text-anchor="middle">%.1f</text>'
|
||||
% (x + bar_w / 2, y - 4, value)
|
||||
)
|
||||
parts.append(
|
||||
'<text x="%.1f" y="%d" font-family="sans-serif" font-size="11" text-anchor="middle">%s</text>'
|
||||
% (x + bar_w / 2, height - 18, label)
|
||||
)
|
||||
parts.append("</svg>")
|
||||
path.write_text("\n".join(parts) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
points = []
|
||||
# Series A: total API budget fixed at 1.0 CPU and 512 MiB. Pool total stays 8.
|
||||
points.append(measure("A1", 1, "1.0", "512m", "1.0", 8))
|
||||
points.append(measure("A2", 2, "0.50", "256m", "1.0", 4))
|
||||
points.append(measure("A4", 4, "0.25", "128m", "1.0", 2))
|
||||
# Series B: each new replica brings the same 0.50 CPU / 256 MiB. Postgres stays at 1 CPU.
|
||||
points.append(measure("B1", 1, "0.50", "256m", "1.0", 4))
|
||||
points.append(measure("B2", 2, "0.50", "256m", "1.0", 4))
|
||||
points.append(measure("B4", 4, "0.50", "256m", "1.0", 4))
|
||||
# Candidate improvement: give the database the extra CPU the API side already has.
|
||||
points.append(measure("C4-pg2", 4, "0.50", "256m", "2.0", 4))
|
||||
# Candidate improvement: stop the 4 pools from opening 16 server connections.
|
||||
points.append(measure("D4-pool2", 4, "0.50", "256m", "1.0", 2))
|
||||
(OUT / "summary.json").write_text(json.dumps(points, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
svg(points, OUT / "rps.svg")
|
||||
# Return the stand to the documented 2 x 0.50 CPU layout.
|
||||
apply_profile(2, "0.50", "256m", "1.0", 4)
|
||||
print("wrote %s" % (OUT / "summary.json"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
# Разбор лабораторной 3
|
||||
|
||||
Автор: Булацев, БПИ2301. Целевая оценка: 5, трек «Цена масштабирования».
|
||||
|
||||
## Суть
|
||||
|
||||
Нужно запустить несколько одинаковых процессов API за одной точкой входа и не потерять корректность, когда один из них останавливают.
|
||||
|
||||
Клиент больше не обращается к конкретному процессу. Он ходит в балансировщик, а тот выбирает живую реплику. Любая реплика должна узнать ту же сессию, увидеть уже подтверждённое сообщение и не создать второе сообщение, если клиент повторил запрос с тем же ключом.
|
||||
|
||||
Источник истины для пользователей, чатов и сообщений — PostgreSQL. Сессии — Redis с AOF. Память процесса для этого не используется.
|
||||
|
||||
## Какие проблемы решались
|
||||
|
||||
### Один процесс нельзя просто скопировать
|
||||
|
||||
Если сессия или кэш лежат в памяти процесса, второй экземпляр пользователя не узнает. Повторный запрос попадёт на другую реплику и получит 401 либо устаревшую историю.
|
||||
|
||||
Решение: сессия хранится в общем Redis, история читается из общего Postgres. Кэш истории тоже в Redis и сбрасывается после успешной записи, поэтому следующая реплика не отдаёт старую страницу. Клиента не приклеиваем к одной реплике.
|
||||
|
||||
### Клиент не должен выбирать процесс сам
|
||||
|
||||
Нужна одна публичная точка входа и при этом несколько процессов. `depends_on` только задаёт порядок запуска контейнеров. Healthcheck контейнера говорит Docker, жив ли он, но не перекладывает HTTP-запрос на другой адрес.
|
||||
|
||||
Решение: HAProxy публикует `127.0.0.1:8080` и сам ходит в DNS-имя `api`. У сервиса Compose нет `container_name`, поэтому реплик может быть сколько угодно. Порты API наружу не опубликованы. HAProxy держит до 8 слотов через `server-template` и обновляет адреса из DNS Docker раз в секунду. Проверка — `GET /health/ready` каждые 1 с, реплика выпадает после двух неудач. Это около 2 с плюс до 1 с устаревшего DNS, если контейнер уже исчез.
|
||||
|
||||
### Остановка процесса не должна обрывать стенд надолго
|
||||
|
||||
Docker посылает SIGTERM. Если процесс сразу умирает, балансировщик ещё считает его живым и шлёт туда запросы.
|
||||
|
||||
Решение согласовано с проверкой готовности. По SIGTERM реплика сразу отвечает 503 на `/health/ready` и на новые бизнес-запросы, а завершается через 2 с (`LB_DRAIN_SECONDS`). За это HAProxy успевает увидеть неготовый backend. `scripts/stop-one.sh` останавливает только контейнер с нужным `X-Instance-Id` и оставляет его остановленным. `X-Instance-Id` равен hostname и не меняется, пока жив процесс. На публичной проверке запись восстановилась за 2.465 с, подтверждённые сообщения и сессия сохранились.
|
||||
|
||||
### Повтор POST после таймаута может создать второе сообщение
|
||||
|
||||
Клиент не знает, успел ли сервер записать сообщение, если ответ потерялся. Повтор без защиты создаёт дубль. Две реплики могут принять один и тот же повтор одновременно.
|
||||
|
||||
Решение: для `POST /messages` обязателен `Idempotency-Key`. Область ключа — пользователь, метод, путь чата и сам ключ. Текст приводится к одному JSON (порядок полей не важен, пустой список вложений равен отсутствию поля) и хешируется. Сообщение и сохранённый ответ 201 пишутся одной транзакцией. Пока транзакция не закоммичена, строки результата нет, поэтому неуспешный запрос можно повторить. Одновременные повторы сериализует advisory lock в Postgres: побеждает одна вставка, остальные получают то же сообщение. Другой текст с тем же ключом — 409. Проверено 16 параллельными запросами: в истории одна копия.
|
||||
|
||||
### Миграции нельзя применять гонкой всех реплик
|
||||
|
||||
Несколько процессов при старте могут одновременно выполнить один и тот же SQL.
|
||||
|
||||
Решение: схему применяет отдельный одноразовый сервис `migrate` до старта API. У реплик `MIGRATIONS_ON_START=0`. Повторный запуск пропускает файлы, уже записанные в `schema_migrations`. На время применения берётся `pg_advisory_lock`.
|
||||
|
||||
### Больше реплик не означает больше пропускной способности
|
||||
|
||||
Гипотеза трека 1: при фиксированном CPU API дробление на процессы не поможет, а при росте числа реплик выигрыш кончится, когда упрёмся не в CPU отдельного API.
|
||||
|
||||
Один и тот же closed-loop, 8 параллельных POST, 15 с. При бюджете 1 CPU / 512 МиБ результат 344, 196 и 124 RPS для 1, 2 и 4 реплик. Postgres был занят на 16–33%: мешала квота CPU на каждом процессе. Когда одна реплика упиралась в 0.50 CPU (98 RPS, p95 167 мс), вторая такая же реплика подняла результат до 224 RPS и p95 81 мс. Это подтверждённое улучшение. Четвёртая реплика дала только 237 RPS, зато сессий Postgres стало 17 вместо 9. Удвоение CPU Postgres и уменьшение пула на этом профиле заметного прироста не дали. Подробные таблицы — в `REPORT.md`.
|
||||
|
||||
## Ответы на вопросы README
|
||||
|
||||
### Чем отказ процесса отличается от отказа узла?
|
||||
|
||||
Отказ процесса — это остановка одного API-контейнера. Балансировщик перестаёт слать ему запросы, вторая реплика на том же хосте продолжает читать и писать в общие Postgres и Redis. Данные не пропадают.
|
||||
|
||||
Отказ узла — это потеря всей машины. Вместе с ней пропадают все реплики API, HAProxy, Postgres и Redis, потому что они запущены на одном Docker-хосте. Несколько процессов API такую потерю не переживают. Отдельно живая реплика базы в этой работе не поднималась, поэтому выживание узла БД не заявляется.
|
||||
|
||||
### Почему depends_on и healthcheck сами по себе не являются балансировщиком?
|
||||
|
||||
`depends_on` лишь откладывает старт контейнера, пока другой не стал healthy. Он не принимает соединения клиентов и не выбирает, какому процессу отдать конкретный HTTP-запрос. Healthcheck Docker перезапускает или помечает свой контейнер, но не знает о соседних репликах и не проксирует трафик. Пока снаружи опубликован сам API, клиент попадает в один процесс в обход остальных. Балансировщик — отдельный процесс: он слушает единственный адрес, смотрит готовность backend-ов и перекладывает запрос на живой.
|
||||
|
||||
### Какие POST можно повторять после timeout и что защищает Idempotency-Key?
|
||||
|
||||
После таймаута можно повторять POST сообщения с тем же ключом и тем же телом. Если запись успела закоммититься, сервер вернёт тот же `201` и то же сообщение. Если не успела, повтор создаст его один раз. Другое тело с тем же ключом получит 409.
|
||||
|
||||
Защищает общая строка в Postgres на пару «пользователь + метод + путь + ключ» вместе с advisory lock и одной транзакцией на сообщение и сохранённый ответ. Ключ не защищает создание пользователя и создание чата: их повтор может создать второй объект, поэтому балансировщик такие POST сам не повторяет.
|
||||
|
||||
### Что сломается при увеличении числа API, если сессии находятся в локальной памяти?
|
||||
|
||||
Вход создаст сессию только в памяти той реплики, которая обработала логин. Следующий запрос балансировщик отдаст другой реплике, и она ответит 401, потому что токена у неё нет. Пользователю придётся входить заново, а при sticky session он останется привязан к одному процессу: отказ этого процесса снова потеряет сессию. Общие сообщения в Postgres это не чинят. Поэтому сессия лежит в Redis, который видят все реплики.
|
||||
@@ -0,0 +1,24 @@
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def base_url():
|
||||
return os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def live(base_url):
|
||||
c = Client(base_url, timeout=15)
|
||||
try:
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
except Failure as exc:
|
||||
pytest.skip("API is not ready: %s" % exc)
|
||||
return c
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Negative, pagination and auth tests for lab 2."""
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
|
||||
from client import Client
|
||||
|
||||
|
||||
PASSWORD = "Example-only-not-a-real-secret-123"
|
||||
|
||||
|
||||
def register(c: Client, username=None):
|
||||
username = username or ("u_" + uuid.uuid4().hex[:10])
|
||||
user = c.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
201,
|
||||
body={"username": username, "display_name": "Test User", "password": PASSWORD},
|
||||
schema="User",
|
||||
)["json"]
|
||||
return user
|
||||
|
||||
|
||||
def login(c: Client, username, password=PASSWORD):
|
||||
import base64
|
||||
|
||||
cred = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
session = c.call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
201,
|
||||
headers={"Authorization": "Basic " + cred},
|
||||
schema="Session",
|
||||
)["json"]
|
||||
return {"id": session["user"]["id"], "user": session["user"], "token": session["token"], "registration": {"username": username, "password": password}}
|
||||
|
||||
|
||||
def actor_of(c: Client, username=None):
|
||||
user = register(c, username)
|
||||
return login(c, user["username"])
|
||||
|
||||
|
||||
def test_malformed_json(live: Client):
|
||||
response = live.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
400,
|
||||
raw=b"{not-json",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
assert response["json"]["error"]["code"] == "invalid_request"
|
||||
assert response["headers"].get("X-Request-Id")
|
||||
|
||||
|
||||
def test_duplicate_username(live: Client):
|
||||
user = register(live, username="dup_" + uuid.uuid4().hex[:8])
|
||||
again = live.call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
409,
|
||||
body={"username": user["username"], "display_name": "Other", "password": PASSWORD},
|
||||
)
|
||||
assert again["json"]["error"]["code"] == "conflict"
|
||||
|
||||
|
||||
def test_empty_message_rejected(live: Client):
|
||||
alice = actor_of(live)
|
||||
bob = actor_of(live)
|
||||
chat = live.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "t", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
response = live.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
400,
|
||||
actor=alice,
|
||||
body={"text": " "},
|
||||
)
|
||||
assert response["status"] == 400
|
||||
|
||||
|
||||
def test_outsider_chat_is_404(live: Client):
|
||||
alice = actor_of(live)
|
||||
bob = actor_of(live)
|
||||
eve = actor_of(live)
|
||||
chat = live.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "secret", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
response = live.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve)
|
||||
assert response["json"]["error"]["code"] == "not_found"
|
||||
|
||||
|
||||
def test_message_pagination_multiple_pages(live: Client):
|
||||
alice = actor_of(live)
|
||||
bob = actor_of(live)
|
||||
chat = live.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "pages", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
created = []
|
||||
for idx in range(5):
|
||||
created.append(live.message(alice, chat, "m%s" % idx)["json"])
|
||||
collected = []
|
||||
cursor = None
|
||||
for _ in range(10):
|
||||
path = "/api/v1/chats/%s/messages?limit=2" % chat["id"]
|
||||
if cursor:
|
||||
path += "&cursor=" + quote(cursor, safe="")
|
||||
page = live.call("GET", path, actor=bob, schema="MessagePage")["json"]
|
||||
collected.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
assert len(collected) == 5
|
||||
ordered = sorted(created, key=lambda x: (x["created_at"], x["id"]))
|
||||
assert collected == ordered
|
||||
|
||||
|
||||
def test_cursor_from_other_chat_rejected(live: Client):
|
||||
alice = actor_of(live)
|
||||
bob = actor_of(live)
|
||||
chat_a = live.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "a", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
chat_b = live.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "b", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
msg = live.message(alice, chat_a, "only in a")["json"]
|
||||
page = live.call(
|
||||
"GET",
|
||||
"/api/v1/chats/%s/messages?limit=1" % chat_a["id"],
|
||||
actor=alice,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
assert page["items"][0]["id"] == msg["id"]
|
||||
cursor = page["next_cursor"]
|
||||
# If history is a single item, next_cursor is null; encode a foreign cursor via first page of A after a dummy extra message.
|
||||
if cursor is None:
|
||||
live.message(alice, chat_a, "second")
|
||||
page = live.call(
|
||||
"GET",
|
||||
"/api/v1/chats/%s/messages?limit=1" % chat_a["id"],
|
||||
actor=alice,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
cursor = page["next_cursor"]
|
||||
response = live.call(
|
||||
"GET",
|
||||
"/api/v1/chats/%s/messages?limit=10&cursor=%s" % (chat_b["id"], quote(cursor, safe="")),
|
||||
400,
|
||||
actor=alice,
|
||||
)
|
||||
assert response["status"] == 400
|
||||
|
||||
|
||||
def test_x_user_id_does_not_authenticate(live: Client):
|
||||
alice = actor_of(live)
|
||||
live.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]})
|
||||
|
||||
|
||||
def test_wrong_password(live: Client):
|
||||
user = register(live)
|
||||
import base64
|
||||
|
||||
bad = base64.b64encode((user["username"] + ":wrong-password-xx").encode()).decode()
|
||||
live.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + bad})
|
||||
|
||||
|
||||
def test_logout(live: Client):
|
||||
alice = actor_of(live)
|
||||
live.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice)
|
||||
live.call("GET", "/api/v1/users/me", 401, actor=alice)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""10 clients × 10 messages = 100 unique messages."""
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
from client import Client
|
||||
|
||||
from student_tests.test_api import actor_of
|
||||
|
||||
|
||||
def test_concurrent_messages(live: Client):
|
||||
users = [actor_of(live) for _ in range(10)]
|
||||
member_ids = [u["id"] for u in users]
|
||||
chat = live.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=users[0],
|
||||
body={"title": "load", "member_ids": member_ids},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
|
||||
errors = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def worker(user):
|
||||
local = Client(live.base, timeout=15)
|
||||
for _ in range(10):
|
||||
try:
|
||||
local.message(user, chat, uuid.uuid4().hex)
|
||||
except Exception as exc:
|
||||
with lock:
|
||||
errors.append(str(exc))
|
||||
|
||||
threads = [threading.Thread(target=worker, args=(u,)) for u in users]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join(timeout=60)
|
||||
|
||||
assert not errors, "unexpected failures: %s" % errors[:5]
|
||||
page = live.call(
|
||||
"GET",
|
||||
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
|
||||
actor=users[0],
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
ids = [item["id"] for item in page["items"]]
|
||||
assert len(ids) == len(set(ids)) == 100
|
||||
@@ -0,0 +1,17 @@
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from client import Client
|
||||
|
||||
|
||||
@pytest.mark.expiry
|
||||
def test_session_expires(live: Client):
|
||||
ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
if os.environ.get("RUN_EXPIRY") != "1":
|
||||
pytest.skip("set RUN_EXPIRY=1 and SESSION_TTL_SECONDS<=3")
|
||||
assert ttl <= 3
|
||||
actor = live.user("ttl")
|
||||
live.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
time.sleep(ttl + 1.5)
|
||||
live.call("GET", "/api/v1/users/me", 401, actor=actor)
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Grade-4: username race and atomic chat rollback."""
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
from client import Client
|
||||
|
||||
from student_tests.test_api import PASSWORD, actor_of
|
||||
|
||||
|
||||
def test_register_race(live: Client):
|
||||
username = "r_" + uuid.uuid4().hex[:12]
|
||||
body = {"username": username, "display_name": "Race", "password": PASSWORD}
|
||||
results = []
|
||||
|
||||
def once():
|
||||
local = Client(live.base, timeout=20)
|
||||
try:
|
||||
results.append(local.call("POST", "/api/v1/users", [201, 409], body=body)["status"])
|
||||
except Exception as exc:
|
||||
results.append(str(exc))
|
||||
|
||||
threads = [threading.Thread(target=once) for _ in range(16)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
assert results.count(201) == 1, results
|
||||
assert results.count(409) == 15, results
|
||||
|
||||
|
||||
def test_chat_rollback(live: Client):
|
||||
alice = actor_of(live)
|
||||
before = live.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"]
|
||||
missing = str(uuid.uuid4())
|
||||
live.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
400,
|
||||
actor=alice,
|
||||
body={"title": "rollback-should-not-exist", "member_ids": [alice["id"], missing]},
|
||||
)
|
||||
after = live.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"]
|
||||
assert len(after) == len(before)
|
||||
assert all(x["title"] != "rollback-should-not-exist" for x in after)
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Dependency failure and Redis restart (AOF + cache rebuild)."""
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from client import Client, Failure
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def compose(*args):
|
||||
env_file = ROOT / ".env"
|
||||
cmd = ["docker", "compose"]
|
||||
if env_file.is_file():
|
||||
cmd.extend(["--env-file", str(env_file)])
|
||||
cmd.extend(args)
|
||||
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
|
||||
if proc.returncode != 0:
|
||||
raise AssertionError("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or "")[-500:]))
|
||||
return proc
|
||||
|
||||
|
||||
def wait_ready(c: Client, seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
return
|
||||
except Failure as exc:
|
||||
last = exc
|
||||
time.sleep(0.5)
|
||||
raise AssertionError("ready not restored: %s" % last)
|
||||
|
||||
|
||||
def wait_status(c: Client, path, expected, seconds=30, actor=None):
|
||||
deadline = time.monotonic() + seconds
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
schema = "Health" if expected == 200 and path.startswith("/health") else None
|
||||
c.call("GET", path, expected, actor=actor, schema=schema)
|
||||
return
|
||||
except Failure as exc:
|
||||
last = exc
|
||||
time.sleep(0.4)
|
||||
raise AssertionError("did not observe %s for %s: %s" % (expected, path, last))
|
||||
|
||||
|
||||
@pytest.mark.infra
|
||||
def test_redis_and_postgres_unavailable(live: Client):
|
||||
if os.environ.get("RUN_INFRA") != "1":
|
||||
pytest.skip("set RUN_INFRA=1 to stop compose services")
|
||||
alice = live.user("infra")
|
||||
compose("stop", "redis")
|
||||
wait_status(live, "/health/live", 200)
|
||||
wait_status(live, "/health/ready", 503)
|
||||
wait_status(live, "/api/v1/users/me", 503, actor=alice)
|
||||
compose("start", "redis")
|
||||
wait_ready(live)
|
||||
live.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
||||
|
||||
compose("stop", "postgres")
|
||||
wait_status(live, "/health/live", 200)
|
||||
wait_status(live, "/health/ready", 503)
|
||||
compose("start", "postgres")
|
||||
wait_ready(live)
|
||||
live.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
||||
|
||||
|
||||
@pytest.mark.infra
|
||||
def test_redis_restart_keeps_session_and_history(live: Client):
|
||||
if os.environ.get("RUN_INFRA") != "1":
|
||||
pytest.skip("set RUN_INFRA=1 to restart redis")
|
||||
alice, bob, _eve, chat = live.bootstrap()
|
||||
msg = live.message(alice, chat, "before-redis-restart")["json"]
|
||||
compose("restart", "redis")
|
||||
wait_ready(live)
|
||||
me = live.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"]
|
||||
assert me["id"] == alice["id"]
|
||||
page = live.call(
|
||||
"GET",
|
||||
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
|
||||
actor=bob,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
assert any(m["id"] == msg["id"] for m in page["items"])
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Track 1: cache consistency — stampede, stale read, read-after-write."""
|
||||
import json
|
||||
import threading
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from client import Client
|
||||
|
||||
|
||||
def _lab_metrics(c: Client):
|
||||
return json.loads(c.call("GET", "/api/v1/_lab/metrics")["bytes"].decode())
|
||||
|
||||
|
||||
def _lab_flags(c: Client, body):
|
||||
return json.loads(c.call("POST", "/api/v1/_lab/flags", 200, body=body)["bytes"].decode())
|
||||
|
||||
|
||||
def test_read_after_write(live: Client):
|
||||
alice, bob, _eve, chat = live.bootstrap()
|
||||
_lab_flags(live, {"invalidate": True, "singleflight": True, "reset_sql": True})
|
||||
first = live.message(alice, chat, "raw-visible-" + uuid.uuid4().hex)["json"]
|
||||
page = live.call(
|
||||
"GET",
|
||||
"/api/v1/chats/%s/messages?limit=100" % chat["id"],
|
||||
actor=bob,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
assert any(m["id"] == first["id"] for m in page["items"])
|
||||
|
||||
|
||||
def test_stale_without_invalidation_then_fixed(live: Client):
|
||||
alice, bob, _eve, chat = live.bootstrap()
|
||||
path = "/api/v1/chats/%s/messages?limit=50" % chat["id"]
|
||||
live.call("GET", path, actor=alice, schema="MessagePage")
|
||||
_lab_flags(live, {"invalidate": False, "singleflight": True})
|
||||
try:
|
||||
stale_text = "stale-probe-" + uuid.uuid4().hex
|
||||
live.message(alice, chat, stale_text)
|
||||
page = live.call("GET", path, actor=bob, schema="MessagePage")["json"]
|
||||
assert all(m["text"] != stale_text for m in page["items"])
|
||||
finally:
|
||||
_lab_flags(live, {"invalidate": True})
|
||||
fresh_text = "fresh-probe-" + uuid.uuid4().hex
|
||||
live.message(alice, chat, fresh_text)
|
||||
page2 = live.call("GET", path, actor=bob, schema="MessagePage")["json"]
|
||||
assert any(m["text"] == fresh_text for m in page2["items"])
|
||||
|
||||
|
||||
def test_stampede_singleflight_reduces_sql(live: Client):
|
||||
alice, _bob, _eve, chat = live.bootstrap()
|
||||
live.message(alice, chat, "stampede-seed")
|
||||
path = "/api/v1/chats/%s/messages?limit=50" % chat["id"]
|
||||
|
||||
def burst(n=24):
|
||||
barrier = threading.Barrier(n)
|
||||
|
||||
def one(_i):
|
||||
local = Client(live.base)
|
||||
barrier.wait()
|
||||
return local.call("GET", path, actor=alice, schema="MessagePage")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=n) as pool:
|
||||
list(pool.map(one, range(n)))
|
||||
|
||||
try:
|
||||
_lab_flags(live, {"singleflight": False, "invalidate": True, "reset_sql": True, "delay_ms": 250})
|
||||
live.message(alice, chat, "miss-" + uuid.uuid4().hex)
|
||||
_lab_flags(live, {"reset_sql": True, "singleflight": False, "delay_ms": 250})
|
||||
burst()
|
||||
unprotected = _lab_metrics(live)["history_sql"]
|
||||
|
||||
_lab_flags(live, {"singleflight": True, "reset_sql": True, "delay_ms": 250})
|
||||
live.message(alice, chat, "miss2-" + uuid.uuid4().hex)
|
||||
_lab_flags(live, {"reset_sql": True, "singleflight": True, "delay_ms": 250})
|
||||
burst()
|
||||
protected = _lab_metrics(live)["history_sql"]
|
||||
finally:
|
||||
_lab_flags(live, {"invalidate": True, "singleflight": True, "delay_ms": 0})
|
||||
|
||||
print(json.dumps({"stampede_sql_without_singleflight": unprotected, "stampede_sql_with_singleflight": protected}))
|
||||
assert unprotected >= 8, unprotected
|
||||
assert protected < unprotected, (protected, unprotected)
|
||||
assert protected <= 4, protected
|
||||
Reference in New Issue
Block a user