ЛР 3 — БВТ2304 — Риц — на 4 #102
+4
-10
@@ -1,13 +1,7 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# Generate a local .env with python scripts/init-env.py. Never commit it.
|
||||
APP_PORT=8081
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
POSTGRES_PASSWORD=
|
||||
CURSOR_SECRET=
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
scripts/*.sh text eol=lf
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM python:3.12.12-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONPATH=/app/src \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080
|
||||
|
||||
# Unprivileged user with a fixed UID; the application never runs as root.
|
||||
RUN groupadd --system --gid 10001 app \
|
||||
&& useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt /app/
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Only the source code goes into the image (no .env, tests, evidence, .git).
|
||||
# Files stay owned by root, so the app user cannot modify its own code.
|
||||
COPY src/ /app/src/
|
||||
|
||||
ARG BUILD_REVISION=dev
|
||||
LABEL org.opencontainers.image.revision=${BUILD_REVISION}
|
||||
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
|
||||
# docker stop sends SIGTERM; the app drains and exits well within the 10 s budget.
|
||||
STOPSIGNAL SIGTERM
|
||||
HEALTHCHECK --interval=2s --timeout=2s --start-period=5s --retries=3 \
|
||||
CMD ["python", "/app/src/healthcheck.py"]
|
||||
|
||||
# Exec form: python is PID 1 and receives signals directly (no shell in between).
|
||||
CMD ["python", "-m", "messenger"]
|
||||
@@ -1,5 +1,5 @@
|
||||
PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
PYTHON ?= python3 -X utf8
|
||||
BASE_URL ?= http://127.0.0.1:8081
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
@@ -1,37 +1,113 @@
|
||||
# Отчёт — лабораторная 3
|
||||
# Лабораторная 3. Балансировка и отказ API-реплики
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
**Риц Дмитрий, БВТ2304**. Целевая оценка — **4**.
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Запуск
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Desktop с Linux Engine или Docker Engine с Compose, а для тестов — Python 3.9+. В контейнерах закреплены Python 3.12.12, PostgreSQL 17.6, Redis 7.4.6 и HAProxy 3.2.12. Команды выполняются из корня репозитория. На Windows можно использовать обычную CMD.
|
||||
|
||||
## Архитектура и решения
|
||||
```bat
|
||||
python scripts\init-env.py
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build --wait
|
||||
docker compose ps -a
|
||||
curl.exe -i http://127.0.0.1:8081/health/ready
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
`init-env.py` создаёт локальный `.env` с паролем PostgreSQL и ключом подписи курсоров. Повторный запуск использует тот же файл. `APP_PORT` по умолчанию равен 8081, чтобы запустить lab3 рядом с lab2. `API_REPLICAS` задаёт число экземпляров одного сервиса, по умолчанию 2. В HAProxy предусмотрено до 8 слотов. `SESSION_TTL_SECONDS` и `IDEMPOTENCY_TTL_SECONDS` по умолчанию равны 3600 и 86400 соответственно.
|
||||
|
||||
## Доказательства критериев
|
||||
`docker compose down` останавливает стенд и сохраняет тома. `docker compose down -v` подходит для намеренного удаления всех учебных данных, включая сессии.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
## Как устроен стенд
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Клиент обращается к HAProxy. Балансировщик раздаёт запросы по кругу готовым API-репликам. `X-Instance-Id` берётся из hostname контейнера, поэтому можно сопоставить заголовок с конкретным контейнером и остановить его. Каждая реплика использует общую PostgreSQL для пользователей, чатов, сообщений и результатов отправки. Redis хранит общие Bearer-сессии с TTL, AOF и именованным томом. Реплики используют один ключ подписи курсоров, поэтому продолжить чтение страницы можно через любой экземпляр.
|
||||
|
||||
## Эксперименты и отказы
|
||||
На хост опубликован только порт HAProxy. Сеть `edge` объединяет балансировщик и API. PostgreSQL, Redis, миграции и API используют отдельную внутреннюю сеть `storage`. Пароль PostgreSQL передаётся БД, API и миграциям, ключ подписи курсоров нужен только API. HAProxy использует один рабочий поток при лимите 0,5 CPU.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
| Контейнер | CPU | RAM |
|
||||
| --- | --- | --- |
|
||||
| API, на реплику | 1 | 384 MiB |
|
||||
| HAProxy | 0,5 | 96 MiB |
|
||||
| PostgreSQL | 1 | 512 MiB |
|
||||
| Redis | 0,5 | 128 MiB |
|
||||
| Миграции | 0,5 | 128 MiB |
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
HAProxy проверяет `GET /health/ready` каждые 500 мс. Два промаха исключают реплику, два успеха возвращают её. В переходном состоянии интервал сокращается до 250 мс. Таймауты подключения и ответа health-check составляют 500 и 700 мс, поэтому бюджет обнаружения отказа по конфигурации — до 3,4 с при нормальной работе планировщика. Docker DNS опрашивается с `hold valid 1s`, чтобы подхватить адрес нового контейнера. Клиентские запросы имеют отдельные таймауты. После SIGTERM API снимает готовность, закрывает вход для новых запросов и до 4 с даёт текущим запросам завершиться. Docker отводит 7 с на остановку. [Параметры HAProxy](https://docs.haproxy.org/3.2/configuration.html)
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
## Что происходит при повторе POST
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
У отправки сообщения есть обязательный `Idempotency-Key`. API проверяет токен и доступ к чату при каждом обращении. Затем PostgreSQL резервирует ключ в `message_requests`. Первичный ключ по `(chat_id, sender_id, idem_key)` сериализует параллельные повторы через разные реплики. Метод и маршрут фиксированы в этом обработчике.
|
||||
|
||||
Запись ключа и сообщения коммитится одной транзакцией. Отложенный внешний ключ позволяет сначала занять ключ, а затем записать сообщение с заранее выбранным UUID. При откате откатываются обе записи. Повтор с тем же текстом до истечения срока возвращает исходный объект и код 201. Если текст другой, API отдаёт 409. После 24 часов ключ можно занять снова, а старое сообщение остаётся в истории. Индекс по `expires_at` пригоден для удаления истёкших записей при обслуживании БД. [Конфликты при INSERT в PostgreSQL](https://www.postgresql.org/docs/17/sql-insert.html)
|
||||
|
||||
Сервис `migrate` один раз применяет `001_initial` и `002_message_requests`. Версии записываются в `schema_migrations`. Миграция 002 добавляет таблицу и индекс поверх прежней схемы. При поочерёдной замене контейнеров команда запускает только API, а обе версии образа используют уже подготовленную схему. Смена схемы с несовместимыми изменениями потребует отдельной последовательности обновления.
|
||||
|
||||
## Проверки
|
||||
|
||||
```bat
|
||||
python -X utf8 scripts\check_contract.py
|
||||
python -X utf8 tests\smoke.py --base-url http://127.0.0.1:8081
|
||||
python -X utf8 tests\resilience.py --base-url http://127.0.0.1:8081 --mode persistence --action-script scripts\restart.cmd
|
||||
python -X utf8 tests\resilience.py --base-url http://127.0.0.1:8081 --mode failover --action-script scripts\stop-one.cmd
|
||||
docker compose start api
|
||||
```
|
||||
|
||||
В тесте persistence перезапускаются HAProxy, обе API-реплики, PostgreSQL и Redis. Тест failover сам передаёт `TARGET_INSTANCE_ID` в скрипт. Скрипт сопоставляет его с hostname запущенного контейнера сервиса API и останавливает только эту реплику. После публичного теста остановленную реплику возвращает команда `docker compose start api`.
|
||||
|
||||
На Linux можно выполнить те же проверки через Makefile.
|
||||
|
||||
```sh
|
||||
chmod +x scripts/restart.sh scripts/stop-one.sh
|
||||
make test
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
docker compose start api
|
||||
```
|
||||
|
||||
Дополнительные проверки запускаются по очереди при двух готовых репликах.
|
||||
|
||||
```bat
|
||||
python -X utf8 student-tests\grade4.py --mode race
|
||||
python -X utf8 student-tests\grade4.py --mode failover
|
||||
python -X utf8 student-tests\grade4.py --mode rolling
|
||||
```
|
||||
|
||||
`race` посылает восемь параллельных запросов с одним ключом. `failover` останавливает каждую реплику по очереди и возвращает предыдущую перед следующим шагом. `rolling` собирает образ с новой OCI-меткой ревизии, заменяет контейнеры по одному и ждёт, пока новый экземпляр начнёт принимать запросы. На каждом шаге оставшийся экземпляр обслуживает запросы. Все запросы теста идут через HAProxy с одной общей сессией.
|
||||
|
||||
## Результаты
|
||||
|
||||
| Что проверялось | Результат |
|
||||
| --- | --- |
|
||||
| Запуск, миграции, smoke | Обе реплики и балансировщик готовы. На новом томе применились две миграции, повтор вернул `Schema is up to date`. Публичный smoke прошёл. [Вывод](evidence/public-checks.txt) |
|
||||
| Полный перезапуск | Изменились `StartedAt` пяти контейнеров. Тест прочитал прежнее сообщение с ранее выданным токеном. [Вывод](evidence/public-checks.txt) |
|
||||
| Публичный failover | Видны две реплики. За 15-секундное окно вокруг остановки тест подтвердил 126 записей и посчитал 2 неудачные попытки. Первая запись после завершения hook подтверждена через 2,024 с от его запуска. Все подтверждённые записи сохранились. [Вывод](evidence/public-checks.txt) |
|
||||
| Конкурентный повтор | Восемь запросов прошли через две реплики и получили один объект сообщения. С другим текстом получен 409. Другой отправитель использовал тот же ключ независимо. [Замер](evidence/race.json) |
|
||||
| Отказ обеих реплик по очереди | Первое чтение после остановки через 1,181 и 1,269 с, первая запись через 0,659 и 0,962 с. В истории остались все 560 подтверждённых сообщений. [Замер](evidence/failover.json) |
|
||||
| Поочерёдная замена | Новые контейнеры стали отвечать через HAProxy через 2,999 и 2,947 с от начала соответствующего шага. По окончании в истории остались все 311 подтверждённых сообщений. [Замер](evidence/rolling.json) |
|
||||
|
||||
| Шаг | Успешные записи/с | Неудачные попытки | Первая запись через оставшуюся реплику, с |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| Остановка первой реплики | 25,20 | 8 | 0,659 |
|
||||
| Остановка второй реплики | 25,17 | 8 | 0,962 |
|
||||
| Замена первой реплики | 21,75 | 13 | 1,143 |
|
||||
| Замена второй реплики | 22,04 | 10 | 0,647 |
|
||||
|
||||
Эксперимент выполнен на Windows с Docker Desktop, 12 логическими CPU AMD Ryzen 5 5560U и 8 011 964 416 байтами RAM, доступными Docker. В каждом нагрузочном опыте четыре клиента отправляли синтетические сообщения через HAProxy, делая паузу 120 мс между запросами. Прогрев — 1 с, базовое окно — 2 с. В опыте с остановкой каждая реплика оставалась выключенной ещё 6 с после первого успешного чтения через оставшуюся. RPS и ошибки посчитаны по завершившимся запросам. p50, p95 и p99 относятся к успешным ответам. После неясного результата клиент повторяет отправку с прежним ключом. В конце проверяется история и все подтверждённые сообщения.
|
||||
|
||||
Это замер работоспособности при отказе процесса и замене контейнеров на одном хосте. У хоста, HAProxy, PostgreSQL и Redis по одному экземпляру. Замена выполнена на образ с новой меткой сборки из того же исходного кода и на той же схеме БД.
|
||||
|
||||
## На защите
|
||||
|
||||
Показать `docker compose ps -a`, две реплики без опубликованных портов и несколько ответов через общий порт.
|
||||
|
||||
```bat
|
||||
for /L %i in (1,1,6) do @curl.exe -isS http://127.0.0.1:8081/health/ready
|
||||
```
|
||||
|
||||
Показать два разных `X-Instance-Id`, затем запустить `--mode race` для одной сессии и одного ключа. Для демонстрации отказа запустить публичный failover, показать один остановленный контейнер через `docker compose ps -a` и вернуть его. Если попросят показать замену под нагрузкой, запустить `--mode rolling`.
|
||||
|
||||
При timeout клиент не знает, успел ли сервер сделать COMMIT. Повтор с тем же ключом отдаёт прежнее сообщение, даже если запрос обслуживает другая реплика. Локальная сессия пропала бы при переключении, поэтому токен проверяет общий Redis. Compose запускает контейнеры и следит за их состоянием, а выбор доступной реплики на каждом запросе делает HAProxy.
|
||||
|
||||
## Источники и помощь ИИ
|
||||
|
||||
Основа — каркас курса и код lab2. При реализации, проверках и подготовке отчёта использовался Codex. [Docker Compose](https://docs.docker.com/reference/cli/docker/compose/up/), [HAProxy](https://docs.haproxy.org/3.2/configuration.html), [PostgreSQL](https://www.postgresql.org/docs/17/sql-insert.html).
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
x-app-image: &app-image
|
||||
image: bruh-lab3-api:local
|
||||
build: .
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.6-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run python scripts/init-env.py}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
networks: [storage]
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
|
||||
redis:
|
||||
image: redis:7.4.6-bookworm
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", ""]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
networks: [storage]
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
|
||||
migrate:
|
||||
<<: *app-image
|
||||
environment:
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run python scripts/init-env.py}
|
||||
command: ["python", "-m", "messenger.migrate"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
networks: [storage]
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
api:
|
||||
<<: *app-image
|
||||
scale: ${API_REPLICAS:-2}
|
||||
environment:
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run python scripts/init-env.py}
|
||||
CURSOR_SECRET: ${CURSOR_SECRET:?Run python scripts/init-env.py}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
SHUTDOWN_TIMEOUT_SECONDS: "4"
|
||||
dns_opt: ["timeout:1", "attempts:1"]
|
||||
stop_grace_period: 7s
|
||||
networks: [edge, storage]
|
||||
cpus: 1.0
|
||||
mem_limit: 384m
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.12-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8081}:8080"
|
||||
networks: [edge]
|
||||
cpus: 0.5
|
||||
mem_limit: 96m
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
|
||||
networks:
|
||||
edge:
|
||||
storage:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
@@ -0,0 +1,80 @@
|
||||
{
|
||||
"mode": "failover",
|
||||
"initial_replicas": [
|
||||
{
|
||||
"id": "120f4ffe45f4b098e6dc30d589050cb6e0c3e5a6cff2221d668b9e1e6eecf51e",
|
||||
"alias": "120f4ffe45f4",
|
||||
"name": "/bruh_lab3-api-3",
|
||||
"image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5",
|
||||
"status": "running"
|
||||
},
|
||||
{
|
||||
"id": "2c9c28e81f35433136326442eeac3328e5f421933d7caa4e544fc5a728cf6d6e",
|
||||
"alias": "2c9c28e81f35",
|
||||
"name": "/bruh_lab3-api-4",
|
||||
"image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5",
|
||||
"status": "running"
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"seconds": 2.001,
|
||||
"successful_writes": 59,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 29.49,
|
||||
"latency_ms": {
|
||||
"p50": 9.86,
|
||||
"p95": 15.74,
|
||||
"p99": 18.17
|
||||
},
|
||||
"serving_aliases": [
|
||||
"120f4ffe45f4",
|
||||
"2c9c28e81f35"
|
||||
]
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"stopped": "/bruh_lab3-api-3",
|
||||
"target_alias": "120f4ffe45f4",
|
||||
"target_status": "exited",
|
||||
"first_survivor_read_seconds": 1.181,
|
||||
"first_survivor_write_seconds": 0.659,
|
||||
"seconds": 7.182,
|
||||
"successful_writes": 181,
|
||||
"failed_attempts": 8,
|
||||
"successful_rps": 25.2,
|
||||
"latency_ms": {
|
||||
"p50": 9.8,
|
||||
"p95": 24.28,
|
||||
"p99": 27.35
|
||||
},
|
||||
"serving_aliases": [
|
||||
"2c9c28e81f35"
|
||||
]
|
||||
},
|
||||
{
|
||||
"stopped": "/bruh_lab3-api-4",
|
||||
"target_alias": "2c9c28e81f35",
|
||||
"target_status": "exited",
|
||||
"first_survivor_read_seconds": 1.269,
|
||||
"first_survivor_write_seconds": 0.962,
|
||||
"seconds": 7.269,
|
||||
"successful_writes": 183,
|
||||
"failed_attempts": 8,
|
||||
"successful_rps": 25.17,
|
||||
"latency_ms": {
|
||||
"p50": 9.55,
|
||||
"p95": 23.23,
|
||||
"p99": 25.36
|
||||
},
|
||||
"serving_aliases": [
|
||||
"120f4ffe45f4",
|
||||
"2c9c28e81f35"
|
||||
]
|
||||
}
|
||||
],
|
||||
"acknowledged_messages": 560,
|
||||
"stored_messages": 560,
|
||||
"workers": 4,
|
||||
"pause_seconds": 0.12,
|
||||
"warmup_seconds": 1
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
bruh_lab3-api-3 bruh-lab3-api:local "python -m messenger" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
bruh_lab3-api-4 bruh-lab3-api:local "python -m messenger" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
bruh_lab3-lb-1 haproxy:3.2.12-alpine "docker-entrypoint.s…" lb 5 seconds ago Up 3 seconds (healthy) 127.0.0.1:8081->8080/tcp
|
||||
bruh_lab3-migrate-1 sha256:3482b32c94c1065cb1df310ccc35f5a0594ca577ecb0e3dd492e5842a624799f "python -m messenger…" migrate 10 minutes ago Exited (0) 6 minutes ago
|
||||
bruh_lab3-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 10 minutes ago Up 6 minutes (healthy) 5432/tcp
|
||||
bruh_lab3-redis-1 redis:7.4.6-bookworm "docker-entrypoint.s…" redis 10 minutes ago Up 6 minutes (healthy) 6379/tcp
|
||||
Applied: 001_initial, 002_message_requests
|
||||
Schema is up to date
|
||||
Schema is up to date
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
Before restart (UTC)
|
||||
/bruh_lab3-api-3 2026-11-05T23:38:49.72159878Z
|
||||
/bruh_lab3-api-4 2026-11-05T23:38:54.359920283Z
|
||||
/bruh_lab3-lb-1 2026-11-05T23:41:37.623541111Z
|
||||
/bruh_lab3-postgres-1 2026-11-05T23:34:57.792671441Z
|
||||
/bruh_lab3-redis-1 2026-11-05T23:34:57.877487028Z
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
After restart (UTC)
|
||||
/bruh_lab3-api-3 2026-11-05T23:41:55.159614054Z
|
||||
/bruh_lab3-api-4 2026-11-05T23:41:55.083328192Z
|
||||
/bruh_lab3-lb-1 2026-11-05T23:41:54.760199725Z
|
||||
/bruh_lab3-postgres-1 2026-11-05T23:41:54.520560824Z
|
||||
/bruh_lab3-redis-1 2026-11-05T23:41:54.610324316Z
|
||||
Before public failover
|
||||
/bruh_lab3-api-3 120f4ffe45f4 running
|
||||
/bruh_lab3-api-4 2c9c28e81f35 running
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 2, "first_recovery_seconds": 2.024}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
After public failover
|
||||
/bruh_lab3-api-3 120f4ffe45f4 running
|
||||
/bruh_lab3-api-4 2c9c28e81f35 exited
|
||||
After rolling replacement
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
bruh_lab3-api-5 bruh-lab3-api:local "python -m messenger" api 8 seconds ago Up 7 seconds (healthy) 8080/tcp
|
||||
bruh_lab3-api-6 bruh-lab3-api:local "python -m messenger" api 4 seconds ago Up 3 seconds (health: starting) 8080/tcp
|
||||
bruh_lab3-lb-1 haproxy:3.2.12-alpine "docker-entrypoint.s…" lb About a minute ago Up About a minute (healthy) 127.0.0.1:8081->8080/tcp
|
||||
bruh_lab3-migrate-1 sha256:3482b32c94c1065cb1df310ccc35f5a0594ca577ecb0e3dd492e5842a624799f "python -m messenger…" migrate 11 minutes ago Exited (0) 7 minutes ago
|
||||
bruh_lab3-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 11 minutes ago Up About a minute (healthy) 5432/tcp
|
||||
bruh_lab3-redis-1 redis:7.4.6-bookworm "docker-entrypoint.s…" redis 11 minutes ago Up About a minute (healthy) 6379/tcp
|
||||
/bruh_lab3-api-5 cpus=1000000000 mem_bytes=402653184 published_ports={}
|
||||
/bruh_lab3-api-6 cpus=1000000000 mem_bytes=402653184 published_ports={}
|
||||
/bruh_lab3-lb-1 cpus=500000000 mem_bytes=100663296 published_ports={"8080/tcp":[{"HostIp":"127.0.0.1","HostPort":"8081"}]}
|
||||
/bruh_lab3-migrate-1 cpus=500000000 mem_bytes=134217728 published_ports={}
|
||||
/bruh_lab3-postgres-1 cpus=1000000000 mem_bytes=536870912 published_ports={}
|
||||
/bruh_lab3-redis-1 cpus=500000000 mem_bytes=134217728 published_ports={}
|
||||
bruh_lab3-api-5 30.36% 28.82MiB / 384MiB
|
||||
bruh_lab3-api-6 28.16% 28.39MiB / 384MiB
|
||||
bruh_lab3-lb-1 2.09% 76.11MiB / 96MiB
|
||||
bruh_lab3-postgres-1 5.40% 31.93MiB / 512MiB
|
||||
bruh_lab3-redis-1 0.42% 4.422MiB / 128MiB
|
||||
Engine 29.3.1, CPUs 12, RAM 8011964416 B, OS Docker Desktop (containerized)
|
||||
Docker Compose version v5.1.1
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
Final ready aliases through HAProxy
|
||||
x-instance-id: 1fc578fde21a
|
||||
x-instance-id: 6143df6729b7
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
bruh_lab3-api-5 bruh-lab3-api:local "python -m messenger" api About a minute ago Up About a minute (healthy) 8080/tcp
|
||||
bruh_lab3-api-6 bruh-lab3-api:local "python -m messenger" api About a minute ago Up About a minute (healthy) 8080/tcp
|
||||
bruh_lab3-lb-1 haproxy:3.2.12-alpine "docker-entrypoint.s…" lb 2 minutes ago Up 2 minutes (healthy) 127.0.0.1:8081->8080/tcp
|
||||
bruh_lab3-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 12 minutes ago Up 2 minutes (healthy) 5432/tcp
|
||||
bruh_lab3-redis-1 redis:7.4.6-bookworm "docker-entrypoint.s…" redis 12 minutes ago Up 2 minutes (healthy) 6379/tcp
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"mode": "race",
|
||||
"initial_replicas": [
|
||||
{
|
||||
"id": "120f4ffe45f4b098e6dc30d589050cb6e0c3e5a6cff2221d668b9e1e6eecf51e",
|
||||
"alias": "120f4ffe45f4",
|
||||
"name": "/bruh_lab3-api-3",
|
||||
"image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5",
|
||||
"status": "running"
|
||||
},
|
||||
{
|
||||
"id": "2c9c28e81f35433136326442eeac3328e5f421933d7caa4e544fc5a728cf6d6e",
|
||||
"alias": "2c9c28e81f35",
|
||||
"name": "/bruh_lab3-api-4",
|
||||
"image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5",
|
||||
"status": "running"
|
||||
}
|
||||
],
|
||||
"requests": 8,
|
||||
"replicas": [
|
||||
"120f4ffe45f4",
|
||||
"2c9c28e81f35"
|
||||
],
|
||||
"original_messages": 1,
|
||||
"different_body_status": 409,
|
||||
"same_key_for_another_sender": 201
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"mode": "rolling",
|
||||
"initial_replicas": [
|
||||
{
|
||||
"id": "120f4ffe45f4b098e6dc30d589050cb6e0c3e5a6cff2221d668b9e1e6eecf51e",
|
||||
"alias": "120f4ffe45f4",
|
||||
"name": "/bruh_lab3-api-3",
|
||||
"image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5",
|
||||
"status": "running"
|
||||
},
|
||||
{
|
||||
"id": "2c9c28e81f35433136326442eeac3328e5f421933d7caa4e544fc5a728cf6d6e",
|
||||
"alias": "2c9c28e81f35",
|
||||
"name": "/bruh_lab3-api-4",
|
||||
"image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5",
|
||||
"status": "running"
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"seconds": 2.0,
|
||||
"successful_writes": 58,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 29.0,
|
||||
"latency_ms": {
|
||||
"p50": 8.64,
|
||||
"p95": 22.53,
|
||||
"p99": 24.44
|
||||
},
|
||||
"serving_aliases": [
|
||||
"120f4ffe45f4",
|
||||
"2c9c28e81f35"
|
||||
]
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"removed": "/bruh_lab3-api-3",
|
||||
"old_alias": "120f4ffe45f4",
|
||||
"old_image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5",
|
||||
"replacement_alias": "1fc578fde21a",
|
||||
"new_image": "sha256:16ac194a8f89e953ccd7ccdcf93ebb091a034b82eb7460d406443309478f0bc6",
|
||||
"replacement_routed_seconds": 2.999,
|
||||
"first_survivor_write_seconds": 1.143,
|
||||
"seconds": 3.999,
|
||||
"successful_writes": 87,
|
||||
"failed_attempts": 13,
|
||||
"successful_rps": 21.75,
|
||||
"latency_ms": {
|
||||
"p50": 10.58,
|
||||
"p95": 26.18,
|
||||
"p99": 32.09
|
||||
},
|
||||
"serving_aliases": [
|
||||
"120f4ffe45f4",
|
||||
"1fc578fde21a",
|
||||
"2c9c28e81f35"
|
||||
]
|
||||
},
|
||||
{
|
||||
"removed": "/bruh_lab3-api-4",
|
||||
"old_alias": "2c9c28e81f35",
|
||||
"old_image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5",
|
||||
"replacement_alias": "6143df6729b7",
|
||||
"new_image": "sha256:16ac194a8f89e953ccd7ccdcf93ebb091a034b82eb7460d406443309478f0bc6",
|
||||
"replacement_routed_seconds": 2.947,
|
||||
"first_survivor_write_seconds": 0.647,
|
||||
"seconds": 3.947,
|
||||
"successful_writes": 87,
|
||||
"failed_attempts": 10,
|
||||
"successful_rps": 22.04,
|
||||
"latency_ms": {
|
||||
"p50": 9.6,
|
||||
"p95": 23.74,
|
||||
"p99": 37.13
|
||||
},
|
||||
"serving_aliases": [
|
||||
"1fc578fde21a",
|
||||
"6143df6729b7"
|
||||
]
|
||||
}
|
||||
],
|
||||
"acknowledged_messages": 311,
|
||||
"stored_messages": 311,
|
||||
"workers": 4,
|
||||
"pause_seconds": 0.12,
|
||||
"warmup_seconds": 1
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
nbthread 1
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
option http-server-close
|
||||
timeout http-request 5s
|
||||
timeout http-keep-alive 5s
|
||||
timeout connect 500ms
|
||||
timeout client 10s
|
||||
timeout server 7s
|
||||
timeout check 700ms
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver embedded 127.0.0.11:53
|
||||
resolve_retries 2
|
||||
timeout resolve 500ms
|
||||
timeout retry 500ms
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend messenger
|
||||
|
||||
backend messenger
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server inter 500ms fastinter 250ms downinter 500ms fall 2 rise 2
|
||||
server-template replica 1-8 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none
|
||||
@@ -0,0 +1,3 @@
|
||||
psycopg[binary]==3.2.10
|
||||
psycopg-pool==3.2.6
|
||||
redis==6.4.0
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Generate local Compose secrets once, without printing them."""
|
||||
|
||||
from pathlib import Path
|
||||
from secrets import token_hex
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
example = (root / ".env.example").read_text(encoding="utf-8")
|
||||
try:
|
||||
with (root / ".env").open("x", encoding="utf-8", newline="\n") as target:
|
||||
target.write(example.replace("POSTGRES_PASSWORD=\n", f"POSTGRES_PASSWORD={token_hex(24)}\n")
|
||||
.replace("CURSOR_SECRET=\n", f"CURSOR_SECRET={token_hex(32)}\n"))
|
||||
except FileExistsError:
|
||||
print("Using the existing .env")
|
||||
else:
|
||||
print("Created local .env")
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Match an API response alias to a container in this Compose project."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import NamedTuple
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class Replica(NamedTuple):
|
||||
id: str
|
||||
alias: str
|
||||
name: str
|
||||
image: str
|
||||
status: str
|
||||
|
||||
|
||||
def docker(*args, timeout=30):
|
||||
return subprocess.run(
|
||||
["docker", *args], cwd=ROOT, text=True, encoding="utf-8",
|
||||
stdout=subprocess.PIPE, check=True, timeout=timeout,
|
||||
).stdout.strip()
|
||||
|
||||
|
||||
def compose(*args, **kwargs):
|
||||
return docker("compose", *args, **kwargs)
|
||||
|
||||
|
||||
def replicas():
|
||||
ids = compose("ps", "-a", "-q", "api").splitlines()
|
||||
if not ids:
|
||||
return []
|
||||
fields = "{{.Id}} {{.Config.Hostname}} {{.Name}} {{.Image}} {{.State.Status}}"
|
||||
return [Replica(*line.split()) for line in docker("inspect", "--format", fields, *ids).splitlines()]
|
||||
|
||||
|
||||
def stop(replica):
|
||||
docker("stop", "--time", "7", replica.id, timeout=9)
|
||||
|
||||
|
||||
def main():
|
||||
target = os.environ.get("TARGET_INSTANCE_ID")
|
||||
running = [replica for replica in replicas() if replica.status == "running"]
|
||||
matched = [replica for replica in running if replica.alias == target]
|
||||
if len(matched) != 1 or len(running) < 2:
|
||||
raise ValueError("TARGET_INSTANCE_ID must match one of at least two running API replicas")
|
||||
replica = matched[0]
|
||||
stop(replica)
|
||||
print(json.dumps({"stopped_container": replica.name, "instance_id": replica.alias}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (ValueError, OSError, subprocess.SubprocessError) as error:
|
||||
print(f"Stop hook failed: {error}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
docker compose --project-directory "%~dp0.." restart postgres redis api lb
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec docker compose restart postgres redis api lb
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
python "%~dp0replicas.py"
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec "${PYTHON:-python3}" "$(dirname "$0")/replicas.py"
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Docker HEALTHCHECK probe: exit 0 if /health/ready answers 200, else 1."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
port = os.environ.get("APP_PORT", "8080")
|
||||
try:
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response:
|
||||
sys.exit(0 if response.status == 200 else 1)
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Replicated messenger for MTUSI Lab 3."""
|
||||
|
||||
__version__ = "3.0.0"
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Run one messenger API replica."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from . import __version__
|
||||
from .config import Config
|
||||
from .http_api import MessengerServer
|
||||
from .limits import Limits
|
||||
from .logs import log
|
||||
from .sessions import Sessions
|
||||
from .store import Store
|
||||
|
||||
|
||||
def main() -> None:
|
||||
config = Config.from_env()
|
||||
limits = Limits()
|
||||
store = Store(config.cursor_secret, config.idempotency_ttl)
|
||||
sessions = Sessions(config.session_ttl)
|
||||
server = MessengerServer((config.host, config.port), store, sessions, limits, config.max_body_bytes)
|
||||
stop_requested = threading.Event()
|
||||
|
||||
def on_signal(signum: int, _frame) -> None:
|
||||
if stop_requested.is_set():
|
||||
return
|
||||
stop_requested.set()
|
||||
server.draining = True # readiness -> 503, new API requests -> 503
|
||||
log("info", "shutdown_started", signal=signal.Signals(signum).name)
|
||||
# shutdown() waits for serve_forever() to return, so it must run in another thread.
|
||||
threading.Thread(target=server.shutdown, name="shutdown", daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, on_signal)
|
||||
signal.signal(signal.SIGINT, on_signal)
|
||||
|
||||
log("info", "started", version=__version__, host=config.host, port=config.port, env=config.env,
|
||||
pid=os.getpid(), instance=server.instance_id, storage="postgres", session_store="redis",
|
||||
max_body_bytes=config.max_body_bytes,
|
||||
**limits.describe())
|
||||
limits.start_sampler()
|
||||
server.serve_forever(poll_interval=0.2)
|
||||
|
||||
stopped_at = time.monotonic()
|
||||
server.server_close() # stop accepting connections: the listening socket is closed
|
||||
finished = server.wait_idle(timeout=config.shutdown_timeout)
|
||||
store.pool.close()
|
||||
sessions.redis.close()
|
||||
log("info" if finished else "warning", "shutdown_complete",
|
||||
in_flight_finished=finished,
|
||||
drain_ms=round((time.monotonic() - stopped_at) * 1000, 1))
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Local configuration from environment variables."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .logs import log
|
||||
|
||||
|
||||
def postgres_options() -> dict:
|
||||
return {
|
||||
"host": os.environ.get("POSTGRES_HOST", "postgres"),
|
||||
"port": env_int("POSTGRES_PORT", 5432, 1, 65535),
|
||||
"dbname": os.environ.get("POSTGRES_DB", "messenger"),
|
||||
"user": os.environ.get("POSTGRES_USER", "messenger"),
|
||||
"password": os.environ.get("POSTGRES_PASSWORD"),
|
||||
"connect_timeout": 2,
|
||||
"options": "-c statement_timeout=3000 -c lock_timeout=1500 -c timezone=UTC",
|
||||
}
|
||||
|
||||
|
||||
def env_int(name: str, default: int, low: int, high: int) -> int:
|
||||
raw = os.environ.get(name, "").strip() or str(default)
|
||||
if not raw.isdigit() or not low <= int(raw) <= high:
|
||||
log("error", "bad_config", variable=name, allowed=f"{low}..{high}")
|
||||
sys.exit(2)
|
||||
return int(raw)
|
||||
|
||||
|
||||
def env_bool(name: str, default: bool) -> bool:
|
||||
raw = os.environ.get(name, "").strip().lower()
|
||||
if not raw:
|
||||
return default
|
||||
if raw in ("1", "true", "yes", "on"):
|
||||
return True
|
||||
if raw in ("0", "false", "no", "off"):
|
||||
return False
|
||||
log("error", "bad_config", variable=name, allowed="0/1")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
host: str
|
||||
port: int
|
||||
env: str
|
||||
shutdown_timeout: int
|
||||
max_body_bytes: int
|
||||
session_ttl: int
|
||||
idempotency_ttl: int
|
||||
cursor_secret: bytes
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "Config":
|
||||
secret = os.environ.get("CURSOR_SECRET", "").encode()
|
||||
if len(secret) < 32:
|
||||
log("error", "bad_config", variable="CURSOR_SECRET", minimum_length=32)
|
||||
sys.exit(2)
|
||||
return cls(
|
||||
host=os.environ.get("APP_HOST", "").strip() or "0.0.0.0",
|
||||
port=env_int("APP_PORT", 8080, 1, 65535),
|
||||
env=os.environ.get("APP_ENV", "lab"),
|
||||
shutdown_timeout=env_int("SHUTDOWN_TIMEOUT_SECONDS", 8, 1, 9),
|
||||
max_body_bytes=env_int("MAX_BODY_BYTES", 64 * 1024, 1024, 16 * 1024 * 1024),
|
||||
session_ttl=env_int("SESSION_TTL_SECONDS", 3600, 1, 31_536_000),
|
||||
idempotency_ttl=env_int("IDEMPOTENCY_TTL_SECONDS", 86400, 1, 31_536_000),
|
||||
cursor_secret=secret,
|
||||
)
|
||||
@@ -0,0 +1,68 @@
|
||||
"""API errors in the common contract format: {"error": {"code", "message", "request_id"}}."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
"""An error that is turned into a JSON error response."""
|
||||
|
||||
def __init__(self, status: int, code: str, message: str, close: bool = False,
|
||||
retry_after: Optional[int] = None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
# close=True: the request body may not have been read, so the connection
|
||||
# must not be reused for another request.
|
||||
self.close = close
|
||||
self.retry_after = retry_after # seconds, sent as the Retry-After header
|
||||
|
||||
|
||||
def bad_request(message: str, close: bool = False) -> ApiError:
|
||||
return ApiError(400, "invalid_request", message, close)
|
||||
|
||||
|
||||
def unauthorized() -> ApiError:
|
||||
return ApiError(401, "unauthorized", "invalid credentials or session")
|
||||
|
||||
|
||||
def not_found(message: str = "resource not found") -> ApiError:
|
||||
return ApiError(404, "not_found", message)
|
||||
|
||||
|
||||
def conflict(message: str) -> ApiError:
|
||||
return ApiError(409, "conflict", message)
|
||||
|
||||
|
||||
def payload_too_large(message: str) -> ApiError:
|
||||
return ApiError(413, "payload_too_large", message, close=True)
|
||||
|
||||
|
||||
def unavailable(message: str) -> ApiError:
|
||||
return ApiError(503, "unavailable", message)
|
||||
|
||||
|
||||
def internal_error() -> ApiError:
|
||||
# Never expose stack traces or internal details to the client.
|
||||
return ApiError(500, "internal_error", "internal server error")
|
||||
|
||||
|
||||
_CODE_BY_STATUS = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
503: "unavailable",
|
||||
}
|
||||
|
||||
|
||||
def code_for_status(status: int) -> str:
|
||||
"""Contract error code for an arbitrary HTTP status (used for low-level HTTP errors)."""
|
||||
if status in _CODE_BY_STATUS:
|
||||
return _CODE_BY_STATUS[status]
|
||||
return "internal_error" if status >= 500 else "invalid_request"
|
||||
@@ -0,0 +1,440 @@
|
||||
"""HTTP layer: routing, session authentication, JSON responses, request logs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import secrets
|
||||
import socket
|
||||
import socketserver
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
from urllib.parse import parse_qs, unquote, urlsplit
|
||||
|
||||
from . import validation as v
|
||||
from .errors import ApiError, bad_request, code_for_status, internal_error, not_found, unauthorized, unavailable
|
||||
from .limits import Limits
|
||||
from .logs import log
|
||||
from .sessions import Sessions, credentials, password_digest, verify_password
|
||||
from .store import Store
|
||||
|
||||
REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
class MessengerServer(ThreadingHTTPServer):
|
||||
"""Thread per connection with bounded admission and graceful shutdown."""
|
||||
|
||||
daemon_threads = True # idle keep-alive connections must not block process exit
|
||||
request_queue_size = 128
|
||||
|
||||
def __init__(self, address: Tuple[str, int], store: Store, sessions: Sessions,
|
||||
limits: Limits, max_body_bytes: int):
|
||||
self.store = store
|
||||
self.sessions = sessions
|
||||
self.limits = limits
|
||||
self.max_body_bytes = max_body_bytes
|
||||
self.instance_id = socket.gethostname()
|
||||
self.draining = False
|
||||
self._in_flight = 0
|
||||
self._connections = 0
|
||||
self._idle = threading.Condition()
|
||||
super().__init__(address, Handler)
|
||||
|
||||
def server_bind(self) -> None:
|
||||
# HTTPServer.server_bind() does a reverse DNS lookup (getfqdn) that can
|
||||
# hang for seconds inside containers; it is not needed for an API.
|
||||
socketserver.TCPServer.server_bind(self)
|
||||
self.server_name, self.server_port = self.server_address[:2]
|
||||
|
||||
# -- connections: one thread each; refuse new ones above MAX_CONNECTIONS --
|
||||
|
||||
def process_request(self, request, client_address) -> None:
|
||||
with self._idle:
|
||||
refuse = self.limits.enabled and self._connections >= self.limits.max_connections
|
||||
if not refuse:
|
||||
self._connections += 1
|
||||
if refuse:
|
||||
_refuse_connection(request, self.instance_id)
|
||||
self.shutdown_request(request)
|
||||
return
|
||||
try:
|
||||
super().process_request(request, client_address)
|
||||
except Exception:
|
||||
with self._idle:
|
||||
self._connections -= 1
|
||||
raise
|
||||
|
||||
def process_request_thread(self, request, client_address) -> None:
|
||||
try:
|
||||
super().process_request_thread(request, client_address)
|
||||
finally:
|
||||
with self._idle:
|
||||
self._connections -= 1
|
||||
|
||||
# -- requests in progress --
|
||||
|
||||
def request_started(self) -> int:
|
||||
with self._idle:
|
||||
self._in_flight += 1
|
||||
return self._in_flight
|
||||
|
||||
def request_finished(self) -> None:
|
||||
with self._idle:
|
||||
self._in_flight -= 1
|
||||
if self._in_flight == 0:
|
||||
self._idle.notify_all()
|
||||
|
||||
def wait_idle(self, timeout: float) -> bool:
|
||||
"""Wait until no request is being processed; False on timeout."""
|
||||
with self._idle:
|
||||
return self._idle.wait_for(lambda: self._in_flight == 0, timeout=max(timeout, 0))
|
||||
|
||||
|
||||
# ----- route handlers --------------------------------------------------------
|
||||
# Each handler gets (request, user, path params, query) and returns (status, body).
|
||||
|
||||
Result = Tuple[int, Optional[Dict[str, Any]]]
|
||||
|
||||
|
||||
def health_live(req: "Handler", user, params, query) -> Result:
|
||||
return 200, {"status": "ok"}
|
||||
|
||||
|
||||
def _refuse_connection(sock, instance_id: str) -> None:
|
||||
"""Answer 503 without starting a thread (connection limit). Runs in the accept loop."""
|
||||
request_id = uuid.uuid4().hex
|
||||
body = json.dumps({"error": {"code": "unavailable", "message": "too many connections, retry later",
|
||||
"request_id": request_id}}).encode()
|
||||
head = (
|
||||
"HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n"
|
||||
f"Content-Length: {len(body)}\r\nX-Request-Id: {request_id}\r\n"
|
||||
f"X-Instance-Id: {instance_id}\r\nRetry-After: 1\r\n"
|
||||
"Connection: close\r\n\r\n"
|
||||
).encode()
|
||||
try:
|
||||
sock.settimeout(0.05)
|
||||
try:
|
||||
sock.recv(65536) # read what already arrived so close() does not reset the connection
|
||||
except OSError:
|
||||
pass
|
||||
sock.sendall(head + body)
|
||||
except OSError:
|
||||
pass
|
||||
log("info", "request", request_id=request_id, method="-", route="-", status=503,
|
||||
duration_ms=0.0, rejected="connections")
|
||||
|
||||
|
||||
def health_ready(req: "Handler", user, params, query) -> Result:
|
||||
if req.server.draining:
|
||||
raise unavailable("server is shutting down")
|
||||
if not req.server.limits.ready():
|
||||
raise unavailable("memory limit reached")
|
||||
req.server.store.ready()
|
||||
req.server.sessions.ready()
|
||||
return 200, {"status": "ok"}
|
||||
|
||||
|
||||
def create_user(req: "Handler", user, params, query) -> Result:
|
||||
fields = ("username", "display_name", "password")
|
||||
data = req.json_body(allowed=fields, required=fields)
|
||||
username = v.username_field(data)
|
||||
display_name = v.text_field(data, "display_name", 100)
|
||||
password = data["password"]
|
||||
if not v.valid_password(password):
|
||||
raise bad_request("password must contain 12..128 valid nonblank characters")
|
||||
salt = secrets.token_bytes(16)
|
||||
return 201, req.server.store.create_user(username, display_name, salt, password_digest(password, salt))
|
||||
|
||||
|
||||
def create_session(req: "Handler", user, params, query) -> Result:
|
||||
if req._read_body():
|
||||
raise bad_request("login does not accept a body")
|
||||
username, password = credentials(req.authorization("basic"))
|
||||
user = req.server.store.credentials(username)
|
||||
if not verify_password(password, user):
|
||||
raise unauthorized()
|
||||
del user["password_hash"], user["password_salt"]
|
||||
token, expires_at = req.server.sessions.create(user["id"])
|
||||
return 201, {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}
|
||||
|
||||
|
||||
def revoke_session(req: "Handler", user, params, query) -> Result:
|
||||
req.server.sessions.revoke(req.authorization("bearer"))
|
||||
return 204, None
|
||||
|
||||
|
||||
def get_me(req: "Handler", user, params, query) -> Result:
|
||||
profile = req.server.store.get_user(user)
|
||||
if profile is None:
|
||||
raise unauthorized()
|
||||
return 200, profile
|
||||
|
||||
|
||||
def create_chat(req: "Handler", user, params, query) -> Result:
|
||||
req.server.limits.check_rate(user) # 429 if this user writes too fast
|
||||
data = req.json_body(allowed=("title", "member_ids"), required=("title", "member_ids"))
|
||||
title = v.text_field(data, "title", 100)
|
||||
members = v.member_ids_field(data)
|
||||
return 201, req.server.store.create_chat(user, title, members)
|
||||
|
||||
|
||||
def list_chats(req: "Handler", user, params, query) -> Result:
|
||||
limit, cursor = v.limit_param(query), v.cursor_param(query)
|
||||
return 200, req.server.store.list_chats(user, limit, cursor)
|
||||
|
||||
|
||||
def get_chat(req: "Handler", user, params, query) -> Result:
|
||||
return 200, req.server.store.get_chat(user, params["chat_id"])
|
||||
|
||||
|
||||
def post_message(req: "Handler", user, params, query) -> Result:
|
||||
keys = req.headers.get_all("Idempotency-Key") or []
|
||||
if len(keys) != 1 or IDEMPOTENCY_KEY_RE.fullmatch(keys[0]) is None:
|
||||
raise bad_request("Idempotency-Key must be a single value matching [A-Za-z0-9._:-]{1,128}")
|
||||
req.server.limits.check_rate(user) # 429 if this user writes too fast
|
||||
# sender_id is not accepted in the body: the sender is always the current user.
|
||||
data = req.json_body(allowed=("text",), required=("text",))
|
||||
text = v.text_field(data, "text", 4000)
|
||||
return 201, req.server.store.add_message(user, params["chat_id"], keys[0], text)
|
||||
|
||||
|
||||
def list_messages(req: "Handler", user, params, query) -> Result:
|
||||
limit, cursor = v.limit_param(query), v.cursor_param(query)
|
||||
return 200, req.server.store.list_messages(user, params["chat_id"], limit, cursor)
|
||||
|
||||
|
||||
def json_value(value):
|
||||
if isinstance(value, uuid.UUID):
|
||||
return str(value)
|
||||
if isinstance(value, datetime):
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
raise TypeError(f"Cannot serialize {type(value).__name__}")
|
||||
|
||||
|
||||
Route = Tuple[str, "re.Pattern[str]", str, Callable[..., Result], bool]
|
||||
|
||||
_CHAT = r"(?P<chat_id>[^/]+)"
|
||||
ROUTES: List[Route] = [
|
||||
# method, path regex, template for logs, handler, requires Bearer
|
||||
("GET", re.compile(r"/health/live"), "/health/live", health_live, False),
|
||||
("GET", re.compile(r"/health/ready"), "/health/ready", health_ready, False),
|
||||
("POST", re.compile(r"/api/v1/users"), "/api/v1/users", create_user, False),
|
||||
("POST", re.compile(r"/api/v1/auth/sessions"), "/api/v1/auth/sessions", create_session, False),
|
||||
("DELETE", re.compile(r"/api/v1/auth/sessions/current"), "/api/v1/auth/sessions/current", revoke_session, False),
|
||||
("GET", re.compile(r"/api/v1/users/me"), "/api/v1/users/me", get_me, True),
|
||||
("POST", re.compile(r"/api/v1/chats"), "/api/v1/chats", create_chat, True),
|
||||
("GET", re.compile(r"/api/v1/chats"), "/api/v1/chats", list_chats, True),
|
||||
("GET", re.compile(r"/api/v1/chats/" + _CHAT), "/api/v1/chats/{chat_id}", get_chat, True),
|
||||
("POST", re.compile(r"/api/v1/chats/" + _CHAT + "/messages"), "/api/v1/chats/{chat_id}/messages", post_message, True),
|
||||
("GET", re.compile(r"/api/v1/chats/" + _CHAT + "/messages"), "/api/v1/chats/{chat_id}/messages", list_messages, True),
|
||||
]
|
||||
|
||||
|
||||
def resolve(method: str, path: str) -> Tuple[Route, Dict[str, str]]:
|
||||
path_matched = False
|
||||
for route in ROUTES:
|
||||
match = route[1].fullmatch(path)
|
||||
if match is None:
|
||||
continue
|
||||
path_matched = True
|
||||
if route[0] == method or (method == "HEAD" and route[0] == "GET"):
|
||||
return route, match.groupdict()
|
||||
if path_matched:
|
||||
raise ApiError(405, "invalid_request", "method not allowed")
|
||||
raise not_found("route not found")
|
||||
|
||||
|
||||
# ----- HTTP handler ----------------------------------------------------------
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server: MessengerServer
|
||||
protocol_version = "HTTP/1.1"
|
||||
server_version = "mtusi-messenger"
|
||||
sys_version = ""
|
||||
timeout = 30 # idle keep-alive connections are closed after this many seconds
|
||||
# TCP_NODELAY: headers and body are written separately; with Nagle's algorithm plus the
|
||||
# client's delayed ACK every keep-alive response waited ~40 ms (found in the track 2 runs).
|
||||
disable_nagle_algorithm = True
|
||||
|
||||
request_id = ""
|
||||
route_template = "-"
|
||||
started = 0.0
|
||||
_body: Optional[bytes] = None
|
||||
|
||||
def do_GET(self) -> None:
|
||||
self._handle()
|
||||
|
||||
def do_POST(self) -> None:
|
||||
self._handle()
|
||||
|
||||
def do_PUT(self) -> None:
|
||||
self._handle()
|
||||
|
||||
def do_PATCH(self) -> None:
|
||||
self._handle()
|
||||
|
||||
def do_DELETE(self) -> None:
|
||||
self._handle()
|
||||
|
||||
def do_HEAD(self) -> None:
|
||||
self._handle()
|
||||
|
||||
def do_OPTIONS(self) -> None:
|
||||
self._handle()
|
||||
|
||||
# -- request processing --
|
||||
|
||||
def _handle(self) -> None:
|
||||
self.started = time.monotonic()
|
||||
incoming = self.headers.get("X-Request-Id", "")
|
||||
self.request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
|
||||
self.route_template = "-"
|
||||
self._body: Optional[bytes] = None
|
||||
self.server.request_started()
|
||||
slot = False
|
||||
try:
|
||||
if self.command in ("POST", "PUT", "PATCH", "DELETE"):
|
||||
# Read the body first: an early error then never leaves unread bytes
|
||||
# on the connection, so keep-alive works even for 429/503 answers.
|
||||
self._read_body()
|
||||
split = urlsplit(self.path)
|
||||
route, params = resolve(self.command, split.path)
|
||||
self.route_template = route[2]
|
||||
is_health = route[2] in ("/health/live", "/health/ready")
|
||||
if self.server.draining and not is_health:
|
||||
raise unavailable("server is shutting down")
|
||||
if not is_health:
|
||||
slot = self.server.limits.acquire_slot() # bounded queue; 503 when overloaded
|
||||
user = self.server.sessions.identify(self.authorization("bearer")) if route[4] else None
|
||||
if "chat_id" in params:
|
||||
params["chat_id"] = v.parse_uuid(unquote(params["chat_id"]), "chat_id")
|
||||
query = parse_qs(split.query, keep_blank_values=True)
|
||||
status, payload = route[3](self, user, params, query)
|
||||
self._send_json(status, payload)
|
||||
except ApiError as error:
|
||||
if error.close:
|
||||
self.close_connection = True
|
||||
self._safe_send_error(error)
|
||||
except (BrokenPipeError, ConnectionResetError, TimeoutError):
|
||||
self.close_connection = True # the client went away; nothing to answer
|
||||
except Exception as error:
|
||||
log("error", "unhandled_exception", request_id=self.request_id,
|
||||
route=self.route_template, exception_type=type(error).__name__)
|
||||
self.close_connection = True
|
||||
self._safe_send_error(internal_error())
|
||||
finally:
|
||||
if slot:
|
||||
self.server.limits.release_slot()
|
||||
self.server.request_finished()
|
||||
|
||||
def _safe_send_error(self, error: ApiError) -> None:
|
||||
try:
|
||||
self._send_error_json(error)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
self.close_connection = True
|
||||
|
||||
def authorization(self, scheme: str) -> str:
|
||||
values = self.headers.get_all("Authorization") or []
|
||||
if len(values) != 1:
|
||||
raise unauthorized()
|
||||
parts = values[0].split()
|
||||
if len(parts) != 2 or parts[0].lower() != scheme:
|
||||
raise unauthorized()
|
||||
return parts[1]
|
||||
|
||||
def _read_body(self) -> bytes:
|
||||
if self._body is not None:
|
||||
return self._body
|
||||
if self.headers.get("Transfer-Encoding"):
|
||||
raise bad_request("chunked request bodies are not supported", close=True)
|
||||
lengths = self.headers.get_all("Content-Length") or []
|
||||
if not lengths:
|
||||
self._body = b""
|
||||
return self._body
|
||||
if len(lengths) != 1 or not re.fullmatch(r"[0-9]{1,16}", lengths[0]):
|
||||
raise bad_request("invalid Content-Length", close=True)
|
||||
length = int(lengths[0])
|
||||
if length > self.server.max_body_bytes:
|
||||
raise ApiError(413, "payload_too_large", "request body is too large", close=True)
|
||||
self._body = self.rfile.read(length)
|
||||
if len(self._body) != length:
|
||||
raise bad_request("incomplete request body", close=True)
|
||||
return self._body
|
||||
|
||||
def json_body(self, allowed, required) -> Dict[str, Any]:
|
||||
raw = self._read_body()
|
||||
if self.headers.get_content_type() != "application/json":
|
||||
raise bad_request("Content-Type must be application/json")
|
||||
charset = self.headers.get_content_charset()
|
||||
if charset not in (None, "utf-8"):
|
||||
raise bad_request("request body must be UTF-8")
|
||||
return v.parse_json_object(raw, allowed, required)
|
||||
|
||||
# -- responses --
|
||||
|
||||
def _send_json(self, status: int, payload: Optional[Dict[str, Any]], retry_after: Optional[int] = None) -> None:
|
||||
body = b"" if payload is None else json.dumps(
|
||||
payload, ensure_ascii=False, separators=(",", ":"), default=json_value
|
||||
).encode("utf-8")
|
||||
if self.command in ("POST", "PUT", "PATCH", "DELETE") and self._body is None:
|
||||
# The body was never read (e.g. error before parsing): do not reuse the connection.
|
||||
self.close_connection = True
|
||||
if self.server.draining:
|
||||
self.close_connection = True
|
||||
self.send_response(status)
|
||||
if status != 204:
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("X-Request-Id", self.request_id)
|
||||
self.send_header("X-Instance-Id", self.server.instance_id)
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
if retry_after is not None:
|
||||
self.send_header("Retry-After", str(retry_after))
|
||||
if status == 401:
|
||||
challenge = 'Bearer realm="messenger"'
|
||||
if self.route_template == "/api/v1/auth/sessions":
|
||||
challenge = 'Basic realm="messenger", charset="UTF-8"'
|
||||
self.send_header("WWW-Authenticate", challenge)
|
||||
if status == 405:
|
||||
path = urlsplit(getattr(self, "path", "") or "").path
|
||||
allow = sorted({r[0] for r in ROUTES if r[1].fullmatch(path)})
|
||||
if allow:
|
||||
self.send_header("Allow", ", ".join(allow))
|
||||
if self.close_connection:
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
log("info", "request", request_id=self.request_id, method=self.command,
|
||||
route=self.route_template, status=status,
|
||||
duration_ms=round((time.monotonic() - self.started) * 1000, 2))
|
||||
|
||||
def _send_error_json(self, error: ApiError) -> None:
|
||||
self._send_json(error.status, {
|
||||
"error": {"code": error.code, "message": error.message, "request_id": self.request_id}
|
||||
}, retry_after=error.retry_after)
|
||||
|
||||
def send_error(self, code: int, message: Optional[str] = None, explain: Optional[str] = None) -> None:
|
||||
"""Low-level HTTP errors (bad request line, unsupported method) in the same JSON format."""
|
||||
self.request_id = uuid.uuid4().hex
|
||||
self.started = time.monotonic()
|
||||
self.route_template = "-"
|
||||
self._body = None
|
||||
self.close_connection = True
|
||||
if getattr(self, "command", None) is None:
|
||||
self.command = "GET"
|
||||
status = 405 if code == 501 else code # 501 = method unknown to http.server
|
||||
self._safe_send_error(ApiError(status, code_for_status(status), "invalid HTTP request"))
|
||||
|
||||
def version_string(self) -> str:
|
||||
return self.server_version
|
||||
|
||||
def log_message(self, format: str, *args: Any) -> None: # noqa: A002
|
||||
pass # replaced by structured logs in _send_json
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Bound HTTP connections, concurrent work and writes per user."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Dict, List, Optional
|
||||
from uuid import UUID
|
||||
|
||||
from .config import env_bool, env_int
|
||||
from .errors import ApiError
|
||||
|
||||
try:
|
||||
_PAGE_SIZE = os.sysconf("SC_PAGE_SIZE")
|
||||
except (AttributeError, ValueError, OSError):
|
||||
_PAGE_SIZE = 4096
|
||||
|
||||
|
||||
def read_rss() -> Optional[int]:
|
||||
"""Resident memory of this process in bytes (Linux /proc), None elsewhere."""
|
||||
try:
|
||||
with open("/proc/self/statm") as f:
|
||||
return int(f.read().split()[1]) * _PAGE_SIZE
|
||||
except (OSError, ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def _read_limit(path: str) -> Optional[int]:
|
||||
try:
|
||||
with open(path) as f:
|
||||
raw = f.read().strip()
|
||||
except OSError:
|
||||
return None
|
||||
if not raw.isdigit():
|
||||
return None # cgroup v2 says "max" when there is no limit
|
||||
value = int(raw)
|
||||
return value if value < (1 << 50) else None # cgroup v1 uses a huge number for "no limit"
|
||||
|
||||
|
||||
def detect_memory_limit() -> Optional[int]:
|
||||
"""Memory limit of the container (cgroup v2 or v1), None if unlimited/unknown."""
|
||||
candidates: List[str] = ["/sys/fs/cgroup/memory.max"]
|
||||
try:
|
||||
with open("/proc/self/cgroup") as f:
|
||||
for line in f:
|
||||
_, controllers, path = line.rstrip("\n").split(":", 2)
|
||||
if controllers == "":
|
||||
candidates.append(f"/sys/fs/cgroup{path}/memory.max")
|
||||
elif "memory" in controllers.split(","):
|
||||
candidates.append(f"/sys/fs/cgroup/memory{path}/memory.limit_in_bytes")
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
candidates.append("/sys/fs/cgroup/memory/memory.limit_in_bytes")
|
||||
for path in candidates:
|
||||
value = _read_limit(path)
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
class Limits:
|
||||
def __init__(self) -> None:
|
||||
self.enabled = env_bool("LIMITS_ENABLED", True)
|
||||
# Bounded request queue: MAX_IN_FLIGHT requests are processed at once, up to
|
||||
# MAX_QUEUE more wait for a slot at most QUEUE_TIMEOUT_MS; the rest get 503 at once.
|
||||
self.max_in_flight = env_int("MAX_IN_FLIGHT", 8, 1, 10_000)
|
||||
self.max_queue = env_int("MAX_QUEUE", 64, 0, 100_000)
|
||||
self.queue_timeout = env_int("QUEUE_TIMEOUT_MS", 1000, 1, 60_000) / 1000
|
||||
self._slots = threading.BoundedSemaphore(self.max_in_flight)
|
||||
self.waiting = 0
|
||||
self.max_connections = env_int("MAX_CONNECTIONS", 256, 1, 100_000)
|
||||
self.user_write_rate = env_int("USER_WRITE_RATE", 50, 1, 100_000) # tokens per second
|
||||
self.user_write_burst = env_int("USER_WRITE_BURST", 100, 1, 100_000)
|
||||
configured = env_int("MEMORY_LIMIT_BYTES", 0, 0, 1 << 50)
|
||||
self.memory_limit = configured or detect_memory_limit()
|
||||
self.hard_percent = env_int("MEMORY_HARD_PERCENT", 90, 10, 99)
|
||||
self.rss = read_rss() or 0
|
||||
self._lock = threading.Lock()
|
||||
self._buckets: Dict[UUID, List[float]] = {}
|
||||
|
||||
# ----- description / stats ------------------------------------------------
|
||||
|
||||
def describe(self) -> Dict[str, object]:
|
||||
return {
|
||||
"limits_enabled": self.enabled,
|
||||
"max_in_flight": self.max_in_flight,
|
||||
"max_queue": self.max_queue,
|
||||
"queue_timeout_ms": int(self.queue_timeout * 1000),
|
||||
"max_connections": self.max_connections,
|
||||
"user_write_rate": self.user_write_rate,
|
||||
"user_write_burst": self.user_write_burst,
|
||||
"memory_limit_bytes": self.memory_limit,
|
||||
"memory_hard_percent": self.hard_percent,
|
||||
}
|
||||
|
||||
def _over(self, percent: int) -> bool:
|
||||
return bool(self.enabled and self.memory_limit and self.rss >= self.memory_limit * percent / 100)
|
||||
|
||||
# ----- checks (each raises ApiError when the limit is exceeded) ------------
|
||||
|
||||
def acquire_slot(self) -> bool:
|
||||
"""Admission for every non-health request, before any work is done.
|
||||
|
||||
Returns True when a processing slot was taken (the caller must call
|
||||
release_slot()), False when limits are off. Raises 503 when overloaded.
|
||||
"""
|
||||
if not self.enabled:
|
||||
return False
|
||||
if self._over(self.hard_percent):
|
||||
raise ApiError(503, "unavailable", "memory limit reached, retry later", retry_after=5)
|
||||
if self._slots.acquire(blocking=False):
|
||||
return True
|
||||
with self._lock:
|
||||
queue_full = self.waiting >= self.max_queue
|
||||
if not queue_full:
|
||||
self.waiting += 1
|
||||
if queue_full:
|
||||
raise ApiError(503, "unavailable", "server is overloaded, retry later", retry_after=1)
|
||||
try:
|
||||
acquired = self._slots.acquire(timeout=self.queue_timeout)
|
||||
finally:
|
||||
with self._lock:
|
||||
self.waiting -= 1
|
||||
if not acquired:
|
||||
raise ApiError(503, "unavailable", "server is overloaded, retry later", retry_after=1)
|
||||
return True
|
||||
|
||||
def release_slot(self) -> None:
|
||||
self._slots.release()
|
||||
|
||||
def ready(self) -> bool:
|
||||
return not self._over(self.hard_percent)
|
||||
|
||||
def check_rate(self, user_id: UUID) -> None:
|
||||
"""Token bucket per user for write requests."""
|
||||
if not self.enabled:
|
||||
return
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
bucket = self._buckets.get(user_id)
|
||||
if bucket is None:
|
||||
bucket = self._buckets[user_id] = [float(self.user_write_burst), now]
|
||||
tokens = min(self.user_write_burst, bucket[0] + (now - bucket[1]) * self.user_write_rate)
|
||||
bucket[1] = now
|
||||
if tokens >= 1:
|
||||
bucket[0] = tokens - 1
|
||||
return
|
||||
bucket[0] = tokens
|
||||
wait = max(1, math.ceil((1 - tokens) / self.user_write_rate))
|
||||
raise ApiError(429, "rate_limited", "too many writes, slow down", retry_after=wait)
|
||||
|
||||
# ----- background sampler -------------------------------------------------
|
||||
|
||||
def start_sampler(self) -> None:
|
||||
"""Refresh RSS for request admission and readiness."""
|
||||
|
||||
def loop() -> None:
|
||||
while True:
|
||||
rss = read_rss()
|
||||
if rss is not None:
|
||||
self.rss = rss
|
||||
time.sleep(0.1)
|
||||
|
||||
threading.Thread(target=loop, name="limits-sampler", daemon=True).start()
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Structured logs: one JSON object per line on stdout.
|
||||
|
||||
Logs never contain message texts, request bodies, query strings or headers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def log(level: str, event: str, **fields: Any) -> None:
|
||||
record = {
|
||||
"ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
|
||||
"level": level,
|
||||
"event": event,
|
||||
**fields,
|
||||
}
|
||||
line = json.dumps(record, ensure_ascii=False, default=str)
|
||||
with _lock:
|
||||
sys.stdout.write(line + "\n")
|
||||
sys.stdout.flush()
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Apply versioned migrations under one PostgreSQL transaction and lock."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from .config import postgres_options
|
||||
|
||||
|
||||
def main() -> None:
|
||||
migrations = Path(__file__).with_name("migrations").glob("*.sql")
|
||||
with psycopg.connect(**postgres_options()) as conn:
|
||||
conn.execute("SELECT pg_advisory_xact_lock(202602)")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
)
|
||||
""")
|
||||
applied = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
pending = []
|
||||
for path in sorted(migrations):
|
||||
if path.stem not in applied:
|
||||
conn.execute(path.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.stem,))
|
||||
pending.append(path.stem)
|
||||
print("Applied: " + ", ".join(pending) if pending else "Schema is up to date")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,34 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash bytea NOT NULL,
|
||||
password_salt bytea NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
position integer NOT NULL CHECK (position BETWEEN 1 AND 100),
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
UNIQUE (chat_id, position)
|
||||
);
|
||||
CREATE INDEX chat_members_by_user ON chat_members (user_id, chat_id);
|
||||
CREATE INDEX chats_order ON chats (created_at, id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX messages_history ON messages (chat_id, created_at, id);
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE message_requests (
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
idem_key text NOT NULL CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED,
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (chat_id, sender_id, idem_key)
|
||||
);
|
||||
|
||||
CREATE INDEX message_requests_expiry ON message_requests (expires_at);
|
||||
@@ -0,0 +1,45 @@
|
||||
"""The lab 1 signed cursor with a stable key and a PostgreSQL sort position."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from .errors import bad_request
|
||||
|
||||
|
||||
class Cursors:
|
||||
def __init__(self, secret: bytes):
|
||||
self.secret = secret
|
||||
|
||||
def _sign(self, payload: bytes) -> bytes:
|
||||
return hmac.new(self.secret, payload, hashlib.sha256).digest()[:16]
|
||||
|
||||
def decode(self, scope: str, cursor: str) -> tuple[datetime, UUID]:
|
||||
try:
|
||||
padded = cursor + "=" * (-len(cursor) % 4)
|
||||
raw = base64.b64decode(padded, altchars=b"-_", validate=True)
|
||||
except (binascii.Error, ValueError):
|
||||
raise bad_request("invalid cursor") from None
|
||||
payload, signature = raw[:-16], raw[-16:]
|
||||
if not payload or not hmac.compare_digest(signature, self._sign(payload)):
|
||||
raise bad_request("invalid cursor")
|
||||
cursor_scope, timestamp, object_id = json.loads(payload)
|
||||
if cursor_scope != scope:
|
||||
raise bad_request("cursor belongs to another resource")
|
||||
return datetime.fromisoformat(timestamp), UUID(object_id)
|
||||
|
||||
def page(self, scope: str, items: list[dict], limit: int) -> dict:
|
||||
cursor = None
|
||||
if len(items) > limit:
|
||||
items = items[:limit]
|
||||
last = items[-1]
|
||||
payload = json.dumps(
|
||||
[scope, last["created_at"].isoformat(), str(last["id"])],
|
||||
separators=(",", ":"),
|
||||
).encode()
|
||||
cursor = base64.urlsafe_b64encode(payload + self._sign(payload)).decode().rstrip("=")
|
||||
return {"items": items, "next_cursor": cursor}
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Basic credentials, scrypt password hashes and opaque Redis sessions."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID
|
||||
|
||||
from redis import Redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.exceptions import RedisError
|
||||
from redis.retry import Retry
|
||||
|
||||
from .errors import unauthorized, unavailable
|
||||
from .validation import USERNAME_RE, valid_password
|
||||
|
||||
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{43}")
|
||||
|
||||
|
||||
def credentials(header: str) -> tuple[str, str]:
|
||||
try:
|
||||
decoded = base64.b64decode(header, validate=True).decode("utf-8")
|
||||
except (binascii.Error, ValueError):
|
||||
raise unauthorized() from None
|
||||
username, _, password = decoded.partition(":")
|
||||
if not USERNAME_RE.fullmatch(username) or not valid_password(password):
|
||||
raise unauthorized()
|
||||
return username, password
|
||||
|
||||
|
||||
def password_digest(password: str, salt: bytes) -> bytes:
|
||||
return hashlib.scrypt(
|
||||
password.encode("utf-8"), salt=salt, n=32768, r=8, p=3, dklen=32,
|
||||
maxmem=64 * 1024 * 1024,
|
||||
)
|
||||
|
||||
|
||||
def verify_password(password: str, user: dict | None) -> bool:
|
||||
# Do equivalent KDF work for unknown usernames.
|
||||
salt, expected = (user["password_salt"], user["password_hash"]) if user else (bytes(16), bytes(32))
|
||||
match = hmac.compare_digest(password_digest(password, salt), expected)
|
||||
return user is not None and match
|
||||
|
||||
|
||||
@contextmanager
|
||||
def redis_operation():
|
||||
try:
|
||||
yield
|
||||
except RedisError:
|
||||
raise unavailable("Redis unavailable") from None
|
||||
|
||||
|
||||
class Sessions:
|
||||
def __init__(self, ttl: int):
|
||||
self.ttl = ttl
|
||||
self.redis = Redis.from_url(
|
||||
os.environ.get("REDIS_URL", "redis://redis:6379/0"),
|
||||
socket_connect_timeout=1, socket_timeout=1, max_connections=16,
|
||||
retry=Retry(NoBackoff(), 0), decode_responses=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def key(token: str) -> str:
|
||||
if not TOKEN_RE.fullmatch(token):
|
||||
raise unauthorized()
|
||||
return "lab2:session:" + hashlib.sha256(token.encode("ascii")).hexdigest()
|
||||
|
||||
def ready(self) -> None:
|
||||
with redis_operation():
|
||||
self.redis.ping()
|
||||
|
||||
def create(self, user_id: UUID) -> tuple[str, datetime]:
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_ms = time.time_ns() // 1_000_000 + self.ttl * 1000
|
||||
with redis_operation():
|
||||
if not self.redis.set(self.key(token), str(user_id), pxat=expires_ms, nx=True):
|
||||
raise unavailable("could not create session")
|
||||
return token, datetime.fromtimestamp(expires_ms / 1000, tz=timezone.utc)
|
||||
|
||||
def identify(self, token: str) -> UUID:
|
||||
with redis_operation():
|
||||
user_id = self.redis.get(self.key(token))
|
||||
if user_id is None:
|
||||
raise unauthorized()
|
||||
return UUID(user_id)
|
||||
|
||||
def revoke(self, token: str) -> None:
|
||||
with redis_operation():
|
||||
if self.redis.delete(self.key(token)) == 0:
|
||||
raise unauthorized()
|
||||
@@ -0,0 +1,167 @@
|
||||
"""PostgreSQL storage for users, chats, membership and message history."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from psycopg import OperationalError
|
||||
from psycopg.errors import ForeignKeyViolation, LockNotAvailable, QueryCanceled
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, TooManyRequests
|
||||
|
||||
from .config import postgres_options
|
||||
from .errors import bad_request, conflict, not_found, unavailable
|
||||
from .pagination import Cursors
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, cursor_secret: bytes, idempotency_ttl: int):
|
||||
self.cursors = Cursors(cursor_secret)
|
||||
self.idempotency_ttl = idempotency_ttl
|
||||
self.pool = ConnectionPool(
|
||||
kwargs={**postgres_options(), "row_factory": dict_row},
|
||||
min_size=1, max_size=8, max_waiting=16, timeout=2, open=False,
|
||||
)
|
||||
self.pool.open()
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
try:
|
||||
# Commits on success and rolls back on any exception before reuse.
|
||||
with self.pool.connection() as conn:
|
||||
yield conn
|
||||
except (OperationalError, TooManyRequests, LockNotAvailable, QueryCanceled):
|
||||
raise unavailable("PostgreSQL unavailable") from None
|
||||
|
||||
def ready(self) -> None:
|
||||
with self.connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
|
||||
def create_user(self, username: str, display_name: str, salt: bytes, password_hash: bytes) -> dict:
|
||||
with self.connection() as conn:
|
||||
user = conn.execute("""
|
||||
INSERT INTO users (username, display_name, password_salt, password_hash)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
ON CONFLICT (username) DO NOTHING
|
||||
RETURNING id, username, display_name, created_at
|
||||
""", (username, display_name, salt, password_hash)).fetchone()
|
||||
if user is None:
|
||||
raise conflict("username is already taken")
|
||||
return user
|
||||
|
||||
def get_user(self, user_id: UUID) -> dict | None:
|
||||
with self.connection() as conn:
|
||||
return conn.execute("""
|
||||
SELECT id, username, display_name, created_at FROM users WHERE id = %s
|
||||
""", (user_id,)).fetchone()
|
||||
|
||||
def credentials(self, username: str) -> dict | None:
|
||||
with self.connection() as conn:
|
||||
return conn.execute("""
|
||||
SELECT id, username, display_name, created_at, password_salt, password_hash
|
||||
FROM users WHERE username = %s
|
||||
""", (username,)).fetchone()
|
||||
|
||||
def create_chat(self, user_id: UUID, title: str, member_ids: list[UUID]) -> dict:
|
||||
if user_id not in member_ids:
|
||||
raise bad_request("member_ids must include the current user")
|
||||
try:
|
||||
with self.connection() as conn:
|
||||
chat = conn.execute("""
|
||||
INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at
|
||||
""", (title,)).fetchone()
|
||||
conn.execute("""
|
||||
INSERT INTO chat_members (chat_id, user_id, position)
|
||||
SELECT %s, user_id, position
|
||||
FROM unnest(%s::uuid[]) WITH ORDINALITY AS members(user_id, position)
|
||||
""", (chat["id"], member_ids))
|
||||
except ForeignKeyViolation:
|
||||
raise bad_request("member_ids contains an unknown user") from None
|
||||
return {**chat, "member_ids": member_ids}
|
||||
|
||||
def get_chat(self, user_id: UUID, chat_id: UUID) -> dict:
|
||||
with self.connection() as conn:
|
||||
chat = conn.execute("""
|
||||
SELECT c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT m.user_id FROM chat_members m
|
||||
WHERE m.chat_id = c.id ORDER BY m.position) AS member_ids
|
||||
FROM chat_members member JOIN chats c ON c.id = member.chat_id
|
||||
WHERE member.user_id = %s AND c.id = %s
|
||||
""", (user_id, chat_id)).fetchone()
|
||||
if chat is None:
|
||||
raise not_found("chat not found")
|
||||
return chat
|
||||
|
||||
def list_chats(self, user_id: UUID, limit: int, cursor: str | None) -> dict:
|
||||
scope = f"chats:{user_id}"
|
||||
query = """
|
||||
SELECT c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT m.user_id FROM chat_members m
|
||||
WHERE m.chat_id = c.id ORDER BY m.position) AS member_ids
|
||||
FROM chat_members member JOIN chats c ON c.id = member.chat_id
|
||||
WHERE member.user_id = %s
|
||||
"""
|
||||
params = (user_id,)
|
||||
if cursor is not None:
|
||||
query += " AND (c.created_at, c.id) > (%s, %s)"
|
||||
params += self.cursors.decode(scope, cursor)
|
||||
with self.connection() as conn:
|
||||
items = conn.execute(
|
||||
query + " ORDER BY c.created_at, c.id LIMIT %s", (*params, limit + 1)
|
||||
).fetchall()
|
||||
return self.cursors.page(scope, items, limit)
|
||||
|
||||
def add_message(self, user_id: UUID, chat_id: UUID, key: str, text: str) -> dict:
|
||||
scope = (chat_id, user_id, key)
|
||||
message_id = uuid4()
|
||||
with self.connection() as conn:
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)
|
||||
).fetchone()
|
||||
if member is None:
|
||||
raise not_found("chat not found")
|
||||
|
||||
# PostgreSQL serializes contenders on the same primary key. The
|
||||
# deferred FK allows reserving the key before inserting the message.
|
||||
claimed = conn.execute("""
|
||||
INSERT INTO message_requests (chat_id, sender_id, idem_key, message_id, expires_at)
|
||||
VALUES (%s, %s, %s, %s, clock_timestamp() + make_interval(secs => %s))
|
||||
ON CONFLICT (chat_id, sender_id, idem_key) DO UPDATE SET
|
||||
message_id = EXCLUDED.message_id,
|
||||
expires_at = clock_timestamp() + make_interval(secs => %s)
|
||||
WHERE message_requests.expires_at <= clock_timestamp()
|
||||
RETURNING message_id
|
||||
""", (*scope, message_id, self.idempotency_ttl, self.idempotency_ttl)).fetchone()
|
||||
if claimed is not None:
|
||||
return conn.execute("""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
RETURNING id, chat_id, sender_id, text, created_at
|
||||
""", (message_id, chat_id, user_id, text)).fetchone()
|
||||
|
||||
# A conflicting live row stays locked until we finish reading its result.
|
||||
message = conn.execute("""
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at
|
||||
FROM message_requests r JOIN messages m ON m.id = r.message_id
|
||||
WHERE r.chat_id = %s AND r.sender_id = %s AND r.idem_key = %s
|
||||
""", scope).fetchone()
|
||||
if message["text"] != text:
|
||||
raise conflict("Idempotency-Key already used for a different message")
|
||||
return message
|
||||
|
||||
def list_messages(self, user_id: UUID, chat_id: UUID, limit: int, cursor: str | None) -> dict:
|
||||
scope = f"messages:{chat_id}"
|
||||
query = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = %s"
|
||||
params = (chat_id,)
|
||||
with self.connection() as conn:
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)
|
||||
).fetchone()
|
||||
if member is None:
|
||||
raise not_found("chat not found")
|
||||
if cursor is not None:
|
||||
query += " AND (created_at, id) > (%s, %s)"
|
||||
params += self.cursors.decode(scope, cursor)
|
||||
items = conn.execute(
|
||||
query + " ORDER BY created_at, id LIMIT %s", (*params, limit + 1)
|
||||
).fetchall()
|
||||
return self.cursors.page(scope, items, limit)
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Strict request validation: bodies, fields, path and query parameters."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
from uuid import UUID
|
||||
|
||||
from .errors import bad_request
|
||||
|
||||
UUID_RE = re.compile(
|
||||
r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
|
||||
)
|
||||
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||
DIGITS_RE = re.compile(r"[0-9]{1,3}")
|
||||
|
||||
DEFAULT_LIMIT = 50
|
||||
MAX_LIMIT = 100
|
||||
MAX_CURSOR_LENGTH = 512
|
||||
|
||||
|
||||
def is_uuid(value: Any) -> bool:
|
||||
return isinstance(value, str) and UUID_RE.fullmatch(value) is not None
|
||||
|
||||
|
||||
def parse_uuid(value: Any, what: str) -> UUID:
|
||||
if not is_uuid(value):
|
||||
raise bad_request(f"{what} must be a UUID")
|
||||
return UUID(value)
|
||||
|
||||
|
||||
def _reject_constant(_name: str) -> Any:
|
||||
raise ValueError("NaN/Infinity are not valid JSON")
|
||||
|
||||
|
||||
def _object_without_duplicates(pairs: List[tuple]) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError("duplicate key in JSON object")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def parse_json_object(raw: bytes, allowed: Iterable[str], required: Iterable[str]) -> Dict[str, Any]:
|
||||
"""Parse a JSON object body. Malformed JSON, extra or missing fields -> 400."""
|
||||
try:
|
||||
data = json.loads(
|
||||
raw.decode("utf-8"),
|
||||
object_pairs_hook=_object_without_duplicates,
|
||||
parse_constant=_reject_constant,
|
||||
)
|
||||
except (UnicodeDecodeError, ValueError, RecursionError):
|
||||
raise bad_request("request body must be valid UTF-8 JSON") from None
|
||||
if not isinstance(data, dict):
|
||||
raise bad_request("request body must be a JSON object")
|
||||
allowed_set = set(allowed)
|
||||
if set(data) - allowed_set:
|
||||
raise bad_request("request body contains unexpected fields")
|
||||
missing = [name for name in required if name not in data]
|
||||
if missing:
|
||||
raise bad_request("missing required fields: " + ", ".join(missing))
|
||||
return data
|
||||
|
||||
|
||||
def valid_text(value: Any, minimum: int, maximum: int) -> bool:
|
||||
if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip():
|
||||
return False
|
||||
try:
|
||||
value.encode("utf-8") # rejects lone surrogates such as "\\ud800"
|
||||
except UnicodeEncodeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def valid_password(value: Any) -> bool:
|
||||
return valid_text(value, 12, 128)
|
||||
|
||||
|
||||
def text_field(data: Dict[str, Any], name: str, max_length: int) -> str:
|
||||
value = data[name]
|
||||
if not valid_text(value, 1, max_length) or "\x00" in value:
|
||||
raise bad_request(f"{name} must contain 1..{max_length} valid nonblank characters")
|
||||
return value
|
||||
|
||||
|
||||
def username_field(data: Dict[str, Any]) -> str:
|
||||
value = data["username"]
|
||||
if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None:
|
||||
raise bad_request("username must match [a-z0-9_]{3,32}")
|
||||
return value
|
||||
|
||||
|
||||
def member_ids_field(data: Dict[str, Any]) -> List[UUID]:
|
||||
value = data["member_ids"]
|
||||
if not isinstance(value, list) or not 1 <= len(value) <= 100:
|
||||
raise bad_request("member_ids must be an array of 1..100 user ids")
|
||||
members = [parse_uuid(item, "member_ids item") for item in value]
|
||||
if len(set(members)) != len(members):
|
||||
raise bad_request("member_ids must not contain duplicates")
|
||||
return members
|
||||
|
||||
|
||||
def _single(params: Dict[str, List[str]], name: str) -> Optional[str]:
|
||||
values = params.get(name)
|
||||
if values is None:
|
||||
return None
|
||||
if len(values) != 1:
|
||||
raise bad_request(f"{name} must be given at most once")
|
||||
return values[0]
|
||||
|
||||
|
||||
def limit_param(params: Dict[str, List[str]]) -> int:
|
||||
raw = _single(params, "limit")
|
||||
if raw is None:
|
||||
return DEFAULT_LIMIT
|
||||
if DIGITS_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT:
|
||||
raise bad_request(f"limit must be an integer 1..{MAX_LIMIT}")
|
||||
return int(raw)
|
||||
|
||||
|
||||
def cursor_param(params: Dict[str, List[str]]) -> Optional[str]:
|
||||
raw = _single(params, "cursor")
|
||||
if raw is None:
|
||||
return None
|
||||
if not raw or len(raw) > MAX_CURSOR_LENGTH:
|
||||
raise bad_request("invalid cursor")
|
||||
return raw
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Measure replica failures and replacement while clients write through HAProxy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from http.client import HTTPException
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
from threading import Barrier, Event, Lock
|
||||
import time
|
||||
from typing import NamedTuple
|
||||
from uuid import uuid4
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
from replicas import compose, docker, replicas, stop
|
||||
from resilience import history
|
||||
|
||||
WORKERS = 4
|
||||
PAUSE_SECONDS = 0.12
|
||||
WARMUP_SECONDS = 1
|
||||
BASELINE_SECONDS = 2
|
||||
OUTAGE_SECONDS = 6
|
||||
|
||||
|
||||
def wait_routed(client, actor, expected):
|
||||
seen = set()
|
||||
deadline = time.monotonic() + 25
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
response = client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
except Failure:
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
check(response["json"] == actor["user"], "session resolved to the wrong user")
|
||||
seen.add(response["headers"]["X-Instance-Id"])
|
||||
if expected <= seen:
|
||||
return
|
||||
time.sleep(0.05)
|
||||
raise Failure(f"LB did not route the shared session to expected replicas: {sorted(expected - seen)}")
|
||||
|
||||
|
||||
def race(client, actor, other, chat):
|
||||
key = uuid4().hex
|
||||
text = "parallel replay " + key
|
||||
count = 8
|
||||
barrier = Barrier(count, timeout=10)
|
||||
|
||||
def send(_):
|
||||
worker = Client(client.base, timeout=5)
|
||||
barrier.wait()
|
||||
return worker.message(actor, chat, text, key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=count) as pool:
|
||||
responses = list(pool.map(send, range(count)))
|
||||
message = responses[0]["json"]
|
||||
seen = {response["headers"]["X-Instance-Id"] for response in responses}
|
||||
check(len(seen) >= 2, "parallel requests did not reach different replicas")
|
||||
check(all(response["json"] == message for response in responses), "replay changed the result")
|
||||
check([m for m in history(client, actor, chat) if m["text"] == text] == [message],
|
||||
"parallel replay must store exactly one message")
|
||||
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
client.call("POST", path, 409, actor=actor, body={"text": "different " + key},
|
||||
headers={"Idempotency-Key": key})
|
||||
another = client.message(other, chat, text, key=key)["json"]
|
||||
check(another["id"] != message["id"], "the key must be scoped to the sender")
|
||||
return {"requests": count, "replicas": sorted(seen), "original_messages": 1,
|
||||
"different_body_status": 409, "same_key_for_another_sender": 201}
|
||||
|
||||
|
||||
class Attempt(NamedTuple):
|
||||
started: float
|
||||
finished: float
|
||||
instance: str | None
|
||||
message: dict | None
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, client, actor, chat):
|
||||
self.base = client.base
|
||||
self.actor = actor
|
||||
self.chat = chat
|
||||
self.done = Event()
|
||||
self.lock = Lock()
|
||||
self.attempts = []
|
||||
|
||||
def worker(self):
|
||||
client = Client(self.base, timeout=2)
|
||||
key = uuid4().hex
|
||||
while not self.done.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = client.message(self.actor, self.chat, "load " + key, key=key)
|
||||
except (Failure, OSError, HTTPException):
|
||||
attempt = Attempt(started, time.monotonic(), None, None)
|
||||
else:
|
||||
attempt = Attempt(started, time.monotonic(), response["headers"]["X-Instance-Id"],
|
||||
response["json"])
|
||||
key = uuid4().hex
|
||||
with self.lock:
|
||||
self.attempts.append(attempt)
|
||||
self.done.wait(PAUSE_SECONDS)
|
||||
|
||||
def window(self, start, end):
|
||||
with self.lock:
|
||||
return [attempt for attempt in self.attempts if start <= attempt.finished < end]
|
||||
|
||||
def verify(self, client):
|
||||
stored = history(client, self.actor, self.chat)
|
||||
by_id = {message["id"]: message for message in stored}
|
||||
check(len({message["text"] for message in stored}) == len(stored), "a retry created a duplicate")
|
||||
acknowledged = [attempt.message for attempt in self.attempts if attempt.message is not None]
|
||||
check(all(by_id.get(message["id"]) == message for message in acknowledged),
|
||||
"an acknowledged message was lost")
|
||||
return {"acknowledged_messages": len(acknowledged), "stored_messages": len(stored)}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def traffic(client, actor, chat):
|
||||
load = Traffic(client, actor, chat)
|
||||
with ThreadPoolExecutor(max_workers=WORKERS) as pool:
|
||||
workers = [pool.submit(load.worker) for _ in range(WORKERS)]
|
||||
try:
|
||||
yield load
|
||||
finally:
|
||||
load.done.set()
|
||||
for worker in workers:
|
||||
worker.result()
|
||||
|
||||
|
||||
def metrics(load, start, end):
|
||||
attempts = load.window(start, end)
|
||||
success = [attempt for attempt in attempts if attempt.message is not None]
|
||||
check(bool(success), "no successful writes during the measured window")
|
||||
latencies = sorted((attempt.finished - attempt.started) * 1000 for attempt in success)
|
||||
return {
|
||||
"seconds": round(end - start, 3),
|
||||
"successful_writes": len(success),
|
||||
"failed_attempts": len(attempts) - len(success),
|
||||
"successful_rps": round(len(success) / (end - start), 2),
|
||||
"latency_ms": {
|
||||
f"p{p}": round(latencies[math.ceil(p * len(latencies) / 100) - 1], 2)
|
||||
for p in (50, 95, 99)
|
||||
},
|
||||
"serving_aliases": sorted({attempt.instance for attempt in success}),
|
||||
}
|
||||
|
||||
|
||||
def first_write(load, started, stopped, end, survivor_aliases):
|
||||
writes = [attempt.finished for attempt in load.window(stopped, end)
|
||||
if attempt.started >= stopped and attempt.instance in survivor_aliases]
|
||||
check(bool(writes), "no confirmed write through a surviving replica after the stop")
|
||||
elapsed = min(writes) - started
|
||||
check(elapsed <= 10, "writes did not recover within 10 seconds")
|
||||
return round(elapsed, 3)
|
||||
|
||||
|
||||
def failover(client, actor, load, initial):
|
||||
for target in initial:
|
||||
survivors = {replica.alias for replica in initial if replica.id != target.id}
|
||||
started = time.monotonic()
|
||||
try:
|
||||
stop(target)
|
||||
stopped = time.monotonic()
|
||||
state = next(replica.status for replica in replicas() if replica.id == target.id)
|
||||
check(state == "exited", "target is still running")
|
||||
wait_routed(client, actor, survivors)
|
||||
first_read = time.monotonic() - started
|
||||
check(first_read <= 10, "reads did not recover within 10 seconds")
|
||||
time.sleep(OUTAGE_SECONDS)
|
||||
end = time.monotonic()
|
||||
during = metrics(load, started, end)
|
||||
check(target.alias not in metrics(load, stopped + 2, end)["serving_aliases"],
|
||||
"stopped replica still serves traffic")
|
||||
recovered = first_write(load, started, stopped, end, survivors)
|
||||
finally:
|
||||
docker("start", target.id)
|
||||
wait_routed(client, actor, survivors | {target.alias})
|
||||
yield {"stopped": target.name, "target_alias": target.alias, "target_status": state,
|
||||
"first_survivor_read_seconds": round(first_read, 3),
|
||||
"first_survivor_write_seconds": recovered, **during}
|
||||
|
||||
|
||||
def roll(client, actor, load, initial, new_image):
|
||||
count = str(len(initial))
|
||||
for target in initial:
|
||||
before = replicas()
|
||||
survivors = [replica for replica in before if replica.id != target.id]
|
||||
survivor_aliases = {replica.alias for replica in survivors}
|
||||
started = time.monotonic()
|
||||
try:
|
||||
stop(target)
|
||||
stopped = time.monotonic()
|
||||
docker("rm", target.id)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={count}", "api")
|
||||
after = replicas()
|
||||
added = [replica for replica in after if replica.id not in {item.id for item in before}]
|
||||
check(len(added) == 1 and added[0].image == new_image, "expected one replacement from the new image")
|
||||
check({item.id for item in survivors} <= {item.id for item in after}, "a survivor was replaced")
|
||||
new = added[0]
|
||||
wait_routed(client, actor, survivor_aliases | {new.alias})
|
||||
ready = time.monotonic()
|
||||
time.sleep(1)
|
||||
except (Failure, OSError, subprocess.SubprocessError):
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={count}", "api")
|
||||
raise
|
||||
end = time.monotonic()
|
||||
during = metrics(load, started, end)
|
||||
check(new.alias in during["serving_aliases"], "new replica did not accept writes")
|
||||
recovered = first_write(load, started, stopped, end, survivor_aliases)
|
||||
yield {"removed": target.name, "old_alias": target.alias, "old_image": target.image,
|
||||
"replacement_alias": new.alias, "new_image": new.image,
|
||||
"replacement_routed_seconds": round(ready - started, 3),
|
||||
"first_survivor_write_seconds": recovered, **during}
|
||||
check(all(replica.image == new_image for replica in replicas()), "rollout left old images running")
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument("--mode", choices=("race", "failover", "rolling"), required=True)
|
||||
parser.add_argument("--output", type=Path)
|
||||
|
||||
args = arguments("Lab 3 grade 4 checks", configure)
|
||||
client = Client(args.base_url or os.environ.get("BASE_URL") or "http://127.0.0.1:8081", timeout=5)
|
||||
actor, other, _, chat = client.bootstrap()
|
||||
initial = replicas()
|
||||
check(len(initial) >= 2 and all(replica.status == "running" for replica in initial),
|
||||
"start all API replicas before the experiment")
|
||||
wait_routed(client, actor, {replica.alias for replica in initial})
|
||||
result = {"mode": args.mode, "initial_replicas": [replica._asdict() for replica in initial]}
|
||||
if args.mode == "race":
|
||||
result.update(race(client, actor, other, chat))
|
||||
else:
|
||||
if args.mode == "rolling":
|
||||
revision = uuid4().hex[:12]
|
||||
compose("build", "--build-arg", f"BUILD_REVISION={revision}", "api", timeout=180)
|
||||
new_image = docker("image", "inspect", "bruh-lab3-api:local", "--format", "{{.Id}}")
|
||||
check(new_image not in {replica.image for replica in initial}, "rollout requires a new image")
|
||||
with traffic(client, actor, chat) as load:
|
||||
time.sleep(WARMUP_SECONDS)
|
||||
start = time.monotonic()
|
||||
time.sleep(BASELINE_SECONDS)
|
||||
result["baseline"] = metrics(load, start, time.monotonic())
|
||||
if args.mode == "failover":
|
||||
result["steps"] = list(failover(client, actor, load, initial))
|
||||
else:
|
||||
result["steps"] = list(roll(client, actor, load, initial, new_image))
|
||||
result.update(load.verify(client), workers=WORKERS, pause_seconds=PAUSE_SECONDS,
|
||||
warmup_seconds=WARMUP_SECONDS)
|
||||
text = json.dumps(result, indent=2)
|
||||
if args.output:
|
||||
args.output.write_text(text + "\n", encoding="utf-8")
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user