diff --git a/.env.example b/.env.example index d32b9be0..5fd572f3 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,7 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# Generate a local .env with python scripts/init-env.py. Never commit it. +APP_PORT=8081 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 +POSTGRES_PASSWORD= +CURSOR_SECRET= diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..0309e02d --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +scripts/*.sh text eol=lf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7a812767 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +# syntax=docker/dockerfile:1 +FROM python:3.12.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/src \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +# Unprivileged user with a fixed UID; the application never runs as root. +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app + +WORKDIR /app +COPY requirements.txt /app/ +RUN pip install --no-cache-dir -r requirements.txt + +# Only the source code goes into the image (no .env, tests, evidence, .git). +# Files stay owned by root, so the app user cannot modify its own code. +COPY src/ /app/src/ + +ARG BUILD_REVISION=dev +LABEL org.opencontainers.image.revision=${BUILD_REVISION} + +USER 10001:10001 +EXPOSE 8080 + +# docker stop sends SIGTERM; the app drains and exits well within the 10 s budget. +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=2s --timeout=2s --start-period=5s --retries=3 \ + CMD ["python", "/app/src/healthcheck.py"] + +# Exec form: python is PID 1 and receives signals directly (no shell in between). +CMD ["python", "-m", "messenger"] diff --git a/Makefile b/Makefile index 5d86d171..ac30765b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +PYTHON ?= python3 -X utf8 +BASE_URL ?= http://127.0.0.1:8081 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) diff --git a/REPORT.md b/REPORT.md index c2679005..07c9ea3c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,113 @@ -# Отчёт — лабораторная 3 +# Лабораторная 3. Балансировка и отказ API-реплики -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Риц Дмитрий, БВТ2304**. Целевая оценка — **4**. -## Запуск из чистого clone +## Запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Desktop с Linux Engine или Docker Engine с Compose, а для тестов — Python 3.9+. В контейнерах закреплены Python 3.12.12, PostgreSQL 17.6, Redis 7.4.6 и HAProxy 3.2.12. Команды выполняются из корня репозитория. На Windows можно использовать обычную CMD. -## Архитектура и решения +```bat +python scripts\init-env.py +docker compose config --quiet +docker compose up -d --build --wait +docker compose ps -a +curl.exe -i http://127.0.0.1:8081/health/ready +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`init-env.py` создаёт локальный `.env` с паролем PostgreSQL и ключом подписи курсоров. Повторный запуск использует тот же файл. `APP_PORT` по умолчанию равен 8081, чтобы запустить lab3 рядом с lab2. `API_REPLICAS` задаёт число экземпляров одного сервиса, по умолчанию 2. В HAProxy предусмотрено до 8 слотов. `SESSION_TTL_SECONDS` и `IDEMPOTENCY_TTL_SECONDS` по умолчанию равны 3600 и 86400 соответственно. -## Доказательства критериев +`docker compose down` останавливает стенд и сохраняет тома. `docker compose down -v` подходит для намеренного удаления всех учебных данных, включая сессии. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +## Как устроен стенд -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Клиент обращается к HAProxy. Балансировщик раздаёт запросы по кругу готовым API-репликам. `X-Instance-Id` берётся из hostname контейнера, поэтому можно сопоставить заголовок с конкретным контейнером и остановить его. Каждая реплика использует общую PostgreSQL для пользователей, чатов, сообщений и результатов отправки. Redis хранит общие Bearer-сессии с TTL, AOF и именованным томом. Реплики используют один ключ подписи курсоров, поэтому продолжить чтение страницы можно через любой экземпляр. -## Эксперименты и отказы +На хост опубликован только порт HAProxy. Сеть `edge` объединяет балансировщик и API. PostgreSQL, Redis, миграции и API используют отдельную внутреннюю сеть `storage`. Пароль PostgreSQL передаётся БД, API и миграциям, ключ подписи курсоров нужен только API. HAProxy использует один рабочий поток при лимите 0,5 CPU. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +| Контейнер | CPU | RAM | +| --- | --- | --- | +| API, на реплику | 1 | 384 MiB | +| HAProxy | 0,5 | 96 MiB | +| PostgreSQL | 1 | 512 MiB | +| Redis | 0,5 | 128 MiB | +| Миграции | 0,5 | 128 MiB | -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +HAProxy проверяет `GET /health/ready` каждые 500 мс. Два промаха исключают реплику, два успеха возвращают её. В переходном состоянии интервал сокращается до 250 мс. Таймауты подключения и ответа health-check составляют 500 и 700 мс, поэтому бюджет обнаружения отказа по конфигурации — до 3,4 с при нормальной работе планировщика. Docker DNS опрашивается с `hold valid 1s`, чтобы подхватить адрес нового контейнера. Клиентские запросы имеют отдельные таймауты. После SIGTERM API снимает готовность, закрывает вход для новых запросов и до 4 с даёт текущим запросам завершиться. Docker отводит 7 с на остановку. [Параметры HAProxy](https://docs.haproxy.org/3.2/configuration.html) -## Ограничения и следующие шаги +## Что происходит при повторе POST -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +У отправки сообщения есть обязательный `Idempotency-Key`. API проверяет токен и доступ к чату при каждом обращении. Затем PostgreSQL резервирует ключ в `message_requests`. Первичный ключ по `(chat_id, sender_id, idem_key)` сериализует параллельные повторы через разные реплики. Метод и маршрут фиксированы в этом обработчике. + +Запись ключа и сообщения коммитится одной транзакцией. Отложенный внешний ключ позволяет сначала занять ключ, а затем записать сообщение с заранее выбранным UUID. При откате откатываются обе записи. Повтор с тем же текстом до истечения срока возвращает исходный объект и код 201. Если текст другой, API отдаёт 409. После 24 часов ключ можно занять снова, а старое сообщение остаётся в истории. Индекс по `expires_at` пригоден для удаления истёкших записей при обслуживании БД. [Конфликты при INSERT в PostgreSQL](https://www.postgresql.org/docs/17/sql-insert.html) + +Сервис `migrate` один раз применяет `001_initial` и `002_message_requests`. Версии записываются в `schema_migrations`. Миграция 002 добавляет таблицу и индекс поверх прежней схемы. При поочерёдной замене контейнеров команда запускает только API, а обе версии образа используют уже подготовленную схему. Смена схемы с несовместимыми изменениями потребует отдельной последовательности обновления. + +## Проверки + +```bat +python -X utf8 scripts\check_contract.py +python -X utf8 tests\smoke.py --base-url http://127.0.0.1:8081 +python -X utf8 tests\resilience.py --base-url http://127.0.0.1:8081 --mode persistence --action-script scripts\restart.cmd +python -X utf8 tests\resilience.py --base-url http://127.0.0.1:8081 --mode failover --action-script scripts\stop-one.cmd +docker compose start api +``` + +В тесте persistence перезапускаются HAProxy, обе API-реплики, PostgreSQL и Redis. Тест failover сам передаёт `TARGET_INSTANCE_ID` в скрипт. Скрипт сопоставляет его с hostname запущенного контейнера сервиса API и останавливает только эту реплику. После публичного теста остановленную реплику возвращает команда `docker compose start api`. + +На Linux можно выполнить те же проверки через Makefile. + +```sh +chmod +x scripts/restart.sh scripts/stop-one.sh +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +docker compose start api +``` + +Дополнительные проверки запускаются по очереди при двух готовых репликах. + +```bat +python -X utf8 student-tests\grade4.py --mode race +python -X utf8 student-tests\grade4.py --mode failover +python -X utf8 student-tests\grade4.py --mode rolling +``` + +`race` посылает восемь параллельных запросов с одним ключом. `failover` останавливает каждую реплику по очереди и возвращает предыдущую перед следующим шагом. `rolling` собирает образ с новой OCI-меткой ревизии, заменяет контейнеры по одному и ждёт, пока новый экземпляр начнёт принимать запросы. На каждом шаге оставшийся экземпляр обслуживает запросы. Все запросы теста идут через HAProxy с одной общей сессией. + +## Результаты + +| Что проверялось | Результат | +| --- | --- | +| Запуск, миграции, smoke | Обе реплики и балансировщик готовы. На новом томе применились две миграции, повтор вернул `Schema is up to date`. Публичный smoke прошёл. [Вывод](evidence/public-checks.txt) | +| Полный перезапуск | Изменились `StartedAt` пяти контейнеров. Тест прочитал прежнее сообщение с ранее выданным токеном. [Вывод](evidence/public-checks.txt) | +| Публичный failover | Видны две реплики. За 15-секундное окно вокруг остановки тест подтвердил 126 записей и посчитал 2 неудачные попытки. Первая запись после завершения hook подтверждена через 2,024 с от его запуска. Все подтверждённые записи сохранились. [Вывод](evidence/public-checks.txt) | +| Конкурентный повтор | Восемь запросов прошли через две реплики и получили один объект сообщения. С другим текстом получен 409. Другой отправитель использовал тот же ключ независимо. [Замер](evidence/race.json) | +| Отказ обеих реплик по очереди | Первое чтение после остановки через 1,181 и 1,269 с, первая запись через 0,659 и 0,962 с. В истории остались все 560 подтверждённых сообщений. [Замер](evidence/failover.json) | +| Поочерёдная замена | Новые контейнеры стали отвечать через HAProxy через 2,999 и 2,947 с от начала соответствующего шага. По окончании в истории остались все 311 подтверждённых сообщений. [Замер](evidence/rolling.json) | + +| Шаг | Успешные записи/с | Неудачные попытки | Первая запись через оставшуюся реплику, с | +| --- | ---: | ---: | ---: | +| Остановка первой реплики | 25,20 | 8 | 0,659 | +| Остановка второй реплики | 25,17 | 8 | 0,962 | +| Замена первой реплики | 21,75 | 13 | 1,143 | +| Замена второй реплики | 22,04 | 10 | 0,647 | + +Эксперимент выполнен на Windows с Docker Desktop, 12 логическими CPU AMD Ryzen 5 5560U и 8 011 964 416 байтами RAM, доступными Docker. В каждом нагрузочном опыте четыре клиента отправляли синтетические сообщения через HAProxy, делая паузу 120 мс между запросами. Прогрев — 1 с, базовое окно — 2 с. В опыте с остановкой каждая реплика оставалась выключенной ещё 6 с после первого успешного чтения через оставшуюся. RPS и ошибки посчитаны по завершившимся запросам. p50, p95 и p99 относятся к успешным ответам. После неясного результата клиент повторяет отправку с прежним ключом. В конце проверяется история и все подтверждённые сообщения. + +Это замер работоспособности при отказе процесса и замене контейнеров на одном хосте. У хоста, HAProxy, PostgreSQL и Redis по одному экземпляру. Замена выполнена на образ с новой меткой сборки из того же исходного кода и на той же схеме БД. + +## На защите + +Показать `docker compose ps -a`, две реплики без опубликованных портов и несколько ответов через общий порт. + +```bat +for /L %i in (1,1,6) do @curl.exe -isS http://127.0.0.1:8081/health/ready +``` + +Показать два разных `X-Instance-Id`, затем запустить `--mode race` для одной сессии и одного ключа. Для демонстрации отказа запустить публичный failover, показать один остановленный контейнер через `docker compose ps -a` и вернуть его. Если попросят показать замену под нагрузкой, запустить `--mode rolling`. + +При timeout клиент не знает, успел ли сервер сделать COMMIT. Повтор с тем же ключом отдаёт прежнее сообщение, даже если запрос обслуживает другая реплика. Локальная сессия пропала бы при переключении, поэтому токен проверяет общий Redis. Compose запускает контейнеры и следит за их состоянием, а выбор доступной реплики на каждом запросе делает HAProxy. + +## Источники и помощь ИИ + +Основа — каркас курса и код lab2. При реализации, проверках и подготовке отчёта использовался Codex. [Docker Compose](https://docs.docker.com/reference/cli/docker/compose/up/), [HAProxy](https://docs.haproxy.org/3.2/configuration.html), [PostgreSQL](https://www.postgresql.org/docs/17/sql-insert.html). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..fa0a800e --- /dev/null +++ b/compose.yaml @@ -0,0 +1,96 @@ +x-app-image: &app-image + image: bruh-lab3-api:local + build: . + +services: + postgres: + image: postgres:17.6-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run python scripts/init-env.py} + volumes: + - postgres_data:/var/lib/postgresql/data + networks: [storage] + cpus: 1.0 + mem_limit: 512m + healthcheck: + test: ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", "messenger", "-d", "messenger"] + interval: 2s + timeout: 2s + retries: 15 + + redis: + image: redis:7.4.6-bookworm + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", ""] + volumes: + - redis_data:/data + networks: [storage] + cpus: 0.5 + mem_limit: 128m + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 15 + + migrate: + <<: *app-image + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run python scripts/init-env.py} + command: ["python", "-m", "messenger.migrate"] + healthcheck: + disable: true + networks: [storage] + cpus: 0.5 + mem_limit: 128m + depends_on: + postgres: + condition: service_healthy + + api: + <<: *app-image + scale: ${API_REPLICAS:-2} + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Run python scripts/init-env.py} + CURSOR_SECRET: ${CURSOR_SECRET:?Run python scripts/init-env.py} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SHUTDOWN_TIMEOUT_SECONDS: "4" + dns_opt: ["timeout:1", "attempts:1"] + stop_grace_period: 7s + networks: [edge, storage] + cpus: 1.0 + mem_limit: 384m + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + + lb: + image: haproxy:3.2.12-alpine + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PORT:-8081}:8080" + networks: [edge] + cpus: 0.5 + mem_limit: 96m + depends_on: + api: + condition: service_started + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"] + interval: 2s + timeout: 2s + retries: 15 + +networks: + edge: + storage: + internal: true + +volumes: + postgres_data: + redis_data: diff --git a/evidence/failover.json b/evidence/failover.json new file mode 100644 index 00000000..dcb0fde7 --- /dev/null +++ b/evidence/failover.json @@ -0,0 +1,80 @@ +{ + "mode": "failover", + "initial_replicas": [ + { + "id": "120f4ffe45f4b098e6dc30d589050cb6e0c3e5a6cff2221d668b9e1e6eecf51e", + "alias": "120f4ffe45f4", + "name": "/bruh_lab3-api-3", + "image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5", + "status": "running" + }, + { + "id": "2c9c28e81f35433136326442eeac3328e5f421933d7caa4e544fc5a728cf6d6e", + "alias": "2c9c28e81f35", + "name": "/bruh_lab3-api-4", + "image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5", + "status": "running" + } + ], + "baseline": { + "seconds": 2.001, + "successful_writes": 59, + "failed_attempts": 0, + "successful_rps": 29.49, + "latency_ms": { + "p50": 9.86, + "p95": 15.74, + "p99": 18.17 + }, + "serving_aliases": [ + "120f4ffe45f4", + "2c9c28e81f35" + ] + }, + "steps": [ + { + "stopped": "/bruh_lab3-api-3", + "target_alias": "120f4ffe45f4", + "target_status": "exited", + "first_survivor_read_seconds": 1.181, + "first_survivor_write_seconds": 0.659, + "seconds": 7.182, + "successful_writes": 181, + "failed_attempts": 8, + "successful_rps": 25.2, + "latency_ms": { + "p50": 9.8, + "p95": 24.28, + "p99": 27.35 + }, + "serving_aliases": [ + "2c9c28e81f35" + ] + }, + { + "stopped": "/bruh_lab3-api-4", + "target_alias": "2c9c28e81f35", + "target_status": "exited", + "first_survivor_read_seconds": 1.269, + "first_survivor_write_seconds": 0.962, + "seconds": 7.269, + "successful_writes": 183, + "failed_attempts": 8, + "successful_rps": 25.17, + "latency_ms": { + "p50": 9.55, + "p95": 23.23, + "p99": 25.36 + }, + "serving_aliases": [ + "120f4ffe45f4", + "2c9c28e81f35" + ] + } + ], + "acknowledged_messages": 560, + "stored_messages": 560, + "workers": 4, + "pause_seconds": 0.12, + "warmup_seconds": 1 +} diff --git a/evidence/public-checks.txt b/evidence/public-checks.txt new file mode 100644 index 00000000..a8eb9b9b --- /dev/null +++ b/evidence/public-checks.txt @@ -0,0 +1,66 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +bruh_lab3-api-3 bruh-lab3-api:local "python -m messenger" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp +bruh_lab3-api-4 bruh-lab3-api:local "python -m messenger" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp +bruh_lab3-lb-1 haproxy:3.2.12-alpine "docker-entrypoint.s…" lb 5 seconds ago Up 3 seconds (healthy) 127.0.0.1:8081->8080/tcp +bruh_lab3-migrate-1 sha256:3482b32c94c1065cb1df310ccc35f5a0594ca577ecb0e3dd492e5842a624799f "python -m messenger…" migrate 10 minutes ago Exited (0) 6 minutes ago +bruh_lab3-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 10 minutes ago Up 6 minutes (healthy) 5432/tcp +bruh_lab3-redis-1 redis:7.4.6-bookworm "docker-entrypoint.s…" redis 10 minutes ago Up 6 minutes (healthy) 6379/tcp +Applied: 001_initial, 002_message_requests +Schema is up to date +Schema is up to date +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. +Before restart (UTC) +/bruh_lab3-api-3 2026-11-05T23:38:49.72159878Z +/bruh_lab3-api-4 2026-11-05T23:38:54.359920283Z +/bruh_lab3-lb-1 2026-11-05T23:41:37.623541111Z +/bruh_lab3-postgres-1 2026-11-05T23:34:57.792671441Z +/bruh_lab3-redis-1 2026-11-05T23:34:57.877487028Z +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +After restart (UTC) +/bruh_lab3-api-3 2026-11-05T23:41:55.159614054Z +/bruh_lab3-api-4 2026-11-05T23:41:55.083328192Z +/bruh_lab3-lb-1 2026-11-05T23:41:54.760199725Z +/bruh_lab3-postgres-1 2026-11-05T23:41:54.520560824Z +/bruh_lab3-redis-1 2026-11-05T23:41:54.610324316Z +Before public failover +/bruh_lab3-api-3 120f4ffe45f4 running +/bruh_lab3-api-4 2c9c28e81f35 running +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 2, "first_recovery_seconds": 2.024} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +After public failover +/bruh_lab3-api-3 120f4ffe45f4 running +/bruh_lab3-api-4 2c9c28e81f35 exited +After rolling replacement +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +bruh_lab3-api-5 bruh-lab3-api:local "python -m messenger" api 8 seconds ago Up 7 seconds (healthy) 8080/tcp +bruh_lab3-api-6 bruh-lab3-api:local "python -m messenger" api 4 seconds ago Up 3 seconds (health: starting) 8080/tcp +bruh_lab3-lb-1 haproxy:3.2.12-alpine "docker-entrypoint.s…" lb About a minute ago Up About a minute (healthy) 127.0.0.1:8081->8080/tcp +bruh_lab3-migrate-1 sha256:3482b32c94c1065cb1df310ccc35f5a0594ca577ecb0e3dd492e5842a624799f "python -m messenger…" migrate 11 minutes ago Exited (0) 7 minutes ago +bruh_lab3-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 11 minutes ago Up About a minute (healthy) 5432/tcp +bruh_lab3-redis-1 redis:7.4.6-bookworm "docker-entrypoint.s…" redis 11 minutes ago Up About a minute (healthy) 6379/tcp +/bruh_lab3-api-5 cpus=1000000000 mem_bytes=402653184 published_ports={} +/bruh_lab3-api-6 cpus=1000000000 mem_bytes=402653184 published_ports={} +/bruh_lab3-lb-1 cpus=500000000 mem_bytes=100663296 published_ports={"8080/tcp":[{"HostIp":"127.0.0.1","HostPort":"8081"}]} +/bruh_lab3-migrate-1 cpus=500000000 mem_bytes=134217728 published_ports={} +/bruh_lab3-postgres-1 cpus=1000000000 mem_bytes=536870912 published_ports={} +/bruh_lab3-redis-1 cpus=500000000 mem_bytes=134217728 published_ports={} +bruh_lab3-api-5 30.36% 28.82MiB / 384MiB +bruh_lab3-api-6 28.16% 28.39MiB / 384MiB +bruh_lab3-lb-1 2.09% 76.11MiB / 96MiB +bruh_lab3-postgres-1 5.40% 31.93MiB / 512MiB +bruh_lab3-redis-1 0.42% 4.422MiB / 128MiB +Engine 29.3.1, CPUs 12, RAM 8011964416 B, OS Docker Desktop (containerized) +Docker Compose version v5.1.1 +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. +Final ready aliases through HAProxy +x-instance-id: 1fc578fde21a +x-instance-id: 6143df6729b7 +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +bruh_lab3-api-5 bruh-lab3-api:local "python -m messenger" api About a minute ago Up About a minute (healthy) 8080/tcp +bruh_lab3-api-6 bruh-lab3-api:local "python -m messenger" api About a minute ago Up About a minute (healthy) 8080/tcp +bruh_lab3-lb-1 haproxy:3.2.12-alpine "docker-entrypoint.s…" lb 2 minutes ago Up 2 minutes (healthy) 127.0.0.1:8081->8080/tcp +bruh_lab3-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 12 minutes ago Up 2 minutes (healthy) 5432/tcp +bruh_lab3-redis-1 redis:7.4.6-bookworm "docker-entrypoint.s…" redis 12 minutes ago Up 2 minutes (healthy) 6379/tcp diff --git a/evidence/race.json b/evidence/race.json new file mode 100644 index 00000000..c300fbbb --- /dev/null +++ b/evidence/race.json @@ -0,0 +1,27 @@ +{ + "mode": "race", + "initial_replicas": [ + { + "id": "120f4ffe45f4b098e6dc30d589050cb6e0c3e5a6cff2221d668b9e1e6eecf51e", + "alias": "120f4ffe45f4", + "name": "/bruh_lab3-api-3", + "image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5", + "status": "running" + }, + { + "id": "2c9c28e81f35433136326442eeac3328e5f421933d7caa4e544fc5a728cf6d6e", + "alias": "2c9c28e81f35", + "name": "/bruh_lab3-api-4", + "image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5", + "status": "running" + } + ], + "requests": 8, + "replicas": [ + "120f4ffe45f4", + "2c9c28e81f35" + ], + "original_messages": 1, + "different_body_status": 409, + "same_key_for_another_sender": 201 +} diff --git a/evidence/rolling.json b/evidence/rolling.json new file mode 100644 index 00000000..c84e7c20 --- /dev/null +++ b/evidence/rolling.json @@ -0,0 +1,86 @@ +{ + "mode": "rolling", + "initial_replicas": [ + { + "id": "120f4ffe45f4b098e6dc30d589050cb6e0c3e5a6cff2221d668b9e1e6eecf51e", + "alias": "120f4ffe45f4", + "name": "/bruh_lab3-api-3", + "image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5", + "status": "running" + }, + { + "id": "2c9c28e81f35433136326442eeac3328e5f421933d7caa4e544fc5a728cf6d6e", + "alias": "2c9c28e81f35", + "name": "/bruh_lab3-api-4", + "image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5", + "status": "running" + } + ], + "baseline": { + "seconds": 2.0, + "successful_writes": 58, + "failed_attempts": 0, + "successful_rps": 29.0, + "latency_ms": { + "p50": 8.64, + "p95": 22.53, + "p99": 24.44 + }, + "serving_aliases": [ + "120f4ffe45f4", + "2c9c28e81f35" + ] + }, + "steps": [ + { + "removed": "/bruh_lab3-api-3", + "old_alias": "120f4ffe45f4", + "old_image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5", + "replacement_alias": "1fc578fde21a", + "new_image": "sha256:16ac194a8f89e953ccd7ccdcf93ebb091a034b82eb7460d406443309478f0bc6", + "replacement_routed_seconds": 2.999, + "first_survivor_write_seconds": 1.143, + "seconds": 3.999, + "successful_writes": 87, + "failed_attempts": 13, + "successful_rps": 21.75, + "latency_ms": { + "p50": 10.58, + "p95": 26.18, + "p99": 32.09 + }, + "serving_aliases": [ + "120f4ffe45f4", + "1fc578fde21a", + "2c9c28e81f35" + ] + }, + { + "removed": "/bruh_lab3-api-4", + "old_alias": "2c9c28e81f35", + "old_image": "sha256:6f51033bf9f56945d6579d807b1fc60c33429eb2cf717fea9acc33e4fbc411d5", + "replacement_alias": "6143df6729b7", + "new_image": "sha256:16ac194a8f89e953ccd7ccdcf93ebb091a034b82eb7460d406443309478f0bc6", + "replacement_routed_seconds": 2.947, + "first_survivor_write_seconds": 0.647, + "seconds": 3.947, + "successful_writes": 87, + "failed_attempts": 10, + "successful_rps": 22.04, + "latency_ms": { + "p50": 9.6, + "p95": 23.74, + "p99": 37.13 + }, + "serving_aliases": [ + "1fc578fde21a", + "6143df6729b7" + ] + } + ], + "acknowledged_messages": 311, + "stored_messages": 311, + "workers": 4, + "pause_seconds": 0.12, + "warmup_seconds": 1 +} diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..03294a9d --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,37 @@ +global + log stdout format raw local0 + nbthread 1 + +defaults + log global + mode http + option httplog + option http-server-close + timeout http-request 5s + timeout http-keep-alive 5s + timeout connect 500ms + timeout client 10s + timeout server 7s + timeout check 700ms + retries 0 + +resolvers docker + nameserver embedded 127.0.0.11:53 + resolve_retries 2 + timeout resolve 500ms + timeout retry 500ms + hold valid 1s + hold obsolete 1s + hold nx 1s + +frontend public + bind :8080 + default_backend messenger + +backend messenger + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 500ms fastinter 250ms downinter 500ms fall 2 rise 2 + server-template replica 1-8 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..0280fec4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +psycopg[binary]==3.2.10 +psycopg-pool==3.2.6 +redis==6.4.0 diff --git a/scripts/init-env.py b/scripts/init-env.py new file mode 100644 index 00000000..3e462b50 --- /dev/null +++ b/scripts/init-env.py @@ -0,0 +1,15 @@ +"""Generate local Compose secrets once, without printing them.""" + +from pathlib import Path +from secrets import token_hex + +root = Path(__file__).resolve().parents[1] +example = (root / ".env.example").read_text(encoding="utf-8") +try: + with (root / ".env").open("x", encoding="utf-8", newline="\n") as target: + target.write(example.replace("POSTGRES_PASSWORD=\n", f"POSTGRES_PASSWORD={token_hex(24)}\n") + .replace("CURSOR_SECRET=\n", f"CURSOR_SECRET={token_hex(32)}\n")) +except FileExistsError: + print("Using the existing .env") +else: + print("Created local .env") diff --git a/scripts/replicas.py b/scripts/replicas.py new file mode 100644 index 00000000..bfb4d638 --- /dev/null +++ b/scripts/replicas.py @@ -0,0 +1,61 @@ +"""Match an API response alias to a container in this Compose project.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +from typing import NamedTuple + + +ROOT = Path(__file__).resolve().parents[1] + + +class Replica(NamedTuple): + id: str + alias: str + name: str + image: str + status: str + + +def docker(*args, timeout=30): + return subprocess.run( + ["docker", *args], cwd=ROOT, text=True, encoding="utf-8", + stdout=subprocess.PIPE, check=True, timeout=timeout, + ).stdout.strip() + + +def compose(*args, **kwargs): + return docker("compose", *args, **kwargs) + + +def replicas(): + ids = compose("ps", "-a", "-q", "api").splitlines() + if not ids: + return [] + fields = "{{.Id}} {{.Config.Hostname}} {{.Name}} {{.Image}} {{.State.Status}}" + return [Replica(*line.split()) for line in docker("inspect", "--format", fields, *ids).splitlines()] + + +def stop(replica): + docker("stop", "--time", "7", replica.id, timeout=9) + + +def main(): + target = os.environ.get("TARGET_INSTANCE_ID") + running = [replica for replica in replicas() if replica.status == "running"] + matched = [replica for replica in running if replica.alias == target] + if len(matched) != 1 or len(running) < 2: + raise ValueError("TARGET_INSTANCE_ID must match one of at least two running API replicas") + replica = matched[0] + stop(replica) + print(json.dumps({"stopped_container": replica.name, "instance_id": replica.alias})) + + +if __name__ == "__main__": + try: + main() + except (ValueError, OSError, subprocess.SubprocessError) as error: + print(f"Stop hook failed: {error}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..1209c401 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,2 @@ +@echo off +docker compose --project-directory "%~dp0.." restart postgres redis api lb diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..ef66f92f --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec docker compose restart postgres redis api lb diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..85423ec6 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,2 @@ +@echo off +python "%~dp0replicas.py" diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..223d97e0 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec "${PYTHON:-python3}" "$(dirname "$0")/replicas.py" diff --git a/src/healthcheck.py b/src/healthcheck.py new file mode 100644 index 00000000..a701b10d --- /dev/null +++ b/src/healthcheck.py @@ -0,0 +1,12 @@ +"""Docker HEALTHCHECK probe: exit 0 if /health/ready answers 200, else 1.""" + +import os +import sys +import urllib.request + +port = os.environ.get("APP_PORT", "8080") +try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response: + sys.exit(0 if response.status == 200 else 1) +except Exception: + sys.exit(1) diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..22c3aac8 --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1,3 @@ +"""Replicated messenger for MTUSI Lab 3.""" + +__version__ = "3.0.0" diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py new file mode 100644 index 00000000..dd08e027 --- /dev/null +++ b/src/messenger/__main__.py @@ -0,0 +1,59 @@ +"""Run one messenger API replica.""" + +from __future__ import annotations + +import os +import signal +import sys +import threading +import time + +from . import __version__ +from .config import Config +from .http_api import MessengerServer +from .limits import Limits +from .logs import log +from .sessions import Sessions +from .store import Store + + +def main() -> None: + config = Config.from_env() + limits = Limits() + store = Store(config.cursor_secret, config.idempotency_ttl) + sessions = Sessions(config.session_ttl) + server = MessengerServer((config.host, config.port), store, sessions, limits, config.max_body_bytes) + stop_requested = threading.Event() + + def on_signal(signum: int, _frame) -> None: + if stop_requested.is_set(): + return + stop_requested.set() + server.draining = True # readiness -> 503, new API requests -> 503 + log("info", "shutdown_started", signal=signal.Signals(signum).name) + # shutdown() waits for serve_forever() to return, so it must run in another thread. + threading.Thread(target=server.shutdown, name="shutdown", daemon=True).start() + + signal.signal(signal.SIGTERM, on_signal) + signal.signal(signal.SIGINT, on_signal) + + log("info", "started", version=__version__, host=config.host, port=config.port, env=config.env, + pid=os.getpid(), instance=server.instance_id, storage="postgres", session_store="redis", + max_body_bytes=config.max_body_bytes, + **limits.describe()) + limits.start_sampler() + server.serve_forever(poll_interval=0.2) + + stopped_at = time.monotonic() + server.server_close() # stop accepting connections: the listening socket is closed + finished = server.wait_idle(timeout=config.shutdown_timeout) + store.pool.close() + sessions.redis.close() + log("info" if finished else "warning", "shutdown_complete", + in_flight_finished=finished, + drain_ms=round((time.monotonic() - stopped_at) * 1000, 1)) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/messenger/config.py b/src/messenger/config.py new file mode 100644 index 00000000..0a714e7e --- /dev/null +++ b/src/messenger/config.py @@ -0,0 +1,70 @@ +"""Local configuration from environment variables.""" + +from __future__ import annotations + +import os +import sys +from dataclasses import dataclass + +from .logs import log + + +def postgres_options() -> dict: + return { + "host": os.environ.get("POSTGRES_HOST", "postgres"), + "port": env_int("POSTGRES_PORT", 5432, 1, 65535), + "dbname": os.environ.get("POSTGRES_DB", "messenger"), + "user": os.environ.get("POSTGRES_USER", "messenger"), + "password": os.environ.get("POSTGRES_PASSWORD"), + "connect_timeout": 2, + "options": "-c statement_timeout=3000 -c lock_timeout=1500 -c timezone=UTC", + } + + +def env_int(name: str, default: int, low: int, high: int) -> int: + raw = os.environ.get(name, "").strip() or str(default) + if not raw.isdigit() or not low <= int(raw) <= high: + log("error", "bad_config", variable=name, allowed=f"{low}..{high}") + sys.exit(2) + return int(raw) + + +def env_bool(name: str, default: bool) -> bool: + raw = os.environ.get(name, "").strip().lower() + if not raw: + return default + if raw in ("1", "true", "yes", "on"): + return True + if raw in ("0", "false", "no", "off"): + return False + log("error", "bad_config", variable=name, allowed="0/1") + sys.exit(2) + + +@dataclass(frozen=True) +class Config: + host: str + port: int + env: str + shutdown_timeout: int + max_body_bytes: int + session_ttl: int + idempotency_ttl: int + cursor_secret: bytes + + @classmethod + def from_env(cls) -> "Config": + secret = os.environ.get("CURSOR_SECRET", "").encode() + if len(secret) < 32: + log("error", "bad_config", variable="CURSOR_SECRET", minimum_length=32) + sys.exit(2) + return cls( + host=os.environ.get("APP_HOST", "").strip() or "0.0.0.0", + port=env_int("APP_PORT", 8080, 1, 65535), + env=os.environ.get("APP_ENV", "lab"), + shutdown_timeout=env_int("SHUTDOWN_TIMEOUT_SECONDS", 8, 1, 9), + max_body_bytes=env_int("MAX_BODY_BYTES", 64 * 1024, 1024, 16 * 1024 * 1024), + session_ttl=env_int("SESSION_TTL_SECONDS", 3600, 1, 31_536_000), + idempotency_ttl=env_int("IDEMPOTENCY_TTL_SECONDS", 86400, 1, 31_536_000), + cursor_secret=secret, + ) diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 00000000..9822a567 --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,68 @@ +"""API errors in the common contract format: {"error": {"code", "message", "request_id"}}.""" + +from __future__ import annotations + +from typing import Optional + + +class ApiError(Exception): + """An error that is turned into a JSON error response.""" + + def __init__(self, status: int, code: str, message: str, close: bool = False, + retry_after: Optional[int] = None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + # close=True: the request body may not have been read, so the connection + # must not be reused for another request. + self.close = close + self.retry_after = retry_after # seconds, sent as the Retry-After header + + +def bad_request(message: str, close: bool = False) -> ApiError: + return ApiError(400, "invalid_request", message, close) + + +def unauthorized() -> ApiError: + return ApiError(401, "unauthorized", "invalid credentials or session") + + +def not_found(message: str = "resource not found") -> ApiError: + return ApiError(404, "not_found", message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, "conflict", message) + + +def payload_too_large(message: str) -> ApiError: + return ApiError(413, "payload_too_large", message, close=True) + + +def unavailable(message: str) -> ApiError: + return ApiError(503, "unavailable", message) + + +def internal_error() -> ApiError: + # Never expose stack traces or internal details to the client. + return ApiError(500, "internal_error", "internal server error") + + +_CODE_BY_STATUS = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", +} + + +def code_for_status(status: int) -> str: + """Contract error code for an arbitrary HTTP status (used for low-level HTTP errors).""" + if status in _CODE_BY_STATUS: + return _CODE_BY_STATUS[status] + return "internal_error" if status >= 500 else "invalid_request" diff --git a/src/messenger/http_api.py b/src/messenger/http_api.py new file mode 100644 index 00000000..b22d2b87 --- /dev/null +++ b/src/messenger/http_api.py @@ -0,0 +1,440 @@ +"""HTTP layer: routing, session authentication, JSON responses, request logs.""" + +from __future__ import annotations + +import json +import re +import secrets +import socket +import socketserver +import threading +import time +import uuid +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Callable, Dict, List, Optional, Tuple +from urllib.parse import parse_qs, unquote, urlsplit + +from . import validation as v +from .errors import ApiError, bad_request, code_for_status, internal_error, not_found, unauthorized, unavailable +from .limits import Limits +from .logs import log +from .sessions import Sessions, credentials, password_digest, verify_password +from .store import Store + +REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +class MessengerServer(ThreadingHTTPServer): + """Thread per connection with bounded admission and graceful shutdown.""" + + daemon_threads = True # idle keep-alive connections must not block process exit + request_queue_size = 128 + + def __init__(self, address: Tuple[str, int], store: Store, sessions: Sessions, + limits: Limits, max_body_bytes: int): + self.store = store + self.sessions = sessions + self.limits = limits + self.max_body_bytes = max_body_bytes + self.instance_id = socket.gethostname() + self.draining = False + self._in_flight = 0 + self._connections = 0 + self._idle = threading.Condition() + super().__init__(address, Handler) + + def server_bind(self) -> None: + # HTTPServer.server_bind() does a reverse DNS lookup (getfqdn) that can + # hang for seconds inside containers; it is not needed for an API. + socketserver.TCPServer.server_bind(self) + self.server_name, self.server_port = self.server_address[:2] + + # -- connections: one thread each; refuse new ones above MAX_CONNECTIONS -- + + def process_request(self, request, client_address) -> None: + with self._idle: + refuse = self.limits.enabled and self._connections >= self.limits.max_connections + if not refuse: + self._connections += 1 + if refuse: + _refuse_connection(request, self.instance_id) + self.shutdown_request(request) + return + try: + super().process_request(request, client_address) + except Exception: + with self._idle: + self._connections -= 1 + raise + + def process_request_thread(self, request, client_address) -> None: + try: + super().process_request_thread(request, client_address) + finally: + with self._idle: + self._connections -= 1 + + # -- requests in progress -- + + def request_started(self) -> int: + with self._idle: + self._in_flight += 1 + return self._in_flight + + def request_finished(self) -> None: + with self._idle: + self._in_flight -= 1 + if self._in_flight == 0: + self._idle.notify_all() + + def wait_idle(self, timeout: float) -> bool: + """Wait until no request is being processed; False on timeout.""" + with self._idle: + return self._idle.wait_for(lambda: self._in_flight == 0, timeout=max(timeout, 0)) + + +# ----- route handlers -------------------------------------------------------- +# Each handler gets (request, user, path params, query) and returns (status, body). + +Result = Tuple[int, Optional[Dict[str, Any]]] + + +def health_live(req: "Handler", user, params, query) -> Result: + return 200, {"status": "ok"} + + +def _refuse_connection(sock, instance_id: str) -> None: + """Answer 503 without starting a thread (connection limit). Runs in the accept loop.""" + request_id = uuid.uuid4().hex + body = json.dumps({"error": {"code": "unavailable", "message": "too many connections, retry later", + "request_id": request_id}}).encode() + head = ( + "HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\nX-Request-Id: {request_id}\r\n" + f"X-Instance-Id: {instance_id}\r\nRetry-After: 1\r\n" + "Connection: close\r\n\r\n" + ).encode() + try: + sock.settimeout(0.05) + try: + sock.recv(65536) # read what already arrived so close() does not reset the connection + except OSError: + pass + sock.sendall(head + body) + except OSError: + pass + log("info", "request", request_id=request_id, method="-", route="-", status=503, + duration_ms=0.0, rejected="connections") + + +def health_ready(req: "Handler", user, params, query) -> Result: + if req.server.draining: + raise unavailable("server is shutting down") + if not req.server.limits.ready(): + raise unavailable("memory limit reached") + req.server.store.ready() + req.server.sessions.ready() + return 200, {"status": "ok"} + + +def create_user(req: "Handler", user, params, query) -> Result: + fields = ("username", "display_name", "password") + data = req.json_body(allowed=fields, required=fields) + username = v.username_field(data) + display_name = v.text_field(data, "display_name", 100) + password = data["password"] + if not v.valid_password(password): + raise bad_request("password must contain 12..128 valid nonblank characters") + salt = secrets.token_bytes(16) + return 201, req.server.store.create_user(username, display_name, salt, password_digest(password, salt)) + + +def create_session(req: "Handler", user, params, query) -> Result: + if req._read_body(): + raise bad_request("login does not accept a body") + username, password = credentials(req.authorization("basic")) + user = req.server.store.credentials(username) + if not verify_password(password, user): + raise unauthorized() + del user["password_hash"], user["password_salt"] + token, expires_at = req.server.sessions.create(user["id"]) + return 201, {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user} + + +def revoke_session(req: "Handler", user, params, query) -> Result: + req.server.sessions.revoke(req.authorization("bearer")) + return 204, None + + +def get_me(req: "Handler", user, params, query) -> Result: + profile = req.server.store.get_user(user) + if profile is None: + raise unauthorized() + return 200, profile + + +def create_chat(req: "Handler", user, params, query) -> Result: + req.server.limits.check_rate(user) # 429 if this user writes too fast + data = req.json_body(allowed=("title", "member_ids"), required=("title", "member_ids")) + title = v.text_field(data, "title", 100) + members = v.member_ids_field(data) + return 201, req.server.store.create_chat(user, title, members) + + +def list_chats(req: "Handler", user, params, query) -> Result: + limit, cursor = v.limit_param(query), v.cursor_param(query) + return 200, req.server.store.list_chats(user, limit, cursor) + + +def get_chat(req: "Handler", user, params, query) -> Result: + return 200, req.server.store.get_chat(user, params["chat_id"]) + + +def post_message(req: "Handler", user, params, query) -> Result: + keys = req.headers.get_all("Idempotency-Key") or [] + if len(keys) != 1 or IDEMPOTENCY_KEY_RE.fullmatch(keys[0]) is None: + raise bad_request("Idempotency-Key must be a single value matching [A-Za-z0-9._:-]{1,128}") + req.server.limits.check_rate(user) # 429 if this user writes too fast + # sender_id is not accepted in the body: the sender is always the current user. + data = req.json_body(allowed=("text",), required=("text",)) + text = v.text_field(data, "text", 4000) + return 201, req.server.store.add_message(user, params["chat_id"], keys[0], text) + + +def list_messages(req: "Handler", user, params, query) -> Result: + limit, cursor = v.limit_param(query), v.cursor_param(query) + return 200, req.server.store.list_messages(user, params["chat_id"], limit, cursor) + + +def json_value(value): + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, datetime): + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + raise TypeError(f"Cannot serialize {type(value).__name__}") + + +Route = Tuple[str, "re.Pattern[str]", str, Callable[..., Result], bool] + +_CHAT = r"(?P[^/]+)" +ROUTES: List[Route] = [ + # method, path regex, template for logs, handler, requires Bearer + ("GET", re.compile(r"/health/live"), "/health/live", health_live, False), + ("GET", re.compile(r"/health/ready"), "/health/ready", health_ready, False), + ("POST", re.compile(r"/api/v1/users"), "/api/v1/users", create_user, False), + ("POST", re.compile(r"/api/v1/auth/sessions"), "/api/v1/auth/sessions", create_session, False), + ("DELETE", re.compile(r"/api/v1/auth/sessions/current"), "/api/v1/auth/sessions/current", revoke_session, False), + ("GET", re.compile(r"/api/v1/users/me"), "/api/v1/users/me", get_me, True), + ("POST", re.compile(r"/api/v1/chats"), "/api/v1/chats", create_chat, True), + ("GET", re.compile(r"/api/v1/chats"), "/api/v1/chats", list_chats, True), + ("GET", re.compile(r"/api/v1/chats/" + _CHAT), "/api/v1/chats/{chat_id}", get_chat, True), + ("POST", re.compile(r"/api/v1/chats/" + _CHAT + "/messages"), "/api/v1/chats/{chat_id}/messages", post_message, True), + ("GET", re.compile(r"/api/v1/chats/" + _CHAT + "/messages"), "/api/v1/chats/{chat_id}/messages", list_messages, True), +] + + +def resolve(method: str, path: str) -> Tuple[Route, Dict[str, str]]: + path_matched = False + for route in ROUTES: + match = route[1].fullmatch(path) + if match is None: + continue + path_matched = True + if route[0] == method or (method == "HEAD" and route[0] == "GET"): + return route, match.groupdict() + if path_matched: + raise ApiError(405, "invalid_request", "method not allowed") + raise not_found("route not found") + + +# ----- HTTP handler ---------------------------------------------------------- + + +class Handler(BaseHTTPRequestHandler): + server: MessengerServer + protocol_version = "HTTP/1.1" + server_version = "mtusi-messenger" + sys_version = "" + timeout = 30 # idle keep-alive connections are closed after this many seconds + # TCP_NODELAY: headers and body are written separately; with Nagle's algorithm plus the + # client's delayed ACK every keep-alive response waited ~40 ms (found in the track 2 runs). + disable_nagle_algorithm = True + + request_id = "" + route_template = "-" + started = 0.0 + _body: Optional[bytes] = None + + def do_GET(self) -> None: + self._handle() + + def do_POST(self) -> None: + self._handle() + + def do_PUT(self) -> None: + self._handle() + + def do_PATCH(self) -> None: + self._handle() + + def do_DELETE(self) -> None: + self._handle() + + def do_HEAD(self) -> None: + self._handle() + + def do_OPTIONS(self) -> None: + self._handle() + + # -- request processing -- + + def _handle(self) -> None: + self.started = time.monotonic() + incoming = self.headers.get("X-Request-Id", "") + self.request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex + self.route_template = "-" + self._body: Optional[bytes] = None + self.server.request_started() + slot = False + try: + if self.command in ("POST", "PUT", "PATCH", "DELETE"): + # Read the body first: an early error then never leaves unread bytes + # on the connection, so keep-alive works even for 429/503 answers. + self._read_body() + split = urlsplit(self.path) + route, params = resolve(self.command, split.path) + self.route_template = route[2] + is_health = route[2] in ("/health/live", "/health/ready") + if self.server.draining and not is_health: + raise unavailable("server is shutting down") + if not is_health: + slot = self.server.limits.acquire_slot() # bounded queue; 503 when overloaded + user = self.server.sessions.identify(self.authorization("bearer")) if route[4] else None + if "chat_id" in params: + params["chat_id"] = v.parse_uuid(unquote(params["chat_id"]), "chat_id") + query = parse_qs(split.query, keep_blank_values=True) + status, payload = route[3](self, user, params, query) + self._send_json(status, payload) + except ApiError as error: + if error.close: + self.close_connection = True + self._safe_send_error(error) + except (BrokenPipeError, ConnectionResetError, TimeoutError): + self.close_connection = True # the client went away; nothing to answer + except Exception as error: + log("error", "unhandled_exception", request_id=self.request_id, + route=self.route_template, exception_type=type(error).__name__) + self.close_connection = True + self._safe_send_error(internal_error()) + finally: + if slot: + self.server.limits.release_slot() + self.server.request_finished() + + def _safe_send_error(self, error: ApiError) -> None: + try: + self._send_error_json(error) + except (BrokenPipeError, ConnectionResetError): + self.close_connection = True + + def authorization(self, scheme: str) -> str: + values = self.headers.get_all("Authorization") or [] + if len(values) != 1: + raise unauthorized() + parts = values[0].split() + if len(parts) != 2 or parts[0].lower() != scheme: + raise unauthorized() + return parts[1] + + def _read_body(self) -> bytes: + if self._body is not None: + return self._body + if self.headers.get("Transfer-Encoding"): + raise bad_request("chunked request bodies are not supported", close=True) + lengths = self.headers.get_all("Content-Length") or [] + if not lengths: + self._body = b"" + return self._body + if len(lengths) != 1 or not re.fullmatch(r"[0-9]{1,16}", lengths[0]): + raise bad_request("invalid Content-Length", close=True) + length = int(lengths[0]) + if length > self.server.max_body_bytes: + raise ApiError(413, "payload_too_large", "request body is too large", close=True) + self._body = self.rfile.read(length) + if len(self._body) != length: + raise bad_request("incomplete request body", close=True) + return self._body + + def json_body(self, allowed, required) -> Dict[str, Any]: + raw = self._read_body() + if self.headers.get_content_type() != "application/json": + raise bad_request("Content-Type must be application/json") + charset = self.headers.get_content_charset() + if charset not in (None, "utf-8"): + raise bad_request("request body must be UTF-8") + return v.parse_json_object(raw, allowed, required) + + # -- responses -- + + def _send_json(self, status: int, payload: Optional[Dict[str, Any]], retry_after: Optional[int] = None) -> None: + body = b"" if payload is None else json.dumps( + payload, ensure_ascii=False, separators=(",", ":"), default=json_value + ).encode("utf-8") + if self.command in ("POST", "PUT", "PATCH", "DELETE") and self._body is None: + # The body was never read (e.g. error before parsing): do not reuse the connection. + self.close_connection = True + if self.server.draining: + self.close_connection = True + self.send_response(status) + if status != 204: + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Instance-Id", self.server.instance_id) + self.send_header("Cache-Control", "no-store") + self.send_header("X-Content-Type-Options", "nosniff") + if retry_after is not None: + self.send_header("Retry-After", str(retry_after)) + if status == 401: + challenge = 'Bearer realm="messenger"' + if self.route_template == "/api/v1/auth/sessions": + challenge = 'Basic realm="messenger", charset="UTF-8"' + self.send_header("WWW-Authenticate", challenge) + if status == 405: + path = urlsplit(getattr(self, "path", "") or "").path + allow = sorted({r[0] for r in ROUTES if r[1].fullmatch(path)}) + if allow: + self.send_header("Allow", ", ".join(allow)) + if self.close_connection: + self.send_header("Connection", "close") + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + log("info", "request", request_id=self.request_id, method=self.command, + route=self.route_template, status=status, + duration_ms=round((time.monotonic() - self.started) * 1000, 2)) + + def _send_error_json(self, error: ApiError) -> None: + self._send_json(error.status, { + "error": {"code": error.code, "message": error.message, "request_id": self.request_id} + }, retry_after=error.retry_after) + + def send_error(self, code: int, message: Optional[str] = None, explain: Optional[str] = None) -> None: + """Low-level HTTP errors (bad request line, unsupported method) in the same JSON format.""" + self.request_id = uuid.uuid4().hex + self.started = time.monotonic() + self.route_template = "-" + self._body = None + self.close_connection = True + if getattr(self, "command", None) is None: + self.command = "GET" + status = 405 if code == 501 else code # 501 = method unknown to http.server + self._safe_send_error(ApiError(status, code_for_status(status), "invalid HTTP request")) + + def version_string(self) -> str: + return self.server_version + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + pass # replaced by structured logs in _send_json diff --git a/src/messenger/limits.py b/src/messenger/limits.py new file mode 100644 index 00000000..45860f5c --- /dev/null +++ b/src/messenger/limits.py @@ -0,0 +1,166 @@ +"""Bound HTTP connections, concurrent work and writes per user.""" + +from __future__ import annotations + +import math +import os +import threading +import time +from typing import Dict, List, Optional +from uuid import UUID + +from .config import env_bool, env_int +from .errors import ApiError + +try: + _PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") +except (AttributeError, ValueError, OSError): + _PAGE_SIZE = 4096 + + +def read_rss() -> Optional[int]: + """Resident memory of this process in bytes (Linux /proc), None elsewhere.""" + try: + with open("/proc/self/statm") as f: + return int(f.read().split()[1]) * _PAGE_SIZE + except (OSError, ValueError, IndexError): + return None + + +def _read_limit(path: str) -> Optional[int]: + try: + with open(path) as f: + raw = f.read().strip() + except OSError: + return None + if not raw.isdigit(): + return None # cgroup v2 says "max" when there is no limit + value = int(raw) + return value if value < (1 << 50) else None # cgroup v1 uses a huge number for "no limit" + + +def detect_memory_limit() -> Optional[int]: + """Memory limit of the container (cgroup v2 or v1), None if unlimited/unknown.""" + candidates: List[str] = ["/sys/fs/cgroup/memory.max"] + try: + with open("/proc/self/cgroup") as f: + for line in f: + _, controllers, path = line.rstrip("\n").split(":", 2) + if controllers == "": + candidates.append(f"/sys/fs/cgroup{path}/memory.max") + elif "memory" in controllers.split(","): + candidates.append(f"/sys/fs/cgroup/memory{path}/memory.limit_in_bytes") + except (OSError, ValueError): + pass + candidates.append("/sys/fs/cgroup/memory/memory.limit_in_bytes") + for path in candidates: + value = _read_limit(path) + if value: + return value + return None + + +class Limits: + def __init__(self) -> None: + self.enabled = env_bool("LIMITS_ENABLED", True) + # Bounded request queue: MAX_IN_FLIGHT requests are processed at once, up to + # MAX_QUEUE more wait for a slot at most QUEUE_TIMEOUT_MS; the rest get 503 at once. + self.max_in_flight = env_int("MAX_IN_FLIGHT", 8, 1, 10_000) + self.max_queue = env_int("MAX_QUEUE", 64, 0, 100_000) + self.queue_timeout = env_int("QUEUE_TIMEOUT_MS", 1000, 1, 60_000) / 1000 + self._slots = threading.BoundedSemaphore(self.max_in_flight) + self.waiting = 0 + self.max_connections = env_int("MAX_CONNECTIONS", 256, 1, 100_000) + self.user_write_rate = env_int("USER_WRITE_RATE", 50, 1, 100_000) # tokens per second + self.user_write_burst = env_int("USER_WRITE_BURST", 100, 1, 100_000) + configured = env_int("MEMORY_LIMIT_BYTES", 0, 0, 1 << 50) + self.memory_limit = configured or detect_memory_limit() + self.hard_percent = env_int("MEMORY_HARD_PERCENT", 90, 10, 99) + self.rss = read_rss() or 0 + self._lock = threading.Lock() + self._buckets: Dict[UUID, List[float]] = {} + + # ----- description / stats ------------------------------------------------ + + def describe(self) -> Dict[str, object]: + return { + "limits_enabled": self.enabled, + "max_in_flight": self.max_in_flight, + "max_queue": self.max_queue, + "queue_timeout_ms": int(self.queue_timeout * 1000), + "max_connections": self.max_connections, + "user_write_rate": self.user_write_rate, + "user_write_burst": self.user_write_burst, + "memory_limit_bytes": self.memory_limit, + "memory_hard_percent": self.hard_percent, + } + + def _over(self, percent: int) -> bool: + return bool(self.enabled and self.memory_limit and self.rss >= self.memory_limit * percent / 100) + + # ----- checks (each raises ApiError when the limit is exceeded) ------------ + + def acquire_slot(self) -> bool: + """Admission for every non-health request, before any work is done. + + Returns True when a processing slot was taken (the caller must call + release_slot()), False when limits are off. Raises 503 when overloaded. + """ + if not self.enabled: + return False + if self._over(self.hard_percent): + raise ApiError(503, "unavailable", "memory limit reached, retry later", retry_after=5) + if self._slots.acquire(blocking=False): + return True + with self._lock: + queue_full = self.waiting >= self.max_queue + if not queue_full: + self.waiting += 1 + if queue_full: + raise ApiError(503, "unavailable", "server is overloaded, retry later", retry_after=1) + try: + acquired = self._slots.acquire(timeout=self.queue_timeout) + finally: + with self._lock: + self.waiting -= 1 + if not acquired: + raise ApiError(503, "unavailable", "server is overloaded, retry later", retry_after=1) + return True + + def release_slot(self) -> None: + self._slots.release() + + def ready(self) -> bool: + return not self._over(self.hard_percent) + + def check_rate(self, user_id: UUID) -> None: + """Token bucket per user for write requests.""" + if not self.enabled: + return + now = time.monotonic() + with self._lock: + bucket = self._buckets.get(user_id) + if bucket is None: + bucket = self._buckets[user_id] = [float(self.user_write_burst), now] + tokens = min(self.user_write_burst, bucket[0] + (now - bucket[1]) * self.user_write_rate) + bucket[1] = now + if tokens >= 1: + bucket[0] = tokens - 1 + return + bucket[0] = tokens + wait = max(1, math.ceil((1 - tokens) / self.user_write_rate)) + raise ApiError(429, "rate_limited", "too many writes, slow down", retry_after=wait) + + # ----- background sampler ------------------------------------------------- + + def start_sampler(self) -> None: + """Refresh RSS for request admission and readiness.""" + + def loop() -> None: + while True: + rss = read_rss() + if rss is not None: + self.rss = rss + time.sleep(0.1) + + threading.Thread(target=loop, name="limits-sampler", daemon=True).start() diff --git a/src/messenger/logs.py b/src/messenger/logs.py new file mode 100644 index 00000000..16cfe779 --- /dev/null +++ b/src/messenger/logs.py @@ -0,0 +1,27 @@ +"""Structured logs: one JSON object per line on stdout. + +Logs never contain message texts, request bodies, query strings or headers. +""" + +from __future__ import annotations + +import json +import sys +import threading +from datetime import datetime, timezone +from typing import Any + +_lock = threading.Lock() + + +def log(level: str, event: str, **fields: Any) -> None: + record = { + "ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + "level": level, + "event": event, + **fields, + } + line = json.dumps(record, ensure_ascii=False, default=str) + with _lock: + sys.stdout.write(line + "\n") + sys.stdout.flush() diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..0d79f9a4 --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,31 @@ +"""Apply versioned migrations under one PostgreSQL transaction and lock.""" + +from pathlib import Path + +import psycopg + +from .config import postgres_options + + +def main() -> None: + migrations = Path(__file__).with_name("migrations").glob("*.sql") + with psycopg.connect(**postgres_options()) as conn: + conn.execute("SELECT pg_advisory_xact_lock(202602)") + conn.execute(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT clock_timestamp() + ) + """) + applied = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + pending = [] + for path in sorted(migrations): + if path.stem not in applied: + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.stem,)) + pending.append(path.stem) + print("Applied: " + ", ".join(pending) if pending else "Schema is up to date") + + +if __name__ == "__main__": + main() diff --git a/src/messenger/migrations/001_initial.sql b/src/messenger/migrations/001_initial.sql new file mode 100644 index 00000000..9c298a2c --- /dev/null +++ b/src/messenger/migrations/001_initial.sql @@ -0,0 +1,34 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash bytea NOT NULL, + password_salt bytea NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position integer NOT NULL CHECK (position BETWEEN 1 AND 100), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); +CREATE INDEX chat_members_by_user ON chat_members (user_id, chat_id); +CREATE INDEX chats_order ON chats (created_at, id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id) +); +CREATE INDEX messages_history ON messages (chat_id, created_at, id); diff --git a/src/messenger/migrations/002_message_requests.sql b/src/messenger/migrations/002_message_requests.sql new file mode 100644 index 00000000..0f90688d --- /dev/null +++ b/src/messenger/migrations/002_message_requests.sql @@ -0,0 +1,10 @@ +CREATE TABLE message_requests ( + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + idem_key text NOT NULL CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED, + expires_at timestamptz NOT NULL, + PRIMARY KEY (chat_id, sender_id, idem_key) +); + +CREATE INDEX message_requests_expiry ON message_requests (expires_at); diff --git a/src/messenger/pagination.py b/src/messenger/pagination.py new file mode 100644 index 00000000..534dc1fd --- /dev/null +++ b/src/messenger/pagination.py @@ -0,0 +1,45 @@ +"""The lab 1 signed cursor with a stable key and a PostgreSQL sort position.""" + +import base64 +import binascii +import hashlib +import hmac +import json +from datetime import datetime +from uuid import UUID + +from .errors import bad_request + + +class Cursors: + def __init__(self, secret: bytes): + self.secret = secret + + def _sign(self, payload: bytes) -> bytes: + return hmac.new(self.secret, payload, hashlib.sha256).digest()[:16] + + def decode(self, scope: str, cursor: str) -> tuple[datetime, UUID]: + try: + padded = cursor + "=" * (-len(cursor) % 4) + raw = base64.b64decode(padded, altchars=b"-_", validate=True) + except (binascii.Error, ValueError): + raise bad_request("invalid cursor") from None + payload, signature = raw[:-16], raw[-16:] + if not payload or not hmac.compare_digest(signature, self._sign(payload)): + raise bad_request("invalid cursor") + cursor_scope, timestamp, object_id = json.loads(payload) + if cursor_scope != scope: + raise bad_request("cursor belongs to another resource") + return datetime.fromisoformat(timestamp), UUID(object_id) + + def page(self, scope: str, items: list[dict], limit: int) -> dict: + cursor = None + if len(items) > limit: + items = items[:limit] + last = items[-1] + payload = json.dumps( + [scope, last["created_at"].isoformat(), str(last["id"])], + separators=(",", ":"), + ).encode() + cursor = base64.urlsafe_b64encode(payload + self._sign(payload)).decode().rstrip("=") + return {"items": items, "next_cursor": cursor} diff --git a/src/messenger/sessions.py b/src/messenger/sessions.py new file mode 100644 index 00000000..f64004ec --- /dev/null +++ b/src/messenger/sessions.py @@ -0,0 +1,96 @@ +"""Basic credentials, scrypt password hashes and opaque Redis sessions.""" + +import base64 +import binascii +import hashlib +import hmac +import os +import re +import secrets +import time +from contextlib import contextmanager +from datetime import datetime, timezone +from uuid import UUID + +from redis import Redis +from redis.backoff import NoBackoff +from redis.exceptions import RedisError +from redis.retry import Retry + +from .errors import unauthorized, unavailable +from .validation import USERNAME_RE, valid_password + +TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{43}") + + +def credentials(header: str) -> tuple[str, str]: + try: + decoded = base64.b64decode(header, validate=True).decode("utf-8") + except (binascii.Error, ValueError): + raise unauthorized() from None + username, _, password = decoded.partition(":") + if not USERNAME_RE.fullmatch(username) or not valid_password(password): + raise unauthorized() + return username, password + + +def password_digest(password: str, salt: bytes) -> bytes: + return hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=32768, r=8, p=3, dklen=32, + maxmem=64 * 1024 * 1024, + ) + + +def verify_password(password: str, user: dict | None) -> bool: + # Do equivalent KDF work for unknown usernames. + salt, expected = (user["password_salt"], user["password_hash"]) if user else (bytes(16), bytes(32)) + match = hmac.compare_digest(password_digest(password, salt), expected) + return user is not None and match + + +@contextmanager +def redis_operation(): + try: + yield + except RedisError: + raise unavailable("Redis unavailable") from None + + +class Sessions: + def __init__(self, ttl: int): + self.ttl = ttl + self.redis = Redis.from_url( + os.environ.get("REDIS_URL", "redis://redis:6379/0"), + socket_connect_timeout=1, socket_timeout=1, max_connections=16, + retry=Retry(NoBackoff(), 0), decode_responses=True, + ) + + @staticmethod + def key(token: str) -> str: + if not TOKEN_RE.fullmatch(token): + raise unauthorized() + return "lab2:session:" + hashlib.sha256(token.encode("ascii")).hexdigest() + + def ready(self) -> None: + with redis_operation(): + self.redis.ping() + + def create(self, user_id: UUID) -> tuple[str, datetime]: + token = secrets.token_urlsafe(32) + expires_ms = time.time_ns() // 1_000_000 + self.ttl * 1000 + with redis_operation(): + if not self.redis.set(self.key(token), str(user_id), pxat=expires_ms, nx=True): + raise unavailable("could not create session") + return token, datetime.fromtimestamp(expires_ms / 1000, tz=timezone.utc) + + def identify(self, token: str) -> UUID: + with redis_operation(): + user_id = self.redis.get(self.key(token)) + if user_id is None: + raise unauthorized() + return UUID(user_id) + + def revoke(self, token: str) -> None: + with redis_operation(): + if self.redis.delete(self.key(token)) == 0: + raise unauthorized() diff --git a/src/messenger/store.py b/src/messenger/store.py new file mode 100644 index 00000000..910f5b8b --- /dev/null +++ b/src/messenger/store.py @@ -0,0 +1,167 @@ +"""PostgreSQL storage for users, chats, membership and message history.""" + +from contextlib import contextmanager +from uuid import UUID, uuid4 + +from psycopg import OperationalError +from psycopg.errors import ForeignKeyViolation, LockNotAvailable, QueryCanceled +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, TooManyRequests + +from .config import postgres_options +from .errors import bad_request, conflict, not_found, unavailable +from .pagination import Cursors + + +class Store: + def __init__(self, cursor_secret: bytes, idempotency_ttl: int): + self.cursors = Cursors(cursor_secret) + self.idempotency_ttl = idempotency_ttl + self.pool = ConnectionPool( + kwargs={**postgres_options(), "row_factory": dict_row}, + min_size=1, max_size=8, max_waiting=16, timeout=2, open=False, + ) + self.pool.open() + + @contextmanager + def connection(self): + try: + # Commits on success and rolls back on any exception before reuse. + with self.pool.connection() as conn: + yield conn + except (OperationalError, TooManyRequests, LockNotAvailable, QueryCanceled): + raise unavailable("PostgreSQL unavailable") from None + + def ready(self) -> None: + with self.connection() as conn: + conn.execute("SELECT 1") + + def create_user(self, username: str, display_name: str, salt: bytes, password_hash: bytes) -> dict: + with self.connection() as conn: + user = conn.execute(""" + INSERT INTO users (username, display_name, password_salt, password_hash) + VALUES (%s, %s, %s, %s) + ON CONFLICT (username) DO NOTHING + RETURNING id, username, display_name, created_at + """, (username, display_name, salt, password_hash)).fetchone() + if user is None: + raise conflict("username is already taken") + return user + + def get_user(self, user_id: UUID) -> dict | None: + with self.connection() as conn: + return conn.execute(""" + SELECT id, username, display_name, created_at FROM users WHERE id = %s + """, (user_id,)).fetchone() + + def credentials(self, username: str) -> dict | None: + with self.connection() as conn: + return conn.execute(""" + SELECT id, username, display_name, created_at, password_salt, password_hash + FROM users WHERE username = %s + """, (username,)).fetchone() + + def create_chat(self, user_id: UUID, title: str, member_ids: list[UUID]) -> dict: + if user_id not in member_ids: + raise bad_request("member_ids must include the current user") + try: + with self.connection() as conn: + chat = conn.execute(""" + INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at + """, (title,)).fetchone() + conn.execute(""" + INSERT INTO chat_members (chat_id, user_id, position) + SELECT %s, user_id, position + FROM unnest(%s::uuid[]) WITH ORDINALITY AS members(user_id, position) + """, (chat["id"], member_ids)) + except ForeignKeyViolation: + raise bad_request("member_ids contains an unknown user") from None + return {**chat, "member_ids": member_ids} + + def get_chat(self, user_id: UUID, chat_id: UUID) -> dict: + with self.connection() as conn: + chat = conn.execute(""" + SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m + WHERE m.chat_id = c.id ORDER BY m.position) AS member_ids + FROM chat_members member JOIN chats c ON c.id = member.chat_id + WHERE member.user_id = %s AND c.id = %s + """, (user_id, chat_id)).fetchone() + if chat is None: + raise not_found("chat not found") + return chat + + def list_chats(self, user_id: UUID, limit: int, cursor: str | None) -> dict: + scope = f"chats:{user_id}" + query = """ + SELECT c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m + WHERE m.chat_id = c.id ORDER BY m.position) AS member_ids + FROM chat_members member JOIN chats c ON c.id = member.chat_id + WHERE member.user_id = %s + """ + params = (user_id,) + if cursor is not None: + query += " AND (c.created_at, c.id) > (%s, %s)" + params += self.cursors.decode(scope, cursor) + with self.connection() as conn: + items = conn.execute( + query + " ORDER BY c.created_at, c.id LIMIT %s", (*params, limit + 1) + ).fetchall() + return self.cursors.page(scope, items, limit) + + def add_message(self, user_id: UUID, chat_id: UUID, key: str, text: str) -> dict: + scope = (chat_id, user_id, key) + message_id = uuid4() + with self.connection() as conn: + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id) + ).fetchone() + if member is None: + raise not_found("chat not found") + + # PostgreSQL serializes contenders on the same primary key. The + # deferred FK allows reserving the key before inserting the message. + claimed = conn.execute(""" + INSERT INTO message_requests (chat_id, sender_id, idem_key, message_id, expires_at) + VALUES (%s, %s, %s, %s, clock_timestamp() + make_interval(secs => %s)) + ON CONFLICT (chat_id, sender_id, idem_key) DO UPDATE SET + message_id = EXCLUDED.message_id, + expires_at = clock_timestamp() + make_interval(secs => %s) + WHERE message_requests.expires_at <= clock_timestamp() + RETURNING message_id + """, (*scope, message_id, self.idempotency_ttl, self.idempotency_ttl)).fetchone() + if claimed is not None: + return conn.execute(""" + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES (%s, %s, %s, %s) + RETURNING id, chat_id, sender_id, text, created_at + """, (message_id, chat_id, user_id, text)).fetchone() + + # A conflicting live row stays locked until we finish reading its result. + message = conn.execute(""" + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at + FROM message_requests r JOIN messages m ON m.id = r.message_id + WHERE r.chat_id = %s AND r.sender_id = %s AND r.idem_key = %s + """, scope).fetchone() + if message["text"] != text: + raise conflict("Idempotency-Key already used for a different message") + return message + + def list_messages(self, user_id: UUID, chat_id: UUID, limit: int, cursor: str | None) -> dict: + scope = f"messages:{chat_id}" + query = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = %s" + params = (chat_id,) + with self.connection() as conn: + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id) + ).fetchone() + if member is None: + raise not_found("chat not found") + if cursor is not None: + query += " AND (created_at, id) > (%s, %s)" + params += self.cursors.decode(scope, cursor) + items = conn.execute( + query + " ORDER BY created_at, id LIMIT %s", (*params, limit + 1) + ).fetchall() + return self.cursors.page(scope, items, limit) diff --git a/src/messenger/validation.py b/src/messenger/validation.py new file mode 100644 index 00000000..707e343b --- /dev/null +++ b/src/messenger/validation.py @@ -0,0 +1,129 @@ +"""Strict request validation: bodies, fields, path and query parameters.""" + +from __future__ import annotations + +import json +import re +from typing import Any, Dict, Iterable, List, Optional +from uuid import UUID + +from .errors import bad_request + +UUID_RE = re.compile( + r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" +) +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +DIGITS_RE = re.compile(r"[0-9]{1,3}") + +DEFAULT_LIMIT = 50 +MAX_LIMIT = 100 +MAX_CURSOR_LENGTH = 512 + + +def is_uuid(value: Any) -> bool: + return isinstance(value, str) and UUID_RE.fullmatch(value) is not None + + +def parse_uuid(value: Any, what: str) -> UUID: + if not is_uuid(value): + raise bad_request(f"{what} must be a UUID") + return UUID(value) + + +def _reject_constant(_name: str) -> Any: + raise ValueError("NaN/Infinity are not valid JSON") + + +def _object_without_duplicates(pairs: List[tuple]) -> Dict[str, Any]: + result: Dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate key in JSON object") + result[key] = value + return result + + +def parse_json_object(raw: bytes, allowed: Iterable[str], required: Iterable[str]) -> Dict[str, Any]: + """Parse a JSON object body. Malformed JSON, extra or missing fields -> 400.""" + try: + data = json.loads( + raw.decode("utf-8"), + object_pairs_hook=_object_without_duplicates, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, ValueError, RecursionError): + raise bad_request("request body must be valid UTF-8 JSON") from None + if not isinstance(data, dict): + raise bad_request("request body must be a JSON object") + allowed_set = set(allowed) + if set(data) - allowed_set: + raise bad_request("request body contains unexpected fields") + missing = [name for name in required if name not in data] + if missing: + raise bad_request("missing required fields: " + ", ".join(missing)) + return data + + +def valid_text(value: Any, minimum: int, maximum: int) -> bool: + if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip(): + return False + try: + value.encode("utf-8") # rejects lone surrogates such as "\\ud800" + except UnicodeEncodeError: + return False + return True + + +def valid_password(value: Any) -> bool: + return valid_text(value, 12, 128) + + +def text_field(data: Dict[str, Any], name: str, max_length: int) -> str: + value = data[name] + if not valid_text(value, 1, max_length) or "\x00" in value: + raise bad_request(f"{name} must contain 1..{max_length} valid nonblank characters") + return value + + +def username_field(data: Dict[str, Any]) -> str: + value = data["username"] + if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None: + raise bad_request("username must match [a-z0-9_]{3,32}") + return value + + +def member_ids_field(data: Dict[str, Any]) -> List[UUID]: + value = data["member_ids"] + if not isinstance(value, list) or not 1 <= len(value) <= 100: + raise bad_request("member_ids must be an array of 1..100 user ids") + members = [parse_uuid(item, "member_ids item") for item in value] + if len(set(members)) != len(members): + raise bad_request("member_ids must not contain duplicates") + return members + + +def _single(params: Dict[str, List[str]], name: str) -> Optional[str]: + values = params.get(name) + if values is None: + return None + if len(values) != 1: + raise bad_request(f"{name} must be given at most once") + return values[0] + + +def limit_param(params: Dict[str, List[str]]) -> int: + raw = _single(params, "limit") + if raw is None: + return DEFAULT_LIMIT + if DIGITS_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT: + raise bad_request(f"limit must be an integer 1..{MAX_LIMIT}") + return int(raw) + + +def cursor_param(params: Dict[str, List[str]]) -> Optional[str]: + raw = _single(params, "cursor") + if raw is None: + return None + if not raw or len(raw) > MAX_CURSOR_LENGTH: + raise bad_request("invalid cursor") + return raw diff --git a/student-tests/grade4.py b/student-tests/grade4.py new file mode 100644 index 00000000..c86f15b4 --- /dev/null +++ b/student-tests/grade4.py @@ -0,0 +1,265 @@ +"""Measure replica failures and replacement while clients write through HAProxy.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from http.client import HTTPException +import json +import math +import os +from pathlib import Path +import subprocess +import sys +from threading import Barrier, Event, Lock +import time +from typing import NamedTuple +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +sys.path.insert(0, str(ROOT / "scripts")) +from client import Client, Failure, arguments, check, run +from replicas import compose, docker, replicas, stop +from resilience import history + +WORKERS = 4 +PAUSE_SECONDS = 0.12 +WARMUP_SECONDS = 1 +BASELINE_SECONDS = 2 +OUTAGE_SECONDS = 6 + + +def wait_routed(client, actor, expected): + seen = set() + deadline = time.monotonic() + 25 + while time.monotonic() < deadline: + try: + response = client.call("GET", "/api/v1/users/me", actor=actor, schema="User") + except Failure: + time.sleep(0.1) + continue + check(response["json"] == actor["user"], "session resolved to the wrong user") + seen.add(response["headers"]["X-Instance-Id"]) + if expected <= seen: + return + time.sleep(0.05) + raise Failure(f"LB did not route the shared session to expected replicas: {sorted(expected - seen)}") + + +def race(client, actor, other, chat): + key = uuid4().hex + text = "parallel replay " + key + count = 8 + barrier = Barrier(count, timeout=10) + + def send(_): + worker = Client(client.base, timeout=5) + barrier.wait() + return worker.message(actor, chat, text, key=key) + + with ThreadPoolExecutor(max_workers=count) as pool: + responses = list(pool.map(send, range(count))) + message = responses[0]["json"] + seen = {response["headers"]["X-Instance-Id"] for response in responses} + check(len(seen) >= 2, "parallel requests did not reach different replicas") + check(all(response["json"] == message for response in responses), "replay changed the result") + check([m for m in history(client, actor, chat) if m["text"] == text] == [message], + "parallel replay must store exactly one message") + + path = "/api/v1/chats/" + chat["id"] + "/messages" + client.call("POST", path, 409, actor=actor, body={"text": "different " + key}, + headers={"Idempotency-Key": key}) + another = client.message(other, chat, text, key=key)["json"] + check(another["id"] != message["id"], "the key must be scoped to the sender") + return {"requests": count, "replicas": sorted(seen), "original_messages": 1, + "different_body_status": 409, "same_key_for_another_sender": 201} + + +class Attempt(NamedTuple): + started: float + finished: float + instance: str | None + message: dict | None + + +class Traffic: + def __init__(self, client, actor, chat): + self.base = client.base + self.actor = actor + self.chat = chat + self.done = Event() + self.lock = Lock() + self.attempts = [] + + def worker(self): + client = Client(self.base, timeout=2) + key = uuid4().hex + while not self.done.is_set(): + started = time.monotonic() + try: + response = client.message(self.actor, self.chat, "load " + key, key=key) + except (Failure, OSError, HTTPException): + attempt = Attempt(started, time.monotonic(), None, None) + else: + attempt = Attempt(started, time.monotonic(), response["headers"]["X-Instance-Id"], + response["json"]) + key = uuid4().hex + with self.lock: + self.attempts.append(attempt) + self.done.wait(PAUSE_SECONDS) + + def window(self, start, end): + with self.lock: + return [attempt for attempt in self.attempts if start <= attempt.finished < end] + + def verify(self, client): + stored = history(client, self.actor, self.chat) + by_id = {message["id"]: message for message in stored} + check(len({message["text"] for message in stored}) == len(stored), "a retry created a duplicate") + acknowledged = [attempt.message for attempt in self.attempts if attempt.message is not None] + check(all(by_id.get(message["id"]) == message for message in acknowledged), + "an acknowledged message was lost") + return {"acknowledged_messages": len(acknowledged), "stored_messages": len(stored)} + + +@contextmanager +def traffic(client, actor, chat): + load = Traffic(client, actor, chat) + with ThreadPoolExecutor(max_workers=WORKERS) as pool: + workers = [pool.submit(load.worker) for _ in range(WORKERS)] + try: + yield load + finally: + load.done.set() + for worker in workers: + worker.result() + + +def metrics(load, start, end): + attempts = load.window(start, end) + success = [attempt for attempt in attempts if attempt.message is not None] + check(bool(success), "no successful writes during the measured window") + latencies = sorted((attempt.finished - attempt.started) * 1000 for attempt in success) + return { + "seconds": round(end - start, 3), + "successful_writes": len(success), + "failed_attempts": len(attempts) - len(success), + "successful_rps": round(len(success) / (end - start), 2), + "latency_ms": { + f"p{p}": round(latencies[math.ceil(p * len(latencies) / 100) - 1], 2) + for p in (50, 95, 99) + }, + "serving_aliases": sorted({attempt.instance for attempt in success}), + } + + +def first_write(load, started, stopped, end, survivor_aliases): + writes = [attempt.finished for attempt in load.window(stopped, end) + if attempt.started >= stopped and attempt.instance in survivor_aliases] + check(bool(writes), "no confirmed write through a surviving replica after the stop") + elapsed = min(writes) - started + check(elapsed <= 10, "writes did not recover within 10 seconds") + return round(elapsed, 3) + + +def failover(client, actor, load, initial): + for target in initial: + survivors = {replica.alias for replica in initial if replica.id != target.id} + started = time.monotonic() + try: + stop(target) + stopped = time.monotonic() + state = next(replica.status for replica in replicas() if replica.id == target.id) + check(state == "exited", "target is still running") + wait_routed(client, actor, survivors) + first_read = time.monotonic() - started + check(first_read <= 10, "reads did not recover within 10 seconds") + time.sleep(OUTAGE_SECONDS) + end = time.monotonic() + during = metrics(load, started, end) + check(target.alias not in metrics(load, stopped + 2, end)["serving_aliases"], + "stopped replica still serves traffic") + recovered = first_write(load, started, stopped, end, survivors) + finally: + docker("start", target.id) + wait_routed(client, actor, survivors | {target.alias}) + yield {"stopped": target.name, "target_alias": target.alias, "target_status": state, + "first_survivor_read_seconds": round(first_read, 3), + "first_survivor_write_seconds": recovered, **during} + + +def roll(client, actor, load, initial, new_image): + count = str(len(initial)) + for target in initial: + before = replicas() + survivors = [replica for replica in before if replica.id != target.id] + survivor_aliases = {replica.alias for replica in survivors} + started = time.monotonic() + try: + stop(target) + stopped = time.monotonic() + docker("rm", target.id) + compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={count}", "api") + after = replicas() + added = [replica for replica in after if replica.id not in {item.id for item in before}] + check(len(added) == 1 and added[0].image == new_image, "expected one replacement from the new image") + check({item.id for item in survivors} <= {item.id for item in after}, "a survivor was replaced") + new = added[0] + wait_routed(client, actor, survivor_aliases | {new.alias}) + ready = time.monotonic() + time.sleep(1) + except (Failure, OSError, subprocess.SubprocessError): + compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={count}", "api") + raise + end = time.monotonic() + during = metrics(load, started, end) + check(new.alias in during["serving_aliases"], "new replica did not accept writes") + recovered = first_write(load, started, stopped, end, survivor_aliases) + yield {"removed": target.name, "old_alias": target.alias, "old_image": target.image, + "replacement_alias": new.alias, "new_image": new.image, + "replacement_routed_seconds": round(ready - started, 3), + "first_survivor_write_seconds": recovered, **during} + check(all(replica.image == new_image for replica in replicas()), "rollout left old images running") + + +def main(): + def configure(parser): + parser.add_argument("--mode", choices=("race", "failover", "rolling"), required=True) + parser.add_argument("--output", type=Path) + + args = arguments("Lab 3 grade 4 checks", configure) + client = Client(args.base_url or os.environ.get("BASE_URL") or "http://127.0.0.1:8081", timeout=5) + actor, other, _, chat = client.bootstrap() + initial = replicas() + check(len(initial) >= 2 and all(replica.status == "running" for replica in initial), + "start all API replicas before the experiment") + wait_routed(client, actor, {replica.alias for replica in initial}) + result = {"mode": args.mode, "initial_replicas": [replica._asdict() for replica in initial]} + if args.mode == "race": + result.update(race(client, actor, other, chat)) + else: + if args.mode == "rolling": + revision = uuid4().hex[:12] + compose("build", "--build-arg", f"BUILD_REVISION={revision}", "api", timeout=180) + new_image = docker("image", "inspect", "bruh-lab3-api:local", "--format", "{{.Id}}") + check(new_image not in {replica.image for replica in initial}, "rollout requires a new image") + with traffic(client, actor, chat) as load: + time.sleep(WARMUP_SECONDS) + start = time.monotonic() + time.sleep(BASELINE_SECONDS) + result["baseline"] = metrics(load, start, time.monotonic()) + if args.mode == "failover": + result["steps"] = list(failover(client, actor, load, initial)) + else: + result["steps"] = list(roll(client, actor, load, initial, new_image)) + result.update(load.verify(client), workers=WORKERS, pause_seconds=PAUSE_SECONDS, + warmup_seconds=WARMUP_SECONDS) + text = json.dumps(result, indent=2) + if args.output: + args.output.write_text(text + "\n", encoding="utf-8") + print(text) + + +if __name__ == "__main__": + run(main)