From 935307d4129448ae27d57ec61d5485a140ec144b Mon Sep 17 00:00:00 2001 From: drunkdiente <145261179+drunkdiente@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:29:02 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D1=82=D1=8C=20=D0=9B=D0=A04:=20S3-=D1=85=D1=80?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D0=BB=D0=B8=D1=89=D0=B5=20=D0=B8=20=D0=B2?= =?UTF-8?q?=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=BD=D0=B0=20?= =?UTF-8?q?=D0=BE=D1=86=D0=B5=D0=BD=D0=BA=D1=83=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Стенд: PostgreSQL 16 + две API-реплики за HAProxy + SeaweedFS S3 в приватном bucket. Метаданные вложений в PostgreSQL, байты в S3, object key назначает сервер. Лимиты 10 MiB и 40 млн пикселей, MIME по магическим байтам, проверка filename. Добавлены smoke/негативные тесты, хуки restart/stop-one, сценарий отказа S3 и отчёт с доказательствами. --- .dockerignore | 2 + .env.example | 36 +- Dockerfile | 20 + Makefile | 11 +- REPORT.md | 183 ++++- app/__init__.py | 1 + app/bootstrap_s3.py | 77 ++ app/config.py | 59 ++ app/db.py | 128 ++++ app/images.py | 56 ++ app/main.py | 4 + app/security.py | 96 +++ app/server.py | 1074 ++++++++++++++++++++++++++++ app/storage.py | 55 ++ compose.yaml | 155 ++++ evidence/check-contract.txt | 1 + evidence/db-schema.txt | 31 + evidence/docker-ps.txt | 6 + evidence/failover.txt | 3 + evidence/negative.txt | 14 + evidence/persistence.txt | 2 + evidence/private-bucket.txt | 6 + evidence/s3-outage.txt | 7 + evidence/smoke.txt | 1 + evidence/unit-tests.txt | 17 + evidence/validate-openapi.txt | 1 + infra/README.md | 23 + infra/haproxy/haproxy.cfg | 47 ++ requirements.txt | 4 + scripts/check_api_error.py | 30 + scripts/evidence_s3_outage.py | 111 +++ scripts/restart.cmd | 29 + scripts/restart.sh | 30 + scripts/stop-one.cmd | 14 + scripts/stop-one.sh | 16 + scripts/verify_private_bucket.py | 50 ++ student-tests/test_api_negative.py | 196 +++++ student-tests/test_core.py | 71 ++ 38 files changed, 2641 insertions(+), 26 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/bootstrap_s3.py create mode 100644 app/config.py create mode 100644 app/db.py create mode 100644 app/images.py create mode 100644 app/main.py create mode 100644 app/security.py create mode 100644 app/server.py create mode 100644 app/storage.py create mode 100644 compose.yaml create mode 100644 evidence/check-contract.txt create mode 100644 evidence/db-schema.txt create mode 100644 evidence/docker-ps.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/negative.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/private-bucket.txt create mode 100644 evidence/s3-outage.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/unit-tests.txt create mode 100644 evidence/validate-openapi.txt create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 requirements.txt create mode 100644 scripts/check_api_error.py create mode 100644 scripts/evidence_s3_outage.py create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.sh create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/stop-one.sh create mode 100644 scripts/verify_private_bucket.py create mode 100644 student-tests/test_api_negative.py create mode 100644 student-tests/test_core.py diff --git a/.dockerignore b/.dockerignore index bc16e55..cd40abf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,4 +17,6 @@ backups __pycache__ node_modules evidence +tests +*.log .DS_Store diff --git a/.env.example b/.env.example index d99d26a..8f16064 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,36 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# The values below are placeholders, not production secrets. APP_HOST=0.0.0.0 APP_PORT=8080 +# Host port for the load balancer; the test BASE_URL must match it. +LAB4_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +API_REPLICAS=2 + +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-me-local-postgres-password +POSTGRES_DB=messenger + +# SeaweedFS administrative identity is used only by the one-shot bootstrap. +S3_ADMIN_ACCESS_KEY_ID=seaweedfs-admin +S3_ADMIN_SECRET_ACCESS_KEY=change-me-local-seaweedfs-admin-password + +# Separate S3 identity used by the API. Its SeaweedFS actions are Read/Write/List +# only; bucket creation and administration use the bootstrap identity. +S3_ENDPOINT=http://seaweedfs:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=change-me-local-api-secret + +# Must be identical for all API replicas so pagination cursors work across the LB. +CURSOR_SECRET=change-me-local-cursor-secret + MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 + +# The application also accepts *_FILE variants (for example DATABASE_URL_FILE +# and S3_SECRET_ACCESS_KEY_FILE) when running outside Compose. +# DATABASE_URL=postgresql://messenger:change-me-local-postgres-password@localhost:5432/messenger diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..66ca0d9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.12.7-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /app + +RUN groupadd --system messenger \ + && useradd --system --gid messenger --home-dir /app --shell /usr/sbin/nologin messenger + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app + +USER messenger +EXPOSE 8080 + +CMD ["python", "-m", "app.main"] diff --git a/Makefile b/Makefile index 5d86d17..3bb0e3e 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,11 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover student-tests test-negative help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "student-tests: unit tests of validation helpers; test-negative: attachment rejection scenarios against a running stand" check: $(PYTHON) scripts/check_contract.py @@ -26,5 +27,11 @@ test-persistence: $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +student-tests: + $(PYTHON) -m unittest discover -s student-tests -v + +test-negative: + $(PYTHON) student-tests/test_api_negative.py --base-url "$(BASE_URL)" diff --git a/REPORT.md b/REPORT.md index 9ad2b84..c7bbfda 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,192 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Ким Владимир, БВТ2302 +- Целевая оценка; 3 +- Commit/PR: ветка `solution/bvt2302-kim`; PR в исходный репозиторий открывается после push. +- Использованные источники, библиотеки и помощь ИИ: контракт и рубрика репозитория; документация S3 + (upload/presigned), SeaweedFS (S3 gateway, `s3.json` identities), boto3/botocore, psycopg 3, Pillow, + HAProxy. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Версии закреплены: + +| Роль | Образ | Версия | +| --- | --- | --- | +| API | `python:3.12.7-slim-bookworm` (digest `sha256:60d9996b6a8a3689d36db740b49f4327be3be09a21122bd02fb8895abb38b50d`) + `requirements.txt` | boto3 1.35.36, Pillow 11.0.0, psycopg 3.2.3, psycopg_pool 3.2.3 | +| PostgreSQL | `postgres:16.4-alpine` | 16.4 | +| S3 | `chrislusf/seaweedfs:3.71` | 3.71 | +| Load balancer | `haproxy:3.0.8-alpine` | 3.0.8 | + +```sh +# 1. Локальная конфигурация: секреты только в .env, файл в Git не попадает. +cp .env.example .env +# Отредактировать пароли/секреты. Для локального стенда достаточно синтетических значений. + +# 2. Проверка структуры и конфигурации Compose. +python scripts/check_contract.py +python scripts/validate_openapi.py # требует requirements-dev.txt +# На Windows глобальный Python может не иметь openapi-spec-validator; +# полная валидация выполнялась интерпретатором из .venv: +# .\.venv\Scripts\python.exe scripts/validate_openapi.py +docker compose --env-file .env config --quiet + +# 3. Сборка и запуск. Bucket создаёт one-shot контейнер `bootstrap`; схему +# PostgreSQL мигрирует каждая API-реплика при старте (идемпотентно, под +# advisory lock). +docker compose --env-file .env up -d --build + +# 4. Готовность (PostgreSQL + S3). LAB4_PORT по умолчанию 8080; +# на машине автора порт 8080 занят стендом lab3, поэтому в .env стоит 8081. +curl -fsS http://127.0.0.1:8081/health/ready + +# 5. Публичный smoke и собственные проверки. +python tests/smoke.py --base-url http://127.0.0.1:8081 +python -m unittest discover -s student-tests -v +python student-tests/test_api_negative.py --base-url http://127.0.0.1:8081 + +# 6. Сценарии отказов (явные, останавливают инфраструктуру). +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8081 +python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8081 +python scripts/evidence_s3_outage.py +``` + +`scripts/restart.sh` / `scripts/stop-one.sh` — POSIX-варианты для Linux/macOS (`chmod +x` нужен перед +запуском). `scripts/restart.cmd` / `scripts/stop-one.cmd` — эквиваленты для Windows: тест запускает hook +без shell interpolation, поэтому на Windows нужен исполняемый файл с расширением `.cmd`. + +Безопасная остановка (данные сохраняются): + +```sh +docker compose --env-file .env stop +``` + +Намеренное удаление учебных данных (необратимо, только для локального стенда): + +```sh +docker compose --env-file .env down -v +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +клиент ──HTTP──> HAProxy :8080 ──> api1:8080 ─┐ + │ ├──> PostgreSQL (users, chats, members, + └────────> api2:8080 ────┘ messages, attachments, idempotency) + └──> SeaweedFS S3 :8333 (приватный bucket, + только байты изображений) +``` + +- **Источник истины** — PostgreSQL. Метаданные вложения (`id`, `chat_id`, `owner_id`, `filename`, + `content_type`, `size_bytes`, `sha256`, `object_key`, `status`, `variants`, `error_code`, таймстемпы) + хранятся только там. Байты — только в приватном bucket; локального каталога с оригиналами у API нет. +- **Object key** назначает сервер: `chats///original`. `filename` — только + отображаемое имя, в путь/ключ не попадает. +- **Транзакционные границы.** Upload: проверка доступа и лимитов → проверка изображения → `PUT` в S3 → + `INSERT` метаданных в PostgreSQL. Если `INSERT` не удался, объект удаляется компенсацией + (`app/server.py`, `_upload_attachment`), поэтому неучтённых объектов не остаётся. Обратный случай + (метаданные без объекта) виден как `404` на download и не превращается в 500. +- **201 ready** возвращается только после успешной записи объекта и метаданных, то есть когда оригинал + действительно доступен любой реплике. +- **Идемпотентность сообщений.** Таблица `idempotency(user_id, chat_id, key)` + транзакционный + advisory lock по SHA-256 области уникальности; сообщение и запись результата создаются в одной + транзакции, повтор с другим body — `409`. +- **Сессии.** Непрозрачный токен (32 случайных байта, base64url), в БД только SHA-256 токена и expiry. + TTL — `SESSION_TTL_SECONDS` (по умолчанию 3600). +- **Пароли** — `scrypt` (N=2^14, r=8, p=1) с солью, сравнение `hmac.compare_digest`; для неизвестного + username выполняется фиктивная проверка, чтобы не выдавать существование пользователя по времени. +- **Курсоры пагинации** — HMAC-подписанный base64url JSON с `(created_at, id)` и привязкой к ресурсу + (`kind`, `chat_id`); чужой/испорченный курсор — `400`. +- **Readiness.** `/health/ready` проверяет PostgreSQL и S3, потому что обязательный синхронный путь + включает скачивание оригинала. При недоступности S3 реплики исключаются из LB, клиент получает `503`; + это осознанный выбор, зафиксированный в `app/server.py`. +- **Лимиты.** `Content-Length`/chunked чтение ограничены `MAX_UPLOAD_BYTES` (10 MiB) до чтения тела; + MIME проверяется по магическим байтам, число пикселей — по заголовку изображения до полного + декодирования (`MAX_IMAGE_PIXELS` = 40 млн). Полное тело ограничено 10 MiB, поэтому RAM одной + загрузки ограничена сверху; download идёт чанками по 64 KiB. +- **Инфраструктура.** SeaweedFS запускается одним процессом `weed server` с master/volume/filer/S3. + Административная идентичность (`S3_ADMIN_*`) используется только one-shot `bootstrap`; API использует + отдельную идентичность с действиями `Read`/`Write`/`List`/`Tagging`. Bucket создаётся идемпотентно + (`app/bootstrap_s3.py`: `head_bucket` → `create_bucket`, повторный запуск безопасен). Порт S3 и + внутренние порты API не публикуются на хост; наружу открыт только LB (loopback). +- **LB.** HAProxy в режиме HTTP с `option http-server-close`: без него клиентские keep-alive соединения + закреплялись за одной репликой и roundrobin не наблюдался. Имена реплик разрешаются через DNS Docker + (`resolvers docker`, `init-addr none`) с периодическим переразрешением: иначе HAProxy запоминал IP, + полученный при одновременном старте, и после пересоздания контейнера мог ходить не туда. + `timeout connect 500ms`, `retries 2`, `option redispatch` и частые health-check дают переключение + на живую реплику за единицы секунд. + +Соответствие переменных `.env.example` реальной конфигурации: + +| Переменная | Где используется | +| --- | --- | +| `APP_HOST`, `APP_PORT` | адрес прослушивания API (`app/config.py`) | +| `DATABASE_URL` | строка подключения psycopg (в Compose собирается из `POSTGRES_*`) | +| `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` | сервис PostgreSQL и сборка `DATABASE_URL` | +| `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET` | клиент S3 и bootstrap | +| `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY` | идентичность API в `s3.json` SeaweedFS | +| `S3_ADMIN_ACCESS_KEY_ID`, `S3_ADMIN_SECRET_ACCESS_KEY` | идентичность one-shot bootstrap | +| `SESSION_TTL_SECONDS`, `IDEMPOTENCY_TTL_SECONDS` | TTL сессий и окно идемпотентности | +| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | лимиты загрузки | +| `CURSOR_SECRET` | HMAC-подпись курсоров, одинаковый у всех реплик | +| `LAB4_PORT` | host-порт LB (по умолчанию 8080, у автора 8081) | + +Приложение также поддерживает `*_FILE`-варианты секретов (`DATABASE_URL_FILE`, +`S3_SECRET_ACCESS_KEY_FILE` и т.д.) на случай запуска вне Compose. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Офлайн-структура контракта | `python scripts/check_contract.py` | `PASS: offline skeleton structure...` | `evidence/check-contract.txt` | +| Полная валидация OpenAPI 3.1 | `python scripts/validate_openapi.py` (после `requirements-dev.txt`) | `PASS: full OpenAPI 3.1 and JSON Schema validation` | `evidence/validate-openapi.txt` | +| Compose-конфигурация | `docker compose --env-file .env config --quiet` | exit 0, ошибок нет | вывод команды в логе работы | +| Публичный smoke (health, users, sessions, chats, messages, pagination, idempotency, upload/download, SHA-256, чужой доступ, logout) | `python tests/smoke.py --base-url http://127.0.0.1:8081` | `PASS: lab4 public API scenario` | `evidence/smoke.txt` | +| S3-сервер в общем стенде, приватный bucket | `docker compose ps`; `python scripts/verify_private_bucket.py` (внутри сети) | объекты в bucket есть; анонимный GET объекта — `403`, анонимный list — `403` | `evidence/docker-ps.txt`, `evidence/private-bucket.txt` | +| Метаданные в PostgreSQL, байты не в БД/локальном каталоге | `\d attachments`; поиск файлов в контейнере API | таблица `attachments` без BLOB-колонок; в `/app` только код, оригиналов нет | `evidence/db-schema.txt` | +| 201 ready после сохранения; побайтное скачивание PNG и JPEG | smoke + `student-tests/test_api_negative.py` | `status=ready`, SHA-256 и размер совпадают, скачанные байты идентичны; отдельный сценарий реального JPEG 640×480 проходит загрузку и скачивание | `evidence/smoke.txt`, `evidence/negative.txt` | +| Message с `attachment_ids` читается участниками чата | `student-tests/test_api_negative.py` (сценарий `attachment_ids survive message history`) | элемент истории совпадает с ответом POST, `attachment_ids` сохранены | `evidence/negative.txt` | +| PNG/JPEG ≤10 MiB и ≤40 млн пикселей; MIME по содержимому; опасные filename; лимиты до декодирования | `python student-tests/test_api_negative.py --base-url ...` | 12/12 сценариев PASS: 413 на размер и пиксели, 415 на битый файл/подмену MIME, 400 на `/`, `\`, NUL, LF | `evidence/negative.txt` | +| Чужой чат и чужое вложение | smoke (`eve` → 404), `test_api_negative.py` | `404` на метаданные и bytes, `404` при привязке вложения другого чата, сообщение не создаётся | `evidence/smoke.txt`, `evidence/negative.txt` | +| Upload → остановка обслужившей реплики → download через другую | `tests/resilience.py --mode failover --action-script scripts/stop-one.cmd` | 2 реплики до отказа, 112 успешных записей, 0 неопределённых ошибок, восстановление 2.33 с, остановленная реплика не обслуживает, вложение скачано через другую | `evidence/failover.txt` | +| Restart стенда с сохранением volumes | `tests/resilience.py --mode persistence --action-script scripts/restart.cmd` | сессия и данные живы, оригинал скачан побайтно после рестарта всех сервисов | `evidence/persistence.txt` | +| Недоступность S3 | `python scripts/evidence_s3_outage.py` | readiness `503`, upload/download `503` (в т.ч. JSON `unavailable` с `X-Request-Id` от реплики), после старта S3 оригинал идентичен | `evidence/s3-outage.txt` | +| Unit-проверки валидаторов | `python -m unittest discover -s student-tests -v` | 6 тестов OK (пароли, курсоры, изображения, filename) | `evidence/unit-tests.txt` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Дополнительно: `make test BASE_URL=http://127.0.0.1:8081`, `make test-negative`, `make student-tests` — +те же сценарии через Makefile. `make check` требует UTF-8 режим Python на Windows (`PYTHONUTF8=1`), +иначе `contracts/openapi.json` читается в cp1251. На машине автора `make` не установлен, поэтому все +результаты выше получены прямыми командами из блока «Запуск из чистого clone»; цели Makefile — +эквивалентные обёртки для POSIX-окружения. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Аппаратура: AMD Ryzen 7 7700 (8C/16T), 31.1 GiB RAM, Docker Desktop 29.4.3, Windows 11. +Лимиты CPU/RAM контейнерам не задавались (учебный локальный стенд). Данные синтетические. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `docker stop` обслужившей API-реплики во время потока POST | LB уводит трафик на вторую реплику, подтверждённые сообщения не теряются, дублей нет | 2 реплики наблюдались до отказа; 112 успешных записей за 15 с, 0 ошибок; первая успешная запись после старта hook — 2.33 с; остановленная реплика отсутствует в последних ответах. В более ранних прогонах (до настройки `timeout connect`/health-check) было 58–59 записей и 2–4 ошибки переходного периода, дублей не возникало ни разу | восстановление 2.3 с (ранее 5.3–7.4 с); потерь подтверждённых данных нет | +| Рестарт `postgres`, `seaweedfs`, `haproxy`, `api1`, `api2` с теми же volumes | сессия, чат, сообщения, метаданные и оригинал сохраняются | `PASS: persistence preserved confirmed data and existing session`; download после рестарта побайтно совпал | простой на время рестарта (~10–20 с); потерь нет | +| Остановка S3 (`docker stop lab4-seaweedfs`) | attachment endpoints отвечают 503, метаданные остаются, данные не теряются | LB `/health/ready` → 503; upload и download → 503; прямой запрос к реплике: метаданные `200`, content `503` с `error_code=unavailable`; после старта S3 оригинал идентичен | после старта S3 готовность вернулась; потерь нет | +| Повторная загрузка с тем же `Idempotency-Key` и другим body | 409 без второго сообщения | `409 conflict` в smoke | — | + +Ограничения измерений: отдельный нагрузочный эксперимент (RPS/p50/p95/p99, пик RAM при ≥10 +одновременных загрузках) **не проводился**, так как он относится к уровню 4. Числа failover — это +наблюдения нескольких локальных прогонов, а не гарантия SLA: LB, PostgreSQL и S3 развёрнуты на одном +Docker-host, поэтому потеря хоста не выдерживается. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Заявлен и реализован уровень **3**. Инженерно закрыты отдельные практики уровня 4 (компенсация при + сбое между S3 и PostgreSQL, отдельная ограниченная идентичность API, проверка приватности bucket, + расширенные негативные тесты), но полный уровень 4 не заявляется: нет документированного reconciler + с grace period и эксперимента с ≥10 одновременными загрузками и измерением пика RAM. +- Изображение читается в память целиком, но не более 10 MiB на запрос; потоковая обработка тела + загрузки и вынесение декодирования в отдельный процесс — следующий шаг. +- Rate limiting (`429`) не реализован: схема допускает код, но рубрика уровня 3 его не требует. +- Сессии хранятся в PostgreSQL; отдельный Redis/Valkey не используется. +- Один Docker-host: отказ диска/host/сети не выдерживается; HA PostgreSQL и LB не заявляются. +- Компенсация покрывает сбой между `PUT` в S3 и `COMMIT` в PostgreSQL. Полная сверка «БД ↔ S3» + (reconciler с dry-run и grace period) — трек 2 уровня 5, он не реализован. +- Ключи и пароли в `.env` — синтетические локальные значения; `.env` в Git не попадает. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..293c3df --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Messenger API for laboratory 4.""" diff --git a/app/bootstrap_s3.py b/app/bootstrap_s3.py new file mode 100644 index 0000000..ee198af --- /dev/null +++ b/app/bootstrap_s3.py @@ -0,0 +1,77 @@ +"""Idempotent S3 bucket bootstrap using an administrative identity.""" +import os +import sys +import time + +import boto3 +from botocore.config import Config as BotoConfig +from botocore.exceptions import BotoCoreError, ClientError + + +def required(name): + value = os.environ.get(name) + if not value: + raise RuntimeError("required configuration is missing: " + name) + return value + + +def main(): + endpoint = required("S3_ENDPOINT") + region = os.environ.get("S3_REGION", "us-east-1") + bucket = required("S3_BUCKET") + access_key = required("S3_ADMIN_ACCESS_KEY_ID") + secret_key = required("S3_ADMIN_SECRET_ACCESS_KEY") + + client = boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=BotoConfig( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=2, + read_timeout=10, + retries={"max_attempts": 2, "mode": "standard"}, + ), + ) + + deadline = time.monotonic() + 90 + while True: + try: + client.head_bucket(Bucket=bucket) + break + except ClientError as error: + code = str(error.response.get("Error", {}).get("Code", "")) + if code in ("404", "NoSuchBucket", "NotFound"): + try: + client.create_bucket(Bucket=bucket) + break + except ClientError as create_error: + create_code = str( + create_error.response.get("Error", {}).get("Code", "") + ) + if create_code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + break + last_error = create_error + else: + last_error = error + except BotoCoreError as error: + last_error = error + if time.monotonic() >= deadline: + raise RuntimeError("S3 bucket bootstrap failed") from last_error + time.sleep(1) + + # SeaweedFS buckets are private by default and anonymous requests have no + # grant. Some S3 implementations close the connection for unsupported + # policy operations, so this best-effort cleanup must not fail bootstrap. + try: + client.delete_bucket_policy(Bucket=bucket) + except (BotoCoreError, ClientError): + pass + sys.stderr.write("bucket is ready: %s\n" % bucket) + + +if __name__ == "__main__": + main() diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..22da056 --- /dev/null +++ b/app/config.py @@ -0,0 +1,59 @@ +"""Runtime configuration loaded from environment variables or *_FILE secrets.""" +import os +from dataclasses import dataclass +from pathlib import Path + + +def _secret(name, default=None, required=False): + value = os.environ.get(name) + if value is None: + file_name = os.environ.get(name + "_FILE") + if file_name: + value = Path(file_name).read_text(encoding="utf-8").strip() + if value is None: + if required: + raise RuntimeError("required configuration is missing: " + name) + return default + return value + + +def _int_secret(name, default): + value = _secret(name) + return default if value is None else int(value) + + +@dataclass(frozen=True) +class Config: + database_url: str + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key_id: str + s3_secret_access_key: str + max_upload_bytes: int + max_image_pixels: int + session_ttl_seconds: int + idempotency_ttl_seconds: int + instance_id: str + cursor_secret: str + host: str + port: int + + @classmethod + def from_env(cls): + return cls( + database_url=_secret("DATABASE_URL", required=True), + s3_endpoint=_secret("S3_ENDPOINT", "http://s3:9000"), + s3_region=_secret("S3_REGION", "us-east-1"), + s3_bucket=_secret("S3_BUCKET", "messenger-attachments"), + s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=True), + s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=True), + max_upload_bytes=_int_secret("MAX_UPLOAD_BYTES", 10_485_760), + max_image_pixels=_int_secret("MAX_IMAGE_PIXELS", 40_000_000), + session_ttl_seconds=_int_secret("SESSION_TTL_SECONDS", 3600), + idempotency_ttl_seconds=_int_secret("IDEMPOTENCY_TTL_SECONDS", 86_400), + instance_id=_secret("INSTANCE_ID", "api-local"), + cursor_secret=_secret("CURSOR_SECRET", "local-dev-only-cursor-secret"), + host=_secret("APP_HOST", "0.0.0.0"), + port=_int_secret("APP_PORT", 8080), + ) diff --git a/app/db.py b/app/db.py new file mode 100644 index 0000000..5ac3c5d --- /dev/null +++ b/app/db.py @@ -0,0 +1,128 @@ +"""PostgreSQL pool and idempotent schema bootstrap.""" +import time + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +MIGRATION_LOCK = 0x4C414234 # "LAB4" + +SCHEMA = [ + """ + CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL + ) + """, + """ + CREATE TABLE IF NOT EXISTS sessions ( + token_hash TEXT PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL + ) + """, + "CREATE INDEX IF NOT EXISTS sessions_expires_idx ON sessions(expires_at)", + """ + CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL + ) + """, + """ + CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) + ) + """, + "CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members(user_id, chat_id)", + """ + CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL + ) + """, + "CREATE INDEX IF NOT EXISTS messages_chat_order_idx ON messages(chat_id, created_at, id)", + """ + CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + object_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL, + variants JSONB NOT NULL DEFAULT '[]'::jsonb, + error_code TEXT NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL + ) + """, + "CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments(chat_id)", + """ + CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id), + position INTEGER NOT NULL, + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position) + ) + """, + "CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx ON message_attachments(attachment_id)", + """ + CREATE TABLE IF NOT EXISTS idempotency ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + created_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (user_id, chat_id, key) + ) + """, + "CREATE INDEX IF NOT EXISTS idempotency_created_idx ON idempotency(created_at)", +] + + +def wait_for_database(dsn, timeout_seconds=90): + deadline = time.monotonic() + timeout_seconds + last_error = None + while time.monotonic() < deadline: + try: + with psycopg.connect(dsn, connect_timeout=3) as connection: + connection.execute("SELECT 1") + return + except psycopg.Error as error: + last_error = error + time.sleep(1) + raise RuntimeError("PostgreSQL did not become available") from last_error + + +def create_pool(dsn): + return ConnectionPool( + conninfo=dsn, + min_size=1, + max_size=10, + timeout=10, + kwargs={"row_factory": dict_row}, + open=False, + ) + + +def migrate(pool): + with pool.connection() as connection: + with connection.transaction(): + with connection.cursor() as cursor: + cursor.execute("SELECT pg_advisory_xact_lock(%s)", (MIGRATION_LOCK,)) + for statement in SCHEMA: + cursor.execute(statement) diff --git a/app/images.py b/app/images.py new file mode 100644 index 0000000..d98855d --- /dev/null +++ b/app/images.py @@ -0,0 +1,56 @@ +"""Bounded image inspection: sniff MIME, enforce pixel limit, verify decoding.""" +from io import BytesIO + +from PIL import Image, UnidentifiedImageError + +# We enforce the configured limit explicitly before any full decode. Disabling +# Pillow's own heuristic keeps the error deterministic for the API contract. +Image.MAX_IMAGE_PIXELS = None + +PNG_MAGIC = b"\x89PNG\r\n\x1a\n" +JPEG_MAGIC = b"\xff\xd8\xff" + + +class InvalidImage(Exception): + pass + + +class ImageTooLarge(Exception): + pass + + +def sniff_content_type(data): + if data.startswith(PNG_MAGIC): + return "image/png" + if data.startswith(JPEG_MAGIC): + return "image/jpeg" + return None + + +def inspect_image(data, declared_content_type, max_pixels): + """Validate bytes and return (actual_content_type, width, height).""" + actual = sniff_content_type(data) + if actual is None or actual != declared_content_type: + raise InvalidImage("unsupported or mismatched image content") + + try: + with Image.open(BytesIO(data)) as image: + width, height = image.size + if width <= 0 or height <= 0: + raise InvalidImage("invalid image dimensions") + if width * height > max_pixels: + raise ImageTooLarge("decoded pixel limit exceeded") + if image.format not in ("PNG", "JPEG"): + raise InvalidImage("unsupported image format") + image.verify() + + # verify() checks structure; load() catches truncated payloads and + # corrupt compressed streams that would otherwise reach the client. + with Image.open(BytesIO(data)) as image: + image.load() + except ImageTooLarge: + raise + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise InvalidImage("corrupt image") from None + + return actual, width, height diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..f5f6e40 --- /dev/null +++ b/app/main.py @@ -0,0 +1,4 @@ +from .server import main + +if __name__ == "__main__": + main() diff --git a/app/security.py b/app/security.py new file mode 100644 index 0000000..d5dd2d5 --- /dev/null +++ b/app/security.py @@ -0,0 +1,96 @@ +"""Password hashing, opaque sessions and signed pagination cursors.""" +import base64 +import hashlib +import hmac +import json +import secrets + +SCRYPT_N = 2 ** 14 +SCRYPT_R = 8 +SCRYPT_P = 1 +SCRYPT_DKLEN = 32 +SCRYPT_MAXMEM = 64 * 1024 * 1024 + +DUMMY_PASSWORD_HASH = None + + +def _b64encode(value): + return base64.urlsafe_b64encode(value).decode("ascii").rstrip("=") + + +def _b64decode(value): + padding = "=" * (-len(value) % 4) + return base64.urlsafe_b64decode(value + padding) + + +def hash_password(password): + salt = secrets.token_bytes(16) + digest = hashlib.scrypt( + password.encode("utf-8"), + salt=salt, + n=SCRYPT_N, + r=SCRYPT_R, + p=SCRYPT_P, + dklen=SCRYPT_DKLEN, + maxmem=SCRYPT_MAXMEM, + ) + return "scrypt${}${}${}${}${}".format( + SCRYPT_N, SCRYPT_R, SCRYPT_P, _b64encode(salt), _b64encode(digest) + ) + + +def verify_password(password, stored): + try: + algorithm, n, r, p, salt, digest = stored.split("$") + if algorithm != "scrypt": + return False + expected = _b64decode(digest) + actual = hashlib.scrypt( + password.encode("utf-8"), + salt=_b64decode(salt), + n=int(n), + r=int(r), + p=int(p), + dklen=len(expected), + maxmem=SCRYPT_MAXMEM, + ) + return hmac.compare_digest(actual, expected) + except (ValueError, TypeError): + return False + + +def dummy_password_hash(): + global DUMMY_PASSWORD_HASH + if DUMMY_PASSWORD_HASH is None: + DUMMY_PASSWORD_HASH = hash_password(secrets.token_urlsafe(24)) + return DUMMY_PASSWORD_HASH + + +def new_session_token(): + return secrets.token_urlsafe(32) + + +def token_hash(token): + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def _signature(secret, payload): + return hmac.new(secret.encode("utf-8"), payload, hashlib.sha256).hexdigest()[:32] + + +def encode_cursor(secret, payload): + raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + return _b64encode(raw) + "." + _signature(secret, raw) + + +def decode_cursor(secret, value): + if not value or "." not in value: + raise ValueError("invalid cursor") + encoded, signature = value.rsplit(".", 1) + raw = _b64decode(encoded) + if not hmac.compare_digest(_signature(secret, raw), signature): + raise ValueError("invalid cursor") + payload = json.loads(raw.decode("utf-8")) + if not isinstance(payload, dict): + raise ValueError("invalid cursor") + return payload diff --git a/app/server.py b/app/server.py new file mode 100644 index 0000000..be3aae9 --- /dev/null +++ b/app/server.py @@ -0,0 +1,1074 @@ +"""HTTP API implementation for laboratory 4.""" +import base64 +import hashlib +import json +import re +import sys +import traceback +import uuid +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, quote, unquote, urlsplit + +import psycopg +from botocore.exceptions import BotoCoreError, ClientError +from psycopg.errors import UniqueViolation +from psycopg.types.json import Jsonb + +from .config import Config +from .db import create_pool, migrate, wait_for_database +from .images import ImageTooLarge, InvalidImage, inspect_image +from .security import ( + decode_cursor, + dummy_password_hash, + encode_cursor, + hash_password, + new_session_token, + token_hash, + verify_password, +) +from .storage import Storage, is_missing_object + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +JSON_BODY_LIMIT = 1024 * 1024 +DEFAULT_PAGE_SIZE = 50 +MAX_PAGE_SIZE = 100 + +ERROR_MESSAGES = { + 400: "Invalid request", + 401: "Unauthorized", + 404: "Not found", + 409: "Conflict", + 413: "Payload too large", + 415: "Unsupported media type", + 429: "Too many requests", + 500: "Internal server error", + 503: "Service unavailable", +} + + +class ApiError(Exception): + def __init__(self, status, code, message=None): + super().__init__(message or ERROR_MESSAGES.get(status, "Error")) + self.status = status + self.code = code + self.message = message or ERROR_MESSAGES.get(status, "Error") + + +def utcnow(): + return datetime.now(timezone.utc) + + +def format_timestamp(value): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + value = value.astimezone(timezone.utc) + return value.isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def parse_timestamp(value): + if not isinstance(value, str) or not value.endswith("Z"): + raise ValueError("invalid timestamp") + parsed = datetime.fromisoformat(value[:-1] + "+00:00") + return parsed.astimezone(timezone.utc) + + +def user_json(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": format_timestamp(row["created_at"]), + } + + +def chat_json(row): + members = row.get("member_ids") or [] + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": sorted(str(member) for member in members), + "created_at": format_timestamp(row["created_at"]), + } + + +def message_json(row): + attachments = row.get("attachment_ids") or [] + if attachments == [None]: + attachments = [] + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": format_timestamp(row["created_at"]), + "attachment_ids": [str(item) for item in attachments], + } + + +def attachment_json(row): + variants = row.get("variants") or [] + if isinstance(variants, str): + variants = json.loads(variants) + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + "sha256": row["sha256"], + "status": row["status"], + "variants": variants, + "error_code": row["error_code"], + "created_at": format_timestamp(row["created_at"]), + "updated_at": format_timestamp(row["updated_at"]), + } + + +class App: + def __init__(self, config, pool, storage): + self.config = config + self.pool = pool + self.storage = storage + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server_version = "MessengerLab4/1.0" + sys_version = "" + + def log_message(self, fmt, *args): + sys.stderr.write("%s %s\n" % (self.address_string(), fmt % args)) + + @property + def app(self): + return self.server.app + + def do_GET(self): + self._handle() + + def do_POST(self): + self._handle() + + def do_DELETE(self): + self._handle() + + def do_PUT(self): + self._handle() + + def do_PATCH(self): + self._handle() + + def _handle(self): + self.request_id = uuid.uuid4().hex + self._response_started = False + self._body_consumed = False + self._path = urlsplit(self.path).path + try: + self._dispatch() + except ApiError as error: + self._send_error(error.status, error.code, error.message) + except (BrokenPipeError, ConnectionResetError): + self.close_connection = True + except Exception: + traceback.print_exc() + self._send_error(500, "internal_error", ERROR_MESSAGES[500]) + + def _common_headers(self): + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Instance-Id", self.app.config.instance_id) + + def _send_json(self, status, payload, challenge=None): + if self._response_started: + self.close_connection = True + return + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self._common_headers() + if challenge: + self.send_header("WWW-Authenticate", challenge) + if self.close_connection: + self.send_header("Connection", "close") + self.end_headers() + self._response_started = True + self.wfile.write(body) + + def _send_empty(self, status): + if self._response_started: + self.close_connection = True + return + self.send_response(status) + self.send_header("Content-Length", "0") + self._common_headers() + if self.close_connection: + self.send_header("Connection", "close") + self.end_headers() + self._response_started = True + + def _send_error(self, status, code, message): + if self._response_started: + self.close_connection = True + return + if self.command in ("POST", "PUT", "PATCH") and not self._body_consumed: + self.close_connection = True + challenge = None + if status == 401: + if self.command == "POST" and self._path == "/api/v1/auth/sessions": + challenge = 'Basic realm="messenger", charset="UTF-8"' + else: + challenge = 'Bearer realm="messenger"' + payload = { + "error": { + "code": code, + "message": message or ERROR_MESSAGES.get(status, "Error"), + "request_id": self.request_id, + } + } + self._send_json(status, payload, challenge) + + def _dispatch(self): + parsed = urlsplit(self.path) + self._path = parsed.path + self._query = parse_qs(parsed.query, keep_blank_values=True) + method = self.command + path = parsed.path + + if path == "/health/live" and method == "GET": + return self._health_live() + if path == "/health/ready" and method == "GET": + return self._health_ready() + if path == "/api/v1/users" and method == "POST": + return self._register_user() + if path == "/api/v1/users/me" and method == "GET": + return self._current_user_endpoint() + if path == "/api/v1/auth/sessions" and method == "POST": + return self._create_session() + if path == "/api/v1/auth/sessions/current" and method == "DELETE": + return self._delete_session() + if path == "/api/v1/chats" and method == "POST": + return self._create_chat() + if path == "/api/v1/chats" and method == "GET": + return self._list_chats() + + parts = [unquote(part) for part in path.strip("/").split("/") if part] + if len(parts) == 4 and parts[:3] == ["api", "v1", "chats"] and method == "GET": + return self._get_chat(self._parse_uuid(parts[3])) + if len(parts) == 5 and parts[:3] == ["api", "v1", "chats"] and parts[4] == "messages": + chat_id = self._parse_uuid(parts[3]) + if method == "GET": + return self._list_messages(chat_id) + if method == "POST": + return self._create_message(chat_id) + if len(parts) == 5 and parts[:3] == ["api", "v1", "chats"] and parts[4] == "attachments": + if method == "POST": + return self._upload_attachment(self._parse_uuid(parts[3])) + if len(parts) == 4 and parts[:3] == ["api", "v1", "attachments"] and method == "GET": + return self._get_attachment(self._parse_uuid(parts[3])) + if len(parts) == 5 and parts[:3] == ["api", "v1", "attachments"] and parts[4] == "content": + if method == "GET": + return self._download_attachment(self._parse_uuid(parts[3])) + + raise ApiError(404, "not_found", "Endpoint not found") + + # ------------------------------------------------------------------ + # request helpers + # ------------------------------------------------------------------ + def _read_body(self, max_bytes): + transfer_encoding = self.headers.get("Transfer-Encoding", "").lower() + if "chunked" in transfer_encoding: + data = self._read_chunked(max_bytes) + self._body_consumed = True + return data + + raw_length = self.headers.get("Content-Length") + if raw_length is None: + self._body_consumed = True + return b"" + try: + length = int(raw_length) + except (TypeError, ValueError): + raise ApiError(400, "invalid_request", "Invalid Content-Length") from None + if length < 0: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if length > max_bytes: + self.close_connection = True + raise ApiError(413, "payload_too_large", "Request body is too large") + data = self.rfile.read(length) + if len(data) != length: + raise ApiError(400, "invalid_request", "Incomplete request body") + self._body_consumed = True + return data + + def _read_chunked(self, max_bytes): + data = bytearray() + while True: + line = self.rfile.readline(1024) + if not line: + raise ApiError(400, "invalid_request", "Incomplete chunked body") + size_text = line.strip().split(b";", 1)[0] + try: + size = int(size_text, 16) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid chunk size") from None + if size == 0: + while True: + trailer = self.rfile.readline(1024) + if trailer in (b"\r\n", b"\n", b""): + break + break + if len(data) + size > max_bytes: + self.close_connection = True + raise ApiError(413, "payload_too_large", "Request body is too large") + chunk = self.rfile.read(size) + if len(chunk) != size: + raise ApiError(400, "invalid_request", "Incomplete chunk") + data.extend(chunk) + if self.rfile.read(2) != b"\r\n": + raise ApiError(400, "invalid_request", "Invalid chunk terminator") + return bytes(data) + + def _json_body(self): + if self.headers.get_content_type() != "application/json": + raise ApiError(400, "invalid_request", "Expected application/json") + data = self._read_body(JSON_BODY_LIMIT) + if not data: + raise ApiError(400, "invalid_request", "JSON body is required") + try: + value = json.loads(data.decode("utf-8")) + except (ValueError, UnicodeError): + raise ApiError(400, "invalid_request", "Malformed JSON") from None + if not isinstance(value, dict): + raise ApiError(400, "invalid_request", "JSON object is required") + return value + + def _check_fields(self, body, allowed): + if set(body) - set(allowed): + raise ApiError(400, "invalid_request", "Unexpected JSON field") + + def _query_value(self, name, required=False): + values = self._query.get(name) + if not values: + if required: + raise ApiError(400, "invalid_request", "Missing query parameter: " + name) + return None + if len(values) != 1: + raise ApiError(400, "invalid_request", "Repeated query parameter: " + name) + return values[0] + + def _parse_limit(self): + raw = self._query_value("limit") + if raw is None: + return DEFAULT_PAGE_SIZE + if not re.fullmatch(r"[0-9]{1,3}", raw): + raise ApiError(400, "invalid_request", "Invalid limit") + value = int(raw) + if value < 1 or value > MAX_PAGE_SIZE: + raise ApiError(400, "invalid_request", "Invalid limit") + return value + + def _parse_cursor(self): + raw = self._query_value("cursor") + if raw is None: + return None + try: + return decode_cursor(self.app.config.cursor_secret, raw) + except (ValueError, TypeError, KeyError): + raise ApiError(400, "invalid_request", "Invalid cursor") from None + + @staticmethod + def _parse_uuid(value): + try: + return uuid.UUID(value) + except (ValueError, TypeError, AttributeError): + raise ApiError(400, "invalid_request", "Invalid UUID") from None + + @staticmethod + def _require_string(value, field, minimum=0, maximum=None): + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "Invalid " + field) + if len(value) < minimum or (maximum is not None and len(value) > maximum): + raise ApiError(400, "invalid_request", "Invalid " + field) + return value + + # ------------------------------------------------------------------ + # authentication + # ------------------------------------------------------------------ + def _bearer_token(self): + header = self.headers.get("Authorization", "") + scheme, separator, value = header.partition(" ") + if not separator or scheme.lower() != "bearer" or not value.strip(): + raise ApiError(401, "unauthorized", "Bearer token is required") + return value.strip() + + def _current_user(self): + token = self._bearer_token() + with self.app.pool.connection() as connection: + row = connection.execute( + """ + SELECT u.id, u.username, u.display_name, u.created_at + FROM sessions s + JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s AND s.expires_at > now() + """, + (token_hash(token),), + ).fetchone() + if row is None: + raise ApiError(401, "unauthorized", "Invalid or expired session") + return row + + @staticmethod + def _require_member(connection, chat_id, user_id): + row = connection.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Chat not found") + + # ------------------------------------------------------------------ + # health and users + # ------------------------------------------------------------------ + def _health_live(self): + self._send_json(200, {"status": "ok"}) + + def _health_ready(self): + # Readiness covers the whole mandatory synchronous path: PostgreSQL for + # metadata/sessions and S3 for attachment bytes. A storage outage makes + # replicas unready, so the load balancer stops routing to them; this is a + # deliberate, documented choice (see REPORT.md) and keeps the public + # persistence check meaningful. + try: + with self.app.pool.connection() as connection: + connection.execute("SELECT 1") + except Exception: + raise ApiError(503, "unavailable", "Service is not ready") from None + if not self.app.storage.ready(): + raise ApiError(503, "unavailable", "Service is not ready") + self._send_json(200, {"status": "ok"}) + + def _register_user(self): + body = self._json_body() + self._check_fields(body, {"username", "display_name", "password"}) + for field in ("username", "display_name", "password"): + if field not in body: + raise ApiError(400, "invalid_request", "Missing field: " + field) + + username = self._require_string(body["username"], "username", 3, 32) + if USERNAME_RE.fullmatch(username) is None: + raise ApiError(400, "invalid_request", "Invalid username") + display_name = self._require_string(body["display_name"], "display_name", 1, 100) + if not display_name.strip(): + raise ApiError(400, "invalid_request", "Invalid display_name") + password = self._require_string(body["password"], "password", 12, 128) + + user_id = uuid.uuid4() + created_at = utcnow() + password_hash = hash_password(password) + try: + with self.app.pool.connection() as connection: + connection.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (user_id, username, display_name, password_hash, created_at), + ) + except UniqueViolation: + raise ApiError(409, "conflict", "Username is already registered") from None + self._send_json( + 201, + user_json( + { + "id": user_id, + "username": username, + "display_name": display_name, + "created_at": created_at, + } + ), + ) + + def _create_session(self): + if self.headers.get("Content-Length") not in (None, "0") or self.headers.get("Transfer-Encoding"): + self.close_connection = True + raise ApiError(400, "invalid_request", "Session creation takes no body") + header = self.headers.get("Authorization", "") + scheme, separator, encoded = header.partition(" ") + if not separator or scheme.lower() != "basic" or not encoded.strip(): + raise ApiError(401, "unauthorized", "Basic credentials are required") + try: + decoded = base64.b64decode(encoded.strip(), validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, UnicodeError): + raise ApiError(401, "unauthorized", "Invalid Basic credentials") from None + if not username or not password: + raise ApiError(401, "unauthorized", "Invalid Basic credentials") + + with self.app.pool.connection() as connection: + row = connection.execute( + """ + SELECT id, username, display_name, created_at, password_hash + FROM users WHERE username = %s + """, + (username,), + ).fetchone() + if row is None: + verify_password(password, dummy_password_hash()) + raise ApiError(401, "unauthorized", "Invalid credentials") + if not verify_password(password, row["password_hash"]): + raise ApiError(401, "unauthorized", "Invalid credentials") + + token = new_session_token() + expires_at = utcnow() + timedelta(seconds=self.app.config.session_ttl_seconds) + with self.app.pool.connection() as connection: + connection.execute( + """ + INSERT INTO sessions (token_hash, user_id, expires_at, created_at) + VALUES (%s, %s, %s, %s) + """, + (token_hash(token), row["id"], expires_at, utcnow()), + ) + self._send_json( + 201, + { + "token": token, + "token_type": "Bearer", + "expires_at": format_timestamp(expires_at), + "user": user_json(row), + }, + ) + + # Separate name from _current_user(), which is the bearer authentication helper. + def _current_user_endpoint(self): + user = self._current_user() + self._send_json(200, user_json(user)) + + def _delete_session(self): + token = self._bearer_token() + with self.app.pool.connection() as connection: + cursor = connection.execute( + "DELETE FROM sessions WHERE token_hash = %s", (token_hash(token),) + ) + deleted = cursor.rowcount + if deleted == 0: + raise ApiError(401, "unauthorized", "Invalid or expired session") + self._send_empty(204) + + # ------------------------------------------------------------------ + # chats + # ------------------------------------------------------------------ + def _create_chat(self): + user = self._current_user() + body = self._json_body() + self._check_fields(body, {"title", "member_ids"}) + if "title" not in body or "member_ids" not in body: + raise ApiError(400, "invalid_request", "title and member_ids are required") + title = self._require_string(body["title"], "title", 1, 100) + if not title.strip(): + raise ApiError(400, "invalid_request", "Invalid title") + raw_members = body["member_ids"] + if not isinstance(raw_members, list) or not 1 <= len(raw_members) <= 100: + raise ApiError(400, "invalid_request", "Invalid member_ids") + members = [] + for value in raw_members: + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "Invalid member_ids") + members.append(str(self._parse_uuid(value))) + if len(set(members)) != len(members): + raise ApiError(400, "invalid_request", "Duplicate member_ids") + if str(user["id"]) not in members: + raise ApiError(400, "invalid_request", "Caller must be a chat member") + + chat_id = uuid.uuid4() + created_at = utcnow() + member_uuids = [uuid.UUID(value) for value in members] + with self.app.pool.connection() as connection: + with connection.transaction(): + found = connection.execute( + "SELECT id FROM users WHERE id = ANY(%s)", (member_uuids,) + ).fetchall() + if len(found) != len(members): + raise ApiError(400, "invalid_request", "Unknown chat member") + connection.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (chat_id, title, created_at), + ) + with connection.cursor() as cursor: + cursor.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat_id, member_id) for member_id in member_uuids], + ) + self._send_json( + 201, + chat_json( + { + "id": chat_id, + "title": title, + "member_ids": member_uuids, + "created_at": created_at, + } + ), + ) + + def _list_chats(self): + user = self._current_user() + limit = self._parse_limit() + payload = self._parse_cursor() + condition = "" + parameters = [user["id"]] + if payload is not None: + if payload.get("kind") != "chats": + raise ApiError(400, "invalid_request", "Invalid cursor") + try: + created_at = parse_timestamp(payload["created_at"]) + chat_id = uuid.UUID(payload["id"]) + except (KeyError, ValueError, TypeError): + raise ApiError(400, "invalid_request", "Invalid cursor") from None + condition = " AND (c.created_at, c.id) > (%s, %s)" + parameters.extend([created_at, chat_id]) + parameters.append(limit + 1) + sql = ( + """ + SELECT c.id, c.title, c.created_at, + (SELECT array_agg(cm2.user_id ORDER BY cm2.user_id) + FROM chat_members cm2 WHERE cm2.chat_id = c.id) AS member_ids + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id AND cm.user_id = %s + WHERE TRUE + """ + + condition + + " ORDER BY c.created_at, c.id LIMIT %s" + ) + with self.app.pool.connection() as connection: + rows = connection.execute(sql, parameters).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + next_cursor = None + if has_more and rows: + last = rows[-1] + next_cursor = encode_cursor( + self.app.config.cursor_secret, + { + "kind": "chats", + "created_at": format_timestamp(last["created_at"]), + "id": str(last["id"]), + }, + ) + self._send_json( + 200, {"items": [chat_json(row) for row in rows], "next_cursor": next_cursor} + ) + + def _get_chat(self, chat_id): + user = self._current_user() + with self.app.pool.connection() as connection: + self._require_member(connection, chat_id, user["id"]) + row = connection.execute( + """ + SELECT c.id, c.title, c.created_at, + (SELECT array_agg(cm2.user_id ORDER BY cm2.user_id) + FROM chat_members cm2 WHERE cm2.chat_id = c.id) AS member_ids + FROM chats c WHERE c.id = %s + """, + (chat_id,), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(200, chat_json(row)) + + # ------------------------------------------------------------------ + # messages + # ------------------------------------------------------------------ + @staticmethod + def _fetch_message(connection, message_id): + row = connection.execute( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) + FROM message_attachments ma + WHERE ma.message_id = m.id), '{}') AS attachment_ids + FROM messages m WHERE m.id = %s + """, + (message_id,), + ).fetchone() + return None if row is None else message_json(row) + + def _create_message(self, chat_id): + user = self._current_user() + with self.app.pool.connection() as connection: + self._require_member(connection, chat_id, user["id"]) + + idempotency_key = self.headers.get("Idempotency-Key") + if not idempotency_key or IDEMPOTENCY_RE.fullmatch(idempotency_key) is None: + raise ApiError(400, "invalid_request", "Valid Idempotency-Key is required") + + body = self._json_body() + self._check_fields(body, {"text", "attachment_ids"}) + if "text" not in body: + raise ApiError(400, "invalid_request", "text is required") + text = self._require_string(body["text"], "text", 0, 4000) + raw_attachments = body.get("attachment_ids", []) + if not isinstance(raw_attachments, list) or len(raw_attachments) > 10: + raise ApiError(400, "invalid_request", "Invalid attachment_ids") + attachment_ids = [] + for value in raw_attachments: + if not isinstance(value, str): + raise ApiError(400, "invalid_request", "Invalid attachment_ids") + attachment_ids.append(str(self._parse_uuid(value))) + if len(set(attachment_ids)) != len(attachment_ids): + raise ApiError(400, "invalid_request", "Duplicate attachment_ids") + if not attachment_ids and not text.strip(): + raise ApiError(400, "invalid_request", "Empty message") + + request_hash = hashlib.sha256( + json.dumps( + {"text": text, "attachment_ids": attachment_ids}, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + ).hexdigest() + scope = "POST|%s|%s|%s" % (user["id"], chat_id, idempotency_key) + lock_id = int.from_bytes( + hashlib.sha256(scope.encode("utf-8")).digest()[:8], "big", signed=True + ) + + with self.app.pool.connection() as connection: + with connection.transaction(): + connection.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,)) + existing = connection.execute( + """ + SELECT request_hash, message_id FROM idempotency + WHERE user_id = %s AND chat_id = %s AND key = %s + """, + (user["id"], chat_id, idempotency_key), + ).fetchone() + if existing is not None: + if existing["request_hash"] != request_hash: + raise ApiError( + 409, "conflict", "Idempotency key was used with a different request" + ) + message = self._fetch_message(connection, existing["message_id"]) + if message is None: + raise ApiError(409, "conflict", "Idempotent result is unavailable") + else: + if attachment_ids: + rows = connection.execute( + """ + SELECT id, chat_id, status FROM attachments + WHERE id = ANY(%s) + """, + ([uuid.UUID(value) for value in attachment_ids],), + ).fetchall() + if len(rows) != len(attachment_ids): + raise ApiError(404, "not_found", "Attachment not found") + by_id = {str(row["id"]): row for row in rows} + for attachment_id in attachment_ids: + row = by_id[attachment_id] + if str(row["chat_id"]) != str(chat_id): + raise ApiError(404, "not_found", "Attachment not found") + if row["status"] == "failed": + raise ApiError(409, "conflict", "Attachment failed processing") + + message_id = uuid.uuid4() + created_at = utcnow() + connection.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (message_id, chat_id, user["id"], text, created_at), + ) + if attachment_ids: + with connection.cursor() as cursor: + cursor.executemany( + """ + INSERT INTO message_attachments + (message_id, attachment_id, position) + VALUES (%s, %s, %s) + """, + [ + (message_id, uuid.UUID(value), position) + for position, value in enumerate(attachment_ids) + ], + ) + connection.execute( + """ + INSERT INTO idempotency + (user_id, chat_id, key, request_hash, message_id, created_at) + VALUES (%s, %s, %s, %s, %s, %s) + """, + ( + user["id"], + chat_id, + idempotency_key, + request_hash, + message_id, + created_at, + ), + ) + message = self._fetch_message(connection, message_id) + self._send_json(201, message) + + def _list_messages(self, chat_id): + user = self._current_user() + with self.app.pool.connection() as connection: + self._require_member(connection, chat_id, user["id"]) + limit = self._parse_limit() + payload = self._parse_cursor() + condition = "" + parameters = [chat_id] + if payload is not None: + if payload.get("kind") != "messages" or str(payload.get("chat_id")) != str(chat_id): + raise ApiError(400, "invalid_request", "Cursor belongs to another resource") + try: + created_at = parse_timestamp(payload["created_at"]) + message_id = uuid.UUID(payload["id"]) + except (KeyError, ValueError, TypeError): + raise ApiError(400, "invalid_request", "Invalid cursor") from None + condition = " AND (m.created_at, m.id) > (%s, %s)" + parameters.extend([created_at, message_id]) + parameters.append(limit + 1) + sql = ( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) + FROM message_attachments ma + WHERE ma.message_id = m.id), '{}') AS attachment_ids + FROM messages m + WHERE m.chat_id = %s + """ + + condition + + " ORDER BY m.created_at, m.id LIMIT %s" + ) + with self.app.pool.connection() as connection: + rows = connection.execute(sql, parameters).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + next_cursor = None + if has_more and rows: + last = rows[-1] + next_cursor = encode_cursor( + self.app.config.cursor_secret, + { + "kind": "messages", + "chat_id": str(chat_id), + "created_at": format_timestamp(last["created_at"]), + "id": str(last["id"]), + }, + ) + self._send_json( + 200, {"items": [message_json(row) for row in rows], "next_cursor": next_cursor} + ) + + # ------------------------------------------------------------------ + # attachments + # ------------------------------------------------------------------ + @staticmethod + def _valid_filename(filename): + if not filename or len(filename) > 255: + return False + for character in filename: + if character in ("/", "\\") or ord(character) < 32 or ord(character) == 127: + return False + return True + + def _upload_attachment(self, chat_id): + user = self._current_user() + try: + with self.app.pool.connection() as connection: + self._require_member(connection, chat_id, user["id"]) + content_type = self.headers.get_content_type() + if content_type not in ("image/png", "image/jpeg"): + self.close_connection = True + raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are accepted") + filename = self._query_value("filename", required=True) + if not self._valid_filename(filename): + self.close_connection = True + raise ApiError(400, "invalid_request", "Invalid filename") + data = self._read_body(self.app.config.max_upload_bytes) + except ApiError: + if not self._body_consumed: + self.close_connection = True + raise + + if not data: + self.close_connection = True + raise ApiError(415, "unsupported_media_type", "Empty image") + try: + actual_content_type, _width, _height = inspect_image( + data, content_type, self.app.config.max_image_pixels + ) + except ImageTooLarge: + self.close_connection = True + raise ApiError(413, "payload_too_large", "Image has too many pixels") from None + except InvalidImage: + self.close_connection = True + raise ApiError(415, "unsupported_media_type", "Invalid or mismatched image") from None + + attachment_id = uuid.uuid4() + object_key = "chats/{}/{}/original".format(chat_id, attachment_id) + created_at = utcnow() + sha256 = hashlib.sha256(data).hexdigest() + try: + self.app.storage.put(object_key, data, actual_content_type) + except (BotoCoreError, ClientError): + raise ApiError(503, "unavailable", "Object storage is unavailable") from None + + try: + with self.app.pool.connection() as connection: + connection.execute( + """ + INSERT INTO attachments + (id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, object_key, status, variants, error_code, + created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'ready', %s, NULL, %s, %s) + """, + ( + attachment_id, + chat_id, + user["id"], + filename, + actual_content_type, + len(data), + sha256, + object_key, + Jsonb([]), + created_at, + created_at, + ), + ) + except psycopg.Error: + try: + self.app.storage.delete(object_key) + except (BotoCoreError, ClientError): + traceback.print_exc() + raise ApiError(503, "unavailable", "Database is unavailable") from None + + self._send_json( + 201, + attachment_json( + { + "id": attachment_id, + "chat_id": chat_id, + "owner_id": user["id"], + "filename": filename, + "content_type": actual_content_type, + "size_bytes": len(data), + "sha256": sha256, + "status": "ready", + "variants": [], + "error_code": None, + "created_at": created_at, + "updated_at": created_at, + } + ), + ) + + def _get_attachment(self, attachment_id): + user = self._current_user() + with self.app.pool.connection() as connection: + row = connection.execute( + """ + SELECT a.* FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s + WHERE a.id = %s + """, + (user["id"], attachment_id), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Attachment not found") + self._send_json(200, attachment_json(row)) + + def _download_attachment(self, attachment_id): + user = self._current_user() + variant = self._query_value("variant") + if variant is None: + variant = "original" + if variant != "original": + raise ApiError(400, "invalid_request", "Unsupported variant") + with self.app.pool.connection() as connection: + row = connection.execute( + """ + SELECT a.* FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s + WHERE a.id = %s + """, + (user["id"], attachment_id), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Attachment not found") + if row["status"] != "ready": + raise ApiError(409, "conflict", "Attachment is not ready") + + try: + response = self.app.storage.get(row["object_key"]) + except (BotoCoreError, ClientError) as error: + if is_missing_object(error): + raise ApiError(404, "not_found", "Attachment content not found") from None + raise ApiError(503, "unavailable", "Object storage is unavailable") from None + + body = response["Body"] + length = int(response.get("ContentLength") or row["size_bytes"]) + self.send_response(200) + self.send_header("Content-Type", row["content_type"]) + self.send_header("Content-Length", str(length)) + self.send_header( + "Content-Disposition", + "inline; filename*=UTF-8''" + quote(row["filename"], safe=""), + ) + self._common_headers() + self.end_headers() + self._response_started = True + try: + for chunk in body.iter_chunks(chunk_size=65536): + if chunk: + self.wfile.write(chunk) + except (BrokenPipeError, ConnectionResetError): + self.close_connection = True + finally: + body.close() + + +class ApiServer(ThreadingHTTPServer): + daemon_threads = True + + def handle_error(self, request, client_address): + error = sys.exc_info()[1] + if isinstance(error, (BrokenPipeError, ConnectionResetError)): + return + super().handle_error(request, client_address) + + +def make_server(app): + server = ApiServer((app.config.host, app.config.port), Handler) + server.app = app + return server + + +def main(): + config = Config.from_env() + wait_for_database(config.database_url) + pool = create_pool(config.database_url) + pool.open(wait=True, timeout=30) + try: + migrate(pool) + storage = Storage( + endpoint=config.s3_endpoint, + region=config.s3_region, + bucket=config.s3_bucket, + access_key_id=config.s3_access_key_id, + secret_access_key=config.s3_secret_access_key, + ) + app = App(config, pool, storage) + server = make_server(app) + sys.stderr.write( + "messenger lab4 listening on %s:%s as %s\n" + % (config.host, config.port, config.instance_id) + ) + server.serve_forever() + finally: + pool.close() diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 0000000..247e99e --- /dev/null +++ b/app/storage.py @@ -0,0 +1,55 @@ +"""Private S3-compatible object storage client.""" +import boto3 +from botocore.config import Config as BotoConfig +from botocore.exceptions import BotoCoreError, ClientError + + +class Storage: + def __init__(self, endpoint, region, bucket, access_key_id, secret_access_key): + self.bucket = bucket + self.client = boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=access_key_id, + aws_secret_access_key=secret_access_key, + config=BotoConfig( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=2, + read_timeout=5, + # Fail fast: a stalled storage call must not hold an API thread + # for the whole client timeout, and retries are the client's job. + retries={"max_attempts": 1, "mode": "standard"}, + ), + ) + + def ready(self): + try: + self.client.head_bucket(Bucket=self.bucket) + return True + except (BotoCoreError, ClientError): + return False + + def put(self, key, data, content_type): + self.client.put_object( + Bucket=self.bucket, + Key=key, + Body=data, + ContentType=content_type, + ContentLength=len(data), + ) + + def get(self, key): + return self.client.get_object(Bucket=self.bucket, Key=key) + + def delete(self, key): + self.client.delete_object(Bucket=self.bucket, Key=key) + + +def is_missing_object(error): + if not isinstance(error, ClientError): + return False + code = str(error.response.get("Error", {}).get("Code", "")) + status = error.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + return code in ("NoSuchKey", "NotFound", "404") or status == 404 diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..390e010 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,155 @@ +x-api-environment: &api-environment + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@postgres:5432/${POSTGRES_DB:-messenger} + S3_ENDPOINT: http://seaweedfs:8333 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?S3_ACCESS_KEY_ID is required} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?S3_SECRET_ACCESS_KEY is required} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + CURSOR_SECRET: ${CURSOR_SECRET:?CURSOR_SECRET is required} + PYTHONUNBUFFERED: "1" + +services: + postgres: + image: postgres:16.4-alpine + container_name: lab4-postgres + environment: + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required} + POSTGRES_DB: ${POSTGRES_DB:-messenger} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 2s + timeout: 3s + retries: 30 + restart: unless-stopped + networks: [lab4] + + seaweedfs: + image: chrislusf/seaweedfs:3.71 + container_name: lab4-seaweedfs + entrypoint: ["/bin/sh", "-c"] + command: + - | + mkdir -p /tmp/seaweedfs + cat > /tmp/seaweedfs/s3.json <8080/tcp +postgres lab4-postgres Up About a minute (healthy) 5432/tcp +seaweedfs lab4-seaweedfs Up 45 seconds 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..b29abbe --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 0, "first_recovery_seconds": 2.328} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/negative.txt b/evidence/negative.txt new file mode 100644 index 0000000..0edf212 --- /dev/null +++ b/evidence/negative.txt @@ -0,0 +1,14 @@ +PASS: oversized upload returns 413 before decoding +PASS: corrupt image returns 415 +PASS: MIME mismatch returns 415 +PASS: unsupported declared type returns 415 +PASS: path-like and control-character filenames return 400 +PASS: pixel limit returns 413 +PASS: attachment from another chat cannot be attached +PASS: unknown attachment id returns 404 +PASS: outsider cannot read metadata or bytes +PASS: chat member downloads identical bytes +PASS: real JPEG upload roundtrip +PASS: attachment_ids survive message history + +PASS: all negative attachment scenarios diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..749fd29 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/private-bucket.txt b/evidence/private-bucket.txt new file mode 100644 index 0000000..b9aacf0 --- /dev/null +++ b/evidence/private-bucket.txt @@ -0,0 +1,6 @@ +{ + "objects_in_bucket": 7, + "first_key": "chats/188fd76c-74c0-4861-a521-8b020b1430e8/95a16c4a-e8c0-4da6-9b7e-581f96762c27/original", + "anonymous_object_get_status": 403, + "anonymous_bucket_list_status": 403 +} diff --git a/evidence/s3-outage.txt b/evidence/s3-outage.txt new file mode 100644 index 0000000..3727a02 --- /dev/null +++ b/evidence/s3-outage.txt @@ -0,0 +1,7 @@ +stage=baseline attachment=9aabb7d3-764a-4d15-b769-21ea2810beb3 sha256=99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 +stage=s3_down lb_readiness_status=503 +stage=s3_down lb_upload_status=503 +stage=s3_down lb_download_status=503 +stage=s3_down api_replica_probe={"metadata": {"status": 200}, "content": {"status": 503, "error_code": "unavailable", "has_request_id": true}} +stage=s3_restored readiness=200 download=identical +PASS: S3 outage surfaced as 503 and the original survived the restart diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..c111ecd --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/unit-tests.txt b/evidence/unit-tests.txt new file mode 100644 index 0000000..11dbdba --- /dev/null +++ b/evidence/unit-tests.txt @@ -0,0 +1,17 @@ +.venv\Scripts\python : test_signed_cursor_roundtrip_and_tampering (test_core.CursorTests) ... ok +строка:1 знак:382 ++ ... ntract.txt; .venv\Scripts\python -m unittest discover -s student-test ... ++ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + CategoryInfo : NotSpecified: (test_signed_cur...orTests) ... ok:String) [], RemoteException + + FullyQualifiedErrorId : NativeCommandError + +test_corrupt_and_oversized (test_core.ImageTests) ... ok +test_filename_rules (test_core.ImageTests) ... ok +test_fixture_png (test_core.ImageTests) ... ok +test_jpeg_and_mime_mismatch (test_core.ImageTests) ... ok +test_password_roundtrip (test_core.PasswordTests) ... ok + +---------------------------------------------------------------------- +Ran 6 tests in 0.099s + +OK diff --git a/evidence/validate-openapi.txt b/evidence/validate-openapi.txt new file mode 100644 index 0000000..248343e --- /dev/null +++ b/evidence/validate-openapi.txt @@ -0,0 +1 @@ +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/infra/README.md b/infra/README.md index 5a39e86..91a3e4c 100644 --- a/infra/README.md +++ b/infra/README.md @@ -5,3 +5,26 @@ В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. + +## Реализованная топология (lab4, Compose) + +| Роль | Service | Образ (закреплён) | Readiness | Volume | Порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| api | `api1`, `api2` | сборка из `Dockerfile`, `lab4-messenger-api:local` | `GET /health/live`, плюс `/health/ready` (PostgreSQL + S3) для LB | нет локального состояния | `8080` только внутри сети | `DATABASE_URL`, `S3_ACCESS_KEY_ID`/`S3_SECRET_ACCESS_KEY` из `.env` | +| postgres | `postgres` | `postgres:16.4-alpine` | `pg_isready` | `pgdata:/var/lib/postgresql/data` | `5432` внутри сети | `POSTGRES_*` из `.env` | +| s3 | `seaweedfs` | `chrislusf/seaweedfs:3.71` | TCP 8333 (проверяется через `bootstrap` и readiness API) | `seaweeddata:/data` | `8333` внутри сети | `s3.json` создаётся из `S3_*` переменных при старте | +| load_balancer | `haproxy` | `haproxy:3.0.8-alpine` | `GET /health/ready` на каждом backend | — | `127.0.0.1:${LAB4_PORT}:8080` | — | +| bootstrap (one-shot) | `bootstrap` | `lab4-messenger-api:local` | завершение с кодом 0 | — | — | `S3_ADMIN_*` только для создания bucket | + +Конфигурации: `infra/haproxy/haproxy.cfg` — балансировка и health-check. `infra/minio/` не используется +(выбран SeaweedFS). + +Bootstrap: + +1. PostgreSQL: DDL в `app/db.py` выполняется каждой репликой при старте под + `pg_advisory_xact_lock`, все операторы `IF NOT EXISTS` — повторный запуск безопасен. +2. S3: `app/bootstrap_s3.py` — `head_bucket`, при `404` `create_bucket`, повторный запуск возвращает 0. + Bucket приватный: анонимные запросы получают `403` (см. `scripts/verify_private_bucket.py`). + +Секреты передаются только через переменные окружения из локального `.env` (в Git не попадает); +поддерживаются `*_FILE`-варианты для запуска вне Compose. diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 0000000..0519735 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,47 @@ +global + log stdout format raw local0 + maxconn 4096 + +# Docker's embedded DNS. Re-resolving the replica names periodically prevents a +# stale address from being used if a container is recreated with another IP +# (a name resolved once at startup can otherwise point to the wrong replica). +resolvers docker + nameserver dns1 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 5s + hold nx 1s + hold timeout 1s + hold refused 1s + hold other 1s + +defaults + mode http + log global + option httplog + option dontlognull + option redispatch + # Close the server-side connection after each response so that roundrobin + # distributes every request instead of reusing one idle backend connection. + option http-server-close + retries 2 + # A stopped container's IP silently drops packets, so a short connect + # timeout bounds how long a request waits before redispatch to a live peer. + timeout connect 500ms + timeout client 60s + timeout server 60s + +frontend public_http + bind *:8080 + default_backend api_backend + +backend api_backend + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + # Fast detection: a stopped replica must leave rotation within a couple of + # seconds, otherwise in-flight writes fail while the LB still routes to it. + default-server check inter 1s fastinter 200ms downinter 500ms fall 2 rise 1 + server api1 api1:8080 check resolvers docker init-addr none + server api2 api2:8080 check resolvers docker init-addr none diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4e83d0a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +boto3==1.35.36 +Pillow==11.0.0 +psycopg[binary]==3.2.3 +psycopg_pool==3.2.3 diff --git a/scripts/check_api_error.py b/scripts/check_api_error.py new file mode 100644 index 0000000..f90ea56 --- /dev/null +++ b/scripts/check_api_error.py @@ -0,0 +1,30 @@ +"""In-container probe: status and error code of attachment endpoints. + +Used as evidence that a replica answers with the common JSON error format while +S3 is unavailable (the load balancer may answer 503 on its own before this). +""" +import json +import os +import urllib.error +import urllib.request + +TOKEN = os.environ["TOKEN"] +ATTACHMENT_ID = os.environ["ATTACHMENT_ID"] + + +def probe(url): + request = urllib.request.Request(url, headers={"Authorization": "Bearer " + TOKEN}) + try: + with urllib.request.urlopen(request, timeout=10) as response: + return {"status": response.status} + except urllib.error.HTTPError as error: + payload = json.loads(error.read().decode("utf-8")) + return { + "status": error.code, + "error_code": payload["error"]["code"], + "has_request_id": bool(error.headers.get("X-Request-Id")), + } + + +base = "http://127.0.0.1:8080/api/v1/attachments/" + ATTACHMENT_ID +print(json.dumps({"metadata": probe(base), "content": probe(base + "/content")})) diff --git a/scripts/evidence_s3_outage.py b/scripts/evidence_s3_outage.py new file mode 100644 index 0000000..c9a278d --- /dev/null +++ b/scripts/evidence_s3_outage.py @@ -0,0 +1,111 @@ +"""Explicit evidence scenario: behaviour when the S3 backend is unavailable. + +Stops the S3 service, observes 503 responses through the load balancer and from +an API replica, then starts it again and verifies that the previously uploaded +original is still downloadable with the same checksum. +This script is NOT part of `make test`: it deliberately stops infrastructure. +""" +import hashlib +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure # noqa: E402 + + +def compose(*arguments): + command = ["docker", "compose", "--env-file", ".env", *arguments] + result = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError("compose command failed: " + " ".join(arguments)) + return result + + +def raw_status(method, url, token=None, body=None, content_type=None): + headers = {} + if token: + headers["Authorization"] = "Bearer " + token + if content_type: + headers["Content-Type"] = content_type + request = urllib.request.Request(url, data=body, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return response.status + except urllib.error.HTTPError as error: + return error.code + + +def main(): + base = os.environ.get("BASE_URL", "http://127.0.0.1:8081") + client = Client(base) + alice, bob, eve, chat = client.bootstrap() + attachment, content = client.attachment(alice, chat) + print( + "stage=baseline attachment=%s sha256=%s" + % (attachment["id"], hashlib.sha256(content).hexdigest()) + ) + + compose("cp", "scripts/check_api_error.py", "api1:/tmp/check_api_error.py") + compose("stop", "seaweedfs") + try: + print("stage=s3_down lb_readiness_status=%d" % raw_status("GET", base + "/health/ready")) + print( + "stage=s3_down lb_upload_status=%d" + % raw_status( + "POST", + base + "/api/v1/chats/" + chat["id"] + "/attachments?filename=after-outage.png", + token=alice["token"], + body=content, + content_type="image/png", + ) + ) + print( + "stage=s3_down lb_download_status=%d" + % raw_status( + "GET", + base + "/api/v1/attachments/" + attachment["id"] + "/content", + token=bob["token"], + ) + ) + probe = compose( + "exec", + "-T", + "-e", + "TOKEN=" + bob["token"], + "-e", + "ATTACHMENT_ID=" + attachment["id"], + "api1", + "python", + "/tmp/check_api_error.py", + ) + print("stage=s3_down api_replica_probe=" + probe.stdout.strip()) + finally: + compose("start", "seaweedfs") + + deadline = time.monotonic() + 90 + while True: + try: + if client.call("GET", "/health/ready", schema="Health")["status"] == 200: + break + except Failure: + pass + if time.monotonic() >= deadline: + raise SystemExit("stack did not become ready within 90 seconds") + time.sleep(1) + + restored = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob) + if restored["bytes"] != content: + raise SystemExit("original changed after S3 restart") + print("stage=s3_restored readiness=200 download=identical") + print("PASS: S3 outage surfaced as 503 and the original survived the restart") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..e92bbb2 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,29 @@ +@echo off +rem Windows equivalent of restart.sh for local evidence collection. +rem Restarts every stateful service and the API replicas without removing volumes. +setlocal +cd /d "%~dp0.." +if not exist ".env" ( + echo missing .env; copy .env.example and fill local values 1>&2 + exit /b 2 +) + +set "LAB4_PORT=" +for /f "usebackq tokens=1,* delims==" %%a in (".env") do ( + if /i "%%a"=="LAB4_PORT" set "LAB4_PORT=%%b" +) +if "%LAB4_PORT%"=="" set "LAB4_PORT=8080" + +docker compose --env-file .env restart postgres seaweedfs haproxy api1 api2 +if errorlevel 1 exit /b 1 + +rem /health/ready covers PostgreSQL and S3, so a 200 means the whole synchronous +rem path is usable again. Wait for it before reporting success. +rem ping is used instead of timeout: timeout fails when stdin is redirected. +for /l %%i in (1,1,90) do ( + powershell -NoProfile -Command "try { if ((Invoke-WebRequest -Uri 'http://127.0.0.1:%LAB4_PORT%/health/ready' -UseBasicParsing -TimeoutSec 3).StatusCode -eq 200) { exit 0 } else { exit 1 } } catch { exit 1 }" >nul 2>&1 + if not errorlevel 1 exit /b 0 + ping -n 2 127.0.0.1 >nul +) +echo stack did not become ready within 90 seconds 1>&2 +exit /b 1 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..39c08c5 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,30 @@ +#!/bin/sh +set -eu + +# Restart every process that owns state, without removing volumes or re-seeding. +cd "$(dirname "$0")/.." + +if [ ! -f .env ]; then + echo "missing .env; copy .env.example and fill local values" >&2 + exit 2 +fi + +docker compose --env-file .env restart postgres seaweedfs haproxy api1 api2 + +# shellcheck disable=SC1091 +. ./.env +port="${LAB4_PORT:-8080}" + +# /health/ready covers PostgreSQL and S3, so a 200 means the whole synchronous +# path is usable again. Wait for it before reporting success to the caller. +i=0 +while [ "$i" -lt 90 ]; do + if curl -fsS "http://127.0.0.1:${port}/health/ready" >/dev/null 2>&1; then + exit 0 + fi + i=$((i + 1)) + sleep 1 +done + +echo "stack did not become ready within 90 seconds" >&2 +exit 1 diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..4fd5acb --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,14 @@ +@echo off +rem Windows equivalent of stop-one.sh: stop exactly one API replica and leave it stopped. +setlocal +if "%TARGET_INSTANCE_ID%"=="" ( + echo test provides target API instance alias 1>&2 + exit /b 2 +) +if /i not "%TARGET_INSTANCE_ID%"=="api1" if /i not "%TARGET_INSTANCE_ID%"=="api2" ( + echo unexpected API instance alias 1>&2 + exit /b 2 +) +docker stop lab4-%TARGET_INSTANCE_ID% >nul +if errorlevel 1 exit /b 1 +exit /b 0 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..938cbeb --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu + +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +case "$TARGET_INSTANCE_ID" in + api1|api2) + ;; + *) + echo "unexpected API instance alias" >&2 + exit 2 + ;; +esac + +# Leave exactly one API replica stopped. The test restores it explicitly later. +docker stop "lab4-${TARGET_INSTANCE_ID}" >/dev/null diff --git a/scripts/verify_private_bucket.py b/scripts/verify_private_bucket.py new file mode 100644 index 0000000..1797396 --- /dev/null +++ b/scripts/verify_private_bucket.py @@ -0,0 +1,50 @@ +"""Evidence helper: the bucket must not serve objects to anonymous clients. + +Runs inside an API container (it needs the API's S3 credentials and network +access to the S3 endpoint). Prints a small JSON summary without secrets. +""" +import json +import os +import sys +import urllib.error +import urllib.request +from urllib.parse import quote + +import boto3 +from botocore.config import Config as BotoConfig + + +def main(): + endpoint = os.environ["S3_ENDPOINT"].rstrip("/") + bucket = os.environ["S3_BUCKET"] + client = boto3.client( + "s3", + endpoint_url=endpoint, + region_name=os.environ.get("S3_REGION", "us-east-1"), + aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"], + config=BotoConfig(signature_version="s3v4", s3={"addressing_style": "path"}), + ) + objects = client.list_objects_v2(Bucket=bucket).get("Contents", []) + summary = {"objects_in_bucket": len(objects), "first_key": None} + if not objects: + print(json.dumps(summary)) + return 0 + key = objects[0]["Key"] + summary["first_key"] = key + + def anonymous(url): + try: + with urllib.request.urlopen(url, timeout=5) as response: + return response.status + except urllib.error.HTTPError as error: + return error.code + + summary["anonymous_object_get_status"] = anonymous(endpoint + "/" + bucket + "/" + quote(key, safe="/")) + summary["anonymous_bucket_list_status"] = anonymous(endpoint + "/" + bucket + "?list-type=2") + print(json.dumps(summary, indent=2)) + return 0 if summary["anonymous_object_get_status"] >= 400 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/test_api_negative.py b/student-tests/test_api_negative.py new file mode 100644 index 0000000..6118f9e --- /dev/null +++ b/student-tests/test_api_negative.py @@ -0,0 +1,196 @@ +"""Negative attachment scenarios against a running stand (lab4). + +Run: python student-tests/test_api_negative.py --base-url http://localhost:8081 +Creates only synthetic users, chats and files. +""" +import os +import sys +import uuid +from io import BytesIO +from pathlib import Path +from urllib.parse import quote + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, check # noqa: E402 + +MAX_UPLOAD_BYTES = 10_485_760 +MAX_PIXELS = 40_000_000 + + +def upload(client, actor, chat, content, content_type, filename="photo.png", expected=201): + path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + quote(filename, safe="") + return client.call( + "POST", + path, + expected, + actor=actor, + raw=content, + headers={"Content-Type": content_type}, + schema="Attachment" if expected == 201 else None, + ) + + +def base_url(): + for index, argument in enumerate(sys.argv): + if argument == "--base-url" and index + 1 < len(sys.argv): + return sys.argv[index + 1] + if argument.startswith("--base-url="): + return argument.split("=", 1)[1] + return os.environ.get("BASE_URL", "http://127.0.0.1:8081") + + +def main(): + base = base_url() + client = Client(base) + alice, bob, eve, chat = client.bootstrap() + other = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Второй чат", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + sample = (ROOT / "tests/fixtures/sample.png").read_bytes() + failures = [] + + def scenario(name, function): + try: + function() + print("PASS:", name) + except AssertionError as error: + failures.append(name + ": " + str(error)) + print("FAIL:", name, "-", error) + + def too_large(): + payload = b"\x89PNG\r\n\x1a\n" + b"\x00" * (MAX_UPLOAD_BYTES + 1 - 8) + response = upload(client, alice, chat, payload, "image/png", expected=413) + check(response["json"]["error"]["code"] == "payload_too_large", "wrong error code") + + def corrupt(): + upload(client, alice, chat, b"\x89PNG\r\n\x1a\n" + b"broken" * 500, "image/png", expected=415) + + def mime_mismatch(): + upload(client, alice, chat, sample, "image/jpeg", expected=415) + + def unsupported_type(): + upload(client, alice, chat, sample, "image/gif", expected=415) + + def dangerous_names(): + for bad in ("a/b.png", "a\b.png", "a\x00b.png", "a\nb.png"): + response = upload(client, alice, chat, sample, "image/png", filename=bad, expected=400) + check(response["json"]["error"]["code"] == "invalid_request", "wrong error code") + + def pixel_limit(): + buffer = BytesIO() + side_width, side_height = 7000, 6000 # 42 Mpx > configured 40 Mpx limit + Image.new("L", (side_width, side_height), 128).save(buffer, "PNG", optimize=True) + payload = buffer.getvalue() + check(len(payload) <= MAX_UPLOAD_BYTES, "fixture exceeds byte limit, adjust the test") + check(side_width * side_height > MAX_PIXELS, "fixture must exceed the pixel limit") + upload(client, alice, chat, payload, "image/png", filename="huge.png", expected=413) + + def cross_chat_attachment(): + attachment = upload(client, alice, chat, sample, "image/png")["json"] + response = client.call( + "POST", + "/api/v1/chats/" + other["id"] + "/messages", + 404, + actor=alice, + body={"text": "", "attachment_ids": [attachment["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + check(response["json"]["error"]["code"] == "not_found", "wrong error code") + page = client.call( + "GET", "/api/v1/chats/" + other["id"] + "/messages?limit=100", actor=alice, schema="MessagePage" + )["json"] + check( + all(attachment["id"] not in item["attachment_ids"] for item in page["items"]), + "rejected attachment leaked into a message", + ) + + def unknown_attachment(): + client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 404, + actor=alice, + body={"text": "нет такого файла", "attachment_ids": [str(uuid.uuid4())]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + + def outsider_access(): + attachment = upload(client, alice, chat, sample, "image/png")["json"] + for suffix in ("", "/content"): + client.call("GET", "/api/v1/attachments/" + attachment["id"] + suffix, 404, actor=eve) + + def member_access_and_checksum(): + import hashlib + + attachment = upload(client, alice, chat, sample, "image/png")["json"] + check(attachment["sha256"] == hashlib.sha256(sample).hexdigest(), "checksum mismatch") + content = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob) + check(content["bytes"] == sample, "download differs from upload") + check(content["headers"].get_content_type() == "image/png", "content type mismatch") + + def real_jpeg_roundtrip(): + import hashlib + + buffer = BytesIO() + Image.new("RGB", (640, 480), "navy").save(buffer, "JPEG", quality=90) + payload = buffer.getvalue() + attachment = upload(client, alice, chat, payload, "image/jpeg", filename="photo.jpg")["json"] + check(attachment["status"] == "ready", "JPEG was not accepted") + check(attachment["content_type"] == "image/jpeg", "JPEG content type mismatch") + check(attachment["size_bytes"] == len(payload), "JPEG size mismatch") + check(attachment["sha256"] == hashlib.sha256(payload).hexdigest(), "JPEG checksum mismatch") + content = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob) + check(content["bytes"] == payload, "JPEG download differs from upload") + check(content["headers"].get_content_type() == "image/jpeg", "JPEG download type mismatch") + + def attachment_ids_survive_history(): + attachment = upload(client, alice, chat, sample, "image/png")["json"] + created = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 201, + actor=alice, + body={"text": "", "attachment_ids": [attachment["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + schema="Message", + )["json"] + page = client.call( + "GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage" + )["json"] + found = [item for item in page["items"] if item["id"] == created["id"]] + check(len(found) == 1, "message with attachment is missing from history") + check(found[0]["attachment_ids"] == [attachment["id"]], "history lost attachment_ids") + check(found[0] == created, "history item differs from the POST response") + + scenario("oversized upload returns 413 before decoding", too_large) + scenario("corrupt image returns 415", corrupt) + scenario("MIME mismatch returns 415", mime_mismatch) + scenario("unsupported declared type returns 415", unsupported_type) + scenario("path-like and control-character filenames return 400", dangerous_names) + scenario("pixel limit returns 413", pixel_limit) + scenario("attachment from another chat cannot be attached", cross_chat_attachment) + scenario("unknown attachment id returns 404", unknown_attachment) + scenario("outsider cannot read metadata or bytes", outsider_access) + scenario("chat member downloads identical bytes", member_access_and_checksum) + scenario("real JPEG upload roundtrip", real_jpeg_roundtrip) + scenario("attachment_ids survive message history", attachment_ids_survive_history) + + if failures: + print("\nFAILED:", len(failures)) + for failure in failures: + print(" -", failure) + raise SystemExit(1) + print("\nPASS: all attachment scenarios (limits, MIME, access, roundtrip, history)") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_core.py b/student-tests/test_core.py new file mode 100644 index 0000000..f65b5b7 --- /dev/null +++ b/student-tests/test_core.py @@ -0,0 +1,71 @@ +"""Small dependency-light checks for pure validation helpers. + +Run with: python -m unittest discover -s student-tests -v +""" +import sys +import unittest +from io import BytesIO +from pathlib import Path + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from app.images import ImageTooLarge, InvalidImage, inspect_image # noqa: E402 +from app.security import ( # noqa: E402 + decode_cursor, + encode_cursor, + hash_password, + verify_password, +) +from app.server import Handler # noqa: E402 + + +class PasswordTests(unittest.TestCase): + def test_password_roundtrip(self): + stored = hash_password("T3st-abcdefghijkl") + self.assertTrue(verify_password("T3st-abcdefghijkl", stored)) + self.assertFalse(verify_password("wrong-password", stored)) + self.assertNotIn("T3st-abcdefghijkl", stored) + + +class CursorTests(unittest.TestCase): + def test_signed_cursor_roundtrip_and_tampering(self): + value = encode_cursor("secret", {"kind": "messages", "id": "1"}) + self.assertEqual(decode_cursor("secret", value)["id"], "1") + with self.assertRaises(ValueError): + decode_cursor("other-secret", value) + with self.assertRaises(ValueError): + decode_cursor("secret", value + "x") + + +class ImageTests(unittest.TestCase): + def test_fixture_png(self): + data = (ROOT / "tests/fixtures/sample.png").read_bytes() + self.assertEqual(inspect_image(data, "image/png", 40_000_000), ("image/png", 320, 200)) + + def test_jpeg_and_mime_mismatch(self): + buffer = BytesIO() + Image.new("RGB", (10, 10), "red").save(buffer, "JPEG") + data = buffer.getvalue() + self.assertEqual(inspect_image(data, "image/jpeg", 40_000_000)[0], "image/jpeg") + with self.assertRaises(InvalidImage): + inspect_image(data, "image/png", 40_000_000) + + def test_corrupt_and_oversized(self): + with self.assertRaises(InvalidImage): + inspect_image(b"not an image", "image/png", 40_000_000) + buffer = BytesIO() + Image.new("RGB", (10, 10), "red").save(buffer, "PNG") + with self.assertRaises(ImageTooLarge): + inspect_image(buffer.getvalue(), "image/png", 99) + + def test_filename_rules(self): + self.assertTrue(Handler._valid_filename("photo.png")) + for bad in ("", "a/b.png", "a\\b.png", "a\x00b", "a\nb"): + self.assertFalse(Handler._valid_filename(bad), repr(bad)) + + +if __name__ == "__main__": + unittest.main()