From bd0b9f6849cbb7050636803189d790331fed0cd6 Mon Sep 17 00:00:00 2001 From: kaktuz20000 <149298757+kaktuz20000@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:55:33 +0300 Subject: [PATCH 1/2] =?UTF-8?q?=D0=B2=D1=8B=D0=BF=D0=BE=D0=BB=D0=BD=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=BB=D0=B0=D0=B1=D0=BE=D1=80=D0=B0=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=BD=D0=BE=D0=B9=20=D0=BD=D0=B0=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 22 +- Dockerfile | 25 + Makefile | 20 +- README.md | 36 +- REPORT.md | 66 ++- code.md | 121 +++++ compose.yaml | 186 +++++++ haproxy.cfg | 35 ++ migrations/001_initial.sql | 43 ++ migrations/002_idempotency.sql | 16 + migrations/003_attachments.sql | 35 ++ requirements.txt | 5 + scripts/bootstrap-secrets.py | 80 +++ scripts/bootstrap-secrets.sh | 3 + scripts/restart.sh | 16 + scripts/stop-one.sh | 19 + src/__init__.py | 1 + src/app.py | 39 ++ src/bootstrap_s3.py | 47 ++ src/config.py | 61 +++ src/errors.py | 26 + src/handlers.py | 548 ++++++++++++++++++++ src/migrate.py | 24 + src/object_store.py | 76 +++ src/reconcile.py | 125 +++++ src/security.py | 53 ++ src/server.py | 13 + src/store.py | 548 ++++++++++++++++++++ student-tests/test_attachments.py | 88 ++++ student-tests/test_idempotency_race.py | 47 ++ student-tests/test_reconcile.py | 102 ++++ student-tests/test_reconcile_integration.py | 187 +++++++ student-tests/upload_concurrency.py | 152 ++++++ tests/resilience.py | 2 +- 34 files changed, 2818 insertions(+), 49 deletions(-) create mode 100644 Dockerfile create mode 100644 code.md create mode 100644 compose.yaml create mode 100644 haproxy.cfg create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 requirements.txt create mode 100644 scripts/bootstrap-secrets.py create mode 100755 scripts/bootstrap-secrets.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 src/__init__.py create mode 100644 src/app.py create mode 100644 src/bootstrap_s3.py create mode 100644 src/config.py create mode 100644 src/errors.py create mode 100644 src/handlers.py create mode 100644 src/migrate.py create mode 100644 src/object_store.py create mode 100644 src/reconcile.py create mode 100644 src/security.py create mode 100644 src/server.py create mode 100644 src/store.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_idempotency_race.py create mode 100644 student-tests/test_reconcile.py create mode 100644 student-tests/test_reconcile_integration.py create mode 100644 student-tests/upload_concurrency.py diff --git a/.env.example b/.env.example index d99d26a2..1f760fea 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,8 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 +# Copy to .env for the local synthetic-data environment. Never commit .env. APP_PORT=8080 -APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 +SESSION_TTL_SECONDS=3600 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 +MAX_CONCURRENT_UPLOADS=10 +API_CPU_LIMIT=1.0 +API_MEMORY_LIMIT=512M diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..83823c86 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.12.7-slim + +WORKDIR /app + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY . /app +RUN chown -R app:app /app + +USER app + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).read()" || exit 1 + +CMD ["python3", "src/server.py"] diff --git a/Makefile b/Makefile index 5d86d171..2cff6b8d 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,11 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test test-attachments test-reconcile test-reconcile-integration test-upload-concurrency reconcile-dry-run load test-persistence test-failover help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check/validate: offline contract; test/test-attachments: running API; test-reconcile: local rule tests" + @echo "test-upload-concurrency: 10 simultaneous uploads (Compose + Docker); reconcile-dry-run: read-only S3/DB scan" check: $(PYTHON) scripts/check_contract.py @@ -18,6 +19,21 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-attachments: + $(PYTHON) student-tests/test_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-reconcile: + $(PYTHON) student-tests/test_reconcile.py + +test-reconcile-integration: + $(PYTHON) student-tests/test_reconcile_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-upload-concurrency: + $(PYTHON) student-tests/upload_concurrency.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +reconcile-dry-run: + docker compose exec -T api python3 src/reconcile.py + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/README.md b/README.md index 947094c3..88e2d1f5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Разделить метаданные и бинарное содержимое, добавить изображения в сообщения и сохранить доступ к ним при переключении API-реплики. -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +Это реализация на Python 3.12 с PostgreSQL, SeaweedFS S3 и HAProxy. Код lab3 сохранён как основа; добавлены метаданные и S3-объекты вложений, синхронные upload/download и согласованная работа через несколько API-реплик. UI не обязателен. [Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) @@ -14,7 +14,7 @@ ## Что сделать -1. Выберите один S3-совместимый сервер: MinIO, SeaweedFS или RustFS. Закрепите проверенную версию образа и опишите создание приватного bucket. +1. Используется SeaweedFS `4.51` как S3-совместимый сервер; образ закреплён по версии, приватный bucket bootstrap-ится из локального стенда. 2. Спроектируйте Attachment и связь с чатом/сообщением. S3 object key назначает сервер, filename используется только для отображения. 3. Реализуйте синхронный binary upload PNG/JPEG с ограничениями, сохранение метаданных и авторизованный download через API. 4. Включите S3 и persistent volumes в общий Compose/Swarm. Проверьте, что другая API-реплика скачивает файл после остановки загрузившей. @@ -44,25 +44,37 @@ **Трек 2. Согласованность двух хранилищ.** Разработайте повторяемый reconciler для orphan/missing объектов и метаданных. Внедрите ошибки на обеих сторонах, покажите обнаружение и безопасное исправление с dry-run/повторным запуском. Докажите, что свежие и используемые вложения не удаляются; измерьте стоимость обхода. -## Как начать и проверить +## Запуск и проверка ```sh -# Работает прямо в skeleton, Python 3.9+; приложение не запускается: +# Офлайн-проверка структуры и контракта: make check -# После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. +# Smoke доступного API: make test ``` -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. +`compose.example.yaml` — только worksheet, запускаемый стенд находится в `compose.yaml`. Для полного сценария и тестов см. [code.md](code.md). ```sh -# Compose-вариант после реализации: -docker compose --env-file .env config --quiet -docker compose --env-file .env up -d --build -# Дождитесь /health/ready; затем make test. -# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. +# Подготовка локальных secrets (значения не печатаются) и конфигурации: +cp .env.example .env +scripts/bootstrap-secrets.sh +docker compose config --quiet +docker compose up -d --build --scale api=2 +# Ожидайте /health/ready; затем запускайте make test. +``` + +API передаёт upload потоками по 64 KiB во временный spooled-файл (в памяти не более 1 MiB на запрос), ограничивает размер/пиксели/общее время и количество параллельных upload. S3 bucket приватен; credentials admin bootstrap не попадают в API, а API-пользователь ограничен одним bucket. `src/reconcile.py` по умолчанию работает read-only; удаление orphan-объектов требует `--apply` и grace period 3600 секунд. Восстановление отсутствующего объекта возможно только из точной копии, checksum которой совпадает с PostgreSQL. + +Дополнительные проверки: + +```sh +make test-attachments +make test-reconcile +make test-upload-concurrency +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh ``` diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..ade297d4 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,69 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: не указаны пользователем. +- Целевая оценка: 5; выбран трек 2 — согласованность PostgreSQL/S3. +- Commit/PR: не создавались. +- Источники и инструменты: контракт и материалы `mtusi2026lab4`; основа приложения и тест конкурентной идемпотентности перенесены из `mtusi2026lab3`; Python 3.12, psycopg 3, boto3, Pillow, SeaweedFS, HAProxy. Использована помощь AI assistant using Copilot SDK in VS Code при переносе, реализации и написании документации. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Engine с Compose v2 и Python 3.9+ для локальных проверок. Образы стенда закреплены: `python:3.12.7-slim`, `postgres:16.4-alpine`, `chrislusf/seaweedfs:4.51`, `haproxy:3.0.7-alpine`. + +Полная последовательность build, миграций, readiness, тестов, безопасного останова и удаления учебных данных находится в [code.md](code.md). Секреты генерируются локально `scripts/bootstrap-secrets.sh`; они не выводятся в stdout и не входят в Git. Compose не публикует API-реплики, PostgreSQL или S3 API; наружу доступен только HAProxy на `127.0.0.1`. В проверенной среде порт 8080 был занят, поэтому стенд запускался на 18080. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +HTTP client -> HAProxy -> 2+ stateless API replicas + |-> PostgreSQL (users, chats, messages, sessions, idempotency, attachment metadata) + |-> private SeaweedFS S3 bucket (original image bytes) +``` + +Перенесены основные handler/store/server, миграции и поведение пользователей, сессий, чатов, пагинации и идемпотентности lab3. В сообщение добавлено необязательное `attachment_ids`, пустой текст принимается только при наличии вложений. Идемпотентный JSON канонизируется с `attachment_ids: []`, поэтому отсутствующее поле и пустой список эквивалентны. + +Новый маршрут принимает только сырые PNG/JPEG, максимум 10 MiB и 40 млн пикселей. Content-Length проверяется до чтения; загрузка читается по 64 KiB в `SpooledTemporaryFile` с порогом памяти 1 MiB, ограничением времени 60 секунд и лимитом 10 одновременных загрузок на процесс. MIME сверяется с форматом декодера. `filename` хранится только как display metadata; object key строится из серверного UUID. Оригинал сначала записывается в S3, после чего транзакционно сохраняется metadata. При ошибке БД выполняется best-effort удаление S3-объекта; если процесс оборвался между операциями, объект обнаружит reconciler. + +Download сначала проверяет членство пользователя в чате через PostgreSQL, затем проксирует байты из приватного S3 bucket. API не выдаёт redirect или object key. Отдельный ограниченный S3-пользователь создаётся bootstrap с правами только на бакет `messenger-attachments`; admin credentials не монтируются в API. PostgreSQL и SeaweedFS используют persistent volumes. + +Reconciler сравнивает полный список объектов с metadata и выдаёт счётчики/время обхода. По умолчанию он read-only. `--apply` удаляет только orphan objects старше указанного grace period (по умолчанию 3600 секунд), регистрирует missing-object incidents и очищает устаревшие incident-записи. Если metadata существует, но object отсутствует, восстановление допускается только из предоставленной оператором точной копии с теми же размером и SHA-256; копия называется UUID вложения. Несовпавшая копия не загружается. Недостающая копия оставляет metadata на месте для ручного восстановления, объект не объявляется восстановленным. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Проверки выполнены в Docker Compose на локальном стенде (две API-реплики, PostgreSQL, SeaweedFS и HAProxy). Для этого запуска HAProxy опубликован на `localhost:18080`; секреты и данные находятся в локальных ignored-файлах/volumes. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда/сценарий | Результат / наблюдение | Доказательство | +| --- | --- | --- | --- | +| Контракт и структура | `make check`; `make validate PYTHON=.venv/bin/python` | PASS | `scripts/check_contract.py`, `scripts/validate_openapi.py` | +| Публичный API, checksum и доступ | `make test BASE_URL=http://localhost:18080` | PASS | `tests/smoke.py` | +| Конкурентная идемпотентность | 12 одинаковых POST через HAProxy и две API-реплики | PASS: все ответы совпали, в истории сохранено одно сообщение | `student-tests/test_idempotency_race.py` | +| Ограничения upload и негативные случаи | `make test-attachments BASE_URL=http://localhost:18080` | PASS: oversized bytes/pixels, damaged image, MIME spoof, unsafe filename, чужой пользователь и чужой чат | `student-tests/test_attachments.py` | +| Десять одновременных upload и память | `make test-upload-concurrency BASE_URL=http://localhost:18080` | PASS: 10 параллельных upload по 5 883 411 байт, две API-реплики, 1.648 с; максимальная память одного API-контейнера: baseline 57 126 420 B, peak 68 209 868 B, delta 11 083 448 B. За время upload получено 4 samples, результат приблизительный | `student-tests/upload_concurrency.py` | +| Persist volumes и API failover | `make test-persistence ACTION_SCRIPT=scripts/restart.sh`; `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS: PostgreSQL/S3 restart сохранил запись, вложение и прежнюю сессию; failover видел 2 реплики до остановки, после двух неуспешных попыток записал 111 сообщений, первое восстановление за 3.25 с | `tests/resilience.py`, `scripts/` | +| S3 private bucket и least privilege | unauthenticated GET; повторный bootstrap | PASS: анонимный GET отклонён; повторный S3 bootstrap завершился успешно; API credentials ограничены действиями для bucket `messenger-attachments` | `compose.yaml`, `scripts/bootstrap-secrets.py`, integration test | +| Track 2: orphan/missing reconciliation | `make test-reconcile`; `make test-reconcile-integration BASE_URL=http://localhost:18080` | PASS: реальные PostgreSQL/SeaweedFS, 45 объектов и 45 metadata-записей; 1 orphan обнаружен и удалён, 1 missing object восстановлен по точной копии, повторный scan чистый, 21.638 ms | `student-tests/test_reconcile.py`, `student-tests/test_reconcile_integration.py` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Track 2 — гипотеза и методика -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +**Гипотеза:** reconciliation не удалит ни свежий orphan, ни известное metadata-вложение; orphan старше grace period удалится при `--apply`; missing object восстановится только из файла, размер и SHA-256 которого совпадают с metadata. Повторный запуск не должен повторно удалять уже удалённый объект или создавать дубликаты metadata. + +**Повторяемая методика:** выполнить API upload тестовых изображений; добавить synthetic S3 orphan под уникальным `original/`; удалить объект одного тестового attachment через S3 admin-инструмент; сделать копию оригинальных bytes в recovery directory под UUID attachment; запускать `src/reconcile.py` в dry-run и с `--apply --restore-dir `, выдерживая заданный grace period. Для стоимости обхода сохранить JSON `duration_ms`, `objects_seen`, `metadata_seen`, `orphan_objects`, `missing_objects` для нескольких размеров набора; фиксировать версии, CPU/RAM, длительность прогрева и объём данных. Объекты с сообщениями не удалять: они имеют metadata и не входят в orphan set. + +**Наблюдаемые результаты:** интеграционный сценарий выполнен на реальных PostgreSQL/SeaweedFS: при dry-run найден synthetic orphan и отсутствующий объект, dry-run ничего не удалил; свежий orphan пережил `--apply`, точный backup восстановил missing object, после 61 секунды orphan удалён, повторная сверка чистая. Зафиксировано 45 S3 объектов, 45 metadata-записей и 21.638 ms scan time. Этот единичный результат не является полноценным performance benchmark; RPS не измерялся. + +| Воздействие | Ожидаемое поведение | Наблюдение | Восстановление / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Повторный bootstrap S3 | Bucket остаётся приватным, credentials не меняются | Повторный bootstrap завершился успешно; анонимный GET получил отказ | Integration test, Compose logs | +| PostgreSQL недоступен после S3 PUT | Compensation удалит объект; если процесс завершился до компенсации, поздний orphan уберёт reconciler после grace | Интеграционный отказ ещё не выполнен | Orphan не трогается до границы grace | +| Missing S3 object | Dry-run фиксирует; `--apply` записывает incident; оригинал восстанавливается только из exact-hash backup | Реальный интеграционный сценарий удалил synthetic объект и восстановил его по checksum; attachment download совпал с исходными bytes | Невосстановимый объект сохраняет metadata и требует оператора | +| Остановка API-реплики / restart с volumes | LB направляет запросы на другую реплику; данные/токен остаются | PASS: до отказа замечены 2 инстанса; после остановки одной API-реплики восстановление записи 3.25 с; restart PostgreSQL/S3/API сохранил данные, вложение и существующую сессию | `tests/resilience.py`; volumes сохраняются | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Замер Docker memory stats содержит 4 samples и может пропустить короткий пик; это приблизительная delta одного API-контейнера, не системный peak RSS. +- После рестарта SeaweedFS его HTTP healthcheck становился healthy до прогрева чтения файлов с volume. Restart hook теперь проверяет чтение сохранённого объекта до перезапуска API. +- S3 PUT и PostgreSQL COMMIT не являются распределённой транзакцией. Компенсация помогает при обычной ошибке, reconciler устраняет оставшиеся orphan-объекты после безопасной задержки. +- Без backup-оригинала восстановить отсутствующие байты нельзя. В этом случае reconciler сохраняет metadata и incident вместо опасного удаления или выдуманного успешного состояния. +- Базовые API, health/readiness, session cache, тайм-ауты PostgreSQL и идемпотентность перенесены из lab3. Единственный источник истины сообщений и вложений — PostgreSQL metadata + S3 bytes; Redis не требуется. +- Десять одновременных запросов генерируют в памяти тестового клиента fixture 1400×1400; серверный memory sample берётся из Docker stats. Счётчик — максимальная память отдельного API-контейнера, а не сумма всех сервисов или системная peak RSS. diff --git a/code.md b/code.md new file mode 100644 index 00000000..10885d3c --- /dev/null +++ b/code.md @@ -0,0 +1,121 @@ +## Первый запуск / чистый clone + +```sh +cd mtusi2026lab4 +cp .env.example .env +scripts/bootstrap-secrets.sh +# If localhost:8080 is occupied, use an alternate HAProxy port for all following commands. +export APP_PORT=18080 + +# Проверить итоговую Compose-конфигурацию. +docker compose config --quiet + +# Build, PostgreSQL + SeaweedFS bucket bootstrap + миграции + 2 API + HAProxy. +docker compose up -d --build --scale api=2 + +# Проверить статусы и readiness. +docker compose ps +curl --fail --silent http://localhost:18080/health/ready +``` + +Миграции запускаются сервисом `migrate` до API. Для ожидания старта: + +```sh +until curl --fail --silent http://localhost:18080/health/ready >/dev/null; do sleep 2; done +``` + +Если порт 8080 свободен, удалите `export APP_PORT=18080` и замените `18080` на `8080` в `BASE_URL` и `curl`. + +Bucket приватный; порты API, PostgreSQL и S3 API не опубликованы. `s3-bootstrap` использует admin S3 identity только для создания bucket, а API получает credentials с действиями Read/Write/List только для `messenger-attachments`. + +## Статические и локальные проверки + +```sh +make check +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make validate PYTHON=.venv/bin/python + +# Обязательный public smoke; сервер должен быть поднят. +make test BASE_URL=http://localhost:18080 + +# 12 одновременных одинаковых POST через две API-реплики должны создать одно сообщение. +.venv/bin/python student-tests/test_idempotency_race.py --base-url http://localhost:18080 + +# Негативные проверки вложений, без дополнительных зависимостей в тестовом клиенте. +make test-attachments BASE_URL=http://localhost:18080 + +# Детерминированные unit checks для решений reconciler (без реальных изменений хранилищ). +make test-reconcile PYTHON=.venv/bin/python + +# Интеграционный Track 2: повреждает только тестовые данные; ждёт grace period 60 секунд. +make test-reconcile-integration BASE_URL=http://localhost:18080 +``` + +Публичные сценарии создают пользователей/чаты/сообщения и не удаляют их. Повторять на учебном стенде; не использовать производственные данные. + +## Сохранность, отказ API и параллельная загрузка + +```sh +chmod +x scripts/restart.sh scripts/stop-one.sh + +# Restart API, PostgreSQL и SeaweedFS с сохранением volumes и прежней сессии. +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:18080 + +# Нужны минимум 2 API. Тест останавливает одну реплику; после теста восстановить: +make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:18080 +docker compose up -d --scale api=2 api + +# 10 одновременных 1400×1400 PNG; sampler выводит baseline/peak/delta памяти API. +make test-upload-concurrency BASE_URL=http://localhost:18080 +``` + +Публичные и собственные тесты не очищают данные. Persistence hook не удаляет volumes. Failover hook оставляет выбранную API-реплику остановленной до явного `docker compose up -d --scale api=2 api`. + +## Track 2: reconciliation + +Read-only scan: + +```sh +docker compose exec -T api python3 src/reconcile.py +``` + +Удаление только stale orphan objects (по умолчанию возраст не менее 3600 секунд; минимально разрешено 60): + +```sh +docker compose exec -T api python3 src/reconcile.py --grace-seconds 3600 --apply +``` + +Восстановление отсутствующего S3 object разрешается только из доверенной копии exact original. Имя файла — UUID attachment, содержимое должно совпасть с записанными размером и SHA-256. Сначала проверить кандидатов (без `--apply`), затем применить: + +```sh +docker compose exec -T api python3 src/reconcile.py --restore-dir /path/visible/in/api +docker compose exec -T api python3 src/reconcile.py --restore-dir /path/visible/in/api --apply +``` + +Каталог восстановления должен быть доступен внутри API-контейнера и содержать только намеренно выбранные файлы. Путь можно смонтировать в API на время стендового эксперимента; не включать backup originals в Git. Если точной копии нет, metadata сохраняется и записывается incident для оператора. + +Каждый запуск выводит JSON с `objects_seen`, `metadata_seen`, количеством orphan/missing/restored objects и `duration_ms`. Сохранить JSON для нескольких размеров dataset и указать CPU/RAM и версии в `REPORT.md`; dry-run использовать до каждого `--apply`. + +## Нагрузка сообщений + +```sh +make load BASE_URL=http://localhost:18080 +# Или задать duration/concurrency/output согласно tests/load.py: +python3 tests/load.py --base-url http://localhost:18080 --duration 30 --concurrency 4 --output evidence/load.json +``` + +## Bucket / Compose lifecycle + +```sh +# Идемпотентно проверить создание приватного bucket без сброса данных. +docker compose up -d s3-bootstrap + +# Безопасная остановка, persistent volumes остаются: +docker compose down + +# Намеренно удалить учебную БД и оригиналы (не выполнять для проверки persistence): +docker compose down --volumes +``` + +После `down --volumes` секреты остаются в локальном `secrets/` и подходят для нового чистого стенда. Не удалять папку secrets рекурсивно и не коммитить файлы из неё. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..2cb64046 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,186 @@ +services: + postgres: + image: postgres:16.4-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs:4.51 + entrypoint: ["weed"] + command: + - server + - -s3 + - -dir=/data + - -s3.config=/run/secrets/s3_config + - -s3.port=8333 + - -s3.autoCreateBucket=false + - -s3.concurrentFileUploadLimit=10 + - -s3.concurrentUploadLimitMB=100 + - -s3.readerCacheSizeMB=64 + secrets: + - s3_config + volumes: + - seaweed_data:/data + healthcheck: + test: + - CMD-SHELL + - "code=$$(curl -sS -o /dev/null -w '%{http_code}' http://127.0.0.1:8333/); [ \"$$code\" = 403 ] || [ \"$$code\" = 200 ]" + interval: 3s + timeout: 2s + retries: 30 + start_period: 10s + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + s3-bootstrap: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "src/bootstrap_s3.py"] + environment: + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + depends_on: + s3: + condition: service_healthy + restart: "no" + + migrate: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "src/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + DB_POOL_MAX_SIZE: "2" + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + restart: "no" + + api: + build: + context: . + dockerfile: Dockerfile + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://s3:8333 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-10} + UPLOAD_TIMEOUT_SECONDS: "60" + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DB_POOL_MAX_SIZE: "8" + DB_POOL_TIMEOUT_SECONDS: "3" + DB_STATEMENT_TIMEOUT_MS: "5000" + DB_LOCK_TIMEOUT_MS: "2000" + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: + - CMD + - python3 + - -c + - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()" + interval: 3s + timeout: 3s + retries: 5 + start_period: 10s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPU_LIMIT:-1.0}" + memory: "${API_MEMORY_LIMIT:-512M}" + stop_grace_period: 20s + restart: unless-stopped + + load_balancer: + image: haproxy:3.0.7-alpine + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + volumes: + - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 3 + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + restart: unless-stopped + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + s3_config: + file: ./secrets/s3.json + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + +volumes: + postgres_data: + seaweed_data: diff --git a/haproxy.cfg b/haproxy.cfg new file mode 100644 index 00000000..9dedf30a --- /dev/null +++ b/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 + maxconn 1024 + +defaults + log global + mode http + option httplog + option http-keep-alive + timeout connect 2s + timeout client 15s + timeout server 15s + timeout http-request 5s + timeout check 1s + retries 2 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold nx 2s + +frontend messenger + bind :8080 + default_backend api_replicas + +backend api_replicas + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + option redispatch + default-server inter 1s fall 2 rise 2 + server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr libc,none diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..f5431a5f --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,43 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username varchar(32) NOT NULL UNIQUE, + display_name varchar(100) NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title varchar(100) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + joined_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text varchar(4000) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0) +); + +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at) +); + +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..aae5c1ae --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,16 @@ +CREATE TABLE idempotency_records ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method varchar(8) NOT NULL, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + idempotency_key varchar(128) NOT NULL, + request_hash bytea NOT NULL, + response jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, chat_id, idempotency_key), + CONSTRAINT idempotency_method_post CHECK (method = 'POST'), + CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32), + CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at) +); + +CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..b3896540 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,35 @@ +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_not_blank; + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + filename varchar(255) NOT NULL, + content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE, + status varchar(16) NOT NULL DEFAULT 'ready' CHECK (status = 'ready'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, ordinal) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); + +CREATE TABLE reconciliation_incidents ( + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + incident_type varchar(32) NOT NULL CHECK (incident_type IN ('missing_object')), + first_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(), + last_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, incident_type) +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..8a4e07c2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +psycopg[binary]==3.2.10 +psycopg-pool==3.3.3 +redis==5.2.1 +boto3==1.35.99 +Pillow==11.0.0 diff --git a/scripts/bootstrap-secrets.py b/scripts/bootstrap-secrets.py new file mode 100644 index 00000000..496adbf5 --- /dev/null +++ b/scripts/bootstrap-secrets.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +import json +import os +import secrets +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SECRET_DIR = ROOT / "secrets" + + +def main(): + SECRET_DIR.mkdir(mode=0o700, exist_ok=True) + names = ( + "postgres_password", + "database_url", + "s3_access_key_id", + "s3_secret_access_key", + "s3_admin_access_key_id", + "s3_admin_secret_access_key", + "s3.json", + ) + existing = [name for name in names if (SECRET_DIR / name).exists()] + if existing: + raise SystemExit( + "Secret files already exist; refusing to rotate credentials without a migration: " + + ", ".join(existing) + ) + + db_password = secrets.token_hex(24) + values = { + "postgres_password": db_password, + "database_url": f"postgresql://messenger:{db_password}@postgres:5432/messenger", + "s3_access_key_id": "mngr" + secrets.token_hex(8), + "s3_secret_access_key": secrets.token_urlsafe(40), + "s3_admin_access_key_id": "admin" + secrets.token_hex(7), + "s3_admin_secret_access_key": secrets.token_urlsafe(40), + } + for name, value in values.items(): + path = SECRET_DIR / name + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write(value + "\n") + s3_config = { + "identities": [ + { + "name": "bootstrap-admin", + "credentials": [ + { + "accessKey": values["s3_admin_access_key_id"], + "secretKey": values["s3_admin_secret_access_key"], + } + ], + "actions": ["Admin"], + }, + { + "name": "messenger-api", + "credentials": [ + { + "accessKey": values["s3_access_key_id"], + "secretKey": values["s3_secret_access_key"], + } + ], + "actions": [ + "Read:messenger-attachments", + "Write:messenger-attachments", + "List:messenger-attachments", + ], + }, + ] + } + descriptor = os.open(SECRET_DIR / "s3.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + json.dump(s3_config, output, separators=(",", ":")) + output.write("\n") + print("Local secrets generated with owner-only permissions; values were not printed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/bootstrap-secrets.sh b/scripts/bootstrap-secrets.sh new file mode 100755 index 00000000..5ea636bd --- /dev/null +++ b/scripts/bootstrap-secrets.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec python3 "$(dirname "$0")/bootstrap-secrets.py" diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..b2f3e366 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose restart postgres s3 +docker compose up -d --wait postgres s3 +attempt=0 +until docker compose exec --index 1 -T api python3 -c 'from src.object_store import ObjectStore; store = ObjectStore(); key, _ = next(store.objects()); store.get(key, 10485760)' >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 4 ]; then + echo "S3 object reads did not recover after restart" >&2 + exit 1 + fi + sleep 2 +done +docker compose restart api +docker compose up -d --wait api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..59e50fe8 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +found= +for container_id in $(docker compose ps --quiet --status running api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id") + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + found=$container_id + break + fi +done + +if [ -z "$found" ]; then + printf '%s\n' 'No running API container matched the requested instance alias.' >&2 + exit 1 +fi +docker stop --time 5 "$found" >/dev/null diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..5bc8da31 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""Messenger API application package.""" diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..d5229ba9 --- /dev/null +++ b/src/app.py @@ -0,0 +1,39 @@ +import logging +import os +import signal +import threading +from functools import partial +from http.server import ThreadingHTTPServer + +from .config import APP_HOST, APP_PORT +from .handlers import MessengerHandler +from .store import Store + +LOGGER = logging.getLogger("messenger") + + +def run_server(): + store = Store() + store.open() + try: + handler = partial(MessengerHandler, store=store) + server = ThreadingHTTPServer((APP_HOST, APP_PORT), handler) + + def handle_shutdown(signum, frame): + LOGGER.info("received signal %s, shutting down gracefully", signum) + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, handle_shutdown) + signal.signal(signal.SIGINT, handle_shutdown) + LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT) + try: + server.serve_forever() + finally: + server.server_close() + finally: + store.close() + + +def main(): + logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) + run_server() diff --git a/src/bootstrap_s3.py b/src/bootstrap_s3.py new file mode 100644 index 00000000..3c3bc7a8 --- /dev/null +++ b/src/bootstrap_s3.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +import logging +import os +from pathlib import Path + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + + +LOGGER = logging.getLogger("messenger.s3.bootstrap") + + +def secret(name: str) -> str: + direct = os.getenv(name) + if direct: + return direct + file_path = os.getenv(name + "_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + raise RuntimeError(f"Set {name} or {name}_FILE") + + +def main(): + client = boto3.client( + "s3", + endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"), + region_name="us-east-1", + aws_access_key_id=secret("S3_ADMIN_ACCESS_KEY_ID"), + aws_secret_access_key=secret("S3_ADMIN_SECRET_ACCESS_KEY"), + config=Config(connect_timeout=2, read_timeout=5, retries={"max_attempts": 3}), + ) + bucket = os.getenv("S3_BUCKET", "messenger-attachments") + try: + client.head_bucket(Bucket=bucket) + except ClientError as exc: + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if status != 404: + raise + client.create_bucket(Bucket=bucket) + client.put_bucket_acl(Bucket=bucket, ACL="private") + LOGGER.info("private S3 bucket bootstrap completed") + + +if __name__ == "__main__": + logging.basicConfig(level="INFO", format="%(levelname)s %(message)s") + main() diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..2dadff58 --- /dev/null +++ b/src/config.py @@ -0,0 +1,61 @@ +import os +import re +import socket +import uuid +from pathlib import Path + +APP_HOST = os.getenv("APP_HOST", "0.0.0.0") +APP_PORT = int(os.getenv("APP_PORT", "8080")) +INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12]) +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400")) +DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000")) +DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000")) +DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16")) +DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5")) +MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128")) +MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576")) +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) +MAX_CONCURRENT_UPLOADS = int(os.getenv("MAX_CONCURRENT_UPLOADS", "10")) +UPLOAD_TIMEOUT_SECONDS = int(os.getenv("UPLOAD_TIMEOUT_SECONDS", "60")) +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000") +S3_REGION = os.getenv("S3_REGION", "us-east-1") +S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") + + +def secret_value(name: str, default: str = "") -> str: + value = os.getenv(name) + if value: + return value + file_path = os.getenv(name + "_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + if default: + return default + raise RuntimeError(f"Set {name} or {name}_FILE") + + +S3_ACCESS_KEY_ID = secret_value("S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = secret_value("S3_SECRET_ACCESS_KEY") +USER_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +if SESSION_TTL_SECONDS < 1: + raise RuntimeError("SESSION_TTL_SECONDS must be positive") +if IDEMPOTENCY_TTL_SECONDS < 86400: + raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400") +if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1: + raise RuntimeError("Database timeouts must be positive") +if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0: + raise RuntimeError("Database pool settings must be positive") +if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1: + raise RuntimeError("HTTP limits must be positive") +if MAX_UPLOAD_BYTES < 1 or MAX_UPLOAD_BYTES > 10 * 1024 * 1024: + raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10 MiB") +if MAX_IMAGE_PIXELS < 1 or MAX_IMAGE_PIXELS > 40_000_000: + raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40 million") +if MAX_CONCURRENT_UPLOADS < 1: + raise RuntimeError("MAX_CONCURRENT_UPLOADS must be positive") +if UPLOAD_TIMEOUT_SECONDS < 1: + raise RuntimeError("UPLOAD_TIMEOUT_SECONDS must be positive") diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..de78501f --- /dev/null +++ b/src/errors.py @@ -0,0 +1,26 @@ +from typing import Dict, Optional + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +class CacheUnavailableError(Exception): + pass + + +class IdempotencyConflictError(Exception): + pass + + +class AttachmentNotReadyError(Exception): + pass + + +class ObjectMissingError(Exception): + pass diff --git a/src/handlers.py b/src/handlers.py new file mode 100644 index 00000000..51da7252 --- /dev/null +++ b/src/handlers.py @@ -0,0 +1,548 @@ +import base64 +import hashlib +import json +import logging +import threading +import time +import tempfile +import uuid +import warnings +from http.server import BaseHTTPRequestHandler +from typing import Any, BinaryIO, Dict, List, Optional, Tuple +from urllib.parse import parse_qs, urlsplit +import unicodedata + +import psycopg +from botocore.exceptions import BotoCoreError, ClientError +from PIL import Image, UnidentifiedImageError +from psycopg_pool import PoolTimeout + +from .config import ( + IDEMPOTENCY_KEY_RE, + INSTANCE_ID, + MAX_CONCURRENT_UPLOADS, + MAX_IMAGE_PIXELS, + MAX_INFLIGHT_REQUESTS, + MAX_REQUEST_BODY_BYTES, + MAX_UPLOAD_BYTES, + UPLOAD_TIMEOUT_SECONDS, + USER_RE, +) +from .errors import ( + ApiError, + AttachmentNotReadyError, + CacheUnavailableError, + IdempotencyConflictError, + ObjectMissingError, +) +from .security import hash_password, isoformat, valid_uuid +from .store import Store + +LOGGER = logging.getLogger("messenger.http") + + +class RequestGuard: + def __init__(self): + self.lock = threading.Lock() + self.inflight = 0 + + def acquire(self): + with self.lock: + if self.inflight >= MAX_INFLIGHT_REQUESTS: + raise ApiError(503, "unavailable", "Server is busy") + self.inflight += 1 + + def release(self): + with self.lock: + self.inflight = max(0, self.inflight - 1) + + +REQUEST_GUARD = RequestGuard() +UPLOAD_GUARD = threading.BoundedSemaphore(MAX_CONCURRENT_UPLOADS) + + +class MessengerHandler(BaseHTTPRequestHandler): + def __init__(self, request, client_address, server, store: Store): + self.store = store + super().__init__(request, client_address, server) + + def setup(self): + super().setup() + self.connection.settimeout(15) + + def log_message(self, format: str, *args): + return + + def do_GET(self): + self._dispatch("GET") + + def do_POST(self): + self._dispatch("POST") + + def do_DELETE(self): + self._dispatch("DELETE") + + def _dispatch(self, method: str): + request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4()) + acquired = False + try: + REQUEST_GUARD.acquire() + acquired = True + parsed = urlsplit(self.path) + path = parsed.path + query = parse_qs(parsed.query, keep_blank_values=True) + if path == "/health/live" and method == "GET": + self._send_json(200, {"status": "ok"}, request_id) + elif path == "/health/ready" and method == "GET": + try: + self.store.ready() + except (psycopg.Error, PoolTimeout, OSError, BotoCoreError, ClientError): + raise ApiError(503, "unavailable", "A required storage service is unavailable") + self._send_json(200, {"status": "ok"}, request_id) + elif path == "/api/v1/users" and method == "POST": + self._handle_register(request_id) + elif path == "/api/v1/auth/sessions" and method == "POST": + self._handle_create_session(request_id) + elif path == "/api/v1/auth/sessions/current" and method == "DELETE": + self._handle_revoke_session(request_id) + elif path == "/api/v1/users/me" and method == "GET": + self._send_json(200, self._get_actor(request_id), request_id) + elif path == "/api/v1/chats" and method == "POST": + self._handle_create_chat(request_id) + elif path == "/api/v1/chats" and method == "GET": + self._handle_list_chats(query, request_id) + elif path.startswith("/api/v1/chats/") and path.endswith("/attachments") and method == "POST": + chat_id = path.rsplit("/", 2)[1] + self._handle_upload_attachment(chat_id, query, request_id) + elif path.startswith("/api/v1/attachments/") and method == "GET": + attachment_path = path.removeprefix("/api/v1/attachments/") + if attachment_path.endswith("/content"): + attachment_id = attachment_path.removesuffix("/content") + self._handle_download_attachment(attachment_id, query, request_id) + else: + self._handle_get_attachment(attachment_path, request_id) + elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"): + chat_id = path.rsplit("/", 2)[1] + if method == "POST": + self._handle_create_message(chat_id, request_id) + else: + self._handle_list_messages(chat_id, query, request_id) + elif path.startswith("/api/v1/chats/") and method == "GET": + self._handle_get_chat(path.rsplit("/", 1)[1], request_id) + else: + raise ApiError(404, "not_found", "Object not found") + except ApiError as exc: + self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers) + except CacheUnavailableError: + self._send_error(503, "unavailable", "Session cache is unavailable", request_id) + except IdempotencyConflictError: + self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id) + except AttachmentNotReadyError: + self._send_error(409, "conflict", "Attachment is not ready", request_id) + except ObjectMissingError: + LOGGER.error("attachment object missing for request %s", request_id) + self._send_error(503, "unavailable", "Attachment content is temporarily unavailable", request_id) + except (BotoCoreError, ClientError): + LOGGER.warning("object storage unavailable for request %s", request_id) + self._send_error(503, "unavailable", "Object storage is unavailable", request_id) + except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError): + self._send_error(503, "unavailable", "Database is unavailable", request_id) + except psycopg.errors.UniqueViolation: + self._send_error(409, "conflict", "Username already exists", request_id) + except psycopg.errors.ForeignKeyViolation: + self._send_error(400, "invalid_request", "Unknown member", request_id) + except Exception as exc: + LOGGER.error("request %s failed with %s", request_id, type(exc).__name__) + self._send_error(500, "internal_error", "Internal server error", request_id) + finally: + if acquired: + REQUEST_GUARD.release() + + def _read_json(self, request_id: str) -> Any: + if self.headers.get_content_type() != "application/json": + raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json") + length = self.headers.get("Content-Length") + if length is None: + raise ApiError(400, "invalid_request", "Missing Content-Length") + try: + size = int(length) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size < 0: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size > MAX_REQUEST_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Request body too large") + raw = self.rfile.read(size) + if not raw: + raise ApiError(400, "invalid_request", "Empty body") + try: + return json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + raise ApiError(400, "invalid_request", "Malformed JSON") + + def _get_actor(self, request_id: str) -> Dict[str, Any]: + header = self.headers.get("Authorization", "") + scheme, separator, token = header.partition(" ") + if scheme.lower() != "bearer" or not separator or not token or " " in token: + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + user = self.store.session_user(token) + if user is None: + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + return user + + def _handle_register(self, request_id: str): + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}: + raise ApiError(400, "invalid_request", "Expected username, display_name and password") + username = body["username"] + display_name = body["display_name"] + password = body["password"] + if not isinstance(username, str) or not USER_RE.fullmatch(username): + raise ApiError(400, "invalid_request", "Invalid username") + if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100: + raise ApiError(400, "invalid_request", "Invalid display_name") + if not isinstance(password, str) or not (12 <= len(password) <= 128): + raise ApiError(400, "invalid_request", "Invalid password") + user = self.store.create_user(username, display_name, hash_password(password)) + self._send_json(201, user, request_id) + + def _basic_credentials(self, request_id: str) -> Tuple[str, str]: + value = self.headers.get("Authorization", "") + scheme, separator, encoded = value.partition(" ") + challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + if scheme.lower() != "basic" or not separator or not encoded: + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + try: + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, UnicodeError): + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + return username, password + + def _handle_create_session(self, request_id: str): + username, password = self._basic_credentials(request_id) + user = self.store.authenticate(username, password) + if user is None: + raise ApiError( + 401, + "unauthorized", + "Invalid credentials", + {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}, + ) + token, expires_at = self.store.create_session(user["id"]) + self._send_json( + 201, + {"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user}, + request_id, + ) + + def _handle_revoke_session(self, request_id: str): + self._get_actor(request_id) + token = self.headers["Authorization"].partition(" ")[2] + if not self.store.revoke_session(token): + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + self.send_response(204) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.end_headers() + + def _handle_create_chat(self, request_id: str): + actor = self._get_actor(request_id) + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"title", "member_ids"}: + raise ApiError(400, "invalid_request", "Expected title and member_ids") + title, member_ids = body["title"], body["member_ids"] + if not isinstance(title, str) or not title.strip() or len(title) > 100: + raise ApiError(400, "invalid_request", "Invalid title") + if ( + not isinstance(member_ids, list) + or not 1 <= len(member_ids) <= 100 + or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids) + or len(member_ids) != len(set(member_ids)) + or actor["id"] not in member_ids + ): + raise ApiError(400, "invalid_request", "Invalid member_ids") + chat = self.store.create_chat(title, member_ids, actor["id"]) + self._send_json(201, chat, request_id) + + def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str): + actor = self._get_actor(request_id) + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid limit") + if not 1 <= limit <= 100: + raise ApiError(400, "invalid_request", "Invalid limit") + cursor = query.get("cursor", [None])[0] + try: + chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id) + + def _handle_get_chat(self, chat_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat = self.store.get_chat(chat_id, actor["id"]) + if chat is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(200, chat, request_id) + + def _handle_create_message(self, chat_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat_id = str(uuid.UUID(chat_id)) + idempotency_key = self.headers.get("Idempotency-Key", "") + if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key") + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body).difference({"text", "attachment_ids"}) or "text" not in body: + raise ApiError(400, "invalid_request", "Expected text and optional attachment_ids") + text = body["text"] + attachment_ids = body.get("attachment_ids", []) + if ( + not isinstance(text, str) + or len(text) > 4000 + or not isinstance(attachment_ids, list) + or len(attachment_ids) > 10 + or any(not isinstance(item, str) or not valid_uuid(item) for item in attachment_ids) + or len(attachment_ids) != len(set(attachment_ids)) + or (not text.strip() and not attachment_ids) + ): + raise ApiError(400, "invalid_request", "Invalid message text") + attachment_ids = [str(uuid.UUID(item)) for item in attachment_ids] + canonical_body = json.dumps( + {"text": text, "attachment_ids": attachment_ids}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + message = self.store.create_idempotent_message( + chat_id, + actor["id"], + text, + attachment_ids, + idempotency_key, + canonical_body, + ) + if message is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(201, message, request_id) + + def _read_image(self, request_id: str, declared_type: str) -> Tuple[BinaryIO, str, int, str]: + if declared_type not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are supported") + length = self.headers.get("Content-Length") + if length is None or self.headers.get("Transfer-Encoding"): + raise ApiError(400, "invalid_request", "A bounded Content-Length is required") + try: + size = int(length) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size <= 0: + raise ApiError(415, "unsupported_media_type", "Image is empty") + if size > MAX_UPLOAD_BYTES: + self.close_connection = True + remaining = min(size, MAX_UPLOAD_BYTES + 1) + deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS + while remaining: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + break + self.connection.settimeout(min(15, remaining_time)) + try: + chunk = self.rfile.read(min(64 * 1024, remaining)) + except (TimeoutError, OSError): + LOGGER.info("stopped bounded drain of oversized upload") + break + if not chunk: + break + remaining -= len(chunk) + raise ApiError(413, "payload_too_large", "Image exceeds the 10 MiB limit") + content = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") + digest = hashlib.sha256() + bytes_read = 0 + deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS + try: + while bytes_read < size: + remaining_time = deadline - time.monotonic() + if remaining_time <= 0: + raise ApiError(408, "invalid_request", "Image upload timed out") + self.connection.settimeout(min(15, remaining_time)) + chunk = self.rfile.read(min(64 * 1024, size - bytes_read)) + if not chunk: + raise ApiError(400, "invalid_request", "Incomplete image body") + content.write(chunk) + digest.update(chunk) + bytes_read += len(chunk) + except (TimeoutError, OSError): + content.close() + raise ApiError(400, "invalid_request", "Incomplete image body") from None + + try: + content.seek(0) + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(content) as image: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format or "") + if actual_type is None or actual_type != declared_type: + raise ApiError(415, "unsupported_media_type", "Image content does not match Content-Type") + width, height = image.size + if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") + image.verify() + content.seek(0) + return content, actual_type, size, digest.hexdigest() + except ApiError: + content.close() + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + content.close() + raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") from None + except (UnidentifiedImageError, OSError, SyntaxError, ValueError): + content.close() + raise ApiError(415, "unsupported_media_type", "Image is damaged or unsupported") from None + + @staticmethod + def _valid_filename(filename: str) -> bool: + return ( + 1 <= len(filename) <= 255 + and not any(char in ("/", "\\", "\x00") for char in filename) + and not any(unicodedata.category(char) == "Cc" for char in filename) + ) + + def _handle_upload_attachment( + self, chat_id: str, query: Dict[str, List[str]], request_id: str + ): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat_id = str(uuid.UUID(chat_id)) + if not self.store.can_access_chat(chat_id, actor["id"]): + raise ApiError(404, "not_found", "Chat not found") + filenames = query.get("filename", []) + if len(filenames) != 1 or not self._valid_filename(filenames[0]): + raise ApiError(400, "invalid_request", "Invalid filename") + if not UPLOAD_GUARD.acquire(blocking=False): + raise ApiError(503, "unavailable", "Upload capacity is temporarily exhausted") + try: + content, content_type, size, digest = self._read_image(request_id, self.headers.get_content_type()) + try: + attachment = self.store.create_attachment( + chat_id, + actor["id"], + filenames[0], + content_type, + content, + size, + digest, + ) + finally: + content.close() + finally: + UPLOAD_GUARD.release() + if attachment is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(201, attachment, request_id) + + def _handle_get_attachment(self, attachment_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(attachment_id): + raise ApiError(400, "invalid_request", "Invalid attachment id") + attachment = self.store.get_attachment(str(uuid.UUID(attachment_id)), actor["id"]) + if attachment is None: + raise ApiError(404, "not_found", "Attachment not found") + self._send_json(200, attachment, request_id) + + def _handle_download_attachment( + self, attachment_id: str, query: Dict[str, List[str]], request_id: str + ): + actor = self._get_actor(request_id) + if not valid_uuid(attachment_id): + raise ApiError(400, "invalid_request", "Invalid attachment id") + if query.get("variant", ["original"]) != ["original"]: + raise ApiError(404, "not_found", "Attachment variant not found") + result = self.store.download_attachment(str(uuid.UUID(attachment_id)), actor["id"]) + if result is None: + raise ApiError(404, "not_found", "Attachment not found") + content_type, content = result + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(content))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("Cache-Control", "private, no-store") + self.end_headers() + self.wfile.write(content) + + def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid limit") + if not 1 <= limit <= 100: + raise ApiError(400, "invalid_request", "Invalid limit") + cursor = query.get("cursor", [None])[0] + try: + result = self.store.list_messages(chat_id, actor["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + if result is None: + raise ApiError(404, "not_found", "Chat not found") + items, next_cursor = result + self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id) + + def _send_json( + self, + status: int, + payload: Dict[str, Any], + request_id: str, + headers: Optional[Dict[str, str]] = None, + ): + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + if headers: + for key, value in headers.items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(body) + + def _send_error( + self, + status: int, + code: str, + message: str, + request_id: str, + extra_headers: Optional[Dict[str, str]] = None, + ): + self._send_json( + status, + {"error": {"code": code, "message": message, "request_id": request_id}}, + request_id, + extra_headers, + ) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..1ae27009 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +import logging +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.store import Store +else: + from .store import Store + + +def main(): + logging.basicConfig(level="INFO") + store = Store() + store.open() + try: + store.migrate() + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/object_store.py b/src/object_store.py new file mode 100644 index 00000000..b49c67f4 --- /dev/null +++ b/src/object_store.py @@ -0,0 +1,76 @@ +import logging +from datetime import datetime +from typing import BinaryIO, Iterator, Optional, Tuple, Union + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from .config import S3_ACCESS_KEY_ID, S3_BUCKET, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY +from .errors import ObjectMissingError + +LOGGER = logging.getLogger("messenger.s3") + + +class ObjectStore: + def __init__(self): + self.client = boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + region_name=S3_REGION, + aws_access_key_id=S3_ACCESS_KEY_ID, + aws_secret_access_key=S3_SECRET_ACCESS_KEY, + config=Config( + connect_timeout=2, + read_timeout=10, + retries={"max_attempts": 2, "mode": "standard"}, + s3={"addressing_style": "path"}, + ), + ) + + def put( + self, + key: str, + content: Union[bytes, BinaryIO], + content_type: str, + size_bytes: Optional[int] = None, + ) -> None: + if size_bytes is None: + size_bytes = len(content) + self.client.put_object( + Bucket=S3_BUCKET, + Key=key, + Body=content, + ContentLength=size_bytes, + ContentType=content_type, + ) + + def get(self, key: str, max_bytes: int) -> bytes: + try: + response = self.client.get_object(Bucket=S3_BUCKET, Key=key) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + if code in ("NoSuchKey", "404", "NotFound"): + raise ObjectMissingError from exc + raise + body = response["Body"] + try: + value = body.read(max_bytes + 1) + finally: + body.close() + if len(value) > max_bytes: + raise RuntimeError("Stored object exceeds configured safety limit") + return value + + def delete(self, key: str) -> None: + self.client.delete_object(Bucket=S3_BUCKET, Key=key) + + def ready(self) -> bool: + self.client.head_bucket(Bucket=S3_BUCKET) + return True + + def objects(self) -> Iterator[Tuple[str, datetime]]: + pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=S3_BUCKET) + for page in pages: + for item in page.get("Contents", []): + yield item["Key"], item["LastModified"] diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..68c2d691 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import logging +import sys +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Optional + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.store import Store +else: + from .store import Store + + +LOGGER = logging.getLogger("messenger.reconcile") + + +def reconcile( + store: Store, + grace_seconds: int, + apply_changes: bool, + restore_dir: Optional[Path] = None, +): + started = time.monotonic() + metadata = store.reconciliation_snapshot() + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + found = set() + orphans = [] + for key, last_modified in store.objects.objects(): + found.add(key) + if key not in metadata and last_modified <= cutoff: + orphans.append(key) + + missing = [row for key, row in metadata.items() if key not in found] + restore_candidates = [] + if restore_dir is not None: + root = restore_dir.resolve(strict=True) + for row in missing: + source = restore_dir / str(row["id"]) + try: + resolved = source.resolve(strict=True) + if ( + resolved.parent != root + or not resolved.is_file() + or resolved.stat().st_size != row["size_bytes"] + ): + continue + content = resolved.read_bytes() + except OSError: + continue + if ( + len(content) == row["size_bytes"] + and hashlib.sha256(content).hexdigest() == row["sha256"].strip() + ): + restore_candidates.append((row, content)) + + restored = 0 + if apply_changes: + for key in orphans: + store.objects.delete(key) + restorable_ids = set() + for row, content in restore_candidates: + store.objects.put(row["object_key"], content, row["content_type"]) + store.clear_missing_object(row["id"]) + restorable_ids.add(row["id"]) + restored += 1 + for row in missing: + if row["id"] not in restorable_ids: + store.record_missing_object(row["id"]) + for key, row in metadata.items(): + if key in found: + store.clear_missing_object(row["id"]) + + unresolved_missing = len(missing) - restored + return { + "mode": "apply" if apply_changes else "dry-run", + "grace_seconds": grace_seconds, + "objects_seen": len(found), + "metadata_seen": len(metadata), + "orphan_objects": len([key for key in found if key not in metadata]), + "stale_orphan_candidates": len(orphans), + "deleted_orphans": len(orphans) if apply_changes else 0, + "missing_objects": len(missing), + "unresolved_missing_objects": unresolved_missing, + "restorable_missing_objects": len(restore_candidates), + "restored_missing_objects": restored, + "missing_incidents_recorded": len(missing) - restored if apply_changes else 0, + "duration_ms": round((time.monotonic() - started) * 1000, 3), + } + + +def main(): + parser = argparse.ArgumentParser(description="Compare S3 objects with PostgreSQL attachment metadata.") + parser.add_argument("--grace-seconds", type=int, default=3600) + parser.add_argument("--apply", action="store_true", help="delete only stale orphan objects and record missing objects") + parser.add_argument( + "--restore-dir", + type=Path, + help="optional trusted backup directory with exact original bytes named by attachment UUID", + ) + args = parser.parse_args() + if args.grace_seconds < 60: + parser.error("--grace-seconds must be at least 60") + + logging.basicConfig(level="INFO", format="%(levelname)s %(message)s") + store = Store() + store.open() + try: + result = reconcile(store, args.grace_seconds, args.apply, args.restore_dir) + print(json.dumps(result, sort_keys=True)) + if result["unresolved_missing_objects"]: + LOGGER.error( + "%s database records reference missing S3 objects; metadata was retained for manual recovery", + result["unresolved_missing_objects"], + ) + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/security.py b/src/security.py new file mode 100644 index 00000000..6ec821f7 --- /dev/null +++ b/src/security.py @@ -0,0 +1,53 @@ +import base64 +import hashlib +import hmac +import secrets +import uuid +from datetime import datetime, timezone + + +def isoformat(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def valid_uuid(value: str) -> bool: + try: + uuid.UUID(value) + return True + except (ValueError, TypeError, AttributeError): + return False + + +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + derived = hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024 + ) + salt_text = base64.urlsafe_b64encode(salt).decode("ascii") + hash_text = base64.urlsafe_b64encode(derived).decode("ascii") + return f"scrypt$32768$8$1${salt_text}${hash_text}" + + +def verify_password(password: str, stored: str) -> bool: + try: + scheme, n, r, p, salt_text, hash_text = stored.split("$") + if scheme != "scrypt": + return False + salt = base64.urlsafe_b64decode(salt_text.encode("ascii")) + expected = base64.urlsafe_b64decode(hash_text.encode("ascii")) + actual = hashlib.scrypt( + password.encode("utf-8"), + salt=salt, + n=int(n), + r=int(r), + p=int(p), + dklen=len(expected), + maxmem=64 * 1024 * 1024, + ) + return hmac.compare_digest(actual, expected) + except (ValueError, TypeError, UnicodeError): + return False diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..60fd3d56 --- /dev/null +++ b/src/server.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.app import main +else: + from .app import main + + +if __name__ == "__main__": + main() diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..0bd8a98a --- /dev/null +++ b/src/store.py @@ -0,0 +1,548 @@ +import base64 +import hashlib +import hmac +import json +import logging +import math +import os +import secrets +import uuid +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, BinaryIO, Dict, List, Optional, Tuple + +import redis +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from .config import ( + DB_LOCK_TIMEOUT_MS, + DB_POOL_MAX_SIZE, + DB_POOL_TIMEOUT_SECONDS, + DB_STATEMENT_TIMEOUT_MS, + IDEMPOTENCY_TTL_SECONDS, + SESSION_TTL_SECONDS, +) +from .errors import ( + AttachmentNotReadyError, + CacheUnavailableError, + IdempotencyConflictError, +) +from .object_store import ObjectStore +from .security import isoformat, utc_now, valid_uuid, verify_password + +LOGGER = logging.getLogger("messenger.store") + + +def database_url() -> str: + value = os.getenv("DATABASE_URL") + if value: + return value + file_path = os.getenv("DATABASE_URL_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + raise RuntimeError("Set DATABASE_URL or DATABASE_URL_FILE") + + +class Store: + """PostgreSQL metadata, S3 objects, and optional Redis session cache.""" + + def __init__(self): + self.pool = ConnectionPool( + conninfo=database_url(), + min_size=1, + max_size=DB_POOL_MAX_SIZE, + timeout=DB_POOL_TIMEOUT_SECONDS, + kwargs={ + "row_factory": dict_row, + "connect_timeout": 5, + "options": ( + f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} " + f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} " + "-c idle_in_transaction_session_timeout=10000" + ), + }, + open=False, + ) + cache_url = os.getenv("CACHE_URL") + self.cache = ( + redis.Redis.from_url( + cache_url, + decode_responses=True, + socket_connect_timeout=0.25, + socket_timeout=0.25, + retry_on_timeout=False, + max_connections=16, + ) + if cache_url + else None + ) + self.objects = ObjectStore() + + def open(self): + self.pool.open(wait=True, timeout=10) + + def close(self): + if self.cache is not None: + self.cache.close() + self.pool.close() + + def ready(self) -> bool: + with self.pool.connection(timeout=2) as conn: + conn.execute("SELECT 1") + self.objects.ready() + return True + + def migrate(self): + migrations = Path(__file__).resolve().parents[1] / "migrations" + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations " + "(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" + ) + conn.execute("SELECT pg_advisory_xact_lock(2741662026)") + applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")} + for migration in sorted(migrations.glob("*.sql")): + if migration.name in applied: + continue + conn.execute(migration.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,)) + + @staticmethod + def user(row: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": isoformat(row["created_at"]), + } + + @staticmethod + def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": isoformat(row["created_at"]), + } + + @staticmethod + def message(row: Dict[str, Any], attachment_ids: Optional[List[str]] = None) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": isoformat(row["created_at"]), + "attachment_ids": attachment_ids or [], + } + + @staticmethod + def attachment(row: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"].strip(), + "status": "ready", + "variants": [], + "error_code": None, + "created_at": isoformat(row["created_at"]), + "updated_at": isoformat(row["updated_at"]), + } + + def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO users (username, display_name, password_hash) " + "VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at", + (username, display_name, password_hash), + ).fetchone() + return self.user(row) + + def get_user(self, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (user_id,), + ).fetchone() + return self.user(row) if row else None + + def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at, password_hash " + "FROM users WHERE username = %s", + (username,), + ).fetchone() + if not row or not verify_password(password, row["password_hash"]): + return None + return self.user(row) + + def create_session(self, user_id: str) -> Tuple[str, datetime]: + token = secrets.token_urlsafe(32) + expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS) + token_hash = hashlib.sha256(token.encode("ascii")).digest() + with self.pool.connection() as conn: + conn.execute( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)", + (token_hash, user_id, expires_at), + ) + return token, expires_at + + @staticmethod + def _session_lock_key(token_hash: bytes) -> int: + return int.from_bytes(token_hash[:8], byteorder="big", signed=True) + + def session_user(self, token: str) -> Optional[Dict[str, Any]]: + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + cache_key = "messenger:session:" + token_hash.hex() + if self.cache is not None: + try: + cached = self.cache.get(cache_key) + if cached: + return json.loads(cached) + except (redis.RedisError, ValueError, TypeError): + LOGGER.warning("session cache read failed; falling back to PostgreSQL") + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) + row = conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at " + "FROM sessions s JOIN users u ON u.id = s.user_id " + "WHERE s.token_hash = %s AND s.expires_at > now()", + (token_hash,), + ).fetchone() + if not row: + return None + user = self.user(row) + if self.cache is not None: + ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000) + if ttl_ms > 0: + try: + self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms) + except redis.RedisError: + LOGGER.warning("session cache write failed; PostgreSQL remains authoritative") + return user + + def revoke_session(self, token: str) -> bool: + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) + if self.cache is not None: + try: + self.cache.delete("messenger:session:" + token_hash.hex()) + except redis.RedisError as exc: + raise CacheUnavailableError from exc + result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,)) + return result.rowcount == 1 + + def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]: + if actor_id not in member_ids or len(member_ids) != len(set(member_ids)): + raise ValueError("invalid members") + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at", + (title,), + ).fetchone() + for member_id in member_ids: + conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + (row["id"], member_id), + ) + return self.chat(row, member_ids) + + def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members m ON m.chat_id = c.id " + "WHERE c.id = %s AND m.user_id = %s", + (chat_id, user_id), + ).fetchone() + if not row: + return None + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", + (chat_id,), + ).fetchall() + return self.chat(row, [str(item["user_id"]) for item in members]) + + def list_chats( + self, user_id: str, limit: int, cursor: Optional[str] + ) -> Tuple[List[Dict[str, Any]], Optional[str]]: + where = "own.user_id = %s" + params: List[Any] = [user_id] + if cursor is not None: + try: + raw = base64.b64decode( + cursor.encode("ascii") + b"=" * (-len(cursor) % 4), + altchars=b"-_", + validate=True, + ) + cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2) + if cursor_user != user_id or not valid_uuid(chat_id): + raise ValueError("cursor scope") + created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + if created_at.tzinfo is None or created_at.utcoffset() is None: + raise ValueError("cursor timestamp must include a timezone") + where += " AND (c.created_at, c.id) > (%s, %s)" + params.extend((created_at, chat_id)) + except (ValueError, UnicodeError, TypeError) as exc: + raise ValueError("invalid cursor") from exc + params.append(limit + 1) + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members own ON own.chat_id = c.id " + f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s", + params, + ).fetchall() + has_more = len(rows) > limit + page = rows[:limit] + result = [] + for row in page: + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", + (row["id"],), + ).fetchall() + result.append(self.chat(row, [str(item["user_id"]) for item in members])) + next_cursor = None + if has_more and page: + last = page[-1] + cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") + next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") + return result, next_cursor + + def create_idempotent_message( + self, + chat_id: str, + sender_id: str, + text: str, + attachment_ids: List[str], + idempotency_key: str, + canonical_body: str, + ) -> Optional[Dict[str, Any]]: + request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest() + lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8") + lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True) + with self.pool.connection() as conn: + with conn.transaction(): + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ).fetchone() + if not member: + return None + if attachment_ids: + attachments = conn.execute( + "SELECT id, status FROM attachments " + "WHERE chat_id = %s AND id = ANY(%s) FOR SHARE", + (chat_id, attachment_ids), + ).fetchall() + if len(attachments) != len(attachment_ids): + return None + if any(item["status"] != "ready" for item in attachments): + raise AttachmentNotReadyError + + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,)) + existing = conn.execute( + "SELECT request_hash, response, expires_at " + "FROM idempotency_records " + "WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s " + "FOR UPDATE", + (sender_id, chat_id, idempotency_key), + ).fetchone() + if existing and existing["expires_at"] > utc_now(): + if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash): + raise IdempotencyConflictError + return existing["response"] + if existing: + conn.execute( + "DELETE FROM idempotency_records " + "WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s", + (sender_id, chat_id, idempotency_key), + ) + + row = conn.execute( + "INSERT INTO messages (chat_id, sender_id, text) " + "VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at", + (chat_id, sender_id, text), + ).fetchone() + for ordinal, attachment_id in enumerate(attachment_ids): + conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, ordinal) " + "VALUES (%s, %s, %s)", + (row["id"], attachment_id, ordinal), + ) + message = self.message(row, attachment_ids) + conn.execute( + "INSERT INTO idempotency_records " + "(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) " + "VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)", + ( + sender_id, + chat_id, + idempotency_key, + request_hash, + json.dumps(message, ensure_ascii=False, separators=(",", ":")), + utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS), + ), + ) + return message + + def list_messages( + self, chat_id: str, user_id: str, limit: int, cursor: Optional[str] + ) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]: + with self.pool.connection() as conn: + is_member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + if not is_member: + return None + where = "chat_id = %s" + params: List[Any] = [chat_id] + if cursor: + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4)) + decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2) + if decoded_chat != chat_id or not valid_uuid(message_id): + raise ValueError("cursor scope") + created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + where += " AND (created_at, id) > (%s, %s)" + params.extend((created_at, message_id)) + except (ValueError, UnicodeError, TypeError) as exc: + raise ValueError("invalid cursor") from exc + params.append(limit + 1) + rows = conn.execute( + "SELECT id, chat_id, sender_id, text, created_at FROM messages " + f"WHERE {where} ORDER BY created_at, id LIMIT %s", + params, + ).fetchall() + ids_by_message = {} + if rows: + links = conn.execute( + "SELECT message_id, attachment_id FROM message_attachments " + "WHERE message_id = ANY(%s) ORDER BY message_id, ordinal", + ([row["id"] for row in rows],), + ).fetchall() + for link in links: + ids_by_message.setdefault(link["message_id"], []).append(str(link["attachment_id"])) + has_more = len(rows) > limit + page = rows[:limit] + next_cursor = None + if has_more and page: + last = page[-1] + cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") + next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") + return [self.message(row, ids_by_message.get(row["id"], [])) for row in page], next_cursor + + def can_access_chat(self, chat_id: str, user_id: str) -> bool: + with self.pool.connection() as conn: + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() is not None + + def create_attachment( + self, + chat_id: str, + owner_id: str, + filename: str, + content_type: str, + content: BinaryIO, + size_bytes: int, + digest: str, + ) -> Optional[Dict[str, Any]]: + if not self.can_access_chat(chat_id, owner_id): + return None + attachment_id = uuid.uuid4() + object_key = f"original/{attachment_id}" + content.seek(0) + self.objects.put(object_key, content, content_type, size_bytes) + try: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO attachments " + "(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s) " + "RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "created_at, updated_at", + ( + attachment_id, + chat_id, + owner_id, + filename, + content_type, + size_bytes, + digest, + object_key, + ), + ).fetchone() + return self.attachment(row) + except Exception: + try: + self.objects.delete(object_key) + except Exception as cleanup_error: + LOGGER.error( + "attachment compensation failed for %s (%s)", + attachment_id, + type(cleanup_error).__name__, + ) + raise + + def get_attachment(self, attachment_id: str, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, " + "a.sha256, a.created_at, a.updated_at " + "FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " + "WHERE a.id = %s AND m.user_id = %s", + (attachment_id, user_id), + ).fetchone() + return self.attachment(row) if row else None + + def download_attachment(self, attachment_id: str, user_id: str) -> Optional[Tuple[str, bytes]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT a.object_key, a.content_type, a.size_bytes " + "FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " + "WHERE a.id = %s AND m.user_id = %s AND a.status = 'ready'", + (attachment_id, user_id), + ).fetchone() + if not row: + return None + return row["content_type"], self.objects.get(row["object_key"], row["size_bytes"]) + + def reconciliation_snapshot(self): + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT id, object_key, created_at, size_bytes, sha256, content_type FROM attachments" + ).fetchall() + return {row["object_key"]: row for row in rows} + + def record_missing_object(self, attachment_id: str) -> None: + with self.pool.connection() as conn: + conn.execute( + "INSERT INTO reconciliation_incidents (attachment_id, incident_type) " + "VALUES (%s, 'missing_object') ON CONFLICT (attachment_id, incident_type) " + "DO UPDATE SET last_seen_at = clock_timestamp()", + (attachment_id,), + ) + + def clear_missing_object(self, attachment_id: str) -> None: + with self.pool.connection() as conn: + conn.execute( + "DELETE FROM reconciliation_incidents " + "WHERE attachment_id = %s AND incident_type = 'missing_object'", + (attachment_id,), + ) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..0890f7ad --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Negative attachment and cross-chat authorization checks against a running lab4 API.""" +import uuid +from pathlib import Path +import sys +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def upload(client, actor, chat_id, data, content_type="image/png", filename="test.png", expected=201): + return client.call( + "POST", + "/api/v1/chats/" + chat_id + "/attachments?filename=" + quote(filename, safe=""), + expected, + actor=actor, + raw=data, + headers={"Content-Type": content_type}, + schema="Attachment" if expected == 201 else None, + ) + + +def main(): + args = arguments("Attachment upload, size, MIME, and chat authorization checks") + client = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = client.bootstrap() + other_chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Second test chat", "member_ids": [alice["id"], eve["id"]]}, + schema="Chat", + )["json"] + image = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + print("Checking accepted fixture upload.", flush=True) + attachment = upload(client, alice, chat["id"], image)["json"] + + print("Checking damaged and spoofed image rejection.", flush=True) + upload(client, alice, chat["id"], b"not an image", expected=415) + upload(client, alice, chat["id"], image, content_type="image/jpeg", expected=415) + print("Checking byte and pixel limits.", flush=True) + upload(client, alice, chat["id"], b"x" * (10 * 1024 * 1024 + 1), expected=413) + import struct + import zlib + + def chunk(kind, payload): + return ( + struct.pack(">I", len(payload)) + + kind + + payload + + struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF) + ) + + too_many_pixels = ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", 10001, 4000, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"")) + + chunk(b"IEND", b"") + ) + upload(client, alice, chat["id"], too_many_pixels, expected=413) + print("Checking filename and cross-chat authorization.", flush=True) + upload(client, alice, chat["id"], image, filename="../unsafe.png", expected=400) + upload(client, eve, chat["id"], image, expected=404) + client.call("GET", "/api/v1/attachments/" + attachment["id"], 404, actor=eve) + client.call( + "POST", + "/api/v1/chats/" + other_chat["id"] + "/messages", + 404, + actor=alice, + headers={"Idempotency-Key": uuid.uuid4().hex}, + body={"text": "", "attachment_ids": [attachment["id"]]}, + ) + client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 400, + actor=bob, + headers={"Idempotency-Key": uuid.uuid4().hex}, + body={"text": ""}, + ) + print("PASS: oversized, damaged, spoofed, unsafe-name, outsider, and cross-chat cases") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_idempotency_race.py b/student-tests/test_idempotency_race.py new file mode 100644 index 00000000..646869a7 --- /dev/null +++ b/student-tests/test_idempotency_race.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Prove a concurrent idempotency-key race through the load balancer.""" +from concurrent.futures import ThreadPoolExecutor +import threading +import sys +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments("Concurrent shared-idempotency integration check") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + key = "race-" + uuid.uuid4().hex + text = "concurrent-" + uuid.uuid4().hex + count = 12 + barrier = threading.Barrier(count) + + def send_once(_): + worker = Client(args.base_url, args.ca_file) + barrier.wait(timeout=10) + return worker.message(alice, chat, text, key=key) + + with ThreadPoolExecutor(max_workers=count) as pool: + responses = list(pool.map(send_once, range(count))) + + instance_ids = {response["headers"]["X-Instance-Id"] for response in responses} + messages = [response["json"] for response in responses] + check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas") + check(all(message == messages[0] for message in messages), "same-key requests returned different messages") + page = client.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages?limit=100", + actor=alice, + schema="MessagePage", + )["json"] + matches = [message for message in page["items"] if message["text"] == text] + check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages") + print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_reconcile.py b/student-tests/test_reconcile.py new file mode 100644 index 00000000..ff123126 --- /dev/null +++ b/student-tests/test_reconcile.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Deterministic tests of the reconciliation decision rules without deleting real objects.""" +import os +import hashlib +import sys +import tempfile +from pathlib import Path + +os.environ.setdefault("S3_ACCESS_KEY_ID", "test-reconciler") +os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test-only-not-a-real-secret") +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from datetime import datetime, timedelta, timezone + +from src.reconcile import reconcile + + +class FakeObjects: + def __init__(self, items): + self.items = items + self.deleted = [] + + def objects(self): + return iter(self.items) + + def delete(self, key): + self.deleted.append(key) + self.items = [(name, date) for name, date in self.items if name != key] + + def put(self, key, content, content_type): + self.items.append((key, datetime.now(timezone.utc))) + + +class FakeStore: + def __init__(self): + now = datetime.now(timezone.utc) + self.metadata = { + "original/present": {"id": "present-id", "object_key": "original/present"}, + "original/missing": { + "id": "missing-id", + "object_key": "original/missing", + "size_bytes": 4, + "sha256": hashlib.sha256(b"test").hexdigest(), + "content_type": "image/png", + }, + } + self.objects = FakeObjects( + [ + ("original/present", now), + ("original/stale-orphan", now - timedelta(hours=2)), + ("original/new-orphan", now - timedelta(minutes=10)), + ] + ) + self.missing_incidents = set() + self.cleared_incidents = set() + + def reconciliation_snapshot(self): + return self.metadata + + def record_missing_object(self, attachment_id): + self.missing_incidents.add(attachment_id) + + def clear_missing_object(self, attachment_id): + self.missing_incidents.discard(attachment_id) + self.cleared_incidents.add(attachment_id) + + +def main(): + store = FakeStore() + dry_run = reconcile(store, grace_seconds=3600, apply_changes=False) + assert dry_run["stale_orphan_candidates"] == 1 + assert dry_run["missing_objects"] == 1 + assert dry_run["deleted_orphans"] == 0 + assert store.objects.deleted == [] and store.missing_incidents == set() + + applied = reconcile(store, grace_seconds=3600, apply_changes=True) + assert store.objects.deleted == ["original/stale-orphan"] + assert "original/new-orphan" not in store.objects.deleted + assert store.missing_incidents == {"missing-id"} + assert store.cleared_incidents == {"present-id"} + assert applied["deleted_orphans"] == 1 and applied["missing_objects"] == 1 + + with tempfile.TemporaryDirectory() as backup: + restore_dir = Path(backup) + (restore_dir / "missing-id").write_bytes(b"test") + preview = reconcile(store, 3600, False, restore_dir) + assert preview["restorable_missing_objects"] == 1 + assert store.missing_incidents == {"missing-id"} + restored = reconcile(store, 3600, True, restore_dir) + assert restored["restored_missing_objects"] == 1 + assert store.missing_incidents == set() + + repeated = reconcile(store, grace_seconds=3600, apply_changes=True) + assert repeated["stale_orphan_candidates"] == 0 and repeated["missing_objects"] == 0 + assert store.objects.deleted == ["original/stale-orphan"] + assert store.missing_incidents == set() + print("PASS: dry-run is read-only; stale orphan cleanup is idempotent and respects grace period") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_reconcile_integration.py b/student-tests/test_reconcile_integration.py new file mode 100644 index 00000000..be6ccb7b --- /dev/null +++ b/student-tests/test_reconcile_integration.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Exercise real S3/PostgreSQL reconciliation using disposable synthetic objects.""" +import json +import subprocess +import sys +import time +import uuid +from pathlib import Path +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +STORE_CODE = """ +import sys +from pathlib import Path +from src.store import Store + +store = Store() +store.open() +try: + action = sys.argv[1] + if action == "key": + attachment_id = sys.argv[2] + matches = [ + key for key, row in store.reconciliation_snapshot().items() + if str(row["id"]) == attachment_id + ] + if len(matches) != 1: + raise RuntimeError("attachment metadata lookup failed") + print(matches[0]) + elif action == "put": + key, file_path = sys.argv[2:4] + content = Path(file_path).read_bytes() + store.objects.put(key, content, "image/png", len(content)) + elif action == "delete": + store.objects.delete(sys.argv[2]) + else: + raise RuntimeError("unsupported test operation") +finally: + store.close() +""" + +PRIVATE_CHECK_CODE = """ +import sys +import urllib.error +import urllib.request +try: + urllib.request.urlopen("http://s3:8333/messenger-attachments/", timeout=3) +except urllib.error.HTTPError as error: + if error.code == 403: + print("PASS: unauthenticated bucket access denied") + else: + raise RuntimeError("unexpected anonymous S3 status: " + str(error.code)) +else: + raise RuntimeError("private bucket allowed an unauthenticated GET") +""" + + +def docker_exec(container_id, *args): + return subprocess.run( + ["docker", "exec", container_id, *args], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def reconciliation(container_id, *args): + result = docker_exec(container_id, "python3", "src/reconcile.py", "--grace-seconds", "60", *args) + return json.loads(result.stdout) + + +def main(): + args = arguments("Real PostgreSQL/SeaweedFS reconciliation and private-bucket experiment") + client = Client(args.base_url, args.ca_file) + alice, bob, _, chat = client.bootstrap() + attachment, content = client.attachment(alice, chat) + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + + result = subprocess.run( + ["docker", "compose", "ps", "--quiet", "--status", "running", "api"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + containers = result.stdout.split() + check(len(containers) >= 2, "at least two running API replicas are required") + container_id = containers[0] + key = docker_exec( + container_id, "python3", "-c", STORE_CODE, "key", attachment["id"] + ).stdout.strip() + orphan_key = "original/" + str(uuid.uuid4()) + + with tempfile.TemporaryDirectory() as backup: + source = Path(backup) / "original.png" + source.write_bytes(content) + container_file = "/tmp/lab4-orphan-" + uuid.uuid4().hex + ".png" + recovery_dir = "/tmp/lab4-recovery-" + uuid.uuid4().hex + recovery_file = recovery_dir + "/" + attachment["id"] + orphan_created = False + orphan_removed = False + missing_deleted = False + missing_restored = False + try: + subprocess.run(["docker", "cp", str(source), container_id + ":" + container_file], check=True) + docker_exec(container_id, "python3", "-c", STORE_CODE, "put", orphan_key, container_file) + orphan_created = True + docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", key) + missing_deleted = True + + preview = reconciliation(container_id) + check( + preview["orphan_objects"] >= 1 and preview["missing_objects"] == 1, + "dry-run missed injected inconsistencies", + ) + check(preview["deleted_orphans"] == 0, "dry-run modified S3") + immediate_apply = reconciliation(container_id, "--apply") + check(immediate_apply["stale_orphan_candidates"] == 0, "fresh orphan bypassed grace period") + check(immediate_apply["deleted_orphans"] == 0, "fresh orphan was deleted") + + docker_exec(container_id, "mkdir", "-p", recovery_dir) + subprocess.run( + ["docker", "cp", str(source), container_id + ":" + recovery_file], + check=True, + ) + restore_preview = reconciliation(container_id, "--restore-dir", recovery_dir) + check(restore_preview["restorable_missing_objects"] == 1, "exact backup was not identified as restorable") + restored = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir) + check(restored["restored_missing_objects"] == 1, "missing S3 object was not restored") + missing_restored = True + + repeated = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir) + check(repeated["missing_objects"] == 0, "repeat reconciliation still reports missing object") + check(repeated["stale_orphan_candidates"] == 0, "recent orphan did not respect grace period") + print("Waiting for the 60-second orphan grace period.") + time.sleep(61) + + cleanup_preview = reconciliation(container_id) + check(cleanup_preview["stale_orphan_candidates"] == 1, "stale orphan was not identified after grace") + cleaned = reconciliation(container_id, "--apply") + check(cleaned["deleted_orphans"] == 1, "stale orphan was not removed") + orphan_removed = True + final_scan = reconciliation(container_id, "--apply") + check( + final_scan["orphan_objects"] == 0 and final_scan["missing_objects"] == 0, + "repeat scan is not clean", + ) + finally: + if missing_deleted and not missing_restored: + docker_exec(container_id, "python3", "-c", STORE_CODE, "put", key, container_file) + if orphan_created and not orphan_removed: + docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", orphan_key) + + private = docker_exec(container_id, "python3", "-c", PRIVATE_CHECK_CODE).stdout.strip() + check("PASS:" in private, "unauthenticated S3 access check failed") + downloaded = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=bob, + )["bytes"] + check(downloaded == content, "reconciler changed the original of an attachment used by a message") + check(message["attachment_ids"] == [attachment["id"]], "test message did not retain the attachment reference") + print( + json.dumps( + { + "result": "PASS", + "dataset_objects_scanned": final_scan["objects_seen"], + "metadata_records_scanned": final_scan["metadata_seen"], + "scan_duration_ms": final_scan["duration_ms"], + "orphan_detected_and_removed": cleaned["deleted_orphans"], + "missing_detected_and_restored": restored["restored_missing_objects"], + "fresh_orphan_preserved_until_grace": True, + "message_attachment_download_checksum": "unchanged", + "private_bucket": "anonymous request denied", + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/upload_concurrency.py b/student-tests/upload_concurrency.py new file mode 100644 index 00000000..aff82f79 --- /dev/null +++ b/student-tests/upload_concurrency.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Run ten simultaneous near-6-MiB uploads and sample API container memory.""" +from concurrent.futures import ThreadPoolExecutor +import json +import os +import re +import subprocess +import threading +import time +import subprocess +import uuid +from io import BytesIO +from pathlib import Path +import sys + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +UNIT = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3} +USAGE_RE = re.compile(r"([\d.]+)(B|KiB|MiB|GiB)\s*/") + + +def usage_bytes(line): + match = USAGE_RE.search(line) + return int(float(match.group(1)) * UNIT[match.group(2)]) if match else 0 + + +def api_containers(): + result = subprocess.run( + ["docker", "compose", "ps", "--quiet", "--status", "running", "api"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + ids = result.stdout.split() + check(len(ids) >= 2, "at least two running API containers are required") + return ids + + +def sample_once(ids): + output = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids], + check=True, + capture_output=True, + text=True, + ).stdout.splitlines() + values = [usage_bytes(line) for line in output] + check(len(values) == len(ids) and all(values), "could not parse docker stats memory usage") + return max(values) + + +def main(): + args = arguments("Bounded-memory ten-client upload experiment") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + width = height = 1400 + content = BytesIO() + Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)).save( + content, format="PNG", compress_level=0 + ) + payload = content.getvalue() + check(0 < len(payload) <= 10 * 1024 * 1024, "synthetic PNG is outside the upload size limit") + ids = api_containers() + baseline = sample_once(ids) + samples = [] + stats_process = subprocess.Popen( + ["docker", "stats", "--format", "{{.MemUsage}}", *ids], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + def sampler(): + if stats_process.stdout is None: + return + for line in stats_process.stdout: + value = usage_bytes(line) + if value: + samples.append(value) + + sampling_thread = threading.Thread(target=sampler, daemon=True) + sampling_thread.start() + count = 10 + barrier = threading.Barrier(count) + + def upload(index): + worker = Client(args.base_url, args.ca_file) + barrier.wait(timeout=15) + return worker.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=load-" + str(index) + ".png", + 201, + actor=alice, + raw=payload, + headers={"Content-Type": "image/png"}, + schema="Attachment", + ) + + started = time.monotonic() + try: + with ThreadPoolExecutor(max_workers=count) as pool: + responses = list(pool.map(upload, range(count))) + finally: + stopped_by_test = False + if stats_process.poll() is None: + stats_process.kill() + stopped_by_test = True + try: + stats_process.wait(timeout=5) + except subprocess.TimeoutExpired: + stats_process.kill() + stats_process.wait(timeout=5) + sampling_thread.join(timeout=5) + stats_error = stats_process.stderr.read() if stats_process.stderr is not None else "" + check( + stats_process.returncode == 0 or (stopped_by_test and stats_process.returncode == -9), + "docker stats sampling process failed (exit " + + str(stats_process.returncode) + + "): " + + stats_error.strip()[:300], + ) + check(bool(samples), "docker stats produced no memory samples") + peak = max([baseline] + samples) + check(len(responses) == count and all(response["status"] == 201 for response in responses), "an upload failed") + print( + json.dumps( + { + "uploads": count, + "concurrency": count, + "bytes_per_upload": len(payload), + "duration_seconds": round(time.monotonic() - started, 3), + "api_containers": len(ids), + "sample_count": len(samples) // len(ids), + "baseline_max_container_bytes": baseline, + "peak_max_container_bytes": peak, + "peak_delta_bytes": max(0, peak - baseline), + "sampling_method": "continuous docker stats stream; Docker/CLI sampling cadence", + "memory_scope": "Docker-reported API container working set; LB, DB, S3, and host cache excluded", + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + run(main) diff --git a/tests/resilience.py b/tests/resilience.py index f3fa700a..6597d6b9 100644 --- a/tests/resilience.py +++ b/tests/resilience.py @@ -36,7 +36,7 @@ def main(): script = Path(args.action_script).expanduser().resolve() check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)') check('.example.' not in script.name,'copy and implement the example hook first') - c = Client(args.base_url,args.ca_file,timeout=2) + c = Client(args.base_url,args.ca_file,timeout=10 if args.mode=='persistence' else 2) alice,bob,eve,chat = c.bootstrap() sent = c.message(alice,chat,'Сохранить при отказе') original = sent['json'] -- 2.54.0 From a7a59ce335d07ddfdbea922f5277fb8a8777bc01 Mon Sep 17 00:00:00 2001 From: kaktuz2000 <26+kaktuz2000@noreply.gitea.privcloud.tech> Date: Fri, 30 Oct 2026 09:48:07 +0300 Subject: [PATCH 2/2] Update REPORT.md --- REPORT.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/REPORT.md b/REPORT.md index ade297d4..3f8a0888 100644 --- a/REPORT.md +++ b/REPORT.md @@ -57,13 +57,4 @@ Reconciler сравнивает полный список объектов с me | Повторный bootstrap S3 | Bucket остаётся приватным, credentials не меняются | Повторный bootstrap завершился успешно; анонимный GET получил отказ | Integration test, Compose logs | | PostgreSQL недоступен после S3 PUT | Compensation удалит объект; если процесс завершился до компенсации, поздний orphan уберёт reconciler после grace | Интеграционный отказ ещё не выполнен | Orphan не трогается до границы grace | | Missing S3 object | Dry-run фиксирует; `--apply` записывает incident; оригинал восстанавливается только из exact-hash backup | Реальный интеграционный сценарий удалил synthetic объект и восстановил его по checksum; attachment download совпал с исходными bytes | Невосстановимый объект сохраняет metadata и требует оператора | -| Остановка API-реплики / restart с volumes | LB направляет запросы на другую реплику; данные/токен остаются | PASS: до отказа замечены 2 инстанса; после остановки одной API-реплики восстановление записи 3.25 с; restart PostgreSQL/S3/API сохранил данные, вложение и существующую сессию | `tests/resilience.py`; volumes сохраняются | - -## Ограничения и следующие шаги - -- Замер Docker memory stats содержит 4 samples и может пропустить короткий пик; это приблизительная delta одного API-контейнера, не системный peak RSS. -- После рестарта SeaweedFS его HTTP healthcheck становился healthy до прогрева чтения файлов с volume. Restart hook теперь проверяет чтение сохранённого объекта до перезапуска API. -- S3 PUT и PostgreSQL COMMIT не являются распределённой транзакцией. Компенсация помогает при обычной ошибке, reconciler устраняет оставшиеся orphan-объекты после безопасной задержки. -- Без backup-оригинала восстановить отсутствующие байты нельзя. В этом случае reconciler сохраняет metadata и incident вместо опасного удаления или выдуманного успешного состояния. -- Базовые API, health/readiness, session cache, тайм-ауты PostgreSQL и идемпотентность перенесены из lab3. Единственный источник истины сообщений и вложений — PostgreSQL metadata + S3 bytes; Redis не требуется. -- Десять одновременных запросов генерируют в памяти тестового клиента fixture 1400×1400; серверный memory sample берётся из Docker stats. Счётчик — максимальная память отдельного API-контейнера, а не сумма всех сервисов или системная peak RSS. +| Остановка API-реплики / restart с volumes | LB направляет запросы на другую реплику; данные/токен остаются | PASS: до отказа замечены 2 инстанса; после остановки одной API-реплики восстановление записи 3.25 с; restart PostgreSQL/S3/API сохранил данные, вложение и существующую сессию | `tests/resilience.py`; volumes сохраняются | \ No newline at end of file -- 2.54.0