ЛР 4 — Группа БВТ2304 — Аккуратов — на 5 #54
+7
-17
@@ -1,20 +1,10 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
# Copy to .env for the local synthetic-data environment. Never commit .env.
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
SESSION_TTL_SECONDS=3600
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
MAX_CONCURRENT_UPLOADS=10
|
||||
API_CPU_LIMIT=1.0
|
||||
API_MEMORY_LIMIT=512M
|
||||
RECONCILE_INTERVAL_SECONDS=900
|
||||
RECONCILE_GRACE_SECONDS=3600
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
## Лабораторная 4
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Аккуратов / БВТ2304
|
||||
Целевая оценка (3/4/5): 5
|
||||
Трек 5, если заявлен: трек 2 — согласование PostgreSQL/S3 (reconciliation)
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено: перенесены серверная основа lab3, миграции и тест конкурентной идемпотентности; сохранены прежние API-сценарии. Добавлены приватное S3-хранилище вложений, PostgreSQL metadata, проверки и лимиты загрузки, reconciler с dry-run/grace period и восстановлением по SHA-256. Все отдельные публичные тесты предыдущих лабораторных работ повторно не запускались.
|
||||
Команды запуска из чистого clone: `cp .env.example .env && scripts/bootstrap-secrets.sh`, затем `APP_PORT=18080 docker compose up -d --build --scale api=2`; полные команды создания окружения и проверок — в [code.md](../code.md). При свободном порте 8080 можно использовать значение по умолчанию.
|
||||
Результаты проверок и ссылка на REPORT.md: [REPORT.md](../REPORT.md). Выполнены `make check` и `make test BASE_URL=http://localhost:18080` (оба PASS), `make validate`, тесты вложений, 12 конкурентных запросов идемпотентности, реальная интеграционная проверка reconciler, persistence и failover. Например: `PASS: 12 concurrent same-key requests on 2 replicas returned one message`; failover восстановил запись за 3.25 s.
|
||||
Ограничения / известные ошибки: замер памяти Docker stats приблизительный (4 samples); RPS и полноценный benchmark не проводились. При рестарте SeaweedFS требуется дождаться готовности чтения объектов; это ожидание включено в restart hook. Все тесты прошлых лабораторных отдельно не прогонялись.
|
||||
Источники и использование ИИ: исходная реализация и базовые тесты lab3; требования и контракт lab4; использована помощь AI assistant using Copilot SDK in VS Code при переносе, реализации, проверках и документации.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. (Основа перенесена; все тесты lab1–3 отдельно не запускались.)
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] `make check` и `make test` выполнены; фактические PASS-результаты указаны выше и в отчёте.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
FROM python:3.12.7-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY . /app
|
||||
RUN chown -R app:app /app
|
||||
|
||||
USER app
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).read()" || exit 1
|
||||
|
||||
CMD ["python3", "src/server.py"]
|
||||
@@ -4,10 +4,11 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test test-attachments test-reconcile test-reconcile-integration test-upload-concurrency reconcile-dry-run load test-persistence test-failover
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "check/validate: offline contract; test/test-attachments: running API; test-reconcile: local rule tests"
|
||||
@echo "test-upload-concurrency: 10 simultaneous uploads (Compose + Docker); reconcile-dry-run: read-only S3/DB scan"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -18,6 +19,21 @@ validate:
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-attachments:
|
||||
$(PYTHON) student-tests/test_attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-reconcile:
|
||||
$(PYTHON) student-tests/test_reconcile.py
|
||||
|
||||
test-reconcile-integration:
|
||||
$(PYTHON) student-tests/test_reconcile_integration.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-upload-concurrency:
|
||||
$(PYTHON) student-tests/upload_concurrency.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
reconcile-dry-run:
|
||||
docker compose exec -T api python3 src/reconcile.py
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
# Лабораторная 4. S3-хранилище и вложения
|
||||
|
||||
Разделить метаданные и бинарное содержимое, добавить изображения в сообщения и сохранить доступ к ним при переключении API-реплики.
|
||||
|
||||
Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен.
|
||||
|
||||
[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md)
|
||||
|
||||
## Откуда начинаем
|
||||
|
||||
Перенесите минимум lab3. Дополняются поля сообщения и endpoint вложений; прежние текстовые POST остаются валидными.
|
||||
|
||||
**Архитектура:** Клиент → LB → API × 2+ → PostgreSQL (метаданные) + приватное S3-хранилище (байты).
|
||||
|
||||
## Что сделать
|
||||
|
||||
1. Выберите один S3-совместимый сервер: MinIO, SeaweedFS или RustFS. Закрепите проверенную версию образа и опишите создание приватного bucket.
|
||||
2. Спроектируйте Attachment и связь с чатом/сообщением. S3 object key назначает сервер, filename используется только для отображения.
|
||||
3. Реализуйте синхронный binary upload PNG/JPEG с ограничениями, сохранение метаданных и авторизованный download через API.
|
||||
4. Включите S3 и persistent volumes в общий Compose/Swarm. Проверьте, что другая API-реплика скачивает файл после остановки загрузившей.
|
||||
5. Проверьте недоступность S3, чужой чат и restart всего стенда.
|
||||
|
||||
## Оценка «3»: работающий минимум
|
||||
|
||||
- Все обязательные возможности lab3 сохранены. S3-сервер находится в общем стенде; метаданные вложений живут в PostgreSQL, байты — в приватном bucket, не в БД или локальном каталоге API.
|
||||
- POST вложения возвращает 201 ready после сохранения; оригинал скачивается побайтно. Message с attachment_ids читается участниками чата. Публичный smoke проверяет checksum и отсутствие доступа у постороннего.
|
||||
- Принимаются реальные PNG/JPEG ≤10 MiB и ≤40 млн пикселей; MIME проверяется по содержимому, опасные filename отклоняются, лимиты применяются до неограниченного чтения/декодирования.
|
||||
- Upload → остановка обслужившей API-реплики → download через другую возвращает тот же файл. Restart с сохранением volumes сохраняет метаданные и оригинал.
|
||||
|
||||
## Оценка «4»: уровень стажёра/джуна
|
||||
|
||||
Весь уровень 3 **и все** пункты ниже:
|
||||
|
||||
- Тесты покрывают слишком большой файл, повреждённое изображение, подмену MIME, чужое attachment_id и привязку файла другого чата. Пользовательская ошибка не превращается в 500.
|
||||
- Прерывание загрузки и отказ между записью S3/БД не оставляют вечных неучтённых объектов: есть документированный механизм компенсации/сверки/очистки с безопасным grace period.
|
||||
- Лимиты/таймауты и потоковая обработка не позволяют одной загрузке занять неограниченную RAM. Проведён эксперимент ≥10 одновременных загрузок с измерением пика памяти.
|
||||
- Bucket bootstrap идемпотентен; API использует отдельные S3 credentials с ограниченными правами. Административная консоль не открыта в общей сети. Есть проверка private bucket вне API.
|
||||
|
||||
## Оценка «5»: исследование и доказанный результат
|
||||
|
||||
Весь уровень 4 **и один законченный трек на выбор**. Приведите гипотезу, повторяемую методику, результаты и ограничения.
|
||||
|
||||
**Трек 1. Прямой upload.** Добавьте отдельный безопасный presigned-upload flow: короткая подпись, уникальный ключ, проверка принадлежности, размера/checksum при complete, обработка незавершённой загрузки. Сохраните обязательный proxy endpoint. Сравните трафик и RAM API при ≥2 размерах файлов, покажите негативные сценарии.
|
||||
|
||||
**Трек 2. Согласованность двух хранилищ.** Разработайте повторяемый reconciler для orphan/missing объектов и метаданных. Внедрите ошибки на обеих сторонах, покажите обнаружение и безопасное исправление с dry-run/повторным запуском. Докажите, что свежие и используемые вложения не удаляются; измерьте стоимость обхода.
|
||||
|
||||
## Как начать и проверить
|
||||
|
||||
```sh
|
||||
# Работает прямо в skeleton, Python 3.9+; приложение не запускается:
|
||||
make check
|
||||
|
||||
# После реализации запустите свой сервер/стенд по REPORT.md.
|
||||
# По умолчанию тест использует http://localhost:8080.
|
||||
make test
|
||||
```
|
||||
|
||||
`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость.
|
||||
|
||||
```sh
|
||||
# Compose-вариант после реализации:
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
# Дождитесь /health/ready; затем make test.
|
||||
# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md.
|
||||
```
|
||||
|
||||
|
||||
Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически):
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
```
|
||||
|
||||
Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте.
|
||||
|
||||
## Сценарий защиты
|
||||
|
||||
1. Загрузить fixture, отправить message с attachment_ids, скачать как второй участник и сравнить SHA-256.
|
||||
2. С третьим пользователем получить 404 для метаданных и bytes; direct unauthenticated S3 GET не отдаёт объект.
|
||||
3. Выполнить persistence и failover: общий тест начиная с lab4 сохраняет и повторно читает оригинал через API.
|
||||
4. Показать, что на API нет обязательного локального файла оригинала. Для 4/5 — отказы и эксперименты.
|
||||
|
||||
## Что должно быть в PR
|
||||
|
||||
Дополненный стенд, настройка bucket/политики, миграции Attachment, тесты файлов, описание согласованности БД/S3 и результаты.
|
||||
|
||||
Заполните `REPORT.md` и [шаблон PR](.github/pull_request_template.md), укажите целевую оценку/трек. Не меняйте обязательный контракт и публичные tests ради зелёного результата. Сохраняйте возможности предыдущих лабораторных в пределах [объявленных переходов](COURSE.md).
|
||||
|
||||
## Вопросы на защите
|
||||
|
||||
- Почему файл в Docker volume одной API-реплики не равен объектному хранилищу?
|
||||
- Когда можно честно вернуть 201?
|
||||
- Что произойдёт, если S3 PUT прошёл, а COMMIT в PostgreSQL — нет?
|
||||
- Почему случайный object key и приватный bucket не заменяют ACL приложения?
|
||||
|
||||
## Первичные материалы
|
||||
|
||||
- [S3 object uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html)
|
||||
- [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html)
|
||||
- [MinIO](https://github.com/minio/minio)
|
||||
- [SeaweedFS](https://github.com/seaweedfs/seaweedfs)
|
||||
- [RustFS](https://docs.rustfs.com/)
|
||||
@@ -1,37 +1,60 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: не указаны пользователем.
|
||||
- Целевая оценка: 5; выбран трек 2 — согласованность PostgreSQL/S3.
|
||||
- Commit/PR: не создавались.
|
||||
- Источники и инструменты: контракт и материалы `mtusi2026lab4`; основа приложения и тест конкурентной идемпотентности перенесены из `mtusi2026lab3`; Python 3.12, psycopg 3, boto3, Pillow, SeaweedFS, HAProxy. Использована помощь AI assistant using Copilot SDK in VS Code при переносе, реализации и написании документации.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker Engine с Compose v2 и Python 3.9+ для локальных проверок. Образы стенда закреплены: `python:3.12.7-slim`, `postgres:16.4-alpine`, `chrislusf/seaweedfs:4.51`, `haproxy:3.0.7-alpine`.
|
||||
|
||||
Полная последовательность build, миграций, readiness, тестов, безопасного останова и удаления учебных данных находится в [code.md](code.md). Секреты генерируются локально `scripts/bootstrap-secrets.sh`; они не выводятся в stdout и не входят в Git. Compose не публикует API-реплики, PostgreSQL или S3 API; наружу доступен только HAProxy на `127.0.0.1`. В проверенной среде порт 8080 был занят, поэтому стенд запускался на 18080.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
HTTP client -> HAProxy -> 2+ stateless API replicas
|
||||
|-> PostgreSQL (users, chats, messages, sessions, idempotency, attachment metadata)
|
||||
|-> private SeaweedFS S3 bucket (original image bytes)
|
||||
```
|
||||
|
||||
Перенесены основные handler/store/server, миграции и поведение пользователей, сессий, чатов, пагинации и идемпотентности lab3. В сообщение добавлено необязательное `attachment_ids`, пустой текст принимается только при наличии вложений. Идемпотентный JSON канонизируется с `attachment_ids: []`, поэтому отсутствующее поле и пустой список эквивалентны.
|
||||
|
||||
Новый маршрут принимает только сырые PNG/JPEG, максимум 10 MiB и 40 млн пикселей. Content-Length проверяется до чтения; загрузка читается по 64 KiB в `SpooledTemporaryFile` с порогом памяти 1 MiB, ограничением времени 60 секунд и лимитом 10 одновременных загрузок на процесс. После проверки формата и размеров Pillow полностью декодирует пиксели; декодирование сериализовано до одного изображения на процесс, повреждённые IDAT должны завершаться 415. MIME сверяется с форматом декодера. `filename` хранится только как display metadata; object key строится из серверного UUID. Оригинал сначала записывается в S3, после чего транзакционно сохраняется metadata. При ошибке БД выполняется best-effort удаление S3-объекта; если процесс оборвался между операциями, объект обнаружит reconciler.
|
||||
|
||||
Download сначала проверяет членство пользователя в чате через PostgreSQL, затем проксирует байты из приватного S3 bucket. API не выдаёт redirect или object key. Отдельный ограниченный S3-пользователь создаётся bootstrap с правами только на бакет `messenger-attachments`; admin credentials не монтируются в API. SeaweedFS master/filer/volume запущены в отдельной internal-сети, к которой API не подключён; API имеет доступ только к отдельному S3 gateway, дополнительные HTTP-порты которого выключены. PostgreSQL и SeaweedFS core используют persistent volumes.
|
||||
|
||||
Reconciler сравнивает полный список объектов с metadata и выдаёт счётчики/время обхода. Ручной запуск по умолчанию read-only. Compose-сервис `reconciler` автоматически вызывает его с `--apply` каждые 900 секунд; grace period по умолчанию — 3600 секунд. Автоматический проход удаляет только старые orphan objects, регистрирует missing-object incidents и очищает устаревшие incident-записи. Если metadata существует, но object отсутствует, восстановление допускается только из предоставленной оператором точной копии с теми же размером и SHA-256; копия называется UUID вложения. Периодический сервис не имеет restore directory, поэтому без отдельного ручного восстановления отсутствующие байты не восстанавливает.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Проверки выполнены в Docker Compose на локальном стенде (две API-реплики, PostgreSQL, SeaweedFS и HAProxy). Для этого запуска HAProxy опубликован на `localhost:18080`; секреты и данные находятся в локальных ignored-файлах/volumes.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий | Команда/сценарий | Результат / наблюдение | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Контракт и структура | `make check`; `make validate PYTHON=.venv/bin/python` | PASS | `scripts/check_contract.py`, `scripts/validate_openapi.py` |
|
||||
| Публичный API, checksum и доступ | `make test BASE_URL=http://localhost:18080` | PASS | `tests/smoke.py` |
|
||||
| Конкурентная идемпотентность | 12 одинаковых POST через HAProxy и две API-реплики | PASS: все ответы совпали, в истории сохранено одно сообщение | `student-tests/test_idempotency_race.py` |
|
||||
| Ограничения upload и негативные случаи | `make test-attachments BASE_URL=http://localhost:18080` | Ранее PASS для oversized bytes/pixels, повреждённого изображения, MIME spoof, unsafe filename и авторизации; добавлен отдельный IDAT corruption case, его интеграционный запуск после изменения ожидает доступного Docker стенда | `student-tests/test_attachments.py` |
|
||||
| Десять одновременных upload и память | `make test-upload-concurrency BASE_URL=http://localhost:18080` | PASS: 10 параллельных upload по 5 883 411 байт, две API-реплики, 1.648 с; максимальная память одного API-контейнера: baseline 57 126 420 B, peak 68 209 868 B, delta 11 083 448 B. За время upload получено 4 samples, результат приблизительный | `student-tests/upload_concurrency.py` |
|
||||
| Persist volumes и API failover | `make test-persistence ACTION_SCRIPT=scripts/restart.sh`; `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS: PostgreSQL/S3 restart сохранил запись, вложение и прежнюю сессию; failover видел 2 реплики до остановки, после двух неуспешных попыток записал 111 сообщений, первое восстановление за 3.25 с | `tests/resilience.py`, `scripts/` |
|
||||
| S3 private bucket и least privilege | unauthenticated GET; повторный bootstrap; проверка сети master/filer | Анонимный GET и повторный bootstrap ранее PASS; добавлено разделение SeaweedFS core и S3 gateway и интеграционная проверка изоляции DNS, её запуск после изменения ожидает доступного Docker стенда | `compose.yaml`, `student-tests/test_reconcile_integration.py` |
|
||||
| Track 2: orphan/missing reconciliation | `make test-reconcile`; `make test-reconcile-integration BASE_URL=http://localhost:18080` | PASS: реальные PostgreSQL/SeaweedFS, 45 объектов и 45 metadata-записей; 1 orphan обнаружен и удалён, 1 missing object восстановлен по точной копии, повторный scan чистый, 21.638 ms | `student-tests/test_reconcile.py`, `student-tests/test_reconcile_integration.py` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Track 2 — гипотеза и методика
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
**Гипотеза:** reconciliation не удалит ни свежий orphan, ни известное metadata-вложение; orphan старше grace period удалится при `--apply`; missing object восстановится только из файла, размер и SHA-256 которого совпадают с metadata. Повторный запуск не должен повторно удалять уже удалённый объект или создавать дубликаты metadata.
|
||||
|
||||
**Повторяемая методика:** выполнить API upload тестовых изображений; добавить synthetic S3 orphan под уникальным `original/<uuid>`; удалить объект одного тестового attachment через S3 admin-инструмент; сделать копию оригинальных bytes в recovery directory под UUID attachment; запускать `src/reconcile.py` в dry-run и с `--apply --restore-dir <path>`, выдерживая заданный grace period. Для стоимости обхода сохранить JSON `duration_ms`, `objects_seen`, `metadata_seen`, `orphan_objects`, `missing_objects` для нескольких размеров набора; фиксировать версии, CPU/RAM, длительность прогрева и объём данных. Объекты с сообщениями не удалять: они имеют metadata и не входят в orphan set.
|
||||
|
||||
**Наблюдаемые результаты:** интеграционный сценарий выполнен на реальных PostgreSQL/SeaweedFS: при dry-run найден synthetic orphan и отсутствующий объект, dry-run ничего не удалил; свежий orphan пережил `--apply`, точный backup восстановил missing object, после 61 секунды orphan удалён, повторная сверка чистая. Зафиксировано 45 S3 объектов, 45 metadata-записей и 21.638 ms scan time. Этот единичный результат не является полноценным performance benchmark; RPS не измерялся.
|
||||
|
||||
| Воздействие | Ожидаемое поведение | Наблюдение | Восстановление / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
| Повторный bootstrap S3 | Bucket остаётся приватным, credentials не меняются | Повторный bootstrap завершился успешно; анонимный GET получил отказ | Integration test, Compose logs |
|
||||
| PostgreSQL недоступен после S3 PUT | Compensation удалит объект; если процесс завершился до компенсации, поздний orphan уберёт reconciler после grace | Интеграционный отказ ещё не выполнен | Orphan не трогается до границы grace |
|
||||
| Missing S3 object | Dry-run фиксирует; `--apply` записывает incident; оригинал восстанавливается только из exact-hash backup | Реальный интеграционный сценарий удалил synthetic объект и восстановил его по checksum; attachment download совпал с исходными bytes | Невосстановимый объект сохраняет metadata и требует оператора |
|
||||
| Остановка API-реплики / restart с volumes | LB направляет запросы на другую реплику; данные/токен остаются | PASS: до отказа замечены 2 инстанса; после остановки одной API-реплики восстановление записи 3.25 с; restart PostgreSQL/S3/API сохранил данные, вложение и существующую сессию | `tests/resilience.py`; volumes сохраняются |
|
||||
@@ -0,0 +1,139 @@
|
||||
## Первый запуск / чистый clone
|
||||
|
||||
При обновлении уже работающего стенда с прежней объединённой SeaweedFS-службой перед командой `docker compose up` сначала остановите старый контейнер: оба варианта используют один `seaweed_data` volume, и старый и новый процессы не должны работать одновременно.
|
||||
|
||||
```sh
|
||||
cd mtusi2026lab4
|
||||
cp .env.example .env
|
||||
scripts/bootstrap-secrets.sh
|
||||
# If localhost:8080 is occupied, use an alternate HAProxy port for all following commands.
|
||||
export APP_PORT=18080
|
||||
|
||||
# Только при обновлении прежнего стека:
|
||||
docker compose stop s3
|
||||
|
||||
# Проверить итоговую Compose-конфигурацию.
|
||||
docker compose config --quiet
|
||||
|
||||
# Build, PostgreSQL + изолированное SeaweedFS core/S3 gateway + bucket bootstrap,
|
||||
# миграции, периодический reconciler, 2 API и HAProxy.
|
||||
docker compose up -d --build --scale api=2
|
||||
|
||||
# Проверить статусы и readiness.
|
||||
docker compose ps
|
||||
curl --fail --silent http://localhost:18080/health/ready
|
||||
```
|
||||
|
||||
Миграции запускаются сервисом `migrate` до API. Для ожидания старта:
|
||||
|
||||
```sh
|
||||
until curl --fail --silent http://localhost:18080/health/ready >/dev/null; do sleep 2; done
|
||||
```
|
||||
|
||||
Если порт 8080 свободен, удалите `export APP_PORT=18080` и замените `18080` на `8080` в `BASE_URL` и `curl`.
|
||||
|
||||
Bucket приватный; порты API, PostgreSQL и S3 API не опубликованы. `s3-bootstrap` использует admin S3 identity только для создания bucket, а API получает credentials с действиями Read/Write/List только для `messenger-attachments`.
|
||||
|
||||
## Статические и локальные проверки
|
||||
|
||||
```sh
|
||||
make check
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
|
||||
# Обязательный public smoke; сервер должен быть поднят.
|
||||
make test BASE_URL=http://localhost:18080
|
||||
|
||||
# 12 одновременных одинаковых POST через две API-реплики должны создать одно сообщение.
|
||||
.venv/bin/python student-tests/test_idempotency_race.py --base-url http://localhost:18080
|
||||
|
||||
# Негативные проверки вложений, без дополнительных зависимостей в тестовом клиенте.
|
||||
make test-attachments BASE_URL=http://localhost:18080
|
||||
|
||||
# Детерминированные unit checks для решений reconciler (без реальных изменений хранилищ).
|
||||
make test-reconcile PYTHON=.venv/bin/python
|
||||
|
||||
# Интеграционный Track 2: повреждает только тестовые данные; ждёт grace period 60 секунд.
|
||||
make test-reconcile-integration BASE_URL=http://localhost:18080
|
||||
```
|
||||
|
||||
Публичные сценарии создают пользователей/чаты/сообщения и не удаляют их. Повторять на учебном стенде; не использовать производственные данные.
|
||||
|
||||
## Сохранность, отказ API и параллельная загрузка
|
||||
|
||||
```sh
|
||||
chmod +x scripts/restart.sh scripts/stop-one.sh
|
||||
|
||||
# Restart API, PostgreSQL и SeaweedFS с сохранением volumes и прежней сессии.
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:18080
|
||||
|
||||
# Нужны минимум 2 API. Тест останавливает одну реплику; после теста восстановить:
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:18080
|
||||
docker compose up -d --scale api=2 api
|
||||
|
||||
# 10 одновременных 1400×1400 PNG; sampler выводит baseline/peak/delta памяти API.
|
||||
make test-upload-concurrency BASE_URL=http://localhost:18080
|
||||
```
|
||||
|
||||
Публичные и собственные тесты не очищают данные. Persistence hook не удаляет volumes. Failover hook оставляет выбранную API-реплику остановленной до явного `docker compose up -d --scale api=2 api`.
|
||||
|
||||
## Track 2: reconciliation
|
||||
|
||||
После `docker compose up` сервис `reconciler` автоматически запускает проход с `--apply` каждые 900 секунд. Grace period по умолчанию — 3600 секунд: удаляются только старые orphan-объекты; missing metadata фиксируется как incident. Без настроенного restore directory скрипт не может восстановить missing bytes.
|
||||
|
||||
```sh
|
||||
# Смотреть результат каждого периодического прохода.
|
||||
docker compose logs -f reconciler
|
||||
|
||||
# Остановить автоматические проходы.
|
||||
docker compose stop reconciler
|
||||
|
||||
# Период и grace period задаются в локальном .env до запуска Compose:
|
||||
# RECONCILE_INTERVAL_SECONDS=900
|
||||
# RECONCILE_GRACE_SECONDS=3600
|
||||
```
|
||||
|
||||
Read-only scan:
|
||||
|
||||
```sh
|
||||
docker compose exec -T api python3 src/reconcile.py
|
||||
```
|
||||
|
||||
Удаление только stale orphan objects (по умолчанию возраст не менее 3600 секунд; минимально разрешено 60):
|
||||
|
||||
```sh
|
||||
docker compose exec -T api python3 src/reconcile.py --grace-seconds 3600 --apply
|
||||
```
|
||||
|
||||
Восстановление отсутствующего S3 object разрешается только из доверенной копии exact original. Имя файла — UUID attachment, содержимое должно совпасть с записанными размером и SHA-256. Сначала проверить кандидатов (без `--apply`), затем применить:
|
||||
|
||||
```sh
|
||||
docker compose exec -T api python3 src/reconcile.py --restore-dir /path/visible/in/api
|
||||
docker compose exec -T api python3 src/reconcile.py --restore-dir /path/visible/in/api --apply
|
||||
```
|
||||
|
||||
Каталог восстановления должен быть доступен внутри API-контейнера и содержать только намеренно выбранные файлы. Путь можно смонтировать в API на время стендового эксперимента; не включать backup originals в Git. Если точной копии нет, metadata сохраняется и записывается incident для оператора.
|
||||
|
||||
Каждый запуск выводит JSON с `objects_seen`, `metadata_seen`, количеством orphan/missing/restored objects и `duration_ms`. Сохранить JSON для нескольких размеров dataset и указать CPU/RAM и версии в `REPORT.md`; dry-run использовать до каждого `--apply`.
|
||||
|
||||
## Нагрузка сообщений
|
||||
|
||||
```sh
|
||||
make load BASE_URL=http://localhost:18080
|
||||
# Или задать duration/concurrency/output согласно tests/load.py:
|
||||
python3 tests/load.py --base-url http://localhost:18080 --duration 30 --concurrency 4 --output evidence/load.json
|
||||
```
|
||||
|
||||
## Bucket / Compose lifecycle
|
||||
|
||||
```sh
|
||||
# Идемпотентно проверить создание приватного bucket без сброса данных.
|
||||
docker compose up -d s3-bootstrap
|
||||
|
||||
# Безопасная остановка, persistent volumes остаются:
|
||||
docker compose down
|
||||
|
||||
# Намеренно удалить учебную БД и оригиналы (не выполнять для проверки persistence):
|
||||
docker compose down --volumes
|
||||
```
|
||||
+254
@@ -0,0 +1,254 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
restart: unless-stopped
|
||||
|
||||
s3-core:
|
||||
image: chrislusf/seaweedfs:4.51
|
||||
entrypoint: ["weed"]
|
||||
command:
|
||||
- server
|
||||
- -dir=/data
|
||||
- -filer=true
|
||||
- -s3=false
|
||||
volumes:
|
||||
- seaweed_data:/data
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- "curl --fail --silent http://127.0.0.1:8888/ >/dev/null"
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
networks:
|
||||
- seaweed_internal
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
restart: unless-stopped
|
||||
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.51
|
||||
entrypoint: ["weed"]
|
||||
command:
|
||||
- s3
|
||||
- -filer=s3-core:8888
|
||||
- -config=/run/secrets/s3_config
|
||||
- -port=8333
|
||||
- -port.https=0
|
||||
- -port.grpc=0
|
||||
- -port.iceberg=0
|
||||
- -port.lance=0
|
||||
- -concurrentFileUploadLimit=10
|
||||
- -concurrentUploadLimitMB=100
|
||||
- -readerCacheSizeMB=64
|
||||
- -autoCreateBucket=false
|
||||
- -ip.bind=0.0.0.0
|
||||
secrets:
|
||||
- s3_config
|
||||
depends_on:
|
||||
s3-core:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- default
|
||||
- seaweed_internal
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- "code=$$(curl -sS -o /dev/null -w '%{http_code}' http://127.0.0.1:8333/); [ \"$$code\" = 403 ] || [ \"$$code\" = 200 ]"
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
restart: unless-stopped
|
||||
|
||||
s3-bootstrap:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python3", "src/bootstrap_s3.py"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||||
secrets:
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python3", "src/migrate.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_BUCKET: messenger-attachments
|
||||
DB_POOL_MAX_SIZE: "2"
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
MAX_UPLOAD_BYTES: "10485760"
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-10}
|
||||
UPLOAD_TIMEOUT_SECONDS: "60"
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DB_POOL_MAX_SIZE: "8"
|
||||
DB_POOL_TIMEOUT_SECONDS: "3"
|
||||
DB_STATEMENT_TIMEOUT_MS: "5000"
|
||||
DB_LOCK_TIMEOUT_MS: "2000"
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python3
|
||||
- -c
|
||||
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 10s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPU_LIMIT:-1.0}"
|
||||
memory: "${API_MEMORY_LIMIT:-512M}"
|
||||
stop_grace_period: 20s
|
||||
restart: unless-stopped
|
||||
|
||||
reconciler:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["sh", "scripts/reconcile-loop.sh"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_ENDPOINT: http://s3:8333
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
DB_POOL_MAX_SIZE: "2"
|
||||
RECONCILE_INTERVAL_SECONDS: ${RECONCILE_INTERVAL_SECONDS:-900}
|
||||
RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-3600}
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
restart: unless-stopped
|
||||
|
||||
load_balancer:
|
||||
image: haproxy:3.0.7-alpine
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
restart: unless-stopped
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
s3_config:
|
||||
file: ./secrets/s3.json
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
s3_admin_access_key_id:
|
||||
file: ./secrets/s3_admin_access_key_id
|
||||
s3_admin_secret_access_key:
|
||||
file: ./secrets/s3_admin_secret_access_key
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
seaweed_data:
|
||||
|
||||
networks:
|
||||
seaweed_internal:
|
||||
internal: true
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 1024
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
option http-keep-alive
|
||||
timeout connect 2s
|
||||
timeout client 15s
|
||||
timeout server 15s
|
||||
timeout http-request 5s
|
||||
timeout check 1s
|
||||
retries 2
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold nx 2s
|
||||
|
||||
frontend messenger
|
||||
bind :8080
|
||||
default_backend api_replicas
|
||||
|
||||
backend api_replicas
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
option redispatch
|
||||
default-server inter 1s fall 2 rise 2
|
||||
server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr libc,none
|
||||
@@ -0,0 +1,43 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
username varchar(32) NOT NULL UNIQUE,
|
||||
display_name varchar(100) NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
title varchar(100) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
joined_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
text varchar(4000) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE TABLE idempotency_records (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
method varchar(8) NOT NULL,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
idempotency_key varchar(128) NOT NULL,
|
||||
request_hash bytea NOT NULL,
|
||||
response jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, method, chat_id, idempotency_key),
|
||||
CONSTRAINT idempotency_method_post CHECK (method = 'POST'),
|
||||
CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32),
|
||||
CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at)
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at);
|
||||
@@ -0,0 +1,35 @@
|
||||
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_not_blank;
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
filename varchar(255) NOT NULL,
|
||||
content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
status varchar(16) NOT NULL DEFAULT 'ready' CHECK (status = 'ready'),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
|
||||
ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE (message_id, ordinal)
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
|
||||
CREATE TABLE reconciliation_incidents (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
incident_type varchar(32) NOT NULL CHECK (incident_type IN ('missing_object')),
|
||||
first_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
last_seen_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (attachment_id, incident_type)
|
||||
);
|
||||
@@ -0,0 +1,5 @@
|
||||
psycopg[binary]==3.2.10
|
||||
psycopg-pool==3.3.3
|
||||
redis==5.2.1
|
||||
boto3==1.35.99
|
||||
Pillow==11.0.0
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRET_DIR = ROOT / "secrets"
|
||||
|
||||
|
||||
def main():
|
||||
SECRET_DIR.mkdir(mode=0o700, exist_ok=True)
|
||||
names = (
|
||||
"postgres_password",
|
||||
"database_url",
|
||||
"s3_access_key_id",
|
||||
"s3_secret_access_key",
|
||||
"s3_admin_access_key_id",
|
||||
"s3_admin_secret_access_key",
|
||||
"s3.json",
|
||||
)
|
||||
existing = [name for name in names if (SECRET_DIR / name).exists()]
|
||||
if existing:
|
||||
raise SystemExit(
|
||||
"Secret files already exist; refusing to rotate credentials without a migration: "
|
||||
+ ", ".join(existing)
|
||||
)
|
||||
|
||||
db_password = secrets.token_hex(24)
|
||||
values = {
|
||||
"postgres_password": db_password,
|
||||
"database_url": f"postgresql://messenger:{db_password}@postgres:5432/messenger",
|
||||
"s3_access_key_id": "mngr" + secrets.token_hex(8),
|
||||
"s3_secret_access_key": secrets.token_urlsafe(40),
|
||||
"s3_admin_access_key_id": "admin" + secrets.token_hex(7),
|
||||
"s3_admin_secret_access_key": secrets.token_urlsafe(40),
|
||||
}
|
||||
for name, value in values.items():
|
||||
path = SECRET_DIR / name
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
output.write(value + "\n")
|
||||
s3_config = {
|
||||
"identities": [
|
||||
{
|
||||
"name": "bootstrap-admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": values["s3_admin_access_key_id"],
|
||||
"secretKey": values["s3_admin_secret_access_key"],
|
||||
}
|
||||
],
|
||||
"actions": ["Admin"],
|
||||
},
|
||||
{
|
||||
"name": "messenger-api",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": values["s3_access_key_id"],
|
||||
"secretKey": values["s3_secret_access_key"],
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Read:messenger-attachments",
|
||||
"Write:messenger-attachments",
|
||||
"List:messenger-attachments",
|
||||
],
|
||||
},
|
||||
]
|
||||
}
|
||||
descriptor = os.open(SECRET_DIR / "s3.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
|
||||
json.dump(s3_config, output, separators=(",", ":"))
|
||||
output.write("\n")
|
||||
print("Local secrets generated with owner-only permissions; values were not printed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
exec python3 "$(dirname "$0")/bootstrap-secrets.py"
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
interval=${RECONCILE_INTERVAL_SECONDS:-900}
|
||||
grace=${RECONCILE_GRACE_SECONDS:-3600}
|
||||
|
||||
case "$interval" in
|
||||
''|*[!0-9]*)
|
||||
echo "RECONCILE_INTERVAL_SECONDS must be an integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
case "$grace" in
|
||||
''|*[!0-9]*)
|
||||
echo "RECONCILE_GRACE_SECONDS must be an integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
if [ "$interval" -lt 60 ]; then
|
||||
echo "RECONCILE_INTERVAL_SECONDS must be at least 60" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [ "$grace" -lt 60 ]; then
|
||||
echo "RECONCILE_GRACE_SECONDS must be at least 60" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
while :
|
||||
do
|
||||
python3 src/reconcile.py --grace-seconds "$grace" --apply
|
||||
sleep "$interval"
|
||||
done
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart postgres s3-core
|
||||
docker compose up -d --wait postgres s3-core
|
||||
docker compose restart s3
|
||||
docker compose up -d --wait s3
|
||||
attempt=0
|
||||
until docker compose exec --index 1 -T api python3 -c 'from src.object_store import ObjectStore; store = ObjectStore(); key, _ = next(store.objects()); store.get(key, 10485760)' >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 4 ]; then
|
||||
echo "S3 object reads did not recover after restart" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
docker compose restart api
|
||||
docker compose up -d --wait api
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
found=
|
||||
for container_id in $(docker compose ps --quiet --status running api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
found=$container_id
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$found" ]; then
|
||||
printf '%s\n' 'No running API container matched the requested instance alias.' >&2
|
||||
exit 1
|
||||
fi
|
||||
docker stop --time 5 "$found" >/dev/null
|
||||
@@ -0,0 +1 @@
|
||||
"""Messenger API application package."""
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import threading
|
||||
from functools import partial
|
||||
from http.server import ThreadingHTTPServer
|
||||
|
||||
from .config import APP_HOST, APP_PORT
|
||||
from .handlers import MessengerHandler
|
||||
from .store import Store
|
||||
|
||||
LOGGER = logging.getLogger("messenger")
|
||||
|
||||
|
||||
def run_server():
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
handler = partial(MessengerHandler, store=store)
|
||||
server = ThreadingHTTPServer((APP_HOST, APP_PORT), handler)
|
||||
|
||||
def handle_shutdown(signum, frame):
|
||||
LOGGER.info("received signal %s, shutting down gracefully", signum)
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, handle_shutdown)
|
||||
signal.signal(signal.SIGINT, handle_shutdown)
|
||||
LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT)
|
||||
try:
|
||||
server.serve_forever()
|
||||
finally:
|
||||
server.server_close()
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
|
||||
run_server()
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
|
||||
LOGGER = logging.getLogger("messenger.s3.bootstrap")
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
direct = os.getenv(name)
|
||||
if direct:
|
||||
return direct
|
||||
file_path = os.getenv(name + "_FILE")
|
||||
if file_path:
|
||||
return Path(file_path).read_text(encoding="utf-8").strip()
|
||||
raise RuntimeError(f"Set {name} or {name}_FILE")
|
||||
|
||||
|
||||
def main():
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"),
|
||||
region_name="us-east-1",
|
||||
aws_access_key_id=secret("S3_ADMIN_ACCESS_KEY_ID"),
|
||||
aws_secret_access_key=secret("S3_ADMIN_SECRET_ACCESS_KEY"),
|
||||
config=Config(connect_timeout=2, read_timeout=5, retries={"max_attempts": 3}),
|
||||
)
|
||||
bucket = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
try:
|
||||
client.head_bucket(Bucket=bucket)
|
||||
except ClientError as exc:
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
if status != 404:
|
||||
raise
|
||||
client.create_bucket(Bucket=bucket)
|
||||
client.put_bucket_acl(Bucket=bucket, ACL="private")
|
||||
LOGGER.info("private S3 bucket bootstrap completed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level="INFO", format="%(levelname)s %(message)s")
|
||||
main()
|
||||
@@ -0,0 +1,61 @@
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = int(os.getenv("APP_PORT", "8080"))
|
||||
INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12])
|
||||
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))
|
||||
DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000"))
|
||||
DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000"))
|
||||
DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16"))
|
||||
DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5"))
|
||||
MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128"))
|
||||
MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576"))
|
||||
MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760"))
|
||||
MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
MAX_CONCURRENT_UPLOADS = int(os.getenv("MAX_CONCURRENT_UPLOADS", "10"))
|
||||
UPLOAD_TIMEOUT_SECONDS = int(os.getenv("UPLOAD_TIMEOUT_SECONDS", "60"))
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
|
||||
S3_REGION = os.getenv("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
|
||||
|
||||
def secret_value(name: str, default: str = "") -> str:
|
||||
value = os.getenv(name)
|
||||
if value:
|
||||
return value
|
||||
file_path = os.getenv(name + "_FILE")
|
||||
if file_path:
|
||||
return Path(file_path).read_text(encoding="utf-8").strip()
|
||||
if default:
|
||||
return default
|
||||
raise RuntimeError(f"Set {name} or {name}_FILE")
|
||||
|
||||
|
||||
S3_ACCESS_KEY_ID = secret_value("S3_ACCESS_KEY_ID")
|
||||
S3_SECRET_ACCESS_KEY = secret_value("S3_SECRET_ACCESS_KEY")
|
||||
USER_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
|
||||
if SESSION_TTL_SECONDS < 1:
|
||||
raise RuntimeError("SESSION_TTL_SECONDS must be positive")
|
||||
if IDEMPOTENCY_TTL_SECONDS < 86400:
|
||||
raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400")
|
||||
if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1:
|
||||
raise RuntimeError("Database timeouts must be positive")
|
||||
if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0:
|
||||
raise RuntimeError("Database pool settings must be positive")
|
||||
if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1:
|
||||
raise RuntimeError("HTTP limits must be positive")
|
||||
if MAX_UPLOAD_BYTES < 1 or MAX_UPLOAD_BYTES > 10 * 1024 * 1024:
|
||||
raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10 MiB")
|
||||
if MAX_IMAGE_PIXELS < 1 or MAX_IMAGE_PIXELS > 40_000_000:
|
||||
raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40 million")
|
||||
if MAX_CONCURRENT_UPLOADS < 1:
|
||||
raise RuntimeError("MAX_CONCURRENT_UPLOADS must be positive")
|
||||
if UPLOAD_TIMEOUT_SECONDS < 1:
|
||||
raise RuntimeError("UPLOAD_TIMEOUT_SECONDS must be positive")
|
||||
@@ -0,0 +1,26 @@
|
||||
from typing import Dict, Optional
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
class CacheUnavailableError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class IdempotencyConflictError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class AttachmentNotReadyError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ObjectMissingError(Exception):
|
||||
pass
|
||||
+556
@@ -0,0 +1,556 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import tempfile
|
||||
import uuid
|
||||
import warnings
|
||||
import zlib
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
import unicodedata
|
||||
|
||||
import psycopg
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from .config import (
|
||||
IDEMPOTENCY_KEY_RE,
|
||||
INSTANCE_ID,
|
||||
MAX_CONCURRENT_UPLOADS,
|
||||
MAX_IMAGE_PIXELS,
|
||||
MAX_INFLIGHT_REQUESTS,
|
||||
MAX_REQUEST_BODY_BYTES,
|
||||
MAX_UPLOAD_BYTES,
|
||||
UPLOAD_TIMEOUT_SECONDS,
|
||||
USER_RE,
|
||||
)
|
||||
from .errors import (
|
||||
ApiError,
|
||||
AttachmentNotReadyError,
|
||||
CacheUnavailableError,
|
||||
IdempotencyConflictError,
|
||||
ObjectMissingError,
|
||||
)
|
||||
from .security import hash_password, isoformat, valid_uuid
|
||||
from .store import Store
|
||||
|
||||
LOGGER = logging.getLogger("messenger.http")
|
||||
|
||||
|
||||
class RequestGuard:
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.inflight = 0
|
||||
|
||||
def acquire(self):
|
||||
with self.lock:
|
||||
if self.inflight >= MAX_INFLIGHT_REQUESTS:
|
||||
raise ApiError(503, "unavailable", "Server is busy")
|
||||
self.inflight += 1
|
||||
|
||||
def release(self):
|
||||
with self.lock:
|
||||
self.inflight = max(0, self.inflight - 1)
|
||||
|
||||
|
||||
REQUEST_GUARD = RequestGuard()
|
||||
UPLOAD_GUARD = threading.BoundedSemaphore(MAX_CONCURRENT_UPLOADS)
|
||||
IMAGE_DECODE_GUARD = threading.BoundedSemaphore(1)
|
||||
|
||||
|
||||
class MessengerHandler(BaseHTTPRequestHandler):
|
||||
def __init__(self, request, client_address, server, store: Store):
|
||||
self.store = store
|
||||
super().__init__(request, client_address, server)
|
||||
|
||||
def setup(self):
|
||||
super().setup()
|
||||
self.connection.settimeout(15)
|
||||
|
||||
def log_message(self, format: str, *args):
|
||||
return
|
||||
|
||||
def do_GET(self):
|
||||
self._dispatch("GET")
|
||||
|
||||
def do_POST(self):
|
||||
self._dispatch("POST")
|
||||
|
||||
def do_DELETE(self):
|
||||
self._dispatch("DELETE")
|
||||
|
||||
def _dispatch(self, method: str):
|
||||
request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4())
|
||||
acquired = False
|
||||
try:
|
||||
REQUEST_GUARD.acquire()
|
||||
acquired = True
|
||||
parsed = urlsplit(self.path)
|
||||
path = parsed.path
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
if path == "/health/live" and method == "GET":
|
||||
self._send_json(200, {"status": "ok"}, request_id)
|
||||
elif path == "/health/ready" and method == "GET":
|
||||
try:
|
||||
self.store.ready()
|
||||
except (psycopg.Error, PoolTimeout, OSError, BotoCoreError, ClientError):
|
||||
raise ApiError(503, "unavailable", "A required storage service is unavailable")
|
||||
self._send_json(200, {"status": "ok"}, request_id)
|
||||
elif path == "/api/v1/users" and method == "POST":
|
||||
self._handle_register(request_id)
|
||||
elif path == "/api/v1/auth/sessions" and method == "POST":
|
||||
self._handle_create_session(request_id)
|
||||
elif path == "/api/v1/auth/sessions/current" and method == "DELETE":
|
||||
self._handle_revoke_session(request_id)
|
||||
elif path == "/api/v1/users/me" and method == "GET":
|
||||
self._send_json(200, self._get_actor(request_id), request_id)
|
||||
elif path == "/api/v1/chats" and method == "POST":
|
||||
self._handle_create_chat(request_id)
|
||||
elif path == "/api/v1/chats" and method == "GET":
|
||||
self._handle_list_chats(query, request_id)
|
||||
elif path.startswith("/api/v1/chats/") and path.endswith("/attachments") and method == "POST":
|
||||
chat_id = path.rsplit("/", 2)[1]
|
||||
self._handle_upload_attachment(chat_id, query, request_id)
|
||||
elif path.startswith("/api/v1/attachments/") and method == "GET":
|
||||
attachment_path = path.removeprefix("/api/v1/attachments/")
|
||||
if attachment_path.endswith("/content"):
|
||||
attachment_id = attachment_path.removesuffix("/content")
|
||||
self._handle_download_attachment(attachment_id, query, request_id)
|
||||
else:
|
||||
self._handle_get_attachment(attachment_path, request_id)
|
||||
elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"):
|
||||
chat_id = path.rsplit("/", 2)[1]
|
||||
if method == "POST":
|
||||
self._handle_create_message(chat_id, request_id)
|
||||
else:
|
||||
self._handle_list_messages(chat_id, query, request_id)
|
||||
elif path.startswith("/api/v1/chats/") and method == "GET":
|
||||
self._handle_get_chat(path.rsplit("/", 1)[1], request_id)
|
||||
else:
|
||||
raise ApiError(404, "not_found", "Object not found")
|
||||
except ApiError as exc:
|
||||
self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers)
|
||||
except CacheUnavailableError:
|
||||
self._send_error(503, "unavailable", "Session cache is unavailable", request_id)
|
||||
except IdempotencyConflictError:
|
||||
self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id)
|
||||
except AttachmentNotReadyError:
|
||||
self._send_error(409, "conflict", "Attachment is not ready", request_id)
|
||||
except ObjectMissingError:
|
||||
LOGGER.error("attachment object missing for request %s", request_id)
|
||||
self._send_error(503, "unavailable", "Attachment content is temporarily unavailable", request_id)
|
||||
except (BotoCoreError, ClientError):
|
||||
LOGGER.warning("object storage unavailable for request %s", request_id)
|
||||
self._send_error(503, "unavailable", "Object storage is unavailable", request_id)
|
||||
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError):
|
||||
self._send_error(503, "unavailable", "Database is unavailable", request_id)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
self._send_error(409, "conflict", "Username already exists", request_id)
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
self._send_error(400, "invalid_request", "Unknown member", request_id)
|
||||
except Exception as exc:
|
||||
LOGGER.error("request %s failed with %s", request_id, type(exc).__name__)
|
||||
self._send_error(500, "internal_error", "Internal server error", request_id)
|
||||
finally:
|
||||
if acquired:
|
||||
REQUEST_GUARD.release()
|
||||
|
||||
def _read_json(self, request_id: str) -> Any:
|
||||
if self.headers.get_content_type() != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json")
|
||||
length = self.headers.get("Content-Length")
|
||||
if length is None:
|
||||
raise ApiError(400, "invalid_request", "Missing Content-Length")
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid Content-Length")
|
||||
if size < 0:
|
||||
raise ApiError(400, "invalid_request", "Invalid Content-Length")
|
||||
if size > MAX_REQUEST_BODY_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "Request body too large")
|
||||
raw = self.rfile.read(size)
|
||||
if not raw:
|
||||
raise ApiError(400, "invalid_request", "Empty body")
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
raise ApiError(400, "invalid_request", "Malformed JSON")
|
||||
|
||||
def _get_actor(self, request_id: str) -> Dict[str, Any]:
|
||||
header = self.headers.get("Authorization", "")
|
||||
scheme, separator, token = header.partition(" ")
|
||||
if scheme.lower() != "bearer" or not separator or not token or " " in token:
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
user = self.store.session_user(token)
|
||||
if user is None:
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
return user
|
||||
|
||||
def _handle_register(self, request_id: str):
|
||||
body = self._read_json(request_id)
|
||||
if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}:
|
||||
raise ApiError(400, "invalid_request", "Expected username, display_name and password")
|
||||
username = body["username"]
|
||||
display_name = body["display_name"]
|
||||
password = body["password"]
|
||||
if not isinstance(username, str) or not USER_RE.fullmatch(username):
|
||||
raise ApiError(400, "invalid_request", "Invalid username")
|
||||
if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid display_name")
|
||||
if not isinstance(password, str) or not (12 <= len(password) <= 128):
|
||||
raise ApiError(400, "invalid_request", "Invalid password")
|
||||
user = self.store.create_user(username, display_name, hash_password(password))
|
||||
self._send_json(201, user, request_id)
|
||||
|
||||
def _basic_credentials(self, request_id: str) -> Tuple[str, str]:
|
||||
value = self.headers.get("Authorization", "")
|
||||
scheme, separator, encoded = value.partition(" ")
|
||||
challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
if scheme.lower() != "basic" or not separator or not encoded:
|
||||
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
|
||||
try:
|
||||
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
except (ValueError, UnicodeError):
|
||||
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
|
||||
return username, password
|
||||
|
||||
def _handle_create_session(self, request_id: str):
|
||||
username, password = self._basic_credentials(request_id)
|
||||
user = self.store.authenticate(username, password)
|
||||
if user is None:
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Invalid credentials",
|
||||
{"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
|
||||
)
|
||||
token, expires_at = self.store.create_session(user["id"])
|
||||
self._send_json(
|
||||
201,
|
||||
{"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user},
|
||||
request_id,
|
||||
)
|
||||
|
||||
def _handle_revoke_session(self, request_id: str):
|
||||
self._get_actor(request_id)
|
||||
token = self.headers["Authorization"].partition(" ")[2]
|
||||
if not self.store.revoke_session(token):
|
||||
raise ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication required",
|
||||
{"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
self.send_response(204)
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
self.end_headers()
|
||||
|
||||
def _handle_create_chat(self, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
body = self._read_json(request_id)
|
||||
if not isinstance(body, dict) or set(body) != {"title", "member_ids"}:
|
||||
raise ApiError(400, "invalid_request", "Expected title and member_ids")
|
||||
title, member_ids = body["title"], body["member_ids"]
|
||||
if not isinstance(title, str) or not title.strip() or len(title) > 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid title")
|
||||
if (
|
||||
not isinstance(member_ids, list)
|
||||
or not 1 <= len(member_ids) <= 100
|
||||
or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids)
|
||||
or len(member_ids) != len(set(member_ids))
|
||||
or actor["id"] not in member_ids
|
||||
):
|
||||
raise ApiError(400, "invalid_request", "Invalid member_ids")
|
||||
chat = self.store.create_chat(title, member_ids, actor["id"])
|
||||
self._send_json(201, chat, request_id)
|
||||
|
||||
def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
try:
|
||||
limit = int(query.get("limit", ["50"])[0])
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
if not 1 <= limit <= 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
cursor = query.get("cursor", [None])[0]
|
||||
try:
|
||||
chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid cursor")
|
||||
self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id)
|
||||
|
||||
def _handle_get_chat(self, chat_id: str, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
chat = self.store.get_chat(chat_id, actor["id"])
|
||||
if chat is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
self._send_json(200, chat, request_id)
|
||||
|
||||
def _handle_create_message(self, chat_id: str, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
chat_id = str(uuid.UUID(chat_id))
|
||||
idempotency_key = self.headers.get("Idempotency-Key", "")
|
||||
if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
|
||||
raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key")
|
||||
body = self._read_json(request_id)
|
||||
if not isinstance(body, dict) or set(body).difference({"text", "attachment_ids"}) or "text" not in body:
|
||||
raise ApiError(400, "invalid_request", "Expected text and optional attachment_ids")
|
||||
text = body["text"]
|
||||
attachment_ids = body.get("attachment_ids", [])
|
||||
if (
|
||||
not isinstance(text, str)
|
||||
or len(text) > 4000
|
||||
or not isinstance(attachment_ids, list)
|
||||
or len(attachment_ids) > 10
|
||||
or any(not isinstance(item, str) or not valid_uuid(item) for item in attachment_ids)
|
||||
or len(attachment_ids) != len(set(attachment_ids))
|
||||
or (not text.strip() and not attachment_ids)
|
||||
):
|
||||
raise ApiError(400, "invalid_request", "Invalid message text")
|
||||
attachment_ids = [str(uuid.UUID(item)) for item in attachment_ids]
|
||||
canonical_body = json.dumps(
|
||||
{"text": text, "attachment_ids": attachment_ids},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
message = self.store.create_idempotent_message(
|
||||
chat_id,
|
||||
actor["id"],
|
||||
text,
|
||||
attachment_ids,
|
||||
idempotency_key,
|
||||
canonical_body,
|
||||
)
|
||||
if message is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
self._send_json(201, message, request_id)
|
||||
|
||||
def _read_image(self, request_id: str, declared_type: str) -> Tuple[BinaryIO, str, int, str]:
|
||||
if declared_type not in ("image/png", "image/jpeg"):
|
||||
raise ApiError(415, "unsupported_media_type", "Only PNG and JPEG are supported")
|
||||
length = self.headers.get("Content-Length")
|
||||
if length is None or self.headers.get("Transfer-Encoding"):
|
||||
raise ApiError(400, "invalid_request", "A bounded Content-Length is required")
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid Content-Length")
|
||||
if size <= 0:
|
||||
raise ApiError(415, "unsupported_media_type", "Image is empty")
|
||||
if size > MAX_UPLOAD_BYTES:
|
||||
self.close_connection = True
|
||||
remaining = min(size, MAX_UPLOAD_BYTES + 1)
|
||||
deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS
|
||||
while remaining:
|
||||
remaining_time = deadline - time.monotonic()
|
||||
if remaining_time <= 0:
|
||||
break
|
||||
self.connection.settimeout(min(15, remaining_time))
|
||||
try:
|
||||
chunk = self.rfile.read(min(64 * 1024, remaining))
|
||||
except (TimeoutError, OSError):
|
||||
LOGGER.info("stopped bounded drain of oversized upload")
|
||||
break
|
||||
if not chunk:
|
||||
break
|
||||
remaining -= len(chunk)
|
||||
raise ApiError(413, "payload_too_large", "Image exceeds the 10 MiB limit")
|
||||
content = tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b")
|
||||
digest = hashlib.sha256()
|
||||
bytes_read = 0
|
||||
deadline = time.monotonic() + UPLOAD_TIMEOUT_SECONDS
|
||||
try:
|
||||
while bytes_read < size:
|
||||
remaining_time = deadline - time.monotonic()
|
||||
if remaining_time <= 0:
|
||||
raise ApiError(408, "invalid_request", "Image upload timed out")
|
||||
self.connection.settimeout(min(15, remaining_time))
|
||||
chunk = self.rfile.read(min(64 * 1024, size - bytes_read))
|
||||
if not chunk:
|
||||
raise ApiError(400, "invalid_request", "Incomplete image body")
|
||||
content.write(chunk)
|
||||
digest.update(chunk)
|
||||
bytes_read += len(chunk)
|
||||
except (TimeoutError, OSError):
|
||||
content.close()
|
||||
raise ApiError(400, "invalid_request", "Incomplete image body") from None
|
||||
|
||||
try:
|
||||
content.seek(0)
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(content) as image:
|
||||
actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format or "")
|
||||
if actual_type is None or actual_type != declared_type:
|
||||
raise ApiError(415, "unsupported_media_type", "Image content does not match Content-Type")
|
||||
width, height = image.size
|
||||
if width <= 0 or height <= 0 or width * height > MAX_IMAGE_PIXELS:
|
||||
raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit")
|
||||
image.verify()
|
||||
content.seek(0)
|
||||
with IMAGE_DECODE_GUARD:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(content) as image:
|
||||
image.load()
|
||||
content.seek(0)
|
||||
return content, actual_type, size, digest.hexdigest()
|
||||
except ApiError:
|
||||
content.close()
|
||||
raise
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
content.close()
|
||||
raise ApiError(413, "payload_too_large", "Image exceeds the 40 million pixel limit") from None
|
||||
except (UnidentifiedImageError, OSError, SyntaxError, ValueError, zlib.error):
|
||||
content.close()
|
||||
raise ApiError(415, "unsupported_media_type", "Image is damaged or unsupported") from None
|
||||
|
||||
@staticmethod
|
||||
def _valid_filename(filename: str) -> bool:
|
||||
return (
|
||||
1 <= len(filename) <= 255
|
||||
and not any(char in ("/", "\\", "\x00") for char in filename)
|
||||
and not any(unicodedata.category(char) == "Cc" for char in filename)
|
||||
)
|
||||
|
||||
def _handle_upload_attachment(
|
||||
self, chat_id: str, query: Dict[str, List[str]], request_id: str
|
||||
):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
chat_id = str(uuid.UUID(chat_id))
|
||||
if not self.store.can_access_chat(chat_id, actor["id"]):
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
filenames = query.get("filename", [])
|
||||
if len(filenames) != 1 or not self._valid_filename(filenames[0]):
|
||||
raise ApiError(400, "invalid_request", "Invalid filename")
|
||||
if not UPLOAD_GUARD.acquire(blocking=False):
|
||||
raise ApiError(503, "unavailable", "Upload capacity is temporarily exhausted")
|
||||
try:
|
||||
content, content_type, size, digest = self._read_image(request_id, self.headers.get_content_type())
|
||||
try:
|
||||
attachment = self.store.create_attachment(
|
||||
chat_id,
|
||||
actor["id"],
|
||||
filenames[0],
|
||||
content_type,
|
||||
content,
|
||||
size,
|
||||
digest,
|
||||
)
|
||||
finally:
|
||||
content.close()
|
||||
finally:
|
||||
UPLOAD_GUARD.release()
|
||||
if attachment is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
self._send_json(201, attachment, request_id)
|
||||
|
||||
def _handle_get_attachment(self, attachment_id: str, request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(attachment_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid attachment id")
|
||||
attachment = self.store.get_attachment(str(uuid.UUID(attachment_id)), actor["id"])
|
||||
if attachment is None:
|
||||
raise ApiError(404, "not_found", "Attachment not found")
|
||||
self._send_json(200, attachment, request_id)
|
||||
|
||||
def _handle_download_attachment(
|
||||
self, attachment_id: str, query: Dict[str, List[str]], request_id: str
|
||||
):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(attachment_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid attachment id")
|
||||
if query.get("variant", ["original"]) != ["original"]:
|
||||
raise ApiError(404, "not_found", "Attachment variant not found")
|
||||
result = self.store.download_attachment(str(uuid.UUID(attachment_id)), actor["id"])
|
||||
if result is None:
|
||||
raise ApiError(404, "not_found", "Attachment not found")
|
||||
content_type, content = result
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(content)))
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("Cache-Control", "private, no-store")
|
||||
self.end_headers()
|
||||
self.wfile.write(content)
|
||||
|
||||
def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str):
|
||||
actor = self._get_actor(request_id)
|
||||
if not valid_uuid(chat_id):
|
||||
raise ApiError(400, "invalid_request", "Invalid chat id")
|
||||
try:
|
||||
limit = int(query.get("limit", ["50"])[0])
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
if not 1 <= limit <= 100:
|
||||
raise ApiError(400, "invalid_request", "Invalid limit")
|
||||
cursor = query.get("cursor", [None])[0]
|
||||
try:
|
||||
result = self.store.list_messages(chat_id, actor["id"], limit, cursor)
|
||||
except ValueError:
|
||||
raise ApiError(400, "invalid_request", "Invalid cursor")
|
||||
if result is None:
|
||||
raise ApiError(404, "not_found", "Chat not found")
|
||||
items, next_cursor = result
|
||||
self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id)
|
||||
|
||||
def _send_json(
|
||||
self,
|
||||
status: int,
|
||||
payload: Dict[str, Any],
|
||||
request_id: str,
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("X-Request-Id", request_id)
|
||||
self.send_header("X-Instance-Id", INSTANCE_ID)
|
||||
if headers:
|
||||
for key, value in headers.items():
|
||||
self.send_header(key, value)
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _send_error(
|
||||
self,
|
||||
status: int,
|
||||
code: str,
|
||||
message: str,
|
||||
request_id: str,
|
||||
extra_headers: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
self._send_json(
|
||||
status,
|
||||
{"error": {"code": code, "message": message, "request_id": request_id}},
|
||||
request_id,
|
||||
extra_headers,
|
||||
)
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from src.store import Store
|
||||
else:
|
||||
from .store import Store
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level="INFO")
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
store.migrate()
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,76 @@
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import BinaryIO, Iterator, Optional, Tuple, Union
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from .config import S3_ACCESS_KEY_ID, S3_BUCKET, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY
|
||||
from .errors import ObjectMissingError
|
||||
|
||||
LOGGER = logging.getLogger("messenger.s3")
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self):
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
region_name=S3_REGION,
|
||||
aws_access_key_id=S3_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
|
||||
config=Config(
|
||||
connect_timeout=2,
|
||||
read_timeout=10,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
s3={"addressing_style": "path"},
|
||||
),
|
||||
)
|
||||
|
||||
def put(
|
||||
self,
|
||||
key: str,
|
||||
content: Union[bytes, BinaryIO],
|
||||
content_type: str,
|
||||
size_bytes: Optional[int] = None,
|
||||
) -> None:
|
||||
if size_bytes is None:
|
||||
size_bytes = len(content)
|
||||
self.client.put_object(
|
||||
Bucket=S3_BUCKET,
|
||||
Key=key,
|
||||
Body=content,
|
||||
ContentLength=size_bytes,
|
||||
ContentType=content_type,
|
||||
)
|
||||
|
||||
def get(self, key: str, max_bytes: int) -> bytes:
|
||||
try:
|
||||
response = self.client.get_object(Bucket=S3_BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code")
|
||||
if code in ("NoSuchKey", "404", "NotFound"):
|
||||
raise ObjectMissingError from exc
|
||||
raise
|
||||
body = response["Body"]
|
||||
try:
|
||||
value = body.read(max_bytes + 1)
|
||||
finally:
|
||||
body.close()
|
||||
if len(value) > max_bytes:
|
||||
raise RuntimeError("Stored object exceeds configured safety limit")
|
||||
return value
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.client.delete_object(Bucket=S3_BUCKET, Key=key)
|
||||
|
||||
def ready(self) -> bool:
|
||||
self.client.head_bucket(Bucket=S3_BUCKET)
|
||||
return True
|
||||
|
||||
def objects(self) -> Iterator[Tuple[str, datetime]]:
|
||||
pages = self.client.get_paginator("list_objects_v2").paginate(Bucket=S3_BUCKET)
|
||||
for page in pages:
|
||||
for item in page.get("Contents", []):
|
||||
yield item["Key"], item["LastModified"]
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from src.store import Store
|
||||
else:
|
||||
from .store import Store
|
||||
|
||||
|
||||
LOGGER = logging.getLogger("messenger.reconcile")
|
||||
|
||||
|
||||
def reconcile(
|
||||
store: Store,
|
||||
grace_seconds: int,
|
||||
apply_changes: bool,
|
||||
restore_dir: Optional[Path] = None,
|
||||
):
|
||||
started = time.monotonic()
|
||||
metadata = store.reconciliation_snapshot()
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
||||
found = set()
|
||||
orphans = []
|
||||
for key, last_modified in store.objects.objects():
|
||||
found.add(key)
|
||||
if key not in metadata and last_modified <= cutoff:
|
||||
orphans.append(key)
|
||||
|
||||
missing = [row for key, row in metadata.items() if key not in found]
|
||||
restore_candidates = []
|
||||
if restore_dir is not None:
|
||||
root = restore_dir.resolve(strict=True)
|
||||
for row in missing:
|
||||
source = restore_dir / str(row["id"])
|
||||
try:
|
||||
resolved = source.resolve(strict=True)
|
||||
if (
|
||||
resolved.parent != root
|
||||
or not resolved.is_file()
|
||||
or resolved.stat().st_size != row["size_bytes"]
|
||||
):
|
||||
continue
|
||||
content = resolved.read_bytes()
|
||||
except OSError:
|
||||
continue
|
||||
if (
|
||||
len(content) == row["size_bytes"]
|
||||
and hashlib.sha256(content).hexdigest() == row["sha256"].strip()
|
||||
):
|
||||
restore_candidates.append((row, content))
|
||||
|
||||
restored = 0
|
||||
if apply_changes:
|
||||
for key in orphans:
|
||||
store.objects.delete(key)
|
||||
restorable_ids = set()
|
||||
for row, content in restore_candidates:
|
||||
store.objects.put(row["object_key"], content, row["content_type"])
|
||||
store.clear_missing_object(row["id"])
|
||||
restorable_ids.add(row["id"])
|
||||
restored += 1
|
||||
for row in missing:
|
||||
if row["id"] not in restorable_ids:
|
||||
store.record_missing_object(row["id"])
|
||||
for key, row in metadata.items():
|
||||
if key in found:
|
||||
store.clear_missing_object(row["id"])
|
||||
|
||||
unresolved_missing = len(missing) - restored
|
||||
return {
|
||||
"mode": "apply" if apply_changes else "dry-run",
|
||||
"grace_seconds": grace_seconds,
|
||||
"objects_seen": len(found),
|
||||
"metadata_seen": len(metadata),
|
||||
"orphan_objects": len([key for key in found if key not in metadata]),
|
||||
"stale_orphan_candidates": len(orphans),
|
||||
"deleted_orphans": len(orphans) if apply_changes else 0,
|
||||
"missing_objects": len(missing),
|
||||
"unresolved_missing_objects": unresolved_missing,
|
||||
"restorable_missing_objects": len(restore_candidates),
|
||||
"restored_missing_objects": restored,
|
||||
"missing_incidents_recorded": len(missing) - restored if apply_changes else 0,
|
||||
"duration_ms": round((time.monotonic() - started) * 1000, 3),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Compare S3 objects with PostgreSQL attachment metadata.")
|
||||
parser.add_argument("--grace-seconds", type=int, default=3600)
|
||||
parser.add_argument("--apply", action="store_true", help="delete only stale orphan objects and record missing objects")
|
||||
parser.add_argument(
|
||||
"--restore-dir",
|
||||
type=Path,
|
||||
help="optional trusted backup directory with exact original bytes named by attachment UUID",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 60:
|
||||
parser.error("--grace-seconds must be at least 60")
|
||||
|
||||
logging.basicConfig(level="INFO", format="%(levelname)s %(message)s")
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
result = reconcile(store, args.grace_seconds, args.apply, args.restore_dir)
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
if result["unresolved_missing_objects"]:
|
||||
LOGGER.error(
|
||||
"%s database records reference missing S3 objects; metadata was retained for manual recovery",
|
||||
result["unresolved_missing_objects"],
|
||||
)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,53 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def isoformat(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def valid_uuid(value: str) -> bool:
|
||||
try:
|
||||
uuid.UUID(value)
|
||||
return True
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
return False
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
salt = secrets.token_bytes(16)
|
||||
derived = hashlib.scrypt(
|
||||
password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024
|
||||
)
|
||||
salt_text = base64.urlsafe_b64encode(salt).decode("ascii")
|
||||
hash_text = base64.urlsafe_b64encode(derived).decode("ascii")
|
||||
return f"scrypt$32768$8$1${salt_text}${hash_text}"
|
||||
|
||||
|
||||
def verify_password(password: str, stored: str) -> bool:
|
||||
try:
|
||||
scheme, n, r, p, salt_text, hash_text = stored.split("$")
|
||||
if scheme != "scrypt":
|
||||
return False
|
||||
salt = base64.urlsafe_b64decode(salt_text.encode("ascii"))
|
||||
expected = base64.urlsafe_b64decode(hash_text.encode("ascii"))
|
||||
actual = hashlib.scrypt(
|
||||
password.encode("utf-8"),
|
||||
salt=salt,
|
||||
n=int(n),
|
||||
r=int(r),
|
||||
p=int(p),
|
||||
dklen=len(expected),
|
||||
maxmem=64 * 1024 * 1024,
|
||||
)
|
||||
return hmac.compare_digest(actual, expected)
|
||||
except (ValueError, TypeError, UnicodeError):
|
||||
return False
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
if __package__ in (None, ""):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from src.app import main
|
||||
else:
|
||||
from .app import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+548
@@ -0,0 +1,548 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, BinaryIO, Dict, List, Optional, Tuple
|
||||
|
||||
import redis
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from .config import (
|
||||
DB_LOCK_TIMEOUT_MS,
|
||||
DB_POOL_MAX_SIZE,
|
||||
DB_POOL_TIMEOUT_SECONDS,
|
||||
DB_STATEMENT_TIMEOUT_MS,
|
||||
IDEMPOTENCY_TTL_SECONDS,
|
||||
SESSION_TTL_SECONDS,
|
||||
)
|
||||
from .errors import (
|
||||
AttachmentNotReadyError,
|
||||
CacheUnavailableError,
|
||||
IdempotencyConflictError,
|
||||
)
|
||||
from .object_store import ObjectStore
|
||||
from .security import isoformat, utc_now, valid_uuid, verify_password
|
||||
|
||||
LOGGER = logging.getLogger("messenger.store")
|
||||
|
||||
|
||||
def database_url() -> str:
|
||||
value = os.getenv("DATABASE_URL")
|
||||
if value:
|
||||
return value
|
||||
file_path = os.getenv("DATABASE_URL_FILE")
|
||||
if file_path:
|
||||
return Path(file_path).read_text(encoding="utf-8").strip()
|
||||
raise RuntimeError("Set DATABASE_URL or DATABASE_URL_FILE")
|
||||
|
||||
|
||||
class Store:
|
||||
"""PostgreSQL metadata, S3 objects, and optional Redis session cache."""
|
||||
|
||||
def __init__(self):
|
||||
self.pool = ConnectionPool(
|
||||
conninfo=database_url(),
|
||||
min_size=1,
|
||||
max_size=DB_POOL_MAX_SIZE,
|
||||
timeout=DB_POOL_TIMEOUT_SECONDS,
|
||||
kwargs={
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": 5,
|
||||
"options": (
|
||||
f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} "
|
||||
f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} "
|
||||
"-c idle_in_transaction_session_timeout=10000"
|
||||
),
|
||||
},
|
||||
open=False,
|
||||
)
|
||||
cache_url = os.getenv("CACHE_URL")
|
||||
self.cache = (
|
||||
redis.Redis.from_url(
|
||||
cache_url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=0.25,
|
||||
socket_timeout=0.25,
|
||||
retry_on_timeout=False,
|
||||
max_connections=16,
|
||||
)
|
||||
if cache_url
|
||||
else None
|
||||
)
|
||||
self.objects = ObjectStore()
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=10)
|
||||
|
||||
def close(self):
|
||||
if self.cache is not None:
|
||||
self.cache.close()
|
||||
self.pool.close()
|
||||
|
||||
def ready(self) -> bool:
|
||||
with self.pool.connection(timeout=2) as conn:
|
||||
conn.execute("SELECT 1")
|
||||
self.objects.ready()
|
||||
return True
|
||||
|
||||
def migrate(self):
|
||||
migrations = Path(__file__).resolve().parents[1] / "migrations"
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"
|
||||
)
|
||||
conn.execute("SELECT pg_advisory_xact_lock(2741662026)")
|
||||
applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
for migration in sorted(migrations.glob("*.sql")):
|
||||
if migration.name in applied:
|
||||
continue
|
||||
conn.execute(migration.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,))
|
||||
|
||||
@staticmethod
|
||||
def user(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": row["title"],
|
||||
"member_ids": member_ids,
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def message(row: Dict[str, Any], attachment_ids: Optional[List[str]] = None) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
"attachment_ids": attachment_ids or [],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def attachment(row: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"].strip(),
|
||||
"status": "ready",
|
||||
"variants": [],
|
||||
"error_code": None,
|
||||
"created_at": isoformat(row["created_at"]),
|
||||
"updated_at": isoformat(row["updated_at"]),
|
||||
}
|
||||
|
||||
def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO users (username, display_name, password_hash) "
|
||||
"VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at",
|
||||
(username, display_name, password_hash),
|
||||
).fetchone()
|
||||
return self.user(row)
|
||||
|
||||
def get_user(self, user_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
|
||||
(user_id,),
|
||||
).fetchone()
|
||||
return self.user(row) if row else None
|
||||
|
||||
def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash "
|
||||
"FROM users WHERE username = %s",
|
||||
(username,),
|
||||
).fetchone()
|
||||
if not row or not verify_password(password, row["password_hash"]):
|
||||
return None
|
||||
return self.user(row)
|
||||
|
||||
def create_session(self, user_id: str) -> Tuple[str, datetime]:
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
|
||||
token_hash = hashlib.sha256(token.encode("ascii")).digest()
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
|
||||
(token_hash, user_id, expires_at),
|
||||
)
|
||||
return token, expires_at
|
||||
|
||||
@staticmethod
|
||||
def _session_lock_key(token_hash: bytes) -> int:
|
||||
return int.from_bytes(token_hash[:8], byteorder="big", signed=True)
|
||||
|
||||
def session_user(self, token: str) -> Optional[Dict[str, Any]]:
|
||||
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
|
||||
cache_key = "messenger:session:" + token_hash.hex()
|
||||
if self.cache is not None:
|
||||
try:
|
||||
cached = self.cache.get(cache_key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
except (redis.RedisError, ValueError, TypeError):
|
||||
LOGGER.warning("session cache read failed; falling back to PostgreSQL")
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
|
||||
row = conn.execute(
|
||||
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at "
|
||||
"FROM sessions s JOIN users u ON u.id = s.user_id "
|
||||
"WHERE s.token_hash = %s AND s.expires_at > now()",
|
||||
(token_hash,),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
user = self.user(row)
|
||||
if self.cache is not None:
|
||||
ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000)
|
||||
if ttl_ms > 0:
|
||||
try:
|
||||
self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms)
|
||||
except redis.RedisError:
|
||||
LOGGER.warning("session cache write failed; PostgreSQL remains authoritative")
|
||||
return user
|
||||
|
||||
def revoke_session(self, token: str) -> bool:
|
||||
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
|
||||
if self.cache is not None:
|
||||
try:
|
||||
self.cache.delete("messenger:session:" + token_hash.hex())
|
||||
except redis.RedisError as exc:
|
||||
raise CacheUnavailableError from exc
|
||||
result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,))
|
||||
return result.rowcount == 1
|
||||
|
||||
def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]:
|
||||
if actor_id not in member_ids or len(member_ids) != len(set(member_ids)):
|
||||
raise ValueError("invalid members")
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at",
|
||||
(title,),
|
||||
).fetchone()
|
||||
for member_id in member_ids:
|
||||
conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
|
||||
(row["id"], member_id),
|
||||
)
|
||||
return self.chat(row, member_ids)
|
||||
|
||||
def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT c.id, c.title, c.created_at FROM chats c "
|
||||
"JOIN chat_members m ON m.chat_id = c.id "
|
||||
"WHERE c.id = %s AND m.user_id = %s",
|
||||
(chat_id, user_id),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
|
||||
(chat_id,),
|
||||
).fetchall()
|
||||
return self.chat(row, [str(item["user_id"]) for item in members])
|
||||
|
||||
def list_chats(
|
||||
self, user_id: str, limit: int, cursor: Optional[str]
|
||||
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
|
||||
where = "own.user_id = %s"
|
||||
params: List[Any] = [user_id]
|
||||
if cursor is not None:
|
||||
try:
|
||||
raw = base64.b64decode(
|
||||
cursor.encode("ascii") + b"=" * (-len(cursor) % 4),
|
||||
altchars=b"-_",
|
||||
validate=True,
|
||||
)
|
||||
cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2)
|
||||
if cursor_user != user_id or not valid_uuid(chat_id):
|
||||
raise ValueError("cursor scope")
|
||||
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
if created_at.tzinfo is None or created_at.utcoffset() is None:
|
||||
raise ValueError("cursor timestamp must include a timezone")
|
||||
where += " AND (c.created_at, c.id) > (%s, %s)"
|
||||
params.extend((created_at, chat_id))
|
||||
except (ValueError, UnicodeError, TypeError) as exc:
|
||||
raise ValueError("invalid cursor") from exc
|
||||
params.append(limit + 1)
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT c.id, c.title, c.created_at FROM chats c "
|
||||
"JOIN chat_members own ON own.chat_id = c.id "
|
||||
f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
result = []
|
||||
for row in page:
|
||||
members = conn.execute(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
|
||||
(row["id"],),
|
||||
).fetchall()
|
||||
result.append(self.chat(row, [str(item["user_id"]) for item in members]))
|
||||
next_cursor = None
|
||||
if has_more and page:
|
||||
last = page[-1]
|
||||
cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
|
||||
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
|
||||
return result, next_cursor
|
||||
|
||||
def create_idempotent_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
attachment_ids: List[str],
|
||||
idempotency_key: str,
|
||||
canonical_body: str,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest()
|
||||
lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8")
|
||||
lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True)
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, sender_id),
|
||||
).fetchone()
|
||||
if not member:
|
||||
return None
|
||||
if attachment_ids:
|
||||
attachments = conn.execute(
|
||||
"SELECT id, status FROM attachments "
|
||||
"WHERE chat_id = %s AND id = ANY(%s) FOR SHARE",
|
||||
(chat_id, attachment_ids),
|
||||
).fetchall()
|
||||
if len(attachments) != len(attachment_ids):
|
||||
return None
|
||||
if any(item["status"] != "ready" for item in attachments):
|
||||
raise AttachmentNotReadyError
|
||||
|
||||
conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,))
|
||||
existing = conn.execute(
|
||||
"SELECT request_hash, response, expires_at "
|
||||
"FROM idempotency_records "
|
||||
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s "
|
||||
"FOR UPDATE",
|
||||
(sender_id, chat_id, idempotency_key),
|
||||
).fetchone()
|
||||
if existing and existing["expires_at"] > utc_now():
|
||||
if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash):
|
||||
raise IdempotencyConflictError
|
||||
return existing["response"]
|
||||
if existing:
|
||||
conn.execute(
|
||||
"DELETE FROM idempotency_records "
|
||||
"WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s",
|
||||
(sender_id, chat_id, idempotency_key),
|
||||
)
|
||||
|
||||
row = conn.execute(
|
||||
"INSERT INTO messages (chat_id, sender_id, text) "
|
||||
"VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at",
|
||||
(chat_id, sender_id, text),
|
||||
).fetchone()
|
||||
for ordinal, attachment_id in enumerate(attachment_ids):
|
||||
conn.execute(
|
||||
"INSERT INTO message_attachments (message_id, attachment_id, ordinal) "
|
||||
"VALUES (%s, %s, %s)",
|
||||
(row["id"], attachment_id, ordinal),
|
||||
)
|
||||
message = self.message(row, attachment_ids)
|
||||
conn.execute(
|
||||
"INSERT INTO idempotency_records "
|
||||
"(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) "
|
||||
"VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)",
|
||||
(
|
||||
sender_id,
|
||||
chat_id,
|
||||
idempotency_key,
|
||||
request_hash,
|
||||
json.dumps(message, ensure_ascii=False, separators=(",", ":")),
|
||||
utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS),
|
||||
),
|
||||
)
|
||||
return message
|
||||
|
||||
def list_messages(
|
||||
self, chat_id: str, user_id: str, limit: int, cursor: Optional[str]
|
||||
) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]:
|
||||
with self.pool.connection() as conn:
|
||||
is_member = conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, user_id),
|
||||
).fetchone()
|
||||
if not is_member:
|
||||
return None
|
||||
where = "chat_id = %s"
|
||||
params: List[Any] = [chat_id]
|
||||
if cursor:
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4))
|
||||
decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2)
|
||||
if decoded_chat != chat_id or not valid_uuid(message_id):
|
||||
raise ValueError("cursor scope")
|
||||
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
where += " AND (created_at, id) > (%s, %s)"
|
||||
params.extend((created_at, message_id))
|
||||
except (ValueError, UnicodeError, TypeError) as exc:
|
||||
raise ValueError("invalid cursor") from exc
|
||||
params.append(limit + 1)
|
||||
rows = conn.execute(
|
||||
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
|
||||
f"WHERE {where} ORDER BY created_at, id LIMIT %s",
|
||||
params,
|
||||
).fetchall()
|
||||
ids_by_message = {}
|
||||
if rows:
|
||||
links = conn.execute(
|
||||
"SELECT message_id, attachment_id FROM message_attachments "
|
||||
"WHERE message_id = ANY(%s) ORDER BY message_id, ordinal",
|
||||
([row["id"] for row in rows],),
|
||||
).fetchall()
|
||||
for link in links:
|
||||
ids_by_message.setdefault(link["message_id"], []).append(str(link["attachment_id"]))
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
next_cursor = None
|
||||
if has_more and page:
|
||||
last = page[-1]
|
||||
cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
|
||||
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
|
||||
return [self.message(row, ids_by_message.get(row["id"], [])) for row in page], next_cursor
|
||||
|
||||
def can_access_chat(self, chat_id: str, user_id: str) -> bool:
|
||||
with self.pool.connection() as conn:
|
||||
return conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, user_id),
|
||||
).fetchone() is not None
|
||||
|
||||
def create_attachment(
|
||||
self,
|
||||
chat_id: str,
|
||||
owner_id: str,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
content: BinaryIO,
|
||||
size_bytes: int,
|
||||
digest: str,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if not self.can_access_chat(chat_id, owner_id):
|
||||
return None
|
||||
attachment_id = uuid.uuid4()
|
||||
object_key = f"original/{attachment_id}"
|
||||
content.seek(0)
|
||||
self.objects.put(object_key, content, content_type, size_bytes)
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO attachments "
|
||||
"(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s) "
|
||||
"RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"created_at, updated_at",
|
||||
(
|
||||
attachment_id,
|
||||
chat_id,
|
||||
owner_id,
|
||||
filename,
|
||||
content_type,
|
||||
size_bytes,
|
||||
digest,
|
||||
object_key,
|
||||
),
|
||||
).fetchone()
|
||||
return self.attachment(row)
|
||||
except Exception:
|
||||
try:
|
||||
self.objects.delete(object_key)
|
||||
except Exception as cleanup_error:
|
||||
LOGGER.error(
|
||||
"attachment compensation failed for %s (%s)",
|
||||
attachment_id,
|
||||
type(cleanup_error).__name__,
|
||||
)
|
||||
raise
|
||||
|
||||
def get_attachment(self, attachment_id: str, user_id: str) -> Optional[Dict[str, Any]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, "
|
||||
"a.sha256, a.created_at, a.updated_at "
|
||||
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
|
||||
"WHERE a.id = %s AND m.user_id = %s",
|
||||
(attachment_id, user_id),
|
||||
).fetchone()
|
||||
return self.attachment(row) if row else None
|
||||
|
||||
def download_attachment(self, attachment_id: str, user_id: str) -> Optional[Tuple[str, bytes]]:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT a.object_key, a.content_type, a.size_bytes "
|
||||
"FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id "
|
||||
"WHERE a.id = %s AND m.user_id = %s AND a.status = 'ready'",
|
||||
(attachment_id, user_id),
|
||||
).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return row["content_type"], self.objects.get(row["object_key"], row["size_bytes"])
|
||||
|
||||
def reconciliation_snapshot(self):
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT id, object_key, created_at, size_bytes, sha256, content_type FROM attachments"
|
||||
).fetchall()
|
||||
return {row["object_key"]: row for row in rows}
|
||||
|
||||
def record_missing_object(self, attachment_id: str) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO reconciliation_incidents (attachment_id, incident_type) "
|
||||
"VALUES (%s, 'missing_object') ON CONFLICT (attachment_id, incident_type) "
|
||||
"DO UPDATE SET last_seen_at = clock_timestamp()",
|
||||
(attachment_id,),
|
||||
)
|
||||
|
||||
def clear_missing_object(self, attachment_id: str) -> None:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM reconciliation_incidents "
|
||||
"WHERE attachment_id = %s AND incident_type = 'missing_object'",
|
||||
(attachment_id,),
|
||||
)
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Negative attachment and cross-chat authorization checks against a running lab4 API."""
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def upload(client, actor, chat_id, data, content_type="image/png", filename="test.png", expected=201):
|
||||
return client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat_id + "/attachments?filename=" + quote(filename, safe=""),
|
||||
expected,
|
||||
actor=actor,
|
||||
raw=data,
|
||||
headers={"Content-Type": content_type},
|
||||
schema="Attachment" if expected == 201 else None,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Attachment upload, size, MIME, and chat authorization checks")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
other_chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Second test chat", "member_ids": [alice["id"], eve["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
image = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
print("Checking accepted fixture upload.", flush=True)
|
||||
attachment = upload(client, alice, chat["id"], image)["json"]
|
||||
|
||||
print("Checking damaged and spoofed image rejection.", flush=True)
|
||||
upload(client, alice, chat["id"], b"not an image", expected=415)
|
||||
upload(client, alice, chat["id"], image, content_type="image/jpeg", expected=415)
|
||||
print("Checking byte and pixel limits.", flush=True)
|
||||
upload(client, alice, chat["id"], b"x" * (10 * 1024 * 1024 + 1), expected=413)
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
def chunk(kind, payload):
|
||||
return (
|
||||
struct.pack(">I", len(payload))
|
||||
+ kind
|
||||
+ payload
|
||||
+ struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF)
|
||||
)
|
||||
|
||||
bad_idat = bytearray(zlib.compress(b"\x00\x10\x20\x30"))
|
||||
bad_idat[-3] ^= 0xFF
|
||||
damaged_idat = (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", bytes(bad_idat))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
upload(client, alice, chat["id"], damaged_idat, expected=415)
|
||||
|
||||
too_many_pixels = (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ chunk(b"IHDR", struct.pack(">IIBBBBB", 10001, 4000, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", zlib.compress(b""))
|
||||
+ chunk(b"IEND", b"")
|
||||
)
|
||||
upload(client, alice, chat["id"], too_many_pixels, expected=413)
|
||||
print("Checking filename and cross-chat authorization.", flush=True)
|
||||
upload(client, alice, chat["id"], image, filename="../unsafe.png", expected=400)
|
||||
upload(client, eve, chat["id"], image, expected=404)
|
||||
client.call("GET", "/api/v1/attachments/" + attachment["id"], 404, actor=eve)
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + other_chat["id"] + "/messages",
|
||||
404,
|
||||
actor=alice,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
body={"text": "", "attachment_ids": [attachment["id"]]},
|
||||
)
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
400,
|
||||
actor=bob,
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
body={"text": ""},
|
||||
)
|
||||
print("PASS: oversized, damaged, spoofed, unsafe-name, outsider, and cross-chat cases")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove a concurrent idempotency-key race through the load balancer."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import threading
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Concurrent shared-idempotency integration check")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
key = "race-" + uuid.uuid4().hex
|
||||
text = "concurrent-" + uuid.uuid4().hex
|
||||
count = 12
|
||||
barrier = threading.Barrier(count)
|
||||
|
||||
def send_once(_):
|
||||
worker = Client(args.base_url, args.ca_file)
|
||||
barrier.wait(timeout=10)
|
||||
return worker.message(alice, chat, text, key=key)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=count) as pool:
|
||||
responses = list(pool.map(send_once, range(count)))
|
||||
|
||||
instance_ids = {response["headers"]["X-Instance-Id"] for response in responses}
|
||||
messages = [response["json"] for response in responses]
|
||||
check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas")
|
||||
check(all(message == messages[0] for message in messages), "same-key requests returned different messages")
|
||||
page = client.call(
|
||||
"GET",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages?limit=100",
|
||||
actor=alice,
|
||||
schema="MessagePage",
|
||||
)["json"]
|
||||
matches = [message for message in page["items"] if message["text"] == text]
|
||||
check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages")
|
||||
print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic tests of the reconciliation decision rules without deleting real objects."""
|
||||
import os
|
||||
import hashlib
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
os.environ.setdefault("S3_ACCESS_KEY_ID", "test-reconciler")
|
||||
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test-only-not-a-real-secret")
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from src.reconcile import reconcile
|
||||
|
||||
|
||||
class FakeObjects:
|
||||
def __init__(self, items):
|
||||
self.items = items
|
||||
self.deleted = []
|
||||
|
||||
def objects(self):
|
||||
return iter(self.items)
|
||||
|
||||
def delete(self, key):
|
||||
self.deleted.append(key)
|
||||
self.items = [(name, date) for name, date in self.items if name != key]
|
||||
|
||||
def put(self, key, content, content_type):
|
||||
self.items.append((key, datetime.now(timezone.utc)))
|
||||
|
||||
|
||||
class FakeStore:
|
||||
def __init__(self):
|
||||
now = datetime.now(timezone.utc)
|
||||
self.metadata = {
|
||||
"original/present": {"id": "present-id", "object_key": "original/present"},
|
||||
"original/missing": {
|
||||
"id": "missing-id",
|
||||
"object_key": "original/missing",
|
||||
"size_bytes": 4,
|
||||
"sha256": hashlib.sha256(b"test").hexdigest(),
|
||||
"content_type": "image/png",
|
||||
},
|
||||
}
|
||||
self.objects = FakeObjects(
|
||||
[
|
||||
("original/present", now),
|
||||
("original/stale-orphan", now - timedelta(hours=2)),
|
||||
("original/new-orphan", now - timedelta(minutes=10)),
|
||||
]
|
||||
)
|
||||
self.missing_incidents = set()
|
||||
self.cleared_incidents = set()
|
||||
|
||||
def reconciliation_snapshot(self):
|
||||
return self.metadata
|
||||
|
||||
def record_missing_object(self, attachment_id):
|
||||
self.missing_incidents.add(attachment_id)
|
||||
|
||||
def clear_missing_object(self, attachment_id):
|
||||
self.missing_incidents.discard(attachment_id)
|
||||
self.cleared_incidents.add(attachment_id)
|
||||
|
||||
|
||||
def main():
|
||||
store = FakeStore()
|
||||
dry_run = reconcile(store, grace_seconds=3600, apply_changes=False)
|
||||
assert dry_run["stale_orphan_candidates"] == 1
|
||||
assert dry_run["missing_objects"] == 1
|
||||
assert dry_run["deleted_orphans"] == 0
|
||||
assert store.objects.deleted == [] and store.missing_incidents == set()
|
||||
|
||||
applied = reconcile(store, grace_seconds=3600, apply_changes=True)
|
||||
assert store.objects.deleted == ["original/stale-orphan"]
|
||||
assert "original/new-orphan" not in store.objects.deleted
|
||||
assert store.missing_incidents == {"missing-id"}
|
||||
assert store.cleared_incidents == {"present-id"}
|
||||
assert applied["deleted_orphans"] == 1 and applied["missing_objects"] == 1
|
||||
|
||||
with tempfile.TemporaryDirectory() as backup:
|
||||
restore_dir = Path(backup)
|
||||
(restore_dir / "missing-id").write_bytes(b"test")
|
||||
preview = reconcile(store, 3600, False, restore_dir)
|
||||
assert preview["restorable_missing_objects"] == 1
|
||||
assert store.missing_incidents == {"missing-id"}
|
||||
restored = reconcile(store, 3600, True, restore_dir)
|
||||
assert restored["restored_missing_objects"] == 1
|
||||
assert store.missing_incidents == set()
|
||||
|
||||
repeated = reconcile(store, grace_seconds=3600, apply_changes=True)
|
||||
assert repeated["stale_orphan_candidates"] == 0 and repeated["missing_objects"] == 0
|
||||
assert store.objects.deleted == ["original/stale-orphan"]
|
||||
assert store.missing_incidents == set()
|
||||
print("PASS: dry-run is read-only; stale orphan cleanup is idempotent and respects grace period")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise real S3/PostgreSQL reconciliation using disposable synthetic objects."""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
STORE_CODE = """
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from src.store import Store
|
||||
|
||||
store = Store()
|
||||
store.open()
|
||||
try:
|
||||
action = sys.argv[1]
|
||||
if action == "key":
|
||||
attachment_id = sys.argv[2]
|
||||
matches = [
|
||||
key for key, row in store.reconciliation_snapshot().items()
|
||||
if str(row["id"]) == attachment_id
|
||||
]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("attachment metadata lookup failed")
|
||||
print(matches[0])
|
||||
elif action == "put":
|
||||
key, file_path = sys.argv[2:4]
|
||||
content = Path(file_path).read_bytes()
|
||||
store.objects.put(key, content, "image/png", len(content))
|
||||
elif action == "delete":
|
||||
store.objects.delete(sys.argv[2])
|
||||
else:
|
||||
raise RuntimeError("unsupported test operation")
|
||||
finally:
|
||||
store.close()
|
||||
"""
|
||||
|
||||
PRIVATE_CHECK_CODE = """
|
||||
import socket
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
try:
|
||||
socket.getaddrinfo("s3-core", 8888)
|
||||
except socket.gaierror:
|
||||
print("PASS: SeaweedFS master/filer network is isolated")
|
||||
else:
|
||||
raise RuntimeError("API network can resolve the SeaweedFS core service")
|
||||
|
||||
for port in (8181, 9101):
|
||||
try:
|
||||
socket.create_connection(("s3", port), timeout=1).close()
|
||||
except OSError:
|
||||
continue
|
||||
raise RuntimeError("unexpected SeaweedFS gateway interface is reachable on port " + str(port))
|
||||
|
||||
try:
|
||||
urllib.request.urlopen("http://s3:8333/messenger-attachments/", timeout=3)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code == 403:
|
||||
print("PASS: unauthenticated bucket access denied")
|
||||
else:
|
||||
raise RuntimeError("unexpected anonymous S3 status: " + str(error.code))
|
||||
else:
|
||||
raise RuntimeError("private bucket allowed an unauthenticated GET")
|
||||
"""
|
||||
|
||||
|
||||
def docker_exec(container_id, *args):
|
||||
return subprocess.run(
|
||||
["docker", "exec", container_id, *args],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def reconciliation(container_id, *args):
|
||||
result = docker_exec(container_id, "python3", "src/reconcile.py", "--grace-seconds", "60", *args)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Real PostgreSQL/SeaweedFS reconciliation and private-bucket experiment")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
attachment, content = client.attachment(alice, chat)
|
||||
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
||||
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
containers = result.stdout.split()
|
||||
check(len(containers) >= 2, "at least two running API replicas are required")
|
||||
container_id = containers[0]
|
||||
key = docker_exec(
|
||||
container_id, "python3", "-c", STORE_CODE, "key", attachment["id"]
|
||||
).stdout.strip()
|
||||
orphan_key = "original/" + str(uuid.uuid4())
|
||||
|
||||
with tempfile.TemporaryDirectory() as backup:
|
||||
source = Path(backup) / "original.png"
|
||||
source.write_bytes(content)
|
||||
container_file = "/tmp/lab4-orphan-" + uuid.uuid4().hex + ".png"
|
||||
recovery_dir = "/tmp/lab4-recovery-" + uuid.uuid4().hex
|
||||
recovery_file = recovery_dir + "/" + attachment["id"]
|
||||
orphan_created = False
|
||||
orphan_removed = False
|
||||
missing_deleted = False
|
||||
missing_restored = False
|
||||
try:
|
||||
subprocess.run(["docker", "cp", str(source), container_id + ":" + container_file], check=True)
|
||||
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", orphan_key, container_file)
|
||||
orphan_created = True
|
||||
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", key)
|
||||
missing_deleted = True
|
||||
|
||||
preview = reconciliation(container_id)
|
||||
check(
|
||||
preview["orphan_objects"] >= 1 and preview["missing_objects"] == 1,
|
||||
"dry-run missed injected inconsistencies",
|
||||
)
|
||||
check(preview["deleted_orphans"] == 0, "dry-run modified S3")
|
||||
immediate_apply = reconciliation(container_id, "--apply")
|
||||
check(immediate_apply["stale_orphan_candidates"] == 0, "fresh orphan bypassed grace period")
|
||||
check(immediate_apply["deleted_orphans"] == 0, "fresh orphan was deleted")
|
||||
|
||||
docker_exec(container_id, "mkdir", "-p", recovery_dir)
|
||||
subprocess.run(
|
||||
["docker", "cp", str(source), container_id + ":" + recovery_file],
|
||||
check=True,
|
||||
)
|
||||
restore_preview = reconciliation(container_id, "--restore-dir", recovery_dir)
|
||||
check(restore_preview["restorable_missing_objects"] == 1, "exact backup was not identified as restorable")
|
||||
restored = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
|
||||
check(restored["restored_missing_objects"] == 1, "missing S3 object was not restored")
|
||||
missing_restored = True
|
||||
|
||||
repeated = reconciliation(container_id, "--apply", "--restore-dir", recovery_dir)
|
||||
check(repeated["missing_objects"] == 0, "repeat reconciliation still reports missing object")
|
||||
check(repeated["stale_orphan_candidates"] == 0, "recent orphan did not respect grace period")
|
||||
print("Waiting for the 60-second orphan grace period.")
|
||||
time.sleep(61)
|
||||
|
||||
cleanup_preview = reconciliation(container_id)
|
||||
check(cleanup_preview["stale_orphan_candidates"] == 1, "stale orphan was not identified after grace")
|
||||
cleaned = reconciliation(container_id, "--apply")
|
||||
check(cleaned["deleted_orphans"] == 1, "stale orphan was not removed")
|
||||
orphan_removed = True
|
||||
final_scan = reconciliation(container_id, "--apply")
|
||||
check(
|
||||
final_scan["orphan_objects"] == 0 and final_scan["missing_objects"] == 0,
|
||||
"repeat scan is not clean",
|
||||
)
|
||||
finally:
|
||||
if missing_deleted and not missing_restored:
|
||||
docker_exec(container_id, "python3", "-c", STORE_CODE, "put", key, container_file)
|
||||
if orphan_created and not orphan_removed:
|
||||
docker_exec(container_id, "python3", "-c", STORE_CODE, "delete", orphan_key)
|
||||
|
||||
private = docker_exec(container_id, "python3", "-c", PRIVATE_CHECK_CODE).stdout.strip()
|
||||
check("PASS:" in private, "unauthenticated S3 access check failed")
|
||||
downloaded = client.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
actor=bob,
|
||||
)["bytes"]
|
||||
check(downloaded == content, "reconciler changed the original of an attachment used by a message")
|
||||
check(message["attachment_ids"] == [attachment["id"]], "test message did not retain the attachment reference")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"result": "PASS",
|
||||
"dataset_objects_scanned": final_scan["objects_seen"],
|
||||
"metadata_records_scanned": final_scan["metadata_seen"],
|
||||
"scan_duration_ms": final_scan["duration_ms"],
|
||||
"orphan_detected_and_removed": cleaned["deleted_orphans"],
|
||||
"missing_detected_and_restored": restored["restored_missing_objects"],
|
||||
"fresh_orphan_preserved_until_grace": True,
|
||||
"message_attachment_download_checksum": "unchanged",
|
||||
"private_bucket": "anonymous request denied",
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run ten simultaneous near-6-MiB uploads and sample API container memory."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import subprocess
|
||||
import uuid
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
UNIT = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
|
||||
USAGE_RE = re.compile(r"([\d.]+)(B|KiB|MiB|GiB)\s*/")
|
||||
|
||||
|
||||
def usage_bytes(line):
|
||||
match = USAGE_RE.search(line)
|
||||
return int(float(match.group(1)) * UNIT[match.group(2)]) if match else 0
|
||||
|
||||
|
||||
def api_containers():
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "ps", "--quiet", "--status", "running", "api"],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
ids = result.stdout.split()
|
||||
check(len(ids) >= 2, "at least two running API containers are required")
|
||||
return ids
|
||||
|
||||
|
||||
def sample_once(ids):
|
||||
output = subprocess.run(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.splitlines()
|
||||
values = [usage_bytes(line) for line in output]
|
||||
check(len(values) == len(ids) and all(values), "could not parse docker stats memory usage")
|
||||
return max(values)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("Bounded-memory ten-client upload experiment")
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
width = height = 1400
|
||||
content = BytesIO()
|
||||
Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)).save(
|
||||
content, format="PNG", compress_level=0
|
||||
)
|
||||
payload = content.getvalue()
|
||||
check(0 < len(payload) <= 10 * 1024 * 1024, "synthetic PNG is outside the upload size limit")
|
||||
ids = api_containers()
|
||||
baseline = sample_once(ids)
|
||||
samples = []
|
||||
stats_process = subprocess.Popen(
|
||||
["docker", "stats", "--format", "{{.MemUsage}}", *ids],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
)
|
||||
|
||||
def sampler():
|
||||
if stats_process.stdout is None:
|
||||
return
|
||||
for line in stats_process.stdout:
|
||||
value = usage_bytes(line)
|
||||
if value:
|
||||
samples.append(value)
|
||||
|
||||
sampling_thread = threading.Thread(target=sampler, daemon=True)
|
||||
sampling_thread.start()
|
||||
count = 10
|
||||
barrier = threading.Barrier(count)
|
||||
|
||||
def upload(index):
|
||||
worker = Client(args.base_url, args.ca_file)
|
||||
barrier.wait(timeout=15)
|
||||
return worker.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/attachments?filename=load-" + str(index) + ".png",
|
||||
201,
|
||||
actor=alice,
|
||||
raw=payload,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=count) as pool:
|
||||
responses = list(pool.map(upload, range(count)))
|
||||
finally:
|
||||
stopped_by_test = False
|
||||
if stats_process.poll() is None:
|
||||
stats_process.kill()
|
||||
stopped_by_test = True
|
||||
try:
|
||||
stats_process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
stats_process.kill()
|
||||
stats_process.wait(timeout=5)
|
||||
sampling_thread.join(timeout=5)
|
||||
stats_error = stats_process.stderr.read() if stats_process.stderr is not None else ""
|
||||
check(
|
||||
stats_process.returncode == 0 or (stopped_by_test and stats_process.returncode == -9),
|
||||
"docker stats sampling process failed (exit "
|
||||
+ str(stats_process.returncode)
|
||||
+ "): "
|
||||
+ stats_error.strip()[:300],
|
||||
)
|
||||
check(bool(samples), "docker stats produced no memory samples")
|
||||
peak = max([baseline] + samples)
|
||||
check(len(responses) == count and all(response["status"] == 201 for response in responses), "an upload failed")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"uploads": count,
|
||||
"concurrency": count,
|
||||
"bytes_per_upload": len(payload),
|
||||
"duration_seconds": round(time.monotonic() - started, 3),
|
||||
"api_containers": len(ids),
|
||||
"sample_count": len(samples) // len(ids),
|
||||
"baseline_max_container_bytes": baseline,
|
||||
"peak_max_container_bytes": peak,
|
||||
"peak_delta_bytes": max(0, peak - baseline),
|
||||
"sampling_method": "continuous docker stats stream; Docker/CLI sampling cadence",
|
||||
"memory_scope": "Docker-reported API container working set; LB, DB, S3, and host cache excluded",
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
+1
-1
@@ -36,7 +36,7 @@ def main():
|
||||
script = Path(args.action_script).expanduser().resolve()
|
||||
check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)')
|
||||
check('.example.' not in script.name,'copy and implement the example hook first')
|
||||
c = Client(args.base_url,args.ca_file,timeout=2)
|
||||
c = Client(args.base_url,args.ca_file,timeout=10 if args.mode=='persistence' else 2)
|
||||
alice,bob,eve,chat = c.bootstrap()
|
||||
sent = c.message(alice,chat,'Сохранить при отказе')
|
||||
original = sent['json']
|
||||
|
||||
Reference in New Issue
Block a user