diff --git a/.dockerignore b/.dockerignore index bc16e556..6d3a8e74 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,4 @@ __pycache__ node_modules evidence .DS_Store +results diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..6d916375 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM maven:3.9.9-eclipse-temurin-21 AS builder +WORKDIR /build + +COPY java/pom.xml . +RUN mvn -B -q dependency:go-offline + + +COPY java/src ./src +RUN mvn -B -q clean package -DskipTests + +FROM eclipse-temurin:21-jre-alpine AS runtime + +RUN addgroup -g 1001 -S appgroup \ + && adduser -u 1001 -S appuser -G appgroup + +WORKDIR /app +COPY --from=builder /build/target/*.jar app.jar +RUN chown -R appuser:appgroup /app + +USER appuser +EXPOSE 8080 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD wget -qO- http://localhost:8080/health/live || exit 1 + +ENTRYPOINT ["java", "-XX:MaxRAMPercentage=75.0", "-jar", "/app/app.jar"] \ No newline at end of file diff --git a/Makefile b/Makefile index 5d86d171..1b0be438 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,9 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +java-test: + cd java && mvnw.cmd test + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) @@ -28,3 +31,10 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-attachments: + $(PYTHON) -X utf8 tests/attachments_negative.py --base-url "$(BASE_URL)" $(TLS_ARGS) + + +test-upload-abort: + $(PYTHON) -X utf8 tests/upload_abort_check.py --base-url "$(BASE_URL)" $(TLS_ARGS) \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..72b97443 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,437 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: TODO: Хуцаева Айса, БПИ2302. +- Целевая оценка: 5. +- Выбранный трек: трек 2 — согласованность двух хранилищ. +- Commit/PR: solution/khutsaeva-bpi2302. -## Запуск из чистого clone - -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +### Процессы и хранилища + +```mermaid +flowchart TD + Client["Клиент"] --> LB["HAProxy"] + LB --> API["API × 2"] + API --> PG["PostgreSQL: метаданные"] + API --> Redis["Redis: сессии"] + API --> S3["MinIO: оригиналы"] + R["Служебный reconciler"] --> PG + R --> S3 + Backup["Проверенные резервные копии"] --> R +``` + +### Что перенесено из lab3 + +- Две API-реплики и HAProxy. +- PostgreSQL и Redis с persistent volumes. +- Авторизация и серверные сессии. +- Проверка участия пользователя в чате. +- Идемпотентная отправка сообщений. +- Health/readiness endpoints. +- Миграции Flyway. +- Проверки перезапуска и отказа API-реплики. + +### Что добавлено в lab4 + +- Приватное S3-хранилище MinIO. +- Метаданные Attachment в PostgreSQL. +- Связь сообщения с `attachment_ids`. +- Синхронный binary upload PNG/JPEG. +- Авторизованное скачивание оригиналов. +- Проверки размера, формата, filename и числа пикселей. +- Ограничение числа одновременно принимаемых загрузок. +- Очистка временных файлов и orphan-объектов. +- Отдельные ограниченные S3 credentials для API. +- Служебный reconciler с dry-run и восстановлением из проверенной копии. +- Эксперименты согласованности и стоимости обхода. + +### Загрузка и ограничения + +API принимает сырое тело PNG/JPEG, а не multipart-запрос. + +Ограничения: + +- размер оригинала — не более 10 MiB; +- число декодированных пикселей — не более 40 миллионов; +- MIME проверяется по содержимому; +- filename используется только для отображения; +- запрещаются разделители пути и управляющие символы в filename; +- object key назначает сервер; +- лишние параллельные загрузки получают `429 rate_limited`. + +Тело запроса читается ограниченными порциями во временный файл. +Размер контролируется при чтении, в том числе при неизвестном +Content-Length. Размеры изображения проверяются до полного декодирования. + +Локальный временный файл API используется только во время обработки. +После успешной загрузки оригинал хранится в S3, а не в каталоге API. + +Ответ `201 ready` возвращается после сохранения оригинала и метаданных. + +### Транзакционные границы + +Общей атомарной транзакции между PostgreSQL и S3 нет. + +При сохранении вложения: + +1. Принимается и проверяется изображение. +2. Открывается транзакция PostgreSQL. +3. Захватывается advisory lock для object key. +4. Оригинал записывается в S3. +5. Метаданные записываются в PostgreSQL. +6. Транзакция фиксируется. +7. API возвращает `201`. + +Если S3-запись прошла, а транзакция PostgreSQL не зафиксировалась, +возможно появление orphan-объекта. Он удаляется последующей сверкой +после grace period. + +Очистка и reconciler используют тот же ключ блокировки, что и upload. +Перед удалением повторно проверяются наличие метаданных и возраст +объекта. Это защищает от удаления объекта незавершённой штатной загрузки. + +### Поведение reconciler + +| Состояние | Dry-run | Apply | +| --- | --- | --- | +| Старый объект без метаданных | `would_delete_orphans` | Удаление после повторной проверки | +| Свежий объект без метаданных | Пропуск | Пропуск | +| Метаданные и оригинал существуют | Проверка наличия и размера | То же | +| Оригинал отсутствует, проверенная копия есть | `would_restore` | Восстановление и проверка SHA-256 | +| Оригинал отсутствует, копии нет | `unresolved_missing` | Метаданные сохраняются | +| БД или S3 недоступны | Проход прерывается | Проход прерывается | +| Объект занят другим процессом | `busy` | Пропуск до следующего запуска | + +Копия для восстановления имеет имя `.bin`. +Перед восстановлением проверяются её размер и SHA-256. +После записи проверяются скачанные из S3 байты. + +Reconciler не создаёт резервные копии автоматически. +В эксперименте копия сохранялась до восстановления намеренно +удалённого тестового оригинала. + +Публичный контракт lab4 не расширялся статусом `missing`. +При отсутствии оригинала сохраняются сообщение и метаданные, +а download возвращает `503 unavailable`. + +### Конфигурация + +| Переменная | Назначение | +| --- | --- | +| `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD` | Инициализация PostgreSQL и подключение приложения через Compose | +| `DB_URL`, `DB_USERNAME`, `DB_PASSWORD` | Настройки PostgreSQL внутри API | +| `REDIS_HOST`, `REDIS_PASSWORD` | Подключение к Redis | +| `S3_ENDPOINT` | Внутренний адрес `http://minio:9000` | +| `S3_BUCKET` | Bucket `messenger-attachments` | +| `S3_ACCESS_KEY`, `S3_SECRET_KEY` | Ограниченные credentials API | +| `MINIO_ROOT_USER`, `MINIO_ROOT_PASSWORD` | Административная инициализация MinIO | +| `API_REPLICAS` | Число API-реплик | +| `API_CPUS`, `API_MEMORY` | Лимиты API-контейнера | +| `DB_POOL_SIZE` | Размер пула соединений API | +| `APP_CLEANUP_ENABLED` | Временное отключение фоновой очистки в эксперименте | + +В Java S3-параметры используются через `app.s3.*`. +Reconciler получает настройки из контейнеров и использует S3 credentials +API, а не root-пользователя MinIO. + +В текущей реализации регион S3 задан как `us-east-1`. +Изменение одной переменной `S3_REGION` не переключает регион автоматически. + +Предложенные skeleton переменные `DATABASE_URL_FILE`, +`S3_ACCESS_KEY_ID_FILE` и `S3_SECRET_ACCESS_KEY_FILE` не следует считать +реализованными автоматически: текущий вариант использует переменные +окружения. + +TODO перед публикацией: сверить актуальный `.env.example` с этой таблицей, +удалить неподдерживаемые настройки либо явно пометить их как примеры. +Лимиты 10 MiB и 40 миллионов пикселей реализованы в валидаторе; +наличие одноимённых переменных в skeleton само по себе их не меняет. ## Доказательства критериев +JSON-отчёты экспериментов сохранены в `results/`. +Обозначения каталогов: + +- `results/reconcile-/` — эксперимент исправления расхождений; +- `results/reconcile-failure-/` — отказы зависимостей; +- `results/reconcile-cost-/` — измерения стоимости обхода. + +Для проверок, выполненных ранее только с выводом в терминал, +ниже явно указана необходимость сохранить файл доказательства. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Валидность контракта | `make validate` | PASS: OpenAPI и JSON Schema валидны; приложение этим не тестируется | TODO: сохранить вывод | +| Структура skeleton | `make check` | PASS: структура, ссылки и синтаксис Python; приложение этим не тестируется | TODO: сохранить вывод | +| Публичный smoke | `make test` | PASS: публичный сценарий lab4 | TODO: сохранить вывод | +| Сохранение возможностей lab3 | `make test`, persistence и failover | Сохраняются сообщения и действующая сессия | TODO: логи smoke и resilience | +| Метаданные в PostgreSQL, оригиналы в S3 | Миграция Attachment, код сохранения, smoke | Оригинал скачивается через API; метаданные отделены от байтов | Миграции, `AttachmentPersistenceService.java`, `S3StorageService.java`; TODO: лог smoke | +| Сообщения с вложениями | Smoke и `tests/reconcile_check.py` | Сообщение и ссылки на вложения сохраняются после сверки и восстановления | `results/reconcile-/`; TODO: итоговый stdout теста | +| Ограничения изображения | `make test-attachments` | Негативный suite прошёл: превышение размера/пикселей, повреждение, MIME и filename | TODO: сохранить stdout negative suite | +| Запрет чужого доступа | Smoke и negative suite | Посторонний не получает чужое вложение; чужой attachment и другой чат отклоняются | TODO: логи smoke и negative suite | +| Переключение API-реплики | `make test-failover ACTION_SCRIPT=./scripts/stop-one.sh` | 2 реплики до воздействия, 74 успешные записи, 2 неудачные попытки; первая успешная запись через 1,833 с | TODO: лог теста и состояние контейнеров | +| Остановка именно загрузившей реплики | Upload → фиксация `X-Instance-Id` → остановка этой реплики → download | Отдельная привязка остановленной реплики к успешному upload пока не зафиксирована в представленном доказательстве | TODO: добавить адресное доказательство | +| Сохранение после restart | `make test-persistence ACTION_SCRIPT=./scripts/restart.sh` | PASS: подтверждённые данные и действующая сессия сохранены | TODO: stdout теста и трасса hook | +| Недоступность S3 | `tests/reconcile_failure_check.py` | При остановке MinIO reconciler завершился без изменения объектов; после восстановления оригинал доступен через API | `02-minio-unavailable.json`, `summary.json` в каталоге failure | +| Пользовательские ошибки не превращаются в 500 | `make test-attachments` | Негативные сценарии завершились ожидаемыми клиентскими ошибками | TODO: stdout negative suite | +| Прерывание HTTP upload | `make test-upload-abort` | Временный файл удалён; обе реплики принимают следующие загрузки; оригинал цел | TODO: сохранить stdout | +| Аварийное завершение API | `python3 -X utf8 tests/upload_crash_check.py` | После SIGKILL файл сохранился; после запуска того же контейнера просроченный файл удалён | TODO: сохранить stdout | +| Отказ БД после S3-записи | `tests/upload_db_failure_check.py` | После исправления кода ошибки на `unavailable` сценарий прошёл; orphan удаляется очисткой | TODO: сохранить stdout | +| Безопасная очистка orphan | `tests/cleanup_check.py --bucket messenger-attachments` | Старый orphan удалён; свежий объект и зарегистрированное вложение сохранены | TODO: сохранить stdout | +| Ограниченная память при ≥10 клиентах | `tests/upload_memory_check.py` | 10 клиентов, 81 успешная загрузка, 661 ответ 429, 0 неожиданных ошибок и OOM | `results/upload-memory.json` | +| Идемпотентный bootstrap | Два запуска `docker compose run --rm --no-deps --pull never minio-init` | Оба завершились успешно; повторное назначение политики обработано | TODO: сохранить вывод обоих запусков | +| Отдельные ограниченные S3 credentials | `infra/minio/check-access.sh` через `minio-init` | API может записывать и читать разрешённые объекты; запрещённые операции отклоняются | TODO: сохранить итоговый stdout проверки | +| Private bucket вне API | Та же проверка доступа | Анонимное скачивание существующего объекта запрещено | TODO: сохранить stdout | +| Изоляция административной консоли | `docker compose port minio 9001` и проверка Compose | Требуется подтвердить отсутствие публикации на общем интерфейсе; при локальной публикации ожидается `127.0.0.1` | TODO: сохранить вывод и фрагмент конфигурации | +| Трек 5: обнаружение orphan и missing | `tests/reconcile_check.py` | Dry-run обнаружил по одному orphan и missing без изменений | `01-dry-run.json` | +| Трек 5: безопасное исправление | Тот же тест, apply | Старый orphan удалён; missing без копии не уничтожает метаданные | `02-apply-no-backup.json` | +| Трек 5: восстановление | Тот же тест с резервной копией | Dry-run запланировал восстановление; apply восстановил оригинал с проверкой байтов | `03-dry-run-with-backup.json`, `04-restore.json` | +| Трек 5: повторный запуск | Повторный apply | Нет повторных удалений или восстановлений | `05-repeat.json` | +| Трек 5: отказы обеих зависимостей | `tests/reconcile_failure_check.py` | MinIO и PostgreSQL останавливались по очереди; ложных missing и изменений данных нет | `results/reconcile-failure-/summary.json` | +| Трек 5: измерение стоимости обхода | `tests/reconcile_cost_check.py --sizes 10 100 1000 --repeats 3` | Все проходы завершены; количество SQL/S3-вызовов совпало с расчётом | `summary.json`, `samples.csv`, `n*-run-*.json` в каталоге cost | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +### Подтверждение инфраструктурных воздействий + +Для persistence недостаточно одного PASS. + +В показанной трассе restart-hook выполнялись: + +- остановка API и LB; +- перезапуск PostgreSQL, Redis и MinIO; +- ожидание готовности хранилищ; +- запуск API; +- запуск LB. + +Volumes при этом сохранялись. + +Для сдачи необходимо приложить трассу фактической версии hook и состояние +контейнеров после выполнения. Аналогично для failover необходимо сохранить +идентификатор остановленной API-реплики. + +В тесте отказов reconciler использовались реальные `docker stop` +и `docker start` для контейнеров MinIO и PostgreSQL. Настройки подключений +фиксировались до остановки, поэтому проверялось поведение worker при +недоступной зависимости, а не только ошибка обнаружения контейнера. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Гипотеза + +Повторяемая сверка PostgreSQL и S3 позволяет обнаруживать и безопасно +исправлять расхождения при следующих условиях: + +- свежие объекты защищены grace period; +- удаление проверяет отсутствие метаданных под общей с upload блокировкой; +- отказ хранилища не считается отсутствием объекта; +- потерянный оригинал восстанавливается только из проверенной копии; +- повторный запуск не создаёт дополнительных изменений. + +Для исправного набора данных ожидается примерно линейный рост числа +SQL-вызовов и операций S3 относительно количества вложений. +Время полного запуска дополнительно включает постоянные накладные расходы. + +### Окружение и лимиты + +Эксперименты выполнялись в Windows с Docker Desktop и WSL. + +| Ресурс | Значение | +| --- | --- | +| CPU, доступные Docker | 12 | +| RAM, доступная Docker | 8 171 163 648 байт, около 7,61 GiB | +| API | 2 реплики, каждая 1 CPU и 768 MiB | +| PostgreSQL | 1 CPU, 512 MiB | +| MinIO | 1 CPU, 768 MiB | +| Reconciler | 1 CPU, 256 MiB | + +Это лимиты контейнеров, а не измеренное потребление CPU/RAM. +Фактическая загрузка CPU в эксперименте сверки отдельно не измерялась. + +Image ID reconciler в эксперименте: + +`sha256:1e2c36d99dfe28d7f9eb8f57dc8b5273ec75e20ae88e8427e560b72f4594bbb4` + +Image ID остальных сервисов сохранены в `environment` итогового JSON. + +### Проверка согласованности + +Созданы: + +- корректное вложение, используемое сообщением; +- вложение, у которого намеренно удалён оригинал; +- старый S3-объект без метаданных; +- свежий S3-объект без метаданных. + +Для старого orphan выполнено реальное ожидание 65 секунд при тестовом +grace period 60 секунд. + +Перед каждым этапом создавался отдельный свежий orphan. Это исключило +накопление времени предыдущих запусков Docker в возрасте проверяемого +свежего объекта. + +| Этап | Результат | Время worker, с | SQL | S3 | +| --- | --- | ---: | ---: | ---: | +| Dry-run без копии | Обнаружены orphan и missing; изменений нет | 0,490 | 13 | 6 | +| Apply без копии | Orphan удалён; missing остаётся нерешённым | 0,600 | 13 | 7 | +| Dry-run с копией | Запланировано восстановление | 0,262 | 11 | 5 | +| Apply с копией | Оригинал восстановлен и проверен | 0,368 | 11 | 7 | +| Повторный apply | Оба оригинала существуют; изменений нет | 0,190 | 11 | 4 | + +`needs_attention` без резервной копии является ожидаемым результатом. +Метаданные не удаляются для сокрытия проблемы. + +### Отказы и восстановление | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка API-реплики | Другая реплика продолжает обслуживание | Failover PASS; 74 успешные записи, 2 неудачные попытки | Первая успешная запись через 1,833 с; подтверждённые данные сохранены | +| Restart стенда с volumes | Сохраняются данные и сессия | Persistence PASS | Полное время восстановления отдельно не зафиксировано; потери проверяемых данных нет | +| Обрыв HTTP upload | Удаляется временный файл | Upload-abort PASS; последующие загрузки работают | Точное время отдельно не сохранено | +| SIGKILL принимающего API | Просроченный временный файл удаляется после запуска | Crash-сценарий PASS | Использовано искусственное старое mtime; полное ожидание штатного grace не измерялось | +| Ошибка БД после S3-записи | Orphan удаляется последующей сверкой | Проверка прошла | Интервал очистки тестовый; точное время не сохранено | +| MinIO остановлен перед запуском reconciler | Проход завершается без изменений | `EndpointConnectionError`; только попытка `head_bucket` | Ошибка обнаружена за 0,343 с; данные и сессия сохранены после запуска MinIO | +| PostgreSQL остановлен перед запуском reconciler | S3 не изменяется | `OperationalError`; операций S3 нет | Ошибка обнаружена за 0,160 с; данные и сессия сохранены после запуска PostgreSQL | +| Зависимости восстановлены | Сверка снова выполняется | Удалён один старый orphan | Проход занял 0,196 с | +| Повторная сверка после восстановления | Нет дополнительных изменений | Чистый повторный apply | Проход занял 0,375 с | + +Время обнаружения ошибки worker не является временем полного +восстановления инфраструктуры. + +### Память при конкурентных загрузках + +Профиль: closed-loop, 10 клиентов, без автоматических повторов POST. +Продолжительность — 30,344 с. + +Изображение: PNG 6000×6000, 36 миллионов пикселей, +157 192 байта в сжатом виде. + +| Метрика | Результат | +| --- | ---: | +| Успешные загрузки | 81 | +| Ответы 429 | 661 | +| Неожиданные ошибки | 0 | +| Успешные RPS | 2,669 | +| p50 успешных запросов | 601,77 мс | +| p95 успешных запросов | 2084,50 мс | +| p99 успешных запросов | 4082,42 мс | +| OOM / OOM kill | 0 / 0 | + +| Реплика | Память до теста, MiB | Наблюдаемый пик, MiB | Пик cgroup за жизнь контейнера после теста, MiB | +| --- | ---: | ---: | ---: | +| API 1 | 356,10 | 375,53 | 377,39 | +| API 2 | 361,88 | 378,82 | 379,07 | + +Примерно 89% запросов отклонены механизмом admission control. +Это не успешные загрузки и не скрытые серверные ошибки. + +Десять клиентов не означают десять одновременно работающих декодеров: +число активных загрузок ограничено приложением. + +Память измерялась по cgroup контейнера, а не только Java heap. +Выборочный мониторинг мог пропустить короткие пики. +Показатель lifetime peak включает запуск и предыдущую работу контейнера. + +### Нагрузка текстовых сообщений + +Публичный `make load` выполнял closed-loop POST сообщений без автоматических +повторов: + +| Метрика | Результат | +| --- | ---: | +| Concurrency | 4 | +| Продолжительность | 30,102 с | +| Успешные запросы | 841 | +| Ошибки | 0 | +| Успешные RPS | 27,938 | +| p50 | 92,299 мс | +| p95 | 408,735 мс | +| p99 | 615,139 мс | + +Это измерение текстовых сообщений, а не пропускной способности upload. +Задержки рассчитаны для успешных запросов. +Closed-loop профиль не задаёт фиксированный внешний arrival rate +и имеет ограничение coordinated omission. + +### Методика измерения стоимости обхода + +Для каждого размера создан отдельный чат с 10, 100 или 1000 вложений. + +Использован публичный fixture: + +- RGB PNG 320×200; +- размер 111 817 байт; +- SHA-256: + `99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75`. + +Всего загружено 1110 файлов, около 118,37 MiB оригиналов. +Подготовка данных заняла 95,224 с и не включалась во время обхода. + +После последней загрузки выполнено ожидание 65 секунд. +Фоновая очистка API была отключена. +Для каждого объёма выполнен один прогревочный и три измеряемых прохода. +Порядок объёмов перемешивался с фиксированным seed 1729. + +Все проходы выполнены в dry-run. +В каждом подтверждено: + +- просмотрено ровно N объектов; +- просмотрено ровно N записей метаданных; +- все N объектов зарегистрированы; +- все N оригиналов найдены при обходе метаданных; +- свежих пропущенных объектов и занятых блокировок нет; +- расхождений и операций записи/удаления нет. + +После измерений проверены неизменность метаданных и сообщений, +а также байты первого и последнего вложения каждого набора. + +### Результаты стоимости обхода + +| Вложения | Worker min, с | Worker median, с | Worker max, с | Медиана полного запуска, с | SQL | S3 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| 10 | 0,381 | 0,413 | 0,604 | 9,580 | 53 | 12 | +| 100 | 1,484 | 1,949 | 2,003 | 10,885 | 503 | 102 | +| 1000 | 11,090 | 14,097 | 14,241 | 23,009 | 5004 | 1003 | + +Во всех трёх повторах одного объёма количество вызовов было одинаковым. + +При размере страницы 500 для текущего алгоритма и исправных старых +вложений получены формулы: + +- SQL-вызовы: `5N + ceil(N / 500) + 2`; +- операции S3: `N + ceil(N / 500) + 1`. + +Измерения полностью совпали с этим расчётом. + +На каждый зарегистрированный старый объект приходятся два SQL-вызова +в проходе S3: блокировка и проверка метаданных. + +На каждую запись метаданных приходятся три SQL-вызова: +чтение ключа, блокировка и повторное чтение строки. +Дополнительно учитываются запросы страниц и глобальная блокировка. + +В S3 выполняются проверка bucket, получение страниц списка и один HEAD +на каждое вложение. + +SQL-счётчик не включает служебные BEGIN/COMMIT. +Счётчик S3 отражает вызовы операций SDK, а не измеренный сетевой трафик. + +### Вывод по гипотезе + +Гипотеза о линейном росте количества обращений к хранилищам подтверждена +для исследованного диапазона: результаты соответствуют формулам алгоритма. + +Время worker увеличивается с объёмом данных, но не строго пропорционально: +на него влияют постоянные расходы, кэширование и состояние локального +стенда. Три повтора недостаточны для универсальной модели задержек. + +Дополнительные расходы полного запуска составили примерно +8,882–9,224 с на проход. Они включают запуск Python-скрипта, +обнаружение контейнеров, Docker и запуск worker. + +Для небольших наборов эти расходы превышают время самой сверки. +Для 1000 вложений медиана работы worker составила 14,097 с, +а полного запуска — 23,009 с. + -## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. diff --git a/compose.reconcile-test.yml b/compose.reconcile-test.yml new file mode 100644 index 00000000..c0542491 --- /dev/null +++ b/compose.reconcile-test.yml @@ -0,0 +1,4 @@ +services: + api: + environment: + APP_CLEANUP_ENABLED: "false" \ No newline at end of file diff --git a/compose.yml b/compose.yml new file mode 100644 index 00000000..6a62b323 --- /dev/null +++ b/compose.yml @@ -0,0 +1,216 @@ +services: + postgres: + image: postgres:16.6-alpine + environment: + POSTGRES_DB: ${POSTGRES_DB:?} + POSTGRES_USER: ${POSTGRES_USER:?} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?} + command: + - postgres + - -c + - max_connections=100 + - -c + - shared_buffers=128MB + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: + - CMD-SHELL + - pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" + interval: 2s + timeout: 1s + retries: 30 + + cpus: 1.0 + mem_limit: 512m + networks: [data] + + redis: + image: redis:7.4.1-alpine + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD:?} + command: + - sh + - -c + - | + exec redis-server \ + --requirepass "$$REDIS_PASSWORD" \ + --appendonly yes \ + --appendfsync always \ + --maxmemory 128mb \ + --maxmemory-policy noeviction + volumes: + - redis_data:/data + healthcheck: + test: + - CMD-SHELL + - REDISCLI_AUTH="$$REDIS_PASSWORD" redis-cli ping | grep -q PONG + interval: 2s + timeout: 1s + retries: 30 + cpus: 0.5 + mem_limit: 192m + networks: [data] + + minio: + build: + context: ./infra/minio + dockerfile: Dockerfile + + image: mtusi4-minio:2025-04-22-local + pull_policy: build + + command: + - server + - /data + - --console-address + - ":9001" + + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?} + MINIO_API_STALE_UPLOADS_EXPIRY: "24h" + MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL: "1h" + + volumes: + - minio_data:/data + + ports: + - "127.0.0.1:9001:9001" + + healthcheck: + test: + - CMD + - curl + - -f + - http://localhost:9000/minio/health/live + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + + cpus: 1.0 + mem_limit: 768m + restart: unless-stopped + networks: [data] + + minio-init: + build: + context: ./infra/minio + dockerfile: Dockerfile + + image: mtusi4-minio:2025-04-22-local + pull_policy: build + + depends_on: + minio: + condition: service_healthy + + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?} + + S3_BUCKET: ${S3_BUCKET:?} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?} + S3_SECRET_KEY: ${S3_SECRET_KEY:?} + + volumes: + - ./infra/minio/bootstrap.sh:/bootstrap/bootstrap.sh:ro + - ./infra/minio/check-access.sh:/bootstrap/check-access.sh:ro + + entrypoint: + - /bin/sh + - /bootstrap/bootstrap.sh + + restart: "no" + networks: [data] + + migrate: + image: flyway/flyway:10.20.1-alpine + command: + - -connectRetries=30 + - migrate + environment: + FLYWAY_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:?} + FLYWAY_USER: ${POSTGRES_USER:?} + FLYWAY_PASSWORD: ${POSTGRES_PASSWORD:?} + volumes: + - ./java/src/main/resources/db/migration:/flyway/sql:ro + depends_on: + postgres: + condition: service_healthy + cpus: 0.5 + mem_limit: 384m + restart: "no" + networks: [data] + + api: + build: . + image: ${APP_IMAGE:-mtusi4-api:local} + environment: + DB_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:?} + DB_USERNAME: ${POSTGRES_USER:?} + DB_PASSWORD: ${POSTGRES_PASSWORD:?} + REDIS_HOST: redis + REDIS_PASSWORD: ${REDIS_PASSWORD:?} + DB_POOL_SIZE: ${DB_POOL_SIZE:-8} + FLYWAY_ENABLED: "false" + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?} + S3_SECRET_KEY: ${S3_SECRET_KEY:?} + S3_BUCKET: ${S3_BUCKET:?} + UPLOAD_MAX_CONCURRENT: ${UPLOAD_MAX_CONCURRENT:-1} + UPLOAD_TEMP_DIR: /tmp/messenger-uploads + ATTACHMENT_CLEANUP_ENABLED: "true" + ATTACHMENT_CLEANUP_GRACE_SECONDS: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:-3600} + ATTACHMENT_CLEANUP_INTERVAL_MS: ${ATTACHMENT_CLEANUP_INTERVAL_MS:-300000} + ATTACHMENT_CLEANUP_INITIAL_DELAY_MS: ${ATTACHMENT_CLEANUP_INITIAL_DELAY_MS:-60000} + + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + minio-init: + condition: service_completed_successfully + deploy: + replicas: ${API_REPLICAS:-2} + cpus: ${API_CPUS:-1.0} + mem_limit: ${API_MEMORY:-768m} + stop_grace_period: 20s + + healthcheck: + test: + - CMD + - wget + - -qO- + - http://localhost:8080/health/ready + interval: 5s + timeout: 15s + start_period: 60s + retries: 3 + networks: [edge, data] + + lb: + image: haproxy:3.0.5-alpine + ports: + - "127.0.0.1:${API_PORT:-8080}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + + cpus: 0.5 + mem_limit: 128m + networks: [edge] + +volumes: + postgres_data: + redis_data: + minio_data: + +networks: + edge: + data: + internal: true \ No newline at end of file diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..c080ca7f --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,33 @@ +global + log stdout format raw local0 + maxconn 2048 +defaults + mode http + log global + option httplog + timeout connect 1s + timeout client 12s + timeout server 10s + timeout check 1s + timeout queue 2s + retries 0 +resolvers docker + nameserver docker 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold refused 1s + hold timeout 1s +frontend public + bind :8080 + http-request deny if { path_beg /actuator } + default_backend api +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + server-template node 1-8 api:8080 check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none diff --git a/infra/minio/Dockerfile b/infra/minio/Dockerfile new file mode 100644 index 00000000..0cab71e9 --- /dev/null +++ b/infra/minio/Dockerfile @@ -0,0 +1,41 @@ +FROM golang:1.24.2-bookworm AS builder + +ENV CGO_ENABLED=0 + +WORKDIR /src + +RUN git clone --depth 1 \ + --branch RELEASE.2025-04-22T22-12-26Z \ + https://github.com/minio/minio.git /src/minio + +WORKDIR /src/minio + +RUN go build -trimpath -o /out/minio . + +WORKDIR /src + +RUN git clone --depth 1 \ + --branch RELEASE.2025-04-16T18-13-26Z \ + https://github.com/minio/mc.git /src/mc + +WORKDIR /src/mc + +RUN go build -trimpath -o /out/mc . + + +FROM debian:bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /out/minio /usr/local/bin/minio +COPY --from=builder /out/mc /usr/local/bin/mc + +RUN mkdir -p /data + +EXPOSE 9000 9001 + +ENTRYPOINT ["minio"] +CMD ["server", "/data", "--console-address", ":9001"] \ No newline at end of file diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh new file mode 100644 index 00000000..b904eb29 --- /dev/null +++ b/infra/minio/bootstrap.sh @@ -0,0 +1,93 @@ +#!/bin/sh +set -eu + +: "${MINIO_ROOT_USER:?}" +: "${MINIO_ROOT_PASSWORD:?}" +: "${S3_BUCKET:?}" +: "${S3_ACCESS_KEY:?}" +: "${S3_SECRET_KEY:?}" + +if [ "$S3_ACCESS_KEY" = "$MINIO_ROOT_USER" ]; then + echo "API must use a separate MinIO user" >&2 + exit 1 +fi + +if [ "$S3_SECRET_KEY" = "$MINIO_ROOT_PASSWORD" ]; then + echo "API and administrator passwords must differ" >&2 + exit 1 +fi + +case "$S3_BUCKET" in + ""|*[!a-z0-9.-]*) + echo "Invalid bucket name" >&2 + exit 1 + ;; +esac + +mc alias set storage http://minio:9000 \ + "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null + +mc mb --ignore-existing "storage/$S3_BUCKET" + +mc anonymous set none "storage/$S3_BUCKET" + +cat > /tmp/attachments-policy.json </dev/null + +mc admin user info --json \ + storage "$S3_ACCESS_KEY" > /tmp/api-user.json + +if grep -Eq \ + '"policyName"[[:space:]]*:[[:space:]]*"messenger-attachments-api"' \ + /tmp/api-user.json +then + echo "API policy is already attached" +else + mc admin policy attach \ + storage "$POLICY_NAME" --user "$S3_ACCESS_KEY" +fi + +mc alias set app http://minio:9000 \ + "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null + +mc stat "app/$S3_BUCKET" >/dev/null + +echo "Bootstrap completed: private bucket and API user are ready" \ No newline at end of file diff --git a/infra/minio/check-access.sh b/infra/minio/check-access.sh new file mode 100644 index 00000000..324c51cb --- /dev/null +++ b/infra/minio/check-access.sh @@ -0,0 +1,98 @@ +#!/bin/sh +set -eu + +: "${S3_BUCKET:?}" +: "${S3_ACCESS_KEY:?}" +: "${S3_SECRET_KEY:?}" + +mc alias set app http://minio:9000 \ + "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null + +TEST_ID="$(cat /proc/sys/kernel/random/uuid)" +OBJECT_KEY="originals/access-check-$TEST_ID.txt" +OUTSIDE_KEY="outside-access-check-$TEST_ID.txt" + +cleanup() { + mc rm --force \ + "app/$S3_BUCKET/$OBJECT_KEY" >/dev/null 2>&1 || true + + mc rm --force \ + "app/$S3_BUCKET/$OUTSIDE_KEY" >/dev/null 2>&1 || true +} + +trap cleanup EXIT + +printf 'private attachment test' > /tmp/expected.txt + +mc cp /tmp/expected.txt \ + "app/$S3_BUCKET/$OBJECT_KEY" >/dev/null + +mc cat "app/$S3_BUCKET/$OBJECT_KEY" > /tmp/actual.txt + +cmp /tmp/expected.txt /tmp/actual.txt + +echo "PASS: API credentials can upload and download originals" + +HTTP_CODE="$( + curl --silent --show-error \ + --connect-timeout 3 \ + --max-time 10 \ + --output /tmp/anonymous-response.xml \ + --write-out '%{http_code}' \ + "http://minio:9000/$S3_BUCKET/$OBJECT_KEY" +)" + +if [ "$HTTP_CODE" != "403" ]; then + echo "FAIL: anonymous GET returned HTTP $HTTP_CODE" >&2 + exit 1 +fi + +if ! grep -q 'AccessDenied' \ + /tmp/anonymous-response.xml +then + echo "FAIL: expected S3 AccessDenied response" >&2 + exit 1 +fi + +echo "PASS: anonymous download of an existing object is denied" + +if mc cp /tmp/expected.txt \ + "app/$S3_BUCKET/$OUTSIDE_KEY" \ + > /tmp/outside-result.txt 2>&1 +then + echo "FAIL: API credentials can write outside originals/" >&2 + exit 1 +fi + +if ! grep -Eiq \ + 'AccessDenied|Access Denied|Insufficient permissions|not authorized|not allowed' \ + /tmp/outside-result.txt +then + echo "FAIL: write failed for a reason other than access denial" >&2 + cat /tmp/outside-result.txt >&2 + exit 1 +fi + +echo "PASS: writing outside originals/ is denied" + +if mc admin info app > /tmp/admin-result.txt 2>&1 +then + echo "FAIL: API credentials have administrative access" >&2 + exit 1 +fi + +if ! grep -Eiq \ + 'AccessDenied|Access Denied|Insufficient permissions|not authorized|not allowed' \ + /tmp/admin-result.txt +then + echo "FAIL: admin command failed for an unexpected reason" >&2 + cat /tmp/admin-result.txt >&2 + exit 1 +fi + +echo "PASS: administrative access is denied" + +mc rm "app/$S3_BUCKET/$OBJECT_KEY" >/dev/null + +echo "PASS: API credentials can delete originals for cleanup" +echo "PASS: S3 access checks completed" \ No newline at end of file diff --git a/infra/reconciler/Dockerfile b/infra/reconciler/Dockerfile new file mode 100644 index 00000000..d1cf26a8 --- /dev/null +++ b/infra/reconciler/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +RUN pip install --no-cache-dir \ + boto3==1.35.99 \ + "psycopg[binary]==3.2.4" + +COPY reconcile_worker.py /app/reconcile_worker.py + +USER 65534:65534 + +ENTRYPOINT ["python", "/app/reconcile_worker.py"] \ No newline at end of file diff --git a/infra/reconciler/reconcile_worker.py b/infra/reconciler/reconcile_worker.py new file mode 100644 index 00000000..d117c4c7 --- /dev/null +++ b/infra/reconciler/reconcile_worker.py @@ -0,0 +1,454 @@ +import base64 +from collections import Counter +from datetime import datetime, timedelta, timezone +import hashlib +import json +from pathlib import Path +import re +import sys +import time + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError +import psycopg +from psycopg.rows import dict_row + + +MAX_BYTES = 10 * 1024 * 1024 + + +class Reconciler: + def __init__(self, config): + self.config = config + self.started = time.monotonic() + + self.cutoff = ( + datetime.now(timezone.utc) + - timedelta(seconds=config["grace_seconds"]) + ) + + self.counts = Counter() + self.sql_calls = 0 + self.s3_calls = Counter() + self.examples = [] + self.db = None + + self.s3 = boto3.client( + "s3", + **config["s3"], + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=3, + read_timeout=15, + retries={"total_max_attempts": 1}, + ), + ) + + def sql(self, statement, params=()): + self.sql_calls += 1 + return self.db.execute(statement, params) + + def call(self, operation, **kwargs): + self.s3_calls[operation] += 1 + + return getattr(self.s3, operation)( + Bucket=self.config["bucket"], + **kwargs, + ) + + def event(self, name, key=None): + self.counts[name] += 1 + + # Ограничиваем размер подробной части отчёта. + if key is not None and len(self.examples) < 100: + self.examples.append({ + "outcome": name, + "object_key": key, + }) + + def deadline(self): + elapsed = time.monotonic() - self.started + + if elapsed > self.config["max_seconds"]: + raise TimeoutError("Scan deadline reached") + + def lock(self, key): + # Тот же ключ блокировки, что используется + # AttachmentRepository в Java. + row = self.sql( + """ + SELECT pg_try_advisory_xact_lock( + hashtextextended(%s, 0) + ) AS ok + """, + ("attachment-object:" + key,), + ).fetchone() + + if not row["ok"]: + self.event("busy", key) + + return row["ok"] + + def head(self, key): + try: + return self.call("head_object", Key=key) + + except ClientError as error: + code = error.response.get("Error", {}).get("Code", "") + + if code not in ( + "404", + "NoSuchKey", + "NoSuchObject", + "NotFound", + ): + raise + + # Проверяем, что доступен сам bucket. + # Исчезновение bucket не считаем потерей + # отдельного оригинала. + self.call("head_bucket") + + return None + + def scan_objects(self): + token = None + chat_id = self.config.get("chat_id") + + prefix = ( + f"originals/{chat_id}/" + if chat_id + else "originals/" + ) + + while True: + self.deadline() + + args = { + "Prefix": prefix, + "MaxKeys": 500, + } + + if token: + args["ContinuationToken"] = token + + page = self.call("list_objects_v2", **args) + + for item in page.get("Contents", []): + self.deadline() + self.event("objects_scanned") + + key = item["Key"] + + if item["LastModified"] >= self.cutoff: + self.event("fresh_objects") + continue + + with self.db.transaction(): + if not self.lock(key): + continue + + # Проверяем всю таблицу, даже если проход + # ограничен одним тестовым чатом. + row = self.sql( + """ + SELECT id + FROM attachments + WHERE object_key = %s + """, + (key,), + ).fetchone() + + if row: + self.event("referenced_objects") + continue + + # Повторная проверка после блокировки. + current = self.head(key) + + if current is None: + self.event("already_absent") + + elif current["LastModified"] >= self.cutoff: + self.event("fresh_objects") + + elif self.config["apply"]: + self.call("delete_object", Key=key) + self.event("deleted_orphans", key) + + else: + self.event("would_delete_orphans", key) + + if not page.get("IsTruncated"): + break + + token = page["NextContinuationToken"] + + def backup(self, row): + checksum = row["sha256"].strip() + + if not re.fullmatch("[a-f0-9]{64}", checksum): + raise ValueError("Invalid metadata checksum") + + # Имя копии определяется SHA-256, а не filename. + path = Path("/backups") / (checksum + ".bin") + + if not path.exists(): + return None + + # Ограничиваем чтение даже для ошибочной копии. + with path.open("rb") as stream: + data = stream.read(MAX_BYTES + 1) + + valid = ( + len(data) == row["size_bytes"] + and hashlib.sha256(data).hexdigest() == checksum + ) + + if not valid: + self.event("invalid_backups", row["object_key"]) + return None + + return data + + def check_attachment(self, attachment_id): + # В текущем протоколе object_key неизменяем. + initial = self.sql( + """ + SELECT object_key + FROM attachments + WHERE id = %s + """, + (attachment_id,), + ).fetchone() + + if not initial: + return + + key = initial["object_key"] + + with self.db.transaction(): + if not self.lock(key): + return + + row = self.sql( + """ + SELECT * + FROM attachments + WHERE id = %s + FOR SHARE + """, + (attachment_id,), + ).fetchone() + + if not row: + return + + if ( + row["object_key"] != key + or not key.startswith("originals/") + ): + raise ValueError("Unsupported object key") + + if row["created_at"] >= self.cutoff: + self.event("fresh_metadata") + return + + current = self.head(key) + + if current is not None: + if current["ContentLength"] == row["size_bytes"]: + self.event("present_metadata") + else: + self.event("size_mismatches", key) + + return + + self.event("missing_objects", key) + + data = self.backup(row) + + if data is None: + # Метаданные и сообщения сохраняем. + self.event("unresolved_missing", key) + return + + if not self.config["apply"]: + self.event("would_restore", key) + return + + # Восстанавливаем отсутствующий объект, + # удерживая ту же блокировку, что и API. + self.call( + "put_object", + Key=key, + Body=data, + ContentType=row["content_type"], + ContentMD5=base64.b64encode( + hashlib.md5(data).digest() + ).decode("ascii"), + ) + + # Проверяем фактические байты после записи. + response = self.call("get_object", Key=key) + + with response["Body"] as stream: + actual = stream.read(MAX_BYTES + 1) + + valid = ( + len(actual) == row["size_bytes"] + and hashlib.sha256(actual).hexdigest() + == row["sha256"].strip() + ) + + if not valid: + raise ValueError( + "Restored object verification failed" + ) + + self.event("restored", key) + + def scan_metadata(self): + cursor = None + chat_id = self.config.get("chat_id") + + while True: + self.deadline() + + rows = self.sql( + """ + SELECT id + FROM attachments + WHERE ( + %s::uuid IS NULL + OR chat_id = %s::uuid + ) + AND ( + %s::uuid IS NULL + OR id > %s::uuid + ) + ORDER BY id + LIMIT 500 + """, + (chat_id, chat_id, cursor, cursor), + ).fetchall() + + if not rows: + return + + for row in rows: + self.deadline() + self.event("metadata_scanned") + self.check_attachment(row["id"]) + + cursor = rows[-1]["id"] + + def run(self): + status = "completed" + error_type = None + + try: + with psycopg.connect( + **self.config["db"], + autocommit=True, + row_factory=dict_row, + connect_timeout=5, + options=( + "-c statement_timeout=10000 " + "-c idle_in_transaction_session_timeout=120000" + ), + ) as db: + self.db = db + + # Только один полный проход этой утилиты + # одновременно в данной БД. + locked = self.sql( + """ + SELECT pg_try_advisory_lock( + hashtextextended( + 'attachment-reconciler:v1', + 0 + ) + ) AS ok + """ + ).fetchone()["ok"] + + if not locked: + raise RuntimeError( + "Another reconciler is running" + ) + + self.call("head_bucket") + + self.scan_objects() + self.scan_metadata() + + # Закрытие соединения освобождает session lock. + # Блокировки объектов освобождаются + # при завершении отдельных транзакций. + + except Exception as error: + status = "failed" + error_type = type(error).__name__ + + finally: + self.s3.close() + + attention_counters = ( + "unresolved_missing", + "size_mismatches", + "invalid_backups", + "busy", + ) + + if status == "completed" and any( + self.counts[name] for name in attention_counters + ): + status = "needs_attention" + + report = { + "status": status, + "mode": ( + "apply" + if self.config["apply"] + else "dry-run" + ), + "chat_id": self.config.get("chat_id"), + "cutoff": self.cutoff.isoformat(), + "elapsed_seconds": round( + time.monotonic() - self.started, + 3, + ), + "counts": dict(self.counts), + "sql_statement_calls": self.sql_calls, + "s3_operation_calls": dict(self.s3_calls), + "examples": self.examples, + "examples_limit": 100, + "error_type": error_type, + } + + print(json.dumps( + report, + ensure_ascii=False, + indent=2, + )) + + return { + "completed": 0, + "needs_attention": 2, + "failed": 1, + }[status] + + +if __name__ == "__main__": + config = json.load(sys.stdin) + + if ( + config["grace_seconds"] < 60 + or config["max_seconds"] < 1 + ): + raise SystemExit("Invalid scan limits") + + raise SystemExit(Reconciler(config).run()) \ No newline at end of file diff --git a/java/.gitignore b/java/.gitignore new file mode 100644 index 00000000..667aaef0 --- /dev/null +++ b/java/.gitignore @@ -0,0 +1,33 @@ +HELP.md +target/ +.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ diff --git a/java/.mvn/wrapper/maven-wrapper.properties b/java/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000..216df058 --- /dev/null +++ b/java/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,3 @@ +wrapperVersion=3.3.4 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.16/apache-maven-3.9.16-bin.zip diff --git a/java/mvnw b/java/mvnw new file mode 100644 index 00000000..bd8896bf --- /dev/null +++ b/java/mvnw @@ -0,0 +1,295 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.4 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +scriptDir="$(dirname "$0")" +scriptName="$(basename "$0")" + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +actualDistributionDir="" + +# First try the expected directory name (for regular distributions) +if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then + if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then + actualDistributionDir="$distributionUrlNameMain" + fi +fi + +# If not found, search for any directory with the Maven executable (for snapshots) +if [ -z "$actualDistributionDir" ]; then + # enable globbing to iterate over items + set +f + for dir in "$TMP_DOWNLOAD_DIR"/*; do + if [ -d "$dir" ]; then + if [ -f "$dir/bin/$MVN_CMD" ]; then + actualDistributionDir="$(basename "$dir")" + break + fi + fi + done + set -f +fi + +if [ -z "$actualDistributionDir" ]; then + verbose "Contents of $TMP_DOWNLOAD_DIR:" + verbose "$(ls -la "$TMP_DOWNLOAD_DIR")" + die "Could not find Maven distribution directory in extracted archive" +fi + +verbose "Found extracted Maven distribution directory: $actualDistributionDir" +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/java/mvnw.cmd b/java/mvnw.cmd new file mode 100644 index 00000000..92450f93 --- /dev/null +++ b/java/mvnw.cmd @@ -0,0 +1,189 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.4 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' + +$MAVEN_M2_PATH = "$HOME/.m2" +if ($env:MAVEN_USER_HOME) { + $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME" +} + +if (-not (Test-Path -Path $MAVEN_M2_PATH)) { + New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null +} + +$MAVEN_WRAPPER_DISTS = $null +if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) { + $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists" +} else { + $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists" +} + +$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain" +$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +$actualDistributionDir = "" + +# First try the expected directory name (for regular distributions) +$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain" +$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD" +if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) { + $actualDistributionDir = $distributionUrlNameMain +} + +# If not found, search for any directory with the Maven executable (for snapshots) +if (!$actualDistributionDir) { + Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object { + $testPath = Join-Path $_.FullName "bin/$MVN_CMD" + if (Test-Path -Path $testPath -PathType Leaf) { + $actualDistributionDir = $_.Name + } + } +} + +if (!$actualDistributionDir) { + Write-Error "Could not find Maven distribution directory in extracted archive" +} + +Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir" +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/java/pom.xml b/java/pom.xml new file mode 100644 index 00000000..cce1d2ba --- /dev/null +++ b/java/pom.xml @@ -0,0 +1,128 @@ + + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 3.3.5 + + + + com.vnp + demo + 0.0.1-SNAPSHOT + mtusi-messenger + + + 21 + 21 + + + + + + + org.springframework.boot + spring-boot-starter-web + + + + + org.springframework.boot + spring-boot-starter-validation + + + + + org.springframework.boot + spring-boot-starter-actuator + + + + + org.springframework.boot + spring-boot-starter-data-jpa + + + + + org.postgresql + postgresql + runtime + + + + org.springframework.boot + spring-boot-starter-data-redis + + + + + org.flywaydb + flyway-core + + + + org.flywaydb + flyway-database-postgresql + + + + + org.springframework.boot + spring-boot-starter-security + + + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.bouncycastle + bcprov-jdk18on + 1.78.1 + + + + + io.rest-assured + rest-assured + test + + + + io.minio + minio + 8.5.17 + + + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + + + + + + + diff --git a/java/src/main/java/com/vnp/demo/BusinessException.java b/java/src/main/java/com/vnp/demo/BusinessException.java new file mode 100644 index 00000000..d1d4a1fb --- /dev/null +++ b/java/src/main/java/com/vnp/demo/BusinessException.java @@ -0,0 +1,63 @@ +package com.vnp.demo; + +import org.springframework.http.HttpStatus; + +public class BusinessException extends RuntimeException { + + private final String code; + private final HttpStatus status; + + public BusinessException(String code, String message, HttpStatus status) { + super(message); + this.code = code; + this.status = status; + } + + public String getCode() { + return code; + } + + public HttpStatus getStatus() { + return status; + } + + public static BusinessException notFound() { + return new BusinessException( + "not_found", + "Объект не найден", + HttpStatus.NOT_FOUND + ); + } + + public static BusinessException conflict() { + return conflict("Конфликт"); + } + + public static BusinessException conflict(String message) { + return new BusinessException( + "conflict", + message, + HttpStatus.CONFLICT + ); + } + + public static BusinessException unauthorized() { + return unauthorized("Пользователь не найден"); + } + + public static BusinessException unauthorized(String message) { + return new BusinessException( + "unauthorized", + message, + HttpStatus.UNAUTHORIZED + ); + } + + public static BusinessException invalid(String message) { + return new BusinessException( + "invalid_request", + message, + HttpStatus.BAD_REQUEST + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/CursorCodec.java b/java/src/main/java/com/vnp/demo/CursorCodec.java new file mode 100644 index 00000000..aa3e1f9d --- /dev/null +++ b/java/src/main/java/com/vnp/demo/CursorCodec.java @@ -0,0 +1,184 @@ +package com.vnp.demo; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.format.DateTimeParseException; +import java.util.Base64; +import java.util.UUID; + +import org.springframework.stereotype.Component; + +@Component +public class CursorCodec { + + public record ChatCursor(Instant createdAt, UUID id) { + } + + public record MessageCursor(Instant createdAt, UUID id) { + } + + public String encodeIndex(int index) { + return Base64.getUrlEncoder() + .withoutPadding() + .encodeToString( + ("i:" + index).getBytes(StandardCharsets.UTF_8) + ); + } + + public int decodeIndex(String cursor, int size) { + try { + String decoded = new String( + Base64.getUrlDecoder().decode(cursor), + StandardCharsets.UTF_8 + ); + + if (!decoded.startsWith("i:")) { + throw new IllegalArgumentException(); + } + + int idx = Integer.parseInt(decoded.substring(2)); + + if (idx < 0 || idx > size) { + throw new IllegalArgumentException(); + } + + return idx; + + } catch (IllegalArgumentException ex) { + throw BusinessException.invalid("Неверный cursor"); + } + } + + public String encodeChat( + UUID userId, + Instant createdAt, + UUID chatId + ) { + String value = "chat:v1:" + + userId + ":" + + createdAt + ":" + + chatId; + + return Base64.getUrlEncoder() + .withoutPadding() + .encodeToString( + value.getBytes(StandardCharsets.UTF_8) + ); + } + + public ChatCursor decodeChat(String cursor, UUID userId) { + try { + String value = decodeBase64(cursor); + + if (!value.startsWith("chat:v1:")) { + throw new IllegalArgumentException(); + } + + String content = value.substring("chat:v1:".length()); + + int firstSeparator = content.indexOf(':'); + int lastSeparator = content.lastIndexOf(':'); + + if (firstSeparator < 0 + || lastSeparator <= firstSeparator) { + throw new IllegalArgumentException(); + } + + UUID cursorUserId = UUID.fromString( + content.substring(0, firstSeparator) + ); + + if (!cursorUserId.equals(userId)) { + throw new IllegalArgumentException(); + } + + Instant createdAt = Instant.parse( + content.substring( + firstSeparator + 1, + lastSeparator + ) + ); + + UUID chatId = UUID.fromString( + content.substring(lastSeparator + 1) + ); + + return new ChatCursor(createdAt, chatId); + + } catch (IllegalArgumentException | DateTimeParseException ex) { + throw BusinessException.invalid("Неверный cursor"); + } + } + + public String encodeMessage( + UUID chatId, + Instant createdAt, + UUID messageId + ) { + String value = "message:v1:" + + chatId + ":" + + createdAt + ":" + + messageId; + + return Base64.getUrlEncoder() + .withoutPadding() + .encodeToString( + value.getBytes(StandardCharsets.UTF_8) + ); + } + + public MessageCursor decodeMessage(String cursor, UUID chatId) { + try { + String value = decodeBase64(cursor); + + if (!value.startsWith("message:v1:")) { + throw new IllegalArgumentException(); + } + + String content = value.substring("message:v1:".length()); + + int firstSeparator = content.indexOf(':'); + int lastSeparator = content.lastIndexOf(':'); + + if (firstSeparator < 0 + || lastSeparator <= firstSeparator) { + throw new IllegalArgumentException(); + } + + UUID cursorChatId = UUID.fromString( + content.substring(0, firstSeparator) + ); + + if (!cursorChatId.equals(chatId)) { + throw new IllegalArgumentException(); + } + + Instant createdAt = Instant.parse( + content.substring( + firstSeparator + 1, + lastSeparator + ) + ); + + UUID messageId = UUID.fromString( + content.substring(lastSeparator + 1) + ); + + return new MessageCursor(createdAt, messageId); + + } catch (IllegalArgumentException | DateTimeParseException ex) { + throw BusinessException.invalid("Неверный cursor"); + } + } + + private String decodeBase64(String cursor) { + if (cursor == null || cursor.isBlank()) { + throw new IllegalArgumentException(); + } + + return new String( + Base64.getUrlDecoder().decode(cursor), + StandardCharsets.UTF_8 + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/GlobalExceptionHandler.java b/java/src/main/java/com/vnp/demo/GlobalExceptionHandler.java new file mode 100644 index 00000000..a6f49115 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/GlobalExceptionHandler.java @@ -0,0 +1,150 @@ + +package com.vnp.demo; + +import org.slf4j.MDC; +import org.springframework.dao.DataAccessException; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.MissingRequestHeaderException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; +import org.springframework.web.server.ResponseStatusException; +import org.springframework.web.servlet.resource.NoResourceFoundException; + +import com.vnp.demo.dto.ErrorResponse; + +import jakarta.servlet.http.HttpServletRequest; + +@RestControllerAdvice +public class GlobalExceptionHandler { + + private ErrorResponse build(String code, String msg) { + return new ErrorResponse( + new ErrorResponse.ErrorBody( + code, + msg, + MDC.get("requestId") + ) + ); + } + + @ExceptionHandler(BusinessException.class) + public ResponseEntity handleBusiness( + BusinessException e, + HttpServletRequest request + ) { + ResponseEntity.BodyBuilder builder = + ResponseEntity.status(e.getStatus()); + + if (e.getStatus() == HttpStatus.TOO_MANY_REQUESTS) { + builder.header("Retry-After", "1"); + } + + if (e.getStatus() == HttpStatus.UNAUTHORIZED) { + String path = request.getRequestURI(); + + String challenge = path.equals("/api/v1/auth/sessions") + ? "Basic realm=\"messenger\", charset=\"UTF-8\"" + : "Bearer realm=\"messenger\""; + + builder.header("WWW-Authenticate", challenge); + } + + return builder.body( + build(e.getCode(), e.getMessage()) + ); + } + + @ExceptionHandler(MethodArgumentNotValidException.class) + public ResponseEntity handleValidation( + MethodArgumentNotValidException e + ) { + String msg = e.getBindingResult() + .getFieldErrors() + .stream() + .findFirst() + .map(f -> f.getField() + ": " + f.getDefaultMessage()) + .orElse("Ошибка валидации"); + + return ResponseEntity.badRequest() + .body(build("invalid_request", msg)); + } + + @ExceptionHandler({ + HttpMessageNotReadableException.class, + MethodArgumentTypeMismatchException.class + }) + public ResponseEntity handleMalformed(Exception e) { + return ResponseEntity.badRequest() + .body(build( + "invalid_request", + "Неверный формат запроса" + )); + } + + @ExceptionHandler(NoResourceFoundException.class) + public ResponseEntity handleNotFound( + NoResourceFoundException e + ) { + return ResponseEntity.status(HttpStatus.NOT_FOUND) + .body(build( + "not_found", + "Ресурс не найден" + )); + } + + @ExceptionHandler(MissingRequestHeaderException.class) + public ResponseEntity handleMissingHeader( + MissingRequestHeaderException e + ) { + return ResponseEntity.badRequest() + .body(build( + "invalid_request", + "Отсутствует обязательный заголовок: " + + e.getHeaderName() + )); + } + + @ExceptionHandler(DataAccessException.class) + public ResponseEntity handleDataAccess( + DataAccessException e + ) { + return ResponseEntity + .status(HttpStatus.SERVICE_UNAVAILABLE) + .body(build( + "unavailable", + "Сервис временно недоступен" + )); + } + + @ExceptionHandler(ResponseStatusException.class) + public ResponseEntity handleStatus( + ResponseStatusException e + ) { + int status = e.getStatusCode().value(); + + String code = status == 503 + ? "unavailable" + : "request_failed"; + + String message = status == 503 + ? "Сервис временно недоступен" + : "Запрос не может быть выполнен"; + + return ResponseEntity.status(status) + .body(build(code, message)); + } + + @ExceptionHandler(Exception.class) + public ResponseEntity handleGeneric(Exception e) { + return ResponseEntity + .status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(build( + "internal_error", + "Внутренняя ошибка" + )); + } +} diff --git a/java/src/main/java/com/vnp/demo/MessengerApplication.java b/java/src/main/java/com/vnp/demo/MessengerApplication.java new file mode 100644 index 00000000..482bc6e0 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/MessengerApplication.java @@ -0,0 +1,13 @@ +package com.vnp.demo; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class MessengerApplication { + + public static void main(String[] args) { + SpringApplication.run(MessengerApplication.class, args); + } + +} diff --git a/java/src/main/java/com/vnp/demo/SessionStoreExceptionHandler.java b/java/src/main/java/com/vnp/demo/SessionStoreExceptionHandler.java new file mode 100644 index 00000000..5f98d7ba --- /dev/null +++ b/java/src/main/java/com/vnp/demo/SessionStoreExceptionHandler.java @@ -0,0 +1,24 @@ +package com.vnp.demo; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +import java.util.Map; + +@RestControllerAdvice +public class SessionStoreExceptionHandler { + + @ExceptionHandler(SessionStoreUnavailableException.class) + public ResponseEntity> handleSessionStoreUnavailable( + SessionStoreUnavailableException ex + ) { + return ResponseEntity + .status(HttpStatus.SERVICE_UNAVAILABLE) + .body(Map.of( + "error", "SERVICE_UNAVAILABLE", + "message", "Session storage is temporarily unavailable" + )); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/SessionStoreUnavailableException.java b/java/src/main/java/com/vnp/demo/SessionStoreUnavailableException.java new file mode 100644 index 00000000..818fe376 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/SessionStoreUnavailableException.java @@ -0,0 +1,8 @@ +package com.vnp.demo; + +public class SessionStoreUnavailableException extends RuntimeException { + + public SessionStoreUnavailableException(Throwable cause) { + super("Session storage is temporarily unavailable", cause); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/config/JacksonConfig.java b/java/src/main/java/com/vnp/demo/config/JacksonConfig.java new file mode 100644 index 00000000..53920141 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/config/JacksonConfig.java @@ -0,0 +1,22 @@ +package main.java.com.vnp.demo.config; + +import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.fasterxml.jackson.databind.cfg.CoercionAction; +import com.fasterxml.jackson.databind.cfg.CoercionInputShape; +import com.fasterxml.jackson.databind.type.LogicalType; + +@Configuration +public class JacksonConfig { + @Bean + public Jackson2ObjectMapperBuilderCustomizer strictTextValues() { + return builder -> builder.postConfigurer(mapper -> + mapper.coercionConfigFor(LogicalType.Textual) + .setCoercion(CoercionInputShape.Integer, CoercionAction.Fail) + .setCoercion(CoercionInputShape.Float, CoercionAction.Fail) + .setCoercion(CoercionInputShape.Boolean, CoercionAction.Fail)); + } +} + diff --git a/java/src/main/java/com/vnp/demo/config/PasswordConfig.java b/java/src/main/java/com/vnp/demo/config/PasswordConfig.java new file mode 100644 index 00000000..7cc12a89 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/config/PasswordConfig.java @@ -0,0 +1,15 @@ +package com.vnp.demo.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.crypto.argon2.Argon2PasswordEncoder; +import org.springframework.security.crypto.password.PasswordEncoder; + +@Configuration +public class PasswordConfig { + + @Bean + public PasswordEncoder passwordEncoder() { + return Argon2PasswordEncoder.defaultsForSpringSecurity_v5_8(); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/config/SchedulingConfig.java b/java/src/main/java/com/vnp/demo/config/SchedulingConfig.java new file mode 100644 index 00000000..8df555d0 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/config/SchedulingConfig.java @@ -0,0 +1,9 @@ +package com.vnp.demo.config; + +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableScheduling; + +@Configuration +@EnableScheduling +public class SchedulingConfig { +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/config/SecurityConfig.java b/java/src/main/java/com/vnp/demo/config/SecurityConfig.java new file mode 100644 index 00000000..4360c6f6 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/config/SecurityConfig.java @@ -0,0 +1,75 @@ +package com.vnp.demo.config; + +import java.util.Map; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.web.SecurityFilterChain; +import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.vnp.demo.filter.BearerAuthenticationFilter; +import com.vnp.demo.service.AuthService; + +import jakarta.servlet.http.HttpServletResponse; + +@Configuration +public class SecurityConfig { + + @Bean + public SecurityFilterChain securityFilterChain( + HttpSecurity http, + AuthService authService, + ObjectMapper objectMapper + ) throws Exception { + + return http + .csrf(csrf -> csrf.disable()) + .httpBasic(basic -> basic.disable()) + .formLogin(form -> form.disable()) + .sessionManagement(session -> session + .sessionCreationPolicy(SessionCreationPolicy.STATELESS) + ) + .authorizeHttpRequests(auth -> auth + .requestMatchers( + "/health/live", + "/health/ready" + ).permitAll() + .requestMatchers( + org.springframework.http.HttpMethod.POST, + "/api/v1/users", + "/api/v1/auth/sessions" + ).permitAll() + .anyRequest().authenticated() + ) + .exceptionHandling(ex -> ex + .authenticationEntryPoint((request, response, exception) -> { + response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + response.setHeader( + HttpHeaders.WWW_AUTHENTICATE, + "Bearer realm=\"messenger\"" + ); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + response.setCharacterEncoding("UTF-8"); + + objectMapper.writeValue( + response.getWriter(), + Map.of("error", Map.of( + "code", "unauthorized", + "message", "Требуется авторизация", + "request_id", response.getHeader("X-Request-Id") + )) + ); + }) + ) + .addFilterBefore( + new BearerAuthenticationFilter(authService, objectMapper), + UsernamePasswordAuthenticationFilter.class + ) + .build(); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/config/UploadTimeoutConfig.java b/java/src/main/java/com/vnp/demo/config/UploadTimeoutConfig.java new file mode 100644 index 00000000..8a8af35e --- /dev/null +++ b/java/src/main/java/com/vnp/demo/config/UploadTimeoutConfig.java @@ -0,0 +1,25 @@ +package com.vnp.demo.config; + +import org.apache.coyote.http11.AbstractHttp11Protocol; +import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory; +import org.springframework.boot.web.server.WebServerFactoryCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class UploadTimeoutConfig { + + @Bean + public WebServerFactoryCustomizer + uploadTimeoutCustomizer() { + + return factory -> factory.addConnectorCustomizers(connector -> { + if (connector.getProtocolHandler() + instanceof AbstractHttp11Protocol protocol) { + + protocol.setDisableUploadTimeout(false); + protocol.setConnectionUploadTimeout(15_000); + } + }); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/controllers/AttachmentController.java b/java/src/main/java/com/vnp/demo/controllers/AttachmentController.java new file mode 100644 index 00000000..5d1bf488 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/controllers/AttachmentController.java @@ -0,0 +1,100 @@ +package com.vnp.demo.controllers; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import org.springframework.http.ContentDisposition; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.dto.AttachmentResponse; +import com.vnp.demo.service.AttachmentService; + +import jakarta.servlet.http.HttpServletRequest; + +@RestController +@RequestMapping("/api/v1") +public class AttachmentController { + + private final AttachmentService attachmentService; + + public AttachmentController( + AttachmentService attachmentService + ) { + this.attachmentService = attachmentService; + } + + @PostMapping("/chats/{chat_id}/attachments") + @ResponseStatus(HttpStatus.CREATED) + public AttachmentResponse upload( + @AuthenticationPrincipal UUID userId, + @PathVariable("chat_id") UUID chatId, + @RequestParam(value = "filename", required = false) + String filename, + HttpServletRequest request + ) { + try { + return attachmentService.upload( + userId, + chatId, + filename, + request.getContentType(), + request.getContentLengthLong(), + request.getInputStream() + ); + } catch (IOException e) { + throw BusinessException.invalid( + "Не удалось прочитать тело запроса" + ); + } + } + + @GetMapping("/attachments/{id}") + public AttachmentResponse metadata( + @AuthenticationPrincipal UUID userId, + @PathVariable("id") UUID id + ) { + return attachmentService.getMetadata(userId, id); + } + + @GetMapping("/attachments/{id}/content") + public ResponseEntity content( + @AuthenticationPrincipal UUID userId, + @PathVariable("id") UUID id, + @RequestParam(defaultValue = "original") + String variant + ) { + AttachmentService.Download download = + attachmentService.download(userId, id, variant); + + String disposition = ContentDisposition.attachment() + .filename( + download.filename(), + StandardCharsets.UTF_8 + ) + .build() + .toString(); + + return ResponseEntity.ok() + .contentType( + MediaType.parseMediaType(download.contentType()) + ) + .contentLength(download.bytes().length) + .header(HttpHeaders.CONTENT_DISPOSITION, disposition) + .header(HttpHeaders.CACHE_CONTROL, "private, no-store") + .header("X-Content-Type-Options", "nosniff") + .body(download.bytes()); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/controllers/AuthController.java b/java/src/main/java/com/vnp/demo/controllers/AuthController.java new file mode 100644 index 00000000..13382916 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/controllers/AuthController.java @@ -0,0 +1,93 @@ +package com.vnp.demo.controllers; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.dto.SessionResponse; +import com.vnp.demo.service.AuthService; + +import jakarta.servlet.http.HttpServletRequest; + +@RestController +@RequestMapping("/api/v1/auth/sessions") +public class AuthController { + + private final AuthService authService; + + public AuthController(AuthService authService) { + this.authService = authService; + } + + @PostMapping + @ResponseStatus(HttpStatus.CREATED) + public SessionResponse login( + @RequestHeader(value = HttpHeaders.AUTHORIZATION, required = false) + String authorization, + HttpServletRequest request + ) throws IOException { + if (request.getInputStream().read() != -1) { + throw BusinessException.invalid("Тело запроса создания сессии должно быть пустым"); + } + if (authorization == null + || !authorization.startsWith("Basic ")) { + throw BusinessException.unauthorized(); + } + + String encoded = authorization.substring(6).trim(); + String credentials; + + try { + byte[] decoded = Base64.getDecoder().decode(encoded); + + credentials = StandardCharsets.UTF_8 + .newDecoder() + .decode(ByteBuffer.wrap(decoded)) + .toString(); + + } catch (IllegalArgumentException | CharacterCodingException ex) { + throw BusinessException.unauthorized(); + } + + int separator = credentials.indexOf(':'); + + if (separator < 0) { + throw BusinessException.unauthorized(); + } + + String username = credentials.substring(0, separator); + String password = credentials.substring(separator + 1); + + if (!username.matches("[a-z0-9_]{3,32}")) { + throw BusinessException.unauthorized(); + } + + return authService.login(username, password); + } + + @DeleteMapping("/current") + @ResponseStatus(HttpStatus.NO_CONTENT) + public void logout( + @RequestHeader(value = HttpHeaders.AUTHORIZATION, required = false) + String authorization + ) { + if (authorization == null + || !authorization.startsWith("Bearer ")) { + throw BusinessException.unauthorized(); + } + + authService.logout(authorization.substring(7).trim()); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/controllers/ChatController.java b/java/src/main/java/com/vnp/demo/controllers/ChatController.java new file mode 100644 index 00000000..7e3baa41 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/controllers/ChatController.java @@ -0,0 +1,58 @@ +package com.vnp.demo.controllers; + +import java.util.UUID; + +import org.springframework.http.HttpStatus; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import com.vnp.demo.dto.ChatCreateRequest; +import com.vnp.demo.dto.ChatPageResponse; +import com.vnp.demo.dto.ChatResponse; +import com.vnp.demo.service.ChatService; + +import jakarta.validation.Valid; + +@RestController +@RequestMapping("/api/v1/chats") +public class ChatController { + + private final ChatService chatService; + + public ChatController(ChatService chatService) { + this.chatService = chatService; + } + + @PostMapping + @ResponseStatus(HttpStatus.CREATED) + public ChatResponse create( + @AuthenticationPrincipal UUID userId, + @Valid @RequestBody ChatCreateRequest request + ) { + return chatService.create(userId, request); + } + + @GetMapping + public ChatPageResponse list( + @AuthenticationPrincipal UUID userId, + @RequestParam(defaultValue = "50") int limit, + @RequestParam(required = false) String cursor + ) { + return chatService.list(userId, limit, cursor); + } + + @GetMapping("/{chat_id}") + public ChatResponse get( + @AuthenticationPrincipal UUID userId, + @PathVariable("chat_id") UUID chatId + ) { + return chatService.get(userId, chatId); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/controllers/HealthController.java b/java/src/main/java/com/vnp/demo/controllers/HealthController.java new file mode 100644 index 00000000..70a92c26 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/controllers/HealthController.java @@ -0,0 +1,49 @@ +package com.vnp.demo.controllers; + +import org.springframework.boot.availability.ApplicationAvailability; +import org.springframework.boot.availability.ReadinessState; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.http.HttpStatus; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import com.vnp.demo.service.S3StorageService; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.dto.HealthResponse; + +@RestController +@RequestMapping("/health") +public class HealthController { + private final JdbcTemplate jdbc; + private final StringRedisTemplate redis; + private final ApplicationAvailability availability; + private final S3StorageService storage; + public HealthController( + JdbcTemplate jdbc, + StringRedisTemplate redis, + ApplicationAvailability availability, + S3StorageService storage + ) { + this.jdbc = jdbc; + this.redis = redis; + this.availability = availability; + this.storage = storage; + } + @GetMapping("/live") + public HealthResponse live() { return HealthResponse.ok(); } + @GetMapping("/ready") + public HealthResponse ready() { + try { + if (availability.getReadinessState() != ReadinessState.ACCEPTING_TRAFFIC) throw new IllegalStateException(); + if (!Integer.valueOf(1).equals(jdbc.queryForObject("SELECT 1", Integer.class))) throw new IllegalStateException(); + String pong = redis.execute((org.springframework.data.redis.core.RedisCallback) c -> c.ping()); + if (!"PONG".equals(pong)) throw new IllegalStateException(); + storage.checkReady(); + return HealthResponse.ok(); + } catch (Exception ex) { + throw new BusinessException("service_unavailable", "Сервис не готов", HttpStatus.SERVICE_UNAVAILABLE); + } + } +} diff --git a/java/src/main/java/com/vnp/demo/controllers/MessageController.java b/java/src/main/java/com/vnp/demo/controllers/MessageController.java new file mode 100644 index 00000000..f048f815 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/controllers/MessageController.java @@ -0,0 +1,53 @@ +package com.vnp.demo.controllers; + +import java.util.UUID; + +import org.springframework.http.HttpStatus; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import com.vnp.demo.dto.MessageCreateRequest; +import com.vnp.demo.dto.MessagePageResponse; +import com.vnp.demo.dto.MessageResponse; +import com.vnp.demo.service.MessageService; + +import jakarta.validation.Valid; + +@RestController +@RequestMapping("/api/v1/chats/{chat_id}/messages") +public class MessageController { + + private final MessageService messageService; + + public MessageController(MessageService messageService) { + this.messageService = messageService; + } + + @PostMapping + @ResponseStatus(HttpStatus.CREATED) + public MessageResponse create( + @AuthenticationPrincipal UUID userId, + @PathVariable("chat_id") UUID chatId, + @org.springframework.web.bind.annotation.RequestHeader("Idempotency-Key") String key, + @Valid @RequestBody MessageCreateRequest request + ) { + return messageService.create(userId, chatId, request, key); + } + + @GetMapping + public MessagePageResponse list( + @AuthenticationPrincipal UUID userId, + @PathVariable("chat_id") UUID chatId, + @RequestParam(defaultValue = "50") int limit, + @RequestParam(required = false) String cursor + ) { + return messageService.list(userId, chatId, limit, cursor); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/controllers/UserController.java b/java/src/main/java/com/vnp/demo/controllers/UserController.java new file mode 100644 index 00000000..36400736 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/controllers/UserController.java @@ -0,0 +1,44 @@ +package com.vnp.demo.controllers; + +import java.util.UUID; + +import org.springframework.http.HttpStatus; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import com.vnp.demo.dto.UserCreateRequest; +import com.vnp.demo.dto.UserResponse; +import com.vnp.demo.service.UserService; + +import jakarta.validation.Valid; + +@RestController +@RequestMapping("/api/v1/users") +public class UserController { + + private final UserService userService; + + public UserController(UserService userService) { + this.userService = userService; + } + + @PostMapping + @ResponseStatus(HttpStatus.CREATED) + public UserResponse register( + @Valid @RequestBody UserCreateRequest request + ) { + return userService.register(request); + } + + @GetMapping("/me") + public UserResponse getCurrent( + @AuthenticationPrincipal UUID userId + ) { + return userService.getCurrent(userId); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/AttachmentResponse.java b/java/src/main/java/com/vnp/demo/dto/AttachmentResponse.java new file mode 100644 index 00000000..78ad37ba --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/AttachmentResponse.java @@ -0,0 +1,56 @@ +package com.vnp.demo.dto; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.vnp.demo.models.Attachment; + +public record AttachmentResponse( + UUID id, + + @JsonProperty("chat_id") + UUID chatId, + + @JsonProperty("owner_id") + UUID ownerId, + + String filename, + + @JsonProperty("content_type") + String contentType, + + @JsonProperty("size_bytes") + long sizeBytes, + + String sha256, + String status, + List variants, + + @JsonProperty("error_code") + String errorCode, + + @JsonProperty("created_at") + Instant createdAt, + + @JsonProperty("updated_at") + Instant updatedAt +) { + public static AttachmentResponse from(Attachment attachment) { + return new AttachmentResponse( + attachment.id(), + attachment.chatId(), + attachment.ownerId(), + attachment.filename(), + attachment.contentType(), + attachment.sizeBytes(), + attachment.sha256(), + attachment.status(), + List.of(), + null, + attachment.createdAt(), + attachment.updatedAt() + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/ChatCreateRequest.java b/java/src/main/java/com/vnp/demo/dto/ChatCreateRequest.java new file mode 100644 index 00000000..12ecd68b --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/ChatCreateRequest.java @@ -0,0 +1,23 @@ +package com.vnp.demo.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; +import org.hibernate.validator.constraints.UniqueElements; + +import java.util.List; +import java.util.UUID; + +public record ChatCreateRequest( + @NotBlank + @Size(max = 100) + String title, + + @NotEmpty + @Size(max = 100) + @UniqueElements + @JsonProperty("member_ids") + List<@NotNull UUID> memberIds +) {} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/ChatPageResponse.java b/java/src/main/java/com/vnp/demo/dto/ChatPageResponse.java new file mode 100644 index 00000000..6c5fbd8e --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/ChatPageResponse.java @@ -0,0 +1,10 @@ +package com.vnp.demo.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record ChatPageResponse( + List items, + @JsonProperty("next_cursor") String nextCursor +) {} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/ChatResponse.java b/java/src/main/java/com/vnp/demo/dto/ChatResponse.java new file mode 100644 index 00000000..f29a6ed9 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/ChatResponse.java @@ -0,0 +1,16 @@ +package com.vnp.demo.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record ChatResponse( + UUID id, + String title, + @JsonProperty("member_ids") + List memberIds, + @JsonProperty("created_at") + Instant createdAt +) {} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/ErrorResponse.java b/java/src/main/java/com/vnp/demo/dto/ErrorResponse.java new file mode 100644 index 00000000..e2aef6a9 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/ErrorResponse.java @@ -0,0 +1,14 @@ +package com.vnp.demo.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record ErrorResponse(ErrorBody error) { + public record ErrorBody( + String code, + + String message, + + @JsonProperty("request_id") + String requestId + ) {} + } \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/HealthResponse.java b/java/src/main/java/com/vnp/demo/dto/HealthResponse.java new file mode 100644 index 00000000..7d6b2f83 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/HealthResponse.java @@ -0,0 +1,6 @@ +package com.vnp.demo.dto; + +public record HealthResponse(String status) { + public static HealthResponse ok() + { return new HealthResponse("ok"); } +} diff --git a/java/src/main/java/com/vnp/demo/dto/MessageCreateRequest.java b/java/src/main/java/com/vnp/demo/dto/MessageCreateRequest.java new file mode 100644 index 00000000..77232447 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/MessageCreateRequest.java @@ -0,0 +1,53 @@ +package com.vnp.demo.dto; + +import java.util.List; +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.annotation.Nulls; + +public class MessageCreateRequest { + + private String text; + private List attachmentIds = List.of(); + + public MessageCreateRequest() { + } + + public MessageCreateRequest(String text) { + this.text = text; + } + + public MessageCreateRequest( + String text, + List attachmentIds + ) { + this.text = text; + this.attachmentIds = attachmentIds; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public String text() { + return text; + } + + public List attachmentIds() { + return attachmentIds; + } + + @JsonSetter( + value = "attachment_ids", + nulls = Nulls.FAIL, + contentNulls = Nulls.FAIL + ) + public void setAttachmentIds(List attachmentIds) { + this.attachmentIds = attachmentIds; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/MessagePageResponse.java b/java/src/main/java/com/vnp/demo/dto/MessagePageResponse.java new file mode 100644 index 00000000..2f5cdf15 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/MessagePageResponse.java @@ -0,0 +1,10 @@ +package com.vnp.demo.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record MessagePageResponse( + List items, + @JsonProperty("next_cursor") String nextCursor +) {} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/MessageResponse.java b/java/src/main/java/com/vnp/demo/dto/MessageResponse.java new file mode 100644 index 00000000..37ff5a17 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/MessageResponse.java @@ -0,0 +1,26 @@ +package com.vnp.demo.dto; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record MessageResponse( + UUID id, + + @JsonProperty("chat_id") + UUID chatId, + + @JsonProperty("sender_id") + UUID senderId, + + String text, + + @JsonProperty("created_at") + Instant createdAt, + + @JsonProperty("attachment_ids") + List attachmentIds +) { +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/SessionResponse.java b/java/src/main/java/com/vnp/demo/dto/SessionResponse.java new file mode 100644 index 00000000..fb847180 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/SessionResponse.java @@ -0,0 +1,13 @@ +package com.vnp.demo.dto; + +import java.time.Instant; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record SessionResponse( + String token, + @JsonProperty("token_type") String tokenType, + @JsonProperty("expires_at") Instant expiresAt, + UserResponse user +) { +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/UserCreateRequest.java b/java/src/main/java/com/vnp/demo/dto/UserCreateRequest.java new file mode 100644 index 00000000..14cbc8ab --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/UserCreateRequest.java @@ -0,0 +1,18 @@ +package com.vnp.demo.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.*; + +public record UserCreateRequest( + + @NotBlank + @Pattern(regexp = "^[a-z0-9_]{3,32}$") + String username, + + @JsonProperty("display_name") + @NotBlank + String displayName, + + @NotNull + String password +) {} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/UserResponse.java b/java/src/main/java/com/vnp/demo/dto/UserResponse.java new file mode 100644 index 00000000..4e5c0088 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/UserResponse.java @@ -0,0 +1,21 @@ +package com.vnp.demo.dto; + +import java.time.Instant; +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record UserResponse( + + UUID id, + + String username, + + @JsonProperty("display_name") + String displayName, + + @JsonProperty("created_at") + Instant createdAt + +) { +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/filter/BearerAuthenticationFilter.java b/java/src/main/java/com/vnp/demo/filter/BearerAuthenticationFilter.java new file mode 100644 index 00000000..e751149d --- /dev/null +++ b/java/src/main/java/com/vnp/demo/filter/BearerAuthenticationFilter.java @@ -0,0 +1,148 @@ + +package com.vnp.demo.filter; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.springframework.dao.DataAccessException; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.transaction.CannotCreateTransactionException; +import org.springframework.web.filter.OncePerRequestFilter; +import org.springframework.web.server.ResponseStatusException; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.vnp.demo.BusinessException; +import com.vnp.demo.SessionStoreUnavailableException; +import com.vnp.demo.models.User; +import com.vnp.demo.service.AuthService; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class BearerAuthenticationFilter extends OncePerRequestFilter { + + private final AuthService authService; + private final ObjectMapper objectMapper; + + public BearerAuthenticationFilter( + AuthService authService, + ObjectMapper objectMapper + ) { + this.authService = authService; + this.objectMapper = objectMapper; + } + + @Override + protected void doFilterInternal( + HttpServletRequest request, + HttpServletResponse response, + FilterChain chain + ) throws ServletException, IOException { + + String authorization = request.getHeader(HttpHeaders.AUTHORIZATION); + + if (authorization == null || !authorization.startsWith("Bearer ")) { + chain.doFilter(request, response); + return; + } + + String token = authorization.substring(7).trim(); + + User user; + + try { + user = authService.authenticate(token); + + } catch (BusinessException ex) { + SecurityContextHolder.clearContext(); + + response.setHeader( + HttpHeaders.WWW_AUTHENTICATE, + "Bearer realm=\"messenger\"" + ); + + writeError( + response, + HttpServletResponse.SC_UNAUTHORIZED, + "unauthorized", + "Недействительный токен" + ); + return; + + } catch (SessionStoreUnavailableException | DataAccessException | CannotCreateTransactionException ex) { + SecurityContextHolder.clearContext(); + + writeError( + response, + HttpServletResponse.SC_SERVICE_UNAVAILABLE, + "unavailable", + "Сервис временно недоступен" + ); + return; + + } catch (ResponseStatusException ex) { + SecurityContextHolder.clearContext(); + + if (ex.getStatusCode().value() == 503) { + writeError( + response, + HttpServletResponse.SC_SERVICE_UNAVAILABLE, + "unavailable", + "Сервис временно недоступен" + ); + return; + } + + throw ex; + } + + try { + UsernamePasswordAuthenticationToken authentication = + new UsernamePasswordAuthenticationToken( + user.getId(), + null, + List.of() + ); + + SecurityContextHolder.getContext() + .setAuthentication(authentication); + + chain.doFilter(request, response); + + } finally { + SecurityContextHolder.clearContext(); + } + } + + private void writeError( + HttpServletResponse response, + int status, + String code, + String message + ) throws IOException { + + response.setStatus(status); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + response.setCharacterEncoding("UTF-8"); + + String requestId = response.getHeader("X-Request-Id"); + + objectMapper.writeValue( + response.getWriter(), + Map.of( + "error", Map.of( + "code", code, + "message", message, + "request_id", + requestId != null ? requestId : "" + ) + ) + ); + } +} diff --git a/java/src/main/java/com/vnp/demo/filter/RequestIdFilter.java b/java/src/main/java/com/vnp/demo/filter/RequestIdFilter.java new file mode 100644 index 00000000..94b7e9f5 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/filter/RequestIdFilter.java @@ -0,0 +1,77 @@ +package com.vnp.demo.filter; + +import java.io.IOException; +import java.util.UUID; + +import org.slf4j.MDC; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +@Component +@Order(Ordered.HIGHEST_PRECEDENCE) +public class RequestIdFilter extends OncePerRequestFilter { + + private final String instanceId = System.getenv().getOrDefault("HOSTNAME", UUID.randomUUID().toString()); + + public static final String HEADER = "X-Request-Id"; + public static final String MDC_KEY = "requestId"; + + private static final int MAX_REQUEST_ID_LENGTH = 128; + + @Override + protected void doFilterInternal( + HttpServletRequest request, + HttpServletResponse response, + FilterChain chain + ) throws ServletException, IOException { + + String requestId = request.getHeader(HEADER); + + if (requestId == null + || requestId.isBlank() + || requestId.length() > MAX_REQUEST_ID_LENGTH + || !isValidRequestId(requestId)) { + + requestId = UUID.randomUUID() + .toString() + .replace("-", ""); + } + + response.setHeader(HEADER, requestId); + response.setHeader("X-Instance-Id", instanceId); + request.setAttribute(MDC_KEY, requestId); + + MDC.put(MDC_KEY, requestId); + + try { + chain.doFilter(request, response); + } finally { + MDC.remove(MDC_KEY); + } + } + + private boolean isValidRequestId(String requestId) { + for (int i = 0; i < requestId.length(); i++) { + char c = requestId.charAt(i); + + if (!( + (c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || c == '-' + || c == '_' + )) { + return false; + } + } + + return true; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/Attachment.java b/java/src/main/java/com/vnp/demo/models/Attachment.java new file mode 100644 index 00000000..ac922bef --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/Attachment.java @@ -0,0 +1,19 @@ +package com.vnp.demo.models; + +import java.time.Instant; +import java.util.UUID; + +public record Attachment( + UUID id, + UUID chatId, + UUID ownerId, + String filename, + String contentType, + long sizeBytes, + String sha256, + String objectKey, + String status, + Instant createdAt, + Instant updatedAt +) { +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/Chat.java b/java/src/main/java/com/vnp/demo/models/Chat.java new file mode 100644 index 00000000..64b98391 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/Chat.java @@ -0,0 +1,46 @@ +package com.vnp.demo.models; + +import java.time.Instant; +import java.util.UUID; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + +@Entity +@Table(name = "chats") +public class Chat { + + @Id + private UUID id = UUID.randomUUID(); + + @Column(nullable = false, length = 400) + private String title; + + @Column(name = "created_at", nullable = false) + private Instant createdAt = Instant.now(); + + protected Chat() { + } + + public Chat(String title) { + this.title = title; + } + + public UUID getId() { + return id; + } + + public String getTitle() { + return title; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public void setTitle(String title) { + this.title = title; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/ChatMember.java b/java/src/main/java/com/vnp/demo/models/ChatMember.java new file mode 100644 index 00000000..9ab0382e --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/ChatMember.java @@ -0,0 +1,45 @@ +package com.vnp.demo.models; + +import jakarta.persistence.*; + +@Entity +@Table(name = "chat_members") +public class ChatMember { + + @EmbeddedId + private ChatMemberId id; + + @MapsId("chatId") + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "chat_id", nullable = false) + private Chat chat; + + @MapsId("userId") + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "user_id", nullable = false) + private User user; + + protected ChatMember() { + } + + public ChatMember(Chat chat, User user) { + this.chat = chat; + this.user = user; + this.id = new ChatMemberId( + chat.getId(), + user.getId() + ); + } + + public ChatMemberId getId() { + return id; + } + + public Chat getChat() { + return chat; + } + + public User getUser() { + return user; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/ChatMemberId.java b/java/src/main/java/com/vnp/demo/models/ChatMemberId.java new file mode 100644 index 00000000..01af0099 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/ChatMemberId.java @@ -0,0 +1,53 @@ +package com.vnp.demo.models; + +import java.io.Serializable; +import java.util.Objects; +import java.util.UUID; + +import jakarta.persistence.Column; +import jakarta.persistence.Embeddable; + +@Embeddable +public class ChatMemberId implements Serializable { + + @Column(name = "chat_id", nullable = false) + private UUID chatId; + + @Column(name = "user_id", nullable = false) + private UUID userId; + + protected ChatMemberId() { + } + + public ChatMemberId(UUID chatId, UUID userId) { + this.chatId = chatId; + this.userId = userId; + } + + public UUID getChatId() { + return chatId; + } + + public UUID getUserId() { + return userId; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (!(obj instanceof ChatMemberId other)) { + return false; + } + + return Objects.equals(chatId, other.chatId) + && Objects.equals(userId, other.userId); + } + + @Override + public int hashCode() { + return Objects.hash(chatId, userId); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/Message.java b/java/src/main/java/com/vnp/demo/models/Message.java new file mode 100644 index 00000000..1ad3c981 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/Message.java @@ -0,0 +1,91 @@ +package com.vnp.demo.models; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import jakarta.persistence.CollectionTable; +import jakarta.persistence.Column; +import jakarta.persistence.ElementCollection; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.OrderColumn; +import jakarta.persistence.Table; + +@Entity +@Table(name = "messages") +public class Message { + + @Id + private UUID id = UUID.randomUUID(); + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "chat_id", nullable = false) + private Chat chat; + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "sender_id", nullable = false) + private User sender; + + @Column(name = "text", nullable = false, columnDefinition = "text") + private String text; + + @Column(name = "created_at", nullable = false) + private Instant createdAt = Instant.now(); + + @ElementCollection(fetch = FetchType.LAZY) + @CollectionTable( + name = "message_attachments", + joinColumns = @JoinColumn(name = "message_id") + ) + @Column(name = "attachment_id", nullable = false) + @OrderColumn(name = "position") + private List attachmentIds = new ArrayList<>(); + + protected Message() { + } + + public Message(Chat chat, User sender, String text) { + this(chat, sender, text, List.of()); + } + + public Message( + Chat chat, + User sender, + String text, + List attachmentIds + ) { + this.chat = chat; + this.sender = sender; + this.text = text; + this.attachmentIds = new ArrayList<>(attachmentIds); + } + + public UUID getId() { + return id; + } + + public Chat getChat() { + return chat; + } + + public User getSender() { + return sender; + } + + public String getText() { + return text; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public List getAttachmentIds() { + return attachmentIds; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/Session.java b/java/src/main/java/com/vnp/demo/models/Session.java new file mode 100644 index 00000000..cdbf6d9c --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/Session.java @@ -0,0 +1,90 @@ +package com.vnp.demo.models; + +import java.sql.Types; +import java.time.Instant; +import java.util.UUID; + +import org.hibernate.annotations.JdbcTypeCode; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; + +@Entity +@Table(name = "sessions") +public class Session { + + @Id + private UUID id = UUID.randomUUID(); + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "user_id", nullable = false) + private User user; + + @JdbcTypeCode(Types.CHAR) + @Column( + name = "token_hash", + nullable = false, + unique = true, + length = 64, + columnDefinition = "CHAR(64)" + ) + private String tokenHash; + + @Column(name = "expires_at", nullable = false) + private Instant expiresAt; + + @Column(name = "revoked_at") + private Instant revokedAt; + + @Column(name = "created_at", nullable = false) + private Instant createdAt = Instant.now(); + + protected Session() { + } + + public Session(User user, String tokenHash, Instant expiresAt) { + this.user = user; + this.tokenHash = tokenHash; + this.expiresAt = expiresAt; + } + + public UUID getId() { + return id; + } + + public User getUser() { + return user; + } + + public String getTokenHash() { + return tokenHash; + } + + public Instant getExpiresAt() { + return expiresAt; + } + + public Instant getRevokedAt() { + return revokedAt; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public boolean isValid() { + return revokedAt == null + && expiresAt.isAfter(Instant.now()); + } + + public void revoke() { + if (revokedAt == null) { + revokedAt = Instant.now(); + } + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/User.java b/java/src/main/java/com/vnp/demo/models/User.java new file mode 100644 index 00000000..f9af6301 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/User.java @@ -0,0 +1,63 @@ +package com.vnp.demo.models; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.UUID; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + +@Entity +@Table(name = "users") +public class User { + + @Id + private UUID id = UUID.randomUUID(); + + @Column(nullable = false, unique = true, length = 32) + private String username; + + @Column(name = "display_name", nullable = false) + private String displayName; + + @Column(name = "password_hash", nullable = false) + private String passwordHash; + + @Column(name = "created_at", nullable = false) + private Instant createdAt = Instant.now().truncatedTo(ChronoUnit.MICROS); + + protected User() { + } + + public User(String username, String displayName, String passwordHash) { + this.username = username; + this.displayName = displayName; + this.passwordHash = passwordHash; + } + + public UUID getId() { + return id; + } + + public String getUsername() { + return username; + } + + public String getDisplayName() { + return displayName; + } + + public String getPasswordHash() { + return passwordHash; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/repositories/AttachmentRepository.java b/java/src/main/java/com/vnp/demo/repositories/AttachmentRepository.java new file mode 100644 index 00000000..1e714517 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/repositories/AttachmentRepository.java @@ -0,0 +1,122 @@ +package com.vnp.demo.repositories; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.Optional; +import java.util.UUID; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import com.vnp.demo.models.Attachment; + +@Repository +public class AttachmentRepository { + + private final JdbcTemplate jdbc; + + public AttachmentRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + public void save(Attachment attachment) { + jdbc.update( + """ + INSERT INTO attachments ( + id, + chat_id, + owner_id, + filename, + content_type, + size_bytes, + sha256, + object_key, + status, + created_at, + updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + attachment.id(), + attachment.chatId(), + attachment.ownerId(), + attachment.filename(), + attachment.contentType(), + attachment.sizeBytes(), + attachment.sha256(), + attachment.objectKey(), + attachment.status(), + Timestamp.from(attachment.createdAt()), + Timestamp.from(attachment.updatedAt()) + ); + } + + public Optional findById(UUID id) { + return jdbc.query( + "SELECT * FROM attachments WHERE id = ?", + this::mapRow, + id + ).stream().findFirst(); + } + + private Attachment mapRow(ResultSet rs, int rowNum) + throws SQLException { + + return new Attachment( + rs.getObject("id", UUID.class), + rs.getObject("chat_id", UUID.class), + rs.getObject("owner_id", UUID.class), + rs.getString("filename"), + rs.getString("content_type"), + rs.getLong("size_bytes"), + rs.getString("sha256"), + rs.getString("object_key"), + rs.getString("status"), + rs.getTimestamp("created_at").toInstant(), + rs.getTimestamp("updated_at").toInstant() + ); + } + + public boolean existsByObjectKey(String objectKey) { + Boolean exists = jdbc.queryForObject( + """ + SELECT EXISTS ( + SELECT 1 + FROM attachments + WHERE object_key = ? + ) + """, + Boolean.class, + objectKey + ); + + return Boolean.TRUE.equals(exists); + } + + public void lockObject(String objectKey) { + jdbc.query( + """ + SELECT pg_advisory_xact_lock( + hashtextextended(?, 0) + ) + """, + (rs, rowNum) -> Boolean.TRUE, + "attachment-object:" + objectKey + ); + } + + public boolean tryLockObject(String objectKey) { + Boolean acquired = jdbc.queryForObject( + """ + SELECT pg_try_advisory_xact_lock( + hashtextextended(?, 0) + ) + """, + Boolean.class, + "attachment-object:" + objectKey + ); + + return Boolean.TRUE.equals(acquired); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/repositories/ChatMemberRepository.java b/java/src/main/java/com/vnp/demo/repositories/ChatMemberRepository.java new file mode 100644 index 00000000..7b68cab3 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/repositories/ChatMemberRepository.java @@ -0,0 +1,19 @@ +package com.vnp.demo.repositories; + +import java.util.List; +import java.util.UUID; + +import org.springframework.data.jpa.repository.JpaRepository; + +import com.vnp.demo.models.ChatMember; +import com.vnp.demo.models.ChatMemberId; + +public interface ChatMemberRepository + extends JpaRepository { + + boolean existsByIdChatIdAndIdUserId(UUID chatId, UUID userId); + + List findByIdChatId(UUID chatId); + + boolean existsByChat_IdAndUser_Id(UUID chatId, UUID userId); +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/repositories/ChatRepository.java b/java/src/main/java/com/vnp/demo/repositories/ChatRepository.java new file mode 100644 index 00000000..320e00ea --- /dev/null +++ b/java/src/main/java/com/vnp/demo/repositories/ChatRepository.java @@ -0,0 +1,56 @@ +package com.vnp.demo.repositories; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import com.vnp.demo.models.Chat; + +public interface ChatRepository extends JpaRepository { + + @Query(""" + SELECT c + FROM Chat c + WHERE EXISTS ( + SELECT 1 + FROM ChatMember cm + WHERE cm.chat.id = c.id + AND cm.user.id = :userId + ) + ORDER BY c.createdAt ASC, c.id ASC + """) + List findFirstPage( + @Param("userId") UUID userId, + Pageable pageable + ); + + @Query(""" + SELECT c + FROM Chat c + WHERE EXISTS ( + SELECT 1 + FROM ChatMember cm + WHERE cm.chat.id = c.id + AND cm.user.id = :userId + ) + AND ( + c.createdAt > :cursorTime + OR ( + c.createdAt = :cursorTime + AND c.id > :cursorId + ) + ) + ORDER BY c.createdAt ASC, c.id ASC + """) + List findNextPage( + @Param("userId") UUID userId, + @Param("cursorTime") Instant cursorTime, + @Param("cursorId") UUID cursorId, + Pageable pageable + ); +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/repositories/MessageRepository.java b/java/src/main/java/com/vnp/demo/repositories/MessageRepository.java new file mode 100644 index 00000000..c726cb60 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/repositories/MessageRepository.java @@ -0,0 +1,40 @@ +package com.vnp.demo.repositories; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import com.vnp.demo.models.Message; + +public interface MessageRepository extends JpaRepository { + + @Query(value = """ + SELECT m.* + FROM messages m + WHERE m.chat_id = :chatId + ORDER BY m.created_at ASC, m.id ASC + """, nativeQuery = true) + List findFirstPage( + @Param("chatId") UUID chatId, + Pageable pageable + ); + + @Query(value = """ + SELECT m.* + FROM messages m + WHERE m.chat_id = :chatId + AND (m.created_at, m.id) > (:cursorTime, :cursorId) + ORDER BY m.created_at ASC, m.id ASC + """, nativeQuery = true) + List findNextPage( + @Param("chatId") UUID chatId, + @Param("cursorTime") Instant cursorTime, + @Param("cursorId") UUID cursorId, + Pageable pageable + ); +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/repositories/SessionRepository.java b/java/src/main/java/com/vnp/demo/repositories/SessionRepository.java new file mode 100644 index 00000000..db182093 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/repositories/SessionRepository.java @@ -0,0 +1,12 @@ +package com.vnp.demo.repositories; + +import com.vnp.demo.models.Session; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; +import java.util.UUID; + +public interface SessionRepository extends JpaRepository { + + Optional findByTokenHash(String tokenHash); +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/repositories/UserRepository.java b/java/src/main/java/com/vnp/demo/repositories/UserRepository.java new file mode 100644 index 00000000..de8c80ab --- /dev/null +++ b/java/src/main/java/com/vnp/demo/repositories/UserRepository.java @@ -0,0 +1,14 @@ +package com.vnp.demo.repositories; + +import com.vnp.demo.models.User; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; +import java.util.UUID; + +public interface UserRepository extends JpaRepository { + + Optional findByUsername(String username); + + boolean existsByUsername(String username); +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/AttachmentCleanupJob.java b/java/src/main/java/com/vnp/demo/service/AttachmentCleanupJob.java new file mode 100644 index 00000000..0c9a40f9 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/AttachmentCleanupJob.java @@ -0,0 +1,119 @@ +package com.vnp.demo.service; + +import java.time.Instant; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import com.vnp.demo.service.AttachmentPersistenceService.CleanupResult; + +import io.minio.Result; +import io.minio.messages.Item; + +@Component +@ConditionalOnProperty( + name = "app.cleanup.enabled", + havingValue = "true", + matchIfMissing = true +) +public class AttachmentCleanupJob { + + private static final Logger log = + LoggerFactory.getLogger(AttachmentCleanupJob.class); + + private final S3StorageService storage; + private final AttachmentPersistenceService persistence; + private final long graceSeconds; + + public AttachmentCleanupJob( + S3StorageService storage, + AttachmentPersistenceService persistence, + @Value("${app.cleanup.grace-seconds:3600}") + long graceSeconds + ) { + if (graceSeconds < 60) { + throw new IllegalArgumentException( + "Grace period должен быть не меньше 60 секунд" + ); + } + + this.storage = storage; + this.persistence = persistence; + this.graceSeconds = graceSeconds; + } + + @Scheduled( + initialDelayString = + "${app.cleanup.initial-delay-ms:60000}", + fixedDelayString = + "${app.cleanup.interval-ms:300000}" + ) + public void cleanup() { + Instant cutoff = Instant.now() + .minusSeconds(graceSeconds); + + long scanned = 0; + long candidates = 0; + long deleted = 0; + + try { + for (Result result : storage.listOriginals()) { + Item item = result.get(); + + if (item.isDir()) { + continue; + } + + scanned++; + + Instant lastModified = item.lastModified() + .toInstant(); + + if (!lastModified.isBefore(cutoff)) { + continue; + } + + candidates++; + + CleanupResult outcome = + persistence.removeIfOrphan( + item.objectName(), + cutoff + ); + + if (outcome == CleanupResult.DELETED) { + deleted++; + + log.info( + "Attachment cleanup removed orphan: {}", + item.objectName() + ); + } + } + + log.info( + "Attachment cleanup completed: " + + "scanned={}, candidates={}, deleted={}", + scanned, + candidates, + deleted + ); + } catch (Exception e) { + // Прерываем текущий проход при ошибке БД или S3. + // Следующий запуск повторит проверку. + // Не выводим credentials или содержимое ответа S3. + log.warn( + "Attachment cleanup interrupted: " + + "scanned={}, candidates={}, deleted={}, errorType={}", + scanned, + candidates, + deleted, + e.getClass().getSimpleName() + ); + } + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/AttachmentPersistenceService.java b/java/src/main/java/com/vnp/demo/service/AttachmentPersistenceService.java new file mode 100644 index 00000000..9d406d87 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/AttachmentPersistenceService.java @@ -0,0 +1,94 @@ +package com.vnp.demo.service; + +import java.nio.file.Path; +import java.time.Instant; + +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Isolation; +import org.springframework.transaction.annotation.Transactional; + +import com.vnp.demo.models.Attachment; +import com.vnp.demo.repositories.AttachmentRepository; + +@Service +public class AttachmentPersistenceService { + + private final AttachmentRepository attachmentRepository; + private final S3StorageService storage; + + public AttachmentPersistenceService( + AttachmentRepository attachmentRepository, + S3StorageService storage + ) { + this.attachmentRepository = attachmentRepository; + this.storage = storage; + } + + @Transactional( + isolation = Isolation.READ_COMMITTED, + timeout = 60 + ) + public void save( + Attachment attachment, + Path temporaryFile + ) { + attachmentRepository.lockObject( + attachment.objectKey() + ); + + storage.put( + attachment.objectKey(), + temporaryFile, + attachment.sizeBytes(), + attachment.contentType() + ); + + attachmentRepository.save(attachment); + } + + @Transactional( + isolation = Isolation.READ_COMMITTED, + timeout = 30 + ) + public CleanupResult removeIfOrphan( + String objectKey, + Instant cutoff + ) { + if (!objectKey.startsWith("originals/")) { + return CleanupResult.SKIPPED; + } + + if (!attachmentRepository.tryLockObject(objectKey)) { + return CleanupResult.BUSY; + } + + if (attachmentRepository.existsByObjectKey(objectKey)) { + return CleanupResult.REFERENCED; + } + + Instant lastModified = storage.lastModifiedOrNull( + objectKey + ); + + if (lastModified == null) { + return CleanupResult.MISSING; + } + + if (!lastModified.isBefore(cutoff)) { + return CleanupResult.RECENT; + } + + storage.remove(objectKey); + + return CleanupResult.DELETED; + } + + public enum CleanupResult { + DELETED, + REFERENCED, + RECENT, + BUSY, + MISSING, + SKIPPED + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/AttachmentService.java b/java/src/main/java/com/vnp/demo/service/AttachmentService.java new file mode 100644 index 00000000..9af595c2 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/AttachmentService.java @@ -0,0 +1,163 @@ +package com.vnp.demo.service; + +import java.io.InputStream; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.UUID; + +import org.springframework.stereotype.Service; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.dto.AttachmentResponse; +import com.vnp.demo.models.Attachment; +import com.vnp.demo.repositories.AttachmentRepository; +import com.vnp.demo.repositories.ChatMemberRepository; +import com.vnp.demo.validation.ImageValidator; +import com.vnp.demo.validation.ImageValidator.ValidatedImage; + +@Service +public class AttachmentService { + + private final AttachmentRepository attachmentRepository; + private final ChatMemberRepository chatMemberRepository; + private final ImageValidator imageValidator; + private final S3StorageService storage; + private final UploadLimiter uploadLimiter; + private final AttachmentPersistenceService persistence; + + public AttachmentService( + AttachmentRepository attachmentRepository, + ChatMemberRepository chatMemberRepository, + ImageValidator imageValidator, + S3StorageService storage, + UploadLimiter uploadLimiter, + AttachmentPersistenceService persistence + ) { + this.attachmentRepository = attachmentRepository; + this.chatMemberRepository = chatMemberRepository; + this.imageValidator = imageValidator; + this.storage = storage; + this.uploadLimiter = uploadLimiter; + this.persistence = persistence; + } + + public AttachmentResponse upload( + UUID userId, + UUID chatId, + String filename, + String contentType, + long contentLength, + InputStream input + ) { + checkChatAccess(userId, chatId); + + imageValidator.validateFilename(filename); + + try (UploadLimiter.Permit permit = uploadLimiter.acquire(); + ValidatedImage image = imageValidator.validate( + input, + contentLength, + contentType + )) { + + UUID attachmentId = UUID.randomUUID(); + + String objectKey = "originals/" + + chatId + + "/" + + attachmentId; + + Instant now = Instant.now() + .truncatedTo(ChronoUnit.MICROS); + + Attachment attachment = new Attachment( + attachmentId, + chatId, + userId, + filename, + image.contentType(), + image.sizeBytes(), + image.sha256(), + objectKey, + "ready", + now, + now + ); + + persistence.save(attachment, image.path()); + + return AttachmentResponse.from(attachment); + } + } + + public AttachmentResponse getMetadata( + UUID userId, + UUID attachmentId + ) { + return AttachmentResponse.from( + getAccessibleAttachment(userId, attachmentId) + ); + } + + public Download download( + UUID userId, + UUID attachmentId, + String variant + ) { + Attachment attachment = getAccessibleAttachment( + userId, + attachmentId + ); + + if (!"ready".equals(attachment.status())) { + throw BusinessException.conflict( + "Вложение ещё не готово" + ); + } + + if (!"original".equals(variant)) { + throw BusinessException.notFound(); + } + + byte[] bytes = storage.get( + attachment.objectKey(), + attachment.sizeBytes(), + attachment.sha256() + ); + + return new Download( + attachment.filename(), + attachment.contentType(), + bytes + ); + } + + private Attachment getAccessibleAttachment( + UUID userId, + UUID attachmentId + ) { + Attachment attachment = attachmentRepository + .findById(attachmentId) + .orElseThrow(BusinessException::notFound); + + checkChatAccess(userId, attachment.chatId()); + + return attachment; + } + + private void checkChatAccess(UUID userId, UUID chatId) { + boolean member = chatMemberRepository + .existsByIdChatIdAndIdUserId(chatId, userId); + + if (!member) { + throw BusinessException.notFound(); + } + } + + public record Download( + String filename, + String contentType, + byte[] bytes + ) { + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/AuthService.java b/java/src/main/java/com/vnp/demo/service/AuthService.java new file mode 100644 index 00000000..37460170 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/AuthService.java @@ -0,0 +1,143 @@ +package com.vnp.demo.service; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.dto.SessionResponse; +import com.vnp.demo.dto.UserResponse; +import com.vnp.demo.models.Session; +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.SessionRepository; +import com.vnp.demo.repositories.UserRepository; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.web.server.ResponseStatusException; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.Instant; +import java.util.Base64; +import java.util.HexFormat; + +@Service +public class AuthService { + private static final SecureRandom RANDOM = new SecureRandom(); + + private final UserRepository userRepository; + private final SessionRepository sessionRepository; + private final SessionCacheService sessionCacheService; + private final PasswordEncoder passwordEncoder; + private final TransactionTemplate transactions; + private final long sessionTtlSeconds; + + public AuthService( + UserRepository userRepository, + SessionRepository sessionRepository, + SessionCacheService sessionCacheService, + PasswordEncoder passwordEncoder, + PlatformTransactionManager transactionManager, + @Value("${app.session.ttl-seconds:3600}") long sessionTtlSeconds + ) { + if (sessionTtlSeconds <= 0) { + throw new IllegalArgumentException("Session TTL must be positive"); + } + this.userRepository = userRepository; + this.sessionRepository = sessionRepository; + this.sessionCacheService = sessionCacheService; + this.passwordEncoder = passwordEncoder; + this.transactions = new TransactionTemplate(transactionManager); + this.sessionTtlSeconds = sessionTtlSeconds; + } + + public SessionResponse login(String username, String password) { + byte[] bytes = new byte[32]; + RANDOM.nextBytes(bytes); + String token = Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + String tokenHash = hashToken(token); + Instant expiresAt = Instant.now().plusSeconds(sessionTtlSeconds); + + // TransactionTemplate returns only after the database commit completes. + UserResponse userResponse = transactions.execute(status -> { + User user = userRepository.findByUsername(username) + .orElseThrow(BusinessException::unauthorized); + if (!passwordEncoder.matches(password, user.getPasswordHash())) { + throw BusinessException.unauthorized(); + } + sessionRepository.save(new Session(user, tokenHash, expiresAt)); + return new UserResponse( + user.getId(), user.getUsername(), + user.getDisplayName(), user.getCreatedAt() + ); + }); + + // Never publish a usable token before the PostgreSQL commit. + // If Redis fails, the DB row remains inert: it cannot authenticate. + try { + sessionCacheService.cache(tokenHash, userResponse.id(), expiresAt); + } catch (RuntimeException activationFailure) { + // The token is never returned. Best-effort cleanup handles an + // uncertain Redis write and records the failed activation in PG. + try { + sessionCacheService.invalidate(tokenHash); + } catch (RuntimeException cleanupFailure) { + activationFailure.addSuppressed(cleanupFailure); + } + try { + transactions.executeWithoutResult(status -> + sessionRepository.findByTokenHash(tokenHash) + .ifPresent(Session::revoke) + ); + } catch (RuntimeException cleanupFailure) { + activationFailure.addSuppressed(cleanupFailure); + } + throw activationFailure; + } + return new SessionResponse(token, "Bearer", expiresAt, userResponse); + } + + public User authenticate(String token) { + if (token == null || token.isBlank()) { + throw BusinessException.unauthorized(); + } + var cached = sessionCacheService.find(hashToken(token)) + .orElseThrow(BusinessException::unauthorized); + return userRepository.findById(cached.userId()) + .orElseThrow(BusinessException::unauthorized); + } + + public void logout(String token) { + if (token == null || token.isBlank()) { + throw BusinessException.unauthorized(); + } + String tokenHash = hashToken(token); + if (sessionCacheService.find(tokenHash).isEmpty()) { + throw BusinessException.unauthorized(); + } + + + if (!sessionCacheService.invalidate(tokenHash)) { + throw BusinessException.unauthorized(); + } + + transactions.executeWithoutResult(status -> { + Session session = sessionRepository.findByTokenHash(tokenHash) + .orElseThrow(() -> new ResponseStatusException( + HttpStatus.SERVICE_UNAVAILABLE, "Session record unavailable")); + session.revoke(); + }); + } + + private String hashToken(String token) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256") + .digest(token.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(hash); + } catch (NoSuchAlgorithmException ex) { + throw new IllegalStateException("SHA-256 unavailable", ex); + } + } +} diff --git a/java/src/main/java/com/vnp/demo/service/ChatService.java b/java/src/main/java/com/vnp/demo/service/ChatService.java new file mode 100644 index 00000000..1b4ce674 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/ChatService.java @@ -0,0 +1,195 @@ +package com.vnp.demo.service; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.CursorCodec; +import com.vnp.demo.dto.ChatCreateRequest; +import com.vnp.demo.dto.ChatPageResponse; +import com.vnp.demo.dto.ChatResponse; +import com.vnp.demo.models.Chat; +import com.vnp.demo.models.ChatMember; +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.ChatMemberRepository; +import com.vnp.demo.repositories.ChatRepository; +import com.vnp.demo.repositories.UserRepository; + +@Service +public class ChatService { + + private final ChatRepository chatRepository; + private final ChatMemberRepository chatMemberRepository; + private final UserRepository userRepository; + private final CursorCodec cursorCodec; + + public ChatService( + ChatRepository chatRepository, + ChatMemberRepository chatMemberRepository, + UserRepository userRepository, + CursorCodec cursorCodec + ) { + this.chatRepository = chatRepository; + this.chatMemberRepository = chatMemberRepository; + this.userRepository = userRepository; + this.cursorCodec = cursorCodec; + } + + @Transactional + public ChatResponse create( + UUID currentUserId, + ChatCreateRequest request + ) { + String title = request.title(); + + if (title == null + || title.isBlank() + || title.codePointCount(0, title.length()) > 100) { + throw BusinessException.invalid( + "Некорректное название чата" + ); + } + + List memberIds = request.memberIds(); + + if (memberIds == null + || memberIds.isEmpty() + || !memberIds.contains(currentUserId)) { + throw BusinessException.invalid( + "Создатель должен быть участником чата" + ); + } + + if (memberIds.stream().distinct().count() != memberIds.size()) { + throw BusinessException.invalid( + "Список участников содержит дубликаты" + ); + } + + List users = userRepository.findAllById(memberIds); + + if (users.size() != memberIds.size()) { + throw BusinessException.invalid("Один или несколько участников не найдены"); + } + + // Создаём чат через конструктор существующей модели. + Chat chat = new Chat(title); + chatRepository.save(chat); + + // Создание чата и участников происходит в одной транзакции. + for (User user : users) { + ChatMember member = new ChatMember(chat, user); + chatMemberRepository.save(member); + } + + return toResponse(chat, memberIds); + } + + @Transactional(readOnly = true) + public ChatResponse get( + UUID currentUserId, + UUID chatId + ) { + Chat chat = chatRepository.findById(chatId) + .orElseThrow(BusinessException::notFound); + + if (!chatMemberRepository.existsByIdChatIdAndIdUserId( + chatId, + currentUserId + )) { + throw BusinessException.notFound(); + } + + return toResponse(chat, getMemberIds(chatId)); + } + + @Transactional(readOnly = true) + public ChatPageResponse list( + UUID currentUserId, + Integer limit, + String cursor + ) { + int pageSize = limit == null ? 20 : limit; + + if (pageSize < 1 || pageSize > 100) { + throw BusinessException.invalid( + "limit должен быть от 1 до 100" + ); + } + + List chats; + + if (cursor == null) { + chats = chatRepository.findFirstPage( + currentUserId, + PageRequest.of(0, pageSize + 1) + ); + } else { + CursorCodec.ChatCursor decoded = + cursorCodec.decodeChat(cursor, currentUserId); + + chats = chatRepository.findNextPage( + currentUserId, + decoded.createdAt(), + decoded.id(), + PageRequest.of(0, pageSize + 1) + ); + } + + boolean hasNext = chats.size() > pageSize; + + List page = hasNext + ? chats.subList(0, pageSize) + : chats; + + List items = new ArrayList<>(); + + for (Chat chat : page) { + items.add( + toResponse( + chat, + getMemberIds(chat.getId()) + ) + ); + } + + String nextCursor = null; + + if (hasNext && !page.isEmpty()) { + Chat last = page.get(page.size() - 1); + + nextCursor = cursorCodec.encodeChat( + currentUserId, + last.getCreatedAt(), + last.getId() + ); + } + + return new ChatPageResponse(items, nextCursor); + } + + private List getMemberIds(UUID chatId) { + return chatMemberRepository.findByIdChatId(chatId) + .stream() + .map(member -> member.getId().getUserId()) + .sorted() + .toList(); + } + + private ChatResponse toResponse( + Chat chat, + List memberIds + ) { + return new ChatResponse( + chat.getId(), + chat.getTitle(), + memberIds, + chat.getCreatedAt() + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/MessageService.java b/java/src/main/java/com/vnp/demo/service/MessageService.java new file mode 100644 index 00000000..1554308f --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/MessageService.java @@ -0,0 +1,323 @@ +package com.vnp.demo.service; + +import java.util.HashSet; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.CursorCodec; +import com.vnp.demo.dto.MessageCreateRequest; +import com.vnp.demo.dto.MessagePageResponse; +import com.vnp.demo.dto.MessageResponse; +import com.vnp.demo.models.Attachment; +import com.vnp.demo.models.Chat; +import com.vnp.demo.models.Message; +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.AttachmentRepository; +import com.vnp.demo.repositories.ChatMemberRepository; +import com.vnp.demo.repositories.ChatRepository; +import com.vnp.demo.repositories.MessageRepository; +import com.vnp.demo.repositories.UserRepository; + +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; + +@Service +public class MessageService { + + private final MessageRepository messageRepository; + private final ChatRepository chatRepository; + private final ChatMemberRepository chatMemberRepository; + private final UserRepository userRepository; + private final AttachmentRepository attachmentRepository; + private final CursorCodec cursorCodec; + + @PersistenceContext + private EntityManager entityManager; + + public MessageService( + MessageRepository messageRepository, + ChatRepository chatRepository, + ChatMemberRepository chatMemberRepository, + UserRepository userRepository, + AttachmentRepository attachmentRepository, + CursorCodec cursorCodec + ) { + this.messageRepository = messageRepository; + this.chatRepository = chatRepository; + this.chatMemberRepository = chatMemberRepository; + this.userRepository = userRepository; + this.attachmentRepository = attachmentRepository; + this.cursorCodec = cursorCodec; + } + + @Transactional + public MessageResponse create( + UUID userId, + UUID chatId, + MessageCreateRequest request, + String key + ) { + if (key == null + || !key.matches("[A-Za-z0-9._:-]{1,128}")) { + throw BusinessException.invalid( + "Некорректный Idempotency-Key" + ); + } + + checkAccess(userId, chatId); + + List attachmentIds = validateRequest(request); + + String attachmentSignature = attachmentIds.stream() + .map(UUID::toString) + .collect(Collectors.joining(",")); + + String scope = userId + ":" + chatId + ":" + key; + + entityManager.createNativeQuery( + """ + SELECT 1 + FROM pg_advisory_xact_lock( + hashtextextended(:scope, 0) + ) + """ + ).setParameter("scope", scope).getResultList(); + + @SuppressWarnings("unchecked") + List existing = entityManager.createNativeQuery( + """ + SELECT message_id, request_text, request_attachment_ids + FROM message_idempotency + WHERE user_id = :user + AND chat_id = :chat + AND key = :key + """ + ) + .setParameter("user", userId) + .setParameter("chat", chatId) + .setParameter("key", key) + .getResultList(); + + if (!existing.isEmpty()) { + Object[] row = existing.get(0); + + boolean sameText = request.text().equals(row[1]); + boolean sameAttachments = + attachmentSignature.equals(row[2]); + + if (!sameText || !sameAttachments) { + throw BusinessException.conflict( + "Ключ уже использован с другим сообщением" + ); + } + + Message original = messageRepository + .findById((UUID) row[0]) + .orElseThrow(BusinessException::notFound); + + return toResponse(original); + } + + validateAttachments(chatId, attachmentIds); + + Chat chat = chatRepository.findById(chatId) + .orElseThrow(BusinessException::notFound); + + User sender = userRepository.findById(userId) + .orElseThrow(BusinessException::unauthorized); + + Message message = new Message( + chat, + sender, + request.text(), + attachmentIds + ); + + Message saved = messageRepository.saveAndFlush(message); + + entityManager.refresh(saved); + + entityManager.createNativeQuery( + """ + INSERT INTO message_idempotency ( + user_id, + chat_id, + key, + request_text, + request_attachment_ids, + message_id + ) + VALUES ( + :user, + :chat, + :key, + :text, + :attachments, + :message + ) + """ + ) + .setParameter("user", userId) + .setParameter("chat", chatId) + .setParameter("key", key) + .setParameter("text", request.text()) + .setParameter("attachments", attachmentSignature) + .setParameter("message", saved.getId()) + .executeUpdate(); + + return toResponse(saved); + } + + private List validateRequest( + MessageCreateRequest request + ) { + if (request == null || request.text() == null) { + throw BusinessException.invalid( + "Поле text обязательно" + ); + } + + if (request.text().codePointCount( + 0, + request.text().length() + ) > 4000) { + throw BusinessException.invalid( + "Текст не должен превышать 4000 символов" + ); + } + + List attachmentIds = request.attachmentIds(); + + if (attachmentIds == null + || attachmentIds.size() > 10 + || attachmentIds.stream().anyMatch(id -> id == null)) { + throw BusinessException.invalid( + "attachment_ids должен содержать от 0 до 10 UUID" + ); + } + + if (new HashSet<>(attachmentIds).size() + != attachmentIds.size()) { + throw BusinessException.invalid( + "Повторяющиеся вложения недопустимы" + ); + } + + if (request.text().isBlank() && attachmentIds.isEmpty()) { + throw BusinessException.invalid( + "Сообщение должно содержать текст или вложение" + ); + } + + return List.copyOf(attachmentIds); + } + + private void validateAttachments( + UUID chatId, + List attachmentIds + ) { + for (UUID attachmentId : attachmentIds) { + Attachment attachment = attachmentRepository + .findById(attachmentId) + .orElseThrow(BusinessException::notFound); + + if (!chatId.equals(attachment.chatId())) { + throw BusinessException.notFound(); + } + + if (!"ready".equals(attachment.status())) { + throw BusinessException.conflict( + "Вложение не готово" + ); + } + } + } + + @Transactional(readOnly = true) + public MessagePageResponse list( + UUID userId, + UUID chatId, + int limit, + String cursor + ) { + checkAccess(userId, chatId); + + if (limit < 1 || limit > 100) { + throw BusinessException.invalid( + "limit должен быть от 1 до 100" + ); + } + + CursorCodec.MessageCursor position = null; + + if (cursor != null) { + position = cursorCodec.decodeMessage(cursor, chatId); + } + + List messages; + + if (position == null) { + messages = messageRepository.findFirstPage( + chatId, + PageRequest.of(0, limit + 1) + ); + } else { + messages = messageRepository.findNextPage( + chatId, + position.createdAt(), + position.id(), + PageRequest.of(0, limit + 1) + ); + } + + boolean hasNext = messages.size() > limit; + + List page = hasNext + ? messages.subList(0, limit) + : messages; + + List items = page.stream() + .map(this::toResponse) + .toList(); + + String nextCursor = null; + + if (hasNext && !page.isEmpty()) { + Message last = page.get(page.size() - 1); + + nextCursor = cursorCodec.encodeMessage( + chatId, + last.getCreatedAt(), + last.getId() + ); + } + + return new MessagePageResponse(items, nextCursor); + } + + private void checkAccess(UUID userId, UUID chatId) { + boolean member = chatMemberRepository + .existsByIdChatIdAndIdUserId(chatId, userId); + + if (!member) { + throw BusinessException.notFound(); + } + } + + private MessageResponse toResponse(Message message) { + return new MessageResponse( + message.getId(), + message.getChat().getId(), + message.getSender().getId(), + message.getText(), + message.getCreatedAt(), + List.copyOf(message.getAttachmentIds()) + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/S3StorageService.java b/java/src/main/java/com/vnp/demo/service/S3StorageService.java new file mode 100644 index 00000000..f1d11856 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/S3StorageService.java @@ -0,0 +1,194 @@ +package com.vnp.demo.service; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.validation.ImageValidator; + +import io.minio.BucketExistsArgs; +import io.minio.GetObjectArgs; +import io.minio.ListObjectsArgs; +import io.minio.MinioClient; +import io.minio.PutObjectArgs; +import io.minio.RemoveObjectArgs; +import io.minio.Result; +import io.minio.StatObjectArgs; +import io.minio.StatObjectResponse; +import io.minio.errors.ErrorResponseException; +import io.minio.messages.Item; +import okhttp3.OkHttpClient; + +@Service +public class S3StorageService { + + private final MinioClient client; + private final String bucket; + + public S3StorageService( + @Value("${app.s3.endpoint}") String endpoint, + @Value("${app.s3.access-key}") String accessKey, + @Value("${app.s3.secret-key}") String secretKey, + @Value("${app.s3.bucket}") String bucket + ) { + this.bucket = bucket; + + OkHttpClient httpClient = new OkHttpClient.Builder() + .connectTimeout(2, TimeUnit.SECONDS) + .writeTimeout(10, TimeUnit.SECONDS) + .readTimeout(10, TimeUnit.SECONDS) + .callTimeout(30, TimeUnit.SECONDS) + .retryOnConnectionFailure(false) + .build(); + + this.client = MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region("us-east-1") + .httpClient(httpClient) + .build(); + } + + public void put( + String objectKey, + Path file, + long sizeBytes, + String contentType + ) { + if (sizeBytes < 1 + || sizeBytes > ImageValidator.MAX_BYTES) { + throw new BusinessException( + "payload_too_large", + "Некорректный размер загружаемого файла", + HttpStatus.PAYLOAD_TOO_LARGE + ); + } + + try (InputStream input = Files.newInputStream(file)) { + client.putObject( + PutObjectArgs.builder() + .bucket(bucket) + .object(objectKey) + .contentType(contentType) + .stream(input, sizeBytes, ImageValidator.MAX_BYTES) + .build() + ); + } catch (Exception e) { + throw unavailable(); + } + } + + public byte[] get( + String objectKey, + long expectedSize, + String expectedSha256 + ) { + if (expectedSize < 1 + || expectedSize > ImageValidator.MAX_BYTES) { + throw unavailable(); + } + + try (InputStream input = client.getObject( + GetObjectArgs.builder() + .bucket(bucket) + .object(objectKey) + .build() + )) { + byte[] bytes = input.readNBytes( + Math.toIntExact(expectedSize) + 1 + ); + + if (bytes.length != expectedSize + || !ImageValidator.sha256(bytes) + .equals(expectedSha256)) { + throw unavailable(); + } + + return bytes; + } catch (BusinessException e) { + throw e; + } catch (Exception e) { + throw unavailable(); + } + } + + public Iterable> listOriginals() { + return client.listObjects( + ListObjectsArgs.builder() + .bucket(bucket) + .prefix("originals/") + .recursive(true) + .build() + ); + } + + public Instant lastModifiedOrNull(String objectKey) { + try { + StatObjectResponse response = client.statObject( + StatObjectArgs.builder() + .bucket(bucket) + .object(objectKey) + .build() + ); + + return response.lastModified().toInstant(); + } catch (ErrorResponseException e) { + String code = e.errorResponse().code(); + + if ("NoSuchKey".equals(code) + || "NoSuchObject".equals(code)) { + return null; + } + + throw unavailable(); + } catch (Exception e) { + throw unavailable(); + } + } + + public void remove(String objectKey) { + try { + client.removeObject( + RemoveObjectArgs.builder() + .bucket(bucket) + .object(objectKey) + .build() + ); + } catch (Exception e) { + throw unavailable(); + } + } + + public void checkReady() { + try { + boolean exists = client.bucketExists( + BucketExistsArgs.builder() + .bucket(bucket) + .build() + ); + + if (!exists) { + throw unavailable(); + } + } catch (BusinessException e) { + throw e; + } catch (Exception e) { + throw unavailable(); + } + } + + private static BusinessException unavailable() { + return new BusinessException( + "unavailable", + "Хранилище файлов временно недоступно", + HttpStatus.SERVICE_UNAVAILABLE + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/SessionCacheService.java b/java/src/main/java/com/vnp/demo/service/SessionCacheService.java new file mode 100644 index 00000000..8f10beee --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/SessionCacheService.java @@ -0,0 +1,72 @@ +package com.vnp.demo.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.vnp.demo.SessionStoreUnavailableException; + +import org.springframework.dao.DataAccessException; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; + +@Service +public class SessionCacheService { + private static final String PREFIX = "session:"; + private final StringRedisTemplate redis; + private final ObjectMapper mapper; + + public SessionCacheService(StringRedisTemplate redis, ObjectMapper mapper) { + this.redis = redis; + this.mapper = mapper; + } + + public record CachedSession(UUID userId, Instant expiresAt) {} + + public Optional find(String tokenHash) { + try { + String json = redis.opsForValue().get(PREFIX + tokenHash); + if (json == null) return Optional.empty(); + CachedSession session = mapper.readValue(json, CachedSession.class); + if (!session.expiresAt().isAfter(Instant.now())) return Optional.empty(); + return Optional.of(session); + } catch (DataAccessException ex) { + throw unavailable(ex); + } catch (JsonProcessingException | IllegalArgumentException ex) { + throw unavailable(ex); + } + } + + public void cache(String tokenHash, UUID userId, Instant expiresAt) { + Duration ttl = Duration.between(Instant.now(), expiresAt); + if (ttl.isNegative() || ttl.isZero()) { + throw new IllegalStateException("Session already expired"); + } + try { + String json = mapper.writeValueAsString(new CachedSession(userId, expiresAt)); + redis.opsForValue().set(PREFIX + tokenHash, json, ttl); + } catch (DataAccessException ex) { + throw unavailable(ex); + } catch (JsonProcessingException ex) { + throw new IllegalStateException("Cannot serialize session", ex); + } + } + + public boolean invalidate(String tokenHash) { + try { + Boolean deleted = redis.delete(PREFIX + tokenHash); + return Boolean.TRUE.equals(deleted); + } catch (DataAccessException ex) { + throw new SessionStoreUnavailableException(ex); + } + } + + private ResponseStatusException unavailable(Exception ex) { + return new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "Session store unavailable", ex); + } +} diff --git a/java/src/main/java/com/vnp/demo/service/UploadLimiter.java b/java/src/main/java/com/vnp/demo/service/UploadLimiter.java new file mode 100644 index 00000000..338fb370 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/UploadLimiter.java @@ -0,0 +1,58 @@ +package com.vnp.demo.service; + +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; + +import com.vnp.demo.BusinessException; + +@Component +public class UploadLimiter { + + private final Semaphore semaphore; + + public UploadLimiter( + @Value("${app.uploads.max-concurrent:1}") + int maxConcurrent + ) { + if (maxConcurrent < 1) { + throw new IllegalArgumentException( + "max-concurrent должен быть больше нуля" + ); + } + + this.semaphore = new Semaphore(maxConcurrent); + } + + public Permit acquire() { + if (!semaphore.tryAcquire()) { + throw new BusinessException( + "rate_limited", + "Все обработчики загрузки заняты. Повторите запрос позже", + HttpStatus.TOO_MANY_REQUESTS + ); + } + + return new Permit(semaphore); + } + + public static final class Permit implements AutoCloseable { + + private final Semaphore semaphore; + private final AtomicBoolean released = new AtomicBoolean(false); + + private Permit(Semaphore semaphore) { + this.semaphore = semaphore; + } + + @Override + public void close() { + if (released.compareAndSet(false, true)) { + semaphore.release(); + } + } + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/UserService.java b/java/src/main/java/com/vnp/demo/service/UserService.java new file mode 100644 index 00000000..9a1163c0 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/UserService.java @@ -0,0 +1,86 @@ + +package com.vnp.demo.service; + +import java.util.UUID; + +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.dto.UserCreateRequest; +import com.vnp.demo.dto.UserResponse; +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.UserRepository; +import com.vnp.demo.validation.UserInputValidator; + +@Service +public class UserService { + + private final UserRepository userRepository; + private final PasswordEncoder passwordEncoder; + private final TransactionTemplate transactionTemplate; + + public UserService( + UserRepository userRepository, + PasswordEncoder passwordEncoder, + TransactionTemplate transactionTemplate + ) { + this.userRepository = userRepository; + this.passwordEncoder = passwordEncoder; + this.transactionTemplate = transactionTemplate; + } + + public UserResponse register(UserCreateRequest request) { + String username = request.username(); + String displayName = request.displayName(); + String password = request.password(); + + UserInputValidator.validate(displayName, password); + + String passwordHash = passwordEncoder.encode(password); + + try { + return transactionTemplate.execute(status -> { + + if (userRepository.existsByUsername(username)) { + throw BusinessException.conflict("Username already exists"); + } + + User user = new User( + username, + displayName, + passwordHash + ); + + User saved = userRepository.saveAndFlush(user); + + return toResponse(saved); + }); + + } catch (DataIntegrityViolationException ex) { + + throw BusinessException.conflict( + "Username already exists" + ); + } + } + + @org.springframework.transaction.annotation.Transactional(readOnly = true) + public UserResponse getCurrent(UUID userId) { + User user = userRepository.findById(userId) + .orElseThrow(BusinessException::notFound); + + return toResponse(user); + } + + public UserResponse toResponse(User user) { + return new UserResponse( + user.getId(), + user.getUsername(), + user.getDisplayName(), + user.getCreatedAt() + ); + } +} diff --git a/java/src/main/java/com/vnp/demo/validation/ImageValidator.java b/java/src/main/java/com/vnp/demo/validation/ImageValidator.java new file mode 100644 index 00000000..e345e212 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/validation/ImageValidator.java @@ -0,0 +1,481 @@ +package com.vnp.demo.validation; + +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.HexFormat; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.imageio.ImageIO; +import javax.imageio.ImageReader; +import javax.imageio.stream.FileImageInputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import com.vnp.demo.BusinessException; + +@Component +public class ImageValidator { + + public static final int MAX_BYTES = 10 * 1024 * 1024; + public static final long MAX_PIXELS = 40_000_000L; + + private static final int BUFFER_SIZE = 64 * 1024; + + private static final long MAX_RECEIVE_NANOS = + TimeUnit.SECONDS.toNanos(60); + + private static final long TEMP_FILE_GRACE_SECONDS = 3600; + + private static final Logger log = + LoggerFactory.getLogger(ImageValidator.class); + + private static final Set ACTIVE_FILES = + ConcurrentHashMap.newKeySet(); + + private final Path tempDirectory; + + public ImageValidator( + @Value("${app.uploads.temp-dir:/tmp/messenger-uploads}") + String tempDirectory + ) { + this.tempDirectory = Path.of(tempDirectory) + .toAbsolutePath() + .normalize(); + + try { + Files.createDirectories(this.tempDirectory); + } catch (IOException e) { + throw new IllegalStateException( + "Не удалось создать каталог временных загрузок", + e + ); + } + } + + public record ValidatedImage( + Path path, + long sizeBytes, + String contentType, + String sha256 + ) implements AutoCloseable { + + @Override + public void close() { + deleteTemporaryFile(path); + } + } + + public void validateFilename(String filename) { + if (filename == null) { + throw BusinessException.invalid( + "Не указано имя файла" + ); + } + + int length = filename.codePointCount( + 0, + filename.length() + ); + + if (length < 1 || length > 255) { + throw BusinessException.invalid( + "Имя файла должно содержать от 1 до 255 символов" + ); + } + + boolean unsafe = filename.codePoints().anyMatch( + codePoint -> codePoint == '/' + || codePoint == '\\' + || Character.isISOControl(codePoint) + ); + + if (unsafe) { + throw BusinessException.invalid( + "Имя файла содержит недопустимые символы" + ); + } + } + + public ValidatedImage validate( + InputStream input, + long contentLength, + String declaredContentType + ) { + if (contentLength > MAX_BYTES) { + throw tooLarge( + "Размер файла превышает 10 MiB" + ); + } + + String declaredMime = normalizeMime( + declaredContentType + ); + + Path temporaryFile = createTemporaryFile(); + + boolean success = false; + + try { + MessageDigest digest = newSha256(); + + long size = copyLimited( + input, + temporaryFile, + digest + ); + + if (size == 0) { + throw unsupported(); + } + + if (contentLength >= 0 && size != contentLength) { + throw BusinessException.invalid( + "Тело запроса получено не полностью" + ); + } + + String actualMime = validateImageContent( + temporaryFile + ); + + if (!actualMime.equals(declaredMime)) { + throw unsupported(); + } + + String checksum = HexFormat.of() + .formatHex(digest.digest()); + + ValidatedImage result = new ValidatedImage( + temporaryFile, + size, + actualMime, + checksum + ); + + success = true; + + return result; + } finally { + if (!success) { + deleteTemporaryFile(temporaryFile); + } + } + } + + private Path createTemporaryFile() { + try { + Path file = Files.createTempFile( + tempDirectory, + "upload-", + ".tmp" + ); + + ACTIVE_FILES.add(file); + + return file; + } catch (IOException e) { + throw temporaryStorageUnavailable(); + } + } + + private long copyLimited( + InputStream input, + Path destination, + MessageDigest digest + ) { + byte[] buffer = new byte[BUFFER_SIZE]; + long total = 0; + long started = System.nanoTime(); + + try (OutputStream output = + Files.newOutputStream(destination)) { + + while (true) { + checkReceiveDeadline(started); + + int requested = (int) Math.min( + buffer.length, + (long) MAX_BYTES - total + 1 + ); + + int count; + + try { + count = input.read(buffer, 0, requested); + } catch (IOException e) { + throw BusinessException.invalid( + "Передача файла прервана или превышено время ожидания" + ); + } + + checkReceiveDeadline(started); + + if (count == -1) { + break; + } + + if (count == 0) { + continue; + } + + total += count; + + if (total > MAX_BYTES) { + throw tooLarge( + "Размер файла превышает 10 MiB" + ); + } + + output.write(buffer, 0, count); + digest.update(buffer, 0, count); + } + + return total; + } catch (IOException e) { + throw temporaryStorageUnavailable(); + } + } + + private static void checkReceiveDeadline(long started) { + if (System.nanoTime() - started >= MAX_RECEIVE_NANOS) { + throw BusinessException.invalid( + "Превышено допустимое время приёма файла" + ); + } + } + + private String normalizeMime(String value) { + if (value == null) { + throw unsupported(); + } + + try { + MediaType type = MediaType.parseMediaType(value); + + if (type.getType().equalsIgnoreCase("image") + && type.getSubtype().equalsIgnoreCase("png")) { + return "image/png"; + } + + if (type.getType().equalsIgnoreCase("image") + && type.getSubtype().equalsIgnoreCase("jpeg")) { + return "image/jpeg"; + } + } catch (IllegalArgumentException e) { + throw unsupported(); + } + + throw unsupported(); + } + + private String validateImageContent(Path file) { + try (FileImageInputStream imageInput = + new FileImageInputStream(file.toFile())) { + + Iterator readers = + ImageIO.getImageReaders(imageInput); + + if (!readers.hasNext()) { + throw unsupported(); + } + + ImageReader reader = readers.next(); + + try { + reader.setInput(imageInput, true, true); + + String format = reader.getFormatName(); + String mime; + + if ("png".equalsIgnoreCase(format)) { + mime = "image/png"; + } else if ("jpeg".equalsIgnoreCase(format) + || "jpg".equalsIgnoreCase(format)) { + mime = "image/jpeg"; + } else { + throw unsupported(); + } + + int width = reader.getWidth(0); + int height = reader.getHeight(0); + + if (width <= 0 || height <= 0) { + throw unsupported(); + } + + long pixels = (long) width * height; + + if (pixels > MAX_PIXELS) { + throw tooLarge( + "Изображение содержит более 40 миллионов пикселей" + ); + } + + AtomicBoolean warning = new AtomicBoolean(false); + + reader.addIIOReadWarningListener( + (source, message) -> warning.set(true) + ); + + BufferedImage image = reader.read(0); + + if (image == null) { + throw unsupported(); + } + + try { + if (warning.get()) { + throw unsupported(); + } + } finally { + image.flush(); + } + + return mime; + } finally { + reader.dispose(); + } + } catch (BusinessException e) { + throw e; + } catch (IOException | RuntimeException e) { + throw unsupported(); + } + } + + @Scheduled( + initialDelay = 60_000, + fixedDelay = 300_000 + ) + public void cleanupTemporaryFiles() { + Instant cutoff = Instant.now() + .minusSeconds(TEMP_FILE_GRACE_SECONDS); + + long deleted = 0; + + try (DirectoryStream files = + Files.newDirectoryStream( + tempDirectory, + "upload-*.tmp" + )) { + + for (Path file : files) { + if (ACTIVE_FILES.contains(file)) { + continue; + } + + if (!Files.isRegularFile( + file, + LinkOption.NOFOLLOW_LINKS + )) { + continue; + } + + try { + Instant modified = Files.getLastModifiedTime( + file, + LinkOption.NOFOLLOW_LINKS + ).toInstant(); + + if (!modified.isBefore(cutoff)) { + continue; + } + + if (Files.deleteIfExists(file)) { + deleted++; + } + } catch (IOException e) { + log.warn( + "Не удалось очистить временный файл {}", + file.getFileName(), + e + ); + } + } + + log.info( + "Temporary upload cleanup completed: deleted={}", + deleted + ); + } catch (IOException e) { + log.warn( + "Не удалось просмотреть каталог временных загрузок", + e + ); + } + } + + private static MessageDigest newSha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException( + "SHA-256 недоступен", + e + ); + } + } + + public static String sha256(byte[] bytes) { + return HexFormat.of().formatHex( + newSha256().digest(bytes) + ); + } + + private static void deleteTemporaryFile(Path file) { + try { + Files.deleteIfExists(file); + } catch (IOException e) { + log.warn( + "Не удалось удалить временный файл {}", + file.getFileName(), + e + ); + } finally { + + ACTIVE_FILES.remove(file); + } + } + + private static BusinessException unsupported() { + return new BusinessException( + "unsupported_media_type", + "Ожидается корректное изображение PNG или JPEG", + HttpStatus.UNSUPPORTED_MEDIA_TYPE + ); + } + + private static BusinessException tooLarge(String message) { + return new BusinessException( + "payload_too_large", + message, + HttpStatus.PAYLOAD_TOO_LARGE + ); + } + + private static BusinessException temporaryStorageUnavailable() { + return new BusinessException( + "unavailable", + "Временное хранилище загрузок недоступно", + HttpStatus.SERVICE_UNAVAILABLE + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/validation/UserInputValidator.java b/java/src/main/java/com/vnp/demo/validation/UserInputValidator.java new file mode 100644 index 00000000..229cb83e --- /dev/null +++ b/java/src/main/java/com/vnp/demo/validation/UserInputValidator.java @@ -0,0 +1,32 @@ +package com.vnp.demo.validation; + +import com.vnp.demo.BusinessException; + +public final class UserInputValidator { + + private UserInputValidator() { + } + + public static void validate(String displayName, String password) { + + if (displayName == null + || displayName.isBlank() + || codePoints(displayName) > 100) { + throw BusinessException.invalid( + "Некорректное отображаемое имя" + ); + } + + if (password == null + || codePoints(password) < 12 + || codePoints(password) > 128) { + throw BusinessException.invalid( + "Пароль должен содержать от 12 до 128 символов" + ); + } + } + + private static int codePoints(String value) { + return value.codePointCount(0, value.length()); + } +} \ No newline at end of file diff --git a/java/src/main/resources/application.yml b/java/src/main/resources/application.yml new file mode 100644 index 00000000..8989eec2 --- /dev/null +++ b/java/src/main/resources/application.yml @@ -0,0 +1,65 @@ +server: + shutdown: graceful + tomcat: + threads: + max: 64 + connection-timeout: 2s +spring: + application: + name: messenger + lifecycle: + timeout-per-shutdown-phase: 15s + jackson: + default-property-inclusion: always + deserialization: + fail-on-unknown-properties: true + fail-on-trailing-tokens: true + mapper: + allow-coercion-of-scalars: false + datasource: + url: ${DB_URL:jdbc:postgresql://localhost:5432/messenger} + username: ${DB_USERNAME:messenger} + password: ${DB_PASSWORD:messenger} + hikari: + maximum-pool-size: ${DB_POOL_SIZE:8} + minimum-idle: 1 + connection-timeout: 1000 + validation-timeout: 500 + data-source-properties: + connectTimeout: 2 + socketTimeout: 8 + options: "-c statement_timeout=5000 -c lock_timeout=3000" + data: + redis: + host: ${REDIS_HOST:localhost} + port: 6379 + password: ${REDIS_PASSWORD:} + timeout: 1s + connect-timeout: 1s + flyway: + enabled: ${FLYWAY_ENABLED:true} + jpa: + hibernate: + ddl-auto: validate + open-in-view: false + transaction: + default-timeout: 8s +app: + session: + ttl-seconds: 86400 + + s3: + endpoint: ${S3_ENDPOINT:http://localhost:9000} + access-key: ${S3_ACCESS_KEY:} + secret-key: ${S3_SECRET_KEY:} + bucket: ${S3_BUCKET:messenger-attachments} + + uploads: + max-concurrent: ${UPLOAD_MAX_CONCURRENT:1} + temp-dir: ${UPLOAD_TEMP_DIR:/tmp/messenger-uploads} + + cleanup: + enabled: ${ATTACHMENT_CLEANUP_ENABLED:true} + grace-seconds: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:3600} + interval-ms: ${ATTACHMENT_CLEANUP_INTERVAL_MS:300000} + initial-delay-ms: ${ATTACHMENT_CLEANUP_INITIAL_DELAY_MS:60000} \ No newline at end of file diff --git a/java/src/main/resources/db/migration/V1__create_tables.sql b/java/src/main/resources/db/migration/V1__create_tables.sql new file mode 100644 index 00000000..4688e46f --- /dev/null +++ b/java/src/main/resources/db/migration/V1__create_tables.sql @@ -0,0 +1,77 @@ + +CREATE TABLE users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(400) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT chk_username + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT chk_display_name + CHECK ( + char_length(display_name) BETWEEN 1 AND 100 + AND length(btrim(display_name)) > 0 + ) +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title VARCHAR(400) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT chk_chat_title + CHECK ( + char_length(title) BETWEEN 1 AND 100 + AND length(btrim(title)) > 0 + ) +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id), + user_id UUID NOT NULL REFERENCES users(id), + + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + sender_id UUID NOT NULL, + text VARCHAR(16000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT fk_message_membership + FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members(chat_id, user_id), + + CONSTRAINT chk_message_text + CHECK ( + char_length(text) BETWEEN 1 AND 4000 + AND length(btrim(text)) > 0 + ) +); + +CREATE TABLE sessions ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id), + token_hash CHAR(64) NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ, + + CONSTRAINT chk_session_expiry + CHECK (expires_at > created_at) +); + +CREATE INDEX idx_chat_members_user + ON chat_members(user_id, chat_id); + +CREATE INDEX idx_chats_history + ON chats(created_at, id); + +CREATE INDEX idx_messages_history + ON messages(chat_id, created_at, id); + +CREATE INDEX idx_sessions_user + ON sessions(user_id); diff --git a/java/src/main/resources/db/migration/V2__message_idempotency.sql b/java/src/main/resources/db/migration/V2__message_idempotency.sql new file mode 100644 index 00000000..63cdd87b --- /dev/null +++ b/java/src/main/resources/db/migration/V2__message_idempotency.sql @@ -0,0 +1,9 @@ +CREATE TABLE message_idempotency ( + user_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + key VARCHAR(128) COLLATE "C" NOT NULL, + request_text TEXT NOT NULL, + message_id UUID NOT NULL UNIQUE REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, key) +); diff --git a/java/src/main/resources/db/migration/V3__attachments.sql b/java/src/main/resources/db/migration/V3__attachments.sql new file mode 100644 index 00000000..56fcbe46 --- /dev/null +++ b/java/src/main/resources/db/migration/V3__attachments.sql @@ -0,0 +1,67 @@ +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + owner_id UUID NOT NULL REFERENCES users(id), + + filename TEXT NOT NULL, + content_type VARCHAR(32) NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 CHAR(64) NOT NULL, + + object_key TEXT NOT NULL UNIQUE, + status VARCHAR(16) NOT NULL DEFAULT 'ready', + + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + + CONSTRAINT chk_attachment_filename + CHECK (char_length(filename) BETWEEN 1 AND 255), + + CONSTRAINT chk_attachment_content_type + CHECK (content_type IN ('image/png', 'image/jpeg')), + + CONSTRAINT chk_attachment_size + CHECK (size_bytes BETWEEN 1 AND 10485760), + + CONSTRAINT chk_attachment_sha256 + CHECK (sha256 ~ '^[a-f0-9]{64}$'), + + CONSTRAINT chk_attachment_status + CHECK (status = 'ready'), + + CONSTRAINT fk_attachment_membership + FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members(chat_id, user_id) +); + +CREATE INDEX idx_attachments_chat + ON attachments(chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id), + position INTEGER NOT NULL, + + PRIMARY KEY (message_id, position), + + CONSTRAINT uq_message_attachment + UNIQUE (message_id, attachment_id), + + CONSTRAINT chk_attachment_position + CHECK (position BETWEEN 0 AND 9) +); + +CREATE INDEX idx_message_attachments_attachment + ON message_attachments(attachment_id); + + +ALTER TABLE messages + DROP CONSTRAINT chk_message_text; + +ALTER TABLE messages + ADD CONSTRAINT chk_message_text + CHECK (char_length(text) BETWEEN 0 AND 4000); + +-- Старые запросы не содержали вложений, поэтому для них значение пустое. +ALTER TABLE message_idempotency + ADD COLUMN request_attachment_ids TEXT NOT NULL DEFAULT ''; \ No newline at end of file diff --git a/java/src/test/java/com/vnp/demo/ConcurrentRegistrationTest.java b/java/src/test/java/com/vnp/demo/ConcurrentRegistrationTest.java new file mode 100644 index 00000000..7468327e --- /dev/null +++ b/java/src/test/java/com/vnp/demo/ConcurrentRegistrationTest.java @@ -0,0 +1,89 @@ +package com.vnp.demo; + +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.http.ResponseEntity; + +import com.vnp.demo.dto.UserCreateRequest; +import com.vnp.demo.repositories.UserRepository; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class ConcurrentRegistrationTest { + + @Autowired + private TestRestTemplate restTemplate; + + @Autowired + private UserRepository userRepository; + + @Test + void concurrentRegistrationAllowsOnlyOneUser() throws Exception { + String username = "race_" + + UUID.randomUUID().toString().substring(0, 8); + + UserCreateRequest request = new UserCreateRequest( + username, + "Race Test", + "Password123!" + ); + + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch start = new CountDownLatch(1); + + try { + Callable> task = () -> { + start.await(); + + return restTemplate.postForEntity( + "/api/v1/users", + request, + String.class + ); + }; + + Future> first = + executor.submit(task); + + Future> second = + executor.submit(task); + + start.countDown(); + + int status1 = first.get(15, TimeUnit.SECONDS) + .getStatusCode().value(); + + int status2 = second.get(15, TimeUnit.SECONDS) + .getStatusCode().value(); + + assertTrue( + (status1 == 201 && status2 == 409) + || (status1 == 409 && status2 == 201), + "Expected 201 and 409, got " + + status1 + " and " + status2 + ); + + assertEquals( + 1, + userRepository.findAll().stream() + .filter(user -> + username.equals(user.getUsername())) + .count() + ); + + } finally { + executor.shutdownNow(); + } + } +} diff --git a/java/src/test/java/com/vnp/demo/InfrastructureFailureIntegrationTest.java b/java/src/test/java/com/vnp/demo/InfrastructureFailureIntegrationTest.java new file mode 100644 index 00000000..f2e64cab --- /dev/null +++ b/java/src/test/java/com/vnp/demo/InfrastructureFailureIntegrationTest.java @@ -0,0 +1,155 @@ + +package com.vnp.demo; + +import java.time.Instant; +import java.util.UUID; + +import static org.hamcrest.Matchers.equalTo; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.dao.DataAccessResourceFailureException; +import org.springframework.security.crypto.password.PasswordEncoder; + +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.UserRepository; +import com.vnp.demo.service.SessionCacheService; + +import io.restassured.RestAssured; +import static io.restassured.RestAssured.given; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class InfrastructureFailureIntegrationTest { + + @LocalServerPort + private int port; + + @SpyBean + private UserRepository userRepository; + + @Autowired + private PasswordEncoder passwordEncoder; + + @SpyBean + private SessionCacheService sessionCacheService; + + private String username; + + private final String password = "Password123!"; + + @BeforeEach + void setUp() { + RestAssured.baseURI = "http://localhost"; + RestAssured.port = port; + + username = "failure_" + UUID.randomUUID() + .toString() + .substring(0, 8); + + User user = new User( + username, + "Failure Test", + passwordEncoder.encode(password) + ); + + userRepository.saveAndFlush(user); + } + + @Test + void redisUnavailableDuringLoginReturns503() { + doThrow(new DataAccessResourceFailureException( + "Redis unavailable" + )).when(sessionCacheService).cache( + anyString(), + any(UUID.class), + any(Instant.class) + ); + + given() + .auth() + .preemptive() + .basic(username, password) + .when() + .post("/api/v1/auth/sessions") + .then() + .statusCode(503) + .body( + "error.code", + equalTo("service_unavailable") + ); + } + + @Test + void anonymousUserCannotBypassAuthentication() { + given() + .header( + "X-User-Id", + UUID.randomUUID().toString() + ) + .when() + .get("/api/v1/users/me") + .then() + .statusCode(401); + } + + @Test + void redisUnavailableDuringAuthenticationDoesNotGrantAccess() { + String token = given() + .auth() + .preemptive() + .basic(username, password) + .when() + .post("/api/v1/auth/sessions") + .then() + .statusCode(201) + .extract() + .path("token"); + + doThrow(new DataAccessResourceFailureException( + "Redis unavailable" + )).when(sessionCacheService).find(anyString()); + + given() + .header( + "Authorization", + "Bearer " + token + ) + .when() + .get("/api/v1/users/me") + .then() + .statusCode(503) + .body( + "error.code", + equalTo("service_unavailable") + ); + } + + @Test + void databaseUnavailableDuringLoginReturns503() { + doThrow(new DataAccessResourceFailureException( + "PostgreSQL unavailable" + )).when(userRepository).findByUsername( + eq(username) + ); + + given() + .auth() + .preemptive() + .basic(username, password) + .when() + .post("/api/v1/auth/sessions") + .then() + .statusCode(503) + .body( + "error.code", + equalTo("service_unavailable") + ); + } +} diff --git a/java/src/test/java/com/vnp/demo/SessionIntegrationTest.java b/java/src/test/java/com/vnp/demo/SessionIntegrationTest.java new file mode 100644 index 00000000..a30b8480 --- /dev/null +++ b/java/src/test/java/com/vnp/demo/SessionIntegrationTest.java @@ -0,0 +1,154 @@ + +package com.vnp.demo; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.UUID; + +import static org.hamcrest.Matchers.equalTo; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.security.crypto.password.PasswordEncoder; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.UserRepository; + +import io.restassured.RestAssured; +import static io.restassured.RestAssured.given; + +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = "app.session.ttl-seconds=2" +) +class SessionIntegrationTest { + + private static final String PASSWORD = "TestPassword123!"; + + @LocalServerPort + private int port; + + @Autowired + private ObjectMapper mapper; + + @Autowired + private UserRepository userRepository; + + @Autowired + private PasswordEncoder passwordEncoder; + + private String username; + + @BeforeEach + void setUp() { + RestAssured.baseURI = "http://localhost"; + RestAssured.port = port; + + username = "test_" + UUID.randomUUID() + .toString().substring(0, 8); + + userRepository.saveAndFlush( + new User( + username, + "Test User", + passwordEncoder.encode(PASSWORD) + ) + ); + } + + private String login(String password) throws Exception { + String credentials = username + ":" + password; + + String basic = Base64.getEncoder().encodeToString( + credentials.getBytes(StandardCharsets.UTF_8) + ); + + var response = given() + .header("Authorization", "Basic " + basic) + .post("/api/v1/auth/sessions"); + + response.then().log().all(); + + response.then().statusCode(201); + + JsonNode json = mapper.readTree(response.asString()); + return json.get("token").asText(); + } + @Test + void validSessionAllowsAccess() throws Exception { + String token = login(PASSWORD); + + given() + .header("Authorization", "Bearer " + token) + .get("/api/v1/users/me") + .then() + .statusCode(200) + .body("username", equalTo(username)); + } + + @Test + void expiredSessionReturns401() throws Exception { + String token = login(PASSWORD); + + Thread.sleep(3000); + + given() + .header("Authorization", "Bearer " + token) + .get("/api/v1/users/me") + .then() + .statusCode(401); + } + + @Test + void logoutRevokesSession() throws Exception { + String token = login(PASSWORD); + + given() + .header("Authorization", "Bearer " + token) + .delete("/api/v1/auth/sessions/current") + .then() + .statusCode(204); + + given() + .header("Authorization", "Bearer " + token) + .get("/api/v1/users/me") + .then() + .statusCode(401); + } + + @Test + void wrongPasswordReturns401() { + String credentials = username + ":WrongPassword123!"; + + String basic = Base64.getEncoder().encodeToString( + credentials.getBytes(StandardCharsets.UTF_8) + ); + + given() + .header("Authorization", "Basic " + basic) + .post("/api/v1/auth/sessions") + .then() + .statusCode(401); + } + + @Test + void missingTokenReturns401() { + given() + .get("/api/v1/users/me") + .then() + .statusCode(401); + } + + @Test + void fakeUserIdHeaderCannotAuthenticate() { + given() + .header("X-User-Id", UUID.randomUUID().toString()) + .get("/api/v1/users/me") + .then() + .statusCode(401); + } +} diff --git a/java/src/test/java/com/vnp/demo/TransactionRollbackTest.java b/java/src/test/java/com/vnp/demo/TransactionRollbackTest.java new file mode 100644 index 00000000..6285d349 --- /dev/null +++ b/java/src/test/java/com/vnp/demo/TransactionRollbackTest.java @@ -0,0 +1,73 @@ +package com.vnp.demo; + +import com.vnp.demo.dto.ChatCreateRequest; +import com.vnp.demo.models.ChatMember; +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.ChatMemberRepository; +import com.vnp.demo.repositories.ChatRepository; +import com.vnp.demo.repositories.UserRepository; +import com.vnp.demo.service.ChatService; + + +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +@SpringBootTest +class TransactionRollbackTest { + + @Autowired + private ChatService chatService; + + @Autowired + private ChatRepository chatRepository; + + @Autowired + private UserRepository userRepository; + + @SpyBean + private ChatMemberRepository chatMemberRepository; + + @Test + void shouldRollbackChatCreationWhenMemberSaveFails() { + User user = userRepository.saveAndFlush( + new User( + "rollback_" + UUID.randomUUID() + .toString() + .substring(0, 8), + "Rollback Test", + "test-password-hash" + ) + ); + + long chatsBefore = chatRepository.count(); + + doThrow(new IllegalStateException("Simulated failure")) + .when(chatMemberRepository) + .save(any(ChatMember.class)); + + ChatCreateRequest request = new ChatCreateRequest( + "Rollback test chat", + List.of(user.getId()) + ); + + assertThrows( + IllegalStateException.class, + () -> chatService.create(user.getId(), request) + ); + + assertEquals( + chatsBefore, + chatRepository.count(), + "Chat must not be saved after transaction rollback" + ); + } +} diff --git a/java/src/test/java/com/vnp/demo/sql/message_index_benchmark.sql b/java/src/test/java/com/vnp/demo/sql/message_index_benchmark.sql new file mode 100644 index 00000000..81217354 --- /dev/null +++ b/java/src/test/java/com/vnp/demo/sql/message_index_benchmark.sql @@ -0,0 +1,89 @@ +BEGIN; + +INSERT INTO users ( + id, + username, + display_name, + password_hash +) +VALUES ( + '11111111-1111-4111-8111-111111111111', + 'benchmark_user', + 'Benchmark User', + 'benchmark-hash' +) +ON CONFLICT (id) DO NOTHING; + +INSERT INTO chats ( + id, + title +) +VALUES ( + '22222222-2222-4222-8222-222222222222', + 'Benchmark Chat' +) +ON CONFLICT (id) DO NOTHING; + +INSERT INTO chat_members ( + chat_id, + user_id +) +VALUES ( + '22222222-2222-4222-8222-222222222222', + '11111111-1111-4111-8111-111111111111' +) +ON CONFLICT DO NOTHING; + +INSERT INTO messages ( + id, + chat_id, + sender_id, + text, + created_at +) +SELECT + ( + '33333333-3333-4333-8333-' || + lpad(g::text, 12, '0') + )::uuid, + '22222222-2222-4222-8222-222222222222'::uuid, + '11111111-1111-4111-8111-111111111111'::uuid, + 'Benchmark message ' || g, + NOW() - INTERVAL '1 day' + + g * INTERVAL '1 second' +FROM generate_series(1, 10000) AS g +ON CONFLICT (id) DO NOTHING; + +COMMIT; + +ANALYZE messages; + +SELECT COUNT(*) +FROM messages +WHERE chat_id = + '22222222-2222-4222-8222-222222222222'; + +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = + '22222222-2222-4222-8222-222222222222' +ORDER BY created_at, id +LIMIT 50; + +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = + '22222222-2222-4222-8222-222222222222' + AND (created_at, id) > ( + ( + SELECT created_at + FROM messages + WHERE id = + '33333333-3333-4333-8333-000000000050' + ), + '33333333-3333-4333-8333-000000000050'::uuid + ) +ORDER BY created_at, id +LIMIT 50; \ No newline at end of file diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100644 index 00000000..709175ad --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,293 @@ +import argparse +import json +from pathlib import Path +import subprocess +import sys +from urllib.parse import unquote, urlsplit +import uuid + + +ROOT = Path(__file__).resolve().parents[1] + + +def output(args): + result = subprocess.run( + args, + cwd=ROOT, + text=True, + capture_output=True, + timeout=30, + ) + + if result.returncode: + raise RuntimeError( + "Docker command failed; " + "check Docker and running services" + ) + + return result.stdout.strip() + + +def service(name): + ids = output([ + "docker", "compose", "ps", "-q", name, + ]).splitlines() + + if not ids: + raise RuntimeError( + "Service is not running: " + name + ) + + return json.loads(output([ + "docker", "inspect", ids[0], + ]))[0] + + +def environment(container): + return dict( + item.split("=", 1) + for item in container["Config"]["Env"] + if "=" in item + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Reconcile PostgreSQL attachment metadata and S3" + ) + + modes = parser.add_mutually_exclusive_group() + modes.add_argument("--apply", action="store_true") + modes.add_argument("--dry-run", action="store_true") + + parser.add_argument( + "--chat-id", + type=lambda value: str(uuid.UUID(value)), + default=None, + ) + + parser.add_argument( + "--grace-seconds", + type=int, + default=3600, + ) + + parser.add_argument( + "--max-seconds", + type=int, + default=600, + ) + + parser.add_argument( + "--backup-dir", + default="backups/attachments", + ) + + parser.add_argument( + "--output", + default="results/reconcile.json", + ) + + args = parser.parse_args() + + if args.grace_seconds < 60 or args.max_seconds < 1: + parser.error( + "grace-seconds >= 60; max-seconds >= 1" + ) + + api = service("api") + postgres = service("postgres") + minio = service("minio") + + env = environment(api) + pg = environment(postgres) + + common = ( + set(api["NetworkSettings"]["Networks"]) + & set(postgres["NetworkSettings"]["Networks"]) + & set(minio["NetworkSettings"]["Networks"]) + ) + + if len(common) != 1: + raise RuntimeError( + "Expected exactly one common Docker network " + "for API, PostgreSQL and MinIO" + ) + + network = next(iter(common)) + + database_url = ( + env.get("SPRING_DATASOURCE_URL") + or env.get("DB_URL") + or env.get("DATABASE_URL") + or "" + ) + + if database_url: + address = urlsplit( + database_url.removeprefix("jdbc:") + ) + + if address.scheme not in ("postgresql", "postgres"): + raise RuntimeError( + "Unsupported database URL scheme" + ) + + db_host = address.hostname + db_port = address.port or 5432 + db_name = unquote(address.path.lstrip("/")) + + url_user = ( + unquote(address.username) + if address.username else None + ) + + url_password = ( + unquote(address.password) + if address.password else None + ) + + else: + db_host = env.get("DB_HOST") or "postgres" + db_port = int(env.get("DB_PORT") or "5432") + + db_name = ( + env.get("DB_NAME") + or pg.get("POSTGRES_DB") + or pg.get("POSTGRES_USER") + or "postgres" + ) + + url_user = None + url_password = None + + db_user = ( + env.get("SPRING_DATASOURCE_USERNAME") + or env.get("DB_USERNAME") + or env.get("DB_USER") + or url_user + or pg.get("POSTGRES_USER") + ) + + db_password = ( + env.get("SPRING_DATASOURCE_PASSWORD") + or env.get("DB_PASSWORD") + or url_password + or pg.get("POSTGRES_PASSWORD") + ) + + if not db_host or not db_name: + raise RuntimeError( + "Cannot determine PostgreSQL host or database" + ) + + if not db_user or not db_password: + raise RuntimeError( + "Database credentials are not available " + "in container environment" + ) + + required = [ + "S3_ENDPOINT", + "S3_ACCESS_KEY", + "S3_SECRET_KEY", + "S3_BUCKET", + ] + + if any(not env.get(key) for key in required): + raise RuntimeError( + "Missing S3 settings in API container environment" + ) + + config = { + "apply": args.apply, + "chat_id": args.chat_id, + "grace_seconds": args.grace_seconds, + "max_seconds": args.max_seconds, + "bucket": env["S3_BUCKET"], + "db": { + "host": db_host, + "port": db_port, + "dbname": db_name, + "user": db_user, + "password": db_password, + }, + "s3": { + "endpoint_url": env["S3_ENDPOINT"], + "aws_access_key_id": env["S3_ACCESS_KEY"], + "aws_secret_access_key": env["S3_SECRET_KEY"], + "region_name": "us-east-1", + }, + } + + backups = (ROOT / args.backup_dir).resolve() + backups.mkdir(parents=True, exist_ok=True) + + command = [ + "docker", "run", + "--rm", + "-i", + "--network", network, + "--read-only", + "--tmpfs", "/tmp:rw,noexec,nosuid,size=16m", + "--memory", "256m", + "--cpus", "1", + "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", + "--mount", + ( + f"type=bind,source={backups}," + "target=/backups,readonly" + ), + "mtusi4-reconciler:local", + ] + + result = subprocess.run( + command, + cwd=ROOT, + input=json.dumps(config), + text=True, + capture_output=True, + ) + + try: + report = json.loads(result.stdout) + + except json.JSONDecodeError: + raise RuntimeError( + "Worker did not return JSON; " + "check image build and backup directory permissions" + ) from None + + target = ROOT / args.output + target.parent.mkdir(parents=True, exist_ok=True) + + rendered = json.dumps( + report, + ensure_ascii=False, + indent=2, + ) + + target.write_text( + rendered + "\n", + encoding="utf-8", + ) + + print(rendered) + print("Report saved:", target, file=sys.stderr) + + return result.returncode + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + + except ( + RuntimeError, + ValueError, + OSError, + subprocess.TimeoutExpired, + ) as error: + print("FAIL:", str(error), file=sys.stderr) + raise SystemExit(1) \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..2422397f --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,48 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +mkdir -p .local-logs +exec > .local-logs/restart.log 2>&1 + +set -x + +date -u + +echo "Stopping API and load balancer" +docker compose stop -t 10 api lb + +echo "Restarting persistent services" +docker compose restart -t 10 postgres redis minio + +echo "Waiting for persistent services" +docker compose up -d \ + --no-build \ + --pull never \ + --no-deps \ + --no-recreate \ + --wait \ + --wait-timeout 25 \ + postgres redis minio + +echo "Starting API" +docker compose up -d \ + --no-build \ + --pull never \ + --no-deps \ + --no-recreate \ + --wait \ + --wait-timeout 45 \ + api + +echo "Starting load balancer without dependencies" +docker compose up -d \ + --no-build \ + --pull never \ + --no-deps \ + --no-recreate \ + lb + +echo "Restart completed" +date -u \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..d8c94e95 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +: "${TARGET_INSTANCE_ID:?Не задан TARGET_INSTANCE_ID}" + +compose() { + docker compose --env-file .env -f compose.yml "$@" +} + +containers=$(compose ps --status running -q api) + +target="" +count=0 + +for container in $containers; do + count=$((count + 1)) + + instance_id=$(docker inspect \ + --format '{{.Config.Hostname}}' "$container") + + if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then + if [ -n "$target" ]; then + printf '%s\n' 'Ошибка: найдено несколько подходящих реплик.' >&2 + exit 1 + fi + + target="$container" + fi +done + +if [ "$count" -lt 2 ]; then + printf '%s\n' 'Ошибка: должны работать минимум две API-реплики.' >&2 + exit 1 +fi + +if [ -z "$target" ]; then + printf '%s\n' 'Ошибка: указанная API-реплика не найдена.' >&2 + exit 1 +fi + + +docker kill --signal=KILL "$target" >/dev/null + +printf '%s\n' "Остановлена API-реплика: $TARGET_INSTANCE_ID" diff --git a/tests/attachments_negative.py b/tests/attachments_negative.py new file mode 100644 index 00000000..4637ad35 --- /dev/null +++ b/tests/attachments_negative.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 + +import hashlib +import os +import struct +import uuid +import zlib +from pathlib import Path +from urllib.parse import quote + +from client import Client, arguments, check, run + + +MAX_BYTES = 10 * 1024 * 1024 + +FIXTURE = ( + Path(__file__).resolve().parent + / "fixtures" + / "sample.png" +) + + +def make_png(width, height, noisy=False): + """Отдельное изображение для проверки лимита байтов или пикселей.""" + + def chunk(kind, data): + return ( + struct.pack(">I", len(data)) + + kind + + data + + struct.pack( + ">I", + zlib.crc32(kind + data) & 0xFFFFFFFF, + ) + ) + + compressor = zlib.compressobj(level=1) + compressed = bytearray() + + for _ in range(height): + row = os.urandom(width) if noisy else bytes(width) + compressed.extend(compressor.compress(b"\x00" + row)) + + compressed.extend(compressor.flush()) + + header = struct.pack( + ">IIBBBBB", + width, height, + 8, 0, 0, 0, 0, + ) + + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", header) + + chunk(b"IDAT", bytes(compressed)) + + chunk(b"IEND", b"") + ) + + +def main(): + args = arguments("Attachment negative tests") + c = Client(args.base_url, args.ca_file, timeout=30) + + alice, bob, eve, chat = c.bootstrap() + chat_path = "/api/v1/chats/" + chat["id"] + + original = FIXTURE.read_bytes() + + def expect_error(label, method, path, status, code, **kwargs): + response = c.call(method, path, status, **kwargs) + + check( + response["json"]["error"]["code"] == code, + label + ": incorrect error code", + ) + + print("PASS:", label, flush=True) + + def reject_upload( + label, + content, + status, + code, + mime="image/png", + filename="sample.png", + actor=None, + ): + expect_error( + label, + "POST", + chat_path + "/attachments?filename=" + + quote(filename, safe=""), + status, + code, + actor=alice if actor is None else actor, + raw=content, + headers={"Content-Type": mime}, + ) + + def reject_message(label, target_chat, ids, status, code): + expect_error( + label, + "POST", + "/api/v1/chats/" + target_chat["id"] + "/messages", + status, + code, + actor=alice, + body={"text": "", "attachment_ids": ids}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + + # 1. Контрольная загрузка исходного sample.png. + attachment, uploaded = c.attachment(alice, chat) + + check(uploaded == original, "Unexpected fixture content") + + downloaded = c.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=bob, + ) + + check(downloaded["bytes"] == original, "Original bytes changed") + check( + attachment["sha256"] == hashlib.sha256(original).hexdigest(), + "SHA-256 mismatch", + ) + + print("PASS: original PNG upload and download", flush=True) + + # 2. Настоящий PNG больше 10 MiB, но меньше 40 млн пикселей. + large = make_png(4096, 2700, noisy=True) + check(len(large) > MAX_BYTES, "Fixture must exceed 10 MiB") + + reject_upload( + "file larger than 10 MiB", + large, + 413, + "payload_too_large", + ) + del large + + # 3. Небольшой по байтам PNG с 40 960 000 пикселями. + many_pixels = make_png(6400, 6400) + check(len(many_pixels) < MAX_BYTES, "Fixture exceeds byte limit") + + reject_upload( + "image larger than 40 million pixels", + many_pixels, + 413, + "payload_too_large", + ) + del many_pixels + + # 4. Повреждение создаётся в памяти, sample.png не меняется. + reject_upload( + "truncated PNG", + original[:len(original) // 2], + 415, + "unsupported_media_type", + ) + + reject_upload( + "text disguised as PNG", + b"This is not an image", + 415, + "unsupported_media_type", + ) + + # 5. Реальный PNG объявлен как JPEG. + reject_upload( + "MIME spoofing", + original, + 415, + "unsupported_media_type", + mime="image/jpeg", + filename="fake.jpg", + ) + + reject_upload( + "empty file", + b"", + 415, + "unsupported_media_type", + ) + + # 6. Опасные и недопустимые имена файлов. + for filename in ( + "../sample.png", + "folder\\sample.png", + "bad\x00name.png", + "bad\nname.png", + "", + "x" * 256, + ): + reject_upload( + "invalid filename " + repr(filename), + original, + 400, + "invalid_request", + filename=filename, + ) + + # 7. Посторонний не может читать метаданные и оригинал. + for suffix in ("", "/content"): + expect_error( + "outsider GET " + (suffix or "metadata"), + "GET", + "/api/v1/attachments/" + attachment["id"] + suffix, + 404, + "not_found", + actor=eve, + ) + + reject_upload( + "outsider upload", + original, + 404, + "not_found", + actor=eve, + ) + + # 8. Участник чата может прикрепить файл другого участника. + message = c.message( + bob, + chat, + "", + attachments=[attachment["id"]], + )["json"] + + check( + message["attachment_ids"] == [attachment["id"]], + "Chat member could not attach the file", + ) + print("PASS: chat member can attach the file", flush=True) + + # 9. Alice состоит в обоих чатах, но перенос вложения запрещён. + second_chat = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={ + "title": "Second test chat", + "member_ids": [alice["id"], bob["id"]], + }, + schema="Chat", + )["json"] + + reject_message( + "attachment from another chat", + second_chat, + [attachment["id"]], + 404, + "not_found", + ) + + reject_message( + "unknown attachment", + chat, + [str(uuid.uuid4())], + 404, + "not_found", + ) + + reject_message( + "duplicate attachment IDs", + chat, + [attachment["id"], attachment["id"]], + 400, + "invalid_request", + ) + + # 10. Идемпотентность учитывает вложения. + key = uuid.uuid4().hex + + first = c.message( + alice, chat, "Photo", + key=key, + attachments=[attachment["id"]], + )["json"] + + replay = c.message( + alice, chat, "Photo", + key=key, + attachments=[attachment["id"]], + )["json"] + + check(first == replay, "Idempotent replay differs") + + expect_error( + "same key with different attachments", + "POST", + chat_path + "/messages", + 409, + "conflict", + actor=alice, + body={"text": "Photo", "attachment_ids": []}, + headers={"Idempotency-Key": key}, + ) + + text_key = uuid.uuid4().hex + + first = c.message( + alice, chat, "Text only", key=text_key, + )["json"] + + replay = c.message( + alice, chat, "Text only", + key=text_key, + attachments=[], + )["json"] + + check(first == replay, "Missing attachment_ids differs from []") + print("PASS: missing attachment_ids equals []", flush=True) + + print("PASS: attachment negative tests completed", flush=True) + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/tests/cleanup_check.py b/tests/cleanup_check.py new file mode 100644 index 00000000..f77a0859 --- /dev/null +++ b/tests/cleanup_check.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import re +import subprocess +import time +import uuid + +from client import Client + + +def check(condition, message): + if not condition: + raise RuntimeError(message) + + +def docker_script(script, *arguments, input_bytes=None): + command = [ + "docker", "compose", "exec", "-T", + "minio", + "sh", "-ec", script, + "cleanup-check", + *arguments, + ] + + result = subprocess.run( + command, + input=input_bytes, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=30, + ) + + if result.returncode != 0: + raise RuntimeError( + "Команда MinIO завершилась ошибкой. " + "Проверь доступность minio и название bucket." + ) + + return result.stdout.decode("utf-8") + + +def configure_minio(): + docker_script( + ''' + mc alias set cleanupcheck http://localhost:9000 \ + "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null + ''' + ) + + +def put_orphan(bucket, key): + docker_script( + 'mc pipe "cleanupcheck/$1/$2" >/dev/null', + bucket, + key, + input_bytes=b"synthetic cleanup test object\n", + ) + + +def list_test_objects(bucket, prefix): + output = docker_script( + 'mc ls --json "cleanupcheck/$1/$2"', + bucket, + prefix, + ) + + names = set() + + for line in output.splitlines(): + if not line.strip(): + continue + + item = json.loads(line) + + check( + item.get("status") != "error", + "MinIO вернул ошибку при получении списка объектов", + ) + + key = item.get("key") + + if key: + names.add(key.rstrip("/").split("/")[-1]) + + return names + + +def remove_test_object(bucket, key): + docker_script( + 'mc rm --force "cleanupcheck/$1/$2" >/dev/null', + bucket, + key, + ) + + +def check_download(client, actor, attachment, original): + result = client.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + actor=actor, + ) + + check( + result["bytes"] == original, + "Корректное вложение потеряно или изменилось", + ) + + check( + hashlib.sha256(result["bytes"]).hexdigest() + == attachment["sha256"], + "SHA-256 корректного вложения не совпадает", + ) + + +def wait_until(deadline): + remaining = deadline - time.monotonic() + + if remaining > 0: + time.sleep(remaining) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--base-url", + default="http://localhost:8080", + ) + parser.add_argument( + "--bucket", + default="messenger-attachments", + ) + args = parser.parse_args() + + check( + re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", args.bucket), + "Некорректное название bucket", + ) + + client = Client(args.base_url, None) + + alice, bob, eve, chat = client.bootstrap() + + # Создаётся через API: в S3 и PostgreSQL. + attachment, original = client.attachment(alice, chat) + + check_download(client, bob, attachment, original) + + configure_minio() + + prefix = "originals/cleanup-test-" + uuid.uuid4().hex + "/" + old_key = prefix + "old.bin" + fresh_key = prefix + "fresh.bin" + + try: + put_orphan(args.bucket, old_key) + old_created = time.monotonic() + + names = list_test_objects(args.bucket, prefix) + + check( + "old.bin" in names, + "Тестовый объект не появился в S3", + ) + + print("Created API attachment and an untracked S3 object", flush=True) + + wait_until(old_created + 25) + + names = list_test_objects(args.bucket, prefix) + + check( + "old.bin" in names, + "Объект удалён раньше grace period", + ) + + print("PASS: object younger than grace period is preserved", flush=True) + + wait_until(old_created + 65) + + put_orphan(args.bucket, fresh_key) + fresh_created = time.monotonic() + + print("Waiting for cleanup of the old object...", flush=True) + + deadline = fresh_created + 30 + + while True: + names = list_test_objects(args.bucket, prefix) + + check( + "fresh.bin" in names, + "Свежий объект удалён раньше grace period", + ) + + if "old.bin" not in names: + break + + check( + time.monotonic() < deadline, + "Старый объект не удалён. Проверь настройки и логи очистки", + ) + + time.sleep(2) + + print("PASS: old object without metadata is deleted", flush=True) + print("PASS: fresh object without metadata is preserved", flush=True) + + client.call( + "GET", + "/api/v1/attachments/" + attachment["id"], + actor=bob, + schema="Attachment", + ) + + check_download(client, bob, attachment, original) + + print("PASS: attachment with metadata and its bytes are preserved", flush=True) + print("PASS: cleanup scenario completed", flush=True) + + finally: + + for key in (old_key, fresh_key): + try: + remove_test_object(args.bucket, key) + except Exception as error: + print( + f"WARNING: final cleanup failed for {key}: {error}", + flush=True, + ) + + +if __name__ == "__main__": + try: + main() + except Exception as error: + print("FAIL:", error, flush=True) + raise SystemExit(1) \ No newline at end of file diff --git a/tests/level4_check.py b/tests/level4_check.py new file mode 100644 index 00000000..23cc2574 --- /dev/null +++ b/tests/level4_check.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Run from tests/ in lab3. Uses the unchanged skeleton client and resilience helpers. +Creates synthetic users/messages, stops only API containers, restores in finally. +Metrics are a small closed-loop workload, not a capacity benchmark. +""" +import csv +import json +import subprocess +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path +from client import Client, check +from resilience import history + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / 'results' / ('level4-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')) + + +def docker(*args): + p = subprocess.run(['docker', *args], cwd=ROOT, text=True, + capture_output=True, timeout=210) + check(p.returncode == 0, 'Docker command failed: ' + ' '.join(args)) + return p.stdout.strip() + + +def state(cid): + return json.loads(docker('inspect', '--format', '{{json .State}}', cid)) + + +def ready(cid): + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + s = state(cid) + if s.get('Running') and s.get('Health', {}).get('Status') == 'healthy': + return + time.sleep(1) + raise RuntimeError('API did not become healthy: ' + cid[:12]) + + +def observe(c, actor, expected): + deadline = time.monotonic() + 15 + seen = set() + while time.monotonic() < deadline: + try: + r = c.call('GET', '/api/v1/users/me', actor=actor, schema='User') + seen.add(r['headers']['X-Instance-Id']) + if expected <= seen: + return + except Exception: + pass + time.sleep(.1) + raise RuntimeError('Both actual API hostnames must be observed through LB') + + +def main(): + ids = docker('compose', 'ps', '--status', 'running', '-q', 'api').splitlines() + check(len(ids) == 2, 'Start exactly two healthy API replicas first') + names = {cid: docker('inspect', '--format', '{{.Config.Hostname}}', cid) for cid in ids} + for cid in ids: + ready(cid) + OUT.mkdir(parents=True) + summary = {'workload': '4 closed-loop workers, POST then GET, 50ms pause; HTTP timeout 2s', + 'replicas': names, 'experiments': []} + c = Client(timeout=2) + alice, bob, _, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' + observe(c, alice, set(names.values())) + + # One key, eight requests released together; a fresh chat isolates this test. + barrier = threading.Barrier(8) + key = uuid.uuid4().hex + def replay(_): + local = Client(timeout=10) + barrier.wait(timeout=15) + r = local.message(alice, chat, 'parallel-' + key, key=key) + return {'instance': r['headers']['X-Instance-Id'], 'message': r['json']} + with ThreadPoolExecutor(max_workers=8) as pool: + replies = list(pool.map(replay, range(8))) + check(all(r['message'] == replies[0]['message'] for r in replies), 'Concurrent replies differ') + check(len({r['instance'] for r in replies}) == 2, 'Concurrency did not cover both replicas; rerun') + check(history(c, bob, chat) == [replies[0]['message']], 'Concurrent replay created duplicates') + summary['concurrency'] = {'pass': True, 'responses': replies} + print('PASS: parallel replay, two replicas, one message', flush=True) + + for cid in ids: + observe(c, alice, set(names.values())) + rows, confirmed = [], {} + halt = threading.Event() + lock = threading.Lock() + start = time.monotonic() + def worker(): + local = Client(timeout=2) + key = uuid.uuid4().hex + while not halt.is_set(): + # Preserve the key/body after any uncertain POST result. + for method in ('POST', 'GET'): + began = time.monotonic() + row = {'method': method, 'start': began - start, 'ok': False, 'instance': '', 'error': ''} + try: + if method == 'POST': + r = local.message(alice, chat, 'load-' + key, key=key) + with lock: + confirmed[r['json']['id']] = r['json'] + key = uuid.uuid4().hex + else: + r = local.call('GET', path, actor=alice, schema='MessagePage') + row.update(ok=True, instance=r['headers']['X-Instance-Id']) + except Exception as exc: + row['error'] = type(exc).__name__ + row['end'] = time.monotonic() - start + with lock: + rows.append(row) + halt.wait(.05) + pool = ThreadPoolExecutor(max_workers=4) + jobs = [pool.submit(worker) for _ in range(4)] + try: + time.sleep(5) + impact = time.monotonic() - start + docker('stop', '--timeout', '20', cid) + stopped = time.monotonic() - start + stopped_state = state(cid) + check(not stopped_state['Running'], 'Target is still running') + print('Stopped ' + names[cid] + '; measuring for 15 seconds', flush=True) + time.sleep(15) + finally: + halt.set() + pool.shutdown(wait=True) + # Restore even if HTTP validation or measurement fails. + docker('start', cid) + ready(cid) + for job in jobs: + job.result() + observe(c, alice, set(names.values())) + stored = history(c, bob, chat) + by_id = {m['id']: m for m in stored} + check(all(by_id.get(k) == v for k, v in confirmed.items()), 'Confirmed message lost') + texts = [m['text'] for m in stored] + check(len(texts) == len(set(texts)), 'Uncertain retry created duplicate') + end = max(r['end'] for r in rows) + def metrics(lo, hi): + selected = [r for r in rows if lo <= r['start'] < hi] + good = sum(r['ok'] for r in selected) + return {'seconds': round(hi-lo, 3), 'attempts': len(selected), 'errors': len(selected)-good, + 'attempt_rps': round(len(selected)/(hi-lo), 2), + 'success_rps': round(good/(hi-lo), 2)} + recovery = {} + for method in ('POST', 'GET'): + hits = [r['end']-impact for r in rows if r['method'] == method and r['ok'] + and r['start'] >= stopped and r['instance'] != names[cid]] + recovery[method] = round(min(hits), 3) if hits else None + result = {'target': names[cid], 'exit_code': stopped_state['ExitCode'], + 'stop_started_seconds': impact, 'stop_finished_seconds': stopped, + 'baseline': metrics(0, impact), 'during_and_after_stop': metrics(impact, end), + 'first_success_after_stop_seconds_from_action_start': recovery, + 'confirmed_messages_preserved': True} + summary['experiments'].append(result) + with (OUT / (names[cid] + '.csv')).open('w', newline='') as f: + w = csv.DictWriter(f, fieldnames=list(rows[0])) + w.writeheader() + w.writerows(rows) + (OUT / 'summary.json').write_text(json.dumps(summary, indent=2), encoding='utf-8') + print(json.dumps(result, indent=2), flush=True) + check(all(v is not None and v <= 10 for v in recovery.values()), 'Read/write recovery exceeded 10s') + print('PASS: replica restored, session and data preserved', flush=True) + print('Results:', OUT) + print('This checks SIGTERM failover and concurrency, NOT rolling replacement.') + + +if __name__ == '__main__': + try: + main() + except Exception as exc: + print('FAIL:', str(exc)) + raise SystemExit(1) diff --git a/tests/reconcile_check.py b/tests/reconcile_check.py new file mode 100644 index 00000000..18bcea1f --- /dev/null +++ b/tests/reconcile_check.py @@ -0,0 +1,686 @@ +import json +from pathlib import Path +import subprocess +import sys +import time +import uuid + +from client import Client, arguments, check, run + + +ROOT = Path(__file__).resolve().parents[1] +BUCKET = None + + +def command(args, data=None, timeout=30): + result = subprocess.run( + args, + cwd=ROOT, + input=data, + capture_output=True, + timeout=timeout, + ) + + check( + result.returncode == 0, + "Command failed: " + " ".join(args[:3]), + ) + + return result.stdout + + +def sql(statement): + result = command( + [ + "docker", + "compose", + "exec", + "-T", + "postgres", + "sh", + "-ec", + ( + 'exec psql -X -A -t -v ON_ERROR_STOP=1 ' + '-U "$POSTGRES_USER" ' + '-d "${POSTGRES_DB:-$POSTGRES_USER}"' + ), + ], + statement.encode(), + ) + + return result.decode().strip() + + +def mc(*args, data=None): + return command( + [ + "docker", + "compose", + "exec", + "-T", + "minio", + "sh", + "-ec", + ( + "mc alias set reconcile_test " + "http://localhost:9000 " + '"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" ' + '>/dev/null; exec mc "$@"' + ), + "sh", + *args, + ], + data, + ) + + +def target(key): + return "reconcile_test/" + BUCKET + "/" + key + + +def object_names(chat_id): + raw = mc( + "ls", + "--recursive", + "--json", + target("originals/" + chat_id + "/"), + ) + + names = set() + + for line in raw.decode().splitlines(): + if not line.strip(): + continue + + item = json.loads(line) + + check( + item.get("status") == "success", + "Cannot list test S3 prefix", + ) + + if item.get("type") == "file": + names.add( + item["key"].rstrip("/").rsplit("/", 1)[-1] + ) + + return names + + +def inspect_api(): + ids = command([ + "docker", + "compose", + "ps", + "-q", + "api", + ]).decode().split() + + check( + len(ids) == 2, + "Start exactly two API replicas", + ) + + containers = json.loads(command([ + "docker", + "inspect", + *ids, + ])) + + buckets = set() + + for container in containers: + env = dict( + value.split("=", 1) + for value in container["Config"]["Env"] + if "=" in value + ) + + check( + env.get("APP_CLEANUP_ENABLED", "").lower() + == "false", + "Disable AttachmentCleanupJob " + "on both replicas first", + ) + + check( + bool(env.get("S3_BUCKET")), + "S3_BUCKET is missing", + ) + + buckets.add(env["S3_BUCKET"]) + + check( + len(buckets) == 1, + "API replicas use different buckets", + ) + + return buckets.pop() + + +def snapshot(chat_id): + # chat_id предварительно проверен через uuid.UUID. + return sql( + "SELECT coalesce(" + "json_agg(a ORDER BY id), '[]'::json" + ") FROM attachments a " + "WHERE chat_id='" + chat_id + "'::uuid;" + ) + + +def reconcile( + chat_id, + backups, + directory, + label, + apply=False, +): + path = directory / (label + ".json") + + result = subprocess.run( + [ + sys.executable, + "-X", + "utf8", + str(ROOT / "scripts/reconcile.py"), + "--apply" if apply else "--dry-run", + "--chat-id", + chat_id, + "--grace-seconds", + "60", + "--max-seconds", + "120", + "--backup-dir", + str(backups), + "--output", + str(path), + ], + cwd=ROOT, + capture_output=True, + text=True, + timeout=180, + ) + + check( + result.returncode in (0, 2), + "Reconciler failed at " + + label + + "; inspect " + + str(path), + ) + + check( + path.exists(), + "Reconciler did not save report", + ) + + report = json.loads( + path.read_text(encoding="utf-8") + ) + + check( + report["status"] in ( + "completed", + "needs_attention", + ), + "Scan failed", + ) + + check( + report["counts"].get("busy", 0) == 0, + "Object lock was busy; repeat experiment", + ) + + check( + report["counts"].get("size_mismatches", 0) == 0, + "Unexpected size mismatch", + ) + + print( + label + ": " + json.dumps(report["counts"]), + flush=True, + ) + + return report + + +def count(report, name): + return report["counts"].get(name, 0) + + +def main(): + global BUCKET + + args = arguments( + "Lab4 reconciler: dry-run, repair " + "and repeated execution" + ) + + BUCKET = inspect_api() + + c = Client( + args.base_url, + args.ca_file, + timeout=30, + ) + + alice, bob, _, chat = c.bootstrap() + chat_id = str(uuid.UUID(chat["id"])) + + stable, content = c.attachment(alice, chat) + missing, missing_bytes = c.attachment(alice, chat) + + message = c.message( + alice, + chat, + "Reconciler experiment", + attachments=[ + stable["id"], + missing["id"], + ], + )["json"] + + missing_id = str(uuid.UUID(missing["id"])) + + key = sql( + "SELECT object_key FROM attachments " + "WHERE id='" + missing_id + "'::uuid;" + ) + + check( + key.startswith("originals/" + chat_id + "/"), + "Unexpected original key", + ) + + prefix = "originals/" + chat_id + "/" + + old_key = ( + prefix + "orphan-old-" + uuid.uuid4().hex + ) + + fresh_key = None + fresh_started = None + + # Запоминаем все созданные orphan-ключи, + # чтобы попытаться убрать их при исключении. + synthetic_keys = {old_key} + + backups = ( + ROOT / "backups" / ("reconcile-" + chat_id) + ) + + directory = ( + ROOT / "results" / ("reconcile-" + chat_id) + ) + + backups.mkdir(parents=True, exist_ok=False) + directory.mkdir(parents=True, exist_ok=False) + + metadata_before = snapshot(chat_id) + + needs_restore = False + completed = False + + def verify_metadata(): + check( + snapshot(chat_id) == metadata_before, + "Attachment metadata changed", + ) + + page = c.call( + "GET", + "/api/v1/chats/" + + chat_id + + "/messages?limit=100", + actor=bob, + schema="MessagePage", + )["json"] + + found = next( + ( + item + for item in page["items"] + if item["id"] == message["id"] + ), + None, + ) + + check( + found == message, + "Message or attachment references changed", + ) + + def prepare_fresh(): + nonlocal fresh_key, fresh_started + + # Предыдущий свежий объект больше не нужен. + # Его сохранность проверена предыдущим этапом. + if fresh_key is not None: + previous_name = fresh_key.rsplit("/", 1)[-1] + + if previous_name in object_names(chat_id): + mc("rm", target(fresh_key)) + + fresh_key = ( + prefix + "orphan-fresh-" + uuid.uuid4().hex + ) + + synthetic_keys.add(fresh_key) + + # Начинаем отсчёт до PUT: это консервативная + # верхняя оценка возраста объекта. + fresh_started = time.monotonic() + + mc( + "pipe", + target(fresh_key), + data=content, + ) + + def verify_fresh(): + check( + fresh_key is not None + and fresh_started is not None, + "Fresh test object was not created", + ) + + names = object_names(chat_id) + elapsed = time.monotonic() - fresh_started + + check( + elapsed < 55, + "A single reconciliation stage exceeded " + "the fresh-object timing budget; " + "repeat on an idle stand", + ) + + check( + fresh_key.rsplit("/", 1)[-1] in names, + "Fresh orphan was removed", + ) + + def download(attachment, expected): + result = c.call( + "GET", + "/api/v1/attachments/" + + attachment["id"] + + "/content", + actor=bob, + ) + + check( + result["bytes"] == expected, + "Original bytes changed", + ) + + try: + mc( + "pipe", + target(old_key), + data=content, + ) + + print( + "Old orphan created. Waiting 65 seconds " + "for the real grace period...", + flush=True, + ) + + time.sleep(65) + + prepare_fresh() + + # Байты пока доступны только тесту. + # Reconciler ещё не получает резервную копию. + needs_restore = True + mc("rm", target(key)) + + before = object_names(chat_id) + + dry = reconcile( + chat_id, + backups, + directory, + "01-dry-run", + ) + + check( + count(dry, "would_delete_orphans") == 1, + "Old orphan not detected", + ) + + check( + count(dry, "unresolved_missing") == 1, + "Missing original not detected", + ) + + check( + object_names(chat_id) == before, + "Dry-run changed S3 objects", + ) + + verify_metadata() + verify_fresh() + download(stable, content) + + print( + "PASS: dry-run found both faults " + "and changed no test data", + flush=True, + ) + + # Для следующего этапа создаём новый свежий объект. + prepare_fresh() + + applied = reconcile( + chat_id, + backups, + directory, + "02-apply-no-backup", + apply=True, + ) + + check( + count(applied, "deleted_orphans") == 1, + "Old orphan not deleted", + ) + + check( + count(applied, "unresolved_missing") == 1, + "Missing original incorrectly repaired", + ) + + names = object_names(chat_id) + + check( + old_key.rsplit("/", 1)[-1] not in names, + "Old orphan still exists", + ) + + check( + key.rsplit("/", 1)[-1] not in names, + "Missing original unexpectedly exists", + ) + + verify_metadata() + verify_fresh() + + c.call( + "GET", + "/api/v1/attachments/" + + missing_id + + "/content", + expected=503, + actor=bob, + ) + + print( + "PASS: old orphan deleted; missing original " + "reported without metadata loss", + flush=True, + ) + + # Предоставляем reconciler достоверную копию. + backup_file = ( + backups / (missing["sha256"] + ".bin") + ) + + backup_file.write_bytes(missing_bytes) + + prepare_fresh() + before = object_names(chat_id) + + planned = reconcile( + chat_id, + backups, + directory, + "03-dry-run-with-backup", + ) + + check( + count(planned, "would_restore") == 1, + "Valid backup not recognized", + ) + + check( + object_names(chat_id) == before, + "Dry-run restored an object", + ) + + verify_metadata() + verify_fresh() + + print( + "PASS: dry-run plans restoration " + "without changing objects", + flush=True, + ) + + prepare_fresh() + + restored = reconcile( + chat_id, + backups, + directory, + "04-restore", + apply=True, + ) + + check( + count(restored, "restored") == 1, + "Original not restored", + ) + + download(missing, missing_bytes) + needs_restore = False + + download(stable, content) + verify_metadata() + verify_fresh() + + print( + "PASS: original restored byte-for-byte; " + "used attachment and message preserved", + flush=True, + ) + + prepare_fresh() + + repeated = reconcile( + chat_id, + backups, + directory, + "05-repeat", + apply=True, + ) + + for name in ( + "deleted_orphans", + "restored", + "unresolved_missing", + "missing_objects", + ): + check( + count(repeated, name) == 0, + "Repeated run is not clean: " + name, + ) + + verify_metadata() + verify_fresh() + + download(stable, content) + download(missing, missing_bytes) + + completed = True + + print( + "PASS: repeated apply makes " + "no additional changes", + flush=True, + ) + + print( + "Reports: " + str(directory), + flush=True, + ) + + finally: + # При исключении пытаемся восстановить оригинал, + # намеренно удалённый этим тестом. + if needs_restore: + try: + mc( + "pipe", + target(key), + data=missing_bytes, + ) + + print( + "Test original restored " + "during final cleanup", + flush=True, + ) + + except Exception: + print( + "WARNING: restore test original manually " + "from backup; key=" + key, + flush=True, + ) + + ( + backups / (missing["sha256"] + ".bin") + ).write_bytes(missing_bytes) + + cleanup_ok = True + + # Удаляем только существующие синтетические объекты. + # Уже удалённый reconciler старый orphan — не ошибка. + for orphan in sorted(synthetic_keys): + try: + name = orphan.rsplit("/", 1)[-1] + + if name in object_names(chat_id): + mc("rm", target(orphan)) + + except Exception: + cleanup_ok = False + + print( + "WARNING: cannot remove synthetic orphan: " + + orphan, + flush=True, + ) + + if completed: + check( + cleanup_ok, + "Scenario passed but final " + "test-object cleanup failed", + ) + + print("PASS: reconciler experiment completed") + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/tests/reconcile_cost_check.py b/tests/reconcile_cost_check.py new file mode 100644 index 00000000..9525460d --- /dev/null +++ b/tests/reconcile_cost_check.py @@ -0,0 +1,561 @@ +import csv +import hashlib +import json +import math +from pathlib import Path +import random +import statistics +import subprocess +import sys +import time +import uuid + +from client import Client, arguments, check, run +import reconcile_check as helpers + + +ROOT = Path(__file__).resolve().parents[1] + + +def options(parser): + parser.add_argument( + "--sizes", + type=int, + nargs="+", + default=[10, 100, 1000], + ) + + parser.add_argument( + "--repeats", + type=int, + default=3, + ) + + parser.add_argument( + "--max-seconds", + type=int, + default=600, + ) + + +def digest_metadata(chat_id): + snapshot = helpers.snapshot(chat_id) + + return hashlib.sha256( + snapshot.encode() + ).hexdigest() + + +def environment_info(): + ids = helpers.command([ + "docker", + "compose", + "ps", + "-q", + "api", + "postgres", + "minio", + ]).decode().split() + + containers = json.loads( + helpers.command([ + "docker", + "inspect", + *ids, + ]) + ) + + cpu_count = int(helpers.command([ + "docker", + "info", + "--format", + "{{.NCPU}}", + ])) + + memory_bytes = int(helpers.command([ + "docker", + "info", + "--format", + "{{.MemTotal}}", + ])) + + worker_image = helpers.command([ + "docker", + "image", + "inspect", + "--format", + "{{.Id}}", + "mtusi4-reconciler:local", + ]).decode().strip() + + # Сохраняем только параметры окружения, + # необходимые для воспроизводимости. + # Credentials и полный docker inspect не записываем. + services = [] + + for container in containers: + services.append({ + "service": container["Config"]["Labels"].get( + "com.docker.compose.service" + ), + "image_id": container["Image"], + "memory_limit_bytes": ( + container["HostConfig"]["Memory"] + ), + "nano_cpus": ( + container["HostConfig"]["NanoCpus"] + ), + }) + + return { + "docker_cpus": cpu_count, + "docker_memory_bytes": memory_bytes, + "worker_image_id": worker_image, + "worker_memory_limit_bytes": 256 * 1024 * 1024, + "worker_cpu_limit": 1, + "services": services, + } + + +def main(): + args = arguments( + "Measure reconciler traversal cost", + options, + ) + + sizes = sorted(set(args.sizes)) + + check( + all(1 <= size <= 5000 for size in sizes), + "Sizes must be between 1 and 5000", + ) + + check( + 1 <= args.repeats <= 10, + "Repeats must be between 1 and 10", + ) + + check( + args.max_seconds >= 1, + "max-seconds must be positive", + ) + + # Проверяет две API-реплики и отключённую очистку. + helpers.BUCKET = helpers.inspect_api() + + fixture = ( + ROOT / "tests/fixtures/sample.png" + ).read_bytes() + + fixture_sha = hashlib.sha256(fixture).hexdigest() + + directory = ( + ROOT + / "results" + / ("reconcile-cost-" + uuid.uuid4().hex) + ) + + backups = directory / "empty-backups" + backups.mkdir(parents=True, exist_ok=False) + + report = { + "passed": False, + "hypothesis": ( + "Traversal calls grow approximately linearly " + "with attachment count; wall time also " + "contains fixed overhead." + ), + "sizes": sizes, + "repeats": args.repeats, + "grace_seconds": 60, + "fixture_size_bytes": len(fixture), + "fixture_sha256": fixture_sha, + "environment": environment_info(), + "groups": [], + "samples": [], + "summary": [], + "method": ( + "Separate chats; API uploads; real grace wait; " + "one warm-up per size; shuffled measured rounds; " + "dry-run only." + ), + } + + groups = [] + + client = Client( + args.base_url, + args.ca_file, + timeout=30, + ) + + def save(): + (directory / "summary.json").write_text( + json.dumps( + report, + ensure_ascii=False, + indent=2, + ) + "\n", + encoding="utf-8", + ) + + fields = [ + "attachments", + "trial", + "worker_seconds", + "launcher_seconds", + "sql_calls", + "s3_calls", + "s3_list_calls", + "s3_head_calls", + ] + + with (directory / "samples.csv").open( + "w", + encoding="utf-8", + newline="", + ) as stream: + writer = csv.DictWriter( + stream, + fieldnames=fields, + ) + + writer.writeheader() + writer.writerows(report["samples"]) + + def scan(group, label): + path = directory / ( + f'n{group["n"]}-{label}.json' + ) + + started = time.monotonic() + + result = subprocess.run( + [ + sys.executable, + "-X", + "utf8", + str(ROOT / "scripts/reconcile.py"), + "--dry-run", + "--chat-id", + group["chat_id"], + "--grace-seconds", + "60", + "--max-seconds", + str(args.max_seconds), + "--backup-dir", + str(backups), + "--output", + str(path), + ], + cwd=ROOT, + capture_output=True, + text=True, + timeout=args.max_seconds + 180, + ) + + wall = time.monotonic() - started + + check( + result.returncode == 0, + "Scan failed: " + str(path), + ) + + data = json.loads( + path.read_text(encoding="utf-8") + ) + + check( + data["status"] == "completed" + and data["mode"] == "dry-run", + "Unexpected scan status", + ) + + check( + data.get("chat_id") == group["chat_id"], + "Wrong scan scope", + ) + + counts = data["counts"] + + # Каждый объект и каждая запись должны пройти + # полноценную проверку после grace period. + for field in ( + "objects_scanned", + "referenced_objects", + "metadata_scanned", + "present_metadata", + ): + check( + counts.get(field, 0) == group["n"], + "Incomplete scan: " + field, + ) + + for field in ( + "fresh_objects", + "fresh_metadata", + "busy", + "missing_objects", + "unresolved_missing", + "size_mismatches", + "would_delete_orphans", + "deleted_orphans", + "restored", + ): + check( + counts.get(field, 0) == 0, + "Unexpected outcome: " + field, + ) + + calls = data["s3_operation_calls"] + + check( + calls.get("put_object", 0) == 0 + and calls.get("delete_object", 0) == 0, + "Dry-run attempted mutations", + ) + + return { + "attachments": group["n"], + "trial": label, + "worker_seconds": data["elapsed_seconds"], + "launcher_seconds": round(wall, 3), + "sql_calls": data["sql_statement_calls"], + "s3_calls": sum(calls.values()), + "s3_list_calls": calls.get( + "list_objects_v2", 0 + ), + "s3_head_calls": calls.get( + "head_object", 0 + ), + } + + save() + + try: + setup_started = time.monotonic() + + # Каждый размер получает отдельный чат. + for size in sizes: + alice, bob, _, chat = client.bootstrap() + chat_id = str(uuid.UUID(chat["id"])) + + entry = { + "attachments": size, + "chat_id": chat_id, + "uploaded": 0, + } + + report["groups"].append(entry) + save() + + sample_attachments = [] + + # Последовательные загрузки. + # Автоматических повторов POST нет. + for index in range(size): + attachment, content = client.attachment( + alice, + chat, + ) + + check( + content == fixture, + "Fixture changed during experiment", + ) + + # Для контрольного скачивания сохраняем + # первое и последнее вложение. + if index in {0, size - 1}: + sample_attachments.append(attachment) + + entry["uploaded"] = index + 1 + + if ( + (index + 1) % 100 == 0 + or index + 1 == size + ): + print( + f"Preparing N={size}: " + f"{index + 1}/{size}", + flush=True, + ) + + save() + + message = client.message( + alice, + chat, + "Reconciler cost experiment", + attachments=[ + attachment["id"] + for attachment in sample_attachments + ], + )["json"] + + groups.append({ + "n": size, + "chat_id": chat_id, + "bob": bob, + "samples": sample_attachments, + "message": message, + "metadata_digest": digest_metadata(chat_id), + }) + + report["setup_seconds"] = round( + time.monotonic() - setup_started, + 3, + ) + + save() + + print( + "All uploads completed. Waiting 65 seconds " + "for the real grace period...", + flush=True, + ) + + time.sleep(65) + + # Прогрев сохраняется отдельными JSON, + # но не включается в итоговую статистику. + for group in groups: + scan(group, "warmup") + + print( + f'Warm-up N={group["n"]}: complete', + flush=True, + ) + + # В каждом раунде меняем порядок объёмов. + # Фиксированный seed делает порядок повторяемым. + rng = random.Random(1729) + + for trial in range(1, args.repeats + 1): + order = list(groups) + rng.shuffle(order) + + for group in order: + sample = scan( + group, + f"run-{trial}", + ) + + report["samples"].append(sample) + save() + + print( + f'N={group["n"]}, run={trial}: ' + f'worker={sample["worker_seconds"]}s, ' + f'launcher={sample["launcher_seconds"]}s, ' + f'SQL={sample["sql_calls"]}, ' + f'S3={sample["s3_calls"]}', + flush=True, + ) + + # Контроль после измерений. + for group in groups: + check( + digest_metadata(group["chat_id"]) + == group["metadata_digest"], + "Metadata changed during benchmark", + ) + + for attachment in group["samples"]: + downloaded = client.call( + "GET", + "/api/v1/attachments/" + + attachment["id"] + + "/content", + actor=group["bob"], + ) + + check( + downloaded["bytes"] == fixture, + "Sample original changed", + ) + + page = client.call( + "GET", + "/api/v1/chats/" + + group["chat_id"] + + "/messages?limit=100", + actor=group["bob"], + schema="MessagePage", + )["json"] + + check( + group["message"] in page["items"], + "Sample message changed", + ) + + for size in sizes: + samples = [ + sample + for sample in report["samples"] + if sample["attachments"] == size + ] + + worker_times = [ + sample["worker_seconds"] + for sample in samples + ] + + launcher_times = [ + sample["launcher_seconds"] + for sample in samples + ] + + report["summary"].append({ + "attachments": size, + "worker_seconds_min": min(worker_times), + "worker_seconds_median": round( + statistics.median(worker_times), + 3, + ), + "worker_seconds_max": max(worker_times), + "launcher_seconds_median": round( + statistics.median(launcher_times), + 3, + ), + "sql_calls_observed": sorted({ + sample["sql_calls"] + for sample in samples + }), + "s3_calls_observed": sorted({ + sample["s3_calls"] + for sample in samples + }), + "sql_calls_expected_for_current_algorithm": ( + 5 * size + + math.ceil(size / 500) + + 2 + ), + "s3_calls_expected_for_current_algorithm": ( + size + + math.ceil(size / 500) + + 1 + ), + }) + + report["passed"] = True + + finally: + save() + + print(json.dumps( + report["summary"], + ensure_ascii=False, + indent=2, + )) + + print("Reports: " + str(directory)) + + print( + "PASS: reconciler traversal cost " + "experiment completed" + ) + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/tests/reconcile_failure_check.py b/tests/reconcile_failure_check.py new file mode 100644 index 00000000..d78f745d --- /dev/null +++ b/tests/reconcile_failure_check.py @@ -0,0 +1,492 @@ +import contextlib +import importlib.util +import io +import json +from pathlib import Path +import sys +import time +import uuid + +from client import Client, Failure, arguments, check, run +import reconcile_check as helpers + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_launcher(): + spec = importlib.util.spec_from_file_location( + "reconciler_launcher", + ROOT / "scripts/reconcile.py", + ) + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + return module + + +def state(container_id): + raw = helpers.command([ + "docker", + "inspect", + "--format", + "{{json .State}}", + container_id, + ]) + + return json.loads(raw) + + +def wait_container(container_id, seconds=90): + deadline = time.monotonic() + seconds + + while time.monotonic() < deadline: + current = state(container_id) + + healthy = ( + current.get("Running") + and current.get("Health", {}).get("Status") + == "healthy" + ) + + if healthy: + return + + time.sleep(2) + + raise Failure( + "Container did not become healthy: " + + container_id[:12] + ) + + +def wait_api(client, seconds=90): + deadline = time.monotonic() + seconds + + while time.monotonic() < deadline: + try: + client.call( + "GET", + "/health/ready", + schema="Health", + ) + + return + + except Failure: + time.sleep(2) + + raise Failure( + "API did not recover after dependency restart" + ) + + +def main(): + args = arguments( + "Reconciler failure checks: " + "MinIO and PostgreSQL outage" + ) + + helpers.BUCKET = helpers.inspect_api() + + launcher = load_launcher() + + saved_services = { + name: launcher.service(name) + for name in ("api", "postgres", "minio") + } + + launcher.service = lambda name: saved_services[name] + + c = Client( + args.base_url, + args.ca_file, + timeout=10, + ) + + alice, bob, _, chat = c.bootstrap() + chat_id = str(uuid.UUID(chat["id"])) + + attachment, content = c.attachment(alice, chat) + + message = c.message( + alice, + chat, + "Reconciler dependency failure", + attachments=[attachment["id"]], + )["json"] + + original_name = str(uuid.UUID(attachment["id"])) + + orphan_key = ( + "originals/" + + chat_id + + "/orphan-failure-" + + uuid.uuid4().hex + ) + + orphan_name = orphan_key.rsplit("/", 1)[-1] + + metadata_before = helpers.snapshot(chat_id) + + directory = ( + ROOT + / "results" + / ("reconcile-failure-" + chat_id) + ) + + backups = ( + ROOT + / "backups" + / ("reconcile-failure-" + chat_id) + ) + + directory.mkdir(parents=True, exist_ok=False) + backups.mkdir(parents=True, exist_ok=False) + + stopped = set() + summary = [] + completed = False + + def worker(label, apply, expected_code): + path = directory / (label + ".json") + previous_argv = sys.argv + + try: + sys.argv = [ + str(ROOT / "scripts/reconcile.py"), + "--apply" if apply else "--dry-run", + "--chat-id", + chat_id, + "--grace-seconds", + "60", + "--max-seconds", + "120", + "--backup-dir", + str(backups), + "--output", + str(path), + ] + + with contextlib.redirect_stdout(io.StringIO()): + code = launcher.main() + + finally: + sys.argv = previous_argv + + check( + code == expected_code, + "Unexpected return code at " + label, + ) + + report = json.loads( + path.read_text(encoding="utf-8") + ) + + summary.append({ + "stage": label, + "status": report["status"], + "elapsed_seconds": report["elapsed_seconds"], + "error_type": report["error_type"], + }) + + print( + label + + ": status=" + + report["status"] + + ", error_type=" + + str(report["error_type"]), + flush=True, + ) + + return report + + def verify_data(expect_orphan): + check( + helpers.snapshot(chat_id) == metadata_before, + "Attachment metadata changed", + ) + + names = helpers.object_names(chat_id) + + expected = {original_name} + + if expect_orphan: + expected.add(orphan_name) + + check( + names == expected, + "Unexpected S3 object set", + ) + + result = c.call( + "GET", + "/api/v1/attachments/" + + attachment["id"] + + "/content", + actor=bob, + ) + + check( + result["bytes"] == content, + "Confirmed original changed", + ) + + page = c.call( + "GET", + "/api/v1/chats/" + + chat_id + + "/messages?limit=100", + actor=bob, + schema="MessagePage", + )["json"] + + check( + message in page["items"], + "Message or attachment references changed", + ) + + def restore(name): + container_id = saved_services[name]["Id"] + + helpers.command( + [ + "docker", + "start", + container_id, + ], + timeout=60, + ) + + wait_container(container_id) + stopped.discard(name) + + wait_api(c) + + try: + helpers.mc( + "pipe", + helpers.target(orphan_key), + data=content, + ) + + print( + "Waiting 65 seconds " + "for the real orphan grace period...", + flush=True, + ) + + time.sleep(65) + + baseline = worker( + "01-baseline", + apply=False, + expected_code=0, + ) + + check( + baseline["counts"].get( + "would_delete_orphans", 0 + ) == 1, + "No deletion candidate in baseline", + ) + + verify_data(expect_orphan=True) + + for number, name in enumerate( + ("minio", "postgres"), + start=2, + ): + container_id = saved_services[name]["Id"] + + print( + "Stopping " + + name + + " for the experiment", + flush=True, + ) + + stopped.add(name) + + helpers.command( + [ + "docker", + "stop", + "-t", + "10", + container_id, + ], + timeout=60, + ) + + check( + not state(container_id).get("Running"), + "Dependency did not stop", + ) + + try: + report = worker( + f"{number:02d}-{name}-unavailable", + apply=True, + expected_code=1, + ) + + check( + report["status"] == "failed", + "Outage was not reported as failed", + ) + + check( + report["error_type"] is not None, + "Missing error type", + ) + + for operation in ( + "delete_object", + "put_object", + ): + check( + report["s3_operation_calls"].get( + operation, 0 + ) == 0, + "Mutation attempted " + "during dependency outage", + ) + + for counter in ( + "missing_objects", + "unresolved_missing", + "deleted_orphans", + "restored", + ): + check( + report["counts"].get( + counter, 0 + ) == 0, + "Outage was confused " + "with a data inconsistency", + ) + + finally: + restore(name) + + verify_data(expect_orphan=True) + + print( + "PASS: " + + name + + " outage aborted reconciliation; " + "data and session survived", + flush=True, + ) + + repaired = worker( + "04-recovery-apply", + apply=True, + expected_code=0, + ) + + check( + repaired["counts"].get( + "deleted_orphans", 0 + ) == 1, + "Orphan was not removed after recovery", + ) + + verify_data(expect_orphan=False) + + repeated = worker( + "05-recovery-repeat", + apply=True, + expected_code=0, + ) + + for counter in ( + "deleted_orphans", + "restored", + "missing_objects", + "unresolved_missing", + ): + check( + repeated["counts"].get( + counter, 0 + ) == 0, + "Repeated run is not clean", + ) + + verify_data(expect_orphan=False) + completed = True + + finally: + recovery_errors = [] + + for name in tuple(stopped): + try: + restore(name) + + except Exception: + recovery_errors.append(name) + + print( + "WARNING: could not restore service: " + + name, + flush=True, + ) + + if not recovery_errors: + try: + if orphan_name in helpers.object_names(chat_id): + helpers.mc( + "rm", + helpers.target(orphan_key), + ) + + except Exception: + recovery_errors.append( + "test orphan cleanup" + ) + + print( + "WARNING: cannot remove synthetic orphan: " + + orphan_key, + flush=True, + ) + + check( + not recovery_errors, + "Recovery incomplete: " + + ", ".join(recovery_errors), + ) + + if completed: + report = { + "passed": True, + "chat_id": chat_id, + "stages": summary, + } + + (directory / "summary.json").write_text( + json.dumps( + report, + ensure_ascii=False, + indent=2, + ) + "\n", + encoding="utf-8", + ) + + print( + "PASS: reconciliation resumes after recovery " + "and repeated apply changes nothing" + ) + + print("Reports: " + str(directory)) + + print( + "PASS: reconciler dependency failure " + "scenario completed" + ) + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/tests/upload_abort_check.py b/tests/upload_abort_check.py new file mode 100644 index 00000000..d5175113 --- /dev/null +++ b/tests/upload_abort_check.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 + +import hashlib +import re +import socket +import subprocess +import time +from pathlib import Path +from urllib.parse import urlsplit + +from client import Client, arguments, check, run + + +FIXTURE = ( + Path(__file__).resolve().parent + / "fixtures" + / "sample.png" +) + + +def command_output(command): + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10, + ) + + check( + result.returncode == 0, + "Docker command failed; check that API containers are running", + ) + + return result.stdout.strip() + + +def api_containers(): + output = command_output([ + "docker", "compose", "ps", "-q", "api", + ]) + + containers = output.splitlines() + + check( + len(containers) == 2, + "This test requires exactly two running API replicas", + ) + + for container in containers: + check( + re.fullmatch(r"[a-f0-9]{12,64}", container) is not None, + "Unexpected container ID", + ) + + return containers + + +def temporary_files(containers): + result = set() + + for container in containers: + output = command_output([ + "docker", "exec", container, + "sh", "-ec", + ''' + directory="${UPLOAD_TEMP_DIR:-/tmp/messenger-uploads}" + + if [ -d "$directory" ]; then + find "$directory" \ + -maxdepth 1 \ + -type f \ + -name 'upload-*.tmp' \ + -print + fi + ''', + ]) + + for filename in output.splitlines(): + if filename: + result.add((container, filename)) + + return result + + +def wait_for_empty_directory(containers, seconds=25): + deadline = time.monotonic() + seconds + + while True: + remaining = temporary_files(containers) + + if not remaining: + return + + check( + time.monotonic() < deadline, + "Temporary upload files were not removed", + ) + + time.sleep(0.5) + + +def main(): + args = arguments("Interrupted upload cleanup check") + + client = Client( + args.base_url, + args.ca_file, + timeout=30, + ) + + address = urlsplit(client.base) + + check( + address.scheme == "http", + "This local test expects an HTTP endpoint", + ) + + containers = api_containers() + + check( + not temporary_files(containers), + "Temporary directory is not empty. " + "Finish other uploads and inspect leftover files first", + ) + + alice, bob, eve, chat = client.bootstrap() + + original = FIXTURE.read_bytes() + + check( + len(original) > 32768, + "Fixture must be larger than the partial upload", + ) + + path = ( + address.path.rstrip("/") + + "/api/v1/chats/" + + chat["id"] + + "/attachments?filename=interrupted.png" + ) + + headers = ( + f"POST {path} HTTP/1.1\r\n" + f"Host: {address.netloc}\r\n" + f"Authorization: Bearer {alice['token']}\r\n" + "Content-Type: image/png\r\n" + f"Content-Length: {len(original)}\r\n" + "Connection: close\r\n" + "\r\n" + ).encode("ascii") + + connection = socket.create_connection( + (address.hostname, address.port or 80), + timeout=10, + ) + + observed = False + + try: + connection.sendall(headers) + connection.sendall(original[:32768]) + + print( + "Partial body sent; waiting for a temporary file", + flush=True, + ) + + deadline = time.monotonic() + 5 + + while time.monotonic() < deadline: + if temporary_files(containers): + observed = True + break + + time.sleep(0.2) + + check( + observed, + "Temporary file was not observed. " + "Check API logs and proxy timeouts", + ) + + print( + "PASS: API started receiving the incomplete upload", + flush=True, + ) + + finally: + # Закрываем соединение до передачи остальных байтов. + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + connection.close() + + print("Client connection closed before upload completion", flush=True) + + wait_for_empty_directory(containers) + + print( + "PASS: temporary file removed after interrupted upload", + flush=True, + ) + + # Проверяем, что разрешение UploadLimiter не осталось занятым. + # Последовательными запросами через LB наблюдаем обе реплики. + instances = set() + last_attachment = None + + for _ in range(20): + response = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + + "/attachments?filename=after-abort.png", + 201, + actor=alice, + raw=original, + headers={"Content-Type": "image/png"}, + schema="Attachment", + ) + + attachment = response["json"] + + check( + attachment["status"] == "ready", + "Upload did not become ready", + ) + + check( + attachment["sha256"] + == hashlib.sha256(original).hexdigest(), + "Uploaded checksum mismatch", + ) + + instances.add(response["headers"]["X-Instance-Id"]) + last_attachment = attachment + + if len(instances) >= 2: + break + + check( + len(instances) == 2, + "Successful uploads were not observed on both API replicas", + ) + + print( + "PASS: both API replicas accept uploads after cancellation", + flush=True, + ) + + downloaded = client.call( + "GET", + "/api/v1/attachments/" + + last_attachment["id"] + + "/content", + actor=bob, + ) + + check( + downloaded["bytes"] == original, + "Downloaded file differs from the original", + ) + + wait_for_empty_directory(containers) + + print("PASS: subsequent upload is intact", flush=True) + print("PASS: interrupted upload scenario completed", flush=True) + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/tests/upload_crash_check.py b/tests/upload_crash_check.py new file mode 100644 index 00000000..2bfa5b7d --- /dev/null +++ b/tests/upload_crash_check.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 + +import socket +import subprocess +import time +from urllib.parse import urlsplit + +from client import Client, arguments, check, run +from upload_abort_check import ( + FIXTURE, + api_containers, + command_output, + temporary_files, +) + + +def container_running(container): + value = command_output([ + "docker", "inspect", + "--format", "{{.State.Running}}", + container, + ]) + + return value == "true" + + +def container_healthy(container): + value = command_output([ + "docker", "inspect", + "--format", "{{.State.Health.Status}}", + container, + ]) + + return value == "healthy" + + +def main(): + args = arguments("API crash during an incomplete upload") + + client = Client( + args.base_url, + args.ca_file, + timeout=30, + ) + + address = urlsplit(client.base) + + check( + address.scheme == "http", + "This local test expects HTTP", + ) + + containers = api_containers() + + check( + not temporary_files(containers), + "Temporary directories must be empty before the test", + ) + + alice, bob, eve, chat = client.bootstrap() + + good_attachment, original = client.attachment(alice, chat) + + content = FIXTURE.read_bytes() + + check( + len(content) > 32768, + "Fixture is too small", + ) + + path = ( + address.path.rstrip("/") + + "/api/v1/chats/" + + chat["id"] + + "/attachments?filename=crash-test.png" + ) + + headers = ( + f"POST {path} HTTP/1.1\r\n" + f"Host: {address.netloc}\r\n" + f"Authorization: Bearer {alice['token']}\r\n" + "Content-Type: image/png\r\n" + f"Content-Length: {len(content)}\r\n" + "Connection: close\r\n" + "\r\n" + ).encode("ascii") + + connection = None + target = None + temporary_path = None + + try: + connection = socket.create_connection( + (address.hostname, address.port or 80), + timeout=10, + ) + + connection.sendall(headers) + connection.sendall(content[:32768]) + + print("Partial upload started", flush=True) + + deadline = time.monotonic() + 5 + + while True: + found = temporary_files(containers) + + if found: + check( + len(found) == 1, + "Expected exactly one active temporary file", + ) + + target, temporary_path = next(iter(found)) + break + + check( + time.monotonic() < deadline, + "Temporary file was not observed", + ) + + time.sleep(0.2) + + + command_output([ + "docker", "exec", target, + "touch", "-t", "200001010000", + temporary_path, + ]) + + print( + "Temporary file found; test timestamp set to the past", + flush=True, + ) + + command_output([ + "docker", "kill", + "--signal", "KILL", + target, + ]) + + check( + not container_running(target), + "Target API container did not stop", + ) + + connection.close() + connection = None + + copied = subprocess.run( + [ + "docker", "cp", + target + ":" + temporary_path, + "-", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=10, + ) + + check( + copied.returncode == 0, + "Temporary file did not survive the process crash", + ) + + print( + "PASS: temporary file remains after SIGKILL", + flush=True, + ) + + command_output([ + "docker", "start", target, + ]) + + ready_deadline = time.monotonic() + 120 + + while not container_healthy(target): + check( + container_running(target), + "API exited after restart", + ) + + check( + time.monotonic() < ready_deadline, + "API did not become healthy after restart", + ) + + time.sleep(2) + + print( + "PASS: the same API container recovered", + flush=True, + ) + + print( + "Waiting for temporary-file cleanup...", + flush=True, + ) + + cleanup_deadline = time.monotonic() + 120 + + while (target, temporary_path) in temporary_files([target]): + check( + time.monotonic() < cleanup_deadline, + "Old temporary file was not removed", + ) + + time.sleep(2) + + print( + "PASS: stale temporary file removed after API restart", + flush=True, + ) + + downloaded = client.call( + "GET", + "/api/v1/attachments/" + + good_attachment["id"] + + "/content", + actor=bob, + ) + + check( + downloaded["bytes"] == original, + "Confirmed attachment was lost or changed", + ) + + print( + "PASS: existing session and confirmed attachment survived", + flush=True, + ) + + client.attachment(alice, chat) + + print("PASS: subsequent upload succeeds", flush=True) + print("PASS: API crash scenario completed", flush=True) + + finally: + if connection is not None: + connection.close() + + if target is not None and not container_running(target): + command_output([ + "docker", "start", target, + ]) + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/tests/upload_db_failure_check.py b/tests/upload_db_failure_check.py new file mode 100644 index 00000000..6e9ba9e9 --- /dev/null +++ b/tests/upload_db_failure_check.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 + +import json +import re +import subprocess +import time +import uuid +from urllib.parse import quote + +from client import Client, arguments, check, run + + +def execute(command, input_text=None): + result = subprocess.run( + command, + input=input_text, + capture_output=True, + text=True, + timeout=30, + ) + + check( + result.returncode == 0, + "Infrastructure command failed. " + "Check PostgreSQL, MinIO and the configured bucket", + ) + + return result.stdout.strip() + + +def sql(query): + return execute( + [ + "docker", "compose", "exec", "-T", + "postgres", "sh", "-ec", + ''' + exec psql -X \ + -v ON_ERROR_STOP=1 \ + -U "$POSTGRES_USER" \ + -d "$POSTGRES_DB" \ + -At + ''', + ], + input_text=query, + ) + + +def minio(script, *arguments): + return execute([ + "docker", "compose", "exec", "-T", + "minio", "sh", "-ec", + script, + "failure-check", + *arguments, + ]) + + +def configure_minio(): + minio( + ''' + mc alias set failurecheck http://localhost:9000 \ + "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null + ''' + ) + + +def list_objects(bucket, prefix): + output = minio( + 'mc ls --json "failurecheck/$1/$2"', + bucket, + prefix, + ) + + objects = set() + + for line in output.splitlines(): + if not line.strip(): + continue + + item = json.loads(line) + + check( + item.get("status") != "error", + "MinIO failed to list test objects", + ) + + if item.get("key"): + objects.add(item["key"]) + + return objects + + +def wait_until(deadline): + remaining = deadline - time.monotonic() + + if remaining > 0: + time.sleep(remaining) + + +def main(): + def configure(parser): + parser.add_argument( + "--bucket", + default="messenger-attachments", + ) + + args = arguments( + "Failure between S3 upload and metadata INSERT", + configure, + ) + + check( + re.fullmatch( + r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", + args.bucket, + ) is not None, + "Invalid bucket name", + ) + + client = Client( + args.base_url, + args.ca_file, + timeout=40, + ) + + alice, bob, eve, chat = client.bootstrap() + + chat_id = str(uuid.UUID(chat["id"])) + + configure_minio() + + good_attachment, original = client.attachment(alice, chat) + + prefix = f"originals/{chat_id}/" + + before = list_objects(args.bucket, prefix) + + check( + len(before) == 1, + "Expected one control object in the new test chat", + ) + + marker = uuid.uuid4().hex + filename = f"db-failure-{marker}.png" + + function_name = f"test_attachment_failure_{marker}" + trigger_name = f"test_attachment_failure_{marker}" + + cleanup_sql = f""" + DROP TRIGGER IF EXISTS {trigger_name} + ON public.attachments; + + DROP FUNCTION IF EXISTS public.{function_name}(); + """ + + try: + sql(f""" + CREATE FUNCTION public.{function_name}() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + RAISE EXCEPTION + 'Synthetic attachment metadata failure'; + END; + $$; + + CREATE TRIGGER {trigger_name} + BEFORE INSERT ON public.attachments + FOR EACH ROW + WHEN (NEW.filename = '{filename}') + EXECUTE FUNCTION public.{function_name}(); + """) + + print( + "Installed a DB failure for one synthetic filename", + flush=True, + ) + + response = client.call( + "POST", + "/api/v1/chats/" + chat_id + + "/attachments?filename=" + + quote(filename, safe=""), + 503, + actor=alice, + raw=original, + headers={"Content-Type": "image/png"}, + ) + + check( + response["json"]["error"]["code"] == "unavailable", + "Expected unavailable error code", + ) + + print("PASS: failed metadata INSERT returns 503", flush=True) + + sql(cleanup_sql) + + count = sql(f""" + SELECT count(*) + FROM public.attachments + WHERE chat_id = '{chat_id}'::uuid + AND filename = '{filename}'; + """) + + check( + count == "0", + "Metadata was unexpectedly committed", + ) + + after = list_objects(args.bucket, prefix) + orphan_objects = after - before + + check( + len(orphan_objects) == 1, + "Expected exactly one S3 object without metadata", + ) + + orphan = next(iter(orphan_objects)) + observed_at = time.monotonic() + + print( + "PASS: S3 object exists, but metadata was rolled back", + flush=True, + ) + + wait_until(observed_at + 25) + + check( + orphan in list_objects(args.bucket, prefix), + "Orphan was deleted before the grace period", + ) + + print("PASS: grace period protects the recent object", flush=True) + print("Waiting for scheduled cleanup...", flush=True) + + deadline = observed_at + 100 + + while orphan in list_objects(args.bucket, prefix): + check( + time.monotonic() < deadline, + "Orphan was not deleted. Check cleanup settings and logs", + ) + + time.sleep(2) + + print( + "PASS: orphan created by a real failed INSERT was deleted", + flush=True, + ) + + downloaded = client.call( + "GET", + "/api/v1/attachments/" + + good_attachment["id"] + + "/content", + actor=bob, + ) + + check( + downloaded["bytes"] == original, + "Control attachment was lost or modified", + ) + + print("PASS: control attachment is intact", flush=True) + + client.attachment(alice, chat) + + print("PASS: normal uploads work after the failure", flush=True) + print("PASS: S3/DB failure scenario completed", flush=True) + + finally: + sql(cleanup_sql) + + +if __name__ == "__main__": + run(main) \ No newline at end of file diff --git a/tests/upload_memory_check.py b/tests/upload_memory_check.py new file mode 100644 index 00000000..6b37e734 --- /dev/null +++ b/tests/upload_memory_check.py @@ -0,0 +1,495 @@ +#!/usr/bin/env python3 + +import hashlib +import json +import subprocess +import threading +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from attachments_negative import make_png +from client import Client, arguments, check, run + + +def docker_output(command): + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=10, + ) + + check( + result.returncode == 0, + "Docker command failed while measuring memory", + ) + + return result.stdout.strip() + + +def get_api_containers(): + output = docker_output([ + "docker", "compose", "ps", "-q", "api", + ]) + + containers = output.splitlines() + + check( + len(containers) == 2, + "Expected exactly two running API replicas", + ) + + return containers + + +def memory_snapshot(container): + # Docker Desktop с cgroup v2. + # memory.current включает память всего контейнера, + # а не только Java heap. + output = docker_output([ + "docker", "exec", container, + "sh", "-ec", + ''' + cat /sys/fs/cgroup/memory.current + + if [ -r /sys/fs/cgroup/memory.peak ]; then + cat /sys/fs/cgroup/memory.peak + else + echo unavailable + fi + + cat /sys/fs/cgroup/memory.events + ''', + ]) + + lines = output.splitlines() + + check( + len(lines) >= 2, + "Cannot read cgroup v2 memory statistics", + ) + + events = {} + + for line in lines[2:]: + key, value = line.split() + events[key] = int(value) + + return { + "current_bytes": int(lines[0]), + "lifetime_peak_bytes": ( + int(lines[1]) + if lines[1].isdigit() + else None + ), + "oom": events.get("oom", 0), + "oom_kill": events.get("oom_kill", 0), + } + + +def mib(value): + if value is None: + return None + + return round(value / 1024 / 1024, 2) + + +def percentile(values, fraction): + if not values: + return None + + ordered = sorted(values) + index = int((len(ordered) - 1) * fraction) + + return round(ordered[index], 2) + + +def main(): + def configure(parser): + parser.add_argument("--concurrency", type=int, default=10) + parser.add_argument("--duration", type=int, default=30) + parser.add_argument( + "--output", + default="results/upload-memory.json", + ) + + args = arguments( + "Concurrent image uploads and API memory measurement", + configure, + ) + + check( + args.concurrency >= 10, + "Use at least 10 concurrent clients", + ) + + check( + args.duration >= 10, + "Duration must be at least 10 seconds", + ) + + controller = Client( + args.base_url, + args.ca_file, + timeout=40, + ) + + alice, bob, eve, chat = controller.bootstrap() + containers = get_api_containers() + + print("Generating PNG 6000x6000...", flush=True) + + image = make_png(6000, 6000) + checksum = hashlib.sha256(image).hexdigest() + + check( + len(image) <= 10 * 1024 * 1024, + "Generated image exceeds upload byte limit", + ) + + print( + f"Image: {len(image)} bytes, 36000000 pixels", + flush=True, + ) + + path = ( + "/api/v1/chats/" + + chat["id"] + + "/attachments?filename=memory-test.png" + ) + + clients = [ + Client(args.base_url, args.ca_file, timeout=40) + for _ in range(args.concurrency) + ] + + baseline = { + container: memory_snapshot(container) + for container in containers + } + + samples = {container: [] for container in containers} + monitor_errors = [] + stop_monitor = threading.Event() + + started = time.monotonic() + + def monitor(): + while not stop_monitor.is_set(): + for container in containers: + try: + snapshot = memory_snapshot(container) + + samples[container].append({ + "elapsed_seconds": round( + time.monotonic() - started, + 3, + ), + **snapshot, + }) + + except Exception as error: + monitor_errors.append( + type(error).__name__ + ) + stop_monitor.set() + return + + stop_monitor.wait(0.5) + + barrier = threading.Barrier(args.concurrency + 1) + + def worker(index): + client = clients[index] + + counters = Counter() + latencies = [] + errors = [] + instances = set() + successful_instances = set() + first_success = None + + barrier.wait() + + while time.monotonic() - started < args.duration: + request_started = time.monotonic() + + try: + response = client.call( + "POST", + path, + expected=[201, 429], + actor=alice, + raw=image, + headers={"Content-Type": "image/png"}, + schema="Attachment", + ) + + instance = response["headers"]["X-Instance-Id"] + instances.add(instance) + + if response["status"] == 429: + check( + response["json"]["error"]["code"] + == "rate_limited", + "Unexpected error code for 429", + ) + + counters["rate_limited"] += 1 + + else: + attachment = response["json"] + + check( + attachment["status"] == "ready", + "Successful upload is not ready", + ) + + check( + attachment["sha256"] == checksum, + "Uploaded SHA-256 mismatch", + ) + + check( + attachment["size_bytes"] == len(image), + "Uploaded size mismatch", + ) + + counters["successful"] += 1 + successful_instances.add(instance) + + latencies.append( + (time.monotonic() - request_started) * 1000 + ) + + if first_success is None: + first_success = attachment + + except Exception as error: + counters["unexpected_errors"] += 1 + + if len(errors) < 3: + errors.append(str(error)) + + time.sleep(0.1) + + return { + "counters": dict(counters), + "latencies": latencies, + "errors": errors, + "instances": instances, + "successful_instances": successful_instances, + "first_success": first_success, + } + + monitor_thread = threading.Thread( + target=monitor, + daemon=True, + ) + + with ThreadPoolExecutor( + max_workers=args.concurrency + ) as executor: + + futures = [ + executor.submit(worker, index) + for index in range(args.concurrency) + ] + + started = time.monotonic() + monitor_thread.start() + + print( + f"Starting {args.concurrency} clients " + f"for {args.duration} seconds...", + flush=True, + ) + + barrier.wait() + + worker_results = [ + future.result() + for future in futures + ] + + elapsed = time.monotonic() - started + + stop_monitor.set() + monitor_thread.join(timeout=25) + + check( + not monitor_thread.is_alive(), + "Memory monitor did not stop", + ) + + final = { + container: memory_snapshot(container) + for container in containers + } + + totals = Counter() + latencies = [] + errors = [] + instances = set() + successful_instances = set() + + for result in worker_results: + totals.update(result["counters"]) + latencies.extend(result["latencies"]) + instances.update(result["instances"]) + successful_instances.update( + result["successful_instances"] + ) + + errors.extend(result["errors"]) + + download_errors = [] + + for result in worker_results: + attachment = result["first_success"] + + if attachment is None: + continue + + try: + response = controller.call( + "GET", + "/api/v1/attachments/" + + attachment["id"] + + "/content", + actor=bob, + ) + + check( + response["bytes"] == image, + "Downloaded image differs from the original", + ) + + except Exception as error: + download_errors.append(str(error)) + + memory_report = {} + + for container in containers: + measurements = [ + baseline[container]["current_bytes"], + final[container]["current_bytes"], + ] + + measurements.extend( + sample["current_bytes"] + for sample in samples[container] + ) + + memory_report[container[:12]] = { + "baseline_mib": mib( + baseline[container]["current_bytes"] + ), + "observed_peak_mib": mib(max(measurements)), + "final_mib": mib( + final[container]["current_bytes"] + ), + "lifetime_peak_before_mib": mib( + baseline[container]["lifetime_peak_bytes"] + ), + "lifetime_peak_after_mib": mib( + final[container]["lifetime_peak_bytes"] + ), + "oom_delta": ( + final[container]["oom"] + - baseline[container]["oom"] + ), + "oom_kill_delta": ( + final[container]["oom_kill"] + - baseline[container]["oom_kill"] + ), + "sample_count": len(samples[container]), + } + + success = totals["successful"] + limited = totals["rate_limited"] + unexpected = totals["unexpected_errors"] + + passed = ( + success > 0 + and unexpected == 0 + and not monitor_errors + and not download_errors + and len(successful_instances) >= 2 + and all( + item["sample_count"] > 0 + and item["oom_delta"] == 0 + and item["oom_kill_delta"] == 0 + for item in memory_report.values() + ) + ) + + report = { + "passed": passed, + "workload": "closed-loop image uploads; no automatic retries", + "concurrency": args.concurrency, + "requested_duration_seconds": args.duration, + "elapsed_seconds": round(elapsed, 3), + "image": { + "width": 6000, + "height": 6000, + "pixels": 36_000_000, + "size_bytes": len(image), + "sha256": checksum, + }, + "successful_uploads": success, + "rate_limited_requests": limited, + "unexpected_errors": unexpected, + "successful_rps": round(success / elapsed, 3), + "observed_instances": sorted(instances), + "successful_instances": sorted(successful_instances), + "successful_latency_ms": { + "p50": percentile(latencies, 0.50), + "p95": percentile(latencies, 0.95), + "p99": percentile(latencies, 0.99), + }, + "memory": memory_report, + "error_examples": errors[:10], + "monitor_errors": monitor_errors, + "download_errors": download_errors, + "notes": [ + "429 responses are admission-control rejections, not successful uploads.", + "Observed memory peak is sampled and may miss short spikes.", + "Lifetime cgroup peak includes earlier container activity and startup.", + "Container memory includes more than Java heap.", + "Concurrency means concurrent clients, not 10 simultaneous decoders.", + ], + } + + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + + saved_report = { + **report, + "memory_samples": samples, + } + + output.write_text( + json.dumps(saved_report, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + print( + json.dumps(report, ensure_ascii=False, indent=2), + flush=True, + ) + + print("Report saved to:", output, flush=True) + + check( + passed, + "Experiment detected errors; inspect the saved report", + ) + + print("PASS: concurrent upload experiment completed", flush=True) + + +if __name__ == "__main__": + run(main) \ No newline at end of file