diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..1639521d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM maven:3.9.9-eclipse-temurin-21 AS build + +WORKDIR /build + +COPY java/pom.xml ./pom.xml + +RUN mvn -B dependency:go-offline + +COPY java/src ./src + +RUN mvn -B package -DskipTests + + +FROM eclipse-temurin:21.0.9_10-jre-jammy + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + python3 \ + python3-venv \ + wget \ + && python3 -m venv /opt/images \ + && /opt/images/bin/pip install --no-cache-dir Pillow==12.0.0 \ + && /opt/images/bin/python -c \ + "from PIL import features; assert features.check('webp')" \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --system --uid 1001 --create-home app + +WORKDIR /app + +COPY --from=build /build/target/*.jar /app/app.jar + +COPY infra/images/process_image.py /app/process_image.py +COPY infra/images/selftest.py /app/selftest.py + +USER 1001 + +EXPOSE 8080 + +ENTRYPOINT ["java", "-XX:MaxRAMPercentage=75", "-jar", "/app/app.jar"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index e98f75e9..59fa72c1 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,276 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Хуцаева Айса, БПИ2302. +- Целевая оценка: 4. +- Commit/PR: solution/khutsaeva-bpi2302 -## Запуск из чистого clone +### Подготовка -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +После клонирования репозитория перейти в его корень. + +Создать локальный файл конфигурации: + +```powershell +Copy-Item .env.example .env +``` + +Заполнить в `.env` пароли PostgreSQL, Redis, RabbitMQ, административной учётной записи MinIO и отдельной учётной записи API. Для генерации каждого нового пароля можно использовать: + +```powershell +python -c "import secrets; print(secrets.token_hex(24))" +``` + +Секреты не добавляются в Git и не включаются в отчёт. В `.env.example` должны находиться только примеры и безопасные значения по умолчанию. + +Для запуска тестов подготовить Python-окружение: + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install jsonschema openapi-spec-validator +$env:PYTHONUTF8 = "1" +``` + +Режим UTF-8 необходим, чтобы чтение контрактов на Windows не зависело от системной кодировки CP1251. Перед запуском проверить, что команда `python3`, используемая Makefile, доступна и видит зависимости тестов. + +### Сборка и запуск + +Проверить Compose: + +```powershell +docker compose config --quiet +``` + +Собрать приложение и MinIO: + +```powershell +docker compose build api minio +``` + +Запустить стенд: + +```powershell +docker compose up -d --no-build --pull never --scale api=2 --wait --wait-timeout 240 +``` + +При первом запуске предварительно загрузить внешние образы: + +```powershell +docker compose pull postgres redis migrate lb rabbitmq +``` + +Если запуск выполняется впервые, эту команду следует выполнить до `docker compose up`. + +Миграции выполняет отдельный сервис `migrate`. Сервис `minio-init` создаёт приватный bucket и настраивает права отдельной учётной записи приложения. Успешное завершение этих двух сервисов со статусом `Exited (0)` является нормальным. + +Проверить состояние: + +```powershell +docker compose ps -a +curl.exe -i http://localhost:8080/health/ready +``` + +Основной адрес API — `http://localhost:8080`. Запросы проходят через HAProxy. Прямой доступ к портам API и worker с хоста не требуется. + +Если стенд лабораторной 4 использует тот же порт, сначала остановить его из каталога соответствующего проекта. + +### Проверки + +Проверка обработчика изображений внутри worker: + +```powershell +docker compose exec -T worker /opt/images/bin/python /app/selftest.py +``` + +Публичные проверки: + +```powershell +make check +make validate +make test +``` + +Сценарии отказов уровня 4: + +```powershell +python3 -X utf8 tests/level4_check.py --base-url http://localhost:8080 --bucket messenger-attachments +``` + +Тест использует `compose.faults.yml`, временно включает внедрение ошибок, перезапускает процессы и затем восстанавливает обычную конфигурацию. Во время его выполнения не следует запускать параллельные тесты или ручные загрузки. + +Результат успешного выполнения сохраняется в: + +```text +results/lab5-level4.json +``` + +Если тест был принудительно прерван, обычную конфигурацию можно восстановить явно: + +```powershell +docker compose -f compose.yml -f compose.override.yml up -d --no-build --pull never --scale api=2 --wait --wait-timeout 240 +``` + +### Остановка + +Безопасная остановка с сохранением данных: + +```powershell +docker compose down +``` + +Намеренное удаление учебных данных, включая persistent volumes: + +```powershell +docker compose down -v +``` + +Вторая команда удаляет данные PostgreSQL, Redis, RabbitMQ и MinIO из томов этого Compose-проекта. Для обычной остановки она не используется. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент обращается к двум API-репликам через HAProxy. PostgreSQL хранит пользователей, чаты, сообщения, метаданные вложений, задания и transactional outbox. Redis используется функциональностью, перенесённой из предыдущих работ. + +MinIO хранит исходные файлы в staging и готовые результаты обработки. RabbitMQ доставляет задания отдельному worker. Сообщения брокера содержат идентификаторы и параметры задания, но не содержат бинарные данные или credentials. + +### Что перенесено из лабораторной 4 + +Перенесены: + +- пользователи, авторизация, чаты и сообщения; +- проверки доступа к чатам и вложениям; +- идемпотентное создание сообщений; +- PostgreSQL для метаданных; +- приватное S3-совместимое хранилище; +- проверка MIME по содержимому и ограничений изображения; +- две API-реплики, балансировщик и persistent volumes. + +Основное изменение: upload возвращает `202 queued` вместо `201 ready`. Клиент получает готовность через запрос метаданных. К сообщению можно привязать вложение своего чата в состоянии `queued` или `processing`. + +### Приём файла и transactional outbox + +API проверяет файл и сохраняет оригинал в устойчивый staging. Метаданные вложения, задание и запись outbox фиксируются в PostgreSQL одной транзакцией. + +Ответ `202` возвращается только после устойчивого сохранения оригинала и успешной фиксации задания. + +PostgreSQL является источником истины о состоянии задания и необходимости его публикации. RabbitMQ используется для доставки работы, но не является единственным местом хранения принятого намерения. + +Публикация выполняется отдельно. Используются persistent-сообщения, durable-очередь и publisher confirms. Если публикация или запись её подтверждения прерывается, outbox позволяет повторить отправку. + +Между S3 и PostgreSQL нет общей распределённой транзакции. При сбое могут остаться неучтённые объекты. Они удаляются механизмом очистки после grace period с проверкой ссылок из метаданных. + +### Обработка и фиксация результата + +Worker получает задание и захватывает его с lease. Доступны состояния: + +- `queued` — ожидает обработки; +- `processing` — выполняется worker; +- `ready` — результаты устойчиво сохранены; +- `failed` — постоянная ошибка или исчерпаны попытки. + +Worker сохраняет оригинал без изменения байтов и создаёт три варианта: + +| Вариант | Формат | Правила | +| --- | --- | --- | +| `thumbnail` | JPEG | Вписывание в 256×256 без увеличения небольших изображений | +| `crop` | JPEG | Центральная обрезка до 128×128, увеличение допускается | +| `optimized` | WebP | Вписывание в 1280×1280 без увеличения | + +Для JPEG используется quality 85, для WebP — quality 80 и method 4. Перед преобразованием учитывается EXIF-ориентация. Прозрачность JPEG заменяется белым фоном. Из производных изображений удаляются EXIF, ICC и XMP. + +После сохранения объектов worker транзакционно фиксирует метаданные вариантов и состояние `ready`. ACK отправляется после устойчивой фиксации результата. + +При повторной доставке готовой задачи worker проверяет уже сохранённый результат. Новая обработка и второй набор вариантов не создаются. + +### Повторы и ограничения ресурсов + +Постоянные ошибки не повторяются автоматически. Для временных ошибок используется ограниченное число попыток и backoff с jitter. + +Настройки решения: + +| Параметр | Значение | +| --- | --- | +| Максимальный размер входного файла | 10 MiB | +| Максимальное число пикселей | 40 млн | +| Обычный бюджет попыток | 3 | +| Конкуренция worker | 1 | +| Prefetch | 1 | +| Таймаут дочернего процесса обработки | 25 секунд | +| Ограничение CPU-времени дочернего процесса | 20 секунд, жёсткий предел 21 секунда | +| Ограничение адресного пространства дочернего процесса | 768 MiB | +| Лимит памяти контейнера worker | 1536 MiB | +| Максимальный Java heap worker | 256 MiB | +| Обычный lease задания | 240 секунд | + +Ограничение адресного пространства процесса не является измерением его фактического RSS. + +При исчерпании попыток задача получает `failed` с безопасным `error_code`. Неуспешные задания остаются в реестре PostgreSQL. Для недопустимых сообщений брокера предусмотрена DLQ. + +Явный replay выполняется оператором. Для `failed` проверяется доступность исходника и сохраняется запись аудита. Replay задачи в `ready` не запускает повторную обработку. + +Очистка staging выполняется после безопасной фиксации результата и истечения grace period. Используемые объекты и исходники активных заданий защищены от удаления. + +### Переменные конфигурации + +Основные переменные `.env.example` и `.env`: + +| Переменные | Назначение | +| --- | --- | +| `COMPOSE_PROJECT_NAME`, `APP_IMAGE`, `API_PORT` | Имя проекта, образ приложения, порт балансировщика | +| `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD` | PostgreSQL | +| `REDIS_PASSWORD`, `SESSION_TTL_SECONDS` | Redis и срок жизни сессии | +| `API_REPLICAS`, `API_CPUS`, `API_MEMORY`, `DB_POOL_SIZE` | Число API-реплик и ограничения ресурсов | +| `MINIO_ROOT_USER`, `MINIO_ROOT_PASSWORD` | Административная учётная запись bootstrap | +| `S3_BUCKET`, `S3_ACCESS_KEY`, `S3_SECRET_KEY` | Приватный bucket и ограниченная учётная запись приложения | +| `RABBITMQ_USER`, `RABBITMQ_PASSWORD` | Подключение к брокеру | +| `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS` | Бюджет попыток и длительность lease | +| `UPLOAD_MAX_CONCURRENT` | Ограничение одновременных upload на API-реплику | +| `ATTACHMENT_CLEANUP_GRACE_SECONDS` | Защитный интервал перед очисткой | +| `ATTACHMENT_CLEANUP_INTERVAL_MS`, `ATTACHMENT_CLEANUP_INITIAL_DELAY_MS` | Расписание очистки | +| `STAGING_RETENTION_SECONDS` | Срок сохранения исходников неуспешных задач | + +`WORKER_ENABLED` задаётся отдельно для API и worker в Compose. + +`FAULT_INJECTION_ENABLED` отключён в обычной конфигурации и включается только тестовым Compose-файлом. В тестах используются сокращённые lease и grace period; их нельзя выдавать за обычные эксплуатационные настройки. ## Доказательства критериев +Указание «вывод терминала» означает наблюдавшийся результат. Такой вывод необходимо сохранить и приложить к репозиторию отдельно; наличие файла лога не предполагается автоматически. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Структура skeleton и локальные ссылки | `make check` | `PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.` | Вывод терминала; отдельный лог пока не приложен | +| Валидность OpenAPI | `make validate` | `PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.` | Вывод терминала; отдельный лог пока не приложен | +| Публичный сценарий API | `make test` | `PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.` | Вывод терминала; отдельный лог пока не приложен | +| Устойчивое принятие задания после ответа upload | `level4_check.py`, сценарий `commit_before_publish` | После реального падения API принятое задание с тем же ID достигло `ready` | `results/lab5-level4.json` | +| Отдельные broker и worker | `docker compose ps -a` | RabbitMQ и worker работают отдельными контейнерами | `compose.yml`, вывод терминала | +| Статусы и готовность вложения | `make test`, `level4_check.py` | Проверены ожидание `ready`, конечный `failed` и восстановление через replay | `results/lab5-level4.json`, вывод smoke | +| Неизменность оригинала и корректность вариантов | `level4_check.py` | Оригинал совпадает побайтно; проверены размеры, MIME, размер файла и SHA-256 вариантов | `results/lab5-level4.json` | +| Ограниченные повторы | Сценарии `permanent_error`, `transient_error` | Постоянная ошибка завершилась после одной попытки, временная — после трёх | `results/lab5-level4.json` | +| Worker остановлен, API и брокер перезапущены с сохранением volumes | Отдельный сценарий уровня 3 | В приведённых результатах ещё не подтверждён | Требуется отдельное доказательство | +| Crop 128×128 и optimized WebP | `level4_check.py` | Для fixture получены crop 128×128 и WebP 320×200 | `results/lab5-level4.json` | +| Ориентация, прозрачность, центрирование, повреждённые и огромные изображения | `docker compose exec -T worker /opt/images/bin/python /app/selftest.py` | PASS для alpha, small, orientation, centered crop; отклонены broken, bytes_limit, pixels before decode | Вывод selftest; отдельный лог пока не приложен | +| Ограничение времени обработки | Сценарий `image_timeout` | Зависший дочерний процесс остановлен; повторная попытка завершилась успешно | `results/lab5-level4.json` | +| Ограничение памяти worker | Конфигурация контейнера и обработчика | Заданы лимиты контейнера, JVM и дочернего процесса; фактический пик памяти отдельно не измерен | `compose.override.yml`, `infra/images/process_image.py` | +| Outbox и crash между COMMIT и publish | Сценарий `commit_before_publish` | Принятое задание восстановлено без ручного создания нового | `results/lab5-level4.json` | +| Backoff с jitter | Сценарий `transient_error` | Задержки повторов попали в предусмотренные границы | `results/lab5-level4.json` | +| Безопасный replay | Сценарии постоянной и временной ошибки | Failed-задача восстановлена; replay готовой задачи не изменил результат | `results/lab5-level4.json` | +| Падение после записи объектов | Сценарий `after_objects` | Объекты существовали до фиксации результата; после восстановления задача завершилась | `results/lab5-level4.json` | +| Падение перед ACK | Сценарий `before_ack` | Повторная доставка подтверждена; ID задания, число попыток и результат не изменились относительно готового состояния до падения | `results/lab5-level4.json` | +| Восстановление истёкшего lease | Сценарий `after_objects` | После остановки worker задание повторно обработано | `results/lab5-level4.json` | +| Ограничение конкуренции и prefetch | Настройки listener | Concurrency 1 и prefetch 1 | `java/src/main/resources/application.yml` | +| Безопасная очистка staging | Сценарий `after_objects` | Staging и объекты незавершённой попытки удалены; подтверждённые результаты доступны | `results/lab5-level4.json` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Методика -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Проверки выполнялись на локальном стенде Docker Desktop: -## Ограничения и следующие шаги +- две API-реплики; +- один worker; +- один RabbitMQ; +- PostgreSQL, Redis и MinIO с persistent volumes; +- HAProxy перед API. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Проверялись восстановление принятого задания после падения API, повторная доставка после остановки worker, ограниченные retry, безопасный replay, таймаут обработки и очистка staging. \ No newline at end of file diff --git a/compose.faults.yml b/compose.faults.yml new file mode 100644 index 00000000..739b3f56 --- /dev/null +++ b/compose.faults.yml @@ -0,0 +1,15 @@ +services: + api: + environment: + FAULT_INJECTION_ENABLED: "true" + JOB_MAX_ATTEMPTS: "3" + JOB_TIMEOUT_SECONDS: "45" + ATTACHMENT_CLEANUP_GRACE_SECONDS: "90" + ATTACHMENT_CLEANUP_INTERVAL_MS: "1000" + + worker: + environment: + FAULT_INJECTION_ENABLED: "true" + JOB_MAX_ATTEMPTS: "3" + JOB_TIMEOUT_SECONDS: "45" + ATTACHMENT_CLEANUP_GRACE_SECONDS: "90" \ No newline at end of file diff --git a/compose.override.yml b/compose.override.yml new file mode 100644 index 00000000..2f088983 --- /dev/null +++ b/compose.override.yml @@ -0,0 +1,13 @@ +services: + api: + environment: + FAULT_INJECTION_ENABLED: "false" + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-240} + + worker: + mem_limit: 1536m + + environment: + FAULT_INJECTION_ENABLED: "false" + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-240} + JAVA_TOOL_OPTIONS: "-Djava.awt.headless=true -Xmx256m" \ No newline at end of file diff --git a/compose.yml b/compose.yml new file mode 100644 index 00000000..b0d9e217 --- /dev/null +++ b/compose.yml @@ -0,0 +1,317 @@ +services: + postgres: + image: postgres:16.6-alpine + + environment: + POSTGRES_DB: ${POSTGRES_DB:?} + POSTGRES_USER: ${POSTGRES_USER:?} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?} + + command: + - postgres + - -c + - max_connections=100 + - -c + - shared_buffers=128MB + + volumes: + - postgres_data:/var/lib/postgresql/data + + healthcheck: + test: + - CMD-SHELL + - pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" + interval: 2s + timeout: 1s + retries: 30 + + cpus: 1.0 + mem_limit: 512m + networks: [data] + + redis: + image: redis:7.4.1-alpine + + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD:?} + + command: + - sh + - -c + - | + exec redis-server \ + --requirepass "$$REDIS_PASSWORD" \ + --appendonly yes \ + --appendfsync always \ + --maxmemory 128mb \ + --maxmemory-policy noeviction + + volumes: + - redis_data:/data + + healthcheck: + test: + - CMD-SHELL + - REDISCLI_AUTH="$$REDIS_PASSWORD" redis-cli ping | grep -q PONG + interval: 2s + timeout: 1s + retries: 30 + + cpus: 0.5 + mem_limit: 192m + networks: [data] + + minio: + build: + context: ./infra/minio + dockerfile: Dockerfile + + image: mtusi5-minio:2025-04-22-local + + command: + - server + - /data + - --console-address + - ":9001" + + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?} + MINIO_API_STALE_UPLOADS_EXPIRY: "24h" + MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL: "1h" + + volumes: + - minio_data:/data + + ports: + - "127.0.0.1:9001:9001" + + healthcheck: + test: + - CMD + - curl + - -f + - http://localhost:9000/minio/health/live + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + + cpus: 1.0 + mem_limit: 768m + restart: unless-stopped + networks: [data] + + minio-init: + image: mtusi5-minio:2025-04-22-local + pull_policy: never + + depends_on: + minio: + condition: service_healthy + + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?} + S3_BUCKET: ${S3_BUCKET:?} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?} + S3_SECRET_KEY: ${S3_SECRET_KEY:?} + + volumes: + - ./infra/minio/bootstrap.sh:/bootstrap/bootstrap.sh:ro + - ./infra/minio/check-access.sh:/bootstrap/check-access.sh:ro + + entrypoint: + - /bin/sh + - /bootstrap/bootstrap.sh + + restart: "no" + networks: [data] + + rabbitmq: + image: rabbitmq:4.1.4-alpine + hostname: rabbitmq + + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:?} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?} + + volumes: + - rabbitmq_data:/var/lib/rabbitmq + + healthcheck: + test: + - CMD + - rabbitmq-diagnostics + - -q + - ping + interval: 5s + timeout: 5s + retries: 30 + start_period: 30s + + cpus: 1.0 + mem_limit: 512m + restart: unless-stopped + networks: [data] + + migrate: + image: flyway/flyway:10.20.1-alpine + + command: + - -connectRetries=30 + - migrate + + environment: + FLYWAY_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:?} + FLYWAY_USER: ${POSTGRES_USER:?} + FLYWAY_PASSWORD: ${POSTGRES_PASSWORD:?} + + volumes: + - ./java/src/main/resources/db/migration:/flyway/sql:ro + + depends_on: + postgres: + condition: service_healthy + + cpus: 0.5 + mem_limit: 384m + restart: "no" + networks: [data] + + api: + build: . + image: ${APP_IMAGE:-mtusi5-api:local} + + environment: &app-environment + DB_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:?} + DB_USERNAME: ${POSTGRES_USER:?} + DB_PASSWORD: ${POSTGRES_PASSWORD:?} + DB_POOL_SIZE: ${DB_POOL_SIZE:-8} + + REDIS_HOST: redis + REDIS_PASSWORD: ${REDIS_PASSWORD:?} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + + FLYWAY_ENABLED: "false" + + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?} + S3_SECRET_KEY: ${S3_SECRET_KEY:?} + S3_BUCKET: ${S3_BUCKET:?} + + RABBITMQ_HOST: rabbitmq + RABBITMQ_USER: ${RABBITMQ_USER:?} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:?} + + WORKER_ENABLED: "false" + + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-3} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-120} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-604800} + + UPLOAD_MAX_CONCURRENT: ${UPLOAD_MAX_CONCURRENT:-1} + UPLOAD_TEMP_DIR: /tmp/messenger-uploads + + ATTACHMENT_CLEANUP_ENABLED: "true" + ATTACHMENT_CLEANUP_GRACE_SECONDS: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:-3600} + ATTACHMENT_CLEANUP_INTERVAL_MS: ${ATTACHMENT_CLEANUP_INTERVAL_MS:-300000} + ATTACHMENT_CLEANUP_INITIAL_DELAY_MS: ${ATTACHMENT_CLEANUP_INITIAL_DELAY_MS:-60000} + + JAVA_TOOL_OPTIONS: "-Djava.awt.headless=true" + + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + minio-init: + condition: service_completed_successfully + + deploy: + replicas: ${API_REPLICAS:-2} + + cpus: ${API_CPUS:-1.0} + mem_limit: ${API_MEMORY:-768m} + + restart: unless-stopped + stop_grace_period: 20s + + healthcheck: + test: + - CMD + - wget + - -qO- + - http://localhost:8080/health/ready + interval: 5s + timeout: 15s + start_period: 60s + retries: 3 + + networks: [edge, data] + + worker: + image: ${APP_IMAGE:-mtusi5-api:local} + pull_policy: never + + environment: + <<: *app-environment + WORKER_ENABLED: "true" + ATTACHMENT_CLEANUP_ENABLED: "false" + + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + minio-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + + cpus: 1.0 + mem_limit: 768m + + restart: unless-stopped + stop_grace_period: 20s + + healthcheck: + test: + - CMD + - wget + - -qO- + - http://localhost:8080/health/live + interval: 5s + timeout: 5s + start_period: 60s + retries: 10 + + networks: [data] + + lb: + image: haproxy:3.0.5-alpine + + ports: + - "127.0.0.1:${API_PORT:-8080}:8080" + + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + + depends_on: + api: + condition: service_healthy + + cpus: 0.5 + mem_limit: 128m + networks: [edge] + +volumes: + postgres_data: + redis_data: + minio_data: + rabbitmq_data: + +networks: + edge: + data: + internal: true \ No newline at end of file diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..c080ca7f --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,33 @@ +global + log stdout format raw local0 + maxconn 2048 +defaults + mode http + log global + option httplog + timeout connect 1s + timeout client 12s + timeout server 10s + timeout check 1s + timeout queue 2s + retries 0 +resolvers docker + nameserver docker 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold refused 1s + hold timeout 1s +frontend public + bind :8080 + http-request deny if { path_beg /actuator } + default_backend api +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + server-template node 1-8 api:8080 check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none diff --git a/infra/images/process_image.py b/infra/images/process_image.py new file mode 100644 index 00000000..34989130 --- /dev/null +++ b/infra/images/process_image.py @@ -0,0 +1,176 @@ +import json +import os +from pathlib import Path +import resource +import sys +import time +import warnings + +from PIL import Image, ImageFile, ImageOps, UnidentifiedImageError + + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 + + +def fit_size(width, height, bound): + denominator = max(bound, width, height) + + return ( + max(1, width * bound // denominator), + max(1, height * bound // denominator), + ) + + +def white_rgb(image): + rgba = image.convert("RGBA") + output = Image.new("RGB", rgba.size, "white") + + output.paste( + rgba, + mask=rgba.getchannel("A"), + ) + + rgba.close() + return output + + +def main(): + # Ограничения дочернего процесса Linux. + resource.setrlimit( + resource.RLIMIT_AS, + (768 * 1024**2,) * 2, + ) + + resource.setrlimit( + resource.RLIMIT_CPU, + (20, 21), + ) + + resource.setrlimit( + resource.RLIMIT_FSIZE, + (11 * 1024**2,) * 2, + ) + + Image.MAX_IMAGE_PIXELS = MAX_PIXELS + ImageFile.LOAD_TRUNCATED_IMAGES = False + + warnings.simplefilter( + "error", + Image.DecompressionBombWarning, + ) + + source, output = map(Path, sys.argv[1:3]) + + if not 0 < source.stat().st_size <= MAX_BYTES: + return 10 + + if os.environ.get("IMAGE_TEST_SLEEP") == "1": + time.sleep(60) + + try: + image = Image.open(source) + + if ( + image.format not in ("PNG", "JPEG") + or image.width * image.height > MAX_PIXELS + ): + image.close() + return 10 + + image.load() + ImageOps.exif_transpose(image, in_place=True) + + except ( + ValueError, + UnidentifiedImageError, + OSError, + SyntaxError, + Image.DecompressionBombWarning, + Image.DecompressionBombError, + ): + return 10 + + with image: + manifest = [] + + variants = [ + ("thumbnail", 256, "image/jpeg", "jpg"), + ("crop", 128, "image/jpeg", "jpg"), + ("optimized", 1280, "image/webp", "webp"), + ] + + for name, bound, mime, extension in variants: + if name == "crop": + scaled = ImageOps.fit( + image, + (128, 128), + method=Image.Resampling.LANCZOS, + centering=(0.5, 0.5), + ) + else: + scaled = image.resize( + fit_size(*image.size, bound), + Image.Resampling.LANCZOS, + ) + + if extension == "jpg": + rendered = white_rgb(scaled) + else: + rendered = scaled.convert("RGBA") + + scaled.close() + + rendered.info.clear() + + path = output / (name + "." + extension) + + if extension == "jpg": + rendered.save( + path, + "JPEG", + quality=85, + subsampling=2, + optimize=True, + exif=b"", + icc_profile=None, + ) + else: + rendered.save( + path, + "WEBP", + quality=80, + method=4, + lossless=False, + exif=b"", + icc_profile=b"", + xmp=b"", + ) + + manifest.append( + { + "name": name, + "content_type": mime, + "width": rendered.width, + "height": rendered.height, + } + ) + + rendered.close() + + if not 0 < path.stat().st_size <= MAX_BYTES: + return 11 + + (output / "manifest.json").write_text( + json.dumps(manifest), + encoding="utf-8", + ) + + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (MemoryError, OSError): + sys.exit(11) \ No newline at end of file diff --git a/infra/images/selftest.py b/infra/images/selftest.py new file mode 100644 index 00000000..8f5ea0f8 --- /dev/null +++ b/infra/images/selftest.py @@ -0,0 +1,184 @@ +from pathlib import Path +import struct +import subprocess +import sys +import tempfile +import zlib + +from PIL import Image + + +SCRIPT = Path(__file__).with_name("process_image.py") + + +def execute(folder, expected): + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(folder / "input.bin"), + str(folder), + ], + capture_output=True, + timeout=30, + ) + + assert result.returncode == expected, ( + folder.name, + result.returncode, + ) + + +def main(): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + + cases = [ + ( + "alpha", + (320, 200), + "RGBA", + (255, 0, 0, 0), + None, + [(256, 160), (128, 128), (320, 200)], + ), + ( + "small", + (20, 10), + "RGB", + "red", + None, + [(20, 10), (128, 128), (20, 10)], + ), + ( + "orientation", + (320, 200), + "RGB", + "blue", + 6, + [(160, 256), (128, 128), (200, 320)], + ), + ] + + for label, size, mode, color, orientation, expected in cases: + folder = root / label + folder.mkdir() + + with Image.new(mode, size, color) as image: + if orientation: + exif = Image.Exif() + exif[274] = orientation + + image.save( + folder / "input.bin", + "JPEG", + exif=exif, + ) + else: + image.save(folder / "input.bin", "PNG") + + execute(folder, 0) + + names = [ + "thumbnail.jpg", + "crop.jpg", + "optimized.webp", + ] + + for name, dimensions in zip(names, expected): + with Image.open(folder / name) as result: + assert result.size == dimensions + assert not result.getexif() + + assert not any( + result.info.get(key) + for key in ["icc_profile", "xmp", "exif"] + ) + + if label == "alpha" and name.endswith(".jpg"): + assert result.getpixel((0, 0)) == ( + 255, + 255, + 255, + ) + + print("PASS:", label) + + folder = root / "center_crop" + folder.mkdir() + + with Image.new("RGB", (400, 200), "blue") as image: + image.paste("red", (0, 0, 100, 200)) + image.paste("green", (300, 0, 400, 200)) + image.save(folder / "input.bin", "PNG") + + execute(folder, 0) + + with Image.open(folder / "crop.jpg") as crop: + for x in [32, 96]: + red, green, blue = crop.getpixel((x, 64)) + + assert blue > 240 + assert red < 15 + assert green < 15 + + print("PASS: centered crop") + + for label, data in [ + ("broken", b"not an image"), + ("bytes_limit", b"x" * (10485760 + 1)), + ]: + folder = root / label + folder.mkdir() + + (folder / "input.bin").write_bytes(data) + + execute(folder, 10) + + print("PASS rejected:", label) + + folder = root / "pixels" + folder.mkdir() + + def chunk(kind, data): + return ( + struct.pack(">I", len(data)) + + kind + + data + + struct.pack( + ">I", + zlib.crc32(kind + data) & 0xffffffff, + ) + ) + + png = ( + b"\x89PNG\r\n\x1a\n" + + chunk( + b"IHDR", + struct.pack( + ">IIBBBBB", + 10000, + 10000, + 8, + 2, + 0, + 0, + 0, + ), + ) + + chunk( + b"IDAT", + zlib.compress(b"\0\0\0\0"), + ) + + chunk(b"IEND", b"") + ) + + (folder / "input.bin").write_bytes(png) + + execute(folder, 10) + + print("PASS rejected: pixels before decode") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/infra/minio/Dockerfile b/infra/minio/Dockerfile new file mode 100644 index 00000000..0cab71e9 --- /dev/null +++ b/infra/minio/Dockerfile @@ -0,0 +1,41 @@ +FROM golang:1.24.2-bookworm AS builder + +ENV CGO_ENABLED=0 + +WORKDIR /src + +RUN git clone --depth 1 \ + --branch RELEASE.2025-04-22T22-12-26Z \ + https://github.com/minio/minio.git /src/minio + +WORKDIR /src/minio + +RUN go build -trimpath -o /out/minio . + +WORKDIR /src + +RUN git clone --depth 1 \ + --branch RELEASE.2025-04-16T18-13-26Z \ + https://github.com/minio/mc.git /src/mc + +WORKDIR /src/mc + +RUN go build -trimpath -o /out/mc . + + +FROM debian:bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /out/minio /usr/local/bin/minio +COPY --from=builder /out/mc /usr/local/bin/mc + +RUN mkdir -p /data + +EXPOSE 9000 9001 + +ENTRYPOINT ["minio"] +CMD ["server", "/data", "--console-address", ":9001"] \ No newline at end of file diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh new file mode 100644 index 00000000..d142a586 --- /dev/null +++ b/infra/minio/bootstrap.sh @@ -0,0 +1,94 @@ +#!/bin/sh +set -eu + +: "${MINIO_ROOT_USER:?}" +: "${MINIO_ROOT_PASSWORD:?}" +: "${S3_BUCKET:?}" +: "${S3_ACCESS_KEY:?}" +: "${S3_SECRET_KEY:?}" + +if [ "$S3_ACCESS_KEY" = "$MINIO_ROOT_USER" ]; then + echo "API must use a separate MinIO user" >&2 + exit 1 +fi + +if [ "$S3_SECRET_KEY" = "$MINIO_ROOT_PASSWORD" ]; then + echo "API and administrator passwords must differ" >&2 + exit 1 +fi + +case "$S3_BUCKET" in + ""|*[!a-z0-9.-]*) + echo "Invalid bucket name" >&2 + exit 1 + ;; +esac + +mc alias set storage http://minio:9000 \ + "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null + +mc mb --ignore-existing "storage/$S3_BUCKET" +mc anonymous set none "storage/$S3_BUCKET" + +cat > /tmp/attachments-policy.json </dev/null + +mc admin user info --json \ + storage "$S3_ACCESS_KEY" > /tmp/api-user.json + +if grep -Eq \ + '"policyName"[[:space:]]*:[[:space:]]*"messenger-attachments-api"' \ + /tmp/api-user.json +then + echo "API policy is already attached" +else + mc admin policy attach \ + storage "$POLICY_NAME" --user "$S3_ACCESS_KEY" +fi + +mc alias set app http://minio:9000 \ + "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null + +mc stat "app/$S3_BUCKET" >/dev/null + +echo "Bootstrap completed: private bucket and application user are ready" \ No newline at end of file diff --git a/infra/minio/check-access.sh b/infra/minio/check-access.sh new file mode 100644 index 00000000..324c51cb --- /dev/null +++ b/infra/minio/check-access.sh @@ -0,0 +1,98 @@ +#!/bin/sh +set -eu + +: "${S3_BUCKET:?}" +: "${S3_ACCESS_KEY:?}" +: "${S3_SECRET_KEY:?}" + +mc alias set app http://minio:9000 \ + "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null + +TEST_ID="$(cat /proc/sys/kernel/random/uuid)" +OBJECT_KEY="originals/access-check-$TEST_ID.txt" +OUTSIDE_KEY="outside-access-check-$TEST_ID.txt" + +cleanup() { + mc rm --force \ + "app/$S3_BUCKET/$OBJECT_KEY" >/dev/null 2>&1 || true + + mc rm --force \ + "app/$S3_BUCKET/$OUTSIDE_KEY" >/dev/null 2>&1 || true +} + +trap cleanup EXIT + +printf 'private attachment test' > /tmp/expected.txt + +mc cp /tmp/expected.txt \ + "app/$S3_BUCKET/$OBJECT_KEY" >/dev/null + +mc cat "app/$S3_BUCKET/$OBJECT_KEY" > /tmp/actual.txt + +cmp /tmp/expected.txt /tmp/actual.txt + +echo "PASS: API credentials can upload and download originals" + +HTTP_CODE="$( + curl --silent --show-error \ + --connect-timeout 3 \ + --max-time 10 \ + --output /tmp/anonymous-response.xml \ + --write-out '%{http_code}' \ + "http://minio:9000/$S3_BUCKET/$OBJECT_KEY" +)" + +if [ "$HTTP_CODE" != "403" ]; then + echo "FAIL: anonymous GET returned HTTP $HTTP_CODE" >&2 + exit 1 +fi + +if ! grep -q 'AccessDenied' \ + /tmp/anonymous-response.xml +then + echo "FAIL: expected S3 AccessDenied response" >&2 + exit 1 +fi + +echo "PASS: anonymous download of an existing object is denied" + +if mc cp /tmp/expected.txt \ + "app/$S3_BUCKET/$OUTSIDE_KEY" \ + > /tmp/outside-result.txt 2>&1 +then + echo "FAIL: API credentials can write outside originals/" >&2 + exit 1 +fi + +if ! grep -Eiq \ + 'AccessDenied|Access Denied|Insufficient permissions|not authorized|not allowed' \ + /tmp/outside-result.txt +then + echo "FAIL: write failed for a reason other than access denial" >&2 + cat /tmp/outside-result.txt >&2 + exit 1 +fi + +echo "PASS: writing outside originals/ is denied" + +if mc admin info app > /tmp/admin-result.txt 2>&1 +then + echo "FAIL: API credentials have administrative access" >&2 + exit 1 +fi + +if ! grep -Eiq \ + 'AccessDenied|Access Denied|Insufficient permissions|not authorized|not allowed' \ + /tmp/admin-result.txt +then + echo "FAIL: admin command failed for an unexpected reason" >&2 + cat /tmp/admin-result.txt >&2 + exit 1 +fi + +echo "PASS: administrative access is denied" + +mc rm "app/$S3_BUCKET/$OBJECT_KEY" >/dev/null + +echo "PASS: API credentials can delete originals for cleanup" +echo "PASS: S3 access checks completed" \ No newline at end of file diff --git a/java/.gitignore b/java/.gitignore new file mode 100644 index 00000000..667aaef0 --- /dev/null +++ b/java/.gitignore @@ -0,0 +1,33 @@ +HELP.md +target/ +.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ diff --git a/java/.mvn/wrapper/maven-wrapper.properties b/java/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 00000000..216df058 --- /dev/null +++ b/java/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,3 @@ +wrapperVersion=3.3.4 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.16/apache-maven-3.9.16-bin.zip diff --git a/java/mvnw b/java/mvnw new file mode 100644 index 00000000..bd8896bf --- /dev/null +++ b/java/mvnw @@ -0,0 +1,295 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.4 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +scriptDir="$(dirname "$0")" +scriptName="$(basename "$0")" + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +actualDistributionDir="" + +# First try the expected directory name (for regular distributions) +if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then + if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then + actualDistributionDir="$distributionUrlNameMain" + fi +fi + +# If not found, search for any directory with the Maven executable (for snapshots) +if [ -z "$actualDistributionDir" ]; then + # enable globbing to iterate over items + set +f + for dir in "$TMP_DOWNLOAD_DIR"/*; do + if [ -d "$dir" ]; then + if [ -f "$dir/bin/$MVN_CMD" ]; then + actualDistributionDir="$(basename "$dir")" + break + fi + fi + done + set -f +fi + +if [ -z "$actualDistributionDir" ]; then + verbose "Contents of $TMP_DOWNLOAD_DIR:" + verbose "$(ls -la "$TMP_DOWNLOAD_DIR")" + die "Could not find Maven distribution directory in extracted archive" +fi + +verbose "Found extracted Maven distribution directory: $actualDistributionDir" +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/java/mvnw.cmd b/java/mvnw.cmd new file mode 100644 index 00000000..92450f93 --- /dev/null +++ b/java/mvnw.cmd @@ -0,0 +1,189 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.4 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' + +$MAVEN_M2_PATH = "$HOME/.m2" +if ($env:MAVEN_USER_HOME) { + $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME" +} + +if (-not (Test-Path -Path $MAVEN_M2_PATH)) { + New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null +} + +$MAVEN_WRAPPER_DISTS = $null +if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) { + $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists" +} else { + $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists" +} + +$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain" +$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +$actualDistributionDir = "" + +# First try the expected directory name (for regular distributions) +$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain" +$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD" +if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) { + $actualDistributionDir = $distributionUrlNameMain +} + +# If not found, search for any directory with the Maven executable (for snapshots) +if (!$actualDistributionDir) { + Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object { + $testPath = Join-Path $_.FullName "bin/$MVN_CMD" + if (Test-Path -Path $testPath -PathType Leaf) { + $actualDistributionDir = $_.Name + } + } +} + +if (!$actualDistributionDir) { + Write-Error "Could not find Maven distribution directory in extracted archive" +} + +Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir" +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/java/pom.xml b/java/pom.xml new file mode 100644 index 00000000..8a233485 --- /dev/null +++ b/java/pom.xml @@ -0,0 +1,139 @@ + + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 3.3.5 + + + + com.vnp + demo + 0.0.1-SNAPSHOT + mtusi-messenger + + + 21 + 21 + + + + + + + org.springframework.boot + spring-boot-starter-web + + + + + org.springframework.boot + spring-boot-starter-validation + + + + + org.springframework.boot + spring-boot-starter-actuator + + + + + org.springframework.boot + spring-boot-starter-data-jpa + + + + + org.postgresql + postgresql + runtime + + + + org.springframework.boot + spring-boot-starter-data-redis + + + + + org.flywaydb + flyway-core + + + + org.flywaydb + flyway-database-postgresql + + + + + org.springframework.boot + spring-boot-starter-security + + + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.bouncycastle + bcprov-jdk18on + 1.78.1 + + + + + io.rest-assured + rest-assured + test + + + + io.minio + minio + 8.5.17 + + + + org.springframework.boot + spring-boot-starter-amqp + + + + com.drewnoakes + metadata-extractor + 2.19.0 + + + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + + + + + + + diff --git a/java/src/main/java/com/vnp/demo/BusinessException.java b/java/src/main/java/com/vnp/demo/BusinessException.java new file mode 100644 index 00000000..d1d4a1fb --- /dev/null +++ b/java/src/main/java/com/vnp/demo/BusinessException.java @@ -0,0 +1,63 @@ +package com.vnp.demo; + +import org.springframework.http.HttpStatus; + +public class BusinessException extends RuntimeException { + + private final String code; + private final HttpStatus status; + + public BusinessException(String code, String message, HttpStatus status) { + super(message); + this.code = code; + this.status = status; + } + + public String getCode() { + return code; + } + + public HttpStatus getStatus() { + return status; + } + + public static BusinessException notFound() { + return new BusinessException( + "not_found", + "Объект не найден", + HttpStatus.NOT_FOUND + ); + } + + public static BusinessException conflict() { + return conflict("Конфликт"); + } + + public static BusinessException conflict(String message) { + return new BusinessException( + "conflict", + message, + HttpStatus.CONFLICT + ); + } + + public static BusinessException unauthorized() { + return unauthorized("Пользователь не найден"); + } + + public static BusinessException unauthorized(String message) { + return new BusinessException( + "unauthorized", + message, + HttpStatus.UNAUTHORIZED + ); + } + + public static BusinessException invalid(String message) { + return new BusinessException( + "invalid_request", + message, + HttpStatus.BAD_REQUEST + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/CursorCodec.java b/java/src/main/java/com/vnp/demo/CursorCodec.java new file mode 100644 index 00000000..aa3e1f9d --- /dev/null +++ b/java/src/main/java/com/vnp/demo/CursorCodec.java @@ -0,0 +1,184 @@ +package com.vnp.demo; + +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.format.DateTimeParseException; +import java.util.Base64; +import java.util.UUID; + +import org.springframework.stereotype.Component; + +@Component +public class CursorCodec { + + public record ChatCursor(Instant createdAt, UUID id) { + } + + public record MessageCursor(Instant createdAt, UUID id) { + } + + public String encodeIndex(int index) { + return Base64.getUrlEncoder() + .withoutPadding() + .encodeToString( + ("i:" + index).getBytes(StandardCharsets.UTF_8) + ); + } + + public int decodeIndex(String cursor, int size) { + try { + String decoded = new String( + Base64.getUrlDecoder().decode(cursor), + StandardCharsets.UTF_8 + ); + + if (!decoded.startsWith("i:")) { + throw new IllegalArgumentException(); + } + + int idx = Integer.parseInt(decoded.substring(2)); + + if (idx < 0 || idx > size) { + throw new IllegalArgumentException(); + } + + return idx; + + } catch (IllegalArgumentException ex) { + throw BusinessException.invalid("Неверный cursor"); + } + } + + public String encodeChat( + UUID userId, + Instant createdAt, + UUID chatId + ) { + String value = "chat:v1:" + + userId + ":" + + createdAt + ":" + + chatId; + + return Base64.getUrlEncoder() + .withoutPadding() + .encodeToString( + value.getBytes(StandardCharsets.UTF_8) + ); + } + + public ChatCursor decodeChat(String cursor, UUID userId) { + try { + String value = decodeBase64(cursor); + + if (!value.startsWith("chat:v1:")) { + throw new IllegalArgumentException(); + } + + String content = value.substring("chat:v1:".length()); + + int firstSeparator = content.indexOf(':'); + int lastSeparator = content.lastIndexOf(':'); + + if (firstSeparator < 0 + || lastSeparator <= firstSeparator) { + throw new IllegalArgumentException(); + } + + UUID cursorUserId = UUID.fromString( + content.substring(0, firstSeparator) + ); + + if (!cursorUserId.equals(userId)) { + throw new IllegalArgumentException(); + } + + Instant createdAt = Instant.parse( + content.substring( + firstSeparator + 1, + lastSeparator + ) + ); + + UUID chatId = UUID.fromString( + content.substring(lastSeparator + 1) + ); + + return new ChatCursor(createdAt, chatId); + + } catch (IllegalArgumentException | DateTimeParseException ex) { + throw BusinessException.invalid("Неверный cursor"); + } + } + + public String encodeMessage( + UUID chatId, + Instant createdAt, + UUID messageId + ) { + String value = "message:v1:" + + chatId + ":" + + createdAt + ":" + + messageId; + + return Base64.getUrlEncoder() + .withoutPadding() + .encodeToString( + value.getBytes(StandardCharsets.UTF_8) + ); + } + + public MessageCursor decodeMessage(String cursor, UUID chatId) { + try { + String value = decodeBase64(cursor); + + if (!value.startsWith("message:v1:")) { + throw new IllegalArgumentException(); + } + + String content = value.substring("message:v1:".length()); + + int firstSeparator = content.indexOf(':'); + int lastSeparator = content.lastIndexOf(':'); + + if (firstSeparator < 0 + || lastSeparator <= firstSeparator) { + throw new IllegalArgumentException(); + } + + UUID cursorChatId = UUID.fromString( + content.substring(0, firstSeparator) + ); + + if (!cursorChatId.equals(chatId)) { + throw new IllegalArgumentException(); + } + + Instant createdAt = Instant.parse( + content.substring( + firstSeparator + 1, + lastSeparator + ) + ); + + UUID messageId = UUID.fromString( + content.substring(lastSeparator + 1) + ); + + return new MessageCursor(createdAt, messageId); + + } catch (IllegalArgumentException | DateTimeParseException ex) { + throw BusinessException.invalid("Неверный cursor"); + } + } + + private String decodeBase64(String cursor) { + if (cursor == null || cursor.isBlank()) { + throw new IllegalArgumentException(); + } + + return new String( + Base64.getUrlDecoder().decode(cursor), + StandardCharsets.UTF_8 + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/GlobalExceptionHandler.java b/java/src/main/java/com/vnp/demo/GlobalExceptionHandler.java new file mode 100644 index 00000000..a6f49115 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/GlobalExceptionHandler.java @@ -0,0 +1,150 @@ + +package com.vnp.demo; + +import org.slf4j.MDC; +import org.springframework.dao.DataAccessException; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.web.bind.MethodArgumentNotValidException; +import org.springframework.web.bind.MissingRequestHeaderException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException; +import org.springframework.web.server.ResponseStatusException; +import org.springframework.web.servlet.resource.NoResourceFoundException; + +import com.vnp.demo.dto.ErrorResponse; + +import jakarta.servlet.http.HttpServletRequest; + +@RestControllerAdvice +public class GlobalExceptionHandler { + + private ErrorResponse build(String code, String msg) { + return new ErrorResponse( + new ErrorResponse.ErrorBody( + code, + msg, + MDC.get("requestId") + ) + ); + } + + @ExceptionHandler(BusinessException.class) + public ResponseEntity handleBusiness( + BusinessException e, + HttpServletRequest request + ) { + ResponseEntity.BodyBuilder builder = + ResponseEntity.status(e.getStatus()); + + if (e.getStatus() == HttpStatus.TOO_MANY_REQUESTS) { + builder.header("Retry-After", "1"); + } + + if (e.getStatus() == HttpStatus.UNAUTHORIZED) { + String path = request.getRequestURI(); + + String challenge = path.equals("/api/v1/auth/sessions") + ? "Basic realm=\"messenger\", charset=\"UTF-8\"" + : "Bearer realm=\"messenger\""; + + builder.header("WWW-Authenticate", challenge); + } + + return builder.body( + build(e.getCode(), e.getMessage()) + ); + } + + @ExceptionHandler(MethodArgumentNotValidException.class) + public ResponseEntity handleValidation( + MethodArgumentNotValidException e + ) { + String msg = e.getBindingResult() + .getFieldErrors() + .stream() + .findFirst() + .map(f -> f.getField() + ": " + f.getDefaultMessage()) + .orElse("Ошибка валидации"); + + return ResponseEntity.badRequest() + .body(build("invalid_request", msg)); + } + + @ExceptionHandler({ + HttpMessageNotReadableException.class, + MethodArgumentTypeMismatchException.class + }) + public ResponseEntity handleMalformed(Exception e) { + return ResponseEntity.badRequest() + .body(build( + "invalid_request", + "Неверный формат запроса" + )); + } + + @ExceptionHandler(NoResourceFoundException.class) + public ResponseEntity handleNotFound( + NoResourceFoundException e + ) { + return ResponseEntity.status(HttpStatus.NOT_FOUND) + .body(build( + "not_found", + "Ресурс не найден" + )); + } + + @ExceptionHandler(MissingRequestHeaderException.class) + public ResponseEntity handleMissingHeader( + MissingRequestHeaderException e + ) { + return ResponseEntity.badRequest() + .body(build( + "invalid_request", + "Отсутствует обязательный заголовок: " + + e.getHeaderName() + )); + } + + @ExceptionHandler(DataAccessException.class) + public ResponseEntity handleDataAccess( + DataAccessException e + ) { + return ResponseEntity + .status(HttpStatus.SERVICE_UNAVAILABLE) + .body(build( + "unavailable", + "Сервис временно недоступен" + )); + } + + @ExceptionHandler(ResponseStatusException.class) + public ResponseEntity handleStatus( + ResponseStatusException e + ) { + int status = e.getStatusCode().value(); + + String code = status == 503 + ? "unavailable" + : "request_failed"; + + String message = status == 503 + ? "Сервис временно недоступен" + : "Запрос не может быть выполнен"; + + return ResponseEntity.status(status) + .body(build(code, message)); + } + + @ExceptionHandler(Exception.class) + public ResponseEntity handleGeneric(Exception e) { + return ResponseEntity + .status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(build( + "internal_error", + "Внутренняя ошибка" + )); + } +} diff --git a/java/src/main/java/com/vnp/demo/MessengerApplication.java b/java/src/main/java/com/vnp/demo/MessengerApplication.java new file mode 100644 index 00000000..482bc6e0 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/MessengerApplication.java @@ -0,0 +1,13 @@ +package com.vnp.demo; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class MessengerApplication { + + public static void main(String[] args) { + SpringApplication.run(MessengerApplication.class, args); + } + +} diff --git a/java/src/main/java/com/vnp/demo/SessionStoreExceptionHandler.java b/java/src/main/java/com/vnp/demo/SessionStoreExceptionHandler.java new file mode 100644 index 00000000..5f98d7ba --- /dev/null +++ b/java/src/main/java/com/vnp/demo/SessionStoreExceptionHandler.java @@ -0,0 +1,24 @@ +package com.vnp.demo; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +import java.util.Map; + +@RestControllerAdvice +public class SessionStoreExceptionHandler { + + @ExceptionHandler(SessionStoreUnavailableException.class) + public ResponseEntity> handleSessionStoreUnavailable( + SessionStoreUnavailableException ex + ) { + return ResponseEntity + .status(HttpStatus.SERVICE_UNAVAILABLE) + .body(Map.of( + "error", "SERVICE_UNAVAILABLE", + "message", "Session storage is temporarily unavailable" + )); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/SessionStoreUnavailableException.java b/java/src/main/java/com/vnp/demo/SessionStoreUnavailableException.java new file mode 100644 index 00000000..818fe376 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/SessionStoreUnavailableException.java @@ -0,0 +1,8 @@ +package com.vnp.demo; + +public class SessionStoreUnavailableException extends RuntimeException { + + public SessionStoreUnavailableException(Throwable cause) { + super("Session storage is temporarily unavailable", cause); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/config/JacksonConfig.java b/java/src/main/java/com/vnp/demo/config/JacksonConfig.java new file mode 100644 index 00000000..53920141 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/config/JacksonConfig.java @@ -0,0 +1,22 @@ +package main.java.com.vnp.demo.config; + +import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.fasterxml.jackson.databind.cfg.CoercionAction; +import com.fasterxml.jackson.databind.cfg.CoercionInputShape; +import com.fasterxml.jackson.databind.type.LogicalType; + +@Configuration +public class JacksonConfig { + @Bean + public Jackson2ObjectMapperBuilderCustomizer strictTextValues() { + return builder -> builder.postConfigurer(mapper -> + mapper.coercionConfigFor(LogicalType.Textual) + .setCoercion(CoercionInputShape.Integer, CoercionAction.Fail) + .setCoercion(CoercionInputShape.Float, CoercionAction.Fail) + .setCoercion(CoercionInputShape.Boolean, CoercionAction.Fail)); + } +} + diff --git a/java/src/main/java/com/vnp/demo/config/PasswordConfig.java b/java/src/main/java/com/vnp/demo/config/PasswordConfig.java new file mode 100644 index 00000000..7cc12a89 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/config/PasswordConfig.java @@ -0,0 +1,15 @@ +package com.vnp.demo.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.crypto.argon2.Argon2PasswordEncoder; +import org.springframework.security.crypto.password.PasswordEncoder; + +@Configuration +public class PasswordConfig { + + @Bean + public PasswordEncoder passwordEncoder() { + return Argon2PasswordEncoder.defaultsForSpringSecurity_v5_8(); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/config/RabbitConfig.java b/java/src/main/java/com/vnp/demo/config/RabbitConfig.java new file mode 100644 index 00000000..29c087fb --- /dev/null +++ b/java/src/main/java/com/vnp/demo/config/RabbitConfig.java @@ -0,0 +1,26 @@ +package com.vnp.demo.config; + +import org.springframework.amqp.core.Declarables; +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.core.QueueBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class RabbitConfig { + + public static final String QUEUE = "attachments.v1"; + public static final String DLQ = "attachments.v1.dlq"; + + @Bean + public Declarables attachmentQueues() { + return new Declarables( + new Queue(DLQ, true), + + QueueBuilder.durable(QUEUE) + .deadLetterExchange("") + .deadLetterRoutingKey(DLQ) + .build() + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/config/SchedulingConfig.java b/java/src/main/java/com/vnp/demo/config/SchedulingConfig.java new file mode 100644 index 00000000..8df555d0 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/config/SchedulingConfig.java @@ -0,0 +1,9 @@ +package com.vnp.demo.config; + +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableScheduling; + +@Configuration +@EnableScheduling +public class SchedulingConfig { +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/config/SecurityConfig.java b/java/src/main/java/com/vnp/demo/config/SecurityConfig.java new file mode 100644 index 00000000..4360c6f6 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/config/SecurityConfig.java @@ -0,0 +1,75 @@ +package com.vnp.demo.config; + +import java.util.Map; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.web.SecurityFilterChain; +import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.vnp.demo.filter.BearerAuthenticationFilter; +import com.vnp.demo.service.AuthService; + +import jakarta.servlet.http.HttpServletResponse; + +@Configuration +public class SecurityConfig { + + @Bean + public SecurityFilterChain securityFilterChain( + HttpSecurity http, + AuthService authService, + ObjectMapper objectMapper + ) throws Exception { + + return http + .csrf(csrf -> csrf.disable()) + .httpBasic(basic -> basic.disable()) + .formLogin(form -> form.disable()) + .sessionManagement(session -> session + .sessionCreationPolicy(SessionCreationPolicy.STATELESS) + ) + .authorizeHttpRequests(auth -> auth + .requestMatchers( + "/health/live", + "/health/ready" + ).permitAll() + .requestMatchers( + org.springframework.http.HttpMethod.POST, + "/api/v1/users", + "/api/v1/auth/sessions" + ).permitAll() + .anyRequest().authenticated() + ) + .exceptionHandling(ex -> ex + .authenticationEntryPoint((request, response, exception) -> { + response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + response.setHeader( + HttpHeaders.WWW_AUTHENTICATE, + "Bearer realm=\"messenger\"" + ); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + response.setCharacterEncoding("UTF-8"); + + objectMapper.writeValue( + response.getWriter(), + Map.of("error", Map.of( + "code", "unauthorized", + "message", "Требуется авторизация", + "request_id", response.getHeader("X-Request-Id") + )) + ); + }) + ) + .addFilterBefore( + new BearerAuthenticationFilter(authService, objectMapper), + UsernamePasswordAuthenticationFilter.class + ) + .build(); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/config/UploadTimeoutConfig.java b/java/src/main/java/com/vnp/demo/config/UploadTimeoutConfig.java new file mode 100644 index 00000000..8a8af35e --- /dev/null +++ b/java/src/main/java/com/vnp/demo/config/UploadTimeoutConfig.java @@ -0,0 +1,25 @@ +package com.vnp.demo.config; + +import org.apache.coyote.http11.AbstractHttp11Protocol; +import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory; +import org.springframework.boot.web.server.WebServerFactoryCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class UploadTimeoutConfig { + + @Bean + public WebServerFactoryCustomizer + uploadTimeoutCustomizer() { + + return factory -> factory.addConnectorCustomizers(connector -> { + if (connector.getProtocolHandler() + instanceof AbstractHttp11Protocol protocol) { + + protocol.setDisableUploadTimeout(false); + protocol.setConnectionUploadTimeout(15_000); + } + }); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/controllers/AttachmentController.java b/java/src/main/java/com/vnp/demo/controllers/AttachmentController.java new file mode 100644 index 00000000..a6f402d2 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/controllers/AttachmentController.java @@ -0,0 +1,100 @@ +package com.vnp.demo.controllers; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +import org.springframework.http.ContentDisposition; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.dto.AttachmentResponse; +import com.vnp.demo.service.AttachmentService; + +import jakarta.servlet.http.HttpServletRequest; + +@RestController +@RequestMapping("/api/v1") +public class AttachmentController { + + private final AttachmentService attachmentService; + + public AttachmentController( + AttachmentService attachmentService + ) { + this.attachmentService = attachmentService; + } + + @PostMapping("/chats/{chat_id}/attachments") + @ResponseStatus(HttpStatus.ACCEPTED) + public AttachmentResponse upload( + @AuthenticationPrincipal UUID userId, + @PathVariable("chat_id") UUID chatId, + @RequestParam(value = "filename", required = false) + String filename, + HttpServletRequest request + ) { + try { + return attachmentService.upload( + userId, + chatId, + filename, + request.getContentType(), + request.getContentLengthLong(), + request.getInputStream() + ); + } catch (IOException e) { + throw BusinessException.invalid( + "Не удалось прочитать тело запроса" + ); + } + } + + @GetMapping("/attachments/{id}") + public AttachmentResponse metadata( + @AuthenticationPrincipal UUID userId, + @PathVariable("id") UUID id + ) { + return attachmentService.getMetadata(userId, id); + } + + @GetMapping("/attachments/{id}/content") + public ResponseEntity content( + @AuthenticationPrincipal UUID userId, + @PathVariable("id") UUID id, + @RequestParam(defaultValue = "original") + String variant + ) { + AttachmentService.Download download = + attachmentService.download(userId, id, variant); + + String disposition = ContentDisposition.attachment() + .filename( + download.filename(), + StandardCharsets.UTF_8 + ) + .build() + .toString(); + + return ResponseEntity.ok() + .contentType( + MediaType.parseMediaType(download.contentType()) + ) + .contentLength(download.bytes().length) + .header(HttpHeaders.CONTENT_DISPOSITION, disposition) + .header(HttpHeaders.CACHE_CONTROL, "private, no-store") + .header("X-Content-Type-Options", "nosniff") + .body(download.bytes()); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/controllers/AuthController.java b/java/src/main/java/com/vnp/demo/controllers/AuthController.java new file mode 100644 index 00000000..13382916 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/controllers/AuthController.java @@ -0,0 +1,93 @@ +package com.vnp.demo.controllers; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.dto.SessionResponse; +import com.vnp.demo.service.AuthService; + +import jakarta.servlet.http.HttpServletRequest; + +@RestController +@RequestMapping("/api/v1/auth/sessions") +public class AuthController { + + private final AuthService authService; + + public AuthController(AuthService authService) { + this.authService = authService; + } + + @PostMapping + @ResponseStatus(HttpStatus.CREATED) + public SessionResponse login( + @RequestHeader(value = HttpHeaders.AUTHORIZATION, required = false) + String authorization, + HttpServletRequest request + ) throws IOException { + if (request.getInputStream().read() != -1) { + throw BusinessException.invalid("Тело запроса создания сессии должно быть пустым"); + } + if (authorization == null + || !authorization.startsWith("Basic ")) { + throw BusinessException.unauthorized(); + } + + String encoded = authorization.substring(6).trim(); + String credentials; + + try { + byte[] decoded = Base64.getDecoder().decode(encoded); + + credentials = StandardCharsets.UTF_8 + .newDecoder() + .decode(ByteBuffer.wrap(decoded)) + .toString(); + + } catch (IllegalArgumentException | CharacterCodingException ex) { + throw BusinessException.unauthorized(); + } + + int separator = credentials.indexOf(':'); + + if (separator < 0) { + throw BusinessException.unauthorized(); + } + + String username = credentials.substring(0, separator); + String password = credentials.substring(separator + 1); + + if (!username.matches("[a-z0-9_]{3,32}")) { + throw BusinessException.unauthorized(); + } + + return authService.login(username, password); + } + + @DeleteMapping("/current") + @ResponseStatus(HttpStatus.NO_CONTENT) + public void logout( + @RequestHeader(value = HttpHeaders.AUTHORIZATION, required = false) + String authorization + ) { + if (authorization == null + || !authorization.startsWith("Bearer ")) { + throw BusinessException.unauthorized(); + } + + authService.logout(authorization.substring(7).trim()); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/controllers/ChatController.java b/java/src/main/java/com/vnp/demo/controllers/ChatController.java new file mode 100644 index 00000000..7e3baa41 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/controllers/ChatController.java @@ -0,0 +1,58 @@ +package com.vnp.demo.controllers; + +import java.util.UUID; + +import org.springframework.http.HttpStatus; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import com.vnp.demo.dto.ChatCreateRequest; +import com.vnp.demo.dto.ChatPageResponse; +import com.vnp.demo.dto.ChatResponse; +import com.vnp.demo.service.ChatService; + +import jakarta.validation.Valid; + +@RestController +@RequestMapping("/api/v1/chats") +public class ChatController { + + private final ChatService chatService; + + public ChatController(ChatService chatService) { + this.chatService = chatService; + } + + @PostMapping + @ResponseStatus(HttpStatus.CREATED) + public ChatResponse create( + @AuthenticationPrincipal UUID userId, + @Valid @RequestBody ChatCreateRequest request + ) { + return chatService.create(userId, request); + } + + @GetMapping + public ChatPageResponse list( + @AuthenticationPrincipal UUID userId, + @RequestParam(defaultValue = "50") int limit, + @RequestParam(required = false) String cursor + ) { + return chatService.list(userId, limit, cursor); + } + + @GetMapping("/{chat_id}") + public ChatResponse get( + @AuthenticationPrincipal UUID userId, + @PathVariable("chat_id") UUID chatId + ) { + return chatService.get(userId, chatId); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/controllers/HealthController.java b/java/src/main/java/com/vnp/demo/controllers/HealthController.java new file mode 100644 index 00000000..70a92c26 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/controllers/HealthController.java @@ -0,0 +1,49 @@ +package com.vnp.demo.controllers; + +import org.springframework.boot.availability.ApplicationAvailability; +import org.springframework.boot.availability.ReadinessState; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.http.HttpStatus; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import com.vnp.demo.service.S3StorageService; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.dto.HealthResponse; + +@RestController +@RequestMapping("/health") +public class HealthController { + private final JdbcTemplate jdbc; + private final StringRedisTemplate redis; + private final ApplicationAvailability availability; + private final S3StorageService storage; + public HealthController( + JdbcTemplate jdbc, + StringRedisTemplate redis, + ApplicationAvailability availability, + S3StorageService storage + ) { + this.jdbc = jdbc; + this.redis = redis; + this.availability = availability; + this.storage = storage; + } + @GetMapping("/live") + public HealthResponse live() { return HealthResponse.ok(); } + @GetMapping("/ready") + public HealthResponse ready() { + try { + if (availability.getReadinessState() != ReadinessState.ACCEPTING_TRAFFIC) throw new IllegalStateException(); + if (!Integer.valueOf(1).equals(jdbc.queryForObject("SELECT 1", Integer.class))) throw new IllegalStateException(); + String pong = redis.execute((org.springframework.data.redis.core.RedisCallback) c -> c.ping()); + if (!"PONG".equals(pong)) throw new IllegalStateException(); + storage.checkReady(); + return HealthResponse.ok(); + } catch (Exception ex) { + throw new BusinessException("service_unavailable", "Сервис не готов", HttpStatus.SERVICE_UNAVAILABLE); + } + } +} diff --git a/java/src/main/java/com/vnp/demo/controllers/MessageController.java b/java/src/main/java/com/vnp/demo/controllers/MessageController.java new file mode 100644 index 00000000..f048f815 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/controllers/MessageController.java @@ -0,0 +1,53 @@ +package com.vnp.demo.controllers; + +import java.util.UUID; + +import org.springframework.http.HttpStatus; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import com.vnp.demo.dto.MessageCreateRequest; +import com.vnp.demo.dto.MessagePageResponse; +import com.vnp.demo.dto.MessageResponse; +import com.vnp.demo.service.MessageService; + +import jakarta.validation.Valid; + +@RestController +@RequestMapping("/api/v1/chats/{chat_id}/messages") +public class MessageController { + + private final MessageService messageService; + + public MessageController(MessageService messageService) { + this.messageService = messageService; + } + + @PostMapping + @ResponseStatus(HttpStatus.CREATED) + public MessageResponse create( + @AuthenticationPrincipal UUID userId, + @PathVariable("chat_id") UUID chatId, + @org.springframework.web.bind.annotation.RequestHeader("Idempotency-Key") String key, + @Valid @RequestBody MessageCreateRequest request + ) { + return messageService.create(userId, chatId, request, key); + } + + @GetMapping + public MessagePageResponse list( + @AuthenticationPrincipal UUID userId, + @PathVariable("chat_id") UUID chatId, + @RequestParam(defaultValue = "50") int limit, + @RequestParam(required = false) String cursor + ) { + return messageService.list(userId, chatId, limit, cursor); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/controllers/UserController.java b/java/src/main/java/com/vnp/demo/controllers/UserController.java new file mode 100644 index 00000000..36400736 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/controllers/UserController.java @@ -0,0 +1,44 @@ +package com.vnp.demo.controllers; + +import java.util.UUID; + +import org.springframework.http.HttpStatus; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import com.vnp.demo.dto.UserCreateRequest; +import com.vnp.demo.dto.UserResponse; +import com.vnp.demo.service.UserService; + +import jakarta.validation.Valid; + +@RestController +@RequestMapping("/api/v1/users") +public class UserController { + + private final UserService userService; + + public UserController(UserService userService) { + this.userService = userService; + } + + @PostMapping + @ResponseStatus(HttpStatus.CREATED) + public UserResponse register( + @Valid @RequestBody UserCreateRequest request + ) { + return userService.register(request); + } + + @GetMapping("/me") + public UserResponse getCurrent( + @AuthenticationPrincipal UUID userId + ) { + return userService.getCurrent(userId); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/AttachmentJobMessage.java b/java/src/main/java/com/vnp/demo/dto/AttachmentJobMessage.java new file mode 100644 index 00000000..c6bc2f87 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/AttachmentJobMessage.java @@ -0,0 +1,81 @@ +package com.vnp.demo.dto; + +import java.time.Instant; +import java.util.Objects; +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record AttachmentJobMessage( + @JsonProperty("schema_version") + int schemaVersion, + + @JsonProperty("job_id") + UUID jobId, + + String type, + + @JsonProperty("attachment_id") + UUID attachmentId, + + @JsonProperty("pipeline_version") + int pipelineVersion, + + int attempt, + + @JsonProperty("created_at") + Instant createdAt +) { + public static final int SCHEMA_VERSION = 1; + + public static final String TYPE = + "attachment.process.v1"; + + public AttachmentJobMessage { + if (schemaVersion != SCHEMA_VERSION) { + throw new IllegalArgumentException( + "Unsupported job schema version" + ); + } + + if (!TYPE.equals(type)) { + throw new IllegalArgumentException( + "Unsupported job type" + ); + } + + Objects.requireNonNull(jobId, "jobId"); + Objects.requireNonNull(attachmentId, "attachmentId"); + Objects.requireNonNull(createdAt, "createdAt"); + + if (pipelineVersion < 1) { + throw new IllegalArgumentException( + "pipelineVersion must be positive" + ); + } + + if (attempt < 1) { + throw new IllegalArgumentException( + "attempt must be positive" + ); + } + } + + public static AttachmentJobMessage create( + UUID jobId, + UUID attachmentId, + int pipelineVersion, + int attempt, + Instant createdAt + ) { + return new AttachmentJobMessage( + SCHEMA_VERSION, + jobId, + TYPE, + attachmentId, + pipelineVersion, + attempt, + createdAt + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/AttachmentResponse.java b/java/src/main/java/com/vnp/demo/dto/AttachmentResponse.java new file mode 100644 index 00000000..78ad37ba --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/AttachmentResponse.java @@ -0,0 +1,56 @@ +package com.vnp.demo.dto; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.vnp.demo.models.Attachment; + +public record AttachmentResponse( + UUID id, + + @JsonProperty("chat_id") + UUID chatId, + + @JsonProperty("owner_id") + UUID ownerId, + + String filename, + + @JsonProperty("content_type") + String contentType, + + @JsonProperty("size_bytes") + long sizeBytes, + + String sha256, + String status, + List variants, + + @JsonProperty("error_code") + String errorCode, + + @JsonProperty("created_at") + Instant createdAt, + + @JsonProperty("updated_at") + Instant updatedAt +) { + public static AttachmentResponse from(Attachment attachment) { + return new AttachmentResponse( + attachment.id(), + attachment.chatId(), + attachment.ownerId(), + attachment.filename(), + attachment.contentType(), + attachment.sizeBytes(), + attachment.sha256(), + attachment.status(), + List.of(), + null, + attachment.createdAt(), + attachment.updatedAt() + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/ChatCreateRequest.java b/java/src/main/java/com/vnp/demo/dto/ChatCreateRequest.java new file mode 100644 index 00000000..12ecd68b --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/ChatCreateRequest.java @@ -0,0 +1,23 @@ +package com.vnp.demo.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; +import org.hibernate.validator.constraints.UniqueElements; + +import java.util.List; +import java.util.UUID; + +public record ChatCreateRequest( + @NotBlank + @Size(max = 100) + String title, + + @NotEmpty + @Size(max = 100) + @UniqueElements + @JsonProperty("member_ids") + List<@NotNull UUID> memberIds +) {} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/ChatPageResponse.java b/java/src/main/java/com/vnp/demo/dto/ChatPageResponse.java new file mode 100644 index 00000000..6c5fbd8e --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/ChatPageResponse.java @@ -0,0 +1,10 @@ +package com.vnp.demo.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record ChatPageResponse( + List items, + @JsonProperty("next_cursor") String nextCursor +) {} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/ChatResponse.java b/java/src/main/java/com/vnp/demo/dto/ChatResponse.java new file mode 100644 index 00000000..f29a6ed9 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/ChatResponse.java @@ -0,0 +1,16 @@ +package com.vnp.demo.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +public record ChatResponse( + UUID id, + String title, + @JsonProperty("member_ids") + List memberIds, + @JsonProperty("created_at") + Instant createdAt +) {} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/ErrorResponse.java b/java/src/main/java/com/vnp/demo/dto/ErrorResponse.java new file mode 100644 index 00000000..e2aef6a9 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/ErrorResponse.java @@ -0,0 +1,14 @@ +package com.vnp.demo.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record ErrorResponse(ErrorBody error) { + public record ErrorBody( + String code, + + String message, + + @JsonProperty("request_id") + String requestId + ) {} + } \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/HealthResponse.java b/java/src/main/java/com/vnp/demo/dto/HealthResponse.java new file mode 100644 index 00000000..7d6b2f83 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/HealthResponse.java @@ -0,0 +1,6 @@ +package com.vnp.demo.dto; + +public record HealthResponse(String status) { + public static HealthResponse ok() + { return new HealthResponse("ok"); } +} diff --git a/java/src/main/java/com/vnp/demo/dto/MessageCreateRequest.java b/java/src/main/java/com/vnp/demo/dto/MessageCreateRequest.java new file mode 100644 index 00000000..77232447 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/MessageCreateRequest.java @@ -0,0 +1,53 @@ +package com.vnp.demo.dto; + +import java.util.List; +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.annotation.Nulls; + +public class MessageCreateRequest { + + private String text; + private List attachmentIds = List.of(); + + public MessageCreateRequest() { + } + + public MessageCreateRequest(String text) { + this.text = text; + } + + public MessageCreateRequest( + String text, + List attachmentIds + ) { + this.text = text; + this.attachmentIds = attachmentIds; + } + + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + + public String text() { + return text; + } + + public List attachmentIds() { + return attachmentIds; + } + + @JsonSetter( + value = "attachment_ids", + nulls = Nulls.FAIL, + contentNulls = Nulls.FAIL + ) + public void setAttachmentIds(List attachmentIds) { + this.attachmentIds = attachmentIds; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/MessagePageResponse.java b/java/src/main/java/com/vnp/demo/dto/MessagePageResponse.java new file mode 100644 index 00000000..2f5cdf15 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/MessagePageResponse.java @@ -0,0 +1,10 @@ +package com.vnp.demo.dto; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record MessagePageResponse( + List items, + @JsonProperty("next_cursor") String nextCursor +) {} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/MessageResponse.java b/java/src/main/java/com/vnp/demo/dto/MessageResponse.java new file mode 100644 index 00000000..37ff5a17 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/MessageResponse.java @@ -0,0 +1,26 @@ +package com.vnp.demo.dto; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record MessageResponse( + UUID id, + + @JsonProperty("chat_id") + UUID chatId, + + @JsonProperty("sender_id") + UUID senderId, + + String text, + + @JsonProperty("created_at") + Instant createdAt, + + @JsonProperty("attachment_ids") + List attachmentIds +) { +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/SessionResponse.java b/java/src/main/java/com/vnp/demo/dto/SessionResponse.java new file mode 100644 index 00000000..fb847180 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/SessionResponse.java @@ -0,0 +1,13 @@ +package com.vnp.demo.dto; + +import java.time.Instant; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record SessionResponse( + String token, + @JsonProperty("token_type") String tokenType, + @JsonProperty("expires_at") Instant expiresAt, + UserResponse user +) { +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/UserCreateRequest.java b/java/src/main/java/com/vnp/demo/dto/UserCreateRequest.java new file mode 100644 index 00000000..14cbc8ab --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/UserCreateRequest.java @@ -0,0 +1,18 @@ +package com.vnp.demo.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.*; + +public record UserCreateRequest( + + @NotBlank + @Pattern(regexp = "^[a-z0-9_]{3,32}$") + String username, + + @JsonProperty("display_name") + @NotBlank + String displayName, + + @NotNull + String password +) {} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/dto/UserResponse.java b/java/src/main/java/com/vnp/demo/dto/UserResponse.java new file mode 100644 index 00000000..4e5c0088 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/UserResponse.java @@ -0,0 +1,21 @@ +package com.vnp.demo.dto; + +import java.time.Instant; +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record UserResponse( + + UUID id, + + String username, + + @JsonProperty("display_name") + String displayName, + + @JsonProperty("created_at") + Instant createdAt + +) { +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/filter/BearerAuthenticationFilter.java b/java/src/main/java/com/vnp/demo/filter/BearerAuthenticationFilter.java new file mode 100644 index 00000000..e751149d --- /dev/null +++ b/java/src/main/java/com/vnp/demo/filter/BearerAuthenticationFilter.java @@ -0,0 +1,148 @@ + +package com.vnp.demo.filter; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.springframework.dao.DataAccessException; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.transaction.CannotCreateTransactionException; +import org.springframework.web.filter.OncePerRequestFilter; +import org.springframework.web.server.ResponseStatusException; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.vnp.demo.BusinessException; +import com.vnp.demo.SessionStoreUnavailableException; +import com.vnp.demo.models.User; +import com.vnp.demo.service.AuthService; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class BearerAuthenticationFilter extends OncePerRequestFilter { + + private final AuthService authService; + private final ObjectMapper objectMapper; + + public BearerAuthenticationFilter( + AuthService authService, + ObjectMapper objectMapper + ) { + this.authService = authService; + this.objectMapper = objectMapper; + } + + @Override + protected void doFilterInternal( + HttpServletRequest request, + HttpServletResponse response, + FilterChain chain + ) throws ServletException, IOException { + + String authorization = request.getHeader(HttpHeaders.AUTHORIZATION); + + if (authorization == null || !authorization.startsWith("Bearer ")) { + chain.doFilter(request, response); + return; + } + + String token = authorization.substring(7).trim(); + + User user; + + try { + user = authService.authenticate(token); + + } catch (BusinessException ex) { + SecurityContextHolder.clearContext(); + + response.setHeader( + HttpHeaders.WWW_AUTHENTICATE, + "Bearer realm=\"messenger\"" + ); + + writeError( + response, + HttpServletResponse.SC_UNAUTHORIZED, + "unauthorized", + "Недействительный токен" + ); + return; + + } catch (SessionStoreUnavailableException | DataAccessException | CannotCreateTransactionException ex) { + SecurityContextHolder.clearContext(); + + writeError( + response, + HttpServletResponse.SC_SERVICE_UNAVAILABLE, + "unavailable", + "Сервис временно недоступен" + ); + return; + + } catch (ResponseStatusException ex) { + SecurityContextHolder.clearContext(); + + if (ex.getStatusCode().value() == 503) { + writeError( + response, + HttpServletResponse.SC_SERVICE_UNAVAILABLE, + "unavailable", + "Сервис временно недоступен" + ); + return; + } + + throw ex; + } + + try { + UsernamePasswordAuthenticationToken authentication = + new UsernamePasswordAuthenticationToken( + user.getId(), + null, + List.of() + ); + + SecurityContextHolder.getContext() + .setAuthentication(authentication); + + chain.doFilter(request, response); + + } finally { + SecurityContextHolder.clearContext(); + } + } + + private void writeError( + HttpServletResponse response, + int status, + String code, + String message + ) throws IOException { + + response.setStatus(status); + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + response.setCharacterEncoding("UTF-8"); + + String requestId = response.getHeader("X-Request-Id"); + + objectMapper.writeValue( + response.getWriter(), + Map.of( + "error", Map.of( + "code", code, + "message", message, + "request_id", + requestId != null ? requestId : "" + ) + ) + ); + } +} diff --git a/java/src/main/java/com/vnp/demo/filter/RequestIdFilter.java b/java/src/main/java/com/vnp/demo/filter/RequestIdFilter.java new file mode 100644 index 00000000..94b7e9f5 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/filter/RequestIdFilter.java @@ -0,0 +1,77 @@ +package com.vnp.demo.filter; + +import java.io.IOException; +import java.util.UUID; + +import org.slf4j.MDC; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +@Component +@Order(Ordered.HIGHEST_PRECEDENCE) +public class RequestIdFilter extends OncePerRequestFilter { + + private final String instanceId = System.getenv().getOrDefault("HOSTNAME", UUID.randomUUID().toString()); + + public static final String HEADER = "X-Request-Id"; + public static final String MDC_KEY = "requestId"; + + private static final int MAX_REQUEST_ID_LENGTH = 128; + + @Override + protected void doFilterInternal( + HttpServletRequest request, + HttpServletResponse response, + FilterChain chain + ) throws ServletException, IOException { + + String requestId = request.getHeader(HEADER); + + if (requestId == null + || requestId.isBlank() + || requestId.length() > MAX_REQUEST_ID_LENGTH + || !isValidRequestId(requestId)) { + + requestId = UUID.randomUUID() + .toString() + .replace("-", ""); + } + + response.setHeader(HEADER, requestId); + response.setHeader("X-Instance-Id", instanceId); + request.setAttribute(MDC_KEY, requestId); + + MDC.put(MDC_KEY, requestId); + + try { + chain.doFilter(request, response); + } finally { + MDC.remove(MDC_KEY); + } + } + + private boolean isValidRequestId(String requestId) { + for (int i = 0; i < requestId.length(); i++) { + char c = requestId.charAt(i); + + if (!( + (c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || c == '-' + || c == '_' + )) { + return false; + } + } + + return true; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/Attachment.java b/java/src/main/java/com/vnp/demo/models/Attachment.java new file mode 100644 index 00000000..ac922bef --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/Attachment.java @@ -0,0 +1,19 @@ +package com.vnp.demo.models; + +import java.time.Instant; +import java.util.UUID; + +public record Attachment( + UUID id, + UUID chatId, + UUID ownerId, + String filename, + String contentType, + long sizeBytes, + String sha256, + String objectKey, + String status, + Instant createdAt, + Instant updatedAt +) { +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/Chat.java b/java/src/main/java/com/vnp/demo/models/Chat.java new file mode 100644 index 00000000..64b98391 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/Chat.java @@ -0,0 +1,46 @@ +package com.vnp.demo.models; + +import java.time.Instant; +import java.util.UUID; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + +@Entity +@Table(name = "chats") +public class Chat { + + @Id + private UUID id = UUID.randomUUID(); + + @Column(nullable = false, length = 400) + private String title; + + @Column(name = "created_at", nullable = false) + private Instant createdAt = Instant.now(); + + protected Chat() { + } + + public Chat(String title) { + this.title = title; + } + + public UUID getId() { + return id; + } + + public String getTitle() { + return title; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public void setTitle(String title) { + this.title = title; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/ChatMember.java b/java/src/main/java/com/vnp/demo/models/ChatMember.java new file mode 100644 index 00000000..9ab0382e --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/ChatMember.java @@ -0,0 +1,45 @@ +package com.vnp.demo.models; + +import jakarta.persistence.*; + +@Entity +@Table(name = "chat_members") +public class ChatMember { + + @EmbeddedId + private ChatMemberId id; + + @MapsId("chatId") + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "chat_id", nullable = false) + private Chat chat; + + @MapsId("userId") + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "user_id", nullable = false) + private User user; + + protected ChatMember() { + } + + public ChatMember(Chat chat, User user) { + this.chat = chat; + this.user = user; + this.id = new ChatMemberId( + chat.getId(), + user.getId() + ); + } + + public ChatMemberId getId() { + return id; + } + + public Chat getChat() { + return chat; + } + + public User getUser() { + return user; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/ChatMemberId.java b/java/src/main/java/com/vnp/demo/models/ChatMemberId.java new file mode 100644 index 00000000..01af0099 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/ChatMemberId.java @@ -0,0 +1,53 @@ +package com.vnp.demo.models; + +import java.io.Serializable; +import java.util.Objects; +import java.util.UUID; + +import jakarta.persistence.Column; +import jakarta.persistence.Embeddable; + +@Embeddable +public class ChatMemberId implements Serializable { + + @Column(name = "chat_id", nullable = false) + private UUID chatId; + + @Column(name = "user_id", nullable = false) + private UUID userId; + + protected ChatMemberId() { + } + + public ChatMemberId(UUID chatId, UUID userId) { + this.chatId = chatId; + this.userId = userId; + } + + public UUID getChatId() { + return chatId; + } + + public UUID getUserId() { + return userId; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (!(obj instanceof ChatMemberId other)) { + return false; + } + + return Objects.equals(chatId, other.chatId) + && Objects.equals(userId, other.userId); + } + + @Override + public int hashCode() { + return Objects.hash(chatId, userId); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/Message.java b/java/src/main/java/com/vnp/demo/models/Message.java new file mode 100644 index 00000000..1ad3c981 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/Message.java @@ -0,0 +1,91 @@ +package com.vnp.demo.models; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import jakarta.persistence.CollectionTable; +import jakarta.persistence.Column; +import jakarta.persistence.ElementCollection; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.OrderColumn; +import jakarta.persistence.Table; + +@Entity +@Table(name = "messages") +public class Message { + + @Id + private UUID id = UUID.randomUUID(); + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "chat_id", nullable = false) + private Chat chat; + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "sender_id", nullable = false) + private User sender; + + @Column(name = "text", nullable = false, columnDefinition = "text") + private String text; + + @Column(name = "created_at", nullable = false) + private Instant createdAt = Instant.now(); + + @ElementCollection(fetch = FetchType.LAZY) + @CollectionTable( + name = "message_attachments", + joinColumns = @JoinColumn(name = "message_id") + ) + @Column(name = "attachment_id", nullable = false) + @OrderColumn(name = "position") + private List attachmentIds = new ArrayList<>(); + + protected Message() { + } + + public Message(Chat chat, User sender, String text) { + this(chat, sender, text, List.of()); + } + + public Message( + Chat chat, + User sender, + String text, + List attachmentIds + ) { + this.chat = chat; + this.sender = sender; + this.text = text; + this.attachmentIds = new ArrayList<>(attachmentIds); + } + + public UUID getId() { + return id; + } + + public Chat getChat() { + return chat; + } + + public User getSender() { + return sender; + } + + public String getText() { + return text; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public List getAttachmentIds() { + return attachmentIds; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/Session.java b/java/src/main/java/com/vnp/demo/models/Session.java new file mode 100644 index 00000000..cdbf6d9c --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/Session.java @@ -0,0 +1,90 @@ +package com.vnp.demo.models; + +import java.sql.Types; +import java.time.Instant; +import java.util.UUID; + +import org.hibernate.annotations.JdbcTypeCode; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.FetchType; +import jakarta.persistence.Id; +import jakarta.persistence.JoinColumn; +import jakarta.persistence.ManyToOne; +import jakarta.persistence.Table; + +@Entity +@Table(name = "sessions") +public class Session { + + @Id + private UUID id = UUID.randomUUID(); + + @ManyToOne(fetch = FetchType.LAZY, optional = false) + @JoinColumn(name = "user_id", nullable = false) + private User user; + + @JdbcTypeCode(Types.CHAR) + @Column( + name = "token_hash", + nullable = false, + unique = true, + length = 64, + columnDefinition = "CHAR(64)" + ) + private String tokenHash; + + @Column(name = "expires_at", nullable = false) + private Instant expiresAt; + + @Column(name = "revoked_at") + private Instant revokedAt; + + @Column(name = "created_at", nullable = false) + private Instant createdAt = Instant.now(); + + protected Session() { + } + + public Session(User user, String tokenHash, Instant expiresAt) { + this.user = user; + this.tokenHash = tokenHash; + this.expiresAt = expiresAt; + } + + public UUID getId() { + return id; + } + + public User getUser() { + return user; + } + + public String getTokenHash() { + return tokenHash; + } + + public Instant getExpiresAt() { + return expiresAt; + } + + public Instant getRevokedAt() { + return revokedAt; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public boolean isValid() { + return revokedAt == null + && expiresAt.isAfter(Instant.now()); + } + + public void revoke() { + if (revokedAt == null) { + revokedAt = Instant.now(); + } + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/models/User.java b/java/src/main/java/com/vnp/demo/models/User.java new file mode 100644 index 00000000..f9af6301 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/models/User.java @@ -0,0 +1,63 @@ +package com.vnp.demo.models; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.UUID; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; + +@Entity +@Table(name = "users") +public class User { + + @Id + private UUID id = UUID.randomUUID(); + + @Column(nullable = false, unique = true, length = 32) + private String username; + + @Column(name = "display_name", nullable = false) + private String displayName; + + @Column(name = "password_hash", nullable = false) + private String passwordHash; + + @Column(name = "created_at", nullable = false) + private Instant createdAt = Instant.now().truncatedTo(ChronoUnit.MICROS); + + protected User() { + } + + public User(String username, String displayName, String passwordHash) { + this.username = username; + this.displayName = displayName; + this.passwordHash = passwordHash; + } + + public UUID getId() { + return id; + } + + public String getUsername() { + return username; + } + + public String getDisplayName() { + return displayName; + } + + public String getPasswordHash() { + return passwordHash; + } + + public Instant getCreatedAt() { + return createdAt; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/repositories/AttachmentRepository.java b/java/src/main/java/com/vnp/demo/repositories/AttachmentRepository.java new file mode 100644 index 00000000..1e714517 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/repositories/AttachmentRepository.java @@ -0,0 +1,122 @@ +package com.vnp.demo.repositories; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.Optional; +import java.util.UUID; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +import com.vnp.demo.models.Attachment; + +@Repository +public class AttachmentRepository { + + private final JdbcTemplate jdbc; + + public AttachmentRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + public void save(Attachment attachment) { + jdbc.update( + """ + INSERT INTO attachments ( + id, + chat_id, + owner_id, + filename, + content_type, + size_bytes, + sha256, + object_key, + status, + created_at, + updated_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + attachment.id(), + attachment.chatId(), + attachment.ownerId(), + attachment.filename(), + attachment.contentType(), + attachment.sizeBytes(), + attachment.sha256(), + attachment.objectKey(), + attachment.status(), + Timestamp.from(attachment.createdAt()), + Timestamp.from(attachment.updatedAt()) + ); + } + + public Optional findById(UUID id) { + return jdbc.query( + "SELECT * FROM attachments WHERE id = ?", + this::mapRow, + id + ).stream().findFirst(); + } + + private Attachment mapRow(ResultSet rs, int rowNum) + throws SQLException { + + return new Attachment( + rs.getObject("id", UUID.class), + rs.getObject("chat_id", UUID.class), + rs.getObject("owner_id", UUID.class), + rs.getString("filename"), + rs.getString("content_type"), + rs.getLong("size_bytes"), + rs.getString("sha256"), + rs.getString("object_key"), + rs.getString("status"), + rs.getTimestamp("created_at").toInstant(), + rs.getTimestamp("updated_at").toInstant() + ); + } + + public boolean existsByObjectKey(String objectKey) { + Boolean exists = jdbc.queryForObject( + """ + SELECT EXISTS ( + SELECT 1 + FROM attachments + WHERE object_key = ? + ) + """, + Boolean.class, + objectKey + ); + + return Boolean.TRUE.equals(exists); + } + + public void lockObject(String objectKey) { + jdbc.query( + """ + SELECT pg_advisory_xact_lock( + hashtextextended(?, 0) + ) + """, + (rs, rowNum) -> Boolean.TRUE, + "attachment-object:" + objectKey + ); + } + + public boolean tryLockObject(String objectKey) { + Boolean acquired = jdbc.queryForObject( + """ + SELECT pg_try_advisory_xact_lock( + hashtextextended(?, 0) + ) + """, + Boolean.class, + "attachment-object:" + objectKey + ); + + return Boolean.TRUE.equals(acquired); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/repositories/ChatMemberRepository.java b/java/src/main/java/com/vnp/demo/repositories/ChatMemberRepository.java new file mode 100644 index 00000000..7b68cab3 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/repositories/ChatMemberRepository.java @@ -0,0 +1,19 @@ +package com.vnp.demo.repositories; + +import java.util.List; +import java.util.UUID; + +import org.springframework.data.jpa.repository.JpaRepository; + +import com.vnp.demo.models.ChatMember; +import com.vnp.demo.models.ChatMemberId; + +public interface ChatMemberRepository + extends JpaRepository { + + boolean existsByIdChatIdAndIdUserId(UUID chatId, UUID userId); + + List findByIdChatId(UUID chatId); + + boolean existsByChat_IdAndUser_Id(UUID chatId, UUID userId); +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/repositories/ChatRepository.java b/java/src/main/java/com/vnp/demo/repositories/ChatRepository.java new file mode 100644 index 00000000..320e00ea --- /dev/null +++ b/java/src/main/java/com/vnp/demo/repositories/ChatRepository.java @@ -0,0 +1,56 @@ +package com.vnp.demo.repositories; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import com.vnp.demo.models.Chat; + +public interface ChatRepository extends JpaRepository { + + @Query(""" + SELECT c + FROM Chat c + WHERE EXISTS ( + SELECT 1 + FROM ChatMember cm + WHERE cm.chat.id = c.id + AND cm.user.id = :userId + ) + ORDER BY c.createdAt ASC, c.id ASC + """) + List findFirstPage( + @Param("userId") UUID userId, + Pageable pageable + ); + + @Query(""" + SELECT c + FROM Chat c + WHERE EXISTS ( + SELECT 1 + FROM ChatMember cm + WHERE cm.chat.id = c.id + AND cm.user.id = :userId + ) + AND ( + c.createdAt > :cursorTime + OR ( + c.createdAt = :cursorTime + AND c.id > :cursorId + ) + ) + ORDER BY c.createdAt ASC, c.id ASC + """) + List findNextPage( + @Param("userId") UUID userId, + @Param("cursorTime") Instant cursorTime, + @Param("cursorId") UUID cursorId, + Pageable pageable + ); +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/repositories/MessageRepository.java b/java/src/main/java/com/vnp/demo/repositories/MessageRepository.java new file mode 100644 index 00000000..c726cb60 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/repositories/MessageRepository.java @@ -0,0 +1,40 @@ +package com.vnp.demo.repositories; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; + +import com.vnp.demo.models.Message; + +public interface MessageRepository extends JpaRepository { + + @Query(value = """ + SELECT m.* + FROM messages m + WHERE m.chat_id = :chatId + ORDER BY m.created_at ASC, m.id ASC + """, nativeQuery = true) + List findFirstPage( + @Param("chatId") UUID chatId, + Pageable pageable + ); + + @Query(value = """ + SELECT m.* + FROM messages m + WHERE m.chat_id = :chatId + AND (m.created_at, m.id) > (:cursorTime, :cursorId) + ORDER BY m.created_at ASC, m.id ASC + """, nativeQuery = true) + List findNextPage( + @Param("chatId") UUID chatId, + @Param("cursorTime") Instant cursorTime, + @Param("cursorId") UUID cursorId, + Pageable pageable + ); +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/repositories/SessionRepository.java b/java/src/main/java/com/vnp/demo/repositories/SessionRepository.java new file mode 100644 index 00000000..db182093 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/repositories/SessionRepository.java @@ -0,0 +1,12 @@ +package com.vnp.demo.repositories; + +import com.vnp.demo.models.Session; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; +import java.util.UUID; + +public interface SessionRepository extends JpaRepository { + + Optional findByTokenHash(String tokenHash); +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/repositories/UserRepository.java b/java/src/main/java/com/vnp/demo/repositories/UserRepository.java new file mode 100644 index 00000000..de8c80ab --- /dev/null +++ b/java/src/main/java/com/vnp/demo/repositories/UserRepository.java @@ -0,0 +1,14 @@ +package com.vnp.demo.repositories; + +import com.vnp.demo.models.User; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.Optional; +import java.util.UUID; + +public interface UserRepository extends JpaRepository { + + Optional findByUsername(String username); + + boolean existsByUsername(String username); +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/AttachmentCleanupJob.java b/java/src/main/java/com/vnp/demo/service/AttachmentCleanupJob.java new file mode 100644 index 00000000..affc5265 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/AttachmentCleanupJob.java @@ -0,0 +1,41 @@ +package com.vnp.demo.service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +@ConditionalOnProperty( + name = "app.cleanup.enabled", + havingValue = "true", + matchIfMissing = true +) +public class AttachmentCleanupJob { + + private static final Logger log = + LoggerFactory.getLogger(AttachmentCleanupJob.class); + + private final AttachmentPipeline pipeline; + + public AttachmentCleanupJob(AttachmentPipeline pipeline) { + this.pipeline = pipeline; + } + + @Scheduled( + initialDelayString = "${app.cleanup.initial-delay-ms:60000}", + fixedDelayString = "${app.cleanup.interval-ms:300000}" + ) + public void cleanup() { + try { + pipeline.cleanup(); + + } catch (Exception error) { + log.warn( + "Attachment cleanup interrupted: {}", + error.getClass().getSimpleName() + ); + } + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/AttachmentPersistenceService.java b/java/src/main/java/com/vnp/demo/service/AttachmentPersistenceService.java new file mode 100644 index 00000000..9d406d87 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/AttachmentPersistenceService.java @@ -0,0 +1,94 @@ +package com.vnp.demo.service; + +import java.nio.file.Path; +import java.time.Instant; + +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Isolation; +import org.springframework.transaction.annotation.Transactional; + +import com.vnp.demo.models.Attachment; +import com.vnp.demo.repositories.AttachmentRepository; + +@Service +public class AttachmentPersistenceService { + + private final AttachmentRepository attachmentRepository; + private final S3StorageService storage; + + public AttachmentPersistenceService( + AttachmentRepository attachmentRepository, + S3StorageService storage + ) { + this.attachmentRepository = attachmentRepository; + this.storage = storage; + } + + @Transactional( + isolation = Isolation.READ_COMMITTED, + timeout = 60 + ) + public void save( + Attachment attachment, + Path temporaryFile + ) { + attachmentRepository.lockObject( + attachment.objectKey() + ); + + storage.put( + attachment.objectKey(), + temporaryFile, + attachment.sizeBytes(), + attachment.contentType() + ); + + attachmentRepository.save(attachment); + } + + @Transactional( + isolation = Isolation.READ_COMMITTED, + timeout = 30 + ) + public CleanupResult removeIfOrphan( + String objectKey, + Instant cutoff + ) { + if (!objectKey.startsWith("originals/")) { + return CleanupResult.SKIPPED; + } + + if (!attachmentRepository.tryLockObject(objectKey)) { + return CleanupResult.BUSY; + } + + if (attachmentRepository.existsByObjectKey(objectKey)) { + return CleanupResult.REFERENCED; + } + + Instant lastModified = storage.lastModifiedOrNull( + objectKey + ); + + if (lastModified == null) { + return CleanupResult.MISSING; + } + + if (!lastModified.isBefore(cutoff)) { + return CleanupResult.RECENT; + } + + storage.remove(objectKey); + + return CleanupResult.DELETED; + } + + public enum CleanupResult { + DELETED, + REFERENCED, + RECENT, + BUSY, + MISSING, + SKIPPED + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/AttachmentPipeline.java b/java/src/main/java/com/vnp/demo/service/AttachmentPipeline.java new file mode 100644 index 00000000..5fe81435 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/AttachmentPipeline.java @@ -0,0 +1,1384 @@ +package com.vnp.demo.service; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +import com.rabbitmq.client.Channel; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.config.RabbitConfig; +import com.vnp.demo.dto.AttachmentJobMessage; +import com.vnp.demo.dto.AttachmentResponse; +import com.vnp.demo.models.Attachment; +import com.vnp.demo.repositories.AttachmentRepository; +import com.vnp.demo.validation.ImageValidator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.amqp.core.Message; +import org.springframework.amqp.core.MessageDeliveryMode; +import org.springframework.amqp.core.MessageProperties; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.amqp.rabbit.connection.CorrelationData; +import org.springframework.amqp.rabbit.core.RabbitTemplate; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.io.IOException; +import java.nio.file.Path; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Service +public class AttachmentPipeline { + + private static final Logger log = + LoggerFactory.getLogger(AttachmentPipeline.class); + + private static final int PIPELINE_VERSION = 1; + + private static final Set REQUIRED_VARIANTS = + Set.of("thumbnail", "crop", "optimized"); + + private final JdbcTemplate jdbc; + private final AttachmentRepository attachments; + private final S3StorageService storage; + private final RabbitTemplate rabbit; + private final ObjectMapper mapper; + + private final ImageProcessor processor; + private final FaultGate faults; // LAB5_LEVEL4 + + private final TransactionTemplate tx; + + private final boolean worker; + private final int maxAttempts; + private final int leaseSeconds; + private final long graceSeconds; + private final long failedRetention; + + public AttachmentPipeline( + JdbcTemplate jdbc, + AttachmentRepository attachments, + S3StorageService storage, + RabbitTemplate rabbit, + ObjectMapper mapper, + ImageProcessor processor, + FaultGate faults, + PlatformTransactionManager manager, + + @Value("${app.worker.enabled:false}") + boolean worker, + + @Value("${app.jobs.max-attempts:3}") + int maxAttempts, + + @Value("${app.jobs.lease-seconds:240}") + int leaseSeconds, + + @Value("${app.cleanup.grace-seconds:3600}") + long graceSeconds, + + @Value("${app.jobs.failed-retention-seconds:604800}") + long failedRetention + ) { + if (maxAttempts < 1 + || leaseSeconds < 30 + || graceSeconds < 2L * leaseSeconds + || failedRetention < graceSeconds) { + throw new IllegalArgumentException( + "Invalid job/cleanup limits" + ); + } + + this.jdbc = jdbc; + this.attachments = attachments; + this.storage = storage; + this.rabbit = rabbit; + this.mapper = mapper; + + this.processor = processor; + this.faults = faults; + + this.worker = worker; + this.maxAttempts = maxAttempts; + this.leaseSeconds = leaseSeconds; + this.graceSeconds = graceSeconds; + this.failedRetention = failedRetention; + + this.tx = new TransactionTemplate(manager); + this.tx.setTimeout(60); + } + + public record Variant( + String name, + + @JsonProperty("content_type") + String contentType, + + @JsonProperty("size_bytes") + long sizeBytes, + + int width, + int height, + String sha256, + + @JsonIgnore + String objectKey + ) { + } + + private record Job( + UUID id, + UUID attachmentId, + int version, + String status, + int attempts, + int limit, + UUID owner, + boolean validLease, + boolean due + ) { + } + + private record Work( + Job job, + Attachment attachment, + String stagingKey, + UUID owner + ) { + } + + private record Publication( + UUID id, + AttachmentJobMessage body + ) { + } + + public void enqueue(Attachment attachment, Path file) { + if (worker) { + throw new BusinessException( + "unavailable", + "Worker does not accept uploads", + HttpStatus.SERVICE_UNAVAILABLE + ); + } + + String staging = "staging/" + + attachment.chatId() + + "/" + + attachment.id(); + + tx.executeWithoutResult(ignored -> { + /* + * Очистка использует такую же блокировку. + * Она не удалит staging между S3 PUT и COMMIT. + */ + attachments.lockObject(staging); + + storage.put( + staging, + file, + attachment.sizeBytes(), + attachment.contentType() + ); + + attachments.save(attachment); + + jdbc.update( + """ + UPDATE attachments + SET staging_key = ?, + pipeline_version = ? + WHERE id = ? + """, + staging, + PIPELINE_VERSION, + attachment.id() + ); + + UUID jobId = UUID.randomUUID(); + + jdbc.update( + """ + INSERT INTO attachment_jobs( + id, + attachment_id, + pipeline_version, + max_attempts + ) + VALUES (?, ?, ?, ?) + """, + jobId, + attachment.id(), + PIPELINE_VERSION, + maxAttempts + ); + + /* + * Метаданные, задание и outbox фиксируются + * в одной транзакции PostgreSQL. + */ + addOutbox(jobId, 1); + }); + } + + public List variants(UUID attachmentId) { + return jdbc.query( + """ + SELECT v.* + FROM attachment_variants v + JOIN attachments a + ON a.id = v.attachment_id + AND a.pipeline_version = v.pipeline_version + WHERE a.id = ? + ORDER BY v.name + """, + (rs, rowNumber) -> new Variant( + rs.getString("name"), + rs.getString("content_type"), + rs.getLong("size_bytes"), + rs.getInt("width"), + rs.getInt("height"), + rs.getString("sha256"), + rs.getString("object_key") + ), + attachmentId + ); + } + + public AttachmentResponse metadata(Attachment attachment) { + return tx.execute(ignored -> { + /* + * Читаем status, error_code и варианты согласованно. + * Worker или replay не изменят строку посреди чтения. + */ + List locked = jdbc.query( + """ + SELECT id + FROM attachments + WHERE id = ? + FOR SHARE + """, + (rs, rowNumber) -> + rs.getObject("id", UUID.class), + attachment.id() + ); + + if (locked.isEmpty()) { + throw BusinessException.notFound(); + } + + Attachment current = attachments + .findById(attachment.id()) + .orElseThrow(BusinessException::notFound); + + String error = jdbc.queryForObject( + """ + SELECT error_code + FROM attachments + WHERE id = ? + """, + String.class, + current.id() + ); + + List responseVariants = + "ready".equals(current.status()) + ? new ArrayList( + variants(current.id()) + ) + : List.of(); + + return new AttachmentResponse( + current.id(), + current.chatId(), + current.ownerId(), + current.filename(), + current.contentType(), + current.sizeBytes(), + current.sha256(), + current.status(), + responseVariants, + "failed".equals(current.status()) + ? error + : null, + current.createdAt(), + current.updatedAt() + ); + }); + } + + private void addOutbox(UUID jobId, int attempt) { + jdbc.update( + """ + INSERT INTO attachment_outbox( + id, + job_id, + attempt + ) + VALUES (?, ?, ?) + ON CONFLICT (job_id, attempt) DO NOTHING + """, + UUID.randomUUID(), + jobId, + attempt + ); + } + + @Scheduled( + fixedDelay = 1000, + initialDelay = 1000 + ) + public void publish() { + if (worker) { + return; + } + + try { + for (int i = 0; i < 10; i++) { + Boolean published = tx.execute( + ignored -> publishOne() + ); + + if (!Boolean.TRUE.equals(published)) { + break; + } + } + } catch (Exception error) { + log.warn( + "Outbox unavailable: {}", + error.getClass().getSimpleName() + ); + } + } + + private boolean publishOne() { + List rows = jdbc.query( + """ + SELECT + o.id, + o.attempt, + j.id AS job_id, + j.attachment_id, + j.pipeline_version, + j.created_at + FROM attachment_outbox o + JOIN attachment_jobs j ON j.id = o.job_id + WHERE o.published_at IS NULL + AND o.next_publish_at <= clock_timestamp() + AND j.status = 'queued' + AND j.available_at <= clock_timestamp() + AND o.attempt = j.attempts + 1 + ORDER BY o.next_publish_at, o.id + LIMIT 1 + FOR UPDATE OF o SKIP LOCKED + """, + (rs, rowNumber) -> new Publication( + rs.getObject("id", UUID.class), + AttachmentJobMessage.create( + rs.getObject("job_id", UUID.class), + rs.getObject( + "attachment_id", + UUID.class + ), + rs.getInt("pipeline_version"), + rs.getInt("attempt"), + rs.getTimestamp("created_at") + .toInstant() + ) + ) + ); + + if (rows.isEmpty()) { + return false; + } + + Publication publication = rows.getFirst(); + + /* + * В обычном режиме paused() всегда возвращает false. + * Тест может удержать публикацию до получения HTTP 202. + */ + if (faults.paused()) { + return false; + } + + /* + * Задание уже сохранено предыдущей транзакцией. + * Здесь тест завершает API перед отправкой в RabbitMQ. + */ + faults.hit( + publication.body().jobId(), + "before_publish" + ); + + boolean confirmed = false; + + try { + MessageProperties properties = + new MessageProperties(); + + properties.setContentType("application/json"); + + properties.setDeliveryMode( + MessageDeliveryMode.PERSISTENT + ); + + properties.setMessageId( + publication.id().toString() + ); + + CorrelationData correlation = new CorrelationData( + UUID.randomUUID().toString() + ); + + rabbit.send( + "", + RabbitConfig.QUEUE, + new Message( + mapper.writeValueAsBytes( + publication.body() + ), + properties + ), + correlation + ); + + confirmed = correlation.getFuture() + .get(5, TimeUnit.SECONDS) + .isAck() + && correlation.getReturned() == null; + + } catch (Exception error) { + if (error instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + + log.warn( + "Job publish not confirmed: " + + "job={}, errorType={}", + publication.body().jobId(), + error.getClass().getSimpleName() + ); + } + + if (confirmed) { + jdbc.update( + """ + UPDATE attachment_outbox + SET published_at = clock_timestamp(), + publish_attempts = publish_attempts + 1, + last_error_code = NULL + WHERE id = ? + """, + publication.id() + ); + } else { + jdbc.update( + """ + UPDATE attachment_outbox + SET publish_attempts = publish_attempts + 1, + next_publish_at = + clock_timestamp() + interval '5 seconds', + last_error_code = 'broker_unavailable' + WHERE id = ? + """, + publication.id() + ); + } + + return confirmed; + } + + private Job mapJob(ResultSet rs, int rowNumber) + throws SQLException { + + return new Job( + rs.getObject("id", UUID.class), + rs.getObject("attachment_id", UUID.class), + rs.getInt("pipeline_version"), + rs.getString("status"), + rs.getInt("attempts"), + rs.getInt("max_attempts"), + rs.getObject("lease_owner", UUID.class), + rs.getBoolean("valid_lease"), + rs.getBoolean("due") + ); + } + + private Job lockJob(UUID id) { + return jdbc.query( + """ + SELECT *, + lease_until > clock_timestamp() AS valid_lease, + available_at <= clock_timestamp() AS due + FROM attachment_jobs + WHERE id = ? + FOR UPDATE + """, + this::mapJob, + id + ).stream().findFirst().orElse(null); + } + + private Work claim(AttachmentJobMessage message) { + return tx.execute(ignored -> { + Job job = lockJob(message.jobId()); + + if (job == null + || !job.attachmentId().equals( + message.attachmentId() + ) + || job.version() != message.pipelineVersion() + || job.version() != PIPELINE_VERSION) { + throw new IllegalArgumentException( + "invalid_job" + ); + } + + Attachment attachment = attachments + .findById(job.attachmentId()) + .orElseThrow(); + + if ("ready".equals(job.status())) { + return new Work( + job, + attachment, + null, + null + ); + } + + /* + * Дубликат processing не создаёт новую попытку. + * После смерти владельца задание восстановит recover(). + * + * Старые доставки предыдущих попыток тоже не исполняются. + */ + if (!"queued".equals(job.status()) + || !job.due() + || message.attempt() != job.attempts() + 1) { + return null; + } + + UUID owner = UUID.randomUUID(); + + jdbc.update( + """ + UPDATE attachment_jobs + SET status = 'processing', + attempts = attempts + 1, + lease_owner = ?, + lease_until = clock_timestamp() + + (? * interval '1 second'), + updated_at = clock_timestamp() + WHERE id = ? + """, + owner, + leaseSeconds, + job.id() + ); + + int changed = jdbc.update( + """ + UPDATE attachments + SET status = 'processing', + error_code = NULL, + updated_at = clock_timestamp() + WHERE id = ? + AND pipeline_version = ? + AND status = 'queued' + """, + job.attachmentId(), + job.version() + ); + + if (changed != 1) { + throw new IllegalStateException( + "attachment_state_mismatch" + ); + } + + String staging = jdbc.queryForObject( + """ + SELECT staging_key + FROM attachments + WHERE id = ? + """, + String.class, + job.attachmentId() + ); + + return new Work( + lockJob(job.id()), + attachment, + staging, + owner + ); + }); + } + + private boolean owns(Job current, Work work) { + return current != null + && "processing".equals(current.status()) + && current.validLease() + && work.owner() != null + && work.owner().equals(current.owner()) + && current.attempts() == work.job().attempts() + && current.version() == work.job().version(); + } + + private boolean finish( + Work work, + String originalKey, + List generatedVariants + ) { + Set names = generatedVariants.stream() + .map(Variant::name) + .collect(Collectors.toSet()); + + if (generatedVariants.size() != 3 + || !names.equals(REQUIRED_VARIANTS)) { + throw new IllegalStateException( + "incomplete_variants" + ); + } + + return Boolean.TRUE.equals( + tx.execute(ignored -> { + Job job = lockJob(work.job().id()); + + if (!owns(job, work)) { + return false; + } + + for (Variant variant : generatedVariants) { + jdbc.update( + """ + INSERT INTO attachment_variants( + attachment_id, + pipeline_version, + name, + object_key, + content_type, + size_bytes, + width, + height, + sha256 + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT ( + attachment_id, + pipeline_version, + name + ) + DO UPDATE SET + object_key = EXCLUDED.object_key, + content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, + width = EXCLUDED.width, + height = EXCLUDED.height, + sha256 = EXCLUDED.sha256 + """, + job.attachmentId(), + job.version(), + variant.name(), + variant.objectKey(), + variant.contentType(), + variant.sizeBytes(), + variant.width(), + variant.height(), + variant.sha256() + ); + } + + int updated = jdbc.update( + """ + UPDATE attachments + SET object_key = ?, + status = 'ready', + error_code = NULL, + updated_at = clock_timestamp() + WHERE id = ? + AND pipeline_version = ? + AND status = 'processing' + """, + originalKey, + job.attachmentId(), + job.version() + ); + + /* + * Повторно проверяем lease непосредственно + * перед завершением транзакции результата. + */ + int completed = jdbc.update( + """ + UPDATE attachment_jobs + SET status = 'ready', + lease_owner = NULL, + lease_until = NULL, + last_error_code = NULL, + updated_at = clock_timestamp() + WHERE id = ? + AND lease_owner = ? + AND lease_until > clock_timestamp() + """, + job.id(), + work.owner() + ); + + if (updated != 1 || completed != 1) { + throw new IllegalStateException( + "lease_or_version_changed" + ); + } + + return true; + }) + ); + } + + private void retryOrFail( + Job job, + String reason, + boolean permanent + ) { + boolean terminal = permanent + || job.attempts() >= job.limit(); + + /* + * Верхняя граница задержки: + * 2, 4, 8, 16, 32, 60, 60... секунд. + * + * Конкретная задержка случайна в диапазоне [1, cap]. + */ + int cap = Math.min( + 60, + 2 << Math.min(5, job.attempts() - 1) + ); + + int delay = terminal + ? 0 + : ThreadLocalRandom.current().nextInt( + 1, + cap + 1 + ); + + String status = terminal ? "failed" : "queued"; + + String code = terminal + ? (permanent ? reason : "retry_exhausted") + : null; + + jdbc.update( + """ + UPDATE attachment_jobs + SET status = ?, + lease_owner = NULL, + lease_until = NULL, + last_error_code = ?, + available_at = clock_timestamp() + + (? * interval '1 second'), + updated_at = clock_timestamp() + WHERE id = ? + """, + status, + reason, + delay, + job.id() + ); + + int updated = jdbc.update( + """ + UPDATE attachments + SET status = ?, + error_code = ?, + updated_at = clock_timestamp() + WHERE id = ? + AND pipeline_version = ? + AND status = 'processing' + """, + status, + code, + job.attachmentId(), + job.version() + ); + + if (updated != 1) { + throw new IllegalStateException( + "attachment_state_mismatch" + ); + } + + jdbc.update( + """ + INSERT INTO attachment_attempt_events( + job_id, + attempt, + state, + error_code, + delay_seconds + ) + VALUES (?, ?, ?, ?, ?) + """, + job.id(), + job.attempts(), + status, + reason, + delay + ); + + if (!terminal) { + addOutbox( + job.id(), + job.attempts() + 1 + ); + } + + log.warn( + "Job transition: job={}, attempt={}, " + + "status={}, reason={}, delay={}", + job.id(), + job.attempts(), + status, + reason, + delay + ); + } + + private void failedAttempt( + Work work, + Exception error + ) { + boolean permanent = + error instanceof ImageProcessor.PermanentFailure; + + String reason; + + if (permanent) { + reason = "invalid_image"; + + } else if ( + error instanceof ImageProcessor.TransientFailure failure + ) { + reason = failure.code(); + + } else { + reason = "processing_unavailable"; + } + + tx.executeWithoutResult(ignored -> { + Job current = lockJob(work.job().id()); + + if (owns(current, work)) { + retryOrFail( + current, + reason, + permanent + ); + } + }); + } + + @Scheduled( + fixedDelay = 5000, + initialDelay = 5000 + ) + public void recover() { + if (worker) { + return; + } + + try { + tx.executeWithoutResult(ignored -> { + List expired = jdbc.query( + """ + SELECT *, + false AS valid_lease, + true AS due + FROM attachment_jobs + WHERE status = 'processing' + AND lease_until <= clock_timestamp() + ORDER BY lease_until + LIMIT 20 + FOR UPDATE SKIP LOCKED + """, + this::mapJob + ); + + for (Job job : expired) { + retryOrFail( + job, + "lease_expired", + false + ); + } + }); + + } catch (Exception error) { + log.warn( + "Job recovery unavailable: {}", + error.getClass().getSimpleName() + ); + } + } + + /* + * Доступен ReplayRunner из того же пакета. + * + * Проверяем не только статус в PostgreSQL, + * но и наличие файлов с ожидаемыми checksum. + */ + void verifyReady(Attachment attachment) { + if (!"ready".equals(attachment.status())) { + throw new IllegalStateException( + "not_ready" + ); + } + + storage.get( + attachment.objectKey(), + attachment.sizeBytes(), + attachment.sha256() + ); + + List items = variants( + attachment.id() + ); + + Set names = items.stream() + .map(Variant::name) + .collect(Collectors.toSet()); + + if (items.size() != 3 + || !names.equals(REQUIRED_VARIANTS)) { + throw new IllegalStateException( + "incomplete_variants" + ); + } + + for (Variant variant : items) { + storage.get( + variant.objectKey(), + variant.sizeBytes(), + variant.sha256() + ); + } + } + + @RabbitListener( + queues = RabbitConfig.QUEUE, + ackMode = "MANUAL", + autoStartup = "${app.worker.enabled:false}" + ) + public void consume( + Message message, + Channel channel + ) throws IOException { + + long tag = message.getMessageProperties() + .getDeliveryTag(); + + AttachmentJobMessage body; + + try { + if (message.getBody().length > 4096) { + throw new IllegalArgumentException( + "job_message_too_large" + ); + } + + body = mapper.readValue( + message.getBody(), + AttachmentJobMessage.class + ); + + if (body == null) { + throw new IllegalArgumentException( + "empty_job_message" + ); + } + + } catch (Exception error) { + log.error( + "Invalid job message sent to DLQ" + ); + + channel.basicReject(tag, false); + return; + } + + Work work; + + try { + work = claim(body); + + } catch (IllegalArgumentException error) { + log.error( + "Unknown/incompatible job sent to DLQ: {}", + body.jobId() + ); + + channel.basicReject(tag, false); + return; + + } catch (Exception error) { + log.warn( + "Cannot claim job: job={}, errorType={}", + body.jobId(), + error.getClass().getSimpleName() + ); + + requeue(channel, tag); + return; + } + + if (work == null) { + /* + * Доставка больше не требует выполнения: + * failed, processing либо старая попытка. + * + * Зависший processing восстанавливается + * через сохранённый lease. + */ + channel.basicAck(tag, false); + return; + } + + if ("ready".equals(work.job().status())) { + try { + verifyReady(work.attachment()); + + } catch (Exception error) { + log.error( + "Ready effect verification failed; " + + "delivery sent to DLQ: {}", + body.jobId() + ); + + channel.basicReject(tag, false); + return; + } + + log.info( + "Duplicate job verified: job={}", + body.jobId() + ); + + channel.basicAck(tag, false); + return; + } + + try { + faults.hit( + body.jobId(), + "permanent_error" + ); + + faults.hit( + body.jobId(), + "transient_error" + ); + + Attachment attachment = work.attachment(); + + byte[] source = storage.get( + work.stagingKey(), + attachment.sizeBytes(), + attachment.sha256() + ); + + boolean testHang = faults.hit( + body.jobId(), + "image_timeout" + ); + + List generated = + processor.render( + source, + testHang + ); + + /* + * Каждая попытка получает собственные S3-ключи. + * + * Запоздавший worker может оставить orphan, + * но не перезапишет результат новой попытки. + * Orphan удалит очистка после grace period. + */ + String suffix = attachment.chatId() + + "/" + + attachment.id() + + "/v" + + work.job().version() + + "/" + + work.owner(); + + String originalKey = "originals/" + suffix; + + storage.putBytes( + originalKey, + source, + attachment.contentType() + ); + + List generatedVariants = + new ArrayList<>(); + + for (ImageProcessor.Generated item : generated) { + ImageProcessor.Info info = item.info(); + + String extension = + "image/webp".equals(info.contentType()) + ? ".webp" + : ".jpg"; + + String key = "variants/" + + suffix + + "/" + + info.name() + + extension; + + storage.putBytes( + key, + item.bytes(), + info.contentType() + ); + + generatedVariants.add( + new Variant( + info.name(), + info.contentType(), + item.bytes().length, + info.width(), + info.height(), + ImageValidator.sha256( + item.bytes() + ), + key + ) + ); + } + + /* + * Все S3 PUT завершились. + * Метаданные результата ещё не закоммичены. + */ + faults.hit( + body.jobId(), + "after_objects" + ); + + boolean completed = finish( + work, + originalKey, + generatedVariants + ); + + if (completed) { + /* + * PostgreSQL уже содержит ready и три варианта, + * но RabbitMQ ещё не получил ACK. + */ + faults.hit( + body.jobId(), + "before_ack" + ); + } else { + log.info( + "Stale attempt ignored: job={}, attempt={}", + body.jobId(), + work.job().attempts() + ); + } + + } catch (Exception error) { + try { + failedAttempt(work, error); + + } catch (Exception databaseError) { + log.warn( + "Cannot record job failure: " + + "job={}, errorType={}", + body.jobId(), + databaseError.getClass() + .getSimpleName() + ); + + requeue(channel, tag); + return; + } + } + + /* + * ACK отправляется после устойчивой фиксации: + * ready, failed или намерения следующей попытки. + * + * При потерянном lease восстановлением занимается recover(). + */ + channel.basicAck(tag, false); + } + + private void requeue( + Channel channel, + long tag + ) throws IOException { + + try { + Thread.sleep(1000); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } + + channel.basicNack( + tag, + false, + true + ); + } + + public void cleanup() throws Exception { + Instant cutoff = Instant.now() + .minusSeconds(graceSeconds); + + long scanned = 0; + long deleted = 0; + + for (String prefix : List.of( + "staging/", + "originals/", + "variants/" + )) { + for (var result : storage.listPrefix(prefix)) { + var item = result.get(); + + if (item.isDir()) { + continue; + } + + scanned++; + + if (!item.lastModified() + .toInstant() + .isBefore(cutoff)) { + continue; + } + + String key = item.objectName(); + + boolean removed = Boolean.TRUE.equals( + tx.execute(ignored -> { + if (!attachments.tryLockObject(key)) { + return false; + } + + /* + * Сохраняем: + * - опубликованные оригиналы; + * - опубликованные варианты; + * - staging активных заданий; + * - staging failed в пределах retention. + */ + Boolean used = jdbc.queryForObject( + """ + SELECT EXISTS( + SELECT 1 + FROM attachments + WHERE object_key = ? + ) + OR EXISTS( + SELECT 1 + FROM attachment_variants + WHERE object_key = ? + ) + OR EXISTS( + SELECT 1 + FROM attachments + WHERE staging_key = ? + AND ( + status IN ( + 'queued', + 'processing' + ) + OR ( + status = 'failed' + AND updated_at > + clock_timestamp() + - ( + ? + * interval '1 second' + ) + ) + ) + ) + """, + Boolean.class, + key, + key, + key, + failedRetention + ); + + if (Boolean.TRUE.equals(used)) { + return false; + } + + Instant modified = + storage.lastModifiedOrNull(key); + + if (modified == null + || !modified.isBefore(cutoff)) { + return false; + } + + storage.remove(key); + + jdbc.update( + """ + UPDATE attachments + SET staging_key = NULL + WHERE staging_key = ? + AND status IN ( + 'ready', + 'failed' + ) + """, + key + ); + + return true; + }) + ); + + if (removed) { + deleted++; + } + } + } + + log.info( + "Attachment cleanup completed: " + + "scanned={}, deleted={}", + scanned, + deleted + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/AttachmentService.java b/java/src/main/java/com/vnp/demo/service/AttachmentService.java new file mode 100644 index 00000000..1db2bc8b --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/AttachmentService.java @@ -0,0 +1,161 @@ +package com.vnp.demo.service; + +import java.io.InputStream; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.UUID; + +import org.springframework.stereotype.Service; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.dto.AttachmentResponse; +import com.vnp.demo.models.Attachment; +import com.vnp.demo.repositories.AttachmentRepository; +import com.vnp.demo.repositories.ChatMemberRepository; +import com.vnp.demo.validation.ImageValidator; + +@Service +public class AttachmentService { + + private final AttachmentRepository attachments; + private final ChatMemberRepository members; + private final ImageValidator validator; + private final UploadLimiter limiter; + private final S3StorageService storage; + private final AttachmentPipeline pipeline; + + public AttachmentService( + AttachmentRepository attachments, + ChatMemberRepository members, + ImageValidator validator, + UploadLimiter limiter, + S3StorageService storage, + AttachmentPipeline pipeline + ) { + this.attachments = attachments; + this.members = members; + this.validator = validator; + this.limiter = limiter; + this.storage = storage; + this.pipeline = pipeline; + } + + public AttachmentResponse upload( + UUID userId, + UUID chatId, + String filename, + String contentType, + long contentLength, + InputStream input + ) { + checkChatAccess(userId, chatId); + validator.validateFilename(filename); + + try (var permit = limiter.acquire(); + var image = validator.validate( + input, + contentLength, + contentType + )) { + + UUID id = UUID.randomUUID(); + + Instant now = Instant.now() + .truncatedTo(ChronoUnit.MICROS); + + Attachment attachment = new Attachment( + id, + chatId, + userId, + filename, + image.contentType(), + image.sizeBytes(), + image.sha256(), + + "originals/" + + chatId + + "/" + + id + + "/pending", + + "queued", + now, + now + ); + + pipeline.enqueue(attachment, image.path()); + + return AttachmentResponse.from(attachment); + } + } + + public AttachmentResponse getMetadata( + UUID userId, + UUID id + ) { + return pipeline.metadata(accessible(userId, id)); + } + + public Download download( + UUID userId, + UUID id, + String variant + ) { + Attachment attachment = accessible(userId, id); + + if (!"ready".equals(attachment.status())) { + throw BusinessException.conflict( + "Вложение ещё не готово" + ); + } + + if ("original".equals(variant)) { + return new Download( + attachment.filename(), + attachment.contentType(), + storage.get( + attachment.objectKey(), + attachment.sizeBytes(), + attachment.sha256() + ) + ); + } + + var selected = pipeline.variants(id).stream() + .filter(item -> item.name().equals(variant)) + .findFirst() + .orElseThrow(BusinessException::notFound); + + return new Download( + selected.name() + ".jpg", + selected.contentType(), + storage.get( + selected.objectKey(), + selected.sizeBytes(), + selected.sha256() + ) + ); + } + + private Attachment accessible(UUID userId, UUID id) { + Attachment attachment = attachments.findById(id) + .orElseThrow(BusinessException::notFound); + + checkChatAccess(userId, attachment.chatId()); + + return attachment; + } + + private void checkChatAccess(UUID userId, UUID chatId) { + if (!members.existsByIdChatIdAndIdUserId(chatId, userId)) { + throw BusinessException.notFound(); + } + } + + public record Download( + String filename, + String contentType, + byte[] bytes + ) { + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/AuthService.java b/java/src/main/java/com/vnp/demo/service/AuthService.java new file mode 100644 index 00000000..37460170 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/AuthService.java @@ -0,0 +1,143 @@ +package com.vnp.demo.service; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.dto.SessionResponse; +import com.vnp.demo.dto.UserResponse; +import com.vnp.demo.models.Session; +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.SessionRepository; +import com.vnp.demo.repositories.UserRepository; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.web.server.ResponseStatusException; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.Instant; +import java.util.Base64; +import java.util.HexFormat; + +@Service +public class AuthService { + private static final SecureRandom RANDOM = new SecureRandom(); + + private final UserRepository userRepository; + private final SessionRepository sessionRepository; + private final SessionCacheService sessionCacheService; + private final PasswordEncoder passwordEncoder; + private final TransactionTemplate transactions; + private final long sessionTtlSeconds; + + public AuthService( + UserRepository userRepository, + SessionRepository sessionRepository, + SessionCacheService sessionCacheService, + PasswordEncoder passwordEncoder, + PlatformTransactionManager transactionManager, + @Value("${app.session.ttl-seconds:3600}") long sessionTtlSeconds + ) { + if (sessionTtlSeconds <= 0) { + throw new IllegalArgumentException("Session TTL must be positive"); + } + this.userRepository = userRepository; + this.sessionRepository = sessionRepository; + this.sessionCacheService = sessionCacheService; + this.passwordEncoder = passwordEncoder; + this.transactions = new TransactionTemplate(transactionManager); + this.sessionTtlSeconds = sessionTtlSeconds; + } + + public SessionResponse login(String username, String password) { + byte[] bytes = new byte[32]; + RANDOM.nextBytes(bytes); + String token = Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + String tokenHash = hashToken(token); + Instant expiresAt = Instant.now().plusSeconds(sessionTtlSeconds); + + // TransactionTemplate returns only after the database commit completes. + UserResponse userResponse = transactions.execute(status -> { + User user = userRepository.findByUsername(username) + .orElseThrow(BusinessException::unauthorized); + if (!passwordEncoder.matches(password, user.getPasswordHash())) { + throw BusinessException.unauthorized(); + } + sessionRepository.save(new Session(user, tokenHash, expiresAt)); + return new UserResponse( + user.getId(), user.getUsername(), + user.getDisplayName(), user.getCreatedAt() + ); + }); + + // Never publish a usable token before the PostgreSQL commit. + // If Redis fails, the DB row remains inert: it cannot authenticate. + try { + sessionCacheService.cache(tokenHash, userResponse.id(), expiresAt); + } catch (RuntimeException activationFailure) { + // The token is never returned. Best-effort cleanup handles an + // uncertain Redis write and records the failed activation in PG. + try { + sessionCacheService.invalidate(tokenHash); + } catch (RuntimeException cleanupFailure) { + activationFailure.addSuppressed(cleanupFailure); + } + try { + transactions.executeWithoutResult(status -> + sessionRepository.findByTokenHash(tokenHash) + .ifPresent(Session::revoke) + ); + } catch (RuntimeException cleanupFailure) { + activationFailure.addSuppressed(cleanupFailure); + } + throw activationFailure; + } + return new SessionResponse(token, "Bearer", expiresAt, userResponse); + } + + public User authenticate(String token) { + if (token == null || token.isBlank()) { + throw BusinessException.unauthorized(); + } + var cached = sessionCacheService.find(hashToken(token)) + .orElseThrow(BusinessException::unauthorized); + return userRepository.findById(cached.userId()) + .orElseThrow(BusinessException::unauthorized); + } + + public void logout(String token) { + if (token == null || token.isBlank()) { + throw BusinessException.unauthorized(); + } + String tokenHash = hashToken(token); + if (sessionCacheService.find(tokenHash).isEmpty()) { + throw BusinessException.unauthorized(); + } + + + if (!sessionCacheService.invalidate(tokenHash)) { + throw BusinessException.unauthorized(); + } + + transactions.executeWithoutResult(status -> { + Session session = sessionRepository.findByTokenHash(tokenHash) + .orElseThrow(() -> new ResponseStatusException( + HttpStatus.SERVICE_UNAVAILABLE, "Session record unavailable")); + session.revoke(); + }); + } + + private String hashToken(String token) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256") + .digest(token.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(hash); + } catch (NoSuchAlgorithmException ex) { + throw new IllegalStateException("SHA-256 unavailable", ex); + } + } +} diff --git a/java/src/main/java/com/vnp/demo/service/ChatService.java b/java/src/main/java/com/vnp/demo/service/ChatService.java new file mode 100644 index 00000000..1b4ce674 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/ChatService.java @@ -0,0 +1,195 @@ +package com.vnp.demo.service; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.CursorCodec; +import com.vnp.demo.dto.ChatCreateRequest; +import com.vnp.demo.dto.ChatPageResponse; +import com.vnp.demo.dto.ChatResponse; +import com.vnp.demo.models.Chat; +import com.vnp.demo.models.ChatMember; +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.ChatMemberRepository; +import com.vnp.demo.repositories.ChatRepository; +import com.vnp.demo.repositories.UserRepository; + +@Service +public class ChatService { + + private final ChatRepository chatRepository; + private final ChatMemberRepository chatMemberRepository; + private final UserRepository userRepository; + private final CursorCodec cursorCodec; + + public ChatService( + ChatRepository chatRepository, + ChatMemberRepository chatMemberRepository, + UserRepository userRepository, + CursorCodec cursorCodec + ) { + this.chatRepository = chatRepository; + this.chatMemberRepository = chatMemberRepository; + this.userRepository = userRepository; + this.cursorCodec = cursorCodec; + } + + @Transactional + public ChatResponse create( + UUID currentUserId, + ChatCreateRequest request + ) { + String title = request.title(); + + if (title == null + || title.isBlank() + || title.codePointCount(0, title.length()) > 100) { + throw BusinessException.invalid( + "Некорректное название чата" + ); + } + + List memberIds = request.memberIds(); + + if (memberIds == null + || memberIds.isEmpty() + || !memberIds.contains(currentUserId)) { + throw BusinessException.invalid( + "Создатель должен быть участником чата" + ); + } + + if (memberIds.stream().distinct().count() != memberIds.size()) { + throw BusinessException.invalid( + "Список участников содержит дубликаты" + ); + } + + List users = userRepository.findAllById(memberIds); + + if (users.size() != memberIds.size()) { + throw BusinessException.invalid("Один или несколько участников не найдены"); + } + + // Создаём чат через конструктор существующей модели. + Chat chat = new Chat(title); + chatRepository.save(chat); + + // Создание чата и участников происходит в одной транзакции. + for (User user : users) { + ChatMember member = new ChatMember(chat, user); + chatMemberRepository.save(member); + } + + return toResponse(chat, memberIds); + } + + @Transactional(readOnly = true) + public ChatResponse get( + UUID currentUserId, + UUID chatId + ) { + Chat chat = chatRepository.findById(chatId) + .orElseThrow(BusinessException::notFound); + + if (!chatMemberRepository.existsByIdChatIdAndIdUserId( + chatId, + currentUserId + )) { + throw BusinessException.notFound(); + } + + return toResponse(chat, getMemberIds(chatId)); + } + + @Transactional(readOnly = true) + public ChatPageResponse list( + UUID currentUserId, + Integer limit, + String cursor + ) { + int pageSize = limit == null ? 20 : limit; + + if (pageSize < 1 || pageSize > 100) { + throw BusinessException.invalid( + "limit должен быть от 1 до 100" + ); + } + + List chats; + + if (cursor == null) { + chats = chatRepository.findFirstPage( + currentUserId, + PageRequest.of(0, pageSize + 1) + ); + } else { + CursorCodec.ChatCursor decoded = + cursorCodec.decodeChat(cursor, currentUserId); + + chats = chatRepository.findNextPage( + currentUserId, + decoded.createdAt(), + decoded.id(), + PageRequest.of(0, pageSize + 1) + ); + } + + boolean hasNext = chats.size() > pageSize; + + List page = hasNext + ? chats.subList(0, pageSize) + : chats; + + List items = new ArrayList<>(); + + for (Chat chat : page) { + items.add( + toResponse( + chat, + getMemberIds(chat.getId()) + ) + ); + } + + String nextCursor = null; + + if (hasNext && !page.isEmpty()) { + Chat last = page.get(page.size() - 1); + + nextCursor = cursorCodec.encodeChat( + currentUserId, + last.getCreatedAt(), + last.getId() + ); + } + + return new ChatPageResponse(items, nextCursor); + } + + private List getMemberIds(UUID chatId) { + return chatMemberRepository.findByIdChatId(chatId) + .stream() + .map(member -> member.getId().getUserId()) + .sorted() + .toList(); + } + + private ChatResponse toResponse( + Chat chat, + List memberIds + ) { + return new ChatResponse( + chat.getId(), + chat.getTitle(), + memberIds, + chat.getCreatedAt() + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/FaultGate.java b/java/src/main/java/com/vnp/demo/service/FaultGate.java new file mode 100644 index 00000000..6e50f779 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/FaultGate.java @@ -0,0 +1,105 @@ +package com.vnp.demo.service; + +import java.util.UUID; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; + +@Component +public class FaultGate { + + private final boolean enabled; + private final JdbcTemplate jdbc; + private final TransactionTemplate tx; + + public FaultGate( + JdbcTemplate jdbc, + PlatformTransactionManager manager, + + @Value("${FAULT_INJECTION_ENABLED:false}") + boolean enabled + ) { + this.jdbc = jdbc; + this.enabled = enabled; + + tx = new TransactionTemplate(manager); + + tx.setPropagationBehavior( + TransactionDefinition.PROPAGATION_REQUIRES_NEW + ); + + tx.setTimeout(5); + } + + public boolean paused() { + if (!enabled) { + return false; + } + + return Boolean.TRUE.equals( + jdbc.queryForObject( + """ + SELECT EXISTS ( + SELECT 1 + FROM attachment_test_faults + WHERE point = 'hold_publish' + AND remaining > 0 + ) + """, + Boolean.class + ) + ); + } + + public boolean hit(UUID job, String point) { + if (!enabled) { + return false; + } + + boolean hit = Boolean.TRUE.equals( + tx.execute(ignored -> + jdbc.update( + """ + UPDATE attachment_test_faults + SET remaining = remaining - 1, + hits = hits + 1, + job_id = ?, + hit_at = clock_timestamp() + WHERE point = ? + AND remaining > 0 + """, + job, + point + ) == 1 + ) + ); + + if (!hit) { + return false; + } + + switch (point) { + case "before_publish", + "after_objects", + "before_ack" -> + Runtime.getRuntime().halt(137); + + case "permanent_error" -> + throw new ImageProcessor.PermanentFailure(); + + case "transient_error" -> + throw new IllegalStateException( + "injected_transient_error" + ); + + default -> { + } + } + + return true; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/ImageProcessor.java b/java/src/main/java/com/vnp/demo/service/ImageProcessor.java new file mode 100644 index 00000000..7b74699a --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/ImageProcessor.java @@ -0,0 +1,274 @@ +package com.vnp.demo.service; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +@Component +public class ImageProcessor { + + public static class PermanentFailure extends RuntimeException { + public PermanentFailure() { + super("invalid_image"); + } + } + + public static class TransientFailure extends RuntimeException { + private final String code; + + public TransientFailure(String code) { + super(code); + this.code = code; + } + + public String code() { + return code; + } + } + + public record Info( + String name, + + @JsonProperty("content_type") + String contentType, + + int width, + int height + ) { + } + + public record Generated( + Info info, + byte[] bytes + ) { + } + + private final ObjectMapper mapper; + + private final Set active = + ConcurrentHashMap.newKeySet(); + + private final Semaphore slots = new Semaphore(1); + private final Path root = Path.of("/tmp/image-jobs"); + + public ImageProcessor(ObjectMapper mapper) { + this.mapper = mapper; + } + + public List render( + byte[] source, + boolean testHang + ) { + if (source.length == 0 || source.length > 10485760) { + throw new PermanentFailure(); + } + + if (!slots.tryAcquire()) { + throw new TransientFailure("processor_busy"); + } + + Path directory = null; + Process process = null; + + try { + Files.createDirectories(root); + + directory = Files.createTempDirectory(root, "job-"); + active.add(directory); + + Path input = directory.resolve("input.bin"); + Files.write(input, source); + + ProcessBuilder builder = new ProcessBuilder( + "/opt/images/bin/python", + "/app/process_image.py", + input.toString(), + directory.toString() + ); + + builder.redirectOutput( + ProcessBuilder.Redirect.DISCARD + ); + + builder.redirectError( + ProcessBuilder.Redirect.DISCARD + ); + + builder.environment().remove("IMAGE_TEST_SLEEP"); + + if (testHang) { + builder.environment().put( + "IMAGE_TEST_SLEEP", + "1" + ); + } + + process = builder.start(); + + if (!process.waitFor(25, TimeUnit.SECONDS)) { + throw new TransientFailure( + "processing_timeout" + ); + } + + if (process.exitValue() == 10) { + throw new PermanentFailure(); + } + + if (process.exitValue() != 0) { + throw new TransientFailure( + "processing_resource_limit" + ); + } + + Path manifest = directory.resolve("manifest.json"); + + if (Files.size(manifest) > 4096) { + throw new TransientFailure("invalid_manifest"); + } + + Info[] infos = mapper.readValue( + Files.readAllBytes(manifest), + Info[].class + ); + + Set names = new HashSet<>(); + List result = new ArrayList<>(); + + for (Info info : infos) { + String mime = switch (info.name()) { + case "thumbnail", "crop" -> "image/jpeg"; + case "optimized" -> "image/webp"; + default -> throw new TransientFailure( + "invalid_variant" + ); + }; + + if (!names.add(info.name()) + || !mime.equals(info.contentType()) + || info.width() < 1 + || info.height() < 1 + || info.width() > 1280 + || info.height() > 1280) { + throw new TransientFailure( + "invalid_manifest" + ); + } + + String extension = mime.equals("image/webp") + ? ".webp" + : ".jpg"; + + Path file = directory.resolve( + info.name() + extension + ); + + long size = Files.size(file); + + if (size < 1 || size > 10485760) { + throw new TransientFailure( + "invalid_variant_size" + ); + } + + result.add( + new Generated( + info, + Files.readAllBytes(file) + ) + ); + } + + if (!names.equals( + Set.of("thumbnail", "crop", "optimized") + )) { + throw new TransientFailure( + "incomplete_variants" + ); + } + + return result; + + } catch (PermanentFailure | TransientFailure error) { + throw error; + + } catch (Exception error) { + if (error instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + + throw new TransientFailure( + "processing_unavailable" + ); + + } finally { + if (process != null && process.isAlive()) { + process.destroyForcibly(); + + try { + process.waitFor(5, TimeUnit.SECONDS); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } + } + + if (directory != null) { + active.remove(directory); + remove(directory); + } + + slots.release(); + } + } + + private void remove(Path directory) { + try (var paths = Files.walk(directory)) { + for (Path path : paths + .sorted(Comparator.reverseOrder()) + .toList()) { + + Files.deleteIfExists(path); + } + } catch (Exception ignored) { + } + } + + @Scheduled( + initialDelay = 60000, + fixedDelay = 300000 + ) + public void cleanTemporaryFiles() { + if (!Files.isDirectory(root)) { + return; + } + + try (var children = Files.list(root)) { + for (Path path : children.toList()) { + if (Files.isDirectory(path) + && !active.contains(path) + && Files.getLastModifiedTime(path) + .toInstant() + .isBefore( + Instant.now().minusSeconds(3600) + )) { + remove(path); + } + } + } catch (Exception ignored) { + } + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/MessageService.java b/java/src/main/java/com/vnp/demo/service/MessageService.java new file mode 100644 index 00000000..6bd51e21 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/MessageService.java @@ -0,0 +1,323 @@ +package com.vnp.demo.service; + +import java.util.HashSet; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.CursorCodec; +import com.vnp.demo.dto.MessageCreateRequest; +import com.vnp.demo.dto.MessagePageResponse; +import com.vnp.demo.dto.MessageResponse; +import com.vnp.demo.models.Attachment; +import com.vnp.demo.models.Chat; +import com.vnp.demo.models.Message; +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.AttachmentRepository; +import com.vnp.demo.repositories.ChatMemberRepository; +import com.vnp.demo.repositories.ChatRepository; +import com.vnp.demo.repositories.MessageRepository; +import com.vnp.demo.repositories.UserRepository; + +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; + +@Service +public class MessageService { + + private final MessageRepository messageRepository; + private final ChatRepository chatRepository; + private final ChatMemberRepository chatMemberRepository; + private final UserRepository userRepository; + private final AttachmentRepository attachmentRepository; + private final CursorCodec cursorCodec; + + @PersistenceContext + private EntityManager entityManager; + + public MessageService( + MessageRepository messageRepository, + ChatRepository chatRepository, + ChatMemberRepository chatMemberRepository, + UserRepository userRepository, + AttachmentRepository attachmentRepository, + CursorCodec cursorCodec + ) { + this.messageRepository = messageRepository; + this.chatRepository = chatRepository; + this.chatMemberRepository = chatMemberRepository; + this.userRepository = userRepository; + this.attachmentRepository = attachmentRepository; + this.cursorCodec = cursorCodec; + } + + @Transactional + public MessageResponse create( + UUID userId, + UUID chatId, + MessageCreateRequest request, + String key + ) { + if (key == null + || !key.matches("[A-Za-z0-9._:-]{1,128}")) { + throw BusinessException.invalid( + "Некорректный Idempotency-Key" + ); + } + + checkAccess(userId, chatId); + + List attachmentIds = validateRequest(request); + + String attachmentSignature = attachmentIds.stream() + .map(UUID::toString) + .collect(Collectors.joining(",")); + + String scope = userId + ":" + chatId + ":" + key; + + entityManager.createNativeQuery( + """ + SELECT 1 + FROM pg_advisory_xact_lock( + hashtextextended(:scope, 0) + ) + """ + ).setParameter("scope", scope).getResultList(); + + @SuppressWarnings("unchecked") + List existing = entityManager.createNativeQuery( + """ + SELECT message_id, request_text, request_attachment_ids + FROM message_idempotency + WHERE user_id = :user + AND chat_id = :chat + AND key = :key + """ + ) + .setParameter("user", userId) + .setParameter("chat", chatId) + .setParameter("key", key) + .getResultList(); + + if (!existing.isEmpty()) { + Object[] row = existing.get(0); + + boolean sameText = request.text().equals(row[1]); + boolean sameAttachments = + attachmentSignature.equals(row[2]); + + if (!sameText || !sameAttachments) { + throw BusinessException.conflict( + "Ключ уже использован с другим сообщением" + ); + } + + Message original = messageRepository + .findById((UUID) row[0]) + .orElseThrow(BusinessException::notFound); + + return toResponse(original); + } + + validateAttachments(chatId, attachmentIds); + + Chat chat = chatRepository.findById(chatId) + .orElseThrow(BusinessException::notFound); + + User sender = userRepository.findById(userId) + .orElseThrow(BusinessException::unauthorized); + + Message message = new Message( + chat, + sender, + request.text(), + attachmentIds + ); + + Message saved = messageRepository.saveAndFlush(message); + + entityManager.refresh(saved); + + entityManager.createNativeQuery( + """ + INSERT INTO message_idempotency ( + user_id, + chat_id, + key, + request_text, + request_attachment_ids, + message_id + ) + VALUES ( + :user, + :chat, + :key, + :text, + :attachments, + :message + ) + """ + ) + .setParameter("user", userId) + .setParameter("chat", chatId) + .setParameter("key", key) + .setParameter("text", request.text()) + .setParameter("attachments", attachmentSignature) + .setParameter("message", saved.getId()) + .executeUpdate(); + + return toResponse(saved); + } + + private List validateRequest( + MessageCreateRequest request + ) { + if (request == null || request.text() == null) { + throw BusinessException.invalid( + "Поле text обязательно" + ); + } + + if (request.text().codePointCount( + 0, + request.text().length() + ) > 4000) { + throw BusinessException.invalid( + "Текст не должен превышать 4000 символов" + ); + } + + List attachmentIds = request.attachmentIds(); + + if (attachmentIds == null + || attachmentIds.size() > 10 + || attachmentIds.stream().anyMatch(id -> id == null)) { + throw BusinessException.invalid( + "attachment_ids должен содержать от 0 до 10 UUID" + ); + } + + if (new HashSet<>(attachmentIds).size() + != attachmentIds.size()) { + throw BusinessException.invalid( + "Повторяющиеся вложения недопустимы" + ); + } + + if (request.text().isBlank() && attachmentIds.isEmpty()) { + throw BusinessException.invalid( + "Сообщение должно содержать текст или вложение" + ); + } + + return List.copyOf(attachmentIds); + } + + private void validateAttachments( + UUID chatId, + List attachmentIds + ) { + for (UUID attachmentId : attachmentIds) { + Attachment attachment = attachmentRepository + .findById(attachmentId) + .orElseThrow(BusinessException::notFound); + + if (!chatId.equals(attachment.chatId())) { + throw BusinessException.notFound(); + } + + if ("failed".equals(attachment.status())) { + throw BusinessException.conflict( + "Обработка вложения завершилась ошибкой" + ); + } + } + } + + @Transactional(readOnly = true) + public MessagePageResponse list( + UUID userId, + UUID chatId, + int limit, + String cursor + ) { + checkAccess(userId, chatId); + + if (limit < 1 || limit > 100) { + throw BusinessException.invalid( + "limit должен быть от 1 до 100" + ); + } + + CursorCodec.MessageCursor position = null; + + if (cursor != null) { + position = cursorCodec.decodeMessage(cursor, chatId); + } + + List messages; + + if (position == null) { + messages = messageRepository.findFirstPage( + chatId, + PageRequest.of(0, limit + 1) + ); + } else { + messages = messageRepository.findNextPage( + chatId, + position.createdAt(), + position.id(), + PageRequest.of(0, limit + 1) + ); + } + + boolean hasNext = messages.size() > limit; + + List page = hasNext + ? messages.subList(0, limit) + : messages; + + List items = page.stream() + .map(this::toResponse) + .toList(); + + String nextCursor = null; + + if (hasNext && !page.isEmpty()) { + Message last = page.get(page.size() - 1); + + nextCursor = cursorCodec.encodeMessage( + chatId, + last.getCreatedAt(), + last.getId() + ); + } + + return new MessagePageResponse(items, nextCursor); + } + + private void checkAccess(UUID userId, UUID chatId) { + boolean member = chatMemberRepository + .existsByIdChatIdAndIdUserId(chatId, userId); + + if (!member) { + throw BusinessException.notFound(); + } + } + + private MessageResponse toResponse(Message message) { + return new MessageResponse( + message.getId(), + message.getChat().getId(), + message.getSender().getId(), + message.getText(), + message.getCreatedAt(), + List.copyOf(message.getAttachmentIds()) + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/ReplayRunner.java b/java/src/main/java/com/vnp/demo/service/ReplayRunner.java new file mode 100644 index 00000000..75f6a9bb --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/ReplayRunner.java @@ -0,0 +1,261 @@ +package com.vnp.demo.service; + +import com.vnp.demo.repositories.AttachmentRepository; + +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.core.env.Environment; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.util.UUID; + +@Component +@ConditionalOnProperty( + name = "replay.enabled", + havingValue = "true" +) +public class ReplayRunner implements ApplicationRunner { + + private final JdbcTemplate jdbc; + private final AttachmentRepository attachments; + private final S3StorageService storage; + private final AttachmentPipeline pipeline; + private final TransactionTemplate tx; + private final Environment env; + private final ConfigurableApplicationContext context; + + public ReplayRunner( + JdbcTemplate jdbc, + AttachmentRepository attachments, + S3StorageService storage, + AttachmentPipeline pipeline, + PlatformTransactionManager manager, + Environment env, + ConfigurableApplicationContext context + ) { + this.jdbc = jdbc; + this.attachments = attachments; + this.storage = storage; + this.pipeline = pipeline; + this.env = env; + this.context = context; + + tx = new TransactionTemplate(manager); + tx.setTimeout(60); + } + + @Override + public void run(ApplicationArguments ignored) { + int code = 0; + + try { + UUID id = UUID.fromString( + env.getRequiredProperty("REPLAY_JOB_ID") + ); + + String actor = env.getRequiredProperty( + "REPLAY_ACTOR" + ); + + String reason = env.getRequiredProperty( + "REPLAY_REASON" + ); + + int budget = env.getProperty( + "app.jobs.max-attempts", + Integer.class, + 3 + ); + + if (!actor.matches("[A-Za-z0-9_.-]{1,64}") + || reason.isBlank() + || reason.length() > 255 + || budget < 1 + || budget > 100) { + throw new IllegalArgumentException( + "invalid_replay_request" + ); + } + + String result = tx.execute(status -> { + var job = jdbc.queryForMap( + """ + SELECT * + FROM attachment_jobs + WHERE id = ? + FOR UPDATE + """, + id + ); + + UUID attachmentId = + (UUID) job.get("attachment_id"); + + var attachment = attachments + .findById(attachmentId) + .orElseThrow(); + + int version = ((Number) job.get( + "pipeline_version" + )).intValue(); + + int attempts = ((Number) job.get( + "attempts" + )).intValue(); + + Integer activeVersion = jdbc.queryForObject( + """ + SELECT pipeline_version + FROM attachments + WHERE id = ? + """, + Integer.class, + attachmentId + ); + + if (version != 1 + || activeVersion == null + || activeVersion != version) { + throw new IllegalStateException( + "obsolete_pipeline" + ); + } + + String action; + + if ("ready".equals(job.get("status"))) { + pipeline.verifyReady(attachment); + action = "already_ready"; + + } else { + if (!"failed".equals(job.get("status")) + || !"failed".equals( + attachment.status() + )) { + throw new IllegalStateException( + "job_is_not_failed" + ); + } + + String staging = jdbc.queryForObject( + """ + SELECT staging_key + FROM attachments + WHERE id = ? + """, + String.class, + attachmentId + ); + + if (staging == null) { + throw new IllegalStateException( + "staging_expired" + ); + } + + /* + * Та же блокировка используется очисткой. + * Staging нельзя удалить во время replay. + */ + attachments.lockObject(staging); + + storage.get( + staging, + attachment.sizeBytes(), + attachment.sha256() + ); + + jdbc.update( + """ + UPDATE attachment_jobs + SET status = 'queued', + max_attempts = ?, + available_at = clock_timestamp(), + lease_owner = NULL, + lease_until = NULL, + last_error_code = NULL, + updated_at = clock_timestamp() + WHERE id = ? + """, + Math.addExact(attempts, budget), + id + ); + + jdbc.update( + """ + UPDATE attachments + SET status = 'queued', + error_code = NULL, + updated_at = clock_timestamp() + WHERE id = ? + """, + attachmentId + ); + + jdbc.update( + """ + INSERT INTO attachment_outbox( + id, + job_id, + attempt + ) + VALUES (?, ?, ?) + """, + UUID.randomUUID(), + id, + attempts + 1 + ); + + action = "enqueued"; + } + + jdbc.update( + """ + INSERT INTO attachment_replay_audit( + job_id, + actor, + reason, + action, + previous_attempts + ) + VALUES (?, ?, ?, ?, ?) + """, + id, + actor, + reason, + action, + attempts + ); + + return action; + }); + + System.out.println( + "REPLAY_RESULT=" + result + ); + + } catch (Exception error) { + code = 1; + + System.err.println( + "REPLAY_FAILED: " + + error.getClass().getSimpleName() + ); + } + + final int exitCode = code; + + System.exit( + SpringApplication.exit( + context, + () -> exitCode + ) + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/S3StorageService.java b/java/src/main/java/com/vnp/demo/service/S3StorageService.java new file mode 100644 index 00000000..611989d1 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/S3StorageService.java @@ -0,0 +1,232 @@ +package com.vnp.demo.service; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.validation.ImageValidator; + +import io.minio.BucketExistsArgs; +import io.minio.GetObjectArgs; +import io.minio.ListObjectsArgs; +import io.minio.MinioClient; +import io.minio.PutObjectArgs; +import io.minio.RemoveObjectArgs; +import io.minio.Result; +import io.minio.StatObjectArgs; +import io.minio.StatObjectResponse; +import io.minio.errors.ErrorResponseException; +import io.minio.messages.Item; +import okhttp3.OkHttpClient; + +@Service +public class S3StorageService { + + private final MinioClient client; + private final String bucket; + + public S3StorageService( + @Value("${app.s3.endpoint}") String endpoint, + @Value("${app.s3.access-key}") String accessKey, + @Value("${app.s3.secret-key}") String secretKey, + @Value("${app.s3.bucket}") String bucket + ) { + this.bucket = bucket; + + OkHttpClient httpClient = new OkHttpClient.Builder() + .connectTimeout(2, TimeUnit.SECONDS) + .writeTimeout(10, TimeUnit.SECONDS) + .readTimeout(10, TimeUnit.SECONDS) + .callTimeout(30, TimeUnit.SECONDS) + .retryOnConnectionFailure(false) + .build(); + + this.client = MinioClient.builder() + .endpoint(endpoint) + .credentials(accessKey, secretKey) + .region("us-east-1") + .httpClient(httpClient) + .build(); + } + + public void put( + String objectKey, + Path file, + long sizeBytes, + String contentType + ) { + if (sizeBytes < 1 + || sizeBytes > ImageValidator.MAX_BYTES) { + throw new BusinessException( + "payload_too_large", + "Некорректный размер загружаемого файла", + HttpStatus.PAYLOAD_TOO_LARGE + ); + } + + try (InputStream input = Files.newInputStream(file)) { + client.putObject( + PutObjectArgs.builder() + .bucket(bucket) + .object(objectKey) + .contentType(contentType) + .stream(input, sizeBytes, ImageValidator.MAX_BYTES) + .build() + ); + } catch (Exception e) { + throw unavailable(); + } + } + + public byte[] get( + String objectKey, + long expectedSize, + String expectedSha256 + ) { + if (expectedSize < 1 + || expectedSize > ImageValidator.MAX_BYTES) { + throw unavailable(); + } + + try (InputStream input = client.getObject( + GetObjectArgs.builder() + .bucket(bucket) + .object(objectKey) + .build() + )) { + byte[] bytes = input.readNBytes( + Math.toIntExact(expectedSize) + 1 + ); + + if (bytes.length != expectedSize + || !ImageValidator.sha256(bytes) + .equals(expectedSha256)) { + throw unavailable(); + } + + return bytes; + } catch (BusinessException e) { + throw e; + } catch (Exception e) { + throw unavailable(); + } + } + + public Iterable> listOriginals() { + return client.listObjects( + ListObjectsArgs.builder() + .bucket(bucket) + .prefix("originals/") + .recursive(true) + .build() + ); + } + + public Instant lastModifiedOrNull(String objectKey) { + try { + StatObjectResponse response = client.statObject( + StatObjectArgs.builder() + .bucket(bucket) + .object(objectKey) + .build() + ); + + return response.lastModified().toInstant(); + } catch (ErrorResponseException e) { + String code = e.errorResponse().code(); + + if ("NoSuchKey".equals(code) + || "NoSuchObject".equals(code)) { + return null; + } + + throw unavailable(); + } catch (Exception e) { + throw unavailable(); + } + } + + public void remove(String objectKey) { + try { + client.removeObject( + RemoveObjectArgs.builder() + .bucket(bucket) + .object(objectKey) + .build() + ); + } catch (Exception e) { + throw unavailable(); + } + } + + public void checkReady() { + try { + boolean exists = client.bucketExists( + BucketExistsArgs.builder() + .bucket(bucket) + .build() + ); + + if (!exists) { + throw unavailable(); + } + } catch (BusinessException e) { + throw e; + } catch (Exception e) { + throw unavailable(); + } + } + + public void putBytes( + String objectKey, + byte[] bytes, + String contentType + ) { + if (bytes.length < 1 + || bytes.length > ImageValidator.MAX_BYTES) { + throw new IllegalArgumentException("Invalid object size"); + } + + try (var input = new java.io.ByteArrayInputStream(bytes)) { + client.putObject( + PutObjectArgs.builder() + .bucket(bucket) + .object(objectKey) + .contentType(contentType) + .stream( + input, + bytes.length, + ImageValidator.MAX_BYTES + ) + .build() + ); + } catch (Exception error) { + throw unavailable(); + } + } + + public Iterable> listPrefix(String prefix) { + return client.listObjects( + ListObjectsArgs.builder() + .bucket(bucket) + .prefix(prefix) + .recursive(true) + .build() + ); + } + + private static BusinessException unavailable() { + return new BusinessException( + "unavailable", + "Хранилище файлов временно недоступно", + HttpStatus.SERVICE_UNAVAILABLE + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/SessionCacheService.java b/java/src/main/java/com/vnp/demo/service/SessionCacheService.java new file mode 100644 index 00000000..8f10beee --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/SessionCacheService.java @@ -0,0 +1,72 @@ +package com.vnp.demo.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.vnp.demo.SessionStoreUnavailableException; + +import org.springframework.dao.DataAccessException; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; + +@Service +public class SessionCacheService { + private static final String PREFIX = "session:"; + private final StringRedisTemplate redis; + private final ObjectMapper mapper; + + public SessionCacheService(StringRedisTemplate redis, ObjectMapper mapper) { + this.redis = redis; + this.mapper = mapper; + } + + public record CachedSession(UUID userId, Instant expiresAt) {} + + public Optional find(String tokenHash) { + try { + String json = redis.opsForValue().get(PREFIX + tokenHash); + if (json == null) return Optional.empty(); + CachedSession session = mapper.readValue(json, CachedSession.class); + if (!session.expiresAt().isAfter(Instant.now())) return Optional.empty(); + return Optional.of(session); + } catch (DataAccessException ex) { + throw unavailable(ex); + } catch (JsonProcessingException | IllegalArgumentException ex) { + throw unavailable(ex); + } + } + + public void cache(String tokenHash, UUID userId, Instant expiresAt) { + Duration ttl = Duration.between(Instant.now(), expiresAt); + if (ttl.isNegative() || ttl.isZero()) { + throw new IllegalStateException("Session already expired"); + } + try { + String json = mapper.writeValueAsString(new CachedSession(userId, expiresAt)); + redis.opsForValue().set(PREFIX + tokenHash, json, ttl); + } catch (DataAccessException ex) { + throw unavailable(ex); + } catch (JsonProcessingException ex) { + throw new IllegalStateException("Cannot serialize session", ex); + } + } + + public boolean invalidate(String tokenHash) { + try { + Boolean deleted = redis.delete(PREFIX + tokenHash); + return Boolean.TRUE.equals(deleted); + } catch (DataAccessException ex) { + throw new SessionStoreUnavailableException(ex); + } + } + + private ResponseStatusException unavailable(Exception ex) { + return new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "Session store unavailable", ex); + } +} diff --git a/java/src/main/java/com/vnp/demo/service/Thumbnailer.java b/java/src/main/java/com/vnp/demo/service/Thumbnailer.java new file mode 100644 index 00000000..bd675400 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/Thumbnailer.java @@ -0,0 +1,182 @@ +package com.vnp.demo.service; + +import com.drew.imaging.ImageMetadataReader; +import com.drew.metadata.exif.ExifIFD0Directory; +import com.vnp.demo.validation.ImageValidator; + +import org.springframework.stereotype.Component; + +import javax.imageio.ImageIO; +import javax.imageio.ImageReader; +import javax.imageio.stream.MemoryCacheImageInputStream; + +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.RenderingHints; +import java.awt.geom.AffineTransform; +import java.awt.image.BufferedImage; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + +@Component +public class Thumbnailer { + + public record Thumbnail( + byte[] bytes, + int width, + int height + ) { + } + + public Thumbnail render(byte[] source) { + BufferedImage original = null; + BufferedImage target = null; + ImageReader reader = null; + + try (var input = new MemoryCacheImageInputStream( + new ByteArrayInputStream(source) + )) { + var readers = ImageIO.getImageReaders(input); + + if (!readers.hasNext()) { + throw new IllegalArgumentException("invalid_image"); + } + + reader = readers.next(); + reader.setInput(input, true, true); + + int width = reader.getWidth(0); + int height = reader.getHeight(0); + + if (width < 1 + || height < 1 + || (long) width * height > ImageValidator.MAX_PIXELS) { + throw new IllegalArgumentException("invalid_image"); + } + + int orientation = 1; + + var metadata = ImageMetadataReader.readMetadata( + new ByteArrayInputStream(source) + ); + + var exif = metadata.getFirstDirectoryOfType( + ExifIFD0Directory.class + ); + + if (exif != null) { + Integer value = exif.getInteger( + ExifIFD0Directory.TAG_ORIENTATION + ); + + if (value != null && value >= 1 && value <= 8) { + orientation = value; + } + } + + original = reader.read(0); + + int orientedWidth = orientation >= 5 ? height : width; + int orientedHeight = orientation >= 5 ? width : height; + + int denominator = Math.max( + 256, + Math.max(orientedWidth, orientedHeight) + ); + + int targetWidth = Math.max( + 1, + (int) ((long) orientedWidth * 256 / denominator) + ); + + int targetHeight = Math.max( + 1, + (int) ((long) orientedHeight * 256 / denominator) + ); + + target = new BufferedImage( + targetWidth, + targetHeight, + BufferedImage.TYPE_INT_RGB + ); + + Graphics2D graphics = target.createGraphics(); + + try { + graphics.setColor(Color.WHITE); + graphics.fillRect(0, 0, targetWidth, targetHeight); + + graphics.setRenderingHint( + RenderingHints.KEY_INTERPOLATION, + RenderingHints.VALUE_INTERPOLATION_BICUBIC + ); + + AffineTransform transform = + AffineTransform.getScaleInstance( + (double) targetWidth / orientedWidth, + (double) targetHeight / orientedHeight + ); + + transform.concatenate( + switch (orientation) { + case 2 -> new AffineTransform( + -1, 0, 0, 1, width, 0 + ); + case 3 -> new AffineTransform( + -1, 0, 0, -1, width, height + ); + case 4 -> new AffineTransform( + 1, 0, 0, -1, 0, height + ); + case 5 -> new AffineTransform( + 0, 1, 1, 0, 0, 0 + ); + case 6 -> new AffineTransform( + 0, 1, -1, 0, height, 0 + ); + case 7 -> new AffineTransform( + 0, -1, -1, 0, height, width + ); + case 8 -> new AffineTransform( + 0, -1, 1, 0, 0, width + ); + default -> new AffineTransform(); + } + ); + + graphics.drawImage(original, transform, null); + } finally { + graphics.dispose(); + } + + var output = new ByteArrayOutputStream(); + + if (!ImageIO.write(target, "jpeg", output)) { + throw new IllegalArgumentException( + "jpeg_encoder_unavailable" + ); + } + + return new Thumbnail( + output.toByteArray(), + targetWidth, + targetHeight + ); + } catch (Exception error) { + throw new IllegalArgumentException("invalid_image", error); + } finally { + if (reader != null) { + reader.dispose(); + } + + if (original != null) { + original.flush(); + } + + if (target != null) { + target.flush(); + } + } + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/UploadLimiter.java b/java/src/main/java/com/vnp/demo/service/UploadLimiter.java new file mode 100644 index 00000000..338fb370 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/UploadLimiter.java @@ -0,0 +1,58 @@ +package com.vnp.demo.service; + +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; + +import com.vnp.demo.BusinessException; + +@Component +public class UploadLimiter { + + private final Semaphore semaphore; + + public UploadLimiter( + @Value("${app.uploads.max-concurrent:1}") + int maxConcurrent + ) { + if (maxConcurrent < 1) { + throw new IllegalArgumentException( + "max-concurrent должен быть больше нуля" + ); + } + + this.semaphore = new Semaphore(maxConcurrent); + } + + public Permit acquire() { + if (!semaphore.tryAcquire()) { + throw new BusinessException( + "rate_limited", + "Все обработчики загрузки заняты. Повторите запрос позже", + HttpStatus.TOO_MANY_REQUESTS + ); + } + + return new Permit(semaphore); + } + + public static final class Permit implements AutoCloseable { + + private final Semaphore semaphore; + private final AtomicBoolean released = new AtomicBoolean(false); + + private Permit(Semaphore semaphore) { + this.semaphore = semaphore; + } + + @Override + public void close() { + if (released.compareAndSet(false, true)) { + semaphore.release(); + } + } + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/UserService.java b/java/src/main/java/com/vnp/demo/service/UserService.java new file mode 100644 index 00000000..9a1163c0 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/UserService.java @@ -0,0 +1,86 @@ + +package com.vnp.demo.service; + +import java.util.UUID; + +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.dto.UserCreateRequest; +import com.vnp.demo.dto.UserResponse; +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.UserRepository; +import com.vnp.demo.validation.UserInputValidator; + +@Service +public class UserService { + + private final UserRepository userRepository; + private final PasswordEncoder passwordEncoder; + private final TransactionTemplate transactionTemplate; + + public UserService( + UserRepository userRepository, + PasswordEncoder passwordEncoder, + TransactionTemplate transactionTemplate + ) { + this.userRepository = userRepository; + this.passwordEncoder = passwordEncoder; + this.transactionTemplate = transactionTemplate; + } + + public UserResponse register(UserCreateRequest request) { + String username = request.username(); + String displayName = request.displayName(); + String password = request.password(); + + UserInputValidator.validate(displayName, password); + + String passwordHash = passwordEncoder.encode(password); + + try { + return transactionTemplate.execute(status -> { + + if (userRepository.existsByUsername(username)) { + throw BusinessException.conflict("Username already exists"); + } + + User user = new User( + username, + displayName, + passwordHash + ); + + User saved = userRepository.saveAndFlush(user); + + return toResponse(saved); + }); + + } catch (DataIntegrityViolationException ex) { + + throw BusinessException.conflict( + "Username already exists" + ); + } + } + + @org.springframework.transaction.annotation.Transactional(readOnly = true) + public UserResponse getCurrent(UUID userId) { + User user = userRepository.findById(userId) + .orElseThrow(BusinessException::notFound); + + return toResponse(user); + } + + public UserResponse toResponse(User user) { + return new UserResponse( + user.getId(), + user.getUsername(), + user.getDisplayName(), + user.getCreatedAt() + ); + } +} diff --git a/java/src/main/java/com/vnp/demo/validation/ImageValidator.java b/java/src/main/java/com/vnp/demo/validation/ImageValidator.java new file mode 100644 index 00000000..e345e212 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/validation/ImageValidator.java @@ -0,0 +1,481 @@ +package com.vnp.demo.validation; + +import java.awt.image.BufferedImage; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.HexFormat; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.imageio.ImageIO; +import javax.imageio.ImageReader; +import javax.imageio.stream.FileImageInputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import com.vnp.demo.BusinessException; + +@Component +public class ImageValidator { + + public static final int MAX_BYTES = 10 * 1024 * 1024; + public static final long MAX_PIXELS = 40_000_000L; + + private static final int BUFFER_SIZE = 64 * 1024; + + private static final long MAX_RECEIVE_NANOS = + TimeUnit.SECONDS.toNanos(60); + + private static final long TEMP_FILE_GRACE_SECONDS = 3600; + + private static final Logger log = + LoggerFactory.getLogger(ImageValidator.class); + + private static final Set ACTIVE_FILES = + ConcurrentHashMap.newKeySet(); + + private final Path tempDirectory; + + public ImageValidator( + @Value("${app.uploads.temp-dir:/tmp/messenger-uploads}") + String tempDirectory + ) { + this.tempDirectory = Path.of(tempDirectory) + .toAbsolutePath() + .normalize(); + + try { + Files.createDirectories(this.tempDirectory); + } catch (IOException e) { + throw new IllegalStateException( + "Не удалось создать каталог временных загрузок", + e + ); + } + } + + public record ValidatedImage( + Path path, + long sizeBytes, + String contentType, + String sha256 + ) implements AutoCloseable { + + @Override + public void close() { + deleteTemporaryFile(path); + } + } + + public void validateFilename(String filename) { + if (filename == null) { + throw BusinessException.invalid( + "Не указано имя файла" + ); + } + + int length = filename.codePointCount( + 0, + filename.length() + ); + + if (length < 1 || length > 255) { + throw BusinessException.invalid( + "Имя файла должно содержать от 1 до 255 символов" + ); + } + + boolean unsafe = filename.codePoints().anyMatch( + codePoint -> codePoint == '/' + || codePoint == '\\' + || Character.isISOControl(codePoint) + ); + + if (unsafe) { + throw BusinessException.invalid( + "Имя файла содержит недопустимые символы" + ); + } + } + + public ValidatedImage validate( + InputStream input, + long contentLength, + String declaredContentType + ) { + if (contentLength > MAX_BYTES) { + throw tooLarge( + "Размер файла превышает 10 MiB" + ); + } + + String declaredMime = normalizeMime( + declaredContentType + ); + + Path temporaryFile = createTemporaryFile(); + + boolean success = false; + + try { + MessageDigest digest = newSha256(); + + long size = copyLimited( + input, + temporaryFile, + digest + ); + + if (size == 0) { + throw unsupported(); + } + + if (contentLength >= 0 && size != contentLength) { + throw BusinessException.invalid( + "Тело запроса получено не полностью" + ); + } + + String actualMime = validateImageContent( + temporaryFile + ); + + if (!actualMime.equals(declaredMime)) { + throw unsupported(); + } + + String checksum = HexFormat.of() + .formatHex(digest.digest()); + + ValidatedImage result = new ValidatedImage( + temporaryFile, + size, + actualMime, + checksum + ); + + success = true; + + return result; + } finally { + if (!success) { + deleteTemporaryFile(temporaryFile); + } + } + } + + private Path createTemporaryFile() { + try { + Path file = Files.createTempFile( + tempDirectory, + "upload-", + ".tmp" + ); + + ACTIVE_FILES.add(file); + + return file; + } catch (IOException e) { + throw temporaryStorageUnavailable(); + } + } + + private long copyLimited( + InputStream input, + Path destination, + MessageDigest digest + ) { + byte[] buffer = new byte[BUFFER_SIZE]; + long total = 0; + long started = System.nanoTime(); + + try (OutputStream output = + Files.newOutputStream(destination)) { + + while (true) { + checkReceiveDeadline(started); + + int requested = (int) Math.min( + buffer.length, + (long) MAX_BYTES - total + 1 + ); + + int count; + + try { + count = input.read(buffer, 0, requested); + } catch (IOException e) { + throw BusinessException.invalid( + "Передача файла прервана или превышено время ожидания" + ); + } + + checkReceiveDeadline(started); + + if (count == -1) { + break; + } + + if (count == 0) { + continue; + } + + total += count; + + if (total > MAX_BYTES) { + throw tooLarge( + "Размер файла превышает 10 MiB" + ); + } + + output.write(buffer, 0, count); + digest.update(buffer, 0, count); + } + + return total; + } catch (IOException e) { + throw temporaryStorageUnavailable(); + } + } + + private static void checkReceiveDeadline(long started) { + if (System.nanoTime() - started >= MAX_RECEIVE_NANOS) { + throw BusinessException.invalid( + "Превышено допустимое время приёма файла" + ); + } + } + + private String normalizeMime(String value) { + if (value == null) { + throw unsupported(); + } + + try { + MediaType type = MediaType.parseMediaType(value); + + if (type.getType().equalsIgnoreCase("image") + && type.getSubtype().equalsIgnoreCase("png")) { + return "image/png"; + } + + if (type.getType().equalsIgnoreCase("image") + && type.getSubtype().equalsIgnoreCase("jpeg")) { + return "image/jpeg"; + } + } catch (IllegalArgumentException e) { + throw unsupported(); + } + + throw unsupported(); + } + + private String validateImageContent(Path file) { + try (FileImageInputStream imageInput = + new FileImageInputStream(file.toFile())) { + + Iterator readers = + ImageIO.getImageReaders(imageInput); + + if (!readers.hasNext()) { + throw unsupported(); + } + + ImageReader reader = readers.next(); + + try { + reader.setInput(imageInput, true, true); + + String format = reader.getFormatName(); + String mime; + + if ("png".equalsIgnoreCase(format)) { + mime = "image/png"; + } else if ("jpeg".equalsIgnoreCase(format) + || "jpg".equalsIgnoreCase(format)) { + mime = "image/jpeg"; + } else { + throw unsupported(); + } + + int width = reader.getWidth(0); + int height = reader.getHeight(0); + + if (width <= 0 || height <= 0) { + throw unsupported(); + } + + long pixels = (long) width * height; + + if (pixels > MAX_PIXELS) { + throw tooLarge( + "Изображение содержит более 40 миллионов пикселей" + ); + } + + AtomicBoolean warning = new AtomicBoolean(false); + + reader.addIIOReadWarningListener( + (source, message) -> warning.set(true) + ); + + BufferedImage image = reader.read(0); + + if (image == null) { + throw unsupported(); + } + + try { + if (warning.get()) { + throw unsupported(); + } + } finally { + image.flush(); + } + + return mime; + } finally { + reader.dispose(); + } + } catch (BusinessException e) { + throw e; + } catch (IOException | RuntimeException e) { + throw unsupported(); + } + } + + @Scheduled( + initialDelay = 60_000, + fixedDelay = 300_000 + ) + public void cleanupTemporaryFiles() { + Instant cutoff = Instant.now() + .minusSeconds(TEMP_FILE_GRACE_SECONDS); + + long deleted = 0; + + try (DirectoryStream files = + Files.newDirectoryStream( + tempDirectory, + "upload-*.tmp" + )) { + + for (Path file : files) { + if (ACTIVE_FILES.contains(file)) { + continue; + } + + if (!Files.isRegularFile( + file, + LinkOption.NOFOLLOW_LINKS + )) { + continue; + } + + try { + Instant modified = Files.getLastModifiedTime( + file, + LinkOption.NOFOLLOW_LINKS + ).toInstant(); + + if (!modified.isBefore(cutoff)) { + continue; + } + + if (Files.deleteIfExists(file)) { + deleted++; + } + } catch (IOException e) { + log.warn( + "Не удалось очистить временный файл {}", + file.getFileName(), + e + ); + } + } + + log.info( + "Temporary upload cleanup completed: deleted={}", + deleted + ); + } catch (IOException e) { + log.warn( + "Не удалось просмотреть каталог временных загрузок", + e + ); + } + } + + private static MessageDigest newSha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException( + "SHA-256 недоступен", + e + ); + } + } + + public static String sha256(byte[] bytes) { + return HexFormat.of().formatHex( + newSha256().digest(bytes) + ); + } + + private static void deleteTemporaryFile(Path file) { + try { + Files.deleteIfExists(file); + } catch (IOException e) { + log.warn( + "Не удалось удалить временный файл {}", + file.getFileName(), + e + ); + } finally { + + ACTIVE_FILES.remove(file); + } + } + + private static BusinessException unsupported() { + return new BusinessException( + "unsupported_media_type", + "Ожидается корректное изображение PNG или JPEG", + HttpStatus.UNSUPPORTED_MEDIA_TYPE + ); + } + + private static BusinessException tooLarge(String message) { + return new BusinessException( + "payload_too_large", + message, + HttpStatus.PAYLOAD_TOO_LARGE + ); + } + + private static BusinessException temporaryStorageUnavailable() { + return new BusinessException( + "unavailable", + "Временное хранилище загрузок недоступно", + HttpStatus.SERVICE_UNAVAILABLE + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/validation/UserInputValidator.java b/java/src/main/java/com/vnp/demo/validation/UserInputValidator.java new file mode 100644 index 00000000..229cb83e --- /dev/null +++ b/java/src/main/java/com/vnp/demo/validation/UserInputValidator.java @@ -0,0 +1,32 @@ +package com.vnp.demo.validation; + +import com.vnp.demo.BusinessException; + +public final class UserInputValidator { + + private UserInputValidator() { + } + + public static void validate(String displayName, String password) { + + if (displayName == null + || displayName.isBlank() + || codePoints(displayName) > 100) { + throw BusinessException.invalid( + "Некорректное отображаемое имя" + ); + } + + if (password == null + || codePoints(password) < 12 + || codePoints(password) > 128) { + throw BusinessException.invalid( + "Пароль должен содержать от 12 до 128 символов" + ); + } + } + + private static int codePoints(String value) { + return value.codePointCount(0, value.length()); + } +} \ No newline at end of file diff --git a/java/src/main/resources/application.yml b/java/src/main/resources/application.yml new file mode 100644 index 00000000..35b1289b --- /dev/null +++ b/java/src/main/resources/application.yml @@ -0,0 +1,111 @@ +server: + shutdown: graceful + tomcat: + threads: + max: 64 + connection-timeout: 2s + +spring: + application: + name: messenger + + lifecycle: + timeout-per-shutdown-phase: 15s + + jackson: + default-property-inclusion: always + deserialization: + fail-on-unknown-properties: true + fail-on-trailing-tokens: true + mapper: + allow-coercion-of-scalars: false + + datasource: + url: ${DB_URL:jdbc:postgresql://localhost:5432/messenger} + username: ${DB_USERNAME:messenger} + password: ${DB_PASSWORD:messenger} + + hikari: + maximum-pool-size: ${DB_POOL_SIZE:8} + minimum-idle: 1 + connection-timeout: 1000 + validation-timeout: 500 + data-source-properties: + connectTimeout: 2 + socketTimeout: 8 + options: "-c statement_timeout=5000 -c lock_timeout=3000" + + data: + redis: + host: ${REDIS_HOST:localhost} + port: 6379 + password: ${REDIS_PASSWORD:} + timeout: 1s + connect-timeout: 1s + + rabbitmq: + host: ${RABBITMQ_HOST:localhost} + port: 5672 + username: ${RABBITMQ_USER:messenger_jobs} + password: ${RABBITMQ_PASSWORD:} + + connection-timeout: 3s + requested-heartbeat: 15s + + publisher-confirm-type: correlated + publisher-returns: true + + template: + mandatory: true + + listener: + simple: + acknowledge-mode: manual + prefetch: 1 + concurrency: 1 + max-concurrency: 1 + default-requeue-rejected: true + + task: + scheduling: + pool: + size: 3 + + flyway: + enabled: ${FLYWAY_ENABLED:true} + + jpa: + hibernate: + ddl-auto: validate + open-in-view: false + + transaction: + default-timeout: 8s + +app: + session: + ttl-seconds: ${SESSION_TTL_SECONDS:3600} + + s3: + endpoint: ${S3_ENDPOINT:http://localhost:9000} + access-key: ${S3_ACCESS_KEY:} + secret-key: ${S3_SECRET_KEY:} + bucket: ${S3_BUCKET:messenger-attachments} + + uploads: + max-concurrent: ${UPLOAD_MAX_CONCURRENT:1} + temp-dir: ${UPLOAD_TEMP_DIR:/tmp/messenger-uploads} + + worker: + enabled: ${WORKER_ENABLED:false} + + jobs: + max-attempts: ${JOB_MAX_ATTEMPTS:3} + lease-seconds: ${JOB_TIMEOUT_SECONDS:120} + failed-retention-seconds: ${STAGING_RETENTION_SECONDS:604800} + + cleanup: + enabled: ${ATTACHMENT_CLEANUP_ENABLED:true} + grace-seconds: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:3600} + interval-ms: ${ATTACHMENT_CLEANUP_INTERVAL_MS:300000} + initial-delay-ms: ${ATTACHMENT_CLEANUP_INITIAL_DELAY_MS:60000} \ No newline at end of file diff --git a/java/src/main/resources/db/migration/V1__create_tables.sql b/java/src/main/resources/db/migration/V1__create_tables.sql new file mode 100644 index 00000000..4688e46f --- /dev/null +++ b/java/src/main/resources/db/migration/V1__create_tables.sql @@ -0,0 +1,77 @@ + +CREATE TABLE users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE, + display_name VARCHAR(400) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT chk_username + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT chk_display_name + CHECK ( + char_length(display_name) BETWEEN 1 AND 100 + AND length(btrim(display_name)) > 0 + ) +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title VARCHAR(400) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT chk_chat_title + CHECK ( + char_length(title) BETWEEN 1 AND 100 + AND length(btrim(title)) > 0 + ) +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id), + user_id UUID NOT NULL REFERENCES users(id), + + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + sender_id UUID NOT NULL, + text VARCHAR(16000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT fk_message_membership + FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members(chat_id, user_id), + + CONSTRAINT chk_message_text + CHECK ( + char_length(text) BETWEEN 1 AND 4000 + AND length(btrim(text)) > 0 + ) +); + +CREATE TABLE sessions ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id), + token_hash CHAR(64) NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ, + + CONSTRAINT chk_session_expiry + CHECK (expires_at > created_at) +); + +CREATE INDEX idx_chat_members_user + ON chat_members(user_id, chat_id); + +CREATE INDEX idx_chats_history + ON chats(created_at, id); + +CREATE INDEX idx_messages_history + ON messages(chat_id, created_at, id); + +CREATE INDEX idx_sessions_user + ON sessions(user_id); diff --git a/java/src/main/resources/db/migration/V2__message_idempotency.sql b/java/src/main/resources/db/migration/V2__message_idempotency.sql new file mode 100644 index 00000000..63cdd87b --- /dev/null +++ b/java/src/main/resources/db/migration/V2__message_idempotency.sql @@ -0,0 +1,9 @@ +CREATE TABLE message_idempotency ( + user_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + key VARCHAR(128) COLLATE "C" NOT NULL, + request_text TEXT NOT NULL, + message_id UUID NOT NULL UNIQUE REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, key) +); diff --git a/java/src/main/resources/db/migration/V3__attachments.sql b/java/src/main/resources/db/migration/V3__attachments.sql new file mode 100644 index 00000000..56fcbe46 --- /dev/null +++ b/java/src/main/resources/db/migration/V3__attachments.sql @@ -0,0 +1,67 @@ +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + owner_id UUID NOT NULL REFERENCES users(id), + + filename TEXT NOT NULL, + content_type VARCHAR(32) NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 CHAR(64) NOT NULL, + + object_key TEXT NOT NULL UNIQUE, + status VARCHAR(16) NOT NULL DEFAULT 'ready', + + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + + CONSTRAINT chk_attachment_filename + CHECK (char_length(filename) BETWEEN 1 AND 255), + + CONSTRAINT chk_attachment_content_type + CHECK (content_type IN ('image/png', 'image/jpeg')), + + CONSTRAINT chk_attachment_size + CHECK (size_bytes BETWEEN 1 AND 10485760), + + CONSTRAINT chk_attachment_sha256 + CHECK (sha256 ~ '^[a-f0-9]{64}$'), + + CONSTRAINT chk_attachment_status + CHECK (status = 'ready'), + + CONSTRAINT fk_attachment_membership + FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members(chat_id, user_id) +); + +CREATE INDEX idx_attachments_chat + ON attachments(chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id), + position INTEGER NOT NULL, + + PRIMARY KEY (message_id, position), + + CONSTRAINT uq_message_attachment + UNIQUE (message_id, attachment_id), + + CONSTRAINT chk_attachment_position + CHECK (position BETWEEN 0 AND 9) +); + +CREATE INDEX idx_message_attachments_attachment + ON message_attachments(attachment_id); + + +ALTER TABLE messages + DROP CONSTRAINT chk_message_text; + +ALTER TABLE messages + ADD CONSTRAINT chk_message_text + CHECK (char_length(text) BETWEEN 0 AND 4000); + +-- Старые запросы не содержали вложений, поэтому для них значение пустое. +ALTER TABLE message_idempotency + ADD COLUMN request_attachment_ids TEXT NOT NULL DEFAULT ''; \ No newline at end of file diff --git a/java/src/main/resources/db/migration/V4__async_attachments.sql b/java/src/main/resources/db/migration/V4__async_attachments.sql new file mode 100644 index 00000000..1e9be45b --- /dev/null +++ b/java/src/main/resources/db/migration/V4__async_attachments.sql @@ -0,0 +1,220 @@ +-- Расширение метаданных вложения для асинхронной обработки. + +ALTER TABLE attachments + DROP CONSTRAINT chk_attachment_status; + +ALTER TABLE attachments + ADD CONSTRAINT chk_attachment_status + CHECK ( + status IN ( + 'queued', + 'processing', + 'ready', + 'failed' + ) + ); + +ALTER TABLE attachments + ADD COLUMN staging_key TEXT, + ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN error_code VARCHAR(64); + +ALTER TABLE attachments + ADD CONSTRAINT uq_attachment_staging_key + UNIQUE (staging_key), + + ADD CONSTRAINT chk_attachment_pipeline_version + CHECK (pipeline_version >= 1), + + ADD CONSTRAINT chk_attachment_error + CHECK ( + ( + status = 'failed' + AND error_code IS NOT NULL + AND length(error_code) > 0 + ) + OR + ( + status <> 'failed' + AND error_code IS NULL + ) + ); + +CREATE INDEX idx_attachments_status + ON attachments(status, updated_at); + + +-- Готовые варианты изображения. +-- object_key используется внутри приложения и не выдаётся клиенту. +-- Ключ эффекта соответствует контракту: +-- attachment_id + pipeline_version + variant. + +CREATE TABLE attachment_variants ( + attachment_id UUID NOT NULL + REFERENCES attachments(id) ON DELETE CASCADE, + + pipeline_version INTEGER NOT NULL, + name VARCHAR(32) NOT NULL, + + object_key TEXT NOT NULL UNIQUE, + content_type VARCHAR(32) NOT NULL, + + size_bytes BIGINT NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + sha256 CHAR(64) NOT NULL, + + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + PRIMARY KEY ( + attachment_id, + pipeline_version, + name + ), + + CONSTRAINT chk_variant_pipeline_version + CHECK (pipeline_version >= 1), + + CONSTRAINT chk_variant_name + CHECK ( + name IN ( + 'thumbnail', + 'crop', + 'optimized' + ) + ), + + CONSTRAINT chk_variant_content_type + CHECK ( + content_type IN ( + 'image/jpeg', + 'image/png', + 'image/webp' + ) + ), + + CONSTRAINT chk_variant_size + CHECK (size_bytes > 0), + + CONSTRAINT chk_variant_dimensions + CHECK (width > 0 AND height > 0), + + CONSTRAINT chk_variant_sha256 + CHECK (sha256 ~ '^[a-f0-9]{64}$') +); + + +-- Одна логическая работа над вложением в рамках версии pipeline. +-- job_id не меняется при повторной доставке. +-- attempts увеличивается при начале управляемой попытки, +-- а не при каждом получении дубликата сообщения. + +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY, + + attachment_id UUID NOT NULL + REFERENCES attachments(id) ON DELETE CASCADE, + + pipeline_version INTEGER NOT NULL DEFAULT 1, + + status VARCHAR(16) NOT NULL DEFAULT 'queued', + + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 3, + + available_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + lease_owner UUID, + lease_until TIMESTAMPTZ, + + last_error_code VARCHAR(64), + + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT uq_attachment_job_pipeline + UNIQUE ( + attachment_id, + pipeline_version + ), + + CONSTRAINT chk_job_pipeline_version + CHECK (pipeline_version >= 1), + + CONSTRAINT chk_job_status + CHECK ( + status IN ( + 'queued', + 'processing', + 'ready', + 'failed' + ) + ), + + CONSTRAINT chk_job_attempts + CHECK ( + max_attempts >= 1 + AND attempts >= 0 + AND attempts <= max_attempts + ), + + CONSTRAINT chk_job_lease + CHECK ( + ( + status = 'processing' + AND lease_owner IS NOT NULL + AND lease_until IS NOT NULL + ) + OR + ( + status <> 'processing' + AND lease_owner IS NULL + AND lease_until IS NULL + ) + ) +); + +CREATE INDEX idx_attachment_jobs_available + ON attachment_jobs(available_at, id) + WHERE status = 'queued'; + +CREATE INDEX idx_attachment_jobs_expired_lease + ON attachment_jobs(lease_until, id) + WHERE status = 'processing'; + + +-- Намерение публикации задания в брокер. +-- Записывается в той же транзакции, что и метаданные задания. +-- published_at устанавливается после подтверждения брокера. +-- Потеря подтверждения может привести к повторной публикации: +-- worker обязан обрабатывать её идемпотентно. + +CREATE TABLE attachment_outbox ( + id UUID PRIMARY KEY, + + job_id UUID NOT NULL + REFERENCES attachment_jobs(id) ON DELETE CASCADE, + + attempt INTEGER NOT NULL, + + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + next_publish_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + published_at TIMESTAMPTZ, + + publish_attempts INTEGER NOT NULL DEFAULT 0, + last_error_code VARCHAR(64), + + CONSTRAINT uq_attachment_outbox_attempt + UNIQUE (job_id, attempt), + + CONSTRAINT chk_outbox_attempt + CHECK (attempt >= 1), + + CONSTRAINT chk_outbox_publish_attempts + CHECK (publish_attempts >= 0) +); + +CREATE INDEX idx_attachment_outbox_pending + ON attachment_outbox(next_publish_at, id) + WHERE published_at IS NULL; \ No newline at end of file diff --git a/java/src/main/resources/db/migration/V5__level4_processing.sql b/java/src/main/resources/db/migration/V5__level4_processing.sql new file mode 100644 index 00000000..6baee3fd --- /dev/null +++ b/java/src/main/resources/db/migration/V5__level4_processing.sql @@ -0,0 +1,94 @@ +CREATE TABLE attachment_replay_audit ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + + job_id UUID NOT NULL REFERENCES attachment_jobs(id), + actor VARCHAR(64) NOT NULL, + reason VARCHAR(255) NOT NULL, + + action VARCHAR(32) NOT NULL + CHECK (action IN ('enqueued', 'already_ready')), + + previous_attempts INTEGER NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE attachment_attempt_events ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + + job_id UUID NOT NULL REFERENCES attachment_jobs(id), + attempt INTEGER NOT NULL, + state VARCHAR(16) NOT NULL, + error_code VARCHAR(64) NOT NULL, + delay_seconds INTEGER NOT NULL, + + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +-- Используется только при FAULT_INJECTION_ENABLED=true. +-- Управления через публичный HTTP API нет. +CREATE TABLE attachment_test_faults ( + point VARCHAR(32) PRIMARY KEY, + remaining INTEGER NOT NULL CHECK (remaining >= 0), + hits INTEGER NOT NULL DEFAULT 0, + + job_id UUID REFERENCES attachment_jobs(id), + hit_at TIMESTAMPTZ +); + +-- Существующие ready-вложения с неполным набором вариантов +-- отправляются на дообработку. +INSERT INTO attachment_jobs ( + id, + attachment_id, + pipeline_version, + status, + max_attempts +) +SELECT + gen_random_uuid(), + a.id, + a.pipeline_version, + 'queued', + 3 +FROM attachments a +WHERE a.status = 'ready' + AND ( + SELECT count(*) + FROM attachment_variants v + WHERE v.attachment_id = a.id + AND v.pipeline_version = a.pipeline_version + AND v.name IN ('thumbnail', 'crop', 'optimized') + ) < 3 +ON CONFLICT (attachment_id, pipeline_version) +DO UPDATE SET + status = 'queued', + max_attempts = greatest( + attachment_jobs.max_attempts, + attachment_jobs.attempts + 3 + ), + available_at = clock_timestamp(), + lease_owner = NULL, + lease_until = NULL, + last_error_code = NULL, + updated_at = clock_timestamp() +WHERE attachment_jobs.status = 'ready'; + +UPDATE attachments a +SET status = 'queued', + staging_key = a.object_key, + error_code = NULL, + updated_at = clock_timestamp() +WHERE a.status = 'ready' + AND EXISTS ( + SELECT 1 + FROM attachment_jobs j + WHERE j.attachment_id = a.id + AND j.pipeline_version = a.pipeline_version + AND j.status = 'queued' + ); + +INSERT INTO attachment_outbox(id, job_id, attempt) +SELECT gen_random_uuid(), id, attempts + 1 +FROM attachment_jobs +WHERE status = 'queued' +ON CONFLICT (job_id, attempt) DO NOTHING; \ No newline at end of file diff --git a/java/src/test/java/com/vnp/demo/ConcurrentRegistrationTest.java b/java/src/test/java/com/vnp/demo/ConcurrentRegistrationTest.java new file mode 100644 index 00000000..7468327e --- /dev/null +++ b/java/src/test/java/com/vnp/demo/ConcurrentRegistrationTest.java @@ -0,0 +1,89 @@ +package com.vnp.demo; + +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.http.ResponseEntity; + +import com.vnp.demo.dto.UserCreateRequest; +import com.vnp.demo.repositories.UserRepository; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class ConcurrentRegistrationTest { + + @Autowired + private TestRestTemplate restTemplate; + + @Autowired + private UserRepository userRepository; + + @Test + void concurrentRegistrationAllowsOnlyOneUser() throws Exception { + String username = "race_" + + UUID.randomUUID().toString().substring(0, 8); + + UserCreateRequest request = new UserCreateRequest( + username, + "Race Test", + "Password123!" + ); + + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch start = new CountDownLatch(1); + + try { + Callable> task = () -> { + start.await(); + + return restTemplate.postForEntity( + "/api/v1/users", + request, + String.class + ); + }; + + Future> first = + executor.submit(task); + + Future> second = + executor.submit(task); + + start.countDown(); + + int status1 = first.get(15, TimeUnit.SECONDS) + .getStatusCode().value(); + + int status2 = second.get(15, TimeUnit.SECONDS) + .getStatusCode().value(); + + assertTrue( + (status1 == 201 && status2 == 409) + || (status1 == 409 && status2 == 201), + "Expected 201 and 409, got " + + status1 + " and " + status2 + ); + + assertEquals( + 1, + userRepository.findAll().stream() + .filter(user -> + username.equals(user.getUsername())) + .count() + ); + + } finally { + executor.shutdownNow(); + } + } +} diff --git a/java/src/test/java/com/vnp/demo/InfrastructureFailureIntegrationTest.java b/java/src/test/java/com/vnp/demo/InfrastructureFailureIntegrationTest.java new file mode 100644 index 00000000..f2e64cab --- /dev/null +++ b/java/src/test/java/com/vnp/demo/InfrastructureFailureIntegrationTest.java @@ -0,0 +1,155 @@ + +package com.vnp.demo; + +import java.time.Instant; +import java.util.UUID; + +import static org.hamcrest.Matchers.equalTo; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.dao.DataAccessResourceFailureException; +import org.springframework.security.crypto.password.PasswordEncoder; + +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.UserRepository; +import com.vnp.demo.service.SessionCacheService; + +import io.restassured.RestAssured; +import static io.restassured.RestAssured.given; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class InfrastructureFailureIntegrationTest { + + @LocalServerPort + private int port; + + @SpyBean + private UserRepository userRepository; + + @Autowired + private PasswordEncoder passwordEncoder; + + @SpyBean + private SessionCacheService sessionCacheService; + + private String username; + + private final String password = "Password123!"; + + @BeforeEach + void setUp() { + RestAssured.baseURI = "http://localhost"; + RestAssured.port = port; + + username = "failure_" + UUID.randomUUID() + .toString() + .substring(0, 8); + + User user = new User( + username, + "Failure Test", + passwordEncoder.encode(password) + ); + + userRepository.saveAndFlush(user); + } + + @Test + void redisUnavailableDuringLoginReturns503() { + doThrow(new DataAccessResourceFailureException( + "Redis unavailable" + )).when(sessionCacheService).cache( + anyString(), + any(UUID.class), + any(Instant.class) + ); + + given() + .auth() + .preemptive() + .basic(username, password) + .when() + .post("/api/v1/auth/sessions") + .then() + .statusCode(503) + .body( + "error.code", + equalTo("service_unavailable") + ); + } + + @Test + void anonymousUserCannotBypassAuthentication() { + given() + .header( + "X-User-Id", + UUID.randomUUID().toString() + ) + .when() + .get("/api/v1/users/me") + .then() + .statusCode(401); + } + + @Test + void redisUnavailableDuringAuthenticationDoesNotGrantAccess() { + String token = given() + .auth() + .preemptive() + .basic(username, password) + .when() + .post("/api/v1/auth/sessions") + .then() + .statusCode(201) + .extract() + .path("token"); + + doThrow(new DataAccessResourceFailureException( + "Redis unavailable" + )).when(sessionCacheService).find(anyString()); + + given() + .header( + "Authorization", + "Bearer " + token + ) + .when() + .get("/api/v1/users/me") + .then() + .statusCode(503) + .body( + "error.code", + equalTo("service_unavailable") + ); + } + + @Test + void databaseUnavailableDuringLoginReturns503() { + doThrow(new DataAccessResourceFailureException( + "PostgreSQL unavailable" + )).when(userRepository).findByUsername( + eq(username) + ); + + given() + .auth() + .preemptive() + .basic(username, password) + .when() + .post("/api/v1/auth/sessions") + .then() + .statusCode(503) + .body( + "error.code", + equalTo("service_unavailable") + ); + } +} diff --git a/java/src/test/java/com/vnp/demo/SessionIntegrationTest.java b/java/src/test/java/com/vnp/demo/SessionIntegrationTest.java new file mode 100644 index 00000000..a30b8480 --- /dev/null +++ b/java/src/test/java/com/vnp/demo/SessionIntegrationTest.java @@ -0,0 +1,154 @@ + +package com.vnp.demo; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.UUID; + +import static org.hamcrest.Matchers.equalTo; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.security.crypto.password.PasswordEncoder; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.UserRepository; + +import io.restassured.RestAssured; +import static io.restassured.RestAssured.given; + +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = "app.session.ttl-seconds=2" +) +class SessionIntegrationTest { + + private static final String PASSWORD = "TestPassword123!"; + + @LocalServerPort + private int port; + + @Autowired + private ObjectMapper mapper; + + @Autowired + private UserRepository userRepository; + + @Autowired + private PasswordEncoder passwordEncoder; + + private String username; + + @BeforeEach + void setUp() { + RestAssured.baseURI = "http://localhost"; + RestAssured.port = port; + + username = "test_" + UUID.randomUUID() + .toString().substring(0, 8); + + userRepository.saveAndFlush( + new User( + username, + "Test User", + passwordEncoder.encode(PASSWORD) + ) + ); + } + + private String login(String password) throws Exception { + String credentials = username + ":" + password; + + String basic = Base64.getEncoder().encodeToString( + credentials.getBytes(StandardCharsets.UTF_8) + ); + + var response = given() + .header("Authorization", "Basic " + basic) + .post("/api/v1/auth/sessions"); + + response.then().log().all(); + + response.then().statusCode(201); + + JsonNode json = mapper.readTree(response.asString()); + return json.get("token").asText(); + } + @Test + void validSessionAllowsAccess() throws Exception { + String token = login(PASSWORD); + + given() + .header("Authorization", "Bearer " + token) + .get("/api/v1/users/me") + .then() + .statusCode(200) + .body("username", equalTo(username)); + } + + @Test + void expiredSessionReturns401() throws Exception { + String token = login(PASSWORD); + + Thread.sleep(3000); + + given() + .header("Authorization", "Bearer " + token) + .get("/api/v1/users/me") + .then() + .statusCode(401); + } + + @Test + void logoutRevokesSession() throws Exception { + String token = login(PASSWORD); + + given() + .header("Authorization", "Bearer " + token) + .delete("/api/v1/auth/sessions/current") + .then() + .statusCode(204); + + given() + .header("Authorization", "Bearer " + token) + .get("/api/v1/users/me") + .then() + .statusCode(401); + } + + @Test + void wrongPasswordReturns401() { + String credentials = username + ":WrongPassword123!"; + + String basic = Base64.getEncoder().encodeToString( + credentials.getBytes(StandardCharsets.UTF_8) + ); + + given() + .header("Authorization", "Basic " + basic) + .post("/api/v1/auth/sessions") + .then() + .statusCode(401); + } + + @Test + void missingTokenReturns401() { + given() + .get("/api/v1/users/me") + .then() + .statusCode(401); + } + + @Test + void fakeUserIdHeaderCannotAuthenticate() { + given() + .header("X-User-Id", UUID.randomUUID().toString()) + .get("/api/v1/users/me") + .then() + .statusCode(401); + } +} diff --git a/java/src/test/java/com/vnp/demo/TransactionRollbackTest.java b/java/src/test/java/com/vnp/demo/TransactionRollbackTest.java new file mode 100644 index 00000000..6285d349 --- /dev/null +++ b/java/src/test/java/com/vnp/demo/TransactionRollbackTest.java @@ -0,0 +1,73 @@ +package com.vnp.demo; + +import com.vnp.demo.dto.ChatCreateRequest; +import com.vnp.demo.models.ChatMember; +import com.vnp.demo.models.User; +import com.vnp.demo.repositories.ChatMemberRepository; +import com.vnp.demo.repositories.ChatRepository; +import com.vnp.demo.repositories.UserRepository; +import com.vnp.demo.service.ChatService; + + +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +@SpringBootTest +class TransactionRollbackTest { + + @Autowired + private ChatService chatService; + + @Autowired + private ChatRepository chatRepository; + + @Autowired + private UserRepository userRepository; + + @SpyBean + private ChatMemberRepository chatMemberRepository; + + @Test + void shouldRollbackChatCreationWhenMemberSaveFails() { + User user = userRepository.saveAndFlush( + new User( + "rollback_" + UUID.randomUUID() + .toString() + .substring(0, 8), + "Rollback Test", + "test-password-hash" + ) + ); + + long chatsBefore = chatRepository.count(); + + doThrow(new IllegalStateException("Simulated failure")) + .when(chatMemberRepository) + .save(any(ChatMember.class)); + + ChatCreateRequest request = new ChatCreateRequest( + "Rollback test chat", + List.of(user.getId()) + ); + + assertThrows( + IllegalStateException.class, + () -> chatService.create(user.getId(), request) + ); + + assertEquals( + chatsBefore, + chatRepository.count(), + "Chat must not be saved after transaction rollback" + ); + } +} diff --git a/java/src/test/java/com/vnp/demo/sql/message_index_benchmark.sql b/java/src/test/java/com/vnp/demo/sql/message_index_benchmark.sql new file mode 100644 index 00000000..81217354 --- /dev/null +++ b/java/src/test/java/com/vnp/demo/sql/message_index_benchmark.sql @@ -0,0 +1,89 @@ +BEGIN; + +INSERT INTO users ( + id, + username, + display_name, + password_hash +) +VALUES ( + '11111111-1111-4111-8111-111111111111', + 'benchmark_user', + 'Benchmark User', + 'benchmark-hash' +) +ON CONFLICT (id) DO NOTHING; + +INSERT INTO chats ( + id, + title +) +VALUES ( + '22222222-2222-4222-8222-222222222222', + 'Benchmark Chat' +) +ON CONFLICT (id) DO NOTHING; + +INSERT INTO chat_members ( + chat_id, + user_id +) +VALUES ( + '22222222-2222-4222-8222-222222222222', + '11111111-1111-4111-8111-111111111111' +) +ON CONFLICT DO NOTHING; + +INSERT INTO messages ( + id, + chat_id, + sender_id, + text, + created_at +) +SELECT + ( + '33333333-3333-4333-8333-' || + lpad(g::text, 12, '0') + )::uuid, + '22222222-2222-4222-8222-222222222222'::uuid, + '11111111-1111-4111-8111-111111111111'::uuid, + 'Benchmark message ' || g, + NOW() - INTERVAL '1 day' + + g * INTERVAL '1 second' +FROM generate_series(1, 10000) AS g +ON CONFLICT (id) DO NOTHING; + +COMMIT; + +ANALYZE messages; + +SELECT COUNT(*) +FROM messages +WHERE chat_id = + '22222222-2222-4222-8222-222222222222'; + +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = + '22222222-2222-4222-8222-222222222222' +ORDER BY created_at, id +LIMIT 50; + +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = + '22222222-2222-4222-8222-222222222222' + AND (created_at, id) > ( + ( + SELECT created_at + FROM messages + WHERE id = + '33333333-3333-4333-8333-000000000050' + ), + '33333333-3333-4333-8333-000000000050'::uuid + ) +ORDER BY created_at, id +LIMIT 50; \ No newline at end of file diff --git a/results/lab5-level4.json b/results/lab5-level4.json new file mode 100644 index 00000000..4b3de76f --- /dev/null +++ b/results/lab5-level4.json @@ -0,0 +1,432 @@ +{ + "passed": true, + "scenarios": [ + { + "scenario": "commit_before_publish", + "job_id": "207a9343-70e4-466e-b481-11e67857199e", + "status": "ready", + "attempts": 1, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/b5f218eb-0085-4eec-a2b5-7dd216e598e0", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/b5f218eb-0085-4eec-a2b5-7dd216e598e0/v1/bac67a3f-0ffd-44cb-8c8b-fc6dbec51390", + "variants": [ + { + "attachment_id": "b5f218eb-0085-4eec-a2b5-7dd216e598e0", + "pipeline_version": 1, + "name": "crop", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/b5f218eb-0085-4eec-a2b5-7dd216e598e0/v1/bac67a3f-0ffd-44cb-8c8b-fc6dbec51390/crop.jpg", + "content_type": "image/jpeg", + "size_bytes": 1820, + "width": 128, + "height": 128, + "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", + "created_at": "2026-11-11T19:49:15.90599+00:00" + }, + { + "attachment_id": "b5f218eb-0085-4eec-a2b5-7dd216e598e0", + "pipeline_version": 1, + "name": "optimized", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/b5f218eb-0085-4eec-a2b5-7dd216e598e0/v1/bac67a3f-0ffd-44cb-8c8b-fc6dbec51390/optimized.webp", + "content_type": "image/webp", + "size_bytes": 2462, + "width": 320, + "height": 200, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "created_at": "2026-11-11T19:49:15.90599+00:00" + }, + { + "attachment_id": "b5f218eb-0085-4eec-a2b5-7dd216e598e0", + "pipeline_version": 1, + "name": "thumbnail", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/b5f218eb-0085-4eec-a2b5-7dd216e598e0/v1/bac67a3f-0ffd-44cb-8c8b-fc6dbec51390/thumbnail.jpg", + "content_type": "image/jpeg", + "size_bytes": 4679, + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "created_at": "2026-11-11T19:49:15.90599+00:00" + } + ], + "events": [] + }, + { + "scenario": "after_objects", + "job_id": "49d51049-a3f9-4f4d-a5be-a764dae8873b", + "status": "ready", + "attempts": 2, + "owner": null, + "staging": null, + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/c75cd648-4491-4c57-b626-11322c195a6f/v1/00e588f1-255c-45f3-a362-32375997971b", + "variants": [ + { + "attachment_id": "c75cd648-4491-4c57-b626-11322c195a6f", + "pipeline_version": 1, + "name": "crop", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/c75cd648-4491-4c57-b626-11322c195a6f/v1/00e588f1-255c-45f3-a362-32375997971b/crop.jpg", + "content_type": "image/jpeg", + "size_bytes": 1820, + "width": 128, + "height": 128, + "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", + "created_at": "2026-11-11T19:50:05.541163+00:00" + }, + { + "attachment_id": "c75cd648-4491-4c57-b626-11322c195a6f", + "pipeline_version": 1, + "name": "optimized", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/c75cd648-4491-4c57-b626-11322c195a6f/v1/00e588f1-255c-45f3-a362-32375997971b/optimized.webp", + "content_type": "image/webp", + "size_bytes": 2462, + "width": 320, + "height": 200, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "created_at": "2026-11-11T19:50:05.541163+00:00" + }, + { + "attachment_id": "c75cd648-4491-4c57-b626-11322c195a6f", + "pipeline_version": 1, + "name": "thumbnail", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/c75cd648-4491-4c57-b626-11322c195a6f/v1/00e588f1-255c-45f3-a362-32375997971b/thumbnail.jpg", + "content_type": "image/jpeg", + "size_bytes": 4679, + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "created_at": "2026-11-11T19:50:05.541163+00:00" + } + ], + "events": [ + { + "id": 5, + "job_id": "49d51049-a3f9-4f4d-a5be-a764dae8873b", + "attempt": 1, + "state": "queued", + "error_code": "lease_expired", + "delay_seconds": 1, + "created_at": "2026-11-11T19:50:03.822834+00:00" + } + ] + }, + { + "scenario": "before_ack", + "job_id": "e4265ea1-b4f8-467c-93cf-61030292f002", + "status": "ready", + "attempts": 2, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/412fda65-d510-4b2d-80f0-c7cd78b7f1ee", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/412fda65-d510-4b2d-80f0-c7cd78b7f1ee/v1/96af9136-89bf-49bd-b2e8-193bc0b1bba9", + "variants": [ + { + "attachment_id": "412fda65-d510-4b2d-80f0-c7cd78b7f1ee", + "pipeline_version": 1, + "name": "crop", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/412fda65-d510-4b2d-80f0-c7cd78b7f1ee/v1/96af9136-89bf-49bd-b2e8-193bc0b1bba9/crop.jpg", + "content_type": "image/jpeg", + "size_bytes": 1820, + "width": 128, + "height": 128, + "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", + "created_at": "2026-11-11T19:51:11.854309+00:00" + }, + { + "attachment_id": "412fda65-d510-4b2d-80f0-c7cd78b7f1ee", + "pipeline_version": 1, + "name": "optimized", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/412fda65-d510-4b2d-80f0-c7cd78b7f1ee/v1/96af9136-89bf-49bd-b2e8-193bc0b1bba9/optimized.webp", + "content_type": "image/webp", + "size_bytes": 2462, + "width": 320, + "height": 200, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "created_at": "2026-11-11T19:51:11.854309+00:00" + }, + { + "attachment_id": "412fda65-d510-4b2d-80f0-c7cd78b7f1ee", + "pipeline_version": 1, + "name": "thumbnail", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/412fda65-d510-4b2d-80f0-c7cd78b7f1ee/v1/96af9136-89bf-49bd-b2e8-193bc0b1bba9/thumbnail.jpg", + "content_type": "image/jpeg", + "size_bytes": 4679, + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "created_at": "2026-11-11T19:51:11.854309+00:00" + } + ], + "events": [ + { + "id": 6, + "job_id": "e4265ea1-b4f8-467c-93cf-61030292f002", + "attempt": 1, + "state": "queued", + "error_code": "processing_unavailable", + "delay_seconds": 2, + "created_at": "2026-11-11T19:51:09.373985+00:00" + } + ] + }, + { + "scenario": "permanent_error", + "failed": { + "job_id": "618544ac-e574-4dd7-9e46-81cf9147f9ef", + "status": "failed", + "attempts": 1, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee/pending", + "variants": [], + "events": [ + { + "id": 7, + "job_id": "618544ac-e574-4dd7-9e46-81cf9147f9ef", + "attempt": 1, + "state": "failed", + "error_code": "invalid_image", + "delay_seconds": 0, + "created_at": "2026-11-11T19:51:45.828306+00:00" + } + ] + }, + "ready": { + "job_id": "618544ac-e574-4dd7-9e46-81cf9147f9ef", + "status": "ready", + "attempts": 2, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee/v1/baf3e4b3-72d9-4ba3-ba13-2ea381d22466", + "variants": [ + { + "attachment_id": "a10ad8db-4c07-43bc-a8ee-be75671095ee", + "pipeline_version": 1, + "name": "crop", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee/v1/baf3e4b3-72d9-4ba3-ba13-2ea381d22466/crop.jpg", + "content_type": "image/jpeg", + "size_bytes": 1820, + "width": 128, + "height": 128, + "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", + "created_at": "2026-11-11T19:52:14.88595+00:00" + }, + { + "attachment_id": "a10ad8db-4c07-43bc-a8ee-be75671095ee", + "pipeline_version": 1, + "name": "optimized", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee/v1/baf3e4b3-72d9-4ba3-ba13-2ea381d22466/optimized.webp", + "content_type": "image/webp", + "size_bytes": 2462, + "width": 320, + "height": 200, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "created_at": "2026-11-11T19:52:14.88595+00:00" + }, + { + "attachment_id": "a10ad8db-4c07-43bc-a8ee-be75671095ee", + "pipeline_version": 1, + "name": "thumbnail", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee/v1/baf3e4b3-72d9-4ba3-ba13-2ea381d22466/thumbnail.jpg", + "content_type": "image/jpeg", + "size_bytes": 4679, + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "created_at": "2026-11-11T19:52:14.88595+00:00" + } + ], + "events": [ + { + "id": 7, + "job_id": "618544ac-e574-4dd7-9e46-81cf9147f9ef", + "attempt": 1, + "state": "failed", + "error_code": "invalid_image", + "delay_seconds": 0, + "created_at": "2026-11-11T19:51:45.828306+00:00" + } + ] + } + }, + { + "scenario": "transient_error", + "failed": { + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "status": "failed", + "attempts": 3, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a/pending", + "variants": [], + "events": [ + { + "id": 8, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 1, + "state": "queued", + "error_code": "processing_unavailable", + "delay_seconds": 2, + "created_at": "2026-11-11T19:52:43.635817+00:00" + }, + { + "id": 9, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 2, + "state": "queued", + "error_code": "processing_unavailable", + "delay_seconds": 2, + "created_at": "2026-11-11T19:52:45.971699+00:00" + }, + { + "id": 10, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 3, + "state": "failed", + "error_code": "processing_unavailable", + "delay_seconds": 0, + "created_at": "2026-11-11T19:52:48.656484+00:00" + } + ] + }, + "ready": { + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "status": "ready", + "attempts": 5, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a/v1/8b9bd5ba-3fc3-4525-b7e3-8e8eeca815e0", + "variants": [ + { + "attachment_id": "772c8541-d5c1-4707-86e5-9ba4a96eca4a", + "pipeline_version": 1, + "name": "crop", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a/v1/8b9bd5ba-3fc3-4525-b7e3-8e8eeca815e0/crop.jpg", + "content_type": "image/jpeg", + "size_bytes": 1820, + "width": 128, + "height": 128, + "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", + "created_at": "2026-11-11T19:53:15.180295+00:00" + }, + { + "attachment_id": "772c8541-d5c1-4707-86e5-9ba4a96eca4a", + "pipeline_version": 1, + "name": "optimized", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a/v1/8b9bd5ba-3fc3-4525-b7e3-8e8eeca815e0/optimized.webp", + "content_type": "image/webp", + "size_bytes": 2462, + "width": 320, + "height": 200, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "created_at": "2026-11-11T19:53:15.180295+00:00" + }, + { + "attachment_id": "772c8541-d5c1-4707-86e5-9ba4a96eca4a", + "pipeline_version": 1, + "name": "thumbnail", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a/v1/8b9bd5ba-3fc3-4525-b7e3-8e8eeca815e0/thumbnail.jpg", + "content_type": "image/jpeg", + "size_bytes": 4679, + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "created_at": "2026-11-11T19:53:15.180295+00:00" + } + ], + "events": [ + { + "id": 8, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 1, + "state": "queued", + "error_code": "processing_unavailable", + "delay_seconds": 2, + "created_at": "2026-11-11T19:52:43.635817+00:00" + }, + { + "id": 9, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 2, + "state": "queued", + "error_code": "processing_unavailable", + "delay_seconds": 2, + "created_at": "2026-11-11T19:52:45.971699+00:00" + }, + { + "id": 10, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 3, + "state": "failed", + "error_code": "processing_unavailable", + "delay_seconds": 0, + "created_at": "2026-11-11T19:52:48.656484+00:00" + }, + { + "id": 11, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 4, + "state": "queued", + "error_code": "processing_unavailable", + "delay_seconds": 2, + "created_at": "2026-11-11T19:53:12.735705+00:00" + } + ] + } + }, + { + "scenario": "image_timeout", + "job_id": "cf2a9648-947a-4111-a100-3939bb467455", + "status": "ready", + "attempts": 2, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/cbaa3d91-69ec-4edb-8804-c815c60d3fe8", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/cbaa3d91-69ec-4edb-8804-c815c60d3fe8/v1/ed7490f5-d0fe-4c37-baa7-b2743ff92d71", + "variants": [ + { + "attachment_id": "cbaa3d91-69ec-4edb-8804-c815c60d3fe8", + "pipeline_version": 1, + "name": "crop", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/cbaa3d91-69ec-4edb-8804-c815c60d3fe8/v1/ed7490f5-d0fe-4c37-baa7-b2743ff92d71/crop.jpg", + "content_type": "image/jpeg", + "size_bytes": 1820, + "width": 128, + "height": 128, + "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", + "created_at": "2026-11-11T19:54:12.013459+00:00" + }, + { + "attachment_id": "cbaa3d91-69ec-4edb-8804-c815c60d3fe8", + "pipeline_version": 1, + "name": "optimized", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/cbaa3d91-69ec-4edb-8804-c815c60d3fe8/v1/ed7490f5-d0fe-4c37-baa7-b2743ff92d71/optimized.webp", + "content_type": "image/webp", + "size_bytes": 2462, + "width": 320, + "height": 200, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "created_at": "2026-11-11T19:54:12.013459+00:00" + }, + { + "attachment_id": "cbaa3d91-69ec-4edb-8804-c815c60d3fe8", + "pipeline_version": 1, + "name": "thumbnail", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/cbaa3d91-69ec-4edb-8804-c815c60d3fe8/v1/ed7490f5-d0fe-4c37-baa7-b2743ff92d71/thumbnail.jpg", + "content_type": "image/jpeg", + "size_bytes": 4679, + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "created_at": "2026-11-11T19:54:12.013459+00:00" + } + ], + "events": [ + { + "id": 12, + "job_id": "cf2a9648-947a-4111-a100-3939bb467455", + "attempt": 1, + "state": "queued", + "error_code": "processing_timeout", + "delay_seconds": 2, + "created_at": "2026-11-11T19:54:09.817644+00:00" + } + ] + } + ] +} diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..2422397f --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,48 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +mkdir -p .local-logs +exec > .local-logs/restart.log 2>&1 + +set -x + +date -u + +echo "Stopping API and load balancer" +docker compose stop -t 10 api lb + +echo "Restarting persistent services" +docker compose restart -t 10 postgres redis minio + +echo "Waiting for persistent services" +docker compose up -d \ + --no-build \ + --pull never \ + --no-deps \ + --no-recreate \ + --wait \ + --wait-timeout 25 \ + postgres redis minio + +echo "Starting API" +docker compose up -d \ + --no-build \ + --pull never \ + --no-deps \ + --no-recreate \ + --wait \ + --wait-timeout 45 \ + api + +echo "Starting load balancer without dependencies" +docker compose up -d \ + --no-build \ + --pull never \ + --no-deps \ + --no-recreate \ + lb + +echo "Restart completed" +date -u \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..d8c94e95 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +: "${TARGET_INSTANCE_ID:?Не задан TARGET_INSTANCE_ID}" + +compose() { + docker compose --env-file .env -f compose.yml "$@" +} + +containers=$(compose ps --status running -q api) + +target="" +count=0 + +for container in $containers; do + count=$((count + 1)) + + instance_id=$(docker inspect \ + --format '{{.Config.Hostname}}' "$container") + + if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then + if [ -n "$target" ]; then + printf '%s\n' 'Ошибка: найдено несколько подходящих реплик.' >&2 + exit 1 + fi + + target="$container" + fi +done + +if [ "$count" -lt 2 ]; then + printf '%s\n' 'Ошибка: должны работать минимум две API-реплики.' >&2 + exit 1 +fi + +if [ -z "$target" ]; then + printf '%s\n' 'Ошибка: указанная API-реплика не найдена.' >&2 + exit 1 +fi + + +docker kill --signal=KILL "$target" >/dev/null + +printf '%s\n' "Остановлена API-реплика: $TARGET_INSTANCE_ID" diff --git a/tests/level4_check.py b/tests/level4_check.py new file mode 100644 index 00000000..7b6d0e38 --- /dev/null +++ b/tests/level4_check.py @@ -0,0 +1,1006 @@ +import hashlib +import json +from pathlib import Path +import subprocess +import time +import uuid + +from client import Client, Failure, arguments, check, run +from image_probe import jpeg_size + + +ROOT = Path(__file__).resolve().parents[1] + +BASE = [ + "docker", + "compose", + "-f", + "compose.yml", + "-f", + "compose.override.yml", +] + +FAULT = BASE + [ + "-f", + "compose.faults.yml", +] + +FAULT_POINTS = { + "hold_publish", + "before_publish", + "after_objects", + "before_ack", + "permanent_error", + "transient_error", + "image_timeout", +} + + +def command(args, required=True): + try: + result = subprocess.run( + args, + cwd=ROOT, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=240, + ) + except subprocess.TimeoutExpired: + raise Failure("Docker command timed out") from None + + if required and result.returncode: + raise Failure( + "Docker command failed; " + "inspect the corresponding service locally" + ) + + return result + + +def sql(query): + result = command( + BASE + + [ + "exec", + "-T", + "postgres", + "sh", + "-ec", + 'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" ' + '-At -v ON_ERROR_STOP=1 -c "$1"', + "sh", + query, + ] + ) + + return result.stdout.strip() + + +def wait(action, label, seconds=180): + deadline = time.monotonic() + seconds + + while time.monotonic() < deadline: + try: + value = action() + + if value: + return value + + except Failure: + pass + + time.sleep(1) + + raise Failure("Timeout: " + label) + + +def state(attachment_id): + attachment_id = str(uuid.UUID(attachment_id)) + + raw = sql( + f""" + SELECT json_build_object( + 'job_id', j.id, + 'status', j.status, + 'attempts', j.attempts, + 'owner', j.lease_owner, + 'staging', a.staging_key, + 'original_key', a.object_key, + + 'variants', COALESCE( + ( + SELECT json_agg(v ORDER BY v.name) + FROM attachment_variants v + WHERE v.attachment_id = a.id + AND v.pipeline_version = a.pipeline_version + ), + '[]'::json + ), + + 'events', COALESCE( + ( + SELECT json_agg(e ORDER BY e.id) + FROM attachment_attempt_events e + WHERE e.job_id = j.id + ), + '[]'::json + ) + ) + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE a.id = '{attachment_id}'::uuid + AND j.pipeline_version = a.pipeline_version + """ + ) + + check(bool(raw), "attachment job not found") + + try: + return json.loads(raw) + except json.JSONDecodeError: + raise Failure("Invalid job state returned by PostgreSQL") from None + + +def arm(point, count=1): + check(point in FAULT_POINTS, "unknown fault") + check(isinstance(count, int) and count >= 0, "invalid fault count") + + sql( + f""" + INSERT INTO attachment_test_faults(point, remaining) + VALUES ('{point}', {count}) + + ON CONFLICT (point) + DO UPDATE SET + remaining = EXCLUDED.remaining, + hits = 0, + job_id = NULL, + hit_at = NULL + """ + ) + + +def hit(point, job_id): + check(point in FAULT_POINTS, "unknown fault") + job_id = str(uuid.UUID(job_id)) + + return sql( + f""" + SELECT count(*) + FROM attachment_test_faults + WHERE point = '{point}' + AND job_id = '{job_id}'::uuid + AND hits > 0 + """ + ) == "1" + + +def restarts(service): + ids = command( + BASE + ["ps", "-a", "-q", service] + ).stdout.split() + + check(bool(ids), "service containers not found: " + service) + + return sum( + int( + command( + [ + "docker", + "inspect", + "--format", + "{{.RestartCount}}", + container, + ] + ).stdout.strip() + ) + for container in ids + ) + + +def object_exists(bucket, key): + result = command( + BASE + + [ + "exec", + "-T", + "minio", + "sh", + "-ec", + "mc alias set grade4 http://localhost:9000 " + '"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null; ' + 'mc stat --json "grade4/$1/$2"', + "sh", + bucket, + key, + ], + required=False, + ) + + if result.returncode == 0: + return True + + error = (result.stdout + result.stderr).lower() + + if any( + value in error + for value in [ + "nosuchkey", + "nosuchobject", + "specified key does not exist", + "object does not exist", + ] + ): + return False + + raise Failure( + "Cannot distinguish missing object from S3 failure" + ) + + +def webp_size(data): + check( + len(data) >= 12 + and data[:4] == b"RIFF" + and data[8:12] == b"WEBP", + "invalid WebP signature", + ) + + check( + int.from_bytes(data[4:8], "little") + 8 == len(data), + "invalid RIFF size", + ) + + offset = 12 + size = None + + while offset + 8 <= len(data): + tag = data[offset:offset + 4] + + length = int.from_bytes( + data[offset + 4:offset + 8], + "little", + ) + + end = offset + 8 + length + check(end <= len(data), "truncated WebP chunk") + + block = data[offset + 8:end] + + check( + tag not in (b"EXIF", b"XMP ", b"ICCP"), + "metadata was retained in WebP", + ) + + if tag == b"VP8X": + check(len(block) == 10, "invalid VP8X chunk") + + size = ( + 1 + int.from_bytes(block[4:7], "little"), + 1 + int.from_bytes(block[7:10], "little"), + ) + + elif tag == b"VP8 " and size is None: + check(len(block) >= 10, "truncated VP8 frame") + + check( + block[3:6] == b"\x9d\x01\x2a", + "invalid VP8 frame", + ) + + size = ( + int.from_bytes(block[6:8], "little") & 0x3fff, + int.from_bytes(block[8:10], "little") & 0x3fff, + ) + + elif tag == b"VP8L" and size is None: + check(len(block) >= 5, "truncated VP8L frame") + check(block[0] == 0x2f, "invalid VP8L frame") + + bits = int.from_bytes(block[1:5], "little") + + size = ( + 1 + (bits & 0x3fff), + 1 + ((bits >> 14) & 0x3fff), + ) + + offset = end + (length & 1) + check(offset <= len(data), "missing WebP padding") + + check(offset == len(data), "invalid WebP trailing data") + check(size is not None, "WebP dimensions missing") + check(size[0] > 0 and size[1] > 0, "invalid WebP dimensions") + + return size + + +def replay(job): + result = command( + FAULT + + [ + "run", + "--rm", + "--no-deps", + "--pull", + "never", + "-e", + "REPLAY_ENABLED=true", + "-e", + "FAULT_INJECTION_ENABLED=false", + "-e", + "REPLAY_JOB_ID=" + str(uuid.UUID(job)), + "-e", + "REPLAY_ACTOR=grade4_test", + "-e", + "REPLAY_REASON=verified_test_replay", + "api", + ] + ) + + return result.stdout + + +def restore_normal_configuration(): + print( + "Restoring normal API and worker configuration...", + flush=True, + ) + + try: + sql( + "UPDATE attachment_test_faults SET remaining = 0" + ) + finally: + command( + BASE + + [ + "up", + "-d", + "--no-build", + "--pull", + "never", + "--no-deps", + "--scale", + "api=2", + "--wait", + "--wait-timeout", + "180", + "api", + "worker", + "lb", + ] + ) +def wait_ready_after_fault(client, actor, attachment, seconds=180): + deadline = time.monotonic() + seconds + path = "/api/v1/attachments/" + attachment["id"] + temporary_errors = 0 + + while time.monotonic() < deadline: + try: + result = client.call( + "GET", + path, + actor=actor, + schema="Attachment", + ) + + except Failure as error: + # Client проверяет HTTP-статус до разбора тела. + # Повторяем только конкретные временные HTTP-ошибки. + message = str(error) + + retryable = any( + message == ( + f"GET {path}: expected [200], received {status}" + ) + for status in (502, 503, 504) + ) + + if not retryable: + raise + + temporary_errors += 1 + + print( + f"Waiting for API recovery: " + f"temporary HTTP errors={temporary_errors}", + flush=True, + ) + + time.sleep(1) + continue + + metadata = result["json"] + + check( + metadata["status"] != "failed", + "attachment reached failed during recovery", + ) + + if metadata["status"] == "ready": + print( + "Attachment reached ready; " + f"temporary HTTP errors={temporary_errors}", + flush=True, + ) + return metadata + + check( + metadata["status"] in ("queued", "processing"), + "unexpected attachment status", + ) + + time.sleep(0.5) + + raise Failure( + "Attachment did not become ready after fault " + f"within {seconds} seconds; " + f"temporary HTTP errors={temporary_errors}" + ) + +def main(): + args = arguments( + "Lab5 level4 fault tests", + lambda parser: parser.add_argument( + "--bucket", + default="messenger-attachments", + ), + ) + + for filename in ( + "compose.yml", + "compose.override.yml", + "compose.faults.yml", + ): + check( + (ROOT / filename).is_file(), + "Required Compose file not found: " + filename, + ) + + client = Client(args.base_url, args.ca_file) + evidence = [] + + try: + print( + "Starting API and worker with test fault injection enabled...", + flush=True, + ) + + command( + FAULT + + [ + "up", + "-d", + "--no-build", + "--pull", + "never", + "--no-deps", + "--scale", + "api=2", + "--wait", + "--wait-timeout", + "180", + "api", + "worker", + "lb", + ] + ) + + wait( + lambda: client.call( + "GET", + "/health/ready", + schema="Health", + ), + "API ready", + ) + + sql( + "UPDATE attachment_test_faults SET remaining = 0" + ) + + wait( + lambda: sql( + """ + SELECT count(*) + FROM attachment_jobs + WHERE status IN ('queued', 'processing') + """ + ) == "0", + "existing work completed", + ) + + alice, bob, _, chat = client.bootstrap() + + def upload(link=False): + attachment, original = client.attachment( + alice, + chat, + ) + + if link: + client.message( + alice, + chat, + "", + attachments=[attachment["id"]], + ) + + return attachment, original + + def verify(attachment, original): + ready = wait_ready_after_fault( + client, + bob, + attachment, + seconds=180, + ) + + path = ( + "/api/v1/attachments/" + + attachment["id"] + + "/content" + ) + + check( + client.call( + "GET", + path, + actor=bob, + )["bytes"] == original, + "original changed", + ) + + expected = { + "thumbnail": ("image/jpeg", (256, 160)), + "crop": ("image/jpeg", (128, 128)), + "optimized": ("image/webp", (320, 200)), + } + + check( + {v["name"] for v in ready["variants"]} + == set(expected) + and len(ready["variants"]) == 3, + "expected three unique variants", + ) + + for variant in ready["variants"]: + mime, dimensions = expected[variant["name"]] + + result = client.call( + "GET", + path + "?variant=" + variant["name"], + actor=bob, + ) + + data = result["bytes"] + + actual = ( + webp_size(data) + if mime == "image/webp" + else jpeg_size(data) + ) + + check( + actual + == dimensions + == ( + variant["width"], + variant["height"], + ), + "wrong actual dimensions", + ) + + check( + result["headers"].get_content_type() + == mime + == variant["content_type"], + "wrong MIME", + ) + + check( + len(data) == variant["size_bytes"] + and hashlib.sha256(data).hexdigest() + == variant["sha256"], + "variant checksum mismatch", + ) + + if mime == "image/jpeg": + check( + b"Exif\x00\x00" not in data, + "JPEG contains EXIF", + ) + + return state(attachment["id"]) + + # Сценарий 1: + # оригинал и задание уже сохранены, + # но API падает до публикации в брокер. + print( + "Testing crash between COMMIT and publish...", + flush=True, + ) + + arm("hold_publish") + + attachment, original = upload(link=True) + before = state(attachment["id"]) + + check( + before["status"] == "queued", + "publish hold did not work", + ) + + baseline = restarts("api") + + arm("before_publish") + arm("hold_publish", 0) + + wait( + lambda: hit( + "before_publish", + before["job_id"], + ), + "crash before publish", + ) + + wait( + lambda: restarts("api") > baseline, + "API actually restarted", + ) + + after = verify(attachment, original) + + check( + after["job_id"] == before["job_id"], + "job was replaced", + ) + + evidence.append( + { + "scenario": "commit_before_publish", + **after, + } + ) + + print( + "PASS: real API crash after COMMIT; " + "the same job reached ready", + flush=True, + ) + + # Сценарии 2 и 3: + # падение после записи объектов; + # падение после фиксации ready, но до ACK. + for point in ["after_objects", "before_ack"]: + print( + "Testing worker crash: " + point + "...", + flush=True, + ) + + baseline = restarts("worker") + + arm(point) + + attachment, original = upload() + before = state(attachment["id"]) + + wait( + lambda: hit(point, before["job_id"]), + point, + ) + + snapshot = state(attachment["id"]) + old_keys = [] + + if point == "after_objects": + check( + snapshot["status"] == "processing" + and not snapshot["variants"], + "result committed before after_objects gate", + ) + + check( + snapshot["owner"] is not None, + "processing lease owner missing", + ) + + suffix = ( + chat["id"] + + "/" + + attachment["id"] + + "/v1/" + + snapshot["owner"] + ) + + old_keys = ["originals/" + suffix] + [ + "variants/" + suffix + "/" + name + for name in [ + "thumbnail.jpg", + "crop.jpg", + "optimized.webp", + ] + ] + + check( + all( + object_exists(args.bucket, key) + for key in old_keys + ), + "S3 writes missing", + ) + + else: + check( + snapshot["status"] == "ready", + "result not committed before before_ack gate", + ) + + wait( + lambda: restarts("worker") > baseline, + "worker actually restarted", + ) + + after = verify(attachment, original) + + if point == "before_ack": + marker = ( + "Duplicate job verified: job=" + + after["job_id"] + ) + + wait( + lambda: marker in command( + BASE + + [ + "logs", + "--tail", + "500", + "worker", + ] + ).stdout, + "duplicate checked after lost ACK", + ) + + comparison = { + "snapshot_status": snapshot["status"], + "after_status": after["status"], + "snapshot_attempts": snapshot["attempts"], + "after_attempts": after["attempts"], + "same_job": ( + after["job_id"] == snapshot["job_id"] + ), + "same_original_key": ( + after["original_key"] + == snapshot["original_key"] + ), + "same_variants": ( + after["variants"] == snapshot["variants"] + ), + "snapshot_variants": snapshot["variants"], + "after_variants": after["variants"], + "attempt_events": after["events"], + } + + print( + "BEFORE_ACK comparison:\n" + + json.dumps( + comparison, + ensure_ascii=False, + indent=2, + ), + flush=True, + ) + + check( + snapshot["status"] == "ready" + and after["status"] == "ready" + and after["attempts"] == snapshot["attempts"] + and after["job_id"] == snapshot["job_id"] + and after["variants"] == snapshot["variants"] + and after["original_key"] == snapshot["original_key"] + and after["events"] == snapshot["events"], + "duplicate changed effect", + ) + + else: + check( + after["attempts"] == 2, + "lease recovery did not create one retry", + ) + + staging = snapshot["staging"] + check(bool(staging), "staging key missing") + + wait( + lambda: state(attachment["id"])["staging"] + is None + and not any( + object_exists(args.bucket, key) + for key in old_keys + [staging] + ), + "safe staging/orphan cleanup", + seconds=240, + ) + + # После очистки подтверждённые результаты + # должны оставаться доступными. + after = verify(attachment, original) + + evidence.append( + { + "scenario": point, + **after, + } + ) + + print( + "PASS: " + + point + + "; durable result verified", + flush=True, + ) + + # Сценарии 4 и 5: + # постоянная ошибка без повторов; + # временная ошибка с ограниченным retry; + # явный replay и повторный replay готовой задачи. + for point, count, attempts in [ + ("permanent_error", 1, 1), + ("transient_error", 3, 3), + ]: + print( + "Testing " + point + " and explicit replay...", + flush=True, + ) + + arm(point, count) + + attachment, original = upload() + + failed = wait( + lambda: ( + value + if ( + value := state(attachment["id"]) + )["status"] == "failed" + else None + ), + "bounded failure", + ) + + check( + failed["attempts"] == attempts, + "incorrect retry count", + ) + + check( + bool(failed["events"]), + "failure attempt events missing", + ) + + for event in failed["events"]: + cap = min( + 60, + 2 ** min(event["attempt"], 6), + ) + + if event["state"] == "queued": + valid_delay = ( + 1 <= event["delay_seconds"] <= cap + ) + else: + valid_delay = ( + event["state"] == "failed" + and event["delay_seconds"] == 0 + ) + + check( + valid_delay, + "invalid backoff/jitter bounds", + ) + + check( + "REPLAY_RESULT=enqueued" + in replay(failed["job_id"]), + "replay failed", + ) + + ready = verify(attachment, original) + + check( + "REPLAY_RESULT=already_ready" + in replay(failed["job_id"]), + "ready replay not a no-op", + ) + + repeated = state(attachment["id"]) + + check( + repeated["attempts"] == ready["attempts"] + and repeated["variants"] == ready["variants"] + and repeated["original_key"] == ready["original_key"], + "replay changed successful effect", + ) + + evidence.append( + { + "scenario": point, + "failed": failed, + "ready": repeated, + } + ) + + print( + "PASS: " + + point + + "; bounded attempts, explicit replay " + "and ready no-op", + flush=True, + ) + + # Сценарий 6: + # зависший процесс обработки должен быть остановлен + # по таймауту, после чего задача повторяется. + print( + "Testing image-process timeout...", + flush=True, + ) + + arm("image_timeout") + + attachment, original = upload() + result = verify(attachment, original) + + check( + result["attempts"] == 2 + and any( + event["error_code"] == "processing_timeout" + for event in result["events"] + ), + "hard image-process timeout not observed", + ) + + evidence.append( + { + "scenario": "image_timeout", + **result, + } + ) + + print( + "PASS: stuck child process killed; retry succeeded", + flush=True, + ) + + finally: + restore_normal_configuration() + + # Отчёт об успехе записывается только после прохождения + # всех сценариев и восстановления обычной конфигурации. + path = ROOT / "results" / "lab5-level4.json" + path.parent.mkdir(parents=True, exist_ok=True) + + path.write_text( + json.dumps( + { + "passed": True, + "scenarios": evidence, + }, + ensure_ascii=False, + indent=2, + ) + "\n", + encoding="utf-8", + ) + + print( + "PASS: level4 scenarios completed; " + "results/lab5-level4.json saved", + flush=True, + ) + + +if __name__ == "__main__": + run(main) \ No newline at end of file