diff --git a/Dockerfile b/Dockerfile index 6d91637..1639521 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,26 +1,39 @@ -FROM maven:3.9.9-eclipse-temurin-21 AS builder +FROM maven:3.9.9-eclipse-temurin-21 AS build + WORKDIR /build -COPY java/pom.xml . -RUN mvn -B -q dependency:go-offline +COPY java/pom.xml ./pom.xml +RUN mvn -B dependency:go-offline COPY java/src ./src -RUN mvn -B -q clean package -DskipTests -FROM eclipse-temurin:21-jre-alpine AS runtime +RUN mvn -B package -DskipTests -RUN addgroup -g 1001 -S appgroup \ - && adduser -u 1001 -S appuser -G appgroup + +FROM eclipse-temurin:21.0.9_10-jre-jammy + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + python3 \ + python3-venv \ + wget \ + && python3 -m venv /opt/images \ + && /opt/images/bin/pip install --no-cache-dir Pillow==12.0.0 \ + && /opt/images/bin/python -c \ + "from PIL import features; assert features.check('webp')" \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --system --uid 1001 --create-home app WORKDIR /app -COPY --from=builder /build/target/*.jar app.jar -RUN chown -R appuser:appgroup /app -USER appuser +COPY --from=build /build/target/*.jar /app/app.jar + +COPY infra/images/process_image.py /app/process_image.py +COPY infra/images/selftest.py /app/selftest.py + +USER 1001 + EXPOSE 8080 -HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ - CMD wget -qO- http://localhost:8080/health/live || exit 1 - -ENTRYPOINT ["java", "-XX:MaxRAMPercentage=75.0", "-jar", "/app/app.jar"] \ No newline at end of file +ENTRYPOINT ["java", "-XX:MaxRAMPercentage=75", "-jar", "/app/app.jar"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index e98f75e..59fa72c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,276 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Хуцаева Айса, БПИ2302. +- Целевая оценка: 4. +- Commit/PR: solution/khutsaeva-bpi2302 -## Запуск из чистого clone +### Подготовка -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +После клонирования репозитория перейти в его корень. + +Создать локальный файл конфигурации: + +```powershell +Copy-Item .env.example .env +``` + +Заполнить в `.env` пароли PostgreSQL, Redis, RabbitMQ, административной учётной записи MinIO и отдельной учётной записи API. Для генерации каждого нового пароля можно использовать: + +```powershell +python -c "import secrets; print(secrets.token_hex(24))" +``` + +Секреты не добавляются в Git и не включаются в отчёт. В `.env.example` должны находиться только примеры и безопасные значения по умолчанию. + +Для запуска тестов подготовить Python-окружение: + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install jsonschema openapi-spec-validator +$env:PYTHONUTF8 = "1" +``` + +Режим UTF-8 необходим, чтобы чтение контрактов на Windows не зависело от системной кодировки CP1251. Перед запуском проверить, что команда `python3`, используемая Makefile, доступна и видит зависимости тестов. + +### Сборка и запуск + +Проверить Compose: + +```powershell +docker compose config --quiet +``` + +Собрать приложение и MinIO: + +```powershell +docker compose build api minio +``` + +Запустить стенд: + +```powershell +docker compose up -d --no-build --pull never --scale api=2 --wait --wait-timeout 240 +``` + +При первом запуске предварительно загрузить внешние образы: + +```powershell +docker compose pull postgres redis migrate lb rabbitmq +``` + +Если запуск выполняется впервые, эту команду следует выполнить до `docker compose up`. + +Миграции выполняет отдельный сервис `migrate`. Сервис `minio-init` создаёт приватный bucket и настраивает права отдельной учётной записи приложения. Успешное завершение этих двух сервисов со статусом `Exited (0)` является нормальным. + +Проверить состояние: + +```powershell +docker compose ps -a +curl.exe -i http://localhost:8080/health/ready +``` + +Основной адрес API — `http://localhost:8080`. Запросы проходят через HAProxy. Прямой доступ к портам API и worker с хоста не требуется. + +Если стенд лабораторной 4 использует тот же порт, сначала остановить его из каталога соответствующего проекта. + +### Проверки + +Проверка обработчика изображений внутри worker: + +```powershell +docker compose exec -T worker /opt/images/bin/python /app/selftest.py +``` + +Публичные проверки: + +```powershell +make check +make validate +make test +``` + +Сценарии отказов уровня 4: + +```powershell +python3 -X utf8 tests/level4_check.py --base-url http://localhost:8080 --bucket messenger-attachments +``` + +Тест использует `compose.faults.yml`, временно включает внедрение ошибок, перезапускает процессы и затем восстанавливает обычную конфигурацию. Во время его выполнения не следует запускать параллельные тесты или ручные загрузки. + +Результат успешного выполнения сохраняется в: + +```text +results/lab5-level4.json +``` + +Если тест был принудительно прерван, обычную конфигурацию можно восстановить явно: + +```powershell +docker compose -f compose.yml -f compose.override.yml up -d --no-build --pull never --scale api=2 --wait --wait-timeout 240 +``` + +### Остановка + +Безопасная остановка с сохранением данных: + +```powershell +docker compose down +``` + +Намеренное удаление учебных данных, включая persistent volumes: + +```powershell +docker compose down -v +``` + +Вторая команда удаляет данные PostgreSQL, Redis, RabbitMQ и MinIO из томов этого Compose-проекта. Для обычной остановки она не используется. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент обращается к двум API-репликам через HAProxy. PostgreSQL хранит пользователей, чаты, сообщения, метаданные вложений, задания и transactional outbox. Redis используется функциональностью, перенесённой из предыдущих работ. + +MinIO хранит исходные файлы в staging и готовые результаты обработки. RabbitMQ доставляет задания отдельному worker. Сообщения брокера содержат идентификаторы и параметры задания, но не содержат бинарные данные или credentials. + +### Что перенесено из лабораторной 4 + +Перенесены: + +- пользователи, авторизация, чаты и сообщения; +- проверки доступа к чатам и вложениям; +- идемпотентное создание сообщений; +- PostgreSQL для метаданных; +- приватное S3-совместимое хранилище; +- проверка MIME по содержимому и ограничений изображения; +- две API-реплики, балансировщик и persistent volumes. + +Основное изменение: upload возвращает `202 queued` вместо `201 ready`. Клиент получает готовность через запрос метаданных. К сообщению можно привязать вложение своего чата в состоянии `queued` или `processing`. + +### Приём файла и transactional outbox + +API проверяет файл и сохраняет оригинал в устойчивый staging. Метаданные вложения, задание и запись outbox фиксируются в PostgreSQL одной транзакцией. + +Ответ `202` возвращается только после устойчивого сохранения оригинала и успешной фиксации задания. + +PostgreSQL является источником истины о состоянии задания и необходимости его публикации. RabbitMQ используется для доставки работы, но не является единственным местом хранения принятого намерения. + +Публикация выполняется отдельно. Используются persistent-сообщения, durable-очередь и publisher confirms. Если публикация или запись её подтверждения прерывается, outbox позволяет повторить отправку. + +Между S3 и PostgreSQL нет общей распределённой транзакции. При сбое могут остаться неучтённые объекты. Они удаляются механизмом очистки после grace period с проверкой ссылок из метаданных. + +### Обработка и фиксация результата + +Worker получает задание и захватывает его с lease. Доступны состояния: + +- `queued` — ожидает обработки; +- `processing` — выполняется worker; +- `ready` — результаты устойчиво сохранены; +- `failed` — постоянная ошибка или исчерпаны попытки. + +Worker сохраняет оригинал без изменения байтов и создаёт три варианта: + +| Вариант | Формат | Правила | +| --- | --- | --- | +| `thumbnail` | JPEG | Вписывание в 256×256 без увеличения небольших изображений | +| `crop` | JPEG | Центральная обрезка до 128×128, увеличение допускается | +| `optimized` | WebP | Вписывание в 1280×1280 без увеличения | + +Для JPEG используется quality 85, для WebP — quality 80 и method 4. Перед преобразованием учитывается EXIF-ориентация. Прозрачность JPEG заменяется белым фоном. Из производных изображений удаляются EXIF, ICC и XMP. + +После сохранения объектов worker транзакционно фиксирует метаданные вариантов и состояние `ready`. ACK отправляется после устойчивой фиксации результата. + +При повторной доставке готовой задачи worker проверяет уже сохранённый результат. Новая обработка и второй набор вариантов не создаются. + +### Повторы и ограничения ресурсов + +Постоянные ошибки не повторяются автоматически. Для временных ошибок используется ограниченное число попыток и backoff с jitter. + +Настройки решения: + +| Параметр | Значение | +| --- | --- | +| Максимальный размер входного файла | 10 MiB | +| Максимальное число пикселей | 40 млн | +| Обычный бюджет попыток | 3 | +| Конкуренция worker | 1 | +| Prefetch | 1 | +| Таймаут дочернего процесса обработки | 25 секунд | +| Ограничение CPU-времени дочернего процесса | 20 секунд, жёсткий предел 21 секунда | +| Ограничение адресного пространства дочернего процесса | 768 MiB | +| Лимит памяти контейнера worker | 1536 MiB | +| Максимальный Java heap worker | 256 MiB | +| Обычный lease задания | 240 секунд | + +Ограничение адресного пространства процесса не является измерением его фактического RSS. + +При исчерпании попыток задача получает `failed` с безопасным `error_code`. Неуспешные задания остаются в реестре PostgreSQL. Для недопустимых сообщений брокера предусмотрена DLQ. + +Явный replay выполняется оператором. Для `failed` проверяется доступность исходника и сохраняется запись аудита. Replay задачи в `ready` не запускает повторную обработку. + +Очистка staging выполняется после безопасной фиксации результата и истечения grace period. Используемые объекты и исходники активных заданий защищены от удаления. + +### Переменные конфигурации + +Основные переменные `.env.example` и `.env`: + +| Переменные | Назначение | +| --- | --- | +| `COMPOSE_PROJECT_NAME`, `APP_IMAGE`, `API_PORT` | Имя проекта, образ приложения, порт балансировщика | +| `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD` | PostgreSQL | +| `REDIS_PASSWORD`, `SESSION_TTL_SECONDS` | Redis и срок жизни сессии | +| `API_REPLICAS`, `API_CPUS`, `API_MEMORY`, `DB_POOL_SIZE` | Число API-реплик и ограничения ресурсов | +| `MINIO_ROOT_USER`, `MINIO_ROOT_PASSWORD` | Административная учётная запись bootstrap | +| `S3_BUCKET`, `S3_ACCESS_KEY`, `S3_SECRET_KEY` | Приватный bucket и ограниченная учётная запись приложения | +| `RABBITMQ_USER`, `RABBITMQ_PASSWORD` | Подключение к брокеру | +| `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS` | Бюджет попыток и длительность lease | +| `UPLOAD_MAX_CONCURRENT` | Ограничение одновременных upload на API-реплику | +| `ATTACHMENT_CLEANUP_GRACE_SECONDS` | Защитный интервал перед очисткой | +| `ATTACHMENT_CLEANUP_INTERVAL_MS`, `ATTACHMENT_CLEANUP_INITIAL_DELAY_MS` | Расписание очистки | +| `STAGING_RETENTION_SECONDS` | Срок сохранения исходников неуспешных задач | + +`WORKER_ENABLED` задаётся отдельно для API и worker в Compose. + +`FAULT_INJECTION_ENABLED` отключён в обычной конфигурации и включается только тестовым Compose-файлом. В тестах используются сокращённые lease и grace period; их нельзя выдавать за обычные эксплуатационные настройки. ## Доказательства критериев +Указание «вывод терминала» означает наблюдавшийся результат. Такой вывод необходимо сохранить и приложить к репозиторию отдельно; наличие файла лога не предполагается автоматически. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Структура skeleton и локальные ссылки | `make check` | `PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.` | Вывод терминала; отдельный лог пока не приложен | +| Валидность OpenAPI | `make validate` | `PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.` | Вывод терминала; отдельный лог пока не приложен | +| Публичный сценарий API | `make test` | `PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.` | Вывод терминала; отдельный лог пока не приложен | +| Устойчивое принятие задания после ответа upload | `level4_check.py`, сценарий `commit_before_publish` | После реального падения API принятое задание с тем же ID достигло `ready` | `results/lab5-level4.json` | +| Отдельные broker и worker | `docker compose ps -a` | RabbitMQ и worker работают отдельными контейнерами | `compose.yml`, вывод терминала | +| Статусы и готовность вложения | `make test`, `level4_check.py` | Проверены ожидание `ready`, конечный `failed` и восстановление через replay | `results/lab5-level4.json`, вывод smoke | +| Неизменность оригинала и корректность вариантов | `level4_check.py` | Оригинал совпадает побайтно; проверены размеры, MIME, размер файла и SHA-256 вариантов | `results/lab5-level4.json` | +| Ограниченные повторы | Сценарии `permanent_error`, `transient_error` | Постоянная ошибка завершилась после одной попытки, временная — после трёх | `results/lab5-level4.json` | +| Worker остановлен, API и брокер перезапущены с сохранением volumes | Отдельный сценарий уровня 3 | В приведённых результатах ещё не подтверждён | Требуется отдельное доказательство | +| Crop 128×128 и optimized WebP | `level4_check.py` | Для fixture получены crop 128×128 и WebP 320×200 | `results/lab5-level4.json` | +| Ориентация, прозрачность, центрирование, повреждённые и огромные изображения | `docker compose exec -T worker /opt/images/bin/python /app/selftest.py` | PASS для alpha, small, orientation, centered crop; отклонены broken, bytes_limit, pixels before decode | Вывод selftest; отдельный лог пока не приложен | +| Ограничение времени обработки | Сценарий `image_timeout` | Зависший дочерний процесс остановлен; повторная попытка завершилась успешно | `results/lab5-level4.json` | +| Ограничение памяти worker | Конфигурация контейнера и обработчика | Заданы лимиты контейнера, JVM и дочернего процесса; фактический пик памяти отдельно не измерен | `compose.override.yml`, `infra/images/process_image.py` | +| Outbox и crash между COMMIT и publish | Сценарий `commit_before_publish` | Принятое задание восстановлено без ручного создания нового | `results/lab5-level4.json` | +| Backoff с jitter | Сценарий `transient_error` | Задержки повторов попали в предусмотренные границы | `results/lab5-level4.json` | +| Безопасный replay | Сценарии постоянной и временной ошибки | Failed-задача восстановлена; replay готовой задачи не изменил результат | `results/lab5-level4.json` | +| Падение после записи объектов | Сценарий `after_objects` | Объекты существовали до фиксации результата; после восстановления задача завершилась | `results/lab5-level4.json` | +| Падение перед ACK | Сценарий `before_ack` | Повторная доставка подтверждена; ID задания, число попыток и результат не изменились относительно готового состояния до падения | `results/lab5-level4.json` | +| Восстановление истёкшего lease | Сценарий `after_objects` | После остановки worker задание повторно обработано | `results/lab5-level4.json` | +| Ограничение конкуренции и prefetch | Настройки listener | Concurrency 1 и prefetch 1 | `java/src/main/resources/application.yml` | +| Безопасная очистка staging | Сценарий `after_objects` | Staging и объекты незавершённой попытки удалены; подтверждённые результаты доступны | `results/lab5-level4.json` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Методика -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Проверки выполнялись на локальном стенде Docker Desktop: -## Ограничения и следующие шаги +- две API-реплики; +- один worker; +- один RabbitMQ; +- PostgreSQL, Redis и MinIO с persistent volumes; +- HAProxy перед API. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Проверялись восстановление принятого задания после падения API, повторная доставка после остановки worker, ограниченные retry, безопасный replay, таймаут обработки и очистка staging. \ No newline at end of file diff --git a/compose.faults.yml b/compose.faults.yml new file mode 100644 index 0000000..739b3f5 --- /dev/null +++ b/compose.faults.yml @@ -0,0 +1,15 @@ +services: + api: + environment: + FAULT_INJECTION_ENABLED: "true" + JOB_MAX_ATTEMPTS: "3" + JOB_TIMEOUT_SECONDS: "45" + ATTACHMENT_CLEANUP_GRACE_SECONDS: "90" + ATTACHMENT_CLEANUP_INTERVAL_MS: "1000" + + worker: + environment: + FAULT_INJECTION_ENABLED: "true" + JOB_MAX_ATTEMPTS: "3" + JOB_TIMEOUT_SECONDS: "45" + ATTACHMENT_CLEANUP_GRACE_SECONDS: "90" \ No newline at end of file diff --git a/compose.override.yml b/compose.override.yml new file mode 100644 index 0000000..2f08898 --- /dev/null +++ b/compose.override.yml @@ -0,0 +1,13 @@ +services: + api: + environment: + FAULT_INJECTION_ENABLED: "false" + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-240} + + worker: + mem_limit: 1536m + + environment: + FAULT_INJECTION_ENABLED: "false" + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-240} + JAVA_TOOL_OPTIONS: "-Djava.awt.headless=true -Xmx256m" \ No newline at end of file diff --git a/compose.yml b/compose.yml index 6a62b32..b0d9e21 100644 --- a/compose.yml +++ b/compose.yml @@ -1,18 +1,22 @@ services: postgres: image: postgres:16.6-alpine + environment: POSTGRES_DB: ${POSTGRES_DB:?} POSTGRES_USER: ${POSTGRES_USER:?} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?} + command: - postgres - -c - max_connections=100 - -c - shared_buffers=128MB + volumes: - postgres_data:/var/lib/postgresql/data + healthcheck: test: - CMD-SHELL @@ -27,8 +31,10 @@ services: redis: image: redis:7.4.1-alpine + environment: REDIS_PASSWORD: ${REDIS_PASSWORD:?} + command: - sh - -c @@ -39,8 +45,10 @@ services: --appendfsync always \ --maxmemory 128mb \ --maxmemory-policy noeviction + volumes: - redis_data:/data + healthcheck: test: - CMD-SHELL @@ -48,97 +56,124 @@ services: interval: 2s timeout: 1s retries: 30 + cpus: 0.5 mem_limit: 192m networks: [data] minio: - build: - context: ./infra/minio - dockerfile: Dockerfile + build: + context: ./infra/minio + dockerfile: Dockerfile - image: mtusi4-minio:2025-04-22-local - pull_policy: build + image: mtusi5-minio:2025-04-22-local - command: - - server - - /data - - --console-address - - ":9001" + command: + - server + - /data + - --console-address + - ":9001" - environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:?} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?} - MINIO_API_STALE_UPLOADS_EXPIRY: "24h" - MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL: "1h" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?} + MINIO_API_STALE_UPLOADS_EXPIRY: "24h" + MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL: "1h" - volumes: - - minio_data:/data + volumes: + - minio_data:/data - ports: - - "127.0.0.1:9001:9001" + ports: + - "127.0.0.1:9001:9001" - healthcheck: - test: - - CMD - - curl - - -f - - http://localhost:9000/minio/health/live - interval: 5s - timeout: 3s - retries: 20 - start_period: 10s + healthcheck: + test: + - CMD + - curl + - -f + - http://localhost:9000/minio/health/live + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s - cpus: 1.0 - mem_limit: 768m - restart: unless-stopped - networks: [data] + cpus: 1.0 + mem_limit: 768m + restart: unless-stopped + networks: [data] minio-init: - build: - context: ./infra/minio - dockerfile: Dockerfile + image: mtusi5-minio:2025-04-22-local + pull_policy: never - image: mtusi4-minio:2025-04-22-local - pull_policy: build + depends_on: + minio: + condition: service_healthy - depends_on: - minio: - condition: service_healthy + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?} + S3_BUCKET: ${S3_BUCKET:?} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?} + S3_SECRET_KEY: ${S3_SECRET_KEY:?} - environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:?} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?} + volumes: + - ./infra/minio/bootstrap.sh:/bootstrap/bootstrap.sh:ro + - ./infra/minio/check-access.sh:/bootstrap/check-access.sh:ro - S3_BUCKET: ${S3_BUCKET:?} - S3_ACCESS_KEY: ${S3_ACCESS_KEY:?} - S3_SECRET_KEY: ${S3_SECRET_KEY:?} + entrypoint: + - /bin/sh + - /bootstrap/bootstrap.sh - volumes: - - ./infra/minio/bootstrap.sh:/bootstrap/bootstrap.sh:ro - - ./infra/minio/check-access.sh:/bootstrap/check-access.sh:ro + restart: "no" + networks: [data] - entrypoint: - - /bin/sh - - /bootstrap/bootstrap.sh + rabbitmq: + image: rabbitmq:4.1.4-alpine + hostname: rabbitmq - restart: "no" - networks: [data] + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:?} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?} + + volumes: + - rabbitmq_data:/var/lib/rabbitmq + + healthcheck: + test: + - CMD + - rabbitmq-diagnostics + - -q + - ping + interval: 5s + timeout: 5s + retries: 30 + start_period: 30s + + cpus: 1.0 + mem_limit: 512m + restart: unless-stopped + networks: [data] migrate: image: flyway/flyway:10.20.1-alpine + command: - -connectRetries=30 - migrate + environment: FLYWAY_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:?} FLYWAY_USER: ${POSTGRES_USER:?} FLYWAY_PASSWORD: ${POSTGRES_PASSWORD:?} + volumes: - ./java/src/main/resources/db/migration:/flyway/sql:ro + depends_on: postgres: condition: service_healthy + cpus: 0.5 mem_limit: 384m restart: "no" @@ -146,26 +181,45 @@ services: api: build: . - image: ${APP_IMAGE:-mtusi4-api:local} - environment: + image: ${APP_IMAGE:-mtusi5-api:local} + + environment: &app-environment DB_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:?} DB_USERNAME: ${POSTGRES_USER:?} DB_PASSWORD: ${POSTGRES_PASSWORD:?} + DB_POOL_SIZE: ${DB_POOL_SIZE:-8} + REDIS_HOST: redis REDIS_PASSWORD: ${REDIS_PASSWORD:?} - DB_POOL_SIZE: ${DB_POOL_SIZE:-8} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + FLYWAY_ENABLED: "false" + S3_ENDPOINT: http://minio:9000 S3_ACCESS_KEY: ${S3_ACCESS_KEY:?} S3_SECRET_KEY: ${S3_SECRET_KEY:?} S3_BUCKET: ${S3_BUCKET:?} + + RABBITMQ_HOST: rabbitmq + RABBITMQ_USER: ${RABBITMQ_USER:?} + RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:?} + + WORKER_ENABLED: "false" + + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-3} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-120} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-604800} + UPLOAD_MAX_CONCURRENT: ${UPLOAD_MAX_CONCURRENT:-1} UPLOAD_TEMP_DIR: /tmp/messenger-uploads + ATTACHMENT_CLEANUP_ENABLED: "true" ATTACHMENT_CLEANUP_GRACE_SECONDS: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:-3600} ATTACHMENT_CLEANUP_INTERVAL_MS: ${ATTACHMENT_CLEANUP_INTERVAL_MS:-300000} ATTACHMENT_CLEANUP_INITIAL_DELAY_MS: ${ATTACHMENT_CLEANUP_INITIAL_DELAY_MS:-60000} - + + JAVA_TOOL_OPTIONS: "-Djava.awt.headless=true" + depends_on: migrate: condition: service_completed_successfully @@ -173,10 +227,14 @@ services: condition: service_healthy minio-init: condition: service_completed_successfully + deploy: replicas: ${API_REPLICAS:-2} + cpus: ${API_CPUS:-1.0} mem_limit: ${API_MEMORY:-768m} + + restart: unless-stopped stop_grace_period: 20s healthcheck: @@ -189,14 +247,56 @@ services: timeout: 15s start_period: 60s retries: 3 + networks: [edge, data] + worker: + image: ${APP_IMAGE:-mtusi5-api:local} + pull_policy: never + + environment: + <<: *app-environment + WORKER_ENABLED: "true" + ATTACHMENT_CLEANUP_ENABLED: "false" + + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + minio-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + + cpus: 1.0 + mem_limit: 768m + + restart: unless-stopped + stop_grace_period: 20s + + healthcheck: + test: + - CMD + - wget + - -qO- + - http://localhost:8080/health/live + interval: 5s + timeout: 5s + start_period: 60s + retries: 10 + + networks: [data] + lb: image: haproxy:3.0.5-alpine + ports: - "127.0.0.1:${API_PORT:-8080}:8080" + volumes: - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: api: condition: service_healthy @@ -209,6 +309,7 @@ volumes: postgres_data: redis_data: minio_data: + rabbitmq_data: networks: edge: diff --git a/infra/images/process_image.py b/infra/images/process_image.py new file mode 100644 index 0000000..3498913 --- /dev/null +++ b/infra/images/process_image.py @@ -0,0 +1,176 @@ +import json +import os +from pathlib import Path +import resource +import sys +import time +import warnings + +from PIL import Image, ImageFile, ImageOps, UnidentifiedImageError + + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 + + +def fit_size(width, height, bound): + denominator = max(bound, width, height) + + return ( + max(1, width * bound // denominator), + max(1, height * bound // denominator), + ) + + +def white_rgb(image): + rgba = image.convert("RGBA") + output = Image.new("RGB", rgba.size, "white") + + output.paste( + rgba, + mask=rgba.getchannel("A"), + ) + + rgba.close() + return output + + +def main(): + # Ограничения дочернего процесса Linux. + resource.setrlimit( + resource.RLIMIT_AS, + (768 * 1024**2,) * 2, + ) + + resource.setrlimit( + resource.RLIMIT_CPU, + (20, 21), + ) + + resource.setrlimit( + resource.RLIMIT_FSIZE, + (11 * 1024**2,) * 2, + ) + + Image.MAX_IMAGE_PIXELS = MAX_PIXELS + ImageFile.LOAD_TRUNCATED_IMAGES = False + + warnings.simplefilter( + "error", + Image.DecompressionBombWarning, + ) + + source, output = map(Path, sys.argv[1:3]) + + if not 0 < source.stat().st_size <= MAX_BYTES: + return 10 + + if os.environ.get("IMAGE_TEST_SLEEP") == "1": + time.sleep(60) + + try: + image = Image.open(source) + + if ( + image.format not in ("PNG", "JPEG") + or image.width * image.height > MAX_PIXELS + ): + image.close() + return 10 + + image.load() + ImageOps.exif_transpose(image, in_place=True) + + except ( + ValueError, + UnidentifiedImageError, + OSError, + SyntaxError, + Image.DecompressionBombWarning, + Image.DecompressionBombError, + ): + return 10 + + with image: + manifest = [] + + variants = [ + ("thumbnail", 256, "image/jpeg", "jpg"), + ("crop", 128, "image/jpeg", "jpg"), + ("optimized", 1280, "image/webp", "webp"), + ] + + for name, bound, mime, extension in variants: + if name == "crop": + scaled = ImageOps.fit( + image, + (128, 128), + method=Image.Resampling.LANCZOS, + centering=(0.5, 0.5), + ) + else: + scaled = image.resize( + fit_size(*image.size, bound), + Image.Resampling.LANCZOS, + ) + + if extension == "jpg": + rendered = white_rgb(scaled) + else: + rendered = scaled.convert("RGBA") + + scaled.close() + + rendered.info.clear() + + path = output / (name + "." + extension) + + if extension == "jpg": + rendered.save( + path, + "JPEG", + quality=85, + subsampling=2, + optimize=True, + exif=b"", + icc_profile=None, + ) + else: + rendered.save( + path, + "WEBP", + quality=80, + method=4, + lossless=False, + exif=b"", + icc_profile=b"", + xmp=b"", + ) + + manifest.append( + { + "name": name, + "content_type": mime, + "width": rendered.width, + "height": rendered.height, + } + ) + + rendered.close() + + if not 0 < path.stat().st_size <= MAX_BYTES: + return 11 + + (output / "manifest.json").write_text( + json.dumps(manifest), + encoding="utf-8", + ) + + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (MemoryError, OSError): + sys.exit(11) \ No newline at end of file diff --git a/infra/images/selftest.py b/infra/images/selftest.py new file mode 100644 index 0000000..8f5ea0f --- /dev/null +++ b/infra/images/selftest.py @@ -0,0 +1,184 @@ +from pathlib import Path +import struct +import subprocess +import sys +import tempfile +import zlib + +from PIL import Image + + +SCRIPT = Path(__file__).with_name("process_image.py") + + +def execute(folder, expected): + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(folder / "input.bin"), + str(folder), + ], + capture_output=True, + timeout=30, + ) + + assert result.returncode == expected, ( + folder.name, + result.returncode, + ) + + +def main(): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + + cases = [ + ( + "alpha", + (320, 200), + "RGBA", + (255, 0, 0, 0), + None, + [(256, 160), (128, 128), (320, 200)], + ), + ( + "small", + (20, 10), + "RGB", + "red", + None, + [(20, 10), (128, 128), (20, 10)], + ), + ( + "orientation", + (320, 200), + "RGB", + "blue", + 6, + [(160, 256), (128, 128), (200, 320)], + ), + ] + + for label, size, mode, color, orientation, expected in cases: + folder = root / label + folder.mkdir() + + with Image.new(mode, size, color) as image: + if orientation: + exif = Image.Exif() + exif[274] = orientation + + image.save( + folder / "input.bin", + "JPEG", + exif=exif, + ) + else: + image.save(folder / "input.bin", "PNG") + + execute(folder, 0) + + names = [ + "thumbnail.jpg", + "crop.jpg", + "optimized.webp", + ] + + for name, dimensions in zip(names, expected): + with Image.open(folder / name) as result: + assert result.size == dimensions + assert not result.getexif() + + assert not any( + result.info.get(key) + for key in ["icc_profile", "xmp", "exif"] + ) + + if label == "alpha" and name.endswith(".jpg"): + assert result.getpixel((0, 0)) == ( + 255, + 255, + 255, + ) + + print("PASS:", label) + + folder = root / "center_crop" + folder.mkdir() + + with Image.new("RGB", (400, 200), "blue") as image: + image.paste("red", (0, 0, 100, 200)) + image.paste("green", (300, 0, 400, 200)) + image.save(folder / "input.bin", "PNG") + + execute(folder, 0) + + with Image.open(folder / "crop.jpg") as crop: + for x in [32, 96]: + red, green, blue = crop.getpixel((x, 64)) + + assert blue > 240 + assert red < 15 + assert green < 15 + + print("PASS: centered crop") + + for label, data in [ + ("broken", b"not an image"), + ("bytes_limit", b"x" * (10485760 + 1)), + ]: + folder = root / label + folder.mkdir() + + (folder / "input.bin").write_bytes(data) + + execute(folder, 10) + + print("PASS rejected:", label) + + folder = root / "pixels" + folder.mkdir() + + def chunk(kind, data): + return ( + struct.pack(">I", len(data)) + + kind + + data + + struct.pack( + ">I", + zlib.crc32(kind + data) & 0xffffffff, + ) + ) + + png = ( + b"\x89PNG\r\n\x1a\n" + + chunk( + b"IHDR", + struct.pack( + ">IIBBBBB", + 10000, + 10000, + 8, + 2, + 0, + 0, + 0, + ), + ) + + chunk( + b"IDAT", + zlib.compress(b"\0\0\0\0"), + ) + + chunk(b"IEND", b"") + ) + + (folder / "input.bin").write_bytes(png) + + execute(folder, 10) + + print("PASS rejected: pixels before decode") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh index b904eb2..d142a58 100644 --- a/infra/minio/bootstrap.sh +++ b/infra/minio/bootstrap.sh @@ -28,7 +28,6 @@ mc alias set storage http://minio:9000 \ "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null mc mb --ignore-existing "storage/$S3_BUCKET" - mc anonymous set none "storage/$S3_BUCKET" cat > /tmp/attachments-policy.json < /tmp/attachments-policy.json < /tmp/attachments-policy.json </dev/null -echo "Bootstrap completed: private bucket and API user are ready" \ No newline at end of file +echo "Bootstrap completed: private bucket and application user are ready" \ No newline at end of file diff --git a/java/pom.xml b/java/pom.xml index cce1d2b..8a23348 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -103,6 +103,17 @@ 8.5.17 + + org.springframework.boot + spring-boot-starter-amqp + + + + com.drewnoakes + metadata-extractor + 2.19.0 + + diff --git a/java/src/main/java/com/vnp/demo/config/RabbitConfig.java b/java/src/main/java/com/vnp/demo/config/RabbitConfig.java new file mode 100644 index 0000000..29c087f --- /dev/null +++ b/java/src/main/java/com/vnp/demo/config/RabbitConfig.java @@ -0,0 +1,26 @@ +package com.vnp.demo.config; + +import org.springframework.amqp.core.Declarables; +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.core.QueueBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class RabbitConfig { + + public static final String QUEUE = "attachments.v1"; + public static final String DLQ = "attachments.v1.dlq"; + + @Bean + public Declarables attachmentQueues() { + return new Declarables( + new Queue(DLQ, true), + + QueueBuilder.durable(QUEUE) + .deadLetterExchange("") + .deadLetterRoutingKey(DLQ) + .build() + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/controllers/AttachmentController.java b/java/src/main/java/com/vnp/demo/controllers/AttachmentController.java index 5d1bf48..a6f402d 100644 --- a/java/src/main/java/com/vnp/demo/controllers/AttachmentController.java +++ b/java/src/main/java/com/vnp/demo/controllers/AttachmentController.java @@ -37,7 +37,7 @@ public class AttachmentController { } @PostMapping("/chats/{chat_id}/attachments") - @ResponseStatus(HttpStatus.CREATED) + @ResponseStatus(HttpStatus.ACCEPTED) public AttachmentResponse upload( @AuthenticationPrincipal UUID userId, @PathVariable("chat_id") UUID chatId, diff --git a/java/src/main/java/com/vnp/demo/dto/AttachmentJobMessage.java b/java/src/main/java/com/vnp/demo/dto/AttachmentJobMessage.java new file mode 100644 index 0000000..c6bc2f8 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/dto/AttachmentJobMessage.java @@ -0,0 +1,81 @@ +package com.vnp.demo.dto; + +import java.time.Instant; +import java.util.Objects; +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record AttachmentJobMessage( + @JsonProperty("schema_version") + int schemaVersion, + + @JsonProperty("job_id") + UUID jobId, + + String type, + + @JsonProperty("attachment_id") + UUID attachmentId, + + @JsonProperty("pipeline_version") + int pipelineVersion, + + int attempt, + + @JsonProperty("created_at") + Instant createdAt +) { + public static final int SCHEMA_VERSION = 1; + + public static final String TYPE = + "attachment.process.v1"; + + public AttachmentJobMessage { + if (schemaVersion != SCHEMA_VERSION) { + throw new IllegalArgumentException( + "Unsupported job schema version" + ); + } + + if (!TYPE.equals(type)) { + throw new IllegalArgumentException( + "Unsupported job type" + ); + } + + Objects.requireNonNull(jobId, "jobId"); + Objects.requireNonNull(attachmentId, "attachmentId"); + Objects.requireNonNull(createdAt, "createdAt"); + + if (pipelineVersion < 1) { + throw new IllegalArgumentException( + "pipelineVersion must be positive" + ); + } + + if (attempt < 1) { + throw new IllegalArgumentException( + "attempt must be positive" + ); + } + } + + public static AttachmentJobMessage create( + UUID jobId, + UUID attachmentId, + int pipelineVersion, + int attempt, + Instant createdAt + ) { + return new AttachmentJobMessage( + SCHEMA_VERSION, + jobId, + TYPE, + attachmentId, + pipelineVersion, + attempt, + createdAt + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/AttachmentCleanupJob.java b/java/src/main/java/com/vnp/demo/service/AttachmentCleanupJob.java index 0c9a40f..affc526 100644 --- a/java/src/main/java/com/vnp/demo/service/AttachmentCleanupJob.java +++ b/java/src/main/java/com/vnp/demo/service/AttachmentCleanupJob.java @@ -1,19 +1,11 @@ package com.vnp.demo.service; -import java.time.Instant; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import com.vnp.demo.service.AttachmentPersistenceService.CleanupResult; - -import io.minio.Result; -import io.minio.messages.Item; - @Component @ConditionalOnProperty( name = "app.cleanup.enabled", @@ -25,94 +17,24 @@ public class AttachmentCleanupJob { private static final Logger log = LoggerFactory.getLogger(AttachmentCleanupJob.class); - private final S3StorageService storage; - private final AttachmentPersistenceService persistence; - private final long graceSeconds; + private final AttachmentPipeline pipeline; - public AttachmentCleanupJob( - S3StorageService storage, - AttachmentPersistenceService persistence, - @Value("${app.cleanup.grace-seconds:3600}") - long graceSeconds - ) { - if (graceSeconds < 60) { - throw new IllegalArgumentException( - "Grace period должен быть не меньше 60 секунд" - ); - } - - this.storage = storage; - this.persistence = persistence; - this.graceSeconds = graceSeconds; + public AttachmentCleanupJob(AttachmentPipeline pipeline) { + this.pipeline = pipeline; } @Scheduled( - initialDelayString = - "${app.cleanup.initial-delay-ms:60000}", - fixedDelayString = - "${app.cleanup.interval-ms:300000}" + initialDelayString = "${app.cleanup.initial-delay-ms:60000}", + fixedDelayString = "${app.cleanup.interval-ms:300000}" ) public void cleanup() { - Instant cutoff = Instant.now() - .minusSeconds(graceSeconds); - - long scanned = 0; - long candidates = 0; - long deleted = 0; - try { - for (Result result : storage.listOriginals()) { - Item item = result.get(); + pipeline.cleanup(); - if (item.isDir()) { - continue; - } - - scanned++; - - Instant lastModified = item.lastModified() - .toInstant(); - - if (!lastModified.isBefore(cutoff)) { - continue; - } - - candidates++; - - CleanupResult outcome = - persistence.removeIfOrphan( - item.objectName(), - cutoff - ); - - if (outcome == CleanupResult.DELETED) { - deleted++; - - log.info( - "Attachment cleanup removed orphan: {}", - item.objectName() - ); - } - } - - log.info( - "Attachment cleanup completed: " - + "scanned={}, candidates={}, deleted={}", - scanned, - candidates, - deleted - ); - } catch (Exception e) { - // Прерываем текущий проход при ошибке БД или S3. - // Следующий запуск повторит проверку. - // Не выводим credentials или содержимое ответа S3. + } catch (Exception error) { log.warn( - "Attachment cleanup interrupted: " - + "scanned={}, candidates={}, deleted={}, errorType={}", - scanned, - candidates, - deleted, - e.getClass().getSimpleName() + "Attachment cleanup interrupted: {}", + error.getClass().getSimpleName() ); } } diff --git a/java/src/main/java/com/vnp/demo/service/AttachmentPipeline.java b/java/src/main/java/com/vnp/demo/service/AttachmentPipeline.java new file mode 100644 index 0000000..5fe8143 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/AttachmentPipeline.java @@ -0,0 +1,1384 @@ +package com.vnp.demo.service; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +import com.rabbitmq.client.Channel; + +import com.vnp.demo.BusinessException; +import com.vnp.demo.config.RabbitConfig; +import com.vnp.demo.dto.AttachmentJobMessage; +import com.vnp.demo.dto.AttachmentResponse; +import com.vnp.demo.models.Attachment; +import com.vnp.demo.repositories.AttachmentRepository; +import com.vnp.demo.validation.ImageValidator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.amqp.core.Message; +import org.springframework.amqp.core.MessageDeliveryMode; +import org.springframework.amqp.core.MessageProperties; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.amqp.rabbit.connection.CorrelationData; +import org.springframework.amqp.rabbit.core.RabbitTemplate; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.io.IOException; +import java.nio.file.Path; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Service +public class AttachmentPipeline { + + private static final Logger log = + LoggerFactory.getLogger(AttachmentPipeline.class); + + private static final int PIPELINE_VERSION = 1; + + private static final Set REQUIRED_VARIANTS = + Set.of("thumbnail", "crop", "optimized"); + + private final JdbcTemplate jdbc; + private final AttachmentRepository attachments; + private final S3StorageService storage; + private final RabbitTemplate rabbit; + private final ObjectMapper mapper; + + private final ImageProcessor processor; + private final FaultGate faults; // LAB5_LEVEL4 + + private final TransactionTemplate tx; + + private final boolean worker; + private final int maxAttempts; + private final int leaseSeconds; + private final long graceSeconds; + private final long failedRetention; + + public AttachmentPipeline( + JdbcTemplate jdbc, + AttachmentRepository attachments, + S3StorageService storage, + RabbitTemplate rabbit, + ObjectMapper mapper, + ImageProcessor processor, + FaultGate faults, + PlatformTransactionManager manager, + + @Value("${app.worker.enabled:false}") + boolean worker, + + @Value("${app.jobs.max-attempts:3}") + int maxAttempts, + + @Value("${app.jobs.lease-seconds:240}") + int leaseSeconds, + + @Value("${app.cleanup.grace-seconds:3600}") + long graceSeconds, + + @Value("${app.jobs.failed-retention-seconds:604800}") + long failedRetention + ) { + if (maxAttempts < 1 + || leaseSeconds < 30 + || graceSeconds < 2L * leaseSeconds + || failedRetention < graceSeconds) { + throw new IllegalArgumentException( + "Invalid job/cleanup limits" + ); + } + + this.jdbc = jdbc; + this.attachments = attachments; + this.storage = storage; + this.rabbit = rabbit; + this.mapper = mapper; + + this.processor = processor; + this.faults = faults; + + this.worker = worker; + this.maxAttempts = maxAttempts; + this.leaseSeconds = leaseSeconds; + this.graceSeconds = graceSeconds; + this.failedRetention = failedRetention; + + this.tx = new TransactionTemplate(manager); + this.tx.setTimeout(60); + } + + public record Variant( + String name, + + @JsonProperty("content_type") + String contentType, + + @JsonProperty("size_bytes") + long sizeBytes, + + int width, + int height, + String sha256, + + @JsonIgnore + String objectKey + ) { + } + + private record Job( + UUID id, + UUID attachmentId, + int version, + String status, + int attempts, + int limit, + UUID owner, + boolean validLease, + boolean due + ) { + } + + private record Work( + Job job, + Attachment attachment, + String stagingKey, + UUID owner + ) { + } + + private record Publication( + UUID id, + AttachmentJobMessage body + ) { + } + + public void enqueue(Attachment attachment, Path file) { + if (worker) { + throw new BusinessException( + "unavailable", + "Worker does not accept uploads", + HttpStatus.SERVICE_UNAVAILABLE + ); + } + + String staging = "staging/" + + attachment.chatId() + + "/" + + attachment.id(); + + tx.executeWithoutResult(ignored -> { + /* + * Очистка использует такую же блокировку. + * Она не удалит staging между S3 PUT и COMMIT. + */ + attachments.lockObject(staging); + + storage.put( + staging, + file, + attachment.sizeBytes(), + attachment.contentType() + ); + + attachments.save(attachment); + + jdbc.update( + """ + UPDATE attachments + SET staging_key = ?, + pipeline_version = ? + WHERE id = ? + """, + staging, + PIPELINE_VERSION, + attachment.id() + ); + + UUID jobId = UUID.randomUUID(); + + jdbc.update( + """ + INSERT INTO attachment_jobs( + id, + attachment_id, + pipeline_version, + max_attempts + ) + VALUES (?, ?, ?, ?) + """, + jobId, + attachment.id(), + PIPELINE_VERSION, + maxAttempts + ); + + /* + * Метаданные, задание и outbox фиксируются + * в одной транзакции PostgreSQL. + */ + addOutbox(jobId, 1); + }); + } + + public List variants(UUID attachmentId) { + return jdbc.query( + """ + SELECT v.* + FROM attachment_variants v + JOIN attachments a + ON a.id = v.attachment_id + AND a.pipeline_version = v.pipeline_version + WHERE a.id = ? + ORDER BY v.name + """, + (rs, rowNumber) -> new Variant( + rs.getString("name"), + rs.getString("content_type"), + rs.getLong("size_bytes"), + rs.getInt("width"), + rs.getInt("height"), + rs.getString("sha256"), + rs.getString("object_key") + ), + attachmentId + ); + } + + public AttachmentResponse metadata(Attachment attachment) { + return tx.execute(ignored -> { + /* + * Читаем status, error_code и варианты согласованно. + * Worker или replay не изменят строку посреди чтения. + */ + List locked = jdbc.query( + """ + SELECT id + FROM attachments + WHERE id = ? + FOR SHARE + """, + (rs, rowNumber) -> + rs.getObject("id", UUID.class), + attachment.id() + ); + + if (locked.isEmpty()) { + throw BusinessException.notFound(); + } + + Attachment current = attachments + .findById(attachment.id()) + .orElseThrow(BusinessException::notFound); + + String error = jdbc.queryForObject( + """ + SELECT error_code + FROM attachments + WHERE id = ? + """, + String.class, + current.id() + ); + + List responseVariants = + "ready".equals(current.status()) + ? new ArrayList( + variants(current.id()) + ) + : List.of(); + + return new AttachmentResponse( + current.id(), + current.chatId(), + current.ownerId(), + current.filename(), + current.contentType(), + current.sizeBytes(), + current.sha256(), + current.status(), + responseVariants, + "failed".equals(current.status()) + ? error + : null, + current.createdAt(), + current.updatedAt() + ); + }); + } + + private void addOutbox(UUID jobId, int attempt) { + jdbc.update( + """ + INSERT INTO attachment_outbox( + id, + job_id, + attempt + ) + VALUES (?, ?, ?) + ON CONFLICT (job_id, attempt) DO NOTHING + """, + UUID.randomUUID(), + jobId, + attempt + ); + } + + @Scheduled( + fixedDelay = 1000, + initialDelay = 1000 + ) + public void publish() { + if (worker) { + return; + } + + try { + for (int i = 0; i < 10; i++) { + Boolean published = tx.execute( + ignored -> publishOne() + ); + + if (!Boolean.TRUE.equals(published)) { + break; + } + } + } catch (Exception error) { + log.warn( + "Outbox unavailable: {}", + error.getClass().getSimpleName() + ); + } + } + + private boolean publishOne() { + List rows = jdbc.query( + """ + SELECT + o.id, + o.attempt, + j.id AS job_id, + j.attachment_id, + j.pipeline_version, + j.created_at + FROM attachment_outbox o + JOIN attachment_jobs j ON j.id = o.job_id + WHERE o.published_at IS NULL + AND o.next_publish_at <= clock_timestamp() + AND j.status = 'queued' + AND j.available_at <= clock_timestamp() + AND o.attempt = j.attempts + 1 + ORDER BY o.next_publish_at, o.id + LIMIT 1 + FOR UPDATE OF o SKIP LOCKED + """, + (rs, rowNumber) -> new Publication( + rs.getObject("id", UUID.class), + AttachmentJobMessage.create( + rs.getObject("job_id", UUID.class), + rs.getObject( + "attachment_id", + UUID.class + ), + rs.getInt("pipeline_version"), + rs.getInt("attempt"), + rs.getTimestamp("created_at") + .toInstant() + ) + ) + ); + + if (rows.isEmpty()) { + return false; + } + + Publication publication = rows.getFirst(); + + /* + * В обычном режиме paused() всегда возвращает false. + * Тест может удержать публикацию до получения HTTP 202. + */ + if (faults.paused()) { + return false; + } + + /* + * Задание уже сохранено предыдущей транзакцией. + * Здесь тест завершает API перед отправкой в RabbitMQ. + */ + faults.hit( + publication.body().jobId(), + "before_publish" + ); + + boolean confirmed = false; + + try { + MessageProperties properties = + new MessageProperties(); + + properties.setContentType("application/json"); + + properties.setDeliveryMode( + MessageDeliveryMode.PERSISTENT + ); + + properties.setMessageId( + publication.id().toString() + ); + + CorrelationData correlation = new CorrelationData( + UUID.randomUUID().toString() + ); + + rabbit.send( + "", + RabbitConfig.QUEUE, + new Message( + mapper.writeValueAsBytes( + publication.body() + ), + properties + ), + correlation + ); + + confirmed = correlation.getFuture() + .get(5, TimeUnit.SECONDS) + .isAck() + && correlation.getReturned() == null; + + } catch (Exception error) { + if (error instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + + log.warn( + "Job publish not confirmed: " + + "job={}, errorType={}", + publication.body().jobId(), + error.getClass().getSimpleName() + ); + } + + if (confirmed) { + jdbc.update( + """ + UPDATE attachment_outbox + SET published_at = clock_timestamp(), + publish_attempts = publish_attempts + 1, + last_error_code = NULL + WHERE id = ? + """, + publication.id() + ); + } else { + jdbc.update( + """ + UPDATE attachment_outbox + SET publish_attempts = publish_attempts + 1, + next_publish_at = + clock_timestamp() + interval '5 seconds', + last_error_code = 'broker_unavailable' + WHERE id = ? + """, + publication.id() + ); + } + + return confirmed; + } + + private Job mapJob(ResultSet rs, int rowNumber) + throws SQLException { + + return new Job( + rs.getObject("id", UUID.class), + rs.getObject("attachment_id", UUID.class), + rs.getInt("pipeline_version"), + rs.getString("status"), + rs.getInt("attempts"), + rs.getInt("max_attempts"), + rs.getObject("lease_owner", UUID.class), + rs.getBoolean("valid_lease"), + rs.getBoolean("due") + ); + } + + private Job lockJob(UUID id) { + return jdbc.query( + """ + SELECT *, + lease_until > clock_timestamp() AS valid_lease, + available_at <= clock_timestamp() AS due + FROM attachment_jobs + WHERE id = ? + FOR UPDATE + """, + this::mapJob, + id + ).stream().findFirst().orElse(null); + } + + private Work claim(AttachmentJobMessage message) { + return tx.execute(ignored -> { + Job job = lockJob(message.jobId()); + + if (job == null + || !job.attachmentId().equals( + message.attachmentId() + ) + || job.version() != message.pipelineVersion() + || job.version() != PIPELINE_VERSION) { + throw new IllegalArgumentException( + "invalid_job" + ); + } + + Attachment attachment = attachments + .findById(job.attachmentId()) + .orElseThrow(); + + if ("ready".equals(job.status())) { + return new Work( + job, + attachment, + null, + null + ); + } + + /* + * Дубликат processing не создаёт новую попытку. + * После смерти владельца задание восстановит recover(). + * + * Старые доставки предыдущих попыток тоже не исполняются. + */ + if (!"queued".equals(job.status()) + || !job.due() + || message.attempt() != job.attempts() + 1) { + return null; + } + + UUID owner = UUID.randomUUID(); + + jdbc.update( + """ + UPDATE attachment_jobs + SET status = 'processing', + attempts = attempts + 1, + lease_owner = ?, + lease_until = clock_timestamp() + + (? * interval '1 second'), + updated_at = clock_timestamp() + WHERE id = ? + """, + owner, + leaseSeconds, + job.id() + ); + + int changed = jdbc.update( + """ + UPDATE attachments + SET status = 'processing', + error_code = NULL, + updated_at = clock_timestamp() + WHERE id = ? + AND pipeline_version = ? + AND status = 'queued' + """, + job.attachmentId(), + job.version() + ); + + if (changed != 1) { + throw new IllegalStateException( + "attachment_state_mismatch" + ); + } + + String staging = jdbc.queryForObject( + """ + SELECT staging_key + FROM attachments + WHERE id = ? + """, + String.class, + job.attachmentId() + ); + + return new Work( + lockJob(job.id()), + attachment, + staging, + owner + ); + }); + } + + private boolean owns(Job current, Work work) { + return current != null + && "processing".equals(current.status()) + && current.validLease() + && work.owner() != null + && work.owner().equals(current.owner()) + && current.attempts() == work.job().attempts() + && current.version() == work.job().version(); + } + + private boolean finish( + Work work, + String originalKey, + List generatedVariants + ) { + Set names = generatedVariants.stream() + .map(Variant::name) + .collect(Collectors.toSet()); + + if (generatedVariants.size() != 3 + || !names.equals(REQUIRED_VARIANTS)) { + throw new IllegalStateException( + "incomplete_variants" + ); + } + + return Boolean.TRUE.equals( + tx.execute(ignored -> { + Job job = lockJob(work.job().id()); + + if (!owns(job, work)) { + return false; + } + + for (Variant variant : generatedVariants) { + jdbc.update( + """ + INSERT INTO attachment_variants( + attachment_id, + pipeline_version, + name, + object_key, + content_type, + size_bytes, + width, + height, + sha256 + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT ( + attachment_id, + pipeline_version, + name + ) + DO UPDATE SET + object_key = EXCLUDED.object_key, + content_type = EXCLUDED.content_type, + size_bytes = EXCLUDED.size_bytes, + width = EXCLUDED.width, + height = EXCLUDED.height, + sha256 = EXCLUDED.sha256 + """, + job.attachmentId(), + job.version(), + variant.name(), + variant.objectKey(), + variant.contentType(), + variant.sizeBytes(), + variant.width(), + variant.height(), + variant.sha256() + ); + } + + int updated = jdbc.update( + """ + UPDATE attachments + SET object_key = ?, + status = 'ready', + error_code = NULL, + updated_at = clock_timestamp() + WHERE id = ? + AND pipeline_version = ? + AND status = 'processing' + """, + originalKey, + job.attachmentId(), + job.version() + ); + + /* + * Повторно проверяем lease непосредственно + * перед завершением транзакции результата. + */ + int completed = jdbc.update( + """ + UPDATE attachment_jobs + SET status = 'ready', + lease_owner = NULL, + lease_until = NULL, + last_error_code = NULL, + updated_at = clock_timestamp() + WHERE id = ? + AND lease_owner = ? + AND lease_until > clock_timestamp() + """, + job.id(), + work.owner() + ); + + if (updated != 1 || completed != 1) { + throw new IllegalStateException( + "lease_or_version_changed" + ); + } + + return true; + }) + ); + } + + private void retryOrFail( + Job job, + String reason, + boolean permanent + ) { + boolean terminal = permanent + || job.attempts() >= job.limit(); + + /* + * Верхняя граница задержки: + * 2, 4, 8, 16, 32, 60, 60... секунд. + * + * Конкретная задержка случайна в диапазоне [1, cap]. + */ + int cap = Math.min( + 60, + 2 << Math.min(5, job.attempts() - 1) + ); + + int delay = terminal + ? 0 + : ThreadLocalRandom.current().nextInt( + 1, + cap + 1 + ); + + String status = terminal ? "failed" : "queued"; + + String code = terminal + ? (permanent ? reason : "retry_exhausted") + : null; + + jdbc.update( + """ + UPDATE attachment_jobs + SET status = ?, + lease_owner = NULL, + lease_until = NULL, + last_error_code = ?, + available_at = clock_timestamp() + + (? * interval '1 second'), + updated_at = clock_timestamp() + WHERE id = ? + """, + status, + reason, + delay, + job.id() + ); + + int updated = jdbc.update( + """ + UPDATE attachments + SET status = ?, + error_code = ?, + updated_at = clock_timestamp() + WHERE id = ? + AND pipeline_version = ? + AND status = 'processing' + """, + status, + code, + job.attachmentId(), + job.version() + ); + + if (updated != 1) { + throw new IllegalStateException( + "attachment_state_mismatch" + ); + } + + jdbc.update( + """ + INSERT INTO attachment_attempt_events( + job_id, + attempt, + state, + error_code, + delay_seconds + ) + VALUES (?, ?, ?, ?, ?) + """, + job.id(), + job.attempts(), + status, + reason, + delay + ); + + if (!terminal) { + addOutbox( + job.id(), + job.attempts() + 1 + ); + } + + log.warn( + "Job transition: job={}, attempt={}, " + + "status={}, reason={}, delay={}", + job.id(), + job.attempts(), + status, + reason, + delay + ); + } + + private void failedAttempt( + Work work, + Exception error + ) { + boolean permanent = + error instanceof ImageProcessor.PermanentFailure; + + String reason; + + if (permanent) { + reason = "invalid_image"; + + } else if ( + error instanceof ImageProcessor.TransientFailure failure + ) { + reason = failure.code(); + + } else { + reason = "processing_unavailable"; + } + + tx.executeWithoutResult(ignored -> { + Job current = lockJob(work.job().id()); + + if (owns(current, work)) { + retryOrFail( + current, + reason, + permanent + ); + } + }); + } + + @Scheduled( + fixedDelay = 5000, + initialDelay = 5000 + ) + public void recover() { + if (worker) { + return; + } + + try { + tx.executeWithoutResult(ignored -> { + List expired = jdbc.query( + """ + SELECT *, + false AS valid_lease, + true AS due + FROM attachment_jobs + WHERE status = 'processing' + AND lease_until <= clock_timestamp() + ORDER BY lease_until + LIMIT 20 + FOR UPDATE SKIP LOCKED + """, + this::mapJob + ); + + for (Job job : expired) { + retryOrFail( + job, + "lease_expired", + false + ); + } + }); + + } catch (Exception error) { + log.warn( + "Job recovery unavailable: {}", + error.getClass().getSimpleName() + ); + } + } + + /* + * Доступен ReplayRunner из того же пакета. + * + * Проверяем не только статус в PostgreSQL, + * но и наличие файлов с ожидаемыми checksum. + */ + void verifyReady(Attachment attachment) { + if (!"ready".equals(attachment.status())) { + throw new IllegalStateException( + "not_ready" + ); + } + + storage.get( + attachment.objectKey(), + attachment.sizeBytes(), + attachment.sha256() + ); + + List items = variants( + attachment.id() + ); + + Set names = items.stream() + .map(Variant::name) + .collect(Collectors.toSet()); + + if (items.size() != 3 + || !names.equals(REQUIRED_VARIANTS)) { + throw new IllegalStateException( + "incomplete_variants" + ); + } + + for (Variant variant : items) { + storage.get( + variant.objectKey(), + variant.sizeBytes(), + variant.sha256() + ); + } + } + + @RabbitListener( + queues = RabbitConfig.QUEUE, + ackMode = "MANUAL", + autoStartup = "${app.worker.enabled:false}" + ) + public void consume( + Message message, + Channel channel + ) throws IOException { + + long tag = message.getMessageProperties() + .getDeliveryTag(); + + AttachmentJobMessage body; + + try { + if (message.getBody().length > 4096) { + throw new IllegalArgumentException( + "job_message_too_large" + ); + } + + body = mapper.readValue( + message.getBody(), + AttachmentJobMessage.class + ); + + if (body == null) { + throw new IllegalArgumentException( + "empty_job_message" + ); + } + + } catch (Exception error) { + log.error( + "Invalid job message sent to DLQ" + ); + + channel.basicReject(tag, false); + return; + } + + Work work; + + try { + work = claim(body); + + } catch (IllegalArgumentException error) { + log.error( + "Unknown/incompatible job sent to DLQ: {}", + body.jobId() + ); + + channel.basicReject(tag, false); + return; + + } catch (Exception error) { + log.warn( + "Cannot claim job: job={}, errorType={}", + body.jobId(), + error.getClass().getSimpleName() + ); + + requeue(channel, tag); + return; + } + + if (work == null) { + /* + * Доставка больше не требует выполнения: + * failed, processing либо старая попытка. + * + * Зависший processing восстанавливается + * через сохранённый lease. + */ + channel.basicAck(tag, false); + return; + } + + if ("ready".equals(work.job().status())) { + try { + verifyReady(work.attachment()); + + } catch (Exception error) { + log.error( + "Ready effect verification failed; " + + "delivery sent to DLQ: {}", + body.jobId() + ); + + channel.basicReject(tag, false); + return; + } + + log.info( + "Duplicate job verified: job={}", + body.jobId() + ); + + channel.basicAck(tag, false); + return; + } + + try { + faults.hit( + body.jobId(), + "permanent_error" + ); + + faults.hit( + body.jobId(), + "transient_error" + ); + + Attachment attachment = work.attachment(); + + byte[] source = storage.get( + work.stagingKey(), + attachment.sizeBytes(), + attachment.sha256() + ); + + boolean testHang = faults.hit( + body.jobId(), + "image_timeout" + ); + + List generated = + processor.render( + source, + testHang + ); + + /* + * Каждая попытка получает собственные S3-ключи. + * + * Запоздавший worker может оставить orphan, + * но не перезапишет результат новой попытки. + * Orphan удалит очистка после grace period. + */ + String suffix = attachment.chatId() + + "/" + + attachment.id() + + "/v" + + work.job().version() + + "/" + + work.owner(); + + String originalKey = "originals/" + suffix; + + storage.putBytes( + originalKey, + source, + attachment.contentType() + ); + + List generatedVariants = + new ArrayList<>(); + + for (ImageProcessor.Generated item : generated) { + ImageProcessor.Info info = item.info(); + + String extension = + "image/webp".equals(info.contentType()) + ? ".webp" + : ".jpg"; + + String key = "variants/" + + suffix + + "/" + + info.name() + + extension; + + storage.putBytes( + key, + item.bytes(), + info.contentType() + ); + + generatedVariants.add( + new Variant( + info.name(), + info.contentType(), + item.bytes().length, + info.width(), + info.height(), + ImageValidator.sha256( + item.bytes() + ), + key + ) + ); + } + + /* + * Все S3 PUT завершились. + * Метаданные результата ещё не закоммичены. + */ + faults.hit( + body.jobId(), + "after_objects" + ); + + boolean completed = finish( + work, + originalKey, + generatedVariants + ); + + if (completed) { + /* + * PostgreSQL уже содержит ready и три варианта, + * но RabbitMQ ещё не получил ACK. + */ + faults.hit( + body.jobId(), + "before_ack" + ); + } else { + log.info( + "Stale attempt ignored: job={}, attempt={}", + body.jobId(), + work.job().attempts() + ); + } + + } catch (Exception error) { + try { + failedAttempt(work, error); + + } catch (Exception databaseError) { + log.warn( + "Cannot record job failure: " + + "job={}, errorType={}", + body.jobId(), + databaseError.getClass() + .getSimpleName() + ); + + requeue(channel, tag); + return; + } + } + + /* + * ACK отправляется после устойчивой фиксации: + * ready, failed или намерения следующей попытки. + * + * При потерянном lease восстановлением занимается recover(). + */ + channel.basicAck(tag, false); + } + + private void requeue( + Channel channel, + long tag + ) throws IOException { + + try { + Thread.sleep(1000); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } + + channel.basicNack( + tag, + false, + true + ); + } + + public void cleanup() throws Exception { + Instant cutoff = Instant.now() + .minusSeconds(graceSeconds); + + long scanned = 0; + long deleted = 0; + + for (String prefix : List.of( + "staging/", + "originals/", + "variants/" + )) { + for (var result : storage.listPrefix(prefix)) { + var item = result.get(); + + if (item.isDir()) { + continue; + } + + scanned++; + + if (!item.lastModified() + .toInstant() + .isBefore(cutoff)) { + continue; + } + + String key = item.objectName(); + + boolean removed = Boolean.TRUE.equals( + tx.execute(ignored -> { + if (!attachments.tryLockObject(key)) { + return false; + } + + /* + * Сохраняем: + * - опубликованные оригиналы; + * - опубликованные варианты; + * - staging активных заданий; + * - staging failed в пределах retention. + */ + Boolean used = jdbc.queryForObject( + """ + SELECT EXISTS( + SELECT 1 + FROM attachments + WHERE object_key = ? + ) + OR EXISTS( + SELECT 1 + FROM attachment_variants + WHERE object_key = ? + ) + OR EXISTS( + SELECT 1 + FROM attachments + WHERE staging_key = ? + AND ( + status IN ( + 'queued', + 'processing' + ) + OR ( + status = 'failed' + AND updated_at > + clock_timestamp() + - ( + ? + * interval '1 second' + ) + ) + ) + ) + """, + Boolean.class, + key, + key, + key, + failedRetention + ); + + if (Boolean.TRUE.equals(used)) { + return false; + } + + Instant modified = + storage.lastModifiedOrNull(key); + + if (modified == null + || !modified.isBefore(cutoff)) { + return false; + } + + storage.remove(key); + + jdbc.update( + """ + UPDATE attachments + SET staging_key = NULL + WHERE staging_key = ? + AND status IN ( + 'ready', + 'failed' + ) + """, + key + ); + + return true; + }) + ); + + if (removed) { + deleted++; + } + } + } + + log.info( + "Attachment cleanup completed: " + + "scanned={}, deleted={}", + scanned, + deleted + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/AttachmentService.java b/java/src/main/java/com/vnp/demo/service/AttachmentService.java index 9af595c..1db2bc8 100644 --- a/java/src/main/java/com/vnp/demo/service/AttachmentService.java +++ b/java/src/main/java/com/vnp/demo/service/AttachmentService.java @@ -13,101 +13,95 @@ import com.vnp.demo.models.Attachment; import com.vnp.demo.repositories.AttachmentRepository; import com.vnp.demo.repositories.ChatMemberRepository; import com.vnp.demo.validation.ImageValidator; -import com.vnp.demo.validation.ImageValidator.ValidatedImage; @Service public class AttachmentService { - private final AttachmentRepository attachmentRepository; - private final ChatMemberRepository chatMemberRepository; - private final ImageValidator imageValidator; + private final AttachmentRepository attachments; + private final ChatMemberRepository members; + private final ImageValidator validator; + private final UploadLimiter limiter; private final S3StorageService storage; - private final UploadLimiter uploadLimiter; - private final AttachmentPersistenceService persistence; + private final AttachmentPipeline pipeline; public AttachmentService( - AttachmentRepository attachmentRepository, - ChatMemberRepository chatMemberRepository, - ImageValidator imageValidator, + AttachmentRepository attachments, + ChatMemberRepository members, + ImageValidator validator, + UploadLimiter limiter, S3StorageService storage, - UploadLimiter uploadLimiter, - AttachmentPersistenceService persistence + AttachmentPipeline pipeline ) { - this.attachmentRepository = attachmentRepository; - this.chatMemberRepository = chatMemberRepository; - this.imageValidator = imageValidator; + this.attachments = attachments; + this.members = members; + this.validator = validator; + this.limiter = limiter; this.storage = storage; - this.uploadLimiter = uploadLimiter; - this.persistence = persistence; + this.pipeline = pipeline; } public AttachmentResponse upload( - UUID userId, - UUID chatId, - String filename, - String contentType, - long contentLength, - InputStream input - ) { - checkChatAccess(userId, chatId); + UUID userId, + UUID chatId, + String filename, + String contentType, + long contentLength, + InputStream input + ) { + checkChatAccess(userId, chatId); + validator.validateFilename(filename); - imageValidator.validateFilename(filename); + try (var permit = limiter.acquire(); + var image = validator.validate( + input, + contentLength, + contentType + )) { - try (UploadLimiter.Permit permit = uploadLimiter.acquire(); - ValidatedImage image = imageValidator.validate( - input, - contentLength, - contentType - )) { + UUID id = UUID.randomUUID(); - UUID attachmentId = UUID.randomUUID(); + Instant now = Instant.now() + .truncatedTo(ChronoUnit.MICROS); - String objectKey = "originals/" - + chatId - + "/" - + attachmentId; + Attachment attachment = new Attachment( + id, + chatId, + userId, + filename, + image.contentType(), + image.sizeBytes(), + image.sha256(), - Instant now = Instant.now() - .truncatedTo(ChronoUnit.MICROS); + "originals/" + + chatId + + "/" + + id + + "/pending", - Attachment attachment = new Attachment( - attachmentId, - chatId, - userId, - filename, - image.contentType(), - image.sizeBytes(), - image.sha256(), - objectKey, - "ready", - now, - now - ); + "queued", + now, + now + ); - persistence.save(attachment, image.path()); + pipeline.enqueue(attachment, image.path()); - return AttachmentResponse.from(attachment); - } + return AttachmentResponse.from(attachment); } + } public AttachmentResponse getMetadata( UUID userId, - UUID attachmentId + UUID id ) { - return AttachmentResponse.from( - getAccessibleAttachment(userId, attachmentId) - ); + return pipeline.metadata(accessible(userId, id)); } public Download download( UUID userId, - UUID attachmentId, + UUID id, String variant ) { - Attachment attachment = getAccessibleAttachment( - userId, - attachmentId - ); + Attachment attachment = accessible(userId, id); if (!"ready".equals(attachment.status())) { throw BusinessException.conflict( @@ -115,29 +109,36 @@ public class AttachmentService { ); } - if (!"original".equals(variant)) { - throw BusinessException.notFound(); + if ("original".equals(variant)) { + return new Download( + attachment.filename(), + attachment.contentType(), + storage.get( + attachment.objectKey(), + attachment.sizeBytes(), + attachment.sha256() + ) + ); } - byte[] bytes = storage.get( - attachment.objectKey(), - attachment.sizeBytes(), - attachment.sha256() - ); + var selected = pipeline.variants(id).stream() + .filter(item -> item.name().equals(variant)) + .findFirst() + .orElseThrow(BusinessException::notFound); return new Download( - attachment.filename(), - attachment.contentType(), - bytes + selected.name() + ".jpg", + selected.contentType(), + storage.get( + selected.objectKey(), + selected.sizeBytes(), + selected.sha256() + ) ); } - private Attachment getAccessibleAttachment( - UUID userId, - UUID attachmentId - ) { - Attachment attachment = attachmentRepository - .findById(attachmentId) + private Attachment accessible(UUID userId, UUID id) { + Attachment attachment = attachments.findById(id) .orElseThrow(BusinessException::notFound); checkChatAccess(userId, attachment.chatId()); @@ -146,10 +147,7 @@ public class AttachmentService { } private void checkChatAccess(UUID userId, UUID chatId) { - boolean member = chatMemberRepository - .existsByIdChatIdAndIdUserId(chatId, userId); - - if (!member) { + if (!members.existsByIdChatIdAndIdUserId(chatId, userId)) { throw BusinessException.notFound(); } } diff --git a/java/src/main/java/com/vnp/demo/service/FaultGate.java b/java/src/main/java/com/vnp/demo/service/FaultGate.java new file mode 100644 index 0000000..6e50f77 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/FaultGate.java @@ -0,0 +1,105 @@ +package com.vnp.demo.service; + +import java.util.UUID; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; + +@Component +public class FaultGate { + + private final boolean enabled; + private final JdbcTemplate jdbc; + private final TransactionTemplate tx; + + public FaultGate( + JdbcTemplate jdbc, + PlatformTransactionManager manager, + + @Value("${FAULT_INJECTION_ENABLED:false}") + boolean enabled + ) { + this.jdbc = jdbc; + this.enabled = enabled; + + tx = new TransactionTemplate(manager); + + tx.setPropagationBehavior( + TransactionDefinition.PROPAGATION_REQUIRES_NEW + ); + + tx.setTimeout(5); + } + + public boolean paused() { + if (!enabled) { + return false; + } + + return Boolean.TRUE.equals( + jdbc.queryForObject( + """ + SELECT EXISTS ( + SELECT 1 + FROM attachment_test_faults + WHERE point = 'hold_publish' + AND remaining > 0 + ) + """, + Boolean.class + ) + ); + } + + public boolean hit(UUID job, String point) { + if (!enabled) { + return false; + } + + boolean hit = Boolean.TRUE.equals( + tx.execute(ignored -> + jdbc.update( + """ + UPDATE attachment_test_faults + SET remaining = remaining - 1, + hits = hits + 1, + job_id = ?, + hit_at = clock_timestamp() + WHERE point = ? + AND remaining > 0 + """, + job, + point + ) == 1 + ) + ); + + if (!hit) { + return false; + } + + switch (point) { + case "before_publish", + "after_objects", + "before_ack" -> + Runtime.getRuntime().halt(137); + + case "permanent_error" -> + throw new ImageProcessor.PermanentFailure(); + + case "transient_error" -> + throw new IllegalStateException( + "injected_transient_error" + ); + + default -> { + } + } + + return true; + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/ImageProcessor.java b/java/src/main/java/com/vnp/demo/service/ImageProcessor.java new file mode 100644 index 0000000..7b74699 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/ImageProcessor.java @@ -0,0 +1,274 @@ +package com.vnp.demo.service; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +@Component +public class ImageProcessor { + + public static class PermanentFailure extends RuntimeException { + public PermanentFailure() { + super("invalid_image"); + } + } + + public static class TransientFailure extends RuntimeException { + private final String code; + + public TransientFailure(String code) { + super(code); + this.code = code; + } + + public String code() { + return code; + } + } + + public record Info( + String name, + + @JsonProperty("content_type") + String contentType, + + int width, + int height + ) { + } + + public record Generated( + Info info, + byte[] bytes + ) { + } + + private final ObjectMapper mapper; + + private final Set active = + ConcurrentHashMap.newKeySet(); + + private final Semaphore slots = new Semaphore(1); + private final Path root = Path.of("/tmp/image-jobs"); + + public ImageProcessor(ObjectMapper mapper) { + this.mapper = mapper; + } + + public List render( + byte[] source, + boolean testHang + ) { + if (source.length == 0 || source.length > 10485760) { + throw new PermanentFailure(); + } + + if (!slots.tryAcquire()) { + throw new TransientFailure("processor_busy"); + } + + Path directory = null; + Process process = null; + + try { + Files.createDirectories(root); + + directory = Files.createTempDirectory(root, "job-"); + active.add(directory); + + Path input = directory.resolve("input.bin"); + Files.write(input, source); + + ProcessBuilder builder = new ProcessBuilder( + "/opt/images/bin/python", + "/app/process_image.py", + input.toString(), + directory.toString() + ); + + builder.redirectOutput( + ProcessBuilder.Redirect.DISCARD + ); + + builder.redirectError( + ProcessBuilder.Redirect.DISCARD + ); + + builder.environment().remove("IMAGE_TEST_SLEEP"); + + if (testHang) { + builder.environment().put( + "IMAGE_TEST_SLEEP", + "1" + ); + } + + process = builder.start(); + + if (!process.waitFor(25, TimeUnit.SECONDS)) { + throw new TransientFailure( + "processing_timeout" + ); + } + + if (process.exitValue() == 10) { + throw new PermanentFailure(); + } + + if (process.exitValue() != 0) { + throw new TransientFailure( + "processing_resource_limit" + ); + } + + Path manifest = directory.resolve("manifest.json"); + + if (Files.size(manifest) > 4096) { + throw new TransientFailure("invalid_manifest"); + } + + Info[] infos = mapper.readValue( + Files.readAllBytes(manifest), + Info[].class + ); + + Set names = new HashSet<>(); + List result = new ArrayList<>(); + + for (Info info : infos) { + String mime = switch (info.name()) { + case "thumbnail", "crop" -> "image/jpeg"; + case "optimized" -> "image/webp"; + default -> throw new TransientFailure( + "invalid_variant" + ); + }; + + if (!names.add(info.name()) + || !mime.equals(info.contentType()) + || info.width() < 1 + || info.height() < 1 + || info.width() > 1280 + || info.height() > 1280) { + throw new TransientFailure( + "invalid_manifest" + ); + } + + String extension = mime.equals("image/webp") + ? ".webp" + : ".jpg"; + + Path file = directory.resolve( + info.name() + extension + ); + + long size = Files.size(file); + + if (size < 1 || size > 10485760) { + throw new TransientFailure( + "invalid_variant_size" + ); + } + + result.add( + new Generated( + info, + Files.readAllBytes(file) + ) + ); + } + + if (!names.equals( + Set.of("thumbnail", "crop", "optimized") + )) { + throw new TransientFailure( + "incomplete_variants" + ); + } + + return result; + + } catch (PermanentFailure | TransientFailure error) { + throw error; + + } catch (Exception error) { + if (error instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + + throw new TransientFailure( + "processing_unavailable" + ); + + } finally { + if (process != null && process.isAlive()) { + process.destroyForcibly(); + + try { + process.waitFor(5, TimeUnit.SECONDS); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + } + } + + if (directory != null) { + active.remove(directory); + remove(directory); + } + + slots.release(); + } + } + + private void remove(Path directory) { + try (var paths = Files.walk(directory)) { + for (Path path : paths + .sorted(Comparator.reverseOrder()) + .toList()) { + + Files.deleteIfExists(path); + } + } catch (Exception ignored) { + } + } + + @Scheduled( + initialDelay = 60000, + fixedDelay = 300000 + ) + public void cleanTemporaryFiles() { + if (!Files.isDirectory(root)) { + return; + } + + try (var children = Files.list(root)) { + for (Path path : children.toList()) { + if (Files.isDirectory(path) + && !active.contains(path) + && Files.getLastModifiedTime(path) + .toInstant() + .isBefore( + Instant.now().minusSeconds(3600) + )) { + remove(path); + } + } + } catch (Exception ignored) { + } + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/MessageService.java b/java/src/main/java/com/vnp/demo/service/MessageService.java index 1554308..6bd51e2 100644 --- a/java/src/main/java/com/vnp/demo/service/MessageService.java +++ b/java/src/main/java/com/vnp/demo/service/MessageService.java @@ -219,25 +219,25 @@ public class MessageService { } private void validateAttachments( - UUID chatId, - List attachmentIds - ) { + UUID chatId, + List attachmentIds + ) { for (UUID attachmentId : attachmentIds) { - Attachment attachment = attachmentRepository - .findById(attachmentId) - .orElseThrow(BusinessException::notFound); + Attachment attachment = attachmentRepository + .findById(attachmentId) + .orElseThrow(BusinessException::notFound); - if (!chatId.equals(attachment.chatId())) { + if (!chatId.equals(attachment.chatId())) { throw BusinessException.notFound(); - } + } - if (!"ready".equals(attachment.status())) { + if ("failed".equals(attachment.status())) { throw BusinessException.conflict( - "Вложение не готово" + "Обработка вложения завершилась ошибкой" ); - } + } + } } - } @Transactional(readOnly = true) public MessagePageResponse list( diff --git a/java/src/main/java/com/vnp/demo/service/ReplayRunner.java b/java/src/main/java/com/vnp/demo/service/ReplayRunner.java new file mode 100644 index 0000000..75f6a9b --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/ReplayRunner.java @@ -0,0 +1,261 @@ +package com.vnp.demo.service; + +import com.vnp.demo.repositories.AttachmentRepository; + +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.core.env.Environment; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.util.UUID; + +@Component +@ConditionalOnProperty( + name = "replay.enabled", + havingValue = "true" +) +public class ReplayRunner implements ApplicationRunner { + + private final JdbcTemplate jdbc; + private final AttachmentRepository attachments; + private final S3StorageService storage; + private final AttachmentPipeline pipeline; + private final TransactionTemplate tx; + private final Environment env; + private final ConfigurableApplicationContext context; + + public ReplayRunner( + JdbcTemplate jdbc, + AttachmentRepository attachments, + S3StorageService storage, + AttachmentPipeline pipeline, + PlatformTransactionManager manager, + Environment env, + ConfigurableApplicationContext context + ) { + this.jdbc = jdbc; + this.attachments = attachments; + this.storage = storage; + this.pipeline = pipeline; + this.env = env; + this.context = context; + + tx = new TransactionTemplate(manager); + tx.setTimeout(60); + } + + @Override + public void run(ApplicationArguments ignored) { + int code = 0; + + try { + UUID id = UUID.fromString( + env.getRequiredProperty("REPLAY_JOB_ID") + ); + + String actor = env.getRequiredProperty( + "REPLAY_ACTOR" + ); + + String reason = env.getRequiredProperty( + "REPLAY_REASON" + ); + + int budget = env.getProperty( + "app.jobs.max-attempts", + Integer.class, + 3 + ); + + if (!actor.matches("[A-Za-z0-9_.-]{1,64}") + || reason.isBlank() + || reason.length() > 255 + || budget < 1 + || budget > 100) { + throw new IllegalArgumentException( + "invalid_replay_request" + ); + } + + String result = tx.execute(status -> { + var job = jdbc.queryForMap( + """ + SELECT * + FROM attachment_jobs + WHERE id = ? + FOR UPDATE + """, + id + ); + + UUID attachmentId = + (UUID) job.get("attachment_id"); + + var attachment = attachments + .findById(attachmentId) + .orElseThrow(); + + int version = ((Number) job.get( + "pipeline_version" + )).intValue(); + + int attempts = ((Number) job.get( + "attempts" + )).intValue(); + + Integer activeVersion = jdbc.queryForObject( + """ + SELECT pipeline_version + FROM attachments + WHERE id = ? + """, + Integer.class, + attachmentId + ); + + if (version != 1 + || activeVersion == null + || activeVersion != version) { + throw new IllegalStateException( + "obsolete_pipeline" + ); + } + + String action; + + if ("ready".equals(job.get("status"))) { + pipeline.verifyReady(attachment); + action = "already_ready"; + + } else { + if (!"failed".equals(job.get("status")) + || !"failed".equals( + attachment.status() + )) { + throw new IllegalStateException( + "job_is_not_failed" + ); + } + + String staging = jdbc.queryForObject( + """ + SELECT staging_key + FROM attachments + WHERE id = ? + """, + String.class, + attachmentId + ); + + if (staging == null) { + throw new IllegalStateException( + "staging_expired" + ); + } + + /* + * Та же блокировка используется очисткой. + * Staging нельзя удалить во время replay. + */ + attachments.lockObject(staging); + + storage.get( + staging, + attachment.sizeBytes(), + attachment.sha256() + ); + + jdbc.update( + """ + UPDATE attachment_jobs + SET status = 'queued', + max_attempts = ?, + available_at = clock_timestamp(), + lease_owner = NULL, + lease_until = NULL, + last_error_code = NULL, + updated_at = clock_timestamp() + WHERE id = ? + """, + Math.addExact(attempts, budget), + id + ); + + jdbc.update( + """ + UPDATE attachments + SET status = 'queued', + error_code = NULL, + updated_at = clock_timestamp() + WHERE id = ? + """, + attachmentId + ); + + jdbc.update( + """ + INSERT INTO attachment_outbox( + id, + job_id, + attempt + ) + VALUES (?, ?, ?) + """, + UUID.randomUUID(), + id, + attempts + 1 + ); + + action = "enqueued"; + } + + jdbc.update( + """ + INSERT INTO attachment_replay_audit( + job_id, + actor, + reason, + action, + previous_attempts + ) + VALUES (?, ?, ?, ?, ?) + """, + id, + actor, + reason, + action, + attempts + ); + + return action; + }); + + System.out.println( + "REPLAY_RESULT=" + result + ); + + } catch (Exception error) { + code = 1; + + System.err.println( + "REPLAY_FAILED: " + + error.getClass().getSimpleName() + ); + } + + final int exitCode = code; + + System.exit( + SpringApplication.exit( + context, + () -> exitCode + ) + ); + } +} \ No newline at end of file diff --git a/java/src/main/java/com/vnp/demo/service/S3StorageService.java b/java/src/main/java/com/vnp/demo/service/S3StorageService.java index f1d1185..611989d 100644 --- a/java/src/main/java/com/vnp/demo/service/S3StorageService.java +++ b/java/src/main/java/com/vnp/demo/service/S3StorageService.java @@ -184,6 +184,44 @@ public class S3StorageService { } } + public void putBytes( + String objectKey, + byte[] bytes, + String contentType + ) { + if (bytes.length < 1 + || bytes.length > ImageValidator.MAX_BYTES) { + throw new IllegalArgumentException("Invalid object size"); + } + + try (var input = new java.io.ByteArrayInputStream(bytes)) { + client.putObject( + PutObjectArgs.builder() + .bucket(bucket) + .object(objectKey) + .contentType(contentType) + .stream( + input, + bytes.length, + ImageValidator.MAX_BYTES + ) + .build() + ); + } catch (Exception error) { + throw unavailable(); + } + } + + public Iterable> listPrefix(String prefix) { + return client.listObjects( + ListObjectsArgs.builder() + .bucket(bucket) + .prefix(prefix) + .recursive(true) + .build() + ); + } + private static BusinessException unavailable() { return new BusinessException( "unavailable", diff --git a/java/src/main/java/com/vnp/demo/service/Thumbnailer.java b/java/src/main/java/com/vnp/demo/service/Thumbnailer.java new file mode 100644 index 0000000..bd67540 --- /dev/null +++ b/java/src/main/java/com/vnp/demo/service/Thumbnailer.java @@ -0,0 +1,182 @@ +package com.vnp.demo.service; + +import com.drew.imaging.ImageMetadataReader; +import com.drew.metadata.exif.ExifIFD0Directory; +import com.vnp.demo.validation.ImageValidator; + +import org.springframework.stereotype.Component; + +import javax.imageio.ImageIO; +import javax.imageio.ImageReader; +import javax.imageio.stream.MemoryCacheImageInputStream; + +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.RenderingHints; +import java.awt.geom.AffineTransform; +import java.awt.image.BufferedImage; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + +@Component +public class Thumbnailer { + + public record Thumbnail( + byte[] bytes, + int width, + int height + ) { + } + + public Thumbnail render(byte[] source) { + BufferedImage original = null; + BufferedImage target = null; + ImageReader reader = null; + + try (var input = new MemoryCacheImageInputStream( + new ByteArrayInputStream(source) + )) { + var readers = ImageIO.getImageReaders(input); + + if (!readers.hasNext()) { + throw new IllegalArgumentException("invalid_image"); + } + + reader = readers.next(); + reader.setInput(input, true, true); + + int width = reader.getWidth(0); + int height = reader.getHeight(0); + + if (width < 1 + || height < 1 + || (long) width * height > ImageValidator.MAX_PIXELS) { + throw new IllegalArgumentException("invalid_image"); + } + + int orientation = 1; + + var metadata = ImageMetadataReader.readMetadata( + new ByteArrayInputStream(source) + ); + + var exif = metadata.getFirstDirectoryOfType( + ExifIFD0Directory.class + ); + + if (exif != null) { + Integer value = exif.getInteger( + ExifIFD0Directory.TAG_ORIENTATION + ); + + if (value != null && value >= 1 && value <= 8) { + orientation = value; + } + } + + original = reader.read(0); + + int orientedWidth = orientation >= 5 ? height : width; + int orientedHeight = orientation >= 5 ? width : height; + + int denominator = Math.max( + 256, + Math.max(orientedWidth, orientedHeight) + ); + + int targetWidth = Math.max( + 1, + (int) ((long) orientedWidth * 256 / denominator) + ); + + int targetHeight = Math.max( + 1, + (int) ((long) orientedHeight * 256 / denominator) + ); + + target = new BufferedImage( + targetWidth, + targetHeight, + BufferedImage.TYPE_INT_RGB + ); + + Graphics2D graphics = target.createGraphics(); + + try { + graphics.setColor(Color.WHITE); + graphics.fillRect(0, 0, targetWidth, targetHeight); + + graphics.setRenderingHint( + RenderingHints.KEY_INTERPOLATION, + RenderingHints.VALUE_INTERPOLATION_BICUBIC + ); + + AffineTransform transform = + AffineTransform.getScaleInstance( + (double) targetWidth / orientedWidth, + (double) targetHeight / orientedHeight + ); + + transform.concatenate( + switch (orientation) { + case 2 -> new AffineTransform( + -1, 0, 0, 1, width, 0 + ); + case 3 -> new AffineTransform( + -1, 0, 0, -1, width, height + ); + case 4 -> new AffineTransform( + 1, 0, 0, -1, 0, height + ); + case 5 -> new AffineTransform( + 0, 1, 1, 0, 0, 0 + ); + case 6 -> new AffineTransform( + 0, 1, -1, 0, height, 0 + ); + case 7 -> new AffineTransform( + 0, -1, -1, 0, height, width + ); + case 8 -> new AffineTransform( + 0, -1, 1, 0, 0, width + ); + default -> new AffineTransform(); + } + ); + + graphics.drawImage(original, transform, null); + } finally { + graphics.dispose(); + } + + var output = new ByteArrayOutputStream(); + + if (!ImageIO.write(target, "jpeg", output)) { + throw new IllegalArgumentException( + "jpeg_encoder_unavailable" + ); + } + + return new Thumbnail( + output.toByteArray(), + targetWidth, + targetHeight + ); + } catch (Exception error) { + throw new IllegalArgumentException("invalid_image", error); + } finally { + if (reader != null) { + reader.dispose(); + } + + if (original != null) { + original.flush(); + } + + if (target != null) { + target.flush(); + } + } + } +} \ No newline at end of file diff --git a/java/src/main/resources/application.yml b/java/src/main/resources/application.yml index 8989eec..35b1289 100644 --- a/java/src/main/resources/application.yml +++ b/java/src/main/resources/application.yml @@ -4,11 +4,14 @@ server: threads: max: 64 connection-timeout: 2s + spring: application: name: messenger + lifecycle: timeout-per-shutdown-phase: 15s + jackson: default-property-inclusion: always deserialization: @@ -16,10 +19,12 @@ spring: fail-on-trailing-tokens: true mapper: allow-coercion-of-scalars: false + datasource: url: ${DB_URL:jdbc:postgresql://localhost:5432/messenger} username: ${DB_USERNAME:messenger} password: ${DB_PASSWORD:messenger} + hikari: maximum-pool-size: ${DB_POOL_SIZE:8} minimum-idle: 1 @@ -29,6 +34,7 @@ spring: connectTimeout: 2 socketTimeout: 8 options: "-c statement_timeout=5000 -c lock_timeout=3000" + data: redis: host: ${REDIS_HOST:localhost} @@ -36,17 +42,49 @@ spring: password: ${REDIS_PASSWORD:} timeout: 1s connect-timeout: 1s + + rabbitmq: + host: ${RABBITMQ_HOST:localhost} + port: 5672 + username: ${RABBITMQ_USER:messenger_jobs} + password: ${RABBITMQ_PASSWORD:} + + connection-timeout: 3s + requested-heartbeat: 15s + + publisher-confirm-type: correlated + publisher-returns: true + + template: + mandatory: true + + listener: + simple: + acknowledge-mode: manual + prefetch: 1 + concurrency: 1 + max-concurrency: 1 + default-requeue-rejected: true + + task: + scheduling: + pool: + size: 3 + flyway: enabled: ${FLYWAY_ENABLED:true} + jpa: hibernate: ddl-auto: validate open-in-view: false + transaction: default-timeout: 8s + app: session: - ttl-seconds: 86400 + ttl-seconds: ${SESSION_TTL_SECONDS:3600} s3: endpoint: ${S3_ENDPOINT:http://localhost:9000} @@ -57,7 +95,15 @@ app: uploads: max-concurrent: ${UPLOAD_MAX_CONCURRENT:1} temp-dir: ${UPLOAD_TEMP_DIR:/tmp/messenger-uploads} - + + worker: + enabled: ${WORKER_ENABLED:false} + + jobs: + max-attempts: ${JOB_MAX_ATTEMPTS:3} + lease-seconds: ${JOB_TIMEOUT_SECONDS:120} + failed-retention-seconds: ${STAGING_RETENTION_SECONDS:604800} + cleanup: enabled: ${ATTACHMENT_CLEANUP_ENABLED:true} grace-seconds: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:3600} diff --git a/java/src/main/resources/db/migration/V4__async_attachments.sql b/java/src/main/resources/db/migration/V4__async_attachments.sql new file mode 100644 index 0000000..1e9be45 --- /dev/null +++ b/java/src/main/resources/db/migration/V4__async_attachments.sql @@ -0,0 +1,220 @@ +-- Расширение метаданных вложения для асинхронной обработки. + +ALTER TABLE attachments + DROP CONSTRAINT chk_attachment_status; + +ALTER TABLE attachments + ADD CONSTRAINT chk_attachment_status + CHECK ( + status IN ( + 'queued', + 'processing', + 'ready', + 'failed' + ) + ); + +ALTER TABLE attachments + ADD COLUMN staging_key TEXT, + ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1, + ADD COLUMN error_code VARCHAR(64); + +ALTER TABLE attachments + ADD CONSTRAINT uq_attachment_staging_key + UNIQUE (staging_key), + + ADD CONSTRAINT chk_attachment_pipeline_version + CHECK (pipeline_version >= 1), + + ADD CONSTRAINT chk_attachment_error + CHECK ( + ( + status = 'failed' + AND error_code IS NOT NULL + AND length(error_code) > 0 + ) + OR + ( + status <> 'failed' + AND error_code IS NULL + ) + ); + +CREATE INDEX idx_attachments_status + ON attachments(status, updated_at); + + +-- Готовые варианты изображения. +-- object_key используется внутри приложения и не выдаётся клиенту. +-- Ключ эффекта соответствует контракту: +-- attachment_id + pipeline_version + variant. + +CREATE TABLE attachment_variants ( + attachment_id UUID NOT NULL + REFERENCES attachments(id) ON DELETE CASCADE, + + pipeline_version INTEGER NOT NULL, + name VARCHAR(32) NOT NULL, + + object_key TEXT NOT NULL UNIQUE, + content_type VARCHAR(32) NOT NULL, + + size_bytes BIGINT NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + sha256 CHAR(64) NOT NULL, + + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + PRIMARY KEY ( + attachment_id, + pipeline_version, + name + ), + + CONSTRAINT chk_variant_pipeline_version + CHECK (pipeline_version >= 1), + + CONSTRAINT chk_variant_name + CHECK ( + name IN ( + 'thumbnail', + 'crop', + 'optimized' + ) + ), + + CONSTRAINT chk_variant_content_type + CHECK ( + content_type IN ( + 'image/jpeg', + 'image/png', + 'image/webp' + ) + ), + + CONSTRAINT chk_variant_size + CHECK (size_bytes > 0), + + CONSTRAINT chk_variant_dimensions + CHECK (width > 0 AND height > 0), + + CONSTRAINT chk_variant_sha256 + CHECK (sha256 ~ '^[a-f0-9]{64}$') +); + + +-- Одна логическая работа над вложением в рамках версии pipeline. +-- job_id не меняется при повторной доставке. +-- attempts увеличивается при начале управляемой попытки, +-- а не при каждом получении дубликата сообщения. + +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY, + + attachment_id UUID NOT NULL + REFERENCES attachments(id) ON DELETE CASCADE, + + pipeline_version INTEGER NOT NULL DEFAULT 1, + + status VARCHAR(16) NOT NULL DEFAULT 'queued', + + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 3, + + available_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + lease_owner UUID, + lease_until TIMESTAMPTZ, + + last_error_code VARCHAR(64), + + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT uq_attachment_job_pipeline + UNIQUE ( + attachment_id, + pipeline_version + ), + + CONSTRAINT chk_job_pipeline_version + CHECK (pipeline_version >= 1), + + CONSTRAINT chk_job_status + CHECK ( + status IN ( + 'queued', + 'processing', + 'ready', + 'failed' + ) + ), + + CONSTRAINT chk_job_attempts + CHECK ( + max_attempts >= 1 + AND attempts >= 0 + AND attempts <= max_attempts + ), + + CONSTRAINT chk_job_lease + CHECK ( + ( + status = 'processing' + AND lease_owner IS NOT NULL + AND lease_until IS NOT NULL + ) + OR + ( + status <> 'processing' + AND lease_owner IS NULL + AND lease_until IS NULL + ) + ) +); + +CREATE INDEX idx_attachment_jobs_available + ON attachment_jobs(available_at, id) + WHERE status = 'queued'; + +CREATE INDEX idx_attachment_jobs_expired_lease + ON attachment_jobs(lease_until, id) + WHERE status = 'processing'; + + +-- Намерение публикации задания в брокер. +-- Записывается в той же транзакции, что и метаданные задания. +-- published_at устанавливается после подтверждения брокера. +-- Потеря подтверждения может привести к повторной публикации: +-- worker обязан обрабатывать её идемпотентно. + +CREATE TABLE attachment_outbox ( + id UUID PRIMARY KEY, + + job_id UUID NOT NULL + REFERENCES attachment_jobs(id) ON DELETE CASCADE, + + attempt INTEGER NOT NULL, + + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + next_publish_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + published_at TIMESTAMPTZ, + + publish_attempts INTEGER NOT NULL DEFAULT 0, + last_error_code VARCHAR(64), + + CONSTRAINT uq_attachment_outbox_attempt + UNIQUE (job_id, attempt), + + CONSTRAINT chk_outbox_attempt + CHECK (attempt >= 1), + + CONSTRAINT chk_outbox_publish_attempts + CHECK (publish_attempts >= 0) +); + +CREATE INDEX idx_attachment_outbox_pending + ON attachment_outbox(next_publish_at, id) + WHERE published_at IS NULL; \ No newline at end of file diff --git a/java/src/main/resources/db/migration/V5__level4_processing.sql b/java/src/main/resources/db/migration/V5__level4_processing.sql new file mode 100644 index 0000000..6baee3f --- /dev/null +++ b/java/src/main/resources/db/migration/V5__level4_processing.sql @@ -0,0 +1,94 @@ +CREATE TABLE attachment_replay_audit ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + + job_id UUID NOT NULL REFERENCES attachment_jobs(id), + actor VARCHAR(64) NOT NULL, + reason VARCHAR(255) NOT NULL, + + action VARCHAR(32) NOT NULL + CHECK (action IN ('enqueued', 'already_ready')), + + previous_attempts INTEGER NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE attachment_attempt_events ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + + job_id UUID NOT NULL REFERENCES attachment_jobs(id), + attempt INTEGER NOT NULL, + state VARCHAR(16) NOT NULL, + error_code VARCHAR(64) NOT NULL, + delay_seconds INTEGER NOT NULL, + + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +-- Используется только при FAULT_INJECTION_ENABLED=true. +-- Управления через публичный HTTP API нет. +CREATE TABLE attachment_test_faults ( + point VARCHAR(32) PRIMARY KEY, + remaining INTEGER NOT NULL CHECK (remaining >= 0), + hits INTEGER NOT NULL DEFAULT 0, + + job_id UUID REFERENCES attachment_jobs(id), + hit_at TIMESTAMPTZ +); + +-- Существующие ready-вложения с неполным набором вариантов +-- отправляются на дообработку. +INSERT INTO attachment_jobs ( + id, + attachment_id, + pipeline_version, + status, + max_attempts +) +SELECT + gen_random_uuid(), + a.id, + a.pipeline_version, + 'queued', + 3 +FROM attachments a +WHERE a.status = 'ready' + AND ( + SELECT count(*) + FROM attachment_variants v + WHERE v.attachment_id = a.id + AND v.pipeline_version = a.pipeline_version + AND v.name IN ('thumbnail', 'crop', 'optimized') + ) < 3 +ON CONFLICT (attachment_id, pipeline_version) +DO UPDATE SET + status = 'queued', + max_attempts = greatest( + attachment_jobs.max_attempts, + attachment_jobs.attempts + 3 + ), + available_at = clock_timestamp(), + lease_owner = NULL, + lease_until = NULL, + last_error_code = NULL, + updated_at = clock_timestamp() +WHERE attachment_jobs.status = 'ready'; + +UPDATE attachments a +SET status = 'queued', + staging_key = a.object_key, + error_code = NULL, + updated_at = clock_timestamp() +WHERE a.status = 'ready' + AND EXISTS ( + SELECT 1 + FROM attachment_jobs j + WHERE j.attachment_id = a.id + AND j.pipeline_version = a.pipeline_version + AND j.status = 'queued' + ); + +INSERT INTO attachment_outbox(id, job_id, attempt) +SELECT gen_random_uuid(), id, attempts + 1 +FROM attachment_jobs +WHERE status = 'queued' +ON CONFLICT (job_id, attempt) DO NOTHING; \ No newline at end of file diff --git a/results/lab5-level4.json b/results/lab5-level4.json new file mode 100644 index 0000000..4b3de76 --- /dev/null +++ b/results/lab5-level4.json @@ -0,0 +1,432 @@ +{ + "passed": true, + "scenarios": [ + { + "scenario": "commit_before_publish", + "job_id": "207a9343-70e4-466e-b481-11e67857199e", + "status": "ready", + "attempts": 1, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/b5f218eb-0085-4eec-a2b5-7dd216e598e0", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/b5f218eb-0085-4eec-a2b5-7dd216e598e0/v1/bac67a3f-0ffd-44cb-8c8b-fc6dbec51390", + "variants": [ + { + "attachment_id": "b5f218eb-0085-4eec-a2b5-7dd216e598e0", + "pipeline_version": 1, + "name": "crop", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/b5f218eb-0085-4eec-a2b5-7dd216e598e0/v1/bac67a3f-0ffd-44cb-8c8b-fc6dbec51390/crop.jpg", + "content_type": "image/jpeg", + "size_bytes": 1820, + "width": 128, + "height": 128, + "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", + "created_at": "2026-11-11T19:49:15.90599+00:00" + }, + { + "attachment_id": "b5f218eb-0085-4eec-a2b5-7dd216e598e0", + "pipeline_version": 1, + "name": "optimized", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/b5f218eb-0085-4eec-a2b5-7dd216e598e0/v1/bac67a3f-0ffd-44cb-8c8b-fc6dbec51390/optimized.webp", + "content_type": "image/webp", + "size_bytes": 2462, + "width": 320, + "height": 200, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "created_at": "2026-11-11T19:49:15.90599+00:00" + }, + { + "attachment_id": "b5f218eb-0085-4eec-a2b5-7dd216e598e0", + "pipeline_version": 1, + "name": "thumbnail", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/b5f218eb-0085-4eec-a2b5-7dd216e598e0/v1/bac67a3f-0ffd-44cb-8c8b-fc6dbec51390/thumbnail.jpg", + "content_type": "image/jpeg", + "size_bytes": 4679, + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "created_at": "2026-11-11T19:49:15.90599+00:00" + } + ], + "events": [] + }, + { + "scenario": "after_objects", + "job_id": "49d51049-a3f9-4f4d-a5be-a764dae8873b", + "status": "ready", + "attempts": 2, + "owner": null, + "staging": null, + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/c75cd648-4491-4c57-b626-11322c195a6f/v1/00e588f1-255c-45f3-a362-32375997971b", + "variants": [ + { + "attachment_id": "c75cd648-4491-4c57-b626-11322c195a6f", + "pipeline_version": 1, + "name": "crop", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/c75cd648-4491-4c57-b626-11322c195a6f/v1/00e588f1-255c-45f3-a362-32375997971b/crop.jpg", + "content_type": "image/jpeg", + "size_bytes": 1820, + "width": 128, + "height": 128, + "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", + "created_at": "2026-11-11T19:50:05.541163+00:00" + }, + { + "attachment_id": "c75cd648-4491-4c57-b626-11322c195a6f", + "pipeline_version": 1, + "name": "optimized", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/c75cd648-4491-4c57-b626-11322c195a6f/v1/00e588f1-255c-45f3-a362-32375997971b/optimized.webp", + "content_type": "image/webp", + "size_bytes": 2462, + "width": 320, + "height": 200, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "created_at": "2026-11-11T19:50:05.541163+00:00" + }, + { + "attachment_id": "c75cd648-4491-4c57-b626-11322c195a6f", + "pipeline_version": 1, + "name": "thumbnail", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/c75cd648-4491-4c57-b626-11322c195a6f/v1/00e588f1-255c-45f3-a362-32375997971b/thumbnail.jpg", + "content_type": "image/jpeg", + "size_bytes": 4679, + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "created_at": "2026-11-11T19:50:05.541163+00:00" + } + ], + "events": [ + { + "id": 5, + "job_id": "49d51049-a3f9-4f4d-a5be-a764dae8873b", + "attempt": 1, + "state": "queued", + "error_code": "lease_expired", + "delay_seconds": 1, + "created_at": "2026-11-11T19:50:03.822834+00:00" + } + ] + }, + { + "scenario": "before_ack", + "job_id": "e4265ea1-b4f8-467c-93cf-61030292f002", + "status": "ready", + "attempts": 2, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/412fda65-d510-4b2d-80f0-c7cd78b7f1ee", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/412fda65-d510-4b2d-80f0-c7cd78b7f1ee/v1/96af9136-89bf-49bd-b2e8-193bc0b1bba9", + "variants": [ + { + "attachment_id": "412fda65-d510-4b2d-80f0-c7cd78b7f1ee", + "pipeline_version": 1, + "name": "crop", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/412fda65-d510-4b2d-80f0-c7cd78b7f1ee/v1/96af9136-89bf-49bd-b2e8-193bc0b1bba9/crop.jpg", + "content_type": "image/jpeg", + "size_bytes": 1820, + "width": 128, + "height": 128, + "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", + "created_at": "2026-11-11T19:51:11.854309+00:00" + }, + { + "attachment_id": "412fda65-d510-4b2d-80f0-c7cd78b7f1ee", + "pipeline_version": 1, + "name": "optimized", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/412fda65-d510-4b2d-80f0-c7cd78b7f1ee/v1/96af9136-89bf-49bd-b2e8-193bc0b1bba9/optimized.webp", + "content_type": "image/webp", + "size_bytes": 2462, + "width": 320, + "height": 200, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "created_at": "2026-11-11T19:51:11.854309+00:00" + }, + { + "attachment_id": "412fda65-d510-4b2d-80f0-c7cd78b7f1ee", + "pipeline_version": 1, + "name": "thumbnail", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/412fda65-d510-4b2d-80f0-c7cd78b7f1ee/v1/96af9136-89bf-49bd-b2e8-193bc0b1bba9/thumbnail.jpg", + "content_type": "image/jpeg", + "size_bytes": 4679, + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "created_at": "2026-11-11T19:51:11.854309+00:00" + } + ], + "events": [ + { + "id": 6, + "job_id": "e4265ea1-b4f8-467c-93cf-61030292f002", + "attempt": 1, + "state": "queued", + "error_code": "processing_unavailable", + "delay_seconds": 2, + "created_at": "2026-11-11T19:51:09.373985+00:00" + } + ] + }, + { + "scenario": "permanent_error", + "failed": { + "job_id": "618544ac-e574-4dd7-9e46-81cf9147f9ef", + "status": "failed", + "attempts": 1, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee/pending", + "variants": [], + "events": [ + { + "id": 7, + "job_id": "618544ac-e574-4dd7-9e46-81cf9147f9ef", + "attempt": 1, + "state": "failed", + "error_code": "invalid_image", + "delay_seconds": 0, + "created_at": "2026-11-11T19:51:45.828306+00:00" + } + ] + }, + "ready": { + "job_id": "618544ac-e574-4dd7-9e46-81cf9147f9ef", + "status": "ready", + "attempts": 2, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee/v1/baf3e4b3-72d9-4ba3-ba13-2ea381d22466", + "variants": [ + { + "attachment_id": "a10ad8db-4c07-43bc-a8ee-be75671095ee", + "pipeline_version": 1, + "name": "crop", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee/v1/baf3e4b3-72d9-4ba3-ba13-2ea381d22466/crop.jpg", + "content_type": "image/jpeg", + "size_bytes": 1820, + "width": 128, + "height": 128, + "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", + "created_at": "2026-11-11T19:52:14.88595+00:00" + }, + { + "attachment_id": "a10ad8db-4c07-43bc-a8ee-be75671095ee", + "pipeline_version": 1, + "name": "optimized", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee/v1/baf3e4b3-72d9-4ba3-ba13-2ea381d22466/optimized.webp", + "content_type": "image/webp", + "size_bytes": 2462, + "width": 320, + "height": 200, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "created_at": "2026-11-11T19:52:14.88595+00:00" + }, + { + "attachment_id": "a10ad8db-4c07-43bc-a8ee-be75671095ee", + "pipeline_version": 1, + "name": "thumbnail", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/a10ad8db-4c07-43bc-a8ee-be75671095ee/v1/baf3e4b3-72d9-4ba3-ba13-2ea381d22466/thumbnail.jpg", + "content_type": "image/jpeg", + "size_bytes": 4679, + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "created_at": "2026-11-11T19:52:14.88595+00:00" + } + ], + "events": [ + { + "id": 7, + "job_id": "618544ac-e574-4dd7-9e46-81cf9147f9ef", + "attempt": 1, + "state": "failed", + "error_code": "invalid_image", + "delay_seconds": 0, + "created_at": "2026-11-11T19:51:45.828306+00:00" + } + ] + } + }, + { + "scenario": "transient_error", + "failed": { + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "status": "failed", + "attempts": 3, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a/pending", + "variants": [], + "events": [ + { + "id": 8, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 1, + "state": "queued", + "error_code": "processing_unavailable", + "delay_seconds": 2, + "created_at": "2026-11-11T19:52:43.635817+00:00" + }, + { + "id": 9, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 2, + "state": "queued", + "error_code": "processing_unavailable", + "delay_seconds": 2, + "created_at": "2026-11-11T19:52:45.971699+00:00" + }, + { + "id": 10, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 3, + "state": "failed", + "error_code": "processing_unavailable", + "delay_seconds": 0, + "created_at": "2026-11-11T19:52:48.656484+00:00" + } + ] + }, + "ready": { + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "status": "ready", + "attempts": 5, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a/v1/8b9bd5ba-3fc3-4525-b7e3-8e8eeca815e0", + "variants": [ + { + "attachment_id": "772c8541-d5c1-4707-86e5-9ba4a96eca4a", + "pipeline_version": 1, + "name": "crop", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a/v1/8b9bd5ba-3fc3-4525-b7e3-8e8eeca815e0/crop.jpg", + "content_type": "image/jpeg", + "size_bytes": 1820, + "width": 128, + "height": 128, + "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", + "created_at": "2026-11-11T19:53:15.180295+00:00" + }, + { + "attachment_id": "772c8541-d5c1-4707-86e5-9ba4a96eca4a", + "pipeline_version": 1, + "name": "optimized", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a/v1/8b9bd5ba-3fc3-4525-b7e3-8e8eeca815e0/optimized.webp", + "content_type": "image/webp", + "size_bytes": 2462, + "width": 320, + "height": 200, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "created_at": "2026-11-11T19:53:15.180295+00:00" + }, + { + "attachment_id": "772c8541-d5c1-4707-86e5-9ba4a96eca4a", + "pipeline_version": 1, + "name": "thumbnail", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/772c8541-d5c1-4707-86e5-9ba4a96eca4a/v1/8b9bd5ba-3fc3-4525-b7e3-8e8eeca815e0/thumbnail.jpg", + "content_type": "image/jpeg", + "size_bytes": 4679, + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "created_at": "2026-11-11T19:53:15.180295+00:00" + } + ], + "events": [ + { + "id": 8, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 1, + "state": "queued", + "error_code": "processing_unavailable", + "delay_seconds": 2, + "created_at": "2026-11-11T19:52:43.635817+00:00" + }, + { + "id": 9, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 2, + "state": "queued", + "error_code": "processing_unavailable", + "delay_seconds": 2, + "created_at": "2026-11-11T19:52:45.971699+00:00" + }, + { + "id": 10, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 3, + "state": "failed", + "error_code": "processing_unavailable", + "delay_seconds": 0, + "created_at": "2026-11-11T19:52:48.656484+00:00" + }, + { + "id": 11, + "job_id": "79ace0c1-ebc6-459f-acf3-190575acaab6", + "attempt": 4, + "state": "queued", + "error_code": "processing_unavailable", + "delay_seconds": 2, + "created_at": "2026-11-11T19:53:12.735705+00:00" + } + ] + } + }, + { + "scenario": "image_timeout", + "job_id": "cf2a9648-947a-4111-a100-3939bb467455", + "status": "ready", + "attempts": 2, + "owner": null, + "staging": "staging/15ee2f51-17cf-4398-8590-a9ae3a9234de/cbaa3d91-69ec-4edb-8804-c815c60d3fe8", + "original_key": "originals/15ee2f51-17cf-4398-8590-a9ae3a9234de/cbaa3d91-69ec-4edb-8804-c815c60d3fe8/v1/ed7490f5-d0fe-4c37-baa7-b2743ff92d71", + "variants": [ + { + "attachment_id": "cbaa3d91-69ec-4edb-8804-c815c60d3fe8", + "pipeline_version": 1, + "name": "crop", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/cbaa3d91-69ec-4edb-8804-c815c60d3fe8/v1/ed7490f5-d0fe-4c37-baa7-b2743ff92d71/crop.jpg", + "content_type": "image/jpeg", + "size_bytes": 1820, + "width": 128, + "height": 128, + "sha256": "97bf1400b30a28c19420a30161c944f57b5e7d613533b76afe6ab2531cba0143", + "created_at": "2026-11-11T19:54:12.013459+00:00" + }, + { + "attachment_id": "cbaa3d91-69ec-4edb-8804-c815c60d3fe8", + "pipeline_version": 1, + "name": "optimized", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/cbaa3d91-69ec-4edb-8804-c815c60d3fe8/v1/ed7490f5-d0fe-4c37-baa7-b2743ff92d71/optimized.webp", + "content_type": "image/webp", + "size_bytes": 2462, + "width": 320, + "height": 200, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "created_at": "2026-11-11T19:54:12.013459+00:00" + }, + { + "attachment_id": "cbaa3d91-69ec-4edb-8804-c815c60d3fe8", + "pipeline_version": 1, + "name": "thumbnail", + "object_key": "variants/15ee2f51-17cf-4398-8590-a9ae3a9234de/cbaa3d91-69ec-4edb-8804-c815c60d3fe8/v1/ed7490f5-d0fe-4c37-baa7-b2743ff92d71/thumbnail.jpg", + "content_type": "image/jpeg", + "size_bytes": 4679, + "width": 256, + "height": 160, + "sha256": "7bc40bf315c8e18e907889b23514d978be213a44223d64b737290188ee808594", + "created_at": "2026-11-11T19:54:12.013459+00:00" + } + ], + "events": [ + { + "id": 12, + "job_id": "cf2a9648-947a-4111-a100-3939bb467455", + "attempt": 1, + "state": "queued", + "error_code": "processing_timeout", + "delay_seconds": 2, + "created_at": "2026-11-11T19:54:09.817644+00:00" + } + ] + } + ] +} diff --git a/tests/level4_check.py b/tests/level4_check.py new file mode 100644 index 0000000..7b6d0e3 --- /dev/null +++ b/tests/level4_check.py @@ -0,0 +1,1006 @@ +import hashlib +import json +from pathlib import Path +import subprocess +import time +import uuid + +from client import Client, Failure, arguments, check, run +from image_probe import jpeg_size + + +ROOT = Path(__file__).resolve().parents[1] + +BASE = [ + "docker", + "compose", + "-f", + "compose.yml", + "-f", + "compose.override.yml", +] + +FAULT = BASE + [ + "-f", + "compose.faults.yml", +] + +FAULT_POINTS = { + "hold_publish", + "before_publish", + "after_objects", + "before_ack", + "permanent_error", + "transient_error", + "image_timeout", +} + + +def command(args, required=True): + try: + result = subprocess.run( + args, + cwd=ROOT, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=240, + ) + except subprocess.TimeoutExpired: + raise Failure("Docker command timed out") from None + + if required and result.returncode: + raise Failure( + "Docker command failed; " + "inspect the corresponding service locally" + ) + + return result + + +def sql(query): + result = command( + BASE + + [ + "exec", + "-T", + "postgres", + "sh", + "-ec", + 'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" ' + '-At -v ON_ERROR_STOP=1 -c "$1"', + "sh", + query, + ] + ) + + return result.stdout.strip() + + +def wait(action, label, seconds=180): + deadline = time.monotonic() + seconds + + while time.monotonic() < deadline: + try: + value = action() + + if value: + return value + + except Failure: + pass + + time.sleep(1) + + raise Failure("Timeout: " + label) + + +def state(attachment_id): + attachment_id = str(uuid.UUID(attachment_id)) + + raw = sql( + f""" + SELECT json_build_object( + 'job_id', j.id, + 'status', j.status, + 'attempts', j.attempts, + 'owner', j.lease_owner, + 'staging', a.staging_key, + 'original_key', a.object_key, + + 'variants', COALESCE( + ( + SELECT json_agg(v ORDER BY v.name) + FROM attachment_variants v + WHERE v.attachment_id = a.id + AND v.pipeline_version = a.pipeline_version + ), + '[]'::json + ), + + 'events', COALESCE( + ( + SELECT json_agg(e ORDER BY e.id) + FROM attachment_attempt_events e + WHERE e.job_id = j.id + ), + '[]'::json + ) + ) + FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id + WHERE a.id = '{attachment_id}'::uuid + AND j.pipeline_version = a.pipeline_version + """ + ) + + check(bool(raw), "attachment job not found") + + try: + return json.loads(raw) + except json.JSONDecodeError: + raise Failure("Invalid job state returned by PostgreSQL") from None + + +def arm(point, count=1): + check(point in FAULT_POINTS, "unknown fault") + check(isinstance(count, int) and count >= 0, "invalid fault count") + + sql( + f""" + INSERT INTO attachment_test_faults(point, remaining) + VALUES ('{point}', {count}) + + ON CONFLICT (point) + DO UPDATE SET + remaining = EXCLUDED.remaining, + hits = 0, + job_id = NULL, + hit_at = NULL + """ + ) + + +def hit(point, job_id): + check(point in FAULT_POINTS, "unknown fault") + job_id = str(uuid.UUID(job_id)) + + return sql( + f""" + SELECT count(*) + FROM attachment_test_faults + WHERE point = '{point}' + AND job_id = '{job_id}'::uuid + AND hits > 0 + """ + ) == "1" + + +def restarts(service): + ids = command( + BASE + ["ps", "-a", "-q", service] + ).stdout.split() + + check(bool(ids), "service containers not found: " + service) + + return sum( + int( + command( + [ + "docker", + "inspect", + "--format", + "{{.RestartCount}}", + container, + ] + ).stdout.strip() + ) + for container in ids + ) + + +def object_exists(bucket, key): + result = command( + BASE + + [ + "exec", + "-T", + "minio", + "sh", + "-ec", + "mc alias set grade4 http://localhost:9000 " + '"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null; ' + 'mc stat --json "grade4/$1/$2"', + "sh", + bucket, + key, + ], + required=False, + ) + + if result.returncode == 0: + return True + + error = (result.stdout + result.stderr).lower() + + if any( + value in error + for value in [ + "nosuchkey", + "nosuchobject", + "specified key does not exist", + "object does not exist", + ] + ): + return False + + raise Failure( + "Cannot distinguish missing object from S3 failure" + ) + + +def webp_size(data): + check( + len(data) >= 12 + and data[:4] == b"RIFF" + and data[8:12] == b"WEBP", + "invalid WebP signature", + ) + + check( + int.from_bytes(data[4:8], "little") + 8 == len(data), + "invalid RIFF size", + ) + + offset = 12 + size = None + + while offset + 8 <= len(data): + tag = data[offset:offset + 4] + + length = int.from_bytes( + data[offset + 4:offset + 8], + "little", + ) + + end = offset + 8 + length + check(end <= len(data), "truncated WebP chunk") + + block = data[offset + 8:end] + + check( + tag not in (b"EXIF", b"XMP ", b"ICCP"), + "metadata was retained in WebP", + ) + + if tag == b"VP8X": + check(len(block) == 10, "invalid VP8X chunk") + + size = ( + 1 + int.from_bytes(block[4:7], "little"), + 1 + int.from_bytes(block[7:10], "little"), + ) + + elif tag == b"VP8 " and size is None: + check(len(block) >= 10, "truncated VP8 frame") + + check( + block[3:6] == b"\x9d\x01\x2a", + "invalid VP8 frame", + ) + + size = ( + int.from_bytes(block[6:8], "little") & 0x3fff, + int.from_bytes(block[8:10], "little") & 0x3fff, + ) + + elif tag == b"VP8L" and size is None: + check(len(block) >= 5, "truncated VP8L frame") + check(block[0] == 0x2f, "invalid VP8L frame") + + bits = int.from_bytes(block[1:5], "little") + + size = ( + 1 + (bits & 0x3fff), + 1 + ((bits >> 14) & 0x3fff), + ) + + offset = end + (length & 1) + check(offset <= len(data), "missing WebP padding") + + check(offset == len(data), "invalid WebP trailing data") + check(size is not None, "WebP dimensions missing") + check(size[0] > 0 and size[1] > 0, "invalid WebP dimensions") + + return size + + +def replay(job): + result = command( + FAULT + + [ + "run", + "--rm", + "--no-deps", + "--pull", + "never", + "-e", + "REPLAY_ENABLED=true", + "-e", + "FAULT_INJECTION_ENABLED=false", + "-e", + "REPLAY_JOB_ID=" + str(uuid.UUID(job)), + "-e", + "REPLAY_ACTOR=grade4_test", + "-e", + "REPLAY_REASON=verified_test_replay", + "api", + ] + ) + + return result.stdout + + +def restore_normal_configuration(): + print( + "Restoring normal API and worker configuration...", + flush=True, + ) + + try: + sql( + "UPDATE attachment_test_faults SET remaining = 0" + ) + finally: + command( + BASE + + [ + "up", + "-d", + "--no-build", + "--pull", + "never", + "--no-deps", + "--scale", + "api=2", + "--wait", + "--wait-timeout", + "180", + "api", + "worker", + "lb", + ] + ) +def wait_ready_after_fault(client, actor, attachment, seconds=180): + deadline = time.monotonic() + seconds + path = "/api/v1/attachments/" + attachment["id"] + temporary_errors = 0 + + while time.monotonic() < deadline: + try: + result = client.call( + "GET", + path, + actor=actor, + schema="Attachment", + ) + + except Failure as error: + # Client проверяет HTTP-статус до разбора тела. + # Повторяем только конкретные временные HTTP-ошибки. + message = str(error) + + retryable = any( + message == ( + f"GET {path}: expected [200], received {status}" + ) + for status in (502, 503, 504) + ) + + if not retryable: + raise + + temporary_errors += 1 + + print( + f"Waiting for API recovery: " + f"temporary HTTP errors={temporary_errors}", + flush=True, + ) + + time.sleep(1) + continue + + metadata = result["json"] + + check( + metadata["status"] != "failed", + "attachment reached failed during recovery", + ) + + if metadata["status"] == "ready": + print( + "Attachment reached ready; " + f"temporary HTTP errors={temporary_errors}", + flush=True, + ) + return metadata + + check( + metadata["status"] in ("queued", "processing"), + "unexpected attachment status", + ) + + time.sleep(0.5) + + raise Failure( + "Attachment did not become ready after fault " + f"within {seconds} seconds; " + f"temporary HTTP errors={temporary_errors}" + ) + +def main(): + args = arguments( + "Lab5 level4 fault tests", + lambda parser: parser.add_argument( + "--bucket", + default="messenger-attachments", + ), + ) + + for filename in ( + "compose.yml", + "compose.override.yml", + "compose.faults.yml", + ): + check( + (ROOT / filename).is_file(), + "Required Compose file not found: " + filename, + ) + + client = Client(args.base_url, args.ca_file) + evidence = [] + + try: + print( + "Starting API and worker with test fault injection enabled...", + flush=True, + ) + + command( + FAULT + + [ + "up", + "-d", + "--no-build", + "--pull", + "never", + "--no-deps", + "--scale", + "api=2", + "--wait", + "--wait-timeout", + "180", + "api", + "worker", + "lb", + ] + ) + + wait( + lambda: client.call( + "GET", + "/health/ready", + schema="Health", + ), + "API ready", + ) + + sql( + "UPDATE attachment_test_faults SET remaining = 0" + ) + + wait( + lambda: sql( + """ + SELECT count(*) + FROM attachment_jobs + WHERE status IN ('queued', 'processing') + """ + ) == "0", + "existing work completed", + ) + + alice, bob, _, chat = client.bootstrap() + + def upload(link=False): + attachment, original = client.attachment( + alice, + chat, + ) + + if link: + client.message( + alice, + chat, + "", + attachments=[attachment["id"]], + ) + + return attachment, original + + def verify(attachment, original): + ready = wait_ready_after_fault( + client, + bob, + attachment, + seconds=180, + ) + + path = ( + "/api/v1/attachments/" + + attachment["id"] + + "/content" + ) + + check( + client.call( + "GET", + path, + actor=bob, + )["bytes"] == original, + "original changed", + ) + + expected = { + "thumbnail": ("image/jpeg", (256, 160)), + "crop": ("image/jpeg", (128, 128)), + "optimized": ("image/webp", (320, 200)), + } + + check( + {v["name"] for v in ready["variants"]} + == set(expected) + and len(ready["variants"]) == 3, + "expected three unique variants", + ) + + for variant in ready["variants"]: + mime, dimensions = expected[variant["name"]] + + result = client.call( + "GET", + path + "?variant=" + variant["name"], + actor=bob, + ) + + data = result["bytes"] + + actual = ( + webp_size(data) + if mime == "image/webp" + else jpeg_size(data) + ) + + check( + actual + == dimensions + == ( + variant["width"], + variant["height"], + ), + "wrong actual dimensions", + ) + + check( + result["headers"].get_content_type() + == mime + == variant["content_type"], + "wrong MIME", + ) + + check( + len(data) == variant["size_bytes"] + and hashlib.sha256(data).hexdigest() + == variant["sha256"], + "variant checksum mismatch", + ) + + if mime == "image/jpeg": + check( + b"Exif\x00\x00" not in data, + "JPEG contains EXIF", + ) + + return state(attachment["id"]) + + # Сценарий 1: + # оригинал и задание уже сохранены, + # но API падает до публикации в брокер. + print( + "Testing crash between COMMIT and publish...", + flush=True, + ) + + arm("hold_publish") + + attachment, original = upload(link=True) + before = state(attachment["id"]) + + check( + before["status"] == "queued", + "publish hold did not work", + ) + + baseline = restarts("api") + + arm("before_publish") + arm("hold_publish", 0) + + wait( + lambda: hit( + "before_publish", + before["job_id"], + ), + "crash before publish", + ) + + wait( + lambda: restarts("api") > baseline, + "API actually restarted", + ) + + after = verify(attachment, original) + + check( + after["job_id"] == before["job_id"], + "job was replaced", + ) + + evidence.append( + { + "scenario": "commit_before_publish", + **after, + } + ) + + print( + "PASS: real API crash after COMMIT; " + "the same job reached ready", + flush=True, + ) + + # Сценарии 2 и 3: + # падение после записи объектов; + # падение после фиксации ready, но до ACK. + for point in ["after_objects", "before_ack"]: + print( + "Testing worker crash: " + point + "...", + flush=True, + ) + + baseline = restarts("worker") + + arm(point) + + attachment, original = upload() + before = state(attachment["id"]) + + wait( + lambda: hit(point, before["job_id"]), + point, + ) + + snapshot = state(attachment["id"]) + old_keys = [] + + if point == "after_objects": + check( + snapshot["status"] == "processing" + and not snapshot["variants"], + "result committed before after_objects gate", + ) + + check( + snapshot["owner"] is not None, + "processing lease owner missing", + ) + + suffix = ( + chat["id"] + + "/" + + attachment["id"] + + "/v1/" + + snapshot["owner"] + ) + + old_keys = ["originals/" + suffix] + [ + "variants/" + suffix + "/" + name + for name in [ + "thumbnail.jpg", + "crop.jpg", + "optimized.webp", + ] + ] + + check( + all( + object_exists(args.bucket, key) + for key in old_keys + ), + "S3 writes missing", + ) + + else: + check( + snapshot["status"] == "ready", + "result not committed before before_ack gate", + ) + + wait( + lambda: restarts("worker") > baseline, + "worker actually restarted", + ) + + after = verify(attachment, original) + + if point == "before_ack": + marker = ( + "Duplicate job verified: job=" + + after["job_id"] + ) + + wait( + lambda: marker in command( + BASE + + [ + "logs", + "--tail", + "500", + "worker", + ] + ).stdout, + "duplicate checked after lost ACK", + ) + + comparison = { + "snapshot_status": snapshot["status"], + "after_status": after["status"], + "snapshot_attempts": snapshot["attempts"], + "after_attempts": after["attempts"], + "same_job": ( + after["job_id"] == snapshot["job_id"] + ), + "same_original_key": ( + after["original_key"] + == snapshot["original_key"] + ), + "same_variants": ( + after["variants"] == snapshot["variants"] + ), + "snapshot_variants": snapshot["variants"], + "after_variants": after["variants"], + "attempt_events": after["events"], + } + + print( + "BEFORE_ACK comparison:\n" + + json.dumps( + comparison, + ensure_ascii=False, + indent=2, + ), + flush=True, + ) + + check( + snapshot["status"] == "ready" + and after["status"] == "ready" + and after["attempts"] == snapshot["attempts"] + and after["job_id"] == snapshot["job_id"] + and after["variants"] == snapshot["variants"] + and after["original_key"] == snapshot["original_key"] + and after["events"] == snapshot["events"], + "duplicate changed effect", + ) + + else: + check( + after["attempts"] == 2, + "lease recovery did not create one retry", + ) + + staging = snapshot["staging"] + check(bool(staging), "staging key missing") + + wait( + lambda: state(attachment["id"])["staging"] + is None + and not any( + object_exists(args.bucket, key) + for key in old_keys + [staging] + ), + "safe staging/orphan cleanup", + seconds=240, + ) + + # После очистки подтверждённые результаты + # должны оставаться доступными. + after = verify(attachment, original) + + evidence.append( + { + "scenario": point, + **after, + } + ) + + print( + "PASS: " + + point + + "; durable result verified", + flush=True, + ) + + # Сценарии 4 и 5: + # постоянная ошибка без повторов; + # временная ошибка с ограниченным retry; + # явный replay и повторный replay готовой задачи. + for point, count, attempts in [ + ("permanent_error", 1, 1), + ("transient_error", 3, 3), + ]: + print( + "Testing " + point + " and explicit replay...", + flush=True, + ) + + arm(point, count) + + attachment, original = upload() + + failed = wait( + lambda: ( + value + if ( + value := state(attachment["id"]) + )["status"] == "failed" + else None + ), + "bounded failure", + ) + + check( + failed["attempts"] == attempts, + "incorrect retry count", + ) + + check( + bool(failed["events"]), + "failure attempt events missing", + ) + + for event in failed["events"]: + cap = min( + 60, + 2 ** min(event["attempt"], 6), + ) + + if event["state"] == "queued": + valid_delay = ( + 1 <= event["delay_seconds"] <= cap + ) + else: + valid_delay = ( + event["state"] == "failed" + and event["delay_seconds"] == 0 + ) + + check( + valid_delay, + "invalid backoff/jitter bounds", + ) + + check( + "REPLAY_RESULT=enqueued" + in replay(failed["job_id"]), + "replay failed", + ) + + ready = verify(attachment, original) + + check( + "REPLAY_RESULT=already_ready" + in replay(failed["job_id"]), + "ready replay not a no-op", + ) + + repeated = state(attachment["id"]) + + check( + repeated["attempts"] == ready["attempts"] + and repeated["variants"] == ready["variants"] + and repeated["original_key"] == ready["original_key"], + "replay changed successful effect", + ) + + evidence.append( + { + "scenario": point, + "failed": failed, + "ready": repeated, + } + ) + + print( + "PASS: " + + point + + "; bounded attempts, explicit replay " + "and ready no-op", + flush=True, + ) + + # Сценарий 6: + # зависший процесс обработки должен быть остановлен + # по таймауту, после чего задача повторяется. + print( + "Testing image-process timeout...", + flush=True, + ) + + arm("image_timeout") + + attachment, original = upload() + result = verify(attachment, original) + + check( + result["attempts"] == 2 + and any( + event["error_code"] == "processing_timeout" + for event in result["events"] + ), + "hard image-process timeout not observed", + ) + + evidence.append( + { + "scenario": "image_timeout", + **result, + } + ) + + print( + "PASS: stuck child process killed; retry succeeded", + flush=True, + ) + + finally: + restore_normal_configuration() + + # Отчёт об успехе записывается только после прохождения + # всех сценариев и восстановления обычной конфигурации. + path = ROOT / "results" / "lab5-level4.json" + path.parent.mkdir(parents=True, exist_ok=True) + + path.write_text( + json.dumps( + { + "passed": True, + "scenarios": evidence, + }, + ensure_ascii=False, + indent=2, + ) + "\n", + encoding="utf-8", + ) + + print( + "PASS: level4 scenarios completed; " + "results/lab5-level4.json saved", + flush=True, + ) + + +if __name__ == "__main__": + run(main) \ No newline at end of file