diff --git a/.env.example b/.env.example index ac8273f..65196d0 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,5 @@ # Скопируйте в .env для локального окружения; .env не коммитится. -# Настройки приложения и Compose; параметры будущей очереди пока не используются. +# Настройки приложения, Compose и очереди lab5. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab @@ -72,10 +72,28 @@ MINIO_ROOT_USER_FILE=/run/secrets/minio_root_user MINIO_ROOT_PASSWORD_FILE=/run/secrets/minio_root_password MINIO_BROWSER=off -# Lab5: будущие параметры очереди/worker, пока без реализации. +# Lab5: отдельный RabbitMQ broker и ограниченный worker. WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 -# Брокер пока не выбран; файл ещё не создаётся и не монтируется. -BROKER_URL_FILE=/run/secrets/broker_url +BROKER_HOST=broker +BROKER_PORT=5672 +BROKER_PASSWORD_FILE=/run/secrets/broker_password +BROKER_QUEUE=attachment-jobs +BROKER_USER=messenger +BROKER_VHOST=/ +JOB_LEASE_SECONDS=90 +JOB_RETRY_SECONDS=5 +PUBLISHER_POLL_SECONDS=1 + +# Качество вариантов и лимиты одного процесса обработки +IMAGE_JPEG_QUALITY=85 +IMAGE_WEBP_QUALITY=80 +IMAGE_WEBP_METHOD=4 +IMAGE_MEMORY_MIB=768 +IMAGE_CPU_SECONDS=20 + +# Retry cap and retained failed sources for explicit replay +JOB_RETRY_MAX_SECONDS=120 +FAILED_STAGING_RETENTION_SECONDS=604800 diff --git a/Makefile b/Makefile index 112af63..a900c25 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,8 @@ endef rolling-update failover-all test-lab3-grade4 help: - @echo "Lab4: init | test-grade4 | test-s3 | load-uploads | cleanup-dry-run | cleanup-apply" + @echo "Lab5: init | worker-stop | worker-start | jobs-logs | cleanup-dry-run | cleanup-apply" + @echo "Lab5: test-grade4 | test-jobs; legacy lab4: test-s3 | load-uploads" @echo "Проверки: check | validate | test | load | test-persistence ACTION_SCRIPT=scripts/restart.sh" @echo " test-failover ACTION_SCRIPT=scripts/stop-one.sh" @echo "Основной: up | down | restart | restore | ps | logs | migrate | instances" @@ -65,10 +66,13 @@ init: test-grade4: $(MAKE) check validate $(MAKE) test-stand-up - $(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover - $(TEST_ENV) $(PYTHON) -m scripts.check_s3 --base-url "$(TEST_BASE_URL)" --with-outage --with-faults - $(TEST_ENV) $(PYTHON) -m scripts.upload_load --base-url "$(TEST_BASE_URL)" --concurrency 10 - $(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)" + $(MAKE) test-docker + $(MAKE) test-stand-smoke + $(MAKE) test-stand-idempotency + $(MAKE) test-stand-persistence + $(MAKE) test-stand-failover + $(TEST_ENV) $(PYTHON) -m scripts.check_outbox_recovery --base-url "$(TEST_BASE_URL)" + $(TEST_ENV) $(PYTHON) -m scripts.check_lab5_jobs --base-url "$(TEST_BASE_URL)" test-s3: $(PYTHON) -m scripts.check_s3 --base-url "$(BASE_URL)" --with-outage --with-faults @@ -226,3 +230,23 @@ expiry-demo: test-stand-up failure-demo: test-stand-up $(TEST_ENV) BASE_URL=$(TEST_BASE_URL) scripts/failure_demo.sh + +# --- Очередь lab5 --- +.PHONY: worker-stop worker-start jobs-logs +worker-stop: + $(COMPOSE) stop worker + +worker-start: + $(COMPOSE) start worker + +jobs-logs: + $(COMPOSE) logs -f publisher worker broker + +# Уже работающий изолированный стенд; сам цель его не поднимает. +.PHONY: test-outbox-recovery +test-outbox-recovery: + $(PYTHON) -m scripts.check_outbox_recovery --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: test-jobs +test-jobs: + $(PYTHON) -m scripts.check_lab5_jobs --base-url "$(BASE_URL)" diff --git a/REPORT.md b/REPORT.md index 037d0fe..94e40ab 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,75 +1,225 @@ # Отчёт — лабораторная 5 -- Автор, группа: Григорян Илья Мурадович, БПИ2303 (из отчёта lab4). -- Целевая оценка: 5; исследовательский трек пока не выбран. -- Этап: перенос базы lab4 и заготовки с docstring, без реализации lab5. -- Дата переноса: 07.11.2026. -- Commit/PR lab5: заполнить после публикации. -- Использованные источники: README-файлы и contracts/ репозитория lab5, - исходники lab4. Использована помощь Codex. Новых зависимостей нет. +Автор: Григорян Илья Мурадович, БПИ2303. +Целевая оценка текущего этапа: **4**. Обновлено 10.11.2026. +Дальнейшая цель — 5; исследовательский трек ещё не выбран. +Использована помощь Codex при реализации, проверках и исправлениях. +Commit/PR: заполнить после публикации. -## Запуск из чистого clone +## Запуск -Нужны Python 3.12, pip, Docker с Compose. Подготовка и запуск: +Нужны Python 3.12, pip, Docker с Compose. Зависимости закреплены в requirements.txt. ```sh python3 -m venv .venv .venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt make init make up +make test ``` -HTTP: http://localhost:18081. `make down` останавливает стенд с сохранением данных. -Тестовый стенд: `make test-stand-up`, остановка: `make test-stand-down`. -Намеренное удаление учебных данных: `make test-stand-destroy` для тестового стенда, -`docker compose --env-file .env down --volumes` для основного. -Версии зависимостей перенесены без обновления. Локальные secrets/.env генерируются заново, не копируются из lab4. -Стенд при переносе не запускался; воспроизводимость после переноса ещё не проверена. +`make init` создаёт отсутствующие .env и файлы secrets/, сохраняя существующие. +Все environment-параметры Compose подставляются из .env; образец — .env.example. +Пароли и S3 credentials читаются из файлов secrets, не включаются в задания. +API не получает credentials брокера. По умолчанию HTTP доступен на +http://localhost:18081. Broker, MinIO и основные PostgreSQL/Redis наружу не открыты. -## Архитектура и решения +При обновлении ранее запущенного стенда сначала остановить старые процессы +через `make down` без удаления volumes, затем выполнить `make up`. +Migrate применяет ревизии до a62c950e18b3 перед стартом приложения. +Миграции добавляют jobs, варианты, бюджет попыток и аудит replay. Готовые +вложения раннего варианта lab5 переобрабатываются для заполнения crop/WebP. +При переходе Redis Streams → RabbitMQ миграция повторно публикует queued jobs +из PostgreSQL; processing восстанавливаются по истечении lease. Старые процессы +обоих брокеров во время перехода должны быть остановлены. -Перенесены приложение FastAPI, PostgreSQL-миграции, общие Redis-сессии, -идемпотентность сообщений, приватный MinIO, две API-реплики за Nginx, собственные -тесты и скрипты эксплуатации lab4. Перенесена и незакоммиченная правка MinIO -bootstrap из исходного рабочего дерева. Исходный lab4 не изменён. +## Инфраструктура и устойчивый приём -Имена Compose-проектов, образы и порты отделены от lab4. Рабочие настройки — -.env.example; там же указаны будущие параметры очереди, пока не используемые. -Compose получает настройки environment из .env, включая пути S3, PostgreSQL -и MinIO к файлам секретов; секретные значения остаются в secrets/. -Пути *_FILE в Compose также задают точки монтирования соответствующих secrets; -MinIO bootstrap читает эти пути из окружения. -Публичные tests/, contracts/, lab.json и README.md lab5 сохранены. +| Компонент | Реализация | Данные | +| --- | --- | --- | +| API | две реплики FastAPI, Python 3.12.15 | PostgreSQL и Redis сессий | +| Балансировка | nginx 1.28.0 | единая HTTP-точка входа | +| База | PostgreSQL 16.10 | сообщения, metadata, jobs, outbox, audit | +| Сессии | Redis 7.4.2, AOF always | redisdata | +| Объекты | приватный MinIO | staging, originals, variants | +| Брокер | RabbitMQ 4.4.0 | durable очередь, rabbitmqdata | +| Publisher | отдельный процесс приложения, aio-pika 10.2.2 | PostgreSQL → RabbitMQ | +| Worker | отдельный процесс приложения | RabbitMQ → MinIO + PostgreSQL | +| Cleanup | отдельный процесс | незавершённые upload и terminal staging | -Будущая архитектура: API → durable staging + PostgreSQL job/outbox → publisher → -broker → worker → конечный MinIO + атомарная фиксация metadata в PostgreSQL. -Текущая реализация продолжает синхронно сохранять оригинал и возвращает 201 ready. -Заготовки lab5 не подключены и вызывают NotImplementedError. Реализация транзакционных границ и восстановления после отказов ещё предстоит. +Redis используется для сессий. Очередь заданий использует RabbitMQ через AMQP. +Очередь durable, сообщения persistent. Publisher ожидает подтверждение брокером +приёма сообщения (publisher confirm) и только затем фиксирует published_attempt. +При неизвестном результате публикации возможен дубль; намерение остаётся в БД. +Подтверждение worker (consumer ack) следует после устойчивого результата или +устойчиво сохранённого перехода retry/failed. При разрыве соединения RabbitMQ +возвращает неподтверждённые доставки в очередь. -## Доказательства критериев +До ответа `202 queued` API полностью принимает и проверяет оригинал, сохраняет +его в приватный staging и фиксирует attachment/job одной транзакцией PostgreSQL. +Job одновременно служит transactional outbox — устойчивым намерением публикации. +Worker и broker не входят в синхронный путь upload и readiness API. Остановка +worker не заставляет API выполнять resize. При компенсации неоднозначного COMMIT +API проверяет БД, чтобы не удалить оригинал уже принятого задания. -| Критерий README | Будущая команда/сценарий | Текущий результат | Доказательство | -| --- | --- | --- | --- | -| Публичный smoke lab5 | make test | Не запускался; 202/thumbnail ещё не реализованы | Нет | -| 3: сохранён минимум lab4 | Собственные тесты, persistence/failover | Перенесён, повторно не проверен | Не подтверждает lab5 | -| 3: durable приём и broker/worker | Worker stop → upload → restart → ready | Не реализовано | Нет | -| 3: состояния, original/thumbnail, retry, дубли | Smoke + собственные сценарии | Не реализовано | Нет | -| 4: crop/optimized, resource limits | Собственные проверки изображений/лимитов | Не реализовано | Нет | -| 4: outbox и восстановление COMMIT/publish | Fault injection | Не реализовано | Нет | -| 4: backoff/jitter, failed/replay | Сбой до исчерпания и явный replay | Не реализовано | Нет | -| 4: crash до ack, lease, concurrency, cleanup | Fault injection и lifecycle | Не реализовано | Нет | -| 5: законченный исследовательский трек | Методика после выбора трека | Трек не выбран | Нет | +Envelope соответствует contracts/jobs.schema.json: job_id, attachment_id, +pipeline_version=1, attempt, created_at, schema_version, type и необязательный +trace context. Bytes, имена объектов S3, URL и credentials туда не помещаются. +Доступ к вложениям проверяется по участию в чате; content до ready возвращает 409. +Аутентификация, пагинация, идемпотентность сообщений и ограничения lab4 сохранены. -## Эксперименты и отказы +## Изображения и ограничения -Пока не проводились. После выбора трека описать гипотезу, hardware/лимиты, -версии, размеры/число файлов, профиль нагрузки, concurrency/prefetch, длительность, -прогрев и повторы. Сохранить raw summary без секретов, queue lag, время до ready, -RSS и другие метрики выбранного трека. +Worker повторно сверяет размер и SHA-256 исходника. Original сохраняется без +перекодирования. Все варианты создаются в отдельном ограниченном процессе: -## Ограничения и следующие шаги +| Вариант | Формат и размеры | Параметры | +| --- | --- | --- | +| thumbnail | JPEG, вписан в 256×256 без увеличения | quality=85, белая подложка alpha | +| crop | JPEG, центральный crop 128×128 | quality=85; маленькое изображение увеличивается | +| optimized | WebP, вписан в 1280×1280 без увеличения | quality=80, method=4, alpha сохраняется | -Сейчас это база для дальнейшей реализации, не завершённая lab5. Очереди и -асинхронной обработки нет; прежние результаты lab4 к этому дереву не относятся. -Сначала просмотр пользователем и выбор трека/брокера, затем реализация; -тестовый прогон и новые evidence — по отдельному указанию после реализации. +Применяется EXIF orientation; EXIF/ICC/XMP и текстовые metadata не переносятся +в варианты. JPEG создаётся из RGB с белой подложкой, WebP — из чистого RGBA. +Алгоритмы используют Pillow 13.0.0. Metadata содержит реальные размеры, bytes и SHA-256. + +Лимиты: PNG/JPEG, 10 MiB, 40 млн пикселей; JOB_TIMEOUT_SECONDS=60; +IMAGE_MEMORY_MIB=768 адресного пространства и IMAGE_CPU_SECONDS=20 CPU на процесс. +Контейнер worker: 2 GiB, 2 CPU, tmpfs 128 MiB. Отмена завершает дочерний процесс; +S3 adapter дожидается SDK-потока перед освобождением блокировки/компенсацией. + +## Повторы, идемпотентность и очистка + +Состояния: queued → processing → ready; временный отказ возвращает queued, +постоянный отказ или исчерпание бюджета переводит failed с безопасным error_code. +JOB_MAX_ATTEMPTS=5 на цикл. Номер attempt монотонен, бюджет закреплён в attempt_limit. +Для попытки n текущего цикла ceiling=min(JOB_RETRY_MAX_SECONDS, +JOB_RETRY_SECONDS × 2^(n−1)); delay равномерно выбирается в [ceiling/2, ceiling]. +База — 5 секунд, максимум — 120 секунд. next_attempt_at сохраняется в БД; +дубликат не пересчитывает задержку. Последняя причина остаётся в job даже при +публичном retry_exhausted. Ошибки изображения/исходника/конфигурации постоянные; +timeout, временные ошибки S3/сети и файловой системы повторяются. + +Job захватывается с lease_token и JOB_LEASE_SECONDS=90. Publisher восстанавливает +истёкший processing в пределах бюджета. Перед PUT worker блокирует строку job, +проверяет актуальность token и lease, держит блокировку до записи original, +трёх вариантов и COMMIT ready. Reaper пропускает заблокированные строки. +Устаревшая попытка не может начать запись. Фиксированные ключи по +(attachment_id, pipeline_version, variant) исключают второй набор объектов. +Дубль ready подтверждается без обработки и PUT. Это доставка как минимум один +раз с идемпотентным эффектом, не exactly-once выполнение. + +WORKER_CONCURRENCY=2, отдельный канал каждого consumer, prefetch=1 на канал. +Publisher и worker используют init:true: Python не является PID 1, поэтому +fault hooks с SIGKILL действительно завершают процесс. SIGTERM отменяет работу +без преждевременного ack. Некорректные envelopes отклоняются без сохранения payload. + +Queued/processing staging не удаляется. Для ready retention — 86400 секунд; +для failed — FAILED_STAGING_RETENTION_SECONDS=604800 (семь суток). +Cleanup и replay блокируют одну строку job. Если staging совпадает с готовым +original, объект сохраняется. Частичные конечные объекты failed автоматически +не удаляются; после replay используются те же ключи. + +## Реестр failed и операторский replay + +Failed registry находится в PostgreSQL. Команды требуют операторского доступа +к Docker/credentials стенда; публичного endpoint для replay нет: + +```sh +docker compose run --rm --no-deps worker python -m app.core.job_admin list-failed +docker compose run --rm --no-deps worker python -m app.core.job_admin show JOB_ID +docker compose run --rm --no-deps worker python -m app.core.job_admin replay JOB_ID --request-id REQUEST_UUID --actor operator --reason dependency_restored +docker compose run --rm --no-deps worker python -m app.core.job_admin history JOB_ID +``` + +JOB_ID/REQUEST_UUID заменить настоящими UUID. Reasons: dependency_restored, +configuration_fixed, manual_review. Actor — аудиторская метка, не система входа. +Replay принимает только failed, сверяет сохранённый исходник по bytes/SHA-256, +атомарно записывает аудит и новый ограниченный бюджет. Счётчик attempt не +сбрасывается, старые envelopes не запускают новый цикл. Повтор того же request-id +с теми же параметрами возвращает already_applied даже после ready; конфликт +параметров отклоняется. После неоднозначного ответа повторяют тот же request-id. +Готовую job с новым request-id повторить нельзя. После удаления staging replay +отклоняется с source_expired; отсутствующий объект даёт source_missing. + +## Проверки уровня 3 и 4 + +Полный воспроизводимый прогон на отдельном стенде: + +```sh +make test-grade4 PYTHON=.venv/bin/python +``` + +Цель последовательно запускает контракт/OpenAPI, тестовый стенд, собственные +тесты с реальными PostgreSQL/Redis, публичный smoke, идемпотентность через две +реплики, persistence/failover, outbox и сценарии worker/replay. Подготовка БД +для pytest выполняется только в выделенной базе *_test, Redis — DB15. +Публичные tests/ и обязательные contracts/ не изменены. + +Отдельные сценарии на уже работающем изолированном стенде: + +```sh +make test-outbox-recovery BASE_URL=http://localhost:28080 +make test-jobs BASE_URL=http://localhost:28080 +``` + +При выборе своего проекта задать COMPOSE_PROJECT_NAME и соответствующие порты +в окружении. Сценарии сверяют BASE_URL с API выбранного Compose-проекта, +создают синтетические данные и возвращают штатные worker/publisher в finally. +Перезапуски сохраняют volumes. Сброса пользовательских данных нет. + +Результаты и сопоставление каждому пункту приведены в evidence/lab5-verification.json; +подробные снимки состояний/объектов/retry — в evidence/lab5-jobs.json. +Секреты, tokens и исходные broker payload в evidence не записываются. + +## Фактический результат проверок + +Все перечисленные проверки завершились успешно на отдельном проекте +`mtusi-lab5-verification`, HTTP http://localhost:28080, без удаления volumes: + +| Проверка | Результат | +| --- | --- | +| Собственные тесты PostgreSQL/Redis/API/миграций/изображений/OS-лимитов | 199 passed, без пропусков | +| Дополнительные timeout и failed retention/SHA-256 | 2 passed | +| Контракт, OpenAPI и Compose config | PASS | +| Публичный smoke | PASS | +| 24 конкурентных повтора сообщения через две API-реплики | PASS | +| Публичные persistence и failover | PASS | +| SIGKILL publisher после COMMIT до publish | 137 → queued → ready; без повторного upload | +| Остановленный worker, рестарт API/брокера/хранилищ | ready того же ID; сообщение RabbitMQ сохранилось с volume | +| SIGKILL после четырёх PUT до COMMIT | processing → ready на попытке 2; тот же набор/checksums объектов | +| SIGKILL после COMMIT до ack | ready-дубль подтверждён без изменения metadata/Last-Modified | +| Временная ошибка и replay | failed после 5 попыток, затем ready; повтор request-id безопасен | +| Постоянная ошибка и replay | failed после 1 попытки, затем ready; новый replay ready отклонён | +| Конкуренция/подтверждения RabbitMQ после сценариев | 2 consumer, prefetch=1; ready=0, unacknowledged=0 | +| Неизменность публичных tests/contracts/README и whitespace | PASS | + +Суммарно выполнен 201 собственный тест двумя прогонами; новые тесты включены +в последующие полные запуски make test-grade4. Тесты намеренно меняют только +выделенную БД *_test и Redis DB15. API и worker в интеграционных сценариях +используют собственную БД тестового Compose-проекта. + +Измеренные задержки пяти неудачных попыток: 4.613, 6.837, 17.323, 30.885, +42.633 секунды; все попали в соответствующие интервалы jitter. Задержка +последней failed-попытки сохраняется для диагностики, автоматический retry +после исчерпания бюджета не выполняется. Ограничение максимума проверено отдельно. + +Во время проверки исправлена проблема fault hooks с PID 1 через init:true. +Старые собственные проверки lab4 адаптированы к обязательному переходу +202 queued → ready; публичные проверки не изменены. Цель make test-grade4 +переведена на lab5 и последовательные сценарии, чтобы рестарты инфраструктуры +не пересекались с регрессией БД. RabbitMQ используется как брокер, Redis — для сессий. + +## Границы результата + +Проверки относятся к одному Docker-хосту и рестартам с сохранением volumes. +PostgreSQL, RabbitMQ и MinIO не реплицированы; потеря диска/хоста не покрыта. +Ожидание в очереди не ограничено; admission control и исследование перегрузки +относятся к будущему уровню 5. Pipeline фиксирован на версии 1; изменение +алгоритмов требует решения о новой версии/миграции результатов. + +Официальные источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), +[publisher confirms и consumer acknowledgements](https://www.rabbitmq.com/docs/confirms), +[aio-pika](https://docs.aio-pika.com/quick-start.html), +[Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), +[Pillow WebP](https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#webp). diff --git a/app/core/background.py b/app/core/background.py new file mode 100644 index 0000000..06a4d9a --- /dev/null +++ b/app/core/background.py @@ -0,0 +1,33 @@ +"""Остановка фоновых процессов и безопасные сообщения об отказах.""" + +import asyncio +import logging +import signal + +from aio_pika.exceptions import AMQPException, ChannelInvalidStateError +from sqlalchemy.exc import InterfaceError, OperationalError +from sqlalchemy.exc import TimeoutError as PoolTimeoutError + +TRANSIENT_JOB_ERRORS = (AMQPException, ChannelInvalidStateError, InterfaceError, OperationalError, PoolTimeoutError, OSError) +logger = logging.getLogger(__name__) + + +def shutdown_event() -> asyncio.Event: + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for signum in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(signum, stop.set) + return stop + + +async def wait_or_stop(stop: asyncio.Event, seconds: float) -> None: + try: + async with asyncio.timeout(seconds): + await stop.wait() + except TimeoutError: + pass + + +def log_background_error(operation: str, error: Exception) -> None: + # SDK/DB exceptions могут содержать параметры и credentials; выводим только тип. + logger.warning("%s: %s", operation, type(error).__name__) diff --git a/app/core/cleanup.py b/app/core/cleanup.py index 7e75507..d520139 100644 --- a/app/core/cleanup.py +++ b/app/core/cleanup.py @@ -6,19 +6,22 @@ from dataclasses import asdict import json from app.core.config import Settings +from app.core.jobs import JobSettings from app.core.database import create_db_engine, create_session_factory from app.core.s3 import S3Settings from app.domain.attachments import reconcile_attachments +from app.domain.attachment_jobs import cleanup_staging from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error from app.storage.s3 import create_s3_storage -async def cleanup_once(factory, storage, apply: bool) -> bool: +async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool: completed = True try: async with factory() as session: result = await reconcile_attachments(session, storage, dry_run=not apply) - print(json.dumps(asdict(result)), flush=True) + staging = await cleanup_staging(session, storage, jobs, apply=apply) + print(json.dumps({**asdict(result), "terminal_staging": staging}), flush=True) except RECOVERABLE_STORAGE_ERRORS as error: log_recoverable_error("Reconciliation", error) completed = False @@ -28,12 +31,13 @@ async def cleanup_once(factory, storage, apply: bool) -> bool: async def run(once: bool, apply: bool) -> None: settings = Settings() # type: ignore s3_settings = S3Settings() + jobs = JobSettings() engine = create_db_engine(settings) factory = create_session_factory(engine) storage = create_s3_storage(s3_settings) try: while True: - completed = await cleanup_once(factory, storage, apply) + completed = await cleanup_once(factory, storage, apply, jobs) if once: if not completed: raise SystemExit(1) diff --git a/app/core/image_processing.py b/app/core/image_processing.py new file mode 100644 index 0000000..9eb46de --- /dev/null +++ b/app/core/image_processing.py @@ -0,0 +1,14 @@ +"""Качество вариантов и пределы отдельного процесса обработки.""" + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class ImageSettings(BaseSettings): + model_config = SettingsConfigDict(env_prefix="IMAGE_", env_file=".env", extra="ignore") + + jpeg_quality: int = Field(default=85, ge=1, le=95) + webp_quality: int = Field(default=80, ge=1, le=100) + webp_method: int = Field(default=4, ge=0, le=6) + memory_mib: int = Field(default=768, ge=256, le=1024) + cpu_seconds: int = Field(default=20, ge=1, le=120) diff --git a/app/core/job_admin.py b/app/core/job_admin.py new file mode 100644 index 0000000..369b9f2 --- /dev/null +++ b/app/core/job_admin.py @@ -0,0 +1,65 @@ +"""Операторский интерфейс failed registry. Доступ требует credentials стенда.""" + +import argparse +import asyncio +import json +from uuid import UUID + +from app.core.background import log_background_error +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.core.jobs import JobSettings +from app.domain.job_admin import REPLAY_REASONS, history, list_failed, replay_failed_job, show_job +from app.errors.exceptions import JobOperationError +from app.storage.s3 import create_s3_storage + + +async def execute(args): + settings = Settings() + engine = create_db_engine(settings) + storage = None + try: + factory = create_session_factory(engine) + async with factory() as session: + if args.command == "list-failed": + result = await list_failed(session, args.limit) + elif args.command == "show": + result = await show_job(session, args.job_id) + elif args.command == "history": + result = await history(session, args.job_id) + else: + storage = create_s3_storage(settings.s3) + result = await replay_failed_job(session, storage, JobSettings(), args.job_id, + request_id=args.request_id, actor=args.actor, reason=args.reason) + print(json.dumps(result, default=str), flush=True) + finally: + if storage is not None: + await storage.aclose() + await engine.dispose() + + +def main(): + parser = argparse.ArgumentParser(description="Реестр failed и явный replay с аудитом") + commands = parser.add_subparsers(dest="command", required=True) + listing = commands.add_parser("list-failed") + listing.add_argument("--limit", type=int, choices=range(1, 501), default=50, metavar="1..500") + for name in ("show", "history", "replay"): + command = commands.add_parser(name) + command.add_argument("job_id", type=UUID) + if name == "replay": + command.add_argument("--request-id", type=UUID, required=True) + command.add_argument("--actor", required=True) + command.add_argument("--reason", choices=REPLAY_REASONS, required=True) + args = parser.parse_args() + try: + asyncio.run(execute(args)) + except JobOperationError as error: + print(json.dumps({"error_code": error.code}), flush=True) + raise SystemExit(2) from None + except Exception as error: + log_background_error("Job administration failed", error) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/app/core/job_worker.py b/app/core/job_worker.py index 14f3629..4c36d0f 100644 --- a/app/core/job_worker.py +++ b/app/core/job_worker.py @@ -1,17 +1,137 @@ -"""Заготовка отдельного worker lab5, не одноразового валидатора image_worker. +"""Отдельные consumers с ограниченной конкуренцией; API не выполняет resize.""" -Здесь будут startup/shutdown ресурсов, цикл получения заданий, ограниченная -concurrency/prefetch, восстановление pending и остановка с сохранением работы. -Каждая задача получает отдельную AsyncSession. Обработка изображений идёт -в процессе с лимитами памяти/CPU/времени; CancelledError не подавляется. -В Compose и API модуль пока не подключён. -""" +import argparse +import asyncio +from dataclasses import dataclass +import json +import logging +import os +import signal +from uuid import UUID, uuid4 + +from PIL import features +from pydantic import ValidationError + +from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.core.jobs import JobSettings +from app.domain.attachment_jobs import process_attachment_job +from app.errors.exceptions import ProcessingError +from app.schemas.job import JobMessage +from app.storage.broker import JobBroker +from app.storage.s3 import create_s3_storage -async def run_worker() -> None: - """Запустить consumer с lease/retry, ограничением ресурсов и корректным ack. +@dataclass(frozen=True) +class Faults: + """Только явные CLI-флаги; обычный worker не включает инъекции сбоев.""" - SIGTERM должен завершить или оставить восстанавливаемую попытку; остановка - worker не заставляет API выполнять resize. Инициализация ресурсов — здесь. - """ - raise NotImplementedError("Lab5: отдельный worker ещё не реализован") + crash_after_object: UUID | None = None + crash_before_ack: UUID | None = None + fail_transient: UUID | None = None + fail_permanent: UUID | None = None + object_name: str = "optimized" + + def kill(self, attachment_id: UUID, phase: str) -> None: + print(json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), flush=True) + os.kill(os.getpid(), signal.SIGKILL) + + def after_object(self, attachment_id: UUID, name: str) -> None: + if attachment_id == self.crash_after_object and name == self.object_name: + self.kill(attachment_id, "after_object:" + name) + + def before_ack(self, attachment_id: UUID) -> None: + if attachment_id == self.crash_before_ack: + self.kill(attachment_id, "before_ack") + + def before_processing(self, attachment_id: UUID) -> None: + if attachment_id == self.fail_transient: + raise TimeoutError("injected transient failure") + if attachment_id == self.fail_permanent: + raise ProcessingError("injected_permanent") + + +async def consume_connected(factory, storage, broker: JobBroker, settings: JobSettings, stop: asyncio.Event, faults: Faults) -> None: + consumer = uuid4().hex + while not stop.is_set(): + try: + await broker.ensure_ready() + delivery = await broker.receive(consumer) + if delivery is None: + continue + try: + message = JobMessage.model_validate_json(delivery.payload) + except ValidationError: + await broker.reject_invalid(delivery.id) + continue + try: + async with factory() as session: + acknowledge = await process_attachment_job( + session, storage, message, settings, + after_object=faults.after_object, before_processing=faults.before_processing, + ) + except ProcessingError: + await broker.reject_invalid(delivery.id) + else: + if acknowledge: + faults.before_ack(message.attachment_id) + await broker.acknowledge_job(delivery.id) + else: + await broker.defer_job(delivery.id) + except TRANSIENT_JOB_ERRORS as error: + log_background_error("Worker delivery deferred", error) + await broker.aclose() + await wait_or_stop(stop, settings.publisher_poll_seconds) + + +async def consume(factory, storage, settings, stop, faults) -> None: + broker = JobBroker(settings) + try: + await consume_connected(factory, storage, broker, settings, stop, faults) + finally: + await broker.aclose() + + +async def run_worker(faults: Faults) -> None: + settings = Settings() + jobs = JobSettings() + logging.basicConfig(level=settings.log_level) + if not features.check("webp"): + logging.error("Pillow requires WebP support for the image worker") + raise RuntimeError("WebP codec is unavailable") + stop = shutdown_event() + engine = create_db_engine(settings) + storage = None + try: + storage = create_s3_storage(settings.s3) + factory = create_session_factory(engine) + async with asyncio.TaskGroup() as group: + tasks = [group.create_task(consume(factory, storage, jobs, stop, faults)) for _ in range(jobs.worker_concurrency)] + await stop.wait() + # Отмена убивает resize-процесс, дожидается SDK и не отправляет ack. + for task in tasks: + task.cancel() + finally: + if storage is not None: + await storage.aclose() + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Image worker с явными fault-injection hooks") + modes = parser.add_mutually_exclusive_group() + for flag in ("crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent"): + modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID") + parser.add_argument("--object-name", choices=("original", "thumbnail", "crop", "optimized"), default="optimized") + args = parser.parse_args() + faults = Faults(**vars(args)) + try: + asyncio.run(run_worker(faults)) + except Exception as error: + log_background_error("Worker stopped", error) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/app/core/jobs.py b/app/core/jobs.py new file mode 100644 index 0000000..e7badd3 --- /dev/null +++ b/app/core/jobs.py @@ -0,0 +1,35 @@ +"""Настройки очереди из .env; credentials брокера читаются отдельно из файла.""" + +from pathlib import Path +from typing import Self + +from pydantic import Field, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class JobSettings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", extra="ignore") + + broker_host: str = "broker" + broker_port: int = Field(default=5672, ge=1, le=65535) + broker_password_file: Path = Path("/run/secrets/broker_password") + broker_queue: str = "attachment-jobs" + broker_user: str = "messenger" + broker_vhost: str = "/" + worker_concurrency: int = Field(default=2, ge=1, le=4) + job_max_attempts: int = Field(default=5, ge=1, le=100) + job_timeout_seconds: int = Field(default=60, ge=10, le=300) + job_lease_seconds: int = Field(default=90, ge=20, le=600) + job_retry_seconds: int = Field(default=5, ge=1, le=300) + job_retry_max_seconds: int = Field(default=120, ge=1, le=3600) + failed_staging_retention_seconds: int = Field(default=604800, ge=300) + publisher_poll_seconds: float = Field(default=1, gt=0, le=10) + staging_retention_seconds: int = Field(default=86400, ge=300) + + @model_validator(mode="after") + def validate_lease(self) -> Self: + if self.job_lease_seconds < self.job_timeout_seconds + 30: + raise ValueError("Lease должен превышать timeout задания минимум на 30 секунд") + if self.job_retry_max_seconds < self.job_retry_seconds: + raise ValueError("Retry maximum must be at least the base delay") + return self diff --git a/app/core/outbox_publisher.py b/app/core/outbox_publisher.py index 04f2cf5..8d23e46 100644 --- a/app/core/outbox_publisher.py +++ b/app/core/outbox_publisher.py @@ -1,15 +1,70 @@ -"""Заготовка процесса доставки PostgreSQL outbox в durable broker. +"""Доставка намерений из PostgreSQL; crash hook включается только явно через CLI.""" -Не держать транзакцию/блокировки во время бесконечного сетевого ожидания. -Согласовать lease publisher, ограниченные пачки и повтор после COMMIT/publish. -Процесс пока не подключён к Compose или жизненному циклу API. -""" +import argparse +import asyncio +import json +import logging +import os +import signal +from uuid import UUID + +from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.core.jobs import JobSettings +from app.domain.attachment_jobs import publish_next_job, recover_expired_jobs +from app.storage.broker import JobBroker -async def run_publisher() -> None: - """Публиковать сохранённые намерения и фиксировать подтверждение брокера. +def crash_before_publish(attachment_id: UUID) -> None: + """Реальный SIGKILL после чтения принятой job, до publisher confirm и отметки publish.""" + print(json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), flush=True) + os.kill(os.getpid(), signal.SIGKILL) - После crash между publish и отметкой допустим дубль того же job_id. - Недоступность брокера не должна терять уже принятые задания. - """ - raise NotImplementedError("Lab5: outbox publisher ещё не реализован") + +async def run_publisher(crash_attachment_id: UUID | None = None) -> None: + settings = Settings() + jobs = JobSettings() + logging.basicConfig(level=settings.log_level) + stop = shutdown_event() + engine = create_db_engine(settings) + broker = None + try: + broker = JobBroker(jobs) + factory = create_session_factory(engine) + while not stop.is_set(): + published = False + try: + await broker.ensure_ready() + async with factory() as session: + await recover_expired_jobs(session, jobs) + published = await publish_next_job( + session, broker, attachment_id=crash_attachment_id, + before_publish=crash_before_publish if crash_attachment_id is not None else None, + ) + except TRANSIENT_JOB_ERRORS as error: + log_background_error("Publisher deferred", error) + if not published: + await wait_or_stop(stop, jobs.publisher_poll_seconds) + finally: + if broker is not None: + await broker.aclose() + await engine.dispose() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Публикация устойчивых jobs в RabbitMQ") + parser.add_argument( + "--crash-before-publish", type=UUID, metavar="ATTACHMENT_ID", + help="аварийно завершить этот процесс перед публикацией только указанного вложения", + ) + args = parser.parse_args() + try: + asyncio.run(run_publisher(args.crash_before_publish)) + except Exception as error: + log_background_error("Publisher stopped", error) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/app/core/thumbnail_worker.py b/app/core/thumbnail_worker.py new file mode 100644 index 0000000..cc3e2be --- /dev/null +++ b/app/core/thumbnail_worker.py @@ -0,0 +1,28 @@ +"""Один pipeline изображений в Linux-процессе с лимитами памяти и CPU.""" + +from pathlib import Path +import resource +import sys + +from app.core.image_processing import ImageSettings +from app.domain.image_pipeline import create_variants +from app.errors.exceptions import IMAGE_WORKER_ERRORS, pipeline_worker_error_code + + +def main() -> int: + settings = ImageSettings() + memory = settings.memory_mib * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + resource.setrlimit(resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds)) + source, directory, metadata, content_type, maximum = sys.argv[1:] + code = 0 + try: + result = create_variants(Path(source), Path(directory), content_type, int(maximum), settings) + Path(metadata).write_text(result.model_dump_json()) + except IMAGE_WORKER_ERRORS as error: + code = pipeline_worker_error_code(error) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/domain/attachment_jobs.py b/app/domain/attachment_jobs.py index 01deb9b..5f6d7dc 100644 --- a/app/domain/attachment_jobs.py +++ b/app/domain/attachment_jobs.py @@ -1,45 +1,198 @@ -"""Заготовки сценариев lab5; не подключены к upload или worker. +"""Приём, durable публикация, попытки и фиксация результата обработки.""" -Здесь появятся транзакционные границы и бизнес-условия; SQL и SDK остаются -в storage/. Одна AsyncSession принадлежит одной последовательной задаче. -""" - -from uuid import UUID +import asyncio +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import UUID, uuid4 +from minio.error import MinioException from sqlalchemy.ext.asyncio import AsyncSession +from urllib3.exceptions import HTTPError as StorageHTTPError + +from app.core.jobs import JobSettings +from app.domain.job_retry import classify_failure, retry_delay +from app.domain.job_files import fetch_original, render_variants +from app.errors.exceptions import ProcessingError +from app.schemas.attachment import VARIANT_FILENAMES +from app.schemas.job import JobMessage +from app.storage import jobs, outbox +from app.storage.broker import JobBroker +from app.storage.jobs import JobWork +from app.storage.s3 import MinioStorage -async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID) -> UUID: - """Создать job и outbox в транзакции queued metadata после durable staging. - - Возвращать стабильный job_id. Commit принадлежит сценарию приёма upload; - 202 допустим только после устойчивого оригинала и успешного commit. - """ - raise NotImplementedError("Lab5: постановка задания ещё не реализована") +@dataclass(frozen=True) +class Claim: + work: JobWork | None = None + acknowledge: bool = False -async def process_attachment_job(session: AsyncSession, job_id: UUID) -> None: - """Захватить job с lease и опубликовать original и обязательные варианты. - - Ready-дубль не меняет результат. Lease и версия защищают от устаревшей - попытки. S3/metadata фиксируются до ack; отмена не подавляется. - """ - raise NotImplementedError("Lab5: обработка задания ещё не реализована") +async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID: + """Включить намерение обработки в транзакцию queued metadata; без commit.""" + return await jobs.create_job(session, attachment_id, staging_key) -async def recover_expired_jobs(session: AsyncSession) -> int: - """Вернуть зависший processing в очередь после истечения lease. - - Учитывать лимит попыток и защиту от конкурентного завершения старым worker. - Точное правило heartbeat/deadline определяется до реализации. - """ - raise NotImplementedError("Lab5: восстановление lease ещё не реализовано") +async def publish_next_job( + session: AsyncSession, broker: JobBroker, *, + attachment_id: UUID | None = None, + before_publish: Callable[[UUID], None] | None = None, +) -> bool: + """Отметка publish следует за durable publisher confirm; crash допускает тот же job-дубль.""" + job = await outbox.next_unpublished_job(session, attachment_id=attachment_id) + published = job is not None + if job is not None: + if before_publish is not None: + before_publish(job.attachment_id) + await broker.publish_job(JobMessage( + job_id=job.id, attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, attempt=job.attempt + 1, + created_at=job.created_at, + )) + outbox.mark_published(job) + await session.commit() + return published -async def replay_failed_job(session: AsyncSession, job_id: UUID) -> None: - """Явно повторить только failed job, сохранив аудит и ключ логического эффекта. +async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSettings) -> Claim: + """Дубликат не захватывает живую аренду и не откатывает terminal-состояние.""" + row = await jobs.lock_job(session, message.job_id) + result = Claim() + if row is None: + raise ProcessingError("unknown_job") + job, attachment = row + if message.attachment_id != attachment.id or message.pipeline_version != job.pipeline_version: + raise ProcessingError("invalid_job") + now = await jobs.database_time(session) + if job.attempt_limit is None: + job.attempt_limit = settings.job_max_attempts + if attachment.status in {"ready", "failed"}: + result = Claim(acknowledge=True) + elif message.attempt <= job.attempt and attachment.status == "queued": + # Предыдущая попытка уже устойчиво заменена новой в PostgreSQL. + result = Claim(acknowledge=True) + elif message.attempt < job.attempt: + result = Claim(acknowledge=True) + elif message.attempt > job.attempt + 1: + raise ProcessingError("invalid_attempt") + elif attachment.status == "queued" and job.next_attempt_at <= now: + if job.attempt >= (job.attempt_limit or settings.job_max_attempts) or job.staging_key is None: + jobs.record_failure(job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0) + result = Claim(acknowledge=True) + else: + result = Claim(work=jobs.start_job(job, attachment, uuid4(), now, settings.job_lease_seconds)) + await session.commit() + return result - Не переисполнять ready. Проверка допустимости, лимиты повторов и способ - операторского вызова будут определены вместе с реестром failed/DLQ. - """ - raise NotImplementedError("Lab5: replay ещё не реализован") + +async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettings, code: str, *, permanent: bool) -> None: + # Неоднозначный COMMIT проверяется повторным чтением: ready сохраняется. + await session.rollback() + row = await jobs.lock_job(session, work.job_id) + if row is not None: + job, attachment = row + if attachment.status == "processing" and job.lease_token == work.lease_token: + exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts) + jobs.record_failure( + job, attachment, await jobs.database_time(session), + "retry_exhausted" if exhausted and not permanent else code, + terminal=permanent or exhausted, + retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt), + ) + job.last_error_code = code + job.last_error_permanent = permanent + await session.commit() + + +async def build_result( + session: AsyncSession, storage: MinioStorage, work: JobWork, + after_object: Callable[[UUID, str], None] | None = None, +) -> bool: + """Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки.""" + with TemporaryDirectory(prefix="messenger-job-") as directory: + source = Path(directory) / "original" + await fetch_original(storage, work, source) + result = await render_variants(storage, work, source, Path(directory)) + row = await jobs.lock_job(session, work.job_id) + if row is None: + await session.rollback() + return True + job, attachment = row + now = await jobs.database_time(session) + if attachment.status != "processing" or job.lease_token != work.lease_token: + await session.rollback() + return True + if job.lease_until is None or job.lease_until <= now: + await session.rollback() + return False + # Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED. + # При отмене S3 adapter дожидается SDK-потока до освобождения блокировки. + await storage.put_file( + work.original_key, source, size_bytes=work.size_bytes, + content_type=work.content_type, sha256=work.sha256, + ) + if after_object is not None: + after_object(work.attachment_id, "original") + for variant in result.variants: + await storage.put_file( + work.variant_key(variant.name), Path(directory) / VARIANT_FILENAMES[variant.name], + size_bytes=variant.size_bytes, content_type=variant.content_type, sha256=variant.sha256, + ) + if after_object is not None: + after_object(work.attachment_id, variant.name) + jobs.complete_job(job, attachment, work, [variant.model_dump() for variant in result.variants], await jobs.database_time(session)) + await session.commit() + return True + + +async def process_attachment_job( + session: AsyncSession, storage: MinioStorage, message: JobMessage, settings: JobSettings, + *, after_object: Callable[[UUID, str], None] | None = None, + before_processing: Callable[[UUID], None] | None = None, +) -> bool: + claim = await claim_job(session, message, settings) + acknowledge = claim.acknowledge + if claim.work is not None: + work = claim.work + try: + async with asyncio.timeout(settings.job_timeout_seconds): + if before_processing is not None: + before_processing(work.attachment_id) + acknowledge = await build_result(session, storage, work, after_object) + except (ProcessingError, MinioException, StorageHTTPError, OSError, TimeoutError) as error: + code, permanent = classify_failure(error) + await save_failure(session, work, settings, code, permanent=permanent) + acknowledge = True + # Ошибка БД/неизвестная ошибка или отмена не подтверждают доставку. + return acknowledge + + +async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) -> int: + rows = await jobs.expired_jobs(session) + now = await jobs.database_time(session) + for job, attachment in rows: + if job.attempt_limit is None: + job.attempt_limit = settings.job_max_attempts + exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts) + jobs.record_failure( + job, attachment, now, "retry_exhausted" if exhausted else "worker_timeout", + terminal=exhausted, retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt), + ) + job.last_error_code = "worker_timeout" + job.last_error_permanent = False + await session.commit() + return len(rows) + + +async def cleanup_staging(session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool) -> int: + rows = await jobs.terminal_staging(session, settings.staging_retention_seconds, settings.failed_staging_retention_seconds) + if apply: + for job, attachment in rows: + if job.staging_key != attachment.object_key: + await storage.delete_object(job.staging_key) + jobs.clear_staging(job) + await session.commit() + else: + await session.rollback() + return len(rows) diff --git a/app/domain/attachments.py b/app/domain/attachments.py index 3216c63..80caf10 100644 --- a/app/domain/attachments.py +++ b/app/domain/attachments.py @@ -13,6 +13,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.request_capacity import RequestCapacity from app.domain.chats import ensure_chat_member +from app.domain.attachment_jobs import enqueue_attachment_job from app.domain.images import validate_filename, validate_image from app.errors.exceptions import ( conflict, @@ -28,7 +29,7 @@ from app.errors.handlers import ( log_recoverable_error, operation_timeout, ) -from app.schemas.attachment import Attachment +from app.schemas.attachment import Attachment, VARIANT_FILENAMES from app.schemas.user import User from app.storage import attachments as repository from app.storage.s3 import MinioStorage @@ -151,7 +152,7 @@ async def recover_upload( await session.rollback() -async def publish_original( +async def stage_attachment( session: AsyncSession, storage: MinioStorage, user: User, @@ -161,8 +162,9 @@ async def publish_original( filename: str, content_type: str, ) -> Attachment: + """Сохранить исходник в staging и атомарно принять metadata/job.""" id = uuid4() - key = f"originals/{uuid4().hex}" + key = f"staging/{uuid4().hex}" # Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT. await repository.reserve_upload(session, id, key) await session.commit() @@ -181,13 +183,16 @@ async def publish_original( id=id, chat_id=chat_id, owner_id=user.id, - object_key=key, + object_key=f"originals/{id.hex}/v1", + status="queued", filename=filename, content_type=content_type, size_bytes=original.size_bytes, sha256=original.sha256, ) + await enqueue_attachment_job(session, id, key) await repository.finish_upload(session, id) + # Снимок queued строится до COMMIT, даже если worker очень быстрый. result = Attachment.model_validate(record) await session.commit() except BaseException: @@ -223,7 +228,7 @@ async def upload_attachment( settings.chunk_bytes, ) await validate_image(original.path, content_type, settings) - result = await publish_original( + result = await stage_attachment( session, storage, user, @@ -245,6 +250,13 @@ async def get_attachment( return Attachment.model_validate(record) +@dataclass(frozen=True) +class DownloadInfo: + filename: str + content_type: str + size_bytes: int + + async def download_attachment( session: AsyncSession, storage: MinioStorage | None, @@ -252,25 +264,30 @@ async def download_attachment( attachment_id: UUID, *, variant: str = "original", -) -> tuple[Attachment, AsyncIterator[bytes]]: +) -> tuple[DownloadInfo, AsyncIterator[bytes]]: result = await get_attachment(session, user, attachment_id) - if variant != "original": - raise not_found() + if result.status != "ready": + raise conflict("Вложение пока недоступно для скачивания") storage = require_storage(storage) record = await repository.get_attachment(session, attachment_id) if record is None: raise not_found() - key = record.object_key + if variant == "original": + key = record.object_key + info = DownloadInfo(result.filename, result.content_type, result.size_bytes) + else: + selected = next((item for item in result.variants if item.name == variant), None) + keys = {"thumbnail": record.thumbnail_key, "crop": record.crop_key, "optimized": record.optimized_key} + key = keys.get(variant) + if selected is None or key is None: + raise not_found() + info = DownloadInfo(VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes) await session.rollback() - info = await storage.stat_object(key) - if ( - info is None - or info.size_bytes != result.size_bytes - or info.content_type != result.content_type - ): - raise unavailable("Оригинал временно недоступен") + stored = await storage.stat_object(key) + if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type: + raise unavailable("Файл временно недоступен") stream = await storage.open_object(key) - return result, stream + return info, stream async def validate_message_attachments( @@ -283,8 +300,8 @@ async def validate_message_attachments( records = await repository.get_for_chat(session, chat_id, attachment_ids) if len(records) != len(attachment_ids): raise not_found() - if any(record.status != "ready" for record in records): - raise conflict("Вложение ещё не готово") + if any(record.status == "failed" for record in records): + raise conflict("Обработка вложения завершилась ошибкой") async def reconcile_attachments( diff --git a/app/domain/image_pipeline.py b/app/domain/image_pipeline.py index 6dffbc0..823ade9 100644 --- a/app/domain/image_pipeline.py +++ b/app/domain/image_pipeline.py @@ -1,34 +1,79 @@ -"""Заготовки преобразований lab5, без доступа к БД, брокеру и HTTP. - -Pipeline запускается внутри ограниченного процесса worker. Исходные bytes -не перекодируются; варианты удаляют метаданные по документированной политике. -Значения качества JPEG/WebP предстоит выбрать и зафиксировать в REPORT.md. -""" +"""Thumbnail, crop и WebP; один ограниченный процесс, последовательная обработка.""" +import hashlib from pathlib import Path +from PIL import Image, ImageOps -def create_thumbnail(source: Path, destination: Path) -> None: - """Создать JPEG, вписанный в 256×256 без увеличения. - - Применить EXIF orientation, alpha положить на белый; округлять размеры - вниз, минимум 1 px. Публичный fixture 320×200 должен дать 256×160. - """ - raise NotImplementedError("Lab5: thumbnail ещё не реализован") +from app.core.image_processing import ImageSettings +from app.domain.image_validation import decode_image +from app.schemas.attachment import PipelineResult, Variant -def create_crop(source: Path, destination: Path) -> None: - """Создать центрированный квадрат JPEG 128×128; увеличение допустимо. - - Учитывать EXIF orientation и прозрачность по той же политике, что thumbnail. - """ - raise NotImplementedError("Lab5: crop ещё не реализован") +def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]: + """Вписать без увеличения, округляя вниз, минимум один пиксель.""" + longest = max(size) + result = size + if longest > maximum: + result = (max(1, size[0] * maximum // longest), max(1, size[1] * maximum // longest)) + return result -def create_optimized(source: Path, destination: Path) -> None: - """Создать WebP, вписанный в 1280×1280 без увеличения. +def describe_variant(name: str, content_type: str, path: Path, size: tuple[int, int]) -> Variant: + checksum = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(65536), b""): + checksum.update(chunk) + return Variant( + name=name, content_type=content_type, size_bytes=path.stat().st_size, + width=size[0], height=size[1], sha256=checksum.hexdigest(), + ) - Удалить метаданные и применить описанную политику качества. Уменьшение - размера любого маленького исходника не гарантируется контрактом. - """ - raise NotImplementedError("Lab5: optimized ещё не реализован") + +def save_jpeg(image: Image.Image, destination: Path, quality: int) -> None: + """Белая подложка и новый image без EXIF/ICC/XMP/текстовых метаданных.""" + with Image.new("RGB", image.size, "white") as clean: + with image.getchannel("A") as alpha: + clean.paste(image, mask=alpha) + clean.save(destination, format="JPEG", quality=quality, exif=b"") + + +def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Variant: + size = fitted_size(image.size, 256) + with image.resize(size, Image.Resampling.LANCZOS) as resized: + save_jpeg(resized, destination, quality) + return describe_variant("thumbnail", "image/jpeg", destination, size) + + +def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant: + """Центрированный JPEG 128×128; маленький исходник допускает увеличение.""" + with ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) as crop: + save_jpeg(crop, destination, quality) + return describe_variant("crop", "image/jpeg", destination, (128, 128)) + + +def create_optimized(image: Image.Image, destination: Path, settings: ImageSettings) -> Variant: + """WebP ≤1280×1280 без увеличения; alpha сохраняется, метаданные удаляются.""" + size = fitted_size(image.size, 1280) + with image.resize(size, Image.Resampling.LANCZOS) as resized: + with Image.new("RGBA", size) as clean: + clean.paste(resized) + clean.save( + destination, format="WEBP", lossless=False, + quality=settings.webp_quality, method=settings.webp_method, + exif=b"", icc_profile=b"", xmp=b"", + ) + return describe_variant("optimized", "image/webp", destination, size) + + +def create_variants(source: Path, directory: Path, content_type: str, max_pixels: int, settings: ImageSettings) -> PipelineResult: + decode_image(source, content_type, max_pixels) + with Image.open(source, formats=["PNG", "JPEG"]) as image: + ImageOps.exif_transpose(image, in_place=True) + with image.convert("RGBA") as pixels: + variants = [ + create_thumbnail(pixels, directory / "thumbnail.jpg", settings.jpeg_quality), + create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality), + create_optimized(pixels, directory / "optimized.webp", settings), + ] + return PipelineResult(variants=variants) diff --git a/app/domain/job_admin.py b/app/domain/job_admin.py new file mode 100644 index 0000000..a1a3494 --- /dev/null +++ b/app/domain/job_admin.py @@ -0,0 +1,93 @@ +"""Replay только failed с проверенным исходником и ограниченным новым бюджетом.""" + +import asyncio +import re +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import uuid4 + +from minio.error import S3Error + +from app.domain.job_files import fetch_original +from app.errors.exceptions import JobOperationError, ProcessingError +from app.storage import job_admin, jobs +from app.storage.jobs import JobWork + +REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review") + + +def snapshot(job, attachment): + return dict(job_id=str(job.id), attachment_id=str(attachment.id), status=attachment.status, + attempt=job.attempt, attempt_limit=job.attempt_limit, cycle_start_attempt=job.cycle_start_attempt, + error_code=attachment.error_code, last_error_code=job.last_error_code, + last_error_permanent=job.last_error_permanent, staging_retained=job.staging_key is not None, + next_attempt_at=job.next_attempt_at, lease_until=job.lease_until, finished_at=job.finished_at) + + +async def list_failed(session, limit): + return [snapshot(job, attachment) for job, attachment in await job_admin.failed_jobs(session, limit)] + + +async def show_job(session, job_id): + row = await jobs.lock_job(session, job_id) + if row is None: + raise JobOperationError("unknown_job") + result = snapshot(*row) + await session.rollback() + return result + + +async def history(session, job_id): + return [dict(request_id=str(row.id), actor=row.actor, reason=row.reason, + previous_attempt=row.previous_attempt, attempt_limit=row.attempt_limit, + previous_error_code=row.previous_error_code, created_at=row.created_at) + for row in await job_admin.replay_history(session, job_id)] + + +def same_request(record, job_id, actor, reason): + if record.job_id != job_id or record.actor != actor or record.reason != reason: + raise JobOperationError("request_conflict") + return {"request_id": str(record.id), "job_id": str(job_id), "already_applied": True} + + +async def replay_failed_job(session, storage, settings, job_id, *, request_id, actor, reason): + if re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None or reason not in REPLAY_REASONS: + raise JobOperationError("invalid_audit_fields") + row = await jobs.lock_job(session, job_id) + if row is None: + raise JobOperationError("unknown_job") + record = await job_admin.replay_record(session, request_id) + if record is not None: + result = same_request(record, job_id, actor, reason) + await session.rollback() + return result + job, attachment = row + if attachment.status != "failed": + raise JobOperationError("job_not_failed") + if job.staging_key is None: + raise JobOperationError("source_expired") + # Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT. + work = JobWork(job.id, attachment.id, uuid4(), job.pipeline_version, job.staging_key, + attachment.content_type, attachment.size_bytes, attachment.sha256) + try: + async with asyncio.timeout(settings.job_timeout_seconds): + with TemporaryDirectory(prefix="replay-source-") as directory: + await fetch_original(storage, work, Path(directory) / "original") + except ProcessingError as error: + raise JobOperationError(error.code) from None + except S3Error as error: + if error.code in {"NoSuchKey", "NoSuchObject"}: + raise JobOperationError("source_missing") from None + raise + limit = job.attempt + settings.job_max_attempts + inserted = await job_admin.insert_replay( + session, id=request_id, job_id=job_id, actor=actor, reason=reason, + previous_attempt=job.attempt, attempt_limit=limit, previous_error_code=job.last_error_code, + ) + if not inserted: + result = same_request(await job_admin.replay_record(session, request_id), job_id, actor, reason) + await session.rollback() + return result + job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit) + await session.commit() + return {"request_id": str(request_id), "job_id": str(job_id), "already_applied": False, "status": "queued", "attempt_limit": limit} diff --git a/app/domain/job_files.py b/app/domain/job_files.py new file mode 100644 index 0000000..b48950d --- /dev/null +++ b/app/domain/job_files.py @@ -0,0 +1,61 @@ +"""Получение исходника и запуск изолированного процесса обработки вариантов.""" + +import asyncio +import errno +import hashlib +from pathlib import Path +import sys + +from anyio import CancelScope + +from app.errors.exceptions import ProcessingError +from app.schemas.attachment import PipelineResult +from app.storage.jobs import JobWork +from app.storage.s3 import MinioStorage + + +async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path) -> None: + if work.size_bytes > storage.settings.max_upload_bytes: + raise ProcessingError("image_limits") + checksum = hashlib.sha256() + size = 0 + stream = await storage.open_object(work.staging_key) + try: + with destination.open("wb") as target: + async for chunk in stream: + size += len(chunk) + if size > work.size_bytes: + raise ProcessingError("source_mismatch") + target.write(chunk) + checksum.update(chunk) + finally: + await stream.aclose() + if size != work.size_bytes or checksum.hexdigest() != work.sha256: + raise ProcessingError("source_mismatch") + + +async def render_variants(storage: MinioStorage, work: JobWork, source: Path, directory: Path) -> PipelineResult: + metadata = directory / "variants.json" + process = await asyncio.create_subprocess_exec( + sys.executable, "-m", "app.core.thumbnail_worker", str(source), str(directory), + str(metadata), work.content_type, str(storage.settings.max_image_pixels), + stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL, + ) + try: + code = await process.wait() + finally: + if process.returncode is None: + process.kill() + with CancelScope(shield=True): + await process.wait() + if code == 2: + raise ProcessingError("invalid_image") + if code == 3 or code < 0: + raise ProcessingError("image_limits") + if code == 4: + raise OSError(errno.EIO, "Pipeline filesystem unavailable") + if code == 5: + raise ProcessingError("processing_configuration") + if code != 0: + raise RuntimeError("Image pipeline subprocess failed") + return PipelineResult.model_validate_json(metadata.read_text()) diff --git a/app/domain/job_retry.py b/app/domain/job_retry.py new file mode 100644 index 0000000..2d9620d --- /dev/null +++ b/app/domain/job_retry.py @@ -0,0 +1,32 @@ +"""Ограниченный exponential backoff: jitter сохраняется вместе с новой попыткой.""" + +import errno +import random + +from minio.error import S3Error + +from app.core.jobs import JobSettings +from app.errors.exceptions import ProcessingError + + +def retry_delay(settings: JobSettings, cycle_attempt: int) -> float: + ceiling = min(settings.job_retry_max_seconds, settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1))) + return random.uniform(ceiling / 2, ceiling) + + +def classify_failure(error: Exception) -> tuple[str, bool]: + if isinstance(error, ProcessingError): + return error.code, True + if isinstance(error, S3Error): + if error.code in {"NoSuchKey", "NoSuchObject"}: + return "source_missing", True + if error.code in {"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "NoSuchBucket"}: + return "storage_configuration", True + if error.code in {"InternalError", "SlowDown", "ServiceUnavailable", "RequestTimeout", "Throttling"}: + return "storage_unavailable", False + return "storage_rejected", True + if isinstance(error, TimeoutError): + return "processing_timeout", False + if isinstance(error, OSError) and error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}: + return "processing_configuration", True + return "processing_unavailable", False diff --git a/app/endpoints/attachments.py b/app/endpoints/attachments.py index 94255c0..8387b3e 100644 --- a/app/endpoints/attachments.py +++ b/app/endpoints/attachments.py @@ -15,7 +15,7 @@ from app.schemas.attachment import Attachment router = APIRouter(prefix='/api/v1', tags=['attachments']) -@router.post('/chats/{chat_id}/attachments', status_code=201) +@router.post('/chats/{chat_id}/attachments', status_code=202) async def upload( request: Request, chat_id: UUID, user: CurrentUser, session: DbSession, storage: S3Storage, capacity: UploadCapacity, diff --git a/app/errors/exceptions.py b/app/errors/exceptions.py index b434529..59d06a9 100644 --- a/app/errors/exceptions.py +++ b/app/errors/exceptions.py @@ -1,3 +1,5 @@ +import errno + from collections.abc import Mapping from PIL import Image, UnidentifiedImageError @@ -107,3 +109,28 @@ def image_validation_error(code: int) -> ApiError: if code == 3 or code < 0: error = payload_too_large("Превышен лимит пикселей или ресурсов изображения") return error + + +class ProcessingError(Exception): + """Постоянная ошибка pipeline с безопасным машинным кодом.""" + + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +class JobOperationError(Exception): + """Безопасный код отказа операторской команды.""" + + def __init__(self, code: str): + self.code = code + super().__init__(code) + + +def pipeline_worker_error_code(error: Exception) -> int: + """Отделить ошибки файловой системы от ошибки декодирования изображения.""" + if isinstance(error, OSError) and error.errno is not None: + if error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}: + return 5 + return 4 + return image_worker_error_code(error) diff --git a/app/main.py b/app/main.py index 7bf806c..de68388 100644 --- a/app/main.py +++ b/app/main.py @@ -45,7 +45,7 @@ def create_app(settings: Settings | None = None) -> FastAPI: await redis.aclose() await engine.dispose() - application = FastAPI(title="Messenger", version="1.4.0", lifespan=lifespan) + application = FastAPI(title="Messenger", version="1.5.0", lifespan=lifespan) application.state.settings = settings application.state.engine = engine diff --git a/app/models/__init__.py b/app/models/__init__.py index 6679c3e..4309785 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,4 +1,6 @@ +from app.models.job_replay import JobReplay from app.models.base import Base +from app.models.job import AttachmentJob from app.models.attachment import Attachment, MessageAttachment, UploadIntent from app.models.chat import Chat from app.models.chat_member import ChatMember @@ -7,4 +9,4 @@ from app.models.idempotency_key import IdempotencyKey from app.models.message import Message from app.models.user import User -__all__ = ["Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"] +__all__ = ["JobReplay", "AttachmentJob", "Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"] diff --git a/app/models/attachment.py b/app/models/attachment.py index 3b7c678..24d075a 100644 --- a/app/models/attachment.py +++ b/app/models/attachment.py @@ -3,7 +3,8 @@ import uuid from datetime import datetime -from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func +from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func, text +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column from app.models.base import Base @@ -17,7 +18,8 @@ class Attachment(Base): "content_type IN ('image/png', 'image/jpeg')", name="content_type" ), CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"), - CheckConstraint("status = 'ready'", name="status"), + CheckConstraint("status IN ('queued', 'processing', 'ready', 'failed')", name="status"), + CheckConstraint("(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", name="error_code"), Index("ix_attachments_chat_id", "chat_id"), ) @@ -29,7 +31,12 @@ class Attachment(Base): content_type: Mapped[str] = mapped_column(String(32)) size_bytes: Mapped[int] sha256: Mapped[str] = mapped_column(String(64)) - status: Mapped[str] = mapped_column(String(16), server_default="ready") + status: Mapped[str] = mapped_column(String(16), server_default="queued") + error_code: Mapped[str | None] = mapped_column(String(64)) + thumbnail_key: Mapped[str | None] = mapped_column(String(128)) + crop_key: Mapped[str | None] = mapped_column(String(128)) + optimized_key: Mapped[str | None] = mapped_column(String(128)) + variants: Mapped[list[dict]] = mapped_column(JSONB, default=list, server_default=text("'[]'::jsonb")) created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) @@ -47,7 +54,7 @@ class MessageAttachment(Base): class UploadIntent(Base): - """Фиксируется до PUT; удаляется атомарно с публикацией ready-метаданных.""" + """Фиксируется до PUT; удаляется атомарно с принятием queued metadata/job.""" __tablename__ = "upload_intents" __table_args__ = (Index("ix_upload_intents_created_at", "created_at"),) diff --git a/app/models/job.py b/app/models/job.py index 95a1af0..00ae6a2 100644 --- a/app/models/job.py +++ b/app/models/job.py @@ -1,16 +1,37 @@ -"""Место для будущих ORM-моделей job, outbox и вариантов lab5. +"""Устойчивая работа и намерение публикации; один pipeline на вложение.""" -Пока моделей и регистрации в Base.metadata нет, чтобы не менять схему lab4 -до реализации миграции. Планируемые данные: +from datetime import datetime +from uuid import UUID, uuid4 -* job: job_id, attachment_id, pipeline_version, attempt/max_attempts, - status, lease token/deadline, next_attempt_at, безопасный error_code, timestamps; -* outbox: event_id, job_id, payload/version, состояние публикации и её lease; -* variant: attachment_id, pipeline_version, variant, приватный object_key, - content_type, size_bytes, sha256, размеры; -* replay audit: операторское действие и связь с прежней логической job. +from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func +from sqlalchemy.orm import Mapped, mapped_column -Ключ эффекта (attachment_id, pipeline_version, variant) уникален. Старый worker -не публикует metadata поверх новой версии; точная схема зависит от трека 5. -Изменение ready-only constraints attachments — отдельная новая миграция. -""" +from app.models.base import Base + + +class AttachmentJob(Base): + __tablename__ = "attachment_jobs" + __table_args__ = ( + CheckConstraint("pipeline_version = 1", name="pipeline_version"), + CheckConstraint("attempt >= 0 AND published_attempt >= 0", name="attempt"), + Index("ix_attachment_jobs_due", "next_attempt_at"), + Index("ix_attachment_jobs_lease", "lease_until"), + ) + + id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4) + attachment_id: Mapped[UUID] = mapped_column( + ForeignKey("attachments.id"), unique=True, + ) + staging_key: Mapped[str | None] = mapped_column(String(128)) + pipeline_version: Mapped[int] = mapped_column(server_default="1") + attempt: Mapped[int] = mapped_column(server_default="0") + cycle_start_attempt: Mapped[int] = mapped_column(server_default="0") + attempt_limit: Mapped[int | None] + last_error_permanent: Mapped[bool | None] + published_attempt: Mapped[int] = mapped_column(server_default="0") + next_attempt_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) + lease_token: Mapped[UUID | None] + lease_until: Mapped[datetime | None] + last_error_code: Mapped[str | None] = mapped_column(String(64)) + finished_at: Mapped[datetime | None] + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) diff --git a/app/models/job_replay.py b/app/models/job_replay.py new file mode 100644 index 0000000..7f2b27a --- /dev/null +++ b/app/models/job_replay.py @@ -0,0 +1,22 @@ +"""Аудит явного повторного запуска failed; request id обеспечивает идемпотентность.""" + +from datetime import datetime +from uuid import UUID + +from sqlalchemy import ForeignKey, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from app.models.base import Base + + +class JobReplay(Base): + __tablename__ = "job_replays" + + id: Mapped[UUID] = mapped_column(primary_key=True) + job_id: Mapped[UUID] = mapped_column(ForeignKey("attachment_jobs.id"), index=True) + actor: Mapped[str] = mapped_column(String(64)) + reason: Mapped[str] = mapped_column(String(32)) + previous_attempt: Mapped[int] + attempt_limit: Mapped[int] + previous_error_code: Mapped[str | None] = mapped_column(String(64)) + created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp()) diff --git a/app/schemas/attachment.py b/app/schemas/attachment.py index 74574dd..93911c1 100644 --- a/app/schemas/attachment.py +++ b/app/schemas/attachment.py @@ -1,14 +1,25 @@ -"""Публичные ready-метаданные lab4; приватный object_key не выдаётся клиенту.""" +"""Публичные состояния и варианты; внутренние ключи S3 не сериализуются.""" from datetime import datetime from typing import Literal from uuid import UUID -from pydantic import Field +from pydantic import BaseModel, ConfigDict, Field, field_validator from app.schemas.base import Base +class Variant(BaseModel): + model_config = ConfigDict(extra="forbid", from_attributes=True) + + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/png", "image/webp"] + size_bytes: int = Field(ge=1) + width: int = Field(ge=1) + height: int = Field(ge=1) + sha256: str = Field(pattern=r"^[a-f0-9]{64}$") + + class Attachment(Base): chat_id: UUID owner_id: UUID @@ -16,8 +27,25 @@ class Attachment(Base): content_type: Literal["image/png", "image/jpeg"] size_bytes: int = Field(ge=1, le=10_485_760) sha256: str = Field(pattern=r"^[a-f0-9]{64}$") - status: Literal["ready"] = "ready" - variants: list[object] = Field(default_factory=list, max_length=0) - error_code: None = None + status: Literal["queued", "processing", "ready", "failed"] + variants: list[Variant] = Field(default_factory=list) + error_code: str | None = None created_at: datetime updated_at: datetime + + +VARIANT_FILENAMES = { + "thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp", +} + + +class PipelineResult(BaseModel): + model_config = ConfigDict(extra="forbid") + variants: list[Variant] = Field(min_length=3, max_length=3) + + @field_validator("variants") + @classmethod + def require_all_variants(cls, variants: list[Variant]) -> list[Variant]: + if {item.name for item in variants} != set(VARIANT_FILENAMES): + raise ValueError("Нужны thumbnail, crop и optimized без повторов") + return variants diff --git a/app/schemas/job.py b/app/schemas/job.py index d9c71ca..7fb7f34 100644 --- a/app/schemas/job.py +++ b/app/schemas/job.py @@ -1,10 +1,20 @@ -"""Место для будущей схемы сообщения contracts/jobs.schema.json. +"""Сообщение брокера: только идентификаторы и служебные поля контракта v1.""" -Пока классы не реализованы и сообщения не принимаются/не публикуются. -Обязательны schema_version=1, type=attachment.process.v1, job_id, attachment_id, -pipeline_version>=1, attempt>=1, created_at (UTC/RFC3339). -Допустимы traceparent и tracestate; дополнительные поля запрещены. -job_id сохраняется при redelivery; очередная управляемая попытка отражается -в attempt. Bytes, secrets, S3 keys и пользовательские пути не передаются. -Pydantic-схема должна соответствовать публичному JSON Schema, не менять его. -""" +from typing import Literal +from uuid import UUID + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field + + +class JobMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: Literal[1] = 1 + type: Literal["attachment.process.v1"] = "attachment.process.v1" + job_id: UUID + attachment_id: UUID + pipeline_version: int = Field(default=1, ge=1) + attempt: int = Field(ge=1) + created_at: AwareDatetime + traceparent: str | None = Field(default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") + tracestate: str | None = Field(default=None, max_length=512) diff --git a/app/storage/broker.py b/app/storage/broker.py index 3145d2f..7358a52 100644 --- a/app/storage/broker.py +++ b/app/storage/broker.py @@ -1,27 +1,78 @@ -"""Заготовки адаптера durable broker; конкретный брокер ещё не выбран. +"""RabbitMQ: durable queue, persistent messages, publisher confirms и manual ack.""" -Сообщения соответствуют contracts/jobs.schema.json, содержат ID и служебные -поля, но не bytes/credentials/произвольные URL. Подключение, consumer identity, -формат delivery handle и интерфейсы уточнить после выбора брокера/версии. -Здесь появятся только SDK-вызовы; решения о retry/ready находятся в domain/. -""" +import asyncio +import logging +from dataclasses import dataclass -from collections.abc import Mapping +import aio_pika +from aio_pika.abc import AbstractIncomingMessage + +from app.core.jobs import JobSettings +from app.schemas.job import JobMessage -async def publish_job(payload: Mapping[str, object]) -> str: - """Опубликовать устойчивое сообщение и вернуть идентификатор доставки. - - Дождаться подтверждения сохранения брокером; неопределённый исход разрешает - повтор с тем же job_id. Не считать публикацию обработкой изображения. - """ - raise NotImplementedError("Lab5: публикация в брокер ещё не реализована") +@dataclass(frozen=True) +class Delivery: + id: AbstractIncomingMessage + payload: bytes -async def acknowledge_job(delivery_id: str) -> None: - """Подтвердить доставку после устойчивого эффекта либо проверенного ready-дубля. +class JobBroker: + """Один consumer/канал на экземпляр; publisher использует отдельный экземпляр.""" - Retry сначала устойчиво планируется, затем закрывается старая доставка. - Конкретный delivery handle зависит от брокера; сигнатура предварительная. - """ - raise NotImplementedError("Lab5: ack ещё не реализован") + def __init__(self, settings: JobSettings) -> None: + self.settings = settings + self.connection = self.channel = self.queue = self.iterator = None + # Transport exceptions могут содержать URL с credentials. + logging.getLogger("aio_pika").setLevel(logging.CRITICAL) + logging.getLogger("aiormq").setLevel(logging.CRITICAL) + + async def ensure_ready(self) -> None: + if self.connection is None or self.connection.is_closed or self.channel is None or self.channel.is_closed: + await self.aclose() + self.connection = await aio_pika.connect( + host=self.settings.broker_host, port=self.settings.broker_port, + login=self.settings.broker_user, + password=self.settings.broker_password_file.read_text().strip(), + virtualhost=self.settings.broker_vhost, timeout=5, heartbeat=30, + ) + self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True) + await self.channel.set_qos(prefetch_count=1) + self.queue = await self.channel.declare_queue(self.settings.broker_queue, durable=True) + + async def publish_job(self, message: JobMessage) -> None: + await self.ensure_ready() + await self.channel.default_exchange.publish( + aio_pika.Message( + body=message.model_dump_json(exclude_none=True).encode(), + content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + message_id=f"{message.job_id}:{message.attempt}", + ), routing_key=self.settings.broker_queue, mandatory=True, timeout=10, + ) + + async def receive(self, consumer: str) -> Delivery: + await self.ensure_ready() + if self.iterator is None: + self.iterator = self.queue.iterator(no_ack=False, consumer_tag=consumer) + await self.iterator.__aenter__() + message = await self.iterator.__anext__() + return Delivery(message, message.body) + + async def acknowledge_job(self, delivery_id: AbstractIncomingMessage) -> None: + await delivery_id.ack() + + async def defer_job(self, delivery_id: AbstractIncomingMessage) -> None: + # Живая чужая lease: не удерживать единственный prefetch slot навсегда. + await asyncio.sleep(self.settings.publisher_poll_seconds) + await delivery_id.nack(requeue=True) + + async def reject_invalid(self, delivery_id: AbstractIncomingMessage) -> None: + # Failed jobs находятся в PostgreSQL; неизвестные payload не копируются в DLQ. + logging.getLogger(__name__).warning("Rejected invalid job envelope") + await delivery_id.reject(requeue=False) + + async def aclose(self) -> None: + connection = self.connection + self.connection = self.channel = self.queue = self.iterator = None + if connection is not None and not connection.is_closed: + await connection.close() diff --git a/app/storage/job_admin.py b/app/storage/job_admin.py new file mode 100644 index 0000000..87782c2 --- /dev/null +++ b/app/storage/job_admin.py @@ -0,0 +1,46 @@ +"""Операторские выборки и журнал replay; транзакция принадлежит domain.""" + +from sqlalchemy import select +from sqlalchemy.dialects.postgresql import insert + +from app.models.attachment import Attachment +from app.models.job import AttachmentJob +from app.models.job_replay import JobReplay + + +async def failed_jobs(session, limit: int): + result = await session.execute( + select(AttachmentJob, Attachment) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where(Attachment.status == "failed") + .order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id).limit(limit) + ) + return result.all() + + +async def replay_record(session, request_id): + return await session.get(JobReplay, request_id) + + +async def insert_replay(session, **values) -> bool: + result = await session.scalar(insert(JobReplay).values(**values).on_conflict_do_nothing(index_elements=[JobReplay.id]).returning(JobReplay.id)) + return result is not None + + +async def replay_history(session, job_id): + return (await session.scalars(select(JobReplay).where(JobReplay.job_id == job_id).order_by(JobReplay.created_at).limit(100))).all() + + +def reset_failed(job, attachment, now, attempt_limit): + job.cycle_start_attempt = job.attempt + job.attempt_limit = attempt_limit + job.published_attempt = job.attempt + job.next_attempt_at = now + job.lease_token = None + job.lease_until = None + job.finished_at = None + job.last_error_code = None + job.last_error_permanent = None + attachment.status = "queued" + attachment.error_code = None + attachment.updated_at = now diff --git a/app/storage/jobs.py b/app/storage/jobs.py index 2f35056..d67545b 100644 --- a/app/storage/jobs.py +++ b/app/storage/jobs.py @@ -1,37 +1,127 @@ -"""Заготовки PostgreSQL-хранилища jobs lab5; скрытых commit здесь нет. - -Будущие ORM-операции захвата, lease, retry и результата должны защищаться -от конкурирующих попыток. Границы транзакций находятся в domain/. -Тип результата захвата уточняется после определения модели job. -""" +"""SQL и изменения jobs/attachments; commit остаётся в domain.""" +from dataclasses import dataclass +from datetime import datetime, timedelta from uuid import UUID +from sqlalchemy import and_, or_, func, select from sqlalchemy.ext.asyncio import AsyncSession - -async def claim_job(session: AsyncSession, job_id: UUID) -> None: - """Условно захватить допустимое задание и сохранить lease/номер попытки. - - Ready и актуальная чужая аренда не переходят обратно в processing. - Будущий результат должен различать захват, завершённый дубль и занятость. - """ - raise NotImplementedError("Lab5: захват job ещё не реализован") +from app.models.attachment import Attachment +from app.models.job import AttachmentJob +from app.schemas.attachment import VARIANT_FILENAMES -async def complete_job(session: AsyncSession, job_id: UUID) -> None: - """Атомарно записать metadata вариантов и ready после сохранения объектов. +@dataclass(frozen=True) +class JobWork: + job_id: UUID + attachment_id: UUID + lease_token: UUID + pipeline_version: int + staging_key: str + content_type: str + size_bytes: int + sha256: str - Предусловия lease/attempt/pipeline_version и структура результата будут - добавлены в сигнатуру вместе с ORM-моделью. Не делать commit в storage. - """ - raise NotImplementedError("Lab5: фиксация результата ещё не реализована") + @property + def original_key(self) -> str: + return f"originals/{self.attachment_id.hex}/v{self.pipeline_version}" + + def variant_key(self, name: str) -> str: + filename = VARIANT_FILENAMES[name] + return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}" -async def record_job_failure(session: AsyncSession, job_id: UUID, error_code: str) -> None: - """Записать безопасный код сбоя без stack trace и secrets. +async def create_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID: + job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key) + session.add(job) + await session.flush() + return job.id - Domain выбирает queued с next_attempt_at либо конечный failed согласно - классификации ошибки и лимиту попыток; storage лишь сохраняет решение. - """ - raise NotImplementedError("Lab5: сохранение сбоя ещё не реализовано") + +async def lock_job(session: AsyncSession, job_id: UUID): + result = await session.execute( + select(AttachmentJob, Attachment) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where(AttachmentJob.id == job_id) + .with_for_update(of=AttachmentJob) + .execution_options(populate_existing=True) + ) + return result.one_or_none() + + +async def database_time(session: AsyncSession) -> datetime: + return await session.scalar(select(func.clock_timestamp())) + + +async def expired_jobs(session: AsyncSession, limit: int = 20): + result = await session.execute( + select(AttachmentJob, Attachment) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where(Attachment.status == "processing", AttachmentJob.lease_until < func.clock_timestamp()) + .order_by(AttachmentJob.lease_until) + .limit(limit) + .with_for_update(of=AttachmentJob, skip_locked=True) + ) + return result.all() + + +def start_job(job, attachment, token: UUID, now: datetime, lease_seconds: int) -> JobWork: + job.attempt += 1 + job.lease_token = token + job.lease_until = now + timedelta(seconds=lease_seconds) + attachment.status = "processing" + attachment.updated_at = now + return JobWork( + job.id, attachment.id, token, job.pipeline_version, job.staging_key, + attachment.content_type, attachment.size_bytes, attachment.sha256, + ) + + +def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: bool, retry_seconds: float) -> None: + job.lease_token = None + job.lease_until = None + job.last_error_code = error_code + job.next_attempt_at = now + timedelta(seconds=retry_seconds) + job.finished_at = now if terminal else None + attachment.status = "failed" if terminal else "queued" + attachment.error_code = error_code if terminal else None + attachment.updated_at = now + + +def complete_job(job, attachment, work: JobWork, variants: list[dict], now: datetime) -> None: + attachment.object_key = work.original_key + attachment.thumbnail_key = work.variant_key("thumbnail") + attachment.crop_key = work.variant_key("crop") + attachment.optimized_key = work.variant_key("optimized") + attachment.variants = variants + attachment.status = "ready" + attachment.error_code = None + attachment.updated_at = now + job.lease_token = None + job.lease_until = None + job.last_error_code = None + job.last_error_permanent = None + job.finished_at = now + + +async def terminal_staging(session: AsyncSession, retention: int, failed_retention: int, limit: int = 20): + result = await session.execute( + select(AttachmentJob, Attachment) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where( + AttachmentJob.staging_key.is_not(None), + or_( + and_(Attachment.status == "ready", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=retention)), + and_(Attachment.status == "failed", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=failed_retention)), + ), + ) + .order_by(AttachmentJob.finished_at) + .limit(limit) + .with_for_update(of=AttachmentJob, skip_locked=True) + ) + return result.all() + + +def clear_staging(job) -> None: + job.staging_key = None diff --git a/app/storage/outbox.py b/app/storage/outbox.py index 7010f4e..e791c84 100644 --- a/app/storage/outbox.py +++ b/app/storage/outbox.py @@ -1,23 +1,31 @@ -"""Заготовки transactional outbox для разрыва PostgreSQL COMMIT → broker. - -Outbox — таблица намерений публикации в той же транзакции, что и job. -Повторная публикация после неопределённого confirm допустима; job_id стабилен. -Форма записей, захват пачки и lease publisher уточняются вместе с моделью. -""" +"""Jobs одновременно хранят намерение публикации очередной попытки.""" from uuid import UUID +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession - -async def add_job_event(session: AsyncSession, job_id: UUID) -> None: - """Сохранить намерение публикации job без commit и обращения к брокеру.""" - raise NotImplementedError("Lab5: запись outbox ещё не реализована") +from app.models.attachment import Attachment +from app.models.job import AttachmentJob -async def mark_event_published(session: AsyncSession, event_id: UUID) -> None: - """Отметить событие опубликованным после устойчивого подтверждения брокера. +async def next_unpublished_job(session: AsyncSession, *, attachment_id: UUID | None = None) -> AttachmentJob | None: + statement = ( + select(AttachmentJob) + .join(Attachment, Attachment.id == AttachmentJob.attachment_id) + .where( + Attachment.status == "queued", + AttachmentJob.next_attempt_at <= func.clock_timestamp(), + AttachmentJob.published_attempt <= AttachmentJob.attempt, + ) + .order_by(AttachmentJob.next_attempt_at, AttachmentJob.id) + .limit(1) + .with_for_update(of=AttachmentJob, skip_locked=True) + ) + if attachment_id is not None: + statement = statement.where(AttachmentJob.attachment_id == attachment_id) + return await session.scalar(statement) - Не помечать job завершённой: publisher confirm отличается от consumer ack. - """ - raise NotImplementedError("Lab5: отметка outbox ещё не реализована") + +def mark_published(job: AttachmentJob) -> None: + job.published_attempt = job.attempt + 1 diff --git a/compose.yaml b/compose.yaml index ac342c5..e17a098 100644 --- a/compose.yaml +++ b/compose.yaml @@ -45,6 +45,28 @@ x-s3-env: &s3-env S3_CLEANUP_GRACE_SECONDS: ${S3_CLEANUP_GRACE_SECONDS:?Set S3_CLEANUP_GRACE_SECONDS in .env} S3_CLEANUP_INTERVAL_SECONDS: ${S3_CLEANUP_INTERVAL_SECONDS:?Set S3_CLEANUP_INTERVAL_SECONDS in .env} +x-jobs-env: &jobs-env + BROKER_HOST: ${BROKER_HOST:?Set BROKER_HOST in .env} + BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env} + BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + BROKER_QUEUE: ${BROKER_QUEUE:?Set BROKER_QUEUE in .env} + BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env} + BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:?Set WORKER_CONCURRENCY in .env} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:?Set JOB_MAX_ATTEMPTS in .env} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:?Set JOB_TIMEOUT_SECONDS in .env} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:?Set JOB_LEASE_SECONDS in .env} + JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:?Set JOB_RETRY_MAX_SECONDS in .env} + JOB_RETRY_SECONDS: ${JOB_RETRY_SECONDS:?Set JOB_RETRY_SECONDS in .env} + PUBLISHER_POLL_SECONDS: ${PUBLISHER_POLL_SECONDS:?Set PUBLISHER_POLL_SECONDS in .env} + FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env} + IMAGE_JPEG_QUALITY: ${IMAGE_JPEG_QUALITY:?Set IMAGE_JPEG_QUALITY in .env} + IMAGE_WEBP_QUALITY: ${IMAGE_WEBP_QUALITY:?Set IMAGE_WEBP_QUALITY in .env} + IMAGE_WEBP_METHOD: ${IMAGE_WEBP_METHOD:?Set IMAGE_WEBP_METHOD in .env} + IMAGE_MEMORY_MIB: ${IMAGE_MEMORY_MIB:?Set IMAGE_MEMORY_MIB in .env} + IMAGE_CPU_SECONDS: ${IMAGE_CPU_SECONDS:?Set IMAGE_CPU_SECONDS in .env} + services: minio: build: @@ -154,6 +176,100 @@ services: cpus: "0.5" memory: 512M + broker: + image: rabbitmq:4.4.0 + hostname: broker + restart: unless-stopped + entrypoint: ["/bin/sh", "/usr/local/bin/start-broker.sh"] + environment: + BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env} + BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env} + BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env} + BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + secrets: + - source: broker_password + target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + volumes: + - ./infra/broker/start-broker.sh:/usr/local/bin/start-broker.sh:ro + - rabbitmqdata:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 20s + deploy: + resources: + limits: + cpus: "1.0" + memory: 768M + + publisher: + <<: *api-image + init: true + command: ["python", "-m", "app.core.outbox_publisher"] + restart: unless-stopped + environment: + <<: [*base-env, *jobs-env] + secrets: + - db_password + - source: broker_password + target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + depends_on: + postgres: + condition: service_healthy + broker: + condition: service_healthy + migrate: + condition: service_completed_successfully + healthcheck: + disable: true + stop_grace_period: 15s + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + + worker: + <<: *api-image + init: true + command: ["python", "-m", "app.core.job_worker"] + restart: unless-stopped + environment: + <<: [*base-env, *s3-env, *jobs-env] + secrets: + - db_password + - source: broker_password + target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env} + - source: s3_access_key_id + target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env} + - source: s3_secret_access_key + target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env} + networks: [default, objects] + tmpfs: + - /tmp:rw,noexec,nosuid,size=128m,mode=1777 + depends_on: + postgres: + condition: service_healthy + broker: + condition: service_healthy + minio: + condition: service_healthy + minio-bootstrap: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + healthcheck: + disable: true + stop_signal: SIGTERM + stop_grace_period: 20s + deploy: + resources: + limits: + cpus: "2.0" + memory: 2048M + migrate: <<: *api-image command: ["alembic", "upgrade", "head"] @@ -229,6 +345,8 @@ services: restart: unless-stopped environment: <<: [*base-env, *s3-env] + FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env} secrets: - db_password - source: s3_access_key_id @@ -274,6 +392,7 @@ volumes: pgdata: redisdata: miniodata: + rabbitmqdata: networks: default: @@ -281,6 +400,8 @@ networks: internal: true secrets: + broker_password: + file: ./secrets/broker_password db_password: file: ./secrets/db_password redis_password: diff --git a/evidence/lab5-jobs.json b/evidence/lab5-jobs.json new file mode 100644 index 0000000..639dc0f --- /dev/null +++ b/evidence/lab5-jobs.json @@ -0,0 +1,378 @@ +{ + "status": "passed", + "results": [ + { + "scenario": "worker_stopped_restart_all", + "attachment_id": "96d4fc78-4e72-473c-a2a9-57db94421707", + "queue_before": { + "name": "attachment-jobs", + "messages_unacknowledged": 0, + "consumers": 0, + "durable": true, + "messages_ready": 1 + }, + "queue_after": { + "name": "attachment-jobs", + "messages_unacknowledged": 0, + "consumers": 0, + "durable": true, + "messages_ready": 1 + }, + "after": { + "job_id": "b3a81ec6-65d6-4201-aeb9-fedbe92fdaaf", + "status": "ready", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-10 03:27:44.556734+00:00", + "next_attempt_at": "2026-11-10 03:26:56.662360+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:27:44.548000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:27:44.552000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:27:44.543000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:27:44.536000+00:00" + } + ] + } + }, + { + "scenario": "crash-after-object", + "attachment_id": "115c5e05-3d09-4f04-9b32-0d27597b65fc", + "exit_code": 137, + "before": { + "job_id": "ba3dc221-5651-4c1d-8bfd-9fd61c8aa854", + "status": "processing", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-10 03:27:49.144648+00:00", + "next_attempt_at": "2026-11-10 03:27:47.218612+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:27:49.912000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:27:49.917000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:27:49.907000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:27:49.894000+00:00" + } + ] + }, + "after": { + "job_id": "ba3dc221-5651-4c1d-8bfd-9fd61c8aa854", + "status": "ready", + "attempt": 2, + "published_attempt": 2, + "limit": 5, + "updated_at": "2026-11-10 03:29:24.765539+00:00", + "next_attempt_at": "2026-11-10 03:29:23.142176+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:29:24.750000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:29:24.758000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:29:24.742000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:29:24.725000+00:00" + } + ] + } + }, + { + "scenario": "crash-before-ack", + "attachment_id": "2f97fc61-fc07-47da-98a8-9d9894ed4cd1", + "exit_code": 137, + "before": { + "job_id": "4a0bc78c-d79a-4e1a-88f7-11ad03045135", + "status": "ready", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-10 03:29:29.833142+00:00", + "next_attempt_at": "2026-11-10 03:29:27.262932+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:29:29.824000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:29:29.829000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:29:29.818000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:29:29.805000+00:00" + } + ] + }, + "after": { + "job_id": "4a0bc78c-d79a-4e1a-88f7-11ad03045135", + "status": "ready", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-10 03:29:29.833142+00:00", + "next_attempt_at": "2026-11-10 03:29:27.262932+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:29:29.824000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:29:29.829000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:29:29.818000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:29:29.805000+00:00" + } + ] + } + }, + { + "scenario": "fail-transient", + "attachment_id": "92c4de14-53cd-4805-a842-9892a8b7a5ea", + "samples": [ + { + "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "status": "queued", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-10 03:29:45.863494+00:00", + "next_attempt_at": "2026-11-10 03:29:50.476742+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + }, + { + "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "status": "queued", + "attempt": 2, + "published_attempt": 2, + "limit": 5, + "updated_at": "2026-11-10 03:29:51.216904+00:00", + "next_attempt_at": "2026-11-10 03:29:58.053724+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + }, + { + "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "status": "queued", + "attempt": 3, + "published_attempt": 3, + "limit": 5, + "updated_at": "2026-11-10 03:29:58.279470+00:00", + "next_attempt_at": "2026-11-10 03:30:15.602067+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + }, + { + "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "status": "queued", + "attempt": 4, + "published_attempt": 4, + "limit": 5, + "updated_at": "2026-11-10 03:30:16.386336+00:00", + "next_attempt_at": "2026-11-10 03:30:47.271081+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + }, + { + "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "status": "failed", + "attempt": 5, + "published_attempt": 5, + "limit": 5, + "updated_at": "2026-11-10 03:30:47.622716+00:00", + "next_attempt_at": "2026-11-10 03:31:30.255216+00:00", + "last_error_code": "processing_timeout", + "permanent": false, + "objects": [] + } + ], + "after": { + "job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154", + "status": "ready", + "attempt": 6, + "published_attempt": 6, + "limit": 10, + "updated_at": "2026-11-10 03:30:59.696883+00:00", + "next_attempt_at": "2026-11-10 03:30:53.135190+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:30:59.687000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:30:59.692000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:30:59.683000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:30:59.670000+00:00" + } + ] + }, + "replay_idempotent": true + }, + { + "scenario": "fail-permanent", + "attachment_id": "92e2baff-5b26-4326-bee6-269382ff80c9", + "samples": [ + { + "job_id": "f462098d-43b7-41cc-afe8-399aaf3ed0bd", + "status": "failed", + "attempt": 1, + "published_attempt": 1, + "limit": 5, + "updated_at": "2026-11-10 03:31:12.333162+00:00", + "next_attempt_at": "2026-11-10 03:31:16.827024+00:00", + "last_error_code": "injected_permanent", + "permanent": true, + "objects": [] + } + ], + "after": { + "job_id": "f462098d-43b7-41cc-afe8-399aaf3ed0bd", + "status": "ready", + "attempt": 2, + "published_attempt": 2, + "limit": 6, + "updated_at": "2026-11-10 03:31:24.282794+00:00", + "next_attempt_at": "2026-11-10 03:31:17.137221+00:00", + "last_error_code": null, + "permanent": null, + "objects": [ + { + "name": "crop.jpg", + "size": 2462, + "sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f", + "last_modified": "2026-11-10 03:31:24.272000+00:00" + }, + { + "name": "optimized.webp", + "size": 2462, + "sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb", + "last_modified": "2026-11-10 03:31:24.278000+00:00" + }, + { + "name": "thumbnail.jpg", + "size": 5813, + "sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546", + "last_modified": "2026-11-10 03:31:24.267000+00:00" + }, + { + "name": "v1", + "size": 111817, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "last_modified": "2026-11-10 03:31:24.254000+00:00" + } + ] + }, + "replay_idempotent": true + } + ] +} diff --git a/evidence/lab5-verification.json b/evidence/lab5-verification.json new file mode 100644 index 0000000..2beb465 --- /dev/null +++ b/evidence/lab5-verification.json @@ -0,0 +1,271 @@ +{ + "date": "2026-11-10", + "status": "passed", + "project": "mtusi-lab5-verification", + "base_url": "http://localhost:28080", + "checks": [ + { + "name": "pytest_regression", + "exit_code": 0, + "stdout": "199 passed in 82.29s (0:01:22)", + "scope": "Dedicated *_test database and Redis DB15; Docker restart tests enabled" + }, + { + "name": "pytest_timeout_retention", + "exit_code": 0, + "stdout": "2 passed, 5 deselected in 1.91s", + "scope": "Dedicated *_test database and Redis DB15; Docker restart tests enabled" + }, + { + "name": "compose_configuration", + "command": [ + "docker", + "compose", + "--env-file", + ".env", + "config", + "--quiet" + ], + "exit_code": 0, + "stdout": "" + }, + { + "name": "contract", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "scripts/check_contract.py" + ], + "exit_code": 0, + "stdout": "PASS: offline skeleton structure, local refs and Python syntax; application NOT tested." + }, + { + "name": "openapi", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "scripts/validate_openapi.py" + ], + "exit_code": 0, + "stdout": "PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested." + }, + { + "name": "public_smoke", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "tests/smoke.py", + "--base-url", + "http://localhost:28080" + ], + "exit_code": 0, + "stdout": "PASS: lab5 public API scenario; infrastructure and grading evidence remain separate." + }, + { + "name": "message_idempotency", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "-m", + "scripts.check_idempotency", + "--base-url", + "http://localhost:28080" + ], + "exit_code": 0, + "stdout": "PASS: 24 concurrent replays, 2 replicas, one message" + }, + { + "name": "outbox_crash", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "-m", + "scripts.check_outbox_recovery", + "--base-url", + "http://localhost:28080" + ], + "exit_code": 0, + "stdout": "{\"scenario\": \"commit_before_publish\", \"attachment_id\": \"0fff1dce-c479-40d8-8627-f28cc85cb6cf\", \"message_id\": \"e79bfb7e-c2af-4011-a87a-52528d7a7840\", \"publisher_exit_code\": 137, \"status_after_crash\": \"queued\", \"status_after_restart\": \"ready\", \"variants\": [\"thumbnail\", \"crop\", \"optimized\"], \"reuploaded\": false}" + }, + { + "name": "public_persistence", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "tests/resilience.py", + "--mode", + "persistence", + "--action-script", + "scripts/restart.sh", + "--base-url", + "http://localhost:28080" + ], + "exit_code": 0, + "stdout": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence." + }, + { + "name": "public_failover", + "command": [ + "/home/gytall/.pyenv/versions/3.12.14/bin/python3", + "tests/resilience.py", + "--mode", + "failover", + "--action-script", + "scripts/stop-one.sh", + "--base-url", + "http://localhost:28080" + ], + "exit_code": 0, + "stdout": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 104, \"failed_attempts\": 5, \"first_recovery_seconds\": 1.907}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence." + }, + { + "name": "rabbitmq_prefetch", + "command": [ + "docker", + "compose", + "--env-file", + ".env", + "exec", + "-T", + "broker", + "rabbitmqctl", + "list_channels", + "--formatter=json", + "consumer_count", + "prefetch_count", + "messages_unacknowledged" + ], + "exit_code": 0, + "stdout": "[\n{\"consumer_count\":1,\"messages_unacknowledged\":0,\"prefetch_count\":1}\n,{\"consumer_count\":0,\"messages_unacknowledged\":0,\"prefetch_count\":1}\n,{\"consumer_count\":1,\"messages_unacknowledged\":0,\"prefetch_count\":1}\n]" + }, + { + "name": "rabbitmq_durability_ack", + "command": [ + "docker", + "compose", + "--env-file", + ".env", + "exec", + "-T", + "broker", + "rabbitmqctl", + "list_queues", + "--formatter=json", + "name", + "durable", + "messages_ready", + "messages_unacknowledged", + "consumers" + ], + "exit_code": 0, + "stdout": "[\n{\"name\":\"attachment-jobs\",\"messages_unacknowledged\":0,\"consumers\":2,\"durable\":true,\"messages_ready\":0}\n]" + }, + { + "name": "worker_fault_retry_replay", + "exit_code": 0, + "evidence": "lab5-jobs.json", + "scenarios": [ + "worker_stopped_restart_all", + "crash-after-object", + "crash-before-ack", + "fail-transient", + "fail-permanent" + ] + }, + { + "name": "whitespace", + "command": [ + "git", + "diff", + "--check" + ], + "exit_code": 0, + "stdout": "" + }, + { + "name": "public_contract_tests_unchanged", + "exit_code": 0 + }, + { + "name": "final_services", + "command": [ + "docker", + "compose", + "--env-file", + ".env", + "ps", + "--format", + "{{.Service}} {{.State}} {{.Health}}" + ], + "exit_code": 0, + "stdout": "api running healthy\napi running healthy\nbroker running healthy\ncleanup running \nminio running healthy\nnginx running healthy\npostgres running healthy\npublisher running \nredis running healthy\nworker running" + } + ], + "git_base_commit": "0a703588290e26d0f148b45d0e6bee2e8eefce7d", + "uncommitted_changes": true, + "environment": { + "test_runner_python": "3.12.14", + "runtime_python": "3.12.15", + "broker": "RabbitMQ 4.4.0", + "sessions": "Redis 7.4.2", + "worker_memory_bytes": 2147483648, + "worker_cpu_limit": 2.0, + "worker_init": true + }, + "profile": { + "data": "Synthetic users/chats only; tests/fixtures/sample.png for fault scenarios; no credentials in evidence", + "worker_concurrency": 2, + "prefetch_per_consumer": 1, + "max_attempts_per_cycle": 5, + "timeout_seconds": 60, + "lease_seconds": 90, + "retry_base_seconds": 5, + "retry_cap_seconds": 120, + "image_memory_mib": 768, + "image_cpu_seconds": 20, + "isolated_limit_tests": "256 MiB / 1 CPU second in child process; timeout cancellation checked separately" + }, + "criteria": { + "3.1_lab4_and_durable_202": [ + "pytest_regression", + "public_smoke", + "public_persistence", + "worker_fault_retry_replay" + ], + "3.2_durable_broker_worker_late_ack": [ + "rabbitmq_durability_ack", + "rabbitmq_prefetch", + "worker_fault_retry_replay" + ], + "3.3_states_original_thumbnail": [ + "public_smoke", + "pytest_regression", + "worker_fault_retry_replay" + ], + "3.4_bounded_retry_idempotency": [ + "pytest_regression", + "worker_fault_retry_replay" + ], + "3.5_stopped_worker_api_broker_restart": [ + "worker_fault_retry_replay", + "public_persistence" + ], + "4.1_crop_webp_and_worker_limits": [ + "pytest_regression", + "worker_fault_retry_replay" + ], + "4.2_commit_publish_recovery": [ + "outbox_crash" + ], + "4.3_backoff_jitter_classification_registry_replay": [ + "pytest_regression", + "pytest_timeout_retention", + "worker_fault_retry_replay" + ], + "4.4_crash_after_put_lease_prefetch_cleanup": [ + "worker_fault_retry_replay", + "rabbitmq_prefetch", + "pytest_regression", + "pytest_timeout_retention" + ] + }, + "limitations": [ + "Single Docker host, no disk-loss or cluster failover claim", + "Grade 5 experiments not performed", + "No production-scale performance/security claim" + ] +} diff --git a/infra/README.md b/infra/README.md index fda1b84..a6b520d 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,17 +1,27 @@ # Инфраструктура студента -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Топология — compose.yaml. compose.test.yaml публикует тестовые PostgreSQL/Redis +только на localhost. Nginx направляет запросы на две API-реплики. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +PostgreSQL хранит сообщения, metadata, jobs/outbox и аудит replay. Redis 7.4.2 +с AOF хранит сессии. RabbitMQ 4.4.0 — отдельный брокер заданий: durable очередь, +persistent сообщения, publisher confirms, manual consumer ack и rabbitmqdata. +Broker и MinIO не публикуют внешние порты. API не получает credentials брокера. -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Publisher читает jobs из PostgreSQL и публикует их в RabbitMQ. Worker сохраняет +original/thumbnail/crop/optimized в приватный MinIO и фиксирует ready до ack. +Конкуренция ограничена WORKER_CONCURRENCY; prefetch=1 на отдельный канал каждого +consumer. Publisher/worker используют init:true для корректных сигналов и +воспроизводимого SIGKILL. Обычный Compose-запуск fault hooks не включает. -## Перенесённая база lab4 +Все environment-настройки Compose берутся из .env; образец — .env.example. +Credentials передаются файлами secrets; make init создаёт только отсутствующие. +Migrate и minio-bootstrap завершаются до старта зависимых сервисов. +Перед обновлением остановить старые процессы без удаления volumes. Миграция +a62c950e18b3 повторно публикует queued jobs при переходе на RabbitMQ. -В compose.yaml перенесены postgres, redis (сессии), minio, minio-bootstrap, -migrate, api (две реплики), nginx и cleanup. Версии сохранены без обновления. -Роли broker, outbox publisher и worker пока не реализованы. -core/image_worker.py — одноразовая проверка изображения lab4, а не consumer очереди. -Имена проектов/образов lab5 отделены от lab4. Запуск и порты — в REPORT.md. -Настройки environment в Compose задаются через .env; образец — .env.example. -Секретные значения монтируются отдельно, пути *_FILE относятся к контейнерам. +Restart hook сохраняет volumes и состояние заранее остановленного worker. +Failover hook останавливает выбранную API-реплику. Retry, retention, операторские +команды, ограничения и результаты проверок описаны в REPORT.md. +Полный прогон lab5 уровня 4 — make test-grade4; отдельные сценарии — +make test-outbox-recovery и make test-jobs на выбранном работающем стенде. diff --git a/infra/broker/start-broker.sh b/infra/broker/start-broker.sh new file mode 100755 index 0000000..8efdf06 --- /dev/null +++ b/infra/broker/start-broker.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +# Secret читается внутри контейнера, не попадает в compose config/аргументы. +export RABBITMQ_DEFAULT_PASS="$(cat "${BROKER_PASSWORD_FILE:?}")" +export RABBITMQ_DEFAULT_USER="${BROKER_USER:?}" +export RABBITMQ_DEFAULT_VHOST="${BROKER_VHOST:?}" +export RABBITMQ_NODE_PORT="${BROKER_PORT:?}" +exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server diff --git a/infra/minio/bootstrap.py b/infra/minio/bootstrap.py index 3b8c4b2..f1c76d7 100644 --- a/infra/minio/bootstrap.py +++ b/infra/minio/bootstrap.py @@ -81,7 +81,7 @@ def main(): { "Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], - "Resource": [f"arn:aws:s3:::{bucket}/originals/*"], + "Resource": [f"arn:aws:s3:::{bucket}/{prefix}/*" for prefix in ("originals", "staging", "variants")], }, ], } diff --git a/migrations/versions/a62c950e18b3_rabbitmq_cutover.py b/migrations/versions/a62c950e18b3_rabbitmq_cutover.py new file mode 100644 index 0000000..3eb0cd8 --- /dev/null +++ b/migrations/versions/a62c950e18b3_rabbitmq_cutover.py @@ -0,0 +1,21 @@ +"""Offline cutover: повторно опубликовать queued jobs в RabbitMQ.""" + +from alembic import op + +revision = "a62c950e18b3" +down_revision = "f9183d27a640" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Старые publisher/worker остановлены. Processing восстановится по lease. + op.execute(""" + UPDATE attachment_jobs j SET published_attempt = attempt + FROM attachments a WHERE a.id = j.attachment_id AND a.status = 'queued' + """) + + +def downgrade() -> None: + # При обратном переключении требуется такая же повторная публикация. + upgrade() diff --git a/migrations/versions/c35e91a2f706_async_attachment_jobs.py b/migrations/versions/c35e91a2f706_async_attachment_jobs.py new file mode 100644 index 0000000..280c7dc --- /dev/null +++ b/migrations/versions/c35e91a2f706_async_attachment_jobs.py @@ -0,0 +1,62 @@ +"""Lab5: состояния, варианты и устойчивые jobs с намерением публикации.""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = "c35e91a2f706" +down_revision = "8f31b94a2d10" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check") + op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status IN ('queued', 'processing', 'ready', 'failed')") + op.alter_column("attachments", "status", server_default="queued") + op.add_column("attachments", sa.Column("error_code", sa.String(64), nullable=True)) + op.add_column("attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True)) + op.add_column("attachments", sa.Column("variants", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb"))) + op.create_check_constraint(op.f("ck_attachments_error_code"), "attachments", "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)") + op.create_table( + "attachment_jobs", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("attachment_id", sa.UUID(), nullable=False), + sa.Column("staging_key", sa.String(128), nullable=True), + sa.Column("pipeline_version", sa.Integer(), server_default="1", nullable=False), + sa.Column("attempt", sa.Integer(), server_default="0", nullable=False), + sa.Column("published_attempt", sa.Integer(), server_default="0", nullable=False), + sa.Column("next_attempt_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.Column("lease_token", sa.UUID(), nullable=True), + sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_error_code", sa.String(64), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_attachment_jobs")), + sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_attachment_jobs_attachment_id_attachments")), + sa.UniqueConstraint("attachment_id", name=op.f("uq_attachment_jobs_attachment_id")), + sa.CheckConstraint("pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version")), + sa.CheckConstraint("attempt >= 0 AND published_attempt >= 0", name=op.f("ck_attachment_jobs_attempt")), + ) + op.create_index("ix_attachment_jobs_due", "attachment_jobs", ["next_attempt_at"]) + op.create_index("ix_attachment_jobs_lease", "attachment_jobs", ["lease_until"]) + # Прежние оригиналы остаются источником; worker создаст отсутствующий thumbnail. + op.execute("INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments") + op.execute("UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()") + + +def downgrade() -> None: + connection = op.get_bind() + unfinished = connection.scalar(sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'")) + if unfinished: + raise RuntimeError("Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения") + op.drop_index("ix_attachment_jobs_lease", table_name="attachment_jobs") + op.drop_index("ix_attachment_jobs_due", table_name="attachment_jobs") + op.drop_table("attachment_jobs") + op.drop_constraint(op.f("ck_attachments_error_code"), "attachments", type_="check") + op.drop_column("attachments", "variants") + op.drop_column("attachments", "thumbnail_key") + op.drop_column("attachments", "error_code") + op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check") + op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status = 'ready'") + op.alter_column("attachments", "status", server_default="ready") diff --git a/migrations/versions/e7048b6da913_attachment_crop_webp.py b/migrations/versions/e7048b6da913_attachment_crop_webp.py new file mode 100644 index 0000000..767fbe7 --- /dev/null +++ b/migrations/versions/e7048b6da913_attachment_crop_webp.py @@ -0,0 +1,49 @@ +"""Уровень 4: ключи crop/WebP и дообработка старых ready-вложений.""" + +from alembic import op +import sqlalchemy as sa + +revision = "e7048b6da913" +down_revision = "c35e91a2f706" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("attachments", sa.Column("crop_key", sa.String(128), nullable=True)) + op.add_column("attachments", sa.Column("optimized_key", sa.String(128), nullable=True)) + # Старый staging сменяется надёжным ready-оригиналом, но не теряется из журнала cleanup. + op.execute(""" + INSERT INTO upload_intents (id, object_key) + SELECT gen_random_uuid(), j.staging_key + FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE a.status = 'ready' AND j.staging_key IS NOT NULL + AND j.staging_key <> a.object_key + ON CONFLICT (object_key) DO NOTHING + """) + op.execute(""" + UPDATE attachment_jobs j + SET staging_key = a.object_key, attempt = 0, published_attempt = 0, + next_attempt_at = clock_timestamp(), lease_token = NULL, lease_until = NULL, + last_error_code = NULL, finished_at = NULL + FROM attachments a + WHERE a.id = j.attachment_id AND a.status = 'ready' + """) + op.execute(""" + UPDATE attachments SET status = 'queued', variants = '[]'::jsonb, + updated_at = clock_timestamp() + WHERE status = 'ready' + """) + + +def downgrade() -> None: + active = op.get_bind().scalar(sa.text("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')")) + if active: + raise RuntimeError("Перед downgrade необходимо завершить активные задания") + op.execute(""" + UPDATE attachments a SET variants = COALESCE( + (SELECT jsonb_agg(v) FROM jsonb_array_elements(a.variants) v + WHERE v->>'name' = 'thumbnail'), '[]'::jsonb) + """) + op.drop_column("attachments", "optimized_key") + op.drop_column("attachments", "crop_key") diff --git a/migrations/versions/f9183d27a640_job_replay.py b/migrations/versions/f9183d27a640_job_replay.py new file mode 100644 index 0000000..7e6ed24 --- /dev/null +++ b/migrations/versions/f9183d27a640_job_replay.py @@ -0,0 +1,34 @@ +"""Бюджет повторов и аудит операторского replay.""" + +from alembic import op +import sqlalchemy as sa + +revision = "f9183d27a640" +down_revision = "e7048b6da913" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("attachment_jobs", sa.Column("cycle_start_attempt", sa.Integer(), nullable=False, server_default="0")) + op.add_column("attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True)) + op.add_column("attachment_jobs", sa.Column("last_error_permanent", sa.Boolean(), nullable=True)) + op.create_table( + "job_replays", + sa.Column("id", sa.Uuid(), primary_key=True), + sa.Column("job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False), + sa.Column("actor", sa.String(64), nullable=False), + sa.Column("reason", sa.String(32), nullable=False), + sa.Column("previous_attempt", sa.Integer(), nullable=False), + sa.Column("attempt_limit", sa.Integer(), nullable=False), + sa.Column("previous_error_code", sa.String(64), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.clock_timestamp()), + ) + op.create_index("ix_job_replays_job_id", "job_replays", ["job_id"]) + + +def downgrade() -> None: + op.drop_table("job_replays") + op.drop_column("attachment_jobs", "last_error_permanent") + op.drop_column("attachment_jobs", "attempt_limit") + op.drop_column("attachment_jobs", "cycle_start_attempt") diff --git a/requirements.txt b/requirements.txt index f171cf5..a3f67b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -50,3 +50,4 @@ pytest-asyncio==1.4.1 httpx==0.28.1 minio==7.2.20 Pillow==13.0.0 +aio-pika==10.2.2 diff --git a/scripts/check_lab5_jobs.py b/scripts/check_lab5_jobs.py new file mode 100644 index 0000000..ebba45f --- /dev/null +++ b/scripts/check_lab5_jobs.py @@ -0,0 +1,210 @@ +"""Fault/recovery lab5 на явно выбранном Compose-стенде; только синтетические данные.""" + +import argparse +from datetime import datetime +import hashlib +from io import BytesIO +import json +from pathlib import Path +import subprocess +import time +from uuid import UUID, uuid4 + +from PIL import Image + +from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose +from scripts.deployment import ComposeProject +from tests.client import Client, check + +PROBE = ''' +import asyncio, hashlib, json +from uuid import UUID +from sqlalchemy import select +from app.core.config import Settings +from app.core.database import create_db_engine, create_session_factory +from app.models.job import AttachmentJob +from app.models.attachment import Attachment +from app.storage.s3 import create_s3_storage +async def main(): + settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3) + try: + async with create_session_factory(engine)() as session: + job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one() + result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent) + objects=[] + for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'): + entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True))) + for obj in entries: + stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256() + try: + async for chunk in stream: checksum.update(chunk) + finally: await stream.aclose() + objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified)) + result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str)) + finally: await storage.aclose();await engine.dispose() +asyncio.run(main()) +''' + + +def probe(attachment_id): + code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE + result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30) + check(result.returncode == 0, 'Probe failed; diagnostic output hidden') + return json.loads(result.stdout) + + +def admin(*arguments, expected=0): + result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments) + check(result.returncode == expected, 'Unexpected operator command exit status') + return json.loads(result.stdout) + + +def verify_result(client, actor, attachment, original): + ready = client.wait_ready(actor, attachment, wait_seconds=180) + check(ready['id'] == attachment['id'], 'Attachment identity changed') + path = '/api/v1/attachments/' + ready['id'] + '/content' + check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed') + check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline') + for variant in ready['variants']: + data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes'] + check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch') + with Image.open(BytesIO(data)) as image: + check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch') + check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format') + check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked') + if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size') + snapshot=probe(ready['id']) + check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants') + return snapshot + + +def crash(client, actor, chat, mode): + name='lab5-worker-fault-'+uuid4().hex + try: + require_compose('stop','worker') + attachment,original=client.attachment(actor,chat) + result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id']) + phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack' + marker=json.dumps({'fault':phase,'attachment_id':attachment['id']}) + check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed') + before=probe(attachment['id']) + check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state') + check(len(before['objects'])==4,'Crash must occur after all four PUTs') + finally: + subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30) + require_compose('start','worker') + after=verify_result(client,actor,attachment,original) + if mode=='crash-before-ack': + # Wait for RabbitMQ to deliver and acknowledge the ready duplicate. + wait_queue_empty() + after=probe(attachment['id']) + check(after==before,'Ready duplicate rewrote result') + else: + check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt') + check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content') + return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after) + + +def retry_replay(client,actor,chat,permanent): + name='lab5-retry-fault-'+uuid4().hex + samples=[] + configuration=ComposeProject().config['services']['worker']['environment'] + retry_base=float(configuration['JOB_RETRY_SECONDS']) + retry_max=float(configuration['JOB_RETRY_MAX_SECONDS']) + try: + require_compose('stop','worker') + attachment,original=client.attachment(actor,chat) + mode='fail-permanent' if permanent else 'fail-transient' + require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id']) + deadline=time.monotonic()+180 + while time.monotonic()=1 and queued_before['durable'], 'No durable queued message before restart') + result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180) + check(result.returncode==0,'Restart hook failed') + check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker') + queued_after=queue_state() + check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart') + client.message(actor,chat,'session and chat survived broker/API restart') + finally:require_compose('start','worker') + after=verify_result(client,actor,attachment,original) + return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after) + + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('--base-url',required=True) + parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all') + parser.add_argument('--output',default='evidence/lab5-jobs.json') + args=parser.parse_args() + client=Client(args.base_url,timeout=40) + aliases={r.instance_id for r in ComposeProject().replicas() if r.running} + check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL') + actor,_,_,chat=client.bootstrap() + results=[] + operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)} + selected=operations.values() if args.scenario=='all' else [operations[args.scenario]] + for operation in selected: + result=operation(client,actor,chat);results.append(result) + Path(args.output).parent.mkdir(parents=True,exist_ok=True) + Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n') + print('PASS: '+result['scenario'],flush=True) + Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n') + + +if __name__=='__main__':main() diff --git a/scripts/check_outbox_recovery.py b/scripts/check_outbox_recovery.py new file mode 100644 index 0000000..e5379af --- /dev/null +++ b/scripts/check_outbox_recovery.py @@ -0,0 +1,83 @@ +"""Сценарий COMMIT → SIGKILL до publish → ready без повторного upload. + +Требуется уже работающий отдельный стенд. Скрипт не выполняется при запуске API. +Публичные tests не изменяет, secrets/tokens не печатает и evidence сам не сохраняет. +""" + +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +from uuid import uuid4 + +from scripts.deployment import ComposeProject +from tests.client import Client, check + +ROOT = Path(__file__).resolve().parents[1] +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess: + return subprocess.run(COMPOSE + list(arguments), cwd=ROOT, text=True, capture_output=True, timeout=timeout) + + +def require_compose(*arguments: str) -> str: + result = compose(*arguments) + check(result.returncode == 0, "Compose command failed; output hidden") + return result.stdout + + +def main() -> None: + parser = argparse.ArgumentParser(description="Отказ publisher после COMMIT перед RabbitMQ publish") + parser.add_argument("--base-url", required=True) + parser.add_argument("--ca-file") + args = parser.parse_args() + running = set(require_compose("ps", "--status", "running", "--services").splitlines()) + check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the lab5 stack before this scenario") + client = Client(args.base_url, args.ca_file, timeout=40) + project = ComposeProject() + aliases = {replica.instance_id for replica in project.replicas() if replica.running} + live = client.call("GET", "/health/live", schema="Health") + check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project") + alice, _, _, chat = client.bootstrap() + attachment = None + fault_container = "lab5-outbox-fault-" + uuid4().hex + try: + require_compose("stop", "publisher") + attachment, original = client.attachment(alice, chat) + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected=409, actor=alice) + killed = compose( + "run", "--rm", "--no-deps", "--name", fault_container, "publisher", "python", "-m", + "app.core.outbox_publisher", "--crash-before-publish", attachment["id"], + ) + marker = json.dumps({"fault": "before_publish", "attachment_id": attachment["id"]}) + check(killed.returncode == 137 and marker in killed.stdout, "Expected targeted SIGKILL before publish was not observed") + pending = client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=alice, schema="Attachment")["json"] + check(pending["status"] == "queued", "Crash must leave accepted job queued") + finally: + # Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется. + try: + subprocess.run(["docker", "rm", "-f", fault_container], capture_output=True, timeout=30) + finally: + require_compose("start", "publisher") + ready = client.wait_ready(alice, attachment, wait_seconds=60) + check(ready["id"] == attachment["id"], "Recovered a different attachment") + check({variant["name"] for variant in ready["variants"]} == {"thumbnail", "crop", "optimized"}, "Missing required variants") + path = f"/api/v1/attachments/{attachment['id']}/content" + check(client.call("GET", path, actor=alice)["bytes"] == original, "Original changed after recovery") + for variant in ready["variants"]: + downloaded = client.call("GET", path + "?variant=" + variant["name"], actor=alice)["bytes"] + check(len(downloaded) == variant["size_bytes"] and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], "Recovered variant checksum mismatch") + print(json.dumps({ + "scenario": "commit_before_publish", "attachment_id": ready["id"], + "message_id": message["id"], "publisher_exit_code": killed.returncode, + "status_after_crash": pending["status"], "status_after_restart": ready["status"], + "variants": [variant["name"] for variant in ready["variants"]], + "reuploaded": False, + })) + + +if __name__ == "__main__": + main() diff --git a/scripts/init_local.py b/scripts/init_local.py index baa1402..a3fb687 100644 --- a/scripts/init_local.py +++ b/scripts/init_local.py @@ -14,7 +14,7 @@ def main() -> None: destination.chmod(0o600) directory = root / "secrets" directory.mkdir(mode=0o700, exist_ok=True) - for name in ("db_password", "redis_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"): + for name in ("db_password", "redis_password", "broker_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"): path = directory / name try: descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444) diff --git a/scripts/restart.sh b/scripts/restart.sh index 1d313c9..493ee34 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -22,11 +22,16 @@ wait_healthy() { done } -compose stop cleanup -compose restart postgres redis minio -wait_healthy postgres redis minio +worker_running=$(compose ps --status running --quiet worker) +compose stop worker publisher cleanup +compose restart postgres redis minio broker +wait_healthy postgres redis minio broker compose restart api wait_healthy api compose restart nginx wait_healthy nginx -compose start cleanup +compose start publisher cleanup +# Сценарий с заранее остановленным worker сохраняет его остановленным. +if [ -n "$worker_running" ]; then + compose start worker +fi diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py index 5661d14..0deb88f 100644 --- a/student-tests/test_attachments.py +++ b/student-tests/test_attachments.py @@ -1,4 +1,4 @@ -"""Lab4: API и транзакции с реальной БД; байты S3 подменены управляемым адаптером.""" +"""Lab5: API и транзакции с реальной БД; байты S3 подменены управляемым адаптером.""" import asyncio import hashlib @@ -17,6 +17,10 @@ from sqlalchemy.ext.asyncio import AsyncSession from starlette.requests import ClientDisconnect from app.core.s3 import S3Settings +from app.core.jobs import JobSettings +from app.domain.attachment_jobs import process_attachment_job +from app.models.job import AttachmentJob +from app.schemas.job import JobMessage from app.domain import attachments as domain from app.errors.exceptions import unavailable from app.models.attachment import Attachment, UploadIntent @@ -79,14 +83,31 @@ async def upload(client, actor, chat, data=PNG, filename='фото.png', content ) +async def finish_attachment(client, storage, actor, attachment_id): + engine = client._transport.app.state.engine + async with AsyncSession(engine) as session: + job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))) + message = JobMessage(job_id=job.id, attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, attempt=job.attempt + 1, created_at=job.created_at) + await session.rollback() + assert await process_attachment_job(session, storage, message, JobSettings(_env_file=None)) + result = await client.get(f'/api/v1/attachments/{attachment_id}', headers=bearer(actor)) + assert result.json()['status'] == 'ready' + return result.json() + + async def test_upload_download_and_attachment_only_message(client, context): storage, alice, bob, eve, chat = context response = await upload(client, alice, chat) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text attachment = response.json() - assert attachment['status'] == 'ready' + assert attachment['status'] == 'queued' + before = await client.get(f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice)) + assert before.status_code == 409 + attachment = await finish_attachment(client, storage, alice, attachment['id']) assert attachment['sha256'] == hashlib.sha256(PNG).hexdigest() - assert attachment['variants'] == [] and attachment['error_code'] is None + assert {v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'} + assert attachment['error_code'] is None assert 'object_key' not in attachment id = attachment['id'] for actor in (alice, bob): @@ -120,13 +141,14 @@ async def test_invalid_image_and_mime_are_415(client, context, data, content_typ async def test_jpeg_is_stored_without_reencoding(client, context): - _, alice, _, _, chat = context + storage, alice, _, _, chat = context output = BytesIO() Image.new('RGB', (32, 32), 'blue').save(output, format='JPEG') original = output.getvalue() response = await upload(client, alice, chat, original, content_type='image/jpeg') - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text id = response.json()['id'] + await finish_attachment(client, storage, alice, id) downloaded = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice)) assert downloaded.content == original @@ -167,7 +189,7 @@ async def test_exact_byte_limit_accepts_valid_png(client, context): padding = struct.pack('!I', len(payload)) + kind + payload + struct.pack('!I', zlib.crc32(kind + payload)) image = PNG[:-12] + padding + PNG[-12:] response = await upload(client, alice, chat, image) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text assert response.json()['size_bytes'] == maximum assert response.json()['sha256'] == hashlib.sha256(image).hexdigest() @@ -178,7 +200,7 @@ async def test_exact_pixel_limit_accepts_valid_png(client, context): with Image.new('RGB', (8000, 5000), 'blue') as image: image.save(output, format='PNG') response = await upload(client, alice, chat, output.getvalue()) - assert response.status_code == 201, response.text + assert response.status_code == 202, response.text async def test_pixel_limit_before_decoding(client, context): @@ -226,6 +248,7 @@ async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context): async def test_s3_outage_is_503(client, context): storage, alice, _, _, chat = context id = (await upload(client, alice, chat)).json()['id'] + await finish_attachment(client, storage, alice, id) storage.down = True response = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice)) assert response.status_code == 503 @@ -353,6 +376,7 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine): storage, alice, _, _, chat = context id = (await upload(client, alice, chat)).json()['id'] + await finish_attachment(client, storage, alice, id) async with AsyncSession(db_engine) as session: attachment = await session.get(Attachment, UUID(id)) key = attachment.object_key diff --git a/student-tests/test_image_pipeline.py b/student-tests/test_image_pipeline.py new file mode 100644 index 0000000..bbed62f --- /dev/null +++ b/student-tests/test_image_pipeline.py @@ -0,0 +1,71 @@ +"""Реальные преобразования и OS-лимиты дочернего процесса worker.""" + +import asyncio +from io import BytesIO +import os +from pathlib import Path +import subprocess +import sys +from uuid import uuid4 + +from PIL import Image +import pytest + +from app.core.image_processing import ImageSettings +from app.domain.image_pipeline import create_variants +from app.domain.job_files import render_variants +from app.storage.jobs import JobWork +from test_attachments import MemoryStorage + + +@pytest.mark.parametrize('size,transparent', [((1600,800),False),((24,12),True)]) +async def test_variant_geometry_metadata_and_alpha(tmp_path,size,transparent): + source=tmp_path/'source.png' + with Image.new('RGBA',size,(255,0,0,0 if transparent else 255)) as image: + exif=Image.Exif();exif[270]='private description' + image.save(source,format='PNG',exif=exif) + result=create_variants(source,tmp_path,'image/png',40_000_000,ImageSettings(_env_file=None)) + variants={v.name:v for v in result.variants} + assert (variants['crop'].width,variants['crop'].height)==(128,128) + assert (variants['thumbnail'].width,variants['thumbnail'].height)==((256,128) if size[0]>256 else size) + assert (variants['optimized'].width,variants['optimized'].height)==((1280,640) if size[0]>1280 else size) + for name in ('thumbnail.jpg','crop.jpg','optimized.webp'): + with Image.open(tmp_path/name) as image: + assert not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp') + if transparent: + if name.endswith('webp'):assert image.getpixel((0,0))[3]==0 + else:assert min(image.getpixel((0,0)))>=250 + + +async def test_exif_orientation_applied(tmp_path): + source=tmp_path/'source.jpg' + with Image.new('RGB',(400,200),'blue') as image: + exif=Image.Exif();exif[274]=6 + image.save(source,exif=exif) + result=create_variants(source,tmp_path,'image/jpeg',40_000_000,ImageSettings(_env_file=None)) + assert (result.variants[0].width,result.variants[0].height)==(128,256) + + +@pytest.mark.parametrize('failure', ['memory','cpu']) +async def test_worker_os_limits(failure,tmp_path): + body='return bytearray(2 * 1024 ** 3)' if failure=='memory' else 'while True: pass' + code="from app.core import thumbnail_worker as w\ndef fail(*args):\n "+body+"\nw.create_variants=fail\nraise SystemExit(w.main())" + result=await asyncio.to_thread(subprocess.run,[sys.executable,'-c',code,'source',str(tmp_path),str(tmp_path/'metadata'),'image/png','40000000'], + env={**os.environ,'IMAGE_MEMORY_MIB':'256','IMAGE_CPU_SECONDS':'1'},capture_output=True,timeout=15) + assert result.returncode==(3 if failure=='memory' else -9) + assert not (tmp_path/'metadata').exists() + + +async def test_render_timeout_kills_child(monkeypatch,tmp_path): + create=asyncio.create_subprocess_exec + children=[] + async def slow_child(*args,**kwargs): + child=await create(sys.executable,'-c','import time;time.sleep(60)',**kwargs) + children.append(child) + return child + monkeypatch.setattr(asyncio,'create_subprocess_exec',slow_child) + work=JobWork(uuid4(),uuid4(),uuid4(),1,'staging/test','image/png',1,'0'*64) + with pytest.raises(TimeoutError): + async with asyncio.timeout(.2): + await render_variants(MemoryStorage(),work,tmp_path/'source',tmp_path) + assert len(children)==1 and children[0].returncode is not None diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py new file mode 100644 index 0000000..aeab649 --- /dev/null +++ b/student-tests/test_jobs.py @@ -0,0 +1,178 @@ +"""Lab5: retry/replay, fencing, cleanup, limits и транзакции реального PostgreSQL.""" + +import asyncio +from datetime import timedelta +import errno +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.jobs import JobSettings +from app.domain import attachment_jobs +from app.domain.attachment_jobs import claim_job, process_attachment_job, cleanup_staging, build_result +from app.domain.job_admin import replay_failed_job +from app.domain.job_retry import classify_failure, retry_delay +from app.errors.exceptions import JobOperationError, ProcessingError +from app.models.attachment import Attachment +from app.models.job import AttachmentJob +from app.models.job_replay import JobReplay +from app.schemas.job import JobMessage +from app.storage import jobs +from test_attachments import context, upload, finish_attachment + + +async def envelope(session, attachment_id): + job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))) + return JobMessage(job_id=job.id, attachment_id=job.attachment_id, pipeline_version=1, + attempt=job.attempt + 1, created_at=job.created_at) + + +async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + settings = JobSettings(_env_file=None, job_max_attempts=3) + def fail(_): + raise TimeoutError + async with AsyncSession(db_engine, expire_on_commit=False) as session: + first = await envelope(session, attachment['id']) + for attempt in range(1, 4): + message = await envelope(session, attachment['id']) + assert await process_attachment_job(session, storage, message, settings, before_processing=fail) + job, item = await jobs.lock_job(session, first.job_id) + assert job.attempt == attempt + delay = (job.next_attempt_at - item.updated_at).total_seconds() + assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1) + assert item.status == ('failed' if attempt == 3 else 'queued') + assert job.last_error_permanent is False + job.next_attempt_at = await jobs.database_time(session) - timedelta(seconds=1) + await session.commit() + assert item.error_code == 'retry_exhausted' + request_id = uuid4() + result = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id, + actor='test', reason='dependency_restored') + assert result['attempt_limit'] == 6 + # Old envelope cannot consume the fresh replay budget. + assert await process_attachment_job(session, storage, first, settings) + job, item = await jobs.lock_job(session, first.job_id) + assert job.attempt == 3 and item.status == 'queued' + await session.commit() + message = await envelope(session, attachment['id']) + assert await process_attachment_job(session, storage, message, settings) + before = dict(storage.objects) + assert await process_attachment_job(session, storage, message, settings) + assert storage.objects == before + repeat = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id, + actor='test', reason='dependency_restored') + assert repeat['already_applied'] + with pytest.raises(JobOperationError, match='job_not_failed'): + await replay_failed_job(session, storage, settings, first.job_id, request_id=uuid4(), actor='test', reason='manual_review') + await session.rollback() + assert len((await session.scalars(select(JobReplay))).all()) == 1 + + +async def test_permanent_failure_and_missing_source_replay(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + def fail(_): + raise ProcessingError('image_limits') + settings = JobSettings(_env_file=None) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment['id']) + await process_attachment_job(session, storage, message, settings, before_processing=fail) + job, item = await jobs.lock_job(session, message.job_id) + assert item.status == 'failed' and job.attempt == 1 and job.last_error_permanent + job.staging_key = None + await session.commit() + with pytest.raises(JobOperationError, match='source_expired'): + await replay_failed_job(session, storage, settings, job.id, request_id=uuid4(), actor='test', reason='manual_review') + + +async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + settings = JobSettings(_env_file=None) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment['id']) + claim = await claim_job(session, message, settings) + job, item = await jobs.lock_job(session, message.job_id) + job.lease_token = uuid4() + job.finished_at = await jobs.database_time(session) - timedelta(days=10) + await session.commit() + assert await cleanup_staging(session, storage, settings, apply=True) == 0 + # Rendering must not be allowed to PUT on a stale lease. + before = dict(storage.objects) + assert await build_result(session, storage, claim.work) + assert storage.objects == before + + +async def test_cleanup_ready_preserves_final_objects(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + await finish_attachment(client, storage, alice, attachment['id']) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment['id']) + job, item = await jobs.lock_job(session, message.job_id) + staging = job.staging_key + job.finished_at = await jobs.database_time(session) - timedelta(days=2) + await session.commit() + assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=False) == 1 + assert staging in storage.objects + assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 1 + assert staging not in storage.objects and len(storage.objects) == 4 + assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 0 + + +async def test_retry_classification_and_cap(): + settings = JobSettings(_env_file=None) + for attempt in (1, 2, 6, 100): + cap = min(120, 5 * 2 ** min(30, attempt - 1)) + assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100)) + assert classify_failure(TimeoutError()) == ('processing_timeout', False) + assert classify_failure(OSError(errno.EACCES, 'denied'))[1] + assert not classify_failure(OSError(errno.ENOSPC, 'full'))[1] + + +async def test_timeout_records_retry_and_clears_lease(client, context, db_engine, monkeypatch): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + async def stalled(*args, **kwargs): + await asyncio.sleep(60) + monkeypatch.setattr(attachment_jobs, 'build_result', stalled) + settings = JobSettings(_env_file=None).model_copy(update={'job_timeout_seconds': .05}) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment['id']) + assert await process_attachment_job(session, storage, message, settings) + job, item = await jobs.lock_job(session, message.job_id) + assert item.status == 'queued' and job.last_error_code == 'processing_timeout' + assert job.lease_token is None and job.lease_until is None + + +async def test_failed_retention_and_source_checksum_protect_replay(client, context, db_engine): + storage, alice, _, _, chat = context + attachment = (await upload(client, alice, chat)).json() + def fail(_): + raise ProcessingError('invalid_image') + settings = JobSettings(_env_file=None) + async with AsyncSession(db_engine, expire_on_commit=False) as session: + message = await envelope(session, attachment['id']) + await process_attachment_job(session, storage, message, settings, before_processing=fail) + job, item = await jobs.lock_job(session, message.job_id) + key = job.staging_key + job.finished_at = await jobs.database_time(session) - timedelta(days=2) + await session.commit() + assert await cleanup_staging(session, storage, settings, apply=True) == 0 + data, metadata = storage.objects[key] + storage.objects[key] = (b'x' * len(data), metadata) + with pytest.raises(JobOperationError, match='source_mismatch'): + await replay_failed_job(session, storage, settings, message.job_id, request_id=uuid4(), actor='test', reason='manual_review') + await session.rollback() + assert not (await session.scalars(select(JobReplay))).all() + job, item = await jobs.lock_job(session, message.job_id) + job.finished_at = await jobs.database_time(session) - timedelta(days=8) + await session.commit() + assert await cleanup_staging(session, storage, settings, apply=True) == 1 + assert key not in storage.objects diff --git a/student-tests/test_migrations.py b/student-tests/test_migrations.py index f13139d..f8efed1 100644 --- a/student-tests/test_migrations.py +++ b/student-tests/test_migrations.py @@ -8,7 +8,7 @@ from sqlalchemy import inspect, text from app.models import Base from conftest import reset_schema, upgrade_head -EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents"} +EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents", "attachment_jobs", "job_replays"} @pytest.fixture