included rabbitmq as broker
This commit is contained in:
+22
-4
@@ -1,5 +1,5 @@
|
||||
# Скопируйте в .env для локального окружения; .env не коммитится.
|
||||
# Настройки приложения и Compose; параметры будущей очереди пока не используются.
|
||||
# Настройки приложения, Compose и очереди lab5.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
@@ -72,10 +72,28 @@ MINIO_ROOT_USER_FILE=/run/secrets/minio_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE=/run/secrets/minio_root_password
|
||||
MINIO_BROWSER=off
|
||||
|
||||
# Lab5: будущие параметры очереди/worker, пока без реализации.
|
||||
# Lab5: отдельный RabbitMQ broker и ограниченный worker.
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
# Брокер пока не выбран; файл ещё не создаётся и не монтируется.
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
BROKER_HOST=broker
|
||||
BROKER_PORT=5672
|
||||
BROKER_PASSWORD_FILE=/run/secrets/broker_password
|
||||
BROKER_QUEUE=attachment-jobs
|
||||
BROKER_USER=messenger
|
||||
BROKER_VHOST=/
|
||||
JOB_LEASE_SECONDS=90
|
||||
JOB_RETRY_SECONDS=5
|
||||
PUBLISHER_POLL_SECONDS=1
|
||||
|
||||
# Качество вариантов и лимиты одного процесса обработки
|
||||
IMAGE_JPEG_QUALITY=85
|
||||
IMAGE_WEBP_QUALITY=80
|
||||
IMAGE_WEBP_METHOD=4
|
||||
IMAGE_MEMORY_MIB=768
|
||||
IMAGE_CPU_SECONDS=20
|
||||
|
||||
# Retry cap and retained failed sources for explicit replay
|
||||
JOB_RETRY_MAX_SECONDS=120
|
||||
FAILED_STAGING_RETENTION_SECONDS=604800
|
||||
|
||||
@@ -43,7 +43,8 @@ endef
|
||||
rolling-update failover-all test-lab3-grade4
|
||||
|
||||
help:
|
||||
@echo "Lab4: init | test-grade4 | test-s3 | load-uploads | cleanup-dry-run | cleanup-apply"
|
||||
@echo "Lab5: init | worker-stop | worker-start | jobs-logs | cleanup-dry-run | cleanup-apply"
|
||||
@echo "Lab5: test-grade4 | test-jobs; legacy lab4: test-s3 | load-uploads"
|
||||
@echo "Проверки: check | validate | test | load | test-persistence ACTION_SCRIPT=scripts/restart.sh"
|
||||
@echo " test-failover ACTION_SCRIPT=scripts/stop-one.sh"
|
||||
@echo "Основной: up | down | restart | restore | ps | logs | migrate | instances"
|
||||
@@ -65,10 +66,13 @@ init:
|
||||
test-grade4:
|
||||
$(MAKE) check validate
|
||||
$(MAKE) test-stand-up
|
||||
$(MAKE) test-docker test-stand-smoke test-stand-idempotency test-stand-persistence test-stand-failover
|
||||
$(TEST_ENV) $(PYTHON) -m scripts.check_s3 --base-url "$(TEST_BASE_URL)" --with-outage --with-faults
|
||||
$(TEST_ENV) $(PYTHON) -m scripts.upload_load --base-url "$(TEST_BASE_URL)" --concurrency 10
|
||||
$(TEST_ENV) $(PYTHON) -m scripts.check_grade4 --base-url "$(TEST_BASE_URL)"
|
||||
$(MAKE) test-docker
|
||||
$(MAKE) test-stand-smoke
|
||||
$(MAKE) test-stand-idempotency
|
||||
$(MAKE) test-stand-persistence
|
||||
$(MAKE) test-stand-failover
|
||||
$(TEST_ENV) $(PYTHON) -m scripts.check_outbox_recovery --base-url "$(TEST_BASE_URL)"
|
||||
$(TEST_ENV) $(PYTHON) -m scripts.check_lab5_jobs --base-url "$(TEST_BASE_URL)"
|
||||
|
||||
test-s3:
|
||||
$(PYTHON) -m scripts.check_s3 --base-url "$(BASE_URL)" --with-outage --with-faults
|
||||
@@ -226,3 +230,23 @@ expiry-demo: test-stand-up
|
||||
|
||||
failure-demo: test-stand-up
|
||||
$(TEST_ENV) BASE_URL=$(TEST_BASE_URL) scripts/failure_demo.sh
|
||||
|
||||
# --- Очередь lab5 ---
|
||||
.PHONY: worker-stop worker-start jobs-logs
|
||||
worker-stop:
|
||||
$(COMPOSE) stop worker
|
||||
|
||||
worker-start:
|
||||
$(COMPOSE) start worker
|
||||
|
||||
jobs-logs:
|
||||
$(COMPOSE) logs -f publisher worker broker
|
||||
|
||||
# Уже работающий изолированный стенд; сам цель его не поднимает.
|
||||
.PHONY: test-outbox-recovery
|
||||
test-outbox-recovery:
|
||||
$(PYTHON) -m scripts.check_outbox_recovery --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: test-jobs
|
||||
test-jobs:
|
||||
$(PYTHON) -m scripts.check_lab5_jobs --base-url "$(BASE_URL)"
|
||||
|
||||
@@ -1,75 +1,225 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: Григорян Илья Мурадович, БПИ2303 (из отчёта lab4).
|
||||
- Целевая оценка: 5; исследовательский трек пока не выбран.
|
||||
- Этап: перенос базы lab4 и заготовки с docstring, без реализации lab5.
|
||||
- Дата переноса: 07.11.2026.
|
||||
- Commit/PR lab5: заполнить после публикации.
|
||||
- Использованные источники: README-файлы и contracts/ репозитория lab5,
|
||||
исходники lab4. Использована помощь Codex. Новых зависимостей нет.
|
||||
Автор: Григорян Илья Мурадович, БПИ2303.
|
||||
Целевая оценка текущего этапа: **4**. Обновлено 10.11.2026.
|
||||
Дальнейшая цель — 5; исследовательский трек ещё не выбран.
|
||||
Использована помощь Codex при реализации, проверках и исправлениях.
|
||||
Commit/PR: заполнить после публикации.
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Запуск
|
||||
|
||||
Нужны Python 3.12, pip, Docker с Compose. Подготовка и запуск:
|
||||
Нужны Python 3.12, pip, Docker с Compose. Зависимости закреплены в requirements.txt.
|
||||
|
||||
```sh
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt
|
||||
make init
|
||||
make up
|
||||
make test
|
||||
```
|
||||
|
||||
HTTP: http://localhost:18081. `make down` останавливает стенд с сохранением данных.
|
||||
Тестовый стенд: `make test-stand-up`, остановка: `make test-stand-down`.
|
||||
Намеренное удаление учебных данных: `make test-stand-destroy` для тестового стенда,
|
||||
`docker compose --env-file .env down --volumes` для основного.
|
||||
Версии зависимостей перенесены без обновления. Локальные secrets/.env генерируются заново, не копируются из lab4.
|
||||
Стенд при переносе не запускался; воспроизводимость после переноса ещё не проверена.
|
||||
`make init` создаёт отсутствующие .env и файлы secrets/, сохраняя существующие.
|
||||
Все environment-параметры Compose подставляются из .env; образец — .env.example.
|
||||
Пароли и S3 credentials читаются из файлов secrets, не включаются в задания.
|
||||
API не получает credentials брокера. По умолчанию HTTP доступен на
|
||||
http://localhost:18081. Broker, MinIO и основные PostgreSQL/Redis наружу не открыты.
|
||||
|
||||
## Архитектура и решения
|
||||
При обновлении ранее запущенного стенда сначала остановить старые процессы
|
||||
через `make down` без удаления volumes, затем выполнить `make up`.
|
||||
Migrate применяет ревизии до a62c950e18b3 перед стартом приложения.
|
||||
Миграции добавляют jobs, варианты, бюджет попыток и аудит replay. Готовые
|
||||
вложения раннего варианта lab5 переобрабатываются для заполнения crop/WebP.
|
||||
При переходе Redis Streams → RabbitMQ миграция повторно публикует queued jobs
|
||||
из PostgreSQL; processing восстанавливаются по истечении lease. Старые процессы
|
||||
обоих брокеров во время перехода должны быть остановлены.
|
||||
|
||||
Перенесены приложение FastAPI, PostgreSQL-миграции, общие Redis-сессии,
|
||||
идемпотентность сообщений, приватный MinIO, две API-реплики за Nginx, собственные
|
||||
тесты и скрипты эксплуатации lab4. Перенесена и незакоммиченная правка MinIO
|
||||
bootstrap из исходного рабочего дерева. Исходный lab4 не изменён.
|
||||
## Инфраструктура и устойчивый приём
|
||||
|
||||
Имена Compose-проектов, образы и порты отделены от lab4. Рабочие настройки —
|
||||
.env.example; там же указаны будущие параметры очереди, пока не используемые.
|
||||
Compose получает настройки environment из .env, включая пути S3, PostgreSQL
|
||||
и MinIO к файлам секретов; секретные значения остаются в secrets/.
|
||||
Пути *_FILE в Compose также задают точки монтирования соответствующих secrets;
|
||||
MinIO bootstrap читает эти пути из окружения.
|
||||
Публичные tests/, contracts/, lab.json и README.md lab5 сохранены.
|
||||
| Компонент | Реализация | Данные |
|
||||
| --- | --- | --- |
|
||||
| API | две реплики FastAPI, Python 3.12.15 | PostgreSQL и Redis сессий |
|
||||
| Балансировка | nginx 1.28.0 | единая HTTP-точка входа |
|
||||
| База | PostgreSQL 16.10 | сообщения, metadata, jobs, outbox, audit |
|
||||
| Сессии | Redis 7.4.2, AOF always | redisdata |
|
||||
| Объекты | приватный MinIO | staging, originals, variants |
|
||||
| Брокер | RabbitMQ 4.4.0 | durable очередь, rabbitmqdata |
|
||||
| Publisher | отдельный процесс приложения, aio-pika 10.2.2 | PostgreSQL → RabbitMQ |
|
||||
| Worker | отдельный процесс приложения | RabbitMQ → MinIO + PostgreSQL |
|
||||
| Cleanup | отдельный процесс | незавершённые upload и terminal staging |
|
||||
|
||||
Будущая архитектура: API → durable staging + PostgreSQL job/outbox → publisher →
|
||||
broker → worker → конечный MinIO + атомарная фиксация metadata в PostgreSQL.
|
||||
Текущая реализация продолжает синхронно сохранять оригинал и возвращает 201 ready.
|
||||
Заготовки lab5 не подключены и вызывают NotImplementedError. Реализация транзакционных границ и восстановления после отказов ещё предстоит.
|
||||
Redis используется для сессий. Очередь заданий использует RabbitMQ через AMQP.
|
||||
Очередь durable, сообщения persistent. Publisher ожидает подтверждение брокером
|
||||
приёма сообщения (publisher confirm) и только затем фиксирует published_attempt.
|
||||
При неизвестном результате публикации возможен дубль; намерение остаётся в БД.
|
||||
Подтверждение worker (consumer ack) следует после устойчивого результата или
|
||||
устойчиво сохранённого перехода retry/failed. При разрыве соединения RabbitMQ
|
||||
возвращает неподтверждённые доставки в очередь.
|
||||
|
||||
## Доказательства критериев
|
||||
До ответа `202 queued` API полностью принимает и проверяет оригинал, сохраняет
|
||||
его в приватный staging и фиксирует attachment/job одной транзакцией PostgreSQL.
|
||||
Job одновременно служит transactional outbox — устойчивым намерением публикации.
|
||||
Worker и broker не входят в синхронный путь upload и readiness API. Остановка
|
||||
worker не заставляет API выполнять resize. При компенсации неоднозначного COMMIT
|
||||
API проверяет БД, чтобы не удалить оригинал уже принятого задания.
|
||||
|
||||
| Критерий README | Будущая команда/сценарий | Текущий результат | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke lab5 | make test | Не запускался; 202/thumbnail ещё не реализованы | Нет |
|
||||
| 3: сохранён минимум lab4 | Собственные тесты, persistence/failover | Перенесён, повторно не проверен | Не подтверждает lab5 |
|
||||
| 3: durable приём и broker/worker | Worker stop → upload → restart → ready | Не реализовано | Нет |
|
||||
| 3: состояния, original/thumbnail, retry, дубли | Smoke + собственные сценарии | Не реализовано | Нет |
|
||||
| 4: crop/optimized, resource limits | Собственные проверки изображений/лимитов | Не реализовано | Нет |
|
||||
| 4: outbox и восстановление COMMIT/publish | Fault injection | Не реализовано | Нет |
|
||||
| 4: backoff/jitter, failed/replay | Сбой до исчерпания и явный replay | Не реализовано | Нет |
|
||||
| 4: crash до ack, lease, concurrency, cleanup | Fault injection и lifecycle | Не реализовано | Нет |
|
||||
| 5: законченный исследовательский трек | Методика после выбора трека | Трек не выбран | Нет |
|
||||
Envelope соответствует contracts/jobs.schema.json: job_id, attachment_id,
|
||||
pipeline_version=1, attempt, created_at, schema_version, type и необязательный
|
||||
trace context. Bytes, имена объектов S3, URL и credentials туда не помещаются.
|
||||
Доступ к вложениям проверяется по участию в чате; content до ready возвращает 409.
|
||||
Аутентификация, пагинация, идемпотентность сообщений и ограничения lab4 сохранены.
|
||||
|
||||
## Эксперименты и отказы
|
||||
## Изображения и ограничения
|
||||
|
||||
Пока не проводились. После выбора трека описать гипотезу, hardware/лимиты,
|
||||
версии, размеры/число файлов, профиль нагрузки, concurrency/prefetch, длительность,
|
||||
прогрев и повторы. Сохранить raw summary без секретов, queue lag, время до ready,
|
||||
RSS и другие метрики выбранного трека.
|
||||
Worker повторно сверяет размер и SHA-256 исходника. Original сохраняется без
|
||||
перекодирования. Все варианты создаются в отдельном ограниченном процессе:
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
| Вариант | Формат и размеры | Параметры |
|
||||
| --- | --- | --- |
|
||||
| thumbnail | JPEG, вписан в 256×256 без увеличения | quality=85, белая подложка alpha |
|
||||
| crop | JPEG, центральный crop 128×128 | quality=85; маленькое изображение увеличивается |
|
||||
| optimized | WebP, вписан в 1280×1280 без увеличения | quality=80, method=4, alpha сохраняется |
|
||||
|
||||
Сейчас это база для дальнейшей реализации, не завершённая lab5. Очереди и
|
||||
асинхронной обработки нет; прежние результаты lab4 к этому дереву не относятся.
|
||||
Сначала просмотр пользователем и выбор трека/брокера, затем реализация;
|
||||
тестовый прогон и новые evidence — по отдельному указанию после реализации.
|
||||
Применяется EXIF orientation; EXIF/ICC/XMP и текстовые metadata не переносятся
|
||||
в варианты. JPEG создаётся из RGB с белой подложкой, WebP — из чистого RGBA.
|
||||
Алгоритмы используют Pillow 13.0.0. Metadata содержит реальные размеры, bytes и SHA-256.
|
||||
|
||||
Лимиты: PNG/JPEG, 10 MiB, 40 млн пикселей; JOB_TIMEOUT_SECONDS=60;
|
||||
IMAGE_MEMORY_MIB=768 адресного пространства и IMAGE_CPU_SECONDS=20 CPU на процесс.
|
||||
Контейнер worker: 2 GiB, 2 CPU, tmpfs 128 MiB. Отмена завершает дочерний процесс;
|
||||
S3 adapter дожидается SDK-потока перед освобождением блокировки/компенсацией.
|
||||
|
||||
## Повторы, идемпотентность и очистка
|
||||
|
||||
Состояния: queued → processing → ready; временный отказ возвращает queued,
|
||||
постоянный отказ или исчерпание бюджета переводит failed с безопасным error_code.
|
||||
JOB_MAX_ATTEMPTS=5 на цикл. Номер attempt монотонен, бюджет закреплён в attempt_limit.
|
||||
Для попытки n текущего цикла ceiling=min(JOB_RETRY_MAX_SECONDS,
|
||||
JOB_RETRY_SECONDS × 2^(n−1)); delay равномерно выбирается в [ceiling/2, ceiling].
|
||||
База — 5 секунд, максимум — 120 секунд. next_attempt_at сохраняется в БД;
|
||||
дубликат не пересчитывает задержку. Последняя причина остаётся в job даже при
|
||||
публичном retry_exhausted. Ошибки изображения/исходника/конфигурации постоянные;
|
||||
timeout, временные ошибки S3/сети и файловой системы повторяются.
|
||||
|
||||
Job захватывается с lease_token и JOB_LEASE_SECONDS=90. Publisher восстанавливает
|
||||
истёкший processing в пределах бюджета. Перед PUT worker блокирует строку job,
|
||||
проверяет актуальность token и lease, держит блокировку до записи original,
|
||||
трёх вариантов и COMMIT ready. Reaper пропускает заблокированные строки.
|
||||
Устаревшая попытка не может начать запись. Фиксированные ключи по
|
||||
(attachment_id, pipeline_version, variant) исключают второй набор объектов.
|
||||
Дубль ready подтверждается без обработки и PUT. Это доставка как минимум один
|
||||
раз с идемпотентным эффектом, не exactly-once выполнение.
|
||||
|
||||
WORKER_CONCURRENCY=2, отдельный канал каждого consumer, prefetch=1 на канал.
|
||||
Publisher и worker используют init:true: Python не является PID 1, поэтому
|
||||
fault hooks с SIGKILL действительно завершают процесс. SIGTERM отменяет работу
|
||||
без преждевременного ack. Некорректные envelopes отклоняются без сохранения payload.
|
||||
|
||||
Queued/processing staging не удаляется. Для ready retention — 86400 секунд;
|
||||
для failed — FAILED_STAGING_RETENTION_SECONDS=604800 (семь суток).
|
||||
Cleanup и replay блокируют одну строку job. Если staging совпадает с готовым
|
||||
original, объект сохраняется. Частичные конечные объекты failed автоматически
|
||||
не удаляются; после replay используются те же ключи.
|
||||
|
||||
## Реестр failed и операторский replay
|
||||
|
||||
Failed registry находится в PostgreSQL. Команды требуют операторского доступа
|
||||
к Docker/credentials стенда; публичного endpoint для replay нет:
|
||||
|
||||
```sh
|
||||
docker compose run --rm --no-deps worker python -m app.core.job_admin list-failed
|
||||
docker compose run --rm --no-deps worker python -m app.core.job_admin show JOB_ID
|
||||
docker compose run --rm --no-deps worker python -m app.core.job_admin replay JOB_ID --request-id REQUEST_UUID --actor operator --reason dependency_restored
|
||||
docker compose run --rm --no-deps worker python -m app.core.job_admin history JOB_ID
|
||||
```
|
||||
|
||||
JOB_ID/REQUEST_UUID заменить настоящими UUID. Reasons: dependency_restored,
|
||||
configuration_fixed, manual_review. Actor — аудиторская метка, не система входа.
|
||||
Replay принимает только failed, сверяет сохранённый исходник по bytes/SHA-256,
|
||||
атомарно записывает аудит и новый ограниченный бюджет. Счётчик attempt не
|
||||
сбрасывается, старые envelopes не запускают новый цикл. Повтор того же request-id
|
||||
с теми же параметрами возвращает already_applied даже после ready; конфликт
|
||||
параметров отклоняется. После неоднозначного ответа повторяют тот же request-id.
|
||||
Готовую job с новым request-id повторить нельзя. После удаления staging replay
|
||||
отклоняется с source_expired; отсутствующий объект даёт source_missing.
|
||||
|
||||
## Проверки уровня 3 и 4
|
||||
|
||||
Полный воспроизводимый прогон на отдельном стенде:
|
||||
|
||||
```sh
|
||||
make test-grade4 PYTHON=.venv/bin/python
|
||||
```
|
||||
|
||||
Цель последовательно запускает контракт/OpenAPI, тестовый стенд, собственные
|
||||
тесты с реальными PostgreSQL/Redis, публичный smoke, идемпотентность через две
|
||||
реплики, persistence/failover, outbox и сценарии worker/replay. Подготовка БД
|
||||
для pytest выполняется только в выделенной базе *_test, Redis — DB15.
|
||||
Публичные tests/ и обязательные contracts/ не изменены.
|
||||
|
||||
Отдельные сценарии на уже работающем изолированном стенде:
|
||||
|
||||
```sh
|
||||
make test-outbox-recovery BASE_URL=http://localhost:28080
|
||||
make test-jobs BASE_URL=http://localhost:28080
|
||||
```
|
||||
|
||||
При выборе своего проекта задать COMPOSE_PROJECT_NAME и соответствующие порты
|
||||
в окружении. Сценарии сверяют BASE_URL с API выбранного Compose-проекта,
|
||||
создают синтетические данные и возвращают штатные worker/publisher в finally.
|
||||
Перезапуски сохраняют volumes. Сброса пользовательских данных нет.
|
||||
|
||||
Результаты и сопоставление каждому пункту приведены в evidence/lab5-verification.json;
|
||||
подробные снимки состояний/объектов/retry — в evidence/lab5-jobs.json.
|
||||
Секреты, tokens и исходные broker payload в evidence не записываются.
|
||||
|
||||
## Фактический результат проверок
|
||||
|
||||
Все перечисленные проверки завершились успешно на отдельном проекте
|
||||
`mtusi-lab5-verification`, HTTP http://localhost:28080, без удаления volumes:
|
||||
|
||||
| Проверка | Результат |
|
||||
| --- | --- |
|
||||
| Собственные тесты PostgreSQL/Redis/API/миграций/изображений/OS-лимитов | 199 passed, без пропусков |
|
||||
| Дополнительные timeout и failed retention/SHA-256 | 2 passed |
|
||||
| Контракт, OpenAPI и Compose config | PASS |
|
||||
| Публичный smoke | PASS |
|
||||
| 24 конкурентных повтора сообщения через две API-реплики | PASS |
|
||||
| Публичные persistence и failover | PASS |
|
||||
| SIGKILL publisher после COMMIT до publish | 137 → queued → ready; без повторного upload |
|
||||
| Остановленный worker, рестарт API/брокера/хранилищ | ready того же ID; сообщение RabbitMQ сохранилось с volume |
|
||||
| SIGKILL после четырёх PUT до COMMIT | processing → ready на попытке 2; тот же набор/checksums объектов |
|
||||
| SIGKILL после COMMIT до ack | ready-дубль подтверждён без изменения metadata/Last-Modified |
|
||||
| Временная ошибка и replay | failed после 5 попыток, затем ready; повтор request-id безопасен |
|
||||
| Постоянная ошибка и replay | failed после 1 попытки, затем ready; новый replay ready отклонён |
|
||||
| Конкуренция/подтверждения RabbitMQ после сценариев | 2 consumer, prefetch=1; ready=0, unacknowledged=0 |
|
||||
| Неизменность публичных tests/contracts/README и whitespace | PASS |
|
||||
|
||||
Суммарно выполнен 201 собственный тест двумя прогонами; новые тесты включены
|
||||
в последующие полные запуски make test-grade4. Тесты намеренно меняют только
|
||||
выделенную БД *_test и Redis DB15. API и worker в интеграционных сценариях
|
||||
используют собственную БД тестового Compose-проекта.
|
||||
|
||||
Измеренные задержки пяти неудачных попыток: 4.613, 6.837, 17.323, 30.885,
|
||||
42.633 секунды; все попали в соответствующие интервалы jitter. Задержка
|
||||
последней failed-попытки сохраняется для диагностики, автоматический retry
|
||||
после исчерпания бюджета не выполняется. Ограничение максимума проверено отдельно.
|
||||
|
||||
Во время проверки исправлена проблема fault hooks с PID 1 через init:true.
|
||||
Старые собственные проверки lab4 адаптированы к обязательному переходу
|
||||
202 queued → ready; публичные проверки не изменены. Цель make test-grade4
|
||||
переведена на lab5 и последовательные сценарии, чтобы рестарты инфраструктуры
|
||||
не пересекались с регрессией БД. RabbitMQ используется как брокер, Redis — для сессий.
|
||||
|
||||
## Границы результата
|
||||
|
||||
Проверки относятся к одному Docker-хосту и рестартам с сохранением volumes.
|
||||
PostgreSQL, RabbitMQ и MinIO не реплицированы; потеря диска/хоста не покрыта.
|
||||
Ожидание в очереди не ограничено; admission control и исследование перегрузки
|
||||
относятся к будущему уровню 5. Pipeline фиксирован на версии 1; изменение
|
||||
алгоритмов требует решения о новой версии/миграции результатов.
|
||||
|
||||
Официальные источники: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability),
|
||||
[publisher confirms и consumer acknowledgements](https://www.rabbitmq.com/docs/confirms),
|
||||
[aio-pika](https://docs.aio-pika.com/quick-start.html),
|
||||
[Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html),
|
||||
[Pillow WebP](https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#webp).
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Остановка фоновых процессов и безопасные сообщения об отказах."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import signal
|
||||
|
||||
from aio_pika.exceptions import AMQPException, ChannelInvalidStateError
|
||||
from sqlalchemy.exc import InterfaceError, OperationalError
|
||||
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
|
||||
|
||||
TRANSIENT_JOB_ERRORS = (AMQPException, ChannelInvalidStateError, InterfaceError, OperationalError, PoolTimeoutError, OSError)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def shutdown_event() -> asyncio.Event:
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for signum in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(signum, stop.set)
|
||||
return stop
|
||||
|
||||
|
||||
async def wait_or_stop(stop: asyncio.Event, seconds: float) -> None:
|
||||
try:
|
||||
async with asyncio.timeout(seconds):
|
||||
await stop.wait()
|
||||
except TimeoutError:
|
||||
pass
|
||||
|
||||
|
||||
def log_background_error(operation: str, error: Exception) -> None:
|
||||
# SDK/DB exceptions могут содержать параметры и credentials; выводим только тип.
|
||||
logger.warning("%s: %s", operation, type(error).__name__)
|
||||
+7
-3
@@ -6,19 +6,22 @@ from dataclasses import asdict
|
||||
import json
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.jobs import JobSettings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.s3 import S3Settings
|
||||
from app.domain.attachments import reconcile_attachments
|
||||
from app.domain.attachment_jobs import cleanup_staging
|
||||
from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error
|
||||
from app.storage.s3 import create_s3_storage
|
||||
|
||||
|
||||
async def cleanup_once(factory, storage, apply: bool) -> bool:
|
||||
async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool:
|
||||
completed = True
|
||||
try:
|
||||
async with factory() as session:
|
||||
result = await reconcile_attachments(session, storage, dry_run=not apply)
|
||||
print(json.dumps(asdict(result)), flush=True)
|
||||
staging = await cleanup_staging(session, storage, jobs, apply=apply)
|
||||
print(json.dumps({**asdict(result), "terminal_staging": staging}), flush=True)
|
||||
except RECOVERABLE_STORAGE_ERRORS as error:
|
||||
log_recoverable_error("Reconciliation", error)
|
||||
completed = False
|
||||
@@ -28,12 +31,13 @@ async def cleanup_once(factory, storage, apply: bool) -> bool:
|
||||
async def run(once: bool, apply: bool) -> None:
|
||||
settings = Settings() # type: ignore
|
||||
s3_settings = S3Settings()
|
||||
jobs = JobSettings()
|
||||
engine = create_db_engine(settings)
|
||||
factory = create_session_factory(engine)
|
||||
storage = create_s3_storage(s3_settings)
|
||||
try:
|
||||
while True:
|
||||
completed = await cleanup_once(factory, storage, apply)
|
||||
completed = await cleanup_once(factory, storage, apply, jobs)
|
||||
if once:
|
||||
if not completed:
|
||||
raise SystemExit(1)
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
"""Качество вариантов и пределы отдельного процесса обработки."""
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class ImageSettings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_prefix="IMAGE_", env_file=".env", extra="ignore")
|
||||
|
||||
jpeg_quality: int = Field(default=85, ge=1, le=95)
|
||||
webp_quality: int = Field(default=80, ge=1, le=100)
|
||||
webp_method: int = Field(default=4, ge=0, le=6)
|
||||
memory_mib: int = Field(default=768, ge=256, le=1024)
|
||||
cpu_seconds: int = Field(default=20, ge=1, le=120)
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Операторский интерфейс failed registry. Доступ требует credentials стенда."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from uuid import UUID
|
||||
|
||||
from app.core.background import log_background_error
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.jobs import JobSettings
|
||||
from app.domain.job_admin import REPLAY_REASONS, history, list_failed, replay_failed_job, show_job
|
||||
from app.errors.exceptions import JobOperationError
|
||||
from app.storage.s3 import create_s3_storage
|
||||
|
||||
|
||||
async def execute(args):
|
||||
settings = Settings()
|
||||
engine = create_db_engine(settings)
|
||||
storage = None
|
||||
try:
|
||||
factory = create_session_factory(engine)
|
||||
async with factory() as session:
|
||||
if args.command == "list-failed":
|
||||
result = await list_failed(session, args.limit)
|
||||
elif args.command == "show":
|
||||
result = await show_job(session, args.job_id)
|
||||
elif args.command == "history":
|
||||
result = await history(session, args.job_id)
|
||||
else:
|
||||
storage = create_s3_storage(settings.s3)
|
||||
result = await replay_failed_job(session, storage, JobSettings(), args.job_id,
|
||||
request_id=args.request_id, actor=args.actor, reason=args.reason)
|
||||
print(json.dumps(result, default=str), flush=True)
|
||||
finally:
|
||||
if storage is not None:
|
||||
await storage.aclose()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Реестр failed и явный replay с аудитом")
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
listing = commands.add_parser("list-failed")
|
||||
listing.add_argument("--limit", type=int, choices=range(1, 501), default=50, metavar="1..500")
|
||||
for name in ("show", "history", "replay"):
|
||||
command = commands.add_parser(name)
|
||||
command.add_argument("job_id", type=UUID)
|
||||
if name == "replay":
|
||||
command.add_argument("--request-id", type=UUID, required=True)
|
||||
command.add_argument("--actor", required=True)
|
||||
command.add_argument("--reason", choices=REPLAY_REASONS, required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
asyncio.run(execute(args))
|
||||
except JobOperationError as error:
|
||||
print(json.dumps({"error_code": error.code}), flush=True)
|
||||
raise SystemExit(2) from None
|
||||
except Exception as error:
|
||||
log_background_error("Job administration failed", error)
|
||||
raise SystemExit(1) from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+133
-13
@@ -1,17 +1,137 @@
|
||||
"""Заготовка отдельного worker lab5, не одноразового валидатора image_worker.
|
||||
"""Отдельные consumers с ограниченной конкуренцией; API не выполняет resize."""
|
||||
|
||||
Здесь будут startup/shutdown ресурсов, цикл получения заданий, ограниченная
|
||||
concurrency/prefetch, восстановление pending и остановка с сохранением работы.
|
||||
Каждая задача получает отдельную AsyncSession. Обработка изображений идёт
|
||||
в процессе с лимитами памяти/CPU/времени; CancelledError не подавляется.
|
||||
В Compose и API модуль пока не подключён.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from PIL import features
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.jobs import JobSettings
|
||||
from app.domain.attachment_jobs import process_attachment_job
|
||||
from app.errors.exceptions import ProcessingError
|
||||
from app.schemas.job import JobMessage
|
||||
from app.storage.broker import JobBroker
|
||||
from app.storage.s3 import create_s3_storage
|
||||
|
||||
|
||||
async def run_worker() -> None:
|
||||
"""Запустить consumer с lease/retry, ограничением ресурсов и корректным ack.
|
||||
@dataclass(frozen=True)
|
||||
class Faults:
|
||||
"""Только явные CLI-флаги; обычный worker не включает инъекции сбоев."""
|
||||
|
||||
SIGTERM должен завершить или оставить восстанавливаемую попытку; остановка
|
||||
worker не заставляет API выполнять resize. Инициализация ресурсов — здесь.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: отдельный worker ещё не реализован")
|
||||
crash_after_object: UUID | None = None
|
||||
crash_before_ack: UUID | None = None
|
||||
fail_transient: UUID | None = None
|
||||
fail_permanent: UUID | None = None
|
||||
object_name: str = "optimized"
|
||||
|
||||
def kill(self, attachment_id: UUID, phase: str) -> None:
|
||||
print(json.dumps({"fault": phase, "attachment_id": str(attachment_id)}), flush=True)
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
|
||||
def after_object(self, attachment_id: UUID, name: str) -> None:
|
||||
if attachment_id == self.crash_after_object and name == self.object_name:
|
||||
self.kill(attachment_id, "after_object:" + name)
|
||||
|
||||
def before_ack(self, attachment_id: UUID) -> None:
|
||||
if attachment_id == self.crash_before_ack:
|
||||
self.kill(attachment_id, "before_ack")
|
||||
|
||||
def before_processing(self, attachment_id: UUID) -> None:
|
||||
if attachment_id == self.fail_transient:
|
||||
raise TimeoutError("injected transient failure")
|
||||
if attachment_id == self.fail_permanent:
|
||||
raise ProcessingError("injected_permanent")
|
||||
|
||||
|
||||
async def consume_connected(factory, storage, broker: JobBroker, settings: JobSettings, stop: asyncio.Event, faults: Faults) -> None:
|
||||
consumer = uuid4().hex
|
||||
while not stop.is_set():
|
||||
try:
|
||||
await broker.ensure_ready()
|
||||
delivery = await broker.receive(consumer)
|
||||
if delivery is None:
|
||||
continue
|
||||
try:
|
||||
message = JobMessage.model_validate_json(delivery.payload)
|
||||
except ValidationError:
|
||||
await broker.reject_invalid(delivery.id)
|
||||
continue
|
||||
try:
|
||||
async with factory() as session:
|
||||
acknowledge = await process_attachment_job(
|
||||
session, storage, message, settings,
|
||||
after_object=faults.after_object, before_processing=faults.before_processing,
|
||||
)
|
||||
except ProcessingError:
|
||||
await broker.reject_invalid(delivery.id)
|
||||
else:
|
||||
if acknowledge:
|
||||
faults.before_ack(message.attachment_id)
|
||||
await broker.acknowledge_job(delivery.id)
|
||||
else:
|
||||
await broker.defer_job(delivery.id)
|
||||
except TRANSIENT_JOB_ERRORS as error:
|
||||
log_background_error("Worker delivery deferred", error)
|
||||
await broker.aclose()
|
||||
await wait_or_stop(stop, settings.publisher_poll_seconds)
|
||||
|
||||
|
||||
async def consume(factory, storage, settings, stop, faults) -> None:
|
||||
broker = JobBroker(settings)
|
||||
try:
|
||||
await consume_connected(factory, storage, broker, settings, stop, faults)
|
||||
finally:
|
||||
await broker.aclose()
|
||||
|
||||
|
||||
async def run_worker(faults: Faults) -> None:
|
||||
settings = Settings()
|
||||
jobs = JobSettings()
|
||||
logging.basicConfig(level=settings.log_level)
|
||||
if not features.check("webp"):
|
||||
logging.error("Pillow requires WebP support for the image worker")
|
||||
raise RuntimeError("WebP codec is unavailable")
|
||||
stop = shutdown_event()
|
||||
engine = create_db_engine(settings)
|
||||
storage = None
|
||||
try:
|
||||
storage = create_s3_storage(settings.s3)
|
||||
factory = create_session_factory(engine)
|
||||
async with asyncio.TaskGroup() as group:
|
||||
tasks = [group.create_task(consume(factory, storage, jobs, stop, faults)) for _ in range(jobs.worker_concurrency)]
|
||||
await stop.wait()
|
||||
# Отмена убивает resize-процесс, дожидается SDK и не отправляет ack.
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
finally:
|
||||
if storage is not None:
|
||||
await storage.aclose()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Image worker с явными fault-injection hooks")
|
||||
modes = parser.add_mutually_exclusive_group()
|
||||
for flag in ("crash-after-object", "crash-before-ack", "fail-transient", "fail-permanent"):
|
||||
modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID")
|
||||
parser.add_argument("--object-name", choices=("original", "thumbnail", "crop", "optimized"), default="optimized")
|
||||
args = parser.parse_args()
|
||||
faults = Faults(**vars(args))
|
||||
try:
|
||||
asyncio.run(run_worker(faults))
|
||||
except Exception as error:
|
||||
log_background_error("Worker stopped", error)
|
||||
raise SystemExit(1) from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Настройки очереди из .env; credentials брокера читаются отдельно из файла."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Self
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class JobSettings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
broker_host: str = "broker"
|
||||
broker_port: int = Field(default=5672, ge=1, le=65535)
|
||||
broker_password_file: Path = Path("/run/secrets/broker_password")
|
||||
broker_queue: str = "attachment-jobs"
|
||||
broker_user: str = "messenger"
|
||||
broker_vhost: str = "/"
|
||||
worker_concurrency: int = Field(default=2, ge=1, le=4)
|
||||
job_max_attempts: int = Field(default=5, ge=1, le=100)
|
||||
job_timeout_seconds: int = Field(default=60, ge=10, le=300)
|
||||
job_lease_seconds: int = Field(default=90, ge=20, le=600)
|
||||
job_retry_seconds: int = Field(default=5, ge=1, le=300)
|
||||
job_retry_max_seconds: int = Field(default=120, ge=1, le=3600)
|
||||
failed_staging_retention_seconds: int = Field(default=604800, ge=300)
|
||||
publisher_poll_seconds: float = Field(default=1, gt=0, le=10)
|
||||
staging_retention_seconds: int = Field(default=86400, ge=300)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_lease(self) -> Self:
|
||||
if self.job_lease_seconds < self.job_timeout_seconds + 30:
|
||||
raise ValueError("Lease должен превышать timeout задания минимум на 30 секунд")
|
||||
if self.job_retry_max_seconds < self.job_retry_seconds:
|
||||
raise ValueError("Retry maximum must be at least the base delay")
|
||||
return self
|
||||
@@ -1,15 +1,70 @@
|
||||
"""Заготовка процесса доставки PostgreSQL outbox в durable broker.
|
||||
"""Доставка намерений из PostgreSQL; crash hook включается только явно через CLI."""
|
||||
|
||||
Не держать транзакцию/блокировки во время бесконечного сетевого ожидания.
|
||||
Согласовать lease publisher, ограниченные пачки и повтор после COMMIT/publish.
|
||||
Процесс пока не подключён к Compose или жизненному циклу API.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
from uuid import UUID
|
||||
|
||||
from app.core.background import TRANSIENT_JOB_ERRORS, log_background_error, shutdown_event, wait_or_stop
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.core.jobs import JobSettings
|
||||
from app.domain.attachment_jobs import publish_next_job, recover_expired_jobs
|
||||
from app.storage.broker import JobBroker
|
||||
|
||||
|
||||
async def run_publisher() -> None:
|
||||
"""Публиковать сохранённые намерения и фиксировать подтверждение брокера.
|
||||
def crash_before_publish(attachment_id: UUID) -> None:
|
||||
"""Реальный SIGKILL после чтения принятой job, до publisher confirm и отметки publish."""
|
||||
print(json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}), flush=True)
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
|
||||
После crash между publish и отметкой допустим дубль того же job_id.
|
||||
Недоступность брокера не должна терять уже принятые задания.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: outbox publisher ещё не реализован")
|
||||
|
||||
async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
|
||||
settings = Settings()
|
||||
jobs = JobSettings()
|
||||
logging.basicConfig(level=settings.log_level)
|
||||
stop = shutdown_event()
|
||||
engine = create_db_engine(settings)
|
||||
broker = None
|
||||
try:
|
||||
broker = JobBroker(jobs)
|
||||
factory = create_session_factory(engine)
|
||||
while not stop.is_set():
|
||||
published = False
|
||||
try:
|
||||
await broker.ensure_ready()
|
||||
async with factory() as session:
|
||||
await recover_expired_jobs(session, jobs)
|
||||
published = await publish_next_job(
|
||||
session, broker, attachment_id=crash_attachment_id,
|
||||
before_publish=crash_before_publish if crash_attachment_id is not None else None,
|
||||
)
|
||||
except TRANSIENT_JOB_ERRORS as error:
|
||||
log_background_error("Publisher deferred", error)
|
||||
if not published:
|
||||
await wait_or_stop(stop, jobs.publisher_poll_seconds)
|
||||
finally:
|
||||
if broker is not None:
|
||||
await broker.aclose()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Публикация устойчивых jobs в RabbitMQ")
|
||||
parser.add_argument(
|
||||
"--crash-before-publish", type=UUID, metavar="ATTACHMENT_ID",
|
||||
help="аварийно завершить этот процесс перед публикацией только указанного вложения",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
asyncio.run(run_publisher(args.crash_before_publish))
|
||||
except Exception as error:
|
||||
log_background_error("Publisher stopped", error)
|
||||
raise SystemExit(1) from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Один pipeline изображений в Linux-процессе с лимитами памяти и CPU."""
|
||||
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import sys
|
||||
|
||||
from app.core.image_processing import ImageSettings
|
||||
from app.domain.image_pipeline import create_variants
|
||||
from app.errors.exceptions import IMAGE_WORKER_ERRORS, pipeline_worker_error_code
|
||||
|
||||
|
||||
def main() -> int:
|
||||
settings = ImageSettings()
|
||||
memory = settings.memory_mib * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds))
|
||||
source, directory, metadata, content_type, maximum = sys.argv[1:]
|
||||
code = 0
|
||||
try:
|
||||
result = create_variants(Path(source), Path(directory), content_type, int(maximum), settings)
|
||||
Path(metadata).write_text(result.model_dump_json())
|
||||
except IMAGE_WORKER_ERRORS as error:
|
||||
code = pipeline_worker_error_code(error)
|
||||
return code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+186
-33
@@ -1,45 +1,198 @@
|
||||
"""Заготовки сценариев lab5; не подключены к upload или worker.
|
||||
"""Приём, durable публикация, попытки и фиксация результата обработки."""
|
||||
|
||||
Здесь появятся транзакционные границы и бизнес-условия; SQL и SDK остаются
|
||||
в storage/. Одна AsyncSession принадлежит одной последовательной задаче.
|
||||
"""
|
||||
|
||||
from uuid import UUID
|
||||
import asyncio
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from minio.error import MinioException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from urllib3.exceptions import HTTPError as StorageHTTPError
|
||||
|
||||
from app.core.jobs import JobSettings
|
||||
from app.domain.job_retry import classify_failure, retry_delay
|
||||
from app.domain.job_files import fetch_original, render_variants
|
||||
from app.errors.exceptions import ProcessingError
|
||||
from app.schemas.attachment import VARIANT_FILENAMES
|
||||
from app.schemas.job import JobMessage
|
||||
from app.storage import jobs, outbox
|
||||
from app.storage.broker import JobBroker
|
||||
from app.storage.jobs import JobWork
|
||||
from app.storage.s3 import MinioStorage
|
||||
|
||||
|
||||
async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID) -> UUID:
|
||||
"""Создать job и outbox в транзакции queued metadata после durable staging.
|
||||
|
||||
Возвращать стабильный job_id. Commit принадлежит сценарию приёма upload;
|
||||
202 допустим только после устойчивого оригинала и успешного commit.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: постановка задания ещё не реализована")
|
||||
@dataclass(frozen=True)
|
||||
class Claim:
|
||||
work: JobWork | None = None
|
||||
acknowledge: bool = False
|
||||
|
||||
|
||||
async def process_attachment_job(session: AsyncSession, job_id: UUID) -> None:
|
||||
"""Захватить job с lease и опубликовать original и обязательные варианты.
|
||||
|
||||
Ready-дубль не меняет результат. Lease и версия защищают от устаревшей
|
||||
попытки. S3/metadata фиксируются до ack; отмена не подавляется.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: обработка задания ещё не реализована")
|
||||
async def enqueue_attachment_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
|
||||
"""Включить намерение обработки в транзакцию queued metadata; без commit."""
|
||||
return await jobs.create_job(session, attachment_id, staging_key)
|
||||
|
||||
|
||||
async def recover_expired_jobs(session: AsyncSession) -> int:
|
||||
"""Вернуть зависший processing в очередь после истечения lease.
|
||||
|
||||
Учитывать лимит попыток и защиту от конкурентного завершения старым worker.
|
||||
Точное правило heartbeat/deadline определяется до реализации.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: восстановление lease ещё не реализовано")
|
||||
async def publish_next_job(
|
||||
session: AsyncSession, broker: JobBroker, *,
|
||||
attachment_id: UUID | None = None,
|
||||
before_publish: Callable[[UUID], None] | None = None,
|
||||
) -> bool:
|
||||
"""Отметка publish следует за durable publisher confirm; crash допускает тот же job-дубль."""
|
||||
job = await outbox.next_unpublished_job(session, attachment_id=attachment_id)
|
||||
published = job is not None
|
||||
if job is not None:
|
||||
if before_publish is not None:
|
||||
before_publish(job.attachment_id)
|
||||
await broker.publish_job(JobMessage(
|
||||
job_id=job.id, attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version, attempt=job.attempt + 1,
|
||||
created_at=job.created_at,
|
||||
))
|
||||
outbox.mark_published(job)
|
||||
await session.commit()
|
||||
return published
|
||||
|
||||
|
||||
async def replay_failed_job(session: AsyncSession, job_id: UUID) -> None:
|
||||
"""Явно повторить только failed job, сохранив аудит и ключ логического эффекта.
|
||||
async def claim_job(session: AsyncSession, message: JobMessage, settings: JobSettings) -> Claim:
|
||||
"""Дубликат не захватывает живую аренду и не откатывает terminal-состояние."""
|
||||
row = await jobs.lock_job(session, message.job_id)
|
||||
result = Claim()
|
||||
if row is None:
|
||||
raise ProcessingError("unknown_job")
|
||||
job, attachment = row
|
||||
if message.attachment_id != attachment.id or message.pipeline_version != job.pipeline_version:
|
||||
raise ProcessingError("invalid_job")
|
||||
now = await jobs.database_time(session)
|
||||
if job.attempt_limit is None:
|
||||
job.attempt_limit = settings.job_max_attempts
|
||||
if attachment.status in {"ready", "failed"}:
|
||||
result = Claim(acknowledge=True)
|
||||
elif message.attempt <= job.attempt and attachment.status == "queued":
|
||||
# Предыдущая попытка уже устойчиво заменена новой в PostgreSQL.
|
||||
result = Claim(acknowledge=True)
|
||||
elif message.attempt < job.attempt:
|
||||
result = Claim(acknowledge=True)
|
||||
elif message.attempt > job.attempt + 1:
|
||||
raise ProcessingError("invalid_attempt")
|
||||
elif attachment.status == "queued" and job.next_attempt_at <= now:
|
||||
if job.attempt >= (job.attempt_limit or settings.job_max_attempts) or job.staging_key is None:
|
||||
jobs.record_failure(job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0)
|
||||
result = Claim(acknowledge=True)
|
||||
else:
|
||||
result = Claim(work=jobs.start_job(job, attachment, uuid4(), now, settings.job_lease_seconds))
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
Не переисполнять ready. Проверка допустимости, лимиты повторов и способ
|
||||
операторского вызова будут определены вместе с реестром failed/DLQ.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: replay ещё не реализован")
|
||||
|
||||
async def save_failure(session: AsyncSession, work: JobWork, settings: JobSettings, code: str, *, permanent: bool) -> None:
|
||||
# Неоднозначный COMMIT проверяется повторным чтением: ready сохраняется.
|
||||
await session.rollback()
|
||||
row = await jobs.lock_job(session, work.job_id)
|
||||
if row is not None:
|
||||
job, attachment = row
|
||||
if attachment.status == "processing" and job.lease_token == work.lease_token:
|
||||
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
|
||||
jobs.record_failure(
|
||||
job, attachment, await jobs.database_time(session),
|
||||
"retry_exhausted" if exhausted and not permanent else code,
|
||||
terminal=permanent or exhausted,
|
||||
retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
|
||||
)
|
||||
job.last_error_code = code
|
||||
job.last_error_permanent = permanent
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def build_result(
|
||||
session: AsyncSession, storage: MinioStorage, work: JobWork,
|
||||
after_object: Callable[[UUID, str], None] | None = None,
|
||||
) -> bool:
|
||||
"""Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки."""
|
||||
with TemporaryDirectory(prefix="messenger-job-") as directory:
|
||||
source = Path(directory) / "original"
|
||||
await fetch_original(storage, work, source)
|
||||
result = await render_variants(storage, work, source, Path(directory))
|
||||
row = await jobs.lock_job(session, work.job_id)
|
||||
if row is None:
|
||||
await session.rollback()
|
||||
return True
|
||||
job, attachment = row
|
||||
now = await jobs.database_time(session)
|
||||
if attachment.status != "processing" or job.lease_token != work.lease_token:
|
||||
await session.rollback()
|
||||
return True
|
||||
if job.lease_until is None or job.lease_until <= now:
|
||||
await session.rollback()
|
||||
return False
|
||||
# Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED.
|
||||
# При отмене S3 adapter дожидается SDK-потока до освобождения блокировки.
|
||||
await storage.put_file(
|
||||
work.original_key, source, size_bytes=work.size_bytes,
|
||||
content_type=work.content_type, sha256=work.sha256,
|
||||
)
|
||||
if after_object is not None:
|
||||
after_object(work.attachment_id, "original")
|
||||
for variant in result.variants:
|
||||
await storage.put_file(
|
||||
work.variant_key(variant.name), Path(directory) / VARIANT_FILENAMES[variant.name],
|
||||
size_bytes=variant.size_bytes, content_type=variant.content_type, sha256=variant.sha256,
|
||||
)
|
||||
if after_object is not None:
|
||||
after_object(work.attachment_id, variant.name)
|
||||
jobs.complete_job(job, attachment, work, [variant.model_dump() for variant in result.variants], await jobs.database_time(session))
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def process_attachment_job(
|
||||
session: AsyncSession, storage: MinioStorage, message: JobMessage, settings: JobSettings,
|
||||
*, after_object: Callable[[UUID, str], None] | None = None,
|
||||
before_processing: Callable[[UUID], None] | None = None,
|
||||
) -> bool:
|
||||
claim = await claim_job(session, message, settings)
|
||||
acknowledge = claim.acknowledge
|
||||
if claim.work is not None:
|
||||
work = claim.work
|
||||
try:
|
||||
async with asyncio.timeout(settings.job_timeout_seconds):
|
||||
if before_processing is not None:
|
||||
before_processing(work.attachment_id)
|
||||
acknowledge = await build_result(session, storage, work, after_object)
|
||||
except (ProcessingError, MinioException, StorageHTTPError, OSError, TimeoutError) as error:
|
||||
code, permanent = classify_failure(error)
|
||||
await save_failure(session, work, settings, code, permanent=permanent)
|
||||
acknowledge = True
|
||||
# Ошибка БД/неизвестная ошибка или отмена не подтверждают доставку.
|
||||
return acknowledge
|
||||
|
||||
|
||||
async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) -> int:
|
||||
rows = await jobs.expired_jobs(session)
|
||||
now = await jobs.database_time(session)
|
||||
for job, attachment in rows:
|
||||
if job.attempt_limit is None:
|
||||
job.attempt_limit = settings.job_max_attempts
|
||||
exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
|
||||
jobs.record_failure(
|
||||
job, attachment, now, "retry_exhausted" if exhausted else "worker_timeout",
|
||||
terminal=exhausted, retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
|
||||
)
|
||||
job.last_error_code = "worker_timeout"
|
||||
job.last_error_permanent = False
|
||||
await session.commit()
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def cleanup_staging(session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool) -> int:
|
||||
rows = await jobs.terminal_staging(session, settings.staging_retention_seconds, settings.failed_staging_retention_seconds)
|
||||
if apply:
|
||||
for job, attachment in rows:
|
||||
if job.staging_key != attachment.object_key:
|
||||
await storage.delete_object(job.staging_key)
|
||||
jobs.clear_staging(job)
|
||||
await session.commit()
|
||||
else:
|
||||
await session.rollback()
|
||||
return len(rows)
|
||||
|
||||
+36
-19
@@ -13,6 +13,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.request_capacity import RequestCapacity
|
||||
from app.domain.chats import ensure_chat_member
|
||||
from app.domain.attachment_jobs import enqueue_attachment_job
|
||||
from app.domain.images import validate_filename, validate_image
|
||||
from app.errors.exceptions import (
|
||||
conflict,
|
||||
@@ -28,7 +29,7 @@ from app.errors.handlers import (
|
||||
log_recoverable_error,
|
||||
operation_timeout,
|
||||
)
|
||||
from app.schemas.attachment import Attachment
|
||||
from app.schemas.attachment import Attachment, VARIANT_FILENAMES
|
||||
from app.schemas.user import User
|
||||
from app.storage import attachments as repository
|
||||
from app.storage.s3 import MinioStorage
|
||||
@@ -151,7 +152,7 @@ async def recover_upload(
|
||||
await session.rollback()
|
||||
|
||||
|
||||
async def publish_original(
|
||||
async def stage_attachment(
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage,
|
||||
user: User,
|
||||
@@ -161,8 +162,9 @@ async def publish_original(
|
||||
filename: str,
|
||||
content_type: str,
|
||||
) -> Attachment:
|
||||
"""Сохранить исходник в staging и атомарно принять metadata/job."""
|
||||
id = uuid4()
|
||||
key = f"originals/{uuid4().hex}"
|
||||
key = f"staging/{uuid4().hex}"
|
||||
# Журнал устойчив до PUT, включая аварию процесса и неоднозначный COMMIT.
|
||||
await repository.reserve_upload(session, id, key)
|
||||
await session.commit()
|
||||
@@ -181,13 +183,16 @@ async def publish_original(
|
||||
id=id,
|
||||
chat_id=chat_id,
|
||||
owner_id=user.id,
|
||||
object_key=key,
|
||||
object_key=f"originals/{id.hex}/v1",
|
||||
status="queued",
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
size_bytes=original.size_bytes,
|
||||
sha256=original.sha256,
|
||||
)
|
||||
await enqueue_attachment_job(session, id, key)
|
||||
await repository.finish_upload(session, id)
|
||||
# Снимок queued строится до COMMIT, даже если worker очень быстрый.
|
||||
result = Attachment.model_validate(record)
|
||||
await session.commit()
|
||||
except BaseException:
|
||||
@@ -223,7 +228,7 @@ async def upload_attachment(
|
||||
settings.chunk_bytes,
|
||||
)
|
||||
await validate_image(original.path, content_type, settings)
|
||||
result = await publish_original(
|
||||
result = await stage_attachment(
|
||||
session,
|
||||
storage,
|
||||
user,
|
||||
@@ -245,6 +250,13 @@ async def get_attachment(
|
||||
return Attachment.model_validate(record)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DownloadInfo:
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
|
||||
|
||||
async def download_attachment(
|
||||
session: AsyncSession,
|
||||
storage: MinioStorage | None,
|
||||
@@ -252,25 +264,30 @@ async def download_attachment(
|
||||
attachment_id: UUID,
|
||||
*,
|
||||
variant: str = "original",
|
||||
) -> tuple[Attachment, AsyncIterator[bytes]]:
|
||||
) -> tuple[DownloadInfo, AsyncIterator[bytes]]:
|
||||
result = await get_attachment(session, user, attachment_id)
|
||||
if variant != "original":
|
||||
raise not_found()
|
||||
if result.status != "ready":
|
||||
raise conflict("Вложение пока недоступно для скачивания")
|
||||
storage = require_storage(storage)
|
||||
record = await repository.get_attachment(session, attachment_id)
|
||||
if record is None:
|
||||
raise not_found()
|
||||
key = record.object_key
|
||||
if variant == "original":
|
||||
key = record.object_key
|
||||
info = DownloadInfo(result.filename, result.content_type, result.size_bytes)
|
||||
else:
|
||||
selected = next((item for item in result.variants if item.name == variant), None)
|
||||
keys = {"thumbnail": record.thumbnail_key, "crop": record.crop_key, "optimized": record.optimized_key}
|
||||
key = keys.get(variant)
|
||||
if selected is None or key is None:
|
||||
raise not_found()
|
||||
info = DownloadInfo(VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes)
|
||||
await session.rollback()
|
||||
info = await storage.stat_object(key)
|
||||
if (
|
||||
info is None
|
||||
or info.size_bytes != result.size_bytes
|
||||
or info.content_type != result.content_type
|
||||
):
|
||||
raise unavailable("Оригинал временно недоступен")
|
||||
stored = await storage.stat_object(key)
|
||||
if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type:
|
||||
raise unavailable("Файл временно недоступен")
|
||||
stream = await storage.open_object(key)
|
||||
return result, stream
|
||||
return info, stream
|
||||
|
||||
|
||||
async def validate_message_attachments(
|
||||
@@ -283,8 +300,8 @@ async def validate_message_attachments(
|
||||
records = await repository.get_for_chat(session, chat_id, attachment_ids)
|
||||
if len(records) != len(attachment_ids):
|
||||
raise not_found()
|
||||
if any(record.status != "ready" for record in records):
|
||||
raise conflict("Вложение ещё не готово")
|
||||
if any(record.status == "failed" for record in records):
|
||||
raise conflict("Обработка вложения завершилась ошибкой")
|
||||
|
||||
|
||||
async def reconcile_attachments(
|
||||
|
||||
@@ -1,34 +1,79 @@
|
||||
"""Заготовки преобразований lab5, без доступа к БД, брокеру и HTTP.
|
||||
|
||||
Pipeline запускается внутри ограниченного процесса worker. Исходные bytes
|
||||
не перекодируются; варианты удаляют метаданные по документированной политике.
|
||||
Значения качества JPEG/WebP предстоит выбрать и зафиксировать в REPORT.md.
|
||||
"""
|
||||
"""Thumbnail, crop и WebP; один ограниченный процесс, последовательная обработка."""
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
def create_thumbnail(source: Path, destination: Path) -> None:
|
||||
"""Создать JPEG, вписанный в 256×256 без увеличения.
|
||||
|
||||
Применить EXIF orientation, alpha положить на белый; округлять размеры
|
||||
вниз, минимум 1 px. Публичный fixture 320×200 должен дать 256×160.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: thumbnail ещё не реализован")
|
||||
from app.core.image_processing import ImageSettings
|
||||
from app.domain.image_validation import decode_image
|
||||
from app.schemas.attachment import PipelineResult, Variant
|
||||
|
||||
|
||||
def create_crop(source: Path, destination: Path) -> None:
|
||||
"""Создать центрированный квадрат JPEG 128×128; увеличение допустимо.
|
||||
|
||||
Учитывать EXIF orientation и прозрачность по той же политике, что thumbnail.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: crop ещё не реализован")
|
||||
def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]:
|
||||
"""Вписать без увеличения, округляя вниз, минимум один пиксель."""
|
||||
longest = max(size)
|
||||
result = size
|
||||
if longest > maximum:
|
||||
result = (max(1, size[0] * maximum // longest), max(1, size[1] * maximum // longest))
|
||||
return result
|
||||
|
||||
|
||||
def create_optimized(source: Path, destination: Path) -> None:
|
||||
"""Создать WebP, вписанный в 1280×1280 без увеличения.
|
||||
def describe_variant(name: str, content_type: str, path: Path, size: tuple[int, int]) -> Variant:
|
||||
checksum = hashlib.sha256()
|
||||
with path.open("rb") as source:
|
||||
for chunk in iter(lambda: source.read(65536), b""):
|
||||
checksum.update(chunk)
|
||||
return Variant(
|
||||
name=name, content_type=content_type, size_bytes=path.stat().st_size,
|
||||
width=size[0], height=size[1], sha256=checksum.hexdigest(),
|
||||
)
|
||||
|
||||
Удалить метаданные и применить описанную политику качества. Уменьшение
|
||||
размера любого маленького исходника не гарантируется контрактом.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: optimized ещё не реализован")
|
||||
|
||||
def save_jpeg(image: Image.Image, destination: Path, quality: int) -> None:
|
||||
"""Белая подложка и новый image без EXIF/ICC/XMP/текстовых метаданных."""
|
||||
with Image.new("RGB", image.size, "white") as clean:
|
||||
with image.getchannel("A") as alpha:
|
||||
clean.paste(image, mask=alpha)
|
||||
clean.save(destination, format="JPEG", quality=quality, exif=b"")
|
||||
|
||||
|
||||
def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Variant:
|
||||
size = fitted_size(image.size, 256)
|
||||
with image.resize(size, Image.Resampling.LANCZOS) as resized:
|
||||
save_jpeg(resized, destination, quality)
|
||||
return describe_variant("thumbnail", "image/jpeg", destination, size)
|
||||
|
||||
|
||||
def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant:
|
||||
"""Центрированный JPEG 128×128; маленький исходник допускает увеличение."""
|
||||
with ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) as crop:
|
||||
save_jpeg(crop, destination, quality)
|
||||
return describe_variant("crop", "image/jpeg", destination, (128, 128))
|
||||
|
||||
|
||||
def create_optimized(image: Image.Image, destination: Path, settings: ImageSettings) -> Variant:
|
||||
"""WebP ≤1280×1280 без увеличения; alpha сохраняется, метаданные удаляются."""
|
||||
size = fitted_size(image.size, 1280)
|
||||
with image.resize(size, Image.Resampling.LANCZOS) as resized:
|
||||
with Image.new("RGBA", size) as clean:
|
||||
clean.paste(resized)
|
||||
clean.save(
|
||||
destination, format="WEBP", lossless=False,
|
||||
quality=settings.webp_quality, method=settings.webp_method,
|
||||
exif=b"", icc_profile=b"", xmp=b"",
|
||||
)
|
||||
return describe_variant("optimized", "image/webp", destination, size)
|
||||
|
||||
|
||||
def create_variants(source: Path, directory: Path, content_type: str, max_pixels: int, settings: ImageSettings) -> PipelineResult:
|
||||
decode_image(source, content_type, max_pixels)
|
||||
with Image.open(source, formats=["PNG", "JPEG"]) as image:
|
||||
ImageOps.exif_transpose(image, in_place=True)
|
||||
with image.convert("RGBA") as pixels:
|
||||
variants = [
|
||||
create_thumbnail(pixels, directory / "thumbnail.jpg", settings.jpeg_quality),
|
||||
create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality),
|
||||
create_optimized(pixels, directory / "optimized.webp", settings),
|
||||
]
|
||||
return PipelineResult(variants=variants)
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Replay только failed с проверенным исходником и ограниченным новым бюджетом."""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from uuid import uuid4
|
||||
|
||||
from minio.error import S3Error
|
||||
|
||||
from app.domain.job_files import fetch_original
|
||||
from app.errors.exceptions import JobOperationError, ProcessingError
|
||||
from app.storage import job_admin, jobs
|
||||
from app.storage.jobs import JobWork
|
||||
|
||||
REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review")
|
||||
|
||||
|
||||
def snapshot(job, attachment):
|
||||
return dict(job_id=str(job.id), attachment_id=str(attachment.id), status=attachment.status,
|
||||
attempt=job.attempt, attempt_limit=job.attempt_limit, cycle_start_attempt=job.cycle_start_attempt,
|
||||
error_code=attachment.error_code, last_error_code=job.last_error_code,
|
||||
last_error_permanent=job.last_error_permanent, staging_retained=job.staging_key is not None,
|
||||
next_attempt_at=job.next_attempt_at, lease_until=job.lease_until, finished_at=job.finished_at)
|
||||
|
||||
|
||||
async def list_failed(session, limit):
|
||||
return [snapshot(job, attachment) for job, attachment in await job_admin.failed_jobs(session, limit)]
|
||||
|
||||
|
||||
async def show_job(session, job_id):
|
||||
row = await jobs.lock_job(session, job_id)
|
||||
if row is None:
|
||||
raise JobOperationError("unknown_job")
|
||||
result = snapshot(*row)
|
||||
await session.rollback()
|
||||
return result
|
||||
|
||||
|
||||
async def history(session, job_id):
|
||||
return [dict(request_id=str(row.id), actor=row.actor, reason=row.reason,
|
||||
previous_attempt=row.previous_attempt, attempt_limit=row.attempt_limit,
|
||||
previous_error_code=row.previous_error_code, created_at=row.created_at)
|
||||
for row in await job_admin.replay_history(session, job_id)]
|
||||
|
||||
|
||||
def same_request(record, job_id, actor, reason):
|
||||
if record.job_id != job_id or record.actor != actor or record.reason != reason:
|
||||
raise JobOperationError("request_conflict")
|
||||
return {"request_id": str(record.id), "job_id": str(job_id), "already_applied": True}
|
||||
|
||||
|
||||
async def replay_failed_job(session, storage, settings, job_id, *, request_id, actor, reason):
|
||||
if re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None or reason not in REPLAY_REASONS:
|
||||
raise JobOperationError("invalid_audit_fields")
|
||||
row = await jobs.lock_job(session, job_id)
|
||||
if row is None:
|
||||
raise JobOperationError("unknown_job")
|
||||
record = await job_admin.replay_record(session, request_id)
|
||||
if record is not None:
|
||||
result = same_request(record, job_id, actor, reason)
|
||||
await session.rollback()
|
||||
return result
|
||||
job, attachment = row
|
||||
if attachment.status != "failed":
|
||||
raise JobOperationError("job_not_failed")
|
||||
if job.staging_key is None:
|
||||
raise JobOperationError("source_expired")
|
||||
# Та же блокировка защищает исходник от cleanup до окончания проверки и COMMIT.
|
||||
work = JobWork(job.id, attachment.id, uuid4(), job.pipeline_version, job.staging_key,
|
||||
attachment.content_type, attachment.size_bytes, attachment.sha256)
|
||||
try:
|
||||
async with asyncio.timeout(settings.job_timeout_seconds):
|
||||
with TemporaryDirectory(prefix="replay-source-") as directory:
|
||||
await fetch_original(storage, work, Path(directory) / "original")
|
||||
except ProcessingError as error:
|
||||
raise JobOperationError(error.code) from None
|
||||
except S3Error as error:
|
||||
if error.code in {"NoSuchKey", "NoSuchObject"}:
|
||||
raise JobOperationError("source_missing") from None
|
||||
raise
|
||||
limit = job.attempt + settings.job_max_attempts
|
||||
inserted = await job_admin.insert_replay(
|
||||
session, id=request_id, job_id=job_id, actor=actor, reason=reason,
|
||||
previous_attempt=job.attempt, attempt_limit=limit, previous_error_code=job.last_error_code,
|
||||
)
|
||||
if not inserted:
|
||||
result = same_request(await job_admin.replay_record(session, request_id), job_id, actor, reason)
|
||||
await session.rollback()
|
||||
return result
|
||||
job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit)
|
||||
await session.commit()
|
||||
return {"request_id": str(request_id), "job_id": str(job_id), "already_applied": False, "status": "queued", "attempt_limit": limit}
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Получение исходника и запуск изолированного процесса обработки вариантов."""
|
||||
|
||||
import asyncio
|
||||
import errno
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
from anyio import CancelScope
|
||||
|
||||
from app.errors.exceptions import ProcessingError
|
||||
from app.schemas.attachment import PipelineResult
|
||||
from app.storage.jobs import JobWork
|
||||
from app.storage.s3 import MinioStorage
|
||||
|
||||
|
||||
async def fetch_original(storage: MinioStorage, work: JobWork, destination: Path) -> None:
|
||||
if work.size_bytes > storage.settings.max_upload_bytes:
|
||||
raise ProcessingError("image_limits")
|
||||
checksum = hashlib.sha256()
|
||||
size = 0
|
||||
stream = await storage.open_object(work.staging_key)
|
||||
try:
|
||||
with destination.open("wb") as target:
|
||||
async for chunk in stream:
|
||||
size += len(chunk)
|
||||
if size > work.size_bytes:
|
||||
raise ProcessingError("source_mismatch")
|
||||
target.write(chunk)
|
||||
checksum.update(chunk)
|
||||
finally:
|
||||
await stream.aclose()
|
||||
if size != work.size_bytes or checksum.hexdigest() != work.sha256:
|
||||
raise ProcessingError("source_mismatch")
|
||||
|
||||
|
||||
async def render_variants(storage: MinioStorage, work: JobWork, source: Path, directory: Path) -> PipelineResult:
|
||||
metadata = directory / "variants.json"
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-m", "app.core.thumbnail_worker", str(source), str(directory),
|
||||
str(metadata), work.content_type, str(storage.settings.max_image_pixels),
|
||||
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
code = await process.wait()
|
||||
finally:
|
||||
if process.returncode is None:
|
||||
process.kill()
|
||||
with CancelScope(shield=True):
|
||||
await process.wait()
|
||||
if code == 2:
|
||||
raise ProcessingError("invalid_image")
|
||||
if code == 3 or code < 0:
|
||||
raise ProcessingError("image_limits")
|
||||
if code == 4:
|
||||
raise OSError(errno.EIO, "Pipeline filesystem unavailable")
|
||||
if code == 5:
|
||||
raise ProcessingError("processing_configuration")
|
||||
if code != 0:
|
||||
raise RuntimeError("Image pipeline subprocess failed")
|
||||
return PipelineResult.model_validate_json(metadata.read_text())
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Ограниченный exponential backoff: jitter сохраняется вместе с новой попыткой."""
|
||||
|
||||
import errno
|
||||
import random
|
||||
|
||||
from minio.error import S3Error
|
||||
|
||||
from app.core.jobs import JobSettings
|
||||
from app.errors.exceptions import ProcessingError
|
||||
|
||||
|
||||
def retry_delay(settings: JobSettings, cycle_attempt: int) -> float:
|
||||
ceiling = min(settings.job_retry_max_seconds, settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)))
|
||||
return random.uniform(ceiling / 2, ceiling)
|
||||
|
||||
|
||||
def classify_failure(error: Exception) -> tuple[str, bool]:
|
||||
if isinstance(error, ProcessingError):
|
||||
return error.code, True
|
||||
if isinstance(error, S3Error):
|
||||
if error.code in {"NoSuchKey", "NoSuchObject"}:
|
||||
return "source_missing", True
|
||||
if error.code in {"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "NoSuchBucket"}:
|
||||
return "storage_configuration", True
|
||||
if error.code in {"InternalError", "SlowDown", "ServiceUnavailable", "RequestTimeout", "Throttling"}:
|
||||
return "storage_unavailable", False
|
||||
return "storage_rejected", True
|
||||
if isinstance(error, TimeoutError):
|
||||
return "processing_timeout", False
|
||||
if isinstance(error, OSError) and error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}:
|
||||
return "processing_configuration", True
|
||||
return "processing_unavailable", False
|
||||
@@ -15,7 +15,7 @@ from app.schemas.attachment import Attachment
|
||||
router = APIRouter(prefix='/api/v1', tags=['attachments'])
|
||||
|
||||
|
||||
@router.post('/chats/{chat_id}/attachments', status_code=201)
|
||||
@router.post('/chats/{chat_id}/attachments', status_code=202)
|
||||
async def upload(
|
||||
request: Request, chat_id: UUID, user: CurrentUser, session: DbSession,
|
||||
storage: S3Storage, capacity: UploadCapacity,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import errno
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
@@ -107,3 +109,28 @@ def image_validation_error(code: int) -> ApiError:
|
||||
if code == 3 or code < 0:
|
||||
error = payload_too_large("Превышен лимит пикселей или ресурсов изображения")
|
||||
return error
|
||||
|
||||
|
||||
class ProcessingError(Exception):
|
||||
"""Постоянная ошибка pipeline с безопасным машинным кодом."""
|
||||
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
class JobOperationError(Exception):
|
||||
"""Безопасный код отказа операторской команды."""
|
||||
|
||||
def __init__(self, code: str):
|
||||
self.code = code
|
||||
super().__init__(code)
|
||||
|
||||
|
||||
def pipeline_worker_error_code(error: Exception) -> int:
|
||||
"""Отделить ошибки файловой системы от ошибки декодирования изображения."""
|
||||
if isinstance(error, OSError) and error.errno is not None:
|
||||
if error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}:
|
||||
return 5
|
||||
return 4
|
||||
return image_worker_error_code(error)
|
||||
|
||||
+1
-1
@@ -45,7 +45,7 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
await redis.aclose()
|
||||
await engine.dispose()
|
||||
|
||||
application = FastAPI(title="Messenger", version="1.4.0", lifespan=lifespan)
|
||||
application = FastAPI(title="Messenger", version="1.5.0", lifespan=lifespan)
|
||||
|
||||
application.state.settings = settings
|
||||
application.state.engine = engine
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
from app.models.job_replay import JobReplay
|
||||
from app.models.base import Base
|
||||
from app.models.job import AttachmentJob
|
||||
from app.models.attachment import Attachment, MessageAttachment, UploadIntent
|
||||
from app.models.chat import Chat
|
||||
from app.models.chat_member import ChatMember
|
||||
@@ -7,4 +9,4 @@ from app.models.idempotency_key import IdempotencyKey
|
||||
from app.models.message import Message
|
||||
from app.models.user import User
|
||||
|
||||
__all__ = ["Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"]
|
||||
__all__ = ["JobReplay", "AttachmentJob", "Base", "Attachment", "MessageAttachment", "UploadIntent", "Chat", "ChatMember", "Counter", "IdempotencyKey", "Message", "User"]
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func
|
||||
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func, text
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base
|
||||
@@ -17,7 +18,8 @@ class Attachment(Base):
|
||||
"content_type IN ('image/png', 'image/jpeg')", name="content_type"
|
||||
),
|
||||
CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
|
||||
CheckConstraint("status = 'ready'", name="status"),
|
||||
CheckConstraint("status IN ('queued', 'processing', 'ready', 'failed')", name="status"),
|
||||
CheckConstraint("(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", name="error_code"),
|
||||
Index("ix_attachments_chat_id", "chat_id"),
|
||||
)
|
||||
|
||||
@@ -29,7 +31,12 @@ class Attachment(Base):
|
||||
content_type: Mapped[str] = mapped_column(String(32))
|
||||
size_bytes: Mapped[int]
|
||||
sha256: Mapped[str] = mapped_column(String(64))
|
||||
status: Mapped[str] = mapped_column(String(16), server_default="ready")
|
||||
status: Mapped[str] = mapped_column(String(16), server_default="queued")
|
||||
error_code: Mapped[str | None] = mapped_column(String(64))
|
||||
thumbnail_key: Mapped[str | None] = mapped_column(String(128))
|
||||
crop_key: Mapped[str | None] = mapped_column(String(128))
|
||||
optimized_key: Mapped[str | None] = mapped_column(String(128))
|
||||
variants: Mapped[list[dict]] = mapped_column(JSONB, default=list, server_default=text("'[]'::jsonb"))
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
|
||||
@@ -47,7 +54,7 @@ class MessageAttachment(Base):
|
||||
|
||||
|
||||
class UploadIntent(Base):
|
||||
"""Фиксируется до PUT; удаляется атомарно с публикацией ready-метаданных."""
|
||||
"""Фиксируется до PUT; удаляется атомарно с принятием queued metadata/job."""
|
||||
|
||||
__tablename__ = "upload_intents"
|
||||
__table_args__ = (Index("ix_upload_intents_created_at", "created_at"),)
|
||||
|
||||
+34
-13
@@ -1,16 +1,37 @@
|
||||
"""Место для будущих ORM-моделей job, outbox и вариантов lab5.
|
||||
"""Устойчивая работа и намерение публикации; один pipeline на вложение."""
|
||||
|
||||
Пока моделей и регистрации в Base.metadata нет, чтобы не менять схему lab4
|
||||
до реализации миграции. Планируемые данные:
|
||||
from datetime import datetime
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
* job: job_id, attachment_id, pipeline_version, attempt/max_attempts,
|
||||
status, lease token/deadline, next_attempt_at, безопасный error_code, timestamps;
|
||||
* outbox: event_id, job_id, payload/version, состояние публикации и её lease;
|
||||
* variant: attachment_id, pipeline_version, variant, приватный object_key,
|
||||
content_type, size_bytes, sha256, размеры;
|
||||
* replay audit: операторское действие и связь с прежней логической job.
|
||||
from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
Ключ эффекта (attachment_id, pipeline_version, variant) уникален. Старый worker
|
||||
не публикует metadata поверх новой версии; точная схема зависит от трека 5.
|
||||
Изменение ready-only constraints attachments — отдельная новая миграция.
|
||||
"""
|
||||
from app.models.base import Base
|
||||
|
||||
|
||||
class AttachmentJob(Base):
|
||||
__tablename__ = "attachment_jobs"
|
||||
__table_args__ = (
|
||||
CheckConstraint("pipeline_version = 1", name="pipeline_version"),
|
||||
CheckConstraint("attempt >= 0 AND published_attempt >= 0", name="attempt"),
|
||||
Index("ix_attachment_jobs_due", "next_attempt_at"),
|
||||
Index("ix_attachment_jobs_lease", "lease_until"),
|
||||
)
|
||||
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4)
|
||||
attachment_id: Mapped[UUID] = mapped_column(
|
||||
ForeignKey("attachments.id"), unique=True,
|
||||
)
|
||||
staging_key: Mapped[str | None] = mapped_column(String(128))
|
||||
pipeline_version: Mapped[int] = mapped_column(server_default="1")
|
||||
attempt: Mapped[int] = mapped_column(server_default="0")
|
||||
cycle_start_attempt: Mapped[int] = mapped_column(server_default="0")
|
||||
attempt_limit: Mapped[int | None]
|
||||
last_error_permanent: Mapped[bool | None]
|
||||
published_attempt: Mapped[int] = mapped_column(server_default="0")
|
||||
next_attempt_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
lease_token: Mapped[UUID | None]
|
||||
lease_until: Mapped[datetime | None]
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(64))
|
||||
finished_at: Mapped[datetime | None]
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Аудит явного повторного запуска failed; request id обеспечивает идемпотентность."""
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import ForeignKey, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.models.base import Base
|
||||
|
||||
|
||||
class JobReplay(Base):
|
||||
__tablename__ = "job_replays"
|
||||
|
||||
id: Mapped[UUID] = mapped_column(primary_key=True)
|
||||
job_id: Mapped[UUID] = mapped_column(ForeignKey("attachment_jobs.id"), index=True)
|
||||
actor: Mapped[str] = mapped_column(String(64))
|
||||
reason: Mapped[str] = mapped_column(String(32))
|
||||
previous_attempt: Mapped[int]
|
||||
attempt_limit: Mapped[int]
|
||||
previous_error_code: Mapped[str | None] = mapped_column(String(64))
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
|
||||
@@ -1,14 +1,25 @@
|
||||
"""Публичные ready-метаданные lab4; приватный object_key не выдаётся клиенту."""
|
||||
"""Публичные состояния и варианты; внутренние ключи S3 не сериализуются."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
from app.schemas.base import Base
|
||||
|
||||
|
||||
class Variant(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", from_attributes=True)
|
||||
|
||||
name: Literal["thumbnail", "crop", "optimized"]
|
||||
content_type: Literal["image/jpeg", "image/png", "image/webp"]
|
||||
size_bytes: int = Field(ge=1)
|
||||
width: int = Field(ge=1)
|
||||
height: int = Field(ge=1)
|
||||
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
|
||||
|
||||
class Attachment(Base):
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
@@ -16,8 +27,25 @@ class Attachment(Base):
|
||||
content_type: Literal["image/png", "image/jpeg"]
|
||||
size_bytes: int = Field(ge=1, le=10_485_760)
|
||||
sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
status: Literal["ready"] = "ready"
|
||||
variants: list[object] = Field(default_factory=list, max_length=0)
|
||||
error_code: None = None
|
||||
status: Literal["queued", "processing", "ready", "failed"]
|
||||
variants: list[Variant] = Field(default_factory=list)
|
||||
error_code: str | None = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
VARIANT_FILENAMES = {
|
||||
"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp",
|
||||
}
|
||||
|
||||
|
||||
class PipelineResult(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
variants: list[Variant] = Field(min_length=3, max_length=3)
|
||||
|
||||
@field_validator("variants")
|
||||
@classmethod
|
||||
def require_all_variants(cls, variants: list[Variant]) -> list[Variant]:
|
||||
if {item.name for item in variants} != set(VARIANT_FILENAMES):
|
||||
raise ValueError("Нужны thumbnail, crop и optimized без повторов")
|
||||
return variants
|
||||
|
||||
+19
-9
@@ -1,10 +1,20 @@
|
||||
"""Место для будущей схемы сообщения contracts/jobs.schema.json.
|
||||
"""Сообщение брокера: только идентификаторы и служебные поля контракта v1."""
|
||||
|
||||
Пока классы не реализованы и сообщения не принимаются/не публикуются.
|
||||
Обязательны schema_version=1, type=attachment.process.v1, job_id, attachment_id,
|
||||
pipeline_version>=1, attempt>=1, created_at (UTC/RFC3339).
|
||||
Допустимы traceparent и tracestate; дополнительные поля запрещены.
|
||||
job_id сохраняется при redelivery; очередная управляемая попытка отражается
|
||||
в attempt. Bytes, secrets, S3 keys и пользовательские пути не передаются.
|
||||
Pydantic-схема должна соответствовать публичному JSON Schema, не менять его.
|
||||
"""
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import AwareDatetime, BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class JobMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
schema_version: Literal[1] = 1
|
||||
type: Literal["attachment.process.v1"] = "attachment.process.v1"
|
||||
job_id: UUID
|
||||
attachment_id: UUID
|
||||
pipeline_version: int = Field(default=1, ge=1)
|
||||
attempt: int = Field(ge=1)
|
||||
created_at: AwareDatetime
|
||||
traceparent: str | None = Field(default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
|
||||
tracestate: str | None = Field(default=None, max_length=512)
|
||||
|
||||
+71
-20
@@ -1,27 +1,78 @@
|
||||
"""Заготовки адаптера durable broker; конкретный брокер ещё не выбран.
|
||||
"""RabbitMQ: durable queue, persistent messages, publisher confirms и manual ack."""
|
||||
|
||||
Сообщения соответствуют contracts/jobs.schema.json, содержат ID и служебные
|
||||
поля, но не bytes/credentials/произвольные URL. Подключение, consumer identity,
|
||||
формат delivery handle и интерфейсы уточнить после выбора брокера/версии.
|
||||
Здесь появятся только SDK-вызовы; решения о retry/ready находятся в domain/.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from collections.abc import Mapping
|
||||
import aio_pika
|
||||
from aio_pika.abc import AbstractIncomingMessage
|
||||
|
||||
from app.core.jobs import JobSettings
|
||||
from app.schemas.job import JobMessage
|
||||
|
||||
|
||||
async def publish_job(payload: Mapping[str, object]) -> str:
|
||||
"""Опубликовать устойчивое сообщение и вернуть идентификатор доставки.
|
||||
|
||||
Дождаться подтверждения сохранения брокером; неопределённый исход разрешает
|
||||
повтор с тем же job_id. Не считать публикацию обработкой изображения.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: публикация в брокер ещё не реализована")
|
||||
@dataclass(frozen=True)
|
||||
class Delivery:
|
||||
id: AbstractIncomingMessage
|
||||
payload: bytes
|
||||
|
||||
|
||||
async def acknowledge_job(delivery_id: str) -> None:
|
||||
"""Подтвердить доставку после устойчивого эффекта либо проверенного ready-дубля.
|
||||
class JobBroker:
|
||||
"""Один consumer/канал на экземпляр; publisher использует отдельный экземпляр."""
|
||||
|
||||
Retry сначала устойчиво планируется, затем закрывается старая доставка.
|
||||
Конкретный delivery handle зависит от брокера; сигнатура предварительная.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: ack ещё не реализован")
|
||||
def __init__(self, settings: JobSettings) -> None:
|
||||
self.settings = settings
|
||||
self.connection = self.channel = self.queue = self.iterator = None
|
||||
# Transport exceptions могут содержать URL с credentials.
|
||||
logging.getLogger("aio_pika").setLevel(logging.CRITICAL)
|
||||
logging.getLogger("aiormq").setLevel(logging.CRITICAL)
|
||||
|
||||
async def ensure_ready(self) -> None:
|
||||
if self.connection is None or self.connection.is_closed or self.channel is None or self.channel.is_closed:
|
||||
await self.aclose()
|
||||
self.connection = await aio_pika.connect(
|
||||
host=self.settings.broker_host, port=self.settings.broker_port,
|
||||
login=self.settings.broker_user,
|
||||
password=self.settings.broker_password_file.read_text().strip(),
|
||||
virtualhost=self.settings.broker_vhost, timeout=5, heartbeat=30,
|
||||
)
|
||||
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
|
||||
await self.channel.set_qos(prefetch_count=1)
|
||||
self.queue = await self.channel.declare_queue(self.settings.broker_queue, durable=True)
|
||||
|
||||
async def publish_job(self, message: JobMessage) -> None:
|
||||
await self.ensure_ready()
|
||||
await self.channel.default_exchange.publish(
|
||||
aio_pika.Message(
|
||||
body=message.model_dump_json(exclude_none=True).encode(),
|
||||
content_type="application/json", delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
message_id=f"{message.job_id}:{message.attempt}",
|
||||
), routing_key=self.settings.broker_queue, mandatory=True, timeout=10,
|
||||
)
|
||||
|
||||
async def receive(self, consumer: str) -> Delivery:
|
||||
await self.ensure_ready()
|
||||
if self.iterator is None:
|
||||
self.iterator = self.queue.iterator(no_ack=False, consumer_tag=consumer)
|
||||
await self.iterator.__aenter__()
|
||||
message = await self.iterator.__anext__()
|
||||
return Delivery(message, message.body)
|
||||
|
||||
async def acknowledge_job(self, delivery_id: AbstractIncomingMessage) -> None:
|
||||
await delivery_id.ack()
|
||||
|
||||
async def defer_job(self, delivery_id: AbstractIncomingMessage) -> None:
|
||||
# Живая чужая lease: не удерживать единственный prefetch slot навсегда.
|
||||
await asyncio.sleep(self.settings.publisher_poll_seconds)
|
||||
await delivery_id.nack(requeue=True)
|
||||
|
||||
async def reject_invalid(self, delivery_id: AbstractIncomingMessage) -> None:
|
||||
# Failed jobs находятся в PostgreSQL; неизвестные payload не копируются в DLQ.
|
||||
logging.getLogger(__name__).warning("Rejected invalid job envelope")
|
||||
await delivery_id.reject(requeue=False)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
connection = self.connection
|
||||
self.connection = self.channel = self.queue = self.iterator = None
|
||||
if connection is not None and not connection.is_closed:
|
||||
await connection.close()
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Операторские выборки и журнал replay; транзакция принадлежит domain."""
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
|
||||
from app.models.attachment import Attachment
|
||||
from app.models.job import AttachmentJob
|
||||
from app.models.job_replay import JobReplay
|
||||
|
||||
|
||||
async def failed_jobs(session, limit: int):
|
||||
result = await session.execute(
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(Attachment.status == "failed")
|
||||
.order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id).limit(limit)
|
||||
)
|
||||
return result.all()
|
||||
|
||||
|
||||
async def replay_record(session, request_id):
|
||||
return await session.get(JobReplay, request_id)
|
||||
|
||||
|
||||
async def insert_replay(session, **values) -> bool:
|
||||
result = await session.scalar(insert(JobReplay).values(**values).on_conflict_do_nothing(index_elements=[JobReplay.id]).returning(JobReplay.id))
|
||||
return result is not None
|
||||
|
||||
|
||||
async def replay_history(session, job_id):
|
||||
return (await session.scalars(select(JobReplay).where(JobReplay.job_id == job_id).order_by(JobReplay.created_at).limit(100))).all()
|
||||
|
||||
|
||||
def reset_failed(job, attachment, now, attempt_limit):
|
||||
job.cycle_start_attempt = job.attempt
|
||||
job.attempt_limit = attempt_limit
|
||||
job.published_attempt = job.attempt
|
||||
job.next_attempt_at = now
|
||||
job.lease_token = None
|
||||
job.lease_until = None
|
||||
job.finished_at = None
|
||||
job.last_error_code = None
|
||||
job.last_error_permanent = None
|
||||
attachment.status = "queued"
|
||||
attachment.error_code = None
|
||||
attachment.updated_at = now
|
||||
+116
-26
@@ -1,37 +1,127 @@
|
||||
"""Заготовки PostgreSQL-хранилища jobs lab5; скрытых commit здесь нет.
|
||||
|
||||
Будущие ORM-операции захвата, lease, retry и результата должны защищаться
|
||||
от конкурирующих попыток. Границы транзакций находятся в domain/.
|
||||
Тип результата захвата уточняется после определения модели job.
|
||||
"""
|
||||
"""SQL и изменения jobs/attachments; commit остаётся в domain."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import and_, or_, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
async def claim_job(session: AsyncSession, job_id: UUID) -> None:
|
||||
"""Условно захватить допустимое задание и сохранить lease/номер попытки.
|
||||
|
||||
Ready и актуальная чужая аренда не переходят обратно в processing.
|
||||
Будущий результат должен различать захват, завершённый дубль и занятость.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: захват job ещё не реализован")
|
||||
from app.models.attachment import Attachment
|
||||
from app.models.job import AttachmentJob
|
||||
from app.schemas.attachment import VARIANT_FILENAMES
|
||||
|
||||
|
||||
async def complete_job(session: AsyncSession, job_id: UUID) -> None:
|
||||
"""Атомарно записать metadata вариантов и ready после сохранения объектов.
|
||||
@dataclass(frozen=True)
|
||||
class JobWork:
|
||||
job_id: UUID
|
||||
attachment_id: UUID
|
||||
lease_token: UUID
|
||||
pipeline_version: int
|
||||
staging_key: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
|
||||
Предусловия lease/attempt/pipeline_version и структура результата будут
|
||||
добавлены в сигнатуру вместе с ORM-моделью. Не делать commit в storage.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: фиксация результата ещё не реализована")
|
||||
@property
|
||||
def original_key(self) -> str:
|
||||
return f"originals/{self.attachment_id.hex}/v{self.pipeline_version}"
|
||||
|
||||
def variant_key(self, name: str) -> str:
|
||||
filename = VARIANT_FILENAMES[name]
|
||||
return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}"
|
||||
|
||||
|
||||
async def record_job_failure(session: AsyncSession, job_id: UUID, error_code: str) -> None:
|
||||
"""Записать безопасный код сбоя без stack trace и secrets.
|
||||
async def create_job(session: AsyncSession, attachment_id: UUID, staging_key: str) -> UUID:
|
||||
job = AttachmentJob(attachment_id=attachment_id, staging_key=staging_key)
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
return job.id
|
||||
|
||||
Domain выбирает queued с next_attempt_at либо конечный failed согласно
|
||||
классификации ошибки и лимиту попыток; storage лишь сохраняет решение.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: сохранение сбоя ещё не реализовано")
|
||||
|
||||
async def lock_job(session: AsyncSession, job_id: UUID):
|
||||
result = await session.execute(
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(AttachmentJob.id == job_id)
|
||||
.with_for_update(of=AttachmentJob)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return result.one_or_none()
|
||||
|
||||
|
||||
async def database_time(session: AsyncSession) -> datetime:
|
||||
return await session.scalar(select(func.clock_timestamp()))
|
||||
|
||||
|
||||
async def expired_jobs(session: AsyncSession, limit: int = 20):
|
||||
result = await session.execute(
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(Attachment.status == "processing", AttachmentJob.lease_until < func.clock_timestamp())
|
||||
.order_by(AttachmentJob.lease_until)
|
||||
.limit(limit)
|
||||
.with_for_update(of=AttachmentJob, skip_locked=True)
|
||||
)
|
||||
return result.all()
|
||||
|
||||
|
||||
def start_job(job, attachment, token: UUID, now: datetime, lease_seconds: int) -> JobWork:
|
||||
job.attempt += 1
|
||||
job.lease_token = token
|
||||
job.lease_until = now + timedelta(seconds=lease_seconds)
|
||||
attachment.status = "processing"
|
||||
attachment.updated_at = now
|
||||
return JobWork(
|
||||
job.id, attachment.id, token, job.pipeline_version, job.staging_key,
|
||||
attachment.content_type, attachment.size_bytes, attachment.sha256,
|
||||
)
|
||||
|
||||
|
||||
def record_failure(job, attachment, now: datetime, error_code: str, *, terminal: bool, retry_seconds: float) -> None:
|
||||
job.lease_token = None
|
||||
job.lease_until = None
|
||||
job.last_error_code = error_code
|
||||
job.next_attempt_at = now + timedelta(seconds=retry_seconds)
|
||||
job.finished_at = now if terminal else None
|
||||
attachment.status = "failed" if terminal else "queued"
|
||||
attachment.error_code = error_code if terminal else None
|
||||
attachment.updated_at = now
|
||||
|
||||
|
||||
def complete_job(job, attachment, work: JobWork, variants: list[dict], now: datetime) -> None:
|
||||
attachment.object_key = work.original_key
|
||||
attachment.thumbnail_key = work.variant_key("thumbnail")
|
||||
attachment.crop_key = work.variant_key("crop")
|
||||
attachment.optimized_key = work.variant_key("optimized")
|
||||
attachment.variants = variants
|
||||
attachment.status = "ready"
|
||||
attachment.error_code = None
|
||||
attachment.updated_at = now
|
||||
job.lease_token = None
|
||||
job.lease_until = None
|
||||
job.last_error_code = None
|
||||
job.last_error_permanent = None
|
||||
job.finished_at = now
|
||||
|
||||
|
||||
async def terminal_staging(session: AsyncSession, retention: int, failed_retention: int, limit: int = 20):
|
||||
result = await session.execute(
|
||||
select(AttachmentJob, Attachment)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(
|
||||
AttachmentJob.staging_key.is_not(None),
|
||||
or_(
|
||||
and_(Attachment.status == "ready", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=retention)),
|
||||
and_(Attachment.status == "failed", AttachmentJob.finished_at < func.clock_timestamp() - timedelta(seconds=failed_retention)),
|
||||
),
|
||||
)
|
||||
.order_by(AttachmentJob.finished_at)
|
||||
.limit(limit)
|
||||
.with_for_update(of=AttachmentJob, skip_locked=True)
|
||||
)
|
||||
return result.all()
|
||||
|
||||
|
||||
def clear_staging(job) -> None:
|
||||
job.staging_key = None
|
||||
|
||||
+23
-15
@@ -1,23 +1,31 @@
|
||||
"""Заготовки transactional outbox для разрыва PostgreSQL COMMIT → broker.
|
||||
|
||||
Outbox — таблица намерений публикации в той же транзакции, что и job.
|
||||
Повторная публикация после неопределённого confirm допустима; job_id стабилен.
|
||||
Форма записей, захват пачки и lease publisher уточняются вместе с моделью.
|
||||
"""
|
||||
"""Jobs одновременно хранят намерение публикации очередной попытки."""
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
async def add_job_event(session: AsyncSession, job_id: UUID) -> None:
|
||||
"""Сохранить намерение публикации job без commit и обращения к брокеру."""
|
||||
raise NotImplementedError("Lab5: запись outbox ещё не реализована")
|
||||
from app.models.attachment import Attachment
|
||||
from app.models.job import AttachmentJob
|
||||
|
||||
|
||||
async def mark_event_published(session: AsyncSession, event_id: UUID) -> None:
|
||||
"""Отметить событие опубликованным после устойчивого подтверждения брокера.
|
||||
async def next_unpublished_job(session: AsyncSession, *, attachment_id: UUID | None = None) -> AttachmentJob | None:
|
||||
statement = (
|
||||
select(AttachmentJob)
|
||||
.join(Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(
|
||||
Attachment.status == "queued",
|
||||
AttachmentJob.next_attempt_at <= func.clock_timestamp(),
|
||||
AttachmentJob.published_attempt <= AttachmentJob.attempt,
|
||||
)
|
||||
.order_by(AttachmentJob.next_attempt_at, AttachmentJob.id)
|
||||
.limit(1)
|
||||
.with_for_update(of=AttachmentJob, skip_locked=True)
|
||||
)
|
||||
if attachment_id is not None:
|
||||
statement = statement.where(AttachmentJob.attachment_id == attachment_id)
|
||||
return await session.scalar(statement)
|
||||
|
||||
Не помечать job завершённой: publisher confirm отличается от consumer ack.
|
||||
"""
|
||||
raise NotImplementedError("Lab5: отметка outbox ещё не реализована")
|
||||
|
||||
def mark_published(job: AttachmentJob) -> None:
|
||||
job.published_attempt = job.attempt + 1
|
||||
|
||||
+121
@@ -45,6 +45,28 @@ x-s3-env: &s3-env
|
||||
S3_CLEANUP_GRACE_SECONDS: ${S3_CLEANUP_GRACE_SECONDS:?Set S3_CLEANUP_GRACE_SECONDS in .env}
|
||||
S3_CLEANUP_INTERVAL_SECONDS: ${S3_CLEANUP_INTERVAL_SECONDS:?Set S3_CLEANUP_INTERVAL_SECONDS in .env}
|
||||
|
||||
x-jobs-env: &jobs-env
|
||||
BROKER_HOST: ${BROKER_HOST:?Set BROKER_HOST in .env}
|
||||
BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env}
|
||||
BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
|
||||
BROKER_QUEUE: ${BROKER_QUEUE:?Set BROKER_QUEUE in .env}
|
||||
BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env}
|
||||
BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:?Set WORKER_CONCURRENCY in .env}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:?Set JOB_MAX_ATTEMPTS in .env}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:?Set JOB_TIMEOUT_SECONDS in .env}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:?Set JOB_LEASE_SECONDS in .env}
|
||||
JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:?Set JOB_RETRY_MAX_SECONDS in .env}
|
||||
JOB_RETRY_SECONDS: ${JOB_RETRY_SECONDS:?Set JOB_RETRY_SECONDS in .env}
|
||||
PUBLISHER_POLL_SECONDS: ${PUBLISHER_POLL_SECONDS:?Set PUBLISHER_POLL_SECONDS in .env}
|
||||
FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env}
|
||||
IMAGE_JPEG_QUALITY: ${IMAGE_JPEG_QUALITY:?Set IMAGE_JPEG_QUALITY in .env}
|
||||
IMAGE_WEBP_QUALITY: ${IMAGE_WEBP_QUALITY:?Set IMAGE_WEBP_QUALITY in .env}
|
||||
IMAGE_WEBP_METHOD: ${IMAGE_WEBP_METHOD:?Set IMAGE_WEBP_METHOD in .env}
|
||||
IMAGE_MEMORY_MIB: ${IMAGE_MEMORY_MIB:?Set IMAGE_MEMORY_MIB in .env}
|
||||
IMAGE_CPU_SECONDS: ${IMAGE_CPU_SECONDS:?Set IMAGE_CPU_SECONDS in .env}
|
||||
|
||||
services:
|
||||
minio:
|
||||
build:
|
||||
@@ -154,6 +176,100 @@ services:
|
||||
cpus: "0.5"
|
||||
memory: 512M
|
||||
|
||||
broker:
|
||||
image: rabbitmq:4.4.0
|
||||
hostname: broker
|
||||
restart: unless-stopped
|
||||
entrypoint: ["/bin/sh", "/usr/local/bin/start-broker.sh"]
|
||||
environment:
|
||||
BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env}
|
||||
BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env}
|
||||
BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env}
|
||||
BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
|
||||
secrets:
|
||||
- source: broker_password
|
||||
target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
|
||||
volumes:
|
||||
- ./infra/broker/start-broker.sh:/usr/local/bin/start-broker.sh:ro
|
||||
- rabbitmqdata:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 20s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 768M
|
||||
|
||||
publisher:
|
||||
<<: *api-image
|
||||
init: true
|
||||
command: ["python", "-m", "app.core.outbox_publisher"]
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
<<: [*base-env, *jobs-env]
|
||||
secrets:
|
||||
- db_password
|
||||
- source: broker_password
|
||||
target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
disable: true
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
|
||||
worker:
|
||||
<<: *api-image
|
||||
init: true
|
||||
command: ["python", "-m", "app.core.job_worker"]
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
<<: [*base-env, *s3-env, *jobs-env]
|
||||
secrets:
|
||||
- db_password
|
||||
- source: broker_password
|
||||
target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
|
||||
- source: s3_access_key_id
|
||||
target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
|
||||
- source: s3_secret_access_key
|
||||
target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
|
||||
networks: [default, objects]
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=128m,mode=1777
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
minio-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
disable: true
|
||||
stop_signal: SIGTERM
|
||||
stop_grace_period: 20s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "2.0"
|
||||
memory: 2048M
|
||||
|
||||
migrate:
|
||||
<<: *api-image
|
||||
command: ["alembic", "upgrade", "head"]
|
||||
@@ -229,6 +345,8 @@ services:
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
<<: [*base-env, *s3-env]
|
||||
FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env}
|
||||
secrets:
|
||||
- db_password
|
||||
- source: s3_access_key_id
|
||||
@@ -274,6 +392,7 @@ volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
miniodata:
|
||||
rabbitmqdata:
|
||||
|
||||
networks:
|
||||
default:
|
||||
@@ -281,6 +400,8 @@ networks:
|
||||
internal: true
|
||||
|
||||
secrets:
|
||||
broker_password:
|
||||
file: ./secrets/broker_password
|
||||
db_password:
|
||||
file: ./secrets/db_password
|
||||
redis_password:
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
{
|
||||
"status": "passed",
|
||||
"results": [
|
||||
{
|
||||
"scenario": "worker_stopped_restart_all",
|
||||
"attachment_id": "96d4fc78-4e72-473c-a2a9-57db94421707",
|
||||
"queue_before": {
|
||||
"name": "attachment-jobs",
|
||||
"messages_unacknowledged": 0,
|
||||
"consumers": 0,
|
||||
"durable": true,
|
||||
"messages_ready": 1
|
||||
},
|
||||
"queue_after": {
|
||||
"name": "attachment-jobs",
|
||||
"messages_unacknowledged": 0,
|
||||
"consumers": 0,
|
||||
"durable": true,
|
||||
"messages_ready": 1
|
||||
},
|
||||
"after": {
|
||||
"job_id": "b3a81ec6-65d6-4201-aeb9-fedbe92fdaaf",
|
||||
"status": "ready",
|
||||
"attempt": 1,
|
||||
"published_attempt": 1,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:27:44.556734+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:26:56.662360+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
{
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:27:44.548000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:27:44.552000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:27:44.543000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:27:44.536000+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "crash-after-object",
|
||||
"attachment_id": "115c5e05-3d09-4f04-9b32-0d27597b65fc",
|
||||
"exit_code": 137,
|
||||
"before": {
|
||||
"job_id": "ba3dc221-5651-4c1d-8bfd-9fd61c8aa854",
|
||||
"status": "processing",
|
||||
"attempt": 1,
|
||||
"published_attempt": 1,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:27:49.144648+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:27:47.218612+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
{
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:27:49.912000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:27:49.917000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:27:49.907000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:27:49.894000+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"after": {
|
||||
"job_id": "ba3dc221-5651-4c1d-8bfd-9fd61c8aa854",
|
||||
"status": "ready",
|
||||
"attempt": 2,
|
||||
"published_attempt": 2,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:29:24.765539+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:29:23.142176+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
{
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:29:24.750000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:29:24.758000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:29:24.742000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:29:24.725000+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "crash-before-ack",
|
||||
"attachment_id": "2f97fc61-fc07-47da-98a8-9d9894ed4cd1",
|
||||
"exit_code": 137,
|
||||
"before": {
|
||||
"job_id": "4a0bc78c-d79a-4e1a-88f7-11ad03045135",
|
||||
"status": "ready",
|
||||
"attempt": 1,
|
||||
"published_attempt": 1,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:29:29.833142+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:29:27.262932+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
{
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:29:29.824000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:29:29.829000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:29:29.818000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:29:29.805000+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"after": {
|
||||
"job_id": "4a0bc78c-d79a-4e1a-88f7-11ad03045135",
|
||||
"status": "ready",
|
||||
"attempt": 1,
|
||||
"published_attempt": 1,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:29:29.833142+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:29:27.262932+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
{
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:29:29.824000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:29:29.829000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:29:29.818000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:29:29.805000+00:00"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "fail-transient",
|
||||
"attachment_id": "92c4de14-53cd-4805-a842-9892a8b7a5ea",
|
||||
"samples": [
|
||||
{
|
||||
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
|
||||
"status": "queued",
|
||||
"attempt": 1,
|
||||
"published_attempt": 1,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:29:45.863494+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:29:50.476742+00:00",
|
||||
"last_error_code": "processing_timeout",
|
||||
"permanent": false,
|
||||
"objects": []
|
||||
},
|
||||
{
|
||||
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
|
||||
"status": "queued",
|
||||
"attempt": 2,
|
||||
"published_attempt": 2,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:29:51.216904+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:29:58.053724+00:00",
|
||||
"last_error_code": "processing_timeout",
|
||||
"permanent": false,
|
||||
"objects": []
|
||||
},
|
||||
{
|
||||
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
|
||||
"status": "queued",
|
||||
"attempt": 3,
|
||||
"published_attempt": 3,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:29:58.279470+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:30:15.602067+00:00",
|
||||
"last_error_code": "processing_timeout",
|
||||
"permanent": false,
|
||||
"objects": []
|
||||
},
|
||||
{
|
||||
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
|
||||
"status": "queued",
|
||||
"attempt": 4,
|
||||
"published_attempt": 4,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:30:16.386336+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:30:47.271081+00:00",
|
||||
"last_error_code": "processing_timeout",
|
||||
"permanent": false,
|
||||
"objects": []
|
||||
},
|
||||
{
|
||||
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
|
||||
"status": "failed",
|
||||
"attempt": 5,
|
||||
"published_attempt": 5,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:30:47.622716+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:31:30.255216+00:00",
|
||||
"last_error_code": "processing_timeout",
|
||||
"permanent": false,
|
||||
"objects": []
|
||||
}
|
||||
],
|
||||
"after": {
|
||||
"job_id": "f44f6bfa-9fde-4616-b3cf-83dde54b1154",
|
||||
"status": "ready",
|
||||
"attempt": 6,
|
||||
"published_attempt": 6,
|
||||
"limit": 10,
|
||||
"updated_at": "2026-11-10 03:30:59.696883+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:30:53.135190+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
{
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:30:59.687000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:30:59.692000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:30:59.683000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:30:59.670000+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"replay_idempotent": true
|
||||
},
|
||||
{
|
||||
"scenario": "fail-permanent",
|
||||
"attachment_id": "92e2baff-5b26-4326-bee6-269382ff80c9",
|
||||
"samples": [
|
||||
{
|
||||
"job_id": "f462098d-43b7-41cc-afe8-399aaf3ed0bd",
|
||||
"status": "failed",
|
||||
"attempt": 1,
|
||||
"published_attempt": 1,
|
||||
"limit": 5,
|
||||
"updated_at": "2026-11-10 03:31:12.333162+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:31:16.827024+00:00",
|
||||
"last_error_code": "injected_permanent",
|
||||
"permanent": true,
|
||||
"objects": []
|
||||
}
|
||||
],
|
||||
"after": {
|
||||
"job_id": "f462098d-43b7-41cc-afe8-399aaf3ed0bd",
|
||||
"status": "ready",
|
||||
"attempt": 2,
|
||||
"published_attempt": 2,
|
||||
"limit": 6,
|
||||
"updated_at": "2026-11-10 03:31:24.282794+00:00",
|
||||
"next_attempt_at": "2026-11-10 03:31:17.137221+00:00",
|
||||
"last_error_code": null,
|
||||
"permanent": null,
|
||||
"objects": [
|
||||
{
|
||||
"name": "crop.jpg",
|
||||
"size": 2462,
|
||||
"sha256": "fae35500205a78065cf7d589f121cc909b017e2a4cde882177e24f5608fb4c1f",
|
||||
"last_modified": "2026-11-10 03:31:24.272000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "optimized.webp",
|
||||
"size": 2462,
|
||||
"sha256": "1090d84ed8411226010ae498b08e1b9e6d81c8eee02ee77143a0dc327b894beb",
|
||||
"last_modified": "2026-11-10 03:31:24.278000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "thumbnail.jpg",
|
||||
"size": 5813,
|
||||
"sha256": "b8e59b0cad429b1c49c27c4a1c79e52bfc23e2fab7c7238d18d5addd4df4b546",
|
||||
"last_modified": "2026-11-10 03:31:24.267000+00:00"
|
||||
},
|
||||
{
|
||||
"name": "v1",
|
||||
"size": 111817,
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"last_modified": "2026-11-10 03:31:24.254000+00:00"
|
||||
}
|
||||
]
|
||||
},
|
||||
"replay_idempotent": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
{
|
||||
"date": "2026-11-10",
|
||||
"status": "passed",
|
||||
"project": "mtusi-lab5-verification",
|
||||
"base_url": "http://localhost:28080",
|
||||
"checks": [
|
||||
{
|
||||
"name": "pytest_regression",
|
||||
"exit_code": 0,
|
||||
"stdout": "199 passed in 82.29s (0:01:22)",
|
||||
"scope": "Dedicated *_test database and Redis DB15; Docker restart tests enabled"
|
||||
},
|
||||
{
|
||||
"name": "pytest_timeout_retention",
|
||||
"exit_code": 0,
|
||||
"stdout": "2 passed, 5 deselected in 1.91s",
|
||||
"scope": "Dedicated *_test database and Redis DB15; Docker restart tests enabled"
|
||||
},
|
||||
{
|
||||
"name": "compose_configuration",
|
||||
"command": [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"config",
|
||||
"--quiet"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": ""
|
||||
},
|
||||
{
|
||||
"name": "contract",
|
||||
"command": [
|
||||
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
|
||||
"scripts/check_contract.py"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": "PASS: offline skeleton structure, local refs and Python syntax; application NOT tested."
|
||||
},
|
||||
{
|
||||
"name": "openapi",
|
||||
"command": [
|
||||
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
|
||||
"scripts/validate_openapi.py"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": "PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested."
|
||||
},
|
||||
{
|
||||
"name": "public_smoke",
|
||||
"command": [
|
||||
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
|
||||
"tests/smoke.py",
|
||||
"--base-url",
|
||||
"http://localhost:28080"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": "PASS: lab5 public API scenario; infrastructure and grading evidence remain separate."
|
||||
},
|
||||
{
|
||||
"name": "message_idempotency",
|
||||
"command": [
|
||||
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
|
||||
"-m",
|
||||
"scripts.check_idempotency",
|
||||
"--base-url",
|
||||
"http://localhost:28080"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": "PASS: 24 concurrent replays, 2 replicas, one message"
|
||||
},
|
||||
{
|
||||
"name": "outbox_crash",
|
||||
"command": [
|
||||
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
|
||||
"-m",
|
||||
"scripts.check_outbox_recovery",
|
||||
"--base-url",
|
||||
"http://localhost:28080"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": "{\"scenario\": \"commit_before_publish\", \"attachment_id\": \"0fff1dce-c479-40d8-8627-f28cc85cb6cf\", \"message_id\": \"e79bfb7e-c2af-4011-a87a-52528d7a7840\", \"publisher_exit_code\": 137, \"status_after_crash\": \"queued\", \"status_after_restart\": \"ready\", \"variants\": [\"thumbnail\", \"crop\", \"optimized\"], \"reuploaded\": false}"
|
||||
},
|
||||
{
|
||||
"name": "public_persistence",
|
||||
"command": [
|
||||
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
"persistence",
|
||||
"--action-script",
|
||||
"scripts/restart.sh",
|
||||
"--base-url",
|
||||
"http://localhost:28080"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence."
|
||||
},
|
||||
{
|
||||
"name": "public_failover",
|
||||
"command": [
|
||||
"/home/gytall/.pyenv/versions/3.12.14/bin/python3",
|
||||
"tests/resilience.py",
|
||||
"--mode",
|
||||
"failover",
|
||||
"--action-script",
|
||||
"scripts/stop-one.sh",
|
||||
"--base-url",
|
||||
"http://localhost:28080"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 104, \"failed_attempts\": 5, \"first_recovery_seconds\": 1.907}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence."
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq_prefetch",
|
||||
"command": [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"exec",
|
||||
"-T",
|
||||
"broker",
|
||||
"rabbitmqctl",
|
||||
"list_channels",
|
||||
"--formatter=json",
|
||||
"consumer_count",
|
||||
"prefetch_count",
|
||||
"messages_unacknowledged"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": "[\n{\"consumer_count\":1,\"messages_unacknowledged\":0,\"prefetch_count\":1}\n,{\"consumer_count\":0,\"messages_unacknowledged\":0,\"prefetch_count\":1}\n,{\"consumer_count\":1,\"messages_unacknowledged\":0,\"prefetch_count\":1}\n]"
|
||||
},
|
||||
{
|
||||
"name": "rabbitmq_durability_ack",
|
||||
"command": [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"exec",
|
||||
"-T",
|
||||
"broker",
|
||||
"rabbitmqctl",
|
||||
"list_queues",
|
||||
"--formatter=json",
|
||||
"name",
|
||||
"durable",
|
||||
"messages_ready",
|
||||
"messages_unacknowledged",
|
||||
"consumers"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": "[\n{\"name\":\"attachment-jobs\",\"messages_unacknowledged\":0,\"consumers\":2,\"durable\":true,\"messages_ready\":0}\n]"
|
||||
},
|
||||
{
|
||||
"name": "worker_fault_retry_replay",
|
||||
"exit_code": 0,
|
||||
"evidence": "lab5-jobs.json",
|
||||
"scenarios": [
|
||||
"worker_stopped_restart_all",
|
||||
"crash-after-object",
|
||||
"crash-before-ack",
|
||||
"fail-transient",
|
||||
"fail-permanent"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "whitespace",
|
||||
"command": [
|
||||
"git",
|
||||
"diff",
|
||||
"--check"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": ""
|
||||
},
|
||||
{
|
||||
"name": "public_contract_tests_unchanged",
|
||||
"exit_code": 0
|
||||
},
|
||||
{
|
||||
"name": "final_services",
|
||||
"command": [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"ps",
|
||||
"--format",
|
||||
"{{.Service}} {{.State}} {{.Health}}"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": "api running healthy\napi running healthy\nbroker running healthy\ncleanup running \nminio running healthy\nnginx running healthy\npostgres running healthy\npublisher running \nredis running healthy\nworker running"
|
||||
}
|
||||
],
|
||||
"git_base_commit": "0a703588290e26d0f148b45d0e6bee2e8eefce7d",
|
||||
"uncommitted_changes": true,
|
||||
"environment": {
|
||||
"test_runner_python": "3.12.14",
|
||||
"runtime_python": "3.12.15",
|
||||
"broker": "RabbitMQ 4.4.0",
|
||||
"sessions": "Redis 7.4.2",
|
||||
"worker_memory_bytes": 2147483648,
|
||||
"worker_cpu_limit": 2.0,
|
||||
"worker_init": true
|
||||
},
|
||||
"profile": {
|
||||
"data": "Synthetic users/chats only; tests/fixtures/sample.png for fault scenarios; no credentials in evidence",
|
||||
"worker_concurrency": 2,
|
||||
"prefetch_per_consumer": 1,
|
||||
"max_attempts_per_cycle": 5,
|
||||
"timeout_seconds": 60,
|
||||
"lease_seconds": 90,
|
||||
"retry_base_seconds": 5,
|
||||
"retry_cap_seconds": 120,
|
||||
"image_memory_mib": 768,
|
||||
"image_cpu_seconds": 20,
|
||||
"isolated_limit_tests": "256 MiB / 1 CPU second in child process; timeout cancellation checked separately"
|
||||
},
|
||||
"criteria": {
|
||||
"3.1_lab4_and_durable_202": [
|
||||
"pytest_regression",
|
||||
"public_smoke",
|
||||
"public_persistence",
|
||||
"worker_fault_retry_replay"
|
||||
],
|
||||
"3.2_durable_broker_worker_late_ack": [
|
||||
"rabbitmq_durability_ack",
|
||||
"rabbitmq_prefetch",
|
||||
"worker_fault_retry_replay"
|
||||
],
|
||||
"3.3_states_original_thumbnail": [
|
||||
"public_smoke",
|
||||
"pytest_regression",
|
||||
"worker_fault_retry_replay"
|
||||
],
|
||||
"3.4_bounded_retry_idempotency": [
|
||||
"pytest_regression",
|
||||
"worker_fault_retry_replay"
|
||||
],
|
||||
"3.5_stopped_worker_api_broker_restart": [
|
||||
"worker_fault_retry_replay",
|
||||
"public_persistence"
|
||||
],
|
||||
"4.1_crop_webp_and_worker_limits": [
|
||||
"pytest_regression",
|
||||
"worker_fault_retry_replay"
|
||||
],
|
||||
"4.2_commit_publish_recovery": [
|
||||
"outbox_crash"
|
||||
],
|
||||
"4.3_backoff_jitter_classification_registry_replay": [
|
||||
"pytest_regression",
|
||||
"pytest_timeout_retention",
|
||||
"worker_fault_retry_replay"
|
||||
],
|
||||
"4.4_crash_after_put_lease_prefetch_cleanup": [
|
||||
"worker_fault_retry_replay",
|
||||
"rabbitmq_prefetch",
|
||||
"pytest_regression",
|
||||
"pytest_timeout_retention"
|
||||
]
|
||||
},
|
||||
"limitations": [
|
||||
"Single Docker host, no disk-loss or cluster failover claim",
|
||||
"Grade 5 experiments not performed",
|
||||
"No production-scale performance/security claim"
|
||||
]
|
||||
}
|
||||
+21
-11
@@ -1,17 +1,27 @@
|
||||
# Инфраструктура студента
|
||||
|
||||
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
|
||||
Топология — compose.yaml. compose.test.yaml публикует тестовые PostgreSQL/Redis
|
||||
только на localhost. Nginx направляет запросы на две API-реплики.
|
||||
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
PostgreSQL хранит сообщения, metadata, jobs/outbox и аудит replay. Redis 7.4.2
|
||||
с AOF хранит сессии. RabbitMQ 4.4.0 — отдельный брокер заданий: durable очередь,
|
||||
persistent сообщения, publisher confirms, manual consumer ack и rabbitmqdata.
|
||||
Broker и MinIO не публикуют внешние порты. API не получает credentials брокера.
|
||||
|
||||
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
|
||||
Publisher читает jobs из PostgreSQL и публикует их в RabbitMQ. Worker сохраняет
|
||||
original/thumbnail/crop/optimized в приватный MinIO и фиксирует ready до ack.
|
||||
Конкуренция ограничена WORKER_CONCURRENCY; prefetch=1 на отдельный канал каждого
|
||||
consumer. Publisher/worker используют init:true для корректных сигналов и
|
||||
воспроизводимого SIGKILL. Обычный Compose-запуск fault hooks не включает.
|
||||
|
||||
## Перенесённая база lab4
|
||||
Все environment-настройки Compose берутся из .env; образец — .env.example.
|
||||
Credentials передаются файлами secrets; make init создаёт только отсутствующие.
|
||||
Migrate и minio-bootstrap завершаются до старта зависимых сервисов.
|
||||
Перед обновлением остановить старые процессы без удаления volumes. Миграция
|
||||
a62c950e18b3 повторно публикует queued jobs при переходе на RabbitMQ.
|
||||
|
||||
В compose.yaml перенесены postgres, redis (сессии), minio, minio-bootstrap,
|
||||
migrate, api (две реплики), nginx и cleanup. Версии сохранены без обновления.
|
||||
Роли broker, outbox publisher и worker пока не реализованы.
|
||||
core/image_worker.py — одноразовая проверка изображения lab4, а не consumer очереди.
|
||||
Имена проектов/образов lab5 отделены от lab4. Запуск и порты — в REPORT.md.
|
||||
Настройки environment в Compose задаются через .env; образец — .env.example.
|
||||
Секретные значения монтируются отдельно, пути *_FILE относятся к контейнерам.
|
||||
Restart hook сохраняет volumes и состояние заранее остановленного worker.
|
||||
Failover hook останавливает выбранную API-реплику. Retry, retention, операторские
|
||||
команды, ограничения и результаты проверок описаны в REPORT.md.
|
||||
Полный прогон lab5 уровня 4 — make test-grade4; отдельные сценарии —
|
||||
make test-outbox-recovery и make test-jobs на выбранном работающем стенде.
|
||||
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Secret читается внутри контейнера, не попадает в compose config/аргументы.
|
||||
export RABBITMQ_DEFAULT_PASS="$(cat "${BROKER_PASSWORD_FILE:?}")"
|
||||
export RABBITMQ_DEFAULT_USER="${BROKER_USER:?}"
|
||||
export RABBITMQ_DEFAULT_VHOST="${BROKER_VHOST:?}"
|
||||
export RABBITMQ_NODE_PORT="${BROKER_PORT:?}"
|
||||
exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server
|
||||
@@ -81,7 +81,7 @@ def main():
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": [f"arn:aws:s3:::{bucket}/originals/*"],
|
||||
"Resource": [f"arn:aws:s3:::{bucket}/{prefix}/*" for prefix in ("originals", "staging", "variants")],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Offline cutover: повторно опубликовать queued jobs в RabbitMQ."""
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision = "a62c950e18b3"
|
||||
down_revision = "f9183d27a640"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Старые publisher/worker остановлены. Processing восстановится по lease.
|
||||
op.execute("""
|
||||
UPDATE attachment_jobs j SET published_attempt = attempt
|
||||
FROM attachments a WHERE a.id = j.attachment_id AND a.status = 'queued'
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# При обратном переключении требуется такая же повторная публикация.
|
||||
upgrade()
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Lab5: состояния, варианты и устойчивые jobs с намерением публикации."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision = "c35e91a2f706"
|
||||
down_revision = "8f31b94a2d10"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check")
|
||||
op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status IN ('queued', 'processing', 'ready', 'failed')")
|
||||
op.alter_column("attachments", "status", server_default="queued")
|
||||
op.add_column("attachments", sa.Column("error_code", sa.String(64), nullable=True))
|
||||
op.add_column("attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True))
|
||||
op.add_column("attachments", sa.Column("variants", postgresql.JSONB(), nullable=False, server_default=sa.text("'[]'::jsonb")))
|
||||
op.create_check_constraint(op.f("ck_attachments_error_code"), "attachments", "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)")
|
||||
op.create_table(
|
||||
"attachment_jobs",
|
||||
sa.Column("id", sa.UUID(), nullable=False),
|
||||
sa.Column("attachment_id", sa.UUID(), nullable=False),
|
||||
sa.Column("staging_key", sa.String(128), nullable=True),
|
||||
sa.Column("pipeline_version", sa.Integer(), server_default="1", nullable=False),
|
||||
sa.Column("attempt", sa.Integer(), server_default="0", nullable=False),
|
||||
sa.Column("published_attempt", sa.Integer(), server_default="0", nullable=False),
|
||||
sa.Column("next_attempt_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
|
||||
sa.Column("lease_token", sa.UUID(), nullable=True),
|
||||
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error_code", sa.String(64), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("clock_timestamp()"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_attachment_jobs")),
|
||||
sa.ForeignKeyConstraint(["attachment_id"], ["attachments.id"], name=op.f("fk_attachment_jobs_attachment_id_attachments")),
|
||||
sa.UniqueConstraint("attachment_id", name=op.f("uq_attachment_jobs_attachment_id")),
|
||||
sa.CheckConstraint("pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version")),
|
||||
sa.CheckConstraint("attempt >= 0 AND published_attempt >= 0", name=op.f("ck_attachment_jobs_attempt")),
|
||||
)
|
||||
op.create_index("ix_attachment_jobs_due", "attachment_jobs", ["next_attempt_at"])
|
||||
op.create_index("ix_attachment_jobs_lease", "attachment_jobs", ["lease_until"])
|
||||
# Прежние оригиналы остаются источником; worker создаст отсутствующий thumbnail.
|
||||
op.execute("INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments")
|
||||
op.execute("UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
connection = op.get_bind()
|
||||
unfinished = connection.scalar(sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'"))
|
||||
if unfinished:
|
||||
raise RuntimeError("Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения")
|
||||
op.drop_index("ix_attachment_jobs_lease", table_name="attachment_jobs")
|
||||
op.drop_index("ix_attachment_jobs_due", table_name="attachment_jobs")
|
||||
op.drop_table("attachment_jobs")
|
||||
op.drop_constraint(op.f("ck_attachments_error_code"), "attachments", type_="check")
|
||||
op.drop_column("attachments", "variants")
|
||||
op.drop_column("attachments", "thumbnail_key")
|
||||
op.drop_column("attachments", "error_code")
|
||||
op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check")
|
||||
op.create_check_constraint(op.f("ck_attachments_status"), "attachments", "status = 'ready'")
|
||||
op.alter_column("attachments", "status", server_default="ready")
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Уровень 4: ключи crop/WebP и дообработка старых ready-вложений."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "e7048b6da913"
|
||||
down_revision = "c35e91a2f706"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("attachments", sa.Column("crop_key", sa.String(128), nullable=True))
|
||||
op.add_column("attachments", sa.Column("optimized_key", sa.String(128), nullable=True))
|
||||
# Старый staging сменяется надёжным ready-оригиналом, но не теряется из журнала cleanup.
|
||||
op.execute("""
|
||||
INSERT INTO upload_intents (id, object_key)
|
||||
SELECT gen_random_uuid(), j.staging_key
|
||||
FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE a.status = 'ready' AND j.staging_key IS NOT NULL
|
||||
AND j.staging_key <> a.object_key
|
||||
ON CONFLICT (object_key) DO NOTHING
|
||||
""")
|
||||
op.execute("""
|
||||
UPDATE attachment_jobs j
|
||||
SET staging_key = a.object_key, attempt = 0, published_attempt = 0,
|
||||
next_attempt_at = clock_timestamp(), lease_token = NULL, lease_until = NULL,
|
||||
last_error_code = NULL, finished_at = NULL
|
||||
FROM attachments a
|
||||
WHERE a.id = j.attachment_id AND a.status = 'ready'
|
||||
""")
|
||||
op.execute("""
|
||||
UPDATE attachments SET status = 'queued', variants = '[]'::jsonb,
|
||||
updated_at = clock_timestamp()
|
||||
WHERE status = 'ready'
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
active = op.get_bind().scalar(sa.text("SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')"))
|
||||
if active:
|
||||
raise RuntimeError("Перед downgrade необходимо завершить активные задания")
|
||||
op.execute("""
|
||||
UPDATE attachments a SET variants = COALESCE(
|
||||
(SELECT jsonb_agg(v) FROM jsonb_array_elements(a.variants) v
|
||||
WHERE v->>'name' = 'thumbnail'), '[]'::jsonb)
|
||||
""")
|
||||
op.drop_column("attachments", "optimized_key")
|
||||
op.drop_column("attachments", "crop_key")
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Бюджет повторов и аудит операторского replay."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "f9183d27a640"
|
||||
down_revision = "e7048b6da913"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("attachment_jobs", sa.Column("cycle_start_attempt", sa.Integer(), nullable=False, server_default="0"))
|
||||
op.add_column("attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True))
|
||||
op.add_column("attachment_jobs", sa.Column("last_error_permanent", sa.Boolean(), nullable=True))
|
||||
op.create_table(
|
||||
"job_replays",
|
||||
sa.Column("id", sa.Uuid(), primary_key=True),
|
||||
sa.Column("job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False),
|
||||
sa.Column("actor", sa.String(64), nullable=False),
|
||||
sa.Column("reason", sa.String(32), nullable=False),
|
||||
sa.Column("previous_attempt", sa.Integer(), nullable=False),
|
||||
sa.Column("attempt_limit", sa.Integer(), nullable=False),
|
||||
sa.Column("previous_error_code", sa.String(64), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.clock_timestamp()),
|
||||
)
|
||||
op.create_index("ix_job_replays_job_id", "job_replays", ["job_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("job_replays")
|
||||
op.drop_column("attachment_jobs", "last_error_permanent")
|
||||
op.drop_column("attachment_jobs", "attempt_limit")
|
||||
op.drop_column("attachment_jobs", "cycle_start_attempt")
|
||||
@@ -50,3 +50,4 @@ pytest-asyncio==1.4.1
|
||||
httpx==0.28.1
|
||||
minio==7.2.20
|
||||
Pillow==13.0.0
|
||||
aio-pika==10.2.2
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Fault/recovery lab5 на явно выбранном Compose-стенде; только синтетические данные."""
|
||||
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
from io import BytesIO
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, check
|
||||
|
||||
PROBE = '''
|
||||
import asyncio, hashlib, json
|
||||
from uuid import UUID
|
||||
from sqlalchemy import select
|
||||
from app.core.config import Settings
|
||||
from app.core.database import create_db_engine, create_session_factory
|
||||
from app.models.job import AttachmentJob
|
||||
from app.models.attachment import Attachment
|
||||
from app.storage.s3 import create_s3_storage
|
||||
async def main():
|
||||
settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3)
|
||||
try:
|
||||
async with create_session_factory(engine)() as session:
|
||||
job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one()
|
||||
result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent)
|
||||
objects=[]
|
||||
for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'):
|
||||
entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True)))
|
||||
for obj in entries:
|
||||
stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256()
|
||||
try:
|
||||
async for chunk in stream: checksum.update(chunk)
|
||||
finally: await stream.aclose()
|
||||
objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified))
|
||||
result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
|
||||
finally: await storage.aclose();await engine.dispose()
|
||||
asyncio.run(main())
|
||||
'''
|
||||
|
||||
|
||||
def probe(attachment_id):
|
||||
code = 'ATTACHMENT_ID = ' + repr(str(UUID(attachment_id))) + '\n' + PROBE
|
||||
result = subprocess.run(COMPOSE + ['exec', '-T', 'api', 'python', '-'], input=code, text=True, capture_output=True, cwd=ROOT, timeout=30)
|
||||
check(result.returncode == 0, 'Probe failed; diagnostic output hidden')
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def admin(*arguments, expected=0):
|
||||
result = compose('run', '--rm', '--no-deps', 'worker', 'python', '-m', 'app.core.job_admin', *arguments)
|
||||
check(result.returncode == expected, 'Unexpected operator command exit status')
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def verify_result(client, actor, attachment, original):
|
||||
ready = client.wait_ready(actor, attachment, wait_seconds=180)
|
||||
check(ready['id'] == attachment['id'], 'Attachment identity changed')
|
||||
path = '/api/v1/attachments/' + ready['id'] + '/content'
|
||||
check(client.call('GET', path, actor=actor)['bytes'] == original, 'Original changed')
|
||||
check({v['name'] for v in ready['variants']} == {'thumbnail','crop','optimized'}, 'Incomplete pipeline')
|
||||
for variant in ready['variants']:
|
||||
data = client.call('GET', path+'?variant='+variant['name'], actor=actor)['bytes']
|
||||
check(hashlib.sha256(data).hexdigest() == variant['sha256'], 'Checksum mismatch')
|
||||
with Image.open(BytesIO(data)) as image:
|
||||
check(image.size == (variant['width'],variant['height']), 'Metadata dimensions mismatch')
|
||||
check(image.format == ('WEBP' if variant['name']=='optimized' else 'JPEG'), 'Wrong format')
|
||||
check(not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp'), 'Metadata leaked')
|
||||
if variant['name']=='crop':check(image.size==(128,128),'Wrong crop size')
|
||||
snapshot=probe(ready['id'])
|
||||
check(len(snapshot['objects']) == 4, 'Expected exactly one original and three variants')
|
||||
return snapshot
|
||||
|
||||
|
||||
def crash(client, actor, chat, mode):
|
||||
name='lab5-worker-fault-'+uuid4().hex
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
result=compose('run','--rm','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
|
||||
phase='after_object:optimized' if mode=='crash-after-object' else 'before_ack'
|
||||
marker=json.dumps({'fault':phase,'attachment_id':attachment['id']})
|
||||
check(result.returncode==137 and marker in result.stdout,'Targeted crash not observed')
|
||||
before=probe(attachment['id'])
|
||||
check(before['status']==('processing' if mode=='crash-after-object' else 'ready'),'Unexpected crash state')
|
||||
check(len(before['objects'])==4,'Crash must occur after all four PUTs')
|
||||
finally:
|
||||
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
|
||||
require_compose('start','worker')
|
||||
after=verify_result(client,actor,attachment,original)
|
||||
if mode=='crash-before-ack':
|
||||
# Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
|
||||
wait_queue_empty()
|
||||
after=probe(attachment['id'])
|
||||
check(after==before,'Ready duplicate rewrote result')
|
||||
else:
|
||||
check(after['attempt']==before['attempt']+1,'Lease did not recover the interrupted attempt')
|
||||
check([(x['name'],x['sha256']) for x in before['objects']]==[(x['name'],x['sha256']) for x in after['objects']],'Replay changed object content')
|
||||
return dict(scenario=mode,attachment_id=attachment['id'],exit_code=137,before=before,after=after)
|
||||
|
||||
|
||||
def retry_replay(client,actor,chat,permanent):
|
||||
name='lab5-retry-fault-'+uuid4().hex
|
||||
samples=[]
|
||||
configuration=ComposeProject().config['services']['worker']['environment']
|
||||
retry_base=float(configuration['JOB_RETRY_SECONDS'])
|
||||
retry_max=float(configuration['JOB_RETRY_MAX_SECONDS'])
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
mode='fail-permanent' if permanent else 'fail-transient'
|
||||
require_compose('run','-d','--no-deps','--name',name,'worker','python','-m','app.core.job_worker','--'+mode,attachment['id'])
|
||||
deadline=time.monotonic()+180
|
||||
while time.monotonic()<deadline:
|
||||
sample=probe(attachment['id'])
|
||||
if sample['attempt'] and sample['status'] in {'queued', 'failed'} and (not samples or samples[-1]['attempt']!=sample['attempt']):samples.append(sample)
|
||||
if sample['status']=='failed':break
|
||||
time.sleep(.5)
|
||||
check(sample['status']=='failed','Retry never reached failed')
|
||||
check(sample['attempt']==(1 if permanent else sample['limit']),'Incorrect attempt budget')
|
||||
check(sample['permanent']==permanent,'Incorrect error classification')
|
||||
if not permanent:
|
||||
check(len(samples)==sample['limit'],'Missed retry samples')
|
||||
for entry in samples:
|
||||
delay=(datetime.fromisoformat(entry['next_attempt_at'])-datetime.fromisoformat(entry['updated_at'])).total_seconds()
|
||||
ceiling=min(retry_max,retry_base*2**(entry['attempt']-1))
|
||||
check(ceiling/2<=delay<=ceiling,'Retry delay outside configured jitter bounds')
|
||||
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
|
||||
job_id=sample['job_id'];request_id=str(uuid4())
|
||||
listing=admin('list-failed')
|
||||
check(any(row['job_id']==job_id for row in listing),'Job missing from failed registry')
|
||||
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30,check=True)
|
||||
first=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
|
||||
check(first['status']=='queued','Replay did not queue job')
|
||||
finally:
|
||||
subprocess.run(['docker','rm','-f',name],capture_output=True,timeout=30)
|
||||
require_compose('start','worker')
|
||||
after=verify_result(client,actor,attachment,original)
|
||||
repeat=admin('replay',job_id,'--request-id',request_id,'--actor','verification','--reason','dependency_restored')
|
||||
check(repeat['already_applied'],'Replay request id not idempotent')
|
||||
refused=admin('replay',job_id,'--request-id',str(uuid4()),'--actor','verification','--reason','manual_review',expected=2)
|
||||
check(refused['error_code']=='job_not_failed','Successful job was replayed')
|
||||
check(probe(attachment['id'])==after,'Operator replay altered ready job')
|
||||
check(len(admin('history',job_id))==1,'Incorrect audit count')
|
||||
return dict(scenario=mode,attachment_id=attachment['id'],samples=samples,after=after,replay_idempotent=True)
|
||||
|
||||
|
||||
def queue_state():
|
||||
output=require_compose('exec','-T','broker','rabbitmqctl','list_queues','--formatter=json','name','messages_ready','messages_unacknowledged','consumers','durable')
|
||||
return next(row for row in json.loads(output) if row['name']=='attachment-jobs')
|
||||
|
||||
|
||||
def wait_queue_empty():
|
||||
deadline=time.monotonic()+20
|
||||
while time.monotonic()<deadline:
|
||||
state=queue_state()
|
||||
if state['messages_ready']==0 and state['messages_unacknowledged']==0:
|
||||
return state
|
||||
time.sleep(.5)
|
||||
check(False,'RabbitMQ did not acknowledge all completed deliveries')
|
||||
|
||||
|
||||
def persistence(client,actor,chat):
|
||||
try:
|
||||
require_compose('stop','worker')
|
||||
attachment,original=client.attachment(actor,chat)
|
||||
client.message(actor,chat,'',attachments=[attachment['id']])
|
||||
client.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=actor,expected=409)
|
||||
time.sleep(2)
|
||||
queued_before=queue_state()
|
||||
check(queued_before['messages_ready']>=1 and queued_before['durable'], 'No durable queued message before restart')
|
||||
result=subprocess.run([str(ROOT/'scripts/restart.sh')],cwd=ROOT,capture_output=True,timeout=180)
|
||||
check(result.returncode==0,'Restart hook failed')
|
||||
check('worker' not in require_compose('ps','--status','running','--services').splitlines(),'Restart started deliberately stopped worker')
|
||||
queued_after=queue_state()
|
||||
check(queued_after['messages_ready']>=1 and queued_after['durable'], 'Broker lost queued delivery across restart')
|
||||
client.message(actor,chat,'session and chat survived broker/API restart')
|
||||
finally:require_compose('start','worker')
|
||||
after=verify_result(client,actor,attachment,original)
|
||||
return dict(scenario='worker_stopped_restart_all',attachment_id=attachment['id'],queue_before=queued_before,queue_after=queued_after,after=after)
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url',required=True)
|
||||
parser.add_argument('--scenario',choices=['all','persistence','crash-after-object','crash-before-ack','fail-transient','fail-permanent'],default='all')
|
||||
parser.add_argument('--output',default='evidence/lab5-jobs.json')
|
||||
args=parser.parse_args()
|
||||
client=Client(args.base_url,timeout=40)
|
||||
aliases={r.instance_id for r in ComposeProject().replicas() if r.running}
|
||||
check(client.call('GET','/health/live',schema='Health')['headers'].get('X-Instance-Id') in aliases,'Wrong Compose project for BASE_URL')
|
||||
actor,_,_,chat=client.bootstrap()
|
||||
results=[]
|
||||
operations={'persistence':persistence,'crash-after-object':lambda *x:crash(*x,'crash-after-object'),'crash-before-ack':lambda *x:crash(*x,'crash-before-ack'),'fail-transient':lambda *x:retry_replay(*x,False),'fail-permanent':lambda *x:retry_replay(*x,True)}
|
||||
selected=operations.values() if args.scenario=='all' else [operations[args.scenario]]
|
||||
for operation in selected:
|
||||
result=operation(client,actor,chat);results.append(result)
|
||||
Path(args.output).parent.mkdir(parents=True,exist_ok=True)
|
||||
Path(args.output).write_text(json.dumps(dict(status='in_progress',results=results),indent=2)+'\n')
|
||||
print('PASS: '+result['scenario'],flush=True)
|
||||
Path(args.output).write_text(json.dumps(dict(status='passed',results=results),indent=2)+'\n')
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Сценарий COMMIT → SIGKILL до publish → ready без повторного upload.
|
||||
|
||||
Требуется уже работающий отдельный стенд. Скрипт не выполняется при запуске API.
|
||||
Публичные tests не изменяет, secrets/tokens не печатает и evidence сам не сохраняет.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
from uuid import uuid4
|
||||
|
||||
from scripts.deployment import ComposeProject
|
||||
from tests.client import Client, check
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(COMPOSE + list(arguments), cwd=ROOT, text=True, capture_output=True, timeout=timeout)
|
||||
|
||||
|
||||
def require_compose(*arguments: str) -> str:
|
||||
result = compose(*arguments)
|
||||
check(result.returncode == 0, "Compose command failed; output hidden")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Отказ publisher после COMMIT перед RabbitMQ publish")
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--ca-file")
|
||||
args = parser.parse_args()
|
||||
running = set(require_compose("ps", "--status", "running", "--services").splitlines())
|
||||
check({"api", "postgres", "minio", "broker", "publisher", "worker"} <= running, "Start the lab5 stack before this scenario")
|
||||
client = Client(args.base_url, args.ca_file, timeout=40)
|
||||
project = ComposeProject()
|
||||
aliases = {replica.instance_id for replica in project.replicas() if replica.running}
|
||||
live = client.call("GET", "/health/live", schema="Health")
|
||||
check(live["headers"].get("X-Instance-Id") in aliases, "BASE_URL does not belong to the selected Compose project")
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
attachment = None
|
||||
fault_container = "lab5-outbox-fault-" + uuid4().hex
|
||||
try:
|
||||
require_compose("stop", "publisher")
|
||||
attachment, original = client.attachment(alice, chat)
|
||||
message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"]
|
||||
client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected=409, actor=alice)
|
||||
killed = compose(
|
||||
"run", "--rm", "--no-deps", "--name", fault_container, "publisher", "python", "-m",
|
||||
"app.core.outbox_publisher", "--crash-before-publish", attachment["id"],
|
||||
)
|
||||
marker = json.dumps({"fault": "before_publish", "attachment_id": attachment["id"]})
|
||||
check(killed.returncode == 137 and marker in killed.stdout, "Expected targeted SIGKILL before publish was not observed")
|
||||
pending = client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=alice, schema="Attachment")["json"]
|
||||
check(pending["status"] == "queued", "Crash must leave accepted job queued")
|
||||
finally:
|
||||
# Если docker compose run оборвался по timeout, одноразовый контейнер тоже удаляется.
|
||||
try:
|
||||
subprocess.run(["docker", "rm", "-f", fault_container], capture_output=True, timeout=30)
|
||||
finally:
|
||||
require_compose("start", "publisher")
|
||||
ready = client.wait_ready(alice, attachment, wait_seconds=60)
|
||||
check(ready["id"] == attachment["id"], "Recovered a different attachment")
|
||||
check({variant["name"] for variant in ready["variants"]} == {"thumbnail", "crop", "optimized"}, "Missing required variants")
|
||||
path = f"/api/v1/attachments/{attachment['id']}/content"
|
||||
check(client.call("GET", path, actor=alice)["bytes"] == original, "Original changed after recovery")
|
||||
for variant in ready["variants"]:
|
||||
downloaded = client.call("GET", path + "?variant=" + variant["name"], actor=alice)["bytes"]
|
||||
check(len(downloaded) == variant["size_bytes"] and hashlib.sha256(downloaded).hexdigest() == variant["sha256"], "Recovered variant checksum mismatch")
|
||||
print(json.dumps({
|
||||
"scenario": "commit_before_publish", "attachment_id": ready["id"],
|
||||
"message_id": message["id"], "publisher_exit_code": killed.returncode,
|
||||
"status_after_crash": pending["status"], "status_after_restart": ready["status"],
|
||||
"variants": [variant["name"] for variant in ready["variants"]],
|
||||
"reuploaded": False,
|
||||
}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -14,7 +14,7 @@ def main() -> None:
|
||||
destination.chmod(0o600)
|
||||
directory = root / "secrets"
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
for name in ("db_password", "redis_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"):
|
||||
for name in ("db_password", "redis_password", "broker_password", "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key"):
|
||||
path = directory / name
|
||||
try:
|
||||
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444)
|
||||
|
||||
+9
-4
@@ -22,11 +22,16 @@ wait_healthy() {
|
||||
done
|
||||
}
|
||||
|
||||
compose stop cleanup
|
||||
compose restart postgres redis minio
|
||||
wait_healthy postgres redis minio
|
||||
worker_running=$(compose ps --status running --quiet worker)
|
||||
compose stop worker publisher cleanup
|
||||
compose restart postgres redis minio broker
|
||||
wait_healthy postgres redis minio broker
|
||||
compose restart api
|
||||
wait_healthy api
|
||||
compose restart nginx
|
||||
wait_healthy nginx
|
||||
compose start cleanup
|
||||
compose start publisher cleanup
|
||||
# Сценарий с заранее остановленным worker сохраняет его остановленным.
|
||||
if [ -n "$worker_running" ]; then
|
||||
compose start worker
|
||||
fi
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Lab4: API и транзакции с реальной БД; байты S3 подменены управляемым адаптером."""
|
||||
"""Lab5: API и транзакции с реальной БД; байты S3 подменены управляемым адаптером."""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
@@ -17,6 +17,10 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from app.core.s3 import S3Settings
|
||||
from app.core.jobs import JobSettings
|
||||
from app.domain.attachment_jobs import process_attachment_job
|
||||
from app.models.job import AttachmentJob
|
||||
from app.schemas.job import JobMessage
|
||||
from app.domain import attachments as domain
|
||||
from app.errors.exceptions import unavailable
|
||||
from app.models.attachment import Attachment, UploadIntent
|
||||
@@ -79,14 +83,31 @@ async def upload(client, actor, chat, data=PNG, filename='фото.png', content
|
||||
)
|
||||
|
||||
|
||||
async def finish_attachment(client, storage, actor, attachment_id):
|
||||
engine = client._transport.app.state.engine
|
||||
async with AsyncSession(engine) as session:
|
||||
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
|
||||
message = JobMessage(job_id=job.id, attachment_id=job.attachment_id,
|
||||
pipeline_version=job.pipeline_version, attempt=job.attempt + 1, created_at=job.created_at)
|
||||
await session.rollback()
|
||||
assert await process_attachment_job(session, storage, message, JobSettings(_env_file=None))
|
||||
result = await client.get(f'/api/v1/attachments/{attachment_id}', headers=bearer(actor))
|
||||
assert result.json()['status'] == 'ready'
|
||||
return result.json()
|
||||
|
||||
|
||||
async def test_upload_download_and_attachment_only_message(client, context):
|
||||
storage, alice, bob, eve, chat = context
|
||||
response = await upload(client, alice, chat)
|
||||
assert response.status_code == 201, response.text
|
||||
assert response.status_code == 202, response.text
|
||||
attachment = response.json()
|
||||
assert attachment['status'] == 'ready'
|
||||
assert attachment['status'] == 'queued'
|
||||
before = await client.get(f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice))
|
||||
assert before.status_code == 409
|
||||
attachment = await finish_attachment(client, storage, alice, attachment['id'])
|
||||
assert attachment['sha256'] == hashlib.sha256(PNG).hexdigest()
|
||||
assert attachment['variants'] == [] and attachment['error_code'] is None
|
||||
assert {v['name'] for v in attachment['variants']} == {'thumbnail', 'crop', 'optimized'}
|
||||
assert attachment['error_code'] is None
|
||||
assert 'object_key' not in attachment
|
||||
id = attachment['id']
|
||||
for actor in (alice, bob):
|
||||
@@ -120,13 +141,14 @@ async def test_invalid_image_and_mime_are_415(client, context, data, content_typ
|
||||
|
||||
|
||||
async def test_jpeg_is_stored_without_reencoding(client, context):
|
||||
_, alice, _, _, chat = context
|
||||
storage, alice, _, _, chat = context
|
||||
output = BytesIO()
|
||||
Image.new('RGB', (32, 32), 'blue').save(output, format='JPEG')
|
||||
original = output.getvalue()
|
||||
response = await upload(client, alice, chat, original, content_type='image/jpeg')
|
||||
assert response.status_code == 201, response.text
|
||||
assert response.status_code == 202, response.text
|
||||
id = response.json()['id']
|
||||
await finish_attachment(client, storage, alice, id)
|
||||
downloaded = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
|
||||
assert downloaded.content == original
|
||||
|
||||
@@ -167,7 +189,7 @@ async def test_exact_byte_limit_accepts_valid_png(client, context):
|
||||
padding = struct.pack('!I', len(payload)) + kind + payload + struct.pack('!I', zlib.crc32(kind + payload))
|
||||
image = PNG[:-12] + padding + PNG[-12:]
|
||||
response = await upload(client, alice, chat, image)
|
||||
assert response.status_code == 201, response.text
|
||||
assert response.status_code == 202, response.text
|
||||
assert response.json()['size_bytes'] == maximum
|
||||
assert response.json()['sha256'] == hashlib.sha256(image).hexdigest()
|
||||
|
||||
@@ -178,7 +200,7 @@ async def test_exact_pixel_limit_accepts_valid_png(client, context):
|
||||
with Image.new('RGB', (8000, 5000), 'blue') as image:
|
||||
image.save(output, format='PNG')
|
||||
response = await upload(client, alice, chat, output.getvalue())
|
||||
assert response.status_code == 201, response.text
|
||||
assert response.status_code == 202, response.text
|
||||
|
||||
|
||||
async def test_pixel_limit_before_decoding(client, context):
|
||||
@@ -226,6 +248,7 @@ async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context):
|
||||
async def test_s3_outage_is_503(client, context):
|
||||
storage, alice, _, _, chat = context
|
||||
id = (await upload(client, alice, chat)).json()['id']
|
||||
await finish_attachment(client, storage, alice, id)
|
||||
storage.down = True
|
||||
response = await client.get(f'/api/v1/attachments/{id}/content', headers=bearer(alice))
|
||||
assert response.status_code == 503
|
||||
@@ -353,6 +376,7 @@ async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypa
|
||||
async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
id = (await upload(client, alice, chat)).json()['id']
|
||||
await finish_attachment(client, storage, alice, id)
|
||||
async with AsyncSession(db_engine) as session:
|
||||
attachment = await session.get(Attachment, UUID(id))
|
||||
key = attachment.object_key
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Реальные преобразования и OS-лимиты дочернего процесса worker."""
|
||||
|
||||
import asyncio
|
||||
from io import BytesIO
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
from uuid import uuid4
|
||||
|
||||
from PIL import Image
|
||||
import pytest
|
||||
|
||||
from app.core.image_processing import ImageSettings
|
||||
from app.domain.image_pipeline import create_variants
|
||||
from app.domain.job_files import render_variants
|
||||
from app.storage.jobs import JobWork
|
||||
from test_attachments import MemoryStorage
|
||||
|
||||
|
||||
@pytest.mark.parametrize('size,transparent', [((1600,800),False),((24,12),True)])
|
||||
async def test_variant_geometry_metadata_and_alpha(tmp_path,size,transparent):
|
||||
source=tmp_path/'source.png'
|
||||
with Image.new('RGBA',size,(255,0,0,0 if transparent else 255)) as image:
|
||||
exif=Image.Exif();exif[270]='private description'
|
||||
image.save(source,format='PNG',exif=exif)
|
||||
result=create_variants(source,tmp_path,'image/png',40_000_000,ImageSettings(_env_file=None))
|
||||
variants={v.name:v for v in result.variants}
|
||||
assert (variants['crop'].width,variants['crop'].height)==(128,128)
|
||||
assert (variants['thumbnail'].width,variants['thumbnail'].height)==((256,128) if size[0]>256 else size)
|
||||
assert (variants['optimized'].width,variants['optimized'].height)==((1280,640) if size[0]>1280 else size)
|
||||
for name in ('thumbnail.jpg','crop.jpg','optimized.webp'):
|
||||
with Image.open(tmp_path/name) as image:
|
||||
assert not image.getexif() and not image.info.get('icc_profile') and not image.info.get('xmp')
|
||||
if transparent:
|
||||
if name.endswith('webp'):assert image.getpixel((0,0))[3]==0
|
||||
else:assert min(image.getpixel((0,0)))>=250
|
||||
|
||||
|
||||
async def test_exif_orientation_applied(tmp_path):
|
||||
source=tmp_path/'source.jpg'
|
||||
with Image.new('RGB',(400,200),'blue') as image:
|
||||
exif=Image.Exif();exif[274]=6
|
||||
image.save(source,exif=exif)
|
||||
result=create_variants(source,tmp_path,'image/jpeg',40_000_000,ImageSettings(_env_file=None))
|
||||
assert (result.variants[0].width,result.variants[0].height)==(128,256)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('failure', ['memory','cpu'])
|
||||
async def test_worker_os_limits(failure,tmp_path):
|
||||
body='return bytearray(2 * 1024 ** 3)' if failure=='memory' else 'while True: pass'
|
||||
code="from app.core import thumbnail_worker as w\ndef fail(*args):\n "+body+"\nw.create_variants=fail\nraise SystemExit(w.main())"
|
||||
result=await asyncio.to_thread(subprocess.run,[sys.executable,'-c',code,'source',str(tmp_path),str(tmp_path/'metadata'),'image/png','40000000'],
|
||||
env={**os.environ,'IMAGE_MEMORY_MIB':'256','IMAGE_CPU_SECONDS':'1'},capture_output=True,timeout=15)
|
||||
assert result.returncode==(3 if failure=='memory' else -9)
|
||||
assert not (tmp_path/'metadata').exists()
|
||||
|
||||
|
||||
async def test_render_timeout_kills_child(monkeypatch,tmp_path):
|
||||
create=asyncio.create_subprocess_exec
|
||||
children=[]
|
||||
async def slow_child(*args,**kwargs):
|
||||
child=await create(sys.executable,'-c','import time;time.sleep(60)',**kwargs)
|
||||
children.append(child)
|
||||
return child
|
||||
monkeypatch.setattr(asyncio,'create_subprocess_exec',slow_child)
|
||||
work=JobWork(uuid4(),uuid4(),uuid4(),1,'staging/test','image/png',1,'0'*64)
|
||||
with pytest.raises(TimeoutError):
|
||||
async with asyncio.timeout(.2):
|
||||
await render_variants(MemoryStorage(),work,tmp_path/'source',tmp_path)
|
||||
assert len(children)==1 and children[0].returncode is not None
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Lab5: retry/replay, fencing, cleanup, limits и транзакции реального PostgreSQL."""
|
||||
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
import errno
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.jobs import JobSettings
|
||||
from app.domain import attachment_jobs
|
||||
from app.domain.attachment_jobs import claim_job, process_attachment_job, cleanup_staging, build_result
|
||||
from app.domain.job_admin import replay_failed_job
|
||||
from app.domain.job_retry import classify_failure, retry_delay
|
||||
from app.errors.exceptions import JobOperationError, ProcessingError
|
||||
from app.models.attachment import Attachment
|
||||
from app.models.job import AttachmentJob
|
||||
from app.models.job_replay import JobReplay
|
||||
from app.schemas.job import JobMessage
|
||||
from app.storage import jobs
|
||||
from test_attachments import context, upload, finish_attachment
|
||||
|
||||
|
||||
async def envelope(session, attachment_id):
|
||||
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id)))
|
||||
return JobMessage(job_id=job.id, attachment_id=job.attachment_id, pipeline_version=1,
|
||||
attempt=job.attempt + 1, created_at=job.created_at)
|
||||
|
||||
|
||||
async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
settings = JobSettings(_env_file=None, job_max_attempts=3)
|
||||
def fail(_):
|
||||
raise TimeoutError
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
first = await envelope(session, attachment['id'])
|
||||
for attempt in range(1, 4):
|
||||
message = await envelope(session, attachment['id'])
|
||||
assert await process_attachment_job(session, storage, message, settings, before_processing=fail)
|
||||
job, item = await jobs.lock_job(session, first.job_id)
|
||||
assert job.attempt == attempt
|
||||
delay = (job.next_attempt_at - item.updated_at).total_seconds()
|
||||
assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1)
|
||||
assert item.status == ('failed' if attempt == 3 else 'queued')
|
||||
assert job.last_error_permanent is False
|
||||
job.next_attempt_at = await jobs.database_time(session) - timedelta(seconds=1)
|
||||
await session.commit()
|
||||
assert item.error_code == 'retry_exhausted'
|
||||
request_id = uuid4()
|
||||
result = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id,
|
||||
actor='test', reason='dependency_restored')
|
||||
assert result['attempt_limit'] == 6
|
||||
# Old envelope cannot consume the fresh replay budget.
|
||||
assert await process_attachment_job(session, storage, first, settings)
|
||||
job, item = await jobs.lock_job(session, first.job_id)
|
||||
assert job.attempt == 3 and item.status == 'queued'
|
||||
await session.commit()
|
||||
message = await envelope(session, attachment['id'])
|
||||
assert await process_attachment_job(session, storage, message, settings)
|
||||
before = dict(storage.objects)
|
||||
assert await process_attachment_job(session, storage, message, settings)
|
||||
assert storage.objects == before
|
||||
repeat = await replay_failed_job(session, storage, settings, first.job_id, request_id=request_id,
|
||||
actor='test', reason='dependency_restored')
|
||||
assert repeat['already_applied']
|
||||
with pytest.raises(JobOperationError, match='job_not_failed'):
|
||||
await replay_failed_job(session, storage, settings, first.job_id, request_id=uuid4(), actor='test', reason='manual_review')
|
||||
await session.rollback()
|
||||
assert len((await session.scalars(select(JobReplay))).all()) == 1
|
||||
|
||||
|
||||
async def test_permanent_failure_and_missing_source_replay(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
def fail(_):
|
||||
raise ProcessingError('image_limits')
|
||||
settings = JobSettings(_env_file=None)
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
await process_attachment_job(session, storage, message, settings, before_processing=fail)
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
assert item.status == 'failed' and job.attempt == 1 and job.last_error_permanent
|
||||
job.staging_key = None
|
||||
await session.commit()
|
||||
with pytest.raises(JobOperationError, match='source_expired'):
|
||||
await replay_failed_job(session, storage, settings, job.id, request_id=uuid4(), actor='test', reason='manual_review')
|
||||
|
||||
|
||||
async def test_stale_worker_cannot_write_and_cleanup_keeps_active(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
settings = JobSettings(_env_file=None)
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
claim = await claim_job(session, message, settings)
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
job.lease_token = uuid4()
|
||||
job.finished_at = await jobs.database_time(session) - timedelta(days=10)
|
||||
await session.commit()
|
||||
assert await cleanup_staging(session, storage, settings, apply=True) == 0
|
||||
# Rendering must not be allowed to PUT on a stale lease.
|
||||
before = dict(storage.objects)
|
||||
assert await build_result(session, storage, claim.work)
|
||||
assert storage.objects == before
|
||||
|
||||
|
||||
async def test_cleanup_ready_preserves_final_objects(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
await finish_attachment(client, storage, alice, attachment['id'])
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
staging = job.staging_key
|
||||
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
|
||||
await session.commit()
|
||||
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=False) == 1
|
||||
assert staging in storage.objects
|
||||
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 1
|
||||
assert staging not in storage.objects and len(storage.objects) == 4
|
||||
assert await cleanup_staging(session, storage, JobSettings(_env_file=None), apply=True) == 0
|
||||
|
||||
|
||||
async def test_retry_classification_and_cap():
|
||||
settings = JobSettings(_env_file=None)
|
||||
for attempt in (1, 2, 6, 100):
|
||||
cap = min(120, 5 * 2 ** min(30, attempt - 1))
|
||||
assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100))
|
||||
assert classify_failure(TimeoutError()) == ('processing_timeout', False)
|
||||
assert classify_failure(OSError(errno.EACCES, 'denied'))[1]
|
||||
assert not classify_failure(OSError(errno.ENOSPC, 'full'))[1]
|
||||
|
||||
|
||||
async def test_timeout_records_retry_and_clears_lease(client, context, db_engine, monkeypatch):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
async def stalled(*args, **kwargs):
|
||||
await asyncio.sleep(60)
|
||||
monkeypatch.setattr(attachment_jobs, 'build_result', stalled)
|
||||
settings = JobSettings(_env_file=None).model_copy(update={'job_timeout_seconds': .05})
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
assert await process_attachment_job(session, storage, message, settings)
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
assert item.status == 'queued' and job.last_error_code == 'processing_timeout'
|
||||
assert job.lease_token is None and job.lease_until is None
|
||||
|
||||
|
||||
async def test_failed_retention_and_source_checksum_protect_replay(client, context, db_engine):
|
||||
storage, alice, _, _, chat = context
|
||||
attachment = (await upload(client, alice, chat)).json()
|
||||
def fail(_):
|
||||
raise ProcessingError('invalid_image')
|
||||
settings = JobSettings(_env_file=None)
|
||||
async with AsyncSession(db_engine, expire_on_commit=False) as session:
|
||||
message = await envelope(session, attachment['id'])
|
||||
await process_attachment_job(session, storage, message, settings, before_processing=fail)
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
key = job.staging_key
|
||||
job.finished_at = await jobs.database_time(session) - timedelta(days=2)
|
||||
await session.commit()
|
||||
assert await cleanup_staging(session, storage, settings, apply=True) == 0
|
||||
data, metadata = storage.objects[key]
|
||||
storage.objects[key] = (b'x' * len(data), metadata)
|
||||
with pytest.raises(JobOperationError, match='source_mismatch'):
|
||||
await replay_failed_job(session, storage, settings, message.job_id, request_id=uuid4(), actor='test', reason='manual_review')
|
||||
await session.rollback()
|
||||
assert not (await session.scalars(select(JobReplay))).all()
|
||||
job, item = await jobs.lock_job(session, message.job_id)
|
||||
job.finished_at = await jobs.database_time(session) - timedelta(days=8)
|
||||
await session.commit()
|
||||
assert await cleanup_staging(session, storage, settings, apply=True) == 1
|
||||
assert key not in storage.objects
|
||||
@@ -8,7 +8,7 @@ from sqlalchemy import inspect, text
|
||||
from app.models import Base
|
||||
from conftest import reset_schema, upgrade_head
|
||||
|
||||
EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents"}
|
||||
EXPECTED_TABLES = {"users", "chats", "chat_members", "messages", "counters", "idempotency_keys", "attachments", "message_attachments", "upload_intents", "attachment_jobs", "job_replays"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
Reference in New Issue
Block a user